Making check_initial_confidence verify return_code bisects.

scripts/slave/recipe_modules/auto_bisect/bisector.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import json
 import re
 import time
 import urllib
 
 from . import depot_config
@@ skipping 469 matching lines @@
     """Checks that the initial range presents a clear enough regression.
 
     We ensure that the good and bad revisions produce significantly different
     results, increasing the sample size until MAX_REQUIRED_SAMPLES is reached
     or REGRESSION_CHECK_TIMEOUT seconds have elapsed.
 
     Returns: True if the revisions produced results that differ from each
     other in a statistically significant manner. False if such difference could
     not be established in the time or sample size allowed.
     """
-    if self.test_type != 'perf':
-      return True
+    if self.test_type == 'return_code':
+      # In return_code bisects, mean_value represents the overall return value
+      # of the test, i.e. 0 if all runs returned 0, 1 otherwise.

[qyearsley, 2016/03/23 20:54:40]

mean_value is a potentially confusing or misleading name.

If we were to rename it (in a separate CL), what would it be? summary_value? Or
might mean_value for return code bisects be ignored, and instead set a separate
attribute called overall_return_code (or something similar)?

[RobertoCN, 2016/03/23 21:52:09]

Created a new member of the revision class called overall_return_code.
Still need to refactor mean_value from the code that posts results back to
bisect.

+      return self.good_rev.mean_value != self.bad_rev.mean_value
+
 
     if self.bypass_stats_check:
       dummy_result = self.good_rev.values != self.bad_rev.values
       if not dummy_result:
         self._set_insufficient_confidence_warning()
       return dummy_result
 
     with self.api.m.step.nest('Re-testing reference range'):
       expiration_time = time.time() + REGRESSION_CHECK_TIMEOUT
       while time.time() < expiration_time:
@@ skipping 342 matching lines @@
         })
     return revision_rows
 
   def _get_build_url(self):
     properties = self.api.m.properties
     bot_url = properties.get('buildbotURL',
                              'http://build.chromium.org/p/chromium/')
     builder_name = urllib.quote(properties.get('buildername', ''))
     builder_number = str(properties.get('buildnumber', ''))
     return '%sbuilders/%s/builds/%s' % (bot_url, builder_name, builder_number)