Making check_initial_confidence verify return_code bisects.

scripts/slave/recipe_modules/auto_bisect/revision_state.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """An interface for holding state and result of revisions in a bisect job.
 
 When implementing support for tests other than perf, one should extend this
 class so that the bisect module and recipe can use it.
 
 See perf_revision_state for an example.
@@ skipping 72 matching lines @@
           self.depot, self.commit_hash)
       self.deps_sha = hashlib.sha1(self.deps_patch).hexdigest()
       self.deps_sha_patch = self.bisector.make_deps_sha_file(self.deps_sha)
       self.deps = dict(base_revision.deps)
       self.deps[self.depot_name] = self.commit_hash
     else:
       self.needs_patch = False
     self.build_url = self.bisector.get_platform_gs_prefix() + self._gs_suffix()
     self.values = []
     self.mean_value = None
+    self.overall_return_code = None
     self.std_dev = None
     self.repeat_count = MINIMUM_SAMPLE_SIZE
     self._test_config = None
     self.build_number = None
 
   @property
   def tested(self):
     return self.status in (RevisionState.TESTED,)
 
   @property
@@ skipping 220 matching lines @@
     # the 'error' key explaining the type of error.
     results = test_results['results']
     if results.get('errors'):
       self.status = RevisionState.FAILED
       if 'MISSING_METRIC' in results.get('errors'):  # pragma: no cover
         self.bisector.surface_result('MISSING_METRIC')
       return
     self.values += results['values']
     if self.bisector.is_return_code_mode():
       retcodes = test_results['retcodes']
-      overall_return_code = 0 if all(v == 0 for v in retcodes) else 1
-      self.mean_value = overall_return_code
+      self.overall_return_code = 0 if all(v == 0 for v in retcodes) else 1
+      # Keeping mean_value for compatibility with dashboard.
+      # TODO(robertocn): refactor mean_value, specially when uploading results
+      # to dashboard.

[qyearsley, 2016/03/23 22:09:36]

What do you think we should do specifically when uploading to the perf
dashboard? I see two main choices:

1. In Bisector._revision_data
(https://code.google.com/p/chromium/codesearch#chromium/build/scripts/slave/re...),
send a field "value" or "overall_return_code" instead of "mean_value", and also
update the perf dashboard to deal with that. (I like this option a bit better)

2. In Bisector._revision_data, send either overall_return_code or mean_value,
but keep using the key "mean_value" so that the perf dashboard doesn't have to
be changed.

Do you agree?

+      self.mean_value = self.overall_return_code
     elif self.values:
       api = self.bisector.api
       self.mean_value = api.m.math_utils.mean(self.values)
       self.std_dev = api.m.math_utils.standard_deviation(self.values)
     # Values were not found, but the test did not otherwise fail.
     else:
       self.status = RevisionState.FAILED
       self.bisector.surface_result('MISSING_METRIC')
       return
     # If we have already decided on the goodness of this revision, we shouldn't
@@ skipping 156 matching lines @@
     significantly different as a precondition.
 
     Returns:
       True if the results of testing this revision are significantly different
       from those of testing the earliest known bad revision.
       False if they are instead significantly different form those of testing
       the latest knwon good revision.
     """
 
     if self.bisector.is_return_code_mode():
-      return self.mean_value == self.bisector.lkgr.mean_value
+      return self.overall_return_code == self.bisector.lkgr.overall_return_code
 
     while True:
       diff_from_good = self.bisector.significantly_different(
           self.bisector.lkgr.values, self.values)
       diff_from_bad = self.bisector.significantly_different(
           self.bisector.fkbr.values, self.values)
 
       if diff_from_good and diff_from_bad:
         # Multiple regressions.
         # For now, proceed bisecting the biggest difference of the means.
@@ skipping 38 matching lines @@
     """
     next_revision_to_test = min(self.bisector.lkgr, self, self.bisector.fkbr,
         key=lambda x: len(x.values))
     if (len(self.bisector.last_tested_revision.values) ==
         next_revision_to_test.values):
       self.bisector.last_tested_revision.retest()
     else:
       next_revision_to_test.retest()
 
   def __repr__(self):
+    if self.overall_return_code is not None:
+      return ('RevisionState(rev=%s, values=%r, overall_return_code=%r, '
+              'std_dev=%r)') % (self.revision_string(), self.values,
+                                self.overall_return_code, self.std_dev)
     return ('RevisionState(rev=%s, values=%r, mean_value=%r, std_dev=%r)' % (
         self.revision_string(), self.values, self.mean_value, self.std_dev))