Iteratively increase sample size for good/bad classification.

scripts/slave/recipe_modules/auto_bisect/revision_state.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """An interface for holding state and result of revisions in a bisect job.
 
 When implementing support for tests other than perf, one should extend this
 class so that the bisect module and recipe can use it.
 
 See perf_revision_state for an example.
@@ skipping 13 matching lines @@
   (
       NEW,  # A revision_state object that has just been initialized.
       BUILDING,  # Requested a build for this revision, waiting for it.
       TESTING,  # A test job for this revision was triggered, waiting for it.
       TESTED,  # The test job completed with non-failing results.
       FAILED,  # Either the build or the test jobs failed or timed out.
       ABORTED,  # The build or test job was aborted. (For use in multi-secting).
       SKIPPED,  # A revision that was not built or tested for a special reason,
                 # such as those ranges that we know are broken, or when nudging
                 # revisions.
-  ) = xrange(7)
+      NEED_MORE_DATA,  # Current number of test values is too small to establish
+                       # a statistically significant difference between this
+                       # revision and the revisions known to be good and bad.
+  ) = xrange(8)
 
   def __init__(self, revision_string, bisector,
                dependency_depot_name=None, base_revision=None,
                deps_revision=None, cp=None):
     """Creates a new instance to track the state of a revision.
 
     There are two use cases for this constructor:
       - Creating a revision state for a chromium revision, OR
       - Creating a revision state for a chromium revision plus an explicitly
         specified revision of a dependency repository (a DEPS change).
@@ skipping 59 matching lines @@
           self.commit_pos = cp
         else:  # pragma: no cover
           # If there is no commit position, use an abbreviated hash instead.
           self.commit_pos = self.commit_hash[:8]
       else:
         self.commit_hash, self.commit_pos = self._commit_from_rev_string()
     self.build_url = self.bisector.get_platform_gs_prefix() + self._gs_suffix()
 
   @property
   def tested(self):
-    return self.status in [RevisionState.TESTED]
+    return self.status in (RevisionState.TESTED,)
 
   @property
   def in_progress(self):
-    return self.status in (RevisionState.BUILDING, RevisionState.TESTING)
+    return self.status in (RevisionState.BUILDING, RevisionState.TESTING,
+                           RevisionState.NEED_MORE_DATA)
 
   @property
   def failed(self):
     return self.status == RevisionState.FAILED
 
   @property
   def aborted(self):
     return self.status == RevisionState.ABORTED
 
   @property
@@ skipping 12 matching lines @@
   def bad(self, value):
     self._good = not value
 
   def start_job(self):
     """Starts a build, or a test job if the build is available."""
     if self.status == RevisionState.NEW and not self._is_build_archived():
       self._request_build()
       self.status = RevisionState.BUILDING
       return
 
-    if self._is_build_archived() and self.status in [RevisionState.NEW,
-                                                     RevisionState.BUILDING]:
+    if self._is_build_archived() and self.status in (
+        RevisionState.NEW, RevisionState.BUILDING,
+        RevisionState.NEED_MORE_DATA):
       self._do_test()
       self.status = RevisionState.TESTING
 
   def abort(self):  # pragma: no cover
     """Aborts the job.
 
     This method is typically called when the bisect no longer requires it. Such
     as when a later good revision or an earlier bad revision have been found in
     parallel.
     """
@@ skipping 77 matching lines @@
     """Checks on the pending jobs and updates status accordingly.
 
     This method will check for the build to complete and then trigger the test,
     or will wait for the test as appropriate.
 
     To wait for the test we try to get the buildbot job url from GS, and if
     available, we query the status of such job.
     """
     if self.status == RevisionState.BUILDING and self._is_build_archived():
       self.start_job()
-    elif self.status == RevisionState.TESTING and self._results_available():
-      self._read_test_results()
+    elif (self.status in (RevisionState.TESTING, RevisionState.NEED_MORE_DATA)
+          and self._results_available()):
+      # If we have already decided whether the revision is good or bad we
+      # shouldn't check again
+      check_revision_goodness = not(self.good or self.bad)
+      self._read_test_results(
+          check_revision_goodness=check_revision_goodness)
       # We assume _read_test_results may have changed the status to a broken
       # state such as FAILED or ABORTED.
-      if self.status == RevisionState.TESTING:
+      if self.status in (RevisionState.TESTING, RevisionState.NEED_MORE_DATA):
         self.status = RevisionState.TESTED
 
   def _is_build_archived(self):
     """Checks if the revision is already built and archived."""
     if not self.build_archived:
       api = self.bisector.api
       self.build_archived = api.gsutil_file_exists(self.build_url)
 
     if self.bisector.dummy_builds:
       self.build_archived = self.in_progress
@@ skipping 59 matching lines @@
 
   def _get_hash_from_pos(self, pos):
     api = self.bisector.api
     try:
       result = api.m.commit_position.chromium_hash_from_commit_position(pos)
     except api.m.step.StepFailure as sf:  # pragma: no cover
       self.bisector.surface_result('BAD_REV')
       api.m.halt('Failed to resolve commit position - ' + sf.reason)
       raise
     return result