Iteratively increase sample size for good/bad classification.

scripts/slave/recipe_modules/auto_bisect/revision_state.py

 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """An interface for holding state and result of revisions in a bisect job.
 
 When implementing support for tests other than perf, one should extend this
 class so that the bisect module and recipe can use it.
 
 See perf_revision_state for an example.
@@ skipping 13 matching lines @@
   (
       NEW,  # A revision_state object that has just been initialized.
       BUILDING,  # Requested a build for this revision, waiting for it.
       TESTING,  # A test job for this revision was triggered, waiting for it.
       TESTED,  # The test job completed with non-failing results.
       FAILED,  # Either the build or the test jobs failed or timed out.
       ABORTED,  # The build or test job was aborted. (For use in multi-secting).
       SKIPPED,  # A revision that was not built or tested for a special reason,
                 # such as those ranges that we know are broken, or when nudging
                 # revisions.
-  ) = xrange(7)
+      NEED_MORE_DATA,

[qyearsley, 2016/01/26 19:26:00]

You could add a comment for this status -- still not enough evidence to classify
this revision as good or bad, waiting for more test runs (although it's fairly
clear from the name, the other statuses are also somewhat clear from the name).

[RobertoCN, 2016/02/01 17:29:51]

Done.

+  ) = xrange(8)
 
   def __init__(self, revision_string, bisector,
                dependency_depot_name=None, base_revision=None,
                deps_revision=None, cp=None):
     """Creates a new instance to track the state of a revision.
 
     There are two use cases for this constructor:
       - Creating a revision state for a chromium revision, OR
       - Creating a revision state for a chromium revision plus an explicitly
         specified revision of a dependency repository (a DEPS change).
@@ skipping 63 matching lines @@
       else:
         self.commit_hash, self.commit_pos = self._commit_from_rev_string()
     self.build_url = self.bisector.get_platform_gs_prefix() + self._gs_suffix()
 
   @property
   def tested(self):
     return self.status in [RevisionState.TESTED]
 
   @property
   def in_progress(self):
-    return self.status in (RevisionState.BUILDING, RevisionState.TESTING)
+    return self.status in (RevisionState.BUILDING, RevisionState.TESTING,
+                           RevisionState.NEED_MORE_DATA)
 
   @property
   def failed(self):
     return self.status == RevisionState.FAILED
 
   @property
   def aborted(self):
     return self.status == RevisionState.ABORTED
 
   @property
@@ skipping 12 matching lines @@
   def bad(self, value):
     self._good = not value
 
   def start_job(self):
     """Starts a build, or a test job if the build is available."""
     if self.status == RevisionState.NEW and not self._is_build_archived():
       self._request_build()
       self.status = RevisionState.BUILDING
       return
 
-    if self._is_build_archived() and self.status in [RevisionState.NEW,
-                                                     RevisionState.BUILDING]:
+    if self._is_build_archived() and self.status in [
+        RevisionState.NEW, RevisionState.BUILDING,
+        RevisionState.NEED_MORE_DATA]:

[qyearsley, 2016/01/26 19:26:00]

Below when checking whether status is one of several things, you use a tuple
rather than list; could also change this to use a tuple.

[RobertoCN, 2016/02/01 17:29:51]

Done.

       self._do_test()
       self.status = RevisionState.TESTING
 
   def abort(self):  # pragma: no cover
     """Aborts the job.
 
     This method is typically called when the bisect no longer requires it. Such
     as when a later good revision or an earlier bad revision have been found in
     parallel.
     """
@@ skipping 73 matching lines @@
     """Checks on the pending jobs and updates status accordingly.
 
     This method will check for the build to complete and then trigger the test,
     or will wait for the test as appropriate.
 
     To wait for the test we try to get the buildbot job url from GS, and if
     available, we query the status of such job.
     """
     if self.status == RevisionState.BUILDING and self._is_build_archived():
       self.start_job()
-    elif self.status == RevisionState.TESTING and self._results_available():
-      self._read_test_results()
+    elif (self.status in (RevisionState.TESTING, RevisionState.NEED_MORE_DATA)
+          and self._results_available()):
+      # if the status is NEED_MORE_DATA, it's a retest, and we should prevent
+      # the check_revision_good from running again.
+      self._read_test_results(
+          check_revision_goodness=self.status == RevisionState.TESTING)
       # We assume _read_test_results may have changed the status to a broken
       # state such as FAILED or ABORTED.
-      if self.status == RevisionState.TESTING:
+      if self.status in (RevisionState.TESTING, RevisionState.NEED_MORE_DATA):
         self.status = RevisionState.TESTED
 
   def _is_build_archived(self):
     """Checks if the revision is already built and archived."""
     if not self.build_archived:
       api = self.bisector.api
       self.build_archived = api.gsutil_file_exists(self.build_url)
 
     if self.bisector.dummy_builds:
       self.build_archived = self.in_progress
@@ skipping 59 matching lines @@
 
   def _get_hash_from_pos(self, pos):
     api = self.bisector.api
     try:
       result = api.m.commit_position.chromium_hash_from_commit_position(pos)
     except api.m.step.StepFailure as sf:  # pragma: no cover
       self.bisector.surface_result('BAD_REV')
       api.m.halt('Failed to resolve commit position - ' + sf.reason)
       raise
     return result