Iteratively increase sample size for good/bad classification.

scripts/slave/recipe_modules/auto_bisect/revision_state.py

Index: scripts/slave/recipe_modules/auto_bisect/revision_state.py
diff --git a/scripts/slave/recipe_modules/auto_bisect/revision_state.py b/scripts/slave/recipe_modules/auto_bisect/revision_state.py
index 174c8609100f437ac2099b18d67545a2d92f657b..c85cf3c97f7017bd40408f413d71079b9daa3f6f 100644
--- a/scripts/slave/recipe_modules/auto_bisect/revision_state.py
+++ b/scripts/slave/recipe_modules/auto_bisect/revision_state.py
@@ -31,7 +31,8 @@ class RevisionState(object):
       SKIPPED,  # A revision that was not built or tested for a special reason,
                 # such as those ranges that we know are broken, or when nudging
                 # revisions.
-  ) = xrange(7)
+      NEED_MORE_DATA,

[qyearsley, 2016/01/26 19:26:00]

You could add a comment for this status -- still not enough evidence to classify
this revision as good or bad, waiting for more test runs (although it's fairly
clear from the name, the other statuses are also somewhat clear from the name).

[RobertoCN, 2016/02/01 17:29:51]

Done.

+  ) = xrange(8)
 
   def __init__(self, revision_string, bisector,
                dependency_depot_name=None, base_revision=None,
@@ -115,7 +116,8 @@ class RevisionState(object):
 
   @property
   def in_progress(self):
-    return self.status in (RevisionState.BUILDING, RevisionState.TESTING)
+    return self.status in (RevisionState.BUILDING, RevisionState.TESTING,
+                           RevisionState.NEED_MORE_DATA)
 
   @property
   def failed(self):
@@ -148,8 +150,9 @@ class RevisionState(object):
       self.status = RevisionState.BUILDING
       return
 
-    if self._is_build_archived() and self.status in [RevisionState.NEW,
-                                                     RevisionState.BUILDING]:
+    if self._is_build_archived() and self.status in [
+        RevisionState.NEW, RevisionState.BUILDING,
+        RevisionState.NEED_MORE_DATA]:

[qyearsley, 2016/01/26 19:26:00]

Below when checking whether status is one of several things, you use a tuple
rather than list; could also change this to use a tuple.

[RobertoCN, 2016/02/01 17:29:51]

Done.

       self._do_test()
       self.status = RevisionState.TESTING
 
@@ -243,11 +246,15 @@ class RevisionState(object):
     """
     if self.status == RevisionState.BUILDING and self._is_build_archived():
       self.start_job()
-    elif self.status == RevisionState.TESTING and self._results_available():
-      self._read_test_results()
+    elif (self.status in (RevisionState.TESTING, RevisionState.NEED_MORE_DATA)
+          and self._results_available()):
+      # if the status is NEED_MORE_DATA, it's a retest, and we should prevent
+      # the check_revision_good from running again.
+      self._read_test_results(
+          check_revision_goodness=self.status == RevisionState.TESTING)
       # We assume _read_test_results may have changed the status to a broken
       # state such as FAILED or ABORTED.
-      if self.status == RevisionState.TESTING:
+      if self.status in (RevisionState.TESTING, RevisionState.NEED_MORE_DATA):
         self.status = RevisionState.TESTED
 
   def _is_build_archived(self):