Obtain confidence score based off last known good and first known bad revision results.

tools/auto_bisect/bisect_perf_regression.py

Index: tools/auto_bisect/bisect_perf_regression.py
diff --git a/tools/auto_bisect/bisect_perf_regression.py b/tools/auto_bisect/bisect_perf_regression.py
index 635adbd3c558c50972f7f35fb3a9ddff03425291..7c10bf016523126bd066baccc4ed49d4677883c3 100755
--- a/tools/auto_bisect/bisect_perf_regression.py
+++ b/tools/auto_bisect/bisect_perf_regression.py
@@ -77,6 +77,10 @@ MAX_LINUX_BUILD_TIME = 14400
 # The confidence percentage we require to consider the initial range a
 # regression based on the test results of the inital good and bad revisions.
 REGRESSION_CONFIDENCE = 80
+# How many times to repeat the test on the last known good and first known bad
+# revisions in order to assess a more accurate confidence score in the
+# regression culprit.
+BORDER_REVISIONS_EXTRA_RUNS = 2
 
 # Patch template to add a new file, DEPS.sha under src folder.
 # This file contains SHA1 value of the DEPS changes made while bisecting
@@ -1233,7 +1237,7 @@ class BisectPerformanceMetrics(object):
 
   def RunPerformanceTestAndParseResults(
       self, command_to_run, metric, reset_on_first_run=False,
-      upload_on_last_run=False, results_label=None):
+      upload_on_last_run=False, results_label=None, extra_test_runs=0):
     """Runs a performance test on the current revision and parses the results.
 
     Args:
@@ -1246,6 +1250,8 @@ class BisectPerformanceMetrics(object):
       results_label: A value for the option flag --results-label.
           The arguments reset_on_first_run, upload_on_last_run and results_label
           are all ignored if the test is not a Telemetry test.
+      extra_test_runs: Factor by which to increase the number of test repeats
+          and the timeout period specified in self.opts.

[qyearsley, 2015/01/16 18:37:03]

Possible suggestion:
Rename this to "repeat_count_multiplier" (or "test_run_multiplier"), give it a
default value of 1 in the args of RunTest, and change below to:
for i in xrange(self.opts.repeat_test_count * repeat_count_multiplier):
  ...
And change the time limit in the same way.

Would this be clearer or no?

[RobertoCN, 2015/01/30 21:23:32]

Done.

 
     Returns:
       (values dict, 0) if --debug_ignore_perf_test was passed.
@@ -1287,7 +1293,8 @@ class BisectPerformanceMetrics(object):
 
     metric_values = []
     output_of_all_runs = ''
-    for i in xrange(self.opts.repeat_test_count):
+    repeat_count = self.opts.repeat_test_count * (1 + extra_test_runs)
+    for i in xrange(repeat_count):
       # Can ignore the return code since if the tests fail, it won't return 0.
       current_args = copy.copy(args)
       if is_telemetry:
@@ -1328,7 +1335,8 @@ class BisectPerformanceMetrics(object):
         metric_values.append(return_code)
 
       elapsed_minutes = (time.time() - start_time) / 60.0
-      if elapsed_minutes >= self.opts.max_time_minutes:
+      time_limit = self.opts.max_time_minutes *  (1 + extra_test_runs)
+      if elapsed_minutes >= time_limit:
         break
 
     if metric and len(metric_values) == 0:
@@ -1433,7 +1441,8 @@ class BisectPerformanceMetrics(object):
     return False
 
   def RunTest(self, revision, depot, command, metric, skippable=False,
-      skip_sync=False, create_patch=False, force_build=False):
+              skip_sync=False, create_patch=False, force_build=False,
+              extra_test_runs=0):
     """Performs a full sync/build/run of the specified revision.
 
     Args:
@@ -1444,6 +1453,8 @@ class BisectPerformanceMetrics(object):
       skip_sync: Skip the sync step.
       create_patch: Create a patch with any locally modified files.
       force_build: Force a local build.
+      extra_test_runs: Factor by which to increase the given number of runs and
+          the set timeout period.
 
     Returns:
       On success, a tuple containing the results of the performance test.
@@ -1485,7 +1496,8 @@ class BisectPerformanceMetrics(object):
     command = self.GetCompatibleCommand(command, revision, depot)
 
     # Run the command and get the results.
-    results = self.RunPerformanceTestAndParseResults(command, metric)
+    results = self.RunPerformanceTestAndParseResults(
+        command, metric, extra_test_runs=extra_test_runs)
 
     # Restore build output directory once the tests are done, to avoid
     # any discrepancies.
@@ -2399,6 +2411,9 @@ class BisectPerformanceMetrics(object):
           self.printer.PrintPartialResults(bisect_state)
           bisect_utils.OutputAnnotationStepClosed()
 
+
+      self._ConfidenceExtraTestRuns(min_revision_state, max_revision_state,
+                                    command_to_run, metric)
       results = BisectResults(bisect_state, self.depot_registry, self.opts,
                               self.warnings)
 
@@ -2412,6 +2427,23 @@ class BisectPerformanceMetrics(object):
                '[%s..%s]' % (good_revision, bad_revision))
       return BisectResults(error=error)
 
+  def _ConfidenceExtraTestRuns(self, good_state, bad_state, command_to_run,
+                               metric):
+    if(bool(good_state.passed) != bool(bad_state.passed)

[qyearsley, 2015/01/16 18:37:03]

Add space after "if"

[RobertoCN, 2015/01/30 21:23:31]

Done.

+       and good_state.passed not in ('Skipped', 'Build Failed')
+       and bad_state.passed not in ('Skipped', 'Build Failed')):
+      for state in (good_state, bad_state):
+        run_results = self.RunTest(
+            state.revision,
+            state.depot,
+            command_to_run,
+            metric,
+            extra_test_runs=BORDER_REVISIONS_EXTRA_RUNS
+        )

[qyearsley, 2015/01/16 18:37:03]

Since this is a function call, the convention is to put the closing parenthesis
on the same line as the last arg.

[RobertoCN, 2015/01/30 21:23:32]

Done.

+        # Is extend the right thing to do here?
+        state.value['values'].extend(run_results[0]['values'])
+      # Get new confidence score

[RobertoCN, 2015/01/14 18:50:47]

This comment will be removed. 

[RobertoCN, 2015/01/30 21:23:32]

Done.

+
 
 def _IsPlatformSupported():
   """Checks that this platform and build system are supported.