Refactor bisect-perf-regression_test.py and add smoke test.

tools/bisect-perf-regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 268 matching lines @@
 
   Returns:
     A number in the range [0, 100].
   """
   if not good_results_lists or not bad_results_lists:
     return 0.0
 
   # Flatten the lists of results lists.
   sample1 = sum(good_results_lists, [])
   sample2 = sum(bad_results_lists, [])
+  if not sample1 or not sample2:
+    return 0.0

[qyearsley, 2014/07/31 22:34:09]

Previously ConfidenceScore was throwing an error if called with an empty list.

 
   # The p-value is approximately the probability of obtaining the given set
   # of good and bad values just by chance.
   _, _, p_value = ttest.WelchsTTest(sample1, sample2)
   return 100.0 * (1.0 - p_value)
 
 
 def GetSHA1HexDigest(contents):
   """Returns SHA1 hex digest of the given string."""
   return hashlib.sha1(contents).hexdigest()
@@ skipping 2748 matching lines @@
           previous_data['depot'], previous_link)
       print
 
   def _GetResultsDict(self, revision_data, revision_data_sorted):
     # Find range where it possibly broke.
     first_working_revision = None
     first_working_revision_index = -1
     last_broken_revision = None
     last_broken_revision_index = -1
 
+    culprit_revisions = []
+    other_regressions = []
+    regression_size = 0.0
+    regression_std_err = 0.0
+    confidence = 0.0

[qyearsley, 2014/07/31 22:34:09]

Previously it was possible that these were referenced before being defined
below.

+
     for i in xrange(len(revision_data_sorted)):
       k, v = revision_data_sorted[i]
       if v['passed'] == 1:
         if not first_working_revision:
           first_working_revision = k
           first_working_revision_index = i
 
       if not v['passed']:
         last_broken_revision = k
         last_broken_revision_index = i
@@ skipping 159 matching lines @@
         self._PrintRevisionInfo(cl, info, depot)
       if results_dict['other_regressions']:
         self._PrintOtherRegressions(results_dict['other_regressions'],
                                     revision_data)
     self._PrintTestedCommitsTable(revision_data_sorted,
                                   results_dict['first_working_revision'],
                                   results_dict['last_broken_revision'],
                                   results_dict['confidence'])
     _PrintStepTime(revision_data_sorted)
     self._PrintReproSteps()
-    self._PrintThankYou()
+    _PrintThankYou()

[qyearsley, 2014/07/31 22:34:09]

The new test testDryRun was able to catch this :-)

     if self.opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
 
   def _PrintBanner(self, results_dict):
     if self._IsBisectModeReturnCode():
       metrics = 'N/A'
       change = 'Yes'
     else:
       metrics = '/'.join(self.opts.metric)
       change = '%.02f%% (+/-%.02f%%)' % (
@@ skipping 405 matching lines @@
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
     print 'Error: %s' % e.message
     if opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
   return 1
 
 
 if __name__ == '__main__':
   sys.exit(main())