Refactor calculation of "other regressions" output.

tools/bisect-perf-regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 2323 matching lines @@
     print
     print 'To reproduce locally:'
     print '$ ' + self.opts.command
     if bisect_utils.IsTelemetryCommand(self.opts.command):
       print
       print 'Also consider passing --profiler=list to see available profilers.'
 
   def _PrintOtherRegressions(self, other_regressions, revision_data):
     print
     print 'Other regressions may have occurred:'
+    print '  %8s  %82s  %10s' % ('Depot'.center(8, ' '),
+        'Range'.center(82, ' '), 'Confidence'.center(10, ' '))
     for regression in other_regressions:
-      current_id, previous_id, percent_change, deviations = regression
+      current_id, previous_id, confidence = regression
       current_data = revision_data[current_id]
       previous_data = revision_data[previous_id]
 
-      if deviations is None:
-        deviations = 'N/A'
-      else:
-        deviations = '%.2f' % deviations
-
-      if percent_change is None:
-        percent_change = 0
-
-      print '  %8s  %s  [%.2f%%, %s x std.dev]' % (
-          previous_data['depot'], previous_id, 100 * percent_change, deviations)
-      print '  %8s  %s' % (current_data['depot'], current_id)
+      print '  %8s  %s..%s %s' % (
+          current_data['depot'], current_id, previous_id,
+          ('%d%%' % confidence).center(10, ' '))
       print
 
   def _PrintStepTime(self, revision_data_sorted):
     step_perf_time_avg = 0.0
     step_build_time_avg = 0.0
     step_count = 0.0
     for _, current_data in revision_data_sorted:
       step_perf_time_avg += current_data['perf_time']
       step_build_time_avg += current_data['build_time']
       step_count += 1
     if step_count:
       step_perf_time_avg = step_perf_time_avg / step_count
       step_build_time_avg = step_build_time_avg / step_count
     print
     print 'Average build time : %s' % datetime.timedelta(
         seconds=int(step_build_time_avg))
     print 'Average test time  : %s' % datetime.timedelta(
         seconds=int(step_perf_time_avg))
 
   def _PrintWarnings(self):
     if not self.warnings:
       return
     print
     print 'WARNINGS:'
     for w in self.warnings:
       print '  !!! %s' % w
 
+  def _FindOtherRegressions(self, revision_data_sorted, bad_greater_than_good):
+    other_regressions = []
+    previous_values = []
+    previous_id = None
+    for current_id, current_data in revision_data_sorted:
+      current_values = current_data['value']
+      if current_values:
+        current_values = current_values['values']
+        if previous_values:
+          confidence = self._CalculateConfidence(previous_values,
+              [current_values])
+          mean_of_prev_runs = CalculateTruncatedMean(
+              sum(previous_values, []), 0)
+          mean_of_current_runs = CalculateTruncatedMean(current_values, 0)
+
+          # Check that the potential regression is in the same direction as
+          # the overall regression. If the mean of the previous runs < the
+          # mean of the current runs, this local regression is in same
+          # direction.
+          prev_less_than_current = mean_of_prev_runs < mean_of_current_runs
+          is_same_direction = (prev_less_than_current if
+              bad_greater_than_good else not prev_less_than_current)
+
+          # Only report potential regressions with high confidence.
+          if is_same_direction and confidence > 50:
+            other_regressions.append([current_id, previous_id, confidence])
+        previous_values.append(current_values)
+        previous_id = current_id
+    return other_regressions
+
+  def _CalculateConfidence(self, working_means, broken_means):
+    bounds_working = []
+    bounds_broken = []
+    for m in working_means:
+      current_mean = CalculateTruncatedMean(m, 0)
+      if bounds_working:
+        bounds_working[0] = min(current_mean, bounds_working[0])
+        bounds_working[1] = max(current_mean, bounds_working[0])
+      else:
+        bounds_working = [current_mean, current_mean]
+    for m in broken_means:
+      current_mean = CalculateTruncatedMean(m, 0)
+      if bounds_broken:
+        bounds_broken[0] = min(current_mean, bounds_broken[0])
+        bounds_broken[1] = max(current_mean, bounds_broken[0])
+      else:
+        bounds_broken = [current_mean, current_mean]
+    dist_between_groups = min(math.fabs(bounds_broken[1] - bounds_working[0]),
+        math.fabs(bounds_broken[0] - bounds_working[1]))
+    working_mean = sum(working_means, [])
+    broken_mean = sum(broken_means, [])
+    len_working_group = CalculateStandardDeviation(working_mean)
+    len_broken_group = CalculateStandardDeviation(broken_mean)
+
+    confidence = (dist_between_groups / (
+        max(0.0001, (len_broken_group + len_working_group ))))
+    confidence = int(min(1.0, max(confidence, 0.0)) * 100.0)
+    return confidence
+
   def _GetResultsDict(self, revision_data, revision_data_sorted):
     # Find range where it possibly broke.
     first_working_revision = None
     first_working_revision_index = -1
     last_broken_revision = None
     last_broken_revision_index = -1
 
     for i in xrange(len(revision_data_sorted)):
       k, v = revision_data_sorted[i]
       if v['passed'] == 1:
         if not first_working_revision:
           first_working_revision = k
           first_working_revision_index = i
 
       if not v['passed']:
         last_broken_revision = k
         last_broken_revision_index = i
 
     if last_broken_revision != None and first_working_revision != None:
-      bounds_broken = [revision_data[last_broken_revision]['value']['mean'],
-          revision_data[last_broken_revision]['value']['mean']]
-      broken_mean = []
+      broken_means = []
       for i in xrange(0, last_broken_revision_index + 1):
         if revision_data_sorted[i][1]['value']:
-          bounds_broken[0] = min(bounds_broken[0],
-              revision_data_sorted[i][1]['value']['mean'])
-          bounds_broken[1] = max(bounds_broken[1],
-              revision_data_sorted[i][1]['value']['mean'])
-          broken_mean.extend(revision_data_sorted[i][1]['value']['values'])
+          broken_means.append(revision_data_sorted[i][1]['value']['values'])
 
-      bounds_working = [revision_data[first_working_revision]['value']['mean'],
-          revision_data[first_working_revision]['value']['mean']]
-      working_mean = []
+      working_means = []
       for i in xrange(first_working_revision_index, len(revision_data_sorted)):
         if revision_data_sorted[i][1]['value']:
-          bounds_working[0] = min(bounds_working[0],
-              revision_data_sorted[i][1]['value']['mean'])
-          bounds_working[1] = max(bounds_working[1],
-              revision_data_sorted[i][1]['value']['mean'])
-          working_mean.extend(revision_data_sorted[i][1]['value']['values'])
+          working_means.append(revision_data_sorted[i][1]['value']['values'])
+
+      # Flatten the lists to calculate mean of all values.
+      working_mean = sum(working_means, [])
+      broken_mean = sum(broken_means, [])
 
       # Calculate the approximate size of the regression
       mean_of_bad_runs = CalculateTruncatedMean(broken_mean, 0.0)
       mean_of_good_runs = CalculateTruncatedMean(working_mean, 0.0)
 
       regression_size = math.fabs(max(mean_of_good_runs, mean_of_bad_runs) /
           max(0.0001, min(mean_of_good_runs, mean_of_bad_runs))) * 100.0 - 100.0
 
       regression_std_err = math.fabs(CalculatePooledStandardError(
           [working_mean, broken_mean]) /
           max(0.0001, min(mean_of_good_runs, mean_of_bad_runs))) * 100.0
 
       # Give a "confidence" in the bisect. At the moment we use how distinct the
       # values are before and after the last broken revision, and how noisy the
       # overall graph is.
-      dist_between_groups = min(math.fabs(bounds_broken[1] - bounds_working[0]),
-          math.fabs(bounds_broken[0] - bounds_working[1]))
-      len_working_group = CalculateStandardDeviation(working_mean)
-      len_broken_group = CalculateStandardDeviation(broken_mean)
-
-      confidence = (dist_between_groups / (
-          max(0.0001, (len_broken_group + len_working_group ))))
-      confidence = int(min(1.0, max(confidence, 0.0)) * 100.0)
+      confidence = self._CalculateConfidence(working_means, broken_means)
 
       culprit_revisions = []
 
       cwd = os.getcwd()
       self.ChangeToDepotWorkingDirectory(
           revision_data[last_broken_revision]['depot'])
 
       if revision_data[last_broken_revision]['depot'] == 'cros':
         # Want to get a list of all the commits and what depots they belong
         # to so that we can grab info about each.
@@ skipping 30 matching lines @@
         for i in xrange(last_broken_revision_index, len(revision_data_sorted)):
           k, v = revision_data_sorted[i]
           if k == first_working_revision:
             break
           self.ChangeToDepotWorkingDirectory(v['depot'])
           info = self.source_control.QueryRevisionInfo(k)
           culprit_revisions.append((k, info, v['depot']))
       os.chdir(cwd)
 
       # Check for any other possible regression ranges
-      good_std_dev = revision_data[first_working_revision]['value']['std_err']
-      good_mean = revision_data[first_working_revision]['value']['mean']
-      bad_mean = revision_data[last_broken_revision]['value']['mean']
-      prev_revision_data = revision_data_sorted[0][1]
-      prev_revision_id = revision_data_sorted[0][0]
-      other_regressions = []
-      for current_id, current_data in revision_data_sorted:
-        if current_data['value']:
-          prev_mean = prev_revision_data['value']['mean']
-          cur_mean = current_data['value']['mean']
-
-          if good_std_dev:
-            deviations = math.fabs(prev_mean - cur_mean) / good_std_dev
-          else:
-            deviations = None
-
-          if good_mean:
-            percent_change = (prev_mean - cur_mean) / good_mean
-
-            # If the "good" valuse are supposed to be higher than the "bad"
-            # values (ie. scores), flip the sign of the percent change so that
-            # a positive value always represents a regression.
-            if bad_mean < good_mean:
-              percent_change *= -1.0
-          else:
-            percent_change = None
-
-          if deviations >= 1.5 or percent_change > 0.01:
-            if current_id != first_working_revision:
-              other_regressions.append(
-                  [current_id, prev_revision_id, percent_change, deviations])
-          prev_revision_data = current_data
-          prev_revision_id = current_id
+      other_regressions = self._FindOtherRegressions(revision_data_sorted,
+          mean_of_bad_runs > mean_of_good_runs)
 
     # Check for warnings:
     if len(culprit_revisions) > 1:
       self.warnings.append('Due to build errors, regression range could '
                            'not be narrowed down to a single commit.')
     if self.opts.repeat_test_count == 1:
       self.warnings.append('Tests were only set to run once. This may '
                            'be insufficient to get meaningful results.')
     if confidence < 100:
       if confidence:
@@ skipping 423 matching lines @@
       # The perf dashboard scrapes the "results" step in order to comment on
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
     print 'Error: %s' % e.message
     if opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
   return 1
 
 if __name__ == '__main__':
   sys.exit(main())