Reformat bisect results output.

tools/bisect-perf-regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 165 matching lines @@
 @@ -0,0 +1 @@
 +%(deps_sha)s
 """
 
 # The possible values of the --bisect_mode flag, which determines what to
 # use when classifying a revision as "good" or "bad".
 BISECT_MODE_MEAN = 'mean'
 BISECT_MODE_STD_DEV = 'std_dev'
 BISECT_MODE_RETURN_CODE = 'return_code'
 
+# The perf dashboard specifically looks for the string
+# "Estimated Confidence: 95%" to decide whether or not
+# to cc the author(s). If you change this, please update the perf
+# dashboard as well.
+RESULTS_BANNER = """
+===== BISECT JOB RESULTS =====
+Status: %(status)s
+
+Test Command: %(command)s
+Test Metric: %(metrics)s
+Relative Change: %(change)s
+Estimated Confidence: %(confidence)d%%"""
+
+# The perf dashboard specifically looks for the string
+# "Author  : " to parse out who to cc on a bug. If you change the
+# formatting here, please update the perf dashboard as well.
+RESULTS_REVISION_INFO = """
+===== SUSPECTED CL(s) =====
+Subject : %(subject)s
+Author  : %(author)s%(email_info)s%(commit_info)s
+Date    : %(cl_date)s"""
+
+REPRO_STEPS_LOCAL = """
+==== INSTRUCTIONS TO REPRODUCE ====
+To run locally:
+$%(command)s"""
+
+REPRO_STEPS_TRYJOB = """
+To reproduce on Performance trybot:
+1. Create new git branch or check out existing branch.
+2. Edit tools/run-perf-test.cfg (instructions in file) or \
+third_party/WebKit/Tools/run-perf-test.cfg.
+  a) Take care to strip any src/ directories from the head of \
+relative path names.
+  b) On desktop, only --browser=release is supported, on android \
+--browser=android-chromium-testshell.
+  c) Test command to use: %(command)s
+3. Upload your patch. --bypass-hooks is necessary to upload the changes you \
+committed locally to run-perf-test.cfg.
+   Note: *DO NOT* commit run-perf-test.cfg changes to the project repository.
+   $ git cl upload --bypass-hooks
+4. Send your try job to the tryserver. \
+[Please make sure to use appropriate bot to reproduce]
+   $ git cl try -m tryserver.chromium.perf -b <bot>
+
+For more details please visit \nhttps://sites.google.com/a/chromium.org/dev/\
+developers/performance-try-bots"""
+
+RESULTS_THANKYOU = """
+===== THANK YOU FOR CHOOSING BISECT AIRLINES =====

[Andrew Hayden (chromium.org), 2014/07/15 10:24:21]

For posterity, this made my perf-sheriffing day.

+Visit http://www.chromium.org/developers/core-principles for Chrome's policy
+on perf regressions.
+Contact chrome-perf-dashboard-team with any questions or suggestions about
+bisecting.
+                    .------.
+      .---.         \       \==)
+      |PERF\         \       \\
+      |     ---------'-------'-----------.
+      . 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 `-.
+      \______________.-------._______________)
+                    /       /
+                   /       /
+                  /       /==)
+                 ._______."""
+
 
 def _AddAdditionalDepotInfo(depot_info):
   """Adds additional depot info to the global depot variables."""
   global DEPOT_DEPS_NAME
   global DEPOT_NAMES
   DEPOT_DEPS_NAME = dict(DEPOT_DEPS_NAME.items() +
       depot_info.items())
   DEPOT_NAMES = DEPOT_DEPS_NAME.keys()
 
 
@@ skipping 3005 matching lines @@
                                   results_dict['last_broken_revision'],
                                   100, final_step=False)
 
   def _PrintConfidence(self, results_dict):
     # The perf dashboard specifically looks for the string
     # "Confidence in Bisection Results: 100%" to decide whether or not
     # to cc the author(s). If you change this, please update the perf
     # dashboard as well.
     print 'Confidence in Bisection Results: %d%%' % results_dict['confidence']
 
+  def _ConfidenceLevelStatus(self, results_dict):
+    if not results_dict['confidence']:
+      return None
+    confidence_status = 'Successful with %(level)s confidence%(warning)s.'
+    if results_dict['confidence'] >= 95:
+      level = 'high'
+    else:
+      level = 'low'
+    warning = ' and warnings'
+    if not self.warnings:
+      warning = ''
+    return confidence_status % {'level': level, 'warning': warning}
+
+  def _PrintThankYou(self):
+    print RESULTS_THANKYOU
+
   def _PrintBanner(self, results_dict):
-    print
-    print " __o_\___          Aw Snap! We hit a speed bump!"
-    print "=-O----O-'__.~.___________________________________"
-    print
     if self._IsBisectModeReturnCode():
-      print ('Bisect reproduced a change in return codes while running the '
-          'performance test.')
+      metrics = 'N/A'
+      change = 'Yes'
     else:
-      print ('Bisect reproduced a %.02f%% (+-%.02f%%) change in the '
-          '%s metric.' % (results_dict['regression_size'],
-          results_dict['regression_std_err'], '/'.join(self.opts.metric)))
-    self._PrintConfidence(results_dict)
+      metrics = '/'.join(self.opts.metric)
+      change = '%.02f%% (+/-%.02f%%)' % (
+          results_dict['regression_size'], results_dict['regression_std_err'])
+
+    if results_dict['culprit_revisions'] and results_dict['confidence']:
+      status = self._ConfidenceLevelStatus(results_dict)
+    else:
+      status = 'Failure, could not reproduce.'
+      change = 'Bisect could not reproduce a change.'
+
+    print RESULTS_BANNER % {
+        'status': status,
+        'command': self.opts.command,
+        'metrics': metrics,
+        'change': change,
+        'confidence': results_dict['confidence'],
+        }
+
 
   def _PrintFailedBanner(self, results_dict):
     print
     if self._IsBisectModeReturnCode():
       print 'Bisect could not reproduce a change in the return code.'
     else:
       print ('Bisect could not reproduce a change in the '
           '%s metric.' % '/'.join(self.opts.metric))
     print
 
   def _GetViewVCLinkFromDepotAndHash(self, cl, depot):
     info = self.source_control.QueryRevisionInfo(cl,
         self._GetDepotDirectory(depot))
     if depot and DEPOT_DEPS_NAME[depot].has_key('viewvc'):
       try:
         # Format is "git-svn-id: svn://....@123456 <other data>"
         svn_line = [i for i in info['body'].splitlines() if 'git-svn-id:' in i]
         svn_revision = svn_line[0].split('@')
         svn_revision = svn_revision[1].split(' ')[0]
         return DEPOT_DEPS_NAME[depot]['viewvc'] + svn_revision
       except IndexError:
         return ''
     return ''
 
   def _PrintRevisionInfo(self, cl, info, depot=None):
-    # The perf dashboard specifically looks for the string
-    # "Author  : " to parse out who to cc on a bug. If you change the
-    # formatting here, please update the perf dashboard as well.
-    print
-    print 'Subject : %s' % info['subject']
-    print 'Author  : %s' % info['author']
+    email_info = ''
     if not info['email'].startswith(info['author']):
-      print 'Email   : %s' % info['email']
+      email_info = '\nEmail   : %s' % info['email']
     commit_link = self._GetViewVCLinkFromDepotAndHash(cl, depot)
     if commit_link:
-      print 'Link    : %s' % commit_link
+      commit_info = '\nLink    : %s' % commit_link
     else:
-      print
-      print 'Failed to parse svn revision from body:'
-      print
-      print info['body']
-      print
-    print 'Commit  : %s' % cl
-    print 'Date    : %s' % info['date']
+      commit_info = ('\nFailed to parse svn revision from body:\n%s' %
+                     info['body'])
+    print RESULTS_REVISION_INFO % {
+        'subject': info['subject'],
+        'author': info['author'],
+        'email_info': email_info,
+        'commit_info': commit_info,
+        'cl_date': info['date']
+        }
 
   def _PrintTableRow(self, column_widths, row_data):
     assert len(column_widths) == len(row_data)
 
     text = ''
     for i in xrange(len(column_widths)):
       current_row_data = row_data[i].center(column_widths[i], ' ')
       text += ('%%%ds' % column_widths[i]) % current_row_data
     print text
 
@@ skipping 33 matching lines @@
       mean = '%d' % current_data['value']['mean']
       self._PrintTableRow(
           [20, 70, 14, 13],
           [current_data['depot'], cl_link, mean, state_str])
 
   def _PrintTestedCommitsTable(self, revision_data_sorted,
       first_working_revision, last_broken_revision, confidence,
       final_step=True):
     print
     if final_step:
-      print 'Tested commits:'
+      print '===== TESTED COMMITS ====='
     else:
-      print 'Partial results:'
+      print '===== PARTIAL RESULTS ====='
     self._PrintTestedCommitsHeader()
     state = 0
     for current_id, current_data in revision_data_sorted:
       if current_data['value']:
         if (current_id == last_broken_revision or
             current_id == first_working_revision):
           # If confidence is too low, don't add this empty line since it's
           # used to put focus on a suspected CL.
           if confidence and final_step:
             print
@@ skipping 13 matching lines @@
           state_str = ''
         state_str = state_str.center(13, ' ')
 
         cl_link = self._GetViewVCLinkFromDepotAndHash(current_id,
             current_data['depot'])
         if not cl_link:
           cl_link = current_id
         self._PrintTestedCommitsEntry(current_data, cl_link, state_str)
 
   def _PrintReproSteps(self):
-    print
-    print 'To reproduce locally:'
-    print '$ ' + self.opts.command
+    command = '$ ' + self.opts.command
     if bisect_utils.IsTelemetryCommand(self.opts.command):
-      print
-      print 'Also consider passing --profiler=list to see available profilers.'
+      command += ('\nAlso consider passing --profiler=list to see available '
+                  'profilers.')
+    print REPRO_STEPS_LOCAL % {'command': command}
+    print REPRO_STEPS_TRYJOB % {'command': command}
 
   def _PrintOtherRegressions(self, other_regressions, revision_data):
     print
     print 'Other regressions may have occurred:'
     print '  %8s  %70s  %10s' % ('Depot'.center(8, ' '),
         'Range'.center(70, ' '), 'Confidence'.center(10, ' '))
     for regression in other_regressions:
       current_id, previous_id, confidence = regression
       current_data = revision_data[current_id]
       previous_data = revision_data[previous_id]
@@ skipping 33 matching lines @@
         seconds=int(step_build_time_avg))
     print 'Average test time  : %s' % datetime.timedelta(
         seconds=int(step_perf_time_avg))
 
   def _PrintWarnings(self):
     if not self.warnings:
       return
     print
     print 'WARNINGS:'
     for w in set(self.warnings):
-      print '  !!! %s' % w
+      print '  ! %s' % w
 
   def _FindOtherRegressions(self, revision_data_sorted, bad_greater_than_good):
     other_regressions = []
     previous_values = []
     previous_id = None
     for current_id, current_data in revision_data_sorted:
       current_values = current_data['value']
       if current_values:
         current_values = current_values['values']
         if previous_values:
@@ skipping 180 matching lines @@
       print '  %20s  %40s  %s' % (current_data['depot'],
                                   current_id, build_status)
     print
 
     if self.opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
       # The perf dashboard scrapes the "results" step in order to comment on
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
 
+    self._PrintBanner(results_dict)
+    self._PrintWarnings()
+
     if results_dict['culprit_revisions'] and results_dict['confidence']:
-      self._PrintBanner(results_dict)
       for culprit in results_dict['culprit_revisions']:
         cl, info, depot = culprit
         self._PrintRevisionInfo(cl, info, depot)
-      self._PrintReproSteps()
       if results_dict['other_regressions']:
         self._PrintOtherRegressions(results_dict['other_regressions'],
                                     revision_data)
-    else:
-      self._PrintFailedBanner(results_dict)
-      self._PrintReproSteps()
-
     self._PrintTestedCommitsTable(revision_data_sorted,
                                   results_dict['first_working_revision'],
                                   results_dict['last_broken_revision'],
                                   results_dict['confidence'])
     self._PrintStepTime(revision_data_sorted)
-    self._PrintWarnings()
-
+    self._PrintReproSteps()
+    self._PrintThankYou()
     if self.opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
 
 
 def DetermineAndCreateSourceControl(opts):
   """Attempts to determine the underlying source control workflow and returns
   a SourceControl object.
 
   Returns:
     An instance of a SourceControl object, or None if the current workflow
@@ skipping 389 matching lines @@
       # The perf dashboard scrapes the "results" step in order to comment on
       # bugs. If you change this, please update the perf dashboard as well.
       bisect_utils.OutputAnnotationStepStart('Results')
     print 'Error: %s' % e.message
     if opts.output_buildbot_annotations:
       bisect_utils.OutputAnnotationStepClosed()
   return 1
 
 if __name__ == '__main__':
   sys.exit(main())