Adding annotation output for trybot.

tools/bisect-perf-regression.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Performance Test Bisect Tool
 
 This script bisects a series of changelists using binary search. It starts at
 a bad revision where a performance metric has regressed, and asks for a last
 known-good revision. It will then binary search across this revision range by
@@ skipping 108 matching lines @@
     True if the string can be converted to an int.
   """
   try:
     int(string_to_check)
 
     return True
   except ValueError:
     return False
 
 
+def OutputAnnotationStepStart(name):
+  """Outputs appropriate annotation to signal the start of a step to
+  a trybot.
+
+  Args:
+    name: The name of the step.
+  """
+  print '@@@SEED_STEP %s@@@' % name
+  print '@@@STEP_CURSOR %s@@@' % name
+  print '@@@STEP_STARTED@@@'
+
+
+def OutputAnnotationStepClosed():
+  """Outputs appropriate annotation to signal the closing of a step to
+  a trybot."""
+  print '@@@STEP_CLOSED@@@'
+
+
 def RunProcess(command):
   """Run an arbitrary command, returning its output and return code.
 
   Args:
     command: A list containing the command and args to execute.
 
   Returns:
     A tuple of the output and return code.
   """
   # On Windows, use shell=True to get PATH interpretation.
@@ skipping 633 matching lines @@
     for i in xrange(num_depot_revisions):
       r = revisions[i]
 
       revision_data[r] = {'revision' : r,
                           'depot' : depot,
                           'value' : None,
                           'passed' : '?',
                           'sort' : i + sort + 1}
 
   def PrintRevisionsToBisectMessage(self, revision_list, depot):
+    if self.opts.output_annotations:
+      OutputAnnotationStepStart('Bisection Range [%s]' % depot)

[tonyg, 2013/02/14 00:17:47]

depot seems like the wrong thing to print as the bisection range. Shouldn't it
be the first and last elem in revision_list?

[shatch, 2013/02/14 01:16:01]

Done.

+
     print
     print 'Revisions to bisect on [%s]:' % depot
     for revision_id in revision_list:
       print '  -> %s' % (revision_id, )
     print
 
+    if self.opts.output_annotations:
+      OutputAnnotationStepClosed()
+
   def Run(self, command_to_run, bad_revision_in, good_revision_in, metric):
     """Given known good and bad revisions, run a binary search on all
     intermediate revisions to determine the CL where the performance regression
     occurred.
 
     Args:
         command_to_run: Specify the command to execute the performance test.
         good_revision: Number/tag of the known good revision.
         bad_revision: Number/tag of the known bad revision.
         metric: The performance metric to monitor.
@@ skipping 41 matching lines @@
                                                           'src')
 
     if bad_revision is None:
       results['error'] = 'Could\'t resolve [%s] to SHA1.' % (bad_revision_in,)
       return results
 
     if good_revision is None:
       results['error'] = 'Could\'t resolve [%s] to SHA1.' % (good_revision_in,)
       return results
 
+    if self.opts.output_annotations:
+      OutputAnnotationStepStart('Gathering Revisions')
+
     print 'Gathering revision range for bisection.'
 
     # Retrieve a list of revisions to do bisection on.
     src_revision_list = self.GetRevisionList(bad_revision, good_revision)
 
+    if self.opts.output_annotations:
+      OutputAnnotationStepClosed()
+
     if src_revision_list:
       # revision_data will store information about a revision such as the
       # depot it came from, the webkit/V8 revision at that time,
       # performance timing, build state, etc...
       revision_data = results['revision_data']
 
       # revision_list is the list we're binary searching through at the moment.
       revision_list = []
 
       sort_key_ids = 0
 
       for current_revision_id in src_revision_list:
         sort_key_ids += 1
 
         revision_data[current_revision_id] = {'value' : None,
                                               'passed' : '?',
                                               'depot' : 'chromium',
                                               'external' : None,
                                               'sort' : sort_key_ids}
         revision_list.append(current_revision_id)
 
       min_revision = 0
       max_revision = len(revision_list) - 1
 
       self.PrintRevisionsToBisectMessage(revision_list, 'src')
 
+      if self.opts.output_annotations:
+        OutputAnnotationStepStart('Gathering Reference Values')
+
       print 'Gathering reference values for bisection.'
 
       # Perform the performance tests on the good and bad revisions, to get
       # reference values.
       (bad_results, good_results) = self.GatherReferenceValues(good_revision,
                                                                bad_revision,
                                                                command_to_run,
                                                                metric)
 
+      if self.opts.output_annotations:
+        OutputAnnotationStepClosed()
+
       if bad_results[1]:
         results['error'] = bad_results[0]
         return results
 
       if good_results[1]:
         results['error'] = good_results[0]
         return results
 
 
       # We need these reference values to determine if later runs should be
@@ skipping 79 matching lines @@
         else:
           next_revision_index = int((max_revision - min_revision) / 2) +\
                                 min_revision
 
         next_revision_id = revision_list[next_revision_index]
         next_revision_data = revision_data[next_revision_id]
         next_revision_depot = next_revision_data['depot']
 
         self.ChangeToDepotWorkingDirectory(next_revision_depot)
 
+        if self.opts.output_annotations:
+          step_name = 'Working on [%s]' % next_revision_id
+          OutputAnnotationStepStart(step_name)
+
         print 'Working on revision: [%s]' % next_revision_id
 
         run_results = self.SyncBuildAndRunRevision(next_revision_id,
                                                    next_revision_depot,
                                                    command_to_run,
                                                    metric)
 
+        if self.opts.output_annotations:
+          OutputAnnotationStepClosed()
+
         # If the build is successful, check whether or not the metric
         # had regressed.
         if not run_results[1]:
           if next_revision_depot == 'chromium':
             next_revision_data['external'] = run_results[2]
 
           passed_regression = self.CheckIfRunPassed(run_results[0],
                                                     known_good_value,
                                                     known_bad_value)
 
@@ skipping 21 matching lines @@
   def FormatAndPrintResults(self, bisect_results):
     """Prints the results from a bisection run in a readable format.
 
     Args
       bisect_results: The results from a bisection test run.
     """
     revision_data = bisect_results['revision_data']
     revision_data_sorted = sorted(revision_data.iteritems(),
                                   key = lambda x: x[1]['sort'])
 
+    if self.opts.output_annotations:
+      OutputAnnotationStepStart('Results')
+
     print
     print 'Full results of bisection:'
     for current_id, current_data  in revision_data_sorted:
       build_status = current_data['passed']
 
       if type(build_status) is bool:
         build_status = int(build_status)
 
       print '  %8s  %s  %s' % (current_data['depot'], current_id, build_status)
     print
 
     # Find range where it possibly broke.
     first_working_revision = None
     last_broken_revision = None
 
     for k, v in revision_data_sorted:
       if v['passed']:
         if first_working_revision is None:
           first_working_revision = k
 
       if not v['passed']:
         last_broken_revision = k
 
     if last_broken_revision != None and first_working_revision != None:
-      print 'Results: Regression was detected as a result of changes on:'
+      print 'Results: Regression may have occurred in range:'
       print '  -> First Bad Revision: [%s] [%s]' %\
             (last_broken_revision,
             revision_data[last_broken_revision]['depot'])
       print '  -> Last Good Revision: [%s] [%s]' %\
             (first_working_revision,
             revision_data[first_working_revision]['depot'])
 
+    if self.opts.output_annotations:
+      OutputAnnotationStepClosed()
+
 
 def DetermineAndCreateSourceControl():
   """Attempts to determine the underlying source control workflow and returns
   a SourceControl object.
 
   Returns:
     An instance of a SourceControl object, or None if the current workflow
     is unsupported.
   """
 
@@ skipping 27 matching lines @@
                     help='A revision to start bisection where performance' +
                     ' test is known to pass. Must be earlier than the ' +
                     'bad revision. May be either a git or svn revision.')
   parser.add_option('-m', '--metric',
                     type='str',
                     help='The desired metric to bisect on. For example ' +
                     '"vm_rss_final_b/vm_rss_f_b"')
   parser.add_option('--use_goma',
                     action="store_true",
                     help='Add a bunch of extra threads for goma.')
+  parser.add_option('--output_annotations',

[tonyg, 2013/02/14 00:17:47]

I'd name this --buildbot_annotations or --output_buildbot_annotations

[shatch, 2013/02/14 01:16:01]

Done.

+                    action="store_true",
+                    help='Add extra annotation output for trybot.')
   parser.add_option('--debug_ignore_build',
                     action="store_true",
                     help='DEBUG: Don\'t perform builds.')
   parser.add_option('--debug_ignore_sync',
                     action="store_true",
                     help='DEBUG: Don\'t perform syncs.')
   parser.add_option('--debug_ignore_perf_test',
                     action="store_true",
                     help='DEBUG: Don\'t perform performance tests.')
   (opts, args) = parser.parse_args()
@@ skipping 60 matching lines @@
   if not(bisect_results['error']):
     bisect_test.FormatAndPrintResults(bisect_results)
     return 0
   else:
     print 'Error: ' + bisect_results['error']
     print
     return 1
 
 if __name__ == '__main__':
   sys.exit(main())