Add a --json option to git cl try-results.

git_cl.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Copyright (C) 2008 Evan Martin <martine@danga.com>
 
 """A git-command for integrating reviews on Rietveld and Gerrit."""
 
 from __future__ import print_function
@@ skipping 380 matching lines @@
       headers={'Content-Type': 'application/json'}
   )
   print_text.append('To see results here, run:        git cl try-results')
   print_text.append('To see results in browser, run:  git cl web')
   print('\n'.join(print_text))
 
 
 def fetch_try_jobs(auth_config, changelist, options):
   """Fetches try jobs from buildbucket.
 
-  Returns a map from build id to build info as json dictionary.
+  Returns a map from build id to build info as a dictionary.
   """
   rietveld_url = settings.GetDefaultServerUrl()
   rietveld_host = urlparse.urlparse(rietveld_url).hostname
   authenticator = auth.get_authenticator_for_host(rietveld_host, auth_config)
   if authenticator.has_cached_credentials():
     http = authenticator.authorize(httplib2.Http())
   else:
     print('Warning: Some results might be missing because %s' %
           # Get the message on how to login.
           (auth.LoginRequiredError(rietveld_host).message,))
@@ skipping 15 matching lines @@
     content = _buildbucket_retry('fetching try jobs', http, url, 'GET')
     for build in content.get('builds', []):
       builds[build['id']] = build
     if 'next_cursor' in content:
       params['start_cursor'] = content['next_cursor']
     else:
       break
   return builds
 
 
-def print_try_jobs(options, builds):
-  """Prints nicely result of fetch_try_jobs."""
-  if not builds:
-    print('No try jobs scheduled')
-    return
+def try_job_buckets(builds):
+  """Groups a set of try jobs into buckets for output.
 
+  Args:
+    builds: A list of dicts, one for each try job. This is the output
+        of fetch_try_jobs.
+
+  Returns:
+      A dict mapping categories of try jobs (e.g. successes, failures,
+      infra failures, etc.) to lists of dicts, one for each try job.
+      The information here is a subset of the information in the original
+      input build dicts.
+  """
   # Make a copy, because we'll be modifying builds dictionary.
   builds = builds.copy()
   builder_names_cache = {}
 
-  def get_builder(b):
+  def get_builder_name(b):
     try:
       return builder_names_cache[b['id']]
     except KeyError:
       try:
         parameters = json.loads(b['parameters_json'])
         name = parameters['builder_name']
       except (ValueError, KeyError) as error:
-        print('WARNING: failed to get builder name for build %s: %s' % (
-              b['id'], error))
+        print('WARNING: failed to get builder name for build %s: %s' %
+              (b['id'], error))
         name = None
       builder_names_cache[b['id']] = name
       return name
 
-  def get_bucket(b):
+  def get_master_name(b):
     bucket = b['bucket']
     if bucket.startswith('master.'):
       return bucket[len('master.'):]
     return bucket
 
-  if options.print_master:
-    name_fmt = '%%-%ds %%-%ds' % (
-        max(len(str(get_bucket(b))) for b in builds.itervalues()),
-        max(len(str(get_builder(b))) for b in builds.itervalues()))
-    def get_name(b):
-      return name_fmt % (get_bucket(b), get_builder(b))
-  else:
-    name_fmt = '%%-%ds' % (
-        max(len(str(get_builder(b))) for b in builds.itervalues()))
-    def get_name(b):
-      return name_fmt % get_builder(b)
+  def sort_key(b):
+    return b['status'], b.get('result'), get_builder_name(b), b.get('url')
 
-  def sort_key(b):
-    return b['status'], b.get('result'), get_name(b), b.get('url')
+  buckets = collections.OrderedDict()
 
-  def pop(title, f, color=None, **kwargs):
-    """Pop matching builds from `builds` dict and print them."""
-
-    if not options.color or color is None:
-      colorize = str
-    else:
-      colorize = lambda x: '%s%s%s' % (color, x, Fore.RESET)
-
+  def pop(title, **kwargs):
+    """Pops matching builds from `builds` and adds it to `buckets`."""
     result = []
     for b in builds.values():
       if all(b.get(k) == v for k, v in kwargs.iteritems()):
         builds.pop(b['id'])
         result.append(b)
     if result:
-      print(colorize(title))
+      buckets[title] = []
       for b in sorted(result, key=sort_key):
-        print(' ', colorize('\t'.join(map(str, f(b)))))
+        buckets[title].append({
+          'builder_name': get_builder_name(b),
+          'master_name': get_master_name(b),
+          'url': b.get('url'),
+          'buildbucket_id': b.get('id'),
+          'status': b.get('status'),
+          'result': b.get('result'),
+          'failure_reason': b.get('failure_reason'),
+        })
 
-  total = len(builds)
-  pop(status='COMPLETED', result='SUCCESS',
-      title='Successes:', color=Fore.GREEN,
-      f=lambda b: (get_name(b), b.get('url')))
-  pop(status='COMPLETED', result='FAILURE', failure_reason='INFRA_FAILURE',
-      title='Infra Failures:', color=Fore.MAGENTA,
-      f=lambda b: (get_name(b), b.get('url')))
-  pop(status='COMPLETED', result='FAILURE', failure_reason='BUILD_FAILURE',
-      title='Failures:', color=Fore.RED,
-      f=lambda b: (get_name(b), b.get('url')))
-  pop(status='COMPLETED', result='CANCELED',
-      title='Canceled:', color=Fore.MAGENTA,
-      f=lambda b: (get_name(b),))
-  pop(status='COMPLETED', result='FAILURE',
-      failure_reason='INVALID_BUILD_DEFINITION',
-      title='Wrong master/builder name:', color=Fore.MAGENTA,
-      f=lambda b: (get_name(b),))
-  pop(status='COMPLETED', result='FAILURE',
-      title='Other failures:',
-      f=lambda b: (get_name(b), b.get('failure_reason'), b.get('url')))
-  pop(status='COMPLETED',
-      title='Other finished:',
-      f=lambda b: (get_name(b), b.get('result'), b.get('url')))
-  pop(status='STARTED',
-      title='Started:', color=Fore.YELLOW,
-      f=lambda b: (get_name(b), b.get('url')))
-  pop(status='SCHEDULED',
-      title='Scheduled:',
-      f=lambda b: (get_name(b), 'id=%s' % b['id']))
+  pop('Successes', status='COMPLETED', result='SUCCESS')
+  pop('Infra Failures', status='COMPLETED', result='FAILURE',
+      failure_reason='INFRA_FAILURE')
+  pop('Failures', status='COMPLETED', result='FAILURE',
+      failure_reason='BUILD_FAILURE')
+  pop('Canceled', status='COMPLETED', result='CANCELED')
+  pop('Wrong master/builder name', status='COMPLETED', result='FAILURE',
+      failure_reason='INVALID_BUILD_DEFINITION')
+  pop('Other failures', status='COMPLETED', result='FAILURE')
+  pop('Other finished', status='COMPLETED')
+  pop('Started', status='STARTED')
+  pop('Scheduled', status='SCHEDULED')
   # The last section is just in case buildbucket API changes OR there is a bug.
-  pop(title='Other:',
-      f=lambda b: (get_name(b), 'id=%s' % b['id']))
+  pop('Other')
   assert len(builds) == 0
-  print('Total: %d try jobs' % total)
+  return buckets
+
+
+def print_try_jobs(options, builds):
+  """Nicely prints the result of fetch_try_jobs."""
+  if not builds:
+    print('No try jobs scheduled')
+    return
+
+  buckets = try_job_buckets(builds)
+
+  def color_for_bucket(title):
+    return {
+      'Successes': Fore.GREEN,
+      'Infra Failures': Fore.MAGENTA,
+      'Failures': Fore.RED,
+      'Canceled': Fore.MAGENTA,
+      'Wrong master/builder name': Fore.MAGENTA,
+      'Started:': Fore.YELLOW,
+    }.get(title)
+
+  def extra_columns_for_bucket(title):
+    return {
+      'Successes': ['url'],
+      'Infra Failures': ['url'],
+      'Failures': ['url'],
+      'Canceled': [],
+      'Wrong master/builder name': ['builder_name'],
+      'Other failures': ['failure_reason', 'url'],
+      'Other finished': ['result', 'url'],
+      'Started': ['url'],
+      'Scheduled': ['buildbucket_id'],
+      'Other': ['buildbucket_id'],
+    }.get(title)
+
+  for title, builds in buckets.iteritems():
+    if not builds:
+      continue
+    color = color_for_bucket(title)
+    if not options.color or color is None:
+      colorize = str
+    else:
+      colorize = lambda x: '%s%s%s' % (color, x, Fore.RESET)
+    print(colorize(title + ':'))
+    for build in builds:
+      row = [build['builder_name']]
+      # TODO: Make sure the spacing is right when master name is printed.
+      if options.print_master:
+        row.append([build['master_name']])
+      for column in extra_columns_for_bucket(title):
+        if column == 'id':
+          row.append('id=%s' % build['id'])
+        else:
+          row.append(build[column])
+      print(' ', colorize('\t'.join(row)))
+  print('Total: %d try jobs' % sum(len(b) for b in buckets))
+
+
+def write_try_results_json(output_file, builds):
+  """This outputs results from fetch_try_jobs as JSON.
+
+  Args:
+    builds: Results fetched from fetch_try_jobs. This is a list of dicts.
+
+  Returns:
+    A dict mapping "status buckets" to lists of builds.
+  """
+  write_json(output_file + '.full', builds)
+  buckets = try_job_buckets(builds)
+  write_json(output_file, buckets)
 
 
 def MatchSvnGlob(url, base_url, glob_spec, allow_wildcards):
   """Return the corresponding git ref if |base_url| together with |glob_spec|
   matches the full |url|.
 
   If |allow_wildcards| is true, |glob_spec| can contain wildcards (see below).
   """
   fetch_suburl, as_ref = glob_spec.split(':')
   if allow_wildcards:
@@ skipping 4206 matching lines @@
   group.add_option(
       "-p", "--patchset", type=int, help="patchset number if not current.")
   group.add_option(
       "--print-master", action='store_true', help="print master name as well.")
   group.add_option(
       "--color", action='store_true', default=setup_color.IS_TTY,
       help="force color output, useful when piping output.")
   group.add_option(
       "--buildbucket-host", default='cr-buildbucket.appspot.com',
       help="Host of buildbucket. The default host is %default.")
+  group.add_option(
+      '--json', help='Path of JSON output file to write try job results to.')
   parser.add_option_group(group)
   auth.add_auth_options(parser)
   options, args = parser.parse_args(args)
   if args:
     parser.error('Unrecognized args: %s' % ' '.join(args))
 
   auth_config = auth.extract_auth_config_from_options(options)
   cl = Changelist(auth_config=auth_config)
   if not cl.GetIssue():
     parser.error('Need to upload first')
 
   if not options.patchset:
     options.patchset = cl.GetMostRecentPatchset()
     if options.patchset and options.patchset != cl.GetPatchset():
       print(
           '\nWARNING Mismatch between local config and server. Did a previous '
           'upload fail?\ngit-cl try always uses latest patchset from rietveld. '
           'Continuing using\npatchset %s.\n' % options.patchset)
   try:
     jobs = fetch_try_jobs(auth_config, cl, options)
   except BuildbucketResponseException as ex:
     print('Buildbucket error: %s' % ex)
     return 1
   except Exception as e:
     stacktrace = (''.join(traceback.format_stack()) + traceback.format_exc())
     print('ERROR: Exception when trying to fetch try jobs: %s\n%s' %
           (e, stacktrace))
     return 1
-  print_try_jobs(options, jobs)
+  if options.json:
+    write_try_results_json(options.json, jobs)
+  else:
+    print_try_jobs(options, jobs)
   return 0
 
 
 @subcommand.usage('[new upstream branch]')
 def CMDupstream(parser, args):
   """Prints or sets the name of the upstream branch, if any."""
   _, args = parser.parse_args(args)
   if len(args) > 1:
     parser.error('Unrecognized args: %s' % ' '.join(args))
 
@@ skipping 419 matching lines @@
 if __name__ == '__main__':
   # These affect sys.stdout so do it outside of main() to simplify mocks in
   # unit testing.
   fix_encoding.fix_encoding()
   setup_color.init()
   try:
     sys.exit(main(sys.argv[1:]))
   except KeyboardInterrupt:
     sys.stderr.write('interrupted\n')
     sys.exit(1)