git cl try uses buildbucket via OAuth2

git_cl.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Copyright (C) 2008 Evan Martin <martine@danga.com>
 
 """A git-command for integrating reviews on Rietveld."""
 
 from distutils.version import LooseVersion
 from multiprocessing.pool import ThreadPool
 import base64
 import glob
+import httplib
 import json
 import logging
 import optparse
 import os
 import Queue
 import re
 import stat
 import sys
 import tempfile
 import textwrap
+import time
+import traceback
 import urllib2
 import urlparse
 import webbrowser
 import zlib
 
 try:
   import readline  # pylint: disable=F0401,W0611
 except ImportError:
   pass
 
+from third_party import colorama
+from third_party import httplib2
+from third_party import upload
 
-from third_party import colorama
-from third_party import upload
+ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(
+    0, os.path.join(ROOT_DIR, 'third_party', 'google_api_python_client'))
+
+import apiclient
 import auth
 import breakpad  # pylint: disable=W0611
 import clang_format
 import dart_format
 import fix_encoding
 import gclient_utils
 import git_common
 import owners
 import owners_finder
 import presubmit_support
 import rietveld
 import scm
 import subcommand
 import subprocess2
 import watchlists
 
 __version__ = '1.0'
 
 DEFAULT_SERVER = 'https://codereview.appspot.com'
 POSTUPSTREAM_HOOK_PATTERN = '.git/hooks/post-cl-%s'
 DESCRIPTION_BACKUP_FILE = '~/.git_cl_description_backup'
 GIT_INSTRUCTIONS_URL = 'http://code.google.com/p/chromium/wiki/UsingGit'
 CHANGE_ID = 'Change-Id:'
 REFS_THAT_ALIAS_TO_OTHER_REFS = {
     'refs/remotes/origin/lkgr': 'refs/remotes/origin/master',
     'refs/remotes/origin/lkcr': 'refs/remotes/origin/master',
 }
 
+# Buildbucket-related constants
+DISCOVERY_URL = (
+    'https://cr-buildbucket.appspot.com/_ah/api/discovery/v1/apis/'
+    '{api}/{apiVersion}/rest')

[nodir, 2015/04/14 16:12:48]

Rename to BUILDBUCKET_DISCOVERY_URL

[sheyang, 2015/04/14 17:29:57]

Done.

+BUILDSET_STR = 'patch/rietveld/{hostname}/{issue}/{patch}'
+
 # Valid extensions for files we want to lint.
 DEFAULT_LINT_REGEX = r"(.*\.cpp|.*\.cc|.*\.h)"
 DEFAULT_LINT_IGNORE_REGEX = r"$^"
 
 # Shortcut since it quickly becomes redundant.
 Fore = colorama.Fore
 
 # Initialized in main()
 settings = None
 
@@ skipping 120 matching lines @@
     else:
       git_set_branch_value('git-find-copies', int(options.find_copies))
 
     print('Using %d%% similarity for rename/copy detection. '
           'Override with --similarity.' % options.similarity)
 
     return options, args
   parser.parse_args = Parse
 
 
+def _prefix_master(master):
+  prefix = 'master.'
+  if master.startswith(prefix):
+    return master
+  else:
+    return '%s%s' % (prefix, master)
+
+
+def _get_buildbucket(rietveld_host, auth_config):
+  authenticator = auth.get_authenticator_for_host(rietveld_host, auth_config)
+  http = httplib2.Http()
+  return apiclient.discovery.build(
+      'buildbucket', 'v1',
+      http=authenticator.authorize(http),
+      discoveryServiceUrl=DISCOVERY_URL,
+  )
+
+
+def trigger_distributed_try_jobs(
+    auth_config, changelist, options, masters, category):
+  rietveld_host = urlparse.urlparse(changelist.GetRietveldServer()).hostname

[nodir, 2015/04/14 16:12:48]

We will need to make sure that all branches on all machines of all chromium devs
have codereview.chromium.org value for `git config
branch.<branch>.rietveldserver` or `git config rietveld.server`. 

[sheyang, 2015/04/14 17:29:57]

As a fallback, rietveld_server will be fetched from codereview.settings, which
should always be available.

+  buildbucket = _get_buildbucket(rietveld_host, auth_config)
+  issue_props = changelist.GetIssueProperties()
+  issue = changelist.GetIssue()
+  patchset = changelist.GetMostRecentPatchset()
+  buildset = BUILDSET_STR.format(
+      hostname=rietveld_host,
+      issue=str(issue),
+      patch=str(patchset))

[nodir, 2015/04/14 16:12:48]

You don't need str calls here

[sheyang, 2015/04/14 17:29:57]

Done.

+  print 'Tried jobs on:'

[nodir, 2015/04/14 16:12:48]

"Trying", as they were not tried yet

[sheyang, 2015/04/14 17:29:57]

This is from the old code... Updated.

+  for (master, builders_and_tests) in masters.iteritems():

[nodir, 2015/04/14 16:12:48]

Parens are unnecessary

[sheyang, 2015/04/14 17:29:57]

Done.

+    print 'Master: %s' % master
+    bucket = _prefix_master(master)
+    for builder, tests in builders_and_tests.iteritems():
+      req = buildbucket.put(body={

[nodir, 2015/04/14 16:12:48]

In https://codereview.chromium.org/1058893004/ esprehn@ said we should send just
one request to schedule many builds. I will add a new endpoint "put_batch" that
schedules many builds
 

[sheyang, 2015/04/14 17:29:57]

Acknowledged.

+          'bucket': bucket,
+          'parameters_json': json.dumps({
+              'builder_name': builder,
+              'changes':[
+                  {'author': {'email': issue_props['owner_email']}},
+              ],
+              'properties': {
+                  'category': category,
+                  'clobber': options.clobber,
+                  'issue': issue,
+                  'master': master,
+                  'patch_project': issue_props['project'],
+                  'patch_storage': 'rietveld',
+                  'patchset': patchset,
+                  'reason': options.name,
+                  'revision': options.revision,
+                  'rietveld': changelist.GetRietveldServer(),

[nodir, 2015/04/14 16:12:48]

why not rietveld_host ?

[sheyang, 2015/04/14 17:29:57]

I think it needs 'https' prefix... adding rietveld_url.

+                  'testfilter': tests,
+              },
+          }),
+          'tags': ['buildset:%s' % buildset,
+                   'master:%s' % master,
+                   'builder:%s' % builder,
+                   'user_agent:git-cl-try']
+      })
+      wait = 1
+      try_count = 3
+      while try_count > 0:
+        try:
+          try_count -= 1
+          response = req.execute()
+          if response.get('error'):
+            msg = 'Error in response. Reason: %s. Message: %s.' % (
+                response['error'].get('reason', ''),
+                response['error'].get('message', ''))
+            raise BuildbucketResponseException(msg)
+          break
+        except apiclient.errors.HttpError as ex:
+          status = ex.resp.status if ex.resp else None
+          if status >= 500:

[nodir, 2015/04/14 16:12:48]

Inverse this condition so you can unindent the ">=500" part

[sheyang, 2015/04/14 17:29:57]

Done.

[sheyang, 2015/04/14 17:29:57]

Done.

+            logging.debug('Transient errors when triggering tryjobs. '
+                          'Will retry in %d seconds.', wait)
+            time.sleep(wait)
+            wait *= 2
+            if try_count <= 0:
+              raise

[nodir, 2015/04/14 16:12:48]

Move these two lines before logging about transient errors and waiting

[sheyang, 2015/04/14 17:29:57]

Done.

[sheyang, 2015/04/14 17:29:57]

Done.

+          else:
+            raise
+      print '  %s: %s' % (builder, tests)
+
+
 def MatchSvnGlob(url, base_url, glob_spec, allow_wildcards):
   """Return the corresponding git ref if |base_url| together with |glob_spec|
   matches the full |url|.
 
   If |allow_wildcards| is true, |glob_spec| can contain wildcards (see below).
   """
   fetch_suburl, as_ref = glob_spec.split(':')
   if allow_wildcards:
     glob_match = re.match('(.+/)?(\*|{[^/]*})(/.+)?', fetch_suburl)
     if glob_match:
@@ skipping 47 matching lines @@
   try:
     stdout = sys.stdout.fileno()
   except AttributeError:
     stdout = None
   return subprocess2.call(
       ['git',
        'diff', '--no-ext-diff', '--stat'] + similarity_options + args,
       stdout=stdout, env=env)
 
 
+class BuildbucketResponseException(Exception):
+  pass
+
+
 class Settings(object):
   def __init__(self):
     self.default_server = None
     self.cc = None
     self.root = None
     self.is_git_svn = None
     self.svn_branch = None
     self.tree_status_url = None
     self.viewvc_url = None
     self.updated = False
@@ skipping 2571 matching lines @@
           'Bot list: %s' % builders)
       return 1
 
   patchset = cl.GetMostRecentPatchset()
   if patchset and patchset != cl.GetPatchset():
     print(
         '\nWARNING Mismatch between local config and server. Did a previous '
         'upload fail?\ngit-cl try always uses latest patchset from rietveld. '
         'Continuing using\npatchset %s.\n' % patchset)
   try:
-    cl.RpcServer().trigger_distributed_try_jobs(
-        cl.GetIssue(), patchset, options.name, options.clobber,
-        options.revision, masters)
-  except urllib2.HTTPError, e:
-    if e.code == 404:
-      print('404 from rietveld; '
-            'did you mean to use "git try" instead of "git cl try"?')
-      return 1
-  print('Tried jobs on:')
-
-  for (master, builders) in masters.iteritems():
-    if master:
-      print 'Master: %s' % master
-    length = max(len(builder) for builder in builders)
-    for builder in sorted(builders):
-      print '  %*s: %s' % (length, builder, ','.join(builders[builder]))
+    trigger_distributed_try_jobs(
+        auth_config, cl, options, masters, 'git cl try')
+  except apiclient.errors.HttpError as ex:
+    status = ex.resp.status if ex.resp else None
+    if status == httplib.FORBIDDEN:
+      print 'ERROR: Access denied. Please verify you have tryjob access.'
+    else:
+      print 'ERROR: Tryjob request failed: %s.' % ex
+    return 1
+  except BuildbucketResponseException as ex:
+    print ex
+    return 1
+  except Exception as e:
+    stacktrace = (''.join(traceback.format_stack()) + traceback.format_exc())
+    print 'ERROR: unexpected error when trying to trigger tryjobs: %s\n%s' % (
+        e, stacktrace)

[nodir, 2015/04/14 16:12:48]

doesn't logging.exception do the same with current exception info (with trace)
for you?

[sheyang, 2015/04/14 17:29:57]

logging.exception only has traceback.format_exc().

+    return 1
   return 0
 
 
 @subcommand.usage('[new upstream branch]')
 def CMDupstream(parser, args):
   """Prints or sets the name of the upstream branch, if any."""
   _, args = parser.parse_args(args)
   if len(args) > 1:
     parser.error('Unrecognized args: %s' % ' '.join(args))
 
@@ skipping 313 matching lines @@
 if __name__ == '__main__':
   # These affect sys.stdout so do it outside of main() to simplify mocks in
   # unit testing.
   fix_encoding.fix_encoding()
   colorama.init()
   try:
     sys.exit(main(sys.argv[1:]))
   except KeyboardInterrupt:
     sys.stderr.write('interrupted\n')
     sys.exit(1)