OAuth2 support in depot_tools

git_cl.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # Copyright (C) 2008 Evan Martin <martine@danga.com>
 
 """A git-command for integrating reviews on Rietveld."""
 
 from distutils.version import LooseVersion
 import base64
 import glob
+import httplib
 import json
 import logging
 import optparse
 import os
 import Queue
 import re
 import stat
 import sys
 import tempfile
 import textwrap
 import threading
 import urllib2
 import urlparse
 import webbrowser
 import zlib
 
 try:
   import readline  # pylint: disable=F0401,W0611
 except ImportError:
   pass
 
 
 from third_party import colorama
+from third_party import httplib2
 from third_party import upload
+from third_party.google_api_python_client import apiclient
 import breakpad  # pylint: disable=W0611
 import clang_format
 import dart_format
 import fix_encoding
 import gclient_utils
+import git_cl_oauth2
 import git_common
 import owners
 import owners_finder
 import presubmit_support
 import rietveld
 import scm
 import subcommand
 import subprocess2
 import watchlists
 
 __version__ = '1.0'
 
 DEFAULT_SERVER = 'https://codereview.appspot.com'
 POSTUPSTREAM_HOOK_PATTERN = '.git/hooks/post-cl-%s'
 DESCRIPTION_BACKUP_FILE = '~/.git_cl_description_backup'
 GIT_INSTRUCTIONS_URL = 'http://code.google.com/p/chromium/wiki/UsingGit'
 CHANGE_ID = 'Change-Id:'
 REFS_THAT_ALIAS_TO_OTHER_REFS = {
     'refs/remotes/origin/lkgr': 'refs/remotes/origin/master',
     'refs/remotes/origin/lkcr': 'refs/remotes/origin/master',
 }
 
+# Buildbucket-related constants
+DISCOVERY_URL = (
+    'https://cr-buildbucket.appspot.com/_ah/api/discovery/v1/apis/'
+    '{api}/{apiVersion}/rest')
+DEFAULT_SCOPES = 'email'
+BUILDSET_STR = 'patch/rietveld/{hostname}/{issue}/{patch}'
+
 # Valid extensions for files we want to lint.
 DEFAULT_LINT_REGEX = r"(.*\.cpp|.*\.cc|.*\.h)"
 DEFAULT_LINT_IGNORE_REGEX = r"$^"
 
 # Shortcut since it quickly becomes redundant.
 Fore = colorama.Fore
 
 # Initialized in main()
 settings = None
 
@@ skipping 134 matching lines @@
   if dirty:
     print 'Cannot %s with a dirty tree. You must commit locally first.' % cmd
     print 'Uncommitted files: (git diff-index --name-status HEAD)'
     print dirty[:4096]
     if len(dirty) > 4096:
       print '... (run "git diff-index --name-status HEAD" to see full output).'
     return True
   return False
 
 
+def _prefix_master(master):
+  prefix = 'master.'
+  if master.startswith(prefix):
+    return master
+  else:
+    return '%s%s' % (prefix, master)
+
+
+def _get_buildbucket(credentials):
+  http = httplib2.Http()
+  http = credentials.authorize(http)
+  return apiclient.discovery.build(
+      'buildbucket', 'v1',
+      http=http,
+      discoveryServiceUrl=DISCOVERY_URL,
+  )
+
+
+def trigger_distributed_try_jobs(

[nodir, 2015/03/05 03:58:55]

what distributed means in this context?

[sheyang, 2015/03/06 00:29:10]

Just keep the same name. 

I guess the original meaning is about the ability to schedule tryjobs across
masters.

+    credentials, changelist, options, masters, category):
+  buildbucket = _get_buildbucket(credentials)
+  cred_props = json.loads(credentials.to_json())
+  requester = cred_props['id_token']['email']
+  issue_props = changelist.GetIssueProperties()
+  rietveld_host = urlparse.urlparse(changelist.GetRietveldServer()).hostname
+  issue = changelist.GetIssue()
+  patchset = changelist.GetMostRecentPatchset()
+  buildset = BUILDSET_STR.format(
+      hostname=rietveld_host,
+      issue=str(issue),
+      patch=str(patchset))
+  print 'Tried jobs on:'
+  for (master, builders_and_tests) in masters.iteritems():
+    print 'Master: %s' % master
+    bb_master = _prefix_master(master)

[nodir, 2015/03/05 03:58:55]

call it bucket

[sheyang, 2015/03/06 00:29:10]

Done.

+    for builder, tests in builders_and_tests.iteritems():
+      req = buildbucket.put(body={
+          'bucket': bb_master,
+          'parameters_json': json.dumps({
+              'builder_name': builder,
+              'changes':[
+                  {'author': {'email': issue_props['owner_email']}},
+              ],
+              'properties': {
+                  'category': category,
+                  'clobber': options.clobber,
+                  'issue': issue,
+                  'master': master,
+                  'patch_project': issue_props['project'],
+                  'patch_storage': 'rietveld',
+                  'patchset': patchset,
+                  'reason': options.name,
+                  'requester': requester,
+                  'revision': options.revision,
+                  'rietveld': changelist.GetRietveldServer(),
+                  'testfilter': tests,
+              },
+          }),
+          'tags': ['buildset:%s' % buildset,
+                   'master:%s' % master,
+                   'builder:%s' % builder,
+                   'requester:%s' % requester]
+      })
+      req.execute()

[nodir, 2015/03/05 03:58:55]

You ignore the result that may contain an error. I'd check and report it.

+      print '  %s: %s' % (builder, tests)
+
+
 def MatchSvnGlob(url, base_url, glob_spec, allow_wildcards):
   """Return the corresponding git ref if |base_url| together with |glob_spec|
   matches the full |url|.
 
   If |allow_wildcards| is true, |glob_spec| can contain wildcards (see below).
   """
   fetch_suburl, as_ref = glob_spec.split(':')
   if allow_wildcards:
     glob_match = re.match('(.+/)?(\*|{[^/]*})(/.+)?', fetch_suburl)
     if glob_match:
@@ skipping 2468 matching lines @@
       "-c", "--clobber", action="store_true", default=False,
       help="Force a clobber before building; e.g. don't do an "
            "incremental build")
   group.add_option(
       "--project",
       help="Override which project to use. Projects are defined "
            "server-side to define what default bot set to use")
   group.add_option(
       "-n", "--name", help="Try job name; default to current branch name")
   parser.add_option_group(group)
+  git_cl_oauth2.add_oauth2_options(parser)
   options, args = parser.parse_args(args)
 
   if args:
     parser.error('Unknown arguments: %s' % args)
 
   cl = Changelist()
   if not cl.GetIssue():
     parser.error('Need to upload first')
 
   props = cl.GetIssueProperties()
@@ skipping 75 matching lines @@
           'Bot list: %s' % builders)
       return 1
 
   patchset = cl.GetMostRecentPatchset()
   if patchset and patchset != cl.GetPatchset():
     print(
         '\nWARNING Mismatch between local config and server. Did a previous '
         'upload fail?\ngit-cl try always uses latest patchset from rietveld. '
         'Continuing using\npatchset %s.\n' % patchset)
   try:
-    cl.RpcServer().trigger_distributed_try_jobs(
-        cl.GetIssue(), patchset, options.name, options.clobber,
-        options.revision, masters)
-  except urllib2.HTTPError, e:
-    if e.code == 404:
-      print('404 from rietveld; '
-            'did you mean to use "git try" instead of "git cl try"?')
-      return 1
-  print('Tried jobs on:')
-
-  for (master, builders) in masters.iteritems():
-    if master:
-      print 'Master: %s' % master
-    length = max(len(builder) for builder in builders)
-    for builder in sorted(builders):
-      print '  %*s: %s' % (length, builder, ','.join(builders[builder]))
+    creds = git_cl_oauth2.get_oauth2_cred(
+        options,
+        urlparse.urlparse(cl.GetRietveldServer()).hostname)
+    trigger_distributed_try_jobs(creds, cl, options, masters, 'git cl try')

[nodir, 2015/03/05 03:58:55]

Please do not commit this CL until we move/replicate try job access list to
chrome-infra-auth, otherwise git-cl-try will stop working for many

[sheyang, 2015/03/06 00:29:10]

I won't - see COMMIT=false in the description.

[nodir, 2015/03/10 22:39:39]

Acknowledged.

+  except apiclient.errors.HttpError as ex:
+    status = ex.resp.status if ex.resp else None
+    if status == httplib.FORBIDDEN:
+      print 'ERROR: Access denied. Please verify you have tryjob access.'
+    else:
+      print 'Tryjob request failed: %s.' % e

[nodir, 2015/03/05 03:58:55]

typo: ex

[sheyang, 2015/03/06 00:29:10]

Done.

+    return 1
+  except Exception as e:
+    print 'Unexcpected error when trying to trigger tryjobs: %s.' % e
+    return 1
   return 0
 
 
 @subcommand.usage('[new upstream branch]')
 def CMDupstream(parser, args):
   """Prints or sets the name of the upstream branch, if any."""
   _, args = parser.parse_args(args)
   if len(args) > 1:
     parser.error('Unrecognized args: %s' % ' '.join(args))
 
@@ skipping 291 matching lines @@
 if __name__ == '__main__':
   # These affect sys.stdout so do it outside of main() to simplify mocks in
   # unit testing.
   fix_encoding.fix_encoding()
   colorama.init()
   try:
     sys.exit(main(sys.argv[1:]))
   except KeyboardInterrupt:
     sys.stderr.write('interrupted\n')
     sys.exit(1)