OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 # Copyright (C) 2008 Evan Martin <martine@danga.com> | 6 # Copyright (C) 2008 Evan Martin <martine@danga.com> |
7 | 7 |
8 """A git-command for integrating reviews on Rietveld.""" | 8 """A git-command for integrating reviews on Rietveld.""" |
9 | 9 |
10 from distutils.version import LooseVersion | 10 from distutils.version import LooseVersion |
11 from multiprocessing.pool import ThreadPool | 11 from multiprocessing.pool import ThreadPool |
12 import base64 | 12 import base64 |
13 import glob | 13 import glob |
| 14 import httplib |
14 import json | 15 import json |
15 import logging | 16 import logging |
16 import optparse | 17 import optparse |
17 import os | 18 import os |
18 import Queue | 19 import Queue |
19 import re | 20 import re |
20 import stat | 21 import stat |
21 import sys | 22 import sys |
22 import tempfile | 23 import tempfile |
23 import textwrap | 24 import textwrap |
| 25 import time |
| 26 import traceback |
24 import urllib2 | 27 import urllib2 |
25 import urlparse | 28 import urlparse |
26 import webbrowser | 29 import webbrowser |
27 import zlib | 30 import zlib |
28 | 31 |
29 try: | 32 try: |
30 import readline # pylint: disable=F0401,W0611 | 33 import readline # pylint: disable=F0401,W0611 |
31 except ImportError: | 34 except ImportError: |
32 pass | 35 pass |
33 | 36 |
34 | |
35 from third_party import colorama | 37 from third_party import colorama |
| 38 from third_party import httplib2 |
36 from third_party import upload | 39 from third_party import upload |
37 import auth | 40 import auth |
38 import breakpad # pylint: disable=W0611 | 41 import breakpad # pylint: disable=W0611 |
39 import clang_format | 42 import clang_format |
40 import dart_format | 43 import dart_format |
41 import fix_encoding | 44 import fix_encoding |
42 import gclient_utils | 45 import gclient_utils |
43 import git_common | 46 import git_common |
44 import owners | 47 import owners |
45 import owners_finder | 48 import owners_finder |
46 import presubmit_support | 49 import presubmit_support |
47 import rietveld | 50 import rietveld |
48 import scm | 51 import scm |
49 import subcommand | 52 import subcommand |
50 import subprocess2 | 53 import subprocess2 |
51 import watchlists | 54 import watchlists |
52 | 55 |
53 __version__ = '1.0' | 56 __version__ = '1.0' |
54 | 57 |
55 DEFAULT_SERVER = 'https://codereview.appspot.com' | 58 DEFAULT_SERVER = 'https://codereview.appspot.com' |
56 POSTUPSTREAM_HOOK_PATTERN = '.git/hooks/post-cl-%s' | 59 POSTUPSTREAM_HOOK_PATTERN = '.git/hooks/post-cl-%s' |
57 DESCRIPTION_BACKUP_FILE = '~/.git_cl_description_backup' | 60 DESCRIPTION_BACKUP_FILE = '~/.git_cl_description_backup' |
58 GIT_INSTRUCTIONS_URL = 'http://code.google.com/p/chromium/wiki/UsingGit' | 61 GIT_INSTRUCTIONS_URL = 'http://code.google.com/p/chromium/wiki/UsingGit' |
59 CHANGE_ID = 'Change-Id:' | 62 CHANGE_ID = 'Change-Id:' |
60 REFS_THAT_ALIAS_TO_OTHER_REFS = { | 63 REFS_THAT_ALIAS_TO_OTHER_REFS = { |
61 'refs/remotes/origin/lkgr': 'refs/remotes/origin/master', | 64 'refs/remotes/origin/lkgr': 'refs/remotes/origin/master', |
62 'refs/remotes/origin/lkcr': 'refs/remotes/origin/master', | 65 'refs/remotes/origin/lkcr': 'refs/remotes/origin/master', |
63 } | 66 } |
64 | 67 |
| 68 # Buildbucket-related constants |
| 69 BUILDBUCKET_HOST = 'cr-buildbucket.appspot.com' |
| 70 |
65 # Valid extensions for files we want to lint. | 71 # Valid extensions for files we want to lint. |
66 DEFAULT_LINT_REGEX = r"(.*\.cpp|.*\.cc|.*\.h)" | 72 DEFAULT_LINT_REGEX = r"(.*\.cpp|.*\.cc|.*\.h)" |
67 DEFAULT_LINT_IGNORE_REGEX = r"$^" | 73 DEFAULT_LINT_IGNORE_REGEX = r"$^" |
68 | 74 |
69 # Shortcut since it quickly becomes redundant. | 75 # Shortcut since it quickly becomes redundant. |
70 Fore = colorama.Fore | 76 Fore = colorama.Fore |
71 | 77 |
72 # Initialized in main() | 78 # Initialized in main() |
73 settings = None | 79 settings = None |
74 | 80 |
(...skipping 120 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
195 else: | 201 else: |
196 git_set_branch_value('git-find-copies', int(options.find_copies)) | 202 git_set_branch_value('git-find-copies', int(options.find_copies)) |
197 | 203 |
198 print('Using %d%% similarity for rename/copy detection. ' | 204 print('Using %d%% similarity for rename/copy detection. ' |
199 'Override with --similarity.' % options.similarity) | 205 'Override with --similarity.' % options.similarity) |
200 | 206 |
201 return options, args | 207 return options, args |
202 parser.parse_args = Parse | 208 parser.parse_args = Parse |
203 | 209 |
204 | 210 |
| 211 def _prefix_master(master): |
| 212 """Convert user-specified master name to full master name. |
| 213 |
| 214 Buildbucket uses full master name(master.tryserver.chromium.linux) as bucket |
| 215 name, while the developers always use shortened master name |
| 216 (tryserver.chromium.linux) by stripping off the prefix 'master.'. This |
| 217 function does the conversion for buildbucket migration. |
| 218 """ |
| 219 prefix = 'master.' |
| 220 if master.startswith(prefix): |
| 221 return master |
| 222 return '%s%s' % (prefix, master) |
| 223 |
| 224 |
| 225 def trigger_try_jobs(auth_config, changelist, options, masters, category): |
| 226 rietveld_url = settings.GetDefaultServerUrl() |
| 227 rietveld_host = urlparse.urlparse(rietveld_url).hostname |
| 228 authenticator = auth.get_authenticator_for_host(rietveld_host, auth_config) |
| 229 http = authenticator.authorize(httplib2.Http()) |
| 230 http.force_exception_to_status_code = True |
| 231 issue_props = changelist.GetIssueProperties() |
| 232 issue = changelist.GetIssue() |
| 233 patchset = changelist.GetMostRecentPatchset() |
| 234 |
| 235 buildbucket_put_url = ( |
| 236 'https://{hostname}/_ah/api/buildbucket/v1/builds/batch'.format( |
| 237 hostname=BUILDBUCKET_HOST)) |
| 238 buildset = 'patch/rietveld/{hostname}/{issue}/{patch}'.format( |
| 239 hostname=rietveld_host, |
| 240 issue=issue, |
| 241 patch=patchset) |
| 242 |
| 243 batch_req_body = {'builds': []} |
| 244 print_text = [] |
| 245 print_text.append('Tried jobs on:') |
| 246 for master, builders_and_tests in sorted(masters.iteritems()): |
| 247 print_text.append('Master: %s' % master) |
| 248 bucket = _prefix_master(master) |
| 249 for builder, tests in sorted(builders_and_tests.iteritems()): |
| 250 print_text.append(' %s: %s' % (builder, tests)) |
| 251 parameters = { |
| 252 'builder_name': builder, |
| 253 'changes': [ |
| 254 {'author': {'email': issue_props['owner_email']}}, |
| 255 ], |
| 256 'properties': { |
| 257 'category': category, |
| 258 'issue': issue, |
| 259 'master': master, |
| 260 'patch_project': issue_props['project'], |
| 261 'patch_storage': 'rietveld', |
| 262 'patchset': patchset, |
| 263 'reason': options.name, |
| 264 'revision': options.revision, |
| 265 'rietveld': rietveld_url, |
| 266 'testfilter': tests, |
| 267 }, |
| 268 } |
| 269 if options.clobber: |
| 270 parameters['properties']['clobber'] = True |
| 271 batch_req_body['builds'].append( |
| 272 { |
| 273 'bucket': bucket, |
| 274 'parameters_json': json.dumps(parameters), |
| 275 'tags': ['builder:%s' % builder, |
| 276 'buildset:%s' % buildset, |
| 277 'master:%s' % master, |
| 278 'user_agent:git_cl_try'] |
| 279 } |
| 280 ) |
| 281 |
| 282 for try_count in xrange(3): |
| 283 response, content = http.request( |
| 284 buildbucket_put_url, |
| 285 'PUT', |
| 286 body=json.dumps(batch_req_body), |
| 287 headers={'Content-Type': 'application/json'}, |
| 288 ) |
| 289 content_json = None |
| 290 try: |
| 291 content_json = json.loads(content) |
| 292 except ValueError: |
| 293 pass |
| 294 |
| 295 # Buildbucket could return an error even if status==200. |
| 296 if content_json and content_json.get('error'): |
| 297 msg = 'Error in response. Code: %d. Reason: %s. Message: %s.' % ( |
| 298 content_json['error'].get('code', ''), |
| 299 content_json['error'].get('reason', ''), |
| 300 content_json['error'].get('message', '')) |
| 301 raise BuildbucketResponseException(msg) |
| 302 |
| 303 if response.status == 200: |
| 304 if not content_json: |
| 305 raise BuildbucketResponseException( |
| 306 'Buildbucket returns invalid json content: %s.\n' |
| 307 'Please file bugs at crbug.com, label "Infra-BuildBucket".' % |
| 308 content) |
| 309 break |
| 310 if response.status < 500 or try_count >= 2: |
| 311 raise httplib2.HttpLib2Error(content) |
| 312 |
| 313 # status >= 500 means transient failures. |
| 314 logging.debug('Transient errors when triggering tryjobs. Will retry.') |
| 315 time.sleep(0.5 + 1.5*try_count) |
| 316 |
| 317 print '\n'.join(print_text) |
| 318 |
| 319 |
205 def MatchSvnGlob(url, base_url, glob_spec, allow_wildcards): | 320 def MatchSvnGlob(url, base_url, glob_spec, allow_wildcards): |
206 """Return the corresponding git ref if |base_url| together with |glob_spec| | 321 """Return the corresponding git ref if |base_url| together with |glob_spec| |
207 matches the full |url|. | 322 matches the full |url|. |
208 | 323 |
209 If |allow_wildcards| is true, |glob_spec| can contain wildcards (see below). | 324 If |allow_wildcards| is true, |glob_spec| can contain wildcards (see below). |
210 """ | 325 """ |
211 fetch_suburl, as_ref = glob_spec.split(':') | 326 fetch_suburl, as_ref = glob_spec.split(':') |
212 if allow_wildcards: | 327 if allow_wildcards: |
213 glob_match = re.match('(.+/)?(\*|{[^/]*})(/.+)?', fetch_suburl) | 328 glob_match = re.match('(.+/)?(\*|{[^/]*})(/.+)?', fetch_suburl) |
214 if glob_match: | 329 if glob_match: |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
262 try: | 377 try: |
263 stdout = sys.stdout.fileno() | 378 stdout = sys.stdout.fileno() |
264 except AttributeError: | 379 except AttributeError: |
265 stdout = None | 380 stdout = None |
266 return subprocess2.call( | 381 return subprocess2.call( |
267 ['git', | 382 ['git', |
268 'diff', '--no-ext-diff', '--stat'] + similarity_options + args, | 383 'diff', '--no-ext-diff', '--stat'] + similarity_options + args, |
269 stdout=stdout, env=env) | 384 stdout=stdout, env=env) |
270 | 385 |
271 | 386 |
| 387 class BuildbucketResponseException(Exception): |
| 388 pass |
| 389 |
| 390 |
272 class Settings(object): | 391 class Settings(object): |
273 def __init__(self): | 392 def __init__(self): |
274 self.default_server = None | 393 self.default_server = None |
275 self.cc = None | 394 self.cc = None |
276 self.root = None | 395 self.root = None |
277 self.is_git_svn = None | 396 self.is_git_svn = None |
278 self.svn_branch = None | 397 self.svn_branch = None |
279 self.tree_status_url = None | 398 self.tree_status_url = None |
280 self.viewvc_url = None | 399 self.viewvc_url = None |
281 self.updated = False | 400 self.updated = False |
(...skipping 2475 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2757 group.add_option( | 2876 group.add_option( |
2758 "-c", "--clobber", action="store_true", default=False, | 2877 "-c", "--clobber", action="store_true", default=False, |
2759 help="Force a clobber before building; e.g. don't do an " | 2878 help="Force a clobber before building; e.g. don't do an " |
2760 "incremental build") | 2879 "incremental build") |
2761 group.add_option( | 2880 group.add_option( |
2762 "--project", | 2881 "--project", |
2763 help="Override which project to use. Projects are defined " | 2882 help="Override which project to use. Projects are defined " |
2764 "server-side to define what default bot set to use") | 2883 "server-side to define what default bot set to use") |
2765 group.add_option( | 2884 group.add_option( |
2766 "-n", "--name", help="Try job name; default to current branch name") | 2885 "-n", "--name", help="Try job name; default to current branch name") |
| 2886 group.add_option( |
| 2887 "--use-buildbucket", action="store_true", default=False, |
| 2888 help="Use buildbucket to trigger try jobs.") |
2767 parser.add_option_group(group) | 2889 parser.add_option_group(group) |
2768 auth.add_auth_options(parser) | 2890 auth.add_auth_options(parser) |
2769 options, args = parser.parse_args(args) | 2891 options, args = parser.parse_args(args) |
2770 auth_config = auth.extract_auth_config_from_options(options) | 2892 auth_config = auth.extract_auth_config_from_options(options) |
2771 | 2893 |
2772 if args: | 2894 if args: |
2773 parser.error('Unknown arguments: %s' % args) | 2895 parser.error('Unknown arguments: %s' % args) |
2774 | 2896 |
2775 cl = Changelist(auth_config=auth_config) | 2897 cl = Changelist(auth_config=auth_config) |
2776 if not cl.GetIssue(): | 2898 if not cl.GetIssue(): |
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2854 'Instead send your job to the parent.\n' | 2976 'Instead send your job to the parent.\n' |
2855 'Bot list: %s' % builders) | 2977 'Bot list: %s' % builders) |
2856 return 1 | 2978 return 1 |
2857 | 2979 |
2858 patchset = cl.GetMostRecentPatchset() | 2980 patchset = cl.GetMostRecentPatchset() |
2859 if patchset and patchset != cl.GetPatchset(): | 2981 if patchset and patchset != cl.GetPatchset(): |
2860 print( | 2982 print( |
2861 '\nWARNING Mismatch between local config and server. Did a previous ' | 2983 '\nWARNING Mismatch between local config and server. Did a previous ' |
2862 'upload fail?\ngit-cl try always uses latest patchset from rietveld. ' | 2984 'upload fail?\ngit-cl try always uses latest patchset from rietveld. ' |
2863 'Continuing using\npatchset %s.\n' % patchset) | 2985 'Continuing using\npatchset %s.\n' % patchset) |
2864 try: | 2986 if options.use_buildbucket: |
2865 cl.RpcServer().trigger_distributed_try_jobs( | 2987 try: |
2866 cl.GetIssue(), patchset, options.name, options.clobber, | 2988 trigger_try_jobs(auth_config, cl, options, masters, 'git_cl_try') |
2867 options.revision, masters) | 2989 except BuildbucketResponseException as ex: |
2868 except urllib2.HTTPError, e: | 2990 print 'ERROR: %s' % ex |
2869 if e.code == 404: | |
2870 print('404 from rietveld; ' | |
2871 'did you mean to use "git try" instead of "git cl try"?') | |
2872 return 1 | 2991 return 1 |
2873 print('Tried jobs on:') | 2992 except Exception as e: |
| 2993 stacktrace = (''.join(traceback.format_stack()) + traceback.format_exc()) |
| 2994 print 'ERROR: Exception when trying to trigger tryjobs: %s\n%s' % ( |
| 2995 e, stacktrace) |
| 2996 return 1 |
| 2997 else: |
| 2998 try: |
| 2999 cl.RpcServer().trigger_distributed_try_jobs( |
| 3000 cl.GetIssue(), patchset, options.name, options.clobber, |
| 3001 options.revision, masters) |
| 3002 except urllib2.HTTPError as e: |
| 3003 if e.code == 404: |
| 3004 print('404 from rietveld; ' |
| 3005 'did you mean to use "git try" instead of "git cl try"?') |
| 3006 return 1 |
| 3007 print('Tried jobs on:') |
2874 | 3008 |
2875 for (master, builders) in masters.iteritems(): | 3009 for (master, builders) in sorted(masters.iteritems()): |
2876 if master: | 3010 if master: |
2877 print 'Master: %s' % master | 3011 print 'Master: %s' % master |
2878 length = max(len(builder) for builder in builders) | 3012 length = max(len(builder) for builder in builders) |
2879 for builder in sorted(builders): | 3013 for builder in sorted(builders): |
2880 print ' %*s: %s' % (length, builder, ','.join(builders[builder])) | 3014 print ' %*s: %s' % (length, builder, ','.join(builders[builder])) |
2881 return 0 | 3015 return 0 |
2882 | 3016 |
2883 | 3017 |
2884 @subcommand.usage('[new upstream branch]') | 3018 @subcommand.usage('[new upstream branch]') |
2885 def CMDupstream(parser, args): | 3019 def CMDupstream(parser, args): |
2886 """Prints or sets the name of the upstream branch, if any.""" | 3020 """Prints or sets the name of the upstream branch, if any.""" |
2887 _, args = parser.parse_args(args) | 3021 _, args = parser.parse_args(args) |
2888 if len(args) > 1: | 3022 if len(args) > 1: |
2889 parser.error('Unrecognized args: %s' % ' '.join(args)) | 3023 parser.error('Unrecognized args: %s' % ' '.join(args)) |
2890 | 3024 |
(...skipping 314 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3205 if __name__ == '__main__': | 3339 if __name__ == '__main__': |
3206 # These affect sys.stdout so do it outside of main() to simplify mocks in | 3340 # These affect sys.stdout so do it outside of main() to simplify mocks in |
3207 # unit testing. | 3341 # unit testing. |
3208 fix_encoding.fix_encoding() | 3342 fix_encoding.fix_encoding() |
3209 colorama.init() | 3343 colorama.init() |
3210 try: | 3344 try: |
3211 sys.exit(main(sys.argv[1:])) | 3345 sys.exit(main(sys.argv[1:])) |
3212 except KeyboardInterrupt: | 3346 except KeyboardInterrupt: |
3213 sys.stderr.write('interrupted\n') | 3347 sys.stderr.write('interrupted\n') |
3214 sys.exit(1) | 3348 sys.exit(1) |
OLD | NEW |