OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 # Copyright (C) 2008 Evan Martin <martine@danga.com> | 6 # Copyright (C) 2008 Evan Martin <martine@danga.com> |
7 | 7 |
8 """A git-command for integrating reviews on Rietveld.""" | 8 """A git-command for integrating reviews on Rietveld.""" |
9 | 9 |
10 from distutils.version import LooseVersion | 10 from distutils.version import LooseVersion |
11 from multiprocessing.pool import ThreadPool | 11 from multiprocessing.pool import ThreadPool |
12 import base64 | 12 import base64 |
13 import glob | 13 import glob |
14 import httplib | |
14 import json | 15 import json |
15 import logging | 16 import logging |
16 import optparse | 17 import optparse |
17 import os | 18 import os |
18 import Queue | 19 import Queue |
19 import re | 20 import re |
20 import stat | 21 import stat |
21 import sys | 22 import sys |
22 import tempfile | 23 import tempfile |
23 import textwrap | 24 import textwrap |
25 import time | |
26 import traceback | |
24 import urllib2 | 27 import urllib2 |
25 import urlparse | 28 import urlparse |
26 import webbrowser | 29 import webbrowser |
27 import zlib | 30 import zlib |
28 | 31 |
29 try: | 32 try: |
30 import readline # pylint: disable=F0401,W0611 | 33 import readline # pylint: disable=F0401,W0611 |
31 except ImportError: | 34 except ImportError: |
32 pass | 35 pass |
33 | 36 |
37 from third_party import colorama | |
38 from third_party import httplib2 | |
39 from third_party import upload | |
34 | 40 |
35 from third_party import colorama | 41 ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) |
36 from third_party import upload | 42 sys.path.insert( |
43 0, os.path.join(ROOT_DIR, 'third_party', 'google_api_python_client')) | |
44 | |
45 import apiclient | |
37 import auth | 46 import auth |
38 import breakpad # pylint: disable=W0611 | 47 import breakpad # pylint: disable=W0611 |
39 import clang_format | 48 import clang_format |
40 import dart_format | 49 import dart_format |
41 import fix_encoding | 50 import fix_encoding |
42 import gclient_utils | 51 import gclient_utils |
43 import git_common | 52 import git_common |
44 import owners | 53 import owners |
45 import owners_finder | 54 import owners_finder |
46 import presubmit_support | 55 import presubmit_support |
47 import rietveld | 56 import rietveld |
48 import scm | 57 import scm |
49 import subcommand | 58 import subcommand |
50 import subprocess2 | 59 import subprocess2 |
51 import watchlists | 60 import watchlists |
52 | 61 |
53 __version__ = '1.0' | 62 __version__ = '1.0' |
54 | 63 |
55 DEFAULT_SERVER = 'https://codereview.appspot.com' | 64 DEFAULT_SERVER = 'https://codereview.appspot.com' |
56 POSTUPSTREAM_HOOK_PATTERN = '.git/hooks/post-cl-%s' | 65 POSTUPSTREAM_HOOK_PATTERN = '.git/hooks/post-cl-%s' |
57 DESCRIPTION_BACKUP_FILE = '~/.git_cl_description_backup' | 66 DESCRIPTION_BACKUP_FILE = '~/.git_cl_description_backup' |
58 GIT_INSTRUCTIONS_URL = 'http://code.google.com/p/chromium/wiki/UsingGit' | 67 GIT_INSTRUCTIONS_URL = 'http://code.google.com/p/chromium/wiki/UsingGit' |
59 CHANGE_ID = 'Change-Id:' | 68 CHANGE_ID = 'Change-Id:' |
60 REFS_THAT_ALIAS_TO_OTHER_REFS = { | 69 REFS_THAT_ALIAS_TO_OTHER_REFS = { |
61 'refs/remotes/origin/lkgr': 'refs/remotes/origin/master', | 70 'refs/remotes/origin/lkgr': 'refs/remotes/origin/master', |
62 'refs/remotes/origin/lkcr': 'refs/remotes/origin/master', | 71 'refs/remotes/origin/lkcr': 'refs/remotes/origin/master', |
63 } | 72 } |
64 | 73 |
74 # Buildbucket-related constants | |
75 BUILDBUCKET_DISCOVERY_URL = ( | |
76 'https://cr-buildbucket.appspot.com/_ah/api/discovery/v1/apis/' | |
77 '{api}/{apiVersion}/rest') | |
78 BUILDSET_STR = 'patch/rietveld/{hostname}/{issue}/{patch}' | |
79 | |
65 # Valid extensions for files we want to lint. | 80 # Valid extensions for files we want to lint. |
66 DEFAULT_LINT_REGEX = r"(.*\.cpp|.*\.cc|.*\.h)" | 81 DEFAULT_LINT_REGEX = r"(.*\.cpp|.*\.cc|.*\.h)" |
67 DEFAULT_LINT_IGNORE_REGEX = r"$^" | 82 DEFAULT_LINT_IGNORE_REGEX = r"$^" |
68 | 83 |
69 # Shortcut since it quickly becomes redundant. | 84 # Shortcut since it quickly becomes redundant. |
70 Fore = colorama.Fore | 85 Fore = colorama.Fore |
71 | 86 |
72 # Initialized in main() | 87 # Initialized in main() |
73 settings = None | 88 settings = None |
74 | 89 |
(...skipping 120 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
195 else: | 210 else: |
196 git_set_branch_value('git-find-copies', int(options.find_copies)) | 211 git_set_branch_value('git-find-copies', int(options.find_copies)) |
197 | 212 |
198 print('Using %d%% similarity for rename/copy detection. ' | 213 print('Using %d%% similarity for rename/copy detection. ' |
199 'Override with --similarity.' % options.similarity) | 214 'Override with --similarity.' % options.similarity) |
200 | 215 |
201 return options, args | 216 return options, args |
202 parser.parse_args = Parse | 217 parser.parse_args = Parse |
203 | 218 |
204 | 219 |
220 def _prefix_master(master): | |
221 prefix = 'master.' | |
222 if master.startswith(prefix): | |
223 return master | |
224 else: | |
225 return '%s%s' % (prefix, master) | |
226 | |
227 | |
228 def _get_buildbucket(rietveld_host, auth_config): | |
229 authenticator = auth.get_authenticator_for_host(rietveld_host, auth_config) | |
230 http = httplib2.Http() | |
231 return apiclient.discovery.build( | |
232 'buildbucket', 'v1', | |
233 http=authenticator.authorize(http), | |
234 discoveryServiceUrl=BUILDBUCKET_DISCOVERY_URL, | |
235 ) | |
236 | |
237 | |
238 def trigger_distributed_try_jobs( | |
239 auth_config, changelist, options, masters, category): | |
240 rietveld_url = settings.GetDefaultServerUrl() | |
241 rietveld_host = urlparse.urlparse(rietveld_url).hostname | |
242 buildbucket = _get_buildbucket(rietveld_host, auth_config) | |
243 issue_props = changelist.GetIssueProperties() | |
244 issue = changelist.GetIssue() | |
245 patchset = changelist.GetMostRecentPatchset() | |
246 buildset = BUILDSET_STR.format( | |
247 hostname=rietveld_host, | |
248 issue=issue, | |
249 patch=patchset) | |
250 print 'Tring jobs on:' | |
nodir
2015/04/18 05:17:55
Typo: trying
| |
251 for master, builders_and_tests in masters.iteritems(): | |
252 print 'Master: %s' % master | |
253 bucket = _prefix_master(master) | |
254 for builder, tests in builders_and_tests.iteritems(): | |
255 req = buildbucket.put(body={ | |
nodir
2015/04/18 05:17:55
You might want to update this to use put_batch API
| |
256 'bucket': bucket, | |
257 'parameters_json': json.dumps({ | |
258 'builder_name': builder, | |
259 'changes':[ | |
260 {'author': {'email': issue_props['owner_email']}}, | |
261 ], | |
262 'properties': { | |
263 'category': category, | |
264 'clobber': options.clobber, | |
265 'issue': issue, | |
266 'master': master, | |
267 'patch_project': issue_props['project'], | |
268 'patch_storage': 'rietveld', | |
269 'patchset': patchset, | |
270 'reason': options.name, | |
271 'revision': options.revision, | |
272 'rietveld': rietveld_url, | |
273 'testfilter': tests, | |
274 }, | |
275 }), | |
276 'tags': ['buildset:%s' % buildset, | |
277 'master:%s' % master, | |
278 'builder:%s' % builder, | |
279 'user_agent:git-cl-try'] | |
280 }) | |
281 wait = 1 | |
282 try_count = 3 | |
283 while try_count > 0: | |
284 try: | |
285 try_count -= 1 | |
286 response = req.execute() | |
287 if response.get('error'): | |
288 msg = 'Error in response. Reason: %s. Message: %s.' % ( | |
289 response['error'].get('reason', ''), | |
290 response['error'].get('message', '')) | |
291 raise BuildbucketResponseException(msg) | |
292 break | |
293 except apiclient.errors.HttpError as ex: | |
294 status = ex.resp.status if ex.resp else None | |
295 if status < 500 or try_count <= 0: | |
296 raise | |
297 logging.debug('Transient errors when triggering tryjobs. ' | |
298 'Will retry in %d seconds.', wait) | |
299 time.sleep(wait) | |
300 wait *= 2 | |
301 print ' %s: %s' % (builder, tests) | |
302 | |
303 | |
205 def MatchSvnGlob(url, base_url, glob_spec, allow_wildcards): | 304 def MatchSvnGlob(url, base_url, glob_spec, allow_wildcards): |
206 """Return the corresponding git ref if |base_url| together with |glob_spec| | 305 """Return the corresponding git ref if |base_url| together with |glob_spec| |
207 matches the full |url|. | 306 matches the full |url|. |
208 | 307 |
209 If |allow_wildcards| is true, |glob_spec| can contain wildcards (see below). | 308 If |allow_wildcards| is true, |glob_spec| can contain wildcards (see below). |
210 """ | 309 """ |
211 fetch_suburl, as_ref = glob_spec.split(':') | 310 fetch_suburl, as_ref = glob_spec.split(':') |
212 if allow_wildcards: | 311 if allow_wildcards: |
213 glob_match = re.match('(.+/)?(\*|{[^/]*})(/.+)?', fetch_suburl) | 312 glob_match = re.match('(.+/)?(\*|{[^/]*})(/.+)?', fetch_suburl) |
214 if glob_match: | 313 if glob_match: |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
262 try: | 361 try: |
263 stdout = sys.stdout.fileno() | 362 stdout = sys.stdout.fileno() |
264 except AttributeError: | 363 except AttributeError: |
265 stdout = None | 364 stdout = None |
266 return subprocess2.call( | 365 return subprocess2.call( |
267 ['git', | 366 ['git', |
268 'diff', '--no-ext-diff', '--stat'] + similarity_options + args, | 367 'diff', '--no-ext-diff', '--stat'] + similarity_options + args, |
269 stdout=stdout, env=env) | 368 stdout=stdout, env=env) |
270 | 369 |
271 | 370 |
371 class BuildbucketResponseException(Exception): | |
372 pass | |
373 | |
374 | |
272 class Settings(object): | 375 class Settings(object): |
273 def __init__(self): | 376 def __init__(self): |
274 self.default_server = None | 377 self.default_server = None |
275 self.cc = None | 378 self.cc = None |
276 self.root = None | 379 self.root = None |
277 self.is_git_svn = None | 380 self.is_git_svn = None |
278 self.svn_branch = None | 381 self.svn_branch = None |
279 self.tree_status_url = None | 382 self.tree_status_url = None |
280 self.viewvc_url = None | 383 self.viewvc_url = None |
281 self.updated = False | 384 self.updated = False |
(...skipping 2571 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2853 'Bot list: %s' % builders) | 2956 'Bot list: %s' % builders) |
2854 return 1 | 2957 return 1 |
2855 | 2958 |
2856 patchset = cl.GetMostRecentPatchset() | 2959 patchset = cl.GetMostRecentPatchset() |
2857 if patchset and patchset != cl.GetPatchset(): | 2960 if patchset and patchset != cl.GetPatchset(): |
2858 print( | 2961 print( |
2859 '\nWARNING Mismatch between local config and server. Did a previous ' | 2962 '\nWARNING Mismatch between local config and server. Did a previous ' |
2860 'upload fail?\ngit-cl try always uses latest patchset from rietveld. ' | 2963 'upload fail?\ngit-cl try always uses latest patchset from rietveld. ' |
2861 'Continuing using\npatchset %s.\n' % patchset) | 2964 'Continuing using\npatchset %s.\n' % patchset) |
2862 try: | 2965 try: |
2863 cl.RpcServer().trigger_distributed_try_jobs( | 2966 trigger_distributed_try_jobs( |
2864 cl.GetIssue(), patchset, options.name, options.clobber, | 2967 auth_config, cl, options, masters, 'git cl try') |
2865 options.revision, masters) | 2968 except apiclient.errors.HttpError as ex: |
2866 except urllib2.HTTPError, e: | 2969 status = ex.resp.status if ex.resp else None |
2867 if e.code == 404: | 2970 if status == httplib.FORBIDDEN: |
2868 print('404 from rietveld; ' | 2971 print 'ERROR: Access denied. Please verify you have tryjob access.' |
2869 'did you mean to use "git try" instead of "git cl try"?') | 2972 else: |
2870 return 1 | 2973 print 'ERROR: Tryjob request failed: %s.' % ex |
2871 print('Tried jobs on:') | 2974 return 1 |
2872 | 2975 except BuildbucketResponseException as ex: |
2873 for (master, builders) in masters.iteritems(): | 2976 print ex |
2874 if master: | 2977 return 1 |
2875 print 'Master: %s' % master | 2978 except Exception as e: |
2876 length = max(len(builder) for builder in builders) | 2979 stacktrace = (''.join(traceback.format_stack()) + traceback.format_exc()) |
2877 for builder in sorted(builders): | 2980 print 'ERROR: unexpected error when trying to trigger tryjobs: %s\n%s' % ( |
2878 print ' %*s: %s' % (length, builder, ','.join(builders[builder])) | 2981 e, stacktrace) |
2982 return 1 | |
2879 return 0 | 2983 return 0 |
2880 | 2984 |
2881 | 2985 |
2882 @subcommand.usage('[new upstream branch]') | 2986 @subcommand.usage('[new upstream branch]') |
2883 def CMDupstream(parser, args): | 2987 def CMDupstream(parser, args): |
2884 """Prints or sets the name of the upstream branch, if any.""" | 2988 """Prints or sets the name of the upstream branch, if any.""" |
2885 _, args = parser.parse_args(args) | 2989 _, args = parser.parse_args(args) |
2886 if len(args) > 1: | 2990 if len(args) > 1: |
2887 parser.error('Unrecognized args: %s' % ' '.join(args)) | 2991 parser.error('Unrecognized args: %s' % ' '.join(args)) |
2888 | 2992 |
(...skipping 313 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
3202 if __name__ == '__main__': | 3306 if __name__ == '__main__': |
3203 # These affect sys.stdout so do it outside of main() to simplify mocks in | 3307 # These affect sys.stdout so do it outside of main() to simplify mocks in |
3204 # unit testing. | 3308 # unit testing. |
3205 fix_encoding.fix_encoding() | 3309 fix_encoding.fix_encoding() |
3206 colorama.init() | 3310 colorama.init() |
3207 try: | 3311 try: |
3208 sys.exit(main(sys.argv[1:])) | 3312 sys.exit(main(sys.argv[1:])) |
3209 except KeyboardInterrupt: | 3313 except KeyboardInterrupt: |
3210 sys.stderr.write('interrupted\n') | 3314 sys.stderr.write('interrupted\n') |
3211 sys.exit(1) | 3315 sys.exit(1) |
OLD | NEW |