Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(32)

Side by Side Diff: tools/roll_deps.py

Issue 126523002: Changes to roll_deps.py (Closed) Base URL: https://skia.googlecode.com/svn/trunk
Patch Set: response to requested changes Created 6 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/python2 1 #!/usr/bin/python2
2 2
3 # Copyright 2014 Google Inc. 3 # Copyright 2014 Google Inc.
4 # 4 #
5 # Use of this source code is governed by a BSD-style license that can be 5 # Use of this source code is governed by a BSD-style license that can be
6 # found in the LICENSE file. 6 # found in the LICENSE file.
7 7
8 """Skia's Chromium DEPS roll script. 8 """Skia's Chromium DEPS roll script.
9 9
10 This script: 10 This script:
11 - searches through the last N Skia git commits to find out the hash that is 11 - searches through the last N Skia git commits to find out the hash that is
12 associated with the SVN revision number. 12 associated with the SVN revision number.
13 - creates a new branch in the Chromium tree, modifies the DEPS file to 13 - creates a new branch in the Chromium tree, modifies the DEPS file to
14 point at the given Skia commit, commits, uploads to Rietveld, and 14 point at the given Skia commit, commits, uploads to Rietveld, and
15 deletes the local copy of the branch. 15 deletes the local copy of the branch.
16 - creates a whitespace-only commit and uploads that to to Rietveld. 16 - creates a whitespace-only commit and uploads that to to Rietveld.
17 - returns the Chromium tree to its previous state. 17 - returns the Chromium tree to its previous state.
18 18
19 Usage: 19 Usage:
20 %prog -c CHROMIUM_PATH -r REVISION [OPTIONAL_OPTIONS] 20 %prog -c CHROMIUM_PATH -r REVISION [OPTIONAL_OPTIONS]
21 """ 21 """
22 22
23 23
24 import optparse 24 import optparse
25 import os 25 import os
26 import re 26 import re
27 import shutil 27 import shutil
28 import subprocess 28 import subprocess
29 from subprocess import check_call
30 import sys 29 import sys
31 import tempfile 30 import tempfile
32 31
33 32
34 class DepsRollConfig(object): 33 class DepsRollConfig(object):
35 """Contains configuration options for this module. 34 """Contains configuration options for this module.
36 35
37 Attributes: 36 Attributes:
38 git: (string) The git executable. 37 git: (string) The git executable.
39 chromium_path: (string) path to a local chromium git repository. 38 chromium_path: (string) path to a local chromium git repository.
40 save_branches: (boolean) iff false, delete temporary branches. 39 save_branches: (boolean) iff false, delete temporary branches.
41 verbose: (boolean) iff false, suppress the output from git-cl. 40 verbose: (boolean) iff false, suppress the output from git-cl.
42 search_depth: (int) how far back to look for the revision. 41 search_depth: (int) how far back to look for the revision.
43 skia_url: (string) Skia's git repository. 42 skia_url: (string) Skia's git repository.
44 self.skip_cl_upload: (boolean) 43 self.skip_cl_upload: (boolean)
45 self.cl_bot_list: (list of strings) 44 self.cl_bot_list: (list of strings)
46 """ 45 """
47 46
48 # pylint: disable=I0011,R0903,R0902 47 # pylint: disable=I0011,R0903,R0902
49 def __init__(self, options=None): 48 def __init__(self, options=None):
50 self.skia_url = 'https://skia.googlesource.com/skia.git' 49 self.skia_url = 'https://skia.googlesource.com/skia.git'
51 self.revision_format = ( 50 self.revision_format = (
52 'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ') 51 'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ')
53 52
54 if not options: 53 if not options:
55 options = DepsRollConfig.GetOptionParser() 54 options = DepsRollConfig.GetOptionParser()
56 # pylint: disable=I0011,E1103 55 # pylint: disable=I0011,E1103
57 self.verbose = options.verbose 56 self.verbose = options.verbose
58 self.save_branches = options.save_branches 57 self.vsp = VerboseSubprocess(self.verbose)
borenet 2014/01/08 19:20:21 This could be named something more descriptive; ev
hal.canary 2014/01/08 19:59:35 That sounds like a terrible namespace collision.
borenet 2014/01/08 20:06:17 Not if you're using it with self.subprocess.* or c
58 self.save_branches = not options.delete_branches
59 self.search_depth = options.search_depth 59 self.search_depth = options.search_depth
60 self.chromium_path = options.chromium_path 60 self.chromium_path = options.chromium_path
61 self.git = options.git_path 61 self.git = options.git_path
62 self.skip_cl_upload = options.skip_cl_upload 62 self.skip_cl_upload = options.skip_cl_upload
63 # Split and remove empty strigns from the bot list. 63 # Split and remove empty strigns from the bot list.
64 self.cl_bot_list = [bot for bot in options.bots.split(',') if bot] 64 self.cl_bot_list = [bot for bot in options.bots.split(',') if bot]
65 self.skia_git_checkout_path = options.skia_git_path 65 self.skia_git_checkout_path = options.skia_git_path
66 self.default_branch_name = 'autogenerated_deps_roll_branch' 66 self.default_branch_name = 'autogenerated_deps_roll_branch'
67 67
68 @staticmethod 68 @staticmethod
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after
103 # Anyone using this script on a regular basis should set the 103 # Anyone using this script on a regular basis should set the
104 # CHROMIUM_CHECKOUT_PATH environment variable. 104 # CHROMIUM_CHECKOUT_PATH environment variable.
105 option_parser.add_option( 105 option_parser.add_option(
106 '-c', '--chromium_path', help='Path to local Chromium Git' 106 '-c', '--chromium_path', help='Path to local Chromium Git'
107 ' repository checkout, defaults to CHROMIUM_CHECKOUT_PATH' 107 ' repository checkout, defaults to CHROMIUM_CHECKOUT_PATH'
108 ' if that environment variable is set.', 108 ' if that environment variable is set.',
109 default=os.environ.get('CHROMIUM_CHECKOUT_PATH')) 109 default=os.environ.get('CHROMIUM_CHECKOUT_PATH'))
110 option_parser.add_option( 110 option_parser.add_option(
111 '-r', '--revision', type='int', default=None, 111 '-r', '--revision', type='int', default=None,
112 help='The Skia SVN revision number, defaults to top of tree.') 112 help='The Skia SVN revision number, defaults to top of tree.')
113 option_parser.add_option(
114 '-g', '--git_hash', default=None,
115 help='A partial Skia Git hash. If set, overrides "--revision".')
116
113 # Anyone using this script on a regular basis should set the 117 # Anyone using this script on a regular basis should set the
114 # SKIA_GIT_CHECKOUT_PATH environment variable. 118 # SKIA_GIT_CHECKOUT_PATH environment variable.
115 option_parser.add_option( 119 option_parser.add_option(
116 '', '--skia_git_path', 120 '', '--skia_git_path',
117 help='Path of a pure-git Skia repository checkout. If empty,' 121 help='Path of a pure-git Skia repository checkout. If empty,'
118 ' a temporary will be cloned. Defaults to SKIA_GIT_CHECKOUT' 122 ' a temporary will be cloned. Defaults to SKIA_GIT_CHECKOUT'
119 '_PATH, if that environment variable is set.', 123 '_PATH, if that environment variable is set.',
120 default=os.environ.get('SKIA_GIT_CHECKOUT_PATH')) 124 default=os.environ.get('SKIA_GIT_CHECKOUT_PATH'))
121 option_parser.add_option( 125 option_parser.add_option(
122 '', '--search_depth', type='int', default=100, 126 '', '--search_depth', type='int', default=100,
123 help='How far back to look for the revision.') 127 help='How far back to look for the revision.')
124 option_parser.add_option( 128 option_parser.add_option(
125 '', '--git_path', help='Git executable, defaults to "git".', 129 '', '--git_path', help='Git executable, defaults to "git".',
126 default='git') 130 default='git')
127 option_parser.add_option( 131 option_parser.add_option(
128 '', '--save_branches', help='Save the temporary branches', 132 '', '--delete_branches', help='Delete the temporary branches',
129 action='store_true', dest='save_branches', default=False) 133 action='store_true', dest='delete_branches', default=False)
130 option_parser.add_option( 134 option_parser.add_option(
131 '', '--verbose', help='Do not suppress the output from `git cl`.', 135 '', '--verbose', help='Do not suppress the output from `git cl`.',
132 action='store_true', dest='verbose', default=False) 136 action='store_true', dest='verbose', default=False)
133 option_parser.add_option( 137 option_parser.add_option(
134 '', '--skip_cl_upload', help='Skip the cl upload step; useful' 138 '', '--skip_cl_upload', help='Skip the cl upload step; useful'
135 ' for testing or with --save_branches.', 139 ' for testing or with --save_branches.',
136 action='store_true', default=False) 140 action='store_true', default=False)
137 141
138 default_bots_help = ( 142 default_bots_help = (
139 'Comma-separated list of bots, defaults to a list of %d bots.' 143 'Comma-separated list of bots, defaults to a list of %d bots.'
(...skipping 20 matching lines...) Expand all
160 except (OSError,): 164 except (OSError,):
161 return False 165 return False
162 return True 166 return True
163 167
164 168
165 class DepsRollError(Exception): 169 class DepsRollError(Exception):
166 """Exceptions specific to this module.""" 170 """Exceptions specific to this module."""
167 pass 171 pass
168 172
169 173
170 def strip_output(*args, **kwargs): 174 class VerboseSubprocess(object):
171 """Wrap subprocess.check_output and str.strip(). 175 """Call subprocess methods, but print out command before executing.
172 176
173 Pass the given arguments into subprocess.check_output() and return 177 Attributes:
174 the results, after stripping any excess whitespace. 178 verbose: (boolean) should we print out the command or not. If
179 not, this is the same as calling the subprocess method
180 quiet: (boolean) suppress stdout on check_call and call.
181 prefix: (string) When verbose, what to print before each command.
182 """
175 183
176 Args: 184 def __init__(self, verbose):
177 *args: to be passed to subprocess.check_output() 185 self.verbose = verbose
178 **kwargs: to be passed to subprocess.check_output() 186 self.quiet = not verbose
187 self.prefix = '~~$ '
179 188
180 Returns: 189 @staticmethod
181 The output of the process as a string without leading or 190 def _fix(string):
182 trailing whitespace. 191 """Quote and escape a string if necessary."""
183 Raises: 192 if ' ' in string or '\n' in string:
184 OSError or subprocess.CalledProcessError: raised by check_output. 193 string = '"%s"' % string.replace('\n', '\\n')
185 """ 194 return string
186 return str(subprocess.check_output(*args, **kwargs)).strip() 195
196 @staticmethod
197 def print_subprocess_args(prefix, *args, **kwargs):
198 """Print out args in a human-readable manner."""
199 if 'cwd' in kwargs:
200 print '%scd %s' % (prefix, kwargs['cwd'])
201 print prefix + ' '.join(VerboseSubprocess._fix(arg) for arg in args[0])
202 if 'cwd' in kwargs:
203 print '%scd -' % prefix
204
205 def check_call(self, *args, **kwargs):
206 """Wrapper for subprocess.check_call().
207
208 Args:
209 *args: to be passed to subprocess.check_call()
210 **kwargs: to be passed to subprocess.check_call()
211 Returns:
212 Whatever subprocess.check_call() returns.
213 Raises:
214 OSError or subprocess.CalledProcessError: raised by check_call.
215 """
216 if self.verbose:
217 self.print_subprocess_args(self.prefix, *args, **kwargs)
218 if self.quiet:
219 with open(os.devnull, 'w') as devnull:
220 return subprocess.check_call(*args, stdout=devnull, **kwargs)
221 else:
222 return subprocess.check_call(*args, **kwargs)
223
224 def call(self, *args, **kwargs):
225 """Wrapper for subprocess.check().
226
227 Args:
228 *args: to be passed to subprocess.check_call()
229 **kwargs: to be passed to subprocess.check_call()
230 Returns:
231 Whatever subprocess.call() returns.
232 Raises:
233 OSError or subprocess.CalledProcessError: raised by call.
234 """
235 if self.verbose:
236 self.print_subprocess_args(self.prefix, *args, **kwargs)
237 if self.quiet:
238 with open(os.devnull, 'w') as devnull:
239 return subprocess.call(*args, stdout=devnull, **kwargs)
240 else:
241 return subprocess.call(*args, **kwargs)
242
243 def check_output(self, *args, **kwargs):
244 """Wrapper for subprocess.check_output().
245
246 Args:
247 *args: to be passed to subprocess.check_output()
248 **kwargs: to be passed to subprocess.check_output()
249 Returns:
250 Whatever subprocess.check_output() returns.
251 Raises:
252 OSError or subprocess.CalledProcessError: raised by check_output.
253 """
254 if self.verbose:
255 self.print_subprocess_args(self.prefix, *args, **kwargs)
256 return subprocess.check_output(*args, **kwargs)
257
258 def strip_output(self, *args, **kwargs):
259 """Wrap subprocess.check_output and str.strip().
260
261 Pass the given arguments into subprocess.check_output() and return
262 the results, after stripping any excess whitespace.
263
264 Args:
265 *args: to be passed to subprocess.check_output()
266 **kwargs: to be passed to subprocess.check_output()
267
268 Returns:
269 The output of the process as a string without leading or
270 trailing whitespace.
271 Raises:
272 OSError or subprocess.CalledProcessError: raised by check_output.
273 """
274 if self.verbose:
275 self.print_subprocess_args(self.prefix, *args, **kwargs)
276 return str(subprocess.check_output(*args, **kwargs)).strip()
277
278 def popen(self, *args, **kwargs):
279 """Wrapper for subprocess.Popen().
280
281 Args:
282 *args: to be passed to subprocess.Popen()
283 **kwargs: to be passed to subprocess.Popen()
284 Returns:
285 The output of subprocess.Popen()
286 Raises:
287 OSError or subprocess.CalledProcessError: raised by Popen.
288 """
289 if self.verbose:
290 self.print_subprocess_args(self.prefix, *args, **kwargs)
291 return subprocess.Popen(*args, **kwargs)
187 292
188 293
189 def create_temp_skia_clone(config, depth): 294 def create_temp_skia_clone(config, depth):
190 """Clones Skia in a temp dir. 295 """Clones Skia in a temp dir.
191 296
192 Args: 297 Args:
193 config: (roll_deps.DepsRollConfig) object containing options. 298 config: (roll_deps.DepsRollConfig) object containing options.
194 depth: (int) how far back to clone the tree. 299 depth: (int) how far back to clone the tree.
195 Returns: 300 Returns:
196 temporary directory path if succcessful. 301 temporary directory path if succcessful.
197 Raises: 302 Raises:
198 OSError, subprocess.CalledProcessError on failure. 303 OSError, subprocess.CalledProcessError on failure.
199 """ 304 """
200 git = config.git 305 git = config.git
201 skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_') 306 skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_')
202 try: 307 try:
203 check_call( 308 config.vsp.check_call(
204 [git, 'clone', '-q', '--depth=%d' % depth, 309 [git, 'clone', '-q', '--depth=%d' % depth,
205 '--single-branch', config.skia_url, skia_dir]) 310 '--single-branch', config.skia_url, skia_dir])
206 return skia_dir 311 return skia_dir
207 except (OSError, subprocess.CalledProcessError) as error: 312 except (OSError, subprocess.CalledProcessError) as error:
208 shutil.rmtree(skia_dir) 313 shutil.rmtree(skia_dir)
209 raise error 314 raise error
210 315
211 316
212 def find_revision_and_hash(config, revision): 317 def get_svn_revision(config, commit):
318 """Works in both git and git-svn. returns a string."""
319 svn_format = (
320 '(git-svn-id: [^@ ]+@|SVN changes up to revision )'
321 '(?P<return>[0-9]+)')
borenet 2014/01/08 19:27:40 Unfortunately, we need to add another case: "LKGR
hal.canary 2014/01/08 19:59:35 Done.
borenet 2014/01/08 20:06:17 Thanks.
322 svn_revision = ReSearch.search_within_output(
323 config.verbose, svn_format, None,
324 [config.git, 'log', '-n', '1', '--format=format:%B', commit])
325 if not svn_revision:
326 raise DepsRollError(
327 'Revision number missing from Chromium origin/master.')
328 return int(svn_revision)
329
330
331 def revision_and_hash(config, revision):
213 """Finds revision number and git hash of origin/master in the Skia tree. 332 """Finds revision number and git hash of origin/master in the Skia tree.
214 333
215 Args: 334 Args:
216 config: (roll_deps.DepsRollConfig) object containing options. 335 config: (roll_deps.DepsRollConfig) object containing options.
217 revision: (int or None) SVN revision number. If None, use 336 revision: (int or None) SVN revision number. If None, use
218 tip-of-tree. 337 tip-of-tree.
219 338
220 Returns: 339 Returns:
221 A tuple (revision, hash) 340 A tuple (revision, hash)
222 revision: (int) SVN revision number. 341 revision: (int) SVN revision number.
223 hash: (string) full Git commit hash. 342 git_hash: (string) full Git commit hash.
224 343
225 Raises: 344 Raises:
226 roll_deps.DepsRollError: if the revision can't be found. 345 roll_deps.DepsRollError: if the revision can't be found.
227 OSError: failed to execute git or git-cl. 346 OSError: failed to execute git or git-cl.
228 subprocess.CalledProcessError: git returned unexpected status. 347 subprocess.CalledProcessError: git returned unexpected status.
229 """ 348 """
230 git = config.git 349 git = config.git
231 use_temp = False
232 skia_dir = None
233 depth = 1 if (revision is None) else config.search_depth 350 depth = 1 if (revision is None) else config.search_depth
234 try: 351 with SkiaGitCheckout(config, depth):
352 if revision is None:
353 revision = get_svn_revision(config, 'origin/master')
354 git_hash = config.vsp.strip_output(
355 [git, 'show-ref', 'origin/master', '--hash'])
356 else:
357 revision_regex = config.revision_format % revision
358 git_hash = config.vsp.strip_output(
359 [git, 'log', '--grep', revision_regex,
360 '--format=format:%H', 'origin/master'])
361 if revision < 0 or not git_hash:
362 raise DepsRollError('Git hash can not be found.')
363 return revision, git_hash
364
365
366 def revision_and_hash_from_partial(config, partial_hash):
367 """Returns the SVN revision number and full git hash.
368
369 Args:
370 config: (roll_deps.DepsRollConfig) object containing options.
371 partial_hash: (string) Partial git commit hash.
372
373 Returns:
374 A tuple (revision, hash)
375 revision: (int) SVN revision number.
376 git_hash: (string) full Git commit hash.
377
378 Raises:
379 roll_deps.DepsRollError: if the revision can't be found.
380 OSError: failed to execute git or git-cl.
381 subprocess.CalledProcessError: git returned unexpected status.
382 """
383 with SkiaGitCheckout(config, config.search_depth):
384 git_hash = config.vsp.strip_output(
385 ['git', 'log', '--format=format:%H', '-n1', partial_hash])
386 if not git_hash:
387 raise DepsRollError('Partial Git hash can not be found.')
388 revision = get_svn_revision(config, git_hash)
389 return revision, git_hash
390
391
392 class SkiaGitCheckout(object):
393 """Class to create a temporary skia git checkout, if necessary.
394 """
395 # pylint: disable=I0011,R0903
396
397 def __init__(self, config, depth):
398 self._config = config
399 self._depth = depth
400 self._use_temp = None
401 self._original_cwd = None
402
403 def __enter__(self):
404 config = self._config
405 git = config.git
406 skia_dir = None
407 self._original_cwd = os.getcwd()
235 if config.skia_git_checkout_path: 408 if config.skia_git_checkout_path:
236 skia_dir = config.skia_git_checkout_path 409 skia_dir = config.skia_git_checkout_path
237 ## Update origin/master if needed. 410 ## Update origin/master if needed.
238 check_call([git, 'fetch', '-q', 'origin'], cwd=skia_dir) 411 if self._config.verbose:
412 print '~~$', 'cd', skia_dir
413 os.chdir(skia_dir)
414 config.vsp.check_call([git, 'fetch', '-q', 'origin'])
415 self._use_temp = None
239 else: 416 else:
240 skia_dir = create_temp_skia_clone(config, depth) 417 skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_')
241 assert skia_dir 418 self._use_temp = skia_dir
242 use_temp = True 419 try:
420 os.chdir(skia_dir)
421 config.vsp.check_call(
422 [git, 'clone', '-q', '--depth=%d' % self._depth,
423 '--single-branch', config.skia_url, '.'])
424 except (OSError, subprocess.CalledProcessError) as error:
425 shutil.rmtree(skia_dir)
426 raise error
243 427
244 if revision is None: 428 finally:
245 message = subprocess.check_output( 429 shutil.rmtree(skia_dir)
borenet 2014/01/08 19:20:21 I'm confused - won't this delete our checkout befo
hal.canary 2014/01/08 19:59:35 Done.
246 [git, 'log', '-n', '1', '--format=format:%B',
247 'origin/master'], cwd=skia_dir)
248 svn_format = (
249 'git-svn-id: http://skia.googlecode.com/svn/trunk@([0-9]+) ')
250 search = re.search(svn_format, message)
251 if not search:
252 raise DepsRollError(
253 'Revision number missing from origin/master.')
254 revision = int(search.group(1))
255 git_hash = strip_output(
256 [git, 'show-ref', 'origin/master', '--hash'], cwd=skia_dir)
257 else:
258 revision_regex = config.revision_format % revision
259 git_hash = strip_output(
260 [git, 'log', '--grep', revision_regex, '--format=format:%H',
261 'origin/master'], cwd=skia_dir)
262 430
263 if revision < 0 or not git_hash: 431 def __exit__(self, etype, value, traceback):
264 raise DepsRollError('Git hash can not be found.') 432 os.chdir(self._original_cwd)
265 return revision, git_hash 433 if self._use_temp:
266 finally: 434 shutil.rmtree(self._use_temp)
267 if use_temp: 435 if self._config.verbose:
268 shutil.rmtree(skia_dir) 436 print '~~$', 'cd', self._original_cwd
borenet 2014/01/08 19:20:21 Please put this before os.chdir so that we get the
hal.canary 2014/01/08 19:59:35 Done.
437
438
439 class ChangeDir(object):
440 """Use with a with-statement to temporarily change directories."""
441 # pylint: disable=I0011,R0903
442
443 def __init__(self, directory, verbose=False):
444 self._directory = directory
445 self._verbose = verbose
446
447 def __enter__(self):
448 if self._verbose:
449 print '~~$ cd %s' % self._directory
450 cwd = os.getcwd()
451 os.chdir(self._directory)
452 self._directory = cwd
453
454 def __exit__(self, etype, value, traceback):
455 if self._verbose:
456 print '~~$ cd %s' % self._directory
457 os.chdir(self._directory)
458
459
460 class ReSearch(object):
borenet 2014/01/08 19:20:21 Now that you've split this into a class with only
hal.canary 2014/01/08 19:59:35 As soon as someone else needs it, we can do that.
461 """A collection of static methods for regexing things."""
462
463 @staticmethod
464 def search_within_stream(input_stream, pattern, default=None):
465 """Search for regular expression in a file-like object.
466
467 Opens a file for reading and searches line by line for a match to
468 the regex and returns the parenthesized group named return for the
469 first match. Does not search across newlines.
470
471 For example:
472 pattern = '^root(:[^:]*){4}:(?P<return>[^:]*)'
473 with open('/etc/passwd', 'r') as stream:
474 return search_within_file(stream, pattern)
475 should return root's home directory (/root on my system).
476
477 Args:
478 input_stream: file-like object to be read
479 pattern: (string) to be passed to re.compile
480 default: what to return if no match
481
482 Returns:
483 A string or whatever default is
484 """
485 pattern_object = re.compile(pattern)
486 for line in input_stream:
487 match = pattern_object.search(line)
488 if match:
489 return match.group('return')
490 return default
491
492 @staticmethod
493 def search_within_string(input_string, pattern, default=None):
494 """Search for regular expression in a string.
495
496 Args:
497 input_string: (string) to be searched
498 pattern: (string) to be passed to re.compile
499 default: what to return if no match
500
501 Returns:
502 A string or whatever default is
503 """
504 match = re.search(pattern, input_string)
505 return match.group('return') if match else default
506
507 @staticmethod
508 def search_within_output(verbose, pattern, default, *args, **kwargs):
509 """Search for regular expression in a process output.
510
511 Does not search across newlines.
512
513 Args:
514 verbose: (boolean) shoule we call
515 VerboseSubprocess.print_subprocess_args?
516 pattern: (string) to be passed to re.compile
517 default: what to return if no match
518 *args: to be passed to subprocess.Popen()
519 **kwargs: to be passed to subprocess.Popen()
520
521 Returns:
522 A string or whatever default is
523 """
524 if verbose:
525 VerboseSubprocess.print_subprocess_args(
526 '~~$ ', *args, **kwargs)
527 proc = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
528 return ReSearch.search_within_stream(proc.stdout, pattern, default)
269 529
270 530
271 class GitBranchCLUpload(object): 531 class GitBranchCLUpload(object):
272 """Class to manage git branches and git-cl-upload. 532 """Class to manage git branches and git-cl-upload.
273 533
274 This class allows one to create a new branch in a repository based 534 This class allows one to create a new branch in a repository based
275 off of origin/master, make changes to the tree inside the 535 off of origin/master, make changes to the tree inside the
276 with-block, upload that new branch to Rietveld, restore the original 536 with-block, upload that new branch to Rietveld, restore the original
277 tree state, and delete the local copy of the new branch. 537 tree state, and delete the local copy of the new branch.
278 538
(...skipping 16 matching lines...) Expand all
295 """ 555 """
296 # pylint: disable=I0011,R0903,R0902 556 # pylint: disable=I0011,R0903,R0902
297 557
298 def __init__(self, config, message, set_branch_name): 558 def __init__(self, config, message, set_branch_name):
299 self._message = message 559 self._message = message
300 self._file_list = [] 560 self._file_list = []
301 self._branch_name = set_branch_name 561 self._branch_name = set_branch_name
302 self._stash = None 562 self._stash = None
303 self._original_branch = None 563 self._original_branch = None
304 self._config = config 564 self._config = config
305 self._svn_info = None
306 self.issue = None 565 self.issue = None
307 566
308 def stage_for_commit(self, *paths): 567 def stage_for_commit(self, *paths):
309 """Calls `git add ...` on each argument. 568 """Calls `git add ...` on each argument.
310 569
311 Args: 570 Args:
312 *paths: (list of strings) list of filenames to pass to `git add`. 571 *paths: (list of strings) list of filenames to pass to `git add`.
313 """ 572 """
314 self._file_list.extend(paths) 573 self._file_list.extend(paths)
315 574
316 def __enter__(self): 575 def __enter__(self):
317 git = self._config.git 576 git = self._config.git
577 vsp = self._config.vsp
318 def branch_exists(branch): 578 def branch_exists(branch):
319 """Return true iff branch exists.""" 579 """Return true iff branch exists."""
320 return 0 == subprocess.call( 580 return 0 == vsp.call([git, 'show-ref', '--quiet', branch])
321 [git, 'show-ref', '--quiet', branch])
322 def has_diff(): 581 def has_diff():
323 """Return true iff repository has uncommited changes.""" 582 """Return true iff repository has uncommited changes."""
324 return bool(subprocess.call([git, 'diff', '--quiet', 'HEAD'])) 583 return bool(vsp.call([git, 'diff', '--quiet', 'HEAD']))
584
325 self._stash = has_diff() 585 self._stash = has_diff()
326 if self._stash: 586 if self._stash:
327 check_call([git, 'stash', 'save']) 587 subprocess.check_call([git, 'stash', 'save'])
borenet 2014/01/08 19:20:21 Why not use your wrapper?
hal.canary 2014/01/08 19:59:35 Done.
328 try: 588 try:
329 self._original_branch = strip_output( 589 self._original_branch = vsp.strip_output(
330 [git, 'symbolic-ref', '--short', 'HEAD']) 590 [git, 'symbolic-ref', '--short', 'HEAD'])
331 except (subprocess.CalledProcessError,): 591 except (subprocess.CalledProcessError,):
332 self._original_branch = strip_output( 592 self._original_branch = vsp.strip_output(
333 [git, 'rev-parse', 'HEAD']) 593 [git, 'rev-parse', 'HEAD'])
334 594
335 if not self._branch_name: 595 if not self._branch_name:
336 self._branch_name = self._config.default_branch_name 596 self._branch_name = self._config.default_branch_name
337 597
338 if branch_exists(self._branch_name): 598 if branch_exists(self._branch_name):
339 check_call([git, 'checkout', '-q', 'master']) 599 vsp.check_call([git, 'checkout', '-q', 'master'])
340 check_call([git, 'branch', '-q', '-D', self._branch_name]) 600 vsp.check_call([git, 'branch', '-q', '-D', self._branch_name])
341 601
342 check_call( 602 vsp.check_call(
343 [git, 'checkout', '-q', '-b', 603 [git, 'checkout', '-q', '-b', self._branch_name, 'origin/master'])
344 self._branch_name, 'origin/master'])
345
346 svn_info = subprocess.check_output(['git', 'svn', 'info'])
347 svn_info_search = re.search(r'Last Changed Rev: ([0-9]+)\W', svn_info)
348 assert svn_info_search
349 self._svn_info = svn_info_search.group(1)
350 604
351 def __exit__(self, etype, value, traceback): 605 def __exit__(self, etype, value, traceback):
352 # pylint: disable=I0011,R0912 606 # pylint: disable=I0011,R0912
353 git = self._config.git 607 git = self._config.git
354 def quiet_check_call(*args, **kwargs): 608 vsp = self._config.vsp
355 """Call check_call, but pipe output to devnull.""" 609 svn_info = str(get_svn_revision(self._config, 'HEAD'))
356 with open(os.devnull, 'w') as devnull:
357 check_call(*args, stdout=devnull, **kwargs)
358 610
359 for filename in self._file_list: 611 for filename in self._file_list:
360 assert os.path.exists(filename) 612 assert os.path.exists(filename)
361 check_call([git, 'add', filename]) 613 vsp.check_call([git, 'add', filename])
362 check_call([git, 'commit', '-q', '-m', self._message]) 614 vsp.check_call([git, 'commit', '-q', '-m', self._message])
363 615
364 git_cl = [git, 'cl', 'upload', '-f', '--cc=skia-team@google.com', 616 git_cl = [git, 'cl', 'upload', '-f', '--cc=skia-team@google.com',
365 '--bypass-hooks', '--bypass-watchlists'] 617 '--bypass-hooks', '--bypass-watchlists']
366 git_try = [git, 'cl', 'try', '--revision', self._svn_info] 618 git_try = [git, 'cl', 'try', '--revision', svn_info]
367 git_try.extend([arg for bot in self._config.cl_bot_list 619 git_try.extend([arg for bot in self._config.cl_bot_list
368 for arg in ('-b', bot)]) 620 for arg in ('-b', bot)])
369 621
370 if self._config.skip_cl_upload: 622 if self._config.skip_cl_upload:
371 print ' '.join(git_cl) 623 print 'You should call:'
624 print ' cd %s' % os.getcwd()
625 VerboseSubprocess.print_subprocess_args(
626 ' ', [git, 'checkout', self._branch_name])
627 VerboseSubprocess.print_subprocess_args(' ', git_cl)
628 if self._config.cl_bot_list:
629 VerboseSubprocess.print_subprocess_args(' ', git_try)
372 print 630 print
373 if self._config.cl_bot_list:
374 print ' '.join(git_try)
375 print
376 self.issue = '' 631 self.issue = ''
377 else: 632 else:
378 if self._config.verbose: 633 vsp.check_call(git_cl)
379 check_call(git_cl) 634 self.issue = vsp.strip_output([git, 'cl', 'issue'])
380 print
381 else:
382 quiet_check_call(git_cl)
383 self.issue = strip_output([git, 'cl', 'issue'])
384 if self._config.cl_bot_list: 635 if self._config.cl_bot_list:
385 if self._config.verbose: 636 vsp.check_call(git_try)
386 check_call(git_try)
387 print
388 else:
389 quiet_check_call(git_try)
390 637
391 # deal with the aftermath of failed executions of this script. 638 # deal with the aftermath of failed executions of this script.
392 if self._config.default_branch_name == self._original_branch: 639 if self._config.default_branch_name == self._original_branch:
393 self._original_branch = 'master' 640 self._original_branch = 'master'
394 check_call([git, 'checkout', '-q', self._original_branch]) 641 vsp.check_call([git, 'checkout', '-q', self._original_branch])
395 642
396 if self._config.default_branch_name == self._branch_name: 643 if self._config.default_branch_name == self._branch_name:
397 check_call([git, 'branch', '-q', '-D', self._branch_name]) 644 vsp.check_call([git, 'branch', '-q', '-D', self._branch_name])
398 if self._stash: 645 if self._stash:
399 check_call([git, 'stash', 'pop']) 646 vsp.check_call([git, 'stash', 'pop'])
400 647
401 648
402 def change_skia_deps(revision, git_hash, depspath): 649 def change_skia_deps(revision, git_hash, depspath):
403 """Update the DEPS file. 650 """Update the DEPS file.
404 651
405 Modify the skia_revision and skia_hash entries in the given DEPS file. 652 Modify the skia_revision and skia_hash entries in the given DEPS file.
406 653
407 Args: 654 Args:
408 revision: (int) Skia SVN revision. 655 revision: (int) Skia SVN revision.
409 git_hash: (string) Skia Git hash. 656 git_hash: (string) Skia Git hash.
(...skipping 11 matching lines...) Expand all
421 with open(depspath, 'r') as input_stream: 668 with open(depspath, 'r') as input_stream:
422 for line in input_stream: 669 for line in input_stream:
423 line = deps_regex_rev.sub(deps_regex_rev_repl, line) 670 line = deps_regex_rev.sub(deps_regex_rev_repl, line)
424 line = deps_regex_hash.sub(deps_regex_hash_repl, line) 671 line = deps_regex_hash.sub(deps_regex_hash_repl, line)
425 temp_file.write(line) 672 temp_file.write(line)
426 finally: 673 finally:
427 temp_file.close() 674 temp_file.close()
428 shutil.move(temp_file.name, depspath) 675 shutil.move(temp_file.name, depspath)
429 676
430 677
431 def branch_name(message):
432 """Return the first line of a commit message to be used as a branch name.
433
434 Args:
435 message: (string)
436
437 Returns:
438 A string derived from message suitable for a branch name.
439 """
440 return message.lstrip().split('\n')[0].rstrip().replace(' ', '_')
441
442
443 def roll_deps(config, revision, git_hash): 678 def roll_deps(config, revision, git_hash):
444 """Upload changed DEPS and a whitespace change. 679 """Upload changed DEPS and a whitespace change.
445 680
446 Given the correct git_hash, create two Reitveld issues. 681 Given the correct git_hash, create two Reitveld issues.
447 682
448 Args: 683 Args:
449 config: (roll_deps.DepsRollConfig) object containing options. 684 config: (roll_deps.DepsRollConfig) object containing options.
450 revision: (int) Skia SVN revision. 685 revision: (int) Skia SVN revision.
451 git_hash: (string) Skia Git hash. 686 git_hash: (string) Skia Git hash.
452 687
453 Returns: 688 Returns:
454 a tuple containing textual description of the two issues. 689 a tuple containing textual description of the two issues.
455 690
456 Raises: 691 Raises:
457 OSError: failed to execute git or git-cl. 692 OSError: failed to execute git or git-cl.
458 subprocess.CalledProcessError: git returned unexpected status. 693 subprocess.CalledProcessError: git returned unexpected status.
459 """ 694 """
695
460 git = config.git 696 git = config.git
461 cwd = os.getcwd() 697 with ChangeDir(config.chromium_path, config.verbose):
462 os.chdir(config.chromium_path) 698 config.vsp.check_call([git, 'fetch', '-q', 'origin'])
463 try: 699
464 check_call([git, 'fetch', '-q', 'origin']) 700 old_revision = ReSearch.search_within_output(
465 master_hash = strip_output( 701 config.verbose, '"skia_revision": "(?P<return>[0-9]+)",', None,
702 [git, 'show', 'origin/master:DEPS'])
703 assert old_revision
704 if revision == int(old_revision):
705 print 'DEPS is up to date!'
706 return None
707
708 master_hash = config.vsp.strip_output(
466 [git, 'show-ref', 'origin/master', '--hash']) 709 [git, 'show-ref', 'origin/master', '--hash'])
467 710
711 branch = None
712
468 # master_hash[8] gives each whitespace CL a unique name. 713 # master_hash[8] gives each whitespace CL a unique name.
469 message = ('whitespace change %s\n\nThis CL was created by' 714 message = ('whitespace change %s\n\nThis CL was created by'
470 ' Skia\'s roll_deps.py script.\n') % master_hash[:8] 715 ' Skia\'s roll_deps.py script.\n') % master_hash[:8]
471 branch = branch_name(message) if config.save_branches else None 716 if config.save_branches:
717 branch = 'control_%s' % master_hash[:8]
472 718
473 codereview = GitBranchCLUpload(config, message, branch) 719 codereview = GitBranchCLUpload(config, message, branch)
474 with codereview: 720 with codereview:
475 with open('build/whitespace_file.txt', 'a') as output_stream: 721 with open('build/whitespace_file.txt', 'a') as output_stream:
476 output_stream.write('\nCONTROL\n') 722 output_stream.write('\nCONTROL\n')
477 codereview.stage_for_commit('build/whitespace_file.txt') 723 codereview.stage_for_commit('build/whitespace_file.txt')
478 whitespace_cl = codereview.issue 724 whitespace_cl = codereview.issue
479 if branch: 725 if branch:
480 whitespace_cl = '%s\n branch: %s' % (whitespace_cl, branch) 726 whitespace_cl = '%s\n branch: %s' % (whitespace_cl, branch)
481 control_url_match = re.search('https?://[^) ]+', codereview.issue) 727
482 if control_url_match: 728 control_url = ReSearch.search_within_string(
483 message = ('roll skia DEPS to %d\n\nThis CL was created by' 729 codereview.issue, '(?P<return>https?://[^) ]+)', '?')
484 ' Skia\'s roll_deps.py script.\n\ncontrol: %s' 730
485 % (revision, control_url_match.group(0))) 731 if config.save_branches:
486 else: 732 branch = 'roll_%d_%s' % (revision, master_hash[:8])
487 message = ('roll skia DEPS to %d\n\nThis CL was created by' 733 message = (
488 ' Skia\'s roll_deps.py script.') % revision 734 'roll skia DEPS to %d\n\nold revision:%s\n'
489 branch = branch_name(message) if config.save_branches else None 735 'new revision:%d\nThis CL was created by'
736 ' Skia\'s roll_deps.py script.\n\ncontrol: %s'
737 % (revision, old_revision, revision, control_url))
490 codereview = GitBranchCLUpload(config, message, branch) 738 codereview = GitBranchCLUpload(config, message, branch)
491 with codereview: 739 with codereview:
492 change_skia_deps(revision, git_hash, 'DEPS') 740 change_skia_deps(revision, git_hash, 'DEPS')
493 codereview.stage_for_commit('DEPS') 741 codereview.stage_for_commit('DEPS')
494 deps_cl = codereview.issue 742 deps_cl = codereview.issue
495 if branch: 743 if branch:
496 deps_cl = '%s\n branch: %s' % (deps_cl, branch) 744 deps_cl = '%s\n branch: %s' % (deps_cl, branch)
497 745
498 return deps_cl, whitespace_cl 746 return deps_cl, whitespace_cl
499 finally:
500 os.chdir(cwd)
501 747
502 748
503 def find_hash_and_roll_deps(config, revision): 749 def find_hash_and_roll_deps(config, revision, partial_hash):
504 """Call find_hash_from_revision() and roll_deps(). 750 """Call find_hash_from_revision() and roll_deps().
505 751
506 The calls to git will be verbose on standard output. After a 752 The calls to git will be verbose on standard output. After a
507 successful upload of both issues, print links to the new 753 successful upload of both issues, print links to the new
508 codereview issues. 754 codereview issues.
509 755
510 Args: 756 Args:
511 config: (roll_deps.DepsRollConfig) object containing options. 757 config: (roll_deps.DepsRollConfig) object containing options.
512 revision: (int or None) the Skia SVN revision number or None 758 revision: (int or None) the Skia SVN revision number or None
513 to use the tip of the tree. 759 to use the tip of the tree.
760 partial_hash: (string) a partial pure-git Skia commit hash.
761 If set, revision is ignored.
borenet 2014/01/08 19:20:21 This and revision should default to None so that o
hal.canary 2014/01/08 19:59:35 Done.
514 762
515 Raises: 763 Raises:
516 roll_deps.DepsRollError: if the revision can't be found. 764 roll_deps.DepsRollError: if the revision can't be found.
517 OSError: failed to execute git or git-cl. 765 OSError: failed to execute git or git-cl.
518 subprocess.CalledProcessError: git returned unexpected status. 766 subprocess.CalledProcessError: git returned unexpected status.
519 """ 767 """
520 revision, git_hash = find_revision_and_hash(config, revision) 768
769 if partial_hash:
770 revision, git_hash = revision_and_hash_from_partial(
771 config, partial_hash)
772 else:
773 revision, git_hash = revision_and_hash(config, revision)
521 774
522 print 'revision=%r\nhash=%r\n' % (revision, git_hash) 775 print 'revision=%r\nhash=%r\n' % (revision, git_hash)
523 776
524 deps_issue, whitespace_issue = roll_deps(config, revision, git_hash) 777 roll = roll_deps(config, revision, git_hash)
525 778
526 print 'DEPS roll:\n %s\n' % deps_issue 779 if roll:
527 print 'Whitespace change:\n %s\n' % whitespace_issue 780 deps_issue, whitespace_issue = roll
781 print 'DEPS roll:\n %s\n' % deps_issue
782 print 'Whitespace change:\n %s\n' % whitespace_issue
528 783
529 784
530 def main(args): 785 def main(args):
531 """main function; see module-level docstring and GetOptionParser help. 786 """main function; see module-level docstring and GetOptionParser help.
532 787
533 Args: 788 Args:
534 args: sys.argv[1:]-type argument list. 789 args: sys.argv[1:]-type argument list.
535 """ 790 """
536 option_parser = DepsRollConfig.GetOptionParser() 791 option_parser = DepsRollConfig.GetOptionParser()
537 options = option_parser.parse_args(args)[0] 792 options = option_parser.parse_args(args)[0]
538 793
539 if not options.chromium_path: 794 if not options.chromium_path:
540 option_parser.error('Must specify chromium_path.') 795 option_parser.error('Must specify chromium_path.')
541 if not os.path.isdir(options.chromium_path): 796 if not os.path.isdir(options.chromium_path):
542 option_parser.error('chromium_path must be a directory.') 797 option_parser.error('chromium_path must be a directory.')
543 if not test_git_executable(options.git_path): 798 if not test_git_executable(options.git_path):
544 option_parser.error('Invalid git executable.') 799 option_parser.error('Invalid git executable.')
545 800
546 config = DepsRollConfig(options) 801 config = DepsRollConfig(options)
547 find_hash_and_roll_deps(config, options.revision) 802 find_hash_and_roll_deps(config, options.revision, options.git_hash)
548 803
549 804
550 if __name__ == '__main__': 805 if __name__ == '__main__':
551 main(sys.argv[1:]) 806 main(sys.argv[1:])
552 807
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698