OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/python2 |
| 2 |
| 3 # Copyright 2014 Google Inc. |
| 4 # |
| 5 # Use of this source code is governed by a BSD-style license that can be |
| 6 # found in the LICENSE file. |
| 7 |
| 8 """Skia's Chromium DEPS roll script. |
| 9 |
| 10 This script: |
| 11 - searches through the last N Skia git commits to find out the hash that is |
| 12 associated with the SVN revision number. |
| 13 - creates a new branch in the Chromium tree, modifies the DEPS file to |
| 14 point at the given Skia commit, commits, uploads to Rietveld, and |
| 15 deletes the local copy of the branch. |
| 16 - creates a whitespace-only commit and uploads that to to Rietveld. |
| 17 - returns the Chromium tree to its previous state. |
| 18 |
| 19 Usage: |
| 20 %prog -c CHROMIUM_PATH -r REVISION [OPTIONAL_OPTIONS] |
| 21 """ |
| 22 |
| 23 |
| 24 import optparse |
| 25 import os |
| 26 import re |
| 27 import shutil |
| 28 import subprocess |
| 29 from subprocess import check_call |
| 30 import sys |
| 31 import tempfile |
| 32 |
| 33 |
| 34 class DepsRollConfig(object): |
| 35 """Contains configuration options for this module. |
| 36 |
| 37 Attributes: |
| 38 git: (string) The git executable. |
| 39 chromium_path: (string) path to a local chromium git repository. |
| 40 save_branches: (boolean) iff false, delete temporary branches. |
| 41 verbose: (boolean) iff false, suppress the output from git-cl. |
| 42 search_depth: (int) how far back to look for the revision. |
| 43 skia_url: (string) Skia's git repository. |
| 44 self.skip_cl_upload: (boolean) |
| 45 self.cl_bot_list: (list of strings) |
| 46 """ |
| 47 |
| 48 # pylint: disable=I0011,R0903,R0902 |
| 49 def __init__(self, options=None): |
| 50 self.skia_url = 'https://skia.googlesource.com/skia.git' |
| 51 self.revision_format = ( |
| 52 'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ') |
| 53 |
| 54 if not options: |
| 55 options = DepsRollConfig.GetOptionParser() |
| 56 # pylint: disable=I0011,E1103 |
| 57 self.verbose = options.verbose |
| 58 self.save_branches = options.save_branches |
| 59 self.search_depth = options.search_depth |
| 60 self.chromium_path = options.chromium_path |
| 61 self.git = options.git_path |
| 62 self.skip_cl_upload = options.skip_cl_upload |
| 63 # Split and remove empty strigns from the bot list. |
| 64 self.cl_bot_list = [bot for bot in options.bots.split(',') if bot] |
| 65 self.skia_git_checkout_path = options.skia_git_path |
| 66 self.default_branch_name = 'autogenerated_deps_roll_branch' |
| 67 |
| 68 @staticmethod |
| 69 def GetOptionParser(): |
| 70 # pylint: disable=I0011,C0103 |
| 71 """Returns an optparse.OptionParser object. |
| 72 |
| 73 Returns: |
| 74 An optparse.OptionParser object. |
| 75 |
| 76 Called by the main() function. |
| 77 """ |
| 78 default_bots_list = [ |
| 79 'android_clang_dbg', |
| 80 'android_dbg', |
| 81 'android_rel', |
| 82 'cros_daisy', |
| 83 'linux', |
| 84 'linux_asan', |
| 85 'linux_chromeos', |
| 86 'linux_chromeos_asan', |
| 87 'linux_gpu', |
| 88 'linux_heapcheck', |
| 89 'linux_layout', |
| 90 'linux_layout_rel', |
| 91 'mac', |
| 92 'mac_asan', |
| 93 'mac_gpu', |
| 94 'mac_layout', |
| 95 'mac_layout_rel', |
| 96 'win', |
| 97 'win_gpu', |
| 98 'win_layout', |
| 99 'win_layout_rel', |
| 100 ] |
| 101 |
| 102 option_parser = optparse.OptionParser(usage=__doc__) |
| 103 # Anyone using this script on a regular basis should set the |
| 104 # CHROMIUM_CHECKOUT_PATH environment variable. |
| 105 option_parser.add_option( |
| 106 '-c', '--chromium_path', help='Path to local Chromium Git' |
| 107 ' repository checkout, defaults to CHROMIUM_CHECKOUT_PATH' |
| 108 ' if that environment variable is set.', |
| 109 default=os.environ.get('CHROMIUM_CHECKOUT_PATH')) |
| 110 option_parser.add_option( |
| 111 '-r', '--revision', type='int', default=None, |
| 112 help='The Skia SVN revision number, defaults to top of tree.') |
| 113 # Anyone using this script on a regular basis should set the |
| 114 # SKIA_GIT_CHECKOUT_PATH environment variable. |
| 115 option_parser.add_option( |
| 116 '', '--skia_git_path', |
| 117 help='Path of a pure-git Skia repository checkout. If empty,' |
| 118 ' a temporary will be cloned. Defaults to SKIA_GIT_CHECKOUT' |
| 119 '_PATH, if that environment variable is set.', |
| 120 default=os.environ.get('SKIA_GIT_CHECKOUT_PATH')) |
| 121 option_parser.add_option( |
| 122 '', '--search_depth', type='int', default=100, |
| 123 help='How far back to look for the revision.') |
| 124 option_parser.add_option( |
| 125 '', '--git_path', help='Git executable, defaults to "git".', |
| 126 default='git') |
| 127 option_parser.add_option( |
| 128 '', '--save_branches', help='Save the temporary branches', |
| 129 action='store_true', dest='save_branches', default=False) |
| 130 option_parser.add_option( |
| 131 '', '--verbose', help='Do not suppress the output from `git cl`.', |
| 132 action='store_true', dest='verbose', default=False) |
| 133 option_parser.add_option( |
| 134 '', '--skip_cl_upload', help='Skip the cl upload step; useful' |
| 135 ' for testing or with --save_branches.', |
| 136 action='store_true', default=False) |
| 137 |
| 138 default_bots_help = ( |
| 139 'Comma-separated list of bots, defaults to a list of %d bots.' |
| 140 ' To skip `git cl try`, set this to an empty string.' |
| 141 % len(default_bots_list)) |
| 142 default_bots = ','.join(default_bots_list) |
| 143 option_parser.add_option( |
| 144 '', '--bots', help=default_bots_help, default=default_bots) |
| 145 |
| 146 return option_parser |
| 147 |
| 148 |
| 149 def test_git_executable(git_executable): |
| 150 """Test the git executable. |
| 151 |
| 152 Args: |
| 153 git_executable: git executable path. |
| 154 Returns: |
| 155 True if test is successful. |
| 156 """ |
| 157 with open(os.devnull, 'w') as devnull: |
| 158 try: |
| 159 subprocess.call([git_executable, '--version'], stdout=devnull) |
| 160 except (OSError,): |
| 161 return False |
| 162 return True |
| 163 |
| 164 |
| 165 class DepsRollError(Exception): |
| 166 """Exceptions specific to this module.""" |
| 167 pass |
| 168 |
| 169 |
| 170 def strip_output(*args, **kwargs): |
| 171 """Wrap subprocess.check_output and str.strip(). |
| 172 |
| 173 Pass the given arguments into subprocess.check_output() and return |
| 174 the results, after stripping any excess whitespace. |
| 175 |
| 176 Args: |
| 177 *args: to be passed to subprocess.check_output() |
| 178 **kwargs: to be passed to subprocess.check_output() |
| 179 |
| 180 Returns: |
| 181 The output of the process as a string without leading or |
| 182 trailing whitespace. |
| 183 Raises: |
| 184 OSError or subprocess.CalledProcessError: raised by check_output. |
| 185 """ |
| 186 return str(subprocess.check_output(*args, **kwargs)).strip() |
| 187 |
| 188 |
| 189 def create_temp_skia_clone(config, depth): |
| 190 """Clones Skia in a temp dir. |
| 191 |
| 192 Args: |
| 193 config: (roll_deps.DepsRollConfig) object containing options. |
| 194 depth: (int) how far back to clone the tree. |
| 195 Returns: |
| 196 temporary directory path if succcessful. |
| 197 Raises: |
| 198 OSError, subprocess.CalledProcessError on failure. |
| 199 """ |
| 200 git = config.git |
| 201 skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_') |
| 202 try: |
| 203 check_call( |
| 204 [git, 'clone', '-q', '--depth=%d' % depth, |
| 205 '--single-branch', config.skia_url, skia_dir]) |
| 206 return skia_dir |
| 207 except (OSError, subprocess.CalledProcessError) as error: |
| 208 shutil.rmtree(skia_dir) |
| 209 raise error |
| 210 |
| 211 |
| 212 def find_revision_and_hash(config, revision): |
| 213 """Finds revision number and git hash of origin/master in the Skia tree. |
| 214 |
| 215 Args: |
| 216 config: (roll_deps.DepsRollConfig) object containing options. |
| 217 revision: (int or None) SVN revision number. If None, use |
| 218 tip-of-tree. |
| 219 |
| 220 Returns: |
| 221 A tuple (revision, hash) |
| 222 revision: (int) SVN revision number. |
| 223 hash: (string) full Git commit hash. |
| 224 |
| 225 Raises: |
| 226 roll_deps.DepsRollError: if the revision can't be found. |
| 227 OSError: failed to execute git or git-cl. |
| 228 subprocess.CalledProcessError: git returned unexpected status. |
| 229 """ |
| 230 git = config.git |
| 231 use_temp = False |
| 232 skia_dir = None |
| 233 depth = 1 if (revision is None) else config.search_depth |
| 234 try: |
| 235 if config.skia_git_checkout_path: |
| 236 skia_dir = config.skia_git_checkout_path |
| 237 ## Update origin/master if needed. |
| 238 check_call([git, 'fetch', '-q', 'origin'], cwd=skia_dir) |
| 239 else: |
| 240 skia_dir = create_temp_skia_clone(config, depth) |
| 241 assert skia_dir |
| 242 use_temp = True |
| 243 |
| 244 if revision is None: |
| 245 message = subprocess.check_output( |
| 246 [git, 'log', '-n', '1', '--format=format:%B', |
| 247 'origin/master'], cwd=skia_dir) |
| 248 svn_format = ( |
| 249 'git-svn-id: http://skia.googlecode.com/svn/trunk@([0-9]+) ') |
| 250 search = re.search(svn_format, message) |
| 251 if not search: |
| 252 raise DepsRollError( |
| 253 'Revision number missing from origin/master.') |
| 254 revision = int(search.group(1)) |
| 255 git_hash = strip_output( |
| 256 [git, 'show-ref', 'origin/master', '--hash'], cwd=skia_dir) |
| 257 else: |
| 258 revision_regex = config.revision_format % revision |
| 259 git_hash = strip_output( |
| 260 [git, 'log', '--grep', revision_regex, '--format=format:%H', |
| 261 'origin/master'], cwd=skia_dir) |
| 262 |
| 263 if revision < 0 or not git_hash: |
| 264 raise DepsRollError('Git hash can not be found.') |
| 265 return revision, git_hash |
| 266 finally: |
| 267 if use_temp: |
| 268 shutil.rmtree(skia_dir) |
| 269 |
| 270 |
| 271 class GitBranchCLUpload(object): |
| 272 """Class to manage git branches and git-cl-upload. |
| 273 |
| 274 This class allows one to create a new branch in a repository based |
| 275 off of origin/master, make changes to the tree inside the |
| 276 with-block, upload that new branch to Rietveld, restore the original |
| 277 tree state, and delete the local copy of the new branch. |
| 278 |
| 279 See roll_deps() for an example of use. |
| 280 |
| 281 Constructor Args: |
| 282 config: (roll_deps.DepsRollConfig) object containing options. |
| 283 message: (string) the commit message, can be multiline. |
| 284 set_brach_name: (string or none) if not None, the name of the |
| 285 branch to use. If None, then use a temporary branch that |
| 286 will be deleted. |
| 287 |
| 288 Attributes: |
| 289 issue: a string describing the codereview issue, after __exit__ |
| 290 has been called, othrwise, None. |
| 291 |
| 292 Raises: |
| 293 OSError: failed to execute git or git-cl. |
| 294 subprocess.CalledProcessError: git returned unexpected status. |
| 295 """ |
| 296 # pylint: disable=I0011,R0903,R0902 |
| 297 |
| 298 def __init__(self, config, message, set_branch_name): |
| 299 self._message = message |
| 300 self._file_list = [] |
| 301 self._branch_name = set_branch_name |
| 302 self._stash = None |
| 303 self._original_branch = None |
| 304 self._config = config |
| 305 self._svn_info = None |
| 306 self.issue = None |
| 307 |
| 308 def stage_for_commit(self, *paths): |
| 309 """Calls `git add ...` on each argument. |
| 310 |
| 311 Args: |
| 312 *paths: (list of strings) list of filenames to pass to `git add`. |
| 313 """ |
| 314 self._file_list.extend(paths) |
| 315 |
| 316 def __enter__(self): |
| 317 git = self._config.git |
| 318 def branch_exists(branch): |
| 319 """Return true iff branch exists.""" |
| 320 return 0 == subprocess.call( |
| 321 [git, 'show-ref', '--quiet', branch]) |
| 322 def has_diff(): |
| 323 """Return true iff repository has uncommited changes.""" |
| 324 return bool(subprocess.call([git, 'diff', '--quiet', 'HEAD'])) |
| 325 self._stash = has_diff() |
| 326 if self._stash: |
| 327 check_call([git, 'stash', 'save']) |
| 328 try: |
| 329 self._original_branch = strip_output( |
| 330 [git, 'symbolic-ref', '--short', 'HEAD']) |
| 331 except (subprocess.CalledProcessError,): |
| 332 self._original_branch = strip_output( |
| 333 [git, 'rev-parse', 'HEAD']) |
| 334 |
| 335 if not self._branch_name: |
| 336 self._branch_name = self._config.default_branch_name |
| 337 |
| 338 if branch_exists(self._branch_name): |
| 339 check_call([git, 'checkout', '-q', 'master']) |
| 340 check_call([git, 'branch', '-q', '-D', self._branch_name]) |
| 341 |
| 342 check_call( |
| 343 [git, 'checkout', '-q', '-b', |
| 344 self._branch_name, 'origin/master']) |
| 345 |
| 346 svn_info = subprocess.check_output(['git', 'svn', 'info']) |
| 347 svn_info_search = re.search(r'Last Changed Rev: ([0-9]+)\W', svn_info) |
| 348 assert svn_info_search |
| 349 self._svn_info = svn_info_search.group(1) |
| 350 |
| 351 def __exit__(self, etype, value, traceback): |
| 352 # pylint: disable=I0011,R0912 |
| 353 git = self._config.git |
| 354 def quiet_check_call(*args, **kwargs): |
| 355 """Call check_call, but pipe output to devnull.""" |
| 356 with open(os.devnull, 'w') as devnull: |
| 357 check_call(*args, stdout=devnull, **kwargs) |
| 358 |
| 359 for filename in self._file_list: |
| 360 assert os.path.exists(filename) |
| 361 check_call([git, 'add', filename]) |
| 362 check_call([git, 'commit', '-q', '-m', self._message]) |
| 363 |
| 364 git_cl = [git, 'cl', 'upload', '-f', '--cc=skia-team@google.com', |
| 365 '--bypass-hooks', '--bypass-watchlists'] |
| 366 git_try = [git, 'cl', 'try', '--revision', self._svn_info] |
| 367 git_try.extend([arg for bot in self._config.cl_bot_list |
| 368 for arg in ('-b', bot)]) |
| 369 |
| 370 if self._config.skip_cl_upload: |
| 371 print ' '.join(git_cl) |
| 372 print |
| 373 if self._config.cl_bot_list: |
| 374 print ' '.join(git_try) |
| 375 print |
| 376 self.issue = '' |
| 377 else: |
| 378 if self._config.verbose: |
| 379 check_call(git_cl) |
| 380 print |
| 381 else: |
| 382 quiet_check_call(git_cl) |
| 383 self.issue = strip_output([git, 'cl', 'issue']) |
| 384 if self._config.cl_bot_list: |
| 385 if self._config.verbose: |
| 386 check_call(git_try) |
| 387 print |
| 388 else: |
| 389 quiet_check_call(git_try) |
| 390 |
| 391 # deal with the aftermath of failed executions of this script. |
| 392 if self._config.default_branch_name == self._original_branch: |
| 393 self._original_branch = 'master' |
| 394 check_call([git, 'checkout', '-q', self._original_branch]) |
| 395 |
| 396 if self._config.default_branch_name == self._branch_name: |
| 397 check_call([git, 'branch', '-q', '-D', self._branch_name]) |
| 398 if self._stash: |
| 399 check_call([git, 'stash', 'pop']) |
| 400 |
| 401 |
| 402 def change_skia_deps(revision, git_hash, depspath): |
| 403 """Update the DEPS file. |
| 404 |
| 405 Modify the skia_revision and skia_hash entries in the given DEPS file. |
| 406 |
| 407 Args: |
| 408 revision: (int) Skia SVN revision. |
| 409 git_hash: (string) Skia Git hash. |
| 410 depspath: (string) path to DEPS file. |
| 411 """ |
| 412 temp_file = tempfile.NamedTemporaryFile(delete=False, |
| 413 prefix='skia_DEPS_ROLL_tmp_') |
| 414 try: |
| 415 deps_regex_rev = re.compile('"skia_revision": "[0-9]*",') |
| 416 deps_regex_hash = re.compile('"skia_hash": "[0-9a-f]*",') |
| 417 |
| 418 deps_regex_rev_repl = '"skia_revision": "%d",' % revision |
| 419 deps_regex_hash_repl = '"skia_hash": "%s",' % git_hash |
| 420 |
| 421 with open(depspath, 'r') as input_stream: |
| 422 for line in input_stream: |
| 423 line = deps_regex_rev.sub(deps_regex_rev_repl, line) |
| 424 line = deps_regex_hash.sub(deps_regex_hash_repl, line) |
| 425 temp_file.write(line) |
| 426 finally: |
| 427 temp_file.close() |
| 428 shutil.move(temp_file.name, depspath) |
| 429 |
| 430 |
| 431 def branch_name(message): |
| 432 """Return the first line of a commit message to be used as a branch name. |
| 433 |
| 434 Args: |
| 435 message: (string) |
| 436 |
| 437 Returns: |
| 438 A string derived from message suitable for a branch name. |
| 439 """ |
| 440 return message.lstrip().split('\n')[0].rstrip().replace(' ', '_') |
| 441 |
| 442 |
| 443 def roll_deps(config, revision, git_hash): |
| 444 """Upload changed DEPS and a whitespace change. |
| 445 |
| 446 Given the correct git_hash, create two Reitveld issues. |
| 447 |
| 448 Args: |
| 449 config: (roll_deps.DepsRollConfig) object containing options. |
| 450 revision: (int) Skia SVN revision. |
| 451 git_hash: (string) Skia Git hash. |
| 452 |
| 453 Returns: |
| 454 a tuple containing textual description of the two issues. |
| 455 |
| 456 Raises: |
| 457 OSError: failed to execute git or git-cl. |
| 458 subprocess.CalledProcessError: git returned unexpected status. |
| 459 """ |
| 460 git = config.git |
| 461 cwd = os.getcwd() |
| 462 os.chdir(config.chromium_path) |
| 463 try: |
| 464 check_call([git, 'fetch', '-q', 'origin']) |
| 465 master_hash = strip_output( |
| 466 [git, 'show-ref', 'origin/master', '--hash']) |
| 467 |
| 468 # master_hash[8] gives each whitespace CL a unique name. |
| 469 message = ('whitespace change %s\n\nThis CL was created by' |
| 470 ' Skia\'s roll_deps.py script.\n') % master_hash[:8] |
| 471 branch = branch_name(message) if config.save_branches else None |
| 472 |
| 473 codereview = GitBranchCLUpload(config, message, branch) |
| 474 with codereview: |
| 475 with open('build/whitespace_file.txt', 'a') as output_stream: |
| 476 output_stream.write('\nCONTROL\n') |
| 477 codereview.stage_for_commit('build/whitespace_file.txt') |
| 478 whitespace_cl = codereview.issue |
| 479 if branch: |
| 480 whitespace_cl = '%s\n branch: %s' % (whitespace_cl, branch) |
| 481 control_url_match = re.search('https?://[^) ]+', codereview.issue) |
| 482 if control_url_match: |
| 483 message = ('roll skia DEPS to %d\n\nThis CL was created by' |
| 484 ' Skia\'s roll_deps.py script.\n\ncontrol: %s' |
| 485 % (revision, control_url_match.group(0))) |
| 486 else: |
| 487 message = ('roll skia DEPS to %d\n\nThis CL was created by' |
| 488 ' Skia\'s roll_deps.py script.') % revision |
| 489 branch = branch_name(message) if config.save_branches else None |
| 490 codereview = GitBranchCLUpload(config, message, branch) |
| 491 with codereview: |
| 492 change_skia_deps(revision, git_hash, 'DEPS') |
| 493 codereview.stage_for_commit('DEPS') |
| 494 deps_cl = codereview.issue |
| 495 if branch: |
| 496 deps_cl = '%s\n branch: %s' % (deps_cl, branch) |
| 497 |
| 498 return deps_cl, whitespace_cl |
| 499 finally: |
| 500 os.chdir(cwd) |
| 501 |
| 502 |
| 503 def find_hash_and_roll_deps(config, revision): |
| 504 """Call find_hash_from_revision() and roll_deps(). |
| 505 |
| 506 The calls to git will be verbose on standard output. After a |
| 507 successful upload of both issues, print links to the new |
| 508 codereview issues. |
| 509 |
| 510 Args: |
| 511 config: (roll_deps.DepsRollConfig) object containing options. |
| 512 revision: (int or None) the Skia SVN revision number or None |
| 513 to use the tip of the tree. |
| 514 |
| 515 Raises: |
| 516 roll_deps.DepsRollError: if the revision can't be found. |
| 517 OSError: failed to execute git or git-cl. |
| 518 subprocess.CalledProcessError: git returned unexpected status. |
| 519 """ |
| 520 revision, git_hash = find_revision_and_hash(config, revision) |
| 521 |
| 522 print 'revision=%r\nhash=%r\n' % (revision, git_hash) |
| 523 |
| 524 deps_issue, whitespace_issue = roll_deps(config, revision, git_hash) |
| 525 |
| 526 print 'DEPS roll:\n %s\n' % deps_issue |
| 527 print 'Whitespace change:\n %s\n' % whitespace_issue |
| 528 |
| 529 |
| 530 def main(args): |
| 531 """main function; see module-level docstring and GetOptionParser help. |
| 532 |
| 533 Args: |
| 534 args: sys.argv[1:]-type argument list. |
| 535 """ |
| 536 option_parser = DepsRollConfig.GetOptionParser() |
| 537 options = option_parser.parse_args(args)[0] |
| 538 |
| 539 if not options.chromium_path: |
| 540 option_parser.error('Must specify chromium_path.') |
| 541 if not os.path.isdir(options.chromium_path): |
| 542 option_parser.error('chromium_path must be a directory.') |
| 543 if not test_git_executable(options.git_path): |
| 544 option_parser.error('Invalid git executable.') |
| 545 |
| 546 config = DepsRollConfig(options) |
| 547 find_hash_and_roll_deps(config, options.revision) |
| 548 |
| 549 |
| 550 if __name__ == '__main__': |
| 551 main(sys.argv[1:]) |
| 552 |
OLD | NEW |