Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(155)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: tools/roll_deps.py

Issue 330423004: Use new common tools in Python scripts (Closed) Base URL: https://skia.googlesource.com/skia.git@master
Patch Set: Pylint Created 6 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« tools/git-sync-deps ('K') | « tools/misc_utils.py ('k') | no next file » | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: tools/roll_deps.py
diff --git a/tools/roll_deps.py b/tools/roll_deps.py
index d280bda11e6e54127ed29a59826bef2702684f95..18f4157bae0577e16db30487bb26e0c27e070964 100755
--- a/tools/roll_deps.py
+++ b/tools/roll_deps.py
@@ -28,12 +28,13 @@ import optparse
import os
import re
import shutil
-import subprocess
import sys
import tempfile
-import git_utils
-import misc_utils
+import fix_pythonpath # pylint: disable=W0611
+from common.py.utils import git_utils
+from common.py.utils import misc
+from common.py.utils import shell_utils
DEFAULT_BOTS_LIST = [
@@ -60,41 +61,32 @@ DEFAULT_BOTS_LIST = [
'win_layout_rel',
]
+REGEXP_SKIA_REVISION = (
+ r'^ "skia_revision": "(?P<revision>[0-9a-fA-F]{2,40})",$')
+
class DepsRollConfig(object):
"""Contains configuration options for this module.
Attributes:
- git: (string) The git executable.
chromium_path: (string) path to a local chromium git repository.
save_branches: (boolean) iff false, delete temporary branches.
verbose: (boolean) iff false, suppress the output from git-cl.
- search_depth: (int) how far back to look for the revision.
- skia_url: (string) Skia's git repository.
- self.skip_cl_upload: (boolean)
- self.cl_bot_list: (list of strings)
+ skip_cl_upload: (boolean)
+ cl_bot_list: (list of strings)
"""
# pylint: disable=I0011,R0903,R0902
def __init__(self, options=None):
- self.skia_url = 'https://skia.googlesource.com/skia.git'
- self.revision_format = (
- 'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ')
-
- self.git = git_utils.git_executable()
-
if not options:
options = DepsRollConfig.GetOptionParser()
# pylint: disable=I0011,E1103
self.verbose = options.verbose
- self.vsp = misc_utils.VerboseSubprocess(self.verbose)
self.save_branches = not options.delete_branches
- self.search_depth = options.search_depth
self.chromium_path = options.chromium_path
self.skip_cl_upload = options.skip_cl_upload
# Split and remove empty strigns from the bot list.
self.cl_bot_list = [bot for bot in options.bots.split(',') if bot]
- self.skia_git_checkout_path = options.skia_git_path
self.default_branch_name = 'autogenerated_deps_roll_branch'
self.reviewers_list = ','.join([
# 'rmistry@google.com',
@@ -125,23 +117,9 @@ class DepsRollConfig(object):
' if that environment variable is set.',
default=os.environ.get('CHROMIUM_CHECKOUT_PATH'))
option_parser.add_option(
- '-r', '--revision', type='int', default=None,
- help='The Skia SVN revision number, defaults to top of tree.')
- option_parser.add_option(
- '-g', '--git_hash', default=None,
- help='A partial Skia Git hash. Do not set this and revision.')
+ '-r', '--revision', default=None,
+ help='The Skia Git commit hash.')
- # Anyone using this script on a regular basis should set the
- # SKIA_GIT_CHECKOUT_PATH environment variable.
- option_parser.add_option(
- '', '--skia_git_path',
- help='Path of a pure-git Skia repository checkout. If empty,'
- ' a temporary will be cloned. Defaults to SKIA_GIT_CHECKOUT'
- '_PATH, if that environment variable is set.',
- default=os.environ.get('SKIA_GIT_CHECKOUT_PATH'))
- option_parser.add_option(
- '', '--search_depth', type='int', default=100,
- help='How far back to look for the revision.')
option_parser.add_option(
'', '--delete_branches', help='Delete the temporary branches',
action='store_true', dest='delete_branches', default=False)
@@ -169,236 +147,58 @@ class DepsRollError(Exception):
pass
-def get_svn_revision(config, commit):
- """Works in both git and git-svn. returns a string."""
- svn_format = (
- '(git-svn-id: [^@ ]+@|SVN changes up to revision |'
- 'LKGR w/ DEPS up to revision )(?P<return>[0-9]+)')
- svn_revision = misc_utils.ReSearch.search_within_output(
- config.verbose, svn_format, None,
- [config.git, 'log', '-n', '1', '--format=format:%B', commit])
- if not svn_revision:
- raise DepsRollError(
- 'Revision number missing from Chromium origin/master.')
- return int(svn_revision)
-
-
-class SkiaGitCheckout(object):
- """Class to create a temporary skia git checkout, if necessary.
- """
- # pylint: disable=I0011,R0903
-
- def __init__(self, config, depth):
- self._config = config
- self._depth = depth
- self._use_temp = None
- self._original_cwd = None
-
- def __enter__(self):
- config = self._config
- git = config.git
- skia_dir = None
- self._original_cwd = os.getcwd()
- if config.skia_git_checkout_path:
- if config.skia_git_checkout_path != os.curdir:
- skia_dir = config.skia_git_checkout_path
- ## Update origin/master if needed.
- if self._config.verbose:
- print '~~$', 'cd', skia_dir
- os.chdir(skia_dir)
- config.vsp.check_call([git, 'fetch', '-q', 'origin'])
- self._use_temp = None
- else:
- skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_')
- self._use_temp = skia_dir
- try:
- os.chdir(skia_dir)
- config.vsp.check_call(
- [git, 'clone', '-q', '--depth=%d' % self._depth,
- '--single-branch', config.skia_url, '.'])
- except (OSError, subprocess.CalledProcessError) as error:
- shutil.rmtree(skia_dir)
- raise error
-
- def __exit__(self, etype, value, traceback):
- if self._config.skia_git_checkout_path != os.curdir:
- if self._config.verbose:
- print '~~$', 'cd', self._original_cwd
- os.chdir(self._original_cwd)
- if self._use_temp:
- shutil.rmtree(self._use_temp)
-
-
-def revision_and_hash(config):
- """Finds revision number and git hash of origin/master in the Skia tree.
-
- Args:
- config: (roll_deps.DepsRollConfig) object containing options.
-
- Returns:
- A tuple (revision, hash)
- revision: (int) SVN revision number.
- git_hash: (string) full Git commit hash.
-
- Raises:
- roll_deps.DepsRollError: if the revision can't be found.
- OSError: failed to execute git or git-cl.
- subprocess.CalledProcessError: git returned unexpected status.
- """
- with SkiaGitCheckout(config, 1):
- revision = get_svn_revision(config, 'origin/master')
- git_hash = config.vsp.strip_output(
- [config.git, 'show-ref', 'origin/master', '--hash'])
- if not git_hash:
- raise DepsRollError('Git hash can not be found.')
- return revision, git_hash
-
-
-def revision_and_hash_from_revision(config, revision):
- """Finds revision number and git hash of a commit in the Skia tree.
-
- Args:
- config: (roll_deps.DepsRollConfig) object containing options.
- revision: (int) SVN revision number.
-
- Returns:
- A tuple (revision, hash)
- revision: (int) SVN revision number.
- git_hash: (string) full Git commit hash.
-
- Raises:
- roll_deps.DepsRollError: if the revision can't be found.
- OSError: failed to execute git or git-cl.
- subprocess.CalledProcessError: git returned unexpected status.
- """
- with SkiaGitCheckout(config, config.search_depth):
- revision_regex = config.revision_format % revision
- git_hash = config.vsp.strip_output(
- [config.git, 'log', '--grep', revision_regex,
- '--format=format:%H', 'origin/master'])
- if not git_hash:
- raise DepsRollError('Git hash can not be found.')
- return revision, git_hash
-
-
-def revision_and_hash_from_partial(config, partial_hash):
- """Returns the SVN revision number and full git hash.
-
- Args:
- config: (roll_deps.DepsRollConfig) object containing options.
- partial_hash: (string) Partial git commit hash.
-
- Returns:
- A tuple (revision, hash)
- revision: (int) SVN revision number.
- git_hash: (string) full Git commit hash.
-
- Raises:
- roll_deps.DepsRollError: if the revision can't be found.
- OSError: failed to execute git or git-cl.
- subprocess.CalledProcessError: git returned unexpected status.
- """
- with SkiaGitCheckout(config, config.search_depth):
- git_hash = config.vsp.strip_output(
- ['git', 'log', '-n', '1', '--format=format:%H', partial_hash])
- if not git_hash:
- raise DepsRollError('Partial Git hash can not be found.')
- revision = get_svn_revision(config, git_hash)
- return revision, git_hash
-
-
-def change_skia_deps(revision, git_hash, depspath):
+def change_skia_deps(revision, depspath):
"""Update the DEPS file.
- Modify the skia_revision and skia_hash entries in the given DEPS file.
+ Modify the skia_revision entry in the given DEPS file.
Args:
- revision: (int) Skia SVN revision.
- git_hash: (string) Skia Git hash.
+ revision: (string) Skia commit hash.
depspath: (string) path to DEPS file.
"""
temp_file = tempfile.NamedTemporaryFile(delete=False,
prefix='skia_DEPS_ROLL_tmp_')
try:
- deps_regex_rev = re.compile('"skia_revision": "[0-9]*",')
- deps_regex_hash = re.compile('"skia_hash": "[0-9a-f]*",')
-
- deps_regex_rev_repl = '"skia_revision": "%d",' % revision
- deps_regex_hash_repl = '"skia_hash": "%s",' % git_hash
+ deps_regex_rev = re.compile(REGEXP_SKIA_REVISION)
+ deps_regex_rev_repl = ' "skia_revision": "%s",' % revision
with open(depspath, 'r') as input_stream:
for line in input_stream:
line = deps_regex_rev.sub(deps_regex_rev_repl, line)
- line = deps_regex_hash.sub(deps_regex_hash_repl, line)
temp_file.write(line)
finally:
temp_file.close()
shutil.move(temp_file.name, depspath)
-def git_cl_uploader(config, message, file_list):
- """Create a commit in the current git branch; upload via git-cl.
-
- Assumes that you are already on the branch you want to be on.
+def submit_tries(bots_to_run, dry_run=False):
+ """Submit try requests for the current branch on the given bots.
Args:
- config: (roll_deps.DepsRollConfig) object containing options.
- message: (string) the commit message, can be multiline.
- file_list: (list of strings) list of filenames to pass to `git add`.
-
- Returns:
- The output of `git cl issue`, if not config.skip_cl_upload, else ''.
+ bots_to_run: (list of strings) bots to request.
+ dry_run: (bool) whether to actually submit the try request.
"""
-
- git, vsp = config.git, config.vsp
- svn_info = str(get_svn_revision(config, 'HEAD'))
-
- for filename in file_list:
- assert os.path.exists(filename)
- vsp.check_call([git, 'add', filename])
-
- vsp.check_call([git, 'commit', '-q', '-m', message])
-
- git_cl = [git, 'cl', 'upload', '-f',
- '--bypass-hooks', '--bypass-watchlists']
- if config.cc_list:
- git_cl.append('--cc=%s' % config.cc_list)
- if config.reviewers_list:
- git_cl.append('--reviewers=%s' % config.reviewers_list)
-
git_try = [
- git, 'cl', 'try', '-m', 'tryserver.chromium', '--revision', svn_info]
- git_try.extend([arg for bot in config.cl_bot_list for arg in ('-b', bot)])
-
- branch_name = git_utils.git_branch_name(vsp.verbose)
+ git_utils.GIT, 'cl', 'try', '-m', 'tryserver.chromium']
+ git_try.extend([arg for bot in bots_to_run for arg in ('-b', bot)])
- if config.skip_cl_upload:
+ if dry_run:
space = ' '
print 'You should call:'
- print '%scd %s' % (space, os.getcwd())
- misc_utils.print_subprocess_args(space, [git, 'checkout', branch_name])
- misc_utils.print_subprocess_args(space, git_cl)
- if config.cl_bot_list:
- misc_utils.print_subprocess_args(space, git_try)
+ print space, git_try
print
- return ''
else:
- vsp.check_call(git_cl)
- issue = vsp.strip_output([git, 'cl', 'issue'])
- if config.cl_bot_list:
- vsp.check_call(git_try)
- return issue
+ shell_utils.run(git_try)
-def roll_deps(config, revision, git_hash):
+def roll_deps(config, revision):
"""Upload changed DEPS and a whitespace change.
Given the correct git_hash, create two Reitveld issues.
Args:
config: (roll_deps.DepsRollConfig) object containing options.
- revision: (int) Skia SVN revision.
- git_hash: (string) Skia Git hash.
+ revision: (string) Skia Git hash.
Returns:
a tuple containing textual description of the two issues.
@@ -408,115 +208,66 @@ def roll_deps(config, revision, git_hash):
subprocess.CalledProcessError: git returned unexpected status.
"""
- git = config.git
- with misc_utils.ChangeDir(config.chromium_path, config.verbose):
- config.vsp.check_call([git, 'fetch', '-q', 'origin'])
-
- old_revision = misc_utils.ReSearch.search_within_output(
- config.verbose, '"skia_revision": "(?P<return>[0-9]+)",', None,
- [git, 'show', 'origin/master:DEPS'])
+ with misc.ChDir(config.chromium_path, verbose=config.verbose):
+ git_utils.Fetch()
+ output = shell_utils.run([git_utils.GIT, 'show', 'origin/master:DEPS'],
+ log_in_real_time=False).rstrip()
+ match = re.search(REGEXP_SKIA_REVISION, output, flags=re.MULTILINE)
+ old_revision = None
+ if match:
+ old_revision = match.group('revision')
assert old_revision
- if revision == int(old_revision):
- print 'DEPS is up to date!'
- return (None, None)
- master_hash = config.vsp.strip_output(
- [git, 'show-ref', 'origin/master', '--hash'])
- master_revision = get_svn_revision(config, 'origin/master')
+ master_hash = git_utils.FullHash('origin/master').rstrip()
# master_hash[8] gives each whitespace CL a unique name.
- if config.save_branches:
- branch = 'control_%s' % master_hash[:8]
- else:
- branch = None
+ branch = 'control_%s' % master_hash[:8]
message = ('whitespace change %s\n\n'
- 'Chromium base revision: %d / %s\n\n'
+ 'Chromium base revision: %s\n\n'
'This CL was created by Skia\'s roll_deps.py script.\n'
- ) % (master_hash[:8], master_revision, master_hash[:8])
- with git_utils.ChangeGitBranch(branch, 'origin/master',
- config.verbose):
- branch = git_utils.git_branch_name(config.vsp.verbose)
-
- with open('build/whitespace_file.txt', 'a') as output_stream:
- output_stream.write('\nCONTROL\n')
-
- whitespace_cl = git_cl_uploader(
- config, message, ['build/whitespace_file.txt'])
-
- control_url = misc_utils.ReSearch.search_within_string(
- whitespace_cl, '(?P<return>https?://[^) ]+)', '?')
+ ) % (master_hash[:8], master_hash[:8])
+ with git_utils.GitBranch(branch, message,
+ delete_when_finished=not config.save_branches,
+ upload=not config.skip_cl_upload
+ ) as whitespace_branch:
+ branch = git_utils.GetCurrentBranch()
+ with open(os.path.join('build', 'whitespace_file.txt'), 'a') as f:
+ f.write('\nCONTROL\n')
+
+ control_url = whitespace_branch.commit_and_upload()
+ if config.cl_bot_list:
+ submit_tries(config.cl_bot_list, dry_run=config.skip_cl_upload)
+ whitespace_cl = control_url
if config.save_branches:
- whitespace_cl = '%s\n branch: %s' % (whitespace_cl, branch)
+ whitespace_cl += '\n branch: %s' % branch
- if config.save_branches:
- branch = 'roll_%d_%s' % (revision, master_hash[:8])
- else:
- branch = None
+ branch = 'roll_%s_%s' % (revision, master_hash[:8])
message = (
- 'roll skia DEPS to %d\n\n'
- 'Chromium base revision: %d / %s\n'
+ 'roll skia DEPS to %s\n\n'
+ 'Chromium base revision: %s\n'
'Old Skia revision: %s\n'
- 'New Skia revision: %d\n'
+ 'New Skia revision: %s\n'
'Control CL: %s\n\n'
'This CL was created by Skia\'s roll_deps.py script.\n\n'
'Bypassing commit queue trybots:\n'
'NOTRY=true\n'
- % (revision, master_revision, master_hash[:8],
- old_revision, revision, control_url))
- with git_utils.ChangeGitBranch(branch, 'origin/master',
- config.verbose):
- branch = git_utils.git_branch_name(config.vsp.verbose)
-
- change_skia_deps(revision, git_hash, 'DEPS')
- deps_cl = git_cl_uploader(config, message, ['DEPS'])
+ % (revision, master_hash[:8],
+ old_revision[:8], revision[:8], control_url))
+ with git_utils.GitBranch(branch, message,
+ delete_when_finished=not config.save_branches,
+ upload=not config.skip_cl_upload
+ ) as roll_branch:
+ change_skia_deps(revision, 'DEPS')
+ deps_url = roll_branch.commit_and_upload()
+ if config.cl_bot_list:
+ submit_tries(config.cl_bot_list, dry_run=config.skip_cl_upload)
+ deps_cl = deps_url
if config.save_branches:
- deps_cl = '%s\n branch: %s' % (deps_cl, branch)
+ deps_cl += '\n branch: %s' % branch
return deps_cl, whitespace_cl
-def find_hash_and_roll_deps(config, revision=None, partial_hash=None):
- """Call find_hash_from_revision() and roll_deps().
-
- The calls to git will be verbose on standard output. After a
- successful upload of both issues, print links to the new
- codereview issues.
-
- Args:
- config: (roll_deps.DepsRollConfig) object containing options.
- revision: (int or None) the Skia SVN revision number or None
- to use the tip of the tree.
- partial_hash: (string or None) a partial pure-git Skia commit
- hash. Don't pass both partial_hash and revision.
-
- Raises:
- roll_deps.DepsRollError: if the revision can't be found.
- OSError: failed to execute git or git-cl.
- subprocess.CalledProcessError: git returned unexpected status.
- """
-
- if revision and partial_hash:
- raise DepsRollError('Pass revision or partial_hash, not both.')
-
- if partial_hash:
- revision, git_hash = revision_and_hash_from_partial(
- config, partial_hash)
- elif revision:
- revision, git_hash = revision_and_hash_from_revision(config, revision)
- else:
- revision, git_hash = revision_and_hash(config)
-
- print 'revision=%r\nhash=%r\n' % (revision, git_hash)
-
- deps_issue, whitespace_issue = roll_deps(config, revision, git_hash)
-
- if deps_issue and whitespace_issue:
- print 'DEPS roll:\n %s\n' % deps_issue
- print 'Whitespace change:\n %s\n' % whitespace_issue
- else:
- print >> sys.stderr, 'No issues created.'
-
-
def main(args):
"""main function; see module-level docstring and GetOptionParser help.
@@ -526,18 +277,23 @@ def main(args):
option_parser = DepsRollConfig.GetOptionParser()
options = option_parser.parse_args(args)[0]
+ if not options.revision:
+ option_parser.error('Must specify a revision.')
if not options.chromium_path:
option_parser.error('Must specify chromium_path.')
if not os.path.isdir(options.chromium_path):
option_parser.error('chromium_path must be a directory.')
- if not git_utils.git_executable():
- option_parser.error('Invalid git executable.')
-
config = DepsRollConfig(options)
- find_hash_and_roll_deps(config, options.revision, options.git_hash)
+ shell_utils.VERBOSE = options.verbose
+ deps_issue, whitespace_issue = roll_deps(config, options.revision)
+
+ if deps_issue and whitespace_issue:
+ print 'DEPS roll:\n %s\n' % deps_issue
+ print 'Whitespace change:\n %s\n' % whitespace_issue
+ else:
+ print >> sys.stderr, 'No issues created.'
if __name__ == '__main__':
main(sys.argv[1:])
-
« tools/git-sync-deps ('K') | « tools/misc_utils.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698