Index: git_drover.py |
diff --git a/git_drover.py b/git_drover.py |
new file mode 100755 |
index 0000000000000000000000000000000000000000..bfd4c83e715966af508b7c57c0f83cddd3c59e59 |
--- /dev/null |
+++ b/git_drover.py |
@@ -0,0 +1,506 @@ |
+#!/usr/bin/env python |
+# Copyright (c) 2014 The Chromium Authors. All rights reserved. |
+# Use of this source code is governed by a BSD-style license that can be |
+# found in the LICENSE file. |
+ |
+"""Merge/Revert changes to Chromium release branches. |
+ |
+This will use the git clone in the current directory if it matches the commit |
+you passed in. Alternately, run this script in an empty directory and it will |
+clone the appropriate repo for you (using `git cache` to do the smallest amount |
+of network IO possible). |
+ |
+This tool is aware of the following repos: |
+""" |
+ |
+import argparse |
+import collections |
+import multiprocessing |
+import os |
+import pprint |
+import re |
+import sys |
+import textwrap |
+import urllib2 |
+import urlparse |
+ |
+from multiprocessing.pool import ThreadPool |
+ |
+import git_cache |
+import git_common as git |
+ |
+from third_party import fancy_urllib |
+ |
+assert fancy_urllib.can_validate_certs() |
+ |
+CA_CERTS_FILE = os.path.abspath(os.path.join( |
+ os.path.dirname(__file__), 'third_party', 'boto', 'cacerts', 'cacerts.txt' |
+)) |
+ |
+urllib2.install_opener(urllib2.build_opener( |
+ fancy_urllib.FancyRedirectHandler(), |
+ fancy_urllib.FancyHTTPSHandler())) |
+ |
+ |
+MISSING = object() |
+ |
+OK_HOST_FMT = '%s.googlesource.com' |
+OK_REPOS = { |
+ 'chrome-internal': ('chrome/src-internal',), |
+ 'chromium': ('chromium/src', 'chromium/blink', |
+ 'native_client/src/native_client') |
+} |
+ |
+def repo_url(host, repo): |
+ assert host in OK_REPOS |
+ assert repo in OK_REPOS[host] |
+ return 'https://%s/%s.git' % (OK_HOST_FMT % host, repo) |
+ |
+# lambda avoids polluting module with variable names, but still executes at |
+# import-time. |
+# 'redefining builtin __doc__' pylint: disable=W0622 |
+__doc__ += (lambda: '\n'.join([ |
+ ' * %s' % repo_url(host, repo) |
+ for host, repos in OK_REPOS.iteritems() |
+ for repo in repos |
+]))() |
+ |
+ |
+def die(msg, *args): |
+ msg = textwrap.dedent(msg) |
+ if args: |
+ msg = msg % args |
+ print >> sys.stderr, msg |
+ sys.exit(1) |
+ |
+ |
+def retry(fn, args=(), kwargs=None, on=(), but_not=(), upto=3): |
+ kwargs = kwargs or {} |
+ for attempt in xrange(upto): |
+ try: |
+ return fn(*args, **kwargs) |
+ except but_not: |
+ raise |
+ except on: |
+ if attempt + 1 == upto: |
+ raise |
+ |
+ |
+################################################################################ |
+ |
+ |
+def announce(msg=None, msg_fn=lambda: None): |
+ print '=' * 80 |
+ if msg: |
+ print textwrap.dedent(msg) |
+ msg_fn() |
+ print '=' * 80 |
+ |
+ |
+def confirm(prompt='Is this correct?', abort='No changes have been made.'): |
+ while True: |
+ v = raw_input('%s (Y/n) ' % prompt) |
+ if v == '' or v in 'Yy': |
+ break |
+ if v in 'Nn': |
+ die('Aborting. %s', abort) |
+ |
+ |
+def summarize_job(correct_url, commits, target_ref, action): |
+ def _msg_fn(): |
+ preposition = 'to' if action == 'merge' else 'from' |
+ print "Planning to %s %d change%s %s branch %s of %s." % ( |
+ action, len(commits), 's' if len(commits) > 1 else '', |
+ preposition, target_ref.num, correct_url) |
+ for commit in commits: |
+ print git.run('show', '-s', '--format=%H\t%s', commit) |
+ announce(msg_fn=_msg_fn) |
+ |
+ |
+def ensure_working_directory(commits, target_ref): |
+ # TODO(iannucci): check all hashes locally after fetching first |
+ |
+ fetch_specs = [ |
+ '%s:%s' % (target_ref.remote_full_ref, target_ref.remote_full_ref) |
+ ] + commits |
+ |
+ if git.check('rev-parse', '--is-inside-work-tree'): |
+ actual_url = git.get_remote_url('origin') |
+ |
+ if not actual_url or not is_ok_repo(actual_url): |
+ die("""\ |
+ Inside a git repo, but origin's remote URL doesn't match one of the |
+ supported git repos. |
+ Current URL: %s""", actual_url) |
+ |
+ s = git.run('status', '--porcelain') |
+ if s: |
+ die("""\ |
+ Your current directory is usable for the command you specified, but it |
+ appears to be dirty (i.e. there are uncommitted changes). Please commit, |
+ freeze, or stash these changes and run this command again. |
+ |
+ %s""", '\n'.join(' '+l for l in s.splitlines())) |
+ |
+ correct_url = get_correct_url(commits, actual_url) |
+ if correct_url != actual_url: |
+ die("""\ |
+ The specified commits appear to be from a different repo than the repo |
+ in the current directory. |
+ Current Repo: %s |
+ Expected Repo: %s |
+ |
+ Please re-run this script in an empty working directory and we'll fetch |
+ the correct repo.""", actual_url, correct_url) |
+ |
+ m = git_cache.Mirror.from_repo('.') |
+ if m: |
+ m.populate(bootstrap=True, verbose=True) |
+ m.populate(fetch_specs=fetch_specs) |
+ |
+ elif len(os.listdir('.')) == 0: |
+ sample_path = '/path/to/cache' |
+ if sys.platform.startswith('win'): |
+ sample_path = r'X:\path\to\cache' |
+ if not git.config('cache.cachepath'): |
+ die("""\ |
+ Automatic drover checkouts require that you configure your global |
+ cachepath to make these automatic checkouts as fast as possible. Do this |
+ by running: |
+ git config --global cache.cachepath "%s" |
+ |
+ We recommend picking a non-network-mounted path with a decent amount of |
+ space (at least 4GB).""", sample_path) |
+ |
+ correct_url = get_correct_url(commits) |
+ |
+ m = git_cache.Mirror(correct_url) |
+ m.populate(bootstrap=True, verbose=True) |
+ m.populate(fetch_specs=fetch_specs) |
+ git.run('clone', '-s', '--no-checkout', m.mirror_path, '.') |
+ git.run('update-ref', '-d', 'refs/heads/master') |
+ else: |
+ die('You must either invoke this from a git repo, or from an empty dir.') |
+ |
+ for s in [target_ref.local_full_ref] + commits: |
+ git.check('fetch', 'origin', s) |
+ |
+ return correct_url |
+ |
+ |
+def find_hash_urls(commits, presumed_url=None): |
+ """Returns {url -> [commits]}. |
+ |
+ Args: |
+ commits - list of 40 char commit hashes |
+ presumed_url - the url to the first repo to try. If commits end up not |
+ existing in this repo, find_hash_urls will try all other |
+ known repos. |
+ """ |
+ pool = ThreadPool() |
+ |
+ def process_async_results(asr, results): |
+ """Resolves async results from |asr| into |results|. |
+ |
+ Args: |
+ asr - {commit -> [multiprocessing.pool.AsyncResult]} |
+ The async results are for a url or MISSING. |
+ results (in,out) - defaultdict({url -> set(commits)}) |
+ |
+ Returns a list of commits which did not have any non-MISSING result. |
agable
2014/04/28 21:38:22
"...which had only MISSING results."
|
+ """ |
+ try: |
+ lost_commits = [] |
+ passes = 0 |
+ while asr and passes <= 10: |
+ new_asr = {} |
+ for commit, attempts in asr.iteritems(): |
+ new_attempts = [] |
+ for attempt in attempts: |
+ try: |
+ attempt = attempt.get(.5) |
+ if attempt is not MISSING: |
+ results[attempt].add(commit) |
+ break |
+ except multiprocessing.TimeoutError: |
+ new_attempts.append(attempt) |
+ else: |
+ if new_attempts: |
+ new_asr[commit] = new_attempts |
+ else: |
+ lost_commits.append(commit) |
+ asr = new_asr |
+ passes += 1 |
+ lost_commits += asr.keys() |
+ return lost_commits |
+ except Exception: |
+ import traceback |
+ traceback.print_exc() |
+ |
+ def exists(url, commit): |
+ """Queries the repo at |url| for |commit|. |
+ |
+ Returns MISSING or the repo |url| |
+ """ |
+ query_url = '%s/+/%s?format=JSON' % (url, commit) |
+ return MISSING if GET(query_url) is MISSING else url |
+ |
+ def go_fish(commit, except_for=()): |
+ """Given a |commit|, search for it in all repos simultaneously, except for |
+ repos indicated by |except_for|. |
+ |
+ Returns the repo url for the commit or None. |
+ """ |
+ async_results = {commit: set()} |
+ for host, repos in OK_REPOS.iteritems(): |
+ for repo in repos: |
+ url = repo_url(host, repo) |
+ if url in except_for: |
+ continue |
+ async_results[commit].add( |
+ pool.apply_async(exists, args=(url, commit))) |
+ |
+ results = collections.defaultdict(set) |
+ lost = process_async_results(async_results, results) |
+ if not lost: |
+ return results.popitem()[0] |
+ |
+ # map of url -> set(commits) |
+ results = collections.defaultdict(set) |
+ |
+ # Try to find one hash which matches some repo |
+ while commits and not presumed_url: |
+ presumed_url = go_fish(commits[0]) |
+ results[presumed_url].add(commits[0]) |
+ commits = commits[1:] |
+ |
+ # map of commit -> attempts |
+ async_results = collections.defaultdict(list) |
+ for commit in commits: |
+ async_results[commit].append( |
+ pool.apply_async(exists, args=(presumed_url, commit))) |
+ |
+ lost = process_async_results(async_results, results) |
+ |
+ if lost: |
+ fishing_pool = ThreadPool() |
+ async_results = collections.defaultdict(list) |
+ for commit in lost: |
+ async_results[commit].append( |
+ fishing_pool.apply_async(go_fish, (commit,), |
+ {'except_for': presumed_url}) |
+ ) |
+ lost = process_async_results(async_results, results) |
+ if lost: |
+ results[None].update(lost) |
+ |
+ return {(k or 'UNKNOWN'): list(v) for k, v in results.iteritems()} |
+ |
+ |
+def GET(url, **kwargs): |
+ try: |
+ kwargs.setdefault('timeout', 5) |
+ request = fancy_urllib.FancyRequest(url) |
+ request.set_ssl_info(ca_certs=CA_CERTS_FILE) |
+ return retry(urllib2.urlopen, [request], kwargs, |
+ on=urllib2.URLError, but_not=urllib2.HTTPError, upto=3) |
+ except urllib2.HTTPError as e: |
+ if e.getcode() / 100 == 4: |
+ return MISSING |
+ raise |
+ |
+ |
+def get_correct_url(commits, presumed_url=None): |
+ unverified = commits |
+ if presumed_url: |
+ unverified = [c for c in unverified if not git.verify_commit(c)] |
+ if not unverified: |
+ return presumed_url |
+ git.cached_fetch(unverified) |
+ unverified = [c for c in unverified if not git.verify_commit(c)] |
+ if not unverified: |
+ return presumed_url |
+ |
+ url_hashes = find_hash_urls(unverified, presumed_url) |
+ if None in url_hashes: |
+ die("""\ |
+ Could not determine what repo the following commits originate from: |
+ %r""", url_hashes[None]) |
+ |
+ if len(url_hashes) > 1: |
+ die("""\ |
+ Ambiguous commits specified. You supplied multiple commits, but they |
+ appear to be from more than one repo? |
+ %s""", pprint.pformat(dict(url_hashes))) |
+ |
+ return url_hashes.popitem()[0] |
+ |
+ |
+def is_ok_repo(url): |
+ parsed = urlparse.urlsplit(url) |
+ |
+ if parsed.scheme == 'https': |
+ host = parsed.netloc[:-len(OK_HOST_FMT % '')] |
+ elif parsed.scheme == 'sso': |
+ host = parsed.netloc |
+ else: |
+ return False |
+ |
+ if host not in OK_REPOS: |
+ return False |
+ |
+ path = parsed.path.strip('/') |
+ if path.endswith('.git'): |
+ path = path[:-4] |
+ |
+ return path in OK_REPOS[host] |
+ |
+ |
+class NumberedBranch(collections.namedtuple('NumberedBranch', 'num')): |
+ # pylint: disable=W0232 |
+ @property |
+ def remote_full_ref(self): |
+ return 'refs/branch-heads/%d' % self.num |
+ |
+ @property |
+ def local_full_ref(self): |
+ return 'refs/origin/branch-heads/%d' % self.num |
+ |
+ |
+def parse_opts(): |
+ epilog = textwrap.dedent("""\ |
+ REF in the above may take the form of: |
+ DDDD - a numbered branch (i.e. refs/branch-heads/DDDD) |
+ """) |
+ |
+ commit_re = re.compile('^[0-9a-fA-F]{40}$') |
+ def commit_type(s): |
+ if not commit_re.match(s): |
+ raise argparse.ArgumentTypeError("%r is not a valid commit hash" % s) |
+ return s |
+ |
+ def ref_type(s): |
+ if not s: |
+ raise argparse.ArgumentTypeError("Empty ref: %r" % s) |
+ if not s.isdigit(): |
agable
2014/04/28 21:38:22
Doesn't handle 1780_21
|
+ raise argparse.ArgumentTypeError("Invalid ref: %r" % s) |
+ return NumberedBranch(int(s)) |
+ |
+ parser = argparse.ArgumentParser( |
+ description=__doc__, epilog=epilog, |
+ formatter_class=argparse.RawDescriptionHelpFormatter |
+ ) |
+ |
+ parser.add_argument('commit', nargs=1, metavar='HASH', |
+ type=commit_type, help='commit hash to revert/merge') |
+ |
+ parser.add_argument('--prep_only', action='store_true', default=False, |
+ help=( |
+ 'Prep and upload the CL (without sending mail) but ' |
+ 'don\'t push.')) |
+ |
+ parser.add_argument('--bug', metavar='NUM', action='append', dest='bugs', |
+ help='optional bug number(s)') |
+ |
+ grp = parser.add_mutually_exclusive_group(required=True) |
+ grp.add_argument('--merge_to', metavar='REF', type=ref_type, |
+ help='branch to merge to') |
+ grp.add_argument('--revert_from', metavar='REF', type=ref_type, |
+ help='branch ref to revert from') |
+ opts = parser.parse_args() |
+ |
+ # TODO(iannucci): Support multiple commits |
+ opts.commits = opts.commit |
+ del opts.commit |
+ |
+ if opts.merge_to: |
+ opts.action = 'merge' |
+ opts.ref = opts.merge_to |
+ elif opts.revert_from: |
+ opts.action = 'revert' |
+ opts.ref = opts.revert_from |
+ else: |
+ parser.error("?confusion? must specify either revert_from or merge_to") |
+ |
+ del opts.merge_to |
+ del opts.revert_from |
+ |
+ return opts |
+ |
+ |
+def main(): |
+ opts = parse_opts() |
+ |
+ announce('Preparing working directory') |
+ |
+ correct_url = ensure_working_directory(opts.commits, opts.ref) |
+ summarize_job(correct_url, opts.commits, opts.ref, opts.action) |
+ confirm() |
+ |
+ announce('Checking out branches to %s changes' % opts.action) |
+ |
+ git.run('fetch', 'origin', |
+ '%s:%s' % (opts.ref.remote_full_ref, opts.ref.local_full_ref)) |
+ git.check('update-ref', '-d', 'refs/heads/__drover_base') |
+ git.run('checkout', '-b', '__drover_base', opts.ref.local_full_ref, |
+ stdout=None, stderr=None) |
+ git.run('config', 'branch.__drover_base.remote', 'origin') |
+ git.run('config', 'branch.__drover_base.merge', opts.ref.remote_full_ref) |
+ git.check('update-ref', '-d', 'refs/heads/__drover_change') |
+ git.run('checkout', '-t', '__drover_base', '-b', '__drover_change', |
+ stdout=None, stderr=None) |
+ |
+ announce('Performing %s' % opts.action) |
+ |
+ # TODO(iannucci): support --signoff ? |
+ authors = [] |
+ for commit in opts.commits: |
+ success = False |
+ if opts.action == 'merge': |
+ success = git.check('cherry-pick', '-x', commit, verbose=True, |
+ stdout=None, stderr=None) |
+ else: # revert |
+ success = git.check('revert', '--no-edit', commit, verbose=True, |
+ stdout=None, stderr=None) |
+ if not success: |
+ die("Aborting. Failed to %s.", opts.action) |
+ |
+ email = git.run('show', '--format=%ae', '-s') |
+ # git-svn email addresses take the form of: |
+ # user@domain.com@<svn id> |
+ authors.append('@'.join(email.split('@', 2)[:2])) |
+ |
+ announce('Success! Uploading codereview...') |
+ |
+ if opts.prep_only: |
+ print "Prep only mode, uploading CL but not sending mail." |
+ mail = [] |
+ else: |
+ mail = ['--send-mail', '--reviewers=' + ','.join(authors)] |
+ |
+ args = [ |
+ '-c', 'gitcl.remotebranch=__drover_base', |
+ '-c', 'branch.__drover_change.base-url=%s' % correct_url, |
+ 'cl', 'upload', '--bypass-hooks' |
+ |
+ # TODO(iannucci): option to not bypass hooks? |
+ git.check(*args, stdout=None, stderr=None, stdin=None) |
+ |
+ if opts.prep_only: |
+ announce('Issue created. To push to the branch, run `git cl push`') |
+ else: |
+ announce('About to push! This will make the commit live.') |
+ confirm(abort=('Issue has been created, ' |
+ 'but change was not pushed to the repo.')) |
+ git.run('cl', 'push', '-f', '--bypass-hooks') |
+ |
+ return 0 |
+ |
+ |
+if __name__ == '__main__': |
+ sys.exit(main()) |