DEPS roll script

tools/roll_deps.py

+#!/usr/bin/python2
+
+# Copyright 2014 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Skia's Chromium DEPS roll script.
+
+This script:
+- searches through the last N Skia git commits to find out the hash that is
+  associated with the SVN revision number.
+- creates a new branch in the Chromium tree, modifies the DEPS file to
+  point at the given Skia commit, commits, uploads to Rietveld, and
+  deletes the local copy of the branch.
+- creates a whitespace-only commit and uploads that to to Rietveld.
+- returns the Chromium tree to its previous state.
+
+Usage:
+  %prog -c CHROMIUM_PATH -r REVISION [OPTIONAL_OPTIONS]
+"""
+
+
+import optparse
+import os
+import re
+import shutil
+import subprocess
+from subprocess import check_call
+import sys
+import tempfile
+
+
+class Config(object):

[borenet, 2014/01/06 19:04:28]

This could be DepsRollConfig or something more descriptive.

[hal.canary, 2014/01/06 19:29:29]

Done.

+    """Contains configuration options for this module.
+
+    Attributes:
+        git: (string) The git executable.
+        chromium_path: (string) path to a local chromium git repository.
+        save_branches: (boolean) iff false, delete temporary branches.
+        verbose: (boolean)  iff false, suppress the output from git-cl.
+        search_depth: (int) how far back to look for the revision.
+        skia_url: (string) Skia's git repository.
+        self.skip_cl_upload: (boolean)
+        self.cl_bot_list: (list of strings)
+    """
+
+    # pylint: disable=I0011,R0903,R0902
+    def __init__(self, options=None):
+        self.skia_url = 'https://skia.googlesource.com/skia.git'
+        self.revision_format = (
+            'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ')
+
+        if not options:
+            options = Config.GetOptionParser()
+        # pylint: disable=I0011,E1103
+        self.verbose = options.verbose
+        self.save_branches = options.save_branches
+        self.search_depth = options.search_depth
+        self.chromium_path = options.chromium_path
+        self.git = options.git_path
+        self.skip_cl_upload = options.skip_cl_upload
+        # Split and remove empty strigns from the bot list.
+        self.cl_bot_list = [bot for bot in options.bots.split(',') if bot]
+        self.skia_git_checkout_path = options.skia_git_path
+        self.default_branch_name = 'autogenerated_deps_roll_branch'
+
+    @staticmethod
+    def GetOptionParser():
+        # pylint: disable=I0011,C0103
+        """Returns an optparse.OptionParser object.
+
+        Returns:
+            An optparse.OptionParser object.
+
+        Called by the main() function.
+        """
+        default_bots_list = [
+            'android_clang_dbg',
+            'android_dbg',
+            'android_rel',
+            'cros_daisy',
+            'linux',
+            'linux_asan',
+            'linux_chromeos',
+            'linux_chromeos_asan',
+            'linux_gpu',
+            'linux_heapcheck',
+            'linux_layout',
+            'linux_layout_rel',
+            'mac',
+            'mac_asan',
+            'mac_gpu',
+            'mac_layout',
+            'mac_layout_rel',
+            'win',
+            'win_gpu',
+            'win_layout',
+            'win_layout_rel',

[borenet, 2014/01/06 19:04:28]

I dislike hard-coded lists like this, but I don't think there's anything we can
do about it.

+            ]
+
+        option_parser = optparse.OptionParser(usage=__doc__)
+        # Anyone using this script on a regular basis should set the
+        # CHROMIUM_REPO_PATH environment variable.
+        option_parser.add_option(
+            '-c', '--chromium_path', help='Path to Chromium Git repository,'
+            ' defaults to CHROMIUM_REPO_PATH if that environment variable'
+            ' is set.',
+            default=os.environ.get('CHROMIUM_REPO_PATH'))
+        option_parser.add_option(
+            '-r', '--revision', type='int', default=-1,
+            help='The Skia SVN revision number, defaults to top of tree.')
+        # Anyone using this script on a regular basis should set the
+        # SKIA_GIT_REPO_PATH environment variable.

[borenet, 2014/01/06 19:04:28]

For clarity, I think this should be "CHECKOUT" instead of "REPO" to indicate
that this is a local checkout rather than the repository URL.

[hal.canary, 2014/01/06 19:29:29]

Done.

+        option_parser.add_option(
+            '', '--skia_git_path',
+            help='Path of a pure-git Skia repo. If empty, a temporary will'
+            ' be cloned.  Defaults to SKIA_GIT_REPO_PATH, if that '
+            'environment variable is set.',
+            default=os.environ.get('SKIA_GIT_REPO_PATH'))
+        option_parser.add_option(
+            '', '--search_depth', help='How far back to look for the revision',
+            type='int', default=100)
+        option_parser.add_option(
+            '', '--git_path', help='Git executable, defaults to "git".',
+            default='git')
+        option_parser.add_option(
+            '', '--save_branches', help='Save the temporary branches',
+            action='store_true', dest='save_branches', default=False)
+        option_parser.add_option(
+            '', '--verbose', help='Do not suppress the output from `git cl`.',
+            action='store_true', dest='verbose', default=False)
+        option_parser.add_option(
+            '', '--skip_cl_upload', help='Skip the cl upload step; useful'
+            ' for testing or with --save_branches.',
+            action='store_true', default=False)
+
+        default_bots_help = (
+            'Comma-separated list of bots, defaults to a list of %d bots.'
+            '  To skip `git cl try`, set this to an empty string.'
+            % len(default_bots_list))
+        default_bots = ','.join(default_bots_list)
+        option_parser.add_option(
+            '', '--bots', help=default_bots_help, default=default_bots)
+
+        return option_parser
+
+
+def test_git_executable(git_executable):
+    """Test the git executable.
+
+    Args:
+        git_executable: git executable path.
+    Returns:
+        True if test is successful.
+    """
+    with open(os.devnull, 'w') as devnull:
+        try:
+            subprocess.call([git_executable, '--version'], stdout=devnull)
+        except (OSError,):
+            return False
+    return True
+
+
+class Error(Exception):

[borenet, 2014/01/06 19:04:28]

Please either name this something more specific or just use Exception
throughout.

[hal.canary, 2014/01/06 19:29:29]

Done.

+    """Exceptions specific to this module."""
+    pass
+
+
+def strip_output(*args, **kwargs):
+    """Wrap subprocess.check_output and str.strip().
+
+    Pass the given arguments into subprocess.check_output() and return
+    the results, after stripping any excess whitespace.
+
+    Args:
+        *args: to be passed to subprocess.check_output()
+        **kwargs: to be passed to subprocess.check_output()
+
+    Returns:
+        The output of the process as a string without leading or
+        trailing whitespace.
+    Raises:
+        OSError or subprocess.CalledProcessError: raised by check_output.
+    """
+    return str(subprocess.check_output(*args, **kwargs)).strip()
+
+
+def create_temp_skia_clone(config, depth):
+    """Clones Skia in a temp dir.
+
+    Args:
+        config: (roll_deps.Config) object containing options.
+        depth: (int) how far back to clone the tree.
+    Returns:
+        temporary directory path if succcessful.
+    Raises:
+        OSError, subprocess.CalledProcessError on failure.
+    """
+    git = config.git
+    skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_')
+    try:
+        check_call(
+            [git, 'clone', '-q', '--depth=%d' % depth,
+             '--single-branch', config.skia_url, skia_dir])
+        return skia_dir
+    except (OSError, subprocess.CalledProcessError) as error:
+        shutil.rmtree(skia_dir)
+        raise error
+
+
+def find_revision_and_hash(config, revision):
+    """Finds revision number and git hash of origin/master in the Skia tree.
+
+    Args:
+        config: (roll_deps.Config) object containing options.
+        revision: (int) SVN revision number.  If -1, use tip-of-tree

[borenet, 2014/01/06 19:04:28]

Why not allow the default to be None?

[hal.canary, 2014/01/06 19:29:29]

Done.

+
+    Returns:
+        A tuple (revision, hash)
+            revision: (int) SVN revision number.
+            hash: (string) full Git commit hash.
+
+    Raises:
+        roll_deps.Error: if the revision can't be found.
+        OSError: failed to execute git or git-cl.
+        subprocess.CalledProcessError: git returned unexpected status.
+    """
+    git = config.git
+    use_temp = False
+    skia_dir = None
+    depth = 1 if (-1 == revision) else config.search_depth
+    try:
+        if config.skia_git_checkout_path:
+            skia_dir = config.skia_git_checkout_path
+            ## Update origin/master if needed.
+            check_call([git, 'fetch', '-q', 'origin'], cwd=skia_dir)
+        else:
+            skia_dir = create_temp_skia_clone(config, depth)
+            assert skia_dir
+            use_temp = True
+
+        if -1 == revision:

[borenet, 2014/01/06 19:04:28]

Again, I'd really rather use None as a magic value than -1.

[hal.canary, 2014/01/06 19:29:29]

Done.

+            message = subprocess.check_output(
+                [git, 'log', '-n', '1', '--format=format:%B', 'origin/master'],
+                cwd=skia_dir)
+            svn_format = (
+                'git-svn-id: http://skia.googlecode.com/svn/trunk@([0-9]+) ')
+            search = re.search(svn_format, message)
+            if not search:
+                raise Error('Revision number missing from origin/master.')
+            revision = int(search.group(1))
+            git_hash = strip_output(
+                [git, 'show-ref', 'origin/master', '--hash'], cwd=skia_dir)
+        else:
+            revision_regex = config.revision_format % revision
+            git_hash = strip_output(
+                [git, 'log', '--grep', revision_regex, '--format=format:%H',
+                 'origin/master'], cwd=skia_dir)
+
+        if revision < 0  or not git_hash:
+            raise Error('Git hash can not be found.')
+        return revision, git_hash
+    finally:
+        if use_temp:
+            shutil.rmtree(skia_dir)
+
+
+class GitBranchCLUpload(object):
+    """Class to manage git branches and git-cl-upload.
+
+    This class allows one to create a new branch in a repository based
+    off of origin/master, make changes to the tree inside the
+    with-block, upload that new branch to Rietveld, restore the original
+    tree state, and delete the local copy of the new branch.
+
+    See roll_deps() for an example of use.
+
+    Constructor Args:
+        config: (roll_deps.Config) object containing options.
+        message: (string) the commit message, can be multiline.
+        set_brach_name: (string or none) if not None, the name of the
+            branch to use.  If None, then use a temporary branch that
+            will be deleted.
+
+    Attributes:
+        issue: a string describing the codereview issue, after __exit__
+            has been called, othrwise, None.
+
+    Raises:
+        OSError: failed to execute git or git-cl.
+        subprocess.CalledProcessError: git returned unexpected status.
+    """
+    # pylint: disable=I0011,R0903,R0902
+
+    def __init__(self, config, message, set_branch_name):
+        self._message = message
+        self._file_list = []
+        self._branch_name = set_branch_name
+        self._stash = None
+        self._original_branch = None
+        self._config = config
+        self._svn_info = None
+        self.issue = None
+
+    def stage_for_commit(self, *paths):
+        """Calls `git add ...` on each argument.
+
+        Args:
+            *paths: (list of strings) list of filenames to pass to `git add`.
+        """
+        self._file_list.extend(paths)
+
+    def __enter__(self):
+        git = self._config.git
+        def branch_exists(branch):
+            """Return true iff branch exists."""
+            return 0 == subprocess.call(
+                [git, 'show-ref', '--quiet', branch])
+        def has_diff():
+            """Return true iff repository has uncommited changes."""
+            return bool(subprocess.call([git, 'diff', '--quiet', 'HEAD']))
+        self._stash = has_diff()
+        if self._stash:
+            check_call([git, 'stash', 'save'])
+        try:
+            self._original_branch = strip_output(
+                [git, 'symbolic-ref', '--short', 'HEAD'])
+        except (subprocess.CalledProcessError,):
+            self._original_branch = strip_output(
+                [git, 'rev-parse', 'HEAD'])
+
+        if not self._branch_name:
+            self._branch_name = self._config.default_branch_name
+
+        if branch_exists(self._branch_name):
+            check_call([git, 'checkout', '-q', 'master'])
+            check_call([git, 'branch', '-q', '-D', self._branch_name])
+
+        check_call(
+            [git, 'checkout', '-q', '-b',
+             self._branch_name, 'origin/master'])
+
+        svn_info = subprocess.check_output(['git', 'svn', 'info'])
+        svn_info_search = re.search(r'Last Changed Rev: ([0-9]+)\W', svn_info)
+        assert svn_info_search
+        self._svn_info = svn_info_search.group(1)
+
+    def __exit__(self, etype, value, traceback):
+        # pylint: disable=I0011,R0912
+        git = self._config.git
+        def quiet_check_call(*args, **kwargs):
+            """Call check_call, but pipe output to devnull."""
+            with open(os.devnull, 'w') as devnull:
+                check_call(*args, stdout=devnull, **kwargs)
+
+        for filename in self._file_list:
+            assert os.path.exists(filename)
+            check_call([git, 'add', filename])
+        check_call([git, 'commit', '-q', '-m', self._message])
+
+        git_cl = [git, 'cl', 'upload', '-f', '--cc=skia-team@google.com',
+                  '--bypass-hooks', '--bypass-watchlists']
+        git_try = [git, 'cl', 'try', '--revision', self._svn_info]
+        git_try.extend([arg for bot in self._config.cl_bot_list
+                        for arg in ('-b', bot)])
+
+        if self._config.skip_cl_upload:
+            print ' '.join(git_cl)
+            print
+            if self._config.cl_bot_list:
+                print ' '.join(git_try)
+                print
+            self.issue = ''
+        else:
+            if self._config.verbose:
+                check_call(git_cl)
+                print
+            else:
+                quiet_check_call(git_cl)
+            self.issue = strip_output([git, 'cl', 'issue'])
+            if self._config.cl_bot_list:
+                if self._config.verbose:
+                    check_call(git_try)
+                    print
+                else:
+                    quiet_check_call(git_try)
+
+        # deal with the aftermath of failed executions of this script.
+        if self._config.default_branch_name == self._original_branch:
+            self._original_branch = 'master'
+        check_call([git, 'checkout', '-q', self._original_branch])
+
+        if self._config.default_branch_name == self._branch_name:
+            check_call([git, 'branch', '-q', '-D', self._branch_name])
+        if self._stash:
+            check_call([git, 'stash', 'pop'])
+
+
+def change_skia_deps(revision, git_hash, depspath):
+    """Update the DEPS file.
+
+    Modify the skia_revision and skia_hash entries in the given DEPS file.
+
+    Args:
+        revision: (int) Skia SVN revision.
+        git_hash: (string) Skia Git hash.
+        depspath: (string) path to DEPS file.
+    """
+    temp_file = tempfile.NamedTemporaryFile(delete=False,
+                                            prefix='skia_DEPS_ROLL_tmp_')
+    try:
+        deps_regex_rev = re.compile('"skia_revision": "[0-9]*",')
+        deps_regex_hash = re.compile('"skia_hash": "[0-9a-f]*",')
+
+        deps_regex_rev_repl = '"skia_revision": "%d",' % revision
+        deps_regex_hash_repl = '"skia_hash": "%s",' % git_hash
+
+        with open(depspath, 'r') as input_stream:
+            for line in input_stream:
+                line = deps_regex_rev.sub(deps_regex_rev_repl, line)
+                line = deps_regex_hash.sub(deps_regex_hash_repl, line)
+                temp_file.write(line)
+    finally:
+        temp_file.close()
+    shutil.move(temp_file.name, depspath)
+
+
+def branch_name(message):
+    """Return the first line of a commit message to be used as a branch name.
+
+    Args:
+        message: (string)
+
+    Returns:
+        A string derived from message suitable for a branch name.
+    """
+    return message.lstrip().split('\n')[0].rstrip().replace(' ', '_')
+
+
+def roll_deps(config, revision, git_hash):
+    """Upload changed DEPS and a whitespace change.
+
+    Given the correct git_hash, create two Reitveld issues.
+
+    Args:
+        config: (roll_deps.Config) object containing options.
+        revision: (int) Skia SVN revision.
+        git_hash: (string) Skia Git hash.
+
+    Returns:
+        a tuple containing textual description of the two issues.
+
+    Raises:
+        OSError: failed to execute git or git-cl.
+        subprocess.CalledProcessError: git returned unexpected status.
+    """
+    git = config.git
+    cwd = os.getcwd()
+    os.chdir(config.chromium_path)
+    try:
+        check_call([git, 'fetch', '-q', 'origin'])
+        master_hash = strip_output(
+            [git, 'show-ref', 'origin/master', '--hash'])
+
+        # master_hash[8] gives each whitespace CL a unique name.
+        message = ('whitespace change %s\n\nThis CL was created by'
+                   ' Skia\'s roll_deps.py script.\n') % master_hash[:8]
+        branch = branch_name(message) if config.save_branches else None
+
+        codereview = GitBranchCLUpload(config, message, branch)
+        with codereview:
+            with open('build/whitespace_file.txt', 'a') as output_stream:
+                output_stream.write('\nCONTROL\n')
+            codereview.stage_for_commit('build/whitespace_file.txt')
+        whitespace_cl = codereview.issue
+        if branch:
+            whitespace_cl = '%s\n    branch: %s' % (whitespace_cl, branch)
+        control_url_match = re.search('https?://[^) ]+', codereview.issue)
+        if control_url_match:
+            message = ('roll skia DEPS to %d\n\nThis CL was created by'
+                       ' Skia\'s roll_deps.py script.\n\n'
+                       'control: %s') % (revision, control_url_match.group(0))
+        else:
+            message = ('roll skia DEPS to %d\n\nThis CL was created by'
+                       ' Skia\'s roll_deps.py script.') % revision
+        branch = branch_name(message) if config.save_branches else None
+        codereview = GitBranchCLUpload(config, message, branch)
+        with codereview:
+            change_skia_deps(revision, git_hash, 'DEPS')
+            codereview.stage_for_commit('DEPS')
+        deps_cl = codereview.issue
+        if branch:
+            deps_cl = '%s\n    branch: %s' % (deps_cl, branch)
+
+        return deps_cl, whitespace_cl
+    finally:
+        os.chdir(cwd)
+
+
+def find_hash_and_roll_deps(config, revision):
+    """Call find_hash_from_revision() and roll_deps().
+
+    The calls to git will be verbose on standard output.  After a
+    successful upload of both issues, print links to the new
+    codereview issues.
+
+    Args:
+        config: (roll_deps.Config) object containing options.
+        revision: (int) the Skia SVN revision number or -1.
+
+    Raises:
+        roll_deps.Error: if the revision can't be found.
+        OSError: failed to execute git or git-cl.
+        subprocess.CalledProcessError: git returned unexpected status.
+    """
+    revision, git_hash = find_revision_and_hash(config, revision)
+
+    print 'revision=%r\nhash=%r\n' % (revision, git_hash)
+
+    deps_issue, whitespace_issue = roll_deps(config, revision, git_hash)
+
+    print 'DEPS roll:\n    %s\n' % deps_issue
+    print 'Whitespace change:\n    %s\n' % whitespace_issue
+
+
+def main(args):
+    """main function; see module-level docstring and GetOptionParser help.
+
+    Args:
+        args: sys.argv[1:]-type argument list.
+    """
+    option_parser = Config.GetOptionParser()
+    options = option_parser.parse_args(args)[0]
+
+    if not options.chromium_path:
+        option_parser.error('Must specify chromium_path.')
+    if not os.path.isdir(options.chromium_path):
+        option_parser.error('chromium_path must be a directory.')
+    if not test_git_executable(options.git_path):
+        option_parser.error('Invalid git executable.')
+
+    config = Config(options)
+    find_hash_and_roll_deps(config, options.revision)
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
+