Use new common tools in Python scripts

tools/roll_deps.py

 #!/usr/bin/python2
 
 # Copyright 2014 Google Inc.
 #
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Skia's Chromium DEPS roll script.
 
 This script:
@@ skipping 10 matching lines @@
 
 Usage:
   %prog -c CHROMIUM_PATH -r REVISION [OPTIONAL_OPTIONS]
 """
 
 
 import optparse
 import os
 import re
 import shutil
-import subprocess
 import sys
 import tempfile
 
-import git_utils
-import misc_utils
+import fix_pythonpath # pylint: disable=W0611
+from common.py.utils import git_utils
+from common.py.utils import misc
+from common.py.utils import shell_utils
 
 
 DEFAULT_BOTS_LIST = [
     'android_clang_dbg',
     'android_dbg',
     'android_rel',
     'cros_daisy',
     'linux',
     'linux_asan',
     'linux_chromeos',
     'linux_chromeos_asan',
     'linux_chromium_gn_dbg',
     'linux_gpu',
     'linux_layout',
     'linux_layout_rel',
     'mac',
     'mac_asan',
     'mac_gpu',
     'mac_layout',
     'mac_layout_rel',
     'win',
     'win_gpu',
     'win_layout',
     'win_layout_rel',
 ]
 
+REGEXP_SKIA_REVISION = (
+    r'^  "skia_revision": "(?P<revision>[0-9a-fA-F]{2,40})",$')
+
 
 class DepsRollConfig(object):
     """Contains configuration options for this module.
 
     Attributes:
-        git: (string) The git executable.
         chromium_path: (string) path to a local chromium git repository.
         save_branches: (boolean) iff false, delete temporary branches.
         verbose: (boolean)  iff false, suppress the output from git-cl.
-        search_depth: (int) how far back to look for the revision.
-        skia_url: (string) Skia's git repository.
-        self.skip_cl_upload: (boolean)
-        self.cl_bot_list: (list of strings)
+        skip_cl_upload: (boolean)
+        cl_bot_list: (list of strings)
     """
 
     # pylint: disable=I0011,R0903,R0902
     def __init__(self, options=None):
-        self.skia_url = 'https://skia.googlesource.com/skia.git'
-        self.revision_format = (
-            'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ')
-
-        self.git = git_utils.git_executable()
-
         if not options:
             options = DepsRollConfig.GetOptionParser()
         # pylint: disable=I0011,E1103
         self.verbose = options.verbose
-        self.vsp = misc_utils.VerboseSubprocess(self.verbose)
         self.save_branches = not options.delete_branches
-        self.search_depth = options.search_depth
         self.chromium_path = options.chromium_path
         self.skip_cl_upload = options.skip_cl_upload
         # Split and remove empty strigns from the bot list.
         self.cl_bot_list = [bot for bot in options.bots.split(',') if bot]
-        self.skia_git_checkout_path = options.skia_git_path
         self.default_branch_name = 'autogenerated_deps_roll_branch'
         self.reviewers_list = ','.join([
             # 'rmistry@google.com',
             # 'reed@google.com',
             # 'bsalomon@google.com',
             # 'robertphillips@google.com',
             ])
         self.cc_list = ','.join([
             # 'skia-team@google.com',
             ])
@@ skipping 10 matching lines @@
         """
         option_parser = optparse.OptionParser(usage=__doc__)
         # Anyone using this script on a regular basis should set the
         # CHROMIUM_CHECKOUT_PATH environment variable.
         option_parser.add_option(
             '-c', '--chromium_path', help='Path to local Chromium Git'
             ' repository checkout, defaults to CHROMIUM_CHECKOUT_PATH'
             ' if that environment variable is set.',
             default=os.environ.get('CHROMIUM_CHECKOUT_PATH'))
         option_parser.add_option(
-            '-r', '--revision', type='int', default=None,
-            help='The Skia SVN revision number, defaults to top of tree.')
-        option_parser.add_option(
-            '-g', '--git_hash', default=None,
-            help='A partial Skia Git hash.  Do not set this and revision.')
+            '-r', '--revision', default=None,
+            help='The Skia Git commit hash.')
 
-        # Anyone using this script on a regular basis should set the
-        # SKIA_GIT_CHECKOUT_PATH environment variable.
-        option_parser.add_option(
-            '', '--skia_git_path',
-            help='Path of a pure-git Skia repository checkout.  If empty,'
-            ' a temporary will be cloned.  Defaults to SKIA_GIT_CHECKOUT'
-            '_PATH, if that environment variable is set.',
-            default=os.environ.get('SKIA_GIT_CHECKOUT_PATH'))
-        option_parser.add_option(
-            '', '--search_depth', type='int', default=100,
-            help='How far back to look for the revision.')
         option_parser.add_option(
             '', '--delete_branches', help='Delete the temporary branches',
             action='store_true', dest='delete_branches', default=False)
         option_parser.add_option(
             '', '--verbose', help='Do not suppress the output from `git cl`.',
             action='store_true', dest='verbose', default=False)
         option_parser.add_option(
             '', '--skip_cl_upload', help='Skip the cl upload step; useful'
             ' for testing.',
             action='store_true', default=False)
 
         default_bots_help = (
             'Comma-separated list of bots, defaults to a list of %d bots.'
             '  To skip `git cl try`, set this to an empty string.'
             % len(DEFAULT_BOTS_LIST))
         default_bots = ','.join(DEFAULT_BOTS_LIST)
         option_parser.add_option(
             '', '--bots', help=default_bots_help, default=default_bots)
 
         return option_parser
 
 
 class DepsRollError(Exception):
     """Exceptions specific to this module."""
     pass
 
 
-def get_svn_revision(config, commit):
-    """Works in both git and git-svn. returns a string."""
-    svn_format = (
-        '(git-svn-id: [^@ ]+@|SVN changes up to revision |'
-        'LKGR w/ DEPS up to revision )(?P<return>[0-9]+)')
-    svn_revision = misc_utils.ReSearch.search_within_output(
-        config.verbose, svn_format, None,
-        [config.git, 'log', '-n', '1', '--format=format:%B', commit])
-    if not svn_revision:
-        raise DepsRollError(
-            'Revision number missing from Chromium origin/master.')
-    return int(svn_revision)
+def change_skia_deps(revision, depspath):
+    """Update the DEPS file.
 
-
-class SkiaGitCheckout(object):
-    """Class to create a temporary skia git checkout, if necessary.
-    """
-    # pylint: disable=I0011,R0903
-
-    def __init__(self, config, depth):
-        self._config = config
-        self._depth = depth
-        self._use_temp = None
-        self._original_cwd = None
-
-    def __enter__(self):
-        config = self._config
-        git = config.git
-        skia_dir = None
-        self._original_cwd = os.getcwd()
-        if config.skia_git_checkout_path:
-            if config.skia_git_checkout_path != os.curdir:
-                skia_dir = config.skia_git_checkout_path
-                ## Update origin/master if needed.
-                if self._config.verbose:
-                    print '~~$', 'cd', skia_dir
-                os.chdir(skia_dir)
-            config.vsp.check_call([git, 'fetch', '-q', 'origin'])
-            self._use_temp = None
-        else:
-            skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_')
-            self._use_temp = skia_dir
-            try:
-                os.chdir(skia_dir)
-                config.vsp.check_call(
-                    [git, 'clone', '-q', '--depth=%d' % self._depth,
-                     '--single-branch', config.skia_url, '.'])
-            except (OSError, subprocess.CalledProcessError) as error:
-                shutil.rmtree(skia_dir)
-                raise error
-
-    def __exit__(self, etype, value, traceback):
-        if self._config.skia_git_checkout_path != os.curdir:
-            if self._config.verbose:
-                print '~~$', 'cd', self._original_cwd
-            os.chdir(self._original_cwd)
-        if self._use_temp:
-            shutil.rmtree(self._use_temp)
-
-
-def revision_and_hash(config):
-    """Finds revision number and git hash of origin/master in the Skia tree.
+    Modify the skia_revision entry in the given DEPS file.
 
     Args:
-        config: (roll_deps.DepsRollConfig) object containing options.
-
-    Returns:
-        A tuple (revision, hash)
-            revision: (int) SVN revision number.
-            git_hash: (string) full Git commit hash.
-
-    Raises:
-        roll_deps.DepsRollError: if the revision can't be found.
-        OSError: failed to execute git or git-cl.
-        subprocess.CalledProcessError: git returned unexpected status.
-    """
-    with SkiaGitCheckout(config, 1):
-        revision = get_svn_revision(config, 'origin/master')
-        git_hash = config.vsp.strip_output(
-            [config.git, 'show-ref', 'origin/master', '--hash'])
-        if not git_hash:
-            raise DepsRollError('Git hash can not be found.')
-    return revision, git_hash
-
-
-def revision_and_hash_from_revision(config, revision):
-    """Finds revision number and git hash of a commit in the Skia tree.
-
-    Args:
-        config: (roll_deps.DepsRollConfig) object containing options.
-        revision: (int) SVN revision number.
-
-    Returns:
-        A tuple (revision, hash)
-            revision: (int) SVN revision number.
-            git_hash: (string) full Git commit hash.
-
-    Raises:
-        roll_deps.DepsRollError: if the revision can't be found.
-        OSError: failed to execute git or git-cl.
-        subprocess.CalledProcessError: git returned unexpected status.
-    """
-    with SkiaGitCheckout(config, config.search_depth):
-        revision_regex = config.revision_format % revision
-        git_hash = config.vsp.strip_output(
-            [config.git, 'log', '--grep', revision_regex,
-             '--format=format:%H', 'origin/master'])
-        if not git_hash:
-            raise DepsRollError('Git hash can not be found.')
-    return revision, git_hash
-
-
-def revision_and_hash_from_partial(config, partial_hash):
-    """Returns the SVN revision number and full git hash.
-
-    Args:
-        config: (roll_deps.DepsRollConfig) object containing options.
-        partial_hash: (string) Partial git commit hash.
-
-    Returns:
-        A tuple (revision, hash)
-            revision: (int) SVN revision number.
-            git_hash: (string) full Git commit hash.
-
-    Raises:
-        roll_deps.DepsRollError: if the revision can't be found.
-        OSError: failed to execute git or git-cl.
-        subprocess.CalledProcessError: git returned unexpected status.
-    """
-    with SkiaGitCheckout(config, config.search_depth):
-        git_hash = config.vsp.strip_output(
-            ['git', 'log', '-n', '1', '--format=format:%H', partial_hash])
-        if not git_hash:
-            raise DepsRollError('Partial Git hash can not be found.')
-        revision = get_svn_revision(config, git_hash)
-    return revision, git_hash
-
-
-def change_skia_deps(revision, git_hash, depspath):
-    """Update the DEPS file.
-
-    Modify the skia_revision and skia_hash entries in the given DEPS file.
-
-    Args:
-        revision: (int) Skia SVN revision.
-        git_hash: (string) Skia Git hash.
+        revision: (string) Skia commit hash.
         depspath: (string) path to DEPS file.
     """
     temp_file = tempfile.NamedTemporaryFile(delete=False,
                                             prefix='skia_DEPS_ROLL_tmp_')
     try:
-        deps_regex_rev = re.compile('"skia_revision": "[0-9]*",')
-        deps_regex_hash = re.compile('"skia_hash": "[0-9a-f]*",')
-
-        deps_regex_rev_repl = '"skia_revision": "%d",' % revision
-        deps_regex_hash_repl = '"skia_hash": "%s",' % git_hash
+        deps_regex_rev = re.compile(REGEXP_SKIA_REVISION)
+        deps_regex_rev_repl = '  "skia_revision": "%s",' % revision
 
         with open(depspath, 'r') as input_stream:
             for line in input_stream:
                 line = deps_regex_rev.sub(deps_regex_rev_repl, line)
-                line = deps_regex_hash.sub(deps_regex_hash_repl, line)
                 temp_file.write(line)
     finally:
         temp_file.close()
     shutil.move(temp_file.name, depspath)
 
 
-def git_cl_uploader(config, message, file_list):
-    """Create a commit in the current git branch; upload via git-cl.
-
-    Assumes that you are already on the branch you want to be on.
+def submit_tries(bots_to_run, dry_run=False):
+    """Submit try requests for the current branch on the given bots.
 
     Args:
-        config: (roll_deps.DepsRollConfig) object containing options.
-        message: (string) the commit message, can be multiline.
-        file_list: (list of strings) list of filenames to pass to `git add`.
+        bots_to_run: (list of strings) bots to request.
+        dry_run: (bool) whether to actually submit the try request.
+    """
+    git_try = [
+        git_utils.GIT, 'cl', 'try', '-m', 'tryserver.chromium']
+    git_try.extend([arg for bot in bots_to_run for arg in ('-b', bot)])
 
-    Returns:
-        The output of `git cl issue`, if not config.skip_cl_upload, else ''.
-    """
-
-    git, vsp = config.git, config.vsp
-    svn_info = str(get_svn_revision(config, 'HEAD'))
-
-    for filename in file_list:
-        assert os.path.exists(filename)
-        vsp.check_call([git, 'add', filename])
-
-    vsp.check_call([git, 'commit', '-q', '-m', message])
-
-    git_cl = [git, 'cl', 'upload', '-f',
-              '--bypass-hooks', '--bypass-watchlists']
-    if config.cc_list:
-        git_cl.append('--cc=%s' % config.cc_list)
-    if config.reviewers_list:
-        git_cl.append('--reviewers=%s' % config.reviewers_list)
-
-    git_try = [
-        git, 'cl', 'try', '-m', 'tryserver.chromium', '--revision', svn_info]
-    git_try.extend([arg for bot in config.cl_bot_list for arg in ('-b', bot)])
-
-    branch_name = git_utils.git_branch_name(vsp.verbose)
-
-    if config.skip_cl_upload:
+    if dry_run:
         space = '   '
         print 'You should call:'
-        print '%scd %s' % (space, os.getcwd())
-        misc_utils.print_subprocess_args(space, [git, 'checkout', branch_name])
-        misc_utils.print_subprocess_args(space, git_cl)
-        if config.cl_bot_list:
-            misc_utils.print_subprocess_args(space, git_try)
+        print space, git_try
         print
-        return ''
     else:
-        vsp.check_call(git_cl)
-        issue = vsp.strip_output([git, 'cl', 'issue'])
-        if config.cl_bot_list:
-            vsp.check_call(git_try)
-        return issue
+        shell_utils.run(git_try)
 
 
-def roll_deps(config, revision, git_hash):
+def roll_deps(config, revision):
     """Upload changed DEPS and a whitespace change.
 
     Given the correct git_hash, create two Reitveld issues.
 
     Args:
         config: (roll_deps.DepsRollConfig) object containing options.
-        revision: (int) Skia SVN revision.
-        git_hash: (string) Skia Git hash.
+        revision: (string) Skia Git hash.
 
     Returns:
         a tuple containing textual description of the two issues.
 
     Raises:
         OSError: failed to execute git or git-cl.
         subprocess.CalledProcessError: git returned unexpected status.
     """
 
-    git = config.git
-    with misc_utils.ChangeDir(config.chromium_path, config.verbose):
-        config.vsp.check_call([git, 'fetch', '-q', 'origin'])
+    with misc.ChDir(config.chromium_path, verbose=config.verbose):
+        git_utils.Fetch()
+        output = shell_utils.run([git_utils.GIT, 'show', 'origin/master:DEPS'],
+                                 log_in_real_time=False).rstrip()
+        match = re.search(REGEXP_SKIA_REVISION, output, flags=re.MULTILINE)
+        old_revision = None
+        if match:
+          old_revision = match.group('revision')
+        assert old_revision
 
-        old_revision = misc_utils.ReSearch.search_within_output(
-            config.verbose, '"skia_revision": "(?P<return>[0-9]+)",', None,
-            [git, 'show', 'origin/master:DEPS'])
-        assert old_revision
-        if revision == int(old_revision):
-            print 'DEPS is up to date!'
-            return (None, None)
-
-        master_hash = config.vsp.strip_output(
-            [git, 'show-ref', 'origin/master', '--hash'])
-        master_revision = get_svn_revision(config, 'origin/master')
+        master_hash = git_utils.FullHash('origin/master').rstrip()
 
         # master_hash[8] gives each whitespace CL a unique name.
-        if config.save_branches:
-            branch = 'control_%s' % master_hash[:8]
-        else:
-            branch = None
+        branch = 'control_%s' % master_hash[:8]
         message = ('whitespace change %s\n\n'
-                   'Chromium base revision: %d / %s\n\n'
+                   'Chromium base revision: %s\n\n'
                    'This CL was created by Skia\'s roll_deps.py script.\n'
-                  ) % (master_hash[:8], master_revision, master_hash[:8])
-        with git_utils.ChangeGitBranch(branch, 'origin/master',
-                                       config.verbose):
-            branch = git_utils.git_branch_name(config.vsp.verbose)
+                   ) % (master_hash[:8], master_hash[:8])
+        with git_utils.GitBranch(branch, message,
+                                 delete_when_finished=not config.save_branches,
+                                 upload=not config.skip_cl_upload
+                                 ) as whitespace_branch:
+            branch = git_utils.GetCurrentBranch()
+            with open(os.path.join('build', 'whitespace_file.txt'), 'a') as f:
+                f.write('\nCONTROL\n')
 
-            with open('build/whitespace_file.txt', 'a') as output_stream:
-                output_stream.write('\nCONTROL\n')
+            control_url = whitespace_branch.commit_and_upload()
+            if config.cl_bot_list:
+                submit_tries(config.cl_bot_list, dry_run=config.skip_cl_upload)
+            whitespace_cl = control_url
+            if config.save_branches:
+                whitespace_cl += '\n    branch: %s' % branch
 
-            whitespace_cl = git_cl_uploader(
-                config, message, ['build/whitespace_file.txt'])
-
-            control_url = misc_utils.ReSearch.search_within_string(
-                whitespace_cl, '(?P<return>https?://[^) ]+)', '?')
-            if config.save_branches:
-                whitespace_cl = '%s\n    branch: %s' % (whitespace_cl, branch)
-
-        if config.save_branches:
-            branch = 'roll_%d_%s' % (revision, master_hash[:8])
-        else:
-            branch = None
+        branch = 'roll_%s_%s' % (revision, master_hash[:8])
         message = (
-            'roll skia DEPS to %d\n\n'
-            'Chromium base revision: %d / %s\n'
+            'roll skia DEPS to %s\n\n'
+            'Chromium base revision: %s\n'
             'Old Skia revision: %s\n'
-            'New Skia revision: %d\n'
+            'New Skia revision: %s\n'
             'Control CL: %s\n\n'
             'This CL was created by Skia\'s roll_deps.py script.\n\n'
             'Bypassing commit queue trybots:\n'
             'NOTRY=true\n'
-            % (revision, master_revision, master_hash[:8],
-               old_revision, revision, control_url))
-        with git_utils.ChangeGitBranch(branch, 'origin/master',
-                                       config.verbose):
-            branch = git_utils.git_branch_name(config.vsp.verbose)
-
-            change_skia_deps(revision, git_hash, 'DEPS')
-            deps_cl = git_cl_uploader(config, message, ['DEPS'])
+            % (revision, master_hash[:8],
+               old_revision[:8], revision[:8], control_url))
+        with git_utils.GitBranch(branch, message,
+                                 delete_when_finished=not config.save_branches,
+                                 upload=not config.skip_cl_upload
+                                 ) as roll_branch:
+            change_skia_deps(revision, 'DEPS')
+            deps_url = roll_branch.commit_and_upload()
+            if config.cl_bot_list:
+                submit_tries(config.cl_bot_list, dry_run=config.skip_cl_upload)
+            deps_cl = deps_url
             if config.save_branches:
-                deps_cl = '%s\n    branch: %s' % (deps_cl, branch)
+                deps_cl += '\n    branch: %s' % branch
 
         return deps_cl, whitespace_cl
 
 
-def find_hash_and_roll_deps(config, revision=None, partial_hash=None):
-    """Call find_hash_from_revision() and roll_deps().
-
-    The calls to git will be verbose on standard output.  After a
-    successful upload of both issues, print links to the new
-    codereview issues.
-
-    Args:
-        config: (roll_deps.DepsRollConfig) object containing options.
-        revision: (int or None) the Skia SVN revision number or None
-            to use the tip of the tree.
-        partial_hash: (string or None) a partial pure-git Skia commit
-            hash.  Don't pass both partial_hash and revision.
-
-    Raises:
-        roll_deps.DepsRollError: if the revision can't be found.
-        OSError: failed to execute git or git-cl.
-        subprocess.CalledProcessError: git returned unexpected status.
-    """
-
-    if revision and partial_hash:
-        raise DepsRollError('Pass revision or partial_hash, not both.')
-
-    if partial_hash:
-        revision, git_hash = revision_and_hash_from_partial(
-            config, partial_hash)
-    elif revision:
-        revision, git_hash = revision_and_hash_from_revision(config, revision)
-    else:
-        revision, git_hash = revision_and_hash(config)
-
-    print 'revision=%r\nhash=%r\n' % (revision, git_hash)
-
-    deps_issue, whitespace_issue = roll_deps(config, revision, git_hash)
-
-    if deps_issue and whitespace_issue:
-        print 'DEPS roll:\n    %s\n' % deps_issue
-        print 'Whitespace change:\n    %s\n' % whitespace_issue
-    else:
-        print >> sys.stderr, 'No issues created.'
-
-
 def main(args):
     """main function; see module-level docstring and GetOptionParser help.
 
     Args:
         args: sys.argv[1:]-type argument list.
     """
     option_parser = DepsRollConfig.GetOptionParser()
     options = option_parser.parse_args(args)[0]
 
+    if not options.revision:
+        option_parser.error('Must specify a revision.')
     if not options.chromium_path:
         option_parser.error('Must specify chromium_path.')
     if not os.path.isdir(options.chromium_path):
         option_parser.error('chromium_path must be a directory.')
 
-    if not git_utils.git_executable():
-        option_parser.error('Invalid git executable.')
+    config = DepsRollConfig(options)
+    shell_utils.VERBOSE = options.verbose
+    deps_issue, whitespace_issue = roll_deps(config, options.revision)
 
-    config = DepsRollConfig(options)
-    find_hash_and_roll_deps(config, options.revision, options.git_hash)
+    if deps_issue and whitespace_issue:
+        print 'DEPS roll:\n    %s\n' % deps_issue
+        print 'Whitespace change:\n    %s\n' % whitespace_issue
+    else:
+        print >> sys.stderr, 'No issues created.'
 
 
 if __name__ == '__main__':
     main(sys.argv[1:])
-