git-sync-deps tool

tools/git-sync-deps

Index: tools/git-sync-deps
diff --git a/tools/git-sync-deps b/tools/git-sync-deps
new file mode 100755
index 0000000000000000000000000000000000000000..7dab3bd8d31aac3a6969c7a4efca6c98d82a39b7
--- /dev/null
+++ b/tools/git-sync-deps
@@ -0,0 +1,201 @@
+#!/usr/bin/python
+# Copyright 2014 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Parse a DEPS file and git checkout all of the dependencies.
+
+Args:
+  An optional list of deps_os values.
+
+Environment Variables:

[epoger, 2014/04/14 19:32:14]

Maybe it would be more straightforward to use argparse/optparse instead of
environment variables?  (Then the script would automatically provide
consistently formatted --help output, with defaults that will stay in sync with
the code rather than be maintained separately, etc.)

I'm OK either way.

+  GIT_EXECUTABLE if git, git.exe, or git.bat isn't in your path, use this 

[epoger, 2014/04/14 19:32:14]

I think this would be more accurate:

GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of ['git',
'git.exe', 'git.bat'] in your default path.

[hal.canary, 2014/04/16 18:46:19]

Done.

+  variable to specify its path.  (optional)
+
+  GIT_SYNC_DEPS_PATH if not set, the DEPS file in this repository

[epoger, 2014/04/14 19:32:14]

This one doesn't quite tell the whole story.  Maybe:

GIT_SYNC_DEPS_PATH: DEPS file to get the dependency list from; if unset, assumes
../DEPS

[hal.canary, 2014/04/16 18:46:19]

Done.

+  will be parsed.  (optional)
+
+  GIT_SYNC_DEPS_QUIET if set to non-empty string, suppress messages.
+
+Git Config:
+  To disable syncing of a single repository:
+      cd path/to/repository
+      git config sync-deps.disable true
+
+  To re-enable sync:
+      cd path/to/repository
+      git config --unset sync-deps.disable
+"""
+
+
+import os
+import subprocess
+import sys
+import threading
+
+from git_utils import git_executable
+
+
+DEFAULT_DEPS_PATH = os.path.normpath(
+  os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS'))
+
+
+def usage(deps_file_path = None):
+  sys.stderr.write(
+    'Usage: run to grab dependencies, with optional platform support:\n')
+  sys.stderr.write('  python %s' % __file__)
+  if deps_file_path:
+    for deps_os in parse_file_to_dict(deps_file_path)['deps_os']:
+      sys.stderr.write(' [%s]' % deps_os)
+  else:
+    sys.stderr.write(' [DEPS_OS...]')
+  sys.stderr.write('\n\n')
+  sys.stderr.write(__doc__)
+
+
+def git_checkout_to_directory(git, repo, checkoutable, directory, verbose):
+  """Checkout (and clone if needed) a Git repository.
+
+  Args:
+    git (string) the git executable
+
+    repo (string) the location of the repository, suitable
+         for passing to `git clone`.
+
+    checkoutable (string) a tag, branch, or commit, suitable for
+                 passing to `git checkout`
+
+    directory (string) the path into which the repository
+              should be checked out.
+
+    verbose (boolean)
+
+  Raises an exception if any calls to git fail.
+  """
+  at_revision = False
+  if os.path.isdir(directory):

[epoger, 2014/04/14 19:32:14]

Could you rearrange it like so, so that the os.path.isdir() branch code is
shorter?  I think that would be easier to follow...

if not os.path.isdir(directory):
  subprocess.check_call([git, 'clone', '--quiet', repo, directory])
  # will fail if repo is bad or if directory is a file

# Check to see if this repo is disabled.  Quick return.
try:
  disable = subprocess.check_output( ...

[hal.canary, 2014/04/16 18:46:19]

Done.

+    # Check to see if this repo is disabled.  Quick return.
+    try:
+      disable = subprocess.check_output(
+        [git, 'config', 'sync-deps.disable'], cwd=directory)
+      if disable.lower().strip() in ['true', '1', 'yes', 'on']:
+        sys.stdout.write('%s\n  SYNC IS DISABLED.\n' % directory)
+        return
+    except subprocess.CalledProcessError:
+      pass
+
+    # Only do a slow git fetch if we are not already on correct revision

[epoger, 2014/04/14 19:32:14]

Sorta-double-negative ("only ... not") and
possibly-parenthetical-possibly-restrictive adjective ("slow") makes this
ambiguous.  Maybe, instead...

# Do a git fetch, unless we are already on the correct revision.
# (git fetch is slow, so it's good to skip it if we can)

[hal.canary, 2014/04/16 18:46:19]

I gave up on saving half a second by checking for correct revision.

+    def revision(rev):

[epoger, 2014/04/14 19:32:14]

Maybe make this a first-class function?  I think that's easier to follow than
declaring it inline...

def get_commithash(directory, checkoutable)
  """Returns the commithash corresponding to this tag/branch/commit.

  Args:
    directory: ...
  """

[hal.canary, 2014/04/16 18:46:19]

Simplified out of existence.

+      return subprocess.check_output(
+        [git, 'rev-list', '-1', rev], cwd=directory)

[epoger, 2014/04/14 19:32:14]

Using the more explicit command-line flag will make it easier to see what's
going on here:

[git, 'rev-list', '--max-count', '1', rev]

[hal.canary, 2014/04/16 18:46:19]

Simplified out of existence.

+    try:
+      at_revision = (revision('HEAD') == revision(checkoutable))

[epoger, 2014/04/14 19:32:14]

I think the logic would be simpler if we resolved checkoutable to a commithash
as close to the top of the function as possible:

desired_commithash = get_commithash(directory, checkoutable)

Then, the rest of this function could just focus on getting
get_commithash(directory, 'HEAD') to equal desired_commithash.

+    except subprocess.CalledProcessError:
+      # if checkoutable is unknown, maybe fetch will fix that.
+      pass 
+    if not at_revision:
+      subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory)
+      # will fail if origin can't be reached.
+
+  else:
+    subprocess.check_call([git, 'clone', '--quiet', repo, directory])
+    # will fail if repo is bad or if directory is a file.
+
+  # we want this repository to be pristine.
+  if (subprocess.check_output(
+      [git, 'diff', '--shortstat'], cwd=directory)
+      or subprocess.check_output([git, 'clean', '-nd'], cwd=directory)):
+    raise Exception("%s has uncommited changes" % directory)
+    # Will fail if `git checkout` can not cleanly be called.
+
+  if not at_revision:

[epoger, 2014/04/14 19:32:14]

Why don't we just do the "git checkout" right after we do the "git fetch", in
the same code branch?

[hal.canary, 2014/04/16 18:46:19]

Done.

+    subprocess.check_call(
+      [git, 'checkout', '--quiet', checkoutable], cwd=directory)
+    # Will fail if checkoutable is unknown.
+
+  if verbose:
+    sys.stdout.write('%s\n  @ %s\n' % (directory, checkoutable))  # Success.
+
+
+def parse_file_to_dict(path):
+  dictionary = {}
+  execfile(path, dictionary)
+  return dictionary
+
+
+class DepsError(Exception):
+  """Raised if deps_os is a bad key.
+  """
+  pass
+
+
+def git_sync_deps(deps_file_path, deps_os_list, verbose):
+  """Grab dependencies, with optional platform support.
+
+  Args:
+    deps_file_path (string) Path to the DEPS file.
+
+    deps_os_list (list of strings) Can be empty list.  List of
+                 strings that should each be a key in the deps_os
+                 dictionary in the DEPS file.
+
+  Raises DepsError exception and git Exceptions.
+  """
+  git = git_executable()
+  assert git
+
+  deps_file_directory = os.path.dirname(deps_file_path)
+  deps = parse_file_to_dict(deps_file_path)
+  dependencies = deps['deps'].copy()
+  for deps_os in deps_os_list:
+    # Add OS-specific dependencies
+    if deps_os not in deps['deps_os']:
+      raise DepsError(
+        'Argument "%s" not found within deps_os keys %r' %
+        (deps_os, deps['deps_os'].keys()))
+    for dep in deps['deps_os'][deps_os]:
+      dependencies[dep] = deps['deps_os'][deps_os][dep]
+  list_of_arg_lists = []
+  for directory in dependencies:
+    if '@' in dependencies[directory]:
+      repo, checkoutable = dependencies[directory].split('@', 1)
+    else:
+      repo, checkoutable = dependencies[directory], 'origin/master'
+
+    relative_directory = os.path.join(deps_file_directory, directory)
+
+    list_of_arg_lists.append(
+      (git, repo, checkoutable, relative_directory, verbose))
+
+  multithread(git_checkout_to_directory, list_of_arg_lists)
+
+
+def multithread(function, list_of_arg_lists):
+  # for args in list_of_arg_lists:
+  #   function(*args)
+  # return
+  threads = []
+  for args in list_of_arg_lists:
+    thread = threading.Thread(None, function, None, args)
+    thread.start()
+    threads.append(thread)
+  for thread in threads:
+    thread.join()
+
+
+def main(argv):
+  deps_file_path = os.environ.get(
+    'GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH)
+  verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False))
+  try:
+    git_sync_deps(deps_file_path, argv, verbose)
+    return 0
+  except DepsError:
+    usage(deps_file_path)
+    return 1
+
+
+if __name__ == '__main__':
+  exit(main(sys.argv[1:]))