Add a way for git cache to bootstrap off Google Storage.

git_cache.py

Index: git_cache.py
diff --git a/git_cache.py b/git_cache.py
index b47b163975d3288a90dafddfdb9128374b23eb5f..d1e2c28b6446b7038240e2fc38884356d5c732e2 100755
--- a/git_cache.py
+++ b/git_cache.py
@@ -14,11 +14,16 @@ import subprocess
 import sys
 import urlparse
 
+import download_from_google_storage
 import gclient_utils
 import subcommand
 
 
 GIT_EXECUTABLE = 'git.bat' if sys.platform.startswith('win') else 'git'
+BOOTSTRAP_BUCKET = 'chromium-git-cache'
+GSUTIL_DEFAULT_PATH = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    'third_party', 'gsutil', 'gsutil')
 
 
 def UrlToCacheDir(url):
@@ -151,6 +156,39 @@ def CMDexists(parser, args):
   return 1
 
 
+@subcommand.usage('[url of repo to create a bootstrap zip file]')
+def CMDupdate_bootstrap(parser, args):
+  """Create and uploads a bootstrap tarball."""

[iannucci, 2014/04/01 05:47:11]

let's just assert this is on !win32 w/ a friendly message

[Ryan Tseng, 2014/04/01 20:18:10]

Done.

+  # First, we need to ensure the cache is populated.
+  args.append('--no_bootstrap')
+  CMDpopulate(parser, args)

[iannucci, 2014/04/01 05:47:11]

you'll want to do a full gc repack

[Ryan Tseng, 2014/04/01 20:18:10]

Done.

+
+  # The files are named <git number>.zip
+  gen_number = subprocess.check_output(['git', 'number']).strip()

[iannucci, 2014/04/01 05:47:11]

git number master

[Ryan Tseng, 2014/04/01 20:18:10]

Done.

+  options, args = parser.parse_args(args)
+
+  if not len(args) == 1:
+    parser.error('git cache update_bootstrap only takes exactly one repo url.')
+  url = args[0]

[iannucci, 2014/04/01 05:47:11]

I don't think these lines are needed b/c CMDpopulate

[Ryan Tseng, 2014/04/01 20:18:10]

Done.

+
+  repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
+  # Creating a temp file and then deleting it ensures we can use this name.
+  _, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
+  os.remove(tmp_zipfile)
+  if sys.platform.startswith('win'):
+    cmd = ['7z', 'a', '-r', '-tzip', tmp_zipfile, '.']
+  else:
+    cmd = ['zip', '-r', tmp_zipfile, '.']
+  subprocess.call(cmd, cwd=repo_dir)
+  gsutil = download_from_google_storage.Gsutil(path=GSUTIL_DEFAULT_PATH,
+                                               boto_path=None)
+  dest_name = 'gs://%s/%s/%s.zip' % (BOOTSTRAP_BUCKET,
+                                     UrlToCacheDir(url),
+                                     gen_number)
+  gsutil.call('cp', tmp_zipfile, dest_name)
+  os.remove(tmp_zipfile)
+
+
 @subcommand.usage('[url of repo to add to or update in cache]')
 def CMDpopulate(parser, args):
   """Ensure that the cache has all up-to-date objects for the given repo."""
@@ -160,6 +198,9 @@ def CMDpopulate(parser, args):
                     help='Only cache 10000 commits of history')
   parser.add_option('--ref', action='append',
                     help='Specify additional refs to be fetched')
+  parser.add_option('--no_bootstrap', action='store_true',
+                    help='Don\'t bootstrap from Google Storage')
+
   options, args = parser.parse_args(args)
   if options.shallow and not options.depth:
     options.depth = 10000
@@ -180,6 +221,77 @@ def CMDpopulate(parser, args):
   if options.depth:
     d = ['--depth', '%d' % options.depth]
 
+  def _find(executable):
+    # Mimics the "which" utility.
+    if sys.platform.startswith('win'):
+      path_folders = os.environ.get('PATH').split(';')
+    else:
+      path_folders = os.environ.get('PATH').split(':')

[iannucci, 2014/04/01 05:47:11]

os.pathsep

[Ryan Tseng, 2014/04/01 20:18:10]

Done.

+
+    for path_folder in path_folders:
+      target = os.path.join(path_folder, executable)
+      # Just incase we have some ~/blah paths.
+      target = os.path.abspath(os.path.expanduser(target))
+      if os.path.isfile(target) and os.access(target, os.X_OK):
+        return target
+    return False
+
+  def _maybe_bootstrap_repo(directory):
+    # Bootstrap the repo from Google Stroage if there is a pre-checked out
+    # version already. Uses 7z on windows to inflate, and unzip
+    # everywhere else.
+    if options.no_bootstrap:
+      return False
+    if sys.platform.startswith('win'):
+      if not _find('7z'):
+        print 'Cannot find 7z in the path.'
+        print 'Install 7z from http://www.7-zip.org/download.html if you want '
+        print 'git cache to bootstrap from Google Storage.'
+        return False
+    else:
+      if not _find('unzip'):
+        print 'Cannot find unzip in the path.'
+        print 'Install unzip if you want to create a git cache to boostrap '
+        print 'from Google Storage.'
+        return False
+
+    folder = UrlToCacheDir(url)
+    gs_folder = 'gs://%s/%s' % (BOOTSTRAP_BUCKET, folder)
+    gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH,
+                                                 boto_path=os.devnull,
+                                                 bypass_prodaccess=True)
+    # Get the most recent version.
+    _, ls_out, _ = gsutil.check_call('ls', gs_folder)
+    ls_out_sorted = sorted(ls_out.splitlines())
+    if not ls_out_sorted:
+      # This repo is not on Google Storage.
+      return False
+    latest_checkout = ls_out_sorted[-1]
+
+    # Download zip file to tempdir.
+    tempdir = tempfile.mkdtemp()
+    print 'Downloading %s...' % latest_checkout
+    code, out, err = gsutil.check_call('cp', latest_checkout, tempdir)
+    if code:
+      print '%s\n%s' % (out, err)
+      return False
+    filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
+
+    # Unpack the file.
+    if sys.platform.startswith('win'):
+      cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
+    else:
+      # Use the "unzip" utility to inflate.
+      cmd = ['unzip', filename, '-d', directory]
+    code = subprocess.call(cmd)
+
+    gclient_utils.rmtree(tempdir)

[iannucci, 2014/04/01 05:47:11]

this whole chunk of code begs to be refactored into a ZipTool class with a zip
and an unzip method...

[Ryan Tseng, 2014/04/01 20:18:10]

A whole new class to house 3 lines for zip and 5 lines for unzip seems
excessive... Maybe you mean for "Download from this gs location and unzip to
there".  But even then I feel like we'd be making classes for the sake of making
classes.

If it was used in one other place I'd agree with the need to refactor into a
generic "ziptool" that searches for 7z, unzip, and falls back to a python
implementation if it can't find either.  But I think thats unnecessarily complex
if this serves the need.

+    if code:
+      print 'Extracting bootstrap zipfile %s failed.' % filename
+      print 'Resuming normal operations'
+      return False
+    return True
+
   def _config(directory):
     RunGit(['config', 'core.deltaBaseCacheLimit',
             gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=directory)
@@ -203,7 +315,9 @@ def CMDpopulate(parser, args):
       gclient_utils.rmtree(repo_dir)
       tempdir = tempfile.mkdtemp(suffix=UrlToCacheDir(url),
                                  dir=options.cache_dir)
-      RunGit(['init', '--bare'], cwd=tempdir)
+      bootstrapped = _maybe_bootstrap_repo(tempdir)
+      if not bootstrapped:
+        RunGit(['init', '--bare'], cwd=tempdir)
       _config(tempdir)
       fetch_cmd = ['fetch'] + v + d + ['origin']
       RunGit(fetch_cmd, filter_fn=filter_fn, cwd=tempdir, retry=True)
@@ -263,7 +377,7 @@ def CMDunlock(parser, args):
       unlocked = True
 
     if unlocked:
-      unlocked.append(repo_dir)      
+      unlocked.append(repo_dir)
     else:
       untouched.append(repo_dir)