Add a way for git cache to bootstrap off Google Storage.

git_cache.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A git command for managing a local cache of git repositories."""
 
 import errno
 import logging
 import optparse
 import os
 import tempfile
 import subprocess
 import sys
 import urlparse
 
+import download_from_google_storage
 import gclient_utils
 import subcommand
 
 
 GIT_EXECUTABLE = 'git.bat' if sys.platform.startswith('win') else 'git'
+BOOTSTRAP_BUCKET = 'chromium-git-cache'
+GSUTIL_DEFAULT_PATH = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    'third_party', 'gsutil', 'gsutil')
 
 
 def UrlToCacheDir(url):
   """Convert a git url to a normalized form for the cache dir path."""
   parsed = urlparse.urlparse(url)
   norm_url = parsed.netloc + parsed.path
   if norm_url.endswith('.git'):
     norm_url = norm_url[:-len('.git')]
   return norm_url.replace('-', '--').replace('/', '-').lower()
 
@@ skipping 112 matching lines @@
     parser.error('git cache exists only takes exactly one repo url.')
   url = args[0]
   repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
   flag_file = os.path.join(repo_dir, 'config')
   if os.path.isdir(repo_dir) and os.path.isfile(flag_file):
     print repo_dir
     return 0
   return 1
 
 
+@subcommand.usage('[url of repo to create a bootstrap zip file]')
+def CMDupdate_bootstrap(parser, args):
+  """Create and uploads a bootstrap tarball."""

[iannucci, 2014/04/01 05:47:11]

let's just assert this is on !win32 w/ a friendly message

[Ryan Tseng, 2014/04/01 20:18:10]

Done.

+  # First, we need to ensure the cache is populated.
+  args.append('--no_bootstrap')
+  CMDpopulate(parser, args)

[iannucci, 2014/04/01 05:47:11]

you'll want to do a full gc repack

[Ryan Tseng, 2014/04/01 20:18:10]

Done.

+
+  # The files are named <git number>.zip
+  gen_number = subprocess.check_output(['git', 'number']).strip()

[iannucci, 2014/04/01 05:47:11]

git number master

[Ryan Tseng, 2014/04/01 20:18:10]

Done.

+  options, args = parser.parse_args(args)
+
+  if not len(args) == 1:
+    parser.error('git cache update_bootstrap only takes exactly one repo url.')
+  url = args[0]

[iannucci, 2014/04/01 05:47:11]

I don't think these lines are needed b/c CMDpopulate

[Ryan Tseng, 2014/04/01 20:18:10]

Done.

+
+  repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
+  # Creating a temp file and then deleting it ensures we can use this name.
+  _, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
+  os.remove(tmp_zipfile)
+  if sys.platform.startswith('win'):
+    cmd = ['7z', 'a', '-r', '-tzip', tmp_zipfile, '.']
+  else:
+    cmd = ['zip', '-r', tmp_zipfile, '.']
+  subprocess.call(cmd, cwd=repo_dir)
+  gsutil = download_from_google_storage.Gsutil(path=GSUTIL_DEFAULT_PATH,
+                                               boto_path=None)
+  dest_name = 'gs://%s/%s/%s.zip' % (BOOTSTRAP_BUCKET,
+                                     UrlToCacheDir(url),
+                                     gen_number)
+  gsutil.call('cp', tmp_zipfile, dest_name)
+  os.remove(tmp_zipfile)
+
+
 @subcommand.usage('[url of repo to add to or update in cache]')
 def CMDpopulate(parser, args):
   """Ensure that the cache has all up-to-date objects for the given repo."""
   parser.add_option('--depth', type='int',
                     help='Only cache DEPTH commits of history')
   parser.add_option('--shallow', '-s', action='store_true',
                     help='Only cache 10000 commits of history')
   parser.add_option('--ref', action='append',
                     help='Specify additional refs to be fetched')
+  parser.add_option('--no_bootstrap', action='store_true',
+                    help='Don\'t bootstrap from Google Storage')
+
   options, args = parser.parse_args(args)
   if options.shallow and not options.depth:
     options.depth = 10000
   if not len(args) == 1:
     parser.error('git cache populate only takes exactly one repo url.')
   url = args[0]
 
   gclient_utils.safe_makedirs(options.cache_dir)
   repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
 
   v = []
   filter_fn = lambda l: '[up to date]' not in l
   if options.verbose:
     v = ['-v', '--progress']
     filter_fn = None
 
   d = []
   if options.depth:
     d = ['--depth', '%d' % options.depth]
 
+  def _find(executable):
+    # Mimics the "which" utility.
+    if sys.platform.startswith('win'):
+      path_folders = os.environ.get('PATH').split(';')
+    else:
+      path_folders = os.environ.get('PATH').split(':')

[iannucci, 2014/04/01 05:47:11]

os.pathsep

[Ryan Tseng, 2014/04/01 20:18:10]

Done.

+
+    for path_folder in path_folders:
+      target = os.path.join(path_folder, executable)
+      # Just incase we have some ~/blah paths.
+      target = os.path.abspath(os.path.expanduser(target))
+      if os.path.isfile(target) and os.access(target, os.X_OK):
+        return target
+    return False
+
+  def _maybe_bootstrap_repo(directory):
+    # Bootstrap the repo from Google Stroage if there is a pre-checked out
+    # version already. Uses 7z on windows to inflate, and unzip
+    # everywhere else.
+    if options.no_bootstrap:
+      return False
+    if sys.platform.startswith('win'):
+      if not _find('7z'):
+        print 'Cannot find 7z in the path.'
+        print 'Install 7z from http://www.7-zip.org/download.html if you want '
+        print 'git cache to bootstrap from Google Storage.'
+        return False
+    else:
+      if not _find('unzip'):
+        print 'Cannot find unzip in the path.'
+        print 'Install unzip if you want to create a git cache to boostrap '
+        print 'from Google Storage.'
+        return False
+
+    folder = UrlToCacheDir(url)
+    gs_folder = 'gs://%s/%s' % (BOOTSTRAP_BUCKET, folder)
+    gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH,
+                                                 boto_path=os.devnull,
+                                                 bypass_prodaccess=True)
+    # Get the most recent version.
+    _, ls_out, _ = gsutil.check_call('ls', gs_folder)
+    ls_out_sorted = sorted(ls_out.splitlines())
+    if not ls_out_sorted:
+      # This repo is not on Google Storage.
+      return False
+    latest_checkout = ls_out_sorted[-1]
+
+    # Download zip file to tempdir.
+    tempdir = tempfile.mkdtemp()
+    print 'Downloading %s...' % latest_checkout
+    code, out, err = gsutil.check_call('cp', latest_checkout, tempdir)
+    if code:
+      print '%s\n%s' % (out, err)
+      return False
+    filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
+
+    # Unpack the file.
+    if sys.platform.startswith('win'):
+      cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
+    else:
+      # Use the "unzip" utility to inflate.
+      cmd = ['unzip', filename, '-d', directory]
+    code = subprocess.call(cmd)
+
+    gclient_utils.rmtree(tempdir)

[iannucci, 2014/04/01 05:47:11]

this whole chunk of code begs to be refactored into a ZipTool class with a zip
and an unzip method...

[Ryan Tseng, 2014/04/01 20:18:10]

A whole new class to house 3 lines for zip and 5 lines for unzip seems
excessive... Maybe you mean for "Download from this gs location and unzip to
there".  But even then I feel like we'd be making classes for the sake of making
classes.

If it was used in one other place I'd agree with the need to refactor into a
generic "ziptool" that searches for 7z, unzip, and falls back to a python
implementation if it can't find either.  But I think thats unnecessarily complex
if this serves the need.

+    if code:
+      print 'Extracting bootstrap zipfile %s failed.' % filename
+      print 'Resuming normal operations'
+      return False
+    return True
+
   def _config(directory):
     RunGit(['config', 'core.deltaBaseCacheLimit',
             gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=directory)
     RunGit(['config', 'remote.origin.url', url],
            cwd=directory)
     RunGit(['config', '--replace-all', 'remote.origin.fetch',
             '+refs/heads/*:refs/heads/*'],
            cwd=directory)
     RunGit(['config', '--add', 'remote.origin.fetch',
             '+refs/tags/*:refs/tags/*'],
            cwd=directory)
     for ref in options.ref or []:
       ref = ref.rstrip('/')
       refspec = '+refs/%s/*:refs/%s/*' % (ref, ref)
       RunGit(['config', '--add', 'remote.origin.fetch', refspec],
              cwd=directory)
 
   with Lockfile(repo_dir):
     # Setup from scratch if the repo is new or is in a bad state.
     if not os.path.exists(os.path.join(repo_dir, 'config')):
       gclient_utils.rmtree(repo_dir)
       tempdir = tempfile.mkdtemp(suffix=UrlToCacheDir(url),
                                  dir=options.cache_dir)
-      RunGit(['init', '--bare'], cwd=tempdir)
+      bootstrapped = _maybe_bootstrap_repo(tempdir)
+      if not bootstrapped:
+        RunGit(['init', '--bare'], cwd=tempdir)
       _config(tempdir)
       fetch_cmd = ['fetch'] + v + d + ['origin']
       RunGit(fetch_cmd, filter_fn=filter_fn, cwd=tempdir, retry=True)
       os.rename(tempdir, repo_dir)
     else:
       _config(repo_dir)
       if options.depth and os.path.exists(os.path.join(repo_dir, 'shallow')):
         logging.warn('Shallow fetch requested, but repo cache already exists.')
       fetch_cmd = ['fetch'] + v + ['origin']
       RunGit(fetch_cmd, filter_fn=filter_fn, cwd=repo_dir, retry=True)
@@ skipping 39 matching lines @@
     lf = Lockfile(repo_dir)
     config_lock = os.path.join(repo_dir, 'config.lock')
     unlocked = False
     if os.path.exists(config_lock):
       os.remove(config_lock)
       unlocked = True
     if lf.break_lock():
       unlocked = True
 
     if unlocked:
-      unlocked.append(repo_dir)      
+      unlocked.append(repo_dir)
     else:
       untouched.append(repo_dir)
 
   if unlocked:
     logging.info('Broke locks on these caches: %s' % unlocked)
   if untouched:
     logging.debug('Did not touch these caches: %s' % untouched)
 
 
 class OptionParser(optparse.OptionParser):
@@ skipping 31 matching lines @@
     return options, args
 
 
 def main(argv):
   dispatcher = subcommand.CommandDispatcher(__name__)
   return dispatcher.execute(OptionParser(), argv)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))