Add a way for git cache to bootstrap off Google Storage.

git_cache.py

 #!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A git command for managing a local cache of git repositories."""
 
 import errno
 import logging
 import optparse
 import os
 import tempfile
 import subprocess
 import sys
 import urlparse
 
+import download_from_google_storage

[agable, 2014/04/01 23:44:52]

from download_from_google_storage import Gsutil? Easier to change that later if
Gsutil gets refactored out of that file.

[Ryan Tseng, 2014/04/02 00:32:19]

Done.

 import gclient_utils
 import subcommand
 
 
 GIT_EXECUTABLE = 'git.bat' if sys.platform.startswith('win') else 'git'
+BOOTSTRAP_BUCKET = 'chromium-git-cache'
+GSUTIL_DEFAULT_PATH = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    'third_party', 'gsutil', 'gsutil')
 
 
 def UrlToCacheDir(url):
   """Convert a git url to a normalized form for the cache dir path."""
   parsed = urlparse.urlparse(url)
   norm_url = parsed.netloc + parsed.path
   if norm_url.endswith('.git'):
     norm_url = norm_url[:-len('.git')]
   return norm_url.replace('-', '--').replace('/', '-').lower()
 
@@ skipping 112 matching lines @@
     parser.error('git cache exists only takes exactly one repo url.')
   url = args[0]
   repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
   flag_file = os.path.join(repo_dir, 'config')
   if os.path.isdir(repo_dir) and os.path.isfile(flag_file):
     print repo_dir
     return 0
   return 1
 
 
+@subcommand.usage('[url of repo to create a bootstrap zip file]')
+def CMDupdate_bootstrap(parser, args):
+  """Create and uploads a bootstrap tarball."""
+  # Lets just assert we can't do this on Windows.
+  if sys.platform.startswith('win'):
+    print >> sys.stderr, 'Sorry, update bootstrap will not work on Windows.'
+    return 1
+
+  # First, we need to ensure the cache is populated.
+  args.append('--no_bootstrap')

[agable, 2014/04/01 23:44:52]

Would prefer to manipulate a copy of args, so that we don't add 'no-bootstrap'
here, send it to populate... and then parse our own 'no-bootstrap' on line 172
but ignore it.

[Ryan Tseng, 2014/04/02 00:32:19]

Done.

+  CMDpopulate(parser, args)
+
+  # Get the repo directory.
+  options, args = parser.parse_args(args)
+  url = args[0]
+  repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
+
+  # The files are named <git number>.zip
+  gen_number = subprocess.check_output(['git', 'number', 'master'],
+                                       cwd=repo_dir).strip()
+  RunGit(['gc'], cwd=repo_dir)  # Run Garbage Collect to compress packfile.
+  # Creating a temp file and then deleting it ensures we can use this name.
+  _, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
+  os.remove(tmp_zipfile)
+  subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=repo_dir)
+  gsutil = download_from_google_storage.Gsutil(path=GSUTIL_DEFAULT_PATH,
+                                               boto_path=None)
+  dest_name = 'gs://%s/%s/%s.zip' % (BOOTSTRAP_BUCKET,
+                                     UrlToCacheDir(url),
+                                     gen_number)
+  gsutil.call('cp', tmp_zipfile, dest_name)
+  os.remove(tmp_zipfile)
+
+
 @subcommand.usage('[url of repo to add to or update in cache]')
 def CMDpopulate(parser, args):
   """Ensure that the cache has all up-to-date objects for the given repo."""
   parser.add_option('--depth', type='int',
                     help='Only cache DEPTH commits of history')
   parser.add_option('--shallow', '-s', action='store_true',
                     help='Only cache 10000 commits of history')
   parser.add_option('--ref', action='append',
                     help='Specify additional refs to be fetched')
+  parser.add_option('--no_bootstrap', action='store_true',
+                    help='Don\'t bootstrap from Google Storage')
+
   options, args = parser.parse_args(args)
   if options.shallow and not options.depth:
     options.depth = 10000
   if not len(args) == 1:
     parser.error('git cache populate only takes exactly one repo url.')
   url = args[0]
 
   gclient_utils.safe_makedirs(options.cache_dir)
   repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
 
   v = []
   filter_fn = lambda l: '[up to date]' not in l
   if options.verbose:
     v = ['-v', '--progress']
     filter_fn = None
 
   d = []
   if options.depth:
     d = ['--depth', '%d' % options.depth]
 
+  def _find(executable):
+    # Mimics the "which" utility.

[agable, 2014/04/01 23:44:52]

Do a real docstring.

[Ryan Tseng, 2014/04/02 00:32:19]

Done.

+    path_folders = os.environ.get('PATH').split(os.pathsep)
+
+    for path_folder in path_folders:
+      target = os.path.join(path_folder, executable)
+      # Just incase we have some ~/blah paths.
+      target = os.path.abspath(os.path.expanduser(target))
+      if os.path.isfile(target) and os.access(target, os.X_OK):
+        return target
+    return False
+
+  def _maybe_bootstrap_repo(directory):
+    # Bootstrap the repo from Google Stroage if there is a pre-checked out

[agable, 2014/04/01 23:44:52]

Do a real docstring.

[Ryan Tseng, 2014/04/02 00:32:19]

Done.

+    # version already. Uses 7z on windows to inflate, and unzip
+    # everywhere else.
+    if options.no_bootstrap:
+      return False
+    if sys.platform.startswith('win'):
+      if not _find('7z'):
+        print 'Cannot find 7z in the path.'
+        print 'Install 7z from http://www.7-zip.org/download.html if you want '

[agable, 2014/04/01 23:44:52]

...if you want git cache to be able to bootstrap...

[Ryan Tseng, 2014/04/02 00:32:19]

Done.

+        print 'git cache to bootstrap from Google Storage.'
+        return False
+    else:
+      if not _find('unzip'):
+        print 'Cannot find unzip in the path.'
+        print 'Install unzip if you want to create a git cache to boostrap '

[agable, 2014/04/01 23:44:52]

...if you want git cache to be able to bootstrap...

[Ryan Tseng, 2014/04/02 00:32:19]

Done.

+        print 'from Google Storage.'
+        return False
+
+    folder = UrlToCacheDir(url)
+    gs_folder = 'gs://%s/%s' % (BOOTSTRAP_BUCKET, folder)
+    gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH,
+                                                 boto_path=os.devnull,
+                                                 bypass_prodaccess=True)
+    # Get the most recent version of the zipfile.
+    _, ls_out, _ = gsutil.check_call('ls', gs_folder)
+    ls_out_sorted = sorted(ls_out.splitlines())
+    if not ls_out_sorted:
+      # This repo is not on Google Storage.
+      return False
+    latest_checkout = ls_out_sorted[-1]
+
+    # Download zip file to a temporary directory.
+    tempdir = tempfile.mkdtemp()
+    print 'Downloading %s...' % latest_checkout
+    code, out, err = gsutil.check_call('cp', latest_checkout, tempdir)
+    if code:
+      print '%s\n%s' % (out, err)
+      return False
+    filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
+
+    # Unpack the file with 7z on Windows, or unzip everywhere else.
+    if sys.platform.startswith('win'):
+      cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
+    else:
+      cmd = ['unzip', filename, '-d', directory]
+    code = subprocess.call(cmd)

[agable, 2014/04/01 23:44:52]

retcode

[Ryan Tseng, 2014/04/02 00:32:19]

Done.

+
+    # Clean up the downloaded zipfile.
+    gclient_utils.rmtree(tempdir)
+    if code:
+      print 'Extracting bootstrap zipfile %s failed.' % filename
+      print 'Resuming normal operations'
+      return False
+    return True
+
   def _config(directory):
     RunGit(['config', 'core.deltaBaseCacheLimit',
             gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=directory)
     RunGit(['config', 'remote.origin.url', url],
            cwd=directory)
     RunGit(['config', '--replace-all', 'remote.origin.fetch',
             '+refs/heads/*:refs/heads/*'],
            cwd=directory)
     RunGit(['config', '--add', 'remote.origin.fetch',
             '+refs/tags/*:refs/tags/*'],
            cwd=directory)
     for ref in options.ref or []:
       ref = ref.rstrip('/')
       refspec = '+refs/%s/*:refs/%s/*' % (ref, ref)
       RunGit(['config', '--add', 'remote.origin.fetch', refspec],
              cwd=directory)
 
   with Lockfile(repo_dir):
     # Setup from scratch if the repo is new or is in a bad state.
     if not os.path.exists(os.path.join(repo_dir, 'config')):
       gclient_utils.rmtree(repo_dir)
       tempdir = tempfile.mkdtemp(suffix=UrlToCacheDir(url),
                                  dir=options.cache_dir)
-      RunGit(['init', '--bare'], cwd=tempdir)
+      bootstrapped = _maybe_bootstrap_repo(tempdir)
+      if not bootstrapped:
+        RunGit(['init', '--bare'], cwd=tempdir)
       _config(tempdir)
       fetch_cmd = ['fetch'] + v + d + ['origin']
       RunGit(fetch_cmd, filter_fn=filter_fn, cwd=tempdir, retry=True)
       os.rename(tempdir, repo_dir)
     else:
       _config(repo_dir)
       if options.depth and os.path.exists(os.path.join(repo_dir, 'shallow')):
         logging.warn('Shallow fetch requested, but repo cache already exists.')
       fetch_cmd = ['fetch'] + v + ['origin']
       RunGit(fetch_cmd, filter_fn=filter_fn, cwd=repo_dir, retry=True)
@@ skipping 39 matching lines @@
     lf = Lockfile(repo_dir)
     config_lock = os.path.join(repo_dir, 'config.lock')
     unlocked = False
     if os.path.exists(config_lock):
       os.remove(config_lock)
       unlocked = True
     if lf.break_lock():
       unlocked = True
 
     if unlocked:
-      unlocked.append(repo_dir)      
+      unlocked.append(repo_dir)
     else:
       untouched.append(repo_dir)
 
   if unlocked:
     logging.info('Broke locks on these caches: %s' % unlocked)
   if untouched:
     logging.debug('Did not touch these caches: %s' % untouched)
 
 
 class OptionParser(optparse.OptionParser):
@@ skipping 31 matching lines @@
     return options, args
 
 
 def main(argv):
   dispatcher = subcommand.CommandDispatcher(__name__)
   return dispatcher.execute(OptionParser(), argv)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))