Add a way for git cache to bootstrap off Google Storage.

git_cache.py

Index: git_cache.py
diff --git a/git_cache.py b/git_cache.py
index b47b163975d3288a90dafddfdb9128374b23eb5f..02009f82985d096ea20eb639adb79af46f6e50ae 100755
--- a/git_cache.py
+++ b/git_cache.py
@@ -14,11 +14,16 @@ import subprocess
 import sys
 import urlparse
 
+import download_from_google_storage

[agable, 2014/04/01 23:44:52]

from download_from_google_storage import Gsutil? Easier to change that later if
Gsutil gets refactored out of that file.

[Ryan Tseng, 2014/04/02 00:32:19]

Done.

 import gclient_utils
 import subcommand
 
 
 GIT_EXECUTABLE = 'git.bat' if sys.platform.startswith('win') else 'git'
+BOOTSTRAP_BUCKET = 'chromium-git-cache'
+GSUTIL_DEFAULT_PATH = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    'third_party', 'gsutil', 'gsutil')
 
 
 def UrlToCacheDir(url):
@@ -151,6 +156,40 @@ def CMDexists(parser, args):
   return 1
 
 
+@subcommand.usage('[url of repo to create a bootstrap zip file]')
+def CMDupdate_bootstrap(parser, args):
+  """Create and uploads a bootstrap tarball."""
+  # Lets just assert we can't do this on Windows.
+  if sys.platform.startswith('win'):
+    print >> sys.stderr, 'Sorry, update bootstrap will not work on Windows.'
+    return 1
+
+  # First, we need to ensure the cache is populated.
+  args.append('--no_bootstrap')

[agable, 2014/04/01 23:44:52]

Would prefer to manipulate a copy of args, so that we don't add 'no-bootstrap'
here, send it to populate... and then parse our own 'no-bootstrap' on line 172
but ignore it.

[Ryan Tseng, 2014/04/02 00:32:19]

Done.

+  CMDpopulate(parser, args)
+
+  # Get the repo directory.
+  options, args = parser.parse_args(args)
+  url = args[0]
+  repo_dir = os.path.join(options.cache_dir, UrlToCacheDir(url))
+
+  # The files are named <git number>.zip
+  gen_number = subprocess.check_output(['git', 'number', 'master'],
+                                       cwd=repo_dir).strip()
+  RunGit(['gc'], cwd=repo_dir)  # Run Garbage Collect to compress packfile.
+  # Creating a temp file and then deleting it ensures we can use this name.
+  _, tmp_zipfile = tempfile.mkstemp(suffix='.zip')
+  os.remove(tmp_zipfile)
+  subprocess.call(['zip', '-r', tmp_zipfile, '.'], cwd=repo_dir)
+  gsutil = download_from_google_storage.Gsutil(path=GSUTIL_DEFAULT_PATH,
+                                               boto_path=None)
+  dest_name = 'gs://%s/%s/%s.zip' % (BOOTSTRAP_BUCKET,
+                                     UrlToCacheDir(url),
+                                     gen_number)
+  gsutil.call('cp', tmp_zipfile, dest_name)
+  os.remove(tmp_zipfile)
+
+
 @subcommand.usage('[url of repo to add to or update in cache]')
 def CMDpopulate(parser, args):
   """Ensure that the cache has all up-to-date objects for the given repo."""
@@ -160,6 +199,9 @@ def CMDpopulate(parser, args):
                     help='Only cache 10000 commits of history')
   parser.add_option('--ref', action='append',
                     help='Specify additional refs to be fetched')
+  parser.add_option('--no_bootstrap', action='store_true',
+                    help='Don\'t bootstrap from Google Storage')
+
   options, args = parser.parse_args(args)
   if options.shallow and not options.depth:
     options.depth = 10000
@@ -180,6 +222,74 @@ def CMDpopulate(parser, args):
   if options.depth:
     d = ['--depth', '%d' % options.depth]
 
+  def _find(executable):
+    # Mimics the "which" utility.

[agable, 2014/04/01 23:44:52]

Do a real docstring.

[Ryan Tseng, 2014/04/02 00:32:19]

Done.

+    path_folders = os.environ.get('PATH').split(os.pathsep)
+
+    for path_folder in path_folders:
+      target = os.path.join(path_folder, executable)
+      # Just incase we have some ~/blah paths.
+      target = os.path.abspath(os.path.expanduser(target))
+      if os.path.isfile(target) and os.access(target, os.X_OK):
+        return target
+    return False
+
+  def _maybe_bootstrap_repo(directory):
+    # Bootstrap the repo from Google Stroage if there is a pre-checked out

[agable, 2014/04/01 23:44:52]

Do a real docstring.

[Ryan Tseng, 2014/04/02 00:32:19]

Done.

+    # version already. Uses 7z on windows to inflate, and unzip
+    # everywhere else.
+    if options.no_bootstrap:
+      return False
+    if sys.platform.startswith('win'):
+      if not _find('7z'):
+        print 'Cannot find 7z in the path.'
+        print 'Install 7z from http://www.7-zip.org/download.html if you want '

[agable, 2014/04/01 23:44:52]

...if you want git cache to be able to bootstrap...

[Ryan Tseng, 2014/04/02 00:32:19]

Done.

+        print 'git cache to bootstrap from Google Storage.'
+        return False
+    else:
+      if not _find('unzip'):
+        print 'Cannot find unzip in the path.'
+        print 'Install unzip if you want to create a git cache to boostrap '

[agable, 2014/04/01 23:44:52]

...if you want git cache to be able to bootstrap...

[Ryan Tseng, 2014/04/02 00:32:19]

Done.

+        print 'from Google Storage.'
+        return False
+
+    folder = UrlToCacheDir(url)
+    gs_folder = 'gs://%s/%s' % (BOOTSTRAP_BUCKET, folder)
+    gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH,
+                                                 boto_path=os.devnull,
+                                                 bypass_prodaccess=True)
+    # Get the most recent version of the zipfile.
+    _, ls_out, _ = gsutil.check_call('ls', gs_folder)
+    ls_out_sorted = sorted(ls_out.splitlines())
+    if not ls_out_sorted:
+      # This repo is not on Google Storage.
+      return False
+    latest_checkout = ls_out_sorted[-1]
+
+    # Download zip file to a temporary directory.
+    tempdir = tempfile.mkdtemp()
+    print 'Downloading %s...' % latest_checkout
+    code, out, err = gsutil.check_call('cp', latest_checkout, tempdir)
+    if code:
+      print '%s\n%s' % (out, err)
+      return False
+    filename = os.path.join(tempdir, latest_checkout.split('/')[-1])
+
+    # Unpack the file with 7z on Windows, or unzip everywhere else.
+    if sys.platform.startswith('win'):
+      cmd = ['7z', 'x', '-o%s' % directory, '-tzip', filename]
+    else:
+      cmd = ['unzip', filename, '-d', directory]
+    code = subprocess.call(cmd)

[agable, 2014/04/01 23:44:52]

retcode

[Ryan Tseng, 2014/04/02 00:32:19]

Done.

+
+    # Clean up the downloaded zipfile.
+    gclient_utils.rmtree(tempdir)
+    if code:
+      print 'Extracting bootstrap zipfile %s failed.' % filename
+      print 'Resuming normal operations'
+      return False
+    return True
+
   def _config(directory):
     RunGit(['config', 'core.deltaBaseCacheLimit',
             gclient_utils.DefaultDeltaBaseCacheLimit()], cwd=directory)
@@ -203,7 +313,9 @@ def CMDpopulate(parser, args):
       gclient_utils.rmtree(repo_dir)
       tempdir = tempfile.mkdtemp(suffix=UrlToCacheDir(url),
                                  dir=options.cache_dir)
-      RunGit(['init', '--bare'], cwd=tempdir)
+      bootstrapped = _maybe_bootstrap_repo(tempdir)
+      if not bootstrapped:
+        RunGit(['init', '--bare'], cwd=tempdir)
       _config(tempdir)
       fetch_cmd = ['fetch'] + v + d + ['origin']
       RunGit(fetch_cmd, filter_fn=filter_fn, cwd=tempdir, retry=True)
@@ -263,7 +375,7 @@ def CMDunlock(parser, args):
       unlocked = True
 
     if unlocked:
-      unlocked.append(repo_dir)      
+      unlocked.append(repo_dir)
     else:
       untouched.append(repo_dir)