Checkout tarball for chrome and blink for initial sync on a bot

gclient_scm.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Gclient-specific SCM-specific operations."""
 
 import collections
 import logging
 import os
 import posixpath
 import re
 import sys
 import tempfile
 import threading
 import time
+import urlparse
 
+import download_from_google_storage
 import gclient_utils
 import scm
 import subprocess2
 
 
 THIS_FILE_PATH = os.path.abspath(__file__)
 
+GSUTIL_DEFAULT_PATH = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    'third_party', 'gsutil', 'gsutil')
+
 
 class DiffFiltererWrapper(object):
   """Simple base class which tracks which file is being diffed and
   replaces instances of its file name in the original and
   working copy lines of the svn/git diff output."""
   index_string = None
   original_prefix = "--- "
   working_prefix = "+++ "
 
   def __init__(self, relpath):
@@ skipping 1129 matching lines @@
       except (gclient_utils.Error, subprocess2.CalledProcessError):
         if options.reset and options.delete_unversioned_trees:
           print 'Removing troublesome path %s' % self.checkout_path
           gclient_utils.rmtree(self.checkout_path)
           exists = False
         else:
           msg = ('Can\'t update/checkout %s if an unversioned directory is '
                  'present. Delete the directory and try again.')
           raise gclient_utils.Error(msg % self.checkout_path)
 
+    BASE_URLS = {
+      '/chrome/trunk/src': {
+          'gs_path': 'gs://chromium-svn-checkout/chrome/'

[cmp, 2014/01/07 02:10:14]

indent mismatch (+2 on line 1180, +4 on line 1181), check the style guide for
the correct indent

append a comma to line 1181 and line 1184

Come to think of it, what else would you ever put into this dict?  If the answer
is 'nothing else', then remove the dict and have /chrome/trunk/src just point to
the string GS URL gs_path points to on line 1181, and then the same change for
line 1183.

[hinoka, 2014/01/07 02:17:10]

Done.

+      },
+      '/blink/trunk' : {

[cmp, 2014/01/07 02:10:14]

remove space before :

[hinoka, 2014/01/07 02:17:10]

Done.

+          'gs_path': 'gs://chromium-svn-checkout/blink/'
+      }
+    }
     if not exists:
+      try:
+        # Split out the revision number since it's not useful for us.
+        base_path = urlparse.urlparse(url).path.split('@')[0]
+        if ('CHROME_HEADLESS' in os.environ
+            and sys.platform == 'linux2'  # TODO(hinoka): Enable for win/mac.
+            and base_path in BASE_URLS):
+          # Use a tarball for initial sync if we are on a bot.
+          # Get an unauthenticated gsutil instance.
+          gsutil = download_from_google_storage.Gsutil(
+              GSUTIL_DEFAULT_PATH, boto_path=os.devnull)
+
+          entry = BASE_URLS[base_path]
+          gs_path = entry['gs_path']
+          _, out, _ = gsutil.check_call('ls', gs_path)
+          # So that we can get the most recent revision.
+          sorted_items = sorted(out.splitlines())
+          latest_checkout = sorted_items[-1]
+
+          tempdir = tempfile.mkdtemp()
+          print 'Downloading %s...' % latest_checkout
+          code, out, err = gsutil.check_call('cp', latest_checkout, tempdir)
+          if code:
+            print '%s\n%s' % (out, err)
+            raise Exception()
+          filename = latest_checkout.split('/')[-1]
+          tarball = os.path.join(tempdir, filename)
+          print 'Unpacking into %s...' % self.checkout_path
+          if sys.platform == 'linux2':

[cmp, 2014/01/07 02:06:25]

This code exists in a branch that already tested sys.platform for if they
compare to 'linux2', so I think line 1215 and lines 1222-1224 should be deleted,
and lines 1216-1221 should be de-indented.

[hinoka, 2014/01/07 02:17:10]

Done.

+            gclient_utils.safe_makedirs(self.checkout_path)
+            cmd = ['tar', '--extract', '--ungzip',
+                   '--directory', self.checkout_path,
+                   '--file', tarball]
+            gclient_utils.CheckCallAndFilter(
+                cmd, stdout=sys.stdout, print_stdout=True)
+          else:
+            # TODO(hinoka): Implement for mac and windows.
+            raise NotImplementedError
+          print 'Deleting temp file'
+          gclient_utils.rmtree(tempdir)
+
+          # Rewrite the repository root to match.
+          tarball_url = scm.SVN.CaptureLocalInfo(
+              ['.'], self.checkout_path)['Repository Root']
+          tarball_parsed = urlparse.urlparse(tarball_url)
+          tarball_root = '%s://%s' % (tarball_parsed.scheme,
+                                      tarball_parsed.netloc)
+          local_parsed = urlparse.urlparse(url)
+          local_root = '%s://%s' % (local_parsed.scheme, local_parsed.netloc)
+
+          if tarball_root != local_root:
+            print 'Switching repository root to %s' % local_root
+            self._Run(['switch', '--relocate', tarball_root,
+                       local_root, self.checkout_path],
+                      options)
+      except Exception as e:
+        print 'We tried to get a source tarball but failed.'
+        print 'Resuming normal operations.'
+        print str(e)
+
       gclient_utils.safe_makedirs(os.path.dirname(self.checkout_path))
       # We need to checkout.
       command = ['checkout', url, self.checkout_path]
       command = self._AddAdditionalUpdateFlags(command, options, revision)
       self._RunAndGetFileList(command, options, file_list, self._root_dir)
       return self.Svnversion()
 
     if not managed:
       print ('________ unmanaged solution; skipping %s' % self.relpath)
       return self.Svnversion()
@@ skipping 268 matching lines @@
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'theirs-conflict'))
     elif options.manually_grab_svn_rev:
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'postpone'))
     elif command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
       new_command.extend(('--accept', 'postpone'))
     return new_command