Checkout tarball for chrome and blink for initial sync on a bot

gclient_scm.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Gclient-specific SCM-specific operations."""
 
 import collections
 import logging
 import os
 import posixpath
 import re
+import shutil
 import sys
+import tarfile
 import tempfile
 import threading
 import time
+import urlparse
 
+import download_from_google_storage
 import gclient_utils
 import scm
 import subprocess2
 
 
 THIS_FILE_PATH = os.path.abspath(__file__)
 
+GSUTIL_DEFAULT_PATH = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    'third_party', 'gsutil', 'gsutil')
+
 
 class DiffFiltererWrapper(object):
   """Simple base class which tracks which file is being diffed and
   replaces instances of its file name in the original and
   working copy lines of the svn/git diff output."""
   index_string = None
   original_prefix = "--- "
   working_prefix = "+++ "
 
   def __init__(self, relpath):
@@ skipping 1129 matching lines @@
       except (gclient_utils.Error, subprocess2.CalledProcessError):
         if options.reset and options.delete_unversioned_trees:
           print 'Removing troublesome path %s' % self.checkout_path
           gclient_utils.rmtree(self.checkout_path)
           exists = False
         else:
           msg = ('Can\'t update/checkout %s if an unversioned directory is '
                  'present. Delete the directory and try again.')
           raise gclient_utils.Error(msg % self.checkout_path)
 
+    BASE_URLS = {
+      '/chrome/trunk/src': {
+          'gs_path': '/chromium-svn-checkout/chrome/'

[cmp, 2013/12/26 19:25:45]

Prepend gs:/ to this and on line 1186.

[hinoka, 2014/01/07 00:57:25]

Done.

+      },
+      '/blink/trunk' : {
+          'gs_path': '/chromium-svn-checkout/blink/'
+      }
+    }
     if not exists:
+      try:
+        # Split out the revision number, its not useful for us.

[cmp, 2013/12/26 19:25:45]

number, its -> number since it's

[hinoka, 2014/01/07 00:57:25]

Done.

+        base_path = urlparse.urlparse(url).path.split('@')[0]
+        if ('CHROME_HEADLESS' in os.environ
+            and sys.platform == 'linux2'  # TODO(hinoka): Enable for win/mac.
+            and base_path in BASE_URLS):
+          # Use a tarball for initial sync if we are on a bot.
+          # Get an unauthenticated gsutil instance.
+          gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH,
+                                                        boto_path=os.devnull)

[cmp, 2013/12/26 19:25:45]

Indent needs to be fixed.  It would be better as:

gsutil = download_from_google_storage.Gsutil(
    GSUTIL_DEFAULT_PATH, boto_path=os.devnull)

[hinoka, 2014/01/07 00:57:25]

Done.

+
+          entry = BASE_URLS[base_path]
+          gs_path = entry['gs_path']
+          _, out, _ = gsutil.check_call('ls', 'gs:/%s' % gs_path)

[cmp, 2013/12/26 19:25:45]

Just use gs_path here (no string shenanigans req'd).

[hinoka, 2014/01/07 00:57:25]

Done.

+          # So that we can get the most recent revision.
+          sorted_items = sorted(out.splitlines())
+          latest_checkout = sorted_items[-1]
+
+          tempdir = tempfile.mkdtemp()
+          print 'Downloading %s...' % latest_checkout
+          code, out, err = gsutil.check_call('cp', latest_checkout, tempdir)
+          if code:
+            print '%s\n%s' % (out, err)
+            raise Exception()
+          filename = latest_checkout.split('/')[-1]
+          tarball = os.path.join(tempdir, filename)
+          print 'Unpacking into %s...' % self.checkout_path
+          with tarfile.open(tarball, 'r:gz') as f:
+            f.extractall(self.checkout_path)

[cmp, 2013/12/26 19:25:45]

Use a local executable to unpack rather than Python.

[hinoka, 2014/01/07 00:57:25]

This would make gclient_scm pretty unportable :(
To do this we'd need to detect the user's platform, and download the appropriate
version of 7zip into depot_tools, as we do with svn/git.  I'm not convince we'd
get anything out of that.

[cmp, 2014/01/07 01:09:52]

You're assuming too much about how well an in-process Python module works to
unpack large files.  If you've tested this on compressed archives that are the
size of the Chrome and Blink checkouts packed, okay.  But if you haven't tested
it, I can tell you that we used to use in-process Python modules for compression
and decompression and it did not work well.  We switched to use 7z.

Such a change to use a separate executable wouldn't make gclient_scm unportable
in general.  It would make it unportable just for the bots, and in those
environments we have control over what's available.  We already have a code
branch here that is Linux specific.  In that branch, why not call out to gzip? 
In the Windows code branch, why not call out to 7z?  etc.

[hinoka, 2014/01/07 02:03:32]

How big of a tarball and what sort of problems are we talking about? Chrome is
500mb compressed.

I'll use "tar -zxf" for linux, since I havn't seen a distro that doesn't include
tar, and add a todo for windows.

+          print 'Deleting temp file'
+          shutil.rmtree(tempdir)

[cmp, 2014/01/07 01:09:52]

We found rmtree() to cause problems on Windows.  We used rd instead.  The same
may be true on other platforms.  This is the same issue as the
compress/decompress executable problem (something we hit a long time ago, solved
with a separate executable, and it's been working great ever since implementing
that decision).

[hinoka, 2014/01/07 02:03:32]

Just realized gclient_utils.rmtree does exactly what you said, switched to using
that.

+
+          # Rewrite the repository root to match.
+          tarball_url = scm.SVN.CaptureLocalInfo(
+              ['.'], self.checkout_path)['Repository Root']
+          tarball_parsed = urlparse.urlparse(tarball_url)
+          tarball_root = '%s://%s' % (tarball_parsed.scheme,
+                                      tarball_parsed.netloc)
+          local_parsed = urlparse.urlparse(url)
+          local_root = '%s://%s' % (local_parsed.scheme, local_parsed.netloc)
+
+          if tarball_root != local_root:
+            print 'Switching repository root to %s' % local_root
+            self._Run(['switch', '--relocate', tarball_root,
+                        local_root, self.checkout_path],

[cmp, 2013/12/26 19:25:45]

Indent needs to be less one character on line 1234.

[hinoka, 2014/01/07 00:57:25]

Done.

+                      options)
+      except Exception as e:
+        print 'We tried to get a source tarball but failed.'
+        print 'Resuming normal operations.'
+        print str(e)
+
       gclient_utils.safe_makedirs(os.path.dirname(self.checkout_path))
       # We need to checkout.
       command = ['checkout', url, self.checkout_path]
       command = self._AddAdditionalUpdateFlags(command, options, revision)
       self._RunAndGetFileList(command, options, file_list, self._root_dir)
       return self.Svnversion()
 
     if not managed:
       print ('________ unmanaged solution; skipping %s' % self.relpath)
       return self.Svnversion()
@@ skipping 268 matching lines @@
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'theirs-conflict'))
     elif options.manually_grab_svn_rev:
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'postpone'))
     elif command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
       new_command.extend(('--accept', 'postpone'))
     return new_command