Make gclient_scm.py use cache_dir

gclient_scm.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Gclient-specific SCM-specific operations."""
 
 import collections
 import logging
 import os
 import posixpath
@@ skipping 129 matching lines @@
           command, self.__class__.__name__))
 
     return getattr(self, command)(options, args, file_list)
 
 
 class GitWrapper(SCMWrapper):
   """Wrapper for Git"""
   name = 'git'
   remote = 'origin'
 
   cache_dir = None

[agable, 2014/02/28 02:06:32]

...I don't see *any* code in this file that assigns to this. Do we *ever* hit
the cache_dir codepaths?

[Ryan Tseng, 2014/02/28 02:22:50]

Well 691 does, since it passes it into git cache populate.

Its sent by bot update using the .gclient file, and probably injected in this
way:
https://code.google.com/p/chromium/codesearch#chromium/tools/depot_tools/gcli...

[agable, 2014/02/28 02:38:24]

No, 691 (thankfully) only uses self.cache_dir if it is already set, but it
doesn't set it to anything.
OH GOD that's so terrifying. Can we please make it a input parameter to __init__
or something?

   # If a given cache is used in a solution more than once, prevent multiple
   # threads from updating it simultaneously.
   cache_locks = collections.defaultdict(threading.Lock)

[agable, 2014/02/28 02:06:32]

This can go away, I believe.

[Ryan Tseng, 2014/02/28 02:22:50]

Done.

 
   def __init__(self, url=None, root_dir=None, relpath=None):
     """Removes 'git+' fake prefix from git URL."""
     if url.startswith('git+http://') or url.startswith('git+https://'):
       url = url[4:]
     SCMWrapper.__init__(self, url, root_dir, relpath)
 
   @staticmethod
   def BinaryExists():
     """Returns true if the command exists."""
@@ skipping 181 matching lines @@
             ['git', 'config', 'remote.%s.gclient-auto-fix-url' % self.remote],
             cwd=self.checkout_path).strip() != 'False'):
       print('_____ switching %s to a new upstream' % self.relpath)
       # Make sure it's clean
       self._CheckClean(rev_str)
       # Switch over to the new upstream
       self._Run(['remote', 'set-url', self.remote, url], options)
       self._FetchAndReset(revision, file_list, options)
       return_early = True
 
-    # Need to do this in the normal path as well as in the post-remote-switch
-    # path.
-    self._PossiblySwitchCache(url, options)
-
     if return_early:
       return self._Capture(['rev-parse', '--verify', 'HEAD'])
 
     cur_branch = self._GetCurrentBranch()
 
     # Cases:
     # 0) HEAD is detached. Probably from our initial clone.
     #   - make sure HEAD is contained by a named ref, then update.
     # Cases 1-4. HEAD is a branch.
     # 1) current branch is not tracking a remote branch (could be git-svn)
@@ skipping 305 matching lines @@
             '#Initial_checkout' ) % rev)
 
     return sha1
 
   def FullUrlForRelativeUrl(self, url):
     # Strip from last '/'
     # Equivalent to unix basename
     base_url = self.url
     return base_url[:base_url.rfind('/')] + url
 
-  @staticmethod
-  def _NormalizeGitURL(url):
-    '''Takes a git url, strips the scheme, and ensures it ends with '.git'.'''
-    idx = url.find('://')
-    if idx != -1:
-      url = url[idx+3:]
-    if not url.endswith('.git'):
-      url += '.git'
-    return url
-
-  def _PossiblySwitchCache(self, url, options):
-    """Handles switching a repo from with-cache to direct, or vice versa.
-
-    When we go from direct to with-cache, the remote url changes from the
-    'real' url to the local file url (in cache_dir). Therefore, this function
-    assumes that |url| points to the correctly-switched-over local file url, if
-    we're in cache_mode.
-
-    When we go from with-cache to direct, assume that the normal url-switching
-    code already flipped the remote over, and we just need to repack and break
-    the dependency to the cache.
-    """
-
-    altfile = os.path.join(
-        self.checkout_path, '.git', 'objects', 'info', 'alternates')
-    if self.cache_dir:
-      if not os.path.exists(altfile):
-        try:
-          with open(altfile, 'w') as f:
-            f.write(os.path.join(url, 'objects'))
-          # pylint: disable=C0301
-          # This dance is necessary according to emperical evidence, also at:
-          # http://lists-archives.com/git/713652-retrospectively-add-alternates-to-a-repository.html
-          self._Run(['repack', '-ad'], options)
-          self._Run(['repack', '-adl'], options)
-        except Exception:
-          # If something goes wrong, try to remove the altfile so we'll go down
-          # this path again next time.
-          try:
-            os.remove(altfile)
-          except OSError as e:
-            print >> sys.stderr, "FAILED: os.remove('%s') -> %s" % (altfile, e)
-          raise
-    else:
-      if os.path.exists(altfile):
-        self._Run(['repack', '-a'], options)
-        os.remove(altfile)
-
   def _CreateOrUpdateCache(self, url, options):
     """Make a new git mirror or update existing mirror for |url|, and return the
     mirror URI to clone from.
 
     If no cache-dir is specified, just return |url| unchanged.
     """
     if not self.cache_dir:
       return url
-
-    # Replace - with -- to avoid ambiguity. / with - to flatten folder structure
-    folder = os.path.join(
-      self.cache_dir,
-      self._NormalizeGitURL(url).replace('-', '--').replace('/', '-'))
-    altfile = os.path.join(folder, 'objects', 'info', 'alternates')
-
-    # If we're bringing an old cache up to date or cloning a new cache, and the
-    # existing repo is currently a direct clone, use its objects to help out
-    # the fetch here.
-    checkout_objects = os.path.join(self.checkout_path, '.git', 'objects')
-    checkout_altfile = os.path.join(checkout_objects, 'info', 'alternates')
-    use_reference = (
-        os.path.exists(checkout_objects) and
-        not os.path.exists(checkout_altfile))
-
     v = ['-v'] if options.verbose else []
-    filter_fn = lambda l: '[up to date]' not in l
-    with self.cache_locks[folder]:
-      gclient_utils.safe_makedirs(self.cache_dir)
-      if not os.path.exists(os.path.join(folder, 'config')):
-        gclient_utils.rmtree(folder)
-        cmd = ['clone'] + v + ['-c', 'core.deltaBaseCacheLimit=2g',
-                               '--progress', '--bare']
-
-        if use_reference:
-          cmd += ['--reference', os.path.abspath(self.checkout_path)]
-
-        self._Run(cmd + [url, folder],
-                  options, filter_fn=filter_fn, cwd=self.cache_dir, retry=True)
-      else:
-        # For now, assert that host/path/to/repo.git is identical. We may want
-        # to relax this restriction in the future to allow for smarter cache
-        # repo update schemes (such as pulling the same repo, but from a
-        # different host).
-        existing_url = self._Capture(['config', 'remote.%s.url' % self.remote],
-                                     cwd=folder)
-        assert self._NormalizeGitURL(existing_url) == self._NormalizeGitURL(url)
-
-        if use_reference:
-          with open(altfile, 'w') as f:
-            f.write(os.path.abspath(checkout_objects))
-
-        # Would normally use `git remote update`, but it doesn't support
-        # --progress, so use fetch instead.
-        self._Run(['fetch'] + v + ['--multiple', '--progress', '--all'],
-                  options, filter_fn=filter_fn, cwd=folder, retry=True)
-
-      # If the clone has an object dependency on the existing repo, break it
-      # with repack and remove the linkage.
-      if os.path.exists(altfile):
-        self._Run(['repack', '-a'], options, cwd=folder)
-        os.remove(altfile)
-    return folder
+    self._Run(['cache', 'populate'] + v +
+              ['--shallow', '--cache-dir', self.cache_dir, url],
+              options, cwd=self._root_dir, retry=True)
+    return self._Run(['cache', 'exists', '--cache-dir', self.cache_dir, url],
+                     options).strip()
 
   def _Clone(self, revision, url, options):
     """Clone a git repository from the given URL.
 
     Once we've cloned the repo, we checkout a working branch if the specified
     revision is a branch head. If it is a tag or a specific commit, then we
     leave HEAD detached as it makes future updates simpler -- in this case the
     user should first create a new branch or switch to an existing branch before
     making changes in the repo."""
     if not options.verbose:
@@ skipping 233 matching lines @@
       # password prompt and simply allow git to fail noisily.  The error
       # message produced by git will be copied to gclient's output.
       env = kwargs.get('env') or kwargs.setdefault('env', os.environ.copy())
       env.setdefault('GIT_ASKPASS', 'true')
       env.setdefault('SSH_ASKPASS', 'true')
     else:
       kwargs.setdefault('print_stdout', True)
     stdout = kwargs.get('stdout', sys.stdout)
     stdout.write('\n________ running \'git %s\' in \'%s\'\n' % (
                  ' '.join(args), kwargs['cwd']))
-    gclient_utils.CheckCallAndFilter(['git'] + args, **kwargs)
+    return gclient_utils.CheckCallAndFilter(['git'] + args, **kwargs)
 
 
 class SVNWrapper(SCMWrapper):
   """ Wrapper for SVN """
   name = 'svn'
 
   @staticmethod
   def BinaryExists():
     """Returns true if the command exists."""
     try:
@@ skipping 448 matching lines @@
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'theirs-conflict'))
     elif options.manually_grab_svn_rev:
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'postpone'))
     elif command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
       new_command.extend(('--accept', 'postpone'))
     return new_command