Add a git cache for gclient sync operations.

gclient_scm.py

Index: gclient_scm.py
diff --git a/gclient_scm.py b/gclient_scm.py
index 5c3929a9639792e358a350e311b65a152664735e..d580d2f7e03544a69e8e86a2ad8597c4d3f7f966 100644
--- a/gclient_scm.py
+++ b/gclient_scm.py
@@ -152,6 +152,35 @@ class SCMWrapper(object):
     return getattr(self, command)(options, args, file_list)
 
 
+class GitFilter(object):
+  PERCENT_RE = re.compile('.* ([0-9]{1,2})% .*')
+
+  def __init__(self, predicate=None):
+    """Create a new GitFilter.
+
+    Args:
+      predicate (f(line)): An optional function which is invoked for every line.
+        The line will be skipped if the predicate is False.
+    """
+    self.last_time = 0
+    self.predicate = predicate
+
+  def __call__(self, line):
+    # git uses an escape sequence to clear the line; elide it.
+    esc = line.find(unichr(033))
+    if esc > -1:
+      line = line[:esc]
+    if self.predicate and not self.predicate(line):
+      return
+    now = time.time()
+    match = self.PERCENT_RE.match(line)
+    if not match:
+      self.last_time = 0
+    if (now - self.last_time) >= 2:

[szager1, 2013/07/01 23:38:29]

Maybe parameterize the suppression interval?  Two seconds may be a good interval
for interactive use; for bots, we might want to make it longer.

[iannucci, 2013/07/01 23:55:52]

I was making it shorter explicitly FOR the bots. Maybe nag_timer / 2?

[szager1, 2013/07/02 00:08:05]

That would be fine, or even something like 10; but two seconds will cause an
awful lot of log file gas for clobber checkouts.

[iannucci, 2013/07/02 00:22:27]

Ok, I set it to nag_timer / 2 (which should be 15 seconds in the default case).

+      self.last_time = now
+      print line
+
+
 class GitWrapper(SCMWrapper):
   """Wrapper for Git"""
 
@@ -297,6 +326,8 @@ class GitWrapper(SCMWrapper):
       verbose = ['--verbose']
       printed_path = True
 
+    url = self._CreateOrUpdateCache(url, options)
+
     if revision.startswith('refs/heads/'):
       rev_type = "branch"
     elif revision.startswith('origin/'):
@@ -691,6 +722,50 @@ class GitWrapper(SCMWrapper):
     base_url = self.url
     return base_url[:base_url.rfind('/')] + url
 
+  @staticmethod
+  def _CacheFolder(url, options):
+    """Transforms a url into the path to the corresponding git cache.
+
+    Ex. (assuming cache_dir == '/cache')
+      IN:  https://chromium.googlesource.com/chromium/src
+      OUT: /cache/chromium.googlesource.com-chromium-src.git

[szager1, 2013/07/01 23:38:29]

Any reason not to mimic the full directory structure?  Admittedly it's unlikely,
but this would be a naming conflict:

https://chromium.googlesource.com/foo/bar.git
https://chromium.googlesource.com/foo-bar.git

[iannucci, 2013/07/01 23:55:52]

Eh... I'm much more inclined to flatten it out, since it makes observing the
caches easier (esp. on windows). I could replace '-' with '--' to deal with this
conflict though.

[szager1, 2013/07/02 00:08:05]

I'd be satisfied with collision detection logic.

[iannucci, 2013/07/02 00:22:27]

Done.

+    """
+    idx = url.find('://')
+    if idx != -1:
+      url = url[idx+3:]
+    url = url.replace('/', '-')
+    if not url.endswith('.git'):
+      url += '.git'
+    return os.path.join(options.cache_dir, url)
+
+  def _CreateOrUpdateCache(self, url, options):
+    """Make a new git mirror or update existing mirror for |url|, and return the
+    mirror URI to clone from.
+
+    If no cache-dir is specified, just return |url| unchanged.
+    """
+    if not options.cache_dir:
+      return url
+    folder = self._CacheFolder(url, options)
+    v = ['-v'] if options.verbose else []
+    filter_fn = lambda l: '[up to date]' not in l
+    with options.cache_lock:
+      lock = options.cache_locks[folder]
+    with lock:
+      gclient_utils.safe_makedirs(options.cache_dir)
+      if not os.path.exists(os.path.join(folder, 'config')):
+        gclient_utils.rmtree(folder)
+        self._Run(['clone'] + v + ['-c', 'core.deltaBaseCacheLimit=2g',
+                                   '--progress', '--mirror', url, folder],
+                  options, git_filter=True, filter_fn=filter_fn,
+                  cwd=options.cache_dir)
+      else:
+        # Would normally use `git remote update`, but it doesn't support
+        # --progress, so use fetch instead.
+        self._Run(['fetch'] + v + ['--multiple', '--progress', '--all'],
+                  options, git_filter=True, filter_fn=filter_fn, cwd=folder)
+    return folder
+
   def _Clone(self, revision, url, options):
     """Clone a git repository from the given URL.
 
@@ -704,6 +779,8 @@ class GitWrapper(SCMWrapper):
       # to stdout
       print('')
     clone_cmd = ['-c', 'core.deltaBaseCacheLimit=2g', 'clone', '--progress']
+    if options.cache_dir:
+      clone_cmd.append('--shared')

[szager1, 2013/07/01 23:38:29]

Hmm... not sure I agree with the use of --shared.  For our use case, it seems to
add risk without providing benefit.

Here's the best explanation I can find online about the difference between
--local (which is implied when cloning from a file path) and --shared:

http://lists-archives.com/git/494327-clarify-git-clone-local-shared-reference...

[iannucci, 2013/07/01 23:55:52]

--local won't work correctly on windows, or cross-device (relies on hardlinks).
I explicitly wanted shared because I want there to only be one copy of the
object per-machine.

The normal pitfalls of --shared won't apply here, since we know that the git
repos in cache are full mirrors of the remote. Any issues there (i.e. force push
of checked out refs) would be exactly the same amount of pain as cloning
directly from the remote, and the remedy is the same (clobber the non-cache
checkout).

[szager1, 2013/07/02 00:08:05]

Fair enough!

     if revision.startswith('refs/heads/'):
       clone_cmd.extend(['-b', revision.replace('refs/heads/', '')])
       detach_head = False
@@ -719,20 +796,9 @@ class GitWrapper(SCMWrapper):
     if not os.path.exists(parent_dir):
       gclient_utils.safe_makedirs(parent_dir)
 
-    percent_re = re.compile('.* ([0-9]{1,2})% .*')
-    def _GitFilter(line):
-      # git uses an escape sequence to clear the line; elide it.
-      esc = line.find(unichr(033))
-      if esc > -1:
-        line = line[:esc]
-      match = percent_re.match(line)
-      if not match or not int(match.group(1)) % 10:
-        print '%s' % line
-
     for _ in range(3):
       try:
-        self._Run(clone_cmd, options, cwd=self._root_dir, filter_fn=_GitFilter,
-                  print_stdout=False)
+        self._Run(clone_cmd, options, cwd=self._root_dir, git_filter=True)
         break
       except subprocess2.CalledProcessError, e:
         # Too bad we don't have access to the actual output yet.
@@ -947,7 +1013,10 @@ class GitWrapper(SCMWrapper):
           time.sleep(backoff_time)
           backoff_time *= 1.3
 
-  def _Run(self, args, options, **kwargs):
+  def _Run(self, args, _options, git_filter=False, **kwargs):
+    if git_filter:
+      kwargs['filter_fn'] = GitFilter(kwargs.get('filter_fn'))
+      kwargs.setdefault('print_stdout', False)
     kwargs.setdefault('cwd', self.checkout_path)
     kwargs.setdefault('print_stdout', True)
     kwargs.setdefault('nag_timer', self.nag_timer)