Eliminate redundant work and control parallelism.

git_tools.py

 #!/usr/bin/python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-
 import cStringIO
 import os
 import re
 import subprocess
 import sys
 import threading
 
+try:
+  import git_cache
+except ImportError:
+  for p in os.environ['PATH'].split(os.pathsep):
+    if (os.path.basename(p) == 'depot_tools' and
+        os.path.exists(os.path.join(p, 'git_cache.py'))):
+      sys.path.append(p)
+  import git_cache
+
 
 # Show more information about the commands being executed.
 VERBOSE = False
 
 # The longest any single subprocess will be allowed to run.
 TIMEOUT = 40 * 60
 
 class AbnormalExit(Exception):
   pass
 
@@ skipping 23 matching lines @@
   def close(self):
     # Empty out the line buffer.
     self.write('\n')
     self.out_q.put(None)
     self.closed = True
 
 
 def GetStatusOutput(cmd, cwd=None, out_buffer=None):
   """Return (status, output) of executing cmd in a shell."""
   if VERBOSE:
-    print ''
-    print '[DEBUG] Running "%s"' % cmd
+    print >> sys.stderr, ''
+    print >> sys.stderr, '[DEBUG] Running "%s"' % cmd
 
   def _thread_main():
     thr = threading.current_thread()
     thr.status = -1
     thr.stdout = ''
     thr.stderr = '<timeout>'
     try:
       if out_buffer:
         proc = subprocess.Popen(cmd, shell=True,
                                 cwd=cwd, stdout=subprocess.PIPE,
@@ skipping 24 matching lines @@
 
   thr = threading.Thread(target=_thread_main)
   thr.daemon = True
   thr.start()
   thr.join(TIMEOUT)
 
   # pylint: disable=E1101
   if VERBOSE:
     short_output = ' '.join(thr.stdout.splitlines())
     short_output = short_output.strip(' \t\n\r')
-    print '[DEBUG] Output: %d, %-60s' % (thr.status, short_output)
+    print >> sys.stderr, (
+        '[DEBUG] Output: %d, %-60s' % (thr.status, short_output))
 
   return (thr.status, thr.stdout)
 
 
 def Git(git_repo, command, is_mirror=False, out_buffer=None):
   """Execute a git command within a local git repo."""
   if is_mirror:
     if git_repo:
       cmd = 'git --git-dir=%s %s' % (git_repo, command)
     else:
       cmd = 'git %s' % command
     cwd = None
   else:
     cmd = 'git %s' % command
     cwd = git_repo
   (status, output) = GetStatusOutput(cmd, cwd, out_buffer)
   # For Abnormal Exit, Windows returns -1, Posix returns 128.
   if status in [-1, 128]:
     raise AbnormalExit('Failed to run %s. Exited Abnormally. output %s' %
                        (cmd, output))
   elif status != 0:
     raise Exception('Failed to run %s. error %d. output %s' % (cmd, status,
                                                                output))
   return (status, output)
 
 
-def GetCacheRepoDir(git_url, cache_dir):
-  """Assuming we used git_cache to populate a cache, get the repo directory."""
-  _, out = Git(None, 'cache exists --cache-dir=%s %s' % (cache_dir, git_url),
-               is_mirror=True)
-  return out.strip()
-
-
-def Clone(git_url, git_repo, is_mirror, out_queue=None, cache_dir=None,
-          shallow=False):
+def Clone(git_url, git_repo, is_mirror, out_buffer=None):
   """Clone a repository."""
-  repo_name = git_url.split('/')[-1]
-  if out_queue:
-    buf = StdioBuffer(repo_name, out_queue)
-  else:
-    buf = None
-
-  if is_mirror == 'bare':
-    if shallow:
-      if 'adobe' in git_url:
-        # --shallow by default checks out 10000 revision, but for really large
-        # repos like adobe ones, we want significantly less than 10000.
-        shallow_arg = '--depth 10 '
-      else:
-        shallow_arg = '--shallow '
-    else:
-      shallow_arg = ''
-    cmd = 'cache populate -v --cache-dir %s %s%s' % (cache_dir, shallow_arg,
-                                                     git_url)
-    return Git(None, cmd, is_mirror=True, out_buffer=buf)
-
   cmd = 'clone'
   if is_mirror:
     cmd += ' --mirror'
   cmd += ' %s %s'  % (git_url, git_repo)
 
   if not is_mirror and not os.path.exists(git_repo):
     os.makedirs(git_repo)
 
-  return Git(None, cmd, is_mirror, buf)
+  return Git(None, cmd, is_mirror=is_mirror, out_buffer=out_buffer)
+
+
+def PopulateCache(git_url, shallow=False):
+  # --shallow by default checks out 10000 revision, but for really large
+  # repos like adobe ones, we want significantly less than 10000.
+  depth = None
+  if shallow and 'adobe' in git_url:
+    depth = 10
+  mirror = git_cache.Mirror(git_url, print_func=lambda *args: None)
+  mirror.populate(depth=depth, shallow=shallow)
+  return mirror.mirror_path
 
 
 def Fetch(git_repo, git_url, is_mirror):
   """Fetch the latest objects for a given git repository."""
   # Always update the upstream url
   Git(git_repo, 'config remote.origin.url %s' % git_url)
   Git(git_repo, 'fetch origin', is_mirror)
 
 
 def Ping(git_repo, verbose=False):
@@ skipping 89 matching lines @@
 
   # Check if svn_rev is newer than the current refspec revision.
   try:
     found_rev = _FindRevForCommitish(git_repo, refspec, is_mirror)
   # Sometimes this fails because it's looking in a branch that hasn't been
   # fetched from upstream yet. Let it fetch and try again.
   except AbnormalExit:
     found_rev = None
   if (not found_rev or found_rev < int(svn_rev)) and fetch_url:
     if VERBOSE:
-      print 'Fetching %s %s [%s < %s]' % (git_repo, refspec, found_rev, svn_rev)
+      print >> sys.stderr, (
+          'Fetching %s %s [%s < %s]' % (git_repo, refspec, found_rev, svn_rev))
     Fetch(git_repo, fetch_url, is_mirror)
     found_rev = _FindRevForCommitish(git_repo, refspec, is_mirror)
 
   # Find the first commit matching the given git-svn-id regex.
   _, output = Git(
       git_repo,
       ('log -E --grep="^git-svn-id: [^@]*@%s [A-Za-z0-9-]*$" '
        '-1 --format="%%H" %s') % (regex, refspec),
       is_mirror)
   output = output.strip()
   if not re.match('^[0-9a-fA-F]{40}$', output):
     raise SearchError('Cannot find revision %s in %s:%s' % (svn_rev, git_repo,
                                                             refspec))
 
   # Check if it actually matched the svn_rev that was requested.
   found_rev = _FindRevForCommitish(git_repo, output, is_mirror)
   found_msg = svn_rev
   if found_rev != int(svn_rev):
     found_msg = '%s [actual: %s]' % (svn_rev, found_rev)
-  print '%s: %s <-> %s' % (git_repo, output, found_msg)
+  print >> sys.stderr, '%s: %s <-> %s' % (git_repo, output, found_msg)
   return output
 
 
 def SearchExact(git_repo, svn_rev, is_mirror, refspec='FETCH_HEAD',
     fetch_url=None):
   """Return the Git commit id exactly matching the given SVN revision.
 
   If fetch_url is not None, will update repo if revision is newer."""
   regex = str(svn_rev)
   return _SearchImpl(git_repo, svn_rev, is_mirror, refspec, fetch_url, regex)
 
 
 def Search(git_repo, svn_rev, is_mirror, refspec='FETCH_HEAD', fetch_url=None):
   """Return the Git commit id fuzzy matching the given SVN revision.
 
   If fetch_url is not None, will update repo if revision is newer."""
   regex = CreateLessThanOrEqualRegex(svn_rev)
   return _SearchImpl(git_repo, svn_rev, is_mirror, refspec, fetch_url, regex)