Eliminate redundant work and control parallelism.

deps2git.py

Index: deps2git.py
diff --git a/deps2git.py b/deps2git.py
index f29e67c989715d640c65a38bb97abe9fcceba76a..2f64ba62fcb0fe1f4bdd33bf170bf95685abbfc6 100755
--- a/deps2git.py
+++ b/deps2git.py
@@ -6,6 +6,7 @@
 """Convert SVN based DEPS into .DEPS.git for use with NewGit."""
 
 import collections
+from cStringIO import StringIO
 import json
 import optparse
 import os
@@ -13,14 +14,29 @@ import Queue
 import shutil
 import subprocess
 import sys
+import threading
 import time
 
-from multiprocessing.pool import ThreadPool
-
 import deps_utils
 import git_tools
 import svn_to_git_public
 
+try:
+  import git_cache
+except ImportError:
+  for p in os.environ['PATH'].split(os.pathsep):
+    if (os.path.basename(p) == 'depot_tools' and
+        os.path.exists(os.path.join(p, 'git_cache.py'))):
+      sys.path.append(p)
+  import git_cache
+
+Job = collections.namedtuple(
+    'Job',
+    ['dep', 'git_url', 'dep_url', 'path', 'git_host', 'dep_rev', 'svn_branch'])
+
+ConversionResults = collections.namedtuple(
+    'ConversionResults',
+    ['new_deps', 'deps_vars', 'bad_git_urls', 'bad_dep_urls', 'bad_git_hash'])
 
 # This is copied from depot_tools/gclient.py
 DEPS_OS_CHOICES = {
@@ -36,6 +52,7 @@ DEPS_OS_CHOICES = {
     "android": "android",
 }
 
+
 def SplitScmUrl(url):
   """Given a repository, return a set containing the URL and the revision."""
   url_split = url.split('@')
@@ -46,14 +63,15 @@ def SplitScmUrl(url):
   return (scm_url, scm_rev)
 
 
-def SvnRevToGitHash(svn_rev, git_url, repos_path, workspace, dep_path,
-                    git_host, svn_branch_name=None, cache_dir=None):
+def SvnRevToGitHash(
+    svn_rev, git_url, repos_path, workspace, dep_path, git_host,
+    svn_branch_name=None, cache_dir=None, outbuf=None, shallow=None):
   """Convert a SVN revision to a Git commit id."""
   git_repo = None
   if git_url.startswith(git_host):
     git_repo = git_url.replace(git_host, '')
   else:
-    raise Exception('Unknown git server %s, host %s' % (git_url, git_host))
+    raise RuntimeError('Unknown git server %s, host %s' % (git_url, git_host))
   if repos_path is None and workspace is None and cache_dir is None:
     # We're running without a repository directory (i.e. no -r option).
     # We cannot actually find the commit id, but this mode is useful
@@ -64,11 +82,10 @@ def SvnRevToGitHash(svn_rev, git_url, repos_path, workspace, dep_path,
     mirror = True
     git_repo_path = os.path.join(repos_path, git_repo)
     if not os.path.exists(git_repo_path) or not os.listdir(git_repo_path):
-      git_tools.Clone(git_url, git_repo_path, mirror)
+      git_tools.Clone(git_url, git_repo_path, mirror, outbuf)
   elif cache_dir:
-    mirror = 'bare'
-    git_tools.Clone(git_url, None, mirror, cache_dir=cache_dir)
-    git_repo_path = git_tools.GetCacheRepoDir(git_url, cache_dir)
+    mirror = True
+    git_repo_path = git_tools.PopulateCache(git_url, shallow)
   else:
     mirror = False
     git_repo_path = os.path.join(workspace, dep_path)
@@ -84,7 +101,7 @@ def SvnRevToGitHash(svn_rev, git_url, repos_path, workspace, dep_path,
       else:
         shutil.rmtree(git_repo_path)
     if not os.path.exists(git_repo_path):
-      git_tools.Clone(git_url, git_repo_path, mirror)
+      git_tools.Clone(git_url, git_repo_path, mirror, outbuf)
 
   if svn_branch_name:
     # svn branches are mirrored with:
@@ -102,110 +119,59 @@ def SvnRevToGitHash(svn_rev, git_url, repos_path, workspace, dep_path,
   # Work-around for:
   #   http://code.google.com/p/chromium/issues/detail?id=362222
   if (git_url.startswith('https://chromium.googlesource.com/external/pefile')
-      and int(svn_rev) == 63):
+      and int(svn_rev) in (63, 141)):
     return '1ceaa279daa62b71e3431e58f68be6a96dd1519a'
 
-  try:
-    return git_tools.Search(git_repo_path, svn_rev, mirror, refspec, git_url)
-  except Exception:
-    print >> sys.stderr, '%s <-> ERROR' % git_repo_path
-    raise
+  return git_tools.Search(git_repo_path, svn_rev, mirror, refspec, git_url)
 
-def ConvertDepsToGit(deps, options, deps_vars, svn_to_git_objs):
-  """Convert a 'deps' section in a DEPS file from SVN to Git."""
-  new_deps = {}
-  bad_git_urls = set([])
-  bad_dep_urls = []
-  bad_override = []
-  bad_git_hash = []
 
-  # Populate our deps list.
-  deps_to_process = {}
-  for dep, dep_url in deps.iteritems():
-    if not dep_url:  # dep is 'None' and emitted to exclude the dep
-      new_deps[dep] = None
+def MessageMain(message_q, threads):
+  while True:
+    try:
+      msg = message_q.get(True, 10)
+    except Queue.Empty:
+      print >> sys.stderr, 'Still working on:'
+      for s in sorted([th.working_on for th in threads if th.working_on]):
+        print >> sys.stderr, '  %s' % s
       continue
+    if msg is Queue.Empty:
+      return
+    if msg:
+      print >> sys.stderr, msg
 
-    # Get the URL and the revision/hash for this dependency.
-    dep_url, dep_rev = SplitScmUrl(deps[dep])
-
-    path = dep
-    git_url = dep_url
-    svn_branch = None
-    git_host = dep_url
-
-    if not dep_url.endswith('.git'):
-      # Convert this SVN URL to a Git URL.
-      for svn_git_converter in svn_to_git_objs:
-        converted_data = svn_git_converter.SvnUrlToGitUrl(dep, dep_url)
-        if converted_data:
-          path, git_url, git_host = converted_data[:3]
-          if len(converted_data) > 3:
-            svn_branch = converted_data[3]
-          break
-      else:
-        # Make all match failures fatal to catch errors early. When a match is
-        # found, we break out of the loop so the exception is not thrown.
-        if options.no_fail_fast:
-          bad_dep_urls.append(dep_url)
-          continue
-        raise Exception('No match found for %s' % dep_url)
-
-    Job = collections.namedtuple('Job', ['git_url', 'dep_url', 'path',
-                                         'git_host', 'dep_rev', 'svn_branch'])
-    deps_to_process[dep] = Job(
-        git_url, dep_url, path, git_host, dep_rev, svn_branch)
 
-  # Lets pre-cache all of the git repos now if we have cache_dir turned on.
-  if options.cache_dir:
-    if not os.path.isdir(options.cache_dir):
-      os.makedirs(options.cache_dir)
-    pool = ThreadPool(processes=len(deps_to_process))
-    output_queue = Queue.Queue()
-    num_threads = 0
-    for git_url, _, _, _, _, _ in deps_to_process.itervalues():
-      print 'Populating cache for %s' % git_url
-      num_threads += 1
-      pool.apply_async(git_tools.Clone, (git_url, None, 'bare',
-                                         output_queue, options.cache_dir,
-                                         options.shallow))
-    pool.close()
-
-    # Stream stdout line by line.
-    sec_since = 0
-    while num_threads > 0:
-      try:
-        line = output_queue.get(block=True, timeout=1)
-        sec_since = 0
-      except Queue.Empty:
-        sec_since += 1
-        line = ('Main> Heartbeat ping. We are still alive!! '
-                'Seconds since last output: %d sec' % sec_since)
-      if line is None:
-        num_threads -= 1
-      else:
-        print line
-    pool.join()
+def ConvertDepMain(dep_q, message_q, options, results):
+  cur_thread = threading.current_thread()
+  while True:
+    try:
+      job = dep_q.get(False)
+      dep, git_url, dep_url, path, git_host, dep_rev, svn_branch = job
+      cur_thread.working_on = dep
+    except Queue.Empty:
+      cur_thread.working_on = None
+      return
+
+    outbuf = StringIO()
+    def _print(s):
+      for l in s.splitlines():
+        outbuf.write('[%s] %s\n' % (dep, l))
 
-  for dep, items in deps_to_process.iteritems():
-    git_url, dep_url, path, git_host, dep_rev, svn_branch = items
     if options.verify:
       delay = 0.5
       success = False
       for try_index in range(1, 6):
-        print >> sys.stderr, 'checking %s (try #%d) ...' % (git_url, try_index),
+        _print('checking %s (try #%d) ...' % (git_url, try_index))
         if git_tools.Ping(git_url, verbose=True):
-          print >> sys.stderr, ' success'
+          _print(' success')
           success = True
           break
-
-        print >> sys.stderr, ' failure'
-        print >> sys.stderr, 'sleeping for %.01f seconds ...' % delay
+        _print(' failure')
+        _print('sleeping for %.01f seconds ...' % delay)
         time.sleep(delay)
         delay *= 2
 
       if not success:
-        bad_git_urls.update([git_url])
+        results.bad_git_urls.add(git_url)
 
     # Get the Git hash based off the SVN rev.
     git_hash = ''
@@ -221,13 +187,13 @@ def ConvertDepsToGit(deps, options, deps_vars, svn_to_git_objs):
               git_host, svn_branch, options.cache_dir)
         except Exception as e:
           if options.no_fail_fast:
-            bad_git_hash.append(e)
+            results.bad_git_hash.append(e)
             continue
           raise
 
     # If this is webkit, we need to add the var for the hash.
     if dep == 'src/third_party/WebKit' and dep_rev:
-      deps_vars['webkit_rev'] = git_hash
+      results.deps_vars['webkit_rev'] = git_hash
       git_hash = 'VAR_WEBKIT_REV'
 
     # Hack to preserve the angle_revision variable in .DEPS.git.
@@ -235,13 +201,78 @@ def ConvertDepsToGit(deps, options, deps_vars, svn_to_git_objs):
     if dep == 'src/third_party/angle' and git_hash:
       # Cut the leading '@' so this variable has the same semantics in
       # DEPS and .DEPS.git.
-      deps_vars['angle_revision'] = git_hash[1:]
+      results.deps_vars['angle_revision'] = git_hash[1:]
       git_hash = 'VAR_ANGLE_REVISION'
 
     # Add this Git dep to the new deps.
-    new_deps[path] = '%s%s' % (git_url, git_hash)
+    results.new_deps[path] = '%s%s' % (git_url, git_hash)
 
-  return new_deps, bad_git_urls, bad_dep_urls, bad_override, bad_git_hash
+    message_q.put(outbuf.getvalue())
+
+
+def ConvertDepsToGit(deps, options, deps_vars, svn_to_git_objs):
+  """Convert a 'deps' section in a DEPS file from SVN to Git."""
+  results = ConversionResults(
+      new_deps={},
+      deps_vars=deps_vars,
+      bad_git_urls=set([]),
+      bad_dep_urls=[],
+      bad_git_hash=[]
+  )
+
+  # Populate our deps list.
+  deps_to_process = Queue.Queue()
+  for dep, dep_url in deps.iteritems():
+    if not dep_url:  # dep is 'None' and emitted to exclude the dep
+      results.new_deps[dep] = None
+      continue
+
+    # Get the URL and the revision/hash for this dependency.
+    dep_url, dep_rev = SplitScmUrl(deps[dep])
+
+    path = dep
+    git_url = dep_url
+    svn_branch = None
+    git_host = dep_url
+
+    if not dep_url.endswith('.git'):
+      # Convert this SVN URL to a Git URL.
+      for svn_git_converter in svn_to_git_objs:
+        converted_data = svn_git_converter.SvnUrlToGitUrl(dep, dep_url)
+        if converted_data:
+          path, git_url, git_host = converted_data[:3]
+          if len(converted_data) > 3:
+            svn_branch = converted_data[3]
+          break
+      else:
+        # Make all match failures fatal to catch errors early. When a match is
+        # found, we break out of the loop so the exception is not thrown.
+        if options.no_fail_fast:
+          results.bad_dep_urls.append(dep_url)
+          continue
+        raise RuntimeError('No match found for %s' % dep_url)
+
+    deps_to_process.put(
+        Job(dep, git_url, dep_url, path, git_host, dep_rev, svn_branch))
+
+  threads = []
+  message_q = Queue.Queue()
+  thread_args = (deps_to_process, message_q, options, results)
+  num_threads = options.num_threads or deps_to_process.qsize()
+  for _ in xrange(num_threads):
+    th = threading.Thread(target=ConvertDepMain, args=thread_args)
+    th.working_on = None
+    th.start()
+    threads.append(th)
+  message_th = threading.Thread(target=MessageMain, args=(message_q, threads))
+  message_th.start()
+
+  for th in threads:
+    th.join()
+  message_q.put(Queue.Empty)
+  message_th.join()
+
+  return results
 
 
 def main():
@@ -250,6 +281,8 @@ def main():
                     help='path to the DEPS file to convert')
   parser.add_option('-o', '--out',
                     help='path to the converted DEPS file (default: stdout)')
+  parser.add_option('-j', '--num-threads', type='int', default=4,
+                    help='Maximum number of threads')
   parser.add_option('-t', '--type',
                     help='[DEPRECATED] type of DEPS file (public, etc)')
   parser.add_option('-x', '--extra-rules',
@@ -290,7 +323,7 @@ def main():
     options.cache_dir = os.path.abspath(options.cache_dir)
 
   if options.extra_rules and not os.path.exists(options.extra_rules):
-    raise Exception('Can\'t locate rules file "%s".' % options.extra_rules)
+    raise RuntimeError('Can\'t locate rules file "%s".' % options.extra_rules)
 
   # Create a var containing the Git and Webkit URL, this will make it easy for
   # people to use a mirror instead.
@@ -332,6 +365,14 @@ def main():
     if not options.cache_dir and 'cache_dir' in gclient_dict:
       options.cache_dir = os.path.abspath(gclient_dict['cache_dir'])
 
+  if options.cache_dir:
+    git_cache.Mirror.SetCachePath(options.cache_dir)
+  else:
+    try:
+      options.cache_dir = git_cache.Mirror.GetCachePath()
+    except RuntimeError:
+      pass
+
   # Do general pre-processing of the DEPS data.
   for svn_git_converter in svn_to_git_objs:
     if hasattr(svn_git_converter, 'CleanDeps'):
@@ -339,44 +380,39 @@ def main():
                                   skip_child_includes, hooks)
 
   # Convert the DEPS file to Git.
-  deps, baddeps, badmaps, badvars, badhashes = ConvertDepsToGit(
+  results = ConvertDepsToGit(
       deps, options, deps_vars, svn_to_git_objs)
   for os_dep in deps_os:
-    result = ConvertDepsToGit(deps_os[os_dep], options, deps_vars,
-                              svn_to_git_objs)
-    deps_os[os_dep] = result[0]
-    baddeps = baddeps.union(result[1])
-    badmaps.extend(result[2])
-    badvars.extend(result[3])
-    badhashes.extend(result[4])
+    os_results = ConvertDepsToGit(deps_os[os_dep], options, deps_vars,
+                                  svn_to_git_objs)
+    deps_os[os_dep] = os_results.new_deps
+    results.bad_git_urls.update(os_results.bad_git_urls)
+    results.bad_dep_urls.extend(os_results.bad_dep_urls)
+    results.bad_git_hash.extend(os_results.bad_git_hash)
 
   if options.json:
     with open(options.json, 'w') as f:
-      json.dump(list(baddeps), f, sort_keys=True, indent=2)
+      json.dump(list(results.bad_git_urls), f, sort_keys=True, indent=2)
 
-  if baddeps:
+  if results.bad_git_urls:
     print >> sys.stderr, ('\nUnable to resolve the following repositories. '
         'Please make sure\nthat any svn URLs have a git mirror associated with '
         'them.\nTo see the exact error, run `git ls-remote [repository]` where'
         '\n[repository] is the URL ending in .git (strip off the @revision\n'
         'number.) For more information, visit http://code.google.com\n'
         '/p/chromium/wiki/UsingGit#Adding_new_repositories_to_DEPS.\n')
-    for dep in baddeps:
+    for dep in results.bad_git_urls:
       print >> sys.stderr, ' ' + dep
-  if badmaps:
+  if results.bad_dep_urls:
     print >> sys.stderr, '\nNo mappings found for the following urls:\n'
-    for bad in badmaps:
-      print >> sys.stderr, ' ' + bad
-  if badvars:
-    print >> sys.stderr, '\nMissing DEPS variables for overrides:\n'
-    for bad in badvars:
+    for bad in results.bad_dep_urls:
       print >> sys.stderr, ' ' + bad
-  if badhashes:
+  if results.bad_git_hash:
     print >> sys.stderr, '\nsvn rev to git hash failures:\n'
-    for bad in badhashes:
+    for bad in results.bad_git_hash:
       print >> sys.stderr, ' ' + str(bad)
 
-  if baddeps or badmaps or badvars or badhashes:
+  if (results.bad_git_urls or results.bad_dep_urls or results.bad_git_hash):
     return 2
 
   if options.verify:
@@ -385,8 +421,8 @@ def main():
     return 0
 
   # Write the DEPS file to disk.
-  deps_utils.WriteDeps(options.out, deps_vars, deps, deps_os, include_rules,
-                       skip_child_includes, hooks)
+  deps_utils.WriteDeps(options.out, deps_vars, results.new_deps, deps_os,
+                       include_rules, skip_child_includes, hooks)
   return 0