Generate index in parallel....

gsd_generate_index.py

 #!/usr/bin/python
 # Copyright (c) 2008-2010 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Generate index.html files for a Google Storage for Developers directory.
 
 Google Storage for Developers provides only a raw set of objects.
 For some buckets we would like to be able to support browsing of the directory
 tree. This utility will generate the needed index and upload/update it.
 """
 
 import optparse
 import posixpath
 import re
 import subprocess
 import sys
 import tempfile
+import threading
 
 
 GENERATED_INDEX = '_index.html'
+NUM_THREADS = 100
 
 
 def PathToLink(path):
   return path.replace('gs://', 'https://sandbox.google.com/storage/')
 
 
 def FixupSize(sz):
   """Convert a size string in bytes to human readable form.
 
   Arguments:
@@ skipping 99 matching lines @@
   # Optionally update acl.
   if options.acl:
     cmd = [options.gsutil, 'setacl', options.acl]
     cmd += [posixpath.join(path, GENERATED_INDEX)]
     p = subprocess.Popen(cmd)
     p.communicate()
     assert p.returncode == 0
   print '%s -- updated index' % path
 
 
+def IndexWorker(index_list, mutex, directories, objects, options):
+  while True:
+    # Pluck out one index to work on, or quit if no more work left.
+    mutex.acquire()
+    if not len(index_list):
+      mutex.release()
+      return
+    d = index_list.pop(0)
+    mutex.release()
+    # Find just this directories children.
+    children = [o for o in objects if posixpath.dirname(o) == d]
+    # Generate it.
+    GenerateIndex(d, children, directories, options)
+
+
 def GenerateIndexes(path, options):
   """Generate all relevant indexes for a given gsd path."""
   # Get a list of objects under this prefix.
   cmd = [options.gsutil, 'ls', posixpath.join(path, '*')]
   p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
   p_stdout, _ = p.communicate()
   assert p.returncode == 0
   objects = str(p_stdout).splitlines()
   objects = [o for o in objects if posixpath.basename(o) != GENERATED_INDEX]
   # Find common prefixes.
   directories = set()
   for o in objects:
     part = posixpath.dirname(o)
     while part.startswith(path):
       directories.add(part)
       part = posixpath.dirname(part)
   objects += list(directories)
   # Generate index for each directory.
-  for d in directories:
-    # Skip directories not on the target path if any.
-    if options.path and not options.path.startswith(d):
-      continue
-    # Find just this directories children.
-    children = [o for o in objects if posixpath.dirname(o) == d]
-    # Generate this directory's index if needed.
-    GenerateIndex(d, children, directories, options)
+  index_list = [i for i in directories
+                if not options.path or options.path.startswith(i)]
+  # Spawn workers
+  mutex = threading.Lock()
+  workers = [threading.Thread(target=IndexWorker,
+                              args=(index_list, mutex,
+                                    directories, objects, options))
+             for _ in range(0, NUM_THREADS)]
+  # Start threads.
+  for w in workers:
+    w.start()
+  # Wait for them to finish.
+  for w in workers:
+    w.join()
   return 0
 
 
 def main(argv):
   parser = optparse.OptionParser(usage='usage: %prog [options] gs://base-dir')
   parser.add_option('-p', '--path', dest='path',
                     help='only update indexes on a given path')
   parser.add_option('-a', dest='acl', help='acl to set on indexes')
   parser.add_option('-f', '--force', action='store_true', default=False,
                     dest='force', help='upload all indexes even on match')
   parser.add_option('', '--gsutil', default='gsutil',
                     dest='gsutil', help='path to gsutil')
   options, args = parser.parse_args(argv)
   if len(args) != 2 or not args[1].startswith('gs://'):
     parser.print_help()
     return 1
   return GenerateIndexes(args[1], options)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))