Generate index in parallel....

gsd_generate_index.py

Index: gsd_generate_index.py
===================================================================
--- gsd_generate_index.py	(revision 52963)
+++ gsd_generate_index.py	(working copy)
@@ -16,9 +16,11 @@
 import subprocess
 import sys
 import tempfile
+import threading
 
 
 GENERATED_INDEX = '_index.html'
+NUM_THREADS = 100
 
 
 def PathToLink(path):
@@ -138,6 +140,21 @@
   print '%s -- updated index' % path
 
 
+def IndexWorker(index_list, mutex, directories, objects, options):
+  while True:
+    # Pluck out one index to work on, or quit if no more work left.
+    mutex.acquire()
+    if not len(index_list):
+      mutex.release()
+      return
+    d = index_list.pop(0)
+    mutex.release()
+    # Find just this directories children.
+    children = [o for o in objects if posixpath.dirname(o) == d]
+    # Generate it.
+    GenerateIndex(d, children, directories, options)
+
+
 def GenerateIndexes(path, options):
   """Generate all relevant indexes for a given gsd path."""
   # Get a list of objects under this prefix.
@@ -156,14 +173,20 @@
       part = posixpath.dirname(part)
   objects += list(directories)
   # Generate index for each directory.
-  for d in directories:
-    # Skip directories not on the target path if any.
-    if options.path and not options.path.startswith(d):
-      continue
-    # Find just this directories children.
-    children = [o for o in objects if posixpath.dirname(o) == d]
-    # Generate this directory's index if needed.
-    GenerateIndex(d, children, directories, options)
+  index_list = [i for i in directories
+                if not options.path or options.path.startswith(i)]
+  # Spawn workers
+  mutex = threading.Lock()
+  workers = [threading.Thread(target=IndexWorker,
+                              args=(index_list, mutex,
+                                    directories, objects, options))
+             for _ in range(0, NUM_THREADS)]
+  # Start threads.
+  for w in workers:
+    w.start()
+  # Wait for them to finish.
+  for w in workers:
+    w.join()
   return 0