Process all input files in parallel

pylib/gyp/input.py

Index: pylib/gyp/input.py
diff --git a/pylib/gyp/input.py b/pylib/gyp/input.py
index 0c303d2335877000b50534feac959c889b463184..98ba1efca19630ed10dbd187e1cc11f1d5f791c8 100644
--- a/pylib/gyp/input.py
+++ b/pylib/gyp/input.py
@@ -553,12 +553,12 @@ class ParallelState(object):
     self.condition.release()
 
 
-def LoadTargetBuildFileParallel(build_file_path, data, aux_data,
-                                variables, includes, depth, check):
+def LoadTargetBuildFilesParallel(build_files, data, aux_data,
+                                 variables, includes, depth, check):
   parallel_state = ParallelState()
   parallel_state.condition = threading.Condition()
-  parallel_state.dependencies = [build_file_path]
-  parallel_state.scheduled = set([build_file_path])
+  parallel_state.dependencies = list(build_files)

[dmazzoni, 2013/08/29 15:04:46]

You're converting |build_files| to both a list and a set...what is it initially?

It might be cleaner to assume the argument is always a list, and create a set
only when needed. When it's a set, call it build_file_set

[Simon Brenner, 2013/08/29 16:15:48]

If |build_files| is already a list, list() serves to make a copy that we can
safely modify. Likewise with set() when the argument is already a set.

Since |dependencies| and |scheduled| will both be updated continuously by the
loop below and by ParallelState.LoadTargetBuildFileCallback, I wanted to work on
copies to avoid surprising the caller.

In this case, I believe the collection is always a set, so I guess
build_file_set? Then again, I like that the code is agnostic in the type of
collection that comes in, it just needs it to be iterable.

+  parallel_state.scheduled = set(build_files)
   parallel_state.pending = 0
   parallel_state.data = data
   parallel_state.aux_data = aux_data
@@ -2589,20 +2589,20 @@ def Load(build_files, variables, includes, depth, generator_input_info, check,
   # track of the keys corresponding to "target" files.
   data = {'target_build_files': set()}
   aux_data = {}
-  for build_file in build_files:
-    # Normalize paths everywhere.  This is important because paths will be
-    # used as keys to the data dict and for references between input files.
-    build_file = os.path.normpath(build_file)
-    try:
-      if parallel:
-        LoadTargetBuildFileParallel(build_file, data, aux_data,
-                                    variables, includes, depth, check)
-      else:
+  # Normalize paths everywhere.  This is important because paths will be
+  # used as keys to the data dict and for references between input files.
+  build_files = set(map(os.path.normpath, build_files))
+  if parallel:
+        LoadTargetBuildFilesParallel(build_files, data, aux_data,

[dmazzoni, 2013/08/29 15:04:46]

Fix indentation

+                                 variables, includes, depth, check)
+  else:
+    for build_file in build_files:
+      try:
         LoadTargetBuildFile(build_file, data, aux_data,
                             variables, includes, depth, check, True)
-    except Exception, e:
-      gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file)
-      raise
+      except Exception, e:
+        gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file)
+        raise
 
   # Build a dict to access each target's subdict by qualified name.
   targets = BuildTargetsDict(data)