Process all input files in parallel

pylib/gyp/input.py

 # Copyright (c) 2012 Google Inc. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from compiler.ast import Const
 from compiler.ast import Dict
 from compiler.ast import Discard
 from compiler.ast import List
 from compiler.ast import Module
 from compiler.ast import Node
@@ skipping 535 matching lines @@
       self.aux_data[key] = aux_data0[key]
     for new_dependency in dependencies0:
       if new_dependency not in self.scheduled:
         self.scheduled.add(new_dependency)
         self.dependencies.append(new_dependency)
     self.pending -= 1
     self.condition.notify()
     self.condition.release()
 
 
-def LoadTargetBuildFileParallel(build_file_path, data, aux_data,
-                                variables, includes, depth, check):
+def LoadTargetBuildFilesParallel(build_files, data, aux_data,
+                                 variables, includes, depth, check):
   parallel_state = ParallelState()
   parallel_state.condition = threading.Condition()
-  parallel_state.dependencies = [build_file_path]
-  parallel_state.scheduled = set([build_file_path])
+  parallel_state.dependencies = list(build_files)

[dmazzoni, 2013/08/29 15:04:46]

You're converting |build_files| to both a list and a set...what is it initially?

It might be cleaner to assume the argument is always a list, and create a set
only when needed. When it's a set, call it build_file_set

[Simon Brenner, 2013/08/29 16:15:48]

If |build_files| is already a list, list() serves to make a copy that we can
safely modify. Likewise with set() when the argument is already a set.

Since |dependencies| and |scheduled| will both be updated continuously by the
loop below and by ParallelState.LoadTargetBuildFileCallback, I wanted to work on
copies to avoid surprising the caller.

In this case, I believe the collection is always a set, so I guess
build_file_set? Then again, I like that the code is agnostic in the type of
collection that comes in, it just needs it to be iterable.

+  parallel_state.scheduled = set(build_files)
   parallel_state.pending = 0
   parallel_state.data = data
   parallel_state.aux_data = aux_data
 
   try:
     parallel_state.condition.acquire()
     while parallel_state.dependencies or parallel_state.pending:
       if parallel_state.error:
         print >>sys.stderr, (
             '\n'
@@ skipping 2010 matching lines @@
 
   # Load build files.  This loads every target-containing build file into
   # the |data| dictionary such that the keys to |data| are build file names,
   # and the values are the entire build file contents after "early" or "pre"
   # processing has been done and includes have been resolved.
   # NOTE: data contains both "target" files (.gyp) and "includes" (.gypi), as
   # well as meta-data (e.g. 'included_files' key). 'target_build_files' keeps
   # track of the keys corresponding to "target" files.
   data = {'target_build_files': set()}
   aux_data = {}
-  for build_file in build_files:
-    # Normalize paths everywhere.  This is important because paths will be
-    # used as keys to the data dict and for references between input files.
-    build_file = os.path.normpath(build_file)
-    try:
-      if parallel:
-        LoadTargetBuildFileParallel(build_file, data, aux_data,
-                                    variables, includes, depth, check)
-      else:
+  # Normalize paths everywhere.  This is important because paths will be
+  # used as keys to the data dict and for references between input files.
+  build_files = set(map(os.path.normpath, build_files))
+  if parallel:
+        LoadTargetBuildFilesParallel(build_files, data, aux_data,

[dmazzoni, 2013/08/29 15:04:46]

Fix indentation

+                                 variables, includes, depth, check)
+  else:
+    for build_file in build_files:
+      try:
         LoadTargetBuildFile(build_file, data, aux_data,
                             variables, includes, depth, check, True)
-    except Exception, e:
-      gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file)
-      raise
+      except Exception, e:
+        gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file)
+        raise
 
   # Build a dict to access each target's subdict by qualified name.
   targets = BuildTargetsDict(data)
 
   # Fully qualify all dependency links.
   QualifyDependencies(targets)
 
   # Remove self-dependencies from targets that have 'prune_self_dependencies'
   # set to 1.
   RemoveSelfDependencies(targets)
@@ skipping 88 matching lines @@
     ValidateRunAsInTarget(target, target_dict, build_file)
     ValidateActionsInTarget(target, target_dict, build_file)
 
   # Generators might not expect ints.  Turn them into strs.
   TurnIntIntoStrInDict(data)
 
   # TODO(mark): Return |data| for now because the generator needs a list of
   # build files that came in.  In the future, maybe it should just accept
   # a list, and not the whole data dict.
   return [flat_list, targets, data]