Load target build files in parallel using Python multiprocessing.

pylib/gyp/input.py

Index: pylib/gyp/input.py
diff --git a/pylib/gyp/input.py b/pylib/gyp/input.py
index 2678bab2c4233f79cfb6fc71b34401e42871d055..d6fa2d338f9e882780e9d2d8a5d278e75bc12092 100644
--- a/pylib/gyp/input.py
+++ b/pylib/gyp/input.py
@@ -12,12 +12,14 @@ from compiler.ast import Stmt
 import compiler
 import copy
 import gyp.common
+import multiprocessing
 import optparse
 import os.path
 import re
 import shlex
 import subprocess
 import sys
+import time
 
 
 # A list of types that are treated as linkable.
@@ -351,6 +353,9 @@ def LoadTargetBuildFile(build_file_path, data, aux_data, variables, includes,
     return
   data['target_build_files'].add(build_file_path)
 
+  data_keys = set(data.keys())

[M-A Ruel, 2012/09/05 19:45:39]

Why double copy?
data_keys = set(data)

[dmazzoni, 2012/09/06 16:40:59]

Done.

+  aux_data_keys = set(aux_data.keys())

[M-A Ruel, 2012/09/05 19:45:39]

same

[dmazzoni, 2012/09/06 16:40:59]

Done.

+
   gyp.DebugOutput(gyp.DEBUG_INCLUDES,
                   "Loading Target Build File '%s'" % build_file_path)
 
@@ -418,23 +423,88 @@ def LoadTargetBuildFile(build_file_path, data, aux_data, variables, includes,
   # in other words, you can't put a "dependencies" section inside a "post"
   # conditional within a target.
 
+  dependencies = []
   if 'targets' in build_file_data:
     for target_dict in build_file_data['targets']:
       if 'dependencies' not in target_dict:
         continue
       for dependency in target_dict['dependencies']:
-        other_build_file = \
-            gyp.common.ResolveTarget(build_file_path, dependency, None)[0]
-        try:
-          LoadTargetBuildFile(other_build_file, data, aux_data, variables,
-                              includes, depth, check)
-        except Exception, e:
-          gyp.common.ExceptionAppend(
-            e, 'while loading dependencies of %s' % build_file_path)
-          raise
-
-  return data
+        dependencies.append(
+            gyp.common.ResolveTarget(build_file_path, dependency, None)[0])
+
+  data_out = {}
+  for key in data:
+    if key not in data_keys:
+      data_out[key] = data[key]
+  aux_data_out = {}
+  for key in aux_data:
+    if key not in aux_data_keys:
+      aux_data_out[key] = aux_data[key]
+
+  return (build_file_path,
+          data_out,
+          aux_data_out,
+          dependencies)
+
+
+def CallLoadTargetBuildFile(global_flags,
+                            build_file_path, input_data,
+                            input_aux_data, input_variables,
+                            includes, depth, check):
+  for key, value in global_flags.iteritems():
+    globals()[key] = value
+  return LoadTargetBuildFile(build_file_path, input_data,
+                             input_aux_data, input_variables,
+                             includes, depth, check)
+
+
+def LoadTargetBuildFileCallback(result):

[M-A Ruel, 2012/09/06 18:02:36]

A bit of docstrings explaining the dataflows would help.

+  (build_file_path0, data0, aux_data0, dependencies0) = result
+  gyp.data['target_build_files'].add(build_file_path0)
+  for key in data0:

[M-A Ruel, 2012/09/06 18:02:36]

gyp.data.update(data0) ?

+    gyp.data[key] = data0[key]
+  for key in aux_data0:
+    gyp.aux_data[key] = aux_data0[key]
+  for new_dependency in dependencies0:
+    if new_dependency not in gyp.scheduled:
+      gyp.scheduled.add(new_dependency)
+      gyp.dependencies.append(new_dependency)
+  gyp.pending -= 1
+
+
+def LoadTargetBuildFileParallel(build_file_path, data, aux_data,
+                                variables, includes, depth, check):
+  gyp.dependencies = [build_file_path]
+  gyp.scheduled = set([build_file_path])
+  gyp.pending = 0
+  gyp.data = data
+  gyp.aux_data = aux_data
+  while gyp.dependencies or gyp.pending:
+    if not gyp.dependencies:
+      time.sleep(0.003)

[M-A Ruel, 2012/09/05 19:45:39]

it'll likely slow down the equivalent of time.sleep(0.01) on Windows, you would
have to confirm manually.

[dmazzoni, 2012/09/06 16:40:59]

On Linux and Mac, any delay is better than none.

I'm having trouble getting it to work on Windows at all - multiprocessing
complains that it can't find gyp_chromium after forking. Any ideas?

[M-A Ruel, 2012/09/06 18:02:36]

It's because fork doesn't exist on Windows. So it is 100% emulated by python.

As such, it is much better if the inner code to be processed can be separated
into a smaller process with a smaller script first. This is cumbersome, so you
may just want to not enable it on Windows.

+      continue
 
+    dependency = gyp.dependencies.pop()
+
+    gyp.pending += 1
+    data_in = {}
+    data_in['target_build_files'] = data['target_build_files']
+    aux_data_in = {}
+    global_flags = {
+      'path_sections': globals()['path_sections'],
+      'non_configuration_keys': globals()['non_configuration_keys'],
+      'absolute_build_file_paths': globals()['absolute_build_file_paths'],
+      'multiple_toolsets': globals()['multiple_toolsets']}
+
+    if 'pool' not in dir(gyp):
+      gyp.pool = multiprocessing.Pool(8)
+    gyp.pool.apply_async(
+        CallLoadTargetBuildFile,
+        args = (global_flags, dependency,
+                data_in, aux_data_in,
+                variables, includes, depth, check),
+        callback = LoadTargetBuildFileCallback)
+    time.sleep(0.003)

[M-A Ruel, 2012/09/06 18:02:36]

You could use a Queue.Queue instead, so you wouldn't have to poll&sleep at all

[M-A Ruel, 2012/09/18 19:42:18]

You didn't tell why you didn't want to use a single Queue for tasks completed in
gyp.dependencies.

In particular, the LoadTargetBuildFileCallback is called in a separate thread
and you are not using locking at all.

[dmazzoni, 2012/09/19 21:14:52]

You're right, I think there was a possible race condition. I fixed it with a
condition variable that gets notified each time a callback completes; this also
eliminates the need for polling or sleeping.

As for a Queue, I think it would be more code to manage subprocesses and results
explicitly and I'm not sure I see the benefit. multiprocessing.Pool already uses
a Queue under the hood, so I think this is giving us the same benefit with the
advantage of using a simpler high-level API.

 
 # Look for the bracket that matches the first bracket seen in a
 # string, and return the start and end as a tuple.  For example, if
@@ -2374,8 +2444,8 @@ def Load(build_files, variables, includes, depth, generator_input_info, check,
     # used as keys to the data dict and for references between input files.
     build_file = os.path.normpath(build_file)
     try:
-      LoadTargetBuildFile(build_file, data, aux_data, variables, includes,
-                          depth, check)
+      LoadTargetBuildFileParallel(build_file, data, aux_data,
+                                  variables, includes, depth, check)
     except Exception, e:
       gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file)
       raise