Index: pylib/gyp/input.py |
diff --git a/pylib/gyp/input.py b/pylib/gyp/input.py |
index 2678bab2c4233f79cfb6fc71b34401e42871d055..d6fa2d338f9e882780e9d2d8a5d278e75bc12092 100644 |
--- a/pylib/gyp/input.py |
+++ b/pylib/gyp/input.py |
@@ -12,12 +12,14 @@ from compiler.ast import Stmt |
import compiler |
import copy |
import gyp.common |
+import multiprocessing |
import optparse |
import os.path |
import re |
import shlex |
import subprocess |
import sys |
+import time |
# A list of types that are treated as linkable. |
@@ -351,6 +353,9 @@ def LoadTargetBuildFile(build_file_path, data, aux_data, variables, includes, |
return |
data['target_build_files'].add(build_file_path) |
+ data_keys = set(data.keys()) |
M-A Ruel
2012/09/05 19:45:39
Why double copy?
data_keys = set(data)
dmazzoni
2012/09/06 16:40:59
Done.
|
+ aux_data_keys = set(aux_data.keys()) |
M-A Ruel
2012/09/05 19:45:39
same
dmazzoni
2012/09/06 16:40:59
Done.
|
+ |
gyp.DebugOutput(gyp.DEBUG_INCLUDES, |
"Loading Target Build File '%s'" % build_file_path) |
@@ -418,23 +423,88 @@ def LoadTargetBuildFile(build_file_path, data, aux_data, variables, includes, |
# in other words, you can't put a "dependencies" section inside a "post" |
# conditional within a target. |
+ dependencies = [] |
if 'targets' in build_file_data: |
for target_dict in build_file_data['targets']: |
if 'dependencies' not in target_dict: |
continue |
for dependency in target_dict['dependencies']: |
- other_build_file = \ |
- gyp.common.ResolveTarget(build_file_path, dependency, None)[0] |
- try: |
- LoadTargetBuildFile(other_build_file, data, aux_data, variables, |
- includes, depth, check) |
- except Exception, e: |
- gyp.common.ExceptionAppend( |
- e, 'while loading dependencies of %s' % build_file_path) |
- raise |
- |
- return data |
+ dependencies.append( |
+ gyp.common.ResolveTarget(build_file_path, dependency, None)[0]) |
+ |
+ data_out = {} |
+ for key in data: |
+ if key not in data_keys: |
+ data_out[key] = data[key] |
+ aux_data_out = {} |
+ for key in aux_data: |
+ if key not in aux_data_keys: |
+ aux_data_out[key] = aux_data[key] |
+ |
+ return (build_file_path, |
+ data_out, |
+ aux_data_out, |
+ dependencies) |
+ |
+ |
+def CallLoadTargetBuildFile(global_flags, |
+ build_file_path, input_data, |
+ input_aux_data, input_variables, |
+ includes, depth, check): |
+ for key, value in global_flags.iteritems(): |
+ globals()[key] = value |
+ return LoadTargetBuildFile(build_file_path, input_data, |
+ input_aux_data, input_variables, |
+ includes, depth, check) |
+ |
+ |
+def LoadTargetBuildFileCallback(result): |
M-A Ruel
2012/09/06 18:02:36
A bit of docstrings explaining the dataflows would
|
+ (build_file_path0, data0, aux_data0, dependencies0) = result |
+ gyp.data['target_build_files'].add(build_file_path0) |
+ for key in data0: |
M-A Ruel
2012/09/06 18:02:36
gyp.data.update(data0) ?
|
+ gyp.data[key] = data0[key] |
+ for key in aux_data0: |
+ gyp.aux_data[key] = aux_data0[key] |
+ for new_dependency in dependencies0: |
+ if new_dependency not in gyp.scheduled: |
+ gyp.scheduled.add(new_dependency) |
+ gyp.dependencies.append(new_dependency) |
+ gyp.pending -= 1 |
+ |
+ |
+def LoadTargetBuildFileParallel(build_file_path, data, aux_data, |
+ variables, includes, depth, check): |
+ gyp.dependencies = [build_file_path] |
+ gyp.scheduled = set([build_file_path]) |
+ gyp.pending = 0 |
+ gyp.data = data |
+ gyp.aux_data = aux_data |
+ while gyp.dependencies or gyp.pending: |
+ if not gyp.dependencies: |
+ time.sleep(0.003) |
M-A Ruel
2012/09/05 19:45:39
it'll likely slow down the equivalent of time.slee
dmazzoni
2012/09/06 16:40:59
On Linux and Mac, any delay is better than none.
M-A Ruel
2012/09/06 18:02:36
It's because fork doesn't exist on Windows. So it
|
+ continue |
+ dependency = gyp.dependencies.pop() |
+ |
+ gyp.pending += 1 |
+ data_in = {} |
+ data_in['target_build_files'] = data['target_build_files'] |
+ aux_data_in = {} |
+ global_flags = { |
+ 'path_sections': globals()['path_sections'], |
+ 'non_configuration_keys': globals()['non_configuration_keys'], |
+ 'absolute_build_file_paths': globals()['absolute_build_file_paths'], |
+ 'multiple_toolsets': globals()['multiple_toolsets']} |
+ |
+ if 'pool' not in dir(gyp): |
+ gyp.pool = multiprocessing.Pool(8) |
+ gyp.pool.apply_async( |
+ CallLoadTargetBuildFile, |
+ args = (global_flags, dependency, |
+ data_in, aux_data_in, |
+ variables, includes, depth, check), |
+ callback = LoadTargetBuildFileCallback) |
+ time.sleep(0.003) |
M-A Ruel
2012/09/06 18:02:36
You could use a Queue.Queue instead, so you wouldn
M-A Ruel
2012/09/18 19:42:18
You didn't tell why you didn't want to use a singl
dmazzoni
2012/09/19 21:14:52
You're right, I think there was a possible race co
|
# Look for the bracket that matches the first bracket seen in a |
# string, and return the start and end as a tuple. For example, if |
@@ -2374,8 +2444,8 @@ def Load(build_files, variables, includes, depth, generator_input_info, check, |
# used as keys to the data dict and for references between input files. |
build_file = os.path.normpath(build_file) |
try: |
- LoadTargetBuildFile(build_file, data, aux_data, variables, includes, |
- depth, check) |
+ LoadTargetBuildFileParallel(build_file, data, aux_data, |
+ variables, includes, depth, check) |
except Exception, e: |
gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file) |
raise |