OLD | NEW |
---|---|
1 # Copyright (c) 2012 Google Inc. All rights reserved. | 1 # Copyright (c) 2012 Google Inc. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 from compiler.ast import Const | 5 from compiler.ast import Const |
6 from compiler.ast import Dict | 6 from compiler.ast import Dict |
7 from compiler.ast import Discard | 7 from compiler.ast import Discard |
8 from compiler.ast import List | 8 from compiler.ast import List |
9 from compiler.ast import Module | 9 from compiler.ast import Module |
10 from compiler.ast import Node | 10 from compiler.ast import Node |
11 from compiler.ast import Stmt | 11 from compiler.ast import Stmt |
12 import compiler | 12 import compiler |
13 import copy | 13 import copy |
14 import gyp.common | 14 import gyp.common |
15 import multiprocessing | |
15 import optparse | 16 import optparse |
16 import os.path | 17 import os.path |
17 import re | 18 import re |
18 import shlex | 19 import shlex |
19 import subprocess | 20 import subprocess |
20 import sys | 21 import sys |
22 import time | |
21 | 23 |
22 | 24 |
23 # A list of types that are treated as linkable. | 25 # A list of types that are treated as linkable. |
24 linkable_types = ['executable', 'shared_library', 'loadable_module'] | 26 linkable_types = ['executable', 'shared_library', 'loadable_module'] |
25 | 27 |
26 # A list of sections that contain links to other targets. | 28 # A list of sections that contain links to other targets. |
27 dependency_sections = ['dependencies', 'export_dependent_settings'] | 29 dependency_sections = ['dependencies', 'export_dependent_settings'] |
28 | 30 |
29 # base_path_sections is a list of sections defined by GYP that contain | 31 # base_path_sections is a list of sections defined by GYP that contain |
30 # pathnames. The generators can provide more keys, the two lists are merged | 32 # pathnames. The generators can provide more keys, the two lists are merged |
(...skipping 313 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
344 | 346 |
345 # If the generator needs absolue paths, then do so. | 347 # If the generator needs absolue paths, then do so. |
346 if absolute_build_file_paths: | 348 if absolute_build_file_paths: |
347 build_file_path = os.path.abspath(build_file_path) | 349 build_file_path = os.path.abspath(build_file_path) |
348 | 350 |
349 if build_file_path in data['target_build_files']: | 351 if build_file_path in data['target_build_files']: |
350 # Already loaded. | 352 # Already loaded. |
351 return | 353 return |
352 data['target_build_files'].add(build_file_path) | 354 data['target_build_files'].add(build_file_path) |
353 | 355 |
356 data_keys = set(data.keys()) | |
M-A Ruel
2012/09/05 19:45:39
Why double copy?
data_keys = set(data)
dmazzoni
2012/09/06 16:40:59
Done.
| |
357 aux_data_keys = set(aux_data.keys()) | |
M-A Ruel
2012/09/05 19:45:39
same
dmazzoni
2012/09/06 16:40:59
Done.
| |
358 | |
354 gyp.DebugOutput(gyp.DEBUG_INCLUDES, | 359 gyp.DebugOutput(gyp.DEBUG_INCLUDES, |
355 "Loading Target Build File '%s'" % build_file_path) | 360 "Loading Target Build File '%s'" % build_file_path) |
356 | 361 |
357 build_file_data = LoadOneBuildFile(build_file_path, data, aux_data, variables, | 362 build_file_data = LoadOneBuildFile(build_file_path, data, aux_data, variables, |
358 includes, True, check) | 363 includes, True, check) |
359 | 364 |
360 # Store DEPTH for later use in generators. | 365 # Store DEPTH for later use in generators. |
361 build_file_data['_DEPTH'] = depth | 366 build_file_data['_DEPTH'] = depth |
362 | 367 |
363 # Set up the included_files key indicating which .gyp files contributed to | 368 # Set up the included_files key indicating which .gyp files contributed to |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
411 "Unable to find targets in build file %s" % build_file_path | 416 "Unable to find targets in build file %s" % build_file_path |
412 | 417 |
413 # No longer needed. | 418 # No longer needed. |
414 del build_file_data['target_defaults'] | 419 del build_file_data['target_defaults'] |
415 | 420 |
416 # Look for dependencies. This means that dependency resolution occurs | 421 # Look for dependencies. This means that dependency resolution occurs |
417 # after "pre" conditionals and variable expansion, but before "post" - | 422 # after "pre" conditionals and variable expansion, but before "post" - |
418 # in other words, you can't put a "dependencies" section inside a "post" | 423 # in other words, you can't put a "dependencies" section inside a "post" |
419 # conditional within a target. | 424 # conditional within a target. |
420 | 425 |
426 dependencies = [] | |
421 if 'targets' in build_file_data: | 427 if 'targets' in build_file_data: |
422 for target_dict in build_file_data['targets']: | 428 for target_dict in build_file_data['targets']: |
423 if 'dependencies' not in target_dict: | 429 if 'dependencies' not in target_dict: |
424 continue | 430 continue |
425 for dependency in target_dict['dependencies']: | 431 for dependency in target_dict['dependencies']: |
426 other_build_file = \ | 432 dependencies.append( |
427 gyp.common.ResolveTarget(build_file_path, dependency, None)[0] | 433 gyp.common.ResolveTarget(build_file_path, dependency, None)[0]) |
428 try: | |
429 LoadTargetBuildFile(other_build_file, data, aux_data, variables, | |
430 includes, depth, check) | |
431 except Exception, e: | |
432 gyp.common.ExceptionAppend( | |
433 e, 'while loading dependencies of %s' % build_file_path) | |
434 raise | |
435 | 434 |
436 return data | 435 data_out = {} |
436 for key in data: | |
437 if key not in data_keys: | |
438 data_out[key] = data[key] | |
439 aux_data_out = {} | |
440 for key in aux_data: | |
441 if key not in aux_data_keys: | |
442 aux_data_out[key] = aux_data[key] | |
437 | 443 |
444 return (build_file_path, | |
445 data_out, | |
446 aux_data_out, | |
447 dependencies) | |
448 | |
449 | |
450 def CallLoadTargetBuildFile(global_flags, | |
451 build_file_path, input_data, | |
452 input_aux_data, input_variables, | |
453 includes, depth, check): | |
454 for key, value in global_flags.iteritems(): | |
455 globals()[key] = value | |
456 return LoadTargetBuildFile(build_file_path, input_data, | |
457 input_aux_data, input_variables, | |
458 includes, depth, check) | |
459 | |
460 | |
461 def LoadTargetBuildFileCallback(result): | |
M-A Ruel
2012/09/06 18:02:36
A bit of docstrings explaining the dataflows would
| |
462 (build_file_path0, data0, aux_data0, dependencies0) = result | |
463 gyp.data['target_build_files'].add(build_file_path0) | |
464 for key in data0: | |
M-A Ruel
2012/09/06 18:02:36
gyp.data.update(data0) ?
| |
465 gyp.data[key] = data0[key] | |
466 for key in aux_data0: | |
467 gyp.aux_data[key] = aux_data0[key] | |
468 for new_dependency in dependencies0: | |
469 if new_dependency not in gyp.scheduled: | |
470 gyp.scheduled.add(new_dependency) | |
471 gyp.dependencies.append(new_dependency) | |
472 gyp.pending -= 1 | |
473 | |
474 | |
475 def LoadTargetBuildFileParallel(build_file_path, data, aux_data, | |
476 variables, includes, depth, check): | |
477 gyp.dependencies = [build_file_path] | |
478 gyp.scheduled = set([build_file_path]) | |
479 gyp.pending = 0 | |
480 gyp.data = data | |
481 gyp.aux_data = aux_data | |
482 while gyp.dependencies or gyp.pending: | |
483 if not gyp.dependencies: | |
484 time.sleep(0.003) | |
M-A Ruel
2012/09/05 19:45:39
it'll likely slow down the equivalent of time.slee
dmazzoni
2012/09/06 16:40:59
On Linux and Mac, any delay is better than none.
M-A Ruel
2012/09/06 18:02:36
It's because fork doesn't exist on Windows. So it
| |
485 continue | |
486 | |
487 dependency = gyp.dependencies.pop() | |
488 | |
489 gyp.pending += 1 | |
490 data_in = {} | |
491 data_in['target_build_files'] = data['target_build_files'] | |
492 aux_data_in = {} | |
493 global_flags = { | |
494 'path_sections': globals()['path_sections'], | |
495 'non_configuration_keys': globals()['non_configuration_keys'], | |
496 'absolute_build_file_paths': globals()['absolute_build_file_paths'], | |
497 'multiple_toolsets': globals()['multiple_toolsets']} | |
498 | |
499 if 'pool' not in dir(gyp): | |
500 gyp.pool = multiprocessing.Pool(8) | |
501 gyp.pool.apply_async( | |
502 CallLoadTargetBuildFile, | |
503 args = (global_flags, dependency, | |
504 data_in, aux_data_in, | |
505 variables, includes, depth, check), | |
506 callback = LoadTargetBuildFileCallback) | |
507 time.sleep(0.003) | |
M-A Ruel
2012/09/06 18:02:36
You could use a Queue.Queue instead, so you wouldn
M-A Ruel
2012/09/18 19:42:18
You didn't tell why you didn't want to use a singl
dmazzoni
2012/09/19 21:14:52
You're right, I think there was a possible race co
| |
438 | 508 |
439 # Look for the bracket that matches the first bracket seen in a | 509 # Look for the bracket that matches the first bracket seen in a |
440 # string, and return the start and end as a tuple. For example, if | 510 # string, and return the start and end as a tuple. For example, if |
441 # the input is something like "<(foo <(bar)) blah", then it would | 511 # the input is something like "<(foo <(bar)) blah", then it would |
442 # return (1, 13), indicating the entire string except for the leading | 512 # return (1, 13), indicating the entire string except for the leading |
443 # "<" and trailing " blah". | 513 # "<" and trailing " blah". |
444 def FindEnclosingBracketGroup(input): | 514 def FindEnclosingBracketGroup(input): |
445 brackets = { '}': '{', | 515 brackets = { '}': '{', |
446 ']': '[', | 516 ']': '[', |
447 ')': '(', } | 517 ')': '(', } |
(...skipping 1919 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2367 # NOTE: data contains both "target" files (.gyp) and "includes" (.gypi), as | 2437 # NOTE: data contains both "target" files (.gyp) and "includes" (.gypi), as |
2368 # well as meta-data (e.g. 'included_files' key). 'target_build_files' keeps | 2438 # well as meta-data (e.g. 'included_files' key). 'target_build_files' keeps |
2369 # track of the keys corresponding to "target" files. | 2439 # track of the keys corresponding to "target" files. |
2370 data = {'target_build_files': set()} | 2440 data = {'target_build_files': set()} |
2371 aux_data = {} | 2441 aux_data = {} |
2372 for build_file in build_files: | 2442 for build_file in build_files: |
2373 # Normalize paths everywhere. This is important because paths will be | 2443 # Normalize paths everywhere. This is important because paths will be |
2374 # used as keys to the data dict and for references between input files. | 2444 # used as keys to the data dict and for references between input files. |
2375 build_file = os.path.normpath(build_file) | 2445 build_file = os.path.normpath(build_file) |
2376 try: | 2446 try: |
2377 LoadTargetBuildFile(build_file, data, aux_data, variables, includes, | 2447 LoadTargetBuildFileParallel(build_file, data, aux_data, |
2378 depth, check) | 2448 variables, includes, depth, check) |
2379 except Exception, e: | 2449 except Exception, e: |
2380 gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file) | 2450 gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file) |
2381 raise | 2451 raise |
2382 | 2452 |
2383 # Build a dict to access each target's subdict by qualified name. | 2453 # Build a dict to access each target's subdict by qualified name. |
2384 targets = BuildTargetsDict(data) | 2454 targets = BuildTargetsDict(data) |
2385 | 2455 |
2386 # Fully qualify all dependency links. | 2456 # Fully qualify all dependency links. |
2387 QualifyDependencies(targets) | 2457 QualifyDependencies(targets) |
2388 | 2458 |
(...skipping 87 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
2476 ValidateRunAsInTarget(target, target_dict, build_file) | 2546 ValidateRunAsInTarget(target, target_dict, build_file) |
2477 ValidateActionsInTarget(target, target_dict, build_file) | 2547 ValidateActionsInTarget(target, target_dict, build_file) |
2478 | 2548 |
2479 # Generators might not expect ints. Turn them into strs. | 2549 # Generators might not expect ints. Turn them into strs. |
2480 TurnIntIntoStrInDict(data) | 2550 TurnIntIntoStrInDict(data) |
2481 | 2551 |
2482 # TODO(mark): Return |data| for now because the generator needs a list of | 2552 # TODO(mark): Return |data| for now because the generator needs a list of |
2483 # build files that came in. In the future, maybe it should just accept | 2553 # build files that came in. In the future, maybe it should just accept |
2484 # a list, and not the whole data dict. | 2554 # a list, and not the whole data dict. |
2485 return [flat_list, targets, data] | 2555 return [flat_list, targets, data] |
OLD | NEW |