OLD | NEW |
1 # Copyright (c) 2012 Google Inc. All rights reserved. | 1 # Copyright (c) 2012 Google Inc. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 from compiler.ast import Const | 5 from compiler.ast import Const |
6 from compiler.ast import Dict | 6 from compiler.ast import Dict |
7 from compiler.ast import Discard | 7 from compiler.ast import Discard |
8 from compiler.ast import List | 8 from compiler.ast import List |
9 from compiler.ast import Module | 9 from compiler.ast import Module |
10 from compiler.ast import Node | 10 from compiler.ast import Node |
(...skipping 30 matching lines...) Expand all Loading... |
41 'destination', | 41 'destination', |
42 'files', | 42 'files', |
43 'include_dirs', | 43 'include_dirs', |
44 'inputs', | 44 'inputs', |
45 'libraries', | 45 'libraries', |
46 'outputs', | 46 'outputs', |
47 'sources', | 47 'sources', |
48 ] | 48 ] |
49 path_sections = set() | 49 path_sections = set() |
50 | 50 |
| 51 # These per-process dictionaries are used to cache build file data when loading |
| 52 # in parallel mode. |
| 53 per_process_data = {} |
| 54 per_process_aux_data = {} |
| 55 |
51 def IsPathSection(section): | 56 def IsPathSection(section): |
52 # If section ends in one of the '=+?!' characters, it's applied to a section | 57 # If section ends in one of the '=+?!' characters, it's applied to a section |
53 # without the trailing characters. '/' is notably absent from this list, | 58 # without the trailing characters. '/' is notably absent from this list, |
54 # because there's no way for a regular expression to be treated as a path. | 59 # because there's no way for a regular expression to be treated as a path. |
55 while section[-1:] in '=+?!': | 60 while section[-1:] in '=+?!': |
56 section = section[:-1] | 61 section = section[:-1] |
57 | 62 |
58 if section in path_sections: | 63 if section in path_sections: |
59 return True | 64 return True |
60 | 65 |
(...skipping 294 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
355 if depth: | 360 if depth: |
356 # TODO(dglazkov) The backslash/forward-slash replacement at the end is a | 361 # TODO(dglazkov) The backslash/forward-slash replacement at the end is a |
357 # temporary measure. This should really be addressed by keeping all paths | 362 # temporary measure. This should really be addressed by keeping all paths |
358 # in POSIX until actual project generation. | 363 # in POSIX until actual project generation. |
359 d = gyp.common.RelativePath(depth, os.path.dirname(build_file_path)) | 364 d = gyp.common.RelativePath(depth, os.path.dirname(build_file_path)) |
360 if d == '': | 365 if d == '': |
361 variables['DEPTH'] = '.' | 366 variables['DEPTH'] = '.' |
362 else: | 367 else: |
363 variables['DEPTH'] = d.replace('\\', '/') | 368 variables['DEPTH'] = d.replace('\\', '/') |
364 | 369 |
365 if build_file_path in data['target_build_files']: | 370 # The 'target_build_files' key is only set when loading target build files in |
366 # Already loaded. | 371 # the non-parallel code path, where LoadTargetBuildFile is called |
367 return False | 372 # recursively. In the parallel code path, we don't need to check whether the |
368 data['target_build_files'].add(build_file_path) | 373 # |build_file_path| has already been loaded, because the 'scheduled' set in |
| 374 # ParallelState guarantees that we never load the same |build_file_path| |
| 375 # twice. |
| 376 if 'target_build_files' in data: |
| 377 if build_file_path in data['target_build_files']: |
| 378 # Already loaded. |
| 379 return False |
| 380 data['target_build_files'].add(build_file_path) |
369 | 381 |
370 gyp.DebugOutput(gyp.DEBUG_INCLUDES, | 382 gyp.DebugOutput(gyp.DEBUG_INCLUDES, |
371 "Loading Target Build File '%s'", build_file_path) | 383 "Loading Target Build File '%s'", build_file_path) |
372 | 384 |
373 build_file_data = LoadOneBuildFile(build_file_path, data, aux_data, | 385 build_file_data = LoadOneBuildFile(build_file_path, data, aux_data, |
374 includes, True, check) | 386 includes, True, check) |
375 | 387 |
376 # Store DEPTH for later use in generators. | 388 # Store DEPTH for later use in generators. |
377 build_file_data['_DEPTH'] = depth | 389 build_file_data['_DEPTH'] = depth |
378 | 390 |
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
449 try: | 461 try: |
450 LoadTargetBuildFile(dependency, data, aux_data, variables, | 462 LoadTargetBuildFile(dependency, data, aux_data, variables, |
451 includes, depth, check, load_dependencies) | 463 includes, depth, check, load_dependencies) |
452 except Exception, e: | 464 except Exception, e: |
453 gyp.common.ExceptionAppend( | 465 gyp.common.ExceptionAppend( |
454 e, 'while loading dependencies of %s' % build_file_path) | 466 e, 'while loading dependencies of %s' % build_file_path) |
455 raise | 467 raise |
456 else: | 468 else: |
457 return (build_file_path, dependencies) | 469 return (build_file_path, dependencies) |
458 | 470 |
459 | |
460 def CallLoadTargetBuildFile(global_flags, | 471 def CallLoadTargetBuildFile(global_flags, |
461 build_file_path, data, | 472 build_file_path, variables, |
462 aux_data, variables, | |
463 includes, depth, check, | 473 includes, depth, check, |
464 generator_input_info): | 474 generator_input_info): |
465 """Wrapper around LoadTargetBuildFile for parallel processing. | 475 """Wrapper around LoadTargetBuildFile for parallel processing. |
466 | 476 |
467 This wrapper is used when LoadTargetBuildFile is executed in | 477 This wrapper is used when LoadTargetBuildFile is executed in |
468 a worker process. | 478 a worker process. |
469 """ | 479 """ |
470 | 480 |
471 try: | 481 try: |
472 signal.signal(signal.SIGINT, signal.SIG_IGN) | 482 signal.signal(signal.SIGINT, signal.SIG_IGN) |
473 | 483 |
474 # Apply globals so that the worker process behaves the same. | 484 # Apply globals so that the worker process behaves the same. |
475 for key, value in global_flags.iteritems(): | 485 for key, value in global_flags.iteritems(): |
476 globals()[key] = value | 486 globals()[key] = value |
477 | 487 |
478 # Save the keys so we can return data that changed. | |
479 data_keys = set(data) | |
480 aux_data_keys = set(aux_data) | |
481 | |
482 SetGeneratorGlobals(generator_input_info) | 488 SetGeneratorGlobals(generator_input_info) |
483 result = LoadTargetBuildFile(build_file_path, data, | 489 result = LoadTargetBuildFile(build_file_path, per_process_data, |
484 aux_data, variables, | 490 per_process_aux_data, variables, |
485 includes, depth, check, False) | 491 includes, depth, check, False) |
486 if not result: | 492 if not result: |
487 return result | 493 return result |
488 | 494 |
489 (build_file_path, dependencies) = result | 495 (build_file_path, dependencies) = result |
490 | 496 |
491 data_out = {} | 497 # We can safely pop the build_file_data from per_process_data because it |
492 for key in data: | 498 # will never be referenced by this process again, so we don't need to keep |
493 if key == 'target_build_files': | 499 # it in the cache. |
494 continue | 500 build_file_data = per_process_data.pop(build_file_path) |
495 if key not in data_keys: | |
496 data_out[key] = data[key] | |
497 aux_data_out = {} | |
498 for key in aux_data: | |
499 if key not in aux_data_keys: | |
500 aux_data_out[key] = aux_data[key] | |
501 | 501 |
502 # This gets serialized and sent back to the main process via a pipe. | 502 # This gets serialized and sent back to the main process via a pipe. |
503 # It's handled in LoadTargetBuildFileCallback. | 503 # It's handled in LoadTargetBuildFileCallback. |
504 return (build_file_path, | 504 return (build_file_path, |
505 data_out, | 505 build_file_data, |
506 aux_data_out, | |
507 dependencies) | 506 dependencies) |
508 except GypError, e: | 507 except GypError, e: |
509 sys.stderr.write("gyp: %s\n" % e) | 508 sys.stderr.write("gyp: %s\n" % e) |
510 return None | 509 return None |
511 except Exception, e: | 510 except Exception, e: |
512 print >>sys.stderr, 'Exception:', e | 511 print >>sys.stderr, 'Exception:', e |
513 print >>sys.stderr, traceback.format_exc() | 512 print >>sys.stderr, traceback.format_exc() |
514 return None | 513 return None |
515 | 514 |
516 | 515 |
(...skipping 10 matching lines...) Expand all Loading... |
527 """ | 526 """ |
528 | 527 |
529 def __init__(self): | 528 def __init__(self): |
530 # The multiprocessing pool. | 529 # The multiprocessing pool. |
531 self.pool = None | 530 self.pool = None |
532 # The condition variable used to protect this object and notify | 531 # The condition variable used to protect this object and notify |
533 # the main loop when there might be more data to process. | 532 # the main loop when there might be more data to process. |
534 self.condition = None | 533 self.condition = None |
535 # The "data" dict that was passed to LoadTargetBuildFileParallel | 534 # The "data" dict that was passed to LoadTargetBuildFileParallel |
536 self.data = None | 535 self.data = None |
537 # The "aux_data" dict that was passed to LoadTargetBuildFileParallel | |
538 self.aux_data = None | |
539 # The number of parallel calls outstanding; decremented when a response | 536 # The number of parallel calls outstanding; decremented when a response |
540 # was received. | 537 # was received. |
541 self.pending = 0 | 538 self.pending = 0 |
542 # The set of all build files that have been scheduled, so we don't | 539 # The set of all build files that have been scheduled, so we don't |
543 # schedule the same one twice. | 540 # schedule the same one twice. |
544 self.scheduled = set() | 541 self.scheduled = set() |
545 # A list of dependency build file paths that haven't been scheduled yet. | 542 # A list of dependency build file paths that haven't been scheduled yet. |
546 self.dependencies = [] | 543 self.dependencies = [] |
547 # Flag to indicate if there was an error in a child process. | 544 # Flag to indicate if there was an error in a child process. |
548 self.error = False | 545 self.error = False |
549 | 546 |
550 def LoadTargetBuildFileCallback(self, result): | 547 def LoadTargetBuildFileCallback(self, result): |
551 """Handle the results of running LoadTargetBuildFile in another process. | 548 """Handle the results of running LoadTargetBuildFile in another process. |
552 """ | 549 """ |
553 self.condition.acquire() | 550 self.condition.acquire() |
554 if not result: | 551 if not result: |
555 self.error = True | 552 self.error = True |
556 self.condition.notify() | 553 self.condition.notify() |
557 self.condition.release() | 554 self.condition.release() |
558 return | 555 return |
559 (build_file_path0, data0, aux_data0, dependencies0) = result | 556 (build_file_path0, build_file_data0, dependencies0) = result |
| 557 self.data[build_file_path0] = build_file_data0 |
560 self.data['target_build_files'].add(build_file_path0) | 558 self.data['target_build_files'].add(build_file_path0) |
561 for key in data0: | |
562 self.data[key] = data0[key] | |
563 for key in aux_data0: | |
564 self.aux_data[key] = aux_data0[key] | |
565 for new_dependency in dependencies0: | 559 for new_dependency in dependencies0: |
566 if new_dependency not in self.scheduled: | 560 if new_dependency not in self.scheduled: |
567 self.scheduled.add(new_dependency) | 561 self.scheduled.add(new_dependency) |
568 self.dependencies.append(new_dependency) | 562 self.dependencies.append(new_dependency) |
569 self.pending -= 1 | 563 self.pending -= 1 |
570 self.condition.notify() | 564 self.condition.notify() |
571 self.condition.release() | 565 self.condition.release() |
572 | 566 |
573 | 567 |
574 def LoadTargetBuildFilesParallel(build_files, data, aux_data, | 568 def LoadTargetBuildFilesParallel(build_files, data, variables, includes, depth, |
575 variables, includes, depth, check, | 569 check, generator_input_info): |
576 generator_input_info): | |
577 parallel_state = ParallelState() | 570 parallel_state = ParallelState() |
578 parallel_state.condition = threading.Condition() | 571 parallel_state.condition = threading.Condition() |
579 # Make copies of the build_files argument that we can modify while working. | 572 # Make copies of the build_files argument that we can modify while working. |
580 parallel_state.dependencies = list(build_files) | 573 parallel_state.dependencies = list(build_files) |
581 parallel_state.scheduled = set(build_files) | 574 parallel_state.scheduled = set(build_files) |
582 parallel_state.pending = 0 | 575 parallel_state.pending = 0 |
583 parallel_state.data = data | 576 parallel_state.data = data |
584 parallel_state.aux_data = aux_data | |
585 | 577 |
586 try: | 578 try: |
587 parallel_state.condition.acquire() | 579 parallel_state.condition.acquire() |
588 while parallel_state.dependencies or parallel_state.pending: | 580 while parallel_state.dependencies or parallel_state.pending: |
589 if parallel_state.error: | 581 if parallel_state.error: |
590 break | 582 break |
591 if not parallel_state.dependencies: | 583 if not parallel_state.dependencies: |
592 parallel_state.condition.wait() | 584 parallel_state.condition.wait() |
593 continue | 585 continue |
594 | 586 |
595 dependency = parallel_state.dependencies.pop() | 587 dependency = parallel_state.dependencies.pop() |
596 | 588 |
597 parallel_state.pending += 1 | 589 parallel_state.pending += 1 |
598 data_in = {} | |
599 data_in['target_build_files'] = data['target_build_files'] | |
600 aux_data_in = {} | |
601 global_flags = { | 590 global_flags = { |
602 'path_sections': globals()['path_sections'], | 591 'path_sections': globals()['path_sections'], |
603 'non_configuration_keys': globals()['non_configuration_keys'], | 592 'non_configuration_keys': globals()['non_configuration_keys'], |
604 'multiple_toolsets': globals()['multiple_toolsets']} | 593 'multiple_toolsets': globals()['multiple_toolsets']} |
605 | 594 |
606 if not parallel_state.pool: | 595 if not parallel_state.pool: |
607 parallel_state.pool = multiprocessing.Pool(multiprocessing.cpu_count()) | 596 parallel_state.pool = multiprocessing.Pool(multiprocessing.cpu_count()) |
608 parallel_state.pool.apply_async( | 597 parallel_state.pool.apply_async( |
609 CallLoadTargetBuildFile, | 598 CallLoadTargetBuildFile, |
610 args = (global_flags, dependency, | 599 args = (global_flags, dependency, |
611 data_in, aux_data_in, | |
612 variables, includes, depth, check, generator_input_info), | 600 variables, includes, depth, check, generator_input_info), |
613 callback = parallel_state.LoadTargetBuildFileCallback) | 601 callback = parallel_state.LoadTargetBuildFileCallback) |
614 except KeyboardInterrupt, e: | 602 except KeyboardInterrupt, e: |
615 parallel_state.pool.terminate() | 603 parallel_state.pool.terminate() |
616 raise e | 604 raise e |
617 | 605 |
618 parallel_state.condition.release() | 606 parallel_state.condition.release() |
619 | 607 |
620 parallel_state.pool.close() | 608 parallel_state.pool.close() |
621 parallel_state.pool.join() | 609 parallel_state.pool.join() |
(...skipping 2105 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2727 extra_sources_for_rules = generator_input_info['extra_sources_for_rules'] | 2715 extra_sources_for_rules = generator_input_info['extra_sources_for_rules'] |
2728 | 2716 |
2729 # Load build files. This loads every target-containing build file into | 2717 # Load build files. This loads every target-containing build file into |
2730 # the |data| dictionary such that the keys to |data| are build file names, | 2718 # the |data| dictionary such that the keys to |data| are build file names, |
2731 # and the values are the entire build file contents after "early" or "pre" | 2719 # and the values are the entire build file contents after "early" or "pre" |
2732 # processing has been done and includes have been resolved. | 2720 # processing has been done and includes have been resolved. |
2733 # NOTE: data contains both "target" files (.gyp) and "includes" (.gypi), as | 2721 # NOTE: data contains both "target" files (.gyp) and "includes" (.gypi), as |
2734 # well as meta-data (e.g. 'included_files' key). 'target_build_files' keeps | 2722 # well as meta-data (e.g. 'included_files' key). 'target_build_files' keeps |
2735 # track of the keys corresponding to "target" files. | 2723 # track of the keys corresponding to "target" files. |
2736 data = {'target_build_files': set()} | 2724 data = {'target_build_files': set()} |
2737 aux_data = {} | |
2738 # Normalize paths everywhere. This is important because paths will be | 2725 # Normalize paths everywhere. This is important because paths will be |
2739 # used as keys to the data dict and for references between input files. | 2726 # used as keys to the data dict and for references between input files. |
2740 build_files = set(map(os.path.normpath, build_files)) | 2727 build_files = set(map(os.path.normpath, build_files)) |
2741 if parallel: | 2728 if parallel: |
2742 LoadTargetBuildFilesParallel(build_files, data, aux_data, | 2729 LoadTargetBuildFilesParallel(build_files, data, variables, includes, depth, |
2743 variables, includes, depth, check, | 2730 check, generator_input_info) |
2744 generator_input_info) | |
2745 else: | 2731 else: |
| 2732 aux_data = {} |
2746 for build_file in build_files: | 2733 for build_file in build_files: |
2747 try: | 2734 try: |
2748 LoadTargetBuildFile(build_file, data, aux_data, | 2735 LoadTargetBuildFile(build_file, data, aux_data, |
2749 variables, includes, depth, check, True) | 2736 variables, includes, depth, check, True) |
2750 except Exception, e: | 2737 except Exception, e: |
2751 gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file) | 2738 gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file) |
2752 raise | 2739 raise |
2753 | 2740 |
2754 # Build a dict to access each target's subdict by qualified name. | 2741 # Build a dict to access each target's subdict by qualified name. |
2755 targets = BuildTargetsDict(data) | 2742 targets = BuildTargetsDict(data) |
(...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2857 ValidateRunAsInTarget(target, target_dict, build_file) | 2844 ValidateRunAsInTarget(target, target_dict, build_file) |
2858 ValidateActionsInTarget(target, target_dict, build_file) | 2845 ValidateActionsInTarget(target, target_dict, build_file) |
2859 | 2846 |
2860 # Generators might not expect ints. Turn them into strs. | 2847 # Generators might not expect ints. Turn them into strs. |
2861 TurnIntIntoStrInDict(data) | 2848 TurnIntIntoStrInDict(data) |
2862 | 2849 |
2863 # TODO(mark): Return |data| for now because the generator needs a list of | 2850 # TODO(mark): Return |data| for now because the generator needs a list of |
2864 # build files that came in. In the future, maybe it should just accept | 2851 # build files that came in. In the future, maybe it should just accept |
2865 # a list, and not the whole data dict. | 2852 # a list, and not the whole data dict. |
2866 return [flat_list, targets, data] | 2853 return [flat_list, targets, data] |
OLD | NEW |