OLD | NEW |
1 # Copyright (c) 2012 Google Inc. All rights reserved. | 1 # Copyright (c) 2012 Google Inc. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 from compiler.ast import Const | 5 from compiler.ast import Const |
6 from compiler.ast import Dict | 6 from compiler.ast import Dict |
7 from compiler.ast import Discard | 7 from compiler.ast import Discard |
8 from compiler.ast import List | 8 from compiler.ast import List |
9 from compiler.ast import Module | 9 from compiler.ast import Module |
10 from compiler.ast import Node | 10 from compiler.ast import Node |
11 from compiler.ast import Stmt | 11 from compiler.ast import Stmt |
12 import compiler | 12 import compiler |
13 import copy | 13 import copy |
14 import gyp.common | 14 import gyp.common |
| 15 import multiprocessing |
15 import optparse | 16 import optparse |
16 import os.path | 17 import os.path |
17 import re | 18 import re |
18 import shlex | 19 import shlex |
19 import subprocess | 20 import subprocess |
20 import sys | 21 import sys |
| 22 import threading |
| 23 import time |
21 from gyp.common import GypError | 24 from gyp.common import GypError |
22 | 25 |
23 | 26 |
24 # A list of types that are treated as linkable. | 27 # A list of types that are treated as linkable. |
25 linkable_types = ['executable', 'shared_library', 'loadable_module'] | 28 linkable_types = ['executable', 'shared_library', 'loadable_module'] |
26 | 29 |
27 # A list of sections that contain links to other targets. | 30 # A list of sections that contain links to other targets. |
28 dependency_sections = ['dependencies', 'export_dependent_settings'] | 31 dependency_sections = ['dependencies', 'export_dependent_settings'] |
29 | 32 |
30 # base_path_sections is a list of sections defined by GYP that contain | 33 # base_path_sections is a list of sections defined by GYP that contain |
(...skipping 292 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
323 for condition in data['conditions']: | 326 for condition in data['conditions']: |
324 if isinstance(condition, list): | 327 if isinstance(condition, list): |
325 for condition_dict in condition[1:]: | 328 for condition_dict in condition[1:]: |
326 ProcessToolsetsInDict(condition_dict) | 329 ProcessToolsetsInDict(condition_dict) |
327 | 330 |
328 | 331 |
329 # TODO(mark): I don't love this name. It just means that it's going to load | 332 # TODO(mark): I don't love this name. It just means that it's going to load |
330 # a build file that contains targets and is expected to provide a targets dict | 333 # a build file that contains targets and is expected to provide a targets dict |
331 # that contains the targets... | 334 # that contains the targets... |
332 def LoadTargetBuildFile(build_file_path, data, aux_data, variables, includes, | 335 def LoadTargetBuildFile(build_file_path, data, aux_data, variables, includes, |
333 depth, check): | 336 depth, check, load_dependencies): |
334 # If depth is set, predefine the DEPTH variable to be a relative path from | 337 # If depth is set, predefine the DEPTH variable to be a relative path from |
335 # this build file's directory to the directory identified by depth. | 338 # this build file's directory to the directory identified by depth. |
336 if depth: | 339 if depth: |
337 # TODO(dglazkov) The backslash/forward-slash replacement at the end is a | 340 # TODO(dglazkov) The backslash/forward-slash replacement at the end is a |
338 # temporary measure. This should really be addressed by keeping all paths | 341 # temporary measure. This should really be addressed by keeping all paths |
339 # in POSIX until actual project generation. | 342 # in POSIX until actual project generation. |
340 d = gyp.common.RelativePath(depth, os.path.dirname(build_file_path)) | 343 d = gyp.common.RelativePath(depth, os.path.dirname(build_file_path)) |
341 if d == '': | 344 if d == '': |
342 variables['DEPTH'] = '.' | 345 variables['DEPTH'] = '.' |
343 else: | 346 else: |
344 variables['DEPTH'] = d.replace('\\', '/') | 347 variables['DEPTH'] = d.replace('\\', '/') |
345 | 348 |
346 # If the generator needs absolue paths, then do so. | 349 # If the generator needs absolue paths, then do so. |
347 if absolute_build_file_paths: | 350 if absolute_build_file_paths: |
348 build_file_path = os.path.abspath(build_file_path) | 351 build_file_path = os.path.abspath(build_file_path) |
349 | 352 |
350 if build_file_path in data['target_build_files']: | 353 if build_file_path in data['target_build_files']: |
351 # Already loaded. | 354 # Already loaded. |
352 return | 355 return False |
353 data['target_build_files'].add(build_file_path) | 356 data['target_build_files'].add(build_file_path) |
354 | 357 |
355 gyp.DebugOutput(gyp.DEBUG_INCLUDES, | 358 gyp.DebugOutput(gyp.DEBUG_INCLUDES, |
356 "Loading Target Build File '%s'" % build_file_path) | 359 "Loading Target Build File '%s'" % build_file_path) |
357 | 360 |
358 build_file_data = LoadOneBuildFile(build_file_path, data, aux_data, variables, | 361 build_file_data = LoadOneBuildFile(build_file_path, data, aux_data, variables, |
359 includes, True, check) | 362 includes, True, check) |
360 | 363 |
361 # Store DEPTH for later use in generators. | 364 # Store DEPTH for later use in generators. |
362 build_file_data['_DEPTH'] = depth | 365 build_file_data['_DEPTH'] = depth |
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
412 index += 1 | 415 index += 1 |
413 | 416 |
414 # No longer needed. | 417 # No longer needed. |
415 del build_file_data['target_defaults'] | 418 del build_file_data['target_defaults'] |
416 | 419 |
417 # Look for dependencies. This means that dependency resolution occurs | 420 # Look for dependencies. This means that dependency resolution occurs |
418 # after "pre" conditionals and variable expansion, but before "post" - | 421 # after "pre" conditionals and variable expansion, but before "post" - |
419 # in other words, you can't put a "dependencies" section inside a "post" | 422 # in other words, you can't put a "dependencies" section inside a "post" |
420 # conditional within a target. | 423 # conditional within a target. |
421 | 424 |
| 425 dependencies = [] |
422 if 'targets' in build_file_data: | 426 if 'targets' in build_file_data: |
423 for target_dict in build_file_data['targets']: | 427 for target_dict in build_file_data['targets']: |
424 if 'dependencies' not in target_dict: | 428 if 'dependencies' not in target_dict: |
425 continue | 429 continue |
426 for dependency in target_dict['dependencies']: | 430 for dependency in target_dict['dependencies']: |
427 other_build_file = \ | 431 dependencies.append( |
428 gyp.common.ResolveTarget(build_file_path, dependency, None)[0] | 432 gyp.common.ResolveTarget(build_file_path, dependency, None)[0]) |
429 try: | |
430 LoadTargetBuildFile(other_build_file, data, aux_data, variables, | |
431 includes, depth, check) | |
432 except Exception, e: | |
433 gyp.common.ExceptionAppend( | |
434 e, 'while loading dependencies of %s' % build_file_path) | |
435 raise | |
436 | 433 |
437 return data | 434 if load_dependencies: |
| 435 for dependency in dependencies: |
| 436 try: |
| 437 LoadTargetBuildFile(dependency, data, aux_data, variables, |
| 438 includes, depth, check, load_dependencies) |
| 439 except Exception, e: |
| 440 gyp.common.ExceptionAppend( |
| 441 e, 'while loading dependencies of %s' % build_file_path) |
| 442 raise |
| 443 else: |
| 444 return (build_file_path, dependencies) |
| 445 |
| 446 |
| 447 def CallLoadTargetBuildFile(global_flags, |
| 448 build_file_path, data, |
| 449 aux_data, variables, |
| 450 includes, depth, check): |
| 451 """Wrapper around LoadTargetBuildFile for parallel processing. |
| 452 |
| 453 This wrapper is used when LoadTargetBuildFile is executed in |
| 454 a worker process. |
| 455 """ |
| 456 |
| 457 # Apply globals so that the worker process behaves the same. |
| 458 for key, value in global_flags.iteritems(): |
| 459 globals()[key] = value |
| 460 |
| 461 # Save the keys so we can return data that changed. |
| 462 data_keys = set(data) |
| 463 aux_data_keys = set(aux_data) |
| 464 |
| 465 result = LoadTargetBuildFile(build_file_path, data, |
| 466 aux_data, variables, |
| 467 includes, depth, check, False) |
| 468 if not result: |
| 469 return result |
| 470 |
| 471 (build_file_path, dependencies) = result |
| 472 |
| 473 data_out = {} |
| 474 for key in data: |
| 475 if key == 'target_build_files': |
| 476 continue |
| 477 if key not in data_keys: |
| 478 data_out[key] = data[key] |
| 479 aux_data_out = {} |
| 480 for key in aux_data: |
| 481 if key not in aux_data_keys: |
| 482 aux_data_out[key] = aux_data[key] |
| 483 |
| 484 # This gets serialized and sent back to the main process via a pipe. |
| 485 # It's handled in LoadTargetBuildFileCallback. |
| 486 return (build_file_path, |
| 487 data_out, |
| 488 aux_data_out, |
| 489 dependencies) |
| 490 |
| 491 |
| 492 class ParallelState(object): |
| 493 """Class to keep track of state when processing input files in parallel. |
| 494 |
| 495 If build files are loaded in parallel, use this to keep track of |
| 496 state during farming out and processing parallel jobs. It's stored |
| 497 in a global so that the callback function can have access to it. |
| 498 """ |
| 499 |
| 500 def __init__(self): |
| 501 # The multiprocessing pool. |
| 502 self.pool = None |
| 503 # The condition variable used to protect this object and notify |
| 504 # the main loop when there might be more data to process. |
| 505 self.condition = None |
| 506 # The "data" dict that was passed to LoadTargetBuildFileParallel |
| 507 self.data = None |
| 508 # The "aux_data" dict that was passed to LoadTargetBuildFileParallel |
| 509 self.aux_data = None |
| 510 # The number of parallel calls outstanding; decremented when a response |
| 511 # was received. |
| 512 self.pending = 0 |
| 513 # The set of all build files that have been scheduled, so we don't |
| 514 # schedule the same one twice. |
| 515 self.scheduled = set() |
| 516 # A list of dependency build file paths that haven't been scheduled yet. |
| 517 self.dependencies = [] |
| 518 |
| 519 def LoadTargetBuildFileCallback(self, result): |
| 520 """Handle the results of running LoadTargetBuildFile in another process. |
| 521 """ |
| 522 (build_file_path0, data0, aux_data0, dependencies0) = result |
| 523 self.condition.acquire() |
| 524 self.data['target_build_files'].add(build_file_path0) |
| 525 for key in data0: |
| 526 self.data[key] = data0[key] |
| 527 for key in aux_data0: |
| 528 self.aux_data[key] = aux_data0[key] |
| 529 for new_dependency in dependencies0: |
| 530 if new_dependency not in self.scheduled: |
| 531 self.scheduled.add(new_dependency) |
| 532 self.dependencies.append(new_dependency) |
| 533 self.pending -= 1 |
| 534 self.condition.notify() |
| 535 self.condition.release() |
| 536 |
| 537 |
| 538 def LoadTargetBuildFileParallel(build_file_path, data, aux_data, |
| 539 variables, includes, depth, check): |
| 540 global parallel_state |
| 541 parallel_state = ParallelState() |
| 542 parallel_state.condition = threading.Condition() |
| 543 parallel_state.dependencies = [build_file_path] |
| 544 parallel_state.scheduled = set([build_file_path]) |
| 545 parallel_state.pending = 0 |
| 546 parallel_state.data = data |
| 547 parallel_state.aux_data = aux_data |
| 548 |
| 549 parallel_state.condition.acquire() |
| 550 while parallel_state.dependencies or parallel_state.pending: |
| 551 if not parallel_state.dependencies: |
| 552 parallel_state.condition.wait() |
| 553 continue |
| 554 |
| 555 dependency = parallel_state.dependencies.pop() |
| 556 |
| 557 parallel_state.pending += 1 |
| 558 data_in = {} |
| 559 data_in['target_build_files'] = data['target_build_files'] |
| 560 aux_data_in = {} |
| 561 global_flags = { |
| 562 'path_sections': globals()['path_sections'], |
| 563 'non_configuration_keys': globals()['non_configuration_keys'], |
| 564 'absolute_build_file_paths': globals()['absolute_build_file_paths'], |
| 565 'multiple_toolsets': globals()['multiple_toolsets']} |
| 566 |
| 567 if not parallel_state.pool: |
| 568 parallel_state.pool = multiprocessing.Pool(8) |
| 569 parallel_state.pool.apply_async( |
| 570 CallLoadTargetBuildFile, |
| 571 args = (global_flags, dependency, |
| 572 data_in, aux_data_in, |
| 573 variables, includes, depth, check), |
| 574 callback = parallel_state.LoadTargetBuildFileCallback) |
| 575 |
| 576 parallel_state.condition.release() |
438 | 577 |
439 | 578 |
440 # Look for the bracket that matches the first bracket seen in a | 579 # Look for the bracket that matches the first bracket seen in a |
441 # string, and return the start and end as a tuple. For example, if | 580 # string, and return the start and end as a tuple. For example, if |
442 # the input is something like "<(foo <(bar)) blah", then it would | 581 # the input is something like "<(foo <(bar)) blah", then it would |
443 # return (1, 13), indicating the entire string except for the leading | 582 # return (1, 13), indicating the entire string except for the leading |
444 # "<" and trailing " blah". | 583 # "<" and trailing " blah". |
445 def FindEnclosingBracketGroup(input): | 584 def FindEnclosingBracketGroup(input): |
446 brackets = { '}': '{', | 585 brackets = { '}': '{', |
447 ']': '[', | 586 ']': '[', |
(...skipping 1880 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2328 # Prepare a key like 'path/to:target_name'. | 2467 # Prepare a key like 'path/to:target_name'. |
2329 key = subdir + ':' + name | 2468 key = subdir + ':' + name |
2330 if key in used: | 2469 if key in used: |
2331 # Complain if this target is already used. | 2470 # Complain if this target is already used. |
2332 raise GypError('Duplicate target name "%s" in directory "%s" used both ' | 2471 raise GypError('Duplicate target name "%s" in directory "%s" used both ' |
2333 'in "%s" and "%s".' % (name, subdir, gyp, used[key])) | 2472 'in "%s" and "%s".' % (name, subdir, gyp, used[key])) |
2334 used[key] = gyp | 2473 used[key] = gyp |
2335 | 2474 |
2336 | 2475 |
2337 def Load(build_files, variables, includes, depth, generator_input_info, check, | 2476 def Load(build_files, variables, includes, depth, generator_input_info, check, |
2338 circular_check): | 2477 circular_check, parallel): |
2339 # Set up path_sections and non_configuration_keys with the default data plus | 2478 # Set up path_sections and non_configuration_keys with the default data plus |
2340 # the generator-specifc data. | 2479 # the generator-specifc data. |
2341 global path_sections | 2480 global path_sections |
2342 path_sections = base_path_sections[:] | 2481 path_sections = base_path_sections[:] |
2343 path_sections.extend(generator_input_info['path_sections']) | 2482 path_sections.extend(generator_input_info['path_sections']) |
2344 | 2483 |
2345 global non_configuration_keys | 2484 global non_configuration_keys |
2346 non_configuration_keys = base_non_configuration_keys[:] | 2485 non_configuration_keys = base_non_configuration_keys[:] |
2347 non_configuration_keys.extend(generator_input_info['non_configuration_keys']) | 2486 non_configuration_keys.extend(generator_input_info['non_configuration_keys']) |
2348 | 2487 |
(...skipping 20 matching lines...) Expand all Loading... |
2369 # NOTE: data contains both "target" files (.gyp) and "includes" (.gypi), as | 2508 # NOTE: data contains both "target" files (.gyp) and "includes" (.gypi), as |
2370 # well as meta-data (e.g. 'included_files' key). 'target_build_files' keeps | 2509 # well as meta-data (e.g. 'included_files' key). 'target_build_files' keeps |
2371 # track of the keys corresponding to "target" files. | 2510 # track of the keys corresponding to "target" files. |
2372 data = {'target_build_files': set()} | 2511 data = {'target_build_files': set()} |
2373 aux_data = {} | 2512 aux_data = {} |
2374 for build_file in build_files: | 2513 for build_file in build_files: |
2375 # Normalize paths everywhere. This is important because paths will be | 2514 # Normalize paths everywhere. This is important because paths will be |
2376 # used as keys to the data dict and for references between input files. | 2515 # used as keys to the data dict and for references between input files. |
2377 build_file = os.path.normpath(build_file) | 2516 build_file = os.path.normpath(build_file) |
2378 try: | 2517 try: |
2379 LoadTargetBuildFile(build_file, data, aux_data, variables, includes, | 2518 if parallel: |
2380 depth, check) | 2519 print >>sys.stderr, 'Using parallel processing (experimental).' |
| 2520 LoadTargetBuildFileParallel(build_file, data, aux_data, |
| 2521 variables, includes, depth, check) |
| 2522 else: |
| 2523 LoadTargetBuildFile(build_file, data, aux_data, |
| 2524 variables, includes, depth, check, True) |
2381 except Exception, e: | 2525 except Exception, e: |
2382 gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file) | 2526 gyp.common.ExceptionAppend(e, 'while trying to load %s' % build_file) |
2383 raise | 2527 raise |
2384 | 2528 |
2385 # Build a dict to access each target's subdict by qualified name. | 2529 # Build a dict to access each target's subdict by qualified name. |
2386 targets = BuildTargetsDict(data) | 2530 targets = BuildTargetsDict(data) |
2387 | 2531 |
2388 # Fully qualify all dependency links. | 2532 # Fully qualify all dependency links. |
2389 QualifyDependencies(targets) | 2533 QualifyDependencies(targets) |
2390 | 2534 |
(...skipping 87 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2478 ValidateRunAsInTarget(target, target_dict, build_file) | 2622 ValidateRunAsInTarget(target, target_dict, build_file) |
2479 ValidateActionsInTarget(target, target_dict, build_file) | 2623 ValidateActionsInTarget(target, target_dict, build_file) |
2480 | 2624 |
2481 # Generators might not expect ints. Turn them into strs. | 2625 # Generators might not expect ints. Turn them into strs. |
2482 TurnIntIntoStrInDict(data) | 2626 TurnIntIntoStrInDict(data) |
2483 | 2627 |
2484 # TODO(mark): Return |data| for now because the generator needs a list of | 2628 # TODO(mark): Return |data| for now because the generator needs a list of |
2485 # build files that came in. In the future, maybe it should just accept | 2629 # build files that came in. In the future, maybe it should just accept |
2486 # a list, and not the whole data dict. | 2630 # a list, and not the whole data dict. |
2487 return [flat_list, targets, data] | 2631 return [flat_list, targets, data] |
OLD | NEW |