OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2015 The LUCI Authors. All rights reserved. | 2 # Copyright 2015 The LUCI Authors. All rights reserved. |
3 # Use of this source code is governed under the Apache License, Version 2.0 | 3 # Use of this source code is governed under the Apache License, Version 2.0 |
4 # that can be found in the LICENSE file. | 4 # that can be found in the LICENSE file. |
5 | 5 |
6 """Tool to interact with recipe repositories. | 6 """Tool to interact with recipe repositories. |
7 | 7 |
8 This tool operates on the nearest ancestor directory containing an | 8 This tool operates on the nearest ancestor directory containing an |
9 infra/config/recipes.cfg. | 9 infra/config/recipes.cfg. |
10 """ | 10 """ |
(...skipping 15 matching lines...) Expand all Loading... |
26 | 26 |
27 ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) | 27 ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) |
28 sys.path.insert(0, ROOT_DIR) | 28 sys.path.insert(0, ROOT_DIR) |
29 | 29 |
30 from recipe_engine import env | 30 from recipe_engine import env |
31 from recipe_engine import arguments_pb2 | 31 from recipe_engine import arguments_pb2 |
32 from recipe_engine import util as recipe_util | 32 from recipe_engine import util as recipe_util |
33 from google.protobuf import json_format as jsonpb | 33 from google.protobuf import json_format as jsonpb |
34 | 34 |
35 | 35 |
36 def test(config_file, package_deps, args, op_args): | 36 def test(config_file, package_deps, args): |
37 try: | 37 try: |
38 from recipe_engine import test | 38 from recipe_engine import test |
39 except ImportError: | 39 except ImportError: |
40 logging.error( | 40 logging.error( |
41 'Error while importing testing libraries. You may be missing the pip' | 41 'Error while importing testing libraries. You may be missing the pip' |
42 ' package "coverage". Install it, or use the --use-bootstrap command' | 42 ' package "coverage". Install it, or use the --use-bootstrap command' |
43 ' line argument when calling into the recipe engine, which will install' | 43 ' line argument when calling into the recipe engine, which will install' |
44 ' it for you.') | 44 ' it for you.') |
45 raise | 45 raise |
46 | 46 |
47 from recipe_engine import loader | 47 from recipe_engine import loader |
48 from recipe_engine import package | 48 from recipe_engine import package |
49 | 49 |
50 universe = loader.RecipeUniverse(package_deps, config_file) | 50 universe = loader.RecipeUniverse(package_deps, config_file) |
51 universe_view = loader.UniverseView(universe, package_deps.root_package) | 51 universe_view = loader.UniverseView(universe, package_deps.root_package) |
52 | 52 |
53 # Prevent flakiness caused by stale pyc files. | 53 # Prevent flakiness caused by stale pyc files. |
54 package.cleanup_pyc(package_deps.root_package.recipes_dir) | 54 package.cleanup_pyc(package_deps.root_package.recipes_dir) |
55 | 55 |
56 return test.main( | 56 return test.main( |
57 universe_view, raw_args=args.args, | 57 universe_view, raw_args=args.args, |
58 engine_flags=op_args.engine_flags) | 58 engine_flags=args.operational_args.engine_flags) |
59 | 59 |
60 | 60 |
61 def lint(config_file, package_deps, args): | 61 def lint(config_file, package_deps, args): |
62 from recipe_engine import lint_test | 62 from recipe_engine import lint_test |
63 from recipe_engine import loader | 63 from recipe_engine import loader |
64 | 64 |
65 universe = loader.RecipeUniverse(package_deps, config_file) | 65 universe = loader.RecipeUniverse(package_deps, config_file) |
66 universe_view = loader.UniverseView(universe, package_deps.root_package) | 66 universe_view = loader.UniverseView(universe, package_deps.root_package) |
67 | 67 |
68 lint_test.main(universe_view, args.whitelist or []) | 68 lint_test.main(universe_view, args.whitelist or []) |
(...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
141 | 141 |
142 with stream_engine.make_step_stream('Failure reason') as s: | 142 with stream_engine.make_step_stream('Failure reason') as s: |
143 with s.new_log_stream('reason') as l: | 143 with s.new_log_stream('reason') as l: |
144 l.write_split(f.human_reason) | 144 l.write_split(f.human_reason) |
145 | 145 |
146 return 1 | 146 return 1 |
147 | 147 |
148 return 0 | 148 return 0 |
149 | 149 |
150 | 150 |
151 def run(config_file, package_deps, args, op_args): | 151 def run(config_file, package_deps, args): |
152 from recipe_engine import run as recipe_run | 152 from recipe_engine import run as recipe_run |
153 from recipe_engine import loader | 153 from recipe_engine import loader |
154 from recipe_engine import step_runner | 154 from recipe_engine import step_runner |
155 from recipe_engine import stream | 155 from recipe_engine import stream |
156 from recipe_engine import stream_logdog | 156 from recipe_engine import stream_logdog |
157 | 157 |
158 if args.props: | 158 if args.props: |
159 for p in args.props: | 159 for p in args.props: |
160 args.properties.update(p) | 160 args.properties.update(p) |
161 | 161 |
162 def get_properties_from_operational_args(op_args): | 162 def get_properties_from_operational_args(op_args): |
163 if not op_args.properties.property: | 163 if not op_args.properties.property: |
164 return None | 164 return None |
165 return _op_properties_to_dict(op_args.properties.property) | 165 return _op_properties_to_dict(op_args.properties.property) |
166 | 166 |
| 167 op_args = args.operational_args |
167 op_properties = get_properties_from_operational_args(op_args) | 168 op_properties = get_properties_from_operational_args(op_args) |
168 if args.properties and op_properties: | 169 if args.properties and op_properties: |
169 raise ValueError( | 170 raise ValueError( |
170 "Got operational args properties as well as CLI properties.") | 171 "Got operational args properties as well as CLI properties.") |
171 | 172 |
172 properties = op_properties | 173 properties = op_properties |
173 if not properties: | 174 if not properties: |
174 properties = args.properties | 175 properties = args.properties |
175 | 176 |
176 properties['recipe'] = args.recipe | 177 properties['recipe'] = args.recipe |
(...skipping 199 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
376 help='Override a project repository path with a local one.') | 377 help='Override a project repository path with a local one.') |
377 parser.add_argument( | 378 parser.add_argument( |
378 # Use "None" as default so that we can recognize when none of the | 379 # Use "None" as default so that we can recognize when none of the |
379 # bootstrap options were passed. | 380 # bootstrap options were passed. |
380 '--use-bootstrap', action='store_true', default=None, | 381 '--use-bootstrap', action='store_true', default=None, |
381 help='Use bootstrap/bootstrap.py to create a isolated python virtualenv' | 382 help='Use bootstrap/bootstrap.py to create a isolated python virtualenv' |
382 ' with required python dependencies.') | 383 ' with required python dependencies.') |
383 parser.add_argument( | 384 parser.add_argument( |
384 '--disable-bootstrap', action='store_false', dest='use_bootstrap', | 385 '--disable-bootstrap', action='store_false', dest='use_bootstrap', |
385 help='Disables bootstrap (see --use-bootstrap)') | 386 help='Disables bootstrap (see --use-bootstrap)') |
| 387 |
| 388 def operational_args_type(value): |
| 389 with open(value) as fd: |
| 390 return jsonpb.ParseDict(json.load(fd), arguments_pb2.Arguments()) |
| 391 |
| 392 parser.set_defaults(operational_args=arguments_pb2.Arguments()) |
| 393 |
386 parser.add_argument( | 394 parser.add_argument( |
387 '--operational-args-path', action='store', | 395 '--operational-args-path', |
388 type=os.path.abspath, | 396 dest='operational_args', |
| 397 type=operational_args_type, |
389 help='The path to an operational Arguments file. If provided, this file ' | 398 help='The path to an operational Arguments file. If provided, this file ' |
390 'must contain a JSONPB-encoded Arguments protobuf message, and will ' | 399 'must contain a JSONPB-encoded Arguments protobuf message, and will ' |
391 'be integrated into the runtime parameters.') | 400 'be integrated into the runtime parameters.') |
392 | 401 |
393 | 402 |
394 def post_process_common_args(parser, args): | 403 def post_process_common_args(parser, args): |
395 if args.command == "remote": | 404 if args.command == "remote": |
396 # TODO(iannucci): this is a hack; remote doesn't behave like ANY other | 405 # TODO(iannucci): this is a hack; remote doesn't behave like ANY other |
397 # commands. A way to solve this will be to allow --package to take a remote | 406 # commands. A way to solve this will be to allow --package to take a remote |
398 # repo and then simply remove the remote subcommand entirely. | 407 # repo and then simply remove the remote subcommand entirely. |
(...skipping 190 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
589 'with their documentation') | 598 'with their documentation') |
590 doc_p.add_argument('recipe', nargs='?', | 599 doc_p.add_argument('recipe', nargs='?', |
591 help='Restrict documentation to this recipe') | 600 help='Restrict documentation to this recipe') |
592 doc_p.add_argument('--kind', default='jsonpb', choices=doc_kinds, | 601 doc_p.add_argument('--kind', default='jsonpb', choices=doc_kinds, |
593 help='Output this kind of documentation') | 602 help='Output this kind of documentation') |
594 doc_p.set_defaults(command='doc') | 603 doc_p.set_defaults(command='doc') |
595 | 604 |
596 args = parser.parse_args() | 605 args = parser.parse_args() |
597 post_process_common_args(parser, args) | 606 post_process_common_args(parser, args) |
598 | 607 |
599 # Load/parse operational arguments. | |
600 op_args = arguments_pb2.Arguments() | |
601 if args.operational_args_path is not None: | |
602 with open(args.operational_args_path) as fd: | |
603 data = fd.read() | |
604 jsonpb.Parse(data, op_args) | |
605 | |
606 # TODO(iannucci): We should always do logging.basicConfig() (probably with | 608 # TODO(iannucci): We should always do logging.basicConfig() (probably with |
607 # logging.WARNING), even if no verbose is passed. However we need to be | 609 # logging.WARNING), even if no verbose is passed. However we need to be |
608 # careful as this could cause issues with spurious/unexpected output. I think | 610 # careful as this could cause issues with spurious/unexpected output. I think |
609 # it's risky enough to do in a different CL. | 611 # it's risky enough to do in a different CL. |
610 | 612 |
611 if args.verbose > 0: | 613 if args.verbose > 0: |
612 logging.basicConfig() | 614 logging.basicConfig() |
613 logging.getLogger().setLevel(logging.INFO) | 615 logging.getLogger().setLevel(logging.INFO) |
614 if args.verbose > 1: | 616 if args.verbose > 1: |
615 logging.getLogger().setLevel(logging.DEBUG) | 617 logging.getLogger().setLevel(logging.DEBUG) |
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
668 return subprocess.call( | 670 return subprocess.call( |
669 [ | 671 [ |
670 os.path.join(env_path, 'bin', python_exe), | 672 os.path.join(env_path, 'bin', python_exe), |
671 '-B', # Don't compile "pyo" binaries. | 673 '-B', # Don't compile "pyo" binaries. |
672 '-E', # Don't use PYTHON* enviornment variables. | 674 '-E', # Don't use PYTHON* enviornment variables. |
673 '-s', # Don't use user 'site.py'. | 675 '-s', # Don't use user 'site.py'. |
674 os.path.join(ROOT_DIR, 'recipes.py'), | 676 os.path.join(ROOT_DIR, 'recipes.py'), |
675 ] + sys.argv[1:]) | 677 ] + sys.argv[1:]) |
676 | 678 |
677 # Standard recipe engine operation. | 679 # Standard recipe engine operation. |
678 return _real_main(args, op_args) | 680 return _real_main(args) |
679 | 681 |
680 finally: | 682 finally: |
681 # If we're using a temporary deps directory, clean it up here. | 683 # If we're using a temporary deps directory, clean it up here. |
682 if temp_deps_dir: | 684 if temp_deps_dir: |
683 logging.info('Cleaning up temporary deps path: %s', temp_deps_dir) | 685 logging.info('Cleaning up temporary deps path: %s', temp_deps_dir) |
684 | 686 |
685 # Remove as much of the temporary directory as we can. If something goes | 687 # Remove as much of the temporary directory as we can. If something goes |
686 # wrong, log the error, but don't actually raise anything. | 688 # wrong, log the error, but don't actually raise anything. |
687 def on_error(_function, path, excinfo): | 689 def on_error(_function, path, excinfo): |
688 logging.error('Error cleaning up temporary deps file: %s', path, | 690 logging.error('Error cleaning up temporary deps file: %s', path, |
689 exc_info=excinfo) | 691 exc_info=excinfo) |
690 shutil.rmtree(temp_deps_dir, onerror=on_error) | 692 shutil.rmtree(temp_deps_dir, onerror=on_error) |
691 | 693 |
692 | 694 |
693 def _real_main(args, op_args): | 695 def _real_main(args): |
694 from recipe_engine import package, package_io | 696 from recipe_engine import package |
695 | 697 |
696 # Commands which do not require config_file, package_deps, and other objects | 698 # Commands which do not require config_file, package_deps, and other objects |
697 # initialized later. | 699 # initialized later. |
698 if args.command == 'remote': | 700 if args.command == 'remote': |
699 return remote(args) | 701 return remote(args) |
700 | 702 |
701 config_file = args.package | 703 config_file = args.package |
702 repo_root = package.InfraRepoConfig().from_recipes_cfg(args.package.path) | 704 repo_root = package.InfraRepoConfig().from_recipes_cfg(args.package.path) |
703 | 705 |
704 try: | 706 try: |
705 # TODO(phajdan.jr): gracefully handle inconsistent deps when rolling. | 707 # TODO(phajdan.jr): gracefully handle inconsistent deps when rolling. |
706 # This fails if the starting point does not have consistent dependency | 708 # This fails if the starting point does not have consistent dependency |
707 # graph. When performing an automated roll, it'd make sense to attempt | 709 # graph. When performing an automated roll, it'd make sense to attempt |
708 # to automatically find a consistent state, rather than bailing out. | 710 # to automatically find a consistent state, rather than bailing out. |
709 # Especially that only some subcommands refer to package_deps. | 711 # Especially that only some subcommands refer to package_deps. |
710 package_deps = package.PackageDeps.create( | 712 package_deps = package.PackageDeps.create( |
711 repo_root, config_file, allow_fetch=not args.no_fetch, | 713 repo_root, config_file, allow_fetch=not args.no_fetch, |
712 deps_path=args.deps_path, overrides=args.project_override) | 714 deps_path=args.deps_path, overrides=args.project_override) |
713 except subprocess.CalledProcessError: | 715 except subprocess.CalledProcessError: |
714 # A git checkout failed somewhere. Return 2, which is the sign that this is | 716 # A git checkout failed somewhere. Return 2, which is the sign that this is |
715 # an infra failure, rather than a test failure. | 717 # an infra failure, rather than a test failure. |
716 return 2 | 718 return 2 |
717 | 719 |
718 if args.command == 'fetch': | 720 if args.command == 'fetch': |
719 # We already did everything in the create() call above. | 721 # We already did everything in the create() call above. |
720 assert not args.no_fetch, 'Fetch? No-fetch? Make up your mind!' | 722 assert not args.no_fetch, 'Fetch? No-fetch? Make up your mind!' |
721 return 0 | 723 return 0 |
722 elif args.command == 'test': | 724 elif args.command == 'test': |
723 return test(config_file, package_deps, args, op_args) | 725 return test(config_file, package_deps, args) |
724 elif args.command == 'bundle': | 726 elif args.command == 'bundle': |
725 return bundle(config_file, package_deps, args) | 727 return bundle(config_file, package_deps, args) |
726 elif args.command == 'lint': | 728 elif args.command == 'lint': |
727 return lint(config_file, package_deps, args) | 729 return lint(config_file, package_deps, args) |
728 elif args.command == 'run': | 730 elif args.command == 'run': |
729 return run(config_file, package_deps, args, op_args) | 731 return run(config_file, package_deps, args) |
730 elif args.command == 'autoroll': | 732 elif args.command == 'autoroll': |
731 return autoroll(repo_root, config_file, args) | 733 return autoroll(repo_root, config_file, args) |
732 elif args.command == 'depgraph': | 734 elif args.command == 'depgraph': |
733 return depgraph(config_file, package_deps, args) | 735 return depgraph(config_file, package_deps, args) |
734 elif args.command == 'refs': | 736 elif args.command == 'refs': |
735 return refs(config_file, package_deps, args) | 737 return refs(config_file, package_deps, args) |
736 elif args.command == 'doc': | 738 elif args.command == 'doc': |
737 return doc(config_file, package_deps, args) | 739 return doc(config_file, package_deps, args) |
738 else: | 740 else: |
739 print """Dear sir or madam, | 741 print """Dear sir or madam, |
(...skipping 27 matching lines...) Expand all Loading... |
767 | 769 |
768 if not isinstance(ret, int): | 770 if not isinstance(ret, int): |
769 if ret is None: | 771 if ret is None: |
770 ret = 0 | 772 ret = 0 |
771 else: | 773 else: |
772 print >> sys.stderr, ret | 774 print >> sys.stderr, ret |
773 ret = 1 | 775 ret = 1 |
774 sys.stdout.flush() | 776 sys.stdout.flush() |
775 sys.stderr.flush() | 777 sys.stderr.flush() |
776 os._exit(ret) | 778 os._exit(ret) |
OLD | NEW |