OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2015 The LUCI Authors. All rights reserved. | 2 # Copyright 2015 The LUCI Authors. All rights reserved. |
3 # Use of this source code is governed under the Apache License, Version 2.0 | 3 # Use of this source code is governed under the Apache License, Version 2.0 |
4 # that can be found in the LICENSE file. | 4 # that can be found in the LICENSE file. |
5 | 5 |
6 """Tool to interact with recipe repositories. | 6 """Tool to interact with recipe repositories. |
7 | 7 |
8 This tool operates on the nearest ancestor directory containing an | 8 This tool operates on the nearest ancestor directory containing an |
9 infra/config/recipes.cfg. | 9 infra/config/recipes.cfg. |
10 """ | 10 """ |
(...skipping 12 matching lines...) Expand all Loading... |
23 reload(sys) | 23 reload(sys) |
24 sys.setdefaultencoding('UTF8') | 24 sys.setdefaultencoding('UTF8') |
25 | 25 |
26 ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) | 26 ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) |
27 sys.path.insert(0, ROOT_DIR) | 27 sys.path.insert(0, ROOT_DIR) |
28 | 28 |
29 from recipe_engine import env | 29 from recipe_engine import env |
30 | 30 |
31 import argparse # this is vendored | 31 import argparse # this is vendored |
32 from recipe_engine import arguments_pb2 | 32 from recipe_engine import arguments_pb2 |
33 from recipe_engine import util as recipe_util | |
34 from google.protobuf import json_format as jsonpb | 33 from google.protobuf import json_format as jsonpb |
35 | 34 |
36 | 35 |
37 def handle_recipe_return(recipe_result, result_filename, stream_engine, | 36 from recipe_engine import fetch, lint_test, bundle, depgraph, autoroll |
38 engine_flags): | 37 from recipe_engine import remote, refs, doc, test, run |
39 if engine_flags and engine_flags.use_result_proto: | |
40 return new_handle_recipe_return( | |
41 recipe_result, result_filename, stream_engine) | |
42 | |
43 if 'recipe_result' in recipe_result.result: | |
44 result_string = json.dumps( | |
45 recipe_result.result['recipe_result'], indent=2) | |
46 if result_filename: | |
47 with open(result_filename, 'w') as f: | |
48 f.write(result_string) | |
49 with stream_engine.make_step_stream('recipe result') as s: | |
50 with s.new_log_stream('result') as l: | |
51 l.write_split(result_string) | |
52 | |
53 if 'traceback' in recipe_result.result: | |
54 with stream_engine.make_step_stream('Uncaught Exception') as s: | |
55 with s.new_log_stream('exception') as l: | |
56 for line in recipe_result.result['traceback']: | |
57 l.write_line(line) | |
58 | |
59 if 'reason' in recipe_result.result: | |
60 with stream_engine.make_step_stream('Failure reason') as s: | |
61 with s.new_log_stream('reason') as l: | |
62 for line in recipe_result.result['reason'].splitlines(): | |
63 l.write_line(line) | |
64 | |
65 if 'status_code' in recipe_result.result: | |
66 return recipe_result.result['status_code'] | |
67 else: | |
68 return 0 | |
69 | |
70 def new_handle_recipe_return(result, result_filename, stream_engine): | |
71 if result_filename: | |
72 with open(result_filename, 'w') as fil: | |
73 fil.write(jsonpb.MessageToJson( | |
74 result, including_default_value_fields=True)) | |
75 | |
76 if result.json_result: | |
77 with stream_engine.make_step_stream('recipe result') as s: | |
78 with s.new_log_stream('result') as l: | |
79 l.write_split(result.json_result) | |
80 | |
81 if result.HasField('failure'): | |
82 f = result.failure | |
83 if f.HasField('exception'): | |
84 with stream_engine.make_step_stream('Uncaught Exception') as s: | |
85 s.add_step_text(f.human_reason) | |
86 with s.new_log_stream('exception') as l: | |
87 for line in f.exception.traceback: | |
88 l.write_line(line) | |
89 # TODO(martiniss): Remove this code once calling code handles these states | |
90 elif f.HasField('timeout'): | |
91 with stream_engine.make_step_stream('Step Timed Out') as s: | |
92 with s.new_log_stream('timeout_s') as l: | |
93 l.write_line(f.timeout.timeout_s) | |
94 elif f.HasField('step_data'): | |
95 with stream_engine.make_step_stream('Invalid Step Data Access') as s: | |
96 with s.new_log_stream('step') as l: | |
97 l.write_line(f.step_data.step) | |
98 | |
99 with stream_engine.make_step_stream('Failure reason') as s: | |
100 with s.new_log_stream('reason') as l: | |
101 l.write_split(f.human_reason) | |
102 | |
103 return 1 | |
104 | |
105 return 0 | |
106 | 38 |
107 | 39 |
108 def run(config_file, package_deps, args): | 40 _SUBCOMMANDS = [ |
109 from recipe_engine import run as recipe_run | 41 autoroll, |
110 from recipe_engine import loader | 42 bundle, |
111 from recipe_engine import step_runner | 43 depgraph, |
112 from recipe_engine import stream | 44 doc, |
113 from recipe_engine import stream_logdog | 45 fetch, |
114 | 46 lint_test, |
115 if args.props: | 47 refs, |
116 for p in args.props: | 48 remote, |
117 args.properties.update(p) | 49 run, |
118 | 50 test, |
119 def get_properties_from_operational_args(op_args): | 51 ] |
120 if not op_args.properties.property: | |
121 return None | |
122 return _op_properties_to_dict(op_args.properties.property) | |
123 | |
124 op_args = args.operational_args | |
125 op_properties = get_properties_from_operational_args(op_args) | |
126 if args.properties and op_properties: | |
127 raise ValueError( | |
128 'Got operational args properties as well as CLI properties.') | |
129 | |
130 properties = op_properties | |
131 if not properties: | |
132 properties = args.properties | |
133 | |
134 properties['recipe'] = args.recipe | |
135 | |
136 properties = recipe_util.strip_unicode(properties) | |
137 | |
138 os.environ['PYTHONUNBUFFERED'] = '1' | |
139 os.environ['PYTHONIOENCODING'] = 'UTF-8' | |
140 | |
141 universe_view = loader.UniverseView( | |
142 loader.RecipeUniverse( | |
143 package_deps, config_file), package_deps.root_package) | |
144 | |
145 workdir = (args.workdir or | |
146 os.path.join(os.path.dirname(os.path.realpath(__file__)), 'workdir')) | |
147 logging.info('Using %s as work directory' % workdir) | |
148 if not os.path.exists(workdir): | |
149 os.makedirs(workdir) | |
150 | |
151 old_cwd = os.getcwd() | |
152 os.chdir(workdir) | |
153 | |
154 # Construct our stream engines. We may want to share stream events with more | |
155 # than one StreamEngine implementation, so we will accumulate them in a | |
156 # "stream_engines" list and compose them into a MultiStreamEngine. | |
157 def build_annotation_stream_engine(): | |
158 return stream.AnnotatorStreamEngine( | |
159 sys.stdout, | |
160 emit_timestamps=(args.timestamps or | |
161 op_args.annotation_flags.emit_timestamp)) | |
162 | |
163 stream_engines = [] | |
164 if op_args.logdog.streamserver_uri: | |
165 logging.debug('Using LogDog with parameters: [%s]', op_args.logdog) | |
166 stream_engines.append(stream_logdog.StreamEngine( | |
167 streamserver_uri=op_args.logdog.streamserver_uri, | |
168 name_base=(op_args.logdog.name_base or None), | |
169 dump_path=op_args.logdog.final_annotation_dump_path, | |
170 )) | |
171 | |
172 # If we're teeing, also fold in a standard annotation stream engine. | |
173 if op_args.logdog.tee: | |
174 stream_engines.append(build_annotation_stream_engine()) | |
175 else: | |
176 # Not using LogDog; use a standard annotation stream engine. | |
177 stream_engines.append(build_annotation_stream_engine()) | |
178 multi_stream_engine = stream.MultiStreamEngine.create(*stream_engines) | |
179 | |
180 emit_initial_properties = op_args.annotation_flags.emit_initial_properties | |
181 engine_flags = op_args.engine_flags | |
182 | |
183 # Have a top-level set of invariants to enforce StreamEngine expectations. | |
184 with stream.StreamEngineInvariants.wrap(multi_stream_engine) as stream_engine: | |
185 try: | |
186 ret = recipe_run.run_steps( | |
187 properties, stream_engine, | |
188 step_runner.SubprocessStepRunner(stream_engine, engine_flags), | |
189 universe_view, engine_flags=engine_flags, | |
190 emit_initial_properties=emit_initial_properties) | |
191 finally: | |
192 os.chdir(old_cwd) | |
193 | |
194 return handle_recipe_return( | |
195 ret, args.output_result_json, stream_engine, engine_flags) | |
196 | |
197 | |
198 class ProjectOverrideAction(argparse.Action): | |
199 def __call__(self, parser, namespace, values, option_string=None): | |
200 p = values.split('=', 2) | |
201 if len(p) != 2: | |
202 raise ValueError('Override must have the form: repo=path') | |
203 project_id, path = p | |
204 | |
205 v = getattr(namespace, self.dest, None) | |
206 if v is None: | |
207 v = {} | |
208 setattr(namespace, self.dest, v) | |
209 | |
210 if v.get(project_id): | |
211 raise ValueError('An override is already defined for [%s] (%s)' % ( | |
212 project_id, v[project_id])) | |
213 path = os.path.abspath(os.path.expanduser(path)) | |
214 if not os.path.isdir(path): | |
215 raise ValueError('Override path [%s] is not a directory' % (path,)) | |
216 v[project_id] = path | |
217 | |
218 | |
219 # Map of arguments_pb2.Property "value" oneof conversion functions. | |
220 # | |
221 # The fields here should be kept in sync with the "value" oneof field names in | |
222 # the arguments_pb2.Arguments.Property protobuf message. | |
223 _OP_PROPERTY_CONV = { | |
224 's': lambda prop: prop.s, | |
225 'int': lambda prop: prop.int, | |
226 'uint': lambda prop: prop.uint, | |
227 'd': lambda prop: prop.d, | |
228 'b': lambda prop: prop.b, | |
229 'data': lambda prop: prop.data, | |
230 'map': lambda prop: _op_properties_to_dict(prop.map.property), | |
231 'list': lambda prop: [_op_property_value(v) for v in prop.list.property], | |
232 } | |
233 | |
234 def _op_property_value(prop): | |
235 """Returns the Python-converted value of an arguments_pb2.Property. | |
236 | |
237 Args: | |
238 prop (arguments_pb2.Property): property to convert. | |
239 Returns: The converted value. | |
240 Raises: | |
241 ValueError: If 'prop' is incomplete or invalid. | |
242 """ | |
243 typ = prop.WhichOneof('value') | |
244 conv = _OP_PROPERTY_CONV.get(typ) | |
245 if not conv: | |
246 raise ValueError('Unknown property field [%s]' % (typ,)) | |
247 return conv(prop) | |
248 | |
249 | |
250 def _op_properties_to_dict(pmap): | |
251 """Creates a properties dictionary from an arguments_pb2.PropertyMap entry. | |
252 | |
253 Args: | |
254 pmap (arguments_pb2.PropertyMap): Map to convert to dictionary form. | |
255 Returns (dict): A dictionary derived from the properties in 'pmap'. | |
256 """ | |
257 return dict((k, _op_property_value(pmap[k])) for k in pmap) | |
258 | 52 |
259 | 53 |
260 def add_common_args(parser): | 54 def add_common_args(parser): |
261 from recipe_engine import package_io | 55 from recipe_engine import package_io |
262 | 56 |
| 57 class ProjectOverrideAction(argparse.Action): |
| 58 def __call__(self, parser, namespace, values, option_string=None): |
| 59 p = values.split('=', 2) |
| 60 if len(p) != 2: |
| 61 raise ValueError('Override must have the form: repo=path') |
| 62 project_id, path = p |
| 63 |
| 64 v = getattr(namespace, self.dest, None) |
| 65 if v is None: |
| 66 v = {} |
| 67 setattr(namespace, self.dest, v) |
| 68 |
| 69 if v.get(project_id): |
| 70 raise ValueError('An override is already defined for [%s] (%s)' % ( |
| 71 project_id, v[project_id])) |
| 72 path = os.path.abspath(os.path.expanduser(path)) |
| 73 if not os.path.isdir(path): |
| 74 raise ValueError('Override path [%s] is not a directory' % (path,)) |
| 75 v[project_id] = path |
| 76 |
263 def package_type(value): | 77 def package_type(value): |
264 if not os.path.isfile(value): | 78 if not os.path.isfile(value): |
265 raise argparse.ArgumentTypeError( | 79 raise argparse.ArgumentTypeError( |
266 'Given recipes config file %r does not exist.' % (value,)) | 80 'Given recipes config file %r does not exist.' % (value,)) |
267 return package_io.PackageFile(value) | 81 return package_io.PackageFile(value) |
268 | 82 |
269 parser.add_argument( | 83 parser.add_argument( |
270 '--package', | 84 '--package', |
271 type=package_type, | 85 type=package_type, |
272 help='Path to recipes.cfg of the recipe package to operate on' | 86 help='Path to recipes.cfg of the recipe package to operate on' |
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
333 | 147 |
334 return post_process_args | 148 return post_process_args |
335 | 149 |
336 | 150 |
337 def main(): | 151 def main(): |
338 parser = argparse.ArgumentParser( | 152 parser = argparse.ArgumentParser( |
339 description='Interact with the recipe system.') | 153 description='Interact with the recipe system.') |
340 | 154 |
341 common_postprocess_func = add_common_args(parser) | 155 common_postprocess_func = add_common_args(parser) |
342 | 156 |
343 from recipe_engine import fetch, lint_test, bundle, depgraph, autoroll | |
344 from recipe_engine import remote, refs, doc, test | |
345 to_add = [ | |
346 fetch, lint_test, bundle, depgraph, autoroll, remote, refs, doc, test, | |
347 ] | |
348 | |
349 subp = parser.add_subparsers() | 157 subp = parser.add_subparsers() |
350 for module in to_add: | 158 for module in _SUBCOMMANDS: |
351 module.add_subparser(subp) | 159 module.add_subparser(subp) |
352 | 160 |
353 | |
354 def properties_file_type(filename): | |
355 with (sys.stdin if filename == '-' else open(filename)) as f: | |
356 obj = json.load(f) | |
357 if not isinstance(obj, dict): | |
358 raise argparse.ArgumentTypeError( | |
359 'must contain a JSON object, i.e. `{}`.') | |
360 return obj | |
361 | |
362 def parse_prop(prop): | |
363 key, val = prop.split('=', 1) | |
364 try: | |
365 val = json.loads(val) | |
366 except (ValueError, SyntaxError): | |
367 pass # If a value couldn't be evaluated, keep the string version | |
368 return {key: val} | |
369 | |
370 def properties_type(value): | |
371 obj = json.loads(value) | |
372 if not isinstance(obj, dict): | |
373 raise argparse.ArgumentTypeError('must contain a JSON object, i.e. `{}`.') | |
374 return obj | |
375 | |
376 run_p = subp.add_parser( | |
377 'run', | |
378 description='Run a recipe locally') | |
379 run_p.set_defaults(command='run', properties={}) | |
380 | |
381 run_p.add_argument( | |
382 '--workdir', | |
383 type=os.path.abspath, | |
384 help='The working directory of recipe execution') | |
385 run_p.add_argument( | |
386 '--output-result-json', | |
387 type=os.path.abspath, | |
388 help='The file to write the JSON serialized returned value \ | |
389 of the recipe to') | |
390 run_p.add_argument( | |
391 '--timestamps', | |
392 action='store_true', | |
393 help='If true, emit CURRENT_TIMESTAMP annotations. ' | |
394 'Default: false. ' | |
395 'CURRENT_TIMESTAMP annotation has one parameter, current time in ' | |
396 'Unix timestamp format. ' | |
397 'CURRENT_TIMESTAMP annotation will be printed at the beginning and ' | |
398 'end of the annotation stream and also immediately before each ' | |
399 'STEP_STARTED and STEP_CLOSED annotations.', | |
400 ) | |
401 prop_group = run_p.add_mutually_exclusive_group() | |
402 prop_group.add_argument( | |
403 '--properties-file', | |
404 dest='properties', | |
405 type=properties_file_type, | |
406 help=('A file containing a json blob of properties. ' | |
407 'Pass "-" to read from stdin')) | |
408 prop_group.add_argument( | |
409 '--properties', | |
410 type=properties_type, | |
411 help='A json string containing the properties') | |
412 | |
413 run_p.add_argument( | |
414 'recipe', | |
415 help='The recipe to execute') | |
416 run_p.add_argument( | |
417 'props', | |
418 nargs=argparse.REMAINDER, | |
419 type=parse_prop, | |
420 help='A list of property pairs; e.g. mastername=chromium.linux ' | |
421 'issue=12345. The property value will be decoded as JSON, but if ' | |
422 'this decoding fails the value will be interpreted as a string.') | |
423 | |
424 args = parser.parse_args() | 161 args = parser.parse_args() |
425 common_postprocess_func(parser, args) | 162 common_postprocess_func(parser, args) |
426 args.postprocess_func(parser, args) | 163 args.postprocess_func(parser, args) |
427 | 164 |
428 # TODO(iannucci): We should always do logging.basicConfig() (probably with | 165 # TODO(iannucci): We should always do logging.basicConfig() (probably with |
429 # logging.WARNING), even if no verbose is passed. However we need to be | 166 # logging.WARNING), even if no verbose is passed. However we need to be |
430 # careful as this could cause issues with spurious/unexpected output. I think | 167 # careful as this could cause issues with spurious/unexpected output. I think |
431 # it's risky enough to do in a different CL. | 168 # it's risky enough to do in a different CL. |
432 | 169 |
433 if args.verbose > 0: | 170 if args.verbose > 0: |
(...skipping 30 matching lines...) Expand all Loading... |
464 # Standard recipe engine operation. | 201 # Standard recipe engine operation. |
465 return _real_main(args) | 202 return _real_main(args) |
466 | 203 |
467 | 204 |
468 def _real_main(args): | 205 def _real_main(args): |
469 from recipe_engine import package | 206 from recipe_engine import package |
470 | 207 |
471 if args.bare_command: | 208 if args.bare_command: |
472 return args.func(None, args) | 209 return args.func(None, args) |
473 | 210 |
474 config_file = args.package | |
475 repo_root = package.InfraRepoConfig().from_recipes_cfg(args.package.path) | 211 repo_root = package.InfraRepoConfig().from_recipes_cfg(args.package.path) |
476 | 212 |
477 try: | 213 try: |
478 # TODO(phajdan.jr): gracefully handle inconsistent deps when rolling. | 214 # TODO(phajdan.jr): gracefully handle inconsistent deps when rolling. |
479 # This fails if the starting point does not have consistent dependency | 215 # This fails if the starting point does not have consistent dependency |
480 # graph. When performing an automated roll, it'd make sense to attempt | 216 # graph. When performing an automated roll, it'd make sense to attempt |
481 # to automatically find a consistent state, rather than bailing out. | 217 # to automatically find a consistent state, rather than bailing out. |
482 # Especially that only some subcommands refer to package_deps. | 218 # Especially that only some subcommands refer to package_deps. |
483 package_deps = package.PackageDeps.create( | 219 package_deps = package.PackageDeps.create( |
484 repo_root, config_file, allow_fetch=not args.no_fetch, | 220 repo_root, args.package, allow_fetch=not args.no_fetch, |
485 deps_path=args.deps_path, overrides=args.project_override) | 221 deps_path=args.deps_path, overrides=args.project_override) |
486 except subprocess.CalledProcessError: | 222 except subprocess.CalledProcessError: |
487 # A git checkout failed somewhere. Return 2, which is the sign that this is | 223 # A git checkout failed somewhere. Return 2, which is the sign that this is |
488 # an infra failure, rather than a test failure. | 224 # an infra failure, rather than a test failure. |
489 return 2 | 225 return 2 |
490 | 226 |
491 if hasattr(args, 'func'): | 227 return args.func(package_deps, args) |
492 return args.func(package_deps, args) | |
493 | 228 |
494 if args.command == 'run': | |
495 return run(config_file, package_deps, args) | |
496 else: | |
497 print """Dear sir or madam, | |
498 It has come to my attention that a quite impossible condition has come | |
499 to pass in the specification you have issued a request for us to fulfill. | |
500 It is with a heavy heart that I inform you that, at the present juncture, | |
501 there is no conceivable next action to be taken upon your request, and as | |
502 such, we have decided to abort the request with a nonzero status code. We | |
503 hope that your larger goals have not been put at risk due to this | |
504 unfortunate circumstance, and wish you the best in deciding the next action | |
505 in your venture and larger life. | |
506 | |
507 Warmly, | |
508 recipes.py | |
509 """ | |
510 return 1 | |
511 | |
512 return 0 | |
513 | 229 |
514 if __name__ == '__main__': | 230 if __name__ == '__main__': |
515 # Use os._exit instead of sys.exit to prevent the python interpreter from | 231 # Use os._exit instead of sys.exit to prevent the python interpreter from |
516 # hanging on threads/processes which may have been spawned and not reaped | 232 # hanging on threads/processes which may have been spawned and not reaped |
517 # (e.g. by a leaky test harness). | 233 # (e.g. by a leaky test harness). |
518 try: | 234 try: |
519 ret = main() | 235 ret = main() |
520 except Exception as e: | 236 except Exception as e: |
521 import traceback | 237 import traceback |
522 traceback.print_exc(file=sys.stderr) | 238 traceback.print_exc(file=sys.stderr) |
523 print >> sys.stderr, 'Uncaught exception (%s): %s' % (type(e).__name__, e) | 239 print >> sys.stderr, 'Uncaught exception (%s): %s' % (type(e).__name__, e) |
524 sys.exit(1) | 240 sys.exit(1) |
525 | 241 |
526 if not isinstance(ret, int): | 242 if not isinstance(ret, int): |
527 if ret is None: | 243 if ret is None: |
528 ret = 0 | 244 ret = 0 |
529 else: | 245 else: |
530 print >> sys.stderr, ret | 246 print >> sys.stderr, ret |
531 ret = 1 | 247 ret = 1 |
532 sys.stdout.flush() | 248 sys.stdout.flush() |
533 sys.stderr.flush() | 249 sys.stderr.flush() |
534 os._exit(ret) | 250 os._exit(ret) |
OLD | NEW |