| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright 2012 The LUCI Authors. All rights reserved. | 2 # Copyright 2012 The LUCI Authors. All rights reserved. |
| 3 # Use of this source code is governed under the Apache License, Version 2.0 | 3 # Use of this source code is governed under the Apache License, Version 2.0 |
| 4 # that can be found in the LICENSE file. | 4 # that can be found in the LICENSE file. |
| 5 | 5 |
| 6 """Runs a command with optional isolated input/output. | 6 """Runs a command with optional isolated input/output. |
| 7 | 7 |
| 8 Despite name "run_isolated", can run a generic non-isolated command specified as | 8 Despite name "run_isolated", can run a generic non-isolated command specified as |
| 9 args. | 9 args. |
| 10 | 10 |
| (...skipping 331 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 342 logging.exception('Had difficulties removing out_dir %s: %s', out_dir, e) | 342 logging.exception('Had difficulties removing out_dir %s: %s', out_dir, e) |
| 343 stats = { | 343 stats = { |
| 344 'duration': time.time() - start, | 344 'duration': time.time() - start, |
| 345 'items_cold': base64.b64encode(large.pack(cold)), | 345 'items_cold': base64.b64encode(large.pack(cold)), |
| 346 'items_hot': base64.b64encode(large.pack(hot)), | 346 'items_hot': base64.b64encode(large.pack(hot)), |
| 347 } | 347 } |
| 348 return outputs_ref, success, stats | 348 return outputs_ref, success, stats |
| 349 | 349 |
| 350 | 350 |
| 351 def map_and_run( | 351 def map_and_run( |
| 352 command, isolated_hash, storage, cache, leak_temp_dir, root_dir, | 352 command, isolated_hash, storage, isolate_cache, leak_temp_dir, root_dir, |
| 353 hard_timeout, grace_period, bot_file, extra_args, install_packages_fn, | 353 hard_timeout, grace_period, bot_file, extra_args, install_packages_fn, |
| 354 use_symlinks): | 354 use_symlinks): |
| 355 """Runs a command with optional isolated input/output. | 355 """Runs a command with optional isolated input/output. |
| 356 | 356 |
| 357 See run_tha_test for argument documentation. | 357 See run_tha_test for argument documentation. |
| 358 | 358 |
| 359 Returns metadata about the result. | 359 Returns metadata about the result. |
| 360 """ | 360 """ |
| 361 assert root_dir or root_dir is None |
| 361 assert bool(command) ^ bool(isolated_hash) | 362 assert bool(command) ^ bool(isolated_hash) |
| 362 result = { | 363 result = { |
| 363 'duration': None, | 364 'duration': None, |
| 364 'exit_code': None, | 365 'exit_code': None, |
| 365 'had_hard_timeout': False, | 366 'had_hard_timeout': False, |
| 366 'internal_failure': None, | 367 'internal_failure': None, |
| 367 'stats': { | 368 'stats': { |
| 368 # 'isolated': { | 369 # 'isolated': { |
| 369 # 'cipd': { | 370 # 'cipd': { |
| 370 # 'duration': 0., | 371 # 'duration': 0., |
| (...skipping 19 matching lines...) Expand all Loading... |
| 390 # ... | 391 # ... |
| 391 # ], | 392 # ], |
| 392 # 'client_package': {'package_name': ..., 'version': ...}, | 393 # 'client_package': {'package_name': ..., 'version': ...}, |
| 393 # }, | 394 # }, |
| 394 'outputs_ref': None, | 395 'outputs_ref': None, |
| 395 'version': 5, | 396 'version': 5, |
| 396 } | 397 } |
| 397 | 398 |
| 398 if root_dir: | 399 if root_dir: |
| 399 file_path.ensure_tree(root_dir, 0700) | 400 file_path.ensure_tree(root_dir, 0700) |
| 400 else: | 401 elif isolate_cache.cache_dir: |
| 401 root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None | 402 root_dir = os.path.dirname(isolate_cache.cache_dir) |
| 402 # See comment for these constants. | 403 # See comment for these constants. |
| 403 run_dir = make_temp_dir(ISOLATED_RUN_DIR, root_dir) | 404 run_dir = make_temp_dir(ISOLATED_RUN_DIR, root_dir) |
| 404 # storage should be normally set but don't crash if it is not. This can happen | 405 # storage should be normally set but don't crash if it is not. This can happen |
| 405 # as Swarming task can run without an isolate server. | 406 # as Swarming task can run without an isolate server. |
| 406 out_dir = make_temp_dir(ISOLATED_OUT_DIR, root_dir) if storage else None | 407 out_dir = make_temp_dir(ISOLATED_OUT_DIR, root_dir) if storage else None |
| 407 tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, root_dir) | 408 tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, root_dir) |
| 408 cwd = run_dir | 409 cwd = run_dir |
| 409 | 410 |
| 410 try: | 411 try: |
| 411 cipd_info = install_packages_fn(run_dir) | 412 cipd_info = install_packages_fn(run_dir) |
| 412 if cipd_info: | 413 if cipd_info: |
| 413 result['stats']['cipd'] = cipd_info['stats'] | 414 result['stats']['cipd'] = cipd_info['stats'] |
| 414 result['cipd_pins'] = cipd_info['cipd_pins'] | 415 result['cipd_pins'] = cipd_info['cipd_pins'] |
| 415 | 416 |
| 416 if isolated_hash: | 417 if isolated_hash: |
| 417 isolated_stats = result['stats'].setdefault('isolated', {}) | 418 isolated_stats = result['stats'].setdefault('isolated', {}) |
| 418 bundle, isolated_stats['download'] = fetch_and_map( | 419 bundle, isolated_stats['download'] = fetch_and_map( |
| 419 isolated_hash=isolated_hash, | 420 isolated_hash=isolated_hash, |
| 420 storage=storage, | 421 storage=storage, |
| 421 cache=cache, | 422 cache=isolate_cache, |
| 422 outdir=run_dir, | 423 outdir=run_dir, |
| 423 use_symlinks=use_symlinks) | 424 use_symlinks=use_symlinks) |
| 424 if not bundle.command: | 425 if not bundle.command: |
| 425 # Handle this as a task failure, not an internal failure. | 426 # Handle this as a task failure, not an internal failure. |
| 426 sys.stderr.write( | 427 sys.stderr.write( |
| 427 '<The .isolated doesn\'t declare any command to run!>\n' | 428 '<The .isolated doesn\'t declare any command to run!>\n' |
| 428 '<Check your .isolate for missing \'command\' variable>\n') | 429 '<Check your .isolate for missing \'command\' variable>\n') |
| 429 if os.environ.get('SWARMING_TASK_ID'): | 430 if os.environ.get('SWARMING_TASK_ID'): |
| 430 # Give an additional hint when running as a swarming task. | 431 # Give an additional hint when running as a swarming task. |
| 431 sys.stderr.write('<This occurs at the \'isolate\' step>\n') | 432 sys.stderr.write('<This occurs at the \'isolate\' step>\n') |
| (...skipping 322 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 754 auth.add_auth_options(parser) | 755 auth.add_auth_options(parser) |
| 755 | 756 |
| 756 parser.set_defaults(cache='cache', cipd_cache='cipd_cache') | 757 parser.set_defaults(cache='cache', cipd_cache='cipd_cache') |
| 757 return parser | 758 return parser |
| 758 | 759 |
| 759 | 760 |
| 760 def main(args): | 761 def main(args): |
| 761 parser = create_option_parser() | 762 parser = create_option_parser() |
| 762 options, args = parser.parse_args(args) | 763 options, args = parser.parse_args(args) |
| 763 | 764 |
| 764 cache = isolateserver.process_cache_options(options) | 765 isolated_cache = isolateserver.process_cache_options(options) |
| 765 if options.clean: | 766 if options.clean: |
| 766 if options.isolated: | 767 if options.isolated: |
| 767 parser.error('Can\'t use --isolated with --clean.') | 768 parser.error('Can\'t use --isolated with --clean.') |
| 768 if options.isolate_server: | 769 if options.isolate_server: |
| 769 parser.error('Can\'t use --isolate-server with --clean.') | 770 parser.error('Can\'t use --isolate-server with --clean.') |
| 770 if options.json: | 771 if options.json: |
| 771 parser.error('Can\'t use --json with --clean.') | 772 parser.error('Can\'t use --json with --clean.') |
| 772 cache.cleanup() | 773 isolated_cache.cleanup() |
| 773 return 0 | 774 return 0 |
| 774 if not options.no_clean: | 775 if not options.no_clean: |
| 775 cache.cleanup() | 776 isolated_cache.cleanup() |
| 776 | 777 |
| 777 if not options.isolated and not args: | 778 if not options.isolated and not args: |
| 778 parser.error('--isolated or command to run is required.') | 779 parser.error('--isolated or command to run is required.') |
| 779 | 780 |
| 780 auth.process_auth_options(parser, options) | 781 auth.process_auth_options(parser, options) |
| 781 | 782 |
| 782 isolateserver.process_isolate_server_options( | 783 isolateserver.process_isolate_server_options( |
| 783 parser, options, True, False) | 784 parser, options, True, False) |
| 784 if not options.isolate_server: | 785 if not options.isolate_server: |
| 785 if options.isolated: | 786 if options.isolated: |
| (...skipping 13 matching lines...) Expand all Loading... |
| 799 run_dir, cipd.parse_package_args(options.cipd_packages), | 800 run_dir, cipd.parse_package_args(options.cipd_packages), |
| 800 options.cipd_server, options.cipd_client_package, | 801 options.cipd_server, options.cipd_client_package, |
| 801 options.cipd_client_version, cache_dir=options.cipd_cache) | 802 options.cipd_client_version, cache_dir=options.cipd_cache) |
| 802 | 803 |
| 803 try: | 804 try: |
| 804 command = [] if options.isolated else args | 805 command = [] if options.isolated else args |
| 805 if options.isolate_server: | 806 if options.isolate_server: |
| 806 storage = isolateserver.get_storage( | 807 storage = isolateserver.get_storage( |
| 807 options.isolate_server, options.namespace) | 808 options.isolate_server, options.namespace) |
| 808 with storage: | 809 with storage: |
| 809 # Hashing schemes used by |storage| and |cache| MUST match. | 810 # Hashing schemes used by |storage| and |isolated_cache| MUST match. |
| 810 assert storage.hash_algo == cache.hash_algo | 811 assert storage.hash_algo == isolated_cache.hash_algo |
| 811 return run_tha_test( | 812 return run_tha_test( |
| 812 command, options.isolated, storage, cache, options.leak_temp_dir, | 813 command, options.isolated, storage, isolated_cache, |
| 813 options.json, options.root_dir, options.hard_timeout, | 814 options.leak_temp_dir, options.json, options.root_dir, |
| 814 options.grace_period, options.bot_file, args, install_packages_fn, | 815 options.hard_timeout, options.grace_period, options.bot_file, args, |
| 815 options.use_symlinks) | 816 install_packages_fn, options.use_symlinks) |
| 816 return run_tha_test( | 817 return run_tha_test( |
| 817 command, options.isolated, None, cache, options.leak_temp_dir, | 818 command, options.isolated, None, isolated_cache, options.leak_temp_dir, |
| 818 options.json, options.root_dir, options.hard_timeout, | 819 options.json, options.root_dir, options.hard_timeout, |
| 819 options.grace_period, options.bot_file, args, install_packages_fn, | 820 options.grace_period, options.bot_file, args, install_packages_fn, |
| 820 options.use_symlinks) | 821 options.use_symlinks) |
| 821 except cipd.Error as ex: | 822 except cipd.Error as ex: |
| 822 print >> sys.stderr, ex.message | 823 print >> sys.stderr, ex.message |
| 823 return 1 | 824 return 1 |
| 824 | 825 |
| 825 | 826 |
| 826 if __name__ == '__main__': | 827 if __name__ == '__main__': |
| 827 subprocess42.inhibit_os_error_reporting() | 828 subprocess42.inhibit_os_error_reporting() |
| 828 # Ensure that we are always running with the correct encoding. | 829 # Ensure that we are always running with the correct encoding. |
| 829 fix_encoding.fix_encoding() | 830 fix_encoding.fix_encoding() |
| 830 file_path.enable_symlink() | 831 file_path.enable_symlink() |
| 831 sys.exit(main(sys.argv[1:])) | 832 sys.exit(main(sys.argv[1:])) |
| OLD | NEW |