Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(292)

Side by Side Diff: client/run_isolated.py

Issue 2411683003: run_isolated: rename cache to isolate_cache (Closed)
Patch Set: unlint Created 4 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2012 The LUCI Authors. All rights reserved. 2 # Copyright 2012 The LUCI Authors. All rights reserved.
3 # Use of this source code is governed under the Apache License, Version 2.0 3 # Use of this source code is governed under the Apache License, Version 2.0
4 # that can be found in the LICENSE file. 4 # that can be found in the LICENSE file.
5 5
6 """Runs a command with optional isolated input/output. 6 """Runs a command with optional isolated input/output.
7 7
8 Despite name "run_isolated", can run a generic non-isolated command specified as 8 Despite name "run_isolated", can run a generic non-isolated command specified as
9 args. 9 args.
10 10
(...skipping 331 matching lines...) Expand 10 before | Expand all | Expand 10 after
342 logging.exception('Had difficulties removing out_dir %s: %s', out_dir, e) 342 logging.exception('Had difficulties removing out_dir %s: %s', out_dir, e)
343 stats = { 343 stats = {
344 'duration': time.time() - start, 344 'duration': time.time() - start,
345 'items_cold': base64.b64encode(large.pack(cold)), 345 'items_cold': base64.b64encode(large.pack(cold)),
346 'items_hot': base64.b64encode(large.pack(hot)), 346 'items_hot': base64.b64encode(large.pack(hot)),
347 } 347 }
348 return outputs_ref, success, stats 348 return outputs_ref, success, stats
349 349
350 350
351 def map_and_run( 351 def map_and_run(
352 command, isolated_hash, storage, cache, leak_temp_dir, root_dir, 352 command, isolated_hash, storage, isolate_cache, leak_temp_dir, root_dir,
353 hard_timeout, grace_period, bot_file, extra_args, install_packages_fn, 353 hard_timeout, grace_period, bot_file, extra_args, install_packages_fn,
354 use_symlinks): 354 use_symlinks):
355 """Runs a command with optional isolated input/output. 355 """Runs a command with optional isolated input/output.
356 356
357 See run_tha_test for argument documentation. 357 See run_tha_test for argument documentation.
358 358
359 Returns metadata about the result. 359 Returns metadata about the result.
360 """ 360 """
361 assert root_dir or root_dir is None
361 assert bool(command) ^ bool(isolated_hash) 362 assert bool(command) ^ bool(isolated_hash)
362 result = { 363 result = {
363 'duration': None, 364 'duration': None,
364 'exit_code': None, 365 'exit_code': None,
365 'had_hard_timeout': False, 366 'had_hard_timeout': False,
366 'internal_failure': None, 367 'internal_failure': None,
367 'stats': { 368 'stats': {
368 # 'isolated': { 369 # 'isolated': {
369 # 'cipd': { 370 # 'cipd': {
370 # 'duration': 0., 371 # 'duration': 0.,
(...skipping 19 matching lines...) Expand all
390 # ... 391 # ...
391 # ], 392 # ],
392 # 'client_package': {'package_name': ..., 'version': ...}, 393 # 'client_package': {'package_name': ..., 'version': ...},
393 # }, 394 # },
394 'outputs_ref': None, 395 'outputs_ref': None,
395 'version': 5, 396 'version': 5,
396 } 397 }
397 398
398 if root_dir: 399 if root_dir:
399 file_path.ensure_tree(root_dir, 0700) 400 file_path.ensure_tree(root_dir, 0700)
400 else: 401 elif isolate_cache.cache_dir:
401 root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None 402 root_dir = os.path.dirname(isolate_cache.cache_dir)
402 # See comment for these constants. 403 # See comment for these constants.
403 run_dir = make_temp_dir(ISOLATED_RUN_DIR, root_dir) 404 run_dir = make_temp_dir(ISOLATED_RUN_DIR, root_dir)
404 # storage should be normally set but don't crash if it is not. This can happen 405 # storage should be normally set but don't crash if it is not. This can happen
405 # as Swarming task can run without an isolate server. 406 # as Swarming task can run without an isolate server.
406 out_dir = make_temp_dir(ISOLATED_OUT_DIR, root_dir) if storage else None 407 out_dir = make_temp_dir(ISOLATED_OUT_DIR, root_dir) if storage else None
407 tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, root_dir) 408 tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, root_dir)
408 cwd = run_dir 409 cwd = run_dir
409 410
410 try: 411 try:
411 cipd_info = install_packages_fn(run_dir) 412 cipd_info = install_packages_fn(run_dir)
412 if cipd_info: 413 if cipd_info:
413 result['stats']['cipd'] = cipd_info['stats'] 414 result['stats']['cipd'] = cipd_info['stats']
414 result['cipd_pins'] = cipd_info['cipd_pins'] 415 result['cipd_pins'] = cipd_info['cipd_pins']
415 416
416 if isolated_hash: 417 if isolated_hash:
417 isolated_stats = result['stats'].setdefault('isolated', {}) 418 isolated_stats = result['stats'].setdefault('isolated', {})
418 bundle, isolated_stats['download'] = fetch_and_map( 419 bundle, isolated_stats['download'] = fetch_and_map(
419 isolated_hash=isolated_hash, 420 isolated_hash=isolated_hash,
420 storage=storage, 421 storage=storage,
421 cache=cache, 422 cache=isolate_cache,
422 outdir=run_dir, 423 outdir=run_dir,
423 use_symlinks=use_symlinks) 424 use_symlinks=use_symlinks)
424 if not bundle.command: 425 if not bundle.command:
425 # Handle this as a task failure, not an internal failure. 426 # Handle this as a task failure, not an internal failure.
426 sys.stderr.write( 427 sys.stderr.write(
427 '<The .isolated doesn\'t declare any command to run!>\n' 428 '<The .isolated doesn\'t declare any command to run!>\n'
428 '<Check your .isolate for missing \'command\' variable>\n') 429 '<Check your .isolate for missing \'command\' variable>\n')
429 if os.environ.get('SWARMING_TASK_ID'): 430 if os.environ.get('SWARMING_TASK_ID'):
430 # Give an additional hint when running as a swarming task. 431 # Give an additional hint when running as a swarming task.
431 sys.stderr.write('<This occurs at the \'isolate\' step>\n') 432 sys.stderr.write('<This occurs at the \'isolate\' step>\n')
(...skipping 322 matching lines...) Expand 10 before | Expand all | Expand 10 after
754 auth.add_auth_options(parser) 755 auth.add_auth_options(parser)
755 756
756 parser.set_defaults(cache='cache', cipd_cache='cipd_cache') 757 parser.set_defaults(cache='cache', cipd_cache='cipd_cache')
757 return parser 758 return parser
758 759
759 760
760 def main(args): 761 def main(args):
761 parser = create_option_parser() 762 parser = create_option_parser()
762 options, args = parser.parse_args(args) 763 options, args = parser.parse_args(args)
763 764
764 cache = isolateserver.process_cache_options(options) 765 isolated_cache = isolateserver.process_cache_options(options)
765 if options.clean: 766 if options.clean:
766 if options.isolated: 767 if options.isolated:
767 parser.error('Can\'t use --isolated with --clean.') 768 parser.error('Can\'t use --isolated with --clean.')
768 if options.isolate_server: 769 if options.isolate_server:
769 parser.error('Can\'t use --isolate-server with --clean.') 770 parser.error('Can\'t use --isolate-server with --clean.')
770 if options.json: 771 if options.json:
771 parser.error('Can\'t use --json with --clean.') 772 parser.error('Can\'t use --json with --clean.')
772 cache.cleanup() 773 isolated_cache.cleanup()
773 return 0 774 return 0
774 if not options.no_clean: 775 if not options.no_clean:
775 cache.cleanup() 776 isolated_cache.cleanup()
776 777
777 if not options.isolated and not args: 778 if not options.isolated and not args:
778 parser.error('--isolated or command to run is required.') 779 parser.error('--isolated or command to run is required.')
779 780
780 auth.process_auth_options(parser, options) 781 auth.process_auth_options(parser, options)
781 782
782 isolateserver.process_isolate_server_options( 783 isolateserver.process_isolate_server_options(
783 parser, options, True, False) 784 parser, options, True, False)
784 if not options.isolate_server: 785 if not options.isolate_server:
785 if options.isolated: 786 if options.isolated:
(...skipping 13 matching lines...) Expand all
799 run_dir, cipd.parse_package_args(options.cipd_packages), 800 run_dir, cipd.parse_package_args(options.cipd_packages),
800 options.cipd_server, options.cipd_client_package, 801 options.cipd_server, options.cipd_client_package,
801 options.cipd_client_version, cache_dir=options.cipd_cache) 802 options.cipd_client_version, cache_dir=options.cipd_cache)
802 803
803 try: 804 try:
804 command = [] if options.isolated else args 805 command = [] if options.isolated else args
805 if options.isolate_server: 806 if options.isolate_server:
806 storage = isolateserver.get_storage( 807 storage = isolateserver.get_storage(
807 options.isolate_server, options.namespace) 808 options.isolate_server, options.namespace)
808 with storage: 809 with storage:
809 # Hashing schemes used by |storage| and |cache| MUST match. 810 # Hashing schemes used by |storage| and |isolated_cache| MUST match.
810 assert storage.hash_algo == cache.hash_algo 811 assert storage.hash_algo == isolated_cache.hash_algo
811 return run_tha_test( 812 return run_tha_test(
812 command, options.isolated, storage, cache, options.leak_temp_dir, 813 command, options.isolated, storage, isolated_cache,
813 options.json, options.root_dir, options.hard_timeout, 814 options.leak_temp_dir, options.json, options.root_dir,
814 options.grace_period, options.bot_file, args, install_packages_fn, 815 options.hard_timeout, options.grace_period, options.bot_file, args,
815 options.use_symlinks) 816 install_packages_fn, options.use_symlinks)
816 return run_tha_test( 817 return run_tha_test(
817 command, options.isolated, None, cache, options.leak_temp_dir, 818 command, options.isolated, None, isolated_cache, options.leak_temp_dir,
818 options.json, options.root_dir, options.hard_timeout, 819 options.json, options.root_dir, options.hard_timeout,
819 options.grace_period, options.bot_file, args, install_packages_fn, 820 options.grace_period, options.bot_file, args, install_packages_fn,
820 options.use_symlinks) 821 options.use_symlinks)
821 except cipd.Error as ex: 822 except cipd.Error as ex:
822 print >> sys.stderr, ex.message 823 print >> sys.stderr, ex.message
823 return 1 824 return 1
824 825
825 826
826 if __name__ == '__main__': 827 if __name__ == '__main__':
827 subprocess42.inhibit_os_error_reporting() 828 subprocess42.inhibit_os_error_reporting()
828 # Ensure that we are always running with the correct encoding. 829 # Ensure that we are always running with the correct encoding.
829 fix_encoding.fix_encoding() 830 fix_encoding.fix_encoding()
830 file_path.enable_symlink() 831 file_path.enable_symlink()
831 sys.exit(main(sys.argv[1:])) 832 sys.exit(main(sys.argv[1:]))
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698