Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(236)

Side by Side Diff: scripts/slave/slave_utils.py

Issue 2442693003: Upload retry summary to desired location instead of uploading then moving. (Closed)
Patch Set: Change slave_utils.GSUtilCopyFile to take an optional dest filename. Created 4 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Functions specific to build slaves, shared by several buildbot scripts. 5 """Functions specific to build slaves, shared by several buildbot scripts.
6 """ 6 """
7 7
8 import datetime 8 import datetime
9 import glob 9 import glob
10 import os 10 import os
(...skipping 370 matching lines...) Expand 10 before | Expand all | Expand 10 after
381 return '%s-%s' % (provider_prefix, name) 381 return '%s-%s' % (provider_prefix, name)
382 382
383 383
384 def GSUtilCopy(source, dest, mimetype=None, gs_acl=None, cache_control=None, 384 def GSUtilCopy(source, dest, mimetype=None, gs_acl=None, cache_control=None,
385 metadata=None, override_gsutil=None): 385 metadata=None, override_gsutil=None):
386 """Copy a file to Google Storage. 386 """Copy a file to Google Storage.
387 387
388 Runs the following command: 388 Runs the following command:
389 gsutil -h Content-Type:<mimetype> \ 389 gsutil -h Content-Type:<mimetype> \
390 -h Cache-Control:<cache_control> \ 390 -h Cache-Control:<cache_control> \
391 cp -a <gs_acl> file://<filename> <gs_base>/<subdir>/<filename w/o path> 391 cp -a <gs_acl> file://<filename> <dest>
392 392
393 Args: 393 Args:
394 source: the source URI 394 source: the source URI
395 dest: the destination URI 395 dest: the destination URI
396 mimetype: optional value to add as a Content-Type header 396 mimetype: optional value to add as a Content-Type header
397 gs_acl: optional value to add as a canned-acl 397 gs_acl: optional value to add as a canned-acl
398 cache_control: optional value to set Cache-Control header 398 cache_control: optional value to set Cache-Control header
399 metadata: (dict) A dictionary of string key/value metadata entries to set 399 metadata: (dict) A dictionary of string key/value metadata entries to set
400 (see `gsutil cp' '-h' option) 400 (see `gsutil cp' '-h' option)
401 override_gsutil (list): optional argv to run gsutil 401 override_gsutil (list): optional argv to run gsutil
402
402 Returns: 403 Returns:
403 The status code returned from running the generated gsutil command. 404 The status code returned from running the generated gsutil command.
404 """ 405 """
405 406
406 if not source.startswith('gs://') and not source.startswith('file://'): 407 if not source.startswith('gs://') and not source.startswith('file://'):
407 source = 'file://' + source 408 source = 'file://' + source
408 if not dest.startswith('gs://') and not dest.startswith('file://'): 409 if not dest.startswith('gs://') and not dest.startswith('file://'):
409 dest = 'file://' + dest 410 dest = 'file://' + dest
410 # The setup also sets up some env variables - for now always run that. 411 # The setup also sets up some env variables - for now always run that.
411 gsutil = GSUtilSetup() 412 gsutil = GSUtilSetup()
(...skipping 12 matching lines...) Expand all
424 param = (field) if v is None else ('%s:%s' % (field, v)) 425 param = (field) if v is None else ('%s:%s' % (field, v))
425 command += ['-h', param] 426 command += ['-h', param]
426 command.extend(['cp']) 427 command.extend(['cp'])
427 if gs_acl: 428 if gs_acl:
428 command.extend(['-a', gs_acl]) 429 command.extend(['-a', gs_acl])
429 command.extend([source, dest]) 430 command.extend([source, dest])
430 return chromium_utils.RunCommand(command) 431 return chromium_utils.RunCommand(command)
431 432
432 433
433 def GSUtilCopyFile(filename, gs_base, subdir=None, mimetype=None, gs_acl=None, 434 def GSUtilCopyFile(filename, gs_base, subdir=None, mimetype=None, gs_acl=None,
434 cache_control=None, metadata=None, override_gsutil=None): 435 cache_control=None, metadata=None, override_gsutil=None,
436 dest_filename=None):
435 """Copy a file to Google Storage. 437 """Copy a file to Google Storage.
436 438
437 Runs the following command: 439 Runs the following command:
438 gsutil -h Content-Type:<mimetype> \ 440 gsutil -h Content-Type:<mimetype> \
439 -h Cache-Control:<cache_control> \ 441 -h Cache-Control:<cache_control> \
440 cp -a <gs_acl> file://<filename> <gs_base>/<subdir>/<filename w/o path> 442 cp -a <gs_acl> file://<filename> <gs_base>/<subdir>/<dest_filename>
441 443
442 Args: 444 Args:
443 filename: the file to upload 445 filename: the file to upload
444 gs_base: the bucket to upload the file to 446 gs_base: the bucket to upload the file to
445 subdir: optional subdirectory withing the bucket 447 subdir: optional subdirectory withing the bucket
446 mimetype: optional value to add as a Content-Type header 448 mimetype: optional value to add as a Content-Type header
447 gs_acl: optional value to add as a canned-acl 449 gs_acl: optional value to add as a canned-acl
448 override_gsutil (list): optional argv to run gsutil 450 override_gsutil (list): optional argv to run gsutil
451 dest_filename: optional destination filename; if not specified, then the
452 destination filename will be the source filename without the path
453
449 Returns: 454 Returns:
450 The status code returned from running the generated gsutil command. 455 The status code returned from running the generated gsutil command.
451 """ 456 """
452 457
453 source = 'file://' + filename 458 source = 'file://' + filename
454 dest = gs_base 459 dest = gs_base
455 if subdir: 460 if subdir:
456 # HACK(nsylvain): We can't use normpath here because it will break the 461 # HACK(nsylvain): We can't use normpath here because it will break the
457 # slashes on Windows. 462 # slashes on Windows.
458 if subdir == '..': 463 if subdir == '..':
459 dest = os.path.dirname(gs_base) 464 dest = os.path.dirname(gs_base)
460 else: 465 else:
461 dest = '/'.join([gs_base, subdir]) 466 dest = '/'.join([gs_base, subdir])
462 dest = '/'.join([dest, os.path.basename(filename)]) 467 if dest_filename is None:
468 dest_filename = os.path.basename(filename)
469 dest = '/'.join([dest, dest_filename])
463 return GSUtilCopy(source, dest, mimetype, gs_acl, cache_control, 470 return GSUtilCopy(source, dest, mimetype, gs_acl, cache_control,
464 metadata=metadata, override_gsutil=override_gsutil) 471 metadata=metadata, override_gsutil=override_gsutil)
465 472
466 473
467 def GSUtilCopyDir(src_dir, gs_base, dest_dir=None, gs_acl=None, 474 def GSUtilCopyDir(src_dir, gs_base, dest_dir=None, gs_acl=None,
468 cache_control=None): 475 cache_control=None):
469 """Upload the directory and its contents to Google Storage.""" 476 """Upload the directory and its contents to Google Storage."""
470 477
471 if os.path.isfile(src_dir): 478 if os.path.isfile(src_dir):
472 assert os.path.isdir(src_dir), '%s must be a directory' % src_dir 479 assert os.path.isdir(src_dir), '%s must be a directory' % src_dir
(...skipping 432 matching lines...) Expand 10 before | Expand all | Expand 10 after
905 dir_path: The directory path to be tested. 912 dir_path: The directory path to be tested.
906 913
907 Returns: 914 Returns:
908 True if given directory is in a git repository, False otherwise. 915 True if given directory is in a git repository, False otherwise.
909 """ 916 """
910 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' 917 git_exe = 'git.bat' if sys.platform.startswith('win') else 'git'
911 with open(os.devnull, 'w') as devnull: 918 with open(os.devnull, 'w') as devnull:
912 p = subprocess.Popen([git_exe, 'rev-parse', '--git-dir'], 919 p = subprocess.Popen([git_exe, 'rev-parse', '--git-dir'],
913 cwd=dir_path, stdout=devnull, stderr=devnull) 920 cwd=dir_path, stdout=devnull, stderr=devnull)
914 return p.wait() == 0 921 return p.wait() == 0
OLDNEW
« no previous file with comments | « scripts/slave/chromium/archive_layout_test_retry_summary.py ('k') | scripts/slave/unittests/slave_utils_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698