Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1184)

Unified Diff: prebuilt.py

Issue 4102001: Add support for prebuilt uploading using rsync. (Closed) Base URL: ssh://git@gitrw.chromium.org:9222/crosutils.git
Patch Set: Fix nits Created 10 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « no previous file | prebuilt_unittest.py » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: prebuilt.py
diff --git a/prebuilt.py b/prebuilt.py
index 0d962666af073d7023fdba1e3a53b1590c70d533..1d5acacd462c86aee08f0463b9c6a2a051b32e8e 100755
--- a/prebuilt.py
+++ b/prebuilt.py
@@ -13,8 +13,10 @@ import tempfile
from chromite.lib import cros_build_lib
"""
-This script is used to upload host prebuilts as well as board BINHOSTS to
-Google Storage.
+This script is used to upload host prebuilts as well as board BINHOSTS.
+
+If the URL starts with 'gs://', we upload using gsutil to Google Storage.
+Otherwise, rsync is used.
After a build is successfully uploaded a file is updated with the proper
BINHOST version as well as the target board. This file is defined in GIT_FILE
@@ -24,11 +26,14 @@ To read more about prebuilts/binhost binary packages please refer to:
http://sites/chromeos/for-team-members/engineering/releng/prebuilt-binaries-for-streamlining-the-build-process
-Example of uploading prebuilt amd64 host files
+Example of uploading prebuilt amd64 host files to Google Storage:
./prebuilt.py -p /b/cbuild/build -s -u gs://chromeos-prebuilt
-Example of uploading x86-dogfood binhosts
+Example of uploading x86-dogfood binhosts to Google Storage:
./prebuilt.py -b x86-dogfood -p /b/cbuild/build/ -u gs://chromeos-prebuilt -g
+
+Example of uploading prebuilt amd64 host files using rsync:
+./prebuilt.py -p /b/cbuild/build -s -u codf30.jail:/tmp
"""
# as per http://crosbug.com/5855 always filter the below packages
@@ -39,10 +44,10 @@ _HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs'
_HOST_TARGET = 'amd64'
_BOARD_PATH = 'chroot/build/%(board)s'
_BOTO_CONFIG = '/home/chrome-bot/external-boto'
-# board/board-target/version'
-_GS_BOARD_PATH = 'board/%(board)s/%(version)s/'
-# We only support amd64 right now
-_GS_HOST_PATH = 'host/%s' % _HOST_TARGET
+# board/board-target/version/packages/'
+_REL_BOARD_PATH = 'board/%(board)s/%(version)s/packages'
+# host/host-target/version/packages/'
+_REL_HOST_PATH = 'host/%(target)s/%(version)s/packages'
# Private overlays to look at for builds to filter
# relative to build path
_PRIVATE_OVERLAY_DIR = 'src/private-overlays'
@@ -249,6 +254,31 @@ def FilterPackagesFile(packages_filename):
return filtered_packages
+def _RetryRun(cmd, print_cmd=True, shell=False):
+ """Run the specified command, retrying if necessary.
+
+ Args:
+ cmd: The command to run.
+ print_cmd: Whether to print out the cmd.
+ shell: Whether to treat the command as a shell.
+
+ Returns:
+ True if the command succeeded. Otherwise, returns False.
+ """
+
+ # TODO(scottz): port to use _Run or similar when it is available in
+ # cros_build_lib.
+ for attempt in range(_RETRIES):
+ try:
+ output = cros_build_lib.RunCommand(cmd, print_cmd=print_cmd, shell=shell)
+ return True
+ except cros_build_lib.RunCommandError:
+ print 'Failed to run %s' % cmd
+ else:
+ print 'Retry failed run %s, giving up' % cmd
+ return False
+
+
def _GsUpload(args):
"""Upload to GS bucket.
@@ -267,20 +297,8 @@ def _GsUpload(args):
local_file = filtered_packages_file.name
cmd = '%s cp -a public-read %s %s' % (_GSUTIL_BIN, local_file, remote_file)
- # TODO(scottz): port to use _Run or similar when it is available in
- # cros_build_lib.
- for attempt in range(_RETRIES):
- try:
- output = cros_build_lib.RunCommand(cmd, print_cmd=False, shell=True)
- break
- except cros_build_lib.RunCommandError:
- print 'Failed to sync %s -> %s, retrying' % (local_file, remote_file)
- else:
- # TODO(scottz): potentially return what failed so we can do something with
- # with it but for now just print an error.
- print 'Retry failed uploading %s -> %s, giving up' % (local_file,
- remote_file)
- return args
+ if not _RetryRun(cmd, print_cmd=False, shell=True):
+ return (local_file, remote_file)
def RemoteUpload(files, pool=10):
@@ -310,14 +328,12 @@ def RemoteUpload(files, pool=10):
pass
-def GenerateUploadDict(local_path, gs_path, strip_str):
+def GenerateUploadDict(local_path, gs_path):
"""Build a dictionary of local remote file key pairs for gsutil to upload.
Args:
local_path: A path to the file on the local hard drive.
gs_path: Path to upload in Google Storage.
- strip_str: String to remove from the local_path so that the relative
- file path can be tacked on to the gs_path.
Returns:
Returns a dictionary of file path/gs_dest_path pairs
@@ -325,7 +341,7 @@ def GenerateUploadDict(local_path, gs_path, strip_str):
files_to_sync = cros_build_lib.ListFiles(local_path)
upload_files = {}
for file_path in files_to_sync:
- filename = file_path.replace(strip_str, '').lstrip('/')
+ filename = file_path.replace(local_path, '').lstrip('/')
gs_file_path = os.path.join(gs_path, filename)
upload_files[file_path] = gs_file_path
@@ -361,12 +377,13 @@ def DetermineMakeConfFile(target):
return os.path.join(make_path)
-def UploadPrebuilt(build_path, bucket, version, board=None, git_sync=False):
+def UploadPrebuilt(build_path, upload_location, version, binhost_base_url,
+ board=None, git_sync=False):
"""Upload Host prebuilt files to Google Storage space.
Args:
build_path: The path to the root of the chroot.
- bucket: The Google Storage bucket to upload to.
+ upload_location: The upload location.
board: The board to upload to Google Storage, if this is None upload
host packages.
git_sync: If set, update make.conf of target to reference the latest
@@ -377,31 +394,35 @@ def UploadPrebuilt(build_path, bucket, version, board=None, git_sync=False):
# We are uploading host packages
# TODO(scottz): eventually add support for different host_targets
package_path = os.path.join(build_path, _HOST_PACKAGES_PATH)
- gs_path = os.path.join(bucket, _GS_HOST_PATH, version)
- strip_pattern = package_path
+ url_suffix = _REL_HOST_PATH % {'version': version, 'target': _HOST_TARGET}
package_string = _HOST_TARGET
git_file = os.path.join(build_path, _PREBUILT_MAKE_CONF[_HOST_TARGET])
- url_suffix = '%s/%s/' % (_GS_HOST_PATH, version)
else:
board_path = os.path.join(build_path, _BOARD_PATH % {'board': board})
package_path = os.path.join(board_path, 'packages')
package_string = board
- strip_pattern = board_path
- remote_board_path = _GS_BOARD_PATH % {'board': board, 'version': version}
- gs_path = os.path.join(bucket, remote_board_path)
+ url_suffix = _REL_BOARD_PATH % {'board': board, 'version': version}
git_file = os.path.join(build_path, DetermineMakeConfFile(board))
- url_suffix = remote_board_path
+ remote_location = os.path.join(upload_location, url_suffix)
- upload_files = GenerateUploadDict(package_path, gs_path, strip_pattern)
+ if upload_location.startswith('gs://'):
+ upload_files = GenerateUploadDict(package_path, remote_location)
- print 'Uploading %s' % package_string
- failed_uploads = RemoteUpload(upload_files)
- if len(failed_uploads) > 1 or (None not in failed_uploads):
- error_msg = ['%s -> %s\n' % args for args in failed_uploads]
- raise UploadFailed('Error uploading:\n%s' % error_msg)
+ print 'Uploading %s' % package_string
+ failed_uploads = RemoteUpload(upload_files)
+ if len(failed_uploads) > 1 or (None not in failed_uploads):
+ error_msg = ['%s -> %s\n' % args for args in failed_uploads]
+ raise UploadFailed('Error uploading:\n%s' % error_msg)
+ else:
+ ssh_server, remote_path = remote_location.split(':', 1)
+ cmds = ['ssh %s mkdir -p %s' % (ssh_server, remote_path),
+ 'rsync -av %s/ %s/' % (package_path, remote_location)]
+ for cmd in cmds:
+ if not _RetryRun(cmd, shell=True):
+ raise UploadFailed('Could not run %s' % cmd)
if git_sync:
- url_value = '%s/%s' % (_BINHOST_BASE_URL, url_suffix)
+ url_value = '%s/%s/' % (binhost_base_url, url_suffix)
RevGitFile(git_file, url_value)
@@ -414,6 +435,9 @@ def usage(parser, msg):
def main():
parser = optparse.OptionParser()
+ parser.add_option('-H', '--binhost-base-url', dest='binhost_base_url',
+ default=_BINHOST_BASE_URL,
+ help='Base URL to use for binhost in make.conf updates')
parser.add_option('-b', '--board', dest='board', default=None,
help='Board type that was built on this machine')
parser.add_option('-p', '--build-path', dest='build_path',
@@ -426,7 +450,7 @@ def main():
help='Enable git version sync (This commits to a repo)')
parser.add_option('-u', '--upload', dest='upload',
default=None,
- help='Upload to GS bucket')
+ help='Upload location')
parser.add_option('-V', '--prepend-version', dest='prepend_version',
default=None,
help='Add an identifier to the front of the version')
@@ -441,9 +465,13 @@ def main():
usage(parser, 'Error: you need provide a chroot path')
if not options.upload:
- usage(parser, 'Error: you need to provide a gsutil upload bucket -u')
+ usage(parser, 'Error: you need to provide an upload location using -u')
if options.filters:
+ # TODO(davidjames): It might be nice to be able to filter private ebuilds
+ # from rsync uploads as well, some day. But for now it's not needed.
+ if not options.upload.startswith("gs://"):
+ usage(parser, 'Error: filtering only works with gs:// paths')
LoadPrivateFilters(options.build_path)
version = GetVersion()
@@ -452,11 +480,12 @@ def main():
if options.sync_host:
UploadPrebuilt(options.build_path, options.upload, version,
- git_sync=options.git_sync)
+ options.binhost_base_url, git_sync=options.git_sync)
if options.board:
UploadPrebuilt(options.build_path, options.upload, version,
- board=options.board, git_sync=options.git_sync)
+ options.binhost_base_url, board=options.board,
+ git_sync=options.git_sync)
if __name__ == '__main__':
« no previous file with comments | « no previous file | prebuilt_unittest.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698