Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(100)

Unified Diff: buildbot/prebuilt.py

Issue 6851021: Update prebuilt.py to support uploading a board tarball to Google Storage. (Closed) Base URL: http://git.chromium.org/git/chromite.git@master
Patch Set: Add more checks Created 9 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « no previous file | buildbot/prebuilt_unittest.py » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: buildbot/prebuilt.py
diff --git a/buildbot/prebuilt.py b/buildbot/prebuilt.py
index cf93078d845d638e50f6ca8d3445991cc719d2cf..c0299c74c4a8e792e29f5c4cd04b0c840cae7037 100755
--- a/buildbot/prebuilt.py
+++ b/buildbot/prebuilt.py
@@ -51,10 +51,10 @@ _HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs'
_CATEGORIES_PATH = 'chroot/etc/portage/categories'
_HOST_TARGET = 'amd64'
_BOARD_PATH = 'chroot/build/%(board)s'
-# board/board-target/version/packages/'
-_REL_BOARD_PATH = 'board/%(board)s/%(version)s/packages'
-# host/host-target/version/packages/'
-_REL_HOST_PATH = 'host/%(target)s/%(version)s/packages'
+# board/board-target/version/'
+_REL_BOARD_PATH = 'board/%(board)s/%(version)s'
+# host/host-target/version/'
+_REL_HOST_PATH = 'host/%(target)s/%(version)s'
# Private overlays to look at for builds to filter
# relative to build path
_PRIVATE_OVERLAY_DIR = 'src/private-overlays'
@@ -515,6 +515,28 @@ class PrebuiltUploader(object):
if not _RetryRun(cmd, shell=True, cwd=package_path):
raise UploadFailed('Could not run %s' % cmd)
+ def _UploadBoardTarball(self, board_path, url_suffix):
+ """Upload a tarball of the board at the specified path to Google Storage.
+
scottz 2011/04/15 17:40:46 Args:
davidjames 2011/04/15 19:18:08 Done.
+ board_path: The path to the board dir.
+ url_suffix: The remote subdirectory where we should upload the packages.
+ """
+ remote_location = '%s/%s' % (self._upload_location.rstrip('/'), url_suffix)
+ assert remote_location.startswith('gs://')
+ cwd, boardname = os.path.split(board_path.rstrip(os.path.sep))
+ tarfile = '/tmp/%s.tbz2' % boardname
scottz 2011/04/15 17:40:46 You should use tempfile here. I prefer not to ass
davidjames 2011/04/15 19:18:08 Done. Now that we switched to tempfile, it will h
+ cmd = ['sudo', 'tar', '-I', 'pbzip2', '-cf', tarfile]
+ for path in ('usr/lib/debug', 'usr/local/autotest', 'packages', 'tmp'):
scottz 2011/04/15 17:40:46 Assigning these to a variable excluded_paths would
davidjames 2011/04/15 19:18:08 Done.
+ cmd.append('--exclude=%s/%s/*' % (boardname, path))
+ cmd.append(boardname)
+ cros_build_lib.RunCommand(cmd, cwd=cwd)
scottz 2011/04/15 17:40:46 this should be in your try, if tar fails we are le
davidjames 2011/04/15 19:18:08 Done.
+ remote_tarfile = '%s/%s.tbz2' % (remote_location.rstrip('/'), boardname)
+ try:
+ if _GsUpload((tarfile, remote_tarfile, self._acl)):
+ sys.exit(1)
+ finally:
+ cros_build_lib.RunCommand(['sudo', 'rm', '-f', tarfile], cwd=cwd)
+
def _SyncHostPrebuilts(self, build_path, version, key, git_sync,
sync_binhost_conf):
"""Synchronize host prebuilt files.
@@ -540,7 +562,8 @@ class PrebuiltUploader(object):
# Upload prebuilts.
package_path = os.path.join(build_path, _HOST_PACKAGES_PATH)
url_suffix = _REL_HOST_PATH % {'version': version, 'target': _HOST_TARGET}
- self._UploadPrebuilt(package_path, url_suffix)
+ packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
+ self._UploadPrebuilt(package_path, packages_url_suffix)
# Record URL where prebuilts were uploaded.
url_value = '%s/%s/' % (self._binhost_base_url.rstrip('/'),
@@ -554,7 +577,7 @@ class PrebuiltUploader(object):
UpdateBinhostConfFile(binhost_conf, key, url_value)
def _SyncBoardPrebuilts(self, board, build_path, version, key, git_sync,
- sync_binhost_conf):
+ sync_binhost_conf, upload_board_tarball):
"""Synchronize board prebuilt files.
Args:
@@ -567,12 +590,24 @@ class PrebuiltUploader(object):
prebuilt packages generated here.
sync_binhost_conf: If set, update binhost config file in
chromiumos-overlay for the current board.
+ upload_board_tarball: Include a tarball of the board in our upload.
"""
# Upload prebuilts.
board_path = os.path.join(build_path, _BOARD_PATH % {'board': board})
package_path = os.path.join(board_path, 'packages')
url_suffix = _REL_BOARD_PATH % {'board': board, 'version': version}
- self._UploadPrebuilt(package_path, url_suffix)
+ packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
+
+ if upload_board_tarball:
scottz 2011/04/15 17:40:46 Please add a comment about you uploading board tar
davidjames 2011/04/15 19:18:08 Done.
+ p = multiprocessing.Process(target=self._UploadBoardTarball,
scottz 2011/04/15 17:40:46 Please consider using something other than p. tar_
davidjames 2011/04/15 19:18:08 Done.
+ args=(board_path, url_suffix))
+ p.start()
+
+ self._UploadPrebuilt(package_path, packages_url_suffix)
+
+ if upload_board_tarball:
+ p.join()
+ assert p.exitcode == 0
# Record URL where prebuilts were uploaded.
url_value = '%s/%s/' % (self._binhost_base_url.rstrip('/'),
@@ -627,6 +662,9 @@ def ParseOptions():
help='Update binhost.conf')
parser.add_option('-P', '--private', dest='private', action='store_true',
default=False, help='Mark gs:// uploads as private.')
+ parser.add_option('', '--upload-board-tarball', dest='upload_board_tarball',
+ action='store_true', default=False,
+ help='Upload board tarball to Google Storage.')
options, args = parser.parse_args()
if not options.build_path:
@@ -634,6 +672,11 @@ def ParseOptions():
if not options.upload:
usage(parser, 'Error: you need to provide an upload location using -u')
+
+ if options.upload_board_tarball and not options.upload.startswith('gs://'):
+ usage(parser, 'Error: --upload-board-tarball only works with gs:// URLs.\n'
+ '--upload must be a gs:// URL.')
+
if options.private:
if options.sync_host:
usage(parser, 'Error: --private and --sync-host/-s cannot be specified '
@@ -684,7 +727,8 @@ def main():
if options.board:
uploader._SyncBoardPrebuilts(options.board, options.build_path, version,
options.key, options.git_sync,
- options.sync_binhost_conf)
+ options.sync_binhost_conf,
+ options.upload_board_tarball)
if __name__ == '__main__':
main()
« no previous file with comments | « no previous file | buildbot/prebuilt_unittest.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698