Update prebuilt.py to support uploading a board tarball to Google Storage.

buildbot/prebuilt.py

Index: buildbot/prebuilt.py
diff --git a/buildbot/prebuilt.py b/buildbot/prebuilt.py
index cf93078d845d638e50f6ca8d3445991cc719d2cf..c0299c74c4a8e792e29f5c4cd04b0c840cae7037 100755
--- a/buildbot/prebuilt.py
+++ b/buildbot/prebuilt.py
@@ -51,10 +51,10 @@ _HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs'
 _CATEGORIES_PATH = 'chroot/etc/portage/categories'
 _HOST_TARGET = 'amd64'
 _BOARD_PATH = 'chroot/build/%(board)s'
-# board/board-target/version/packages/'
-_REL_BOARD_PATH = 'board/%(board)s/%(version)s/packages'
-# host/host-target/version/packages/'
-_REL_HOST_PATH = 'host/%(target)s/%(version)s/packages'
+# board/board-target/version/'
+_REL_BOARD_PATH = 'board/%(board)s/%(version)s'
+# host/host-target/version/'
+_REL_HOST_PATH = 'host/%(target)s/%(version)s'
 # Private overlays to look at for builds to filter
 # relative to build path
 _PRIVATE_OVERLAY_DIR = 'src/private-overlays'
@@ -515,6 +515,28 @@ class PrebuiltUploader(object):
         if not _RetryRun(cmd, shell=True, cwd=package_path):
           raise UploadFailed('Could not run %s' % cmd)
 
+  def _UploadBoardTarball(self, board_path, url_suffix):
+    """Upload a tarball of the board at the specified path to Google Storage.
+

[scottz, 2011/04/15 17:40:46]

Args:

[davidjames, 2011/04/15 19:18:08]

Done.

+      board_path: The path to the board dir.
+      url_suffix: The remote subdirectory where we should upload the packages.
+    """
+    remote_location = '%s/%s' % (self._upload_location.rstrip('/'), url_suffix)
+    assert remote_location.startswith('gs://')
+    cwd, boardname = os.path.split(board_path.rstrip(os.path.sep))
+    tarfile = '/tmp/%s.tbz2' % boardname

[scottz, 2011/04/15 17:40:46]

You should use tempfile here.

I prefer not to assume we can dump gigs of files in /tmp. I would much rather
you create a temporary directory in the dir where prebuilt is executed and wipe
it if anything bad happens. 

[davidjames, 2011/04/15 19:18:08]

Done.

Now that we switched to tempfile, it will honor the TMPDIR / TEMP / TMP
environment variables and so if the buildbot sets those variables, Python will
use one of those directories.

+    cmd = ['sudo', 'tar', '-I', 'pbzip2', '-cf', tarfile]
+    for path in ('usr/lib/debug', 'usr/local/autotest', 'packages', 'tmp'):

[scottz, 2011/04/15 17:40:46]

Assigning these to a variable excluded_paths would make it a lot more clear what
you are doing here at the cost of a var added. 

[davidjames, 2011/04/15 19:18:08]

Done.

+      cmd.append('--exclude=%s/%s/*' % (boardname, path))
+    cmd.append(boardname)
+    cros_build_lib.RunCommand(cmd, cwd=cwd)

[scottz, 2011/04/15 17:40:46]

this should be in your try, if tar fails we are left with a tarball in /tmp,
this could fail easily if /tmp has no more space. 

[davidjames, 2011/04/15 19:18:08]

Done.

+    remote_tarfile = '%s/%s.tbz2' % (remote_location.rstrip('/'), boardname)
+    try:
+      if _GsUpload((tarfile, remote_tarfile, self._acl)):
+        sys.exit(1)
+    finally:
+      cros_build_lib.RunCommand(['sudo', 'rm', '-f', tarfile], cwd=cwd)
+
   def _SyncHostPrebuilts(self, build_path, version, key, git_sync,
                         sync_binhost_conf):
     """Synchronize host prebuilt files.
@@ -540,7 +562,8 @@ class PrebuiltUploader(object):
     # Upload prebuilts.
     package_path = os.path.join(build_path, _HOST_PACKAGES_PATH)
     url_suffix = _REL_HOST_PATH % {'version': version, 'target': _HOST_TARGET}
-    self._UploadPrebuilt(package_path, url_suffix)
+    packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
+    self._UploadPrebuilt(package_path, packages_url_suffix)
 
     # Record URL where prebuilts were uploaded.
     url_value = '%s/%s/' % (self._binhost_base_url.rstrip('/'),
@@ -554,7 +577,7 @@ class PrebuiltUploader(object):
       UpdateBinhostConfFile(binhost_conf, key, url_value)
 
   def _SyncBoardPrebuilts(self, board, build_path, version, key, git_sync,
-                         sync_binhost_conf):
+                          sync_binhost_conf, upload_board_tarball):
     """Synchronize board prebuilt files.
 
     Args:
@@ -567,12 +590,24 @@ class PrebuiltUploader(object):
           prebuilt packages generated here.
       sync_binhost_conf: If set, update binhost config file in
           chromiumos-overlay for the current board.
+      upload_board_tarball: Include a tarball of the board in our upload.
     """
     # Upload prebuilts.
     board_path = os.path.join(build_path, _BOARD_PATH % {'board': board})
     package_path = os.path.join(board_path, 'packages')
     url_suffix = _REL_BOARD_PATH % {'board': board, 'version': version}
-    self._UploadPrebuilt(package_path, url_suffix)
+    packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
+
+    if upload_board_tarball:

[scottz, 2011/04/15 17:40:46]

Please add a comment about you uploading board tarballs in the background 

[davidjames, 2011/04/15 19:18:08]

Done.

+      p = multiprocessing.Process(target=self._UploadBoardTarball,

[scottz, 2011/04/15 17:40:46]

Please consider using something other than p. tar_process?

[davidjames, 2011/04/15 19:18:08]

Done.

+                                  args=(board_path, url_suffix))
+      p.start()
+
+    self._UploadPrebuilt(package_path, packages_url_suffix)
+
+    if upload_board_tarball:
+      p.join()
+      assert p.exitcode == 0
 
     # Record URL where prebuilts were uploaded.
     url_value = '%s/%s/' % (self._binhost_base_url.rstrip('/'),
@@ -627,6 +662,9 @@ def ParseOptions():
                     help='Update binhost.conf')
   parser.add_option('-P', '--private', dest='private', action='store_true',
                     default=False, help='Mark gs:// uploads as private.')
+  parser.add_option('', '--upload-board-tarball', dest='upload_board_tarball',
+                    action='store_true', default=False,
+                    help='Upload board tarball to Google Storage.')
 
   options, args = parser.parse_args()
   if not options.build_path:
@@ -634,6 +672,11 @@ def ParseOptions():
   if not options.upload:
     usage(parser, 'Error: you need to provide an upload location using -u')
 
+
+  if options.upload_board_tarball and not options.upload.startswith('gs://'):
+    usage(parser, 'Error: --upload-board-tarball only works with gs:// URLs.\n'
+                  '--upload must be a gs:// URL.')
+
   if options.private:
     if options.sync_host:
       usage(parser, 'Error: --private and --sync-host/-s cannot be specified '
@@ -684,7 +727,8 @@ def main():
   if options.board:
     uploader._SyncBoardPrebuilts(options.board, options.build_path, version,
                                  options.key, options.git_sync,
-                                 options.sync_binhost_conf)
+                                 options.sync_binhost_conf,
+                                 options.upload_board_tarball)
 
 if __name__ == '__main__':
   main()