Update prebuilt.py to support uploading a board tarball to Google Storage.

buildbot/prebuilt.py

 #!/usr/bin/python
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import datetime
 import multiprocessing
 import optparse
 import os
 import re
@@ skipping 33 matching lines @@
 """
 
 # as per http://crosbug.com/5855 always filter the below packages
 _FILTER_PACKAGES = set()
 _RETRIES = 3
 _GSUTIL_BIN = '/b/build/third_party/gsutil/gsutil'
 _HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs'
 _CATEGORIES_PATH = 'chroot/etc/portage/categories'
 _HOST_TARGET = 'amd64'
 _BOARD_PATH = 'chroot/build/%(board)s'
-# board/board-target/version/packages/'
-_REL_BOARD_PATH = 'board/%(board)s/%(version)s/packages'
-# host/host-target/version/packages/'
-_REL_HOST_PATH = 'host/%(target)s/%(version)s/packages'
+# board/board-target/version/'
+_REL_BOARD_PATH = 'board/%(board)s/%(version)s'
+# host/host-target/version/'
+_REL_HOST_PATH = 'host/%(target)s/%(version)s'
 # Private overlays to look at for builds to filter
 # relative to build path
 _PRIVATE_OVERLAY_DIR = 'src/private-overlays'
 _GOOGLESTORAGE_ACL_FILE = 'googlestorage_acl.xml'
 _BINHOST_BASE_URL = 'http://commondatastorage.googleapis.com/chromeos-prebuilt'
 _PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/'
 # Created in the event of new host targets becoming available
 _PREBUILT_MAKE_CONF = {'amd64': os.path.join(_PREBUILT_BASE_DIR,
                                              'make.conf.amd64-host')}
 _BINHOST_CONF_DIR = 'src/third_party/chromiumos-overlay/chromeos/binhost'
@@ skipping 440 matching lines @@
             'remote_location': remote_location.rstrip('/'),
             'ssh_server': ssh_server }
       cmds = ['ssh %(ssh_server)s mkdir -p %(remote_path)s' % d,
               'rsync -av --chmod=a+r %(pkg_index)s %(remote_packages)s' % d]
       if pkgs:
         cmds.append('rsync -Rav %(pkgs)s %(remote_location)s/' % d)
       for cmd in cmds:
         if not _RetryRun(cmd, shell=True, cwd=package_path):
           raise UploadFailed('Could not run %s' % cmd)
 
+  def _UploadBoardTarball(self, board_path, url_suffix):
+    """Upload a tarball of the board at the specified path to Google Storage.
+

[scottz, 2011/04/15 17:40:46]

Args:

[davidjames, 2011/04/15 19:18:08]

Done.

+      board_path: The path to the board dir.
+      url_suffix: The remote subdirectory where we should upload the packages.
+    """
+    remote_location = '%s/%s' % (self._upload_location.rstrip('/'), url_suffix)
+    assert remote_location.startswith('gs://')
+    cwd, boardname = os.path.split(board_path.rstrip(os.path.sep))
+    tarfile = '/tmp/%s.tbz2' % boardname

[scottz, 2011/04/15 17:40:46]

You should use tempfile here.

I prefer not to assume we can dump gigs of files in /tmp. I would much rather
you create a temporary directory in the dir where prebuilt is executed and wipe
it if anything bad happens. 

[davidjames, 2011/04/15 19:18:08]

Done.

Now that we switched to tempfile, it will honor the TMPDIR / TEMP / TMP
environment variables and so if the buildbot sets those variables, Python will
use one of those directories.

+    cmd = ['sudo', 'tar', '-I', 'pbzip2', '-cf', tarfile]
+    for path in ('usr/lib/debug', 'usr/local/autotest', 'packages', 'tmp'):

[scottz, 2011/04/15 17:40:46]

Assigning these to a variable excluded_paths would make it a lot more clear what
you are doing here at the cost of a var added. 

[davidjames, 2011/04/15 19:18:08]

Done.

+      cmd.append('--exclude=%s/%s/*' % (boardname, path))
+    cmd.append(boardname)
+    cros_build_lib.RunCommand(cmd, cwd=cwd)

[scottz, 2011/04/15 17:40:46]

this should be in your try, if tar fails we are left with a tarball in /tmp,
this could fail easily if /tmp has no more space. 

[davidjames, 2011/04/15 19:18:08]

Done.

+    remote_tarfile = '%s/%s.tbz2' % (remote_location.rstrip('/'), boardname)
+    try:
+      if _GsUpload((tarfile, remote_tarfile, self._acl)):
+        sys.exit(1)
+    finally:
+      cros_build_lib.RunCommand(['sudo', 'rm', '-f', tarfile], cwd=cwd)
+
   def _SyncHostPrebuilts(self, build_path, version, key, git_sync,
                         sync_binhost_conf):
     """Synchronize host prebuilt files.
 
     This function will sync both the standard host packages, plus the host
     packages associated with all targets that have been "setup" with the
     current host's chroot. For instance, if this host has been used to build
     x86-generic, it will sync the host packages associated with
     'i686-pc-linux-gnu'. If this host has also been used to build arm-generic,
     it will also sync the host packages associated with
     'armv7a-cros-linux-gnueabi'.
 
     Args:
       build_path: The path to the directory containing the chroot.
       version: A unique string, intended to be included in the upload path,
           which identifies the version number of the uploaded prebuilts.
       key: The variable key to update in the git file.
       git_sync: If set, update make.conf of target to reference the latest
           prebuilt packages generated here.
       sync_binhost_conf: If set, update binhost config file in
           chromiumos-overlay for the host.
     """
     # Upload prebuilts.
     package_path = os.path.join(build_path, _HOST_PACKAGES_PATH)
     url_suffix = _REL_HOST_PATH % {'version': version, 'target': _HOST_TARGET}
-    self._UploadPrebuilt(package_path, url_suffix)
+    packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
+    self._UploadPrebuilt(package_path, packages_url_suffix)
 
     # Record URL where prebuilts were uploaded.
     url_value = '%s/%s/' % (self._binhost_base_url.rstrip('/'),
                             url_suffix.rstrip('/'))
     if git_sync:
       git_file = os.path.join(build_path, _PREBUILT_MAKE_CONF[_HOST_TARGET])
       RevGitFile(git_file, url_value, key=key)
     if sync_binhost_conf:
       binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'host',
           '%s-%s.conf' % (_HOST_TARGET, key))
       UpdateBinhostConfFile(binhost_conf, key, url_value)
 
   def _SyncBoardPrebuilts(self, board, build_path, version, key, git_sync,
-                         sync_binhost_conf):
+                          sync_binhost_conf, upload_board_tarball):
     """Synchronize board prebuilt files.
 
     Args:
       board: The board to upload to Google Storage.
       build_path: The path to the directory containing the chroot.
       version: A unique string, intended to be included in the upload path,
           which identifies the version number of the uploaded prebuilts.
       key: The variable key to update in the git file.
       git_sync: If set, update make.conf of target to reference the latest
           prebuilt packages generated here.
       sync_binhost_conf: If set, update binhost config file in
           chromiumos-overlay for the current board.
+      upload_board_tarball: Include a tarball of the board in our upload.
     """
     # Upload prebuilts.
     board_path = os.path.join(build_path, _BOARD_PATH % {'board': board})
     package_path = os.path.join(board_path, 'packages')
     url_suffix = _REL_BOARD_PATH % {'board': board, 'version': version}
-    self._UploadPrebuilt(package_path, url_suffix)
+    packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
+
+    if upload_board_tarball:

[scottz, 2011/04/15 17:40:46]

Please add a comment about you uploading board tarballs in the background 

[davidjames, 2011/04/15 19:18:08]

Done.

+      p = multiprocessing.Process(target=self._UploadBoardTarball,

[scottz, 2011/04/15 17:40:46]

Please consider using something other than p. tar_process?

[davidjames, 2011/04/15 19:18:08]

Done.

+                                  args=(board_path, url_suffix))
+      p.start()
+
+    self._UploadPrebuilt(package_path, packages_url_suffix)
+
+    if upload_board_tarball:
+      p.join()
+      assert p.exitcode == 0
 
     # Record URL where prebuilts were uploaded.
     url_value = '%s/%s/' % (self._binhost_base_url.rstrip('/'),
                             url_suffix.rstrip('/'))
     if git_sync:
       git_file = DeterminePrebuiltConfFile(build_path, board)
       RevGitFile(git_file, url_value, key=key)
     if sync_binhost_conf:
       binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'target',
           '%s-%s.conf' % (board, key))
@@ skipping 34 matching lines @@
                     default=False,
                     help='Turn on filtering of private ebuild packages')
   parser.add_option('-k', '--key', dest='key',
                     default='PORTAGE_BINHOST',
                     help='Key to update in make.conf / binhost.conf')
   parser.add_option('', '--sync-binhost-conf', dest='sync_binhost_conf',
                     default=False, action='store_true',
                     help='Update binhost.conf')
   parser.add_option('-P', '--private', dest='private', action='store_true',
                     default=False, help='Mark gs:// uploads as private.')
+  parser.add_option('', '--upload-board-tarball', dest='upload_board_tarball',
+                    action='store_true', default=False,
+                    help='Upload board tarball to Google Storage.')
 
   options, args = parser.parse_args()
   if not options.build_path:
     usage(parser, 'Error: you need provide a chroot path')
   if not options.upload:
     usage(parser, 'Error: you need to provide an upload location using -u')
 
+
+  if options.upload_board_tarball and not options.upload.startswith('gs://'):
+    usage(parser, 'Error: --upload-board-tarball only works with gs:// URLs.\n'
+                  '--upload must be a gs:// URL.')
+
   if options.private:
     if options.sync_host:
       usage(parser, 'Error: --private and --sync-host/-s cannot be specified '
                     'together, we do not support private host prebuilts')
 
     if not options.upload.startswith('gs://'):
       usage(parser, 'Error: --private is only valid for gs:// URLs.\n'
                     '--upload must be a gs:// URL.')
 
     if options.binhost_base_url != _BINHOST_BASE_URL:
@@ skipping 30 matching lines @@
   uploader = PrebuiltUploader(options.upload, acl, binhost_base_url,
                               pkg_indexes)
 
   if options.sync_host:
     uploader._SyncHostPrebuilts(options.build_path, version, options.key,
                                 options.git_sync, options.sync_binhost_conf)
 
   if options.board:
     uploader._SyncBoardPrebuilts(options.board, options.build_path, version,
                                  options.key, options.git_sync,
-                                 options.sync_binhost_conf)
+                                 options.sync_binhost_conf,
+                                 options.upload_board_tarball)
 
 if __name__ == '__main__':
   main()