Update prebuilt.py to support uploading a board tarball to Google Storage.

buildbot/prebuilt.py

 #!/usr/bin/python
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import datetime
 import multiprocessing
 import optparse
 import os
 import re
@@ skipping 33 matching lines @@
 """
 
 # as per http://crosbug.com/5855 always filter the below packages
 _FILTER_PACKAGES = set()
 _RETRIES = 3
 _GSUTIL_BIN = '/b/build/third_party/gsutil/gsutil'
 _HOST_PACKAGES_PATH = 'chroot/var/lib/portage/pkgs'
 _CATEGORIES_PATH = 'chroot/etc/portage/categories'
 _HOST_TARGET = 'amd64'
 _BOARD_PATH = 'chroot/build/%(board)s'
-# board/board-target/version/packages/'
-_REL_BOARD_PATH = 'board/%(board)s/%(version)s/packages'
-# host/host-target/version/packages/'
-_REL_HOST_PATH = 'host/%(target)s/%(version)s/packages'
+# board/board-target/version/'
+_REL_BOARD_PATH = 'board/%(board)s/%(version)s'
+# host/host-target/version/'
+_REL_HOST_PATH = 'host/%(target)s/%(version)s'
 # Private overlays to look at for builds to filter
 # relative to build path
 _PRIVATE_OVERLAY_DIR = 'src/private-overlays'
 _GOOGLESTORAGE_ACL_FILE = 'googlestorage_acl.xml'
 _BINHOST_BASE_URL = 'http://commondatastorage.googleapis.com/chromeos-prebuilt'
 _PREBUILT_BASE_DIR = 'src/third_party/chromiumos-overlay/chromeos/config/'
 # Created in the event of new host targets becoming available
 _PREBUILT_MAKE_CONF = {'amd64': os.path.join(_PREBUILT_BASE_DIR,
                                              'make.conf.amd64-host')}
 _BINHOST_CONF_DIR = 'src/third_party/chromiumos-overlay/chromeos/binhost'
@@ skipping 440 matching lines @@
             'remote_location': remote_location.rstrip('/'),
             'ssh_server': ssh_server }
       cmds = ['ssh %(ssh_server)s mkdir -p %(remote_path)s' % d,
               'rsync -av --chmod=a+r %(pkg_index)s %(remote_packages)s' % d]
       if pkgs:
         cmds.append('rsync -Rav %(pkgs)s %(remote_location)s/' % d)
       for cmd in cmds:
         if not _RetryRun(cmd, shell=True, cwd=package_path):
           raise UploadFailed('Could not run %s' % cmd)
 
+  def _UploadBoardTarball(self, board_path, url_suffix):
+    """Upload a tarball of the board at the specified path to Google Storage.
+
+    Args:
+      board_path: The path to the board dir.
+      url_suffix: The remote subdirectory where we should upload the packages.
+    """
+    remote_location = '%s/%s' % (self._upload_location.rstrip('/'), url_suffix)
+    assert remote_location.startswith('gs://')
+    cwd, boardname = os.path.split(board_path.rstrip(os.path.sep))
+    tmpdir = tempfile.mkdtemp()
+    try:
+      tarfile = os.path.join(tmpdir, '%s.tbz2' % boardname)
+      cmd = ['sudo', 'tar', '-I', 'pbzip2', '-cf', tarfile]
+      excluded_paths = ('usr/lib/debug', 'usr/local/autotest', 'packages',
+                        'tmp')
+      for path in excluded_paths:
+        cmd.append('--exclude=%s/%s/*' % (boardname, path))
+      cmd.append(boardname)
+      cros_build_lib.RunCommand(cmd, cwd=cwd)
+      remote_tarfile = '%s/%s.tbz2' % (remote_location.rstrip('/'), boardname)
+      if _GsUpload((tarfile, remote_tarfile, self._acl)):
+        sys.exit(1)
+    finally:
+      cros_build_lib.RunCommand(['sudo', 'rm', '-rf', tmpdir], cwd=cwd)
+
   def _SyncHostPrebuilts(self, build_path, version, key, git_sync,
                         sync_binhost_conf):
     """Synchronize host prebuilt files.
 
     This function will sync both the standard host packages, plus the host
     packages associated with all targets that have been "setup" with the
     current host's chroot. For instance, if this host has been used to build
     x86-generic, it will sync the host packages associated with
     'i686-pc-linux-gnu'. If this host has also been used to build arm-generic,
     it will also sync the host packages associated with
     'armv7a-cros-linux-gnueabi'.
 
     Args:
       build_path: The path to the directory containing the chroot.
       version: A unique string, intended to be included in the upload path,
           which identifies the version number of the uploaded prebuilts.
       key: The variable key to update in the git file.
       git_sync: If set, update make.conf of target to reference the latest
           prebuilt packages generated here.
       sync_binhost_conf: If set, update binhost config file in
           chromiumos-overlay for the host.
     """
     # Upload prebuilts.
     package_path = os.path.join(build_path, _HOST_PACKAGES_PATH)
     url_suffix = _REL_HOST_PATH % {'version': version, 'target': _HOST_TARGET}
-    self._UploadPrebuilt(package_path, url_suffix)
+    packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
+    self._UploadPrebuilt(package_path, packages_url_suffix)
 
     # Record URL where prebuilts were uploaded.
     url_value = '%s/%s/' % (self._binhost_base_url.rstrip('/'),
                             url_suffix.rstrip('/'))
     if git_sync:
       git_file = os.path.join(build_path, _PREBUILT_MAKE_CONF[_HOST_TARGET])
       RevGitFile(git_file, url_value, key=key)
     if sync_binhost_conf:
       binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'host',
           '%s-%s.conf' % (_HOST_TARGET, key))
       UpdateBinhostConfFile(binhost_conf, key, url_value)
 
   def _SyncBoardPrebuilts(self, board, build_path, version, key, git_sync,
-                         sync_binhost_conf):
+                          sync_binhost_conf, upload_board_tarball):
     """Synchronize board prebuilt files.
 
     Args:
       board: The board to upload to Google Storage.
       build_path: The path to the directory containing the chroot.
       version: A unique string, intended to be included in the upload path,
           which identifies the version number of the uploaded prebuilts.
       key: The variable key to update in the git file.
       git_sync: If set, update make.conf of target to reference the latest
           prebuilt packages generated here.
       sync_binhost_conf: If set, update binhost config file in
           chromiumos-overlay for the current board.
+      upload_board_tarball: Include a tarball of the board in our upload.
     """
-    # Upload prebuilts.
     board_path = os.path.join(build_path, _BOARD_PATH % {'board': board})
     package_path = os.path.join(board_path, 'packages')
     url_suffix = _REL_BOARD_PATH % {'board': board, 'version': version}
-    self._UploadPrebuilt(package_path, url_suffix)
+    packages_url_suffix = '%s/packages' % url_suffix.rstrip('/')
+
+    # Upload board tarballs in the background.
+    if upload_board_tarball:
+      tar_process = multiprocessing.Process(target=self._UploadBoardTarball,
+                                            args=(board_path, url_suffix))
+      tar_process.start()
+
+    # Upload prebuilts.
+    self._UploadPrebuilt(package_path, packages_url_suffix)
+
+    # Make sure we finished uploading the board tarballs.
+    if upload_board_tarball:
+      tar_process.join()
+      assert tar_process.exitcode == 0
 
     # Record URL where prebuilts were uploaded.
     url_value = '%s/%s/' % (self._binhost_base_url.rstrip('/'),
                             url_suffix.rstrip('/'))
     if git_sync:
       git_file = DeterminePrebuiltConfFile(build_path, board)
       RevGitFile(git_file, url_value, key=key)
     if sync_binhost_conf:
       binhost_conf = os.path.join(build_path, _BINHOST_CONF_DIR, 'target',
           '%s-%s.conf' % (board, key))
@@ skipping 34 matching lines @@
                     default=False,
                     help='Turn on filtering of private ebuild packages')
   parser.add_option('-k', '--key', dest='key',
                     default='PORTAGE_BINHOST',
                     help='Key to update in make.conf / binhost.conf')
   parser.add_option('', '--sync-binhost-conf', dest='sync_binhost_conf',
                     default=False, action='store_true',
                     help='Update binhost.conf')
   parser.add_option('-P', '--private', dest='private', action='store_true',
                     default=False, help='Mark gs:// uploads as private.')
+  parser.add_option('', '--upload-board-tarball', dest='upload_board_tarball',
+                    action='store_true', default=False,
+                    help='Upload board tarball to Google Storage.')
 
   options, args = parser.parse_args()
   if not options.build_path:
     usage(parser, 'Error: you need provide a chroot path')
   if not options.upload:
     usage(parser, 'Error: you need to provide an upload location using -u')
 
+
+  if options.upload_board_tarball and not options.upload.startswith('gs://'):
+    usage(parser, 'Error: --upload-board-tarball only works with gs:// URLs.\n'
+                  '--upload must be a gs:// URL.')
+
   if options.private:
     if options.sync_host:
       usage(parser, 'Error: --private and --sync-host/-s cannot be specified '
                     'together, we do not support private host prebuilts')
 
     if not options.upload.startswith('gs://'):
       usage(parser, 'Error: --private is only valid for gs:// URLs.\n'
                     '--upload must be a gs:// URL.')
 
     if options.binhost_base_url != _BINHOST_BASE_URL:
@@ skipping 30 matching lines @@
   uploader = PrebuiltUploader(options.upload, acl, binhost_base_url,
                               pkg_indexes)
 
   if options.sync_host:
     uploader._SyncHostPrebuilts(options.build_path, version, options.key,
                                 options.git_sync, options.sync_binhost_conf)
 
   if options.board:
     uploader._SyncBoardPrebuilts(options.board, options.build_path, version,
                                  options.key, options.git_sync,
-                                 options.sync_binhost_conf)
+                                 options.sync_binhost_conf,
+                                 options.upload_board_tarball)
 
 if __name__ == '__main__':
   main()