Update cbuildbot.py to upload prebuilts from preflight buildbot.

bin/cbuildbot.py

 #!/usr/bin/python
 
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """CBuildbot is wrapper around the build process used by the pre-flight queue"""
 
 import errno
 import heapq
 import re
 import optparse
 import os
 import shutil
 import sys
 
 import cbuildbot_comm
 from cbuildbot_config import config
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '../lib'))
 from cros_build_lib import (Die, Info, ReinterpretPathForChroot, RunCommand,
                             Warning)
 
 _DEFAULT_RETRIES = 3
 _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list'
 ARCHIVE_BASE = '/var/www/archive'
 ARCHIVE_COUNT = 10
 
+# Currently, both the full buildbot and the preflight buildbot store their
+# data in a variable named PORTAGE_BINHOST, but they're in different files.
+# We're planning on joining the two files soon and renaming the full binhost
+# to FULL_BINHOST.
+_FULL_BINHOST = 'PORTAGE_BINHOST'
+_PREFLIGHT_BINHOST = 'PORTAGE_BINHOST'
+
 # ======================== Utility functions ================================
 
 def MakeDir(path, parents=False):
   """Basic wrapper around os.mkdirs.
 
   Keyword arguments:
   path -- Path to create.
   parents -- Follow mkdir -p logic.
 
   """
@@ skipping 252 matching lines @@
   """Performs a checkout without clobbering previous checkout."""
   RepoSync(buildroot, rw_checkout, retries)
 
 
 def _MakeChroot(buildroot):
   """Wrapper around make_chroot."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./make_chroot', '--fast'], cwd=cwd)
 
 
+def _GetPortageEnvVar(buildroot, board, envvar):
+  """Get a portage environment variable for the specified board, if any.
+
+  buildroot: The root directory where the build occurs. Must be an absolute
+             path.
+  board: Board type that was built on this machine. E.g. x86-generic.
+  envvar: The environment variable to get. E.g. "PORTAGE_BINHOST".
+
+  Returns:
+    The value of the environment variable, as a string. If no such variable
+    can be found, return the empty string.
+  """
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  binhost = RunCommand(['portageq-%s' % board, 'envvar', envvar],
+                       cwd=cwd, redirect_stdout=True, enter_chroot=True,
+                       error_ok=True)
+  return binhost.rstrip('\n')
+
+
 def _SetupBoard(buildroot, board='x86-generic'):
   """Wrapper around setup_board."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./setup_board', '--fast', '--default', '--board=%s' % board],
              cwd=cwd, enter_chroot=True)
 
 
 def _Build(buildroot):
   """Wrapper around build_packages."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
@@ skipping 178 matching lines @@
     paths = [private_overlay]
   elif overlays == 'public':
     paths = [public_overlay]
   elif overlays == 'both':
     paths = [public_overlay, private_overlay]
   else:
     Die('Incorrect overlay configuration: %s' % overlays)
   return paths
 
 
+def _UploadPrebuilts(buildroot, board, overlay_config):
+  """Upload prebuilts.
+
+  Args:
+    buildroot: The root directory where the build occurs.
+    board: Board type that was built on this machine
+    overlay_config: A string describing which overlays you want.
+                    'private': Just the private overlay.
+                    'public': Just the public overlay.
+                    'both': Both the public and private overlays.
+  """
+
+  cmd = ['%s/src/scripts/prebuilt.py' % buildroot, '--sync-binhost-conf',

[sosa, 2010/11/23 23:56:21]

Can you put each of this long options on their own line?  It's much easier to
read option=arg and also modify.  Otherwise for example someone may come back
later and accidentally add a new option between --key and _PREFLIGHT_BINHOST 

[davidjames, 2010/11/24 00:29:23]

Done.

+         '--build-path', buildroot, '--board', board,
+         '--prepend-version', 'preflight', '--key', _PREFLIGHT_BINHOST]
+  if overlay_config == 'public':
+    cmd.extend(['--upload', 'gs://chromeos-prebuilt'])
+  else:
+    assert overlay_config in ('private', 'both')
+    cmd.extend(['--upload', 'chromeos-images:/var/www/prebuilt/',
+                '--binhost-base-url', 'http://chromeos-prebuilt'])
+
+  RunCommand(cmd)

[sosa, 2010/11/23 23:56:21]

Nit: Most other code sets cwd, is it possible to use cwd like other funcs?

[davidjames, 2010/11/24 00:29:23]

Done.

+
+
 def main():
   # Parse options
   usage = "usage: %prog [options] cbuildbot_config"
   parser = optparse.OptionParser(usage=usage)
   parser.add_option('-r', '--buildroot',
                     help='root directory where build occurs', default=".")
   parser.add_option('-n', '--buildnumber',
                     help='build number', type='int', default=0)
   parser.add_option('-f', '--revisionfile',
                     help='file where new revisions are stored')
@@ skipping 17 matching lines @@
 
   if len(args) >= 1:
     buildconfig = _GetConfig(args[-1])
   else:
     Warning('Missing configuration description')
     parser.print_usage()
     sys.exit(1)
 
   # Calculate list of overlay directories.
   overlays = _ResolveOverlays(buildroot, buildconfig['overlays'])
+  board = buildconfig['board']
 
   try:
     _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays)
+    chroot_path = os.path.join(buildroot, 'chroot')
+    boardpath = os.path.join(chroot_path, 'build', board)
     if options.clobber or not os.path.isdir(buildroot):
       _FullCheckout(buildroot, tracking_branch, url=options.url)
     else:
+      old_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
       _IncrementalCheckout(buildroot)
+      new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
+      if old_binhost != new_binhost:
+        RunCommand(['sudo', 'rm', '-rf', boardpath])

[sosa, 2010/11/23 23:56:21]

Is sudo necessary?  Is shutil.rmtree not sufficient?

[davidjames, 2010/11/24 00:29:23]

Yeah, it's necessary unfortunately. There are some files in the board that are
owned by root.

 
     # Check that all overlays can be found.
     for path in overlays:
       assert ':' not in path, 'Overlay must not contain colons: %s' % path
       if not os.path.isdir(path):
         Die('Missing overlay: %s' % path)
 
-    chroot_path = os.path.join(buildroot, 'chroot')
     if not os.path.isdir(chroot_path):
       _MakeChroot(buildroot)
 
-    boardpath = os.path.join(chroot_path, 'build', buildconfig['board'])
     if not os.path.isdir(boardpath):
       _SetupBoard(buildroot, board=buildconfig['board'])
 
     if buildconfig['uprev']:
       _UprevPackages(buildroot, tracking_branch, revisionfile,
                      buildconfig['board'], overlays)
 
     _EnableLocalAccount(buildroot)
     _Build(buildroot)
     if buildconfig['unittests']:
@@ skipping 10 matching lines @@
         _ArchiveTestResults(buildroot, buildconfig['board'],
                             archive_dir=options.buildnumber,
                             test_results_dir=test_results_dir)
 
     if buildconfig['uprev']:
       # Don't push changes for developers.
       if not options.debug:
         if buildconfig['master']:
           # Master bot needs to check if the other slaves completed.
           if cbuildbot_comm.HaveSlavesCompleted(config):
+            _UploadPrebuilts(buildroot, board, buildconfig['overlays'])
             _UprevPush(buildroot, tracking_branch, buildconfig['board'],
                        overlays)
           else:
             Die('CBUILDBOT - One of the slaves has failed!!!')
 
         else:
           # Publish my status to the master if its expecting it.
           if buildconfig['important']:
             cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)

[sosa, 2010/11/23 23:56:21]

Scott has a if/else convention of a newline after any control block.

[davidjames, 2010/11/24 00:29:23]

OK, added the newline back.

-
   except:
     # Send failure to master bot.
     if not buildconfig['master'] and buildconfig['important']:
       cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED)
 
     raise
 
 
 if __name__ == '__main__':
     main()