Update cbuildbot.py to upload prebuilts from preflight buildbot.

bin/cbuildbot.py

Index: bin/cbuildbot.py
diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py
index aef3ec27191741e024e46526260462a37a643248..d089ac1447cf78d9fa8b443ed8c82d4bc9a923e9 100755
--- a/bin/cbuildbot.py
+++ b/bin/cbuildbot.py
@@ -23,6 +23,8 @@ from cros_build_lib import (Die, Info, ReinterpretPathForChroot, RunCommand,
 
 _DEFAULT_RETRIES = 3
 _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list'

[dianders, 2010/11/23 21:47:39]

Please add comments about what these are: the names of portage environment
variables.

[davidjames, 2010/11/23 22:53:14]

Done.

+_FULL_BINHOST = 'PORTAGE_BINHOST'
+_PREFLIGHT_BINHOST = 'PREFLIGHT_BINHOST'
 ARCHIVE_BASE = '/var/www/archive'
 ARCHIVE_COUNT = 10
 
@@ -298,6 +300,17 @@ def _MakeChroot(buildroot):
   RunCommand(['./make_chroot', '--fast'], cwd=cwd)
 
 
+def _GetPortageEnvVar(buildroot, board, envvar):
+  """Get the current portage binhost for the specified board, if any.
+

[dianders, 2010/11/23 21:47:39]

Describe Args and Returns according to standard.

Args:
  buildroot: The current sysroot, like "/home/xyz/trunk"
  board: The name of the board, like "x86-generic".
  envvar: The environment variable to get, like "PORTAGE_BINHOST".

Returns:
  The value of the environment variable, as a string.

[davidjames, 2010/11/23 22:53:14]

Done.

+  If no binhost can be found, return the empty string."""
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  binhost = RunCommand(['portageq-%s' % board, 'envvar', envvar],
+                       cwd=cwd, redirect_stdout=True, enter_chroot=True,
+                       error_ok=True)
+  return binhost.rstrip('\n')
+
+
 def _SetupBoard(buildroot, board='x86-generic'):
   """Wrapper around setup_board."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
@@ -496,6 +509,36 @@ def _ResolveOverlays(buildroot, overlays):
   return paths
 
 
+def _UploadPrebuilts(buildroot, board, overlay_config, binhosts):
+  """Upload prebuilts.
+
+  Args:
+    buildroot: The root directory where the build occurs.
+    board: Board type that was built on this machine
+    overlay_config: A string describing which overlays you want.
+                    'private': Just the private overlay.
+                    'public': Just the public overlay.
+                    'both': Both the public and private overlays.
+    binhosts: The URLs of the current binhosts. Binaries that are already
+              present will not be uploaded twice.

[dianders, 2010/11/23 21:47:39]

Add comment about binhosts:

Note: it's expected that some of the strings in this list will simply be the
empty string.  We'll act as if those empty strings weren't in the list.

+  """
+
+  cmd = ['%s/src/scripts/prebuilt.py' % buildroot, '--git-sync',
+         '--build-path', buildroot, '--board', board,
+         '--prepend-version', 'preflight', '--key', _PREFLIGHT_BINHOST]

[dianders, 2010/11/23 21:47:39]

I'm curious why you don't just import a function from prebuilt.py and call it. 
Going through the command line seems awkward...

[davidjames, 2010/11/23 22:53:14]

That's a good point.

I like having a little separation between programs so that there's no risk the
two can affect each other. Integrating using the command-line ensures that
there's no tight coupling between the two programs.

+  for binhost in binhosts:
+    if binhost:
+      cmd.extend(['--previous-board-binhost-url', binhost])
+  if overlay_config == 'public':
+    cmd.extend(['--upload', 'gs://chromeos-prebuilt'])
+  else:
+    assert overlay_config in ('private', 'both')
+    cmd.extend(['--upload', 'chromeos-images:/var/www/prebuilt/',
+                '--binhost-base-url', 'http://chromeos-prebuilt'])
+
+  RunCommand(cmd)
+
+
 def main():
   # Parse options
   usage = "usage: %prog [options] cbuildbot_config"
@@ -533,13 +576,20 @@ def main():
 
   # Calculate list of overlay directories.
   overlays = _ResolveOverlays(buildroot, buildconfig['overlays'])
+  board = buildconfig['board']
 
   try:
     _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays)
+    chroot_path = os.path.join(buildroot, 'chroot')
+    boardpath = os.path.join(chroot_path, 'build', board)
     if options.clobber or not os.path.isdir(buildroot):
       _FullCheckout(buildroot, tracking_branch, url=options.url)
     else:
+      old_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
       _IncrementalCheckout(buildroot)
+      new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
+      if old_binhost != new_binhost:
+        RunCommand(['sudo', 'rm', '-rf', boardpath])
 
     # Check that all overlays can be found.
     for path in overlays:
@@ -547,11 +597,9 @@ def main():
       if not os.path.isdir(path):
         Die('Missing overlay: %s' % path)
 
-    chroot_path = os.path.join(buildroot, 'chroot')
     if not os.path.isdir(chroot_path):
       _MakeChroot(buildroot)
 
-    boardpath = os.path.join(chroot_path, 'build', buildconfig['board'])
     if not os.path.isdir(boardpath):
       _SetupBoard(buildroot, board=buildconfig['board'])
 
@@ -582,8 +630,14 @@ def main():
         if buildconfig['master']:
           # Master bot needs to check if the other slaves completed.
           if cbuildbot_comm.HaveSlavesCompleted(config):
+            binhosts = [
+                _GetPortageEnvVar(buildroot, board, _PREFLIGHT_BINHOST),
+                _GetPortageEnvVar(buildroot, board, _FULL_BINHOST),
+            ]
             _UprevPush(buildroot, tracking_branch, buildconfig['board'],
                        overlays)
+            _UploadPrebuilts(buildroot, board, buildconfig['overlays'],
+                             binhosts)
           else:
             Die('CBUILDBOT - One of the slaves has failed!!!')
 
@@ -591,7 +645,6 @@ def main():
           # Publish my status to the master if its expecting it.
           if buildconfig['important']:
             cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)
-
   except:
     # Send failure to master bot.
     if not buildconfig['master'] and buildconfig['important']: