Update cbuildbot.py to upload prebuilts from preflight buildbot.

bin/cbuildbot.py

 #!/usr/bin/python
 
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """CBuildbot is wrapper around the build process used by the pre-flight queue"""
 
 import errno
 import heapq
 import re
 import optparse
 import os
 import shutil
 import sys
 
 import cbuildbot_comm
 from cbuildbot_config import config
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '../lib'))
 from cros_build_lib import (Die, Info, ReinterpretPathForChroot, RunCommand,
                             Warning)
 
 _DEFAULT_RETRIES = 3
 _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list'
 ARCHIVE_BASE = '/var/www/archive'
 ARCHIVE_COUNT = 10
 
+# Currently, both the full buildbot and the preflight buildbot store their
+# data in a variable named PORTAGE_BINHOST, but they're in different files.
+# We're planning on joining the two files soon and renaming the full binhost
+# to FULL_BINHOST.
+_FULL_BINHOST = 'PORTAGE_BINHOST'
+_PREFLIGHT_BINHOST = 'PORTAGE_BINHOST'
+
 # ======================== Utility functions ================================
 
 def MakeDir(path, parents=False):
   """Basic wrapper around os.mkdirs.
 
   Keyword arguments:
   path -- Path to create.
   parents -- Follow mkdir -p logic.
 
   """
@@ skipping 252 matching lines @@
   """Performs a checkout without clobbering previous checkout."""
   RepoSync(buildroot, rw_checkout, retries)
 
 
 def _MakeChroot(buildroot):
   """Wrapper around make_chroot."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./make_chroot', '--fast'], cwd=cwd)
 
 
+def _GetPortageEnvVar(buildroot, board, envvar):
+  """Get a portage environment variable for the specified board, if any.
+
+  buildroot: The root directory where the build occurs. Must be an absolute
+             path.
+  board: Board type that was built on this machine. E.g. x86-generic.
+  envvar: The environment variable to get. E.g. "PORTAGE_BINHOST".
+
+  Returns:
+    The value of the environment variable, as a string. If no such variable
+    can be found, return the empty string.
+  """
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  binhost = RunCommand(['portageq-%s' % board, 'envvar', envvar],
+                       cwd=cwd, redirect_stdout=True, enter_chroot=True,
+                       error_ok=True)
+  return binhost.rstrip('\n')
+
+
 def _SetupBoard(buildroot, board='x86-generic'):
   """Wrapper around setup_board."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./setup_board', '--fast', '--default', '--board=%s' % board],
              cwd=cwd, enter_chroot=True)
 
 
 def _Build(buildroot):
   """Wrapper around build_packages."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
@@ skipping 178 matching lines @@
     paths = [private_overlay]
   elif overlays == 'public':
     paths = [public_overlay]
   elif overlays == 'both':
     paths = [public_overlay, private_overlay]
   else:
     Die('Incorrect overlay configuration: %s' % overlays)
   return paths
 
 
+def _UploadPrebuilts(buildroot, board, overlay_config):
+  """Upload prebuilts.
+
+  Args:
+    buildroot: The root directory where the build occurs.
+    board: Board type that was built on this machine
+    overlay_config: A string describing which overlays you want.
+                    'private': Just the private overlay.
+                    'public': Just the public overlay.
+                    'both': Both the public and private overlays.
+  """
+
+  cmd = ['%s/src/scripts/prebuilt.py' % buildroot,
+         '--sync-binhost-conf',
+         '--build-path', buildroot,
+         '--board', board,
+         '--prepend-version', 'preflight',
+         '--key', _PREFLIGHT_BINHOST]

[diandersAtChromium, 2010/11/24 01:30:31]

Still a little curious why this is done through RunCommand rather than simply
importing a function from prebuilt.py.

Also a NIT that the two paths in this function should be using os.path.join(),
though not a big deal.

[scottz, 2010/11/29 17:56:39]

Agreed on both, if we want to convert to importing later that is fine but
os.path.join should be used for the path.

On 2010/11/24 01:30:31, diandersAtChromium wrote:

[davidjames, 2010/11/29 21:18:55]

Which of the following do you mean?
  1. os.path.join(buildroot, 'src/scripts/prebuilt.py')
  2. os.path.join(buildroot, 'src', 'scripts', 'prebuilt.py')

If it's the former, I agree, because it's helpful to catch the case where
buildroot maybe ends in a slash and ensures we don't have two slashes in the
path.

If it's the latter, should we convert all similar examples to use os.path.join?
E.g. the _SetupBoard function uses './setup_board'. Should that be
os.path.join('.', 'setup_board') ? That seems a lot more verbose.

importing is a bit of a pain because the two scripts use different versions of
cros_build_lib (one uses the chromite version, the other does not.) If you
import one into the other you might confuse Python because it changes the import
path.

That should be cleaned up but we haven't merged in chromite yet.

One advantage of using command-line interaction between the two scripts is it
ensures we can't have any complex interdependencies between the two scripts such
as the above. But there are also advantages to importing, so once we fix the
above problem I'd be happy with either approach.

+  if overlay_config == 'public':
+    cmd.extend(['--upload', 'gs://chromeos-prebuilt'])
+  else:
+    assert overlay_config in ('private', 'both')
+    cmd.extend(['--upload', 'chromeos-images:/var/www/prebuilt/',
+                '--binhost-base-url', 'http://chromeos-prebuilt'])
+
+  RunCommand(cmd, cwd='%s/src/scripts' % buildroot)
+
+
 def main():
   # Parse options
   usage = "usage: %prog [options] cbuildbot_config"
   parser = optparse.OptionParser(usage=usage)
   parser.add_option('-r', '--buildroot',
                     help='root directory where build occurs', default=".")
   parser.add_option('-n', '--buildnumber',
                     help='build number', type='int', default=0)
   parser.add_option('-f', '--revisionfile',
                     help='file where new revisions are stored')
@@ skipping 17 matching lines @@
 
   if len(args) >= 1:
     buildconfig = _GetConfig(args[-1])
   else:
     Warning('Missing configuration description')
     parser.print_usage()
     sys.exit(1)
 
   # Calculate list of overlay directories.
   overlays = _ResolveOverlays(buildroot, buildconfig['overlays'])
+  board = buildconfig['board']

[diandersAtChromium, 2010/11/24 01:30:31]

Curious about whether these two lines should be in the try block.

[davidjames, 2010/11/29 21:18:55]

Done.

 
   try:
     _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays)
+    chroot_path = os.path.join(buildroot, 'chroot')
+    boardpath = os.path.join(chroot_path, 'build', board)
     if options.clobber or not os.path.isdir(buildroot):
       _FullCheckout(buildroot, tracking_branch, url=options.url)
     else:
+      old_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
       _IncrementalCheckout(buildroot)
+      new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
+      if old_binhost != new_binhost:
+        RunCommand(['sudo', 'rm', '-rf', boardpath])
 
     # Check that all overlays can be found.
     for path in overlays:
       assert ':' not in path, 'Overlay must not contain colons: %s' % path
       if not os.path.isdir(path):
         Die('Missing overlay: %s' % path)
 
-    chroot_path = os.path.join(buildroot, 'chroot')
     if not os.path.isdir(chroot_path):
       _MakeChroot(buildroot)
 
-    boardpath = os.path.join(chroot_path, 'build', buildconfig['board'])
     if not os.path.isdir(boardpath):
       _SetupBoard(buildroot, board=buildconfig['board'])
 
     if buildconfig['uprev']:
       _UprevPackages(buildroot, tracking_branch, revisionfile,
                      buildconfig['board'], overlays)
 
     _EnableLocalAccount(buildroot)
     _Build(buildroot)
     if buildconfig['unittests']:
@@ skipping 10 matching lines @@
         _ArchiveTestResults(buildroot, buildconfig['board'],
                             archive_dir=options.buildnumber,
                             test_results_dir=test_results_dir)
 
     if buildconfig['uprev']:
       # Don't push changes for developers.
       if not options.debug:
         if buildconfig['master']:
           # Master bot needs to check if the other slaves completed.
           if cbuildbot_comm.HaveSlavesCompleted(config):
+            _UploadPrebuilts(buildroot, board, buildconfig['overlays'])
             _UprevPush(buildroot, tracking_branch, buildconfig['board'],
                        overlays)
           else:
             Die('CBUILDBOT - One of the slaves has failed!!!')
 
         else:
           # Publish my status to the master if its expecting it.
           if buildconfig['important']:
             cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)
 
   except:
     # Send failure to master bot.
     if not buildconfig['master'] and buildconfig['important']:
       cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED)
 
     raise
 
 
 if __name__ == '__main__':
     main()