Update cbuildbot.py to upload prebuilts from preflight buildbot.

bin/cbuildbot.py

Index: bin/cbuildbot.py
diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py
index aef3ec27191741e024e46526260462a37a643248..0c2f2eaa4f4a1a7e8aecafdd46ef6f14b5a98ca2 100755
--- a/bin/cbuildbot.py
+++ b/bin/cbuildbot.py
@@ -26,6 +26,13 @@ _PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list'
 ARCHIVE_BASE = '/var/www/archive'
 ARCHIVE_COUNT = 10
 
+# Currently, both the full buildbot and the preflight buildbot store their
+# data in a variable named PORTAGE_BINHOST, but they're in different files.
+# We're planning on joining the two files soon and renaming the full binhost
+# to FULL_BINHOST.
+_FULL_BINHOST = 'PORTAGE_BINHOST'
+_PREFLIGHT_BINHOST = 'PORTAGE_BINHOST'
+
 # ======================== Utility functions ================================
 
 def MakeDir(path, parents=False):
@@ -298,6 +305,25 @@ def _MakeChroot(buildroot):
   RunCommand(['./make_chroot', '--fast'], cwd=cwd)
 
 
+def _GetPortageEnvVar(buildroot, board, envvar):
+  """Get a portage environment variable for the specified board, if any.
+
+  buildroot: The root directory where the build occurs. Must be an absolute
+             path.
+  board: Board type that was built on this machine. E.g. x86-generic.
+  envvar: The environment variable to get. E.g. "PORTAGE_BINHOST".
+
+  Returns:
+    The value of the environment variable, as a string. If no such variable
+    can be found, return the empty string.
+  """
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  binhost = RunCommand(['portageq-%s' % board, 'envvar', envvar],
+                       cwd=cwd, redirect_stdout=True, enter_chroot=True,
+                       error_ok=True)
+  return binhost.rstrip('\n')
+
+
 def _SetupBoard(buildroot, board='x86-generic'):
   """Wrapper around setup_board."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
@@ -496,6 +522,34 @@ def _ResolveOverlays(buildroot, overlays):
   return paths
 
 
+def _UploadPrebuilts(buildroot, board, overlay_config):
+  """Upload prebuilts.
+
+  Args:
+    buildroot: The root directory where the build occurs.
+    board: Board type that was built on this machine
+    overlay_config: A string describing which overlays you want.
+                    'private': Just the private overlay.
+                    'public': Just the public overlay.
+                    'both': Both the public and private overlays.
+  """
+
+  cmd = ['%s/src/scripts/prebuilt.py' % buildroot,
+         '--sync-binhost-conf',
+         '--build-path', buildroot,
+         '--board', board,
+         '--prepend-version', 'preflight',
+         '--key', _PREFLIGHT_BINHOST]

[diandersAtChromium, 2010/11/24 01:30:31]

Still a little curious why this is done through RunCommand rather than simply
importing a function from prebuilt.py.

Also a NIT that the two paths in this function should be using os.path.join(),
though not a big deal.

[scottz, 2010/11/29 17:56:39]

Agreed on both, if we want to convert to importing later that is fine but
os.path.join should be used for the path.

On 2010/11/24 01:30:31, diandersAtChromium wrote:

[davidjames, 2010/11/29 21:18:55]

Which of the following do you mean?
  1. os.path.join(buildroot, 'src/scripts/prebuilt.py')
  2. os.path.join(buildroot, 'src', 'scripts', 'prebuilt.py')

If it's the former, I agree, because it's helpful to catch the case where
buildroot maybe ends in a slash and ensures we don't have two slashes in the
path.

If it's the latter, should we convert all similar examples to use os.path.join?
E.g. the _SetupBoard function uses './setup_board'. Should that be
os.path.join('.', 'setup_board') ? That seems a lot more verbose.

importing is a bit of a pain because the two scripts use different versions of
cros_build_lib (one uses the chromite version, the other does not.) If you
import one into the other you might confuse Python because it changes the import
path.

That should be cleaned up but we haven't merged in chromite yet.

One advantage of using command-line interaction between the two scripts is it
ensures we can't have any complex interdependencies between the two scripts such
as the above. But there are also advantages to importing, so once we fix the
above problem I'd be happy with either approach.

+  if overlay_config == 'public':
+    cmd.extend(['--upload', 'gs://chromeos-prebuilt'])
+  else:
+    assert overlay_config in ('private', 'both')
+    cmd.extend(['--upload', 'chromeos-images:/var/www/prebuilt/',
+                '--binhost-base-url', 'http://chromeos-prebuilt'])
+
+  RunCommand(cmd, cwd='%s/src/scripts' % buildroot)
+
+
 def main():
   # Parse options
   usage = "usage: %prog [options] cbuildbot_config"
@@ -533,13 +587,20 @@ def main():
 
   # Calculate list of overlay directories.
   overlays = _ResolveOverlays(buildroot, buildconfig['overlays'])
+  board = buildconfig['board']

[diandersAtChromium, 2010/11/24 01:30:31]

Curious about whether these two lines should be in the try block.

[davidjames, 2010/11/29 21:18:55]

Done.

 
   try:
     _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays)
+    chroot_path = os.path.join(buildroot, 'chroot')
+    boardpath = os.path.join(chroot_path, 'build', board)
     if options.clobber or not os.path.isdir(buildroot):
       _FullCheckout(buildroot, tracking_branch, url=options.url)
     else:
+      old_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
       _IncrementalCheckout(buildroot)
+      new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
+      if old_binhost != new_binhost:
+        RunCommand(['sudo', 'rm', '-rf', boardpath])
 
     # Check that all overlays can be found.
     for path in overlays:
@@ -547,11 +608,9 @@ def main():
       if not os.path.isdir(path):
         Die('Missing overlay: %s' % path)
 
-    chroot_path = os.path.join(buildroot, 'chroot')
     if not os.path.isdir(chroot_path):
       _MakeChroot(buildroot)
 
-    boardpath = os.path.join(chroot_path, 'build', buildconfig['board'])
     if not os.path.isdir(boardpath):
       _SetupBoard(buildroot, board=buildconfig['board'])
 
@@ -582,6 +641,7 @@ def main():
         if buildconfig['master']:
           # Master bot needs to check if the other slaves completed.
           if cbuildbot_comm.HaveSlavesCompleted(config):
+            _UploadPrebuilts(buildroot, board, buildconfig['overlays'])
             _UprevPush(buildroot, tracking_branch, buildconfig['board'],
                        overlays)
           else: