Update cbuildbot.py and prebuilt.py to deduplicate preflight prebuilts.

bin/cbuildbot.py

 #!/usr/bin/python
 
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """CBuildbot is wrapper around the build process used by the pre-flight queue"""
 
 import errno
 import heapq
@@ skipping 292 matching lines @@
   """Wrapper around make_chroot."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./make_chroot', '--fast'], cwd=cwd)
 
 
 def _GetPortageEnvVar(buildroot, board, envvar):
   """Get a portage environment variable for the specified board, if any.
 
   buildroot: The root directory where the build occurs. Must be an absolute
              path.
-  board: Board type that was built on this machine. E.g. x86-generic.
-  envvar: The environment variable to get. E.g. "PORTAGE_BINHOST".
+  board: Board type that was built on this machine. E.g. x86-generic. If this
+         is None, get the env var from the host.
+  envvar: The environment variable to get. E.g. 'PORTAGE_BINHOST'.
 
   Returns:
     The value of the environment variable, as a string. If no such variable
     can be found, return the empty string.
   """
   cwd = os.path.join(buildroot, 'src', 'scripts')
-  binhost = RunCommand(['portageq-%s' % board, 'envvar', envvar],
-                       cwd=cwd, redirect_stdout=True, enter_chroot=True,
-                       error_ok=True)
+  portageq = 'portageq'
+  if board:
+    portageq += '-%s' % board
+  binhost = RunCommand([portageq, 'envvar', envvar], cwd=cwd,
+                       redirect_stdout=True, enter_chroot=True, error_ok=True)
   return binhost.rstrip('\n')
 
 
 def _SetupBoard(buildroot, board='x86-generic'):
   """Wrapper around setup_board."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./setup_board', '--fast', '--default', '--board=%s' % board],
              cwd=cwd, enter_chroot=True)
 
 
@@ skipping 181 matching lines @@
     paths = [private_overlay]
   elif overlays == 'public':
     paths = [public_overlay]
   elif overlays == 'both':
     paths = [public_overlay, private_overlay]
   else:
     Die('Incorrect overlay configuration: %s' % overlays)
   return paths
 
 
-def _UploadPrebuilts(buildroot, board, overlay_config):
+def _UploadPrebuilts(buildroot, board, overlay_config, binhosts):
   """Upload prebuilts.
 
   Args:
     buildroot: The root directory where the build occurs.
     board: Board type that was built on this machine
     overlay_config: A string describing which overlays you want.
                     'private': Just the private overlay.
                     'public': Just the public overlay.
                     'both': Both the public and private overlays.
+    binhosts: The URLs of the current binhosts. Binaries that are already
+              present will not be uploaded twice. Empty URLs will be ignored.
   """
 
   cmd = ['%s/src/scripts/prebuilt.py' % buildroot,
          '--sync-binhost-conf',
+         '--sync-host',
          '--build-path', buildroot,
          '--board', board,
          '--prepend-version', 'preflight',
          '--key', _PREFLIGHT_BINHOST]
+  for binhost in binhosts:
+    if binhost:
+      cmd.extend(['--previous-binhost-url', binhost])
   if overlay_config == 'public':
     cmd.extend(['--upload', 'gs://chromeos-prebuilt'])
   else:
     assert overlay_config in ('private', 'both')
     cmd.extend(['--upload', 'chromeos-images:/var/www/prebuilt/',
                 '--binhost-base-url', 'http://chromeos-prebuilt'])
 
   RunCommand(cmd, cwd='%s/src/scripts' % buildroot)
 
 
@@ skipping 35 matching lines @@
   # Calculate list of overlay directories.
   overlays = _ResolveOverlays(buildroot, buildconfig['overlays'])
   board = buildconfig['board']
 
   try:
     _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch, overlays)
     chroot_path = os.path.join(buildroot, 'chroot')
     boardpath = os.path.join(chroot_path, 'build', board)
     if options.clobber or not os.path.isdir(buildroot):
       _FullCheckout(buildroot, tracking_branch, url=options.url)
+      new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST),
     else:
       old_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
       _IncrementalCheckout(buildroot)
       new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
       if old_binhost != new_binhost:
         RunCommand(['sudo', 'rm', '-rf', boardpath])
+    binhosts = [new_binhost, _GetPortageEnvVar(buildroot, None, _FULL_BINHOST)]
 
     # Check that all overlays can be found.
     for path in overlays:
       assert ':' not in path, 'Overlay must not contain colons: %s' % path
       if not os.path.isdir(path):
         Die('Missing overlay: %s' % path)
 
     if not os.path.isdir(chroot_path):
       _MakeChroot(buildroot)
 
@@ skipping 20 matching lines @@
         _ArchiveTestResults(buildroot, buildconfig['board'],
                             archive_dir=options.buildnumber,
                             test_results_dir=test_results_dir)
 
     if buildconfig['uprev']:
       # Don't push changes for developers.
       if not options.debug:
         if buildconfig['master']:
           # Master bot needs to check if the other slaves completed.
           if cbuildbot_comm.HaveSlavesCompleted(config):
-            _UploadPrebuilts(buildroot, board, buildconfig['overlays'])
+            _UploadPrebuilts(buildroot, board, buildconfig['overlays'],
+                             binhosts)
             _UprevPush(buildroot, tracking_branch, buildconfig['board'],
                        overlays)
           else:
             Die('CBUILDBOT - One of the slaves has failed!!!')
 
         else:
           # Publish my status to the master if its expecting it.
           if buildconfig['important']:
             cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)
 
   except:
     # Send failure to master bot.
     if not buildconfig['master'] and buildconfig['important']:
       cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED)
 
     raise
 
 
 if __name__ == '__main__':
     main()