Add buildbot functionality to cbuildbot.

bin/cbuildbot.py

 #!/usr/bin/python
 
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """CBuildbot is wrapper around the build process used by the pre-flight queue"""
 
 import errno
 import heapq
@@ skipping 311 matching lines @@
   """Performs a checkout without clobbering previous checkout."""
   RepoSync(buildroot, retries)
 
 
 def _MakeChroot(buildroot):
   """Wrapper around make_chroot."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./make_chroot', '--fast'], cwd=cwd)
 
 
+def _CachePackages(buildroot, board, package_cache, restore):

[sosa, 2011/01/25 22:54:48]

As per discussion, let's remove this.

[dgarrett, 2011/01/25 23:38:51]

Done.

+  """Copy pre-built packages from the pre-flight into our chroot
+
+  buildroot: The root directory where the build occurs. Must be an absolute
+             path.
+  
+  package_cache: Directory into which built packages are copied off or
+                 restored from. Usually /var/buildbot-package-cache.
+                 
+  restore: Boolean. True to restore packages, False to save them off.
+  """
+  chroot_host_dir = os.path.join(buildroot,
+                                 'chroot/var/lib/portage/pkgs/')
+  chroot_board_dir = os.path.join(buildroot,
+                                  'chroot/build/',
+                                  board,
+                                  'packages/')
+
+  cache_host_dir = os.path.join(package_cache, 'host/')
+  cache_board_dir = os.path.join(package_cache, board+'/')
+
+  if restore:
+    copies = { cache_host_dir : chroot_host_dir,
+               cache_board_dir : chroot_board_dir }
+  else:
+    copies = { chroot_host_dir : cache_host_dir,
+               chroot_board_dir : cache_board_dir }
+
+  for src, dest in copies.items():
+    
+    print "***   _CachePackages %s %s" % (src, dest)
+    
+    if not os.path.exists(src):
+      continue
+
+    # Make sure the src permissions work
+    # Make sure the dest exists
+    # copy the files
+    RunCommand(['sudo', 'chmod', 'a+rwx', src])
+    RunCommand(['sudo', 'mkdir', '-p', dest])
+    RunCommand(['sudo', 'rsync', '-az', src, dest])
+
+
 def _GetPortageEnvVar(buildroot, board, envvar):
   """Get a portage environment variable for the specified board, if any.
 
   buildroot: The root directory where the build occurs. Must be an absolute
              path.
   board: Board type that was built on this machine. E.g. x86-generic. If this
          is None, get the env var from the host.
   envvar: The environment variable to get. E.g. 'PORTAGE_BINHOST'.
 
   Returns:
@@ skipping 136 matching lines @@
          '--overlays=%s' % ':'.join(overlays),
          '--tracking_branch=%s' % tracking_branch
         ]
   if dryrun:
     cmd.append('--dryrun')
 
   cmd.append('push')
   RunCommand(cmd, cwd=cwd)
 
 
+def _ArchiveBuild(bot_id, buildconfig, buildnumber):

[sosa, 2011/01/25 22:54:48]

Can you rename to _OldArchiveBuild or LegacyArchiveBuild.  We're moving away
from that script per say so let's call this something 

[dgarrett, 2011/01/25 23:38:51]

Done.

+  """Adds a step to the factory to archive a build."""
+
+  # Fixed properties
+  keep_max = 3
+  gsutil_archive = 'gs://chromeos-archive/' + bot_id
+
+  # TODO: Currently, --to is based on a known config for the buildbot

[sosa, 2011/01/25 22:54:48]

Remove this TODO.  It is obselete (we use gsutil from this to dir to archive
things)

+  # slaves storing their build results locally.  When we can store the
+  # results on chrome-web (after open source release), this needs
+  # refactoring.
+
+  cmd = ['./archive_build.sh',
+         '--build_number', str(buildnumber),
+         '--to', '/var/www/archive/' + bot_id,
+         '--keep_max', str(keep_max),
+         '--prebuilt_upload',
+         '--board', buildconfig['board'],
+
+         '--acl', '/home/chrome-bot/slave_archive_acl',
+         '--gsutil_archive', gsutil_archive,
+         '--gsd_gen_index',
+           '/b/scripts/gsd_generate_index/gsd_generate_index.py',
+         '--gsutil', '/b/scripts/slave/gsutil',
+         '--test_mod'
+  ]
+
+  if buildconfig.get('test_mod', True):

[sosa, 2011/01/25 22:54:48]

isn't this equivalent to buildconfig.get('testmod')?

[dgarrett, 2011/01/25 23:38:51]

No. This gives it a default value, so that it doesn't have to explicitly be
defined for every config.

+    cmd += ['--test_mod']

[sosa, 2011/01/25 22:54:48]

append rather than +=

[dgarrett, 2011/01/25 23:38:51]

Done.

+  
+  if buildconfig.get('factory_install_mod', True):
+    cmd += ['--factory_install_mod']

[sosa, 2011/01/25 22:54:48]

append rather than +=

[dgarrett, 2011/01/25 23:38:51]

Done.

+  
+  if buildconfig.get('factory_test_mod', True):
+    cmd += ['--factory_test_mod']

[sosa, 2011/01/25 22:54:48]

append rather than +=

[dgarrett, 2011/01/25 23:38:51]

Done.

+
+  Warning('***** ***** ArchiveBuild CMD: ' + ' '.join(cmd))
+  #RunCommand(cmd)

[sosa, 2011/01/25 22:54:48]

if not debug rather than comment out

[dgarrett, 2011/01/25 23:38:51]

Makes sense.

+
 def _ArchiveTestResults(buildroot, board, test_results_dir,
                         gsutil, archive_dir, acl):
   """Archives the test results into Google Storage
 
   Takes the results from the test_results_dir and the last qemu image and
   uploads them to Google Storage.
 
   Arguments:
     buildroot: Root directory where build occurs
     board: Board to find the qemu image.
@@ skipping 103 matching lines @@
 
   RunCommand(cmd, cwd=cwd)
 
 
 def main():
   # Parse options
   usage = "usage: %prog [options] cbuildbot_config"
   parser = optparse.OptionParser(usage=usage)
   parser.add_option('-a', '--acl', default='private',
                     help='ACL to set on GSD archives')
+  parser.add_option('--archive_build', action='store_true', default=False,
+                    help='Run the archive_build script.')
   parser.add_option('-r', '--buildroot',
                     help='root directory where build occurs', default=".")
   parser.add_option('-n', '--buildnumber',
                     help='build number', type='int', default=0)
   parser.add_option('--chrome_rev', default=None, type='string',
                     dest='chrome_rev',
                     help=('Chrome_rev of type [tot|latest_release|'
                           'sticky_release]'))
   parser.add_option('-g', '--gsutil', default='', help='Location of gsutil')
   parser.add_option('-c', '--gsutil_archive', default='',
                     help='Datastore archive location')
   parser.add_option('--clobber', action='store_true', dest='clobber',
                     default=False,
                     help='Clobbers an old checkout before syncing')
   parser.add_option('--debug', action='store_true', dest='debug',
                     default=False,
                     help='Override some options to run as a developer.')
+  parser.add_option('--nobuild', action='store_false', dest='build',
+                    default=True,
+                    help="Don't actually build (for cbuildbot devel")

[sosa, 2011/01/25 22:54:48]

dev*

[dgarrett, 2011/01/25 23:38:51]

Done.

   parser.add_option('--noprebuilts', action='store_false', dest='prebuilts',
                     default=True,
                     help="Don't upload prebuilts.")
   parser.add_option('--nosync', action='store_false', dest='sync',
                     default=True,
                     help="Don't sync before building.")
   parser.add_option('--notests', action='store_false', dest='tests',
                     default=True,
                     help='Override values from buildconfig and run no tests.')
+  parser.add_option('--package-cache', dest='package_cache',

[sosa, 2011/01/25 22:54:48]

As per our discussions with scottz / raja ... let's remove this option.

[dgarrett, 2011/01/25 23:38:51]

Done.

+                    default=None,
+                    help='Directory to save/restore packages (for buildbot).')
   parser.add_option('-f', '--revisionfile',
                     help='file where new revisions are stored')
   parser.add_option('-t', '--tracking-branch', dest='tracking_branch',
                     default='cros/master', help='Run the buildbot on a branch')
   parser.add_option('-u', '--url', dest='url',
                     default='http://git.chromium.org/git/manifest',
                     help='Run the buildbot on internal manifest')
 
   (options, args) = parser.parse_args()
 
   buildroot = os.path.abspath(options.buildroot)
   revisionfile = options.revisionfile
   tracking_branch = options.tracking_branch
   chrome_atom_to_build = None
 
   if len(args) >= 1:
-    buildconfig = _GetConfig(args[-1])
+    bot_id = args[-1]
+    buildconfig = _GetConfig(bot_id)
   else:
     Warning('Missing configuration description')
     parser.print_usage()
     sys.exit(1)
 
   try:
     # Calculate list of overlay directories.
     rev_overlays = _ResolveOverlays(buildroot, buildconfig['rev_overlays'])
     push_overlays = _ResolveOverlays(buildroot, buildconfig['push_overlays'])
     # We cannot push to overlays that we don't rev.
@@ skipping 19 matching lines @@
     emptytree = (old_binhost and old_binhost != new_binhost)
 
     # Check that all overlays can be found.
     for path in rev_overlays:
       if not os.path.isdir(path):
         Die('Missing overlay: %s' % path)
 
     if not os.path.isdir(chroot_path):
       _MakeChroot(buildroot)
 
+    if options.package_cache:
+      _CachePackages(buildroot, board,

[sosa, 2011/01/25 22:54:48]

Remove

[dgarrett, 2011/01/25 23:38:51]

Done.

+                     options.package_cache,
+                     restore=True)
+
     if not os.path.isdir(boardpath):
       _SetupBoard(buildroot, board=buildconfig['board'])
 
     # Perform uprev.  If chrome_uprev is set, rev Chrome ebuilds.
     if options.chrome_rev:
       chrome_atom_to_build = _MarkChromeAsStable(buildroot, tracking_branch,
                                                  options.chrome_rev, board)
       # If we found nothing to rev, we're done here.
       if not chrome_atom_to_build:
         return
 
     elif buildconfig['uprev']:
       _UprevPackages(buildroot, tracking_branch, revisionfile,
                      buildconfig['board'], rev_overlays)
 
     _EnableLocalAccount(buildroot)
-    _Build(buildroot, emptytree)
+    
+    if options.build:
+      _Build(buildroot, emptytree)
+
+    if options.package_cache:

[sosa, 2011/01/25 22:54:48]

Remove

[dgarrett, 2011/01/25 23:38:51]

Done.

+      _CachePackages(buildroot, board,
+                     options.package_cache,
+                     restore=False)
 
     if buildconfig['unittests'] and options.tests:
       _RunUnitTests(buildroot)
 
-    _BuildImage(buildroot)
+    if options.build:

[sosa, 2011/01/25 22:54:48]

Merge into other build cmd

[dgarrett, 2011/01/25 23:38:51]

Done.

+      _BuildImage(buildroot)
 
     if buildconfig['tests'] and options.tests:
       _BuildVMImageForTesting(buildroot)
       test_results_dir = '/tmp/run_remote_tests.%s' % options.buildnumber
       try:
         _RunSmokeSuite(buildroot, test_results_dir)
         _RunAUTest(buildroot, buildconfig['board'])
       finally:
         if not options.debug:
           archive_full_path = os.path.join(options.gsutil_archive,
@@ skipping 14 matching lines @@
                              [new_binhost])
           _UprevPush(buildroot, tracking_branch, buildconfig['board'],
                      push_overlays, options.debug)
         else:
           Die('CBUILDBOT - One of the slaves has failed!!!')
 
       else:
         # Publish my status to the master if its expecting it.
         if buildconfig['important'] and not options.debug:
           cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)
-
+    
+    if options.archive_build:
+      _ArchiveBuild(bot_id, buildconfig, options.buildnumber)
   except:
     # Send failure to master bot.
     if not buildconfig['master'] and buildconfig['important']:
       cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED)
 
     raise
 
 
 if __name__ == '__main__':
     main()