Add buildbot functionality to cbuildbot.

bin/cbuildbot.py

Index: bin/cbuildbot.py
diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py
index 168d6212df414aa2557050d13fbdc84b57693855..617f2b771060f900c85b8fc39904b647185655dd 100755
--- a/bin/cbuildbot.py
+++ b/bin/cbuildbot.py
@@ -329,6 +329,49 @@ def _MakeChroot(buildroot):
   RunCommand(['./make_chroot', '--fast'], cwd=cwd)
 
 
+def _CachePackages(buildroot, board, package_cache, restore):

[sosa, 2011/01/25 22:54:48]

As per discussion, let's remove this.

[dgarrett, 2011/01/25 23:38:51]

Done.

+  """Copy pre-built packages from the pre-flight into our chroot
+
+  buildroot: The root directory where the build occurs. Must be an absolute
+             path.
+  
+  package_cache: Directory into which built packages are copied off or
+                 restored from. Usually /var/buildbot-package-cache.
+                 
+  restore: Boolean. True to restore packages, False to save them off.
+  """
+  chroot_host_dir = os.path.join(buildroot,
+                                 'chroot/var/lib/portage/pkgs/')
+  chroot_board_dir = os.path.join(buildroot,
+                                  'chroot/build/',
+                                  board,
+                                  'packages/')
+
+  cache_host_dir = os.path.join(package_cache, 'host/')
+  cache_board_dir = os.path.join(package_cache, board+'/')
+
+  if restore:
+    copies = { cache_host_dir : chroot_host_dir,
+               cache_board_dir : chroot_board_dir }
+  else:
+    copies = { chroot_host_dir : cache_host_dir,
+               chroot_board_dir : cache_board_dir }
+
+  for src, dest in copies.items():
+    
+    print "***   _CachePackages %s %s" % (src, dest)
+    
+    if not os.path.exists(src):
+      continue
+
+    # Make sure the src permissions work
+    # Make sure the dest exists
+    # copy the files
+    RunCommand(['sudo', 'chmod', 'a+rwx', src])
+    RunCommand(['sudo', 'mkdir', '-p', dest])
+    RunCommand(['sudo', 'rsync', '-az', src, dest])
+
+
 def _GetPortageEnvVar(buildroot, board, envvar):
   """Get a portage environment variable for the specified board, if any.
 
@@ -485,6 +528,45 @@ def _UprevPush(buildroot, tracking_branch, board, overlays, dryrun):
   RunCommand(cmd, cwd=cwd)
 
 
+def _ArchiveBuild(bot_id, buildconfig, buildnumber):

[sosa, 2011/01/25 22:54:48]

Can you rename to _OldArchiveBuild or LegacyArchiveBuild.  We're moving away
from that script per say so let's call this something 

[dgarrett, 2011/01/25 23:38:51]

Done.

+  """Adds a step to the factory to archive a build."""
+
+  # Fixed properties
+  keep_max = 3
+  gsutil_archive = 'gs://chromeos-archive/' + bot_id
+
+  # TODO: Currently, --to is based on a known config for the buildbot

[sosa, 2011/01/25 22:54:48]

Remove this TODO.  It is obselete (we use gsutil from this to dir to archive
things)

+  # slaves storing their build results locally.  When we can store the
+  # results on chrome-web (after open source release), this needs
+  # refactoring.
+
+  cmd = ['./archive_build.sh',
+         '--build_number', str(buildnumber),
+         '--to', '/var/www/archive/' + bot_id,
+         '--keep_max', str(keep_max),
+         '--prebuilt_upload',
+         '--board', buildconfig['board'],
+
+         '--acl', '/home/chrome-bot/slave_archive_acl',
+         '--gsutil_archive', gsutil_archive,
+         '--gsd_gen_index',
+           '/b/scripts/gsd_generate_index/gsd_generate_index.py',
+         '--gsutil', '/b/scripts/slave/gsutil',
+         '--test_mod'
+  ]
+
+  if buildconfig.get('test_mod', True):

[sosa, 2011/01/25 22:54:48]

isn't this equivalent to buildconfig.get('testmod')?

[dgarrett, 2011/01/25 23:38:51]

No. This gives it a default value, so that it doesn't have to explicitly be
defined for every config.

+    cmd += ['--test_mod']

[sosa, 2011/01/25 22:54:48]

append rather than +=

[dgarrett, 2011/01/25 23:38:51]

Done.

+  
+  if buildconfig.get('factory_install_mod', True):
+    cmd += ['--factory_install_mod']

[sosa, 2011/01/25 22:54:48]

append rather than +=

[dgarrett, 2011/01/25 23:38:51]

Done.

+  
+  if buildconfig.get('factory_test_mod', True):
+    cmd += ['--factory_test_mod']

[sosa, 2011/01/25 22:54:48]

append rather than +=

[dgarrett, 2011/01/25 23:38:51]

Done.

+
+  Warning('***** ***** ArchiveBuild CMD: ' + ' '.join(cmd))
+  #RunCommand(cmd)

[sosa, 2011/01/25 22:54:48]

if not debug rather than comment out

[dgarrett, 2011/01/25 23:38:51]

Makes sense.

+
 def _ArchiveTestResults(buildroot, board, test_results_dir,
                         gsutil, archive_dir, acl):
   """Archives the test results into Google Storage
@@ -608,6 +690,8 @@ def main():
   parser = optparse.OptionParser(usage=usage)
   parser.add_option('-a', '--acl', default='private',
                     help='ACL to set on GSD archives')
+  parser.add_option('--archive_build', action='store_true', default=False,
+                    help='Run the archive_build script.')
   parser.add_option('-r', '--buildroot',
                     help='root directory where build occurs', default=".")
   parser.add_option('-n', '--buildnumber',
@@ -625,6 +709,9 @@ def main():
   parser.add_option('--debug', action='store_true', dest='debug',
                     default=False,
                     help='Override some options to run as a developer.')
+  parser.add_option('--nobuild', action='store_false', dest='build',
+                    default=True,
+                    help="Don't actually build (for cbuildbot devel")

[sosa, 2011/01/25 22:54:48]

dev*

[dgarrett, 2011/01/25 23:38:51]

Done.

   parser.add_option('--noprebuilts', action='store_false', dest='prebuilts',
                     default=True,
                     help="Don't upload prebuilts.")
@@ -634,6 +721,9 @@ def main():
   parser.add_option('--notests', action='store_false', dest='tests',
                     default=True,
                     help='Override values from buildconfig and run no tests.')
+  parser.add_option('--package-cache', dest='package_cache',

[sosa, 2011/01/25 22:54:48]

As per our discussions with scottz / raja ... let's remove this option.

[dgarrett, 2011/01/25 23:38:51]

Done.

+                    default=None,
+                    help='Directory to save/restore packages (for buildbot).')
   parser.add_option('-f', '--revisionfile',
                     help='file where new revisions are stored')
   parser.add_option('-t', '--tracking-branch', dest='tracking_branch',
@@ -650,7 +740,8 @@ def main():
   chrome_atom_to_build = None
 
   if len(args) >= 1:
-    buildconfig = _GetConfig(args[-1])
+    bot_id = args[-1]
+    buildconfig = _GetConfig(bot_id)
   else:
     Warning('Missing configuration description')
     parser.print_usage()
@@ -690,6 +781,11 @@ def main():
     if not os.path.isdir(chroot_path):
       _MakeChroot(buildroot)
 
+    if options.package_cache:
+      _CachePackages(buildroot, board,

[sosa, 2011/01/25 22:54:48]

Remove

[dgarrett, 2011/01/25 23:38:51]

Done.

+                     options.package_cache,
+                     restore=True)
+
     if not os.path.isdir(boardpath):
       _SetupBoard(buildroot, board=buildconfig['board'])
 
@@ -706,12 +802,20 @@ def main():
                      buildconfig['board'], rev_overlays)
 
     _EnableLocalAccount(buildroot)
-    _Build(buildroot, emptytree)
+    
+    if options.build:
+      _Build(buildroot, emptytree)
+
+    if options.package_cache:

[sosa, 2011/01/25 22:54:48]

Remove

[dgarrett, 2011/01/25 23:38:51]

Done.

+      _CachePackages(buildroot, board,
+                     options.package_cache,
+                     restore=False)
 
     if buildconfig['unittests'] and options.tests:
       _RunUnitTests(buildroot)
 
-    _BuildImage(buildroot)
+    if options.build:

[sosa, 2011/01/25 22:54:48]

Merge into other build cmd

[dgarrett, 2011/01/25 23:38:51]

Done.

+      _BuildImage(buildroot)
 
     if buildconfig['tests'] and options.tests:
       _BuildVMImageForTesting(buildroot)
@@ -746,7 +850,9 @@ def main():
         # Publish my status to the master if its expecting it.
         if buildconfig['important'] and not options.debug:
           cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)
-
+    
+    if options.archive_build:
+      _ArchiveBuild(bot_id, buildconfig, options.buildnumber)
   except:
     # Send failure to master bot.
     if not buildconfig['master'] and buildconfig['important']: