Add all pfq configurations and add build_image / run_unit_tests.

bin/cbuildbot.py

 #!/usr/bin/python
 
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """CBuildbot is wrapper around the build process used by the pre-flight queue"""
 
 import errno
 import re
@@ skipping 39 matching lines @@
 
   # Print out the command before running.
   if print_cmd:
     print >> sys.stderr, 'CBUILDBOT -- RunCommand: ', ' '.join(cmd)
 
   proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin,
                           stdout=stdout, stderr=stderr)
   (output, error) = proc.communicate(input)
   if exit_code:
     return proc.returncode
+
   if not error_ok and proc.returncode != 0:
     raise Exception('Command "%s" failed.\n' % (' '.join(cmd)) +
                     (error_message or error or output or ''))
+
   return output
 
 
 def MakeDir(path, parents=False):
   """Basic wrapper around os.mkdirs.
 
   Keyword arguments:
   path -- Path to create.
   parents -- Follow mkdir -p logic.
 
@@ skipping 17 matching lines @@
   """
   while retries > 0:
     try:
       RunCommand(['repo', 'sync'], cwd=buildroot)
       if rw_checkout:
         # Always re-run in case of new git repos or repo sync
         # failed in a previous run because of a forced Stop Build.
         RunCommand(['repo', 'forall', '-c', 'git', 'config',
                     'url.ssh://git@gitrw.chromium.org:9222.pushinsteadof',
                     'http://git.chromium.org/git'], cwd=buildroot)
+
       retries = 0
     except:
       retries -= 1
       if retries > 0:
         print >> sys.stderr, 'CBUILDBOT -- Repo Sync Failed, retrying'
       else:
         print >> sys.stderr, 'CBUILDBOT -- Retries exhausted'
         raise
 
 # =========================== Command Helpers =================================
@@ skipping 13 matching lines @@
   # Parse line using re to get tuple.
   result_array = re.findall('.+name=\"([\w-]+)\".+path=\"(\S+)".+', output)
 
   # Create the array.
   for result in result_array:
     if len(result) != 2:
       print >> sys.stderr, 'Found in correct xml object %s', result
     else:
       # Remove pre-pended src directory from manifest.
       manifest_tuples.append([result[0], result[1].replace('src/', '')])
+
   return manifest_tuples
 
 
 def _GetCrosWorkOnSrcPath(buildroot, board, package, debug=False):
   """Returns ${CROS_WORKON_SRC_PATH} for given package."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   equery_cmd = ('equery-%s which %s' % (board, package)).split()
   ebuild_path = RunCommand(equery_cmd, cwd=cwd, redirect_stdout=True,
                              redirect_stderr=True, enter_chroot=True,
                              error_ok=True, print_cmd=debug)
   if ebuild_path:
     ebuild_cmd = ('ebuild-%s %s info' % (board, ebuild_path)).split()
     cros_workon_output = RunCommand(ebuild_cmd, cwd=cwd,
                                     redirect_stdout=True, redirect_stderr=True,
                                     enter_chroot=True, print_cmd=debug)
 
     temp = re.findall('CROS_WORKON_SRCDIR="(\S+)"', cros_workon_output)
     if temp:
       return temp[0]
+
   return None
 
 
 def _CreateRepoDictionary(buildroot, board, debug=False):
   """Returns the repo->list_of_ebuilds dictionary."""
   repo_dictionary = {}
   manifest_tuples = _GetAllGitRepos(buildroot)
-  print >> sys.stderr, 'Creating dictionary of git repos to portage packages ...'
+  print >> sys.stderr, (
+               'Creating dictionary of git repos to portage packages ...')
 
   cwd = os.path.join(buildroot, 'src', 'scripts')
   get_all_workon_pkgs_cmd = './cros_workon list --all'.split()
   packages = RunCommand(get_all_workon_pkgs_cmd, cwd=cwd,
                         redirect_stdout=True, redirect_stderr=True,
                         enter_chroot=True, print_cmd=debug)
   for package in packages.split():
     cros_workon_src_path = _GetCrosWorkOnSrcPath(buildroot, board, package)
     if cros_workon_src_path:
       for tuple in manifest_tuples:
         # This path tends to have the user's home_dir prepended to it.
         if cros_workon_src_path.endswith(tuple[1]):
           print >> sys.stderr, ('For %s found matching package %s' %
               (tuple[0], package))
           if repo_dictionary.has_key(tuple[0]):
             repo_dictionary[tuple[0]] += [package]
           else:
             repo_dictionary[tuple[0]] = [package]
+
   return repo_dictionary
 
 
 def _ParseRevisionString(revision_string, repo_dictionary):
   """Parses the given revision_string into a revision dictionary.
 
   Returns a list of tuples that contain [portage_package_name, commit_id] to
   update.
 
   Keyword arguments:
   revision_string -- revision_string with format
       'repo1.git@commit_1 repo2.git@commit2 ...'.
   repo_dictionary -- dictionary with git repository names as keys (w/out git)
       to portage package names.
 
   """
   # Using a dictionary removes duplicates.
   revisions = {}
   for revision in revision_string.split():
     # Format 'package@commit-id'.
     revision_tuple = revision.split('@')
     if len(revision_tuple) != 2:
       print >> sys.stderr, 'Incorrectly formatted revision %s' % revision
+
     repo_name = revision_tuple[0].replace('.git', '')
     # Might not have entry if no matching ebuild.
     if repo_dictionary.has_key(repo_name):
       # May be many corresponding packages to a given git repo e.g. kernel).
       for package in repo_dictionary[repo_name]:
         revisions[package] = revision_tuple[1]
+
   return revisions.items()
 
 
 def _UprevFromRevisionList(buildroot, revision_list):
   """Uprevs based on revision list."""
   if not revision_list:
     print >> sys.stderr, 'No packages found to uprev'
     return
+
   package_str = ''
   commit_str = ''
   for package, revision in revision_list:
     package_str += package + ' '
     commit_str += revision + ' '
+
   package_str = package_str.strip()
   commit_str = commit_str.strip()
 
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./cros_mark_as_stable',
               '--tracking_branch="cros/master"',
               '--packages="%s"' % package_str,
               '--commit_ids="%s"' % commit_str,
               'commit'],
               cwd=cwd, enter_chroot=True)
@@ skipping 32 matching lines @@
 def _SetupBoard(buildroot, board='x86-generic'):
   """Wrapper around setup_board."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./setup_board', '--fast', '--default', '--board=%s' % board],
              cwd=cwd)
 
 
 def _Build(buildroot):
   """Wrapper around build_packages."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
-  RunCommand(['./build_packages'], cwd=cwd)
+  RunCommand(['./build_packages'], cwd=cwd, enter_chroot=True)
+
+def _BuildImage(buildroot):
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  RunCommand(['./build_image'], cwd=cwd)
+
+def _RunUnitTests(buildroot):
+  cwd = os.path.join(buildroot, 'src', 'scripts')
+  RunCommand(['./cros_run_unit_tests'], cwd=cwd)
 
 
 def _UprevPackages(buildroot, revisionfile, board):
   """Uprevs a package based on given revisionfile.
 
   If revisionfile is set to None or does not resolve to an actual file, this
   function will uprev all packages.
 
   Keyword arguments:
   revisionfile -- string specifying a file that contains a list of revisions to
@@ skipping 38 matching lines @@
   RunCommand(['./cros_mark_as_stable', '--srcroot=..',
               '--tracking_branch="cros/master"',
               '--push_options', '--bypass-hooks -f', 'push'],
              cwd=cwd)
 
 
 def _GetConfig(config_name):
   """Gets the configuration for the build"""
   default = config['default']
   buildconfig = {}
-  if config.has_key(config_name):
-    buildconfig = config[config_name]
+  if not config.has_key(config_name):
+    print >> sys.stderr, 'Non-existent configuration specified.'
+    print >> sys.stderr, 'Please specify one of:'
+    config_names = config.keys()
+    config_names.sort()
+    for name in config_names:
+      print >> sys.stderr, '  %s' % name
+    sys.exit(1)
+
+  buildconfig = config[config_name]
+
   for key in default.iterkeys():
     if not buildconfig.has_key(key):
       buildconfig[key] = default[key]
+
   return buildconfig
 
 
 def main():
   # Parse options
   usage = "usage: %prog [options] cbuildbot_config"
   parser = optparse.OptionParser(usage=usage)
   parser.add_option('-r', '--buildroot',
                     help='root directory where build occurs', default=".")
   parser.add_option('-n', '--buildnumber',
                     help='build number', type='int', default=0)
   parser.add_option('-f', '--revisionfile',
                     help='file where new revisions are stored')
   parser.add_option('--noclobber', action='store_false', dest='clobber',
                     default=True,
                     help='Disables clobbering the buildroot on failure')
   (options, args) = parser.parse_args()
 
   buildroot = options.buildroot
   revisionfile = options.revisionfile
   clobber = options.clobber
 
   if len(args) == 1:
     buildconfig = _GetConfig(args[0])
   else:
     print >> sys.stderr, "Missing configuration description"
     parser.print_usage()
     sys.exit(1)
+
   try:
     if not os.path.isdir(buildroot):
       _FullCheckout(buildroot)
     else:
       _IncrementalCheckout(buildroot)
+
     chroot_path = os.path.join(buildroot, 'chroot')
     if not os.path.isdir(chroot_path):
       _MakeChroot(buildroot)
+
     boardpath = os.path.join(chroot_path, 'build', buildconfig['board'])
     if not os.path.isdir(boardpath):
       _SetupBoard(buildroot, board=buildconfig['board'])
+
     if buildconfig['uprev']:
       _UprevPackages(buildroot, revisionfile, board=buildconfig['board'])
+
     _Build(buildroot)
+    if buildconfig['unittests']:
+      _RunUnitTests(buildroot)
+
+    _BuildImage(buildroot)
     if buildconfig['uprev']:
       if buildconfig['master']:
         # Master bot needs to check if the other slaves completed.
         if cbuildbot_comm.HaveSlavesCompleted(config):
           _UprevPush(buildroot)
           _UprevCleanup(buildroot)
         else:
           # At least one of the slaves failed or we timed out.
           _UprevCleanup(buildroot)
           sys.stderr('CBUILDBOT - One of the slaves has failed!!!')
           sys.exit(1)
       else:
-        # Publish my status to the master.
-        cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)
+        # Publish my status to the master if its expecting it.
+        if buildconfig['important']:
+          cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)
+
         _UprevCleanup(buildroot)
   except:
     # Something went wrong, cleanup (being paranoid) for next build.
     if clobber:
       RunCommand(['sudo', 'rm', '-rf', buildroot], print_cmd=False)
+
     # Send failure to master bot.
-    if not buildconfig['master']:
+    if not buildconfig['master'] and buildconfig['important']:
       cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED)
+
     raise
 
 
 if __name__ == '__main__':
     main()