Move RunCommand, and Info/Warning/Die into common pylib

bin/cbuildbot.py

 #!/usr/bin/python
 
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """CBuildbot is wrapper around the build process used by the pre-flight queue"""
 
 import errno
 import re
 import optparse
 import os
-import subprocess
 import sys
 
 import cbuildbot_comm
 from cbuildbot_config import config
 
+sys.path.append(os.path.join(os.path.dirname(__file__), '../lib'))
+from cros_build_lib import Die, Info, RunCommand, Warning
+
 _DEFAULT_RETRIES = 3
 
 # ======================== Utility functions ================================
 
-def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None,
-               exit_code=False, redirect_stdout=False, redirect_stderr=False,
-               cwd=None, input=None, enter_chroot=False):
-  """Runs a shell command.
-
-  Keyword arguments:
-  print_cmd -- prints the command before running it.
-  error_ok -- does not raise an exception on error.
-  error_message -- prints out this message when an error occurrs.
-  exit_code -- returns the return code of the shell command.
-  redirect_stdout -- returns the stdout.
-  redirect_stderr -- holds stderr output until input is communicated.
-  cwd -- the working directory to run this cmd.
-  input -- input to pipe into this command through stdin.
-  enter_chroot -- this command should be run from within the chroot.
-
-  """
-  # Set default for variables.
-  stdout = None
-  stderr = None
-  stdin = None
-
-  # Modify defaults based on parameters.
-  if redirect_stdout:  stdout = subprocess.PIPE
-  if redirect_stderr:  stderr = subprocess.PIPE
-  if input:  stdin = subprocess.PIPE
-  if enter_chroot:  cmd = ['./enter_chroot.sh', '--'] + cmd
-
-  # Print out the command before running.
-  if print_cmd:
-    print >> sys.stderr, 'CBUILDBOT -- RunCommand: ', ' '.join(cmd)
-
-  proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin,
-                          stdout=stdout, stderr=stderr)
-  (output, error) = proc.communicate(input)
-  if exit_code:
-    return proc.returncode
-
-  if not error_ok and proc.returncode != 0:
-    raise Exception('Command "%s" failed.\n' % (' '.join(cmd)) +
-                    (error_message or error or output or ''))
-
-  return output
-
-
 def MakeDir(path, parents=False):
   """Basic wrapper around os.mkdirs.
 
   Keyword arguments:
   path -- Path to create.
   parents -- Follow mkdir -p logic.
 
   """
   try:
     os.makedirs(path)
@@ skipping 19 matching lines @@
         # Always re-run in case of new git repos or repo sync
         # failed in a previous run because of a forced Stop Build.
         RunCommand(['repo', 'forall', '-c', 'git', 'config',
                     'url.ssh://git@gitrw.chromium.org:9222.pushinsteadof',
                     'http://git.chromium.org/git'], cwd=buildroot)
 
       retries = 0
     except:
       retries -= 1
       if retries > 0:
-        print >> sys.stderr, 'CBUILDBOT -- Repo Sync Failed, retrying'
+        Warning('CBUILDBOT -- Repo Sync Failed, retrying')
       else:
-        print >> sys.stderr, 'CBUILDBOT -- Retries exhausted'
+        Warning('CBUILDBOT -- Retries exhausted')
         raise
 
 # =========================== Command Helpers =================================
 
 def _GetAllGitRepos(buildroot, debug=False):
   """Returns a list of tuples containing [git_repo, src_path]."""
   manifest_tuples = []
   # Gets all the git repos from a full repo manifest.
   repo_cmd = "repo manifest -o -".split()
   output = RunCommand(repo_cmd, cwd=buildroot, redirect_stdout=True,
                       redirect_stderr=True, print_cmd=debug)
 
   # Extract all lines containg a project.
   extract_cmd = ["grep", "project name="]
   output = RunCommand(extract_cmd, cwd=buildroot, input=output,
                       redirect_stdout=True, print_cmd=debug)
   # Parse line using re to get tuple.
   result_array = re.findall('.+name=\"([\w-]+)\".+path=\"(\S+)".+', output)
 
   # Create the array.
   for result in result_array:
     if len(result) != 2:
-      print >> sys.stderr, 'Found in correct xml object %s', result
+      Warning('Found incorrect xml object %s' % result)
     else:
       # Remove pre-pended src directory from manifest.
       manifest_tuples.append([result[0], result[1].replace('src/', '')])
 
   return manifest_tuples
 
 
 def _GetCrosWorkOnSrcPath(buildroot, board, package, debug=False):
   """Returns ${CROS_WORKON_SRC_PATH} for given package."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
@@ skipping 11 matching lines @@
     if temp:
       return temp[0]
 
   return None
 
 
 def _CreateRepoDictionary(buildroot, board, debug=False):
   """Returns the repo->list_of_ebuilds dictionary."""
   repo_dictionary = {}
   manifest_tuples = _GetAllGitRepos(buildroot)
-  print >> sys.stderr, (
-               'Creating dictionary of git repos to portage packages ...')
+  Info('Creating dictionary of git repos to portage packages ...')
 
   cwd = os.path.join(buildroot, 'src', 'scripts')
   get_all_workon_pkgs_cmd = './cros_workon list --all'.split()
   packages = RunCommand(get_all_workon_pkgs_cmd, cwd=cwd,
                         redirect_stdout=True, redirect_stderr=True,
                         enter_chroot=True, print_cmd=debug)
   for package in packages.split():
     cros_workon_src_path = _GetCrosWorkOnSrcPath(buildroot, board, package)
     if cros_workon_src_path:
       for tuple in manifest_tuples:
         # This path tends to have the user's home_dir prepended to it.
         if cros_workon_src_path.endswith(tuple[1]):
-          print >> sys.stderr, ('For %s found matching package %s' %
-              (tuple[0], package))
+          Info('For %s found matching package %s' % (tuple[0], package))
           if repo_dictionary.has_key(tuple[0]):
             repo_dictionary[tuple[0]] += [package]
           else:
             repo_dictionary[tuple[0]] = [package]
 
   return repo_dictionary
 
 
 def _ParseRevisionString(revision_string, repo_dictionary):
   """Parses the given revision_string into a revision dictionary.
 
   Returns a list of tuples that contain [portage_package_name, commit_id] to
   update.
 
   Keyword arguments:
   revision_string -- revision_string with format
       'repo1.git@commit_1 repo2.git@commit2 ...'.
   repo_dictionary -- dictionary with git repository names as keys (w/out git)
       to portage package names.
 
   """
   # Using a dictionary removes duplicates.
   revisions = {}
   for revision in revision_string.split():
     # Format 'package@commit-id'.
     revision_tuple = revision.split('@')
     if len(revision_tuple) != 2:
-      print >> sys.stderr, 'Incorrectly formatted revision %s' % revision
+      Warning('Incorrectly formatted revision %s' % revision)
 
     repo_name = revision_tuple[0].replace('.git', '')
     # Might not have entry if no matching ebuild.
     if repo_dictionary.has_key(repo_name):
       # May be many corresponding packages to a given git repo e.g. kernel).
       for package in repo_dictionary[repo_name]:
         revisions[package] = revision_tuple[1]
 
   return revisions.items()
 
 
 def _UprevFromRevisionList(buildroot, revision_list):
   """Uprevs based on revision list."""
   if not revision_list:
-    print >> sys.stderr, 'No packages found to uprev'
+    Info('No packages found to uprev')
     return
 
   package_str = ''
   commit_str = ''
   for package, revision in revision_list:
     package_str += package + ' '
     commit_str += revision + ' '
 
   package_str = package_str.strip()
   commit_str = commit_str.strip()
@@ skipping 69 matching lines @@
       uprev.
   """
   # Purposefully set to None as it means Force Build was pressed.
   revisions = 'None'
   if (revisionfile):
     try:
       rev_file = open(revisionfile)
       revisions = rev_file.read()
       rev_file.close()
     except Exception, e:
-      print >> sys.stderr, 'Error reading %s, revving all' % revisionfile
-      print e
+      Warning('Error reading %s, revving all' % revisionfile)
       revisions = 'None'
 
   revisions = revisions.strip()
 
   # TODO(sosa): Un-comment once we close individual trees.
-  # Revisions == "None" indicates a Force Build.
+  # revisions == "None" indicates a Force Build.
   #if revisions != 'None':
   #  print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions
   #  revision_list = _ParseRevisionString(revisions,
   #      _CreateRepoDictionary(buildroot, board))
   #  _UprevFromRevisionList(buildroot, revision_list)
   #else:
-  print >> sys.stderr, 'CBUILDBOT Revving all'
+  Info('CBUILDBOT Revving all')
   _UprevAllPackages(buildroot)
 
 
 def _UprevCleanup(buildroot):
   """Clean up after a previous uprev attempt."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./cros_mark_as_stable', '--srcroot=..',
               '--tracking_branch="cros/master"', 'clean'],
              cwd=cwd)
 
 
 def _UprevPush(buildroot):
   """Pushes uprev changes to the main line."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   RunCommand(['./cros_mark_as_stable', '--srcroot=..',
               '--tracking_branch="cros/master"',
               '--push_options', '--bypass-hooks -f', 'push'],
              cwd=cwd)
 
 
 def _GetConfig(config_name):
   """Gets the configuration for the build"""
   default = config['default']
   buildconfig = {}
   if not config.has_key(config_name):
-    print >> sys.stderr, 'Non-existent configuration specified.'
-    print >> sys.stderr, 'Please specify one of:'
+    Warning('Non-existent configuration specified.')
+    Warning('Please specify one of:')
     config_names = config.keys()
     config_names.sort()
     for name in config_names:
-      print >> sys.stderr, '  %s' % name
+      Warning('  %s' % name)
     sys.exit(1)
 
   buildconfig = config[config_name]
 
   for key in default.iterkeys():
     if not buildconfig.has_key(key):
       buildconfig[key] = default[key]
 
   return buildconfig
 
@@ skipping 16 matching lines @@
   buildroot = options.buildroot
   revisionfile = options.revisionfile
 
   # Passed option to clobber.
   if options.clobber:
     RunCommand(['sudo', 'rm', '-rf', buildroot])
 
   if len(args) == 1:
     buildconfig = _GetConfig(args[0])
   else:
-    print >> sys.stderr, "Missing configuration description"
+    Warning('Missing configuration description')
     parser.print_usage()
     sys.exit(1)
 
   try:
     if not os.path.isdir(buildroot):
       _FullCheckout(buildroot)
     else:
       _IncrementalCheckout(buildroot)
 
     chroot_path = os.path.join(buildroot, 'chroot')
@@ skipping 14 matching lines @@
     _BuildImage(buildroot)
     if buildconfig['uprev']:
       if buildconfig['master']:
         # Master bot needs to check if the other slaves completed.
         if cbuildbot_comm.HaveSlavesCompleted(config):
           _UprevPush(buildroot)
           _UprevCleanup(buildroot)
         else:
           # At least one of the slaves failed or we timed out.
           _UprevCleanup(buildroot)
-          sys.stderr('CBUILDBOT - One of the slaves has failed!!!')
-          sys.exit(1)
+          Die('CBUILDBOT - One of the slaves has failed!!!')
       else:
         # Publish my status to the master if its expecting it.
         if buildconfig['important']:
           cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)
 
         _UprevCleanup(buildroot)
   except:
     # Send failure to master bot.
     if not buildconfig['master'] and buildconfig['important']:
       cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED)
 
     raise
 
 
 if __name__ == '__main__':
     main()