Index: bin/cbuildbot.py |
diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py |
index da99f5b243d106a2b2c8e392c69da057a862acf7..e9d2f152db30bc57e460edaa56439485afc7a51a 100755 |
--- a/bin/cbuildbot.py |
+++ b/bin/cbuildbot.py |
@@ -4,9 +4,12 @@ |
# Use of this source code is governed by a BSD-style license that can be |
# found in the LICENSE file. |
+"""CBuildbot is wrapper around the build process used by the pre-flight queue""" |
+ |
import errno |
import optparse |
import os |
+import re |
import shutil |
import subprocess |
import sys |
@@ -15,26 +18,40 @@ from cbuildbot_config import config |
_DEFAULT_RETRIES = 3 |
-# Utility functions |
+# ======================== Utility functions ================================ |
def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None, |
exit_code=False, redirect_stdout=False, redirect_stderr=False, |
- cwd=None, input=None): |
+ cwd=None, input=None, enter_chroot=False): |
+ """Runs a shell command. |
+ |
+ Keyword arguments: |
+ print_cmd -- prints the command before running it. |
+ error_ok -- does not raise an exception on error. |
+ error_message -- prints out this message when an error occurrs. |
+ exit_code -- returns the return code of the shell command. |
+ redirect_stdout -- returns the stdout. |
+ redirect_stderr -- holds stderr output until input is communicated. |
+ cwd -- the working directory to run this cmd. |
+ input -- input to pipe into this command through stdin. |
+ enter_chroot -- this command should be run from within the chroot. |
+ |
+ """ |
+ # Set default for variables. |
+ stdout = None |
+ stderr = None |
+ stdin = None |
+ |
+ # Modify defaults based on parameters. |
+ if redirect_stdout: stdout = subprocess.PIPE |
+ if redirect_stderr: stderr = subprocess.PIPE |
+ if input: stdin = subprocess.PIPE |
+ if enter_chroot: cmd = ['./enter_chroot.sh', '--'] + cmd |
+ |
# Print out the command before running. |
if print_cmd: |
- print >> sys.stderr, "CBUILDBOT -- RunCommand:", ' '.join(cmd) |
- if redirect_stdout: |
- stdout = subprocess.PIPE |
- else: |
- stdout = None |
- if redirect_stderr: |
- stderr = subprocess.PIPE |
- else: |
- stderr = None |
- if input: |
- stdin = subprocess.PIPE |
- else: |
- stdin = None |
+ print >> sys.stderr, 'CBUILDBOT -- RunCommand: ', ' '.join(cmd) |
+ |
proc = subprocess.Popen(cmd, cwd=cwd, stdin=stdin, |
stdout=stdout, stderr=stderr) |
(output, error) = proc.communicate(input) |
@@ -45,7 +62,15 @@ def RunCommand(cmd, print_cmd=True, error_ok=False, error_message=None, |
(error_message or error or output or '')) |
return output |
+ |
def MakeDir(path, parents=False): |
+ """Basic wrapper around os.mkdirs. |
+ |
+ Keyword arguments: |
+ path -- Path to create. |
+ parents -- Follow mkdir -p logic. |
+ |
+ """ |
try: |
os.makedirs(path) |
except OSError, e: |
@@ -54,7 +79,15 @@ def MakeDir(path, parents=False): |
else: |
raise |
-def RepoSync(buildroot, rw_checkout, retries=_DEFAULT_RETRIES): |
+ |
+def RepoSync(buildroot, rw_checkout=False, retries=_DEFAULT_RETRIES): |
+ """Uses repo to checkout the source code. |
+ |
+ Keyword arguments: |
+ rw_checkout -- Reconfigure repo after sync'ing to read-write. |
+ retries -- Number of retries to try before failing on the sync. |
+ |
+ """ |
while retries > 0: |
try: |
RunCommand(['repo', 'sync'], cwd=buildroot) |
@@ -73,70 +106,221 @@ def RepoSync(buildroot, rw_checkout, retries=_DEFAULT_RETRIES): |
print >> sys.stderr, 'CBUILDBOT -- Retries exhausted' |
raise |
-# Main functions |
+# =========================== Command Helpers ================================= |
+ |
+def _GetAllGitRepos(buildroot, debug=False): |
+ """Returns a list of tuples containing [git_repo, src_path].""" |
+ manifest_tuples = [] |
+ # Gets all the git repos from a full repo manifest. |
+ repo_cmd = "repo manifest -o -".split() |
+ output = RunCommand(repo_cmd, cwd=buildroot, redirect_stdout=True, |
+ redirect_stderr=True, print_cmd=debug) |
+ |
+ # Extract all lines containg a project. |
+ extract_cmd = ["grep", "project name="] |
+ output = RunCommand(extract_cmd, cwd=buildroot, input=output, |
+ redirect_stdout=True, print_cmd=debug) |
+ # Parse line using re to get tuple. |
+ result_array = re.findall('.+name=\"([\w-]+)\".+path=\"(\S+)".+', output) |
+ |
+ # Create the array. |
+ for result in result_array: |
+ if len(result) != 2: |
+ print >> sys.stderr, 'Found in correct xml object %s', result |
+ else: |
+ # Remove pre-pended src directory from manifest. |
+ manifest_tuples.append([result[0], result[1].replace('src/', '')]) |
+ return manifest_tuples |
+ |
+ |
+def _GetCrosWorkOnSrcPath(buildroot, board, package, debug=False): |
+ """Returns ${CROS_WORKON_SRC_PATH} for given package.""" |
+ cwd = os.path.join(buildroot, 'src', 'scripts') |
+ equery_cmd = ('equery-%s which %s' % (board, package)).split() |
+ ebuild_path = RunCommand(equery_cmd, cwd=cwd, redirect_stdout=True, |
+ redirect_stderr=True, enter_chroot=True, |
+ error_ok=True, print_cmd=debug) |
+ if ebuild_path: |
+ ebuild_cmd = ('ebuild-%s %s info' % (board, ebuild_path)).split() |
+ cros_workon_output = RunCommand(ebuild_cmd, cwd=cwd, |
+ redirect_stdout=True, redirect_stderr=True, |
+ enter_chroot=True, print_cmd=debug) |
+ |
+ temp = re.findall('CROS_WORKON_SRCDIR="(\S+)"', cros_workon_output) |
+ if temp: |
+ return temp[0] |
+ return None |
+ |
+ |
+def _CreateRepoDictionary(buildroot, board, debug=False): |
+ """Returns the repo->list_of_ebuilds dictionary.""" |
+ repo_dictionary = {} |
+ manifest_tuples = _GetAllGitRepos(buildroot) |
+ print >> sys.stderr, 'Creating dictionary of git repos to portage packages ...' |
+ |
+ cwd = os.path.join(buildroot, 'src', 'scripts') |
+ get_all_workon_pkgs_cmd = './cros_workon list --all'.split() |
+ packages = RunCommand(get_all_workon_pkgs_cmd, cwd=cwd, |
+ redirect_stdout=True, redirect_stderr=True, |
+ enter_chroot=True, print_cmd=debug) |
+ for package in packages.split(): |
+ cros_workon_src_path = _GetCrosWorkOnSrcPath(buildroot, board, package) |
+ if cros_workon_src_path: |
+ for tuple in manifest_tuples: |
+ # This path tends to have the user's home_dir prepended to it. |
+ if cros_workon_src_path.endswith(tuple[1]): |
+ print >> sys.stderr, ('For %s found matching package %s' % |
+ (tuple[0], package)) |
+ if repo_dictionary.has_key(tuple[0]): |
+ repo_dictionary[tuple[0]] += [package] |
+ else: |
+ repo_dictionary[tuple[0]] = [package] |
+ return repo_dictionary |
+ |
+ |
+def _ParseRevisionString(revision_string, repo_dictionary): |
+ """Parses the given revision_string into a revision dictionary. |
+ |
+ Returns a list of tuples that contain [portage_package_name, commit_id] to |
+ update. |
+ |
+ Keyword arguments: |
+ revision_string -- revision_string with format |
+ 'repo1.git@commit_1 repo2.git@commit2 ...'. |
+ repo_dictionary -- dictionary with git repository names as keys (w/out git) |
+ to portage package names. |
+ |
+ """ |
+ # Using a dictionary removes duplicates. |
+ revisions = {} |
+ for revision in revision_string.split(): |
+ # Format 'package@commit-id'. |
+ revision_tuple = revision.split('@') |
+ if len(revision_tuple) != 2: |
+ print >> sys.stderr, 'Incorrectly formatted revision %s' % revision |
+ repo_name = revision_tuple[0].replace('.git', '') |
+ # May be many corresponding packages to a given git repo e.g. kernel) |
+ for package in repo_dictionary[repo_name]: |
+ revisions[package] = revision_tuple[1] |
+ return revisions.items() |
+ |
+ |
+def _UprevFromRevisionList(buildroot, revision_list): |
+ """Uprevs based on revision list.""" |
+ package_str = '' |
+ commit_str = '' |
+ for package, revision in revision_list: |
+ package_str += package + ' ' |
+ commit_str += revision + ' ' |
+ package_str = package_str.strip() |
+ commit_str = commit_str.strip() |
+ |
+ cwd = os.path.join(buildroot, 'src', 'scripts') |
+ RunCommand(['./cros_mark_as_stable', |
+ '--tracking_branch="cros/master"', |
+ '--packages="%s"' % package_str, |
+ '--commit_ids="%s"' % commit_str, |
+ 'commit'], |
+ cwd=cwd, enter_chroot=True) |
+ |
+ |
+def _UprevAllPackages(buildroot): |
+ """Uprevs all packages that have been updated since last uprev.""" |
+ cwd = os.path.join(buildroot, 'src', 'scripts') |
+ RunCommand(['./cros_mark_all_as_stable', |
+ '--tracking_branch="cros/master"'], |
+ cwd=cwd, enter_chroot=True) |
+ |
+# =========================== Main Commands =================================== |
def _FullCheckout(buildroot, rw_checkout=True, retries=_DEFAULT_RETRIES): |
+ """Performs a full checkout and clobbers any previous checkouts.""" |
RunCommand(['sudo', 'rm', '-rf', buildroot]) |
MakeDir(buildroot, parents=True) |
RunCommand(['repo', 'init', '-u', 'http://src.chromium.org/git/manifest'], |
cwd=buildroot, input='\n\ny\n') |
RepoSync(buildroot, rw_checkout, retries) |
+ |
def _IncrementalCheckout(buildroot, rw_checkout=True, |
retries=_DEFAULT_RETRIES): |
+ """Performs a checkout without clobbering previous checkout.""" |
RepoSync(buildroot, rw_checkout, retries) |
+ |
def _MakeChroot(buildroot): |
+ """Wrapper around make_chroot.""" |
cwd = os.path.join(buildroot, 'src', 'scripts') |
RunCommand(['./make_chroot', '--fast'], cwd=cwd) |
+ |
def _SetupBoard(buildroot, board='x86-generic'): |
+ """Wrapper around setup_board.""" |
cwd = os.path.join(buildroot, 'src', 'scripts') |
RunCommand(['./setup_board', '--fast', '--default', '--board=%s' % board], |
cwd=cwd) |
+ |
def _Build(buildroot): |
+ """Wrapper around build_packages.""" |
cwd = os.path.join(buildroot, 'src', 'scripts') |
RunCommand(['./build_packages'], cwd=cwd) |
-def _UprevAllPackages(buildroot): |
- cwd = os.path.join(buildroot, 'src', 'scripts') |
- RunCommand(['./enter_chroot.sh', '--', './cros_mark_all_as_stable', |
- '--tracking_branch="cros/master"'], |
- cwd=cwd) |
-def _UprevPackages(buildroot, revisionfile): |
- revisions = None |
+def _UprevPackages(buildroot, revisionfile, board): |
+ """Uprevs a package based on given revisionfile. |
+ |
+ If revisionfile is set to None or does not resolve to an actual file, this |
+ function will uprev all packages. |
+ |
+ Keyword arguments: |
+ revisionfile -- string specifying a file that contains a list of revisions to |
+ uprev. |
+ """ |
+ # Purposefully set to None as it means Force Build was pressed. |
+ revisions = 'None' |
if (revisionfile): |
try: |
rev_file = open(revisionfile) |
revisions = rev_file.read() |
rev_file.close() |
- except: |
- print >> sys.stderr, 'Error reading %s' % revisionfile |
- revisions = None |
+ except Exception, e: |
+ print >> sys.stderr, 'Error reading %s, revving all' % revisionfile |
+ print e |
+ revisions = 'None' |
+ |
+ revisions = revisions.strip() |
- # Note: Revisions == "None" indicates a Force Build. |
- if revisions and revisions != 'None': |
- print 'CBUILDBOT - Revision list found %s' % revisions |
- print 'Revision list not yet propagating to build, marking all instead' |
+ # Revisions == "None" indicates a Force Build. |
+ if revisions != 'None': |
+ print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions |
+ revision_list = _ParseRevisionString(revisions, |
+ _CreateRepoDictionary(buildroot, board)) |
+ _UprevFromRevisionList(buildroot, revision_list) |
+ else: |
+ print >> sys.stderr, 'CBUILDBOT Revving all' |
+ _UprevAllPackages(buildroot) |
- _UprevAllPackages(buildroot) |
def _UprevCleanup(buildroot): |
+ """Clean up after a previous uprev attempt.""" |
cwd = os.path.join(buildroot, 'src', 'scripts') |
RunCommand(['./cros_mark_as_stable', '--srcroot=..', |
'--tracking_branch="cros/master"', 'clean'], |
cwd=cwd) |
+ |
def _UprevPush(buildroot): |
+ """Pushes uprev changes to the main line.""" |
cwd = os.path.join(buildroot, 'src', 'scripts') |
RunCommand(['./cros_mark_as_stable', '--srcroot=..', |
'--tracking_branch="cros/master"', |
'--push_options', '--bypass-hooks -f', 'push'], |
cwd=cwd) |
+ |
def _GetConfig(config_name): |
+ """Gets the configuration for the build""" |
default = config['default'] |
buildconfig = {} |
if config.has_key(config_name): |
@@ -146,6 +330,7 @@ def _GetConfig(config_name): |
buildconfig[key] = default[key] |
return buildconfig |
+ |
def main(): |
# Parse options |
usage = "usage: %prog [options] cbuildbot_config" |
@@ -183,7 +368,7 @@ def main(): |
if not os.path.isdir(boardpath): |
_SetupBoard(buildroot, board=buildconfig['board']) |
if buildconfig['uprev']: |
- _UprevPackages(buildroot, revisionfile) |
+ _UprevPackages(buildroot, revisionfile, board=buildconfig['board']) |
_Build(buildroot) |
if buildconfig['uprev']: |
_UprevPush(buildroot) |
@@ -194,5 +379,6 @@ def main(): |
RunCommand(['sudo', 'rm', '-rf', buildroot], print_cmd=False) |
raise |
+ |
if __name__ == '__main__': |
main() |