| Index: bin/cbuildbot.py
|
| diff --git a/bin/cbuildbot.py b/bin/cbuildbot.py
|
| deleted file mode 100755
|
| index be71830d7a38ad67a02f2071d1bff26bd847b3b7..0000000000000000000000000000000000000000
|
| --- a/bin/cbuildbot.py
|
| +++ /dev/null
|
| @@ -1,817 +0,0 @@
|
| -#!/usr/bin/python
|
| -
|
| -# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
|
| -# Use of this source code is governed by a BSD-style license that can be
|
| -# found in the LICENSE file.
|
| -
|
| -"""CBuildbot is wrapper around the build process used by the pre-flight queue"""
|
| -
|
| -import errno
|
| -import heapq
|
| -import re
|
| -import optparse
|
| -import os
|
| -import shutil
|
| -import sys
|
| -
|
| -import cbuildbot_comm
|
| -from cbuildbot_config import config
|
| -
|
| -sys.path.append(os.path.join(os.path.dirname(__file__), '../lib'))
|
| -from cros_build_lib import (Die, Info, ReinterpretPathForChroot, RunCommand,
|
| - Warning)
|
| -
|
| -_DEFAULT_RETRIES = 3
|
| -_PACKAGE_FILE = '%(buildroot)s/src/scripts/cbuildbot_package.list'
|
| -ARCHIVE_BASE = '/var/www/archive'
|
| -ARCHIVE_COUNT = 10
|
| -PUBLIC_OVERLAY = '%(buildroot)s/src/third_party/chromiumos-overlay'
|
| -PRIVATE_OVERLAY = '%(buildroot)s/src/private-overlays/chromeos-overlay'
|
| -CHROME_KEYWORDS_FILE = ('/build/%(board)s/etc/portage/package.keywords/chrome')
|
| -
|
| -# Currently, both the full buildbot and the preflight buildbot store their
|
| -# data in a variable named PORTAGE_BINHOST, but they're in different files.
|
| -# We're planning on joining the two files soon and renaming the full binhost
|
| -# to FULL_BINHOST.
|
| -_FULL_BINHOST = 'PORTAGE_BINHOST'
|
| -_PREFLIGHT_BINHOST = 'PORTAGE_BINHOST'
|
| -
|
| -# ======================== Utility functions ================================
|
| -
|
| -def _PrintFile(path):
|
| - """Prints out the contents of a file to stderr."""
|
| - file_handle = open(path)
|
| - print >> sys.stderr, file_handle.read()
|
| - file_handle.close()
|
| - sys.stderr.flush()
|
| -
|
| -
|
| -def MakeDir(path, parents=False):
|
| - """Basic wrapper around os.mkdirs.
|
| -
|
| - Keyword arguments:
|
| - path -- Path to create.
|
| - parents -- Follow mkdir -p logic.
|
| -
|
| - """
|
| - try:
|
| - os.makedirs(path)
|
| - except OSError, e:
|
| - if e.errno == errno.EEXIST and parents:
|
| - pass
|
| - else:
|
| - raise
|
| -
|
| -
|
| -def RepoSync(buildroot, retries=_DEFAULT_RETRIES):
|
| - """Uses repo to checkout the source code.
|
| -
|
| - Keyword arguments:
|
| - retries -- Number of retries to try before failing on the sync.
|
| - """
|
| - while retries > 0:
|
| - try:
|
| - # The --trace option ensures that repo shows the output from git. This
|
| - # is needed so that the buildbot can kill us if git is not making
|
| - # progress.
|
| - RunCommand(['repo', '--trace', 'sync'], cwd=buildroot)
|
| - RunCommand(['repo', 'forall', '-c', 'git', 'config',
|
| - 'url.ssh://git@gitrw.chromium.org:9222.insteadof',
|
| - 'http://git.chromium.org/git'], cwd=buildroot)
|
| - retries = 0
|
| - except:
|
| - retries -= 1
|
| - if retries > 0:
|
| - Warning('CBUILDBOT -- Repo Sync Failed, retrying')
|
| - else:
|
| - Warning('CBUILDBOT -- Retries exhausted')
|
| - raise
|
| -
|
| - RunCommand(['repo', 'manifest', '-r', '-o', '/dev/stderr'], cwd=buildroot)
|
| -
|
| -# =========================== Command Helpers =================================
|
| -
|
| -def _GetAllGitRepos(buildroot, debug=False):
|
| - """Returns a list of tuples containing [git_repo, src_path]."""
|
| - manifest_tuples = []
|
| - # Gets all the git repos from a full repo manifest.
|
| - repo_cmd = "repo manifest -o -".split()
|
| - output = RunCommand(repo_cmd, cwd=buildroot, redirect_stdout=True,
|
| - redirect_stderr=True, print_cmd=debug)
|
| -
|
| - # Extract all lines containg a project.
|
| - extract_cmd = ['grep', 'project name=']
|
| - output = RunCommand(extract_cmd, cwd=buildroot, input=output,
|
| - redirect_stdout=True, print_cmd=debug)
|
| - # Parse line using re to get tuple.
|
| - result_array = re.findall('.+name=\"([\w-]+)\".+path=\"(\S+)".+', output)
|
| -
|
| - # Create the array.
|
| - for result in result_array:
|
| - if len(result) != 2:
|
| - Warning('Found incorrect xml object %s' % result)
|
| - else:
|
| - # Remove pre-pended src directory from manifest.
|
| - manifest_tuples.append([result[0], result[1].replace('src/', '')])
|
| -
|
| - return manifest_tuples
|
| -
|
| -
|
| -def _GetCrosWorkOnSrcPath(buildroot, board, package, debug=False):
|
| - """Returns ${CROS_WORKON_SRC_PATH} for given package."""
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - equery_cmd = ('equery-%s which %s' % (board, package)).split()
|
| - ebuild_path = RunCommand(equery_cmd, cwd=cwd, redirect_stdout=True,
|
| - redirect_stderr=True, enter_chroot=True,
|
| - error_ok=True, print_cmd=debug)
|
| - if ebuild_path:
|
| - ebuild_cmd = ('ebuild-%s %s info' % (board, ebuild_path)).split()
|
| - cros_workon_output = RunCommand(ebuild_cmd, cwd=cwd,
|
| - redirect_stdout=True, redirect_stderr=True,
|
| - enter_chroot=True, print_cmd=debug)
|
| -
|
| - temp = re.findall('CROS_WORKON_SRCDIR="(\S+)"', cros_workon_output)
|
| - if temp:
|
| - return temp[0]
|
| -
|
| - return None
|
| -
|
| -
|
| -def _CreateRepoDictionary(buildroot, board, debug=False):
|
| - """Returns the repo->list_of_ebuilds dictionary."""
|
| - repo_dictionary = {}
|
| - manifest_tuples = _GetAllGitRepos(buildroot)
|
| - Info('Creating dictionary of git repos to portage packages ...')
|
| -
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - get_all_workon_pkgs_cmd = './cros_workon list --all'.split()
|
| - packages = RunCommand(get_all_workon_pkgs_cmd, cwd=cwd,
|
| - redirect_stdout=True, redirect_stderr=True,
|
| - enter_chroot=True, print_cmd=debug)
|
| - for package in packages.split():
|
| - cros_workon_src_path = _GetCrosWorkOnSrcPath(buildroot, board, package)
|
| - if cros_workon_src_path:
|
| - for tuple in manifest_tuples:
|
| - # This path tends to have the user's home_dir prepended to it.
|
| - if cros_workon_src_path.endswith(tuple[1]):
|
| - Info('For %s found matching package %s' % (tuple[0], package))
|
| - if repo_dictionary.has_key(tuple[0]):
|
| - repo_dictionary[tuple[0]] += [package]
|
| - else:
|
| - repo_dictionary[tuple[0]] = [package]
|
| -
|
| - return repo_dictionary
|
| -
|
| -
|
| -def _ParseRevisionString(revision_string, repo_dictionary):
|
| - """Parses the given revision_string into a revision dictionary.
|
| -
|
| - Returns a list of tuples that contain [portage_package_name, commit_id] to
|
| - update.
|
| -
|
| - Keyword arguments:
|
| - revision_string -- revision_string with format
|
| - 'repo1.git@commit_1 repo2.git@commit2 ...'.
|
| - repo_dictionary -- dictionary with git repository names as keys (w/out git)
|
| - to portage package names.
|
| -
|
| - """
|
| - # Using a dictionary removes duplicates.
|
| - revisions = {}
|
| - for revision in revision_string.split():
|
| - # Format 'package@commit-id'.
|
| - revision_tuple = revision.split('@')
|
| - if len(revision_tuple) != 2:
|
| - Warning('Incorrectly formatted revision %s' % revision)
|
| -
|
| - repo_name = revision_tuple[0].replace('.git', '')
|
| - # Might not have entry if no matching ebuild.
|
| - if repo_dictionary.has_key(repo_name):
|
| - # May be many corresponding packages to a given git repo e.g. kernel).
|
| - for package in repo_dictionary[repo_name]:
|
| - revisions[package] = revision_tuple[1]
|
| -
|
| - return revisions.items()
|
| -
|
| -
|
| -def _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board,
|
| - overlays):
|
| - """Uprevs based on revision list."""
|
| - if not revision_list:
|
| - Info('No packages found to uprev')
|
| - return
|
| -
|
| - packages = []
|
| - for package, revision in revision_list:
|
| - assert ':' not in package, 'Invalid package name: %s' % package
|
| - packages.append(package)
|
| -
|
| - chroot_overlays = [ReinterpretPathForChroot(path) for path in overlays]
|
| -
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - RunCommand(['./cros_mark_as_stable',
|
| - '--board=%s' % board,
|
| - '--tracking_branch=%s' % tracking_branch,
|
| - '--overlays=%s' % ':'.join(chroot_overlays),
|
| - '--packages=%s' % ':'.join(packages),
|
| - '--drop_file=%s' % ReinterpretPathForChroot(_PACKAGE_FILE %
|
| - {'buildroot': buildroot}),
|
| - 'commit'],
|
| - cwd=cwd, enter_chroot=True)
|
| -
|
| -
|
| -def _MarkChromeAsStable(buildroot, tracking_branch, chrome_rev, board):
|
| - """Returns the portage atom for the revved chrome ebuild - see man emerge."""
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - portage_atom_string = RunCommand(['bin/cros_mark_chrome_as_stable',
|
| - '--tracking_branch=%s' % tracking_branch,
|
| - chrome_rev], cwd=cwd, redirect_stdout=True,
|
| - enter_chroot=True).rstrip()
|
| - if not portage_atom_string:
|
| - Info('Found nothing to rev.')
|
| - return None
|
| - else:
|
| - chrome_atom = portage_atom_string.split('=')[1]
|
| - keywords_file = CHROME_KEYWORDS_FILE % {'board': board}
|
| - # TODO(sosa): Workaround to build unstable chrome ebuild we uprevved.
|
| - RunCommand(['sudo', 'mkdir', '-p', os.path.dirname(keywords_file)],
|
| - enter_chroot=True, cwd=cwd)
|
| - RunCommand(['sudo', 'tee', keywords_file], input='=%s\n' % chrome_atom,
|
| - enter_chroot=True, cwd=cwd)
|
| - return chrome_atom
|
| -
|
| -
|
| -def _UprevAllPackages(buildroot, tracking_branch, board, overlays):
|
| - """Uprevs all packages that have been updated since last uprev."""
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - chroot_overlays = [ReinterpretPathForChroot(path) for path in overlays]
|
| - RunCommand(['./cros_mark_as_stable', '--all',
|
| - '--board=%s' % board,
|
| - '--overlays=%s' % ':'.join(chroot_overlays),
|
| - '--tracking_branch=%s' % tracking_branch,
|
| - '--drop_file=%s' % ReinterpretPathForChroot(_PACKAGE_FILE %
|
| - {'buildroot': buildroot}),
|
| - 'commit'],
|
| - cwd=cwd, enter_chroot=True)
|
| -
|
| -
|
| -def _GetVMConstants(buildroot):
|
| - """Returns minimum (vdisk_size, statefulfs_size) recommended for VM's."""
|
| - cwd = os.path.join(buildroot, 'src', 'scripts', 'lib')
|
| - source_cmd = 'source %s/cros_vm_constants.sh' % cwd
|
| - vdisk_size = RunCommand([
|
| - '/bin/bash', '-c', '%s && echo $MIN_VDISK_SIZE_FULL' % source_cmd],
|
| - redirect_stdout=True)
|
| - statefulfs_size = RunCommand([
|
| - '/bin/bash', '-c', '%s && echo $MIN_STATEFUL_FS_SIZE_FULL' % source_cmd],
|
| - redirect_stdout=True)
|
| - return (vdisk_size.strip(), statefulfs_size.strip())
|
| -
|
| -
|
| -def _GitCleanup(buildroot, board, tracking_branch, overlays):
|
| - """Clean up git branch after previous uprev attempt."""
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - if os.path.exists(cwd):
|
| - RunCommand(['./cros_mark_as_stable', '--srcroot=..',
|
| - '--board=%s' % board,
|
| - '--overlays=%s' % ':'.join(overlays),
|
| - '--tracking_branch=%s' % tracking_branch, 'clean'],
|
| - cwd=cwd, error_ok=True)
|
| -
|
| -
|
| -def _CleanUpMountPoints(buildroot):
|
| - """Cleans up any stale mount points from previous runs."""
|
| - mount_output = RunCommand(['mount'], redirect_stdout=True)
|
| - mount_pts_in_buildroot = RunCommand(['grep', buildroot], input=mount_output,
|
| - redirect_stdout=True, error_ok=True)
|
| -
|
| - for mount_pt_str in mount_pts_in_buildroot.splitlines():
|
| - mount_pt = mount_pt_str.rpartition(' type ')[0].partition(' on ')[2]
|
| - RunCommand(['sudo', 'umount', '-l', mount_pt], error_ok=True)
|
| -
|
| -
|
| -def _WipeOldOutput(buildroot):
|
| - """Wipes out build output directories."""
|
| - RunCommand(['rm', '-rf', 'src/build/images'], cwd=buildroot)
|
| -
|
| -
|
| -# =========================== Main Commands ===================================
|
| -
|
| -
|
| -def _PreFlightRinse(buildroot, board, tracking_branch, overlays):
|
| - """Cleans up any leftover state from previous runs."""
|
| - _GitCleanup(buildroot, board, tracking_branch, overlays)
|
| - _CleanUpMountPoints(buildroot)
|
| - RunCommand(['sudo', 'killall', 'kvm'], error_ok=True)
|
| -
|
| -
|
| -def _FullCheckout(buildroot, tracking_branch,
|
| - retries=_DEFAULT_RETRIES,
|
| - url='http://git.chromium.org/git/manifest'):
|
| - """Performs a full checkout and clobbers any previous checkouts."""
|
| - RunCommand(['sudo', 'rm', '-rf', buildroot])
|
| - MakeDir(buildroot, parents=True)
|
| - branch = tracking_branch.split('/');
|
| - RunCommand(['repo', 'init', '-u',
|
| - url, '-b',
|
| - '%s' % branch[-1]], cwd=buildroot, input='\n\ny\n')
|
| - RepoSync(buildroot, retries)
|
| -
|
| -
|
| -def _IncrementalCheckout(buildroot, retries=_DEFAULT_RETRIES):
|
| - """Performs a checkout without clobbering previous checkout."""
|
| - RepoSync(buildroot, retries)
|
| -
|
| -
|
| -def _MakeChroot(buildroot, replace=False):
|
| - """Wrapper around make_chroot."""
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| -
|
| - cmd = ['./make_chroot', '--fast']
|
| -
|
| - if replace:
|
| - cmd.append('--replace')
|
| -
|
| - RunCommand(cmd, cwd=cwd)
|
| -
|
| -
|
| -def _GetPortageEnvVar(buildroot, board, envvar):
|
| - """Get a portage environment variable for the specified board, if any.
|
| -
|
| - buildroot: The root directory where the build occurs. Must be an absolute
|
| - path.
|
| - board: Board type that was built on this machine. E.g. x86-generic. If this
|
| - is None, get the env var from the host.
|
| - envvar: The environment variable to get. E.g. 'PORTAGE_BINHOST'.
|
| -
|
| - Returns:
|
| - The value of the environment variable, as a string. If no such variable
|
| - can be found, return the empty string.
|
| - """
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - portageq = 'portageq'
|
| - if board:
|
| - portageq += '-%s' % board
|
| - binhost = RunCommand([portageq, 'envvar', envvar], cwd=cwd,
|
| - redirect_stdout=True, enter_chroot=True, error_ok=True)
|
| - return binhost.rstrip('\n')
|
| -
|
| -
|
| -def _SetupBoard(buildroot, board='x86-generic'):
|
| - """Wrapper around setup_board."""
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - RunCommand(['./setup_board', '--fast', '--default', '--board=%s' % board],
|
| - cwd=cwd, enter_chroot=True)
|
| -
|
| -
|
| -def _Build(buildroot, emptytree, build_autotest=True, usepkg=True):
|
| - """Wrapper around build_packages."""
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - if emptytree:
|
| - cmd = ['sh', '-c', 'EXTRA_BOARD_FLAGS=--emptytree ./build_packages']
|
| - else:
|
| - cmd = ['./build_packages']
|
| -
|
| - if not build_autotest:
|
| - cmd.append('--nowithautotest')
|
| -
|
| - if not usepkg:
|
| - cmd.append('--nousepkg')
|
| -
|
| - RunCommand(cmd, cwd=cwd, enter_chroot=True)
|
| -
|
| -
|
| -def _EnableLocalAccount(buildroot):
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - # Set local account for test images.
|
| - RunCommand(['./enable_localaccount.sh',
|
| - 'chronos'],
|
| - print_cmd=False, cwd=cwd)
|
| -
|
| -
|
| -def _BuildImage(buildroot):
|
| - _WipeOldOutput(buildroot)
|
| -
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - RunCommand(['./build_image', '--replace'], cwd=cwd, enter_chroot=True)
|
| -
|
| -
|
| -def _BuildVMImageForTesting(buildroot):
|
| - (vdisk_size, statefulfs_size) = _GetVMConstants(buildroot)
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - RunCommand(['./image_to_vm.sh',
|
| - '--test_image',
|
| - '--full',
|
| - '--vdisk_size=%s' % vdisk_size,
|
| - '--statefulfs_size=%s' % statefulfs_size,
|
| - ], cwd=cwd, enter_chroot=True)
|
| -
|
| -
|
| -def _RunUnitTests(buildroot):
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - RunCommand(['./cros_run_unit_tests',
|
| - '--package_file=%s' % ReinterpretPathForChroot(_PACKAGE_FILE %
|
| - {'buildroot': buildroot}),
|
| - ], cwd=cwd, enter_chroot=True)
|
| -
|
| -
|
| -def _RunSmokeSuite(buildroot, results_dir):
|
| - results_dir_in_chroot = os.path.join(buildroot, 'chroot',
|
| - results_dir.lstrip('/'))
|
| - if os.path.exists(results_dir_in_chroot):
|
| - shutil.rmtree(results_dir_in_chroot)
|
| -
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - RunCommand(['bin/cros_run_vm_test',
|
| - '--no_graphics',
|
| - '--results_dir_root=%s' % results_dir,
|
| - 'suite_Smoke',
|
| - ], cwd=cwd, error_ok=False)
|
| -
|
| -
|
| -def _RunAUTest(buildroot, board):
|
| - """Runs a basic update test from the au test harness."""
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - image_path = os.path.join(buildroot, 'src', 'build', 'images', board,
|
| - 'latest', 'chromiumos_test_image.bin')
|
| - RunCommand(['bin/cros_au_test_harness',
|
| - '--no_graphics',
|
| - '--no_delta',
|
| - '--board=%s' % board,
|
| - '--test_prefix=SimpleTest',
|
| - '--verbose',
|
| - '--base_image=%s' % image_path,
|
| - '--target_image=%s' % image_path,
|
| - ], cwd=cwd, error_ok=False)
|
| -
|
| -
|
| -def _UprevPackages(buildroot, tracking_branch, revisionfile, board, overlays):
|
| - """Uprevs a package based on given revisionfile.
|
| -
|
| - If revisionfile is set to None or does not resolve to an actual file, this
|
| - function will uprev all packages.
|
| -
|
| - Keyword arguments:
|
| - revisionfile -- string specifying a file that contains a list of revisions to
|
| - uprev.
|
| - """
|
| - # Purposefully set to None as it means Force Build was pressed.
|
| - revisions = 'None'
|
| - if (revisionfile):
|
| - try:
|
| - rev_file = open(revisionfile)
|
| - revisions = rev_file.read()
|
| - rev_file.close()
|
| - except Exception, e:
|
| - Warning('Error reading %s, revving all' % revisionfile)
|
| - revisions = 'None'
|
| -
|
| - revisions = revisions.strip()
|
| -
|
| - # TODO(sosa): Un-comment once we close individual trees.
|
| - # revisions == "None" indicates a Force Build.
|
| - #if revisions != 'None':
|
| - # print >> sys.stderr, 'CBUILDBOT Revision list found %s' % revisions
|
| - # revision_list = _ParseRevisionString(revisions,
|
| - # _CreateRepoDictionary(buildroot, board))
|
| - # _UprevFromRevisionList(buildroot, tracking_branch, revision_list, board,
|
| - # overlays)
|
| - #else:
|
| - Info('CBUILDBOT Revving all')
|
| - _UprevAllPackages(buildroot, tracking_branch, board, overlays)
|
| -
|
| -
|
| -def _UprevPush(buildroot, tracking_branch, board, overlays, dryrun):
|
| - """Pushes uprev changes to the main line."""
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - cmd = ['./cros_mark_as_stable',
|
| - '--srcroot=%s' % os.path.join(buildroot, 'src'),
|
| - '--board=%s' % board,
|
| - '--overlays=%s' % ':'.join(overlays),
|
| - '--tracking_branch=%s' % tracking_branch
|
| - ]
|
| - if dryrun:
|
| - cmd.append('--dryrun')
|
| -
|
| - cmd.append('push')
|
| - RunCommand(cmd, cwd=cwd)
|
| -
|
| -
|
| -def _LegacyArchiveBuild(buildroot, bot_id, buildconfig, buildnumber,
|
| - debug=False):
|
| - """Adds a step to the factory to archive a build."""
|
| -
|
| - # Fixed properties
|
| - keep_max = 3
|
| - gsutil_archive = 'gs://chromeos-archive/' + bot_id
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| -
|
| - cmd = ['./archive_build.sh',
|
| - '--build_number', str(buildnumber),
|
| - '--to', '/var/www/archive/' + bot_id,
|
| - '--keep_max', str(keep_max),
|
| - '--prebuilt_upload',
|
| - '--board', buildconfig['board'],
|
| -
|
| - '--acl', '/home/chrome-bot/slave_archive_acl',
|
| - '--gsutil_archive', gsutil_archive,
|
| - '--gsd_gen_index',
|
| - '/b/scripts/gsd_generate_index/gsd_generate_index.py',
|
| - '--gsutil', '/b/scripts/slave/gsutil',
|
| - '--test_mod'
|
| - ]
|
| -
|
| - if buildconfig.get('test_mod', True):
|
| - cmd.append('--test_mod')
|
| -
|
| - if buildconfig.get('factory_install_mod', True):
|
| - cmd.append('--factory_install_mod')
|
| -
|
| - if buildconfig.get('factory_test_mod', True):
|
| - cmd.append('--factory_test_mod')
|
| -
|
| - if debug:
|
| - Warning('***** ***** LegacyArchiveBuild CMD: ' + ' '.join(cmd))
|
| - else:
|
| - RunCommand(cmd, cwd=cwd)
|
| -
|
| -def _ArchiveTestResults(buildroot, board, test_results_dir,
|
| - gsutil, archive_dir, acl):
|
| - """Archives the test results into Google Storage
|
| -
|
| - Takes the results from the test_results_dir and the last qemu image and
|
| - uploads them to Google Storage.
|
| -
|
| - Arguments:
|
| - buildroot: Root directory where build occurs
|
| - board: Board to find the qemu image.
|
| - test_results_dir: Path from buildroot/chroot to find test results.
|
| - This must a subdir of /tmp.
|
| - gsutil: Location of gsutil
|
| - archive_dir: Google Storage path to store the archive
|
| - acl: ACL to set on archive in Google Storage
|
| - """
|
| - num_gsutil_retries = 5
|
| - test_results_dir = test_results_dir.lstrip('/')
|
| - results_path = os.path.join(buildroot, 'chroot', test_results_dir)
|
| - RunCommand(['sudo', 'chmod', '-R', '+r', results_path])
|
| - try:
|
| - # gsutil has the ability to resume an upload when the command is retried
|
| - RunCommand([gsutil, 'cp', '-R', results_path, archive_dir],
|
| - num_retries=num_gsutil_retries)
|
| - RunCommand([gsutil, 'setacl', acl, archive_dir])
|
| -
|
| - image_name = 'chromiumos_qemu_image.bin'
|
| - image_path = os.path.join(buildroot, 'src', 'build', 'images', board,
|
| - 'latest', image_name)
|
| - RunCommand(['gzip', '-f', '--fast', image_path])
|
| - RunCommand([gsutil, 'cp', image_path + '.gz', archive_dir],
|
| - num_retries=num_gsutil_retries)
|
| - except Exception, e:
|
| - Warning('Could not archive test results (error=%s)' % str(e))
|
| -
|
| -
|
| -def _GetConfig(config_name):
|
| - """Gets the configuration for the build"""
|
| - buildconfig = {}
|
| - if not config.has_key(config_name):
|
| - Warning('Non-existent configuration specified.')
|
| - Warning('Please specify one of:')
|
| - config_names = config.keys()
|
| - config_names.sort()
|
| - for name in config_names:
|
| - Warning(' %s' % name)
|
| - sys.exit(1)
|
| -
|
| - return config[config_name]
|
| -
|
| -
|
| -def _ResolveOverlays(buildroot, overlays):
|
| - """Return the list of overlays to use for a given buildbot.
|
| -
|
| - Args:
|
| - buildroot: The root directory where the build occurs. Must be an absolute
|
| - path.
|
| - overlays: A string describing which overlays you want.
|
| - 'private': Just the private overlay.
|
| - 'public': Just the public overlay.
|
| - 'both': Both the public and private overlays.
|
| - """
|
| - public_overlay = PUBLIC_OVERLAY % {'buildroot': buildroot}
|
| - private_overlay = PRIVATE_OVERLAY % {'buildroot': buildroot}
|
| - if overlays == 'private':
|
| - paths = [private_overlay]
|
| - elif overlays == 'public':
|
| - paths = [public_overlay]
|
| - elif overlays == 'both':
|
| - paths = [public_overlay, private_overlay]
|
| - else:
|
| - Info('No overlays found.')
|
| - paths = []
|
| - return paths
|
| -
|
| -
|
| -def _UploadPrebuilts(buildroot, board, overlay_config, binhosts):
|
| - """Upload prebuilts.
|
| -
|
| - Args:
|
| - buildroot: The root directory where the build occurs.
|
| - board: Board type that was built on this machine
|
| - overlay_config: A string describing which overlays you want.
|
| - 'private': Just the private overlay.
|
| - 'public': Just the public overlay.
|
| - 'both': Both the public and private overlays.
|
| - binhosts: The URLs of the current binhosts. Binaries that are already
|
| - present will not be uploaded twice. Empty URLs will be ignored.
|
| - """
|
| -
|
| - cwd = os.path.join(buildroot, 'src', 'scripts')
|
| - cmd = [os.path.join(cwd, 'prebuilt.py'),
|
| - '--sync-binhost-conf',
|
| - '--build-path', buildroot,
|
| - '--board', board,
|
| - '--prepend-version', 'preflight',
|
| - '--key', _PREFLIGHT_BINHOST]
|
| - for binhost in binhosts:
|
| - if binhost:
|
| - cmd.extend(['--previous-binhost-url', binhost])
|
| - if overlay_config == 'public':
|
| - cmd.extend(['--upload', 'gs://chromeos-prebuilt'])
|
| - else:
|
| - assert overlay_config in ('private', 'both')
|
| - cmd.extend(['--upload', 'chromeos-images:/var/www/prebuilt/',
|
| - '--binhost-base-url', 'http://chromeos-prebuilt'])
|
| -
|
| - RunCommand(cmd, cwd=cwd)
|
| -
|
| -
|
| -def main():
|
| - # Parse options
|
| - usage = "usage: %prog [options] cbuildbot_config"
|
| - parser = optparse.OptionParser(usage=usage)
|
| - parser.add_option('-a', '--acl', default='private',
|
| - help='ACL to set on GSD archives')
|
| - parser.add_option('-r', '--buildroot',
|
| - help='root directory where build occurs', default=".")
|
| - parser.add_option('-n', '--buildnumber',
|
| - help='build number', type='int', default=0)
|
| - parser.add_option('--chrome_rev', default=None, type='string',
|
| - dest='chrome_rev',
|
| - help=('Chrome_rev of type [tot|latest_release|'
|
| - 'sticky_release]'))
|
| - parser.add_option('-g', '--gsutil', default='', help='Location of gsutil')
|
| - parser.add_option('-c', '--gsutil_archive', default='',
|
| - help='Datastore archive location')
|
| - parser.add_option('--clobber', action='store_true', dest='clobber',
|
| - default=False,
|
| - help='Clobbers an old checkout before syncing')
|
| - parser.add_option('--debug', action='store_true', dest='debug',
|
| - default=False,
|
| - help='Override some options to run as a developer.')
|
| - parser.add_option('--nobuild', action='store_false', dest='build',
|
| - default=True,
|
| - help="Don't actually build (for cbuildbot dev")
|
| - parser.add_option('--noprebuilts', action='store_false', dest='prebuilts',
|
| - default=True,
|
| - help="Don't upload prebuilts.")
|
| - parser.add_option('--nosync', action='store_false', dest='sync',
|
| - default=True,
|
| - help="Don't sync before building.")
|
| - parser.add_option('--notests', action='store_false', dest='tests',
|
| - default=True,
|
| - help='Override values from buildconfig and run no tests.')
|
| - parser.add_option('-f', '--revisionfile',
|
| - help='file where new revisions are stored')
|
| - parser.add_option('-t', '--tracking-branch', dest='tracking_branch',
|
| - default='cros/master', help='Run the buildbot on a branch')
|
| - parser.add_option('-u', '--url', dest='url',
|
| - default='http://git.chromium.org/git/manifest',
|
| - help='Run the buildbot on internal manifest')
|
| -
|
| - (options, args) = parser.parse_args()
|
| -
|
| - buildroot = os.path.abspath(options.buildroot)
|
| - revisionfile = options.revisionfile
|
| - tracking_branch = options.tracking_branch
|
| - chrome_atom_to_build = None
|
| -
|
| - if len(args) >= 1:
|
| - bot_id = args[-1]
|
| - buildconfig = _GetConfig(bot_id)
|
| - else:
|
| - Warning('Missing configuration description')
|
| - parser.print_usage()
|
| - sys.exit(1)
|
| -
|
| - try:
|
| - # Calculate list of overlay directories.
|
| - rev_overlays = _ResolveOverlays(buildroot, buildconfig['rev_overlays'])
|
| - push_overlays = _ResolveOverlays(buildroot, buildconfig['push_overlays'])
|
| - # We cannot push to overlays that we don't rev.
|
| - assert set(push_overlays).issubset(set(rev_overlays))
|
| - # Either has to be a master or not have any push overlays.
|
| - assert buildconfig['master'] or not push_overlays
|
| -
|
| - board = buildconfig['board']
|
| - old_binhost = None
|
| -
|
| - _PreFlightRinse(buildroot, buildconfig['board'], tracking_branch,
|
| - rev_overlays)
|
| - chroot_path = os.path.join(buildroot, 'chroot')
|
| - boardpath = os.path.join(chroot_path, 'build', board)
|
| - if options.sync:
|
| - if options.clobber or not os.path.isdir(buildroot):
|
| - _FullCheckout(buildroot, tracking_branch, url=options.url)
|
| - else:
|
| - old_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
|
| - _IncrementalCheckout(buildroot)
|
| -
|
| - new_binhost = _GetPortageEnvVar(buildroot, board, _FULL_BINHOST)
|
| - emptytree = (old_binhost and old_binhost != new_binhost)
|
| -
|
| - # Check that all overlays can be found.
|
| - for path in rev_overlays:
|
| - if not os.path.isdir(path):
|
| - Die('Missing overlay: %s' % path)
|
| -
|
| - if not os.path.isdir(chroot_path) or buildconfig['chroot_replace']:
|
| - _MakeChroot(buildroot, buildconfig['chroot_replace'])
|
| -
|
| - if not os.path.isdir(boardpath):
|
| - _SetupBoard(buildroot, board=buildconfig['board'])
|
| -
|
| - # Perform chrome uprev.
|
| - if options.chrome_rev:
|
| - chrome_atom_to_build = _MarkChromeAsStable(buildroot, tracking_branch,
|
| - options.chrome_rev, board)
|
| - # Perform other uprevs.
|
| - if buildconfig['uprev']:
|
| - _UprevPackages(buildroot, tracking_branch, revisionfile,
|
| - buildconfig['board'], rev_overlays)
|
| - elif options.chrome_rev and not chrome_atom_to_build:
|
| - # We found nothing to rev, we're done here.
|
| - return
|
| -
|
| - _EnableLocalAccount(buildroot)
|
| -
|
| - if options.build:
|
| - _Build(buildroot,
|
| - emptytree,
|
| - build_autotest=(buildconfig['vm_tests'] and options.tests),
|
| - usepkg=buildconfig['usepkg'])
|
| -
|
| - if buildconfig['unittests'] and options.tests:
|
| - _RunUnitTests(buildroot)
|
| -
|
| - _BuildImage(buildroot)
|
| -
|
| - if buildconfig['vm_tests'] and options.tests:
|
| - _BuildVMImageForTesting(buildroot)
|
| - test_results_dir = '/tmp/run_remote_tests.%s' % options.buildnumber
|
| - try:
|
| - _RunSmokeSuite(buildroot, test_results_dir)
|
| - _RunAUTest(buildroot, buildconfig['board'])
|
| - finally:
|
| - if not options.debug:
|
| - archive_full_path = os.path.join(options.gsutil_archive,
|
| - str(options.buildnumber))
|
| - _ArchiveTestResults(buildroot, buildconfig['board'],
|
| - test_results_dir=test_results_dir,
|
| - gsutil=options.gsutil,
|
| - archive_dir=archive_full_path,
|
| - acl=options.acl)
|
| -
|
| - if buildconfig['uprev']:
|
| - # Don't push changes for developers.
|
| - if buildconfig['master']:
|
| - # Master bot needs to check if the other slaves completed.
|
| - if cbuildbot_comm.HaveSlavesCompleted(config):
|
| - if not options.debug and options.prebuilts:
|
| - _UploadPrebuilts(buildroot, board, buildconfig['rev_overlays'],
|
| - [new_binhost])
|
| - _UprevPush(buildroot, tracking_branch, buildconfig['board'],
|
| - push_overlays, options.debug)
|
| - else:
|
| - Die('CBUILDBOT - One of the slaves has failed!!!')
|
| -
|
| - else:
|
| - # Publish my status to the master if its expecting it.
|
| - if buildconfig['important'] and not options.debug:
|
| - cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)
|
| -
|
| - if buildconfig['archive_build']:
|
| - _LegacyArchiveBuild(buildroot,
|
| - bot_id,
|
| - buildconfig,
|
| - options.buildnumber,
|
| - options.debug)
|
| - except:
|
| - # Send failure to master bot.
|
| - if not buildconfig['master'] and buildconfig['important']:
|
| - cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_FAILED)
|
| -
|
| - raise
|
| -
|
| -
|
| -if __name__ == '__main__':
|
| - main()
|
|
|