Index: build/build.py |
diff --git a/build/build.py b/build/build.py |
index cb280ab64ad71267c0f068e8838b42c30cb61c93..95d6370b2f183abb5c5e30b744910cd56c79a860 100755 |
--- a/build/build.py |
+++ b/build/build.py |
@@ -7,15 +7,18 @@ |
invokes CIPD client to package and upload chunks of it to the CIPD repository as |
individual packages. |
-See build/packages/*.yaml for definition of packages. |
+See build/packages/*.yaml for definition of packages and README.md for more |
+details. |
""" |
import argparse |
+import contextlib |
import glob |
+import hashlib |
import json |
import os |
import platform |
-import shutil |
+import socket |
import subprocess |
import sys |
import tempfile |
@@ -42,6 +45,56 @@ class UploadException(Exception): |
"""Raised on errors during package upload step.""" |
+class PackageDef(object): |
+ """Represents parsed package *.yaml file.""" |
+ |
+ def __init__(self, path, pkg_def): |
+ self.path = path |
+ self.pkg_def = pkg_def |
+ |
+ @property |
+ def name(self): |
+ """Returns name of YAML file (without the directory path and extension).""" |
+ return os.path.splitext(os.path.basename(self.path))[0] |
+ |
+ @property |
+ def uses_python_env(self): |
+ """Returns True if 'uses_python_env' in the YAML file is set.""" |
+ return bool(self.pkg_def.get('uses_python_env')) |
+ |
+ @property |
+ def go_packages(self): |
+ """Returns a list of Go packages that must be installed for this package.""" |
+ return self.pkg_def.get('go_packages') or [] |
+ |
+ def should_build(self, builder): |
+ """Returns True if package should be built in the current environment. |
+ |
+ Takes into account 'builders' and 'supports_cross_compilation' properties of |
+ the package definition file. |
+ """ |
+ # If '--builder' is not specified, ignore 'builders' property. Otherwise, if |
+ # 'builders' YAML attribute it not empty, verify --builder is listed there. |
+ builders = self.pkg_def.get('builders') |
+ if builder and builders and builder not in builders: |
+ return False |
+ |
+ # If cross-compiling, pick only packages that support cross-compilation. |
+ if is_cross_compiling(): |
+ return bool(self.pkg_def.get('supports_cross_compilation')) |
+ |
+ return True |
+ |
+ |
+def is_cross_compiling(): |
+ """Returns True if using GOOS or GOARCH env vars. |
+ |
+ We also check at the start of the script that if one of them is used, then |
+ the other is specified as well. |
+ """ |
+ return bool(os.environ.get('GOOS')) or bool(os.environ.get('GOARCH')) |
+ |
+ |
def run_python(script, args): |
"""Invokes a python script. |
@@ -53,11 +106,11 @@ def run_python(script, args): |
args=['python', '-u', script] + list(args), executable=sys.executable) |
-def run_cipd(go_workspace, cmd, args): |
+def run_cipd(cipd_exe, cmd, args): |
"""Invokes CIPD, parsing -json-output result. |
Args: |
- go_workspace: path to 'infra/go' or 'infra_internal/go'. |
+ cipd_exe: path to cipd client binary to run. |
cmd: cipd subcommand to run. |
args: list of command line arguments to pass to the subcommand. |
@@ -69,10 +122,7 @@ def run_cipd(go_workspace, cmd, args): |
fd, temp_file = tempfile.mkstemp(suffix='.json', prefix='cipd_%s' % cmd) |
os.close(fd) |
- cmd_line = [ |
- os.path.join(go_workspace, 'bin', 'cipd' + EXE_SUFFIX), |
- cmd, '-json-output', temp_file, |
- ] + list(args) |
+ cmd_line = [cipd_exe, cmd, '-json-output', temp_file] + list(args) |
print 'Running %s' % ' '.join(cmd_line) |
exit_code = subprocess.call(args=cmd_line, executable=cmd_line[0]) |
@@ -101,20 +151,32 @@ def print_title(title): |
print '-' * 80 |
-def build_go(go_workspace, packages): |
- """Bootstraps go environment and rebuilds (and installs) Go packages. |
+def print_go_step_title(title): |
+ """Same as 'print_title', but also appends values of GOOS and GOARCH.""" |
+ if is_cross_compiling(): |
+ title += '\n' + '-' * 80 |
+ title += '\n GOOS=%s' % os.environ['GOOS'] |
+ title += '\n GOARCH=%s' % os.environ['GOARCH'] |
+ if 'GOARM' in os.environ: |
+ title += '\n GOARM=%s' % os.environ['GOARM'] |
+ print_title(title) |
- Compiles and installs packages into default GOBIN, which is <path>/go/bin/ |
- (it is setup by go/env.py and depends on what workspace is used). |
+ |
+@contextlib.contextmanager |
+def hacked_workspace(go_workspace, goos=None, goarch=None): |
+ """Symlinks Go workspace into new root, modifies os.environ. |
+ |
+ Go toolset embeds absolute paths to *.go files into the executable. Use |
+ symlink with stable path to make executables independent of checkout path. |
Args: |
go_workspace: path to 'infra/go' or 'infra_internal/go'. |
- packages: list of packages to build (can include '...' patterns). |
- """ |
- print_title('Compiling Go code: %s' % ', '.join(packages)) |
+ goos: if set, overrides GOOS environment variable (removes it if ''). |
+ goarch: if set, overrides GOARCH environment variable (removes it if ''). |
- # Go toolchain embeds absolute paths to *.go files into the executable. Use |
- # symlink with stable path to make executables independent of checkout path. |
+ Yields: |
+ Path where go_workspace is symlinked to. |
+ """ |
new_root = None |
new_workspace = go_workspace |
if sys.platform != 'win32': |
@@ -127,49 +189,218 @@ def build_go(go_workspace, packages): |
assert not rel.startswith('..'), rel |
new_workspace = os.path.join(new_root, rel) |
- # Remove any stale binaries and libraries. |
- shutil.rmtree(os.path.join(new_workspace, 'bin'), ignore_errors=True) |
- shutil.rmtree(os.path.join(new_workspace, 'pkg'), ignore_errors=True) |
+ orig_environ = os.environ.copy() |
+ |
+ if goos is not None: |
+ if goos == '': |
+ os.environ.pop('GOOS', None) |
+ else: |
+ os.environ['GOOS'] = goos |
+ if goarch is not None: |
+ if goarch == '': |
+ os.environ.pop('GOARCH', None) |
+ else: |
+ os.environ['GOARCH'] = goarch |
+ |
+ # Make sure we build ARMv6 code even if the host is ARMv7. See the comment in |
+ # get_host_package_vars for reasons why. Also explicitly set GOARM to 6 when |
+ # cross-compiling (it should be '6' in this case by default anyway). |
+ plat = platform.machine().lower() |
+ if plat.startswith('arm') or os.environ.get('GOARCH') == 'arm': |
+ os.environ['GOARM'] = '6' |
+ else: |
+ os.environ.pop('GOARM', None) |
- # Recompile ('-a'). |
try: |
+ yield new_workspace |
+ finally: |
+ # Apparently 'os.environ = orig_environ' doesn't actually modify process |
+ # environment, only modifications of os.environ object itself do. |
+ for k, v in orig_environ.iteritems(): |
+ os.environ[k] = v |
+ for k in os.environ.keys(): |
+ if k not in orig_environ: |
+ os.environ.pop(k) |
+ if new_root: |
+ os.remove(new_root) |
+ |
+ |
+def bootstrap_go_toolset(go_workspace): |
+ """Makes sure go toolset is installed and returns its 'go env' environment. |
+ |
+ Used to verify that our platform detection in get_host_package_vars() matches |
+ the Go toolset being used. |
+ """ |
+ with hacked_workspace(go_workspace) as new_workspace: |
+ print_go_step_title('Making sure Go toolset is installed') |
+ # env.py does the actual job of bootstrapping if the toolset is missing. |
+ output = subprocess.check_output( |
+ args=[ |
+ 'python', '-u', os.path.join(new_workspace, 'env.py'), |
+ 'go', 'env', |
+ ], |
+ executable=sys.executable) |
+ # See https://github.com/golang/go/blob/master/src/cmd/go/env.go for format |
+ # of the output. |
+ print 'Go environ:' |
+ print output.strip() |
+ env = {} |
+ for line in output.splitlines(): |
+ k, _, v = line.lstrip('set ').partition('=') |
+ if v.startswith('"') and v.endswith('"'): |
+ v = v.strip('"') |
+ env[k] = v |
+ return env |
+ |
+ |
+def run_go_clean(go_workspace, packages, goos=None, goarch=None): |
+ """Removes object files and executables left from building given packages. |
+ |
+ Transitively cleans all dependencies (including stdlib!) and removes |
+ executables from GOBIN. |
+ |
+ Args: |
+ go_workspace: path to 'infra/go' or 'infra_internal/go'. |
+ packages: list of go packages to clean (can include '...' patterns). |
+ goos: if set, overrides GOOS environment variable (removes it if ''). |
+ goarch: if set, overrides GOARCH environment variable (removes it if ''). |
+ """ |
+ with hacked_workspace(go_workspace, goos, goarch) as new_workspace: |
+ print_go_step_title('Cleaning:\n %s' % '\n '.join(packages)) |
subprocess.check_call( |
args=[ |
'python', '-u', os.path.join(new_workspace, 'env.py'), |
- 'go', 'install', '-a', '-v', |
+ 'go', 'clean', '-i', '-r', |
] + list(packages), |
executable=sys.executable, |
stderr=subprocess.STDOUT) |
- finally: |
- if new_root: |
- os.remove(new_root) |
+ # Above command is either silent (with '-x') or too verbose (with '-x'). |
nodir
2016/06/27 22:41:56
did you mean it is silent without -x?
Vadim Sh.
2016/06/27 22:54:00
Done.
|
+ # Prefer silent version, but add a note that it's alright. |
+ print '<nothing to see here>' |
nodir
2016/06/27 22:41:56
'Done'?
Vadim Sh.
2016/06/27 22:54:00
Done.
|
+ |
+ |
+def run_go_install( |
+ go_workspace, packages, rebuild=False, goos=None, goarch=None): |
+ """Builds (and installs) Go packages into GOBIN via 'go install ...'. |
+ |
+ Compiles and installs packages into default GOBIN, which is <go_workspace>/bin |
+ (it is setup by go/env.py). |
+ |
+ Args: |
+ go_workspace: path to 'infra/go' or 'infra_internal/go'. |
+ packages: list of go packages to build (can include '...' patterns). |
+ rebuild: if True, will forcefully rebuild all dependences. |
+ goos: if set, overrides GOOS environment variable (removes it if ''). |
+ goarch: if set, overrides GOARCH environment variable (removes it if ''). |
+ """ |
+ rebuild_opt = ['-a'] if rebuild else [] |
+ title = 'Rebuilding' if rebuild else 'Building' |
+ with hacked_workspace(go_workspace, goos, goarch) as new_workspace: |
+ print_go_step_title('%s:\n %s' % (title, '\n '.join(packages))) |
+ subprocess.check_call( |
+ args=[ |
+ 'python', '-u', os.path.join(new_workspace, 'env.py'), |
+ 'go', 'install', '-v', |
+ ] + rebuild_opt + list(packages), |
+ executable=sys.executable, |
+ stderr=subprocess.STDOUT) |
+ |
+ |
+def run_go_build( |
+ go_workspace, package, output, rebuild=False, goos=None, goarch=None): |
+ """Builds single Go package. |
+ |
+ Args: |
+ go_workspace: path to 'infra/go' or 'infra_internal/go'. |
+ package: go package to build. |
+ output: where to put the resulting binary. |
+ rebuild: if True, will forcefully rebuild all dependences. |
+ goos: if set, overrides GOOS environment variable (removes it if ''). |
+ goarch: if set, overrides GOARCH environment variable (removes it if ''). |
+ """ |
+ rebuild_opt = ['-a'] if rebuild else [] |
+ title = 'Rebuilding' if rebuild else 'Building' |
+ with hacked_workspace(go_workspace, goos, goarch) as new_workspace: |
+ print_go_step_title('%s %s' % (title, package)) |
+ subprocess.check_call( |
+ args=[ |
+ 'python', '-u', os.path.join(new_workspace, 'env.py'), |
+ 'go', 'build', |
+ ] + rebuild_opt + ['-v', '-o', output, package], |
+ executable=sys.executable, |
+ stderr=subprocess.STDOUT) |
+ |
+ |
+def build_go_code(go_workspace, pkg_defs): |
+ """Builds and installs all Go packages used by the given PackageDefs. |
+ Understands GOOS and GOARCH and uses slightly different build strategy when |
+ cross-compiling. In the end <go_workspace>/bin will have all built binaries, |
+ and only them (regardless of whether we are cross-compiling or not). |
-def enumerate_packages_to_build(package_def_dir, package_def_files=None): |
- """Returns a list of absolute paths to files in build/packages/*.yaml. |
+ Args: |
+ go_workspace: path to 'infra/go' or 'infra_internal/go'. |
+ pkg_defs: list of PackageDef objects that define what to build. |
+ """ |
+ # Grab a set of all go packages we need to build and install into GOBIN. |
+ to_install = [] |
+ for p in pkg_defs: |
+ to_install.extend(p.go_packages) |
+ to_install = sorted(set(to_install)) |
+ if not to_install: |
+ return |
+ |
+ # Make sure there are no stale files in the workspace. |
+ run_go_clean(go_workspace, to_install) |
+ |
+ if not is_cross_compiling(): |
+ # If not cross-compiling, build all Go code in a single "go install" step, |
+ # it's faster that way. We can't do that when cross-compiling, since |
+ # 'go install' isn't supposed to be used for cross-compilation and the |
+ # toolset actively complains with "go install: cannot install cross-compiled |
+ # binaries when GOBIN is set". |
+ run_go_install(go_workspace, to_install) |
+ else: |
+ # Build packages one by one and put the resulting binaries into GOBIN, as if |
+ # they were installed there. It's where the rest of the build.py code |
+ # expects them to be (see also 'root' property in package definition YAMLs). |
+ go_bin = os.path.join(go_workspace, 'bin') |
+ exe_suffix = get_target_package_vars()['exe_suffix'] |
+ for pkg in to_install: |
+ name = pkg[pkg.rfind('/')+1:] |
+ run_go_build(go_workspace, pkg, os.path.join(go_bin, name + exe_suffix)) |
+ |
+ |
+def enumerate_packages(py_venv, package_def_dir, package_def_files): |
+ """Returns a list PackageDef instances for files in build/packages/*.yaml. |
Args: |
+ py_env: path to python ENV where to look for YAML parser. |
package_def_dir: path to build/packages dir to search for *.yaml. |
package_def_files: optional list of filenames to limit results to. |
Returns: |
- List of absolute paths to *.yaml files under packages_dir. |
+ List of PackageDef instances parsed from *.yaml files under packages_dir. |
""" |
- # All existing package by default. |
- if not package_def_files: |
- return sorted(glob.glob(os.path.join(package_def_dir, '*.yaml'))) |
paths = [] |
- for name in package_def_files: |
- abs_path = os.path.join(package_def_dir, name) |
- if not os.path.isfile(abs_path): |
- raise BuildException('No such package definition file: %s' % name) |
- paths.append(abs_path) |
- return sorted(paths) |
+ if not package_def_files: |
+ # All existing package by default. |
+ paths = glob.glob(os.path.join(package_def_dir, '*.yaml')) |
+ else: |
+ # Otherwise pick only the ones in 'package_def_files' list. |
+ for name in package_def_files: |
+ abs_path = os.path.abspath(os.path.join(package_def_dir, name)) |
+ if not os.path.isfile(abs_path): |
+ raise BuildException('No such package definition file: %s' % name) |
+ paths.append(abs_path) |
+ return [PackageDef(p, read_yaml(py_venv, p)) for p in sorted(paths)] |
def read_yaml(py_venv, path): |
"""Returns content of YAML file as python dict.""" |
# YAML lib is in venv, not activated here. Go through hoops. |
+ # TODO(vadimsh): Doesn't work on ARM, since we have no working infra_python |
+ # venv there. Replace this hack with vendored pure-python PyYAML. |
oneliner = ( |
'import json, sys, yaml; ' |
'json.dump(yaml.safe_load(sys.stdin), sys.stdout)') |
@@ -193,21 +424,72 @@ def read_yaml(py_venv, path): |
return json.loads(out) |
-def should_process_on_builder(pkg_def_file, py_venv, builder): |
- """Returns True if package should be processed by current CI builder.""" |
- if not builder: |
- return True |
- builders = read_yaml(py_venv, pkg_def_file).get('builders') |
- return not builders or builder in builders |
- |
- |
def get_package_vars(): |
- """Returns a dict with variables that describe the current environment. |
+ """Returns a dict with variables that describe the package target environment. |
Variables can be referenced in the package definition YAML as |
${variable_name}. It allows to reuse exact same definition file for similar |
packages (e.g. packages with same cross platform binary, but for different |
platforms). |
+ |
+ If running in cross-compilation mode, uses GOOS and GOARCH to figure out the |
+ target platform instead of examining the host environment. |
+ """ |
+ if is_cross_compiling(): |
+ return get_target_package_vars() |
+ return get_host_package_vars() |
+ |
+ |
+def get_target_package_vars(): |
+ """Returns a dict with variables that describe cross-compilation target env. |
+ |
+ Examines os.environ for GOOS, GOARCH and GOARM. |
+ |
+ The returned dict contains only 'platform' and 'exe_suffix' entries. |
+ """ |
+ assert is_cross_compiling() |
+ goos = os.environ['GOOS'] |
+ goarch = os.environ['GOARCH'] |
+ |
+ if goarch not in ('386', 'amd64', 'arm'): |
+ raise BuildException('Unsupported GOARCH %s' % goarch) |
+ |
+ # There are many ARMs, pick the concrete instruction set. 'v6' is the default, |
+ # don't try to support other variants for now. |
+ # |
+ # See https://golang.org/doc/install/source#environment. |
+ if goarch == 'arm': |
+ goarm = os.environ.get('GOARM', '6') |
+ if goarm != '6': |
+ raise BuildException('Unsupported GOARM value %s' % goarm) |
+ arch = 'armv6l' |
+ else: |
+ arch = goarch |
+ |
+ # We use 'mac' instead of 'darwin'. |
+ if goos == 'darwin': |
+ goos = 'mac' |
+ |
+ return { |
+ 'exe_suffix': '.exe' if goos == 'windows' else '', |
+ 'platform': '%s-%s' % (goos, arch), |
+ } |
+ |
+ |
+def get_host_package_vars(): |
+ """Returns a dict with variables that describe the current host environment. |
+ |
+ The returned platform may not match the machine environment exactly, but it is |
+ compatible with it. |
+ |
+ For example, on ARMv7 machines we claim that we are in fact running ARMv6 |
+ (which is subset of ARMv7), since we don't really care about v7 over v6 |
+ difference and want to reduce the variability in supported architectures |
+ instead. |
+ |
+ Similarly, if running on 64-bit Linux with 32-bit user space (based on python |
+ interpreter bitness), we claim that machine is 32-bit, since most 32-bit Linux |
+ Chrome Infra bots are in fact running 64-bit kernels with 32-bit userlands. |
""" |
# linux, mac or windows. |
platform_variant = { |
@@ -240,10 +522,20 @@ def get_package_vars(): |
'i686': '386', |
'x86': '386', |
'x86_64': 'amd64', |
+ 'armv6l': 'armv6l', |
+ 'armv7l': 'armv6l', # we prefer to use older instruction set for builds |
}.get(platform.machine().lower()) |
if not platform_arch: |
raise ValueError('Unknown machine arch: %s' % platform.machine()) |
+ # Most 32-bit Linux Chrome Infra bots are in fact running 64-bit kernel with |
+ # 32-bit userland. Detect this case (based on bitness of the python |
+ # interpreter) and report the bot as '386'. |
+ if (platform_variant == 'linux' and |
+ platform_arch == 'amd64' and |
+ sys.maxsize == (2 ** 31) - 1): |
+ platform_arch = '386' |
+ |
return { |
# e.g. '.exe' or ''. |
'exe_suffix': EXE_SUFFIX, |
@@ -256,12 +548,12 @@ def get_package_vars(): |
} |
-def build_pkg(go_workspace, pkg_def_file, out_file, package_vars): |
+def build_pkg(cipd_exe, pkg_def, out_file, package_vars): |
"""Invokes CIPD client to build a package. |
Args: |
- go_workspace: path to 'infra/go' or 'infra_internal/go'. |
- pkg_def_file: path to *.yaml file with package definition. |
+ cipd_exe: path to cipd client binary to use. |
+ pkg_def: instance of PackageDef representing this package. |
out_file: where to store the built package. |
package_vars: dict with variables to pass as -pkg-var to cipd. |
@@ -271,18 +563,18 @@ def build_pkg(go_workspace, pkg_def_file, out_file, package_vars): |
Raises: |
BuildException on error. |
""" |
- print_title('Building: %s' % os.path.basename(pkg_def_file)) |
+ print_title('Building: %s' % os.path.basename(out_file)) |
# Make sure not stale output remains. |
if os.path.isfile(out_file): |
os.remove(out_file) |
# Build the package. |
- args = ['-pkg-def', pkg_def_file] |
+ args = ['-pkg-def', pkg_def.path] |
for k, v in sorted(package_vars.items()): |
args.extend(['-pkg-var', '%s:%s' % (k, v)]) |
args.extend(['-out', out_file]) |
- exit_code, json_output = run_cipd(go_workspace, 'pkg-build', args) |
+ exit_code, json_output = run_cipd(cipd_exe, 'pkg-build', args) |
if exit_code: |
print >> sys.stderr, 'FAILED! ' * 10 |
@@ -294,11 +586,11 @@ def build_pkg(go_workspace, pkg_def_file, out_file, package_vars): |
return info |
-def upload_pkg(go_workspace, pkg_file, service_url, tags, service_account): |
+def upload_pkg(cipd_exe, pkg_file, service_url, tags, service_account): |
"""Uploads existing *.cipd file to the storage and tags it. |
Args: |
- go_workspace: path to 'infra/go' or 'infra_internal/go'. |
+ cipd_exe: path to cipd client binary to use. |
pkg_file: path to *.cipd file to upload. |
service_url: URL of a package repository service. |
tags: a list of tags to attach to uploaded package instance. |
@@ -319,7 +611,7 @@ def upload_pkg(go_workspace, pkg_file, service_url, tags, service_account): |
if service_account: |
args.extend(['-service-account-json', service_account]) |
args.append(pkg_file) |
- exit_code, json_output = run_cipd(go_workspace, 'pkg-register', args) |
+ exit_code, json_output = run_cipd(cipd_exe, 'pkg-register', args) |
if exit_code: |
print >> sys.stderr, 'FAILED! ' * 10 |
@@ -329,6 +621,79 @@ def upload_pkg(go_workspace, pkg_file, service_url, tags, service_account): |
return info |
+def build_cipd_client(go_workspace, out_dir): |
+ """Builds cipd client binary for the host platform. |
+ |
+ Ignores GOOS and GOARCH env vars. Puts the client binary into |
+ '<out_dir>/.cipd_client/cipd_<digest>'. |
+ |
+ This binary is used by build.py itself and later by test_packages.py. |
+ |
+ Args: |
+ go_workspace: path to Go workspace root (contains 'env.py', 'src', etc). |
+ out_dir: build output directory, will be used to store the binary. |
+ |
+ Returns: |
+ Path to the built binary. |
+ """ |
+ # To avoid rebuilding cipd client all the time, we cache it in out/*, using |
+ # a combination of DEPS+deps.lock+bootstrap.py as a cache key (they define |
+ # exact set of sources used to build the cipd binary). |
+ # |
+ # We can't just use the client in infra.git/cipd/* because it is built by this |
+ # script itself: it introduced bootstrap dependency cycle in case we need to |
+ # add a new platform or if we wipe cipd backend storage. |
+ seed_paths = [ |
+ os.path.join(ROOT, 'DEPS'), |
+ os.path.join(ROOT, 'go', 'deps.lock'), |
+ os.path.join(ROOT, 'go', 'bootstrap.py'), |
+ ] |
+ digest = hashlib.sha1() |
+ for p in seed_paths: |
+ with open(p, 'rb') as f: |
+ digest.update(f.read()) |
+ cache_key = digest.hexdigest()[:20] |
+ |
+ # Already have it? |
+ cipd_out_dir = os.path.join(out_dir, '.cipd_client') |
+ cipd_exe = os.path.join(cipd_out_dir, 'cipd_%s%s' % (cache_key, EXE_SUFFIX)) |
+ if os.path.exists(cipd_exe): |
+ return cipd_exe |
+ |
+ # Nuke all previous copies, make sure out_dir exists. |
+ if os.path.exists(cipd_out_dir): |
+ for p in glob.glob(os.path.join(cipd_out_dir, 'cipd_*')): |
+ os.remove(p) |
+ else: |
+ os.makedirs(cipd_out_dir) |
+ |
+ # Build cipd client binary for the host platform. |
+ run_go_build( |
+ go_workspace, |
+ package='github.com/luci/luci-go/client/cmd/cipd', |
+ output=cipd_exe, |
+ rebuild=True, |
+ goos='', |
+ goarch='') |
+ |
+ return cipd_exe |
+ |
+ |
+def get_build_out_file(package_out_dir, pkg_def): |
+ """Returns a path where to put built *.cipd package file. |
+ |
+ Args: |
+ package_out_dir: root directory where to put *.cipd files. |
+ pkg_def: instance of PackageDef being built. |
+ """ |
+ # When cross-compiling, append a suffix to package file name to indicate that |
+ # it's for foreign platform. |
+ sfx = '' |
+ if is_cross_compiling(): |
+ sfx = '+' + get_target_package_vars()['platform'] |
+ return os.path.join(package_out_dir, pkg_def.name + sfx + '.cipd') |
+ |
+ |
def run( |
py_venv, |
go_workspace, |
@@ -343,7 +708,7 @@ def run( |
tags, |
service_account_json, |
json_output): |
- """Rebuild python and Go universes and CIPD packages. |
+ """Rebuilds python and Go universes and CIPD packages. |
Args: |
py_venv: path to 'infra/ENV' or 'infra_internal/ENV'. |
@@ -365,26 +730,39 @@ def run( |
""" |
assert build or upload, 'Both build and upload are False, nothing to do' |
- # Remove stale output so that test_packages.py do not test old files when |
- # invoked without arguments. |
- if build: |
- for path in glob.glob(os.path.join(package_out_dir, '*.cipd')): |
- os.remove(path) |
- |
- packages_to_build = [ |
- p for p in enumerate_packages_to_build(package_def_dir, package_def_files) |
- if should_process_on_builder(p, py_venv, builder) |
- ] |
+ # We need both GOOS and GOARCH or none. |
+ if is_cross_compiling(): |
+ if not os.environ.get('GOOS') or not os.environ.get('GOARCH'): |
+ print >> sys.stderr, ( |
+ 'When cross-compiling both GOOS and GOARCH environment variables ' |
+ 'must be set.') |
+ return 1 |
+ if os.environ.get('GOARM', '6') != '6': |
+ print >> sys.stderr, 'Only GOARM=6 is supported for now.' |
+ return 1 |
+ |
+ # Append tags related to the build host. They are especially important when |
+ # cross-compiling: cross-compiled packages can be identified by comparing the |
+ # platform in the package name with value of 'build_host_platform' tag. |
+ tags = list(tags) |
+ host_vars = get_host_package_vars() |
+ tags.append('build_host_hostname:' + socket.gethostname().split('.')[0]) |
+ tags.append('build_host_platform:' + host_vars['platform']) |
+ tags.append('build_host_os_ver:' + host_vars['os_ver']) |
+ |
+ all_packages = enumerate_packages(py_venv, package_def_dir, package_def_files) |
+ packages_to_build = [p for p in all_packages if p.should_build(builder)] |
print_title('Overview') |
- print 'Service URL: %s' % service_url |
+ if upload: |
+ print 'Service URL: %s' % service_url |
if builder: |
print 'Package definition files to process on %s:' % builder |
else: |
print 'Package definition files to process:' |
- for pkg_def_file in packages_to_build: |
- print ' %s' % os.path.basename(pkg_def_file) |
+ for pkg_def in packages_to_build: |
+ print ' %s' % pkg_def.name |
if not packages_to_build: |
print ' <none>' |
@@ -392,7 +770,7 @@ def run( |
package_vars = get_package_vars() |
for k, v in sorted(package_vars.items()): |
print ' %s = %s' % (k, v) |
- if tags: |
+ if upload and tags: |
print 'Tags to attach to uploaded packages:' |
for tag in sorted(tags): |
@@ -402,32 +780,62 @@ def run( |
print 'Nothing to do.' |
return 0 |
+ # Remove old build artifacts to avoid stale files in case the script crashes |
+ # for some reason. |
+ if build: |
+ print_title('Cleaning %s' % package_out_dir) |
+ if not os.path.exists(package_out_dir): |
+ os.makedirs(package_out_dir) |
+ cleaned = False |
+ for pkg_def in packages_to_build: |
+ out_file = get_build_out_file(package_out_dir, pkg_def) |
+ if os.path.exists(out_file): |
+ print 'Removing stale %s' % os.path.basename(out_file) |
+ os.remove(out_file) |
+ cleaned = True |
+ if not cleaned: |
+ print 'Nothing to clean' |
+ |
+ # Make sure we have a Go toolset and it matches the host platform we detected |
+ # in get_host_package_vars(). Otherwise we may end up uploading wrong binaries |
+ # under host platform CIPD package suffix. It's important on Linux with 64-bit |
+ # kernel and 32-bit userland (we must use 32-bit Go in that case, even if |
+ # 64-bit Go works too). |
+ go_env = bootstrap_go_toolset(go_workspace) |
+ expected_arch = host_vars['platform'].split('-')[1] |
+ if go_env['GOHOSTARCH'] != expected_arch: |
+ print >> sys.stderr, ( |
+ 'Go toolset GOHOSTARCH (%s) doesn\'t match expected architecture (%s)' % |
+ (go_env['GOHOSTARCH'], expected_arch)) |
+ return 1 |
+ |
+ # Build the cipd client needed later to build or upload packages. |
+ cipd_exe = build_cipd_client(go_workspace, package_out_dir) |
+ |
# Build the world. |
if build: |
- build_callback() |
+ build_callback(packages_to_build) |
# Package it. |
failed = [] |
succeeded = [] |
- for pkg_def_file in packages_to_build: |
- # path/name.yaml -> out/name.cipd. |
- name = os.path.splitext(os.path.basename(pkg_def_file))[0] |
- out_file = os.path.join(package_out_dir, name + '.cipd') |
+ for pkg_def in packages_to_build: |
+ out_file = get_build_out_file(package_out_dir, pkg_def) |
try: |
info = None |
if build: |
- info = build_pkg(go_workspace, pkg_def_file, out_file, package_vars) |
+ info = build_pkg(cipd_exe, pkg_def, out_file, package_vars) |
if upload: |
info = upload_pkg( |
- go_workspace, |
+ cipd_exe, |
out_file, |
service_url, |
tags, |
service_account_json) |
assert info is not None |
- succeeded.append({'pkg_def_name': name, 'info': info}) |
+ succeeded.append({'pkg_def_name': pkg_def.name, 'info': info}) |
except (BuildException, UploadException) as e: |
- failed.append({'pkg_def_name': name, 'error': str(e)}) |
+ failed.append({'pkg_def_name': pkg_def.name, 'error': str(e)}) |
print_title('Summary') |
for d in failed: |
@@ -448,23 +856,24 @@ def run( |
return 1 if failed else 0 |
-def build_infra(): |
- """Builds infra.git multiverse.""" |
- # Python side. |
- print_title('Making sure python virtual environment is fresh') |
- run_python( |
- script=os.path.join(ROOT, 'bootstrap', 'bootstrap.py'), |
- args=[ |
- '--deps_file', |
- os.path.join(ROOT, 'bootstrap', 'deps.pyl'), |
- os.path.join(ROOT, 'ENV'), |
- ]) |
- # Go side. |
- build_go(os.path.join(ROOT, 'go'), [ |
- 'infra/...', |
- 'github.com/luci/luci-go/client/...', |
- 'github.com/luci/luci-go/tools/...', |
- ]) |
+def build_infra(pkg_defs): |
+ """Builds infra.git multiverse. |
+ |
+ Args: |
+ pkg_defs: list of PackageDef instances for packages being built. |
+ """ |
+ # Skip building python if not used or if cross-compiling. |
+ if any(p.uses_python_env for p in pkg_defs) and not is_cross_compiling(): |
+ print_title('Making sure python virtual environment is fresh') |
+ run_python( |
+ script=os.path.join(ROOT, 'bootstrap', 'bootstrap.py'), |
+ args=[ |
+ '--deps_file', |
+ os.path.join(ROOT, 'bootstrap', 'deps.pyl'), |
+ os.path.join(ROOT, 'ENV'), |
+ ]) |
+ # Build all necessary go binaries. |
+ build_go_code(os.path.join(ROOT, 'go'), pkg_defs) |
def main( |