OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/env python |
| 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. |
| 5 |
| 6 import argparse |
| 7 import contextlib |
| 8 import hashlib |
| 9 import os |
| 10 import shutil |
| 11 import subprocess |
| 12 import sys |
| 13 import tempfile |
| 14 |
| 15 import bootstrap |
| 16 |
| 17 |
| 18 ROOT = os.path.dirname(os.path.abspath(__file__)) |
| 19 WHEELHOUSE = os.path.join(ROOT, 'wheelhouse') |
| 20 |
| 21 GIT_REPO = 'git+https://chromium.googlesource.com/infra/third_party/{}' |
| 22 SOURCE_URL = 'gs://{}/sources/{{}}'.format(bootstrap.BUCKET) |
| 23 WHEELS_URL = 'gs://{}/wheels/'.format(bootstrap.BUCKET) |
| 24 |
| 25 |
| 26 @contextlib.contextmanager |
| 27 def tempdir(*args, **kwargs): |
| 28 tdir = None |
| 29 try: |
| 30 tdir = tempfile.mkdtemp(*args, **kwargs) |
| 31 yield tdir |
| 32 finally: |
| 33 if tdir: |
| 34 shutil.rmtree(tdir, ignore_errors=True) |
| 35 |
| 36 |
| 37 @contextlib.contextmanager |
| 38 def tempname(*args, **kwargs): |
| 39 tmp = None |
| 40 try: |
| 41 tmp = tempfile.mktemp(*args, **kwargs) |
| 42 yield tmp |
| 43 finally: |
| 44 if tmp: |
| 45 try: |
| 46 os.unlink(tmp) |
| 47 except OSError: |
| 48 pass |
| 49 |
| 50 |
| 51 def has_custom_build(name): |
| 52 return os.path.exists(os.path.join(ROOT, 'custom_builds', name + '.py')) |
| 53 |
| 54 |
| 55 def pip(*args, **kwargs): |
| 56 subprocess.check_call( |
| 57 (os.path.join(sys.prefix, 'bin', 'pip'),) + args, **kwargs) |
| 58 |
| 59 |
| 60 def wheel(arg, source_sha, build): |
| 61 with tempdir() as tdir: |
| 62 pip('wheel', '--no-index', '--no-deps', '--wheel-dir', tdir, arg) |
| 63 grab_wheel(tdir, WHEELHOUSE, source_sha, build) |
| 64 |
| 65 |
| 66 def grab_wheel(src, dst, source_sha, build): |
| 67 # late import lets us grab the virtualenv pip |
| 68 from pip.wheel import Wheel |
| 69 |
| 70 items = os.listdir(src) |
| 71 assert len(items) == 1, 'Wrong number of files in wheel directory: %r' % items |
| 72 |
| 73 wheelfile = items[0] |
| 74 wheel_info = Wheel.wheel_file_re.match(wheelfile) |
| 75 |
| 76 assert wheel_info is not None, 'Not a wheel file? %r' % wheelfile |
| 77 |
| 78 os.rename(os.path.join(src, wheelfile), |
| 79 os.path.join(dst, '%s-%s_%s%s' % ( |
| 80 wheel_info.group('namever'), |
| 81 build, |
| 82 source_sha, |
| 83 wheel_info.group(4) |
| 84 ))) |
| 85 |
| 86 |
| 87 def run_custom_build(name, link, sha, build): |
| 88 from pip.index import Link |
| 89 from pip.download import unpack_file_url, unpack_vcs_link, is_vcs_url |
| 90 |
| 91 assert has_custom_build(name) |
| 92 |
| 93 link = Link(link, trusted=True) |
| 94 unpack = unpack_vcs_link if is_vcs_url(link) else unpack_file_url |
| 95 |
| 96 with tempdir() as tmpd, tempdir() as wheeld: # pylint: disable=C0321 |
| 97 unpack(link, tmpd) |
| 98 m = getattr(__import__('custom_builds.%s' % (name,)), name) |
| 99 m.Build(tmpd, wheeld) |
| 100 grab_wheel(wheeld, WHEELHOUSE, sha, build) |
| 101 |
| 102 |
| 103 def process_git(name, rev, build): |
| 104 print |
| 105 print 'Processing (git)', name, rev |
| 106 |
| 107 url = GIT_REPO.format(name) + '@' + rev |
| 108 |
| 109 if not has_custom_build(name): |
| 110 wheel(url, rev, build) |
| 111 else: |
| 112 run_custom_build(name, url, rev, build) |
| 113 |
| 114 |
| 115 def process_gs(name, sha_ext, build): |
| 116 print |
| 117 print 'Processing (gs)', name, sha_ext |
| 118 |
| 119 sha, ext = sha_ext.split('.', 1) |
| 120 with tempname(ext) as tmp: |
| 121 link = 'file://' + tmp |
| 122 subprocess.check_call(['gsutil', 'cp', SOURCE_URL.format(sha_ext), tmp]) |
| 123 with open(tmp, 'rb') as f: |
| 124 assert hashlib.sha1(f.read()).hexdigest() == sha |
| 125 if not has_custom_build(name): |
| 126 wheel(link, sha, build) |
| 127 else: |
| 128 run_custom_build(name, link, sha, build) |
| 129 |
| 130 |
| 131 def clear_wheelhouse(): |
| 132 shutil.rmtree(WHEELHOUSE, ignore_errors=True) |
| 133 try: |
| 134 os.makedirs(WHEELHOUSE) |
| 135 except OSError: |
| 136 pass |
| 137 |
| 138 |
| 139 def push_wheelhouse(): |
| 140 return subprocess.call(['gsutil', '-m', 'cp', WHEELHOUSE + '/*', WHEELS_URL]) |
| 141 |
| 142 |
| 143 def main(args): |
| 144 parser = argparse.ArgumentParser() |
| 145 parser.add_argument( |
| 146 'to_build', nargs='*', |
| 147 help='Names of packages to build. Defaults to all packages.') |
| 148 opts = parser.parse_args(args) |
| 149 to_build = set(opts.to_build) |
| 150 |
| 151 build_env = os.path.join(ROOT, 'BUILD_ENV') |
| 152 |
| 153 print 'Parsing deps.pyl' |
| 154 deps = bootstrap.read_deps(os.path.join(ROOT, 'deps.pyl')) |
| 155 bootstrap.activate_env(build_env, {'wheel': deps.pop('wheel')}) |
| 156 |
| 157 print 'Finding missing deps' |
| 158 missing_deps = {} |
| 159 for name, entry in deps.iteritems(): |
| 160 if to_build and name not in to_build: |
| 161 continue |
| 162 try: |
| 163 bootstrap.get_links({name: entry}) |
| 164 except bootstrap.NoWheelException: |
| 165 missing_deps[name] = entry |
| 166 |
| 167 if not missing_deps: |
| 168 print 'Nothing to process' |
| 169 return |
| 170 |
| 171 print 'Processing deps:' |
| 172 bootstrap.print_deps(missing_deps) |
| 173 |
| 174 for name, options in missing_deps.iteritems(): |
| 175 clear_wheelhouse() |
| 176 # TODO(iannucci): skip entries which already exist in gs |
| 177 if 'rev' in options: |
| 178 process_git(name, options['rev'], options['build']) |
| 179 elif 'gs' in options: |
| 180 process_gs(name, options['gs'], options['build']) |
| 181 else: |
| 182 raise Exception('Invalid options %r for %r' % (options, name)) |
| 183 push_wheelhouse() |
| 184 |
| 185 |
| 186 if __name__ == '__main__': |
| 187 sys.exit(main(sys.argv[1:])) |
OLD | NEW |