| OLD | NEW |
|---|
| (Empty) | |
| 1 #!/usr/bin/env python |
| 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. |
| 5 |
| 6 import argparse |
| 7 import ast |
| 8 import contextlib |
| 9 import logging |
| 10 import os |
| 11 import shutil |
| 12 import subprocess |
| 13 import sys |
| 14 import tempfile |
| 15 |
| 16 LOGGER = logging.getLogger(__name__) |
| 17 LOGGER.setLevel(logging.DEBUG) |
| 18 |
| 19 # /path/to/infra |
| 20 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 21 |
| 22 BUCKET = 'chrome-python-wheelhouse' |
| 23 STORAGE_URL = 'https://www.googleapis.com/storage/v1/b/{}/o'.format(BUCKET) |
| 24 OBJECT_URL = 'https://storage.googleapis.com/{}/{{}}#md5={{}}'.format(BUCKET) |
| 25 |
| 26 |
| 27 class NoWheelException(Exception): |
| 28 def __init__(self, name, version, build, source_sha): |
| 29 super(NoWheelException, self).__init__( |
| 30 'No matching wheel found for (%s==%s (build %s_%s))' % |
| 31 (name, version, build, source_sha)) |
| 32 |
| 33 |
| 34 def ls(prefix): |
| 35 from pip._vendor import requests |
| 36 data = requests.get(STORAGE_URL, params=dict( |
| 37 prefix=prefix, |
| 38 fields='items(name,md5Hash)' |
| 39 )).json() |
| 40 return data.get('items', ()) |
| 41 |
| 42 |
| 43 def sha_for(deps_entry): |
| 44 if 'rev' in deps_entry: |
| 45 return deps_entry['rev'] |
| 46 else: |
| 47 return deps_entry['gs'].split('.')[0] |
| 48 |
| 49 |
| 50 def print_deps(deps, indent=1, with_implicit=True): |
| 51 for dep, entry in deps.iteritems(): |
| 52 if not with_implicit and entry.get('implicit'): |
| 53 continue |
| 54 print ' ' * indent + '%s: %r' % (dep, entry) |
| 55 print |
| 56 |
| 57 |
| 58 def read_deps(path): |
| 59 if os.path.exists(path): |
| 60 with open(path, 'rb') as f: |
| 61 return ast.literal_eval(f.read()) |
| 62 |
| 63 |
| 64 def get_links(deps): |
| 65 import pip.wheel |
| 66 |
| 67 links = [] |
| 68 |
| 69 for name, entry in deps.iteritems(): |
| 70 version, source_sha = entry['version'] , sha_for(entry) |
| 71 prefix = 'wheels/{}-{}-{}_{}'.format(name, version, entry['build'], |
| 72 source_sha) |
| 73 link = None |
| 74 count = 0 |
| 75 |
| 76 for entry in ls(prefix): |
| 77 fname = entry['name'].split('/')[-1] |
| 78 md5hash = entry['md5Hash'].decode('base64').encode('hex') |
| 79 wheel_info = pip.wheel.Wheel.wheel_file_re.match(fname) |
| 80 if not wheel_info: |
| 81 LOGGER.warn('Skipping invalid wheel: %r', fname) |
| 82 continue |
| 83 |
| 84 if pip.wheel.Wheel(fname).supported(): |
| 85 count += 1 |
| 86 if count > 1: |
| 87 LOGGER.error('Found more than one matching wheel for %r: %r', |
| 88 prefix, entry) |
| 89 continue |
| 90 link = OBJECT_URL.format(entry['name'], md5hash) |
| 91 |
| 92 if link is None: |
| 93 raise NoWheelException(name, version, entry['build'], source_sha) |
| 94 |
| 95 links.append(link) |
| 96 |
| 97 return links |
| 98 |
| 99 |
| 100 @contextlib.contextmanager |
| 101 def html_index(links): |
| 102 tf = tempfile.mktemp('.html') |
| 103 try: |
| 104 with open(tf, 'w') as f: |
| 105 print >> f, '<html><body>' |
| 106 for link in links: |
| 107 print >> f, '<a href="%s">wat</a>' % link |
| 108 print >> f, '</body></html>' |
| 109 yield tf |
| 110 finally: |
| 111 os.unlink(tf) |
| 112 |
| 113 |
| 114 def install(deps): |
| 115 py = os.path.join(sys.prefix, 'bin', 'python') |
| 116 pip = os.path.join(sys.prefix, 'bin', 'pip') |
| 117 |
| 118 links = get_links(deps) |
| 119 with html_index(links) as ipath: |
| 120 requirements = [] |
| 121 # TODO(iannucci): Do this as a requirements.txt |
| 122 for name, deps_entry in deps.iteritems(): |
| 123 if not deps_entry.get('implicit'): |
| 124 requirements.append('%s==%s' % (name, deps_entry['version'])) |
| 125 subprocess.check_call( |
| 126 [py, pip, 'install', '--no-index', '--download-cache', |
| 127 os.path.join(ROOT, '.wheelcache'), '-f', ipath] + requirements) |
| 128 |
| 129 |
| 130 def activate_env(env, deps): |
| 131 if hasattr(sys, 'real_prefix'): |
| 132 LOGGER.error('Already activated environment!') |
| 133 return |
| 134 |
| 135 print 'Activating environment: %r' % env |
| 136 if isinstance(deps, basestring): |
| 137 deps = read_deps(deps) |
| 138 assert deps is not None |
| 139 |
| 140 manifest_path = os.path.join(env, 'manifest.pyl') |
| 141 cur_deps = read_deps(manifest_path) |
| 142 if cur_deps != deps: |
| 143 print ' Removing old environment: %r' % cur_deps |
| 144 shutil.rmtree(env, ignore_errors=True) |
| 145 cur_deps = None |
| 146 |
| 147 if cur_deps is None: |
| 148 print ' Building new environment' |
| 149 # Add in bundled virtualenv lib |
| 150 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'virtualenv')) |
| 151 import virtualenv # pylint: disable=F0401 |
| 152 virtualenv.create_environment( |
| 153 env, search_dirs=virtualenv.file_search_dirs()) |
| 154 |
| 155 print ' Activating environment' |
| 156 activate_this = os.path.join(env, 'bin', 'activate_this.py') |
| 157 execfile(activate_this, dict(__file__=activate_this)) |
| 158 |
| 159 if cur_deps is None: |
| 160 print ' Installing deps' |
| 161 print_deps(deps, indent=2, with_implicit=False) |
| 162 install(deps) |
| 163 with open(manifest_path, 'wb') as f: |
| 164 f.write(repr(deps) + '\n') |
| 165 |
| 166 print 'Done creating environment' |
| 167 |
| 168 |
| 169 def main(args): |
| 170 parser = argparse.ArgumentParser() |
| 171 parser.add_argument('--deps_file', |
| 172 help='Path to python deps file (default: %(default)s)') |
| 173 parser.add_argument('env_path', nargs='?', |
| 174 help='Path to place environment (default: %(default)s)', |
| 175 default='ENV') |
| 176 opts = parser.parse_args(args) |
| 177 |
| 178 activate_env(opts.env_path, opts.deps_file or {}) |
| 179 |
| 180 |
| 181 if __name__ == '__main__': |
| 182 logging.basicConfig() |
| 183 sys.exit(main(sys.argv[1:])) |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 381043002:
Add a virtualenv-based python bootstrapping service to infra. (Closed)
Base URL: https://chromium.googlesource.com/infra/infra@master