OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/env python |
| 2 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. |
| 5 |
| 6 import argparse |
| 7 import contextlib |
| 8 import logging |
| 9 import os |
| 10 import shutil |
| 11 import subprocess |
| 12 import sys |
| 13 import tempfile |
| 14 |
| 15 from util import STORAGE_URL, OBJECT_URL, read_deps, print_deps, platform_tag |
| 16 |
| 17 LOGGER = logging.getLogger(__name__) |
| 18 |
| 19 # /path/to/infra |
| 20 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 21 |
| 22 |
| 23 class NoWheelException(Exception): |
| 24 def __init__(self, name, version, build, source_sha): |
| 25 super(NoWheelException, self).__init__( |
| 26 'No matching wheel found for (%s==%s (build %s_%s))' % |
| 27 (name, version, build, source_sha)) |
| 28 |
| 29 |
| 30 def ls(prefix): |
| 31 from pip._vendor import requests |
| 32 data = requests.get(STORAGE_URL, params=dict( |
| 33 prefix=prefix, |
| 34 fields='items(name,md5Hash)' |
| 35 )).json() |
| 36 return data.get('items', ()) |
| 37 |
| 38 |
| 39 def sha_for(deps_entry): |
| 40 if 'rev' in deps_entry: |
| 41 return deps_entry['rev'] |
| 42 else: |
| 43 return deps_entry['gs'].split('.')[0] |
| 44 |
| 45 |
| 46 def get_links(deps): |
| 47 import pip.wheel |
| 48 |
| 49 plat_tag = platform_tag() |
| 50 |
| 51 links = [] |
| 52 |
| 53 for name, dep in deps.iteritems(): |
| 54 version, source_sha = dep['version'] , sha_for(dep) |
| 55 prefix = 'wheels/{}-{}-{}_{}'.format(name, version, dep['build'], |
| 56 source_sha) |
| 57 generic_link = None |
| 58 binary_link = None |
| 59 |
| 60 for entry in ls(prefix): |
| 61 fname = entry['name'].split('/')[-1] |
| 62 md5hash = entry['md5Hash'].decode('base64').encode('hex') |
| 63 wheel_info = pip.wheel.Wheel.wheel_file_re.match(fname) |
| 64 if not wheel_info: |
| 65 LOGGER.warn('Skipping invalid wheel: %r', fname) |
| 66 continue |
| 67 |
| 68 if pip.wheel.Wheel(fname).supported(): |
| 69 link = OBJECT_URL.format(entry['name'], md5hash) |
| 70 if plat_tag and plat_tag in fname: |
| 71 if binary_link: |
| 72 LOGGER.error( |
| 73 'Found more than one binary matching wheel for %r: %r', |
| 74 prefix, dep) |
| 75 continue |
| 76 binary_link = link |
| 77 else: |
| 78 if generic_link: |
| 79 LOGGER.error( |
| 80 'Found more than one generic matching wheel for %r: %r', |
| 81 prefix, dep) |
| 82 continue |
| 83 generic_link = link |
| 84 |
| 85 if not binary_link and not generic_link: |
| 86 raise NoWheelException(name, version, dep['build'], source_sha) |
| 87 |
| 88 links.append(binary_link or generic_link) |
| 89 |
| 90 return links |
| 91 |
| 92 |
| 93 @contextlib.contextmanager |
| 94 def html_index(links): |
| 95 tf = tempfile.mktemp('.html') |
| 96 try: |
| 97 with open(tf, 'w') as f: |
| 98 print >> f, '<html><body>' |
| 99 for link in links: |
| 100 print >> f, '<a href="%s">wat</a>' % link |
| 101 print >> f, '</body></html>' |
| 102 yield tf |
| 103 finally: |
| 104 os.unlink(tf) |
| 105 |
| 106 |
| 107 def install(deps): |
| 108 py = os.path.join(sys.prefix, 'bin', 'python') |
| 109 pip = os.path.join(sys.prefix, 'bin', 'pip') |
| 110 |
| 111 links = get_links(deps) |
| 112 with html_index(links) as ipath: |
| 113 requirements = [] |
| 114 # TODO(iannucci): Do this as a requirements.txt |
| 115 for name, deps_entry in deps.iteritems(): |
| 116 if not deps_entry.get('implicit'): |
| 117 requirements.append('%s==%s' % (name, deps_entry['version'])) |
| 118 subprocess.check_call( |
| 119 [py, pip, 'install', '--no-index', '--download-cache', |
| 120 os.path.join(ROOT, '.wheelcache'), '-f', ipath] + requirements) |
| 121 |
| 122 |
| 123 def activate_env(env, deps): |
| 124 if hasattr(sys, 'real_prefix'): |
| 125 LOGGER.error('Already activated environment!') |
| 126 return |
| 127 |
| 128 print 'Activating environment: %r' % env |
| 129 if isinstance(deps, basestring): |
| 130 deps = read_deps(deps) |
| 131 assert deps is not None |
| 132 |
| 133 manifest_path = os.path.join(env, 'manifest.pyl') |
| 134 cur_deps = read_deps(manifest_path) |
| 135 if cur_deps != deps: |
| 136 print ' Removing old environment: %r' % cur_deps |
| 137 shutil.rmtree(env, ignore_errors=True) |
| 138 cur_deps = None |
| 139 |
| 140 if cur_deps is None: |
| 141 print ' Building new environment' |
| 142 # Add in bundled virtualenv lib |
| 143 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'virtualenv')) |
| 144 import virtualenv # pylint: disable=F0401 |
| 145 virtualenv.create_environment( |
| 146 env, search_dirs=virtualenv.file_search_dirs()) |
| 147 |
| 148 print ' Activating environment' |
| 149 activate_this = os.path.join(env, 'bin', 'activate_this.py') |
| 150 execfile(activate_this, dict(__file__=activate_this)) |
| 151 |
| 152 if cur_deps is None: |
| 153 print ' Installing deps' |
| 154 print_deps(deps, indent=2, with_implicit=False) |
| 155 install(deps) |
| 156 with open(manifest_path, 'wb') as f: |
| 157 f.write(repr(deps) + '\n') |
| 158 |
| 159 print 'Done creating environment' |
| 160 |
| 161 |
| 162 def main(args): |
| 163 parser = argparse.ArgumentParser() |
| 164 parser.add_argument('--deps_file', |
| 165 help='Path to python deps file (default: %(default)s)') |
| 166 parser.add_argument('env_path', nargs='?', |
| 167 help='Path to place environment (default: %(default)s)', |
| 168 default='ENV') |
| 169 opts = parser.parse_args(args) |
| 170 |
| 171 activate_env(opts.env_path, opts.deps_file or {}) |
| 172 |
| 173 |
| 174 if __name__ == '__main__': |
| 175 logging.basicConfig() |
| 176 LOGGER.setLevel(logging.DEBUG) |
| 177 sys.exit(main(sys.argv[1:])) |
OLD | NEW |