OLD | NEW |
---|---|
(Empty) | |
1 #!/usr/bin/env python | |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 import argparse | |
7 import ast | |
8 import contextlib | |
9 import logging | |
10 import os | |
11 import shutil | |
12 import subprocess | |
13 import sys | |
14 import tempfile | |
15 | |
16 logging.basicConfig() | |
dnj
2014/07/10 16:06:17
I like to put the 'basicConfig' and initial 'setLe
agable
2014/07/10 17:12:57
+1. This also lets you only set the loglevel once
iannucci
2014/07/11 02:14:45
Done.
| |
17 LOGGER = logging.getLogger(__name__) | |
18 LOGGER.setLevel(logging.DEBUG) | |
19 | |
20 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
agable
2014/07/10 17:12:57
Comment on the expected path of root (i.e. /path/t
iannucci
2014/07/11 02:14:45
Done.
| |
21 | |
22 BUCKET = 'chrome-python-wheelhouse' | |
23 STORAGE_URL = 'https://www.googleapis.com/storage/v1/b/{}/o'.format(BUCKET) | |
24 OBJECT_URL = 'https://storage.googleapis.com/{}/{{}}#md5={{}}'.format(BUCKET) | |
25 | |
26 | |
27 class NoWheelException(Exception): | |
28 def __init__(self, name, version, build, source_sha): | |
29 super(NoWheelException, self).__init__( | |
30 'No matching wheel found for (%s==%s-%s_%s)' % | |
agable
2014/07/10 17:12:56
I think this version-build_sha format (and why is
iannucci
2014/07/11 02:14:45
Unfortunately I don't think I can... they're const
| |
31 (name, version, build, source_sha)) | |
32 | |
33 | |
34 def ls(prefix): | |
35 from pip._vendor import requests | |
36 data = requests.get(STORAGE_URL, params=dict( | |
37 prefix=prefix, | |
38 fields='items(name,md5Hash)' | |
39 )).json() | |
40 return data.get('items', ()) | |
41 | |
42 | |
43 def sha_for(deps_entry): | |
44 if 'rev' in deps_entry: | |
45 return deps_entry['rev'] | |
46 else: | |
47 return deps_entry['gs'].split('.')[0] | |
48 | |
49 | |
50 def print_deps(deps, indent=1, with_implicit=True): | |
dnj
2014/07/10 16:06:17
If you anticipate any reuse, consider allowing a s
agable
2014/07/10 17:12:56
If you do this, this might be a good opportunity f
iannucci
2014/07/11 02:14:45
¯\_(ツ)_/¯
I'll refactor it if something needs it.
| |
51 for dep, entry in deps.iteritems(): | |
52 if not with_implicit and entry.get('implicit'): | |
53 continue | |
54 print ' ' * indent + '%s: %r' % (dep, entry) | |
55 print | |
56 | |
57 | |
58 def read_deps(path): | |
59 if os.path.exists(path): | |
60 with open(path, 'rb') as f: | |
dnj
2014/07/10 16:06:17
Maybe pass a 'local' boolean that optionally toggl
iannucci
2014/07/11 02:14:45
nah, it's just for newlines. It's all single-line
| |
61 return ast.literal_eval(f.read()) | |
62 | |
63 | |
64 def get_links(deps): | |
65 import pip.wheel | |
Vadim Sh.
2014/07/10 18:18:24
nit: from pip import wheel
| |
66 | |
67 links = [] | |
68 | |
69 for name, entry in deps.iteritems(): | |
70 version, source_sha = entry['version'] , sha_for(entry) | |
Vadim Sh.
2014/07/10 18:18:24
nit: remove space before ','
| |
71 prefix = 'wheels/{}-{}-{}_{}'.format(name, version, entry['build'], | |
iannucci
2014/07/11 02:14:45
from vadim: add a platform tag for linux (ubuntu)
| |
72 source_sha) | |
73 link = None | |
74 | |
agable
2014/07/10 17:12:56
nit: no newline
iannucci
2014/07/11 02:14:45
Done.
| |
75 count = 0 | |
76 | |
77 for entry in ls(prefix): | |
78 fname = entry['name'].split('/')[-1] | |
79 md5hash = entry['md5Hash'].decode('base64').encode('hex') | |
80 wheel_info = pip.wheel.Wheel.wheel_file_re.match(fname) | |
81 if not wheel_info: | |
82 LOGGER.warn('Skipping invalid wheel: %r', fname) | |
83 continue | |
84 | |
85 if pip.wheel.Wheel(fname).supported(): | |
86 count += 1 | |
87 if count > 1: | |
88 LOGGER.error('Found more than one matching wheel? %r', entry) | |
agable
2014/07/10 17:12:56
... matching wheel for %r.
iannucci
2014/07/11 02:14:45
Done.
| |
89 continue | |
90 link = OBJECT_URL.format(entry['name'], md5hash) | |
91 | |
92 if link is None: | |
93 raise NoWheelException(name, version, entry['build'], source_sha) | |
94 | |
95 links.append(link) | |
96 | |
97 return links | |
98 | |
99 | |
100 @contextlib.contextmanager | |
101 def html_index(links): | |
102 tf = tempfile.mktemp('.html') | |
dnj
2014/07/10 16:06:17
Consider using 'tempfile.NamedTemporaryFile', whic
iannucci
2014/07/11 02:14:45
Yep, doesn't work on winders b/c the file can't be
| |
103 try: | |
104 with open(tf, 'w') as f: | |
105 print >> f, '<html><body>' | |
106 for link in links: | |
107 print >> f, '<a href="%s">wat</a>' % link | |
dnj
2014/07/10 16:06:17
Is "wat" going in the final version? :)
iannucci
2014/07/11 02:14:45
Yep, link text is ignored. It's dumb, dumb format.
agable
2014/07/11 18:38:16
Maybe include a comment to that effect.
| |
108 print >> f, '</body></html>' | |
109 yield tf | |
110 finally: | |
111 os.unlink(tf) | |
112 | |
113 | |
114 def install(deps): | |
115 py = os.path.join(sys.prefix, 'bin', 'python') | |
Vadim Sh.
2014/07/10 18:18:24
Does this work on windows? Also why not sys.execut
| |
116 pip = os.path.join(sys.prefix, 'bin', 'pip') | |
117 | |
118 links = get_links(deps) | |
119 with html_index(links) as ipath: | |
120 requirements = [] | |
121 # TODO(iannucci): Do this as a requirements.txt | |
122 for name, deps_entry in deps.iteritems(): | |
123 if not deps_entry.get('implicit'): | |
124 requirements.append('%s==%s' % (name, deps_entry['version'])) | |
125 subprocess.check_call( | |
126 [py, pip, 'install', '--no-index', '--download-cache', | |
127 os.path.join(ROOT, '.wheelcache'), '-f', ipath] + requirements) | |
128 | |
129 | |
130 def activate_env(env, deps): | |
131 print 'Activating environment: %r' % env | |
Vadim Sh.
2014/07/10 18:18:24
nit: use logging for this? Even logging.debug(...)
| |
132 if isinstance(deps, basestring): | |
dnj
2014/07/10 16:06:17
Is there a difference between 'basestring' and 'ty
agable
2014/07/10 17:12:57
Official python docs disagree, sorry :)
iannucci
2014/07/11 02:14:45
Acknowledged.
| |
133 deps = read_deps(deps) | |
134 assert deps is not None | |
dnj
2014/07/10 16:06:17
This seems useful to attach an error message to, l
iannucci
2014/07/11 02:14:45
Icing for later, this needs to land.
| |
135 | |
136 if hasattr(sys, 'real_prefix'): | |
dnj
2014/07/10 16:06:17
Do this check before loading 'deps'? Since you sho
iannucci
2014/07/11 02:14:45
Done.
| |
137 LOGGER.error('Already activated environment!') | |
138 return | |
139 | |
140 manifest_path = os.path.join(env, 'manifest.pst') | |
141 cur_deps = read_deps(manifest_path) | |
dnj
2014/07/10 16:06:17
If you use the local 'read_deps' idea, this would
iannucci
2014/07/11 02:14:45
Acknowledged.
| |
142 if cur_deps != deps: | |
143 print ' Removing old environment: %r' % cur_deps | |
144 shutil.rmtree(env, ignore_errors=True) | |
dnj
2014/07/10 16:06:17
You may want to do a check to ensure that none of
iannucci
2014/07/11 02:14:45
I already check at the top that we're not in a vir
| |
145 cur_deps = None | |
146 | |
147 if cur_deps is None: | |
148 print ' Building new environment' | |
149 # Add in bundled virtualenv lib | |
150 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'virtualenv')) | |
151 import virtualenv # pylint: disable=F0401 | |
152 virtualenv.create_environment( | |
153 env, search_dirs=virtualenv.file_search_dirs()) | |
154 | |
155 print ' Activating environment' | |
156 activate_this = os.path.join(env, 'bin', 'activate_this.py') | |
157 execfile(activate_this, dict(__file__=activate_this)) | |
158 | |
159 if cur_deps is None: | |
160 print ' Installing deps' | |
161 print_deps(deps, indent=2, with_implicit=False) | |
162 install(deps) | |
163 with open(manifest_path, 'wb') as f: | |
dnj
2014/07/10 16:06:17
If you use the local 'read_deps' flag idea, this w
iannucci
2014/07/11 02:14:45
Acknowledged.
| |
164 f.write(repr(deps) + '\n') | |
dnj
2014/07/10 16:06:17
Any love for 'pprint'? Also could emit the 'vim' l
iannucci
2014/07/11 02:14:45
mer. no one is going to poke at the manifest. can
| |
165 | |
166 print 'Done creating environment' | |
167 | |
168 | |
169 def main(args): | |
170 parser = argparse.ArgumentParser() | |
171 parser.add_argument('--deps_file', | |
172 help='Path to python deps file (default: %(default)s)') | |
173 parser.add_argument('env_path', nargs='?', | |
174 help='Path to place environment (default: %(default)s)', | |
175 default='ENV') | |
176 opts = parser.parse_args(args) | |
177 | |
178 activate_env(opts.env_path, opts.deps_file or {}) | |
179 | |
180 | |
181 if __name__ == '__main__': | |
182 sys.exit(main(sys.argv[1:])) | |
dnj
2014/07/10 16:06:17
Logger-friendly version would catch uncaught Excep
| |
OLD | NEW |