OLD | NEW |
---|---|
(Empty) | |
1 #!/usr/bin/env python | |
2 # Copyright 2014 The Chromium Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 import hashlib | |
7 import os | |
8 import shutil | |
9 import subprocess | |
10 import sys | |
11 import tempfile | |
12 | |
13 import bootstrap | |
14 | |
15 | |
16 GIT_REPO = 'git+https://chromium.googlesource.com/infra/third_party/{}' | |
17 SOURCE_URL = 'gs://{}/sources/{{}}'.format(bootstrap.BUCKET) | |
18 WHEELS_URL = 'gs://{}/wheels/'.format(bootstrap.BUCKET) | |
19 | |
20 MISSING_REPOS = { | |
21 'coverage': 'git+https://github.com/nedbat/coveragepy/', | |
agable
2014/07/10 17:12:57
Why aren't we mirroring this yet? We should mirror
iannucci
2014/07/11 02:14:46
Filed a bug: crbug.com/393015
| |
22 'python_dateutil': | |
23 'git+https://chromium.googlesource.com/infra/third_party/dateutil/' | |
agable
2014/07/10 17:12:57
Is this only 'missing' because its name is dateuti
iannucci
2014/07/11 02:14:46
Rename the repo. The name should match what's in s
| |
24 } | |
25 | |
26 | |
27 def has_custom_build(name): | |
28 return os.path.exists(os.path.join('custom_builds', name + '.py')) | |
29 | |
30 | |
31 def pip(*args, **kwargs): | |
32 subprocess.check_call((os.path.join(sys.prefix, 'bin', 'pip'),) + args, | |
33 **kwargs) | |
34 | |
35 | |
36 def wheel(arg, source_sha, build): | |
37 tdir = tempfile.mkdtemp() | |
38 try: | |
39 pip('wheel', '--no-index', '--no-deps', '--wheel-dir', tdir, arg) | |
40 grab_wheel(tdir, 'wheelhouse', source_sha, build) | |
41 finally: | |
42 shutil.rmtree(tdir) | |
43 | |
44 | |
45 def grab_wheel(src, dst, source_sha, build): | |
46 # late import lets us grab the virtualenv pip | |
47 from pip.wheel import Wheel | |
48 | |
49 items = os.listdir(src) | |
50 assert len(items) == 1, 'Wrong number of files in wheel directory: %r' % items | |
51 | |
52 wheelfile = items[0] | |
53 wheel_info = Wheel.wheel_file_re.match(wheelfile) | |
54 | |
55 assert wheel_info is not None, 'Not a wheel file? %r' % wheelfile | |
56 | |
57 os.rename(os.path.join(src, wheelfile), | |
58 os.path.join(dst, '%s-%s_%s%s' % ( | |
59 wheel_info.group('namever'), | |
60 build, | |
61 source_sha, | |
62 wheel_info.group(4) | |
63 ))) | |
64 | |
65 | |
66 def run_custom_build(name, link, sha, build): | |
67 from pip.index import Link | |
68 from pip.download import unpack_file_url, unpack_vcs_link, is_vcs_url | |
69 | |
70 assert has_custom_build(name) | |
71 | |
72 link = Link(link, trusted=True) | |
73 unpack = unpack_vcs_link if is_vcs_url(link) else unpack_file_url | |
74 | |
75 tmpd = tempfile.mkdtemp() | |
76 wheeld = tempfile.mkdtemp() | |
dnj
2014/07/10 16:06:18
This could fail after 'tmpd'. Since you have dual-
iannucci
2014/07/11 02:14:46
Did it moar betterz
| |
77 try: | |
78 unpack(link, tmpd) | |
79 m = getattr(__import__('custom_builds.%s' % name), name) | |
dnj
2014/07/10 16:06:18
'custom_builds.%s' % (name,)
iannucci
2014/07/11 02:14:46
Right. Done.
| |
80 m.Build(tmpd, wheeld) | |
81 grab_wheel(wheeld, 'wheelhouse', sha, build) | |
82 finally: | |
83 shutil.rmtree(tmpd) | |
84 shutil.rmtree(wheeld) | |
85 | |
86 | |
87 def process_git(name, rev, build): | |
88 print | |
89 print 'Processing (git)', name, rev | |
90 | |
91 if name not in MISSING_REPOS: | |
92 url = GIT_REPO.format(name) | |
93 else: | |
94 url = MISSING_REPOS[name] | |
95 | |
96 url += '@' + rev | |
97 | |
98 if not has_custom_build(name): | |
99 wheel(url, rev, build) | |
100 else: | |
101 run_custom_build(name, url, rev, build) | |
102 | |
103 | |
104 def process_gs(name, sha_ext, build): | |
105 print | |
106 print 'Processing (gs)', name, sha_ext | |
107 | |
108 sha, ext = sha_ext.split('.', 1) | |
109 tmp = tempfile.mktemp(ext) | |
dnj
2014/07/10 16:06:18
Consider 'with tempfile.NamedTemporaryFile(mode='w
iannucci
2014/07/11 02:14:46
Nop. Winders. Made more better tho.
Don't want t
| |
110 link = 'file://' + tmp | |
111 try: | |
112 subprocess.check_call(['gsutil', 'cp', SOURCE_URL.format(sha_ext), tmp]) | |
113 with open(tmp, 'rb') as f: | |
114 assert hashlib.sha1(f.read()).hexdigest() == sha | |
115 if not has_custom_build(name): | |
116 wheel(link, sha, build) | |
117 else: | |
118 run_custom_build(name, link, sha, build) | |
119 finally: | |
120 try: | |
121 os.unlink(tmp) | |
122 except OSError: | |
123 pass | |
124 | |
125 | |
126 def push_wheelhouse(): | |
127 return subprocess.call(['gsutil', '-m', 'cp', 'wheelhouse/*', WHEELS_URL]) | |
128 | |
129 | |
130 def main(to_build): | |
131 os.chdir(os.path.dirname(__file__)) | |
dnj
2014/07/10 16:06:18
I'm generally avoid 'chdir' when possible. I under
iannucci
2014/07/11 02:14:46
In general, I agree.
So I fixed it. :P
| |
132 build_env = os.path.abspath('BUILD_ENV') | |
133 | |
134 print 'Parsing deps.pst' | |
135 deps = bootstrap.read_deps('deps.pst') | |
136 bootstrap.activate_env(build_env, {'wheel': deps.pop('wheel')}) | |
137 | |
138 shutil.rmtree('wheelhouse', ignore_errors=True) | |
139 try: | |
140 os.makedirs('wheelhouse') | |
141 except OSError: | |
142 pass | |
143 | |
144 print 'Finding missing deps' | |
145 missing_deps = {} | |
146 for name, entry in deps.iteritems(): | |
147 if to_build and name not in to_build: | |
dnj
2014/07/10 16:06:18
It would be worth doing an "argparse" w/ "to_build
iannucci
2014/07/11 02:14:46
Fiiiiiine.... Done.
| |
148 continue | |
149 try: | |
150 bootstrap.get_links({name: entry}) | |
151 except bootstrap.NoWheelException: | |
152 missing_deps[name] = entry | |
153 | |
154 if not missing_deps: | |
155 print 'Nothing to process' | |
156 return | |
157 | |
158 print 'Processing deps:' | |
159 bootstrap.print_deps(missing_deps) | |
160 | |
161 for name, options in missing_deps.iteritems(): | |
162 # TODO(iannucci): skip entries which already exist in gs | |
163 if 'rev' in options: | |
164 process_git(name, options['rev'], options['build']) | |
165 elif 'gs' in options: | |
166 process_gs(name, options['gs'], options['build']) | |
167 else: | |
168 raise Exception('Invalid options %r for %r' % (options, name)) | |
169 | |
170 return push_wheelhouse() | |
171 | |
172 | |
173 if __name__ == '__main__': | |
174 sys.exit(main(sys.argv[1:])) | |
dnj
2014/07/10 16:06:18
If you are doing top-level logging initialization
| |
OLD | NEW |