Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(161)

Side by Side Diff: bootstrap/build_deps.py

Issue 381043002: Add a virtualenv-based python bootstrapping service to infra. (Closed) Base URL: https://chromium.googlesource.com/infra/infra@master
Patch Set: I fixed stuff Created 6 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 import argparse
7 import contextlib
8 import hashlib
9 import os
10 import shutil
11 import subprocess
12 import sys
13 import tempfile
14
15 import bootstrap
16
17
18 ROOT = os.path.dirname(os.path.abspath(__file__))
19 WHEELHOUSE = os.path.join(ROOT, 'wheelhouse')
20
21 GIT_REPO = 'git+https://chromium.googlesource.com/infra/third_party/{}'
22 SOURCE_URL = 'gs://{}/sources/{{}}'.format(bootstrap.BUCKET)
23 WHEELS_URL = 'gs://{}/wheels/'.format(bootstrap.BUCKET)
24
25
26 @contextlib.contextmanager
27 def tempdir(*args, **kwargs):
28 tdir = None
29 try:
30 tdir = tempfile.mkdtemp(*args, **kwargs)
31 yield tdir
32 finally:
33 if tdir:
34 shutil.rmtree(tdir, ignore_errors=True)
35
36
37 @contextlib.contextmanager
38 def tempname(*args, **kwargs):
39 tmp = None
40 try:
41 tmp = tempfile.mktemp(*args, **kwargs)
42 yield tmp
43 finally:
44 if tmp:
45 try:
46 os.unlink(tmp)
47 except OSError:
48 pass
49
50
51 def has_custom_build(name):
52 return os.path.exists(os.path.join(ROOT, 'custom_builds', name + '.py'))
53
54
55 def pip(*args, **kwargs):
56 subprocess.check_call(
57 (os.path.join(sys.prefix, 'bin', 'pip'),) + args, **kwargs)
58
59
60 def wheel(arg, source_sha, build):
61 with tempdir() as tdir:
62 pip('wheel', '--no-index', '--no-deps', '--wheel-dir', tdir, arg)
63 grab_wheel(tdir, WHEELHOUSE, source_sha, build)
64
65
66 def grab_wheel(src, dst, source_sha, build):
67 # late import lets us grab the virtualenv pip
68 from pip.wheel import Wheel
69
70 items = os.listdir(src)
71 assert len(items) == 1, 'Wrong number of files in wheel directory: %r' % items
72
73 wheelfile = items[0]
74 wheel_info = Wheel.wheel_file_re.match(wheelfile)
75
76 assert wheel_info is not None, 'Not a wheel file? %r' % wheelfile
77
78 os.rename(os.path.join(src, wheelfile),
79 os.path.join(dst, '%s-%s_%s%s' % (
80 wheel_info.group('namever'),
81 build,
82 source_sha,
83 wheel_info.group(4)
84 )))
85
86
87 def run_custom_build(name, link, sha, build):
88 from pip.index import Link
89 from pip.download import unpack_file_url, unpack_vcs_link, is_vcs_url
90
91 assert has_custom_build(name)
92
93 link = Link(link, trusted=True)
94 unpack = unpack_vcs_link if is_vcs_url(link) else unpack_file_url
95
96 with tempdir() as tmpd, tempdir() as wheeld: # pylint: disable=C0321
97 unpack(link, tmpd)
98 m = getattr(__import__('custom_builds.%s' % (name,)), name)
99 m.Build(tmpd, wheeld)
100 grab_wheel(wheeld, WHEELHOUSE, sha, build)
101
102
103 def process_git(name, rev, build):
104 print
105 print 'Processing (git)', name, rev
106
107 url = GIT_REPO.format(name) + '@' + rev
108
109 if not has_custom_build(name):
110 wheel(url, rev, build)
111 else:
112 run_custom_build(name, url, rev, build)
113
114
115 def process_gs(name, sha_ext, build):
116 print
117 print 'Processing (gs)', name, sha_ext
118
119 sha, ext = sha_ext.split('.', 1)
120 with tempname(ext) as tmp:
121 link = 'file://' + tmp
122 subprocess.check_call(['gsutil', 'cp', SOURCE_URL.format(sha_ext), tmp])
123 with open(tmp, 'rb') as f:
124 assert hashlib.sha1(f.read()).hexdigest() == sha
125 if not has_custom_build(name):
126 wheel(link, sha, build)
127 else:
128 run_custom_build(name, link, sha, build)
129
130
131 def clear_wheelhouse():
132 shutil.rmtree(WHEELHOUSE, ignore_errors=True)
133 try:
134 os.makedirs(WHEELHOUSE)
135 except OSError:
136 pass
137
138
139 def push_wheelhouse():
140 return subprocess.call(['gsutil', '-m', 'cp', WHEELHOUSE + '/*', WHEELS_URL])
141
142
143 def main(args):
144 parser = argparse.ArgumentParser()
145 parser.add_argument(
146 'to_build', nargs='*',
147 help='Names of packages to build. Defaults to all packages.')
148 opts = parser.parse_args(args)
149 to_build = set(opts.to_build)
150
151 build_env = os.path.join(ROOT, 'BUILD_ENV')
152
153 print 'Parsing deps.pyl'
154 deps = bootstrap.read_deps(os.path.join(ROOT, 'deps.pyl'))
155 bootstrap.activate_env(build_env, {'wheel': deps.pop('wheel')})
156
157 print 'Finding missing deps'
158 missing_deps = {}
159 for name, entry in deps.iteritems():
160 if to_build and name not in to_build:
161 continue
162 try:
163 bootstrap.get_links({name: entry})
164 except bootstrap.NoWheelException:
165 missing_deps[name] = entry
166
167 if not missing_deps:
168 print 'Nothing to process'
169 return
170
171 print 'Processing deps:'
172 bootstrap.print_deps(missing_deps)
173
174 for name, options in missing_deps.iteritems():
175 clear_wheelhouse()
176 # TODO(iannucci): skip entries which already exist in gs
177 if 'rev' in options:
178 process_git(name, options['rev'], options['build'])
179 elif 'gs' in options:
180 process_gs(name, options['gs'], options['build'])
181 else:
182 raise Exception('Invalid options %r for %r' % (options, name))
183 push_wheelhouse()
184
185
186 if __name__ == '__main__':
187 sys.exit(main(sys.argv[1:]))
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698