Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1079)

Side by Side Diff: bootstrap/build_deps.py

Issue 381043002: Add a virtualenv-based python bootstrapping service to infra. (Closed) Base URL: https://chromium.googlesource.com/infra/infra@master
Patch Set: Created 6 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 import hashlib
7 import os
8 import shutil
9 import subprocess
10 import sys
11 import tempfile
12
13 import bootstrap
14
15
16 GIT_REPO = 'git+https://chromium.googlesource.com/infra/third_party/{}'
17 SOURCE_URL = 'gs://{}/sources/{{}}'.format(bootstrap.BUCKET)
18 WHEELS_URL = 'gs://{}/wheels/'.format(bootstrap.BUCKET)
19
20 MISSING_REPOS = {
21 'coverage': 'git+https://github.com/nedbat/coveragepy/',
agable 2014/07/10 17:12:57 Why aren't we mirroring this yet? We should mirror
iannucci 2014/07/11 02:14:46 Filed a bug: crbug.com/393015
22 'python_dateutil':
23 'git+https://chromium.googlesource.com/infra/third_party/dateutil/'
agable 2014/07/10 17:12:57 Is this only 'missing' because its name is dateuti
iannucci 2014/07/11 02:14:46 Rename the repo. The name should match what's in s
24 }
25
26
27 def has_custom_build(name):
28 return os.path.exists(os.path.join('custom_builds', name + '.py'))
29
30
31 def pip(*args, **kwargs):
32 subprocess.check_call((os.path.join(sys.prefix, 'bin', 'pip'),) + args,
33 **kwargs)
34
35
36 def wheel(arg, source_sha, build):
37 tdir = tempfile.mkdtemp()
38 try:
39 pip('wheel', '--no-index', '--no-deps', '--wheel-dir', tdir, arg)
40 grab_wheel(tdir, 'wheelhouse', source_sha, build)
41 finally:
42 shutil.rmtree(tdir)
43
44
45 def grab_wheel(src, dst, source_sha, build):
46 # late import lets us grab the virtualenv pip
47 from pip.wheel import Wheel
48
49 items = os.listdir(src)
50 assert len(items) == 1, 'Wrong number of files in wheel directory: %r' % items
51
52 wheelfile = items[0]
53 wheel_info = Wheel.wheel_file_re.match(wheelfile)
54
55 assert wheel_info is not None, 'Not a wheel file? %r' % wheelfile
56
57 os.rename(os.path.join(src, wheelfile),
58 os.path.join(dst, '%s-%s_%s%s' % (
59 wheel_info.group('namever'),
60 build,
61 source_sha,
62 wheel_info.group(4)
63 )))
64
65
66 def run_custom_build(name, link, sha, build):
67 from pip.index import Link
68 from pip.download import unpack_file_url, unpack_vcs_link, is_vcs_url
69
70 assert has_custom_build(name)
71
72 link = Link(link, trusted=True)
73 unpack = unpack_vcs_link if is_vcs_url(link) else unpack_file_url
74
75 tmpd = tempfile.mkdtemp()
76 wheeld = tempfile.mkdtemp()
dnj 2014/07/10 16:06:18 This could fail after 'tmpd'. Since you have dual-
iannucci 2014/07/11 02:14:46 Did it moar betterz
77 try:
78 unpack(link, tmpd)
79 m = getattr(__import__('custom_builds.%s' % name), name)
dnj 2014/07/10 16:06:18 'custom_builds.%s' % (name,)
iannucci 2014/07/11 02:14:46 Right. Done.
80 m.Build(tmpd, wheeld)
81 grab_wheel(wheeld, 'wheelhouse', sha, build)
82 finally:
83 shutil.rmtree(tmpd)
84 shutil.rmtree(wheeld)
85
86
87 def process_git(name, rev, build):
88 print
89 print 'Processing (git)', name, rev
90
91 if name not in MISSING_REPOS:
92 url = GIT_REPO.format(name)
93 else:
94 url = MISSING_REPOS[name]
95
96 url += '@' + rev
97
98 if not has_custom_build(name):
99 wheel(url, rev, build)
100 else:
101 run_custom_build(name, url, rev, build)
102
103
104 def process_gs(name, sha_ext, build):
105 print
106 print 'Processing (gs)', name, sha_ext
107
108 sha, ext = sha_ext.split('.', 1)
109 tmp = tempfile.mktemp(ext)
dnj 2014/07/10 16:06:18 Consider 'with tempfile.NamedTemporaryFile(mode='w
iannucci 2014/07/11 02:14:46 Nop. Winders. Made more better tho. Don't want t
110 link = 'file://' + tmp
111 try:
112 subprocess.check_call(['gsutil', 'cp', SOURCE_URL.format(sha_ext), tmp])
113 with open(tmp, 'rb') as f:
114 assert hashlib.sha1(f.read()).hexdigest() == sha
115 if not has_custom_build(name):
116 wheel(link, sha, build)
117 else:
118 run_custom_build(name, link, sha, build)
119 finally:
120 try:
121 os.unlink(tmp)
122 except OSError:
123 pass
124
125
126 def push_wheelhouse():
127 return subprocess.call(['gsutil', '-m', 'cp', 'wheelhouse/*', WHEELS_URL])
128
129
130 def main(to_build):
131 os.chdir(os.path.dirname(__file__))
dnj 2014/07/10 16:06:18 I'm generally avoid 'chdir' when possible. I under
iannucci 2014/07/11 02:14:46 In general, I agree. So I fixed it. :P
132 build_env = os.path.abspath('BUILD_ENV')
133
134 print 'Parsing deps.pst'
135 deps = bootstrap.read_deps('deps.pst')
136 bootstrap.activate_env(build_env, {'wheel': deps.pop('wheel')})
137
138 shutil.rmtree('wheelhouse', ignore_errors=True)
139 try:
140 os.makedirs('wheelhouse')
141 except OSError:
142 pass
143
144 print 'Finding missing deps'
145 missing_deps = {}
146 for name, entry in deps.iteritems():
147 if to_build and name not in to_build:
dnj 2014/07/10 16:06:18 It would be worth doing an "argparse" w/ "to_build
iannucci 2014/07/11 02:14:46 Fiiiiiine.... Done.
148 continue
149 try:
150 bootstrap.get_links({name: entry})
151 except bootstrap.NoWheelException:
152 missing_deps[name] = entry
153
154 if not missing_deps:
155 print 'Nothing to process'
156 return
157
158 print 'Processing deps:'
159 bootstrap.print_deps(missing_deps)
160
161 for name, options in missing_deps.iteritems():
162 # TODO(iannucci): skip entries which already exist in gs
163 if 'rev' in options:
164 process_git(name, options['rev'], options['build'])
165 elif 'gs' in options:
166 process_gs(name, options['gs'], options['build'])
167 else:
168 raise Exception('Invalid options %r for %r' % (options, name))
169
170 return push_wheelhouse()
171
172
173 if __name__ == '__main__':
174 sys.exit(main(sys.argv[1:]))
dnj 2014/07/10 16:06:18 If you are doing top-level logging initialization
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698