Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(287)

Side by Side Diff: bootstrap/bootstrap.py

Issue 381043002: Add a virtualenv-based python bootstrapping service to infra. (Closed) Base URL: https://chromium.googlesource.com/infra/infra@master
Patch Set: Address comments, add warning while building on Ubuntu Created 6 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « bootstrap/README.md ('k') | bootstrap/bootstrap_wheel_wheel.sh » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 import argparse
7 import contextlib
8 import logging
9 import os
10 import shutil
11 import subprocess
12 import sys
13 import tempfile
14
15 from util import STORAGE_URL, OBJECT_URL, read_deps, print_deps, platform_tag
16
17 LOGGER = logging.getLogger(__name__)
18
19 # /path/to/infra
20 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
21
22
23 class NoWheelException(Exception):
24 def __init__(self, name, version, build, source_sha):
25 super(NoWheelException, self).__init__(
26 'No matching wheel found for (%s==%s (build %s_%s))' %
27 (name, version, build, source_sha))
28
29
30 def ls(prefix):
31 from pip._vendor import requests
32 data = requests.get(STORAGE_URL, params=dict(
33 prefix=prefix,
34 fields='items(name,md5Hash)'
35 )).json()
36 return data.get('items', ())
37
38
39 def sha_for(deps_entry):
40 if 'rev' in deps_entry:
41 return deps_entry['rev']
42 else:
43 return deps_entry['gs'].split('.')[0]
44
45
46 def get_links(deps):
47 import pip.wheel
48
49 plat_tag = platform_tag()
50
51 links = []
52
53 for name, dep in deps.iteritems():
54 version, source_sha = dep['version'] , sha_for(dep)
55 prefix = 'wheels/{}-{}-{}_{}'.format(name, version, dep['build'],
56 source_sha)
57 generic_link = None
58 binary_link = None
59
60 for entry in ls(prefix):
61 fname = entry['name'].split('/')[-1]
62 md5hash = entry['md5Hash'].decode('base64').encode('hex')
63 wheel_info = pip.wheel.Wheel.wheel_file_re.match(fname)
64 if not wheel_info:
65 LOGGER.warn('Skipping invalid wheel: %r', fname)
66 continue
67
68 if pip.wheel.Wheel(fname).supported():
69 link = OBJECT_URL.format(entry['name'], md5hash)
70 if plat_tag and plat_tag in fname:
71 if binary_link:
72 LOGGER.error(
73 'Found more than one binary matching wheel for %r: %r',
74 prefix, dep)
75 continue
76 binary_link = link
77 else:
78 if generic_link:
79 LOGGER.error(
80 'Found more than one generic matching wheel for %r: %r',
81 prefix, dep)
82 continue
83 generic_link = link
84
85 if not binary_link and not generic_link:
86 raise NoWheelException(name, version, dep['build'], source_sha)
87
88 links.append(binary_link or generic_link)
89
90 return links
91
92
93 @contextlib.contextmanager
94 def html_index(links):
95 tf = tempfile.mktemp('.html')
96 try:
97 with open(tf, 'w') as f:
98 print >> f, '<html><body>'
99 for link in links:
100 print >> f, '<a href="%s">wat</a>' % link
101 print >> f, '</body></html>'
102 yield tf
103 finally:
104 os.unlink(tf)
105
106
107 def install(deps):
108 py = os.path.join(sys.prefix, 'bin', 'python')
109 pip = os.path.join(sys.prefix, 'bin', 'pip')
110
111 links = get_links(deps)
112 with html_index(links) as ipath:
113 requirements = []
114 # TODO(iannucci): Do this as a requirements.txt
115 for name, deps_entry in deps.iteritems():
116 if not deps_entry.get('implicit'):
117 requirements.append('%s==%s' % (name, deps_entry['version']))
118 subprocess.check_call(
119 [py, pip, 'install', '--no-index', '--download-cache',
120 os.path.join(ROOT, '.wheelcache'), '-f', ipath] + requirements)
121
122
123 def activate_env(env, deps):
124 if hasattr(sys, 'real_prefix'):
125 LOGGER.error('Already activated environment!')
126 return
127
128 print 'Activating environment: %r' % env
129 if isinstance(deps, basestring):
130 deps = read_deps(deps)
131 assert deps is not None
132
133 manifest_path = os.path.join(env, 'manifest.pyl')
134 cur_deps = read_deps(manifest_path)
135 if cur_deps != deps:
136 print ' Removing old environment: %r' % cur_deps
137 shutil.rmtree(env, ignore_errors=True)
138 cur_deps = None
139
140 if cur_deps is None:
141 print ' Building new environment'
142 # Add in bundled virtualenv lib
143 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'virtualenv'))
144 import virtualenv # pylint: disable=F0401
145 virtualenv.create_environment(
146 env, search_dirs=virtualenv.file_search_dirs())
147
148 print ' Activating environment'
149 activate_this = os.path.join(env, 'bin', 'activate_this.py')
150 execfile(activate_this, dict(__file__=activate_this))
151
152 if cur_deps is None:
153 print ' Installing deps'
154 print_deps(deps, indent=2, with_implicit=False)
155 install(deps)
156 with open(manifest_path, 'wb') as f:
157 f.write(repr(deps) + '\n')
158
159 print 'Done creating environment'
160
161
162 def main(args):
163 parser = argparse.ArgumentParser()
164 parser.add_argument('--deps_file',
165 help='Path to python deps file (default: %(default)s)')
166 parser.add_argument('env_path', nargs='?',
167 help='Path to place environment (default: %(default)s)',
168 default='ENV')
169 opts = parser.parse_args(args)
170
171 activate_env(opts.env_path, opts.deps_file or {})
172
173
174 if __name__ == '__main__':
175 logging.basicConfig()
176 LOGGER.setLevel(logging.DEBUG)
177 sys.exit(main(sys.argv[1:]))
OLDNEW
« no previous file with comments | « bootstrap/README.md ('k') | bootstrap/bootstrap_wheel_wheel.sh » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698