Chromium Code Reviews Chromium Code Reviews
Issue 1200843003:
Added virtualenv for depot_tools (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/depot_tools| OLD | NEW |
|---|---|
| (Empty) | |
| 1 #!/usr/bin/env python | |
| 2 # Copyright 2014 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 import argparse | |
| 7 import contextlib | |
| 8 import glob | |
| 9 import logging | |
| 10 import os | |
| 11 import shutil | |
| 12 import subprocess | |
| 13 import sys | |
| 14 import tempfile | |
| 15 | |
| 16 from util import STORAGE_URL, OBJECT_URL, LOCAL_STORAGE_PATH, LOCAL_OBJECT_URL | |
| 17 from util import read_deps, merge_deps, print_deps, platform_tag | |
| 18 | |
| 19 LOGGER = logging.getLogger(__name__) | |
| 20 | |
| 21 # /path/to/infra | |
| 22 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| 23 | |
| 24 PYTHON_BAT_WIN = '@%~dp0\\..\\Scripts\\python.exe %*' | |
| 25 | |
| 26 | |
| 27 class NoWheelException(Exception): | |
| 28 def __init__(self, name, version, build, source_sha): | |
| 29 super(NoWheelException, self).__init__( | |
| 30 'No matching wheel found for (%s==%s (build %s_%s))' % | |
| 31 (name, version, build, source_sha)) | |
| 32 | |
| 33 | |
| 34 def check_pydistutils(): | |
| 35 if os.path.exists(os.path.expanduser('~/.pydistutils.cfg')): | |
| 36 print >> sys.stderr, '\n'.join([ | |
| 37 '', | |
| 38 '', | |
| 39 '=========== ERROR ===========', | |
| 40 'You have a ~/.pydistutils.cfg file, which interferes with the ', | |
| 41 'infra virtualenv environment. Please move it to the side and bootstrap ', | |
| 42 'again. Once infra has bootstrapped, you may move it back.', | |
| 43 '', | |
| 44 'Upstream bug: https://github.com/pypa/virtualenv/issues/88/', | |
| 45 '' | |
| 46 ]) | |
| 47 sys.exit(1) | |
| 48 | |
| 49 | |
| 50 def ls(prefix): | |
| 51 from pip._vendor import requests # pylint: disable=E0611 | |
| 52 data = requests.get(STORAGE_URL, params=dict( | |
| 53 prefix=prefix, | |
| 54 fields='items(name,md5Hash)' | |
| 55 )).json() | |
| 56 entries = data.get('items', []) | |
| 57 for entry in entries: | |
| 58 entry['md5Hash'] = entry['md5Hash'].decode('base64').encode('hex') | |
| 59 entry['local'] = False | |
| 60 # Also look in the local cache | |
| 61 entries.extend([ | |
| 62 {'name': fname, 'md5Hash': None, 'local': True} | |
| 63 for fname in glob.glob(os.path.join(LOCAL_STORAGE_PATH, | |
| 64 prefix.split('/')[-1] + '*'))]) | |
| 65 return entries | |
| 66 | |
| 67 | |
| 68 def sha_for(deps_entry): | |
| 69 if 'rev' in deps_entry: | |
| 70 return deps_entry['rev'] | |
| 71 else: | |
| 72 return deps_entry['gs'].split('.')[0] | |
| 73 | |
| 74 | |
| 75 def get_links(deps): | |
| 76 import pip.wheel # pylint: disable=E0611 | |
| 77 plat_tag = platform_tag() | |
| 78 | |
| 79 links = [] | |
| 80 | |
| 81 for name, dep in deps.iteritems(): | |
| 82 version, source_sha = dep['version'] , sha_for(dep) | |
| 83 prefix = 'wheels/{}-{}-{}_{}'.format(name, version, dep['build'], | |
| 84 source_sha) | |
| 85 generic_link = None | |
| 86 binary_link = None | |
| 87 local_link = None | |
| 88 | |
| 89 for entry in ls(prefix): | |
| 90 fname = entry['name'].split('/')[-1] | |
| 91 md5hash = entry['md5Hash'] | |
| 92 wheel_info = pip.wheel.Wheel.wheel_file_re.match(fname) | |
| 93 if not wheel_info: | |
| 94 LOGGER.warn('Skipping invalid wheel: %r', fname) | |
| 95 continue | |
| 96 | |
| 97 if pip.wheel.Wheel(fname).supported(): | |
| 98 if entry['local']: | |
| 99 link = LOCAL_OBJECT_URL.format(entry['name']) | |
| 100 local_link = link | |
| 101 continue | |
| 102 else: | |
| 103 link = OBJECT_URL.format(entry['name'], md5hash) | |
| 104 if fname.endswith('none-any.whl'): | |
| 105 if generic_link: | |
| 106 LOGGER.error( | |
| 107 'Found more than one generic matching wheel for %r: %r', | |
| 108 prefix, dep) | |
| 109 continue | |
| 110 generic_link = link | |
| 111 elif plat_tag in fname: | |
| 112 if binary_link: | |
| 113 LOGGER.error( | |
| 114 'Found more than one binary matching wheel for %r: %r', | |
| 115 prefix, dep) | |
| 116 continue | |
| 117 binary_link = link | |
| 118 | |
| 119 if not binary_link and not generic_link and not local_link: | |
| 120 raise NoWheelException(name, version, dep['build'], source_sha) | |
| 121 | |
| 122 links.append(local_link or binary_link or generic_link) | |
| 123 | |
| 124 return links | |
| 125 | |
| 126 | |
| 127 @contextlib.contextmanager | |
| 128 def html_index(links): | |
| 129 tf = tempfile.mktemp('.html') | |
| 130 try: | |
| 131 with open(tf, 'w') as f: | |
| 132 print >> f, '<html><body>' | |
| 133 for link in links: | |
| 134 print >> f, '<a href="%s">wat</a>' % link | |
| 135 print >> f, '</body></html>' | |
| 136 yield tf | |
| 137 finally: | |
| 138 os.unlink(tf) | |
| 139 | |
| 140 | |
| 141 def install(deps): | |
| 142 bin_dir = 'Scripts' if sys.platform.startswith('win') else 'bin' | |
| 143 pip = os.path.join(sys.prefix, bin_dir, 'pip') | |
| 144 | |
| 145 links = get_links(deps) | |
| 146 with html_index(links) as ipath: | |
| 147 requirements = [] | |
| 148 # TODO(iannucci): Do this as a requirements.txt | |
| 149 for name, deps_entry in deps.iteritems(): | |
| 150 if not deps_entry.get('implicit'): | |
| 151 requirements.append('%s==%s' % (name, deps_entry['version'])) | |
| 152 subprocess.check_call( | |
| 153 [pip, 'install', '--no-index', '--download-cache', | |
| 154 os.path.join(ROOT, '.wheelcache'), '-f', ipath] + requirements) | |
| 155 | |
| 156 | |
| 157 def activate_env(env, deps): | |
| 158 if hasattr(sys, 'real_prefix'): | |
| 159 LOGGER.error('Already activated environment!') | |
| 160 return | |
| 161 | |
| 162 print 'Activating environment: %r' % env | |
| 163 assert isinstance(deps, dict) | |
| 164 | |
| 165 manifest_path = os.path.join(env, 'manifest.pyl') | |
| 166 cur_deps = read_deps(manifest_path) | |
| 167 if cur_deps != deps: | |
| 168 print ' Removing old environment: %r' % cur_deps | |
| 169 shutil.rmtree(env, ignore_errors=True) | |
| 170 cur_deps = None | |
| 171 | |
| 172 if cur_deps is None: | |
| 173 check_pydistutils() | |
| 174 | |
| 175 print ' Building new environment' | |
| 176 # Add in bundled virtualenv lib | |
| 177 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'virtualenv')) | |
| 178 import virtualenv # pylint: disable=F0401 | |
| 179 virtualenv.create_environment( | |
| 180 env, search_dirs=virtualenv.file_search_dirs()) | |
| 181 | |
| 182 print ' Activating environment' | |
| 183 # Ensure hermeticity during activation. | |
| 184 os.environ.pop('PYTHONPATH', None) | |
| 185 bin_dir = 'Scripts' if sys.platform.startswith('win') else 'bin' | |
| 186 activate_this = os.path.join(env, bin_dir, 'activate_this.py') | |
| 187 execfile(activate_this, dict(__file__=activate_this)) | |
| 188 | |
| 189 if cur_deps is None: | |
| 190 print ' Installing deps' | |
| 191 print_deps(deps, indent=2, with_implicit=False) | |
| 192 install(deps) | |
| 193 virtualenv.make_environment_relocatable(env) | |
| 194 with open(manifest_path, 'wb') as f: | |
| 195 f.write(repr(deps) + '\n') | |
| 196 | |
| 197 # Create bin\python.bat on Windows to unify path where Python is found. | |
|
Sergiy Byelozyorov
2015/06/22 17:26:32
This block below is new and wan't present in infra
| |
| 198 if sys.platform.startswith('win'): | |
| 199 bin_path = os.path.join(env, 'bin') | |
| 200 if not os.path.isdir(bin_path): | |
| 201 os.makedirs(bin_path) | |
| 202 python_bat_path = os.path.join(bin_path, 'python.bat') | |
| 203 if not os.path.isfile(python_bat_path): | |
| 204 with open(python_bat_path, 'w') as python_bat_file: | |
| 205 python_bat_file.write(PYTHON_BAT_WIN) | |
| 206 | |
| 207 print 'Done creating environment' | |
| 208 | |
| 209 | |
| 210 def main(args): | |
| 211 parser = argparse.ArgumentParser() | |
| 212 parser.add_argument('--deps-file', '--deps_file', action='append', | |
| 213 help='Path to deps.pyl file (may be used multiple times)') | |
| 214 parser.add_argument('env_path', | |
| 215 help='Path to place environment (default: %(default)s)', | |
| 216 default='ENV') | |
| 217 opts = parser.parse_args(args) | |
| 218 | |
| 219 deps = merge_deps(opts.deps_file) | |
| 220 activate_env(opts.env_path, deps) | |
| 221 | |
| 222 | |
| 223 if __name__ == '__main__': | |
| 224 logging.basicConfig() | |
| 225 LOGGER.setLevel(logging.DEBUG) | |
| 226 sys.exit(main(sys.argv[1:])) | |
| OLD | NEW |
