OLD | NEW |
(Empty) | |
| 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. |
| 4 |
| 5 import argparse |
| 6 import errno |
| 7 import hashlib |
| 8 import json |
| 9 import os |
| 10 import sys |
| 11 import tempfile |
| 12 import time |
| 13 import traceback |
| 14 import urllib |
| 15 import urllib2 |
| 16 |
| 17 |
| 18 # Default package repository URL. |
| 19 CIPD_BACKEND_URL = 'https://chrome-infra-packages.appspot.com' |
| 20 |
| 21 # ./cipd resolve \ |
| 22 # infra/tools/cipd/ \ |
| 23 # -version=git_revision:768586a9aa72fe6a41a8a0205ed56ceb1495625c |
| 24 CLIENT_VERSIONS = { |
| 25 'linux-386': '3f5d6d3906c3b2c6a057d1ab1634ac06aa418708', |
| 26 'linux-amd64': '1ea0b6b254ad3f546c826dd3e437798ace2c2480', |
| 27 'mac-amd64': 'c111be343c692e5285113a6b1c999887adbb268e', |
| 28 'windows-386': 'dc3d1bd5b4b93945640bac4bb047c333a8fa12fd', |
| 29 'windows-amd64': 'bccdb9a605037e3dd2a8a64e79e08f691a6f159d', |
| 30 } |
| 31 |
| 32 |
| 33 class CipdBootstrapError(Exception): |
| 34 """Raised by install_cipd_client on fatal error.""" |
| 35 |
| 36 def install_cipd_client(path, package, version): |
| 37 """Installs CIPD client to <path>/cipd. |
| 38 |
| 39 Args: |
| 40 path: root directory to install CIPD client into. |
| 41 package: cipd client package name, e.g. infra/tools/cipd/linux-amd64. |
| 42 version: version of the package to install. |
| 43 |
| 44 Returns: |
| 45 Absolute path to CIPD executable. |
| 46 """ |
| 47 print 'Ensuring CIPD client is up-to-date' |
| 48 version_file = os.path.join(path, 'VERSION') |
| 49 bin_file = os.path.join(path, 'cipd') |
| 50 |
| 51 # Resolve version to concrete instance ID, e.g "live" -> "abcdef0123....". |
| 52 instance_id = call_cipd_api( |
| 53 'repo/v1/instance/resolve', |
| 54 {'package_name': package, 'version': version})['instance_id'] |
| 55 print 'CIPD client %s => %s' % (version, instance_id) |
| 56 |
| 57 # Already installed? |
| 58 installed_instance_id = (read_file(version_file) or '').strip() |
| 59 if installed_instance_id == instance_id and os.path.exists(bin_file): |
| 60 return bin_file, instance_id |
| 61 |
| 62 # Resolve instance ID to an URL to fetch client binary from. |
| 63 client_info = call_cipd_api( |
| 64 'repo/v1/client', |
| 65 {'package_name': package, 'instance_id': instance_id}) |
| 66 print 'CIPD client binary info:\n%s' % dump_json(client_info) |
| 67 |
| 68 # Fetch the client. It is ~10 MB, so don't bother and fetch it into memory. |
| 69 status, raw_client_bin = fetch_url(client_info['client_binary']['fetch_url']) |
| 70 if status != 200: |
| 71 print 'Failed to fetch client binary, HTTP %d' % status |
| 72 raise CipdBootstrapError('Failed to fetch client binary, HTTP %d' % status) |
| 73 digest = hashlib.sha1(raw_client_bin).hexdigest() |
| 74 if digest != client_info['client_binary']['sha1']: |
| 75 raise CipdBootstrapError('Client SHA1 mismatch') |
| 76 |
| 77 # Success. |
| 78 print 'Fetched CIPD client %s:%s at %s' % (package, instance_id, bin_file) |
| 79 write_file(bin_file, raw_client_bin) |
| 80 os.chmod(bin_file, 0755) |
| 81 write_file(version_file, instance_id + '\n') |
| 82 return bin_file, instance_id |
| 83 |
| 84 |
| 85 def call_cipd_api(endpoint, query): |
| 86 """Sends GET request to CIPD backend, parses JSON response.""" |
| 87 url = '%s/_ah/api/%s' % (CIPD_BACKEND_URL, endpoint) |
| 88 if query: |
| 89 url += '?' + urllib.urlencode(query) |
| 90 status, body = fetch_url(url) |
| 91 if status != 200: |
| 92 raise CipdBootstrapError('Server replied with HTTP %d' % status) |
| 93 try: |
| 94 body = json.loads(body) |
| 95 except ValueError: |
| 96 raise CipdBootstrapError('Server returned invalid JSON') |
| 97 status = body.get('status') |
| 98 if status != 'SUCCESS': |
| 99 m = body.get('error_message') or '<no error message>' |
| 100 raise CipdBootstrapError('Server replied with error %s: %s' % (status, m)) |
| 101 return body |
| 102 |
| 103 |
| 104 def fetch_url(url, headers=None): |
| 105 """Sends GET request (with retries). |
| 106 |
| 107 Args: |
| 108 url: URL to fetch. |
| 109 headers: dict with request headers. |
| 110 |
| 111 Returns: |
| 112 (200, reply body) on success. |
| 113 (HTTP code, None) on HTTP 401, 403, or 404 reply. |
| 114 |
| 115 Raises: |
| 116 Whatever urllib2 raises. |
| 117 """ |
| 118 req = urllib2.Request(url) |
| 119 req.add_header('User-Agent', 'cipd recipe bootstrap.py') |
| 120 for k, v in (headers or {}).iteritems(): |
| 121 req.add_header(str(k), str(v)) |
| 122 i = 0 |
| 123 while True: |
| 124 i += 1 |
| 125 try: |
| 126 print 'GET %s' % url |
| 127 return 200, urllib2.urlopen(req, timeout=60).read() |
| 128 except Exception as e: |
| 129 if isinstance(e, urllib2.HTTPError): |
| 130 print 'Failed to fetch %s, server returned HTTP %d' % (url, e.code) |
| 131 if e.code in (401, 403, 404): |
| 132 return e.code, None |
| 133 else: |
| 134 print 'Failed to fetch %s' % url |
| 135 if i == 20: |
| 136 raise |
| 137 print 'Retrying in %d sec.' % i |
| 138 time.sleep(i) |
| 139 |
| 140 |
| 141 def ensure_directory(path): |
| 142 """Creates a directory.""" |
| 143 # Handle a case where a file is being converted into a directory. |
| 144 chunks = path.split(os.sep) |
| 145 for i in xrange(len(chunks)): |
| 146 p = os.sep.join(chunks[:i+1]) |
| 147 if os.path.exists(p) and not os.path.isdir(p): |
| 148 os.remove(p) |
| 149 break |
| 150 try: |
| 151 os.makedirs(path) |
| 152 except OSError as e: |
| 153 if e.errno != errno.EEXIST: |
| 154 raise |
| 155 |
| 156 |
| 157 def read_file(path): |
| 158 """Returns contents of a file or None if missing.""" |
| 159 try: |
| 160 with open(path, 'r') as f: |
| 161 return f.read() |
| 162 except IOError as e: |
| 163 if e.errno == errno.ENOENT: |
| 164 return None |
| 165 raise |
| 166 |
| 167 |
| 168 def write_file(path, data): |
| 169 """Puts a file on disk, atomically.""" |
| 170 ensure_directory(os.path.dirname(path)) |
| 171 fd, temp_file = tempfile.mkstemp(dir=os.path.dirname(path)) |
| 172 with os.fdopen(fd, 'w') as f: |
| 173 f.write(data) |
| 174 if not sys.platform in ('linux2', 'darwin'): |
| 175 # On windows we should remove destination file if it exists. |
| 176 if os.path.exists(path): |
| 177 os.remove(path) |
| 178 # At this point we may crash, and it's OK, as next time we'll just |
| 179 # re-install CIPD from scratch. |
| 180 os.rename(temp_file, path) |
| 181 |
| 182 |
| 183 def dump_json(obj): |
| 184 """Pretty-formats object to JSON.""" |
| 185 return json.dumps(obj, indent=2, sort_keys=True, separators=(',',':')) |
| 186 |
| 187 |
| 188 def main(args): |
| 189 parser = argparse.ArgumentParser('bootstrap cipd') |
| 190 parser.add_argument('--json-output', default=None) |
| 191 parser.add_argument('--version', default=None) |
| 192 parser.add_argument('--platform', required=True) |
| 193 parser.add_argument('--dest-directory', required=True) |
| 194 opts = parser.parse_args(args) |
| 195 |
| 196 package = "infra/tools/cipd/%s" % opts.platform |
| 197 version = opts.version or CLIENT_VERSIONS[opts.platform] |
| 198 |
| 199 try: |
| 200 exe_path, instance_id = install_cipd_client(opts.dest_directory, |
| 201 package, version) |
| 202 result = { |
| 203 'executable': exe_path, |
| 204 'instance_id': instance_id |
| 205 } |
| 206 if opts.json_output: |
| 207 with open(opts.json_output, 'w') as f: |
| 208 json.dump(result, f) |
| 209 except Exception as e: |
| 210 print 'Exception installing cipd: %s' % e |
| 211 exc_type, exc_value, exc_traceback = sys.exc_info() |
| 212 traceback.print_tb(exc_traceback) |
| 213 return 1 |
| 214 |
| 215 return 0 |
| 216 |
| 217 if __name__ == '__main__': |
| 218 sys.exit(main(sys.argv[1:])) |
OLD | NEW |