OLD | NEW |
| (Empty) |
1 # Copyright 2015 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 | |
5 import argparse | |
6 import errno | |
7 import hashlib | |
8 import json | |
9 import os | |
10 import sys | |
11 import tempfile | |
12 import time | |
13 import traceback | |
14 import urllib | |
15 import urllib2 | |
16 | |
17 | |
18 # Default package repository URL. | |
19 CIPD_BACKEND_URL = 'https://chrome-infra-packages.appspot.com' | |
20 | |
21 # ./cipd resolve \ | |
22 # infra/tools/cipd/ \ | |
23 # -version=git_revision:768586a9aa72fe6a41a8a0205ed56ceb1495625c | |
24 CLIENT_VERSIONS = { | |
25 'linux-386': '3f5d6d3906c3b2c6a057d1ab1634ac06aa418708', | |
26 'linux-amd64': '1ea0b6b254ad3f546c826dd3e437798ace2c2480', | |
27 'mac-amd64': 'c111be343c692e5285113a6b1c999887adbb268e', | |
28 'windows-386': 'dc3d1bd5b4b93945640bac4bb047c333a8fa12fd', | |
29 'windows-amd64': 'bccdb9a605037e3dd2a8a64e79e08f691a6f159d', | |
30 } | |
31 | |
32 | |
33 class CipdBootstrapError(Exception): | |
34 """Raised by install_cipd_client on fatal error.""" | |
35 | |
36 def install_cipd_client(path, package, version): | |
37 """Installs CIPD client to <path>/cipd. | |
38 | |
39 Args: | |
40 path: root directory to install CIPD client into. | |
41 package: cipd client package name, e.g. infra/tools/cipd/linux-amd64. | |
42 version: version of the package to install. | |
43 | |
44 Returns: | |
45 Absolute path to CIPD executable. | |
46 """ | |
47 print 'Ensuring CIPD client is up-to-date' | |
48 version_file = os.path.join(path, 'VERSION') | |
49 bin_file = os.path.join(path, 'cipd') | |
50 | |
51 # Resolve version to concrete instance ID, e.g "live" -> "abcdef0123....". | |
52 instance_id = call_cipd_api( | |
53 'repo/v1/instance/resolve', | |
54 {'package_name': package, 'version': version})['instance_id'] | |
55 print 'CIPD client %s => %s' % (version, instance_id) | |
56 | |
57 # Already installed? | |
58 installed_instance_id = (read_file(version_file) or '').strip() | |
59 if installed_instance_id == instance_id and os.path.exists(bin_file): | |
60 return bin_file, instance_id | |
61 | |
62 # Resolve instance ID to an URL to fetch client binary from. | |
63 client_info = call_cipd_api( | |
64 'repo/v1/client', | |
65 {'package_name': package, 'instance_id': instance_id}) | |
66 print 'CIPD client binary info:\n%s' % dump_json(client_info) | |
67 | |
68 # Fetch the client. It is ~10 MB, so don't bother and fetch it into memory. | |
69 status, raw_client_bin = fetch_url(client_info['client_binary']['fetch_url']) | |
70 if status != 200: | |
71 print 'Failed to fetch client binary, HTTP %d' % status | |
72 raise CipdBootstrapError('Failed to fetch client binary, HTTP %d' % status) | |
73 digest = hashlib.sha1(raw_client_bin).hexdigest() | |
74 if digest != client_info['client_binary']['sha1']: | |
75 raise CipdBootstrapError('Client SHA1 mismatch') | |
76 | |
77 # Success. | |
78 print 'Fetched CIPD client %s:%s at %s' % (package, instance_id, bin_file) | |
79 write_file(bin_file, raw_client_bin) | |
80 os.chmod(bin_file, 0755) | |
81 write_file(version_file, instance_id + '\n') | |
82 return bin_file, instance_id | |
83 | |
84 | |
85 def call_cipd_api(endpoint, query): | |
86 """Sends GET request to CIPD backend, parses JSON response.""" | |
87 url = '%s/_ah/api/%s' % (CIPD_BACKEND_URL, endpoint) | |
88 if query: | |
89 url += '?' + urllib.urlencode(query) | |
90 status, body = fetch_url(url) | |
91 if status != 200: | |
92 raise CipdBootstrapError('Server replied with HTTP %d' % status) | |
93 try: | |
94 body = json.loads(body) | |
95 except ValueError: | |
96 raise CipdBootstrapError('Server returned invalid JSON') | |
97 status = body.get('status') | |
98 if status != 'SUCCESS': | |
99 m = body.get('error_message') or '<no error message>' | |
100 raise CipdBootstrapError('Server replied with error %s: %s' % (status, m)) | |
101 return body | |
102 | |
103 | |
104 def fetch_url(url, headers=None): | |
105 """Sends GET request (with retries). | |
106 | |
107 Args: | |
108 url: URL to fetch. | |
109 headers: dict with request headers. | |
110 | |
111 Returns: | |
112 (200, reply body) on success. | |
113 (HTTP code, None) on HTTP 401, 403, or 404 reply. | |
114 | |
115 Raises: | |
116 Whatever urllib2 raises. | |
117 """ | |
118 req = urllib2.Request(url) | |
119 req.add_header('User-Agent', 'cipd recipe bootstrap.py') | |
120 for k, v in (headers or {}).iteritems(): | |
121 req.add_header(str(k), str(v)) | |
122 i = 0 | |
123 while True: | |
124 i += 1 | |
125 try: | |
126 print 'GET %s' % url | |
127 return 200, urllib2.urlopen(req, timeout=60).read() | |
128 except Exception as e: | |
129 if isinstance(e, urllib2.HTTPError): | |
130 print 'Failed to fetch %s, server returned HTTP %d' % (url, e.code) | |
131 if e.code in (401, 403, 404): | |
132 return e.code, None | |
133 else: | |
134 print 'Failed to fetch %s' % url | |
135 if i == 20: | |
136 raise | |
137 print 'Retrying in %d sec.' % i | |
138 time.sleep(i) | |
139 | |
140 | |
141 def ensure_directory(path): | |
142 """Creates a directory.""" | |
143 # Handle a case where a file is being converted into a directory. | |
144 chunks = path.split(os.sep) | |
145 for i in xrange(len(chunks)): | |
146 p = os.sep.join(chunks[:i+1]) | |
147 if os.path.exists(p) and not os.path.isdir(p): | |
148 os.remove(p) | |
149 break | |
150 try: | |
151 os.makedirs(path) | |
152 except OSError as e: | |
153 if e.errno != errno.EEXIST: | |
154 raise | |
155 | |
156 | |
157 def read_file(path): | |
158 """Returns contents of a file or None if missing.""" | |
159 try: | |
160 with open(path, 'r') as f: | |
161 return f.read() | |
162 except IOError as e: | |
163 if e.errno == errno.ENOENT: | |
164 return None | |
165 raise | |
166 | |
167 | |
168 def write_file(path, data): | |
169 """Puts a file on disk, atomically.""" | |
170 ensure_directory(os.path.dirname(path)) | |
171 fd, temp_file = tempfile.mkstemp(dir=os.path.dirname(path)) | |
172 with os.fdopen(fd, 'w') as f: | |
173 f.write(data) | |
174 if not sys.platform in ('linux2', 'darwin'): | |
175 # On windows we should remove destination file if it exists. | |
176 if os.path.exists(path): | |
177 os.remove(path) | |
178 # At this point we may crash, and it's OK, as next time we'll just | |
179 # re-install CIPD from scratch. | |
180 os.rename(temp_file, path) | |
181 | |
182 | |
183 def dump_json(obj): | |
184 """Pretty-formats object to JSON.""" | |
185 return json.dumps(obj, indent=2, sort_keys=True, separators=(',',':')) | |
186 | |
187 | |
188 def main(args): | |
189 parser = argparse.ArgumentParser('bootstrap cipd') | |
190 parser.add_argument('--json-output', default=None) | |
191 parser.add_argument('--version', default=None) | |
192 parser.add_argument('--platform', required=True) | |
193 parser.add_argument('--dest-directory', required=True) | |
194 opts = parser.parse_args(args) | |
195 | |
196 package = "infra/tools/cipd/%s" % opts.platform | |
197 version = opts.version or CLIENT_VERSIONS[opts.platform] | |
198 | |
199 try: | |
200 exe_path, instance_id = install_cipd_client(opts.dest_directory, | |
201 package, version) | |
202 result = { | |
203 'executable': exe_path, | |
204 'instance_id': instance_id | |
205 } | |
206 if opts.json_output: | |
207 with open(opts.json_output, 'w') as f: | |
208 json.dump(result, f) | |
209 except Exception as e: | |
210 print 'Exception installing cipd: %s' % e | |
211 exc_type, exc_value, exc_traceback = sys.exc_info() | |
212 traceback.print_tb(exc_traceback) | |
213 return 1 | |
214 | |
215 return 0 | |
216 | |
217 if __name__ == '__main__': | |
218 sys.exit(main(sys.argv[1:])) | |
OLD | NEW |