OLD | NEW |
---|---|
1 # Copyright 2016 The LUCI Authors. All rights reserved. | 1 # Copyright 2016 The LUCI Authors. All rights reserved. |
2 # Use of this source code is governed under the Apache License, Version 2.0 | 2 # Use of this source code is governed under the Apache License, Version 2.0 |
3 # that can be found in the LICENSE file. | 3 # that can be found in the LICENSE file. |
4 | 4 |
5 """Fetches CIPD client and installs packages.""" | 5 """Fetches CIPD client and installs packages.""" |
6 | 6 |
7 __version__ = '0.2' | 7 __version__ = '0.2' |
M-A Ruel
2016/08/24 02:13:44
bump
iannucci
2016/08/24 02:31:57
done
| |
8 | 8 |
9 import collections | 9 import collections |
10 import contextlib | 10 import contextlib |
11 import hashlib | 11 import hashlib |
12 import json | |
12 import logging | 13 import logging |
13 import optparse | 14 import optparse |
14 import os | 15 import os |
15 import platform | 16 import platform |
16 import sys | 17 import sys |
17 import tempfile | 18 import tempfile |
18 import time | 19 import time |
19 import urllib | 20 import urllib |
20 | 21 |
21 from utils import file_path | 22 from utils import file_path |
(...skipping 100 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
122 Blocking call. | 123 Blocking call. |
123 | 124 |
124 Args: | 125 Args: |
125 site_root (str): where to install packages. | 126 site_root (str): where to install packages. |
126 packages: list of (package_template, version) tuples. | 127 packages: list of (package_template, version) tuples. |
127 cache_dir (str): if set, cache dir for cipd binary own cache. | 128 cache_dir (str): if set, cache dir for cipd binary own cache. |
128 Typically contains packages and tags. | 129 Typically contains packages and tags. |
129 tmp_dir (str): if not None, dir for temp files. | 130 tmp_dir (str): if not None, dir for temp files. |
130 timeout (int): if not None, timeout in seconds for this function to run. | 131 timeout (int): if not None, timeout in seconds for this function to run. |
131 | 132 |
133 Returns: | |
134 Pinned packages in the form of [(package_name, package_id)], which | |
135 correspond 1:1 with the input packages argument. | |
136 | |
132 Raises: | 137 Raises: |
133 Error if could not install packages or timed out. | 138 Error if could not install packages or timed out. |
134 """ | 139 """ |
135 timeoutfn = tools.sliding_timeout(timeout) | 140 timeoutfn = tools.sliding_timeout(timeout) |
136 logging.info('Installing packages %r into %s', packages, site_root) | 141 logging.info('Installing packages %r into %s', packages, site_root) |
137 | 142 |
138 list_file_handle, list_file_path = tempfile.mkstemp( | 143 list_file_handle, list_file_path = tempfile.mkstemp( |
139 dir=tmp_dir, prefix=u'cipd-ensure-list-', suffix='.txt') | 144 dir=tmp_dir, prefix=u'cipd-ensure-list-', suffix='.txt') |
145 json_out_file_handle, json_file_path = tempfile.mkstemp( | |
146 dir=tmp_dir, prefix=u'cipd-ensure-result-', suffix='.json') | |
147 os.close(json_out_file_handle) | |
148 | |
140 try: | 149 try: |
141 try: | 150 try: |
142 for pkg, version in packages: | 151 for pkg, version in packages: |
143 pkg = render_package_name_template(pkg) | 152 pkg = render_package_name_template(pkg) |
144 os.write(list_file_handle, '%s %s\n' % (pkg, version)) | 153 os.write(list_file_handle, '%s %s\n' % (pkg, version)) |
M-A Ruel
2016/08/24 02:13:44
it's weird that input is txt but output is json
iannucci
2016/08/24 02:31:57
I think the text file is supposed to be easier or
| |
145 finally: | 154 finally: |
146 os.close(list_file_handle) | 155 os.close(list_file_handle) |
147 | 156 |
148 cmd = [ | 157 cmd = [ |
149 self.binary_path, 'ensure', | 158 self.binary_path, 'ensure', |
150 '-root', site_root, | 159 '-root', site_root, |
151 '-list', list_file_path, | 160 '-list', list_file_path, |
152 '-verbose', # this is safe because cipd-ensure does not print a lot | 161 '-verbose', # this is safe because cipd-ensure does not print a lot |
162 '-json-output', json_file_path, | |
153 ] | 163 ] |
154 if cache_dir: | 164 if cache_dir: |
155 cmd += ['-cache-dir', cache_dir] | 165 cmd += ['-cache-dir', cache_dir] |
156 if self.service_url: | 166 if self.service_url: |
157 cmd += ['-service-url', self.service_url] | 167 cmd += ['-service-url', self.service_url] |
158 | 168 |
159 logging.debug('Running %r', cmd) | 169 logging.debug('Running %r', cmd) |
160 process = subprocess42.Popen( | 170 process = subprocess42.Popen( |
161 cmd, stdout=subprocess42.PIPE, stderr=subprocess42.PIPE) | 171 cmd, stdout=subprocess42.PIPE, stderr=subprocess42.PIPE) |
162 output = [] | 172 output = [] |
(...skipping 10 matching lines...) Expand all Loading... | |
173 if pipe_name == 'stderr': | 183 if pipe_name == 'stderr': |
174 logging.debug('cipd client: %s', line) | 184 logging.debug('cipd client: %s', line) |
175 else: | 185 else: |
176 logging.info('cipd client: %s', line) | 186 logging.info('cipd client: %s', line) |
177 | 187 |
178 exit_code = process.wait(timeout=timeoutfn()) | 188 exit_code = process.wait(timeout=timeoutfn()) |
179 if exit_code != 0: | 189 if exit_code != 0: |
180 raise Error( | 190 raise Error( |
181 'Could not install packages; exit code %d\noutput:%s' % ( | 191 'Could not install packages; exit code %d\noutput:%s' % ( |
182 exit_code, '\n'.join(output))) | 192 exit_code, '\n'.join(output))) |
193 with open(json_file_path) as jfile: | |
194 result_json = json.load(jfile) | |
195 return [(x['package'], x['instance_id']) for x in result_json['result']] | |
183 finally: | 196 finally: |
184 fs.remove(list_file_path) | 197 fs.remove(list_file_path) |
198 fs.remove(json_file_path) | |
185 | 199 |
186 | 200 |
187 def get_platform(): | 201 def get_platform(): |
188 """Returns ${platform} parameter value. | 202 """Returns ${platform} parameter value. |
189 | 203 |
190 Borrowed from | 204 Borrowed from |
191 https://chromium.googlesource.com/infra/infra/+/aaf9586/build/build.py#204 | 205 https://chromium.googlesource.com/infra/infra/+/aaf9586/build/build.py#204 |
192 """ | 206 """ |
193 # linux, mac or windows. | 207 # linux, mac or windows. |
194 platform_variant = { | 208 platform_variant = { |
(...skipping 200 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
395 binary_path = unicode(os.path.join(cache_dir, 'cipd' + EXECUTABLE_SUFFIX)) | 409 binary_path = unicode(os.path.join(cache_dir, 'cipd' + EXECUTABLE_SUFFIX)) |
396 if fs.isfile(binary_path): | 410 if fs.isfile(binary_path): |
397 file_path.remove(binary_path) | 411 file_path.remove(binary_path) |
398 | 412 |
399 with instance_cache.getfileobj(instance_id) as f: | 413 with instance_cache.getfileobj(instance_id) as f: |
400 isolateserver.putfile(f, binary_path, 0511) # -r-x--x--x | 414 isolateserver.putfile(f, binary_path, 0511) # -r-x--x--x |
401 | 415 |
402 yield CipdClient(binary_path) | 416 yield CipdClient(binary_path) |
403 | 417 |
404 | 418 |
405 def parse_package_args(packages): | 419 def parse_package_args(packages, with_index=False): |
406 """Parses --cipd-package arguments. | 420 """Parses --cipd-package arguments. |
407 | 421 |
408 Assumes |packages| were validated by validate_cipd_options. | 422 Assumes |packages| were validated by validate_cipd_options. |
409 | 423 |
410 Returns: | 424 Returns: |
411 A map {path: [(package, version)]}. | 425 A map {path: [(package, version)]}. If with_index is True, the tuples in the |
426 map have a third parameter which is the original command line index of that | |
427 pin. | |
412 """ | 428 """ |
413 result = collections.defaultdict(list) | 429 result = collections.defaultdict(list) |
414 for pkg in packages: | 430 for i, pkg in enumerate(packages): |
415 path, name, version = pkg.split(':', 2) | 431 path, name, version = pkg.split(':', 2) |
416 path = path.replace('/', os.path.sep) | 432 path = path.replace('/', os.path.sep) |
417 if not name: | 433 if not name: |
418 raise Error('Invalid package "%s": package name is not specified' % pkg) | 434 raise Error('Invalid package "%s": package name is not specified' % pkg) |
419 if not version: | 435 if not version: |
420 raise Error('Invalid package "%s": version is not specified' % pkg) | 436 raise Error('Invalid package "%s": version is not specified' % pkg) |
421 result[path].append((name, version)) | 437 if with_index: |
438 result[path].append((name, version, i)) | |
439 else: | |
440 result[path].append((name, version)) | |
422 return result | 441 return result |
OLD | NEW |