OLD | NEW |
---|---|
1 # Copyright 2016 The LUCI Authors. All rights reserved. | 1 # Copyright 2016 The LUCI Authors. All rights reserved. |
2 # Use of this source code is governed under the Apache License, Version 2.0 | 2 # Use of this source code is governed under the Apache License, Version 2.0 |
3 # that can be found in the LICENSE file. | 3 # that can be found in the LICENSE file. |
4 | 4 |
5 """Fetches CIPD client and installs packages.""" | 5 """Fetches CIPD client and installs packages.""" |
6 | 6 |
7 __version__ = '0.2' | 7 __version__ = '0.3' |
8 | 8 |
9 import collections | 9 import collections |
10 import contextlib | 10 import contextlib |
11 import hashlib | 11 import hashlib |
12 import json | |
12 import logging | 13 import logging |
13 import optparse | 14 import optparse |
14 import os | 15 import os |
15 import platform | 16 import platform |
16 import sys | 17 import sys |
17 import tempfile | 18 import tempfile |
18 import time | 19 import time |
19 import urllib | 20 import urllib |
20 | 21 |
21 from utils import file_path | 22 from utils import file_path |
(...skipping 75 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
97 parser.error( | 98 parser.error( |
98 '--cipd-package requires non-empty --cipd-client-package') | 99 '--cipd-package requires non-empty --cipd-client-package') |
99 if not options.cipd_client_version: | 100 if not options.cipd_client_version: |
100 parser.error( | 101 parser.error( |
101 '--cipd-package requires non-empty --cipd-client-version') | 102 '--cipd-package requires non-empty --cipd-client-version') |
102 | 103 |
103 | 104 |
104 class CipdClient(object): | 105 class CipdClient(object): |
105 """Installs packages.""" | 106 """Installs packages.""" |
106 | 107 |
107 def __init__(self, binary_path, service_url=None): | 108 def __init__(self, binary_path, package_name, instance_id, service_url): |
108 """Initializes CipdClient. | 109 """Initializes CipdClient. |
109 | 110 |
110 Args: | 111 Args: |
111 binary_path (str): path to the CIPD client binary. | 112 binary_path (str): path to the CIPD client binary. |
113 package_name (str): the CIPD package name for the client itself. | |
114 instance_id (str): the CIPD instance_id for the client itself. | |
112 service_url (str): if not None, URL of the CIPD backend that overrides | 115 service_url (str): if not None, URL of the CIPD backend that overrides |
113 the default one. | 116 the default one. |
114 """ | 117 """ |
115 self.binary_path = binary_path | 118 self.binary_path = binary_path |
119 self.package_name = package_name | |
120 self.instance_id = instance_id | |
116 self.service_url = service_url | 121 self.service_url = service_url |
117 | 122 |
118 def ensure( | 123 def ensure( |
119 self, site_root, packages, cache_dir=None, tmp_dir=None, timeout=None): | 124 self, site_root, packages, cache_dir=None, tmp_dir=None, timeout=None): |
120 """Ensures that packages installed in |site_root| equals |packages| set. | 125 """Ensures that packages installed in |site_root| equals |packages| set. |
121 | 126 |
122 Blocking call. | 127 Blocking call. |
123 | 128 |
124 Args: | 129 Args: |
125 site_root (str): where to install packages. | 130 site_root (str): where to install packages. |
126 packages: list of (package_template, version) tuples. | 131 packages: list of (package_template, version) tuples. |
127 cache_dir (str): if set, cache dir for cipd binary own cache. | 132 cache_dir (str): if set, cache dir for cipd binary own cache. |
128 Typically contains packages and tags. | 133 Typically contains packages and tags. |
129 tmp_dir (str): if not None, dir for temp files. | 134 tmp_dir (str): if not None, dir for temp files. |
130 timeout (int): if not None, timeout in seconds for this function to run. | 135 timeout (int): if not None, timeout in seconds for this function to run. |
131 | 136 |
137 Returns: | |
138 Pinned packages in the form of [(package_name, package_id)], which | |
139 correspond 1:1 with the input packages argument. | |
140 | |
132 Raises: | 141 Raises: |
133 Error if could not install packages or timed out. | 142 Error if could not install packages or timed out. |
134 """ | 143 """ |
135 timeoutfn = tools.sliding_timeout(timeout) | 144 timeoutfn = tools.sliding_timeout(timeout) |
136 logging.info('Installing packages %r into %s', packages, site_root) | 145 logging.info('Installing packages %r into %s', packages, site_root) |
137 | 146 |
138 list_file_handle, list_file_path = tempfile.mkstemp( | 147 list_file_handle, list_file_path = tempfile.mkstemp( |
139 dir=tmp_dir, prefix=u'cipd-ensure-list-', suffix='.txt') | 148 dir=tmp_dir, prefix=u'cipd-ensure-list-', suffix='.txt') |
149 json_out_file_handle, json_file_path = tempfile.mkstemp( | |
150 dir=tmp_dir, prefix=u'cipd-ensure-result-', suffix='.json') | |
151 os.close(json_out_file_handle) | |
152 | |
140 try: | 153 try: |
141 try: | 154 try: |
142 for pkg, version in packages: | 155 for pkg, version in packages: |
143 pkg = render_package_name_template(pkg) | 156 pkg = render_package_name_template(pkg) |
144 os.write(list_file_handle, '%s %s\n' % (pkg, version)) | 157 os.write(list_file_handle, '%s %s\n' % (pkg, version)) |
145 finally: | 158 finally: |
146 os.close(list_file_handle) | 159 os.close(list_file_handle) |
147 | 160 |
148 cmd = [ | 161 cmd = [ |
149 self.binary_path, 'ensure', | 162 self.binary_path, 'ensure', |
150 '-root', site_root, | 163 '-root', site_root, |
151 '-list', list_file_path, | 164 '-list', list_file_path, |
152 '-verbose', # this is safe because cipd-ensure does not print a lot | 165 '-verbose', # this is safe because cipd-ensure does not print a lot |
166 '-json-output', json_file_path, | |
153 ] | 167 ] |
154 if cache_dir: | 168 if cache_dir: |
155 cmd += ['-cache-dir', cache_dir] | 169 cmd += ['-cache-dir', cache_dir] |
156 if self.service_url: | 170 if self.service_url: |
157 cmd += ['-service-url', self.service_url] | 171 cmd += ['-service-url', self.service_url] |
158 | 172 |
159 logging.debug('Running %r', cmd) | 173 logging.debug('Running %r', cmd) |
160 process = subprocess42.Popen( | 174 process = subprocess42.Popen( |
161 cmd, stdout=subprocess42.PIPE, stderr=subprocess42.PIPE) | 175 cmd, stdout=subprocess42.PIPE, stderr=subprocess42.PIPE) |
162 output = [] | 176 output = [] |
(...skipping 10 matching lines...) Expand all Loading... | |
173 if pipe_name == 'stderr': | 187 if pipe_name == 'stderr': |
174 logging.debug('cipd client: %s', line) | 188 logging.debug('cipd client: %s', line) |
175 else: | 189 else: |
176 logging.info('cipd client: %s', line) | 190 logging.info('cipd client: %s', line) |
177 | 191 |
178 exit_code = process.wait(timeout=timeoutfn()) | 192 exit_code = process.wait(timeout=timeoutfn()) |
179 if exit_code != 0: | 193 if exit_code != 0: |
180 raise Error( | 194 raise Error( |
181 'Could not install packages; exit code %d\noutput:%s' % ( | 195 'Could not install packages; exit code %d\noutput:%s' % ( |
182 exit_code, '\n'.join(output))) | 196 exit_code, '\n'.join(output))) |
197 with open(json_file_path) as jfile: | |
198 result_json = json.load(jfile) | |
199 return [(x['package'], x['instance_id']) for x in result_json['result']] | |
183 finally: | 200 finally: |
184 fs.remove(list_file_path) | 201 fs.remove(list_file_path) |
202 fs.remove(json_file_path) | |
185 | 203 |
186 | 204 |
187 def get_platform(): | 205 def get_platform(): |
188 """Returns ${platform} parameter value. | 206 """Returns ${platform} parameter value. |
189 | 207 |
190 Borrowed from | 208 Borrowed from |
191 https://chromium.googlesource.com/infra/infra/+/aaf9586/build/build.py#204 | 209 https://chromium.googlesource.com/infra/infra/+/aaf9586/build/build.py#204 |
192 """ | 210 """ |
193 # linux, mac or windows. | 211 # linux, mac or windows. |
194 platform_variant = { | 212 platform_variant = { |
(...skipping 197 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
392 # A single host can run multiple swarming bots, but ATM they do not share | 410 # A single host can run multiple swarming bots, but ATM they do not share |
393 # same root bot directory. Thus, it is safe to use the same name for the | 411 # same root bot directory. Thus, it is safe to use the same name for the |
394 # binary. | 412 # binary. |
395 binary_path = unicode(os.path.join(cache_dir, 'cipd' + EXECUTABLE_SUFFIX)) | 413 binary_path = unicode(os.path.join(cache_dir, 'cipd' + EXECUTABLE_SUFFIX)) |
396 if fs.isfile(binary_path): | 414 if fs.isfile(binary_path): |
397 file_path.remove(binary_path) | 415 file_path.remove(binary_path) |
398 | 416 |
399 with instance_cache.getfileobj(instance_id) as f: | 417 with instance_cache.getfileobj(instance_id) as f: |
400 isolateserver.putfile(f, binary_path, 0511) # -r-x--x--x | 418 isolateserver.putfile(f, binary_path, 0511) # -r-x--x--x |
401 | 419 |
402 yield CipdClient(binary_path) | 420 yield CipdClient(binary_path, package_name=package_name, |
421 instance_id=instance_id, service_url=service_url) | |
403 | 422 |
404 | 423 |
405 def parse_package_args(packages): | 424 def parse_package_args(packages, with_index=False): |
406 """Parses --cipd-package arguments. | 425 """Parses --cipd-package arguments. |
407 | 426 |
408 Assumes |packages| were validated by validate_cipd_options. | 427 Assumes |packages| were validated by validate_cipd_options. |
409 | 428 |
410 Returns: | 429 Returns: |
411 A map {path: [(package, version)]}. | 430 A map {path: [(package, version)]}. If with_index is True, the tuples in the |
431 map have a third parameter which is the original command line index of that | |
432 pin. | |
412 """ | 433 """ |
413 result = collections.defaultdict(list) | 434 result = collections.defaultdict(list) |
M-A Ruel
2016/08/26 23:19:51
Can it be a flat list instead of a dict, then orde
iannucci
2016/08/29 22:08:41
yeah, that felt more invasive than I wanted at the
| |
414 for pkg in packages: | 435 for i, pkg in enumerate(packages): |
415 path, name, version = pkg.split(':', 2) | 436 path, name, version = pkg.split(':', 2) |
416 path = path.replace('/', os.path.sep) | 437 path = path.replace('/', os.path.sep) |
417 if not name: | 438 if not name: |
418 raise Error('Invalid package "%s": package name is not specified' % pkg) | 439 raise Error('Invalid package "%s": package name is not specified' % pkg) |
419 if not version: | 440 if not version: |
420 raise Error('Invalid package "%s": version is not specified' % pkg) | 441 raise Error('Invalid package "%s": version is not specified' % pkg) |
421 result[path].append((name, version)) | 442 if with_index: |
M-A Ruel
2016/08/26 23:19:51
can we not make this a condition?
iannucci
2016/08/29 22:08:41
will make it a list, no condition needed.
| |
443 result[path].append((name, version, i)) | |
444 else: | |
445 result[path].append((name, version)) | |
422 return result | 446 return result |
OLD | NEW |