| OLD | NEW |
| 1 # Copyright 2016 The LUCI Authors. All rights reserved. | 1 # Copyright 2016 The LUCI Authors. All rights reserved. |
| 2 # Use of this source code is governed under the Apache License, Version 2.0 | 2 # Use of this source code is governed under the Apache License, Version 2.0 |
| 3 # that can be found in the LICENSE file. | 3 # that can be found in the LICENSE file. |
| 4 | 4 |
| 5 """Fetches CIPD client and installs packages.""" | 5 """Fetches CIPD client and installs packages.""" |
| 6 | 6 |
| 7 import contextlib | 7 import contextlib |
| 8 import hashlib | 8 import hashlib |
| 9 import json | 9 import json |
| 10 import logging | 10 import logging |
| (...skipping 391 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 402 # Resolve version to instance id. | 402 # Resolve version to instance id. |
| 403 # Is it an instance id already? They look like HEX SHA1. | 403 # Is it an instance id already? They look like HEX SHA1. |
| 404 if isolated_format.is_valid_hash(version, hashlib.sha1): | 404 if isolated_format.is_valid_hash(version, hashlib.sha1): |
| 405 instance_id = version | 405 instance_id = version |
| 406 elif ':' in version: # it's an immutable tag | 406 elif ':' in version: # it's an immutable tag |
| 407 # version_cache is {version_digest -> instance id} mapping. | 407 # version_cache is {version_digest -> instance id} mapping. |
| 408 # It does not take a lot of disk space. | 408 # It does not take a lot of disk space. |
| 409 version_cache = isolateserver.DiskCache( | 409 version_cache = isolateserver.DiskCache( |
| 410 unicode(os.path.join(cache_dir, 'versions')), | 410 unicode(os.path.join(cache_dir, 'versions')), |
| 411 isolateserver.CachePolicies(0, 0, 300), | 411 isolateserver.CachePolicies(0, 0, 300), |
| 412 hashlib.sha1) | 412 hashlib.sha1, |
| 413 trim=True) |
| 413 with version_cache: | 414 with version_cache: |
| 414 version_cache.cleanup() | 415 version_cache.cleanup() |
| 415 # Convert |version| to a string that may be used as a filename in disk | 416 # Convert |version| to a string that may be used as a filename in disk |
| 416 # cache by hashing it. | 417 # cache by hashing it. |
| 417 version_digest = hashlib.sha1(version).hexdigest() | 418 version_digest = hashlib.sha1(version).hexdigest() |
| 418 try: | 419 try: |
| 419 with version_cache.getfileobj(version_digest) as f: | 420 with version_cache.getfileobj(version_digest) as f: |
| 420 instance_id = f.read() | 421 instance_id = f.read() |
| 421 except isolateserver.CacheMiss: | 422 except isolateserver.CacheMiss: |
| 422 instance_id = resolve_version( | 423 instance_id = resolve_version( |
| 423 service_url, package_name, version, timeout=timeoutfn()) | 424 service_url, package_name, version, timeout=timeoutfn()) |
| 424 version_cache.write(version_digest, instance_id) | 425 version_cache.write(version_digest, instance_id) |
| 425 else: # it's a ref | 426 else: # it's a ref |
| 426 instance_id = resolve_version( | 427 instance_id = resolve_version( |
| 427 service_url, package_name, version, timeout=timeoutfn()) | 428 service_url, package_name, version, timeout=timeoutfn()) |
| 428 | 429 |
| 429 # instance_cache is {instance_id -> client binary} mapping. | 430 # instance_cache is {instance_id -> client binary} mapping. |
| 430 # It is bounded by 5 client versions. | 431 # It is bounded by 5 client versions. |
| 431 instance_cache = isolateserver.DiskCache( | 432 instance_cache = isolateserver.DiskCache( |
| 432 unicode(os.path.join(cache_dir, 'clients')), | 433 unicode(os.path.join(cache_dir, 'clients')), |
| 433 isolateserver.CachePolicies(0, 0, 5), | 434 isolateserver.CachePolicies(0, 0, 5), |
| 434 hashlib.sha1) | 435 hashlib.sha1, |
| 436 trim=True) |
| 435 with instance_cache: | 437 with instance_cache: |
| 436 instance_cache.cleanup() | 438 instance_cache.cleanup() |
| 437 if instance_id not in instance_cache: | 439 if instance_id not in instance_cache: |
| 438 logging.info('Fetching CIPD client %s:%s', package_name, instance_id) | 440 logging.info('Fetching CIPD client %s:%s', package_name, instance_id) |
| 439 fetch_url = get_client_fetch_url( | 441 fetch_url = get_client_fetch_url( |
| 440 service_url, package_name, instance_id, timeout=timeoutfn()) | 442 service_url, package_name, instance_id, timeout=timeoutfn()) |
| 441 _fetch_cipd_client(instance_cache, instance_id, fetch_url, timeoutfn) | 443 _fetch_cipd_client(instance_cache, instance_id, fetch_url, timeoutfn) |
| 442 | 444 |
| 443 # A single host can run multiple swarming bots, but ATM they do not share | 445 # A single host can run multiple swarming bots, but ATM they do not share |
| 444 # same root bot directory. Thus, it is safe to use the same name for the | 446 # same root bot directory. Thus, it is safe to use the same name for the |
| (...skipping 27 matching lines...) Expand all Loading... |
| 472 """ | 474 """ |
| 473 result = [] | 475 result = [] |
| 474 for pkg in packages: | 476 for pkg in packages: |
| 475 path, name, version = pkg.split(':', 2) | 477 path, name, version = pkg.split(':', 2) |
| 476 if not name: | 478 if not name: |
| 477 raise Error('Invalid package "%s": package name is not specified' % pkg) | 479 raise Error('Invalid package "%s": package name is not specified' % pkg) |
| 478 if not version: | 480 if not version: |
| 479 raise Error('Invalid package "%s": version is not specified' % pkg) | 481 raise Error('Invalid package "%s": version is not specified' % pkg) |
| 480 result.append((path, name, version)) | 482 result.append((path, name, version)) |
| 481 return result | 483 return result |
| OLD | NEW |