Index: client/run_isolated.py |
diff --git a/client/run_isolated.py b/client/run_isolated.py |
index a879c221e39bdbcfb91a88c15837c2e9c90cd6b8..fa97cf5b69e53c3a93185fac8be0c6845a044183 100755 |
--- a/client/run_isolated.py |
+++ b/client/run_isolated.py |
@@ -27,7 +27,7 @@ state of the host to tasks. It is written to by the swarming bot's |
on_before_task() hook in the swarming server's custom bot_config.py. |
""" |
-__version__ = '0.8.4' |
+__version__ = '0.8.5' |
import base64 |
import logging |
@@ -383,6 +383,13 @@ def map_and_run( |
# }, |
# }, |
}, |
+ # 'cipd_pins': { |
+ # 'packages': [ |
+ # {'package_name': ..., 'version': ..., 'path': ...}, |
+ # ... |
+ # ], |
+ # 'client_package': {'package_name': ..., 'version': ...}, |
+ # }, |
'outputs_ref': None, |
'version': 5, |
} |
@@ -400,9 +407,10 @@ def map_and_run( |
cwd = run_dir |
try: |
- cipd_stats = install_packages_fn(run_dir) |
- if cipd_stats: |
- result['stats']['cipd'] = cipd_stats |
+ cipd_info = install_packages_fn(run_dir) |
+ if cipd_info: |
+ result['stats']['cipd'] = cipd_info['stats'] |
+ result['cipd_pins'] = cipd_info['cipd_pins'] |
if isolated_hash: |
isolated_stats = result['stats'].setdefault('isolated', {}) |
@@ -539,7 +547,8 @@ def run_tha_test( |
grace_period: number of seconds to wait between SIGTERM and SIGKILL. |
extra_args: optional arguments to add to the command stated in the .isolate |
file. Ignored if isolate_hash is empty. |
- install_packages_fn: function (dir) => cipd_stats. Installs packages. |
+ install_packages_fn: function (dir) => {"stats": cipd_stats, "pins": |
+ cipd_pins}. Installs packages. |
use_symlinks: create tree with symlinks instead of hardlinks. |
Returns: |
@@ -595,11 +604,25 @@ def run_tha_test( |
def install_packages( |
run_dir, packages, service_url, client_package_name, |
client_version, cache_dir=None, timeout=None): |
- """Installs packages. Returns stats. |
+ """Installs packages. Returns stats, cipd client info and pins. |
+ |
+ pins and the cipd client info are in the form of: |
+ [ |
+ { |
+ "path": path, "package_name": package_name, "version": version, |
+ }, |
+ ... |
+ ] |
+ (the cipd client info is a single dictionary instead of a list) |
+ |
+ such that they correspond 1:1 to all input package arguments from the command |
+ line. These dictionaries make their all the way back to swarming, where they |
+ become the arguments of CipdPackage. |
Args: |
run_dir (str): root of installation. |
- packages: packages to install, dict {path: [(package_name, version)]. |
+ packages: packages to install, dict |
+ {path: [(package_name, version, cmd_line_idx)]. |
service_url (str): CIPD server url, e.g. |
"https://chrome-infra-packages.appspot.com." |
client_package_name (str): CIPD package name of CIPD client. |
@@ -617,11 +640,25 @@ def install_packages( |
run_dir = os.path.abspath(run_dir) |
+ package_pins = [] |
+ def insert_pin(path, name, version, idx): |
+ if idx > len(package_pins)-1: |
+ package_pins.extend([None] * (idx - len(package_pins) + 1)) |
+ package_pins[idx] = { |
+ 'package_name': name, |
+ 'version': version, |
+ 'path': path, |
+ } |
+ |
get_client_start = time.time() |
client_manager = cipd.get_client( |
service_url, client_package_name, client_version, cache_dir, |
timeout=timeoutfn()) |
with client_manager as client: |
+ client_package = { |
+ 'package_name': client.package_name, |
+ 'version': client.instance_id, |
+ } |
get_client_duration = time.time() - get_client_start |
for path, packages in sorted(packages.iteritems()): |
site_root = os.path.abspath(os.path.join(run_dir, path)) |
@@ -631,19 +668,29 @@ def install_packages( |
# Do not clean site_root before installation because it may contain other |
# site roots. |
file_path.ensure_tree(site_root, 0770) |
- client.ensure( |
- site_root, packages, |
+ pins = client.ensure( |
+ site_root, [(name, vers) for name, vers, _ in packages], |
cache_dir=os.path.join(cache_dir, 'cipd_internal'), |
timeout=timeoutfn()) |
+ for i, pin in enumerate(pins): |
+ insert_pin(path, pin[0], pin[1], packages[i][2]) |
file_path.make_tree_files_read_only(site_root) |
total_duration = time.time() - start |
logging.info( |
'Installing CIPD client and packages took %d seconds', total_duration) |
+ assert None not in package_pins |
+ |
return { |
- 'duration': total_duration, |
- 'get_client_duration': get_client_duration, |
+ 'stats': { |
+ 'duration': total_duration, |
+ 'get_client_duration': get_client_duration, |
+ }, |
+ 'cipd_pins': { |
+ 'packages': package_pins, |
+ 'client_package': client_package, |
+ } |
} |
@@ -744,7 +791,7 @@ def main(args): |
cipd.validate_cipd_options(parser, options) |
install_packages_fn = lambda run_dir: install_packages( |
- run_dir, cipd.parse_package_args(options.cipd_packages), |
+ run_dir, cipd.parse_package_args(options.cipd_packages, with_index=True), |
options.cipd_server, options.cipd_client_package, |
options.cipd_client_version, cache_dir=options.cipd_cache) |