Add CIPD pin reporting to swarming.

client/run_isolated.py

 #!/usr/bin/env python
 # Copyright 2012 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Runs a command with optional isolated input/output.
 
 Despite name "run_isolated", can run a generic non-isolated command specified as
 args.
 
 If input isolated hash is provided, fetches it, creates a tree of hard links,
 appends args to the command in the fetched isolated and runs it.
 To improve performance, keeps a local cache.
 The local cache can safely be deleted.
 
 Any ${EXECUTABLE_SUFFIX} on the command line will be replaced with ".exe" string
 on Windows and "" on other platforms.
 
 Any ${ISOLATED_OUTDIR} on the command line will be replaced by the location of a
 temporary directory upon execution of the command specified in the .isolated
 file. All content written to this directory will be uploaded upon termination
 and the .isolated file describing this directory will be printed to stdout.
 
 Any ${SWARMING_BOT_FILE} on the command line will be replaced by the value of
 the --bot-file parameter. This file is used by a swarming bot to communicate
 state of the host to tasks. It is written to by the swarming bot's
 on_before_task() hook in the swarming server's custom bot_config.py.
 """
 
 __version__ = '0.8.4'

[M-A Ruel, 2016/08/24 02:40:23]

bump too

[iannucci, 2016/08/24 16:01:11]

done

Can we make a PRESUBMIT hook for these?

[M-A Ruel, 2016/08/24 19:34:07]

Please upload again.

Also please test it on staging so I can look at the results.
It's tricky, some changes may not be in the script itself yet still warrant a
bump and the reverse is more often true, small changes do not warrant a bump.

 
 import base64
 import logging
 import optparse
 import os
 import sys
 import tempfile
 import time
 
 from third_party.depot_tools import fix_encoding
@@ skipping 352 matching lines @@
     root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None
   # See comment for these constants.
   run_dir = make_temp_dir(ISOLATED_RUN_DIR, root_dir)
   # storage should be normally set but don't crash if it is not. This can happen
   # as Swarming task can run without an isolate server.
   out_dir = make_temp_dir(ISOLATED_OUT_DIR, root_dir) if storage else None
   tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, root_dir)
   cwd = run_dir
 
   try:
-    cipd_stats = install_packages_fn(run_dir)
-    if cipd_stats:
-      result['stats']['cipd'] = cipd_stats
+    cipd_info = install_packages_fn(run_dir)
+    if cipd_info:
+      result['stats']['cipd'] = cipd_info['stats']
+      result['cipd_pins'] = cipd_info['pins']
 
     if isolated_hash:
       isolated_stats = result['stats'].setdefault('isolated', {})
       bundle, isolated_stats['download'] = fetch_and_map(
           isolated_hash=isolated_hash,
           storage=storage,
           cache=cache,
           outdir=run_dir,
           use_symlinks=use_symlinks)
       if not bundle.command:
@@ skipping 116 matching lines @@
                    for later examination.
     result_json: file path to dump result metadata into. If set, the process
                  exit code is always 0 unless an internal error occurred.
     root_dir: path to the directory to use to create the temporary directory. If
               not specified, a random temporary directory is created.
     hard_timeout: kills the process if it lasts more than this amount of
                   seconds.
     grace_period: number of seconds to wait between SIGTERM and SIGKILL.
     extra_args: optional arguments to add to the command stated in the .isolate
                 file. Ignored if isolate_hash is empty.
-    install_packages_fn: function (dir) => cipd_stats. Installs packages.
+    install_packages_fn: function (dir) => {"stats": cipd_stats, "pins":
+                         cipd_pins}. Installs packages.
     use_symlinks: create tree with symlinks instead of hardlinks.
 
   Returns:
     Process exit code that should be used.
   """
   assert bool(command) ^ bool(isolated_hash)
   extra_args = extra_args or []
 
   if any(ISOLATED_OUTDIR_PARAMETER in a for a in (command or extra_args)):
     assert storage is not None, 'storage is None although outdir is specified'
@@ skipping 35 matching lines @@
     print(
         '[run_isolated_out_hack]%s[/run_isolated_out_hack]' %
         tools.format_json(data, dense=True))
     sys.stdout.flush()
   return result['exit_code'] or int(bool(result['internal_failure']))
 
 
 def install_packages(
     run_dir, packages, service_url, client_package_name,
     client_version, cache_dir=None, timeout=None):
-  """Installs packages. Returns stats.
+  """Installs packages. Returns stats and pins.
+
+  pins are in the form of:
+    [
+      {
+        "path": path, "package_name": package_name, "version": version,
+      },
+      ...
+    ]
+
+  such that they correspond 1:1 to all input package arguments from the command
+  line. These dictionaries make their all the way back to swarming, where they
+  become the arguments of CipdPackage.
 
   Args:
     run_dir (str): root of installation.
-    packages: packages to install, dict {path: [(package_name, version)].
+    packages: packages to install, dict
+      {path: [(package_name, version, cmd_line_idx)].
     service_url (str): CIPD server url, e.g.
       "https://chrome-infra-packages.appspot.com."
     client_package_name (str): CIPD package name of CIPD client.
     client_version (str): Version of CIPD client.
     cache_dir (str): where to keep cache of cipd clients, packages and tags.
     timeout: max duration in seconds that this function can take.
   """
   assert cache_dir
   if not packages:
     return None
 
   timeoutfn = tools.sliding_timeout(timeout)
   start = time.time()
   cache_dir = os.path.abspath(cache_dir)
 
   run_dir = os.path.abspath(run_dir)
 
+  all_pins = []
+  def insert_pin(path, name, version, idx):
+    if idx > len(all_pins)-1:
+      all_pins.extend([None] * (len(all_pins) - idx + 1))
+    all_pins[idx] = {
+      'package_name': name,
+      'version': version,
+      'path': path,
+    }
+
   get_client_start = time.time()
   client_manager = cipd.get_client(
       service_url, client_package_name, client_version, cache_dir,
       timeout=timeoutfn())
   with client_manager as client:
     get_client_duration = time.time() - get_client_start
     for path, packages in sorted(packages.iteritems()):
       site_root = os.path.abspath(os.path.join(run_dir, path))
       if not site_root.startswith(run_dir):
         raise cipd.Error('Invalid CIPD package path "%s"' % path)
 
       # Do not clean site_root before installation because it may contain other
       # site roots.
       file_path.ensure_tree(site_root, 0770)
-      client.ensure(
-          site_root, packages,
+      pins = client.ensure(
+          site_root, [(name, vers) for name, vers, _ in packages],
           cache_dir=os.path.join(cache_dir, 'cipd_internal'),
           timeout=timeoutfn())
+      for i, pin in enumerate(pins):
+        insert_pin(path, pin[0], pin[1], packages[i])
       file_path.make_tree_files_read_only(site_root)
 
   total_duration = time.time() - start
   logging.info(
       'Installing CIPD client and packages took %d seconds', total_duration)
 
+  assert None not in all_pins
+
   return {
-    'duration': total_duration,
-    'get_client_duration': get_client_duration,
+    'stats': {
+      'duration': total_duration,
+      'get_client_duration': get_client_duration,
+    },
+    'pins': all_pins,
   }
 
 
 def create_option_parser():
   parser = logging_utils.OptionParserWithLogging(
       usage='%prog <options> [command to run or extra args]',
       version=__version__,
       log_file=RUN_ISOLATED_LOG_FILE)
   parser.add_option(
       '--clean', action='store_true',
@@ skipping 80 matching lines @@
         '%s in args requires --isolate-server' % ISOLATED_OUTDIR_PARAMETER)
 
   if options.root_dir:
     options.root_dir = unicode(os.path.abspath(options.root_dir))
   if options.json:
     options.json = unicode(os.path.abspath(options.json))
 
   cipd.validate_cipd_options(parser, options)
 
   install_packages_fn = lambda run_dir: install_packages(
-      run_dir, cipd.parse_package_args(options.cipd_packages),
+      run_dir, cipd.parse_package_args(options.cipd_packages, with_index=True),
       options.cipd_server, options.cipd_client_package,
       options.cipd_client_version, cache_dir=options.cipd_cache)
 
   try:
     command = [] if options.isolated else args
     if options.isolate_server:
       storage = isolateserver.get_storage(
           options.isolate_server, options.namespace)
       with storage:
         # Hashing schemes used by |storage| and |cache| MUST match.
@@ skipping 12 matching lines @@
     print >> sys.stderr, ex.message
     return 1
 
 
 if __name__ == '__main__':
   subprocess42.inhibit_os_error_reporting()
   # Ensure that we are always running with the correct encoding.
   fix_encoding.fix_encoding()
   file_path.enable_symlink()
   sys.exit(main(sys.argv[1:]))