Cache/retrieve extracted CIPD packages in local isolate cache

client/run_isolated.py

 #!/usr/bin/env python
 # Copyright 2012 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Runs a command with optional isolated input/output.
 
 Despite name "run_isolated", can run a generic non-isolated command specified as
 args.
 
@@ skipping 75 matching lines @@
 RUN_ISOLATED_LOG_FILE = 'run_isolated.log'
 
 
 # The name of the log to use for the run_test_cases.py command
 RUN_TEST_CASES_LOG = 'run_test_cases.log'
 
 
 # Use short names for temporary directories. This is driven by Windows, which
 # imposes a relatively short maximum path length of 260 characters, often
 # referred to as MAX_PATH. It is relatively easy to create files with longer
-# path length. A use case is with recursive depedency treesV like npm packages.
+# path length. A use case is with recursive dependency trees like npm packages.
 #
 # It is recommended to start the script with a `root_dir` as short as
 # possible.
 # - ir stands for isolated_run
 # - io stands for isolated_out
 # - it stands for isolated_tmp
 ISOLATED_RUN_DIR = u'ir'
 ISOLATED_OUT_DIR = u'io'
 ISOLATED_TMP_DIR = u'it'
 
 
 def get_as_zip_package(executable=True):
   """Returns ZipPackage with this module and all its dependencies.
 
   If |executable| is True will store run_isolated.py as __main__.py so that
   zip package is directly executable be python.
   """
   # Building a zip package when running from another zip package is
   # unsupported and probably unneeded.
   assert not zip_package.is_zipped_module(sys.modules[__name__])
   assert THIS_FILE_PATH
   assert BASE_DIR
   package = zip_package.ZipPackage(root=BASE_DIR)
   package.add_python_file(THIS_FILE_PATH, '__main__.py' if executable else None)
   package.add_python_file(os.path.join(BASE_DIR, 'isolate_storage.py'))
   package.add_python_file(os.path.join(BASE_DIR, 'isolated_format.py'))
   package.add_python_file(os.path.join(BASE_DIR, 'isolateserver.py'))
+  package.add_python_file(os.path.join(BASE_DIR, 'isolate_format.py'))
   package.add_python_file(os.path.join(BASE_DIR, 'auth.py'))
   package.add_python_file(os.path.join(BASE_DIR, 'cipd.py'))
   package.add_python_file(os.path.join(BASE_DIR, 'named_cache.py'))
+  package.add_python_file(os.path.join(BASE_DIR, 'run_isolated.py'))

[Vadim Sh., 2017/05/11 03:09:09]

note that this file may already be included in the zip as __main__.py

(depending on 'executable').

Also, I think run_isolated.zip is kind of dead code now. Nothing is using it in
production (Swarming bot has run_isolated.py copy itself). So I'm surprised you
are modifying it.

   package.add_directory(os.path.join(BASE_DIR, 'libs'))
   package.add_directory(os.path.join(BASE_DIR, 'third_party'))
   package.add_directory(os.path.join(BASE_DIR, 'utils'))
   return package
 
 
 def make_temp_dir(prefix, root_dir):
   """Returns a new unique temporary directory."""
   return unicode(tempfile.mkdtemp(prefix=prefix, dir=root_dir))
 
@@ skipping 138 matching lines @@
             exit_code = proc.wait(grace_period or None)
           except subprocess42.TimeoutExpired:
             # Now kill for real. The user can distinguish between the
             # following states:
             # - signal but process exited within grace period,
             #   hard_timed_out will be set but the process exit code will be
             #   script provided.
             # - processed exited late, exit code will be -9 on posix.
             logging.warning('Grace exhausted; sending SIGKILL')
             proc.kill()
-      logging.info('Waiting for proces exit')
+      logging.info('Waiting for process exit')
       exit_code = proc.wait()
     except OSError:
       # This is not considered to be an internal error. The executable simply
       # does not exit.
       sys.stderr.write(
           '<The executable does not exist or a dependent library is missing>\n'
           '<Check for missing .so/.dll in the .isolate or GN file>\n'
           '<Command: %s>\n' % command)
       if os.environ.get('SWARMING_TASK_ID'):
         # Give an additional hint when running as a swarming task.
@@ skipping 383 matching lines @@
   'pins',       # dict with installed cipd pins to return to the server
 ])
 
 
 @contextlib.contextmanager
 def noop_install_packages(_run_dir):
   """Placeholder for 'install_client_and_packages' if cipd is disabled."""
   yield None
 
 
-def _install_packages(run_dir, cipd_cache_dir, client, packages, timeout):
+def _install_packages(run_dir, cipd_cache_dir, client, packages, timeout,
+                      isolate_cache=None):

[Vadim Sh., 2017/05/11 03:09:09]

please document in Args, at least its type (in all other places where it is
added too)

   """Calls 'cipd ensure' for packages.
 
   Args:
     run_dir (str): root of installation.
     cipd_cache_dir (str): the directory to use for the cipd package cache.
     client (CipdClient): the cipd client to use
     packages: packages to install, list [(path, package_name, version), ...].
     timeout: max duration in seconds that this function can take.
 
   Returns: list of pinned packages.  Looks like [
@@ skipping 22 matching lines @@
     by_path[path].append((name, version, i))
 
   pins = client.ensure(
     run_dir,
     {
       subdir: [(name, vers) for name, vers, _ in pkgs]
       for subdir, pkgs in by_path.iteritems()
     },
     cache_dir=cipd_cache_dir,
     timeout=timeout,
+    isolate_cache=isolate_cache,
   )
 
   for subdir, pin_list in sorted(pins.iteritems()):
     this_subdir = by_path[subdir]
     for i, (name, version) in enumerate(pin_list):
       insert_pin(subdir, name, version, this_subdir[i][2])
 
   assert None not in package_pins
 
   return package_pins
 
 
 @contextlib.contextmanager
 def install_client_and_packages(
     run_dir, packages, service_url, client_package_name,
-    client_version, cache_dir, timeout=None):
+    client_version, cache_dir, timeout=None, isolate_cache=None):
   """Bootstraps CIPD client and installs CIPD packages.
 
   Yields CipdClient, stats, client info and pins (as single CipdInfo object).
 
   Pins and the CIPD client info are in the form of:
     [
       {
         "path": path, "package_name": package_name, "version": version,
       },
       ...
@@ skipping 20 matching lines @@
     cache_dir (str): where to keep cache of cipd clients, packages and tags.
     timeout: max duration in seconds that this function can take.
   """
   assert cache_dir
 
   timeoutfn = tools.sliding_timeout(timeout)
   start = time.time()
 
   cache_dir = os.path.abspath(cache_dir)
   cipd_cache_dir = os.path.join(cache_dir, 'cache')  # tag and instance caches
+  file_path.ensure_tree(unicode(cipd_cache_dir))
+
   run_dir = os.path.abspath(run_dir)
   packages = packages or []
 
   get_client_start = time.time()
   client_manager = cipd.get_client(
       service_url, client_package_name, client_version, cache_dir,
       timeout=timeoutfn())
 
   with client_manager as client:
     get_client_duration = time.time() - get_client_start
 
     package_pins = []
     if packages:
       package_pins = _install_packages(
-        run_dir, cipd_cache_dir, client, packages, timeoutfn())
+        run_dir, cipd_cache_dir, client, packages, timeoutfn(), isolate_cache)
 
     file_path.make_tree_files_read_only(run_dir)
 
     total_duration = time.time() - start
     logging.info(
         'Installing CIPD client and packages took %d seconds', total_duration)
 
     yield CipdInfo(
       client=client,
       cache_dir=cipd_cache_dir,
@@ skipping 185 matching lines @@
   if options.json:
     options.json = unicode(os.path.abspath(options.json))
 
   cipd.validate_cipd_options(parser, options)
 
   install_packages_fn = noop_install_packages
   if options.cipd_enabled:
     install_packages_fn = lambda run_dir: install_client_and_packages(
         run_dir, cipd.parse_package_args(options.cipd_packages),
         options.cipd_server, options.cipd_client_package,
-        options.cipd_client_version, cache_dir=options.cipd_cache)
+        options.cipd_client_version, cache_dir=options.cipd_cache,
+        isolate_cache=isolate_cache)
 
   @contextlib.contextmanager
   def init_named_caches(run_dir):
     # WARNING: this function depends on "options" variable defined in the outer
     # function.
     with named_cache_manager.open():
       named_cache_manager.create_symlinks(run_dir, options.named_caches)
     try:
       yield
     finally:
@@ skipping 41 matching lines @@
     return 1
 
 
 if __name__ == '__main__':
   subprocess42.inhibit_os_error_reporting()
   # Ensure that we are always running with the correct encoding.
   fix_encoding.fix_encoding()
   file_path.enable_symlink()
 
   sys.exit(main(sys.argv[1:]))