Cache/retrieve extracted CIPD packages in local isolate cache

client/run_isolated.py

Index: client/run_isolated.py
diff --git a/client/run_isolated.py b/client/run_isolated.py
index ff4397fe2e3277933db8265bf4464f868d720dcc..b1ea1b24fd9afe81eb867b95287fe9febd9de7ec 100755
--- a/client/run_isolated.py
+++ b/client/run_isolated.py
@@ -93,7 +93,7 @@ RUN_TEST_CASES_LOG = 'run_test_cases.log'
 # Use short names for temporary directories. This is driven by Windows, which
 # imposes a relatively short maximum path length of 260 characters, often
 # referred to as MAX_PATH. It is relatively easy to create files with longer
-# path length. A use case is with recursive depedency treesV like npm packages.
+# path length. A use case is with recursive dependency trees like npm packages.
 #
 # It is recommended to start the script with a `root_dir` as short as
 # possible.
@@ -121,9 +121,11 @@ def get_as_zip_package(executable=True):
   package.add_python_file(os.path.join(BASE_DIR, 'isolate_storage.py'))
   package.add_python_file(os.path.join(BASE_DIR, 'isolated_format.py'))
   package.add_python_file(os.path.join(BASE_DIR, 'isolateserver.py'))
+  package.add_python_file(os.path.join(BASE_DIR, 'isolate_format.py'))
   package.add_python_file(os.path.join(BASE_DIR, 'auth.py'))
   package.add_python_file(os.path.join(BASE_DIR, 'cipd.py'))
   package.add_python_file(os.path.join(BASE_DIR, 'named_cache.py'))
+  package.add_python_file(os.path.join(BASE_DIR, 'run_isolated.py'))

[Vadim Sh., 2017/05/11 03:09:09]

note that this file may already be included in the zip as __main__.py

(depending on 'executable').

Also, I think run_isolated.zip is kind of dead code now. Nothing is using it in
production (Swarming bot has run_isolated.py copy itself). So I'm surprised you
are modifying it.

   package.add_directory(os.path.join(BASE_DIR, 'libs'))
   package.add_directory(os.path.join(BASE_DIR, 'third_party'))
   package.add_directory(os.path.join(BASE_DIR, 'utils'))
@@ -282,7 +284,7 @@ def run_command(command, cwd, env, hard_timeout, grace_period):
             # - processed exited late, exit code will be -9 on posix.
             logging.warning('Grace exhausted; sending SIGKILL')
             proc.kill()
-      logging.info('Waiting for proces exit')
+      logging.info('Waiting for process exit')
       exit_code = proc.wait()
     except OSError:
       # This is not considered to be an internal error. The executable simply
@@ -686,7 +688,8 @@ def noop_install_packages(_run_dir):
   yield None
 
 
-def _install_packages(run_dir, cipd_cache_dir, client, packages, timeout):
+def _install_packages(run_dir, cipd_cache_dir, client, packages, timeout,
+                      isolate_cache=None):

[Vadim Sh., 2017/05/11 03:09:09]

please document in Args, at least its type (in all other places where it is
added too)

   """Calls 'cipd ensure' for packages.
 
   Args:
@@ -729,6 +732,7 @@ def _install_packages(run_dir, cipd_cache_dir, client, packages, timeout):
     },
     cache_dir=cipd_cache_dir,
     timeout=timeout,
+    isolate_cache=isolate_cache,
   )
 
   for subdir, pin_list in sorted(pins.iteritems()):
@@ -744,7 +748,7 @@ def _install_packages(run_dir, cipd_cache_dir, client, packages, timeout):
 @contextlib.contextmanager
 def install_client_and_packages(
     run_dir, packages, service_url, client_package_name,
-    client_version, cache_dir, timeout=None):
+    client_version, cache_dir, timeout=None, isolate_cache=None):
   """Bootstraps CIPD client and installs CIPD packages.
 
   Yields CipdClient, stats, client info and pins (as single CipdInfo object).
@@ -785,6 +789,8 @@ def install_client_and_packages(
 
   cache_dir = os.path.abspath(cache_dir)
   cipd_cache_dir = os.path.join(cache_dir, 'cache')  # tag and instance caches
+  file_path.ensure_tree(unicode(cipd_cache_dir))
+
   run_dir = os.path.abspath(run_dir)
   packages = packages or []
 
@@ -799,7 +805,7 @@ def install_client_and_packages(
     package_pins = []
     if packages:
       package_pins = _install_packages(
-        run_dir, cipd_cache_dir, client, packages, timeoutfn())
+        run_dir, cipd_cache_dir, client, packages, timeoutfn(), isolate_cache)
 
     file_path.make_tree_files_read_only(run_dir)
 
@@ -1005,7 +1011,8 @@ def main(args):
     install_packages_fn = lambda run_dir: install_client_and_packages(
         run_dir, cipd.parse_package_args(options.cipd_packages),
         options.cipd_server, options.cipd_client_package,
-        options.cipd_client_version, cache_dir=options.cipd_cache)
+        options.cipd_client_version, cache_dir=options.cipd_cache,
+        isolate_cache=isolate_cache)
 
   @contextlib.contextmanager
   def init_named_caches(run_dir):