Reland "named caches: move instead of symlinking"

appengine/swarming/swarming_bot/bot_code/task_runner_test.py

 #!/usr/bin/env python
 # coding=utf-8
 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 import base64
+import contextlib
 import json
 import logging
 import os
 import re
 import signal
 import sys
 import tempfile
 import time
 import unittest
 
 import test_env_bot_code
 test_env_bot_code.setup_test_env()
 
 # Creates a server mock for functions in net.py.
 import net_utils
 
 from depot_tools import fix_encoding
 from utils import file_path
 from utils import large
 from utils import logging_utils
 from utils import subprocess42
 from libs import luci_context
 import bot_auth
+import fake_swarming
+import named_cache
 import remote_client
-import fake_swarming
 import task_runner
 
 CLIENT_DIR = os.path.normpath(
     os.path.join(test_env_bot_code.BOT_DIR, '..', '..', '..', 'client'))
 
 sys.path.insert(0, os.path.join(CLIENT_DIR, 'tests'))
 import isolateserver_mock
 
 
 def get_manifest(script=None, isolated=None, **kwargs):
@@ skipping 537 matching lines @@
     expected = {
       u'exit_code': -1,
       u'hard_timeout': False,
       u'io_timeout': False,
       u'must_signal_internal_failure': None,
       u'version': 3,
     }
     self.assertEqual(expected, self._run_command(task_details))
 
   def test_run_command_caches(self):
+    # Put a file into a named cache.
+    cache_manager = named_cache.CacheManager(os.path.join(self.root_dir, u'c'))
+    install_dir = os.path.join(self.root_dir, u'install')
+
+    @contextlib.contextmanager
+    def tmpinstall():
+      with cache_manager.open():
+        cache_manager.install(install_dir, 'foo')
+        yield
+        cache_manager.uninstall(install_dir, 'foo')
+
+    with tmpinstall(), open(os.path.join(install_dir, 'bar'), 'wb') as f:

[M-A Ruel, 2017/05/11 19:55:13]

Why not:
with cache_manager.open():
  cache_manager.install(install_dir, 'foo')
  with open(os.path.join(install_dir, 'bar'), 'wb') as f:
    f.write('thecache')
  cache_manager.uninstall(install_dir, 'foo')

it's shorter and easier to read.

+      f.write('thecache')
+
     # This runs the command for real.
-    self.requests(cost_usd=1, exit_code=0);
+    self.requests(cost_usd=1, exit_code=0)
     script = (
       'import os\n'
       'print "hi"\n'
-      'with open("../../result", "w") as f:\n'
-      '  print >> f, os.path.abspath(os.readlink("git_cache"))'
+      'with open("cache_foo/bar", "rb") as f:\n'
+      '  cached = f.read()\n'
+      'with open("../../result", "wb") as f:\n'
+      '  f.write(cached)\n'
+      'with open("cache_foo/bar", "wb") as f:\n'
+      '  f.write("updated_cache")\n'
     )
     task_details = self.get_task_details(
         script,
-        caches=[{'name': 'git_chromium', 'path': 'git_cache'}])
+        caches=[{'name': 'foo', 'path': 'cache_foo'}])
     expected = {
       u'exit_code': 0,
       u'hard_timeout': False,
       u'io_timeout': False,
       u'must_signal_internal_failure': None,
       u'version': task_runner.OUT_VERSION,
     }
     self.assertEqual(expected, self._run_command(task_details))
 
-    with open(os.path.join(self.root_dir, 'result')) as f:
-      rundir_cache_path = f.read().strip()
-    named_caches = os.path.join(self.root_dir, 'c', 'named')
-    self.assertTrue(os.path.isdir(named_caches))
-    named_cache_path = os.path.abspath(
-        os.readlink(os.path.join(named_caches, 'git_chromium')))
-    self.assertEqual(rundir_cache_path, named_cache_path)
+    with open(os.path.join(self.root_dir, 'result'), 'rb') as f:
+      self.assertEqual('thecache', f.read())
+
+    with tmpinstall(), open(os.path.join(install_dir, 'bar'), 'rb') as f:
+      self.assertEqual(f.read(), 'updated_cache')

[M-A Ruel, 2017/05/11 19:55:13]

self.assertEqual('updated_cache', f.read())

 
   def test_main(self):
     def load_and_run(
         manifest, swarming_server, is_grpc, cost_usd_hour, start,
         json_file, run_isolated_flags, bot_file, auth_params_file):
       self.assertEqual('foo', manifest)
       self.assertEqual('http://localhost', swarming_server)
       self.assertFalse(is_grpc)
       self.assertEqual(3600., cost_usd_hour)
       self.assertEqual(time.time(), start)
@@ skipping 646 matching lines @@
   fix_encoding.fix_encoding()
   if '-v' in sys.argv:
     unittest.TestCase.maxDiff = None
   logging_utils.prepare_logging(None)
   logging_utils.set_console_level(
       logging.DEBUG if '-v' in sys.argv else logging.CRITICAL+1)
   # Fix litteral text expectation.
   os.environ['LANG'] = 'en_US.UTF-8'
   os.environ['LANGUAGE'] = 'en_US.UTF-8'
   unittest.main()