run_isolated.py: install CIPD packages

client/tests/run_isolated_test.py

 #!/usr/bin/env python
 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 # pylint: disable=R0201
 
 import StringIO
 import base64
 import functools
 import json
 import logging
 import os
 import sys
 import tempfile
 import unittest
 
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, ROOT_DIR)
 sys.path.insert(0, os.path.join(ROOT_DIR, 'third_party'))
 
+import cipd
 import isolated_format
 import isolateserver
 import run_isolated
 from depot_tools import auto_stub
 from depot_tools import fix_encoding
 from utils import file_path
+from utils import fs
 from utils import large
 from utils import logging_utils
 from utils import on_error
 from utils import subprocess42
 from utils import tools
 
 import isolateserver_mock
+import cipdserver_mock
 
 
 def write_content(filepath, content):
   with open(filepath, 'wb') as f:
     f.write(content)
 
 
 def json_dumps(data):
   return json.dumps(data, sort_keys=True, separators=(',', ':'))
 
@@ skipping 27 matching lines @@
   def setUp(self):
     super(RunIsolatedTestBase, self).setUp()
     self.tempdir = tempfile.mkdtemp(prefix=u'run_isolated_test')
     logging.debug(self.tempdir)
     self.mock(run_isolated, 'make_temp_dir', self.fake_make_temp_dir)
     self.mock(run_isolated.auth, 'ensure_logged_in', lambda _: None)
     self.mock(
         logging_utils.OptionParserWithLogging, 'logger_root',
         logging.Logger('unittest'))
 
+    self.cipd_server = cipdserver_mock.MockCipdServer()
+
   def tearDown(self):
     for dirpath, dirnames, filenames in os.walk(self.tempdir, topdown=True):
       for filename in filenames:
         file_path.set_read_only(os.path.join(dirpath, filename), False)
       for dirname in dirnames:
         file_path.set_read_only(os.path.join(dirpath, dirname), False)
     file_path.rmtree(self.tempdir)
+    self.cipd_server.close()
     super(RunIsolatedTestBase, self).tearDown()
 
   @property
   def run_test_temp_dir(self):
     """Where to map all files in run_isolated.run_tha_test."""
     return os.path.join(self.tempdir, 'isolated_run')
 
   def fake_make_temp_dir(self, prefix, _root_dir=None):
     """Predictably returns directory for run_tha_test (one per test case)."""
-    self.assertIn(prefix, ('isolated_out', 'isolated_run', 'isolated_tmp'))
+    self.assertIn(
+        prefix,
+        ('isolated_out', 'isolated_run', 'isolated_tmp', 'cipd_site_root'))
     temp_dir = os.path.join(self.tempdir, prefix)
     self.assertFalse(os.path.isdir(temp_dir))
     os.makedirs(temp_dir)
     return temp_dir
 
   def temp_join(self, *args):
     """Shortcut for joining path with self.run_test_temp_dir."""
     return os.path.join(self.run_test_temp_dir, *args)
 
 
 class RunIsolatedTest(RunIsolatedTestBase):
   def setUp(self):
     super(RunIsolatedTest, self).setUp()
     self.popen_calls = []
     # pylint: disable=no-self-argument
     class Popen(object):
       def __init__(self2, args, **kwargs):
         kwargs.pop('cwd', None)
         kwargs.pop('env', None)
         self.popen_calls.append((args, kwargs))
         self2.returncode = None
 
+      def yield_any_line(self, timeout=None):  # pylint: disable=unused-argument
+        return ()
+
       def wait(self, timeout=None):  # pylint: disable=unused-argument
         self.returncode = 0
         return self.returncode
 
       def kill(self):
         pass
 
     self.mock(subprocess42, 'Popen', Popen)
 
   def test_main(self):
@@ skipping 57 matching lines @@
     ret = run_isolated.run_tha_test(
         command,
         isolated_hash,
         StorageFake(files),
         isolateserver.MemoryCache(),
         False,
         None,
         None,
         None,
         None,
+        None,
+        None,
         None)
     self.assertEqual(0, ret)
     return make_tree_call
 
   def test_run_tha_test_naked(self):
     isolated = json_dumps({'command': ['invalid', 'command']})
     isolated_hash = isolateserver_mock.hash_content(isolated)
     files = {isolated_hash:isolated}
     make_tree_call = self._run_tha_test(isolated_hash, files)
     self.assertEqual(
@@ skipping 105 matching lines @@
       'hello',
       'world',
     ]
     ret = run_isolated.main(cmd)
     self.assertEqual(1, ret)
     self.assertEqual(1, len(self.popen_calls))
     self.assertEqual(
         [([u'/bin/echo', u'hello', u'world'], {'detached': True})],
         self.popen_calls)
 
+  def test_main_naked_with_packages(self):
+    cmd = [
+      '--no-log',
+      '--cipd-package', 'infra/tools/echo/${PLATFORM}:latest',
+      '--cipd-server', self.cipd_server.url,
+      '${CIPD_PATH}/echo',
+      'hello',
+      'world',
+    ]
+    ret = run_isolated.main(cmd)
+    self.assertEqual(0, ret)
+
+    self.assertEqual(2, len(self.popen_calls))
+
+    # Test cipd-ensure command.
+    cipd_ensure_cmd, _ = self.popen_calls[0]
+    self.assertEqual(cipd_ensure_cmd[:2], [
+      os.path.abspath(
+          'cipd_cache/cipd' + cipd.EXECUTABLE_SUFFIX),
+      'ensure',
+    ])
+    cache_dir_index = cipd_ensure_cmd.index('-cache-dir')
+    self.assertEqual(
+        cipd_ensure_cmd[cache_dir_index+1],
+        os.path.abspath('cipd_cache/cipd_internal'))
+
+    # Test cipd cache.
+    version_file = unicode(os.path.abspath(
+        'cipd_cache/versions/1481d0a0ceb16ea4672fed76a0710306eb9f3a33'))
+    self.assertTrue(fs.isfile(version_file))
+    with open(version_file) as f:
+      self.assertEqual(f.read(), 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')
+
+    client_binary_file = unicode(os.path.abspath(
+        'cipd_cache/clients/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'))
+    self.assertTrue(fs.isfile(client_binary_file))
+
+    # Test echo call.
+    echo_cmd, _ = self.popen_calls[1]
+    self.assertTrue(echo_cmd[0].endswith('cipd_site_root/echo'))
+    self.assertEqual(echo_cmd[1:], ['hello', 'world'])
+
   def test_modified_cwd(self):
     isolated = json_dumps({
         'command': ['../out/some.exe', 'arg'],
         'relative_cwd': 'some',
     })
     isolated_hash = isolateserver_mock.hash_content(isolated)
     files = {isolated_hash:isolated}
     _ = self._run_tha_test(isolated_hash, files)
     self.assertEqual(1, len(self.popen_calls))
     self.assertEqual(
@@ skipping 58 matching lines @@
       ret = run_isolated.run_tha_test(
           None,
           isolated_hash,
           store,
           isolateserver.MemoryCache(),
           False,
           None,
           None,
           None,
           None,
-          [])
+          [],
+          None,
+          None)
       self.assertEqual(0, ret)
 
       # It uploaded back. Assert the store has a new item containing foo.
       hashes = {isolated_hash, script_hash}
       output_hash = isolateserver_mock.hash_content('bar')
       hashes.add(output_hash)
       isolated =  {
         'algo': 'sha-1',
         'files': {
           'foo': {
@@ skipping 110 matching lines @@
             u'initial_size': 0,
             u'items_cold': [len(isolated_in_json)],
             u'items_hot': [],
           },
           u'upload': {
             u'items_cold': [len(isolated_out_json)],
             u'items_hot': [15],
           },
         },
       },
-      u'version': 4,
+      u'version': 5,
     }
     actual = tools.read_json(out)
     # duration can be exactly 0 due to low timer resolution, especially but not
     # exclusively on Windows.
     self.assertLessEqual(0, actual.pop(u'duration'))
     actual_isolated_stats = actual[u'stats'][u'isolated']
     self.assertLessEqual(0, actual_isolated_stats[u'download'].pop(u'duration'))
     self.assertLessEqual(0, actual_isolated_stats[u'upload'].pop(u'duration'))
     for i in (u'download', u'upload'):
       for j in (u'items_cold', u'items_hot'):
         actual_isolated_stats[i][j] = large.unpack(
             base64.b64decode(actual_isolated_stats[i][j]))
     self.assertEqual(expected, actual)
 
 
 if __name__ == '__main__':
   fix_encoding.fix_encoding()
   if '-v' in sys.argv:
     unittest.TestCase.maxDiff = None
   logging.basicConfig(
       level=logging.DEBUG if '-v' in sys.argv else logging.ERROR)
   unittest.main()