luci-py/isolateserver.py: Add archive support when downloading.

client/tests/isolateserver_test.py

Index: client/tests/isolateserver_test.py
diff --git a/client/tests/isolateserver_test.py b/client/tests/isolateserver_test.py
index 12c8f1c62cf0b305a077ee3a64cda74e16dc1fdf..f38b08a7e7e6f1a7b35aa0598504e333a0cc5e39 100755
--- a/client/tests/isolateserver_test.py
+++ b/client/tests/isolateserver_test.py
@@ -724,7 +724,7 @@ class IsolateServerStorageSmokeTest(unittest.TestCase):
     # Ensure fetched same data as was pushed.
     self.assertEqual(
         [i.buffer for i in items],
-        [cache.read(i.digest) for i in items])
+        [cache.getfileobj(i.digest).read() for i in items])

[M-A Ruel, 2016/06/21 13:31:09]

please close everywhere

[mithro, 2016/06/22 12:03:11]

Done.

 
   def test_push_and_fetch(self):
     self.run_push_and_fetch_test('default')
@@ -838,12 +838,12 @@ class IsolateServerDownloadTest(TestCase):
     }
     self.assertEqual(expected, actual)
 
-  def test_download_isolated(self):
+  def test_download_isolated_simple(self):
     # Test downloading an isolated tree.
     actual = {}
-    def file_write_mock(key, generator):
-      actual[key] = ''.join(generator)
-    self.mock(isolateserver, 'file_write', file_write_mock)
+    def putfile_mock(srcfileobj, dstpath, file_mode=None):
+      actual[dstpath] = srcfileobj.read()
+    self.mock(isolateserver, 'putfile', putfile_mock)
     self.mock(os, 'makedirs', lambda _: None)
     server = 'http://example.com'
     files = {
@@ -896,6 +896,99 @@ class IsolateServerDownloadTest(TestCase):
         % os.path.join(self.tempdir, 'a'))
     self.checkOutput(expected_stdout, '')
 
+  def test_download_isolated_archive(self):
+    # Test downloading an isolated tree.
+    actual = {}
+    def putfile_mock(srcfileobj, dstpath, file_mode=None, limit=-1):
+      actual[dstpath] = srcfileobj.read(limit)
+    self.mock(isolateserver, 'putfile', putfile_mock)
+    self.mock(os, 'makedirs', lambda _: None)
+    server = 'http://example.com'
+    files = {
+      os.path.join('a', 'foo'): 'Content',
+      'b': 'More content',
+      }
+
+    archive = (
+      # ar file header
+      '!<arch>\n'
+      # File 1 -------------------------
+      # (16 bytes) filename len
+      '#1/5            '
+      # file metadata
+      '1447140471  1000  1000  100640  '
+      # (10 bytes) Data size
+      '12        '
+      # (2 bytes) File magic
+      '\x60\n'
+      # (5 bytes) File name
+      'a/foo'
+      # (7 bytes) File data
+      'Content'
+      # File 2 -------------------------
+      # (16 bytes) filename
+      'b               '
+      # file metadata
+      '1447140471  1000  1000  100640  '
+      # (12 bytes) Data size
+      '12        '
+      # (2 bytes) File magic
+      '\x60\n'
+      # (12 bytes) File data
+      'More content'
+      '')
+
+    isolated = {
+      'command': ['Absurb', 'command'],
+      'relative_cwd': 'a',
+      'files': {
+        'archive1': {
+          'h': isolateserver_mock.hash_content(archive),
+          's': len(archive),
+          'a': 'ar',
+        },
+      },
+      'version': isolated_format.ISOLATED_FILE_VERSION,
+    }
+    isolated_data = json.dumps(isolated, sort_keys=True, separators=(',',':'))
+    isolated_hash = isolateserver_mock.hash_content(isolated_data)
+    requests = [(isolated['files']['archive1']['h'], archive)]
+    requests.append((isolated_hash, isolated_data))
+    requests = [
+      (
+        server + '/_ah/api/isolateservice/v1/retrieve',
+        {
+            'data': {
+                'digest': h.encode('utf-8'),
+                'namespace': {
+                    'namespace': 'default-gzip',
+                    'digest_hash': 'sha-1',
+                    'compression': 'flate',
+                },
+                'offset': 0,
+            },
+            'read_timeout': 60,
+        },
+        {'content': base64.b64encode(zlib.compress(v))},
+      ) for h, v in requests
+    ]
+    cmd = [
+      'download',
+      '--isolate-server', server,
+      '--target', self.tempdir,
+      '--isolated', isolated_hash,
+    ]
+    self.expected_requests(requests)
+    self.assertEqual(0, isolateserver.main(cmd))
+    expected = dict(
+        (os.path.join(self.tempdir, k), v) for k, v in files.iteritems())
+    self.assertEqual(expected, actual)
+    expected_stdout = (
+        'To run this test please run from the directory %s:\n  Absurb command\n'
+        % os.path.join(self.tempdir, 'a'))
+    self.checkOutput(expected_stdout, '')
+
+
 
 def get_storage(_isolate_server, namespace):
   class StorageFake(object):