luci-py/isolateserver.py: Add support for tar archives when downloading.

client/tests/isolateserver_test.py

 #!/usr/bin/env python
 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 # pylint: disable=W0212,W0223,W0231,W0613
 
 import base64
 import collections
 import hashlib
 import json
 import logging
 import io
 import os
 import StringIO
 import sys
+import tarfile
 import tempfile
 import unittest
 import urllib
 import zlib
 
 # net_utils adjusts sys.path.
 import net_utils
 
 import auth
 import isolated_format
@@ skipping 996 matching lines @@
     self.expected_requests(requests)
     self.assertEqual(0, isolateserver.main(cmd))
     expected = dict(
         (os.path.join(self.tempdir, k), v) for k, v in files.iteritems())
     self.assertEqual(expected, actual)
     expected_stdout = (
         'To run this test please run from the directory %s:\n  Absurb command\n'
         % os.path.join(self.tempdir, 'a'))
     self.checkOutput(expected_stdout, '')
 
-  def test_download_isolated_archive(self):
+  def test_download_isolated_ar_archive(self):
     # Test downloading an isolated tree.
     actual = {}
     def putfile_mock(
         srcfileobj, dstpath, file_mode=None, size=-1, use_symlink=False):
       actual[dstpath] = srcfileobj.read(size)
     self.mock(isolateserver, 'putfile', putfile_mock)
     self.mock(os, 'makedirs', lambda _: None)
     server = 'http://example.com'
 
     files = {
@@ skipping 81 matching lines @@
     self.expected_requests(requests)
     self.assertEqual(0, isolateserver.main(cmd))
     expected = dict(
         (os.path.join(self.tempdir, k), v) for k, v in files.iteritems())
     self.assertEqual(expected, actual)
     expected_stdout = (
         'To run this test please run from the directory %s:\n  Absurb command\n'
         % os.path.join(self.tempdir, 'a'))
     self.checkOutput(expected_stdout, '')
 
+  maxDiff = None

[M-A Ruel, 2016/11/08 14:33:44]

remove

[mithro, 2016/11/09 00:19:35]

Sorry, left over from debugging.

+
+  def test_download_isolated_tar_archive(self):
+    # Test downloading an isolated tree.
+    actual = {}
+    def putfile_mock(
+        srcfileobj, dstpath, file_mode=None, size=-1, use_symlink=False):
+      actual[dstpath] = srcfileobj.read(size)
+    self.mock(isolateserver, 'putfile', putfile_mock)
+    self.mock(os, 'makedirs', lambda _: None)
+    server = 'http://example.com'
+
+    files = {
+      os.path.join('a', 'foo'): 'Content',
+      'b': 'More content',
+      'c': 'Even more content!',
+    }
+
+    # Generate a tar archive
+    tf = io.BytesIO()
+    with tarfile.TarFile(mode='w', fileobj=tf) as tar:
+        f1 = tarfile.TarInfo()

[M-A Ruel, 2016/11/08 14:33:44]

indent at +2

[mithro, 2016/11/09 00:19:35]

Done.

+        f1.type = tarfile.REGTYPE
+        f1.name = 'a/foo'
+        f1.size = 7
+        tar.addfile(f1, io.BytesIO('Content'))
+
+        f2 = tarfile.TarInfo()
+        f2.type = tarfile.REGTYPE
+        f2.name = 'b'
+        f2.size = 12
+        tar.addfile(f2, io.BytesIO('More content'))
+    archive = tf.getvalue()
+
+    isolated = {
+      'command': ['Absurb', 'command'],
+      'relative_cwd': 'a',
+      'files': {
+        'archive1': {
+          'h': isolateserver_mock.hash_content(archive),
+          's': len(archive),
+          't': 'tar',
+        },
+        'c': {
+          'h': isolateserver_mock.hash_content(files['c']),
+          's': len(files['c']),
+        },
+      },
+      'version': isolated_format.ISOLATED_FILE_VERSION,
+    }
+    isolated_data = json.dumps(isolated, sort_keys=True, separators=(',',':'))
+    isolated_hash = isolateserver_mock.hash_content(isolated_data)
+    requests = [
+      (isolated['files']['archive1']['h'], archive),
+      (isolated['files']['c']['h'], files['c']),
+    ]
+    requests.append((isolated_hash, isolated_data))
+    requests = [
+      (
+        server + '/api/isolateservice/v1/retrieve',
+        {
+            'data': {
+                'digest': h.encode('utf-8'),
+                'namespace': {
+                    'namespace': 'default-gzip',
+                    'digest_hash': 'sha-1',
+                    'compression': 'flate',
+                },
+                'offset': 0,
+            },
+            'read_timeout': 60,
+        },
+        {'content': base64.b64encode(zlib.compress(v))},
+      ) for h, v in requests
+    ]
+    cmd = [
+      'download',
+      '--isolate-server', server,
+      '--target', self.tempdir,
+      '--isolated', isolated_hash,
+    ]
+    self.expected_requests(requests)
+    self.assertEqual(0, isolateserver.main(cmd))
+    expected = dict(
+        (os.path.join(self.tempdir, k), v) for k, v in files.iteritems())
+    self.assertEqual(expected, actual)
+    expected_stdout = (
+        'To run this test please run from the directory %s:\n  Absurb command\n'
+        % os.path.join(self.tempdir, 'a'))
+    self.checkOutput(expected_stdout, '')
+

[M-A Ruel, 2016/11/08 14:33:44]

only 2 lines between file level symbols

[mithro, 2016/11/09 00:19:35]

Done.

 
 
 def get_storage(_isolate_server, namespace):
   class StorageFake(object):
     def __enter__(self, *_):
       return self
 
     def __exit__(self, *_):
       pass
 
@@ skipping 218 matching lines @@
 
 
 if __name__ == '__main__':
   fix_encoding.fix_encoding()
   if '-v' in sys.argv:
     unittest.TestCase.maxDiff = None
   logging.basicConfig(
       level=(logging.DEBUG if '-v' in sys.argv else logging.CRITICAL))
   clear_env_vars()
   unittest.main()