OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 # Copyright 2013 The Chromium Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 import binascii | |
7 import hashlib | |
8 import logging | |
9 import os | |
10 import subprocess | |
11 import sys | |
12 import tempfile | |
13 import time | |
14 import unittest | |
15 import urllib | |
16 | |
17 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
18 sys.path.insert(0, ROOT_DIR) | |
19 | |
20 import isolateserver | |
21 | |
22 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
23 | |
24 # Ensure that the testing machine has access to this server. | |
25 ISOLATE_SERVER = 'https://isolateserver.appspot.com/' | |
26 | |
27 # The directory containing the test data files. | |
28 TEST_DATA_DIR = os.path.join(ROOT_DIR, 'tests', 'isolateserver') | |
29 | |
30 | |
31 # TODO(vadimsh): This test is a bit frankensteinish now. It uses new /content-gs | |
32 # protocol for uploads via 'isolateserver.py archive', but uses old /content | |
33 # protocol for validity check and fetches. | |
34 | |
35 class IsolateServerArchiveSmokeTest(unittest.TestCase): | |
36 def setUp(self): | |
37 # The namespace must end in '-gzip' since all files are now compressed | |
38 # before being uploaded. | |
39 self.namespace = ('temporary' + str(long(time.time())).split('.', 1)[0] | |
40 + '-gzip') | |
41 url = ISOLATE_SERVER + '/content/get_token?from_smoke_test=1' | |
42 self.token = urllib.quote(isolateserver.net.url_read(url)) | |
43 | |
44 def _archive_given_files(self, files): | |
45 """Given a list of files, call isolateserver.py with them. Then | |
46 verify they are all on the server.""" | |
47 args = [ | |
48 sys.executable, | |
49 os.path.join(ROOT_DIR, 'isolateserver.py'), | |
50 'archive', | |
51 '--isolate-server', ISOLATE_SERVER, | |
52 '--namespace', self.namespace | |
53 ] | |
54 if '-v' in sys.argv: | |
55 args.append('--verbose') | |
56 args.extend(os.path.join(TEST_DATA_DIR, filename) for filename in files) | |
57 | |
58 self.assertEqual(0, subprocess.call(args)) | |
59 | |
60 # Try to download the files from the server. | |
61 file_hashes = [ | |
62 isolateserver.hash_file(os.path.join(TEST_DATA_DIR, f), hashlib.sha1) | |
63 for f in files | |
64 ] | |
65 for i in range(len(files)): | |
66 download_url = '%scontent/retrieve/%s/%s' % ( | |
67 ISOLATE_SERVER, self.namespace, file_hashes[i]) | |
68 | |
69 downloaded_file = isolateserver.net.url_read(download_url, retry_404=True) | |
70 self.assertTrue(downloaded_file is not None, | |
71 'File %s was missing from the server' % files[i]) | |
72 | |
73 # Ensure the files are listed as present on the server. | |
74 contains_hash_url = '%scontent/contains/%s?token=%s&from_smoke_test=1' % ( | |
75 ISOLATE_SERVER, self.namespace, self.token) | |
76 | |
77 body = ''.join(binascii.unhexlify(h) for h in file_hashes) | |
78 expected = chr(1) * len(files) | |
79 MAX_ATTEMPTS = 10 | |
80 for i in xrange(MAX_ATTEMPTS): | |
81 # AppEngine's database is eventually consistent and isolateserver do not | |
82 # use transaction for performance reasons, so even if one request was able | |
83 # to retrieve the file, an subsequent may not see it! So retry a few time | |
84 # until the database becomes consistent with regard to these entities. | |
85 response = isolateserver.net.url_read( | |
86 contains_hash_url, | |
87 data=body, | |
88 content_type='application/octet-stream') | |
89 if response == expected: | |
90 break | |
91 # GAE is exposing its internal data inconsistency. | |
92 if i != (MAX_ATTEMPTS - 1): | |
93 print('Visible datastore inconsistency, retrying.') | |
94 time.sleep(0.1) | |
95 self.assertEqual(expected, response) | |
96 | |
97 def test_archive_empty_file(self): | |
98 self._archive_given_files(['empty_file.txt']) | |
99 | |
100 def test_archive_small_file(self): | |
101 self._archive_given_files(['small_file.txt']) | |
102 | |
103 def disabled_test_archive_huge_file(self): | |
104 # Create a file over 2gbs. | |
105 # TODO(maruel): Temporarily disabled until the server is fixed. | |
106 filepath = None | |
107 try: | |
108 try: | |
109 handle, filepath = tempfile.mkstemp(prefix='isolateserver') | |
110 # Write 2.1gb. | |
111 chunk = chr(0) + chr(57) + chr(128) + chr(255) | |
112 chunk1mb = chunk * (1024 * 1024 / len(chunk)) | |
113 for _ in xrange(1280): | |
114 os.write(handle, chunk1mb) | |
115 finally: | |
116 os.close(handle) | |
117 | |
118 self._archive_given_files([filepath]) | |
119 finally: | |
120 if filepath: | |
121 os.remove(filepath) | |
122 | |
123 | |
124 if __name__ == '__main__': | |
125 if len(sys.argv) > 1 and sys.argv[1].startswith('http'): | |
126 ISOLATE_SERVER = sys.argv.pop(1).rstrip('/') + '/' | |
127 logging.basicConfig( | |
128 level=logging.DEBUG if '-v' in sys.argv else logging.ERROR) | |
129 unittest.main() | |
OLD | NEW |