OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 | 2 |
3 """Tests for gs_utils.py. | 3 """Tests for gs_utils.py. |
4 | 4 |
5 TODO(epoger): How should we exercise these self-tests? See http://skbug.com/2751 | 5 TODO(epoger): How should we exercise these self-tests? See http://skbug.com/2751 |
6 """ | 6 """ |
7 | 7 |
8 # System-level imports. | 8 # System-level imports. |
9 import os | 9 import os |
10 import posixpath | 10 import posixpath |
(...skipping 27 matching lines...) Expand all Loading... |
38 """Returns a unique directory name suitable for use in Google Storage.""" | 38 """Returns a unique directory name suitable for use in Google Storage.""" |
39 return 'gs_utils_manualtest/%d' % random.randint(0, sys.maxint) | 39 return 'gs_utils_manualtest/%d' % random.randint(0, sys.maxint) |
40 | 40 |
41 | 41 |
42 def _test_public_read(): | 42 def _test_public_read(): |
43 """Make sure we can read from public files without .boto file credentials.""" | 43 """Make sure we can read from public files without .boto file credentials.""" |
44 gs = gs_utils.GSUtils() | 44 gs = gs_utils.GSUtils() |
45 gs.list_bucket_contents(bucket=TEST_BUCKET, subdir=None) | 45 gs.list_bucket_contents(bucket=TEST_BUCKET, subdir=None) |
46 | 46 |
47 | 47 |
48 def _test_only_if_modified(): | 48 def _test_upload_if(): |
49 """Test only_if_modified param within upload_file().""" | 49 """Test upload_if param within upload_file().""" |
50 gs = _get_authenticated_gs_handle() | 50 gs = _get_authenticated_gs_handle() |
51 filename = 'filename' | 51 filename = 'filename' |
52 remote_dir = _get_unique_posix_dir() | 52 remote_dir = _get_unique_posix_dir() |
53 dest_path = posixpath.join(remote_dir, filename) | 53 dest_path = posixpath.join(remote_dir, filename) |
54 local_dir = tempfile.mkdtemp() | 54 local_dir = tempfile.mkdtemp() |
55 try: | 55 try: |
56 # Create a file on local disk, and upload it for the first time. | 56 # Create a file on local disk, and upload it for the first time. |
57 local_path = os.path.join(local_dir, filename) | 57 local_path = os.path.join(local_dir, filename) |
58 with open(local_path, 'w') as f: | 58 with open(local_path, 'w') as f: |
59 f.write('original contents') | 59 f.write('original contents') |
60 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, | 60 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, |
61 dest_path=dest_path, only_if_modified=True) | 61 dest_path=dest_path, upload_if=gs.UploadIf.IF_NEW) |
62 try: | 62 try: |
63 # Re-upload the same file one second later, with only_if_modified=False; | 63 # Re-upload the same file, with upload_if=gs.UploadIf.ALWAYS; |
64 # the timestamp should change. | 64 # the timestamp should change. |
65 old_timestamp = gs.get_last_modified_time( | 65 old_timestamp = gs.get_last_modified_time( |
66 bucket=TEST_BUCKET, path=dest_path) | 66 bucket=TEST_BUCKET, path=dest_path) |
67 time.sleep(2) | 67 time.sleep(2) |
68 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, | 68 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, |
69 dest_path=dest_path, only_if_modified=False) | 69 dest_path=dest_path, upload_if=gs.UploadIf.ALWAYS) |
70 new_timestamp = gs.get_last_modified_time( | 70 new_timestamp = gs.get_last_modified_time( |
71 bucket=TEST_BUCKET, path=dest_path) | 71 bucket=TEST_BUCKET, path=dest_path) |
72 assert old_timestamp != new_timestamp, '%s != %s' % ( | 72 assert old_timestamp != new_timestamp, '%s != %s' % ( |
73 old_timestamp, new_timestamp) | 73 old_timestamp, new_timestamp) |
74 | 74 |
75 # Re-upload the same file one second later, with only_if_modified=True; | 75 # Re-upload the same file, with upload_if=gs.UploadIf.IF_MODIFIED; |
76 # the timestamp should NOT change. | 76 # the timestamp should NOT change. |
77 old_timestamp = new_timestamp | 77 old_timestamp = new_timestamp |
78 time.sleep(2) | 78 time.sleep(2) |
79 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, | 79 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, |
80 dest_path=dest_path, only_if_modified=True) | 80 dest_path=dest_path, upload_if=gs.UploadIf.IF_MODIFIED) |
81 new_timestamp = gs.get_last_modified_time( | 81 new_timestamp = gs.get_last_modified_time( |
82 bucket=TEST_BUCKET, path=dest_path) | 82 bucket=TEST_BUCKET, path=dest_path) |
83 assert old_timestamp == new_timestamp, '%s == %s' % ( | 83 assert old_timestamp == new_timestamp, '%s == %s' % ( |
84 old_timestamp, new_timestamp) | 84 old_timestamp, new_timestamp) |
85 | 85 |
86 # MODIFY and re-upload the file one second later, with | 86 # Modify and re-upload the file, with upload_if=gs.UploadIf.IF_NEW; |
87 # only_if_modified=True; the timestamp SHOULD change. | 87 # the timestamp should still not change. |
88 old_timestamp = new_timestamp | 88 old_timestamp = new_timestamp |
89 with open(local_path, 'w') as f: | 89 with open(local_path, 'w') as f: |
90 f.write('modified contents') | 90 f.write('modified contents') |
91 time.sleep(2) | 91 time.sleep(2) |
92 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, | 92 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, |
93 dest_path=dest_path, only_if_modified=True) | 93 dest_path=dest_path, upload_if=gs.UploadIf.IF_NEW) |
| 94 new_timestamp = gs.get_last_modified_time( |
| 95 bucket=TEST_BUCKET, path=dest_path) |
| 96 assert old_timestamp == new_timestamp, '%s == %s' % ( |
| 97 old_timestamp, new_timestamp) |
| 98 |
| 99 # Re-upload the modified file, with upload_if=gs.UploadIf.IF_MODIFIED; |
| 100 # now the timestamp SHOULD change. |
| 101 old_timestamp = new_timestamp |
| 102 time.sleep(2) |
| 103 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, |
| 104 dest_path=dest_path, upload_if=gs.UploadIf.IF_MODIFIED) |
94 new_timestamp = gs.get_last_modified_time( | 105 new_timestamp = gs.get_last_modified_time( |
95 bucket=TEST_BUCKET, path=dest_path) | 106 bucket=TEST_BUCKET, path=dest_path) |
96 assert old_timestamp != new_timestamp, '%s != %s' % ( | 107 assert old_timestamp != new_timestamp, '%s != %s' % ( |
97 old_timestamp, new_timestamp) | 108 old_timestamp, new_timestamp) |
98 finally: | 109 finally: |
99 # Clean up the remote_dir. | 110 # Clean up the remote_dir. |
100 gs.delete_file(bucket=TEST_BUCKET, path=dest_path) | 111 gs.delete_file(bucket=TEST_BUCKET, path=dest_path) |
101 finally: | 112 finally: |
102 # Clean up the local dir. | 113 # Clean up the local dir. |
103 shutil.rmtree(local_dir) | 114 shutil.rmtree(local_dir) |
(...skipping 153 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
257 assert file_contents == 'contents of %s\n' % filename, ( | 268 assert file_contents == 'contents of %s\n' % filename, ( |
258 '%s == "contents of %s\n"' % (file_contents, filename)) | 269 '%s == "contents of %s\n"' % (file_contents, filename)) |
259 finally: | 270 finally: |
260 shutil.rmtree(local_dest_dir) | 271 shutil.rmtree(local_dest_dir) |
261 for filename in filenames: | 272 for filename in filenames: |
262 gs.delete_file(bucket=TEST_BUCKET, | 273 gs.delete_file(bucket=TEST_BUCKET, |
263 path=posixpath.join(remote_dir, subdir, filename)) | 274 path=posixpath.join(remote_dir, subdir, filename)) |
264 | 275 |
265 | 276 |
266 if __name__ == '__main__': | 277 if __name__ == '__main__': |
267 _test_only_if_modified() | 278 _test_upload_if() |
268 _test_public_read() | 279 _test_public_read() |
269 _test_authenticated_round_trip() | 280 _test_authenticated_round_trip() |
270 _test_dir_upload_and_download() | 281 _test_dir_upload_and_download() |
271 # TODO(epoger): Add _test_unauthenticated_access() to make sure we raise | 282 # TODO(epoger): Add _test_unauthenticated_access() to make sure we raise |
272 # an exception when we try to access without needed credentials. | 283 # an exception when we try to access without needed credentials. |
OLD | NEW |