| OLD | NEW |
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 | 2 |
| 3 """Tests for gs_utils.py. | 3 """Tests for gs_utils.py. |
| 4 | 4 |
| 5 TODO(epoger): How should we exercise these self-tests? See http://skbug.com/2751 | 5 TODO(epoger): How should we exercise these self-tests? See http://skbug.com/2751 |
| 6 """ | 6 """ |
| 7 | 7 |
| 8 # System-level imports. | 8 # System-level imports. |
| 9 import os | 9 import os |
| 10 import posixpath | 10 import posixpath |
| (...skipping 27 matching lines...) Expand all Loading... |
| 38 """Returns a unique directory name suitable for use in Google Storage.""" | 38 """Returns a unique directory name suitable for use in Google Storage.""" |
| 39 return 'gs_utils_manualtest/%d' % random.randint(0, sys.maxint) | 39 return 'gs_utils_manualtest/%d' % random.randint(0, sys.maxint) |
| 40 | 40 |
| 41 | 41 |
| 42 def _test_public_read(): | 42 def _test_public_read(): |
| 43 """Make sure we can read from public files without .boto file credentials.""" | 43 """Make sure we can read from public files without .boto file credentials.""" |
| 44 gs = gs_utils.GSUtils() | 44 gs = gs_utils.GSUtils() |
| 45 gs.list_bucket_contents(bucket=TEST_BUCKET, subdir=None) | 45 gs.list_bucket_contents(bucket=TEST_BUCKET, subdir=None) |
| 46 | 46 |
| 47 | 47 |
| 48 def _test_only_if_modified(): | 48 def _test_upload_if(): |
| 49 """Test only_if_modified param within upload_file().""" | 49 """Test upload_if param within upload_file().""" |
| 50 gs = _get_authenticated_gs_handle() | 50 gs = _get_authenticated_gs_handle() |
| 51 filename = 'filename' | 51 filename = 'filename' |
| 52 remote_dir = _get_unique_posix_dir() | 52 remote_dir = _get_unique_posix_dir() |
| 53 dest_path = posixpath.join(remote_dir, filename) | 53 dest_path = posixpath.join(remote_dir, filename) |
| 54 local_dir = tempfile.mkdtemp() | 54 local_dir = tempfile.mkdtemp() |
| 55 try: | 55 try: |
| 56 # Create a file on local disk, and upload it for the first time. | 56 # Create a file on local disk, and upload it for the first time. |
| 57 local_path = os.path.join(local_dir, filename) | 57 local_path = os.path.join(local_dir, filename) |
| 58 with open(local_path, 'w') as f: | 58 with open(local_path, 'w') as f: |
| 59 f.write('original contents') | 59 f.write('original contents') |
| 60 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, | 60 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, |
| 61 dest_path=dest_path, only_if_modified=True) | 61 dest_path=dest_path, upload_if=gs.UploadIf.IF_NEW) |
| 62 try: | 62 try: |
| 63 # Re-upload the same file one second later, with only_if_modified=False; | 63 # Re-upload the same file, with upload_if=gs.UploadIf.ALWAYS; |
| 64 # the timestamp should change. | 64 # the timestamp should change. |
| 65 old_timestamp = gs.get_last_modified_time( | 65 old_timestamp = gs.get_last_modified_time( |
| 66 bucket=TEST_BUCKET, path=dest_path) | 66 bucket=TEST_BUCKET, path=dest_path) |
| 67 time.sleep(2) | 67 time.sleep(2) |
| 68 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, | 68 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, |
| 69 dest_path=dest_path, only_if_modified=False) | 69 dest_path=dest_path, upload_if=gs.UploadIf.ALWAYS) |
| 70 new_timestamp = gs.get_last_modified_time( | 70 new_timestamp = gs.get_last_modified_time( |
| 71 bucket=TEST_BUCKET, path=dest_path) | 71 bucket=TEST_BUCKET, path=dest_path) |
| 72 assert old_timestamp != new_timestamp, '%s != %s' % ( | 72 assert old_timestamp != new_timestamp, '%s != %s' % ( |
| 73 old_timestamp, new_timestamp) | 73 old_timestamp, new_timestamp) |
| 74 | 74 |
| 75 # Re-upload the same file one second later, with only_if_modified=True; | 75 # Re-upload the same file, with upload_if=gs.UploadIf.IF_MODIFIED; |
| 76 # the timestamp should NOT change. | 76 # the timestamp should NOT change. |
| 77 old_timestamp = new_timestamp | 77 old_timestamp = new_timestamp |
| 78 time.sleep(2) | 78 time.sleep(2) |
| 79 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, | 79 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, |
| 80 dest_path=dest_path, only_if_modified=True) | 80 dest_path=dest_path, upload_if=gs.UploadIf.IF_MODIFIED) |
| 81 new_timestamp = gs.get_last_modified_time( | 81 new_timestamp = gs.get_last_modified_time( |
| 82 bucket=TEST_BUCKET, path=dest_path) | 82 bucket=TEST_BUCKET, path=dest_path) |
| 83 assert old_timestamp == new_timestamp, '%s == %s' % ( | 83 assert old_timestamp == new_timestamp, '%s == %s' % ( |
| 84 old_timestamp, new_timestamp) | 84 old_timestamp, new_timestamp) |
| 85 | 85 |
| 86 # MODIFY and re-upload the file one second later, with | 86 # Modify and re-upload the file, with upload_if=gs.UploadIf.IF_NEW; |
| 87 # only_if_modified=True; the timestamp SHOULD change. | 87 # the timestamp should still not change. |
| 88 old_timestamp = new_timestamp | 88 old_timestamp = new_timestamp |
| 89 with open(local_path, 'w') as f: | 89 with open(local_path, 'w') as f: |
| 90 f.write('modified contents') | 90 f.write('modified contents') |
| 91 time.sleep(2) | 91 time.sleep(2) |
| 92 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, | 92 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, |
| 93 dest_path=dest_path, only_if_modified=True) | 93 dest_path=dest_path, upload_if=gs.UploadIf.IF_NEW) |
| 94 new_timestamp = gs.get_last_modified_time( |
| 95 bucket=TEST_BUCKET, path=dest_path) |
| 96 assert old_timestamp == new_timestamp, '%s == %s' % ( |
| 97 old_timestamp, new_timestamp) |
| 98 |
| 99 # Re-upload the modified file, with upload_if=gs.UploadIf.IF_MODIFIED; |
| 100 # now the timestamp SHOULD change. |
| 101 old_timestamp = new_timestamp |
| 102 time.sleep(2) |
| 103 gs.upload_file(source_path=local_path, dest_bucket=TEST_BUCKET, |
| 104 dest_path=dest_path, upload_if=gs.UploadIf.IF_MODIFIED) |
| 94 new_timestamp = gs.get_last_modified_time( | 105 new_timestamp = gs.get_last_modified_time( |
| 95 bucket=TEST_BUCKET, path=dest_path) | 106 bucket=TEST_BUCKET, path=dest_path) |
| 96 assert old_timestamp != new_timestamp, '%s != %s' % ( | 107 assert old_timestamp != new_timestamp, '%s != %s' % ( |
| 97 old_timestamp, new_timestamp) | 108 old_timestamp, new_timestamp) |
| 98 finally: | 109 finally: |
| 99 # Clean up the remote_dir. | 110 # Clean up the remote_dir. |
| 100 gs.delete_file(bucket=TEST_BUCKET, path=dest_path) | 111 gs.delete_file(bucket=TEST_BUCKET, path=dest_path) |
| 101 finally: | 112 finally: |
| 102 # Clean up the local dir. | 113 # Clean up the local dir. |
| 103 shutil.rmtree(local_dir) | 114 shutil.rmtree(local_dir) |
| (...skipping 153 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 257 assert file_contents == 'contents of %s\n' % filename, ( | 268 assert file_contents == 'contents of %s\n' % filename, ( |
| 258 '%s == "contents of %s\n"' % (file_contents, filename)) | 269 '%s == "contents of %s\n"' % (file_contents, filename)) |
| 259 finally: | 270 finally: |
| 260 shutil.rmtree(local_dest_dir) | 271 shutil.rmtree(local_dest_dir) |
| 261 for filename in filenames: | 272 for filename in filenames: |
| 262 gs.delete_file(bucket=TEST_BUCKET, | 273 gs.delete_file(bucket=TEST_BUCKET, |
| 263 path=posixpath.join(remote_dir, subdir, filename)) | 274 path=posixpath.join(remote_dir, subdir, filename)) |
| 264 | 275 |
| 265 | 276 |
| 266 if __name__ == '__main__': | 277 if __name__ == '__main__': |
| 267 _test_only_if_modified() | 278 _test_upload_if() |
| 268 _test_public_read() | 279 _test_public_read() |
| 269 _test_authenticated_round_trip() | 280 _test_authenticated_round_trip() |
| 270 _test_dir_upload_and_download() | 281 _test_dir_upload_and_download() |
| 271 # TODO(epoger): Add _test_unauthenticated_access() to make sure we raise | 282 # TODO(epoger): Add _test_unauthenticated_access() to make sure we raise |
| 272 # an exception when we try to access without needed credentials. | 283 # an exception when we try to access without needed credentials. |
| OLD | NEW |