Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(2110)

Unified Diff: boto/tests/test_resumable_uploads.py

Issue 8386013: Merging in latest boto. (Closed) Base URL: svn://svn.chromium.org/boto
Patch Set: Redoing vendor drop by deleting and then merging. Created 9 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « boto/tests/test_resumable_downloads.py ('k') | boto/tests/test_s3connection.py » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: boto/tests/test_resumable_uploads.py
diff --git a/boto/tests/test_resumable_uploads.py b/boto/tests/test_resumable_uploads.py
deleted file mode 100755
index da7b0865cadcd6745a36962487acddf5b8230e01..0000000000000000000000000000000000000000
--- a/boto/tests/test_resumable_uploads.py
+++ /dev/null
@@ -1,551 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2010 Google Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish, dis-
-# tribute, sublicense, and/or sell copies of the Software, and to permit
-# persons to whom the Software is furnished to do so, subject to the fol-
-# lowing conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-
-"""
-Tests of resumable uploads.
-"""
-
-import errno
-import getopt
-import os
-import random
-import re
-import shutil
-import socket
-import StringIO
-import sys
-import tempfile
-import time
-import unittest
-
-import boto
-from boto.exception import GSResponseError
-from boto.gs.resumable_upload_handler import ResumableUploadHandler
-from boto.exception import ResumableTransferDisposition
-from boto.exception import ResumableUploadException
-from boto.exception import StorageResponseError
-from boto.tests.cb_test_harnass import CallbackTestHarnass
-
-
-class ResumableUploadTests(unittest.TestCase):
- """
- Resumable upload test suite.
- """
-
- def get_suite_description(self):
- return 'Resumable upload test suite'
-
- @classmethod
- def setUp(cls):
- """
- Creates dst_key needed by all tests.
-
- This method's namingCase is required by the unittest framework.
- """
- cls.dst_key = cls.dst_key_uri.new_key(validate=False)
-
- @classmethod
- def tearDown(cls):
- """
- Deletes any objects or files created by last test run.
-
- This method's namingCase is required by the unittest framework.
- """
- try:
- cls.dst_key_uri.delete_key()
- except GSResponseError:
- # Ignore possible not-found error.
- pass
- # Recursively delete dst dir and then re-create it, so in effect we
- # remove all dirs and files under that directory.
- shutil.rmtree(cls.tmp_dir)
- os.mkdir(cls.tmp_dir)
-
- @staticmethod
- def build_test_input_file(size):
- buf = []
- # I manually construct the random data here instead of calling
- # os.urandom() because I want to constrain the range of data (in
- # this case to 0'..'9') so the test
- # code can easily overwrite part of the StringIO file with
- # known-to-be-different values.
- for i in range(size):
- buf.append(str(random.randint(0, 9)))
- file_as_string = ''.join(buf)
- return (file_as_string, StringIO.StringIO(file_as_string))
-
- @classmethod
- def set_up_class(cls, debug):
- """
- Initializes test suite.
- """
-
- # Use a designated tmpdir prefix to make it easy to find the end of
- # the tmp path.
- cls.tmpdir_prefix = 'tmp_resumable_upload_test'
-
- # Create test source file data.
- cls.empty_src_file_size = 0
- (cls.empty_src_file_as_string, cls.empty_src_file) = (
- cls.build_test_input_file(cls.empty_src_file_size))
- cls.small_src_file_size = 2 * 1024 # 2 KB.
- (cls.small_src_file_as_string, cls.small_src_file) = (
- cls.build_test_input_file(cls.small_src_file_size))
- cls.larger_src_file_size = 500 * 1024 # 500 KB.
- (cls.larger_src_file_as_string, cls.larger_src_file) = (
- cls.build_test_input_file(cls.larger_src_file_size))
- cls.largest_src_file_size = 1024 * 1024 # 1 MB.
- (cls.largest_src_file_as_string, cls.largest_src_file) = (
- cls.build_test_input_file(cls.largest_src_file_size))
-
- # Create temp dir.
- cls.tmp_dir = tempfile.mkdtemp(prefix=cls.tmpdir_prefix)
-
- # Create the test bucket.
- hostname = socket.gethostname().split('.')[0]
- cls.uri_base_str = 'gs://res_upload_test_%s_%s_%s' % (
- hostname, os.getpid(), int(time.time()))
- cls.dst_bucket_uri = boto.storage_uri('%s_dst' %
- cls.uri_base_str, debug=debug)
- cls.dst_bucket_uri.create_bucket()
- cls.dst_key_uri = cls.dst_bucket_uri.clone_replace_name('obj')
-
- cls.tracker_file_name = '%s%suri_tracker' % (cls.tmp_dir, os.sep)
-
- cls.syntactically_invalid_tracker_file_name = (
- '%s%ssynt_invalid_uri_tracker' % (cls.tmp_dir, os.sep))
- f = open(cls.syntactically_invalid_tracker_file_name, 'w')
- f.write('ftp://example.com')
- f.close()
-
- cls.invalid_upload_id = (
- 'http://pub.commondatastorage.googleapis.com/?upload_id='
- 'AyzB2Uo74W4EYxyi5dp_-r68jz8rtbvshsv4TX7srJVkJ57CxTY5Dw2')
- cls.invalid_upload_id_tracker_file_name = (
- '%s%sinvalid_upload_id_tracker' % (cls.tmp_dir, os.sep))
- f = open(cls.invalid_upload_id_tracker_file_name, 'w')
- f.write(cls.invalid_upload_id)
- f.close()
-
- cls.created_test_data = True
-
- @classmethod
- def tear_down_class(cls):
- """
- Deletes bucket and tmp dir created by set_up_class.
- """
- if not hasattr(cls, 'created_test_data'):
- return
- # Call cls.tearDown() in case the tests got interrupted, to ensure
- # dst objects get deleted.
- cls.tearDown()
-
- # Retry (for up to 2 minutes) the bucket gets deleted (it may not
- # the first time round, due to eventual consistency of bucket delete
- # operations).
- for i in range(60):
- try:
- cls.dst_bucket_uri.delete_bucket()
- break
- except StorageResponseError:
- print 'Test bucket (%s) not yet deleted, still trying' % (
- cls.dst_bucket_uri.uri)
- time.sleep(2)
- shutil.rmtree(cls.tmp_dir)
- cls.tmp_dir = tempfile.mkdtemp(prefix=cls.tmpdir_prefix)
-
- def test_non_resumable_upload(self):
- """
- Tests that non-resumable uploads work
- """
- self.dst_key.set_contents_from_file(self.small_src_file)
- self.assertEqual(self.small_src_file_size, self.dst_key.size)
- self.assertEqual(self.small_src_file_as_string,
- self.dst_key.get_contents_as_string())
-
- def test_upload_without_persistent_tracker(self):
- """
- Tests a single resumable upload, with no tracker URI persistence
- """
- res_upload_handler = ResumableUploadHandler()
- self.dst_key.set_contents_from_file(
- self.small_src_file, res_upload_handler=res_upload_handler)
- self.assertEqual(self.small_src_file_size, self.dst_key.size)
- self.assertEqual(self.small_src_file_as_string,
- self.dst_key.get_contents_as_string())
-
- def test_failed_upload_with_persistent_tracker(self):
- """
- Tests that failed resumable upload leaves a correct tracker URI file
- """
- harnass = CallbackTestHarnass()
- res_upload_handler = ResumableUploadHandler(
- tracker_file_name=self.tracker_file_name, num_retries=0)
- try:
- self.dst_key.set_contents_from_file(
- self.small_src_file, cb=harnass.call,
- res_upload_handler=res_upload_handler)
- self.fail('Did not get expected ResumableUploadException')
- except ResumableUploadException, e:
- # We'll get a ResumableUploadException at this point because
- # of CallbackTestHarnass (above). Check that the tracker file was
- # created correctly.
- self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
- self.assertTrue(os.path.exists(self.tracker_file_name))
- f = open(self.tracker_file_name)
- uri_from_file = f.readline().strip()
- f.close()
- self.assertEqual(uri_from_file,
- res_upload_handler.get_tracker_uri())
-
- def test_retryable_exception_recovery(self):
- """
- Tests handling of a retryable exception
- """
- # Test one of the RETRYABLE_EXCEPTIONS.
- exception = ResumableUploadHandler.RETRYABLE_EXCEPTIONS[0]
- harnass = CallbackTestHarnass(exception=exception)
- res_upload_handler = ResumableUploadHandler(num_retries=1)
- self.dst_key.set_contents_from_file(
- self.small_src_file, cb=harnass.call,
- res_upload_handler=res_upload_handler)
- # Ensure uploaded object has correct content.
- self.assertEqual(self.small_src_file_size, self.dst_key.size)
- self.assertEqual(self.small_src_file_as_string,
- self.dst_key.get_contents_as_string())
-
- def test_non_retryable_exception_handling(self):
- """
- Tests a resumable upload that fails with a non-retryable exception
- """
- harnass = CallbackTestHarnass(
- exception=OSError(errno.EACCES, 'Permission denied'))
- res_upload_handler = ResumableUploadHandler(num_retries=1)
- try:
- self.dst_key.set_contents_from_file(
- self.small_src_file, cb=harnass.call,
- res_upload_handler=res_upload_handler)
- self.fail('Did not get expected OSError')
- except OSError, e:
- # Ensure the error was re-raised.
- self.assertEqual(e.errno, 13)
-
- def test_failed_and_restarted_upload_with_persistent_tracker(self):
- """
- Tests resumable upload that fails once and then completes, with tracker
- file
- """
- harnass = CallbackTestHarnass()
- res_upload_handler = ResumableUploadHandler(
- tracker_file_name=self.tracker_file_name, num_retries=1)
- self.dst_key.set_contents_from_file(
- self.small_src_file, cb=harnass.call,
- res_upload_handler=res_upload_handler)
- # Ensure uploaded object has correct content.
- self.assertEqual(self.small_src_file_size, self.dst_key.size)
- self.assertEqual(self.small_src_file_as_string,
- self.dst_key.get_contents_as_string())
- # Ensure tracker file deleted.
- self.assertFalse(os.path.exists(self.tracker_file_name))
-
- def test_multiple_in_process_failures_then_succeed(self):
- """
- Tests resumable upload that fails twice in one process, then completes
- """
- res_upload_handler = ResumableUploadHandler(num_retries=3)
- self.dst_key.set_contents_from_file(
- self.small_src_file, res_upload_handler=res_upload_handler)
- # Ensure uploaded object has correct content.
- self.assertEqual(self.small_src_file_size, self.dst_key.size)
- self.assertEqual(self.small_src_file_as_string,
- self.dst_key.get_contents_as_string())
-
- def test_multiple_in_process_failures_then_succeed_with_tracker_file(self):
- """
- Tests resumable upload that fails completely in one process,
- then when restarted completes, using a tracker file
- """
- # Set up test harnass that causes more failures than a single
- # ResumableUploadHandler instance will handle, writing enough data
- # before the first failure that some of it survives that process run.
- harnass = CallbackTestHarnass(
- fail_after_n_bytes=self.larger_src_file_size/2, num_times_to_fail=2)
- res_upload_handler = ResumableUploadHandler(
- tracker_file_name=self.tracker_file_name, num_retries=1)
- try:
- self.dst_key.set_contents_from_file(
- self.larger_src_file, cb=harnass.call,
- res_upload_handler=res_upload_handler)
- self.fail('Did not get expected ResumableUploadException')
- except ResumableUploadException, e:
- self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
- # Ensure a tracker file survived.
- self.assertTrue(os.path.exists(self.tracker_file_name))
- # Try it one more time; this time should succeed.
- self.dst_key.set_contents_from_file(
- self.larger_src_file, cb=harnass.call,
- res_upload_handler=res_upload_handler)
- self.assertEqual(self.larger_src_file_size, self.dst_key.size)
- self.assertEqual(self.larger_src_file_as_string,
- self.dst_key.get_contents_as_string())
- self.assertFalse(os.path.exists(self.tracker_file_name))
- # Ensure some of the file was uploaded both before and after failure.
- self.assertTrue(len(harnass.transferred_seq_before_first_failure) > 1
- and
- len(harnass.transferred_seq_after_first_failure) > 1)
-
- def test_upload_with_inital_partial_upload_before_failure(self):
- """
- Tests resumable upload that successfully uploads some content
- before it fails, then restarts and completes
- """
- # Set up harnass to fail upload after several hundred KB so upload
- # server will have saved something before we retry.
- harnass = CallbackTestHarnass(
- fail_after_n_bytes=self.larger_src_file_size/2)
- res_upload_handler = ResumableUploadHandler(num_retries=1)
- self.dst_key.set_contents_from_file(
- self.larger_src_file, cb=harnass.call,
- res_upload_handler=res_upload_handler)
- # Ensure uploaded object has correct content.
- self.assertEqual(self.larger_src_file_size, self.dst_key.size)
- self.assertEqual(self.larger_src_file_as_string,
- self.dst_key.get_contents_as_string())
- # Ensure some of the file was uploaded both before and after failure.
- self.assertTrue(len(harnass.transferred_seq_before_first_failure) > 1
- and
- len(harnass.transferred_seq_after_first_failure) > 1)
-
- def test_empty_file_upload(self):
- """
- Tests uploading an empty file (exercises boundary conditions).
- """
- res_upload_handler = ResumableUploadHandler()
- self.dst_key.set_contents_from_file(
- self.empty_src_file, res_upload_handler=res_upload_handler)
- self.assertEqual(0, self.dst_key.size)
-
- def test_upload_retains_metadata(self):
- """
- Tests that resumable upload correctly sets passed metadata
- """
- res_upload_handler = ResumableUploadHandler()
- headers = {'Content-Type' : 'text/plain', 'Content-Encoding' : 'gzip',
- 'x-goog-meta-abc' : 'my meta', 'x-goog-acl' : 'public-read'}
- self.dst_key.set_contents_from_file(
- self.small_src_file, headers=headers,
- res_upload_handler=res_upload_handler)
- self.assertEqual(self.small_src_file_size, self.dst_key.size)
- self.assertEqual(self.small_src_file_as_string,
- self.dst_key.get_contents_as_string())
- self.dst_key.open_read()
- self.assertEqual('text/plain', self.dst_key.content_type)
- self.assertEqual('gzip', self.dst_key.content_encoding)
- self.assertTrue('abc' in self.dst_key.metadata)
- self.assertEqual('my meta', str(self.dst_key.metadata['abc']))
- acl = self.dst_key.get_acl()
- for entry in acl.entries.entry_list:
- if str(entry.scope) == '<AllUsers>':
- self.assertEqual('READ', str(acl.entries.entry_list[1].permission))
- return
- self.fail('No <AllUsers> scope found')
-
- def test_upload_with_file_size_change_between_starts(self):
- """
- Tests resumable upload on a file that changes sizes between inital
- upload start and restart
- """
- harnass = CallbackTestHarnass(
- fail_after_n_bytes=self.larger_src_file_size/2)
- # Set up first process' ResumableUploadHandler not to do any
- # retries (initial upload request will establish expected size to
- # upload server).
- res_upload_handler = ResumableUploadHandler(
- tracker_file_name=self.tracker_file_name, num_retries=0)
- try:
- self.dst_key.set_contents_from_file(
- self.larger_src_file, cb=harnass.call,
- res_upload_handler=res_upload_handler)
- self.fail('Did not get expected ResumableUploadException')
- except ResumableUploadException, e:
- self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
- # Ensure a tracker file survived.
- self.assertTrue(os.path.exists(self.tracker_file_name))
- # Try it again, this time with different size source file.
- # Wait 1 second between retry attempts, to give upload server a
- # chance to save state so it can respond to changed file size with
- # 500 response in the next attempt.
- time.sleep(1)
- try:
- self.dst_key.set_contents_from_file(
- self.largest_src_file, res_upload_handler=res_upload_handler)
- self.fail('Did not get expected ResumableUploadException')
- except ResumableUploadException, e:
- self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
- self.assertNotEqual(
- e.message.find('attempt to upload a different size file'), -1)
-
- def test_upload_with_file_size_change_during_upload(self):
- """
- Tests resumable upload on a file that changes sizes while upload
- in progress
- """
- # Create a file we can change during the upload.
- test_file_size = 500 * 1024 # 500 KB.
- test_file = self.build_test_input_file(test_file_size)[1]
- harnass = CallbackTestHarnass(fp_to_change=test_file,
- fp_change_pos=test_file_size)
- res_upload_handler = ResumableUploadHandler(num_retries=1)
- try:
- self.dst_key.set_contents_from_file(
- test_file, cb=harnass.call,
- res_upload_handler=res_upload_handler)
- self.fail('Did not get expected ResumableUploadException')
- except ResumableUploadException, e:
- self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
- self.assertNotEqual(
- e.message.find('File changed during upload'), -1)
-
- def test_upload_with_file_content_change_during_upload(self):
- """
- Tests resumable upload on a file that changes one byte of content
- (so, size stays the same) while upload in progress
- """
- test_file_size = 500 * 1024 # 500 KB.
- test_file = self.build_test_input_file(test_file_size)[1]
- harnass = CallbackTestHarnass(fail_after_n_bytes=test_file_size/2,
- fp_to_change=test_file,
- # Writing at file_size-5 won't change file
- # size because CallbackTestHarnass only
- # writes 3 bytes.
- fp_change_pos=test_file_size-5)
- res_upload_handler = ResumableUploadHandler(num_retries=1)
- try:
- self.dst_key.set_contents_from_file(
- test_file, cb=harnass.call,
- res_upload_handler=res_upload_handler)
- self.fail('Did not get expected ResumableUploadException')
- except ResumableUploadException, e:
- self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
- # Ensure the file size didn't change.
- test_file.seek(0, os.SEEK_END)
- self.assertEqual(test_file_size, test_file.tell())
- self.assertNotEqual(
- e.message.find('md5 signature doesn\'t match etag'), -1)
- # Ensure the bad data wasn't left around.
- all_keys = self.dst_key_uri.get_all_keys()
- self.assertEqual(0, len(all_keys))
-
- def test_upload_with_content_length_header_set(self):
- """
- Tests resumable upload on a file when the user supplies a
- Content-Length header. This is used by gsutil, for example,
- to set the content length when gzipping a file.
- """
- res_upload_handler = ResumableUploadHandler()
- try:
- self.dst_key.set_contents_from_file(
- self.small_src_file, res_upload_handler=res_upload_handler,
- headers={'Content-Length' : self.small_src_file_size})
- self.fail('Did not get expected ResumableUploadException')
- except ResumableUploadException, e:
- self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
- self.assertNotEqual(
- e.message.find('Attempt to specify Content-Length header'), -1)
-
- def test_upload_with_syntactically_invalid_tracker_uri(self):
- """
- Tests resumable upload with a syntactically invalid tracker URI
- """
- res_upload_handler = ResumableUploadHandler(
- tracker_file_name=self.syntactically_invalid_tracker_file_name)
- # An error should be printed about the invalid URI, but then it
- # should run the update successfully.
- self.dst_key.set_contents_from_file(
- self.small_src_file, res_upload_handler=res_upload_handler)
- self.assertEqual(self.small_src_file_size, self.dst_key.size)
- self.assertEqual(self.small_src_file_as_string,
- self.dst_key.get_contents_as_string())
-
- def test_upload_with_invalid_upload_id_in_tracker_file(self):
- """
- Tests resumable upload with invalid upload ID
- """
- res_upload_handler = ResumableUploadHandler(
- tracker_file_name=self.invalid_upload_id_tracker_file_name)
- # An error should occur, but then the tracker URI should be
- # regenerated and the the update should succeed.
- self.dst_key.set_contents_from_file(
- self.small_src_file, res_upload_handler=res_upload_handler)
- self.assertEqual(self.small_src_file_size, self.dst_key.size)
- self.assertEqual(self.small_src_file_as_string,
- self.dst_key.get_contents_as_string())
- self.assertNotEqual(self.invalid_upload_id,
- res_upload_handler.get_tracker_uri())
-
- def test_upload_with_unwritable_tracker_file(self):
- """
- Tests resumable upload with an unwritable tracker file
- """
- # Make dir where tracker_file lives temporarily unwritable.
- save_mod = os.stat(self.tmp_dir).st_mode
- try:
- os.chmod(self.tmp_dir, 0)
- res_upload_handler = ResumableUploadHandler(
- tracker_file_name=self.tracker_file_name)
- except ResumableUploadException, e:
- self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
- self.assertNotEqual(
- e.message.find('Couldn\'t write URI tracker file'), -1)
- finally:
- # Restore original protection of dir where tracker_file lives.
- os.chmod(self.tmp_dir, save_mod)
-
-if __name__ == '__main__':
- if sys.version_info[:3] < (2, 5, 1):
- sys.exit('These tests must be run on at least Python 2.5.1\n')
-
- # Use -d to see more HTTP protocol detail during tests.
- debug = 0
- opts, args = getopt.getopt(sys.argv[1:], 'd', ['debug'])
- for o, a in opts:
- if o in ('-d', '--debug'):
- debug = 2
-
- test_loader = unittest.TestLoader()
- test_loader.testMethodPrefix = 'test_'
- suite = test_loader.loadTestsFromTestCase(ResumableUploadTests)
- # Seems like there should be a cleaner way to find the test_class.
- test_class = suite.__getattribute__('_tests')[0]
- # We call set_up_class() and tear_down_class() ourselves because we
- # don't assume the user has Python 2.7 (which supports classmethods
- # that do it, with camelCase versions of these names).
- try:
- print 'Setting up %s...' % test_class.get_suite_description()
- test_class.set_up_class(debug)
- print 'Running %s...' % test_class.get_suite_description()
- unittest.TextTestRunner(verbosity=2).run(suite)
- finally:
- print 'Cleaning up after %s...' % test_class.get_suite_description()
- test_class.tear_down_class()
- print ''
« no previous file with comments | « boto/tests/test_resumable_downloads.py ('k') | boto/tests/test_s3connection.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698