Index: tools/telemetry/third_party/gsutil/third_party/boto/boto/glacier/job.py |
diff --git a/tools/telemetry/third_party/gsutil/third_party/boto/boto/glacier/job.py b/tools/telemetry/third_party/gsutil/third_party/boto/boto/glacier/job.py |
deleted file mode 100644 |
index 33e66a196c49b7e7ba0af2139e0e7da0633ea50b..0000000000000000000000000000000000000000 |
--- a/tools/telemetry/third_party/gsutil/third_party/boto/boto/glacier/job.py |
+++ /dev/null |
@@ -1,177 +0,0 @@ |
-# -*- coding: utf-8 -*- |
-# Copyright (c) 2012 Thomas Parslow http://almostobsolete.net/ |
-# |
-# Permission is hereby granted, free of charge, to any person obtaining a |
-# copy of this software and associated documentation files (the |
-# "Software"), to deal in the Software without restriction, including |
-# without limitation the rights to use, copy, modify, merge, publish, dis- |
-# tribute, sublicense, and/or sell copies of the Software, and to permit |
-# persons to whom the Software is furnished to do so, subject to the fol- |
-# lowing conditions: |
-# |
-# The above copyright notice and this permission notice shall be included |
-# in all copies or substantial portions of the Software. |
-# |
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS |
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- |
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT |
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, |
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
-# IN THE SOFTWARE. |
-# |
-import math |
-import socket |
- |
-from boto.glacier.exceptions import TreeHashDoesNotMatchError, \ |
- DownloadArchiveError |
-from boto.glacier.utils import tree_hash_from_str |
- |
- |
-class Job(object): |
- |
- DefaultPartSize = 4 * 1024 * 1024 |
- |
- ResponseDataElements = (('Action', 'action', None), |
- ('ArchiveId', 'archive_id', None), |
- ('ArchiveSizeInBytes', 'archive_size', 0), |
- ('Completed', 'completed', False), |
- ('CompletionDate', 'completion_date', None), |
- ('CreationDate', 'creation_date', None), |
- ('InventorySizeInBytes', 'inventory_size', 0), |
- ('JobDescription', 'description', None), |
- ('JobId', 'id', None), |
- ('SHA256TreeHash', 'sha256_treehash', None), |
- ('SNSTopic', 'sns_topic', None), |
- ('StatusCode', 'status_code', None), |
- ('StatusMessage', 'status_message', None), |
- ('VaultARN', 'arn', None)) |
- |
- def __init__(self, vault, response_data=None): |
- self.vault = vault |
- if response_data: |
- for response_name, attr_name, default in self.ResponseDataElements: |
- setattr(self, attr_name, response_data[response_name]) |
- else: |
- for response_name, attr_name, default in self.ResponseDataElements: |
- setattr(self, attr_name, default) |
- |
- def __repr__(self): |
- return 'Job(%s)' % self.arn |
- |
- def get_output(self, byte_range=None, validate_checksum=False): |
- """ |
- This operation downloads the output of the job. Depending on |
- the job type you specified when you initiated the job, the |
- output will be either the content of an archive or a vault |
- inventory. |
- |
- You can download all the job output or download a portion of |
- the output by specifying a byte range. In the case of an |
- archive retrieval job, depending on the byte range you |
- specify, Amazon Glacier returns the checksum for the portion |
- of the data. You can compute the checksum on the client and |
- verify that the values match to ensure the portion you |
- downloaded is the correct data. |
- |
- :type byte_range: tuple |
- :param range: A tuple of integer specifying the slice (in bytes) |
- of the archive you want to receive |
- |
- :type validate_checksum: bool |
- :param validate_checksum: Specify whether or not to validate |
- the associate tree hash. If the response does not contain |
- a TreeHash, then no checksum will be verified. |
- |
- """ |
- response = self.vault.layer1.get_job_output(self.vault.name, |
- self.id, |
- byte_range) |
- if validate_checksum and 'TreeHash' in response: |
- data = response.read() |
- actual_tree_hash = tree_hash_from_str(data) |
- if response['TreeHash'] != actual_tree_hash: |
- raise TreeHashDoesNotMatchError( |
- "The calculated tree hash %s does not match the " |
- "expected tree hash %s for the byte range %s" % ( |
- actual_tree_hash, response['TreeHash'], byte_range)) |
- return response |
- |
- def _calc_num_chunks(self, chunk_size): |
- return int(math.ceil(self.archive_size / float(chunk_size))) |
- |
- def download_to_file(self, filename, chunk_size=DefaultPartSize, |
- verify_hashes=True, retry_exceptions=(socket.error,)): |
- """Download an archive to a file by name. |
- |
- :type filename: str |
- :param filename: The name of the file where the archive |
- contents will be saved. |
- |
- :type chunk_size: int |
- :param chunk_size: The chunk size to use when downloading |
- the archive. |
- |
- :type verify_hashes: bool |
- :param verify_hashes: Indicates whether or not to verify |
- the tree hashes for each downloaded chunk. |
- |
- """ |
- num_chunks = self._calc_num_chunks(chunk_size) |
- with open(filename, 'wb') as output_file: |
- self._download_to_fileob(output_file, num_chunks, chunk_size, |
- verify_hashes, retry_exceptions) |
- |
- def download_to_fileobj(self, output_file, chunk_size=DefaultPartSize, |
- verify_hashes=True, |
- retry_exceptions=(socket.error,)): |
- """Download an archive to a file object. |
- |
- :type output_file: file |
- :param output_file: The file object where the archive |
- contents will be saved. |
- |
- :type chunk_size: int |
- :param chunk_size: The chunk size to use when downloading |
- the archive. |
- |
- :type verify_hashes: bool |
- :param verify_hashes: Indicates whether or not to verify |
- the tree hashes for each downloaded chunk. |
- |
- """ |
- num_chunks = self._calc_num_chunks(chunk_size) |
- self._download_to_fileob(output_file, num_chunks, chunk_size, |
- verify_hashes, retry_exceptions) |
- |
- def _download_to_fileob(self, fileobj, num_chunks, chunk_size, verify_hashes, |
- retry_exceptions): |
- for i in range(num_chunks): |
- byte_range = ((i * chunk_size), ((i + 1) * chunk_size) - 1) |
- data, expected_tree_hash = self._download_byte_range( |
- byte_range, retry_exceptions) |
- if verify_hashes: |
- actual_tree_hash = tree_hash_from_str(data) |
- if expected_tree_hash != actual_tree_hash: |
- raise TreeHashDoesNotMatchError( |
- "The calculated tree hash %s does not match the " |
- "expected tree hash %s for the byte range %s" % ( |
- actual_tree_hash, expected_tree_hash, byte_range)) |
- fileobj.write(data) |
- |
- def _download_byte_range(self, byte_range, retry_exceptions): |
- # You can occasionally get socket.errors when downloading |
- # chunks from Glacier, so each chunk can be retried up |
- # to 5 times. |
- for _ in range(5): |
- try: |
- response = self.get_output(byte_range) |
- data = response.read() |
- expected_tree_hash = response['TreeHash'] |
- return data, expected_tree_hash |
- except retry_exceptions as e: |
- continue |
- else: |
- raise DownloadArchiveError("There was an error downloading" |
- "byte range %s: %s" % (byte_range, |
- e)) |