| Index: tools/telemetry/third_party/gsutilz/third_party/boto/boto/glacier/job.py
|
| diff --git a/tools/telemetry/third_party/gsutilz/third_party/boto/boto/glacier/job.py b/tools/telemetry/third_party/gsutilz/third_party/boto/boto/glacier/job.py
|
| deleted file mode 100644
|
| index 33e66a196c49b7e7ba0af2139e0e7da0633ea50b..0000000000000000000000000000000000000000
|
| --- a/tools/telemetry/third_party/gsutilz/third_party/boto/boto/glacier/job.py
|
| +++ /dev/null
|
| @@ -1,177 +0,0 @@
|
| -# -*- coding: utf-8 -*-
|
| -# Copyright (c) 2012 Thomas Parslow http://almostobsolete.net/
|
| -#
|
| -# Permission is hereby granted, free of charge, to any person obtaining a
|
| -# copy of this software and associated documentation files (the
|
| -# "Software"), to deal in the Software without restriction, including
|
| -# without limitation the rights to use, copy, modify, merge, publish, dis-
|
| -# tribute, sublicense, and/or sell copies of the Software, and to permit
|
| -# persons to whom the Software is furnished to do so, subject to the fol-
|
| -# lowing conditions:
|
| -#
|
| -# The above copyright notice and this permission notice shall be included
|
| -# in all copies or substantial portions of the Software.
|
| -#
|
| -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
| -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
|
| -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
| -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
| -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
| -# IN THE SOFTWARE.
|
| -#
|
| -import math
|
| -import socket
|
| -
|
| -from boto.glacier.exceptions import TreeHashDoesNotMatchError, \
|
| - DownloadArchiveError
|
| -from boto.glacier.utils import tree_hash_from_str
|
| -
|
| -
|
| -class Job(object):
|
| -
|
| - DefaultPartSize = 4 * 1024 * 1024
|
| -
|
| - ResponseDataElements = (('Action', 'action', None),
|
| - ('ArchiveId', 'archive_id', None),
|
| - ('ArchiveSizeInBytes', 'archive_size', 0),
|
| - ('Completed', 'completed', False),
|
| - ('CompletionDate', 'completion_date', None),
|
| - ('CreationDate', 'creation_date', None),
|
| - ('InventorySizeInBytes', 'inventory_size', 0),
|
| - ('JobDescription', 'description', None),
|
| - ('JobId', 'id', None),
|
| - ('SHA256TreeHash', 'sha256_treehash', None),
|
| - ('SNSTopic', 'sns_topic', None),
|
| - ('StatusCode', 'status_code', None),
|
| - ('StatusMessage', 'status_message', None),
|
| - ('VaultARN', 'arn', None))
|
| -
|
| - def __init__(self, vault, response_data=None):
|
| - self.vault = vault
|
| - if response_data:
|
| - for response_name, attr_name, default in self.ResponseDataElements:
|
| - setattr(self, attr_name, response_data[response_name])
|
| - else:
|
| - for response_name, attr_name, default in self.ResponseDataElements:
|
| - setattr(self, attr_name, default)
|
| -
|
| - def __repr__(self):
|
| - return 'Job(%s)' % self.arn
|
| -
|
| - def get_output(self, byte_range=None, validate_checksum=False):
|
| - """
|
| - This operation downloads the output of the job. Depending on
|
| - the job type you specified when you initiated the job, the
|
| - output will be either the content of an archive or a vault
|
| - inventory.
|
| -
|
| - You can download all the job output or download a portion of
|
| - the output by specifying a byte range. In the case of an
|
| - archive retrieval job, depending on the byte range you
|
| - specify, Amazon Glacier returns the checksum for the portion
|
| - of the data. You can compute the checksum on the client and
|
| - verify that the values match to ensure the portion you
|
| - downloaded is the correct data.
|
| -
|
| - :type byte_range: tuple
|
| - :param range: A tuple of integer specifying the slice (in bytes)
|
| - of the archive you want to receive
|
| -
|
| - :type validate_checksum: bool
|
| - :param validate_checksum: Specify whether or not to validate
|
| - the associate tree hash. If the response does not contain
|
| - a TreeHash, then no checksum will be verified.
|
| -
|
| - """
|
| - response = self.vault.layer1.get_job_output(self.vault.name,
|
| - self.id,
|
| - byte_range)
|
| - if validate_checksum and 'TreeHash' in response:
|
| - data = response.read()
|
| - actual_tree_hash = tree_hash_from_str(data)
|
| - if response['TreeHash'] != actual_tree_hash:
|
| - raise TreeHashDoesNotMatchError(
|
| - "The calculated tree hash %s does not match the "
|
| - "expected tree hash %s for the byte range %s" % (
|
| - actual_tree_hash, response['TreeHash'], byte_range))
|
| - return response
|
| -
|
| - def _calc_num_chunks(self, chunk_size):
|
| - return int(math.ceil(self.archive_size / float(chunk_size)))
|
| -
|
| - def download_to_file(self, filename, chunk_size=DefaultPartSize,
|
| - verify_hashes=True, retry_exceptions=(socket.error,)):
|
| - """Download an archive to a file by name.
|
| -
|
| - :type filename: str
|
| - :param filename: The name of the file where the archive
|
| - contents will be saved.
|
| -
|
| - :type chunk_size: int
|
| - :param chunk_size: The chunk size to use when downloading
|
| - the archive.
|
| -
|
| - :type verify_hashes: bool
|
| - :param verify_hashes: Indicates whether or not to verify
|
| - the tree hashes for each downloaded chunk.
|
| -
|
| - """
|
| - num_chunks = self._calc_num_chunks(chunk_size)
|
| - with open(filename, 'wb') as output_file:
|
| - self._download_to_fileob(output_file, num_chunks, chunk_size,
|
| - verify_hashes, retry_exceptions)
|
| -
|
| - def download_to_fileobj(self, output_file, chunk_size=DefaultPartSize,
|
| - verify_hashes=True,
|
| - retry_exceptions=(socket.error,)):
|
| - """Download an archive to a file object.
|
| -
|
| - :type output_file: file
|
| - :param output_file: The file object where the archive
|
| - contents will be saved.
|
| -
|
| - :type chunk_size: int
|
| - :param chunk_size: The chunk size to use when downloading
|
| - the archive.
|
| -
|
| - :type verify_hashes: bool
|
| - :param verify_hashes: Indicates whether or not to verify
|
| - the tree hashes for each downloaded chunk.
|
| -
|
| - """
|
| - num_chunks = self._calc_num_chunks(chunk_size)
|
| - self._download_to_fileob(output_file, num_chunks, chunk_size,
|
| - verify_hashes, retry_exceptions)
|
| -
|
| - def _download_to_fileob(self, fileobj, num_chunks, chunk_size, verify_hashes,
|
| - retry_exceptions):
|
| - for i in range(num_chunks):
|
| - byte_range = ((i * chunk_size), ((i + 1) * chunk_size) - 1)
|
| - data, expected_tree_hash = self._download_byte_range(
|
| - byte_range, retry_exceptions)
|
| - if verify_hashes:
|
| - actual_tree_hash = tree_hash_from_str(data)
|
| - if expected_tree_hash != actual_tree_hash:
|
| - raise TreeHashDoesNotMatchError(
|
| - "The calculated tree hash %s does not match the "
|
| - "expected tree hash %s for the byte range %s" % (
|
| - actual_tree_hash, expected_tree_hash, byte_range))
|
| - fileobj.write(data)
|
| -
|
| - def _download_byte_range(self, byte_range, retry_exceptions):
|
| - # You can occasionally get socket.errors when downloading
|
| - # chunks from Glacier, so each chunk can be retried up
|
| - # to 5 times.
|
| - for _ in range(5):
|
| - try:
|
| - response = self.get_output(byte_range)
|
| - data = response.read()
|
| - expected_tree_hash = response['TreeHash']
|
| - return data, expected_tree_hash
|
| - except retry_exceptions as e:
|
| - continue
|
| - else:
|
| - raise DownloadArchiveError("There was an error downloading"
|
| - "byte range %s: %s" % (byte_range,
|
| - e))
|
|
|