| Index: tools/telemetry/third_party/gsutilz/third_party/boto/boto/glacier/utils.py
|
| diff --git a/tools/telemetry/third_party/gsutilz/third_party/boto/boto/glacier/utils.py b/tools/telemetry/third_party/gsutilz/third_party/boto/boto/glacier/utils.py
|
| deleted file mode 100644
|
| index 98847e3f16ddc81681bd0a0f6c43595ab580abe7..0000000000000000000000000000000000000000
|
| --- a/tools/telemetry/third_party/gsutilz/third_party/boto/boto/glacier/utils.py
|
| +++ /dev/null
|
| @@ -1,175 +0,0 @@
|
| -# Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved
|
| -#
|
| -# Permission is hereby granted, free of charge, to any person obtaining a
|
| -# copy of this software and associated documentation files (the
|
| -# "Software"), to deal in the Software without restriction, including
|
| -# without limitation the rights to use, copy, modify, merge, publish, dis-
|
| -# tribute, sublicense, and/or sell copies of the Software, and to permit
|
| -# persons to whom the Software is furnished to do so, subject to the fol-
|
| -# lowing conditions:
|
| -#
|
| -# The above copyright notice and this permission notice shall be included
|
| -# in all copies or substantial portions of the Software.
|
| -#
|
| -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
| -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
|
| -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
| -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
| -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
| -# IN THE SOFTWARE.
|
| -#
|
| -import hashlib
|
| -import math
|
| -import binascii
|
| -
|
| -from boto.compat import six
|
| -
|
| -
|
| -_MEGABYTE = 1024 * 1024
|
| -DEFAULT_PART_SIZE = 4 * _MEGABYTE
|
| -MAXIMUM_NUMBER_OF_PARTS = 10000
|
| -
|
| -
|
| -def minimum_part_size(size_in_bytes, default_part_size=DEFAULT_PART_SIZE):
|
| - """Calculate the minimum part size needed for a multipart upload.
|
| -
|
| - Glacier allows a maximum of 10,000 parts per upload. It also
|
| - states that the maximum archive size is 10,000 * 4 GB, which means
|
| - the part size can range from 1MB to 4GB (provided it is one 1MB
|
| - multiplied by a power of 2).
|
| -
|
| - This function will compute what the minimum part size must be in
|
| - order to upload a file of size ``size_in_bytes``.
|
| -
|
| - It will first check if ``default_part_size`` is sufficient for
|
| - a part size given the ``size_in_bytes``. If this is not the case,
|
| - then the smallest part size than can accomodate a file of size
|
| - ``size_in_bytes`` will be returned.
|
| -
|
| - If the file size is greater than the maximum allowed archive
|
| - size of 10,000 * 4GB, a ``ValueError`` will be raised.
|
| -
|
| - """
|
| - # The default part size (4 MB) will be too small for a very large
|
| - # archive, as there is a limit of 10,000 parts in a multipart upload.
|
| - # This puts the maximum allowed archive size with the default part size
|
| - # at 40,000 MB. We need to do a sanity check on the part size, and find
|
| - # one that works if the default is too small.
|
| - part_size = _MEGABYTE
|
| - if (default_part_size * MAXIMUM_NUMBER_OF_PARTS) < size_in_bytes:
|
| - if size_in_bytes > (4096 * _MEGABYTE * 10000):
|
| - raise ValueError("File size too large: %s" % size_in_bytes)
|
| - min_part_size = size_in_bytes / 10000
|
| - power = 3
|
| - while part_size < min_part_size:
|
| - part_size = math.ldexp(_MEGABYTE, power)
|
| - power += 1
|
| - part_size = int(part_size)
|
| - else:
|
| - part_size = default_part_size
|
| - return part_size
|
| -
|
| -
|
| -def chunk_hashes(bytestring, chunk_size=_MEGABYTE):
|
| - chunk_count = int(math.ceil(len(bytestring) / float(chunk_size)))
|
| - hashes = []
|
| - for i in range(chunk_count):
|
| - start = i * chunk_size
|
| - end = (i + 1) * chunk_size
|
| - hashes.append(hashlib.sha256(bytestring[start:end]).digest())
|
| - if not hashes:
|
| - return [hashlib.sha256(b'').digest()]
|
| - return hashes
|
| -
|
| -
|
| -def tree_hash(fo):
|
| - """
|
| - Given a hash of each 1MB chunk (from chunk_hashes) this will hash
|
| - together adjacent hashes until it ends up with one big one. So a
|
| - tree of hashes.
|
| - """
|
| - hashes = []
|
| - hashes.extend(fo)
|
| - while len(hashes) > 1:
|
| - new_hashes = []
|
| - while True:
|
| - if len(hashes) > 1:
|
| - first = hashes.pop(0)
|
| - second = hashes.pop(0)
|
| - new_hashes.append(hashlib.sha256(first + second).digest())
|
| - elif len(hashes) == 1:
|
| - only = hashes.pop(0)
|
| - new_hashes.append(only)
|
| - else:
|
| - break
|
| - hashes.extend(new_hashes)
|
| - return hashes[0]
|
| -
|
| -
|
| -def compute_hashes_from_fileobj(fileobj, chunk_size=1024 * 1024):
|
| - """Compute the linear and tree hash from a fileobj.
|
| -
|
| - This function will compute the linear/tree hash of a fileobj
|
| - in a single pass through the fileobj.
|
| -
|
| - :param fileobj: A file like object.
|
| -
|
| - :param chunk_size: The size of the chunks to use for the tree
|
| - hash. This is also the buffer size used to read from
|
| - `fileobj`.
|
| -
|
| - :rtype: tuple
|
| - :return: A tuple of (linear_hash, tree_hash). Both hashes
|
| - are returned in hex.
|
| -
|
| - """
|
| - # Python 3+, not binary
|
| - if six.PY3 and hasattr(fileobj, 'mode') and 'b' not in fileobj.mode:
|
| - raise ValueError('File-like object must be opened in binary mode!')
|
| -
|
| - linear_hash = hashlib.sha256()
|
| - chunks = []
|
| - chunk = fileobj.read(chunk_size)
|
| - while chunk:
|
| - # It's possible to get a file-like object that has no mode (checked
|
| - # above) and returns something other than bytes (e.g. str). So here
|
| - # we try to catch that and encode to bytes.
|
| - if not isinstance(chunk, bytes):
|
| - chunk = chunk.encode(getattr(fileobj, 'encoding', '') or 'utf-8')
|
| - linear_hash.update(chunk)
|
| - chunks.append(hashlib.sha256(chunk).digest())
|
| - chunk = fileobj.read(chunk_size)
|
| - if not chunks:
|
| - chunks = [hashlib.sha256(b'').digest()]
|
| - return linear_hash.hexdigest(), bytes_to_hex(tree_hash(chunks))
|
| -
|
| -
|
| -def bytes_to_hex(str_as_bytes):
|
| - return binascii.hexlify(str_as_bytes)
|
| -
|
| -
|
| -def tree_hash_from_str(str_as_bytes):
|
| - """
|
| -
|
| - :type str_as_bytes: str
|
| - :param str_as_bytes: The string for which to compute the tree hash.
|
| -
|
| - :rtype: str
|
| - :return: The computed tree hash, returned as hex.
|
| -
|
| - """
|
| - return bytes_to_hex(tree_hash(chunk_hashes(str_as_bytes)))
|
| -
|
| -
|
| -class ResettingFileSender(object):
|
| - def __init__(self, archive):
|
| - self._archive = archive
|
| - self._starting_offset = archive.tell()
|
| -
|
| - def __call__(self, connection, method, path, body, headers):
|
| - try:
|
| - connection.request(method, path, self._archive, headers)
|
| - return connection.getresponse()
|
| - finally:
|
| - self._archive.seek(self._starting_offset)
|
|
|