Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1707)

Unified Diff: tools/telemetry/third_party/gsutil/third_party/boto/boto/glacier/utils.py

Issue 1260493004: Revert "Add gsutil 4.13 to telemetry/third_party" (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 5 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: tools/telemetry/third_party/gsutil/third_party/boto/boto/glacier/utils.py
diff --git a/tools/telemetry/third_party/gsutil/third_party/boto/boto/glacier/utils.py b/tools/telemetry/third_party/gsutil/third_party/boto/boto/glacier/utils.py
deleted file mode 100644
index 98847e3f16ddc81681bd0a0f6c43595ab580abe7..0000000000000000000000000000000000000000
--- a/tools/telemetry/third_party/gsutil/third_party/boto/boto/glacier/utils.py
+++ /dev/null
@@ -1,175 +0,0 @@
-# Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish, dis-
-# tribute, sublicense, and/or sell copies of the Software, and to permit
-# persons to whom the Software is furnished to do so, subject to the fol-
-# lowing conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-#
-import hashlib
-import math
-import binascii
-
-from boto.compat import six
-
-
-_MEGABYTE = 1024 * 1024
-DEFAULT_PART_SIZE = 4 * _MEGABYTE
-MAXIMUM_NUMBER_OF_PARTS = 10000
-
-
-def minimum_part_size(size_in_bytes, default_part_size=DEFAULT_PART_SIZE):
- """Calculate the minimum part size needed for a multipart upload.
-
- Glacier allows a maximum of 10,000 parts per upload. It also
- states that the maximum archive size is 10,000 * 4 GB, which means
- the part size can range from 1MB to 4GB (provided it is one 1MB
- multiplied by a power of 2).
-
- This function will compute what the minimum part size must be in
- order to upload a file of size ``size_in_bytes``.
-
- It will first check if ``default_part_size`` is sufficient for
- a part size given the ``size_in_bytes``. If this is not the case,
- then the smallest part size than can accomodate a file of size
- ``size_in_bytes`` will be returned.
-
- If the file size is greater than the maximum allowed archive
- size of 10,000 * 4GB, a ``ValueError`` will be raised.
-
- """
- # The default part size (4 MB) will be too small for a very large
- # archive, as there is a limit of 10,000 parts in a multipart upload.
- # This puts the maximum allowed archive size with the default part size
- # at 40,000 MB. We need to do a sanity check on the part size, and find
- # one that works if the default is too small.
- part_size = _MEGABYTE
- if (default_part_size * MAXIMUM_NUMBER_OF_PARTS) < size_in_bytes:
- if size_in_bytes > (4096 * _MEGABYTE * 10000):
- raise ValueError("File size too large: %s" % size_in_bytes)
- min_part_size = size_in_bytes / 10000
- power = 3
- while part_size < min_part_size:
- part_size = math.ldexp(_MEGABYTE, power)
- power += 1
- part_size = int(part_size)
- else:
- part_size = default_part_size
- return part_size
-
-
-def chunk_hashes(bytestring, chunk_size=_MEGABYTE):
- chunk_count = int(math.ceil(len(bytestring) / float(chunk_size)))
- hashes = []
- for i in range(chunk_count):
- start = i * chunk_size
- end = (i + 1) * chunk_size
- hashes.append(hashlib.sha256(bytestring[start:end]).digest())
- if not hashes:
- return [hashlib.sha256(b'').digest()]
- return hashes
-
-
-def tree_hash(fo):
- """
- Given a hash of each 1MB chunk (from chunk_hashes) this will hash
- together adjacent hashes until it ends up with one big one. So a
- tree of hashes.
- """
- hashes = []
- hashes.extend(fo)
- while len(hashes) > 1:
- new_hashes = []
- while True:
- if len(hashes) > 1:
- first = hashes.pop(0)
- second = hashes.pop(0)
- new_hashes.append(hashlib.sha256(first + second).digest())
- elif len(hashes) == 1:
- only = hashes.pop(0)
- new_hashes.append(only)
- else:
- break
- hashes.extend(new_hashes)
- return hashes[0]
-
-
-def compute_hashes_from_fileobj(fileobj, chunk_size=1024 * 1024):
- """Compute the linear and tree hash from a fileobj.
-
- This function will compute the linear/tree hash of a fileobj
- in a single pass through the fileobj.
-
- :param fileobj: A file like object.
-
- :param chunk_size: The size of the chunks to use for the tree
- hash. This is also the buffer size used to read from
- `fileobj`.
-
- :rtype: tuple
- :return: A tuple of (linear_hash, tree_hash). Both hashes
- are returned in hex.
-
- """
- # Python 3+, not binary
- if six.PY3 and hasattr(fileobj, 'mode') and 'b' not in fileobj.mode:
- raise ValueError('File-like object must be opened in binary mode!')
-
- linear_hash = hashlib.sha256()
- chunks = []
- chunk = fileobj.read(chunk_size)
- while chunk:
- # It's possible to get a file-like object that has no mode (checked
- # above) and returns something other than bytes (e.g. str). So here
- # we try to catch that and encode to bytes.
- if not isinstance(chunk, bytes):
- chunk = chunk.encode(getattr(fileobj, 'encoding', '') or 'utf-8')
- linear_hash.update(chunk)
- chunks.append(hashlib.sha256(chunk).digest())
- chunk = fileobj.read(chunk_size)
- if not chunks:
- chunks = [hashlib.sha256(b'').digest()]
- return linear_hash.hexdigest(), bytes_to_hex(tree_hash(chunks))
-
-
-def bytes_to_hex(str_as_bytes):
- return binascii.hexlify(str_as_bytes)
-
-
-def tree_hash_from_str(str_as_bytes):
- """
-
- :type str_as_bytes: str
- :param str_as_bytes: The string for which to compute the tree hash.
-
- :rtype: str
- :return: The computed tree hash, returned as hex.
-
- """
- return bytes_to_hex(tree_hash(chunk_hashes(str_as_bytes)))
-
-
-class ResettingFileSender(object):
- def __init__(self, archive):
- self._archive = archive
- self._starting_offset = archive.tell()
-
- def __call__(self, connection, method, path, body, headers):
- try:
- connection.request(method, path, self._archive, headers)
- return connection.getresponse()
- finally:
- self._archive.seek(self._starting_offset)

Powered by Google App Engine
This is Rietveld 408576698