Index: tools/telemetry/third_party/gsutil/third_party/boto/boto/gs/key.py |
diff --git a/tools/telemetry/third_party/gsutil/third_party/boto/boto/gs/key.py b/tools/telemetry/third_party/gsutil/third_party/boto/boto/gs/key.py |
deleted file mode 100644 |
index c4fcf01f091582fede729e7c647d5d749bf4a7c6..0000000000000000000000000000000000000000 |
--- a/tools/telemetry/third_party/gsutil/third_party/boto/boto/gs/key.py |
+++ /dev/null |
@@ -1,946 +0,0 @@ |
-# Copyright 2010 Google Inc. |
-# |
-# Permission is hereby granted, free of charge, to any person obtaining a |
-# copy of this software and associated documentation files (the |
-# "Software"), to deal in the Software without restriction, including |
-# without limitation the rights to use, copy, modify, merge, publish, dis- |
-# tribute, sublicense, and/or sell copies of the Software, and to permit |
-# persons to whom the Software is furnished to do so, subject to the fol- |
-# lowing conditions: |
-# |
-# The above copyright notice and this permission notice shall be included |
-# in all copies or substantial portions of the Software. |
-# |
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS |
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- |
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT |
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, |
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
-# IN THE SOFTWARE. |
- |
-import base64 |
-import binascii |
-import os |
-import re |
- |
-from boto.compat import StringIO |
-from boto.exception import BotoClientError |
-from boto.s3.key import Key as S3Key |
-from boto.s3.keyfile import KeyFile |
-from boto.utils import compute_hash |
-from boto.utils import get_utf8_value |
- |
-class Key(S3Key): |
- """ |
- Represents a key (object) in a GS bucket. |
- |
- :ivar bucket: The parent :class:`boto.gs.bucket.Bucket`. |
- :ivar name: The name of this Key object. |
- :ivar metadata: A dictionary containing user metadata that you |
- wish to store with the object or that has been retrieved from |
- an existing object. |
- :ivar cache_control: The value of the `Cache-Control` HTTP header. |
- :ivar content_type: The value of the `Content-Type` HTTP header. |
- :ivar content_encoding: The value of the `Content-Encoding` HTTP header. |
- :ivar content_disposition: The value of the `Content-Disposition` HTTP |
- header. |
- :ivar content_language: The value of the `Content-Language` HTTP header. |
- :ivar etag: The `etag` associated with this object. |
- :ivar last_modified: The string timestamp representing the last |
- time this object was modified in GS. |
- :ivar owner: The ID of the owner of this object. |
- :ivar storage_class: The storage class of the object. Currently, one of: |
- STANDARD | DURABLE_REDUCED_AVAILABILITY. |
- :ivar md5: The MD5 hash of the contents of the object. |
- :ivar size: The size, in bytes, of the object. |
- :ivar generation: The generation number of the object. |
- :ivar metageneration: The generation number of the object metadata. |
- :ivar encrypted: Whether the object is encrypted while at rest on |
- the server. |
- :ivar cloud_hashes: Dictionary of checksums as supplied by the storage |
- provider. |
- """ |
- |
- def __init__(self, bucket=None, name=None, generation=None): |
- super(Key, self).__init__(bucket=bucket, name=name) |
- self.generation = generation |
- self.meta_generation = None |
- self.cloud_hashes = {} |
- self.component_count = None |
- |
- def __repr__(self): |
- if self.generation and self.metageneration: |
- ver_str = '#%s.%s' % (self.generation, self.metageneration) |
- else: |
- ver_str = '' |
- if self.bucket: |
- return '<Key: %s,%s%s>' % (self.bucket.name, self.name, ver_str) |
- else: |
- return '<Key: None,%s%s>' % (self.name, ver_str) |
- |
- def endElement(self, name, value, connection): |
- if name == 'Key': |
- self.name = value |
- elif name == 'ETag': |
- self.etag = value |
- elif name == 'IsLatest': |
- if value == 'true': |
- self.is_latest = True |
- else: |
- self.is_latest = False |
- elif name == 'LastModified': |
- self.last_modified = value |
- elif name == 'Size': |
- self.size = int(value) |
- elif name == 'StorageClass': |
- self.storage_class = value |
- elif name == 'Owner': |
- pass |
- elif name == 'VersionId': |
- self.version_id = value |
- elif name == 'Generation': |
- self.generation = value |
- elif name == 'MetaGeneration': |
- self.metageneration = value |
- else: |
- setattr(self, name, value) |
- |
- def handle_version_headers(self, resp, force=False): |
- self.metageneration = resp.getheader('x-goog-metageneration', None) |
- self.generation = resp.getheader('x-goog-generation', None) |
- |
- def handle_restore_headers(self, response): |
- return |
- |
- def handle_addl_headers(self, headers): |
- for key, value in headers: |
- if key == 'x-goog-hash': |
- for hash_pair in value.split(','): |
- alg, b64_digest = hash_pair.strip().split('=', 1) |
- self.cloud_hashes[alg] = binascii.a2b_base64(b64_digest) |
- elif key == 'x-goog-component-count': |
- self.component_count = int(value) |
- elif key == 'x-goog-generation': |
- self.generation = value |
- # Use x-goog-stored-content-encoding and |
- # x-goog-stored-content-length to indicate original content length |
- # and encoding, which are transcoding-invariant (so are preferable |
- # over using content-encoding and size headers). |
- elif key == 'x-goog-stored-content-encoding': |
- self.content_encoding = value |
- elif key == 'x-goog-stored-content-length': |
- self.size = int(value) |
- |
- def open_read(self, headers=None, query_args='', |
- override_num_retries=None, response_headers=None): |
- """ |
- Open this key for reading |
- |
- :type headers: dict |
- :param headers: Headers to pass in the web request |
- |
- :type query_args: string |
- :param query_args: Arguments to pass in the query string |
- (ie, 'torrent') |
- |
- :type override_num_retries: int |
- :param override_num_retries: If not None will override configured |
- num_retries parameter for underlying GET. |
- |
- :type response_headers: dict |
- :param response_headers: A dictionary containing HTTP |
- headers/values that will override any headers associated |
- with the stored object in the response. See |
- http://goo.gl/EWOPb for details. |
- """ |
- # For GCS we need to include the object generation in the query args. |
- # The rest of the processing is handled in the parent class. |
- if self.generation: |
- if query_args: |
- query_args += '&' |
- query_args += 'generation=%s' % self.generation |
- super(Key, self).open_read(headers=headers, query_args=query_args, |
- override_num_retries=override_num_retries, |
- response_headers=response_headers) |
- |
- def get_file(self, fp, headers=None, cb=None, num_cb=10, |
- torrent=False, version_id=None, override_num_retries=None, |
- response_headers=None, hash_algs=None): |
- query_args = None |
- if self.generation: |
- query_args = ['generation=%s' % self.generation] |
- self._get_file_internal(fp, headers=headers, cb=cb, num_cb=num_cb, |
- override_num_retries=override_num_retries, |
- response_headers=response_headers, |
- hash_algs=hash_algs, |
- query_args=query_args) |
- |
- def get_contents_to_file(self, fp, headers=None, |
- cb=None, num_cb=10, |
- torrent=False, |
- version_id=None, |
- res_download_handler=None, |
- response_headers=None, |
- hash_algs=None): |
- """ |
- Retrieve an object from GCS using the name of the Key object as the |
- key in GCS. Write the contents of the object to the file pointed |
- to by 'fp'. |
- |
- :type fp: File -like object |
- :param fp: |
- |
- :type headers: dict |
- :param headers: additional HTTP headers that will be sent with |
- the GET request. |
- |
- :type cb: function |
- :param cb: a callback function that will be called to report |
- progress on the upload. The callback should accept two |
- integer parameters, the first representing the number of |
- bytes that have been successfully transmitted to GCS and |
- the second representing the size of the to be transmitted |
- object. |
- |
- :type cb: int |
- :param num_cb: (optional) If a callback is specified with the |
- cb parameter this parameter determines the granularity of |
- the callback by defining the maximum number of times the |
- callback will be called during the file transfer. |
- |
- :type torrent: bool |
- :param torrent: If True, returns the contents of a torrent |
- file as a string. |
- |
- :type res_upload_handler: ResumableDownloadHandler |
- :param res_download_handler: If provided, this handler will |
- perform the download. |
- |
- :type response_headers: dict |
- :param response_headers: A dictionary containing HTTP |
- headers/values that will override any headers associated |
- with the stored object in the response. See |
- http://goo.gl/sMkcC for details. |
- """ |
- if self.bucket is not None: |
- if res_download_handler: |
- res_download_handler.get_file(self, fp, headers, cb, num_cb, |
- torrent=torrent, |
- version_id=version_id, |
- hash_algs=hash_algs) |
- else: |
- self.get_file(fp, headers, cb, num_cb, torrent=torrent, |
- version_id=version_id, |
- response_headers=response_headers, |
- hash_algs=hash_algs) |
- |
- def compute_hash(self, fp, algorithm, size=None): |
- """ |
- :type fp: file |
- :param fp: File pointer to the file to hash. The file |
- pointer will be reset to the same position before the |
- method returns. |
- |
- :type algorithm: zero-argument constructor for hash objects that |
- implements update() and digest() (e.g. hashlib.md5) |
- |
- :type size: int |
- :param size: (optional) The Maximum number of bytes to read |
- from the file pointer (fp). This is useful when uploading |
- a file in multiple parts where the file is being split |
- in place into different parts. Less bytes may be available. |
- """ |
- hex_digest, b64_digest, data_size = compute_hash( |
- fp, size=size, hash_algorithm=algorithm) |
- # The internal implementation of compute_hash() needs to return the |
- # data size, but we don't want to return that value to the external |
- # caller because it changes the class interface (i.e. it might |
- # break some code), so we consume the third tuple value here and |
- # return the remainder of the tuple to the caller, thereby preserving |
- # the existing interface. |
- self.size = data_size |
- return (hex_digest, b64_digest) |
- |
- def send_file(self, fp, headers=None, cb=None, num_cb=10, |
- query_args=None, chunked_transfer=False, size=None, |
- hash_algs=None): |
- """ |
- Upload a file to GCS. |
- |
- :type fp: file |
- :param fp: The file pointer to upload. The file pointer must |
- point point at the offset from which you wish to upload. |
- ie. if uploading the full file, it should point at the |
- start of the file. Normally when a file is opened for |
- reading, the fp will point at the first byte. See the |
- bytes parameter below for more info. |
- |
- :type headers: dict |
- :param headers: The headers to pass along with the PUT request |
- |
- :type num_cb: int |
- :param num_cb: (optional) If a callback is specified with the |
- cb parameter this parameter determines the granularity of |
- the callback by defining the maximum number of times the |
- callback will be called during the file |
- transfer. Providing a negative integer will cause your |
- callback to be called with each buffer read. |
- |
- :type query_args: string |
- :param query_args: Arguments to pass in the query string. |
- |
- :type chunked_transfer: boolean |
- :param chunked_transfer: (optional) If true, we use chunked |
- Transfer-Encoding. |
- |
- :type size: int |
- :param size: (optional) The Maximum number of bytes to read |
- from the file pointer (fp). This is useful when uploading |
- a file in multiple parts where you are splitting the file |
- up into different ranges to be uploaded. If not specified, |
- the default behaviour is to read all bytes from the file |
- pointer. Less bytes may be available. |
- |
- :type hash_algs: dictionary |
- :param hash_algs: (optional) Dictionary of hash algorithms and |
- corresponding hashing class that implements update() and digest(). |
- Defaults to {'md5': hashlib.md5}. |
- """ |
- self._send_file_internal(fp, headers=headers, cb=cb, num_cb=num_cb, |
- query_args=query_args, |
- chunked_transfer=chunked_transfer, size=size, |
- hash_algs=hash_algs) |
- |
- def delete(self, headers=None): |
- return self.bucket.delete_key(self.name, version_id=self.version_id, |
- generation=self.generation, |
- headers=headers) |
- |
- def add_email_grant(self, permission, email_address): |
- """ |
- Convenience method that provides a quick way to add an email grant to a |
- key. This method retrieves the current ACL, creates a new grant based on |
- the parameters passed in, adds that grant to the ACL and then PUT's the |
- new ACL back to GS. |
- |
- :type permission: string |
- :param permission: The permission being granted. Should be one of: |
- READ|FULL_CONTROL |
- See http://code.google.com/apis/storage/docs/developer-guide.html#authorization |
- for more details on permissions. |
- |
- :type email_address: string |
- :param email_address: The email address associated with the Google |
- account to which you are granting the permission. |
- """ |
- acl = self.get_acl() |
- acl.add_email_grant(permission, email_address) |
- self.set_acl(acl) |
- |
- def add_user_grant(self, permission, user_id): |
- """ |
- Convenience method that provides a quick way to add a canonical user |
- grant to a key. This method retrieves the current ACL, creates a new |
- grant based on the parameters passed in, adds that grant to the ACL and |
- then PUT's the new ACL back to GS. |
- |
- :type permission: string |
- :param permission: The permission being granted. Should be one of: |
- READ|FULL_CONTROL |
- See http://code.google.com/apis/storage/docs/developer-guide.html#authorization |
- for more details on permissions. |
- |
- :type user_id: string |
- :param user_id: The canonical user id associated with the GS account to |
- which you are granting the permission. |
- """ |
- acl = self.get_acl() |
- acl.add_user_grant(permission, user_id) |
- self.set_acl(acl) |
- |
- def add_group_email_grant(self, permission, email_address, headers=None): |
- """ |
- Convenience method that provides a quick way to add an email group |
- grant to a key. This method retrieves the current ACL, creates a new |
- grant based on the parameters passed in, adds that grant to the ACL and |
- then PUT's the new ACL back to GS. |
- |
- :type permission: string |
- :param permission: The permission being granted. Should be one of: |
- READ|FULL_CONTROL |
- See http://code.google.com/apis/storage/docs/developer-guide.html#authorization |
- for more details on permissions. |
- |
- :type email_address: string |
- :param email_address: The email address associated with the Google |
- Group to which you are granting the permission. |
- """ |
- acl = self.get_acl(headers=headers) |
- acl.add_group_email_grant(permission, email_address) |
- self.set_acl(acl, headers=headers) |
- |
- def add_group_grant(self, permission, group_id): |
- """ |
- Convenience method that provides a quick way to add a canonical group |
- grant to a key. This method retrieves the current ACL, creates a new |
- grant based on the parameters passed in, adds that grant to the ACL and |
- then PUT's the new ACL back to GS. |
- |
- :type permission: string |
- :param permission: The permission being granted. Should be one of: |
- READ|FULL_CONTROL |
- See http://code.google.com/apis/storage/docs/developer-guide.html#authorization |
- for more details on permissions. |
- |
- :type group_id: string |
- :param group_id: The canonical group id associated with the Google |
- Groups account you are granting the permission to. |
- """ |
- acl = self.get_acl() |
- acl.add_group_grant(permission, group_id) |
- self.set_acl(acl) |
- |
- def set_contents_from_file(self, fp, headers=None, replace=True, |
- cb=None, num_cb=10, policy=None, md5=None, |
- res_upload_handler=None, size=None, rewind=False, |
- if_generation=None): |
- """ |
- Store an object in GS using the name of the Key object as the |
- key in GS and the contents of the file pointed to by 'fp' as the |
- contents. |
- |
- :type fp: file |
- :param fp: The file whose contents are to be uploaded. |
- |
- :type headers: dict |
- :param headers: (optional) Additional HTTP headers to be sent with the |
- PUT request. |
- |
- :type replace: bool |
- :param replace: (optional) If this parameter is False, the method will |
- first check to see if an object exists in the bucket with the same |
- key. If it does, it won't overwrite it. The default value is True |
- which will overwrite the object. |
- |
- :type cb: function |
- :param cb: (optional) Callback function that will be called to report |
- progress on the upload. The callback should accept two integer |
- parameters, the first representing the number of bytes that have |
- been successfully transmitted to GS and the second representing the |
- total number of bytes that need to be transmitted. |
- |
- :type num_cb: int |
- :param num_cb: (optional) If a callback is specified with the cb |
- parameter, this parameter determines the granularity of the callback |
- by defining the maximum number of times the callback will be called |
- during the file transfer. |
- |
- :type policy: :class:`boto.gs.acl.CannedACLStrings` |
- :param policy: (optional) A canned ACL policy that will be applied to |
- the new key in GS. |
- |
- :type md5: tuple |
- :param md5: (optional) A tuple containing the hexdigest version of the |
- MD5 checksum of the file as the first element and the |
- Base64-encoded version of the plain checksum as the second element. |
- This is the same format returned by the compute_md5 method. |
- |
- If you need to compute the MD5 for any reason prior to upload, it's |
- silly to have to do it twice so this param, if present, will be |
- used as the MD5 values of the file. Otherwise, the checksum will be |
- computed. |
- |
- :type res_upload_handler: :py:class:`boto.gs.resumable_upload_handler.ResumableUploadHandler` |
- :param res_upload_handler: (optional) If provided, this handler will |
- perform the upload. |
- |
- :type size: int |
- :param size: (optional) The Maximum number of bytes to read from the |
- file pointer (fp). This is useful when uploading a file in multiple |
- parts where you are splitting the file up into different ranges to |
- be uploaded. If not specified, the default behaviour is to read all |
- bytes from the file pointer. Less bytes may be available. |
- |
- Notes: |
- |
- 1. The "size" parameter currently cannot be used when a |
- resumable upload handler is given but is still useful for |
- uploading part of a file as implemented by the parent class. |
- 2. At present Google Cloud Storage does not support multipart |
- uploads. |
- |
- :type rewind: bool |
- :param rewind: (optional) If True, the file pointer (fp) will be |
- rewound to the start before any bytes are read from it. The default |
- behaviour is False which reads from the current position of the |
- file pointer (fp). |
- |
- :type if_generation: int |
- :param if_generation: (optional) If set to a generation number, the |
- object will only be written to if its current generation number is |
- this value. If set to the value 0, the object will only be written |
- if it doesn't already exist. |
- |
- :rtype: int |
- :return: The number of bytes written to the key. |
- |
- TODO: At some point we should refactor the Bucket and Key classes, |
- to move functionality common to all providers into a parent class, |
- and provider-specific functionality into subclasses (rather than |
- just overriding/sharing code the way it currently works). |
- """ |
- provider = self.bucket.connection.provider |
- if res_upload_handler and size: |
- # could use size instead of file_length if provided but... |
- raise BotoClientError( |
- '"size" param not supported for resumable uploads.') |
- headers = headers or {} |
- if policy: |
- headers[provider.acl_header] = policy |
- |
- if rewind: |
- # caller requests reading from beginning of fp. |
- fp.seek(0, os.SEEK_SET) |
- else: |
- # The following seek/tell/seek logic is intended |
- # to detect applications using the older interface to |
- # set_contents_from_file(), which automatically rewound the |
- # file each time the Key was reused. This changed with commit |
- # 14ee2d03f4665fe20d19a85286f78d39d924237e, to support uploads |
- # split into multiple parts and uploaded in parallel, and at |
- # the time of that commit this check was added because otherwise |
- # older programs would get a success status and upload an empty |
- # object. Unfortuantely, it's very inefficient for fp's implemented |
- # by KeyFile (used, for example, by gsutil when copying between |
- # providers). So, we skip the check for the KeyFile case. |
- # TODO: At some point consider removing this seek/tell/seek |
- # logic, after enough time has passed that it's unlikely any |
- # programs remain that assume the older auto-rewind interface. |
- if not isinstance(fp, KeyFile): |
- spos = fp.tell() |
- fp.seek(0, os.SEEK_END) |
- if fp.tell() == spos: |
- fp.seek(0, os.SEEK_SET) |
- if fp.tell() != spos: |
- # Raise an exception as this is likely a programming |
- # error whereby there is data before the fp but nothing |
- # after it. |
- fp.seek(spos) |
- raise AttributeError('fp is at EOF. Use rewind option ' |
- 'or seek() to data start.') |
- # seek back to the correct position. |
- fp.seek(spos) |
- |
- if hasattr(fp, 'name'): |
- self.path = fp.name |
- if self.bucket is not None: |
- if isinstance(fp, KeyFile): |
- # Avoid EOF seek for KeyFile case as it's very inefficient. |
- key = fp.getkey() |
- size = key.size - fp.tell() |
- self.size = size |
- # At present both GCS and S3 use MD5 for the etag for |
- # non-multipart-uploaded objects. If the etag is 32 hex |
- # chars use it as an MD5, to avoid having to read the file |
- # twice while transferring. |
- if (re.match('^"[a-fA-F0-9]{32}"$', key.etag)): |
- etag = key.etag.strip('"') |
- md5 = (etag, base64.b64encode(binascii.unhexlify(etag))) |
- if size: |
- self.size = size |
- else: |
- # If md5 is provided, still need to size so |
- # calculate based on bytes to end of content |
- spos = fp.tell() |
- fp.seek(0, os.SEEK_END) |
- self.size = fp.tell() - spos |
- fp.seek(spos) |
- size = self.size |
- |
- if md5 is None: |
- md5 = self.compute_md5(fp, size) |
- self.md5 = md5[0] |
- self.base64md5 = md5[1] |
- |
- if self.name is None: |
- self.name = self.md5 |
- |
- if not replace: |
- if self.bucket.lookup(self.name): |
- return |
- |
- if if_generation is not None: |
- headers['x-goog-if-generation-match'] = str(if_generation) |
- |
- if res_upload_handler: |
- res_upload_handler.send_file(self, fp, headers, cb, num_cb) |
- else: |
- # Not a resumable transfer so use basic send_file mechanism. |
- self.send_file(fp, headers, cb, num_cb, size=size) |
- |
- def set_contents_from_filename(self, filename, headers=None, replace=True, |
- cb=None, num_cb=10, policy=None, md5=None, |
- reduced_redundancy=None, |
- res_upload_handler=None, |
- if_generation=None): |
- """ |
- Store an object in GS using the name of the Key object as the |
- key in GS and the contents of the file named by 'filename'. |
- See set_contents_from_file method for details about the |
- parameters. |
- |
- :type filename: string |
- :param filename: The name of the file that you want to put onto GS. |
- |
- :type headers: dict |
- :param headers: (optional) Additional headers to pass along with the |
- request to GS. |
- |
- :type replace: bool |
- :param replace: (optional) If True, replaces the contents of the file |
- if it already exists. |
- |
- :type cb: function |
- :param cb: (optional) Callback function that will be called to report |
- progress on the upload. The callback should accept two integer |
- parameters, the first representing the number of bytes that have |
- been successfully transmitted to GS and the second representing the |
- total number of bytes that need to be transmitted. |
- |
- :type num_cb: int |
- :param num_cb: (optional) If a callback is specified with the cb |
- parameter this parameter determines the granularity of the callback |
- by defining the maximum number of times the callback will be called |
- during the file transfer. |
- |
- :type policy: :py:attribute:`boto.gs.acl.CannedACLStrings` |
- :param policy: (optional) A canned ACL policy that will be applied to |
- the new key in GS. |
- |
- :type md5: tuple |
- :param md5: (optional) A tuple containing the hexdigest version of the |
- MD5 checksum of the file as the first element and the |
- Base64-encoded version of the plain checksum as the second element. |
- This is the same format returned by the compute_md5 method. |
- |
- If you need to compute the MD5 for any reason prior to upload, it's |
- silly to have to do it twice so this param, if present, will be |
- used as the MD5 values of the file. Otherwise, the checksum will be |
- computed. |
- |
- :type res_upload_handler: :py:class:`boto.gs.resumable_upload_handler.ResumableUploadHandler` |
- :param res_upload_handler: (optional) If provided, this handler will |
- perform the upload. |
- |
- :type if_generation: int |
- :param if_generation: (optional) If set to a generation number, the |
- object will only be written to if its current generation number is |
- this value. If set to the value 0, the object will only be written |
- if it doesn't already exist. |
- """ |
- # Clear out any previously computed hashes, since we are setting the |
- # content. |
- self.local_hashes = {} |
- |
- with open(filename, 'rb') as fp: |
- self.set_contents_from_file(fp, headers, replace, cb, num_cb, |
- policy, md5, res_upload_handler, |
- if_generation=if_generation) |
- |
- def set_contents_from_string(self, s, headers=None, replace=True, |
- cb=None, num_cb=10, policy=None, md5=None, |
- if_generation=None): |
- """ |
- Store an object in GCS using the name of the Key object as the |
- key in GCS and the string 's' as the contents. |
- See set_contents_from_file method for details about the |
- parameters. |
- |
- :type headers: dict |
- :param headers: Additional headers to pass along with the |
- request to AWS. |
- |
- :type replace: bool |
- :param replace: If True, replaces the contents of the file if |
- it already exists. |
- |
- :type cb: function |
- :param cb: a callback function that will be called to report |
- progress on the upload. The callback should accept |
- two integer parameters, the first representing the |
- number of bytes that have been successfully |
- transmitted to GCS and the second representing the |
- size of the to be transmitted object. |
- |
- :type cb: int |
- :param num_cb: (optional) If a callback is specified with |
- the cb parameter this parameter determines the |
- granularity of the callback by defining |
- the maximum number of times the callback will |
- be called during the file transfer. |
- |
- :type policy: :class:`boto.gs.acl.CannedACLStrings` |
- :param policy: A canned ACL policy that will be applied to the |
- new key in GCS. |
- |
- :type md5: A tuple containing the hexdigest version of the MD5 |
- checksum of the file as the first element and the |
- Base64-encoded version of the plain checksum as the |
- second element. This is the same format returned by |
- the compute_md5 method. |
- :param md5: If you need to compute the MD5 for any reason prior |
- to upload, it's silly to have to do it twice so this |
- param, if present, will be used as the MD5 values |
- of the file. Otherwise, the checksum will be computed. |
- |
- :type if_generation: int |
- :param if_generation: (optional) If set to a generation number, the |
- object will only be written to if its current generation number is |
- this value. If set to the value 0, the object will only be written |
- if it doesn't already exist. |
- """ |
- |
- # Clear out any previously computed md5 hashes, since we are setting the content. |
- self.md5 = None |
- self.base64md5 = None |
- |
- fp = StringIO(get_utf8_value(s)) |
- r = self.set_contents_from_file(fp, headers, replace, cb, num_cb, |
- policy, md5, |
- if_generation=if_generation) |
- fp.close() |
- return r |
- |
- def set_contents_from_stream(self, *args, **kwargs): |
- """ |
- Store an object using the name of the Key object as the key in |
- cloud and the contents of the data stream pointed to by 'fp' as |
- the contents. |
- |
- The stream object is not seekable and total size is not known. |
- This has the implication that we can't specify the |
- Content-Size and Content-MD5 in the header. So for huge |
- uploads, the delay in calculating MD5 is avoided but with a |
- penalty of inability to verify the integrity of the uploaded |
- data. |
- |
- :type fp: file |
- :param fp: the file whose contents are to be uploaded |
- |
- :type headers: dict |
- :param headers: additional HTTP headers to be sent with the |
- PUT request. |
- |
- :type replace: bool |
- :param replace: If this parameter is False, the method will first check |
- to see if an object exists in the bucket with the same key. If it |
- does, it won't overwrite it. The default value is True which will |
- overwrite the object. |
- |
- :type cb: function |
- :param cb: a callback function that will be called to report |
- progress on the upload. The callback should accept two integer |
- parameters, the first representing the number of bytes that have |
- been successfully transmitted to GS and the second representing the |
- total number of bytes that need to be transmitted. |
- |
- :type num_cb: int |
- :param num_cb: (optional) If a callback is specified with the |
- cb parameter, this parameter determines the granularity of |
- the callback by defining the maximum number of times the |
- callback will be called during the file transfer. |
- |
- :type policy: :class:`boto.gs.acl.CannedACLStrings` |
- :param policy: A canned ACL policy that will be applied to the new key |
- in GS. |
- |
- :type size: int |
- :param size: (optional) The Maximum number of bytes to read from |
- the file pointer (fp). This is useful when uploading a |
- file in multiple parts where you are splitting the file up |
- into different ranges to be uploaded. If not specified, |
- the default behaviour is to read all bytes from the file |
- pointer. Less bytes may be available. |
- |
- :type if_generation: int |
- :param if_generation: (optional) If set to a generation number, the |
- object will only be written to if its current generation number is |
- this value. If set to the value 0, the object will only be written |
- if it doesn't already exist. |
- """ |
- if_generation = kwargs.pop('if_generation', None) |
- if if_generation is not None: |
- headers = kwargs.get('headers', {}) |
- headers['x-goog-if-generation-match'] = str(if_generation) |
- kwargs['headers'] = headers |
- super(Key, self).set_contents_from_stream(*args, **kwargs) |
- |
- def set_acl(self, acl_or_str, headers=None, generation=None, |
- if_generation=None, if_metageneration=None): |
- """Sets the ACL for this object. |
- |
- :type acl_or_str: string or :class:`boto.gs.acl.ACL` |
- :param acl_or_str: A canned ACL string (see |
- :data:`~.gs.acl.CannedACLStrings`) or an ACL object. |
- |
- :type headers: dict |
- :param headers: Additional headers to set during the request. |
- |
- :type generation: int |
- :param generation: If specified, sets the ACL for a specific generation |
- of a versioned object. If not specified, the current version is |
- modified. |
- |
- :type if_generation: int |
- :param if_generation: (optional) If set to a generation number, the acl |
- will only be updated if its current generation number is this value. |
- |
- :type if_metageneration: int |
- :param if_metageneration: (optional) If set to a metageneration number, |
- the acl will only be updated if its current metageneration number is |
- this value. |
- """ |
- if self.bucket is not None: |
- self.bucket.set_acl(acl_or_str, self.name, headers=headers, |
- generation=generation, |
- if_generation=if_generation, |
- if_metageneration=if_metageneration) |
- |
- def get_acl(self, headers=None, generation=None): |
- """Returns the ACL of this object. |
- |
- :param dict headers: Additional headers to set during the request. |
- |
- :param int generation: If specified, gets the ACL for a specific |
- generation of a versioned object. If not specified, the current |
- version is returned. |
- |
- :rtype: :class:`.gs.acl.ACL` |
- """ |
- if self.bucket is not None: |
- return self.bucket.get_acl(self.name, headers=headers, |
- generation=generation) |
- |
- def get_xml_acl(self, headers=None, generation=None): |
- """Returns the ACL string of this object. |
- |
- :param dict headers: Additional headers to set during the request. |
- |
- :param int generation: If specified, gets the ACL for a specific |
- generation of a versioned object. If not specified, the current |
- version is returned. |
- |
- :rtype: str |
- """ |
- if self.bucket is not None: |
- return self.bucket.get_xml_acl(self.name, headers=headers, |
- generation=generation) |
- |
- def set_xml_acl(self, acl_str, headers=None, generation=None, |
- if_generation=None, if_metageneration=None): |
- """Sets this objects's ACL to an XML string. |
- |
- :type acl_str: string |
- :param acl_str: A string containing the ACL XML. |
- |
- :type headers: dict |
- :param headers: Additional headers to set during the request. |
- |
- :type generation: int |
- :param generation: If specified, sets the ACL for a specific generation |
- of a versioned object. If not specified, the current version is |
- modified. |
- |
- :type if_generation: int |
- :param if_generation: (optional) If set to a generation number, the acl |
- will only be updated if its current generation number is this value. |
- |
- :type if_metageneration: int |
- :param if_metageneration: (optional) If set to a metageneration number, |
- the acl will only be updated if its current metageneration number is |
- this value. |
- """ |
- if self.bucket is not None: |
- return self.bucket.set_xml_acl(acl_str, self.name, headers=headers, |
- generation=generation, |
- if_generation=if_generation, |
- if_metageneration=if_metageneration) |
- |
- def set_canned_acl(self, acl_str, headers=None, generation=None, |
- if_generation=None, if_metageneration=None): |
- """Sets this objects's ACL using a predefined (canned) value. |
- |
- :type acl_str: string |
- :param acl_str: A canned ACL string. See |
- :data:`~.gs.acl.CannedACLStrings`. |
- |
- :type headers: dict |
- :param headers: Additional headers to set during the request. |
- |
- :type generation: int |
- :param generation: If specified, sets the ACL for a specific generation |
- of a versioned object. If not specified, the current version is |
- modified. |
- |
- :type if_generation: int |
- :param if_generation: (optional) If set to a generation number, the acl |
- will only be updated if its current generation number is this value. |
- |
- :type if_metageneration: int |
- :param if_metageneration: (optional) If set to a metageneration number, |
- the acl will only be updated if its current metageneration number is |
- this value. |
- """ |
- if self.bucket is not None: |
- return self.bucket.set_canned_acl( |
- acl_str, |
- self.name, |
- headers=headers, |
- generation=generation, |
- if_generation=if_generation, |
- if_metageneration=if_metageneration |
- ) |
- |
- def compose(self, components, content_type=None, headers=None): |
- """Create a new object from a sequence of existing objects. |
- |
- The content of the object representing this Key will be the |
- concatenation of the given object sequence. For more detail, visit |
- |
- https://developers.google.com/storage/docs/composite-objects |
- |
- :type components list of Keys |
- :param components List of gs.Keys representing the component objects |
- |
- :type content_type (optional) string |
- :param content_type Content type for the new composite object. |
- """ |
- compose_req = [] |
- for key in components: |
- if key.bucket.name != self.bucket.name: |
- raise BotoClientError( |
- 'GCS does not support inter-bucket composing') |
- |
- generation_tag = '' |
- if key.generation: |
- generation_tag = ('<Generation>%s</Generation>' |
- % str(key.generation)) |
- compose_req.append('<Component><Name>%s</Name>%s</Component>' % |
- (key.name, generation_tag)) |
- compose_req_xml = ('<ComposeRequest>%s</ComposeRequest>' % |
- ''.join(compose_req)) |
- headers = headers or {} |
- if content_type: |
- headers['Content-Type'] = content_type |
- resp = self.bucket.connection.make_request( |
- 'PUT', get_utf8_value(self.bucket.name), get_utf8_value(self.name), |
- headers=headers, query_args='compose', |
- data=get_utf8_value(compose_req_xml)) |
- if resp.status < 200 or resp.status > 299: |
- raise self.bucket.connection.provider.storage_response_error( |
- resp.status, resp.reason, resp.read()) |
- |
- # Return the generation so that the result URI can be built with this |
- # for automatic parallel uploads. |
- return resp.getheader('x-goog-generation') |