[catapult] - Copy Telemetry's gsutilz over to third_party.

third_party/gsutil/third_party/boto/boto/gs/key.py

Index: third_party/gsutil/third_party/boto/boto/gs/key.py
diff --git a/third_party/gsutil/third_party/boto/boto/gs/key.py b/third_party/gsutil/third_party/boto/boto/gs/key.py
new file mode 100644
index 0000000000000000000000000000000000000000..c4fcf01f091582fede729e7c647d5d749bf4a7c6
--- /dev/null
+++ b/third_party/gsutil/third_party/boto/boto/gs/key.py
@@ -0,0 +1,946 @@
+# Copyright 2010 Google Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+import base64
+import binascii
+import os
+import re
+
+from boto.compat import StringIO
+from boto.exception import BotoClientError
+from boto.s3.key import Key as S3Key
+from boto.s3.keyfile import KeyFile
+from boto.utils import compute_hash
+from boto.utils import get_utf8_value
+
+class Key(S3Key):
+    """
+    Represents a key (object) in a GS bucket.
+
+    :ivar bucket: The parent :class:`boto.gs.bucket.Bucket`.
+    :ivar name: The name of this Key object.
+    :ivar metadata: A dictionary containing user metadata that you
+        wish to store with the object or that has been retrieved from
+        an existing object.
+    :ivar cache_control: The value of the `Cache-Control` HTTP header.
+    :ivar content_type: The value of the `Content-Type` HTTP header.
+    :ivar content_encoding: The value of the `Content-Encoding` HTTP header.
+    :ivar content_disposition: The value of the `Content-Disposition` HTTP
+        header.
+    :ivar content_language: The value of the `Content-Language` HTTP header.
+    :ivar etag: The `etag` associated with this object.
+    :ivar last_modified: The string timestamp representing the last
+        time this object was modified in GS.
+    :ivar owner: The ID of the owner of this object.
+    :ivar storage_class: The storage class of the object. Currently, one of:
+        STANDARD | DURABLE_REDUCED_AVAILABILITY.
+    :ivar md5: The MD5 hash of the contents of the object.
+    :ivar size: The size, in bytes, of the object.
+    :ivar generation: The generation number of the object.
+    :ivar metageneration: The generation number of the object metadata.
+    :ivar encrypted: Whether the object is encrypted while at rest on
+        the server.
+    :ivar cloud_hashes: Dictionary of checksums as supplied by the storage
+        provider.
+    """
+
+    def __init__(self, bucket=None, name=None, generation=None):
+        super(Key, self).__init__(bucket=bucket, name=name)
+        self.generation = generation
+        self.meta_generation = None
+        self.cloud_hashes = {}
+        self.component_count = None
+
+    def __repr__(self):
+        if self.generation and self.metageneration:
+            ver_str = '#%s.%s' % (self.generation, self.metageneration)
+        else:
+            ver_str = ''
+        if self.bucket:
+            return '<Key: %s,%s%s>' % (self.bucket.name, self.name, ver_str)
+        else:
+            return '<Key: None,%s%s>' % (self.name, ver_str)
+
+    def endElement(self, name, value, connection):
+        if name == 'Key':
+            self.name = value
+        elif name == 'ETag':
+            self.etag = value
+        elif name == 'IsLatest':
+            if value == 'true':
+                self.is_latest = True
+            else:
+                self.is_latest = False
+        elif name == 'LastModified':
+            self.last_modified = value
+        elif name == 'Size':
+            self.size = int(value)
+        elif name == 'StorageClass':
+            self.storage_class = value
+        elif name == 'Owner':
+            pass
+        elif name == 'VersionId':
+            self.version_id = value
+        elif name == 'Generation':
+            self.generation = value
+        elif name == 'MetaGeneration':
+            self.metageneration = value
+        else:
+            setattr(self, name, value)
+
+    def handle_version_headers(self, resp, force=False):
+        self.metageneration = resp.getheader('x-goog-metageneration', None)
+        self.generation = resp.getheader('x-goog-generation', None)
+
+    def handle_restore_headers(self, response):
+        return
+
+    def handle_addl_headers(self, headers):
+        for key, value in headers:
+            if key == 'x-goog-hash':
+                for hash_pair in value.split(','):
+                    alg, b64_digest = hash_pair.strip().split('=', 1)
+                    self.cloud_hashes[alg] = binascii.a2b_base64(b64_digest)
+            elif key == 'x-goog-component-count':
+                self.component_count = int(value)
+            elif key == 'x-goog-generation':
+                self.generation = value
+            # Use x-goog-stored-content-encoding and
+            # x-goog-stored-content-length to indicate original content length
+            # and encoding, which are transcoding-invariant (so are preferable
+            # over using content-encoding and size headers).
+            elif key == 'x-goog-stored-content-encoding':
+                self.content_encoding = value
+            elif key == 'x-goog-stored-content-length':
+                self.size = int(value)
+
+    def open_read(self, headers=None, query_args='',
+                  override_num_retries=None, response_headers=None):
+        """
+        Open this key for reading
+
+        :type headers: dict
+        :param headers: Headers to pass in the web request
+
+        :type query_args: string
+        :param query_args: Arguments to pass in the query string
+            (ie, 'torrent')
+
+        :type override_num_retries: int
+        :param override_num_retries: If not None will override configured
+            num_retries parameter for underlying GET.
+
+        :type response_headers: dict
+        :param response_headers: A dictionary containing HTTP
+            headers/values that will override any headers associated
+            with the stored object in the response.  See
+            http://goo.gl/EWOPb for details.
+        """
+        # For GCS we need to include the object generation in the query args.
+        # The rest of the processing is handled in the parent class.
+        if self.generation:
+            if query_args:
+                query_args += '&'
+            query_args += 'generation=%s' % self.generation
+        super(Key, self).open_read(headers=headers, query_args=query_args,
+                                   override_num_retries=override_num_retries,
+                                   response_headers=response_headers)
+
+    def get_file(self, fp, headers=None, cb=None, num_cb=10,
+                 torrent=False, version_id=None, override_num_retries=None,
+                 response_headers=None, hash_algs=None):
+        query_args = None
+        if self.generation:
+            query_args = ['generation=%s' % self.generation]
+        self._get_file_internal(fp, headers=headers, cb=cb, num_cb=num_cb,
+                                override_num_retries=override_num_retries,
+                                response_headers=response_headers,
+                                hash_algs=hash_algs,
+                                query_args=query_args)
+
+    def get_contents_to_file(self, fp, headers=None,
+                             cb=None, num_cb=10,
+                             torrent=False,
+                             version_id=None,
+                             res_download_handler=None,
+                             response_headers=None,
+                             hash_algs=None):
+        """
+        Retrieve an object from GCS using the name of the Key object as the
+        key in GCS. Write the contents of the object to the file pointed
+        to by 'fp'.
+
+        :type fp: File -like object
+        :param fp:
+
+        :type headers: dict
+        :param headers: additional HTTP headers that will be sent with
+            the GET request.
+
+        :type cb: function
+        :param cb: a callback function that will be called to report
+            progress on the upload. The callback should accept two
+            integer parameters, the first representing the number of
+            bytes that have been successfully transmitted to GCS and
+            the second representing the size of the to be transmitted
+            object.
+
+        :type cb: int
+        :param num_cb: (optional) If a callback is specified with the
+            cb parameter this parameter determines the granularity of
+            the callback by defining the maximum number of times the
+            callback will be called during the file transfer.
+
+        :type torrent: bool
+        :param torrent: If True, returns the contents of a torrent
+            file as a string.
+
+        :type res_upload_handler: ResumableDownloadHandler
+        :param res_download_handler: If provided, this handler will
+            perform the download.
+
+        :type response_headers: dict
+        :param response_headers: A dictionary containing HTTP
+            headers/values that will override any headers associated
+            with the stored object in the response. See
+            http://goo.gl/sMkcC for details.
+        """
+        if self.bucket is not None:
+            if res_download_handler:
+                res_download_handler.get_file(self, fp, headers, cb, num_cb,
+                                              torrent=torrent,
+                                              version_id=version_id,
+                                              hash_algs=hash_algs)
+            else:
+                self.get_file(fp, headers, cb, num_cb, torrent=torrent,
+                              version_id=version_id,
+                              response_headers=response_headers,
+                              hash_algs=hash_algs)
+
+    def compute_hash(self, fp, algorithm, size=None):
+        """
+        :type fp: file
+        :param fp: File pointer to the file to hash. The file
+            pointer will be reset to the same position before the
+            method returns.
+
+        :type algorithm: zero-argument constructor for hash objects that
+            implements update() and digest() (e.g. hashlib.md5)
+
+        :type size: int
+        :param size: (optional) The Maximum number of bytes to read
+            from the file pointer (fp). This is useful when uploading
+            a file in multiple parts where the file is being split
+            in place into different parts. Less bytes may be available.
+        """
+        hex_digest, b64_digest, data_size = compute_hash(
+            fp, size=size, hash_algorithm=algorithm)
+        # The internal implementation of compute_hash() needs to return the
+        # data size, but we don't want to return that value to the external
+        # caller because it changes the class interface (i.e. it might
+        # break some code), so we consume the third tuple value here and
+        # return the remainder of the tuple to the caller, thereby preserving
+        # the existing interface.
+        self.size = data_size
+        return (hex_digest, b64_digest)
+
+    def send_file(self, fp, headers=None, cb=None, num_cb=10,
+                  query_args=None, chunked_transfer=False, size=None,
+                  hash_algs=None):
+        """
+        Upload a file to GCS.
+
+        :type fp: file
+        :param fp: The file pointer to upload. The file pointer must
+            point point at the offset from which you wish to upload.
+            ie. if uploading the full file, it should point at the
+            start of the file. Normally when a file is opened for
+            reading, the fp will point at the first byte. See the
+            bytes parameter below for more info.
+
+        :type headers: dict
+        :param headers: The headers to pass along with the PUT request
+
+        :type num_cb: int
+        :param num_cb: (optional) If a callback is specified with the
+            cb parameter this parameter determines the granularity of
+            the callback by defining the maximum number of times the
+            callback will be called during the file
+            transfer. Providing a negative integer will cause your
+            callback to be called with each buffer read.
+
+        :type query_args: string
+        :param query_args: Arguments to pass in the query string.
+
+        :type chunked_transfer: boolean
+        :param chunked_transfer: (optional) If true, we use chunked
+            Transfer-Encoding.
+
+        :type size: int
+        :param size: (optional) The Maximum number of bytes to read
+            from the file pointer (fp). This is useful when uploading
+            a file in multiple parts where you are splitting the file
+            up into different ranges to be uploaded. If not specified,
+            the default behaviour is to read all bytes from the file
+            pointer. Less bytes may be available.
+
+        :type hash_algs: dictionary
+        :param hash_algs: (optional) Dictionary of hash algorithms and
+            corresponding hashing class that implements update() and digest().
+            Defaults to {'md5': hashlib.md5}.
+        """
+        self._send_file_internal(fp, headers=headers, cb=cb, num_cb=num_cb,
+                                 query_args=query_args,
+                                 chunked_transfer=chunked_transfer, size=size,
+                                 hash_algs=hash_algs)
+
+    def delete(self, headers=None):
+        return self.bucket.delete_key(self.name, version_id=self.version_id,
+                                      generation=self.generation,
+                                      headers=headers)
+
+    def add_email_grant(self, permission, email_address):
+        """
+        Convenience method that provides a quick way to add an email grant to a
+        key. This method retrieves the current ACL, creates a new grant based on
+        the parameters passed in, adds that grant to the ACL and then PUT's the
+        new ACL back to GS.
+
+        :type permission: string
+        :param permission: The permission being granted. Should be one of:
+            READ|FULL_CONTROL
+            See http://code.google.com/apis/storage/docs/developer-guide.html#authorization
+            for more details on permissions.
+
+        :type email_address: string
+        :param email_address: The email address associated with the Google
+                              account to which you are granting the permission.
+        """
+        acl = self.get_acl()
+        acl.add_email_grant(permission, email_address)
+        self.set_acl(acl)
+
+    def add_user_grant(self, permission, user_id):
+        """
+        Convenience method that provides a quick way to add a canonical user
+        grant to a key. This method retrieves the current ACL, creates a new
+        grant based on the parameters passed in, adds that grant to the ACL and
+        then PUT's the new ACL back to GS.
+
+        :type permission: string
+        :param permission: The permission being granted. Should be one of:
+            READ|FULL_CONTROL
+            See http://code.google.com/apis/storage/docs/developer-guide.html#authorization
+            for more details on permissions.
+
+        :type user_id: string
+        :param user_id: The canonical user id associated with the GS account to
+             which you are granting the permission.
+        """
+        acl = self.get_acl()
+        acl.add_user_grant(permission, user_id)
+        self.set_acl(acl)
+
+    def add_group_email_grant(self, permission, email_address, headers=None):
+        """
+        Convenience method that provides a quick way to add an email group
+        grant to a key. This method retrieves the current ACL, creates a new
+        grant based on the parameters passed in, adds that grant to the ACL and
+        then PUT's the new ACL back to GS.
+
+        :type permission: string
+        :param permission: The permission being granted. Should be one of:
+            READ|FULL_CONTROL
+            See http://code.google.com/apis/storage/docs/developer-guide.html#authorization
+            for more details on permissions.
+
+        :type email_address: string
+        :param email_address: The email address associated with the Google
+            Group to which you are granting the permission.
+        """
+        acl = self.get_acl(headers=headers)
+        acl.add_group_email_grant(permission, email_address)
+        self.set_acl(acl, headers=headers)
+
+    def add_group_grant(self, permission, group_id):
+        """
+        Convenience method that provides a quick way to add a canonical group
+        grant to a key. This method retrieves the current ACL, creates a new
+        grant based on the parameters passed in, adds that grant to the ACL and
+        then PUT's the new ACL back to GS.
+
+        :type permission: string
+        :param permission: The permission being granted. Should be one of:
+            READ|FULL_CONTROL
+            See http://code.google.com/apis/storage/docs/developer-guide.html#authorization
+            for more details on permissions.
+
+        :type group_id: string
+        :param group_id: The canonical group id associated with the Google
+            Groups account you are granting the permission to.
+        """
+        acl = self.get_acl()
+        acl.add_group_grant(permission, group_id)
+        self.set_acl(acl)
+
+    def set_contents_from_file(self, fp, headers=None, replace=True,
+                               cb=None, num_cb=10, policy=None, md5=None,
+                               res_upload_handler=None, size=None, rewind=False,
+                               if_generation=None):
+        """
+        Store an object in GS using the name of the Key object as the
+        key in GS and the contents of the file pointed to by 'fp' as the
+        contents.
+
+        :type fp: file
+        :param fp: The file whose contents are to be uploaded.
+
+        :type headers: dict
+        :param headers: (optional) Additional HTTP headers to be sent with the
+            PUT request.
+
+        :type replace: bool
+        :param replace: (optional) If this parameter is False, the method will
+            first check to see if an object exists in the bucket with the same
+            key. If it does, it won't overwrite it. The default value is True
+            which will overwrite the object.
+
+        :type cb: function
+        :param cb: (optional) Callback function that will be called to report
+            progress on the upload. The callback should accept two integer
+            parameters, the first representing the number of bytes that have
+            been successfully transmitted to GS and the second representing the
+            total number of bytes that need to be transmitted.
+
+        :type num_cb: int
+        :param num_cb: (optional) If a callback is specified with the cb
+            parameter, this parameter determines the granularity of the callback
+            by defining the maximum number of times the callback will be called
+            during the file transfer.
+
+        :type policy: :class:`boto.gs.acl.CannedACLStrings`
+        :param policy: (optional) A canned ACL policy that will be applied to
+            the new key in GS.
+
+        :type md5: tuple
+        :param md5: (optional) A tuple containing the hexdigest version of the
+            MD5 checksum of the file as the first element and the
+            Base64-encoded version of the plain checksum as the second element.
+            This is the same format returned by the compute_md5 method.
+
+            If you need to compute the MD5 for any reason prior to upload, it's
+            silly to have to do it twice so this param, if present, will be
+            used as the MD5 values of the file. Otherwise, the checksum will be
+            computed.
+
+        :type res_upload_handler: :py:class:`boto.gs.resumable_upload_handler.ResumableUploadHandler`
+        :param res_upload_handler: (optional) If provided, this handler will
+            perform the upload.
+
+        :type size: int
+        :param size: (optional) The Maximum number of bytes to read from the
+            file pointer (fp). This is useful when uploading a file in multiple
+            parts where you are splitting the file up into different ranges to
+            be uploaded. If not specified, the default behaviour is to read all
+            bytes from the file pointer. Less bytes may be available.
+
+            Notes:
+
+                1. The "size" parameter currently cannot be used when a
+                   resumable upload handler is given but is still useful for
+                   uploading part of a file as implemented by the parent class.
+                2. At present Google Cloud Storage does not support multipart
+                   uploads.
+
+        :type rewind: bool
+        :param rewind: (optional) If True, the file pointer (fp) will be
+            rewound to the start before any bytes are read from it. The default
+            behaviour is False which reads from the current position of the
+            file pointer (fp).
+
+        :type if_generation: int
+        :param if_generation: (optional) If set to a generation number, the
+            object will only be written to if its current generation number is
+            this value. If set to the value 0, the object will only be written
+            if it doesn't already exist.
+
+        :rtype: int
+        :return: The number of bytes written to the key.
+
+        TODO: At some point we should refactor the Bucket and Key classes,
+        to move functionality common to all providers into a parent class,
+        and provider-specific functionality into subclasses (rather than
+        just overriding/sharing code the way it currently works).
+        """
+        provider = self.bucket.connection.provider
+        if res_upload_handler and size:
+            # could use size instead of file_length if provided but...
+            raise BotoClientError(
+                '"size" param not supported for resumable uploads.')
+        headers = headers or {}
+        if policy:
+            headers[provider.acl_header] = policy
+
+        if rewind:
+            # caller requests reading from beginning of fp.
+            fp.seek(0, os.SEEK_SET)
+        else:
+            # The following seek/tell/seek logic is intended
+            # to detect applications using the older interface to
+            # set_contents_from_file(), which automatically rewound the
+            # file each time the Key was reused. This changed with commit
+            # 14ee2d03f4665fe20d19a85286f78d39d924237e, to support uploads
+            # split into multiple parts and uploaded in parallel, and at
+            # the time of that commit this check was added because otherwise
+            # older programs would get a success status and upload an empty
+            # object. Unfortuantely, it's very inefficient for fp's implemented
+            # by KeyFile (used, for example, by gsutil when copying between
+            # providers). So, we skip the check for the KeyFile case.
+            # TODO: At some point consider removing this seek/tell/seek
+            # logic, after enough time has passed that it's unlikely any
+            # programs remain that assume the older auto-rewind interface.
+            if not isinstance(fp, KeyFile):
+                spos = fp.tell()
+                fp.seek(0, os.SEEK_END)
+                if fp.tell() == spos:
+                    fp.seek(0, os.SEEK_SET)
+                    if fp.tell() != spos:
+                        # Raise an exception as this is likely a programming
+                        # error whereby there is data before the fp but nothing
+                        # after it.
+                        fp.seek(spos)
+                        raise AttributeError('fp is at EOF. Use rewind option '
+                                             'or seek() to data start.')
+                # seek back to the correct position.
+                fp.seek(spos)
+
+        if hasattr(fp, 'name'):
+            self.path = fp.name
+        if self.bucket is not None:
+            if isinstance(fp, KeyFile):
+                # Avoid EOF seek for KeyFile case as it's very inefficient.
+                key = fp.getkey()
+                size = key.size - fp.tell()
+                self.size = size
+                # At present both GCS and S3 use MD5 for the etag for
+                # non-multipart-uploaded objects. If the etag is 32 hex
+                # chars use it as an MD5, to avoid having to read the file
+                # twice while transferring.
+                if (re.match('^"[a-fA-F0-9]{32}"$', key.etag)):
+                    etag = key.etag.strip('"')
+                    md5 = (etag, base64.b64encode(binascii.unhexlify(etag)))
+            if size:
+                self.size = size
+            else:
+                # If md5 is provided, still need to size so
+                # calculate based on bytes to end of content
+                spos = fp.tell()
+                fp.seek(0, os.SEEK_END)
+                self.size = fp.tell() - spos
+                fp.seek(spos)
+                size = self.size
+
+            if md5 is None:
+                md5 = self.compute_md5(fp, size)
+            self.md5 = md5[0]
+            self.base64md5 = md5[1]
+
+            if self.name is None:
+                self.name = self.md5
+
+            if not replace:
+                if self.bucket.lookup(self.name):
+                    return
+
+            if if_generation is not None:
+                headers['x-goog-if-generation-match'] = str(if_generation)
+
+            if res_upload_handler:
+                res_upload_handler.send_file(self, fp, headers, cb, num_cb)
+            else:
+                # Not a resumable transfer so use basic send_file mechanism.
+                self.send_file(fp, headers, cb, num_cb, size=size)
+
+    def set_contents_from_filename(self, filename, headers=None, replace=True,
+                                   cb=None, num_cb=10, policy=None, md5=None,
+                                   reduced_redundancy=None,
+                                   res_upload_handler=None,
+                                   if_generation=None):
+        """
+        Store an object in GS using the name of the Key object as the
+        key in GS and the contents of the file named by 'filename'.
+        See set_contents_from_file method for details about the
+        parameters.
+
+        :type filename: string
+        :param filename: The name of the file that you want to put onto GS.
+
+        :type headers: dict
+        :param headers: (optional) Additional headers to pass along with the
+            request to GS.
+
+        :type replace: bool
+        :param replace: (optional) If True, replaces the contents of the file
+            if it already exists.
+
+        :type cb: function
+        :param cb: (optional) Callback function that will be called to report
+            progress on the upload. The callback should accept two integer
+            parameters, the first representing the number of bytes that have
+            been successfully transmitted to GS and the second representing the
+            total number of bytes that need to be transmitted.
+
+        :type num_cb: int
+        :param num_cb: (optional) If a callback is specified with the cb
+            parameter this parameter determines the granularity of the callback
+            by defining the maximum number of times the callback will be called
+            during the file transfer.
+
+        :type policy: :py:attribute:`boto.gs.acl.CannedACLStrings`
+        :param policy: (optional) A canned ACL policy that will be applied to
+            the new key in GS.
+
+        :type md5: tuple
+        :param md5: (optional) A tuple containing the hexdigest version of the
+            MD5 checksum of the file as the first element and the
+            Base64-encoded version of the plain checksum as the second element.
+            This is the same format returned by the compute_md5 method.
+
+            If you need to compute the MD5 for any reason prior to upload, it's
+            silly to have to do it twice so this param, if present, will be
+            used as the MD5 values of the file. Otherwise, the checksum will be
+            computed.
+
+        :type res_upload_handler: :py:class:`boto.gs.resumable_upload_handler.ResumableUploadHandler`
+        :param res_upload_handler: (optional) If provided, this handler will
+            perform the upload.
+
+        :type if_generation: int
+        :param if_generation: (optional) If set to a generation number, the
+            object will only be written to if its current generation number is
+            this value. If set to the value 0, the object will only be written
+            if it doesn't already exist.
+        """
+        # Clear out any previously computed hashes, since we are setting the
+        # content.
+        self.local_hashes = {}
+
+        with open(filename, 'rb') as fp:
+            self.set_contents_from_file(fp, headers, replace, cb, num_cb,
+                                        policy, md5, res_upload_handler,
+                                        if_generation=if_generation)
+
+    def set_contents_from_string(self, s, headers=None, replace=True,
+                                 cb=None, num_cb=10, policy=None, md5=None,
+                                 if_generation=None):
+        """
+        Store an object in GCS using the name of the Key object as the
+        key in GCS and the string 's' as the contents.
+        See set_contents_from_file method for details about the
+        parameters.
+
+        :type headers: dict
+        :param headers: Additional headers to pass along with the
+                        request to AWS.
+
+        :type replace: bool
+        :param replace: If True, replaces the contents of the file if
+                        it already exists.
+
+        :type cb: function
+        :param cb: a callback function that will be called to report
+                   progress on the upload. The callback should accept
+                   two integer parameters, the first representing the
+                   number of bytes that have been successfully
+                   transmitted to GCS and the second representing the
+                   size of the to be transmitted object.
+
+        :type cb: int
+        :param num_cb: (optional) If a callback is specified with
+                       the cb parameter this parameter determines the
+                       granularity of the callback by defining
+                       the maximum number of times the callback will
+                       be called during the file transfer.
+
+        :type policy: :class:`boto.gs.acl.CannedACLStrings`
+        :param policy: A canned ACL policy that will be applied to the
+                       new key in GCS.
+
+        :type md5: A tuple containing the hexdigest version of the MD5
+                   checksum of the file as the first element and the
+                   Base64-encoded version of the plain checksum as the
+                   second element. This is the same format returned by
+                   the compute_md5 method.
+        :param md5: If you need to compute the MD5 for any reason prior
+                    to upload, it's silly to have to do it twice so this
+                    param, if present, will be used as the MD5 values
+                    of the file. Otherwise, the checksum will be computed.
+
+        :type if_generation: int
+        :param if_generation: (optional) If set to a generation number, the
+            object will only be written to if its current generation number is
+            this value. If set to the value 0, the object will only be written
+            if it doesn't already exist.
+        """
+
+        # Clear out any previously computed md5 hashes, since we are setting the content.
+        self.md5 = None
+        self.base64md5 = None
+
+        fp = StringIO(get_utf8_value(s))
+        r = self.set_contents_from_file(fp, headers, replace, cb, num_cb,
+                                        policy, md5,
+                                        if_generation=if_generation)
+        fp.close()
+        return r
+
+    def set_contents_from_stream(self, *args, **kwargs):
+        """
+        Store an object using the name of the Key object as the key in
+        cloud and the contents of the data stream pointed to by 'fp' as
+        the contents.
+
+        The stream object is not seekable and total size is not known.
+        This has the implication that we can't specify the
+        Content-Size and Content-MD5 in the header. So for huge
+        uploads, the delay in calculating MD5 is avoided but with a
+        penalty of inability to verify the integrity of the uploaded
+        data.
+
+        :type fp: file
+        :param fp: the file whose contents are to be uploaded
+
+        :type headers: dict
+        :param headers: additional HTTP headers to be sent with the
+            PUT request.
+
+        :type replace: bool
+        :param replace: If this parameter is False, the method will first check
+            to see if an object exists in the bucket with the same key. If it
+            does, it won't overwrite it. The default value is True which will
+            overwrite the object.
+
+        :type cb: function
+        :param cb: a callback function that will be called to report
+            progress on the upload. The callback should accept two integer
+            parameters, the first representing the number of bytes that have
+            been successfully transmitted to GS and the second representing the
+            total number of bytes that need to be transmitted.
+
+        :type num_cb: int
+        :param num_cb: (optional) If a callback is specified with the
+            cb parameter, this parameter determines the granularity of
+            the callback by defining the maximum number of times the
+            callback will be called during the file transfer.
+
+        :type policy: :class:`boto.gs.acl.CannedACLStrings`
+        :param policy: A canned ACL policy that will be applied to the new key
+            in GS.
+
+        :type size: int
+        :param size: (optional) The Maximum number of bytes to read from
+            the file pointer (fp). This is useful when uploading a
+            file in multiple parts where you are splitting the file up
+            into different ranges to be uploaded. If not specified,
+            the default behaviour is to read all bytes from the file
+            pointer. Less bytes may be available.
+
+        :type if_generation: int
+        :param if_generation: (optional) If set to a generation number, the
+            object will only be written to if its current generation number is
+            this value. If set to the value 0, the object will only be written
+            if it doesn't already exist.
+        """
+        if_generation = kwargs.pop('if_generation', None)
+        if if_generation is not None:
+            headers = kwargs.get('headers', {})
+            headers['x-goog-if-generation-match'] = str(if_generation)
+            kwargs['headers'] = headers
+        super(Key, self).set_contents_from_stream(*args, **kwargs)
+
+    def set_acl(self, acl_or_str, headers=None, generation=None,
+                 if_generation=None, if_metageneration=None):
+        """Sets the ACL for this object.
+
+        :type acl_or_str: string or :class:`boto.gs.acl.ACL`
+        :param acl_or_str: A canned ACL string (see
+            :data:`~.gs.acl.CannedACLStrings`) or an ACL object.
+
+        :type headers: dict
+        :param headers: Additional headers to set during the request.
+
+        :type generation: int
+        :param generation: If specified, sets the ACL for a specific generation
+            of a versioned object. If not specified, the current version is
+            modified.
+
+        :type if_generation: int
+        :param if_generation: (optional) If set to a generation number, the acl
+            will only be updated if its current generation number is this value.
+
+        :type if_metageneration: int
+        :param if_metageneration: (optional) If set to a metageneration number,
+            the acl will only be updated if its current metageneration number is
+            this value.
+        """
+        if self.bucket is not None:
+            self.bucket.set_acl(acl_or_str, self.name, headers=headers,
+                                generation=generation,
+                                if_generation=if_generation,
+                                if_metageneration=if_metageneration)
+
+    def get_acl(self, headers=None, generation=None):
+        """Returns the ACL of this object.
+
+        :param dict headers: Additional headers to set during the request.
+
+        :param int generation: If specified, gets the ACL for a specific
+            generation of a versioned object. If not specified, the current
+            version is returned.
+
+        :rtype: :class:`.gs.acl.ACL`
+        """
+        if self.bucket is not None:
+            return self.bucket.get_acl(self.name, headers=headers,
+                                       generation=generation)
+
+    def get_xml_acl(self, headers=None, generation=None):
+        """Returns the ACL string of this object.
+
+        :param dict headers: Additional headers to set during the request.
+
+        :param int generation: If specified, gets the ACL for a specific
+            generation of a versioned object. If not specified, the current
+            version is returned.
+
+        :rtype: str
+        """
+        if self.bucket is not None:
+            return self.bucket.get_xml_acl(self.name, headers=headers,
+                                           generation=generation)
+
+    def set_xml_acl(self, acl_str, headers=None, generation=None,
+                     if_generation=None, if_metageneration=None):
+        """Sets this objects's ACL to an XML string.
+
+        :type acl_str: string
+        :param acl_str: A string containing the ACL XML.
+
+        :type headers: dict
+        :param headers: Additional headers to set during the request.
+
+        :type generation: int
+        :param generation: If specified, sets the ACL for a specific generation
+            of a versioned object. If not specified, the current version is
+            modified.
+
+        :type if_generation: int
+        :param if_generation: (optional) If set to a generation number, the acl
+            will only be updated if its current generation number is this value.
+
+        :type if_metageneration: int
+        :param if_metageneration: (optional) If set to a metageneration number,
+            the acl will only be updated if its current metageneration number is
+            this value.
+        """
+        if self.bucket is not None:
+            return self.bucket.set_xml_acl(acl_str, self.name, headers=headers,
+                                           generation=generation,
+                                           if_generation=if_generation,
+                                           if_metageneration=if_metageneration)
+
+    def set_canned_acl(self, acl_str, headers=None, generation=None,
+                       if_generation=None, if_metageneration=None):
+        """Sets this objects's ACL using a predefined (canned) value.
+
+        :type acl_str: string
+        :param acl_str: A canned ACL string. See
+            :data:`~.gs.acl.CannedACLStrings`.
+
+        :type headers: dict
+        :param headers: Additional headers to set during the request.
+
+        :type generation: int
+        :param generation: If specified, sets the ACL for a specific generation
+            of a versioned object. If not specified, the current version is
+            modified.
+
+        :type if_generation: int
+        :param if_generation: (optional) If set to a generation number, the acl
+            will only be updated if its current generation number is this value.
+
+        :type if_metageneration: int
+        :param if_metageneration: (optional) If set to a metageneration number,
+            the acl will only be updated if its current metageneration number is
+            this value.
+        """
+        if self.bucket is not None:
+            return self.bucket.set_canned_acl(
+                acl_str,
+                self.name,
+                headers=headers,
+                generation=generation,
+                if_generation=if_generation,
+                if_metageneration=if_metageneration
+            )
+
+    def compose(self, components, content_type=None, headers=None):
+        """Create a new object from a sequence of existing objects.
+
+        The content of the object representing this Key will be the
+        concatenation of the given object sequence. For more detail, visit
+
+            https://developers.google.com/storage/docs/composite-objects
+
+        :type components list of Keys
+        :param components List of gs.Keys representing the component objects
+
+        :type content_type (optional) string
+        :param content_type Content type for the new composite object.
+        """
+        compose_req = []
+        for key in components:
+            if key.bucket.name != self.bucket.name:
+                raise BotoClientError(
+                    'GCS does not support inter-bucket composing')
+
+            generation_tag = ''
+            if key.generation:
+                generation_tag = ('<Generation>%s</Generation>'
+                                  % str(key.generation))
+            compose_req.append('<Component><Name>%s</Name>%s</Component>' %
+                               (key.name, generation_tag))
+        compose_req_xml = ('<ComposeRequest>%s</ComposeRequest>' %
+                         ''.join(compose_req))
+        headers = headers or {}
+        if content_type:
+            headers['Content-Type'] = content_type
+        resp = self.bucket.connection.make_request(
+            'PUT', get_utf8_value(self.bucket.name), get_utf8_value(self.name),
+            headers=headers, query_args='compose',
+            data=get_utf8_value(compose_req_xml))
+        if resp.status < 200 or resp.status > 299:
+            raise self.bucket.connection.provider.storage_response_error(
+                resp.status, resp.reason, resp.read())
+
+        # Return the generation so that the result URI can be built with this
+        # for automatic parallel uploads.
+        return resp.getheader('x-goog-generation')