Merging in latest boto.

boto/s3/connection.py

 # Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/
 # Copyright (c) 2010, Eucalyptus Systems, Inc.
 # All rights reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the
 # "Software"), to deal in the Software without restriction, including
 # without limitation the rights to use, copy, modify, merge, publish, dis-
 # tribute, sublicense, and/or sell copies of the Software, and to permit
 # persons to whom the Software is furnished to do so, subject to the fol-
 # lowing conditions:
 #
 # The above copyright notice and this permission notice shall be included
 # in all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
 
 import xml.sax
 import urllib, base64
 import time
 import boto.utils
 from boto.connection import AWSAuthConnection
 from boto import handler
-from boto.provider import Provider
 from boto.s3.bucket import Bucket
 from boto.s3.key import Key
 from boto.resultset import ResultSet
 from boto.exception import BotoClientError
 
 def check_lowercase_bucketname(n):
     """
     Bucket names must not contain uppercase characters. We check for
     this by appending a lowercase character and testing with islower().
     Note this also covers cases like numeric bucket names with dashes.
@@ skipping 16 matching lines @@
             "hosting calling format.")
     return True
 
 def assert_case_insensitive(f):
     def wrapper(*args, **kwargs):
         if len(args) == 3 and check_lowercase_bucketname(args[2]):
             pass
         return f(*args, **kwargs)
     return wrapper
 
-class _CallingFormat:
+class _CallingFormat(object):
+
+    def get_bucket_server(self, server, bucket):
+        return ''
 
     def build_url_base(self, connection, protocol, server, bucket, key=''):
         url_base = '%s://' % protocol
         url_base += self.build_host(server, bucket)
         url_base += connection.get_path(self.build_path_base(bucket, key))
         return url_base
 
     def build_host(self, server, bucket):
         if bucket == '':
             return server
         else:
             return self.get_bucket_server(server, bucket)
 
     def build_auth_path(self, bucket, key=''):
+        key = boto.utils.get_utf8_value(key)
         path = ''
         if bucket != '':
             path = '/' + bucket
         return path + '/%s' % urllib.quote(key)
 
     def build_path_base(self, bucket, key=''):
+        key = boto.utils.get_utf8_value(key)
         return '/%s' % urllib.quote(key)
 
 class SubdomainCallingFormat(_CallingFormat):
 
     @assert_case_insensitive
     def get_bucket_server(self, server, bucket):
         return '%s.%s' % (bucket, server)
 
 class VHostCallingFormat(_CallingFormat):
 
     @assert_case_insensitive
     def get_bucket_server(self, server, bucket):
         return bucket
 
 class OrdinaryCallingFormat(_CallingFormat):
     
     def get_bucket_server(self, server, bucket):
         return server
 
     def build_path_base(self, bucket, key=''):
+        key = boto.utils.get_utf8_value(key)
         path_base = '/'
         if bucket:
             path_base += "%s/" % bucket
         return path_base + urllib.quote(key)
 
+class ProtocolIndependentOrdinaryCallingFormat(OrdinaryCallingFormat):
+    
+    def build_url_base(self, connection, protocol, server, bucket, key=''):
+        url_base = '//'
+        url_base += self.build_host(server, bucket)
+        url_base += connection.get_path(self.build_path_base(bucket, key))
+        return url_base
+
 class Location:
     DEFAULT = '' # US Classic Region
     EU = 'EU'
     USWest = 'us-west-1'
+    APNortheast = 'ap-northeast-1'
     APSoutheast = 'ap-southeast-1'
 
 class S3Connection(AWSAuthConnection):
 
     DefaultHost = 's3.amazonaws.com'
     QueryString = 'Signature=%s&Expires=%d&AWSAccessKeyId=%s'
 
     def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
                  is_secure=True, port=None, proxy=None, proxy_port=None,
                  proxy_user=None, proxy_pass=None,
                  host=DefaultHost, debug=0, https_connection_factory=None,
-                 calling_format=SubdomainCallingFormat(), path='/', provider='aws',
-                 bucket_class=Bucket):
+                 calling_format=SubdomainCallingFormat(), path='/',
+                 provider='aws', bucket_class=Bucket, security_token=None):
         self.calling_format = calling_format
         self.bucket_class = bucket_class
         AWSAuthConnection.__init__(self, host,
                 aws_access_key_id, aws_secret_access_key,
                 is_secure, port, proxy, proxy_port, proxy_user, proxy_pass,
                 debug=debug, https_connection_factory=https_connection_factory,
-                path=path, provider=provider)
+                path=path, provider=provider, security_token=security_token)
 
     def _required_auth_capability(self):
         return ['s3']
 
     def __iter__(self):
         for bucket in self.get_all_buckets():
             yield bucket
 
     def __contains__(self, bucket_name):
-       return not (self.lookup(bucket_name) is None)
+        return not (self.lookup(bucket_name) is None)
 
     def set_bucket_class(self, bucket_class):
         """
         Set the Bucket class associated with this bucket.  By default, this
         would be the boto.s3.key.Bucket class but if you want to subclass that
         for some reason this allows you to associate your new class.
         
         :type bucket_class: class
         :param bucket_class: A subclass of Bucket that can be more specific
         """
         self.bucket_class = bucket_class
 
     def build_post_policy(self, expiration_time, conditions):
         """
         Taken from the AWS book Python examples and modified for use with boto
         """
         assert type(expiration_time) == time.struct_time, \
             'Policy document must include a valid expiration Time object'
 
         # Convert conditions object mappings to condition statements
 
         return '{"expiration": "%s",\n"conditions": [%s]}' % \
             (time.strftime(boto.utils.ISO8601, expiration_time), ",".join(conditions))
 
 
     def build_post_form_args(self, bucket_name, key, expires_in = 6000,
-                        acl = None, success_action_redirect = None, max_content_length = None,
-                        http_method = "http", fields=None, conditions=None):
+                             acl = None, success_action_redirect = None,
+                             max_content_length = None,
+                             http_method = "http", fields=None,
+                             conditions=None):
         """
         Taken from the AWS book Python examples and modified for use with boto
         This only returns the arguments required for the post form, not the actual form
         This does not return the file input field which also needs to be added
         
         :param bucket_name: Bucket to submit to
         :type bucket_name: string 
         
         :param key:  Key name, optionally add ${filename} to the end to attach the submitted filename
         :type key: string
@@ skipping 69 matching lines @@
         signature = self._auth_handler.sign_string(policy_b64)
         fields.append({"name": "signature", "value": signature})
         fields.append({"name": "key", "value": key})
 
         # HTTPS protocol will be used if the secure HTTP option is enabled.
         url = '%s://%s/' % (http_method, self.calling_format.build_host(self.server_name(), bucket_name))
 
         return {"action": url, "fields": fields}
 
 
-    def generate_url(self, expires_in, method, bucket='', key='',
-                     headers=None, query_auth=True, force_http=False):
+    def generate_url(self, expires_in, method, bucket='', key='', headers=None,
+                     query_auth=True, force_http=False, response_headers=None):
         if not headers:
             headers = {}
         expires = int(time.time() + expires_in)
         auth_path = self.calling_format.build_auth_path(bucket, key)
         auth_path = self.get_path(auth_path)
+        # Arguments to override response headers become part of the canonical
+        # string to be signed.
+        if response_headers:
+            response_hdrs = ["%s=%s" % (k, v) for k, v in
+                             response_headers.items()]
+            delimiter = '?' if '?' not in auth_path else '&'
+            auth_path = "%s%s%s" % (auth_path, delimiter, '&'.join(response_hdrs))
+        else:
+            response_headers = {}
         c_string = boto.utils.canonical_string(method, auth_path, headers,
                                                expires, self.provider)
         b64_hmac = self._auth_handler.sign_string(c_string)
         encoded_canonical = urllib.quote_plus(b64_hmac)
         self.calling_format.build_path_base(bucket, key)
         if query_auth:
             query_part = '?' + self.QueryString % (encoded_canonical, expires,
-                                             self.aws_access_key_id)
-            sec_hdr = self.provider.security_token_header
-            if sec_hdr in headers:
-                query_part += ('&%s=%s' % (sec_hdr,
-                                           urllib.quote(headers[sec_hdr])));
+                                                   self.aws_access_key_id)
+            # The response headers must also be GET parameters in the URL.
+            headers.update(response_headers)
+            hdrs = [ '%s=%s'%(name, urllib.quote(val)) for name,val in headers.items() ]
+            q_str = '&'.join(hdrs)
+            if q_str:
+                query_part += '&' + q_str
         else:
             query_part = ''
         if force_http:
             protocol = 'http'
             port = 80
         else:
             protocol = self.protocol
             port = self.port
         return self.calling_format.build_url_base(self, protocol, self.server_name(port),
                                                   bucket, key) + query_part
 
     def get_all_buckets(self, headers=None):
         response = self.make_request('GET', headers=headers)
         body = response.read()
         if response.status > 300:
             raise self.provider.storage_response_error(
                 response.status, response.reason, body)
         rs = ResultSet([('Bucket', self.bucket_class)])
         h = handler.XmlHandler(rs, self)
         xml.sax.parseString(body, h)
         return rs
 
     def get_canonical_user_id(self, headers=None):
         """
-        Convenience method that returns the "CanonicalUserID" of the user who's credentials
-        are associated with the connection.  The only way to get this value is to do a GET
-        request on the service which returns all buckets associated with the account.  As part
-        of that response, the canonical userid is returned.  This method simply does all of
-        that and then returns just the user id.
+        Convenience method that returns the "CanonicalUserID" of the
+        user who's credentials are associated with the connection.
+        The only way to get this value is to do a GET request on the
+        service which returns all buckets associated with the account.
+        As part of that response, the canonical userid is returned.
+        This method simply does all of that and then returns just the
+        user id.
 
         :rtype: string
         :return: A string containing the canonical user id.
         """
         rs = self.get_all_buckets(headers=headers)
         return rs.ID
 
     def get_bucket(self, bucket_name, validate=True, headers=None):
         bucket = self.bucket_class(self, bucket_name)
         if validate:
@@ skipping 70 matching lines @@
         host = self.calling_format.build_host(self.server_name(), bucket)
         if query_args:
             path += '?' + query_args
             boto.log.debug('path=%s' % path)
             auth_path += '?' + query_args
             boto.log.debug('auth_path=%s' % auth_path)
         return AWSAuthConnection.make_request(self, method, path, headers,
                 data, host, auth_path, sender,
                 override_num_retries=override_num_retries)