Merging in latest boto.

boto/sdb/db/manager/sdbmanager.py

 # Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/
 # Copyright (c) 2010 Chris Moyer http://coredumped.org/
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the
 # "Software"), to deal in the Software without restriction, including
 # without limitation the rights to use, copy, modify, merge, publish, dis-
 # tribute, sublicense, and/or sell copies of the Software, and to permit
 # persons to whom the Software is furnished to do so, subject to the fol-
 # lowing conditions:
 #
 # The above copyright notice and this permission notice shall be included
 # in all copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
 import boto
 import re
 from boto.utils import find_class
 import uuid
 from boto.sdb.db.key import Key
 from boto.sdb.db.model import Model
 from boto.sdb.db.blob import Blob
 from boto.sdb.db.property import ListProperty, MapProperty
-from datetime import datetime, date
-from boto.exception import SDBPersistenceError
+from datetime import datetime, date, time
+from boto.exception import SDBPersistenceError, S3ResponseError
 
 ISO8601 = '%Y-%m-%dT%H:%M:%SZ'
 
+class TimeDecodeError(Exception):
+    pass
 
-class SDBConverter:
+class SDBConverter(object):
     """
     Responsible for converting base Python types to format compatible with underlying
     database.  For SimpleDB, that means everything needs to be converted to a string
     when stored in SimpleDB and from a string when retrieved.
 
     To convert a value, pass it to the encode or decode method.  The encode method
     will take a Python native value and convert to DB format.  The decode method will
     take a DB format value and convert it to Python native format.  To find the appropriate
     method to call, the generic encode/decode methods will look for the type-specific
     method by searching for a method called "encode_<type name>" or "decode_<type name>".
     """
     def __init__(self, manager):
         self.manager = manager
         self.type_map = { bool : (self.encode_bool, self.decode_bool),
                           int : (self.encode_int, self.decode_int),
                           long : (self.encode_long, self.decode_long),
                           float : (self.encode_float, self.decode_float),
                           Model : (self.encode_reference, self.decode_reference),
                           Key : (self.encode_reference, self.decode_reference),
                           datetime : (self.encode_datetime, self.decode_datetime),
                           date : (self.encode_date, self.decode_date),
+                          time : (self.encode_time, self.decode_time),
                           Blob: (self.encode_blob, self.decode_blob),
+                          str: (self.encode_string, self.decode_string),
                       }
 
     def encode(self, item_type, value):
         try:
             if Model in item_type.mro():
                 item_type = Model
         except:
             pass
         if item_type in self.type_map:
             encode = self.type_map[item_type][0]
@@ skipping 17 matching lines @@
         # Just enumerate(value) won't work here because
         # we need to add in some zero padding
         # We support lists up to 1,000 attributes, since
         # SDB technically only supports 1024 attributes anyway
         values = {}
         for k,v in enumerate(value):
             values["%03d" % k] = v
         return self.encode_map(prop, values)
 
     def encode_map(self, prop, value):
+        import urllib
         if value == None:
             return None
         if not isinstance(value, dict):
             raise ValueError, 'Expected a dict value, got %s' % type(value)
         new_value = []
         for key in value:
             item_type = getattr(prop, "item_type")
             if Model in item_type.mro():
                 item_type = Model
             encoded_value = self.encode(item_type, value[key])
             if encoded_value != None:
-                new_value.append('%s:%s' % (key, encoded_value))
+                new_value.append('%s:%s' % (urllib.quote(key), encoded_value))
         return new_value
 
     def encode_prop(self, prop, value):
         if isinstance(prop, ListProperty):
             return self.encode_list(prop, value)
         elif isinstance(prop, MapProperty):
             return self.encode_map(prop, value)
         else:
             return self.encode(prop.data_type, value)
 
@@ skipping 19 matching lines @@
             value = [value]
         ret_value = {}
         item_type = getattr(prop, "item_type")
         for val in value:
             k,v = self.decode_map_element(item_type, val)
             ret_value[k] = v
         return ret_value
 
     def decode_map_element(self, item_type, value):
         """Decode a single element for a map"""
+        import urllib
         key = value
         if ":" in value:
             key, value = value.split(':',1)
+            key = urllib.unquote(key)
         if Model in item_type.mro():
             value = item_type(id=value)
         else:
             value = self.decode(item_type, value)
         return (key, value)
 
     def decode_prop(self, prop, value):
         if isinstance(prop, ListProperty):
             return self.decode_list(prop, value)
         elif isinstance(prop, MapProperty):
@@ skipping 103 matching lines @@
             return value
         return value.isoformat()
 
     def decode_date(self, value):
         try:
             value = value.split("-")
             return date(int(value[0]), int(value[1]), int(value[2]))
         except:
             return None
 
+    encode_time = encode_date
+
+    def decode_time(self, value):
+        """ converts strings in the form of HH:MM:SS.mmmmmm
+            (created by datetime.time.isoformat()) to
+            datetime.time objects.
+
+            Timzone-aware strings ("HH:MM:SS.mmmmmm+HH:MM") won't
+            be handled right now and will raise TimeDecodeError.
+        """
+        if '-' in value or '+' in value:
+            # TODO: Handle tzinfo
+            raise TimeDecodeError("Can't handle timezone aware objects: %r" % value)
+        tmp = value.split('.')
+        arg = map(int, tmp[0].split(':'))
+        if len(tmp) == 2:
+            arg.append(int(tmp[1]))
+        return time(*arg)
+
     def encode_reference(self, value):
         if value in (None, 'None', '', ' '):
             return None
         if isinstance(value, str) or isinstance(value, unicode):
             return value
         else:
             return value.id
 
     def decode_reference(self, value):
         if not value or value == "None":
@@ skipping 24 matching lines @@
         return value.id
 
 
     def decode_blob(self, value):
         if not value:
             return None
         match = re.match("^s3:\/\/([^\/]*)\/(.*)$", value)
         if match:
             s3 = self.manager.get_s3_connection()
             bucket = s3.get_bucket(match.group(1), validate=False)
-            key = bucket.get_key(match.group(2))
+            try:
+                key = bucket.get_key(match.group(2))
+            except S3ResponseError, e:
+                if e.reason != "Forbidden":
+                    raise
+                return None
         else:
             return None
         if key:
             return Blob(file=key, id="s3://%s/%s" % (key.bucket.name, key.name))
         else:
             return None
 
+    def encode_string(self, value):
+        """Convert ASCII, Latin-1 or UTF-8 to pure Unicode"""
+        if not isinstance(value, str): return value
+        try:
+            return unicode(value, 'utf-8')
+        except: # really, this should throw an exception.
+                # in the interest of not breaking current
+                # systems, however:
+            arr = []
+            for ch in value:
+                arr.append(unichr(ord(ch)))
+            return u"".join(arr)
+
+    def decode_string(self, value):
+        """Decoding a string is really nothing, just
+        return the value as-is"""
+        return value
+
 class SDBManager(object):
     
     def __init__(self, cls, db_name, db_user, db_passwd,
                  db_host, db_port, db_table, ddl_dir, enable_ssl, consistent=None):
         self.cls = cls
         self.db_name = db_name
         self.db_user = db_user
         self.db_passwd = db_passwd
         self.db_host = db_host
         self.db_port = db_port
@@ skipping 15 matching lines @@
             self._connect()
         return self._sdb
 
     @property
     def domain(self):
         if self._domain is None:
             self._connect()
         return self._domain
 
     def _connect(self):
-        self._sdb = boto.connect_sdb(aws_access_key_id=self.db_user,
-                                    aws_secret_access_key=self.db_passwd,
-                                    is_secure=self.enable_ssl)
+        args = dict(aws_access_key_id=self.db_user,
+                    aws_secret_access_key=self.db_passwd,
+                    is_secure=self.enable_ssl)
+        try:
+            region = [x for x in boto.sdb.regions() if x.endpoint == self.db_host][0]
+            args['region'] = region
+        except IndexError:
+            pass
+        self._sdb = boto.connect_sdb(**args)
         # This assumes that the domain has already been created
         # It's much more efficient to do it this way rather than
         # having this make a roundtrip each time to validate.
         # The downside is that if the domain doesn't exist, it breaks
         self._domain = self._sdb.lookup(self.db_name, validate=False)
         if not self._domain:
             self._domain = self._sdb.create_domain(self.db_name)
 
     def _object_lister(self, cls, query_lister):
         for item in query_lister:
@@ skipping 106 matching lines @@
             if not(op in ["like", "not like"] and val.startswith("%")):
                 val = "%%:%s" % val
         return "%s %s '%s'" % (name, op, val.replace("'", "''"))
 
     def _build_filter_part(self, cls, filters, order_by=None, select=None):
         """
         Build the filter part
         """
         import types
         query_parts = []
+
         order_by_filtered = False
+
         if order_by:
             if order_by[0] == "-":
                 order_by_method = "DESC";
                 order_by = order_by[1:]
             else:
                 order_by_method = "ASC";
+
+        if select:
+            if order_by and order_by in select:
+                order_by_filtered = True
+            query_parts.append("(%s)" % select)
+
         if isinstance(filters, str) or isinstance(filters, unicode):
-            query = "WHERE `__type__` = '%s' AND %s" % (cls.__name__, filters)
-            if order_by != None:
+            query = "WHERE %s AND `__type__` = '%s'" % (filters, cls.__name__)
+            if order_by in ["__id__", "itemName()"]:
+                query += " ORDER BY itemName() %s" % order_by_method
+            elif order_by != None:
                 query += " ORDER BY `%s` %s" % (order_by, order_by_method)
             return query
 
         for filter in filters:
             filter_parts = []
             filter_props = filter[0]
             if type(filter_props) != list:
                 filter_props = [filter_props]
             for filter_prop in filter_props:
                 (name, op) = filter_prop.strip().split(" ", 1)
@@ skipping 21 matching lines @@
             query_parts.append("(%s)" % (" or ".join(filter_parts)))
 
 
         type_query = "(`__type__` = '%s'" % cls.__name__
         for subclass in self._get_all_decendents(cls).keys():
             type_query += " or `__type__` = '%s'" % subclass
         type_query +=")"
         query_parts.append(type_query)
 
         order_by_query = ""
+
         if order_by:
             if not order_by_filtered:
                 query_parts.append("`%s` LIKE '%%'" % order_by)
-            order_by_query = " ORDER BY `%s` %s" % (order_by, order_by_method)
-
-        if select:
-            query_parts.append("(%s)" % select)
+            if order_by in ["__id__", "itemName()"]:
+                order_by_query = " ORDER BY itemName() %s" % order_by_method
+            else:
+                order_by_query = " ORDER BY `%s` %s" % (order_by, order_by_method)
 
         if len(query_parts) > 0:
             return "WHERE %s %s" % (" AND ".join(query_parts), order_by_query)
         else:
             return ""
 
 
     def _get_all_decendents(self, cls):
         """Get all decendents for a given class"""
         decendents = {}
         for sc in cls.__sub_classes__:
             decendents[sc.__name__] = sc
             decendents.update(self._get_all_decendents(sc))
         return decendents
 
     def query_gql(self, query_string, *args, **kwds):
         raise NotImplementedError, "GQL queries not supported in SimpleDB"
 
-    def save_object(self, obj):
+    def save_object(self, obj, expected_value=None):
         if not obj.id:
             obj.id = str(uuid.uuid4())
 
         attrs = {'__type__' : obj.__class__.__name__,
                  '__module__' : obj.__class__.__module__,
                  '__lineage__' : obj.get_lineage()}
         del_attrs = []
         for property in obj.properties(hidden=False):
             value = property.get_value_for_datastore(obj)
             if value is not None:
                 value = self.encode_value(property, value)
             if value == []:
                 value = None
             if value == None:
                 del_attrs.append(property.name)
                 continue
             attrs[property.name] = value
             if property.unique:
                 try:
                     args = {property.name: value}
                     obj2 = obj.find(**args).next()
                     if obj2.id != obj.id:
                         raise SDBPersistenceError("Error: %s must be unique!" % property.name)
                 except(StopIteration):
                     pass
-        self.domain.put_attributes(obj.id, attrs, replace=True)
+        # Convert the Expected value to SDB format
+        if expected_value:
+            prop = obj.find_property(expected_value[0])
+            v = expected_value[1]
+            if v is not None and not type(v) == bool:
+                v = self.encode_value(prop, v)
+            expected_value[1] = v
+        self.domain.put_attributes(obj.id, attrs, replace=True, expected_value=expected_value)
         if len(del_attrs) > 0:
             self.domain.delete_attributes(obj.id, del_attrs)
         return obj
 
     def delete_object(self, obj):
         self.domain.delete_attributes(obj.id)
 
     def set_property(self, prop, obj, name, value):
+        setattr(obj, name, value)
         value = prop.get_value_for_datastore(obj)
         value = self.encode_value(prop, value)
         if prop.unique:
             try:
                 args = {prop.name: value}
                 obj2 = obj.find(**args).next()
                 if obj2.id != obj.id:
                     raise SDBPersistenceError("Error: %s must be unique!" % prop.name)
             except(StopIteration):
                 pass
@@ skipping 19 matching lines @@
     def get_key_value(self, obj, name):
         a = self.domain.get_attributes(obj.id, name,consistent_read=self.consistent)
         if a.has_key(name):
             return a[name]
         else:
             return None
     
     def get_raw_item(self, obj):
         return self.domain.get_item(obj.id)