mojo: Starting serialization for python bindings.

mojo/public/python/mojo/bindings/descriptor.py

Index: mojo/public/python/mojo/bindings/descriptor.py
diff --git a/mojo/public/python/mojo/bindings/descriptor.py b/mojo/public/python/mojo/bindings/descriptor.py
index 1a466b8c6a661d10cb1c89ec4ba5a1caf0fc081f..cd3aae7baebe60a28abc4ebe650e6241e23a4b15 100644
--- a/mojo/public/python/mojo/bindings/descriptor.py
+++ b/mojo/public/python/mojo/bindings/descriptor.py
@@ -7,9 +7,12 @@ The descriptors used to define generated elements of the mojo python bindings.
 """
 
 import array
+import struct
 
 # pylint: disable=F0401
-from mojo.system import Handle
+import mojo.bindings.serialization as serialization
+import mojo.system
+
 
 class Type(object):
   """Describes the type of a struct field or a method parameter,"""
@@ -29,6 +32,40 @@ class Type(object):
     return self.Convert(value)
 
 
+class SerializableType(Type):
+  """Describe a type that can be serialized by itself."""
+
+  def GetTypeCode(self):
+    """
+    Returns the type code (as defined by the struct module) used to encode
+    this type.
+    """
+    raise NotImplementedError()
+
+  def GetByteSize(self):
+    """
+    Returns the size of the encoding of this type.
+    """
+    raise NotImplementedError()
+
+  def Serialize(self, value, data_offset, data, handle_offset):
+    """
+    Serialize an value of this type.
+
+    Args:
+      value: the value to serialize.
+      data_offset: the offset to the end of the data bytearray. Used to encode
+                   pointers.
+      data: the bytearray to append additional data to.
+      handle_offset: the offset to use to encode handles.
+
+    Returns a a tuple where the first element is the value to encode, and the
+    second is the array of handles to add to the message.
+    """
+    raise NotImplementedError()
+

[sdefresne, 2014/09/15 08:46:54]

nit: only two blank lines

[qsr, 2014/09/15 11:01:53]

Done.

+
+
 class BooleanType(Type):

[sdefresne, 2014/09/15 08:46:54]

Are BooleanType really unserializable or did you forget making them so?

[qsr, 2014/09/15 11:01:52]

 You cannot serialize a boolean by itself, because boolean are packed into
bytes. Serialization is done via BooleanGroup

[sdefresne, 2014/09/15 11:14:46]

OK, this make sense.

[qsr, 2014/09/15 11:42:00]

Acknowledged.

   """Type object for booleans"""
 
@@ -36,20 +73,32 @@ class BooleanType(Type):
     return bool(value)
 
 
-class NumericType(Type):
+class NumericType(SerializableType):
   """Base Type object for all numeric types"""
 
+  def __init__(self, type_code):
+    SerializableType.__init__(self)
+    self.type_code = type_code
+
   def GetDefaultValue(self, value):
     if value is None:
-      return self.convert(0)
+      return self.Convert(0)
     return self.Convert(value)
 
+  def GetTypeCode(self):
+    return self.type_code
+
+  def GetByteSize(self):
+    raise NotImplementedError()

[sdefresne, 2014/09/15 08:46:54]

Why don't you implement this using struct.calcsize(self.GetTypeCode())?

[qsr, 2014/09/15 11:01:53]

 Refactored to take the type code in the constructor, and initialize the size
there too.

+
+  def Serialize(self, value, data_offset, data, handle_offset):
+    return (value, [])
 
 class IntegerType(NumericType):
   """Type object for integer types."""
 
-  def __init__(self, size, signed):
-    NumericType.__init__(self)
+  def __init__(self, type_code, size, signed):
+    NumericType.__init__(self, type_code)
     self.size = size
     self.signed = signed
     if self.signed:
@@ -69,13 +118,16 @@ class IntegerType(NumericType):
                           (value, self._min_value, self._max_value))
     return value
 
+  def GetByteSize(self):
+    return self.size / 8
+
 
 class FloatType(NumericType):
   """Type object for floating point number types."""
 
-  def __init__(self, size):
-    NumericType.__init__(self)
-    self.size = size
+  def __init__(self, type_code, byte_size):
+    NumericType.__init__(self, type_code)
+    self.byte_size = byte_size
 
   def Convert(self, value):
     if value is None:
@@ -84,8 +136,37 @@ class FloatType(NumericType):
       raise TypeError('%r is not a numeric type' % value)
     return float(value)
 
+  def GetByteSize(self):
+    return self.byte_size
 
-class StringType(Type):
+
+class PointerType(SerializableType):
+  """Base Type object for pointers."""
+
+  def __init__(self, nullable=False):
+    SerializableType.__init__(self)
+    self.nullable = nullable
+
+  def GetTypeCode(self):
+    return 'Q'
+
+  def GetByteSize(self):
+    return 8
+
+  def Serialize(self, value, data_offset, data, handle_offset):
+    if value is None and not self.nullable:
+      raise serialization.SerializationException(
+          "Trying to serialize null for non nullable type.")
+    if value is None:
+      return (0, [])
+    return self.SerializePointer(value, data_offset, data, handle_offset)
+
+  def SerializePointer(self, value, data_offset, data, handle_offset):
+    """Serialize the not null value."""
+    raise NotImplementedError()
+
+
+class StringType(PointerType):
   """
   Type object for strings.
 
@@ -94,8 +175,8 @@ class StringType(Type):
   """
 
   def __init__(self, nullable=False):
-    Type.__init__(self)
-    self.nullable = nullable
+    PointerType.__init__(self, nullable)
+    self._array_type = NativeArrayType('B', nullable)
 
   def Convert(self, value):
     if value is None or isinstance(value, unicode):
@@ -104,36 +185,82 @@ class StringType(Type):
       return unicode(value)
     raise TypeError('%r is not a string' % value)
 
+  def SerializePointer(self, value, data_offset, data, handle_offset):
+    string_array = array.array('b')
+    string_array.fromstring(value.encode('utf8'))
+    return self._array_type.SerializeArray(
+        string_array, data_offset, data, handle_offset)
+
 
-class HandleType(Type):
+class HandleType(SerializableType):
   """Type object for handles."""
 
   def __init__(self, nullable=False):
-    Type.__init__(self)
+    SerializableType.__init__(self)
     self.nullable = nullable
 
   def Convert(self, value):
     if value is None:
-      return Handle()
-    if not isinstance(value, Handle):
+      return mojo.system.Handle()
+    if not isinstance(value, mojo.system.Handle):
       raise TypeError('%r is not a handle' % value)
     return value
 
+  def GetTypeCode(self):
+    return 'i'
+
+  def GetByteSize(self):
+    return 4
+
+  def Serialize(self, value, data_offset, data, handle_offset):
+    if not value.IsValid() and not self.nullable:
+      raise serialization.SerializationException(
+          "Trying to serialize null for non nullable type.")
+    if not value.IsValid():
+      return (-1, [])
+    return (handle_offset, [value])
 
-class GenericArrayType(Type):
+
+class BaseArrayType(PointerType):
   """Abstract Type object for arrays."""
 
   def __init__(self, nullable=False, length=0):
-    Type.__init__(self)
-    self.nullable = nullable
+    PointerType.__init__(self, nullable)
     self.length = length
 
+  def SerializePointer(self, value, data_offset, data, handle_offset):
+    if self.length != 0 and len(value) != self.length:
+      raise serialization.SerializationException("Incorrect array size")
+    return self.SerializeArray(value, data_offset, data, handle_offset)
+
+  def SerializeArray(self, value, data_offset, data, handle_offset):
+    """Serialize the not null array."""
+    raise NotImplementedError()
+
+class BooleanArrayType(BaseArrayType):
 
-class PointerArrayType(GenericArrayType):
+  def __init__(self, nullable=False, length=0):
+    BaseArrayType.__init__(self, nullable, length)
+    self._array_type = NativeArrayType('B', nullable)
+
+  def Convert(self, value):
+    if value is None:
+      return value
+    return [TYPE_BOOL.Convert(x) for x in value]
+
+  def SerializeArray(self, value, data_offset, data, handle_offset):
+    groups = [value[i:i+8] for i in range(0, len(value), 8)]
+    converted = array.array('B', [_ConvertBooleansToByte(x) for x in groups])
+    return self._array_type.SerializeArray(
+        converted, data_offset, data, handle_offset)
+
+
+class GenericArrayType(BaseArrayType):
   """Type object for arrays of pointers."""
 
   def __init__(self, sub_type, nullable=False, length=0):
-    GenericArrayType.__init__(self, nullable, length)
+    BaseArrayType.__init__(self, nullable, length)
+    assert sub_type != TYPE_BOOL

[sdefresne, 2014/09/15 08:46:54]

Use "is not" instead of "!=" here as types are singleton in python.

[qsr, 2014/09/15 11:01:53]

Except that TYPE_BOOL is an instance, not a type. Am I missing something stupid.
Anyway, changed that to check that sub_type is serializable.

[sdefresne, 2014/09/15 11:14:46]

Well it would have been a singleton nonetheless. Changing to checking that the
type is serializable is probably better. Good.

[qsr, 2014/09/15 11:42:00]

Acknowledged.

     self.sub_type = sub_type
 
   def Convert(self, value):
@@ -141,12 +268,38 @@ class PointerArrayType(GenericArrayType):
       return value
     return [self.sub_type.Convert(x) for x in value]
 
-
-class NativeArrayType(GenericArrayType):
+  def SerializeArray(self, value, data_offset, data, handle_offset):
+    size = (serialization.HEADER_STRUCT.size +
+            self.sub_type.GetByteSize() * len(value))
+    data_end = len(data)
+    position = len(data) - data_offset
+    data.extend(bytearray(size +
+                          serialization.NeededPaddingForAlignment(size)))
+    returned_handles = []
+    to_pack = []
+    position = position + 2
+    for item in value:
+      (new_data, new_handles) = self.sub_type.Serialize(
+          item,
+          len(data) - position,
+          data,
+          handle_offset + len(returned_handles))
+      to_pack.append(new_data)
+      returned_handles.extend(new_handles)
+      position = position + self.sub_type.GetByteSize()
+    serialization.HEADER_STRUCT.pack_into(data, data_end, size, len(value))
+    struct.pack_into("%d%s" % (len(value), self.sub_type.GetTypeCode()),
+                     data,
+                     data_end + serialization.HEADER_STRUCT.size,
+                     *to_pack)
+    return (data_offset, returned_handles)
+
+
+class NativeArrayType(BaseArrayType):
   """Type object for arrays of native types."""
 
   def __init__(self, typecode, nullable=False, length=0):
-    GenericArrayType.__init__(self, nullable, length)
+    BaseArrayType.__init__(self, nullable, length)
     self.typecode = typecode
 
   def Convert(self, value):
@@ -156,12 +309,23 @@ class NativeArrayType(GenericArrayType):
       return value
     return array.array(self.typecode, value)
 
+  def SerializeArray(self, value, data_offset, data, handle_offset):
+    data_size = len(data)
+    data.extend(bytearray(serialization.HEADER_STRUCT.size))
+    data.extend(value.tostring())
+    data_length = len(data) - data_size
+    data.extend(bytearray(
+        serialization.NeededPaddingForAlignment(data_length)))
+    serialization.HEADER_STRUCT.pack_into(
+        data, data_size, data_length, len(value))
+    return (data_offset, [])
+
 
-class StructType(Type):
+class StructType(PointerType):
   """Type object for structs."""
 
   def __init__(self, struct_type, nullable=False):
-    Type.__init__(self)
+    PointerType.__init__(self)
     self.struct_type = struct_type
     self.nullable = nullable
 
@@ -175,30 +339,44 @@ class StructType(Type):
       return self.struct_type()
     return None
 
+  def SerializePointer(self, value, data_offset, data, handle_offset):
+    (new_data, new_handles) = value.Serialize(handle_offset)
+    data.extend(new_data)
+    return (data_offset, new_handles)
 
-class NoneType(Type):
+
+class NoneType(SerializableType):
   """Placeholder type, used temporarily until all mojo types are handled."""
 
   def Convert(self, value):
     return None
 
+  def GetTypeCode(self):
+    return 'B'
+
+  def GetByteSize(self):
+    return 1
+
+  def Serialize(self, value, data_offset, data, handle_offset):
+    return (0, [])
+
 
 TYPE_NONE = NoneType()
 
 TYPE_BOOL = BooleanType()
 
-TYPE_INT8 = IntegerType(8, True)
-TYPE_INT16 = IntegerType(16, True)
-TYPE_INT32 = IntegerType(32, True)
-TYPE_INT64 = IntegerType(64, True)
+TYPE_INT8 = IntegerType('b', 8, True)
+TYPE_INT16 = IntegerType('h', 16, True)
+TYPE_INT32 = IntegerType('i', 32, True)
+TYPE_INT64 = IntegerType('q', 64, True)
 
-TYPE_UINT8 = IntegerType(8, False)
-TYPE_UINT16 = IntegerType(16, False)
-TYPE_UINT32 = IntegerType(32, False)
-TYPE_UINT64 = IntegerType(64, False)
+TYPE_UINT8 = IntegerType('B', 8, False)
+TYPE_UINT16 = IntegerType('H', 16, False)
+TYPE_UINT32 = IntegerType('I', 32, False)
+TYPE_UINT64 = IntegerType('Q', 64, False)
 
-TYPE_FLOAT = FloatType(4)
-TYPE_DOUBLE = FloatType(8)
+TYPE_FLOAT = FloatType('f', 4)
+TYPE_DOUBLE = FloatType('d', 8)
 
 TYPE_STRING = StringType()
 TYPE_NULLABLE_STRING = StringType(True)
@@ -210,13 +388,88 @@ TYPE_NULLABLE_HANDLE = HandleType(True)
 class FieldDescriptor(object):
   """Describes a field in a generated struct."""
 
-  def __init__(self, name, field_type, offset,
-               bit_offset=None, default_value=None):
+  def __init__(self, name, field_type, field_number, default_value=None):
     self.name = name
     self.field_type = field_type
-    self.offset = offset
-    self.bit_offset = bit_offset
+    self.field_number = field_number
     self._default_value = default_value
 
   def GetDefaultValue(self):
     return self.field_type.GetDefaultValue(self._default_value)
+
+
+class FieldGroup(object):
+  """
+  Describe a list of field in the generated struct that must be
+  serialized/deserialized together.
+  """
+  def __init__(self, descriptors):
+    self.descriptors = descriptors
+
+  def GetDescriptors(self):
+    return self.descriptors
+
+  def GetTypeCode(self):
+    raise NotImplementedError()
+
+  def GetByteSize(self):
+    raise NotImplementedError()
+
+  def GetVersion(self):
+    raise NotImplementedError()
+
+  def Serialize(self, obj, data_offset, data, handle_offset):
+    raise NotImplementedError()
+
+
+class SingleFieldGroup(FieldGroup, FieldDescriptor):
+  """A FieldGroup that contains a single FieldDescriptor."""
+
+  def __init__(self, name, field_type, field_number, default_value=None):
+    FieldDescriptor.__init__(
+        self, name, field_type, field_number, default_value)
+    FieldGroup.__init__(self, [self])
+
+  def GetTypeCode(self):
+    return self.field_type.GetTypeCode()
+
+  def GetByteSize(self):
+    return self.field_type.GetByteSize()
+
+  def GetVersion(self):
+    return self.field_number
+
+  def Serialize(self, obj, data_offset, data, handle_offset):
+    value = getattr(obj, self.name)
+    return self.field_type.Serialize(value, data_offset, data, handle_offset)
+
+
+class BooleanGroup(FieldGroup):
+  """A FieldGroup to pack booleans."""
+  def __init__(self, descriptors):
+    FieldGroup.__init__(self, descriptors)
+    self.version = min([descriptor.field_number  for descriptor in descriptors])
+
+  def GetTypeCode(self):
+    return "B"
+
+  def GetByteSize(self):
+    return 1
+
+  def GetVersion(self):
+    return self.version
+
+  def Serialize(self, obj, data_offset, data, handle_offset):
+    value = _ConvertBooleansToByte(
+        [getattr(obj, field.name) for field in self.GetDescriptors()])
+    return (value, [])
+
+
+def _ConvertBooleansToByte(booleans):
+  """Pack a list of booleans into an integer."""
+  value = 0

[sdefresne, 2014/09/15 08:46:54]

This could be rewritten as "reduce(lambda x, y: x * 2 + y, booleans, 0)" but
maybe this is too compact...

[qsr, 2014/09/15 11:01:52]

Done.

+  for boolean in reversed(booleans):
+    value = value * 2
+    if boolean:
+      value = value + 1
+  return value