Isolates: allow sending of arbitrary objects in dart2js.

sdk/lib/_internal/compiler/js_lib/isolate_serialization.dart

+// Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+part of _isolate_helper;
+
+/// Serialize [message].
+_serializeMessage(message) {
+  return new _Serializer().serialize(message);
+}
+
+/// Deserialize [message].
+_deserializeMessage(message) {
+  return new _Deserializer().deserialize(message);
+}
+
+/// Clones the message.
+///
+/// Contrary to a `_deserializeMessage(_serializeMessage(x))` the `_clone`
+/// function will not try to adjust SendPort values and pass them through.
+_clone(message) {
+  _Serializer serializer = new _Serializer(serializeSendPorts: false);
+  _Deserializer deserializer = new _Deserializer();
+  return deserializer.deserialize(serializer.serialize(message));
+}
+
+class _Serializer {
+  final bool _serializeSendPorts;
+  Map<dynamic, int> serializedObjectIds = new Map<dynamic, int>.identity();
+
+  _Serializer({serializeSendPorts: true})
+      : _serializeSendPorts = serializeSendPorts;
+
+  /// Returns a message that can be transmitted through web-worker channels.
+  serialize(x) {
+    if (isPrimitive(x)) return serializePrimitive(x);
+
+    int id = serializedObjectIds[x];
+    if (id != null) return makeRef(id);
+
+    id = serializedObjectIds.length;
+    serializedObjectIds[x] = id;
+
+    if (x is JSIndexable) return serializeJSIndexable(x, id);
+    if (x is InternalMap) return serializeMap(x, id);
+
+    if (x is JSObject) return serializeJSObject(x, id);
+
+    // We should not have any interceptors any more.
+    if (x is Interceptor) unsupported(x);
+
+    if (x is RawReceivePort) {
+      unsupported(x, "RawReceivePorts can't be transmitted:");
+    }
+
+    // SendPorts need their workerIds adjusted (either during serialization or
+    // deserialization).
+    if (x is _NativeJsSendPort) return serializeJsSendPort(x, id);
+    if (x is _WorkerSendPort) return serializeWorkerSendPort(x, id);
+
+    if (x is Closure) return serializeClosure(x, id);
+
+    return serializeAny(x, id);
+  }
+
+  void unsupported(x, [String message]) {
+    if (message == null) message = "Can't transmit:";
+    throw new UnsupportedError("$message $x");
+  }
+
+  makeRef(int id) => ["ref", id];
+
+  bool isPrimitive(x) => x == null || x is String || x is num || x is bool;
+  serializePrimitive(primitive) => primitive;
+
+  serializeJSIndexable(JSIndexable indexable, int id) {
+    // Strings are JSIndexable but should have been treated earlier.
+    assert(indexable is! String);
+    if (indexable is TypedData) {
+      return ["typed", id, indexable];
+    }
+    List serialized = serializeArray(indexable);
+    if (indexable is JSFixedArray) return ["fixed", id, serialized];
+    if (indexable is JSExtendableArray) return ["extendable", id, serialized];
+    // MutableArray check must be last, since JSFixedArray and JSExtendableArray
+    // extend JSMutableArray.
+    if (indexable is JSMutableArray) return ["mutable", id, serialized];
+    if (indexable is JSArray) return ["const", id, serialized];

[sigurdm, 2014/11/24 09:06:09]

Maybe use a different keyword. 'const' has a certain meaning in dart.

[Lasse Reichstein Nielsen, 2014/11/24 12:55:41]

This is a const array (created using `const[...]`), right?

[floitsch, 2014/11/24 15:06:57]

This is the const List (as in const [1, 2]). Since we already tested for the
Mutable version this one must be the const one. Added comment.

[floitsch, 2014/11/24 15:06:57]

Yes. Since it is not a JSMutableArray it must be const.

+    unsupported(indexable, "Can't serialize indexable: ");
+    return null;
+  }
+
+  serializeArray(JSArray x) {
+    List serialized = [];
+    serialized.length = x.length;
+    for (int i = 0; i < x.length; i++) {
+      serialized[i] = serialize(x[i]);
+    }
+    return serialized;
+  }
+
+  serializeArrayInPlace(JSArray x) {
+    for (int i = 0; i < x.length; i++) {
+      x[i] = serialize(x[i]);
+    }
+    return x;
+  }
+
+  serializeMap(Map x, int id) {
+    return ['map', id,

[Lasse Reichstein Nielsen, 2014/11/24 12:55:42]

Do you have to send the id with the object - can't it be reconstructed based on
the order of objects in the serialized form? (I.e., a counter matching the
counter on the serializing side)?

[floitsch, 2014/11/24 15:06:57]

I thought it would be less brittle with the ids, but in the end we still rely on
the order of deserialization, since the 'ref' references code that must have
been deserialized before.
Changed the code and added test.

+            x.keys.map(serialize).toList(),
+            x.values.map(serialize).toList()];
+  }
+
+  serializeJSObject(JSObject x, int id) {
+    // TODO(floitsch): weed out more JS objects.

[Lasse Reichstein Nielsen, 2014/11/24 12:55:41]

Document what you are trying to guard against. The code is *not* obvious.

[floitsch, 2014/11/24 15:06:57]

Changed to checking the constructor, which is what I should have been doing in
the first place.

+    if (JS('bool', '(typeof #.__proto__) !== "undefined"', x) &&
+        JS('bool', '(typeof #.__proto__) !== "object"', x)) {

[Lasse Reichstein Nielsen, 2014/11/24 12:55:41]

Are there any prototypes that don't have typeof "object"?
For example `typeof new Date().__proto__` is "object".
And `typeof new Date()` is also "object", so using that as prototype won't be
caught.

[floitsch, 2014/11/24 15:06:56]

Functions have "function" as type for the prototype.
But obsolete, since I changed the test to look for the constructor property.

+      unsupported(x, "Only JS Objects without custom prototype are supported:");
+    }
+    List keys = JS('JSArray', 'Object.keys(#)', x);
+    List values = [];
+    values.length = keys.length;
+    for (int i = 0; i < keys.length; i++) {
+      values[i] = serialize(JS('', '#[#]', x, keys[i]));
+    }
+    return ['js-object', id, keys, values];
+  }
+
+  serializeWorkerSendPort(_WorkerSendPort x, int id) {
+    if (_serializeSendPorts) {
+      return ['sendport', id, x._workerId, x._isolateId, x._receivePortId];
+    }
+    return ['raw sendport', x];
+  }
+
+  serializeJsSendPort(_NativeJsSendPort x, int id) {
+    if (_serializeSendPorts) {
+      int workerId = _globalState.currentManagerId;
+      return ['sendport', id, workerId, x._isolateId, x._receivePort._id];
+    }
+    return ['raw sendport', x];
+  }
+
+  serializeCapability(CapabilityImpl x, int id) => ['capability', id, x._id];
+
+  serializeClosure(Closure x, int id) {
+    final name = IsolateNatives._getJSFunctionName(x);
+    if (name == null) {
+      unsupported(x, "Closures can't be transmitted:");
+    }
+    return ['function', id, name];
+  }
+
+  serializeAny(x, int id) {
+    var classExtractor = JS_EMBEDDED_GLOBAL('', CLASS_ID_EXTRACTOR);
+    var fieldsExtractor = JS_EMBEDDED_GLOBAL('', CLASS_FIELDS_EXTRACTOR);
+    String classId = JS('String', '#(#)', classExtractor, x);
+    List fields = JS('JSArray', '#(#)', fieldsExtractor, x);
+    return ['any', id, classId, serializeArrayInPlace(fields)];
+  }
+}
+
+class _Deserializer {
+  /// When `true`, encodes sendports specially so that they can be adjusted on
+  /// the receiving end.
+  ///
+  /// When `false`, sendports are cloned like any other object.
+  final bool _adjustSendPorts;
+
+  Map<int, dynamic> deserializedObjects = new Map<int, dynamic>();
+
+  _Deserializer({adjustSendPorts: true}) : _adjustSendPorts = adjustSendPorts;
+
+  /// Returns a message that can be transmitted through web-worker channels.
+  deserialize(x) {
+    if (isPrimitive(x)) return deserializePrimitive(x);
+
+    assert(x is List);
+    switch (x.first) {
+      case "ref": return deserializeRef(x);
+      case "typed": return deserializeTyped(x);
+      case "fixed": return deserializeFixed(x);
+      case "extendable": return deserializeExtendable(x);
+      case "mutable": return deserializeMutable(x);
+      case "const": return deserializeConst(x);
+      case "map": return deserializeMap(x);
+      case "sendport": return deserializeSendPort(x);
+      case "raw sendport": return deserializeRawSendPort(x);
+      case "js-object": return deserializeJSObject(x);
+      case "function": return deserializeClosure(x);
+      case "any": return deserializeAny(x);
+      default: throw "couldn't deserialize: $x";
+    }
+  }
+
+  bool isPrimitive(x) => x == null || x is String || x is num || x is bool;
+  deserializePrimitive(x) => x;
+
+  // ['ref', id].
+  deserializeRef(x) {
+    assert(x[0] == 'ref');
+    int id = x[1];
+    return deserializedObjects[id];
+  }
+
+  // ['typed', id, <typed array>].
+  TypedData deserializeTyped(x) {
+    assert(x[0] == 'typed');
+    int id = x[1];
+    TypedData result = x[2];
+    deserializedObjects[id] = result;
+    return result;
+  }
+
+  // Updates the given array in place with its deserialized content.
+  List deserializeArrayInPlace(JSArray x) {
+    for (int i = 0; i < x.length; i++) {
+      x[i] = deserialize(x[i]);
+    }
+    return x;
+  }
+
+  // ['fixed', id, <array>].
+  List deserializeFixed(x) {
+    assert(x[0] == 'fixed');
+    int id = x[1];
+    List result = x[2];
+    deserializedObjects[id] = result;
+    return new JSArray.markFixed(deserializeArrayInPlace(result));
+  }
+
+  // ['extendable', id, <array>].
+  List deserializeExtendable(x) {
+    assert(x[0] == 'extendable');
+    int id = x[1];
+    List result = x[2];
+    deserializedObjects[id] = result;
+    return new JSArray.markGrowable(deserializeArrayInPlace(result));
+  }
+
+  // ['mutable', id, <array>].
+  List deserializeMutable(x) {
+    assert(x[0] == 'mutable');
+    int id = x[1];
+    List result = x[2];
+    deserializedObjects[id] = result;
+    return deserializeArrayInPlace(result);
+  }
+
+  // ['const', id, <array>].
+  List deserializeConst(x) {
+    assert(x[0] == 'const');
+    int id = x[1];
+    List result = x[2];
+    deserializedObjects[id] = result;
+    // TODO(floitsch): need to mark list as non-changeable.
+    return new JSArray.markFixed(deserializeArrayInPlace(result));
+  }
+
+  // ['map', id, <key-list>, <value-list>].
+  Map deserializeMap(x) {
+    assert(x[0] == 'map');
+    int id = x[1];
+    List keys = x[2];
+    List values = x[3];
+    Map result = {};
+    deserializedObjects[x[1]] = result;
+    for (int i = 0; i < keys.length; i++) {
+      result[deserialize(keys[i])] = deserialize(values[i]);
+    }
+    return result;

[Lasse Reichstein Nielsen, 2014/11/24 12:55:41]

This special casing of just a single Map implementation isn't really enough.
In practice we need fixes for HashMap, [Linked]IdentityHashMap, [Linked]HashSet,
[Linked]IdentityHashSet, and even [Linked]CustomHashSet/Map (if the custom
functions are serializable).

And then there's all the other things that don't work.

I'm a little wary at adding this, because I don't want anyone to think that it
is actually even half a solution. I'd almost rather that it just doesn't work
until we fix it properly. Half (or less) solutions can live for years with
people working around it or just not using it :(

[floitsch, 2014/11/24 15:06:56]

This is just a special case for the extremely common hashmap. Otherwise sending
maps is very expensive. It wasn't added with the intent of fixing anything.

+  }
+
+  // ['sendport', id, <managerId>, <isolateId>, <receivePortId>].
+  SendPort deserializeSendPort(x) {
+    assert(x[0] == 'sendport');
+    int id = x[1];
+    int managerId = x[2];
+    int isolateId = x[3];
+    int receivePortId = x[4];
+    SendPort result;
+    // If two isolates are in the same manager, we use NativeJsSendPorts to
+    // deliver messages directly without using postMessage.
+    if (managerId == _globalState.currentManagerId) {
+      var isolate = _globalState.isolates[isolateId];
+      if (isolate == null) return null; // Isolate has been closed.
+      var receivePort = isolate.lookup(receivePortId);
+      if (receivePort == null) return null; // Port has been closed.
+      result = new _NativeJsSendPort(receivePort, isolateId);
+    } else {
+      result = new _WorkerSendPort(managerId, isolateId, receivePortId);
+    }
+    deserializedObjects[id] = result;
+    return result;
+  }
+
+  // ['raw sendport', <sendport>].
+  SendPort deserializeRawSendPort(x) {
+    assert(x[0] == 'raw sendport');
+    return x[1];

[Lasse Reichstein Nielsen, 2014/11/24 12:55:41]

This means the SendPort preserves its identity when sent through another
SendPort, unlike everything else.

[floitsch, 2014/11/24 15:06:57]

Yes. But anything else would have required big refactorings. Once we don't have
local isolates we can simplify lots of the code (including this part).

+  }
+
+  // ['js-object', id, <key-list>, <value-list>].
+  deserializeJSObject(x) {
+    assert(x[0] == 'js-object');
+    int id = x[1];
+    List keys = x[2];
+    List values = x[3];
+    var o = JS('', '{}');
+    deserializedObjects[id] = o;
+    for (int i = 0; i < keys.length; i++) {
+      JS('', '#[#]=#', o, keys[i], deserialize(values[i]));
+    }
+    return o;
+  }
+
+  // ['function', id, <name>].
+  Function deserializeClosure(x) {
+    assert(x[0] == 'function');
+    int id = x[1];
+    String name = x[2];
+    Function result = IsolateNatives._getJSFunctionFromName(name);
+    deserializedObjects[id] = result;
+    return result;
+  }
+
+  // ['any', id, <class-id>, <field-list>].
+  deserializeAny(x) {

[sigurdm, 2014/11/24 09:06:09]

Maybe name this 'deserializeDartObject'

[floitsch, 2014/11/24 15:06:56]

Done.

+    assert(x[0] == 'any');
+    int id = x[1];
+    String classId = x[2];
+    List fields = x[3];
+    var instanceFromClassId = JS_EMBEDDED_GLOBAL('', INSTANCE_FROM_CLASS_ID);
+    var initializeObject = JS_EMBEDDED_GLOBAL('', INITIALIZE_EMPTY_INSTANCE);
+
+    var emptyInstance = JS('', '#(#)', instanceFromClassId, classId);
+    deserializedObjects[id] = emptyInstance;
+    deserializeArrayInPlace(fields);
+    return JS('', '#(#, #, #)',
+              initializeObject, classId, emptyInstance, fields);
+  }
+}