Support per-library serialization.

pkg/compiler/lib/src/serialization/serialization.dart

 // Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library dart2js.serialization;
 
 import '../elements/elements.dart';
 import '../constants/expressions.dart';
 import '../dart_types.dart';
 
@@ skipping 70 matching lines @@
 
   /// The name of the encoder kind. Use for error reporting.
   String get _name;
 
   void _checkKey(K key) {
     if (_map.containsKey(key)) {
       throw new StateError("$_name value '$key' already in $_map.");
     }
   }
 
+  /// Maps the [key] entry to the [value] in the encoded object.
+  void setValue(K key, Value value) {
+    _checkKey(key);
+    _map[key] = value;
+  }
+
+
   /// Maps the [key] entry to the enum [value] in the encoded object.
   void setEnum(K key, var value) {
     _checkKey(key);
     _map[key] = new EnumValue(value);
   }
 
   /// Maps the [key] entry to the [element] in the encoded object.
   void setElement(K key, Element element) {
     _checkKey(key);
     _map[key] = _serializer.createElementValue(element);
@@ skipping 470 matching lines @@
   final ObjectValue objectValue;
 
   DataObject(Value id, EnumValue kind)
       : this.id = id,
         this.objectValue =
             new ObjectValue(<Key, Value>{Key.ID: id, Key.KIND: kind});
 
   Map<Key, Value> get map => objectValue.map;
 }
 
+/// Function used to filter which element serialized.
+typedef bool IncludeElementFunction(Element element);

[Siggi Cherem (dart-lang), 2016/03/18 20:00:20]

IncludeElementFunction => ElementMatcher

[Johnni Winther, 2016/03/29 09:05:54]

Done.

+
+bool includeAllElements(Element element) => true;
+
 /// Serializer for the transitive closure of a collection of libraries.
 ///
 /// The serializer creates an [ObjectValue] model of the [Element], [DartType]
 /// and [ConstantExpression] values in the transitive closure of the serialized
 /// libraries.
 ///
 /// The model layout of the produced [objectValue] is:
 ///
 ///     { // Header object
 ///       Key.ELEMENTS: [
 ///         {...}, // [ObjectValue] of the 0th [Element].
 ///         ...
 ///         {...}, // [ObjectValue] of the n-th [Element].
 ///       ],
 ///       Key.TYPES: [
 ///         {...}, // [ObjectValue] of the 0th [DartType].
 ///         ...
 ///         {...}, // [ObjectValue] of the n-th [DartType].
 ///       ],
 ///       Key.CONSTANTS: [
 ///         {...}, // [ObjectValue] of the 0th [ConstantExpression].
 ///         ...
 ///         {...}, // [ObjectValue] of the n-th [ConstantExpression].
 ///       ],
 ///     }
 ///
-// TODO(johnniwinther): Support per-library serialization and dependencies
-// between serialized subcomponent.
+// TODO(johnniwinther): Support dependencies between serialized subcomponent.
 class Serializer {
-  final SerializationEncoder _encoder;
   List<SerializerPlugin> plugins = <SerializerPlugin>[];
 
+  Map<Uri, dynamic> _dependencyMap = <Uri, dynamic>{};
   Map<Element, DataObject> _elementMap = <Element, DataObject>{};
   Map<ConstantExpression, DataObject> _constantMap =
       <ConstantExpression, DataObject>{};
   Map<DartType, DataObject> _typeMap = <DartType, DataObject>{};
   List _pendingList = [];
+  IncludeElementFunction includeElement;

[Siggi Cherem (dart-lang), 2016/03/18 20:00:20]

rename to `shouldInclude`:

  ElementMatcher shouldInclude;

[Johnni Winther, 2016/03/29 09:05:54]

Done.

 
-  Serializer(this._encoder);
+  // TODO(johnniwinther): Replace [includeElement] with a general strategy.
+  Serializer({this.includeElement: includeAllElements});
 
   /// Add the transitive closure of [library] to this serializer.
   void serialize(LibraryElement library) {
-    // Call [_getElementDataObject] for its side-effect: To create a
+    // Call [_getElementId] for its side-effect: To create a
     // [DataObject] for [library]. If not already created, this will
     // put the serialization of [library] in the work queue.
-    _getElementDataObject(library);
+    _getElementId(library);
   }
 
   void _emptyWorklist() {
     while (_pendingList.isNotEmpty) {
       _pendingList.removeLast()();
     }
   }
 
-  /// Returns the [DataObject] for [element].
+  /// Returns the id [Value] for [element].
   ///
-  /// If [constant] has no [DataObject], a new [DataObject] is created and
-  /// encoding the [ObjectValue] for [constant] is put into the work queue of
+  /// If [element] has no [DataObject], a new [DataObject] is created and
+  /// encoding the [ObjectValue] for [element] is put into the work queue of
   /// this serializer.
-  DataObject _getElementDataObject(Element element) {
+  Value _getElementId(Element element) {
     if (element == null) {
       throw new ArgumentError('Serializer._getElementDataObject(null)');
     }
     DataObject dataObject = _elementMap[element];
     if (dataObject == null) {
-      // Run through [ELEMENT_SERIALIZERS] sequentially to find the one that
-      // deals with [element].
-      for (ElementSerializer serializer in ELEMENT_SERIALIZERS) {
-        SerializedElementKind kind = serializer.getSerializedKind(element);
-        if (kind != null) {
-          dataObject = new DataObject(
-              new IntValue(_elementMap.length), new EnumValue(kind));
-          _elementMap[element] = dataObject;
-          // Delay the serialization of the element itself to avoid loops, and
-          // to keep the call stack small.
-          _pendingList.add(() {
-            ObjectEncoder encoder = new ObjectEncoder(this, dataObject.map);
-            serializer.serialize(element, encoder, kind);
+      if (!includeElement(element)) {
+        if (element.isLibrary) {
+          LibraryElement library = element;
+          _elementMap[element] = dataObject = new DataObject(
+              new IntValue(_elementMap.length),
+              new EnumValue(SerializedElementKind.EXTERNAL_LIBRARY));
+          ObjectEncoder encoder = new ObjectEncoder(this, dataObject.map);
+          encoder.setUri(Key.URI, library.canonicalUri, library.canonicalUri);
+        } else if (element.isStatic) {
+          Value classId =_getElementId(element.enclosingClass);
+          _elementMap[element] = dataObject = new DataObject(
+              new IntValue(_elementMap.length),
+              new EnumValue(SerializedElementKind.EXTERNAL_STATIC_MEMBER));
+          ObjectEncoder encoder = new ObjectEncoder(this, dataObject.map);
+          encoder.setValue(Key.CLASS, classId);
+          encoder.setString(Key.NAME, element.name);
+        } else if (element.isConstructor) {
+          Value classId =_getElementId(element.enclosingClass);
+          _elementMap[element] = dataObject = new DataObject(
+              new IntValue(_elementMap.length),
+              new EnumValue(SerializedElementKind.EXTERNAL_CONSTRUCTOR));
+          ObjectEncoder encoder = new ObjectEncoder(this, dataObject.map);
+          encoder.setValue(Key.CLASS, classId);

[Siggi Cherem (dart-lang), 2016/03/18 20:00:20]

Just to check: does the classId include the library id?

[Johnni Winther, 2016/03/29 09:05:54]

The class will be an EXTERNAL_LIBRARY_MEMBER, which stores a library id.

+          encoder.setString(Key.NAME, element.name);
+        } else {
+          Value libraryId =_getElementId(element.library);
+          _elementMap[element] = dataObject = new DataObject(
+              new IntValue(_elementMap.length),
+              new EnumValue(SerializedElementKind.EXTERNAL_LIBRARY_MEMBER));
+          ObjectEncoder encoder = new ObjectEncoder(this, dataObject.map);
+          encoder.setValue(Key.LIBRARY, libraryId);
+          encoder.setString(Key.NAME, element.name);
+        }
+      } else {
+        // Run through [ELEMENT_SERIALIZERS] sequentially to find the one that
+        // deals with [element].
+        for (ElementSerializer serializer in ELEMENT_SERIALIZERS) {
+          SerializedElementKind kind = serializer.getSerializedKind(element);
+          if (kind != null) {
+            _elementMap[element] = dataObject = new DataObject(
+                new IntValue(_elementMap.length), new EnumValue(kind));
+            // Delay the serialization of the element itself to avoid loops, and
+            // to keep the call stack small.
+            _pendingList.add(() {
+              ObjectEncoder encoder = new ObjectEncoder(this, dataObject.map);
+              serializer.serialize(element, encoder, kind);
 
-            MapEncoder pluginData;
-            for (SerializerPlugin plugin in plugins) {
-              plugin.onElement(element, (String tag) {
-                if (pluginData == null) {
-                  pluginData = encoder.createMap(Key.DATA);
-                }
-                return pluginData.createObject(tag);
-              });
-            }
-          });
+              MapEncoder pluginData;
+              for (SerializerPlugin plugin in plugins) {
+                plugin.onElement(element, (String tag) {
+                  if (pluginData == null) {
+                    pluginData = encoder.createMap(Key.DATA);
+                  }
+                  return pluginData.createObject(tag);
+                });
+              }
+            });
+          }
         }
       }
     }
     if (dataObject == null) {
       throw new UnsupportedError(
           'Unsupported element: $element (${element.kind})');
     }
-    return dataObject;
+    return dataObject.id;
   }
 
   /// Creates the [ElementValue] for [element].
   ///
   /// If [element] has not already been serialized, it is added to the work
   /// queue of this serializer.
   ElementValue createElementValue(Element element) {
-    return new ElementValue(element, _getElementDataObject(element).id);
+    return new ElementValue(element, _getElementId(element));
   }
 
-  /// Returns the [DataObject] for [constant].
+  /// Returns the id [Value] for [constant].
   ///
   /// If [constant] has no [DataObject], a new [DataObject] is created and
   /// encoding the [ObjectValue] for [constant] is put into the work queue of
   /// this serializer.
-  DataObject _getConstantDataObject(ConstantExpression constant) {
+  Value _getConstantId(ConstantExpression constant) {
     return _constantMap.putIfAbsent(constant, () {
       DataObject dataObject = new DataObject(
           new IntValue(_constantMap.length), new EnumValue(constant.kind));
       // Delay the serialization of the constant itself to avoid loops, and to
       // keep the call stack small.
       _pendingList.add(() => _encodeConstant(constant, dataObject));
       return dataObject;
-    });
+    }).id;
   }
 
   /// Encodes [constant] into the [ObjectValue] of [dataObject].
   void _encodeConstant(ConstantExpression constant, DataObject dataObject) {
     const ConstantSerializer().visit(constant,
         new ObjectEncoder(this, dataObject.map));
   }
 
   /// Creates the [ConstantValue] for [constant].
   ///
   /// If [constant] has not already been serialized, it is added to the work
   /// queue of this serializer.
   ConstantValue createConstantValue(ConstantExpression constant) {
-    return new ConstantValue(constant, _getConstantDataObject(constant).id);
+    return new ConstantValue(constant, _getConstantId(constant));
   }
 
-  /// Returns the [DataObject] for [type].
+  /// Returns the id [Value] for [type].
   ///
   /// If [type] has no [DataObject], a new [DataObject] is created and
   /// encoding the [ObjectValue] for [type] is put into the work queue of this
   /// serializer.
-  DataObject _getTypeDataObject(DartType type) {
-    return _typeMap.putIfAbsent(type, () {
-      DataObject dataObject = new DataObject(
+  Value _getTypeId(DartType type) {
+    DataObject dataObject = _typeMap[type];
+    if (dataObject == null) {
+      _typeMap[type] = dataObject = new DataObject(
           new IntValue(_typeMap.length), new EnumValue(type.kind));
       // Delay the serialization of the type itself to avoid loops, and to keep
       // the call stack small.
       _pendingList.add(() => _encodeType(type, dataObject));
-      return dataObject;
-    });
+    }
+    return dataObject.id;
   }
 
   /// Encodes [type] into the [ObjectValue] of [dataObject].
   void _encodeType(DartType type, DataObject dataObject) {
     const TypeSerializer().visit(type, new ObjectEncoder(this, dataObject.map));
   }
 
   /// Creates the [TypeValue] for [type].
   ///
   /// If [type] has not already been serialized, it is added to the work
   /// queue of this serializer.
   TypeValue createTypeValue(DartType type) {
-    return new TypeValue(type, _getTypeDataObject(type).id);
+    return new TypeValue(type, _getTypeId(type));
   }
 
   ObjectValue get objectValue {
     _emptyWorklist();
 
     Map<Key, Value> map = <Key, Value>{};
     map[Key.ELEMENTS] =
         new ListValue(_elementMap.values.map((l) => l.objectValue).toList());
     if (_typeMap.isNotEmpty) {
       map[Key.TYPES] =
           new ListValue(_typeMap.values.map((l) => l.objectValue).toList());
     }
     if (_constantMap.isNotEmpty) {
       map[Key.CONSTANTS] =
           new ListValue(_constantMap.values.map((l) => l.objectValue).toList());
     }
     return new ObjectValue(map);
   }
 
-  String toText() {
-    return _encoder.encode(objectValue);
+  String toText(SerializationEncoder encoder) {
+    return encoder.encode(objectValue);
   }
 
   String prettyPrint() {
     PrettyPrintEncoder encoder = new PrettyPrintEncoder();
     return encoder.toText(objectValue);
   }
 }
 
 /// Plugin for serializing additional data for an [Element].
 class SerializerPlugin {
@@ skipping 10 matching lines @@
 class DeserializerPlugin {
   const DeserializerPlugin();
 
   /// Called upon the deserialization of [element].
   ///
   /// Use [getDecoder] to retrieve the data object with id [tag] stored for
   /// [element]. If not object is stored for [tag], [getDecoder] returns `null`.
   void onElement(Element element, ObjectDecoder getDecoder(String tag)) {}
 }
 
+/// Context for parallel deserialization.
+class DeserializationContext {
+  Map<Uri, LibraryElement> _uriMap = <Uri, LibraryElement>{};
+  List<Deserializer> deserializers = <Deserializer>[];
+
+  LibraryElement lookupLibrary(Uri uri) {
+    return _uriMap.putIfAbsent(uri, () {
+      for (Deserializer deserializer in deserializers) {
+        LibraryElement library = deserializer.lookupLibrary(uri);
+        if (library != null) {
+          return library;
+        }
+      }
+      return null;
+    });
+  }
+}
+
 /// Deserializer for a closed collection of libraries.
 // TODO(johnniwinther): Support per-library deserialization and dependencies
 // between deserialized subcomponent.
 class Deserializer {
+  final DeserializationContext context;
   final SerializationDecoder decoder;
   List<DeserializerPlugin> plugins = <DeserializerPlugin>[];
   ObjectDecoder _headerObject;
   ListDecoder _elementList;
   ListDecoder _typeList;
   ListDecoder _constantList;
   Map<int, Element> _elementMap = {};
   Map<int, DartType> _typeMap = {};
   Map<int, ConstantExpression> _constantMap = {};
 
-  Deserializer.fromText(String text, this.decoder) {
+  Deserializer.fromText(this.context, String text, this.decoder) {
     _headerObject = new ObjectDecoder(this, decoder.decode(text));
+    context.deserializers.add(this);
   }
 
   /// Returns the [ListDecoder] for the [Element]s in this deserializer.
   ListDecoder get elements {
     if (_elementList == null) {
       _elementList = _headerObject.getList(Key.ELEMENTS);
     }
     return _elementList;
   }
 
@@ skipping 31 matching lines @@
     }
     return null;
   }
 
   /// Returns the deserialized [Element] for [id].
   Element deserializeElement(int id) {
     if (id == null) throw new ArgumentError('Deserializer.getElement(null)');
     Element element = _elementMap[id];
     if (element == null) {
       ObjectDecoder decoder = elements.getObject(id);
-      element = ElementDeserializer.deserialize(decoder);
+      SerializedElementKind elementKind =
+          decoder.getEnum(Key.KIND, SerializedElementKind.values);
+      if (elementKind == SerializedElementKind.EXTERNAL_LIBRARY) {
+        Uri uri = decoder.getUri(Key.URI);
+        element = context.lookupLibrary(uri);
+        if (element == null) {
+          throw new StateError("Missing library for $uri.");
+        }
+      } else if (elementKind == SerializedElementKind.EXTERNAL_LIBRARY_MEMBER) {
+        LibraryElement library = decoder.getElement(Key.LIBRARY);
+        String name = decoder.getString(Key.NAME);
+        element = library.find(name);
+        if (element == null) {
+          throw new StateError("Missing library member for $name in $library.");
+        }
+      } else if (elementKind == SerializedElementKind.EXTERNAL_STATIC_MEMBER) {
+        ClassElement cls = decoder.getElement(Key.CLASS);
+        String name = decoder.getString(Key.NAME);
+        element = cls.lookupLocalMember(name);
+        if (element == null) {
+          throw new StateError("Missing static member for $name in $cls.");
+        }
+      } else if (elementKind == SerializedElementKind.EXTERNAL_CONSTRUCTOR) {
+        ClassElement cls = decoder.getElement(Key.CLASS);
+        String name = decoder.getString(Key.NAME);
+        element = cls.lookupConstructor(name);
+        if (element == null) {
+          throw new StateError("Missing constructor for $name in $cls.");
+        }
+      } else {
+        element = ElementDeserializer.deserialize(decoder, elementKind);
+      }
       _elementMap[id] = element;
 
       MapDecoder pluginData = decoder.getMap(Key.DATA, isOptional: true);
       // Call plugins even when there is no data, so they can take action in
       // this case.
       for (DeserializerPlugin plugin in plugins) {
         plugin.onElement(element,
           (String tag) => pluginData?.getObject(tag, isOptional: true));
       }
     }
@@ skipping 32 matching lines @@
 
   /// Returns the value used to store [key] as a property in the encoding an
   /// [ObjectValue].
   ///
   /// Different encodings have different restrictions and capabilities as how
   /// to store a [Key] value. For instance: A JSON encoding needs to convert
   /// [Key] to a [String] to store it in a JSON object; a Dart encoding can
   /// choose to store a [Key] as an [int] or as the [Key] itself.
   getObjectPropertyValue(Key key);
 }