Make input/output formats pluggable, adapt to new libraries

pkg/serialization/lib/src/format.dart

Index: pkg/serialization/lib/src/format.dart
diff --git a/pkg/serialization/lib/src/format.dart b/pkg/serialization/lib/src/format.dart
new file mode 100644
index 0000000000000000000000000000000000000000..4ab20d43f5343f9d4800f4d560cee1ac2a9b5ab5
--- /dev/null
+++ b/pkg/serialization/lib/src/format.dart
@@ -0,0 +1,423 @@
+part of serialization;
+
+/**
+ * An abstract class for serialization formats. Subclasses define how data
+ * is read or written to a particular output mechanism.
+ */
+abstract class Format {
+  /**
+   * Return true if this format stores primitives in their own area and uses
+   * references to them (e.g. [SimpleFlatFormat]) and false if primitives
+   * are stored directly (e.g. [SimpleJsonFormat], [SimpleMapFormat]).
+   */
+  bool get shouldUseReferencesForPrimitives => false;
+
+  /**
+   * Generate output for [w] and return it. The particular form of the output
+   * will depend on the format. The format can assume that [w] has data
+   * generated by rules in a series of lists, and that each list will contain
+   * either primitives (null, bool, num, String), Lists or Maps. The Lists or
+   * Maps may contain any of the same things recursively, or may contain
+   * Reference objects. For lists and maps the rule will tell us if they can
+   * be of variable length or not. The format is allowed to operate
+   * destructively on the rule data.
+   */
+  generateOutput(Writer w);
+
+  /**
+   * Read the data from [input] in the context of [reader] and return it as a
+   * Map with entries for "roots", "data" and "rules", which the reader knows
+   * how to interpret. The type of [input] will depend on the particular format.
+   */
+  Map<String, dynamic> read(input, Reader reader);
+}
+
+/**
+ * A format that stores the data in maps which are converted into a JSON
+ * string. Note that the maps aren't nested, and it handles cyclic references
+ * by converting object references to [Reference] objects. If you want simple
+ * acyclic JSON look at [SimpleJsonFormat].
+ */
+class SimpleMapFormat extends Format {
+
+  /**
+   * Generate output for this format from [w] and return it as a String which
+   * is the [json] representation of a nested Map structure. The top level has
+   * 3 fields, "rules" which may hold a definition of the rules used,
+   * "data" which holds the serialized data, and "roots", which holds
+   * [Reference] objects indicating the root objects. Note that roots are
+   * necessary because the data is organized in the same way as the object
+   * structure, it's a list of lists holding self-contained maps which only
+   * refer to other parts via [Reference] objects.
+   * This effectively defines a custom JSON serialization format, although
+   * the details of the format vary depending which rules were used.
+   */
+  String generateOutput(Writer w) {
+    var result = {
+      "rules" : w.serializedRules(),
+      "data" : w.states,
+      "roots" : w._rootReferences()
+    };
+    return json.stringify(result);
+  }
+
+  /**
+   * Read a [json] encoded string representing serialized data in this format
+   * and return the nested Map representation described in [generateOutput]. If
+   * the data also includes rule definitions, then these will replace the rules
+   * in the [Serialization] for [reader].
+   */
+  Map<String, dynamic> read(String input, Reader reader) {
+    var topLevel = json.parse(input);
+    var ruleString = topLevel["rules"];
+    reader.readRules(ruleString);
+    return topLevel;
+  }
+}
+
+/**
+ * A format for "normal" JSON representation of objects. It stores
+ * the fields of the objects as nested maps, and doesn't allow cycles. This can
+ * be useful in talking to existing APIs that expect JSON format data. However,
+ * note that since the classes of objects aren't stored, this isn't enough
+ * information to read back the objects. This format also doesn't support the
+ * [selfDescriptive] option on the [Serialization], as storing the rules.
+ * If the [storeRoundTripData] field of the format is set to true, then this
+ * will store the rule number along with the data, allowing reconstruction.
+ */
+class SimpleJsonFormat extends Format {
+
+  /**
+   * Indicate if we should store rule numbers with map/list data so that we
+   * will know how to reconstruct it with a read operation. If we don't, this
+   * will be more compliant with things that expect known format JSON as input,
+   * but we won't be able to read back the objects.
+   */
+  final bool storeRoundTripInfo;
+
+  SimpleJsonFormat({this.storeRoundTripInfo : false});
+
+  /**
+   * Generate output for this format from [w] and return it as a String which
+   * is the [json] representation of a nested Map structure.
+   */
+  String generateOutput(Writer w) {
+    jsonify(w);
+    return json.stringify(w.stateForReference(w._rootReferences().first));
+  }
+
+  /**
+   * Convert the data generated by the rules to have nested maps instead
+   * of Reference objects and to add rule numbers if [storeRoundTripInfo]
+   * is true.
+   */
+  jsonify(Writer w) {
+    for (var eachRule in w.rules) {
+      var ruleData = w.states[eachRule.number];
+      jsonifyForRule(ruleData, w, eachRule);
+    }
+  }
+
+  /**
+   * For a particular [rule] modify the [ruleData] to conform to this format.
+   */
+  jsonifyForRule(List ruleData, Writer w, SerializationRule rule) {
+    for (var i = 0; i < ruleData.length; i++) {
+      var each = ruleData[i];
+      if (each is List) {
+        jsonifyEntry(each, w);
+        if (storeRoundTripInfo) ruleData[i].add(rule.number);
+      } else if (each is Map) {
+        jsonifyEntry(each, w);
+        if (storeRoundTripInfo) each["__rule"] = rule.number;
+      }
+    }
+  }
+
+  /**
+   * For one particular entry, which is either a Map or a List, update it
+   * to turn References into a nested List/Map.
+   */
+  jsonifyEntry(map, Writer w) {
+    keysAndValues(map).forEach((key, value) {
+      if (value is Reference) map[key] = w.stateForReference(value);
+    });
+  }
+
+  /**
+   * Read a [json] encoded string representing serialized data in this format
+   * and return the Map representation that the reader expects, with top-level
+   * entries for "rules", "data", and "roots". Nested lists/maps will be
+   * converted into Reference objects. Note that if the data was not written
+   * with [storeRoundTripInfo] true this will fail.
+   */
+  Map<String, dynamic> read(String input, Reader r) {
+    var data = json.parse(input);
+    var result = {};
+    result["rules"] = null;
+    var ruleData =
+        new List(r.serialization.rules.length).mappedBy((x) => []).toList();
+    var rootRule = data["__rule"];
+    var top = recursivelyFixUp(data, r, ruleData);
+    result["data"] = ruleData;
+    result["roots"] = [top];
+    return result;
+  }
+
+  /**
+   * Convert nested references in [data] into [Reference] objects.
+   */
+  recursivelyFixUp(data, Reader r, List result) {
+    if (isPrimitive(data)) {
+      result[r._primitiveRule().number].add(data);
+      return data;
+    }
+    var ruleNumber =
+        (data is List) ? data.removeLast() : data.remove("__rule");
+    var newData = values(data).mappedBy(
+        (x) => recursivelyFixUp(x, r, result));
+    result[ruleNumber].add(newData);
+    return new Reference(this, ruleNumber, result[ruleNumber].length - 1);
+  }
+}
+
+/**
+ * Writes to a simple mostly-flat format. Details are subject to change.
+ * Right now this produces a List containing null, num, and String. This is
+ * more space-efficient than the map formats, but much less human-readable.
+ * Simple usage is to turn this into JSON for transmission.
+ */
+class SimpleFlatFormat extends Format {
+  bool get shouldUseReferencesForPrimitives => true;
+
+  /**
+   * For each rule we store data to indicate whether it will be reconstructed
+   * as a primitive, a list or a map.
+   */
+  static final int STORED_AS_LIST = 1;
+  static final int STORED_AS_MAP = 2;
+  static final int STORED_AS_PRIMITIVE = 3;
+
+  /**
+   * Generate output for this format from [w]. This will return a List with
+   * three entries, corresponding to the "rules", "data", and "roots" from
+   * [SimpleMapFormat]. The data is stored as a single List containing
+   * primitives.
+   */
+  List generateOutput(Writer w) {
+    var result = new List(3);
+    var flatData = [];
+    for (var eachRule in w.rules) {
+      var ruleData = w.states[eachRule.number];
+      flatData.add(ruleData.length);
+      writeStateInto(eachRule, ruleData, flatData);
+    }
+    result[0] = w.serializedRules();
+    result[1] = flatData;
+    result[2] = new List();
+    w._rootReferences().forEach((x) => x.writeToList(result[2]));
+    return result;
+  }
+
+  /**
+   * Writes the data from [rule] into the [target] list.
+   */
+  void writeStateInto(SerializationRule rule, List ruleData, List target) {
+    if (!ruleData.isEmpty) {
+      var sample = ruleData.first;
+      if (sample is List) {
+        writeLists(rule, ruleData, target);
+      } else if (sample is Map) {
+        writeMaps(rule, ruleData, target);
+      } else {
+        writeObjects(ruleData, target);
+      }
+    } else {
+      // If there is no data, write a zero for the length.
+      target.add(0);
+    }
+  }
+
+  /**
+   * Write [entries], which contains Lists. Either the lists are variable
+   * length, in which case we add a length field, or they are fixed length, in
+   * which case we don't, and assume the [rule] will know how to read the
+   * right length when we read it back. We expect everything in the list to be
+   * a reference, which is stored as two numbers.
+   */
+  writeLists(SerializationRule rule, List<List> entries, List target) {
+    target.add(STORED_AS_LIST);
+    for (var eachEntry in entries) {
+      if (rule.hasVariableLengthEntries) {
+        target.add(eachEntry.length);
+      }
+      for (var eachReference in eachEntry) {
+        writeReference(eachReference, target);
+      }
+    }
+  }
+
+  /**
+   * Write [entries], which contains Maps. Either the Maps are variable
+   * length, in which case we add a length field, or they are fixed length, in
+   * which case we don't, and assume the [rule] will know how to read the
+   * right length when we read it back. Then we write alternating keys and
+   * values. We expect the values to be references, which we store as
+   * two numbers.
+   */
+  writeMaps(SerializationRule rule, List<Map> entries, List target) {
+    target.add(STORED_AS_MAP);
+    for (var eachEntry in entries) {
+      if (rule.hasVariableLengthEntries) {
+        target.add(eachEntry.length);
+      }
+      // We take advantage of this being only a semi-flat format, and expecting
+      // that the keys here are field names, i.e. strings. So we write
+      // the keys as literals and the values as references. This duplicates the
+      // keys, so is quite inefficient. But generating maps rather than lists is
+      // not very efficient in the first place.
+      eachEntry.forEach((key, value) {
+        target.add(key);
+        writeReference(value, target);
+      });
+    }
+  }
+
+  /**
+   * Write [entries], which contains simple objects which we can put directly
+   * into [target].
+   */
+  writeObjects(List entries, List target) {
+    target.add(STORED_AS_PRIMITIVE);
+    target.addAll(entries);
+  }
+
+  /**
+   * Write [eachRef] to [target]. It will be written as two ints. If [eachRef]
+   * is null it will be written as two nulls.
+   */
+  void writeReference(Reference eachRef, List target) {
+    // TODO(alanknight): Writing nulls is problematic in a real flat format.
+    if (eachRef == null) {
+      target..add(null)..add(null);
+    } else {
+      eachRef.writeToList(target);
+    }
+  }
+
+  /**
+   * Read the data from [rawInput] in the context of [r] and return it as a
+   * Map with entries for "roots", "data" and "rules", which the reader knows
+   * how to interpret. We expect [rawInput] to have been generated from this
+   * format.
+   */
+  Map<String, dynamic> read(List rawInput, Reader r) {
+    var input = {};
+    input["rules"] = rawInput[0];
+    r.readRules(input["rules"]);
+
+    var flatData = rawInput[1];
+    var stream = flatData.iterator;
+    var tempData = new List(r.rules.length);
+    for (var eachRule in r.rules) {
+       tempData[eachRule.number] = readRuleDataFrom(stream, eachRule);
+    }
+    input["data"] = tempData;
+
+    var roots = [];
+    var rootsAsInts = rawInput[2].iterator;
+    do {
+      roots.add(nextReferenceFrom(rootsAsInts));
+    } while (rootsAsInts.current != null);
+
+    input["roots"] = roots;
+    return input;
+  }
+
+  /**
+   * Read the data for [rule] from [input] and return it.
+   */
+  readRuleDataFrom(Iterator input, SerializationRule rule) {
+    var numberOfEntries = _next(input);
+    var entryType = _next(input);
+    if (entryType == STORED_AS_LIST) {
+      return readLists(input, rule, numberOfEntries);
+    }
+    if (entryType == STORED_AS_MAP) {
+      return readMaps(input, rule, numberOfEntries);
+    }
+    if (entryType == STORED_AS_PRIMITIVE) {
+      return readPrimitives(input, rule, numberOfEntries);
+    }
+    if (numberOfEntries == 0) {
+      return [];
+    } else {
+      throw new SerializationException("Invalid data in serialization");
+    }
+  }
+
+  /**
+   * Read data for [rule] from [input] with [length] number of entries,
+   * creating lists from the results.
+   */
+  readLists(Iterator input, SerializationRule rule, int length) {
+    var ruleData = [];
+    for (var i = 0; i < length; i++) {
+      var subLength =
+          rule.hasVariableLengthEntries ? _next(input) : rule.dataLength;
+      var subList = [];
+      ruleData.add(subList);
+      for (var j = 0; j < subLength; j++) {
+        subList.add(nextReferenceFrom(input));
+      }
+    }
+    return ruleData;
+  }
+
+  /**
+   * Read data for [rule] from [input] with [length] number of entries,
+   * creating maps from the results.
+   */
+  readMaps(Iterator input, SerializationRule rule, int length) {
+    var ruleData = [];
+    for (var i = 0; i < length; i++) {
+      var subLength =
+          rule.hasVariableLengthEntries ? _next(input) : rule.dataLength;
+      var map = {};
+      ruleData.add(map);
+      for (var j = 0; j < subLength; j++) {
+        map[_next(input)] = nextReferenceFrom(input);
+      }
+    }
+    return ruleData;
+  }
+
+  /**
+   * Read data for [rule] from [input] with [length] number of entries,
+   * treating the data as primitives that can be returned directly.
+   */
+  readPrimitives(Iterator input, SerializationRule rule, int length) {
+    var ruleData = [];
+    for (var i = 0; i < length; i++) {
+      ruleData.add(_next(input));
+    }
+    return ruleData;
+  }
+
+  /** Read the next Reference from the input. */
+  nextReferenceFrom(Iterator input) {
+    var a = _next(input);
+    var b = _next(input);
+    if (a == null) {
+      return null;
+    } else {
+      return new Reference(this, a, b);
+    }
+  }
+
+  /** Return the next element from the input. */
+  _next(Iterator input) {
+    input.moveNext();
+    return input.current;
+  }
+}