Add codegen support in smoke

pkg/smoke/lib/codegen/generator.dart

+// Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/// Library to generate code that can initialize the `StaticConfiguration` in
+/// `package:smoke/static.dart`.
+///
+/// This library doesn't have any specific logic to extract information from
+/// Dart source code. To extract code using the analyzer, take a look at the
+/// `smoke.codegen.recorder` library.
+library smoke.codegen.generator;

[Jennifer Messerly, 2014/03/17 22:38:48]

is this all intended as public api? SmokeCodeGenerator kind of looks that way
but I'm not sure about the AST-ish types

[Siggi Cherem (dart-lang), 2014/03/18 01:52:29]

Yes.

SmokeCodeGenerator will be heavily used extenrally. The main thing about the AST
is that it's needed for the 'addDeclaration', which is also public. I expect
it's mainly going to be used by recorder.dart, but it needs to be exposed
anyways.

+
+import 'dart:collection' show SplayTreeMap; // To keep generated code sorted.
+
+/// Collects the necessary information and generates code to initialize a
+/// `StaticConfiguration`. After setting up the generator by calling
+/// [addGetter], [addSetter], [addSymbol], and [addDeclaration], you can
+/// retrieve the generated code using the following three methods:
+///
+///   * [generateImports] returns a list of imports directives,
+///   * [generateTopLevelDeclarations] returns additional declarations used to
+///     represent mixin classes by name in the generated code.
+///   * [generateInitCall] returns the actual code that allocates the static
+///     configuration.
+///
+/// You'd need to include all three in your generated code, since the
+/// initialization code refers to symbols that are only available from the
+/// generated imports or the generated top-level declarations.
+class SmokeCodeGenerator {
+  /// Names used as getters via smoke.
+  List<String> _getters = [];

[Jennifer Messerly, 2014/03/17 22:38:48]

final here & below?

[Siggi Cherem (dart-lang), 2014/03/18 01:52:29]

Done.

+
+  /// Names used as setters via smoke.
+  List<String> _setters = [];
+
+  /// Subclass relations needed to run smoke queries.
+  Map<TypeIdentifier, TypeIdentifier> _parents =

[Jennifer Messerly, 2014/03/17 22:38:48]

does it work to just say "final _parents" here instead of repeating the type
twice?

same for _declarations

maybe we need to file some bugs, but I thought for "final" it was supposed to do
the right thing.

(alternative if you care more about tooling support, assuming that it is an
issue, keeping the type annotation and the runtime type "new SplayTreeMap()"
would work too.)

[Siggi Cherem (dart-lang), 2014/03/18 01:52:29]

Done.

+      new SplayTreeMap<TypeIdentifier, TypeIdentifier>();
+
+  /// Declarations requested via `smoke.getDeclaration or `smoke.query`.
+  Map<TypeIdentifier, Map<String, _DeclarationCode>> _declarations =
+      new SplayTreeMap<TypeIdentifier, Map<String, _DeclarationCode>>();
+
+  /// Names that are used both as strings and symbols.
+  List<String> _names = [];
+
+  /// Prefixes associated with imported libraries.
+  Map<String, String> _libraryPrefix = {};
+
+  /// Register that [name] is used as a getter in the code.
+  void addGetter(String name) => _getters.add(name);
+
+  /// Register that [name] is used as a setter in the code.
+  void addSetter(String name) => _setters.add(name);
+
+  /// Register that [name] might be needed as a symbol.
+  void addSymbol(String name) => _names.add(name);
+
+  int _mixins = 0;
+
+  /// Creates a new type to represent a mixin. Use [comment] to help users
+  /// figure out what mixin is being represented.
+  TypeIdentifier createMixinType([String comment = '']) =>
+      new TypeIdentifier(null, '_M${_mixins++}', comment);
+
+  /// Register that we care to know that [child] extends [parent].
+  void addParent(TypeIdentifier child, TypeIdentifier parent) {
+    var existing = _parents[child];
+    if (existing != null) {
+      if (existing == parent) return;
+      throw new StateError('$child already has a different parent associated'
+          '($existing instead of $parent)');
+    }
+    _addLibrary(child.importUrl);
+    _addLibrary(parent.importUrl);
+    _parents[child] = parent;
+  }
+
+  /// Register a declaration of a field, property, or method. Note that one and
+  /// only one of [isField], [isMethod], or [isProperty] should be set at a
+  /// given time.
+  void addDeclaration(TypeIdentifier cls, String name, TypeIdentifier type,
+      {bool isField: false, bool isProperty: false, bool isMethod: false,
+      bool isFinal: false, bool isStatic: false,
+      List<ConstExpression> annotations: const []}) {
+    final count = (isField ? 1 : 0) + (isProperty ? 1 : 0) + (isMethod ? 1 : 0);
+    if (count != 1) {
+      throw new ArgumentError('Declaration must be one (and only one) of the '
+          'following: a field, a property, or a method.');
+    }
+    var kind = isField ? 'FIELD' : isProperty ? 'PROPERTY' : 'METHOD';
+    _declarations.putIfAbsent(cls,
+        () => new SplayTreeMap<String, _DeclarationCode>());
+    _addLibrary(cls.importUrl);
+    var map = _declarations[cls];
+
+    for (var exp in annotations) {
+      for (var lib in exp.librariesUsed) {
+        _addLibrary(lib);
+      }
+    }
+
+    _addLibrary(type.importUrl);
+    var decl = new _DeclarationCode(name, type, kind, isFinal, isStatic,
+        annotations);
+    if (map.containsKey(name) && map[name] != decl) {
+      throw new StateError('$type.$name already has a different declaration'
+          ' (${map[name]} instead of $decl).');
+    }
+    map[name] = decl;
+  }
+
+  /// Register that we might try to read declarations of [type], even if no
+  /// declaration exists. This informs the smoke system that querying for a
+  /// member in this class might be intentional and not an error.
+  void addEmptyDeclaration(TypeIdentifier type) {
+    _addLibrary(type.importUrl);
+    _declarations.putIfAbsent(type,
+        () => new SplayTreeMap<String, _DeclarationCode>());
+  }
+
+  /// Returns a list of imports that are needed for the code returned by
+  /// [generateInitCall].
+  List<String> generateImports() {
+    var imports = []..addAll(DEFAULT_IMPORTS);
+    _libraryPrefix.forEach((url, prefix) {
+      imports.add("import '$url' as $prefix;");
+    });
+    return imports;
+  }
+
+  /// Returns a top-level declarations that are used by the code in
+  /// [generateInitCall]. These are typically declarations of empty classes that
+  /// are then used as placeholder for mixin superclasses.
+  String generateTopLevelDeclarations() {

[Jennifer Messerly, 2014/03/17 22:38:48]

hmmm, I wonder if these should be methods that accept a StringBuffer. that's
usually a more performant pattern when composing a bunch of them

[Siggi Cherem (dart-lang), 2014/03/18 01:52:29]

good idea, done.

+    var types = new Set()
+        ..addAll(_parents.keys)
+        ..addAll(_parents.values)
+        ..addAll(_declarations.keys)
+        ..addAll(_declarations.values.expand(
+              (m) => m.values.map((d) => d.type)))
+        ..removeWhere((t) => t.importUrl != null);
+    final sb = new StringBuffer();
+    for (var type in types) {
+      sb.write('abstract class ${type.name} {}');
+      if (type.comment != null) sb.write(' // ${type.comment}');
+      sb.writeln();
+    }
+    return sb.toString();
+  }
+
+  /// Returns code that will initialize smoke's static configuration. For
+  /// example, the code might be of the form:
+  ///
+  ///    useGeneratedCode(new StaticConfiguration(
+  ///      getters: {
+  ///         #i: (o) => o.i,
+  ///         ...
+  ///      names: {
+  ///         #i: "i",
+  ///      }));
+  ///
+  /// The optional [indent] argument is used for formatting purposes. All
+  /// entries in each map (getters, setters, names, declarations, parents) are
+  /// sorted alphabetically.
+  ///
+  /// **Note**: this code assumes that imports returned by [generateImports] and
+  /// top-level declarations from [generateTopLevelDeclarations] are included in
+  /// the same library where this code will live.
+  String generateInitCall([int indent = 2]) {
+    final spaces = new List.filled(indent, ' ').join('');
+    var args = {};
+
+    if (_getters.isNotEmpty) {
+      args['getters'] = (_getters..sort()).map((n) => '#$n: (o) => o.$n');
+    }
+    if (_setters.isNotEmpty) {
+      args['setters'] = (_setters..sort()).map(
+          (n) => '#$n: (o, v) { o.$n = v; }');
+    }
+
+    if (_parents.isNotEmpty) {
+      var parentsMap = [];
+      _parents.forEach((child, parent) {
+        var parent = _parents[child];
+        parentsMap.add('${child.asCode(_libraryPrefix)}: '
+          '${parent.asCode(_libraryPrefix)}');
+      });
+      args['parents'] = parentsMap;
+    }
+
+    if (_declarations.isNotEmpty) {
+      var declarations = [];
+      _declarations.forEach((type, members) {
+        final sb = new StringBuffer()
+            ..write(type.asCode(_libraryPrefix))
+            ..write(': ');
+        if (members.isEmpty) {
+          sb.write('const {}');
+        } else {
+          sb.write('{\n');
+          members.forEach((name, decl) {
+            var decl = members[name].asCode(_libraryPrefix);
+            sb.write('$spaces        #$name: $decl,\n');
+          });
+          sb.write('$spaces      }');
+        }
+        declarations.add(sb.toString());
+      });
+      args['declarations'] = declarations;
+    }
+
+    if (_names.isNotEmpty) {
+      args['names'] = (_names..sort()).map((n) => "#$n: '$n'");
+    }
+
+    final sb = new StringBuffer()
+        ..write(spaces)
+        ..writeln('useGeneratedCode(new StaticConfiguration(')
+        ..write('$spaces    checkedMode: false');
+
+    args.forEach((name, mapContents) {
+      sb.writeln(',');
+      // TODO(sigmund): use const map when Type can be keys (dartbug.com/17123)
+      sb.writeln('$spaces    $name: {');
+      for (var entry in mapContents) {
+        sb.writeln('$spaces      $entry,');
+      }
+      sb.write('$spaces    }');
+    });
+    sb.writeln('));');
+    return sb.toString();
+  }
+
+  /// Adds a library that needs to be imported.
+  void _addLibrary(String url) {
+    if (url == null || url == 'dart:core') return;
+    _libraryPrefix.putIfAbsent(url, () => 'smoke_${_libraryPrefix.length}');
+  }
+}
+
+/// Information used to generate code that allocates a `Declaration` object.
+class _DeclarationCode extends ConstExpression {
+  String name;

[Jennifer Messerly, 2014/03/17 22:38:48]

final for these fields?

[Siggi Cherem (dart-lang), 2014/03/18 01:52:29]

Done.

+  TypeIdentifier type;
+  String kind;
+  bool isFinal;
+  bool isStatic;
+  List<ConstExpression> annotations;
+
+  _DeclarationCode(this.name, this.type, this.kind, this.isFinal, this.isStatic,
+      this.annotations);
+
+  List<String> get librariesUsed => []
+      ..addAll(type.librariesUsed)
+      ..addAll(annotations.expand((a) => a.librariesUsed));
+
+  String asCode(Map<String, String> libraryPrefixes) {
+    var sb = new StringBuffer();
+    sb.write("const Declaration(#$name, ${type.asCode(libraryPrefixes)}");
+    if (kind != 'FIELD') sb.write(', kind: $kind');
+    if (isFinal) sb.write(', isFinal: true');
+    if (isStatic) sb.write(', isStatic: true');
+    if (annotations != null && annotations.isNotEmpty) {
+      sb.write(', annotations: const [');
+      bool first = true;
+      for (var e in annotations) {
+        if (!first) sb.write(', ');
+        first = false;
+        sb.write(e.asCode(libraryPrefixes));
+      }
+      sb.write(']');
+    }
+    sb.write(')');
+    return sb.toString();
+  }
+
+  String toString() =>
+      '(decl: $type.$name - $kind, $isFinal, $isStatic, $annotations)';
+  operator== (other) => other is _DeclarationCode && name == other.name && 
+      type == other.type && kind == other.kind && isFinal == other.isFinal &&
+      isStatic == other.isStatic &&
+      _compareLists(annotations, other.annotations);
+  int get hashCode => name.hashCode + (31 * type.hashCode);
+}
+
+/// A constant expression that can be used as an annotation.
+abstract class ConstExpression {
+
+  List<String> get librariesUsed;

[Jennifer Messerly, 2014/03/17 22:38:48]

doc comment?

[Siggi Cherem (dart-lang), 2014/03/18 01:52:29]

Done.

+
+  /// Return a string representation of the code in this expression.
+  /// [libraryPrefixes] describes what prefix has been associated
+  String asCode(Map<String, String> libraryPrefixes);
+
+  ConstExpression();
+
+  /// Create a string expression of the form `"string"`.

[Jennifer Messerly, 2014/03/17 22:38:48]

maybe explain that it will be normalized

[Siggi Cherem (dart-lang), 2014/03/18 01:52:29]

Done.

+  factory ConstExpression.string(String string) {
+    var value = string.replaceAll(r'\', r'\\').replaceAll('\'', '\\\'');

[Jennifer Messerly, 2014/03/17 22:38:48]

for second replace, maybe write it as:

.replaceAll("'", r"\'");

IOW, double quotes on outside then you don't need to escape single quote inside

[Siggi Cherem (dart-lang), 2014/03/18 01:52:29]

Done.

+    return new CodeAsConstExpression("'$value'");
+  }
+
+  /// Create an expression of the form `prefix.variable_name`.
+  factory ConstExpression.identifier(String importUrl, String name) =>
+      new TopLevelIdentifier(importUrl, name);
+
+  /// Create an expression of the form `prefix.Constructor(v1, v2, p3: v3)`.
+  factory ConstExpression.constructor(String importUrl, String name,
+      List<ConstExpression> positionalArgs,
+      Map<String, ConstExpression> namedArgs) =>
+      new ConstructorExpression(importUrl, name, positionalArgs, namedArgs);
+}
+
+/// A constant expression written as a String. Used when the code is self
+/// contained and it doesn't depend on any imported libraries.
+class CodeAsConstExpression extends ConstExpression {
+  String code;
+  List<String> get librariesUsed => const [];
+
+  CodeAsConstExpression(this.code);
+
+  String asCode(Map<String, String> libraryPrefixes) => code;
+
+  String toString() => '(code: $code)';
+  operator== (other) => other is CodeAsConstExpression && code == other.code;
+  int get hashCode => code.hashCode;
+}
+
+/// Describes a reference to some symbol that is exported from a library. This
+/// is typically used to refer to a type or a top-level variable from that
+/// library.
+class TopLevelIdentifier extends ConstExpression {
+  final String importUrl;
+  final String name;
+  TopLevelIdentifier(this.importUrl, this.name);
+
+  List<String> get librariesUsed => [importUrl];
+  String asCode(Map<String, String> libraryPrefixes) {
+    if (importUrl == 'dart:core' || importUrl == null) return name;
+    return '${libraryPrefixes[importUrl]}.$name';
+  }
+
+  String toString() => '(identifier: $importUrl, $name)';
+  operator== (other) => other is TopLevelIdentifier && name == other.name
+      && importUrl == other.importUrl;
+  int get hashCode => 31 * importUrl.hashCode + name.hashCode;
+}
+
+/// Represents an expression that invokes a const constructor.
+class ConstructorExpression extends ConstExpression {
+  final String importUrl;
+  final String name;
+  final List<ConstExpression> positionalArgs;
+  final Map<String, ConstExpression> namedArgs;
+  ConstructorExpression(this.importUrl, this.name, this.positionalArgs,
+      this.namedArgs);
+
+  List<String> get librariesUsed => [importUrl]
+      ..addAll(positionalArgs.expand((e) => e.librariesUsed))
+      ..addAll(namedArgs.values.expand((e) => e.librariesUsed));
+
+  String asCode(Map<String, String> libraryPrefixes) {
+    var sb = new StringBuffer();
+    sb.write('const ');
+    if (importUrl != 'dart:core' && importUrl != null) {
+      sb.write('${libraryPrefixes[importUrl]}.');
+    }
+    sb.write('$name(');
+    bool first = true;
+    for (var e in positionalArgs) {
+      if (!first) sb.write(', ');
+      first = false;
+      sb.write(e.asCode(libraryPrefixes));
+    }
+    namedArgs.forEach((name, value) {
+      if (!first) sb.write(', ');
+      first = false;
+      sb.write('$name: ');
+      sb.write(value.asCode(libraryPrefixes));
+    });
+    sb.write(')');
+    return sb.toString();
+  }
+
+  String toString() => '(ctor: $importUrl, $name, $positionalArgs, $namedArgs)';
+  operator== (other) => other is ConstructorExpression && name == other.name
+      && importUrl == other.importUrl &&
+      _compareLists(positionalArgs, other.positionalArgs) &&
+      _compareMaps(namedArgs, other.namedArgs);
+  int get hashCode => 31 * importUrl.hashCode + name.hashCode;
+}
+
+
+/// Describes a type identifier, with the library URL where the type is defined.
+// TODO(sigmund): consider adding support for imprecise TypeIdentifiers, which
+// may be used by tools that want to generate code without using the analyzer
+// (they can syntactically tell the type comes from one of N imports).
+class TypeIdentifier extends TopLevelIdentifier
+    implements Comparable<TypeIdentifier> {
+  final String comment;
+  TypeIdentifier(importUrl, typeName, [this.comment])
+      : super(importUrl, typeName);
+
+  // We implement [Comparable] to sort out entries in the generated code.
+  int compareTo(TypeIdentifier other) {
+    if (importUrl == null && other.importUrl != null) return 1;
+    if (importUrl != null && other.importUrl == null) return -1;
+    var c1 = importUrl == null ? 0 : importUrl.compareTo(other.importUrl);
+    return c1 != 0 ? c1 : name.compareTo(other.name);
+  }
+
+  String toString() => '(type-identifier: $importUrl, $name, $comment)';
+  bool operator ==(other) => other is TypeIdentifier &&
+      importUrl == other.importUrl && name == other.name &&
+      comment == other.comment;
+  int get hashCode => super.hashCode;
+}
+
+/// Default set of imports added by [SmokeCodeGenerator].
+const DEFAULT_IMPORTS = const [
+    "import 'package:smoke/smoke.dart' show Declaration, PROPERTY, METHOD;",
+    "import 'package:smoke/static.dart' show "
+        "useGeneratedCode, StaticConfiguration;",
+  ];
+
+/// Shallow comparison of two lists.
+bool _compareLists(List a, List b) {
+  if (a == null && b != null) return false;
+  if (a != null && b == null) return false;
+  if (a.length != b.length) return false;
+  for (int i = 0; i < a.length; i++) {
+    if (a[i] != b[i]) return false;
+  }
+  return true;
+}
+
+/// Shallow comparison of two maps.
+bool _compareMaps(Map a, Map b) {
+  if (a == null && b != null) return false;
+  if (a != null && b == null) return false;
+  if (a.length != b.length) return false;
+  for (var k in a.keys) {
+    if (!b.containsKey(k) || a[k] != b[k]) return false;
+  }
+  return true;
+}