Plurals and Genders

pkg/intl/lib/extract_messages.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * This is for use in extracting messages from a Dart program
  * using the Intl.message() mechanism and writing them to a file for
  * translation. This provides only the stub of a mechanism, because it
  * doesn't define how the file should be written. It provides an
  * [IntlMessage] class that holds the extracted data and [parseString]
@@ skipping 18 matching lines @@
 /**
  * If this is true, print warnings for skipped messages. Otherwise, warnings
  * are suppressed.
  */
 bool suppressWarnings = false;
 
 /**
  * Parse the source of the Dart program file [file] and return a Map from
  * message names to [IntlMessage] instances.
  */
-Map<String, IntlMessage> parseFile(File file) {
-  var unit = parseDartFile(file.path);
-  var visitor = new MessageFindingVisitor(unit, file.path);
-  unit.accept(visitor);
+Map<String, MainMessage> parseFile(File file) {

[Emily Fortuna, 2013/07/03 17:52:33]

why change the name to MainMessage? 

[Alan Knight, 2013/07/03 18:41:07]

I was getting too many classes that were all Intl something, so I thought the
Intl was all implied by context and removed it everywhere, and I used Message as
the name for the abstract class at the top of the hierarchy, so wanted to
differentiate the top-level one created by the Intl.message call.

+  _root = parseDartFile(file.path);
+  _origin = file.path;
+  var visitor = new MessageFindingVisitor();
+  _root.accept(visitor);
   return visitor.messages;
 }
 
+

[Emily Fortuna, 2013/07/03 17:52:33]

remove line here

[Alan Knight, 2013/07/03 18:41:07]

Done.

+/**
+ * The root of the compilation unit, and the first node we visit. We hold
+ * on to this for error reporting, as it can give us line numbers of other
+ * nodes.
+ */
+CompilationUnit _root;
+
+/**
+ * An arbitrary string describing where the source code came from. Most
+ * obviously, this could be a file path. We use this when reporting
+ * invalid messages.
+ */
+String _origin;
+
+void _reportErrorLocation(ASTNode node) {
+  if (_origin != null) print("    from $_origin");
+  var info = _root.lineInfo;
+  if (info != null) {
+    var line = info.getLocation(node.offset);
+    print("    line: ${line.lineNumber}, column: ${line.columnNumber}");
+  }
+}
+
 /**
  * This visits the program source nodes looking for Intl.message uses
- * that conform to its pattern and then finding the
+ * that conform to its pattern and then creating the corresponding
+ * IntlMessage objects. We have to find both the enclosing function, and
+ * the Intl.message invocation.
  */
 class MessageFindingVisitor extends GeneralizingASTVisitor {
 
-  /**
-   * The root of the compilation unit, and the first node we visit. We hold
-   * on to this for error reporting, as it can give us line numbers of other
-   * nodes.
-   */
-  final CompilationUnit root;
+  MessageFindingVisitor();
 
   /**
-   * An arbitrary string describing where the source code came from. Most
-   * obviously, this could be a file path. We use this when reporting
-   * invalid messages.
+   * Accumulates the messages we have found, keyed by name.
    */
-  final String origin;
-
-  MessageFindingVisitor(this.root, this.origin);
-
-  /**
-   * Accumulates the messages we have found.
-   */
-  final Map<String, IntlMessage> messages = new Map<String, IntlMessage>();
+  final Map<String, MainMessage> messages = new Map<String, MainMessage>();
 
   /**
    * We keep track of the data from the last MethodDeclaration,
    * FunctionDeclaration or FunctionExpression that we saw on the way down,
    * as that will be the nearest parent of the Intl.message invocation.
    */
   FormalParameterList parameters;
   String name;
 
   /** Return true if [node] matches the pattern we expect for Intl.message() */
@@ skipping 82 matching lines @@
   /**
    * Check that the node looks like an Intl.message invocation, and create
    * the [IntlMessage] object from it and store it in [messages].
    */
   void addIntlMessage(MethodInvocation node) {
     if (!looksLikeIntlMessage(node)) return;
     var reason = checkValidity(node);
     if (reason != null && !suppressWarnings) {
       print("Skipping invalid Intl.message invocation\n    <$node>");
       print("    reason: $reason");
-      reportErrorLocation(node);
+      _reportErrorLocation(node);
       return;
     }
     var message = messageFromMethodInvocation(node);
     if (message != null) messages[message.name] = message;
   }
 
   /**
    * Create an IntlMessage from [node] using the name and
    * parameters of the last function/method declaration we encountered
    * and the parameters to the Intl.message call.
    */
-  IntlMessage messageFromMethodInvocation(MethodInvocation node) {
-    var message = new IntlMessage();
+  MainMessage messageFromMethodInvocation(MethodInvocation node) {
+    var message = new MainMessage();
     message.name = name;
     message.arguments = parameters.parameters.elements.map(
         (x) => x.identifier.name).toList();
+    var arguments = node.argumentList.arguments.elements;
     try {
-      node.accept(new MessageVisitor(message));
+      var interpolation = new InterpolationVisitor(message);
+      arguments.first.accept(interpolation);
+      message.messagePieces.addAll(interpolation.pieces);
     } on IntlMessageExtractionException catch (e) {
       message = null;
       print("Error $e");
       print("Processing <$node>");
-      reportErrorLocation(node);
+      _reportErrorLocation(node);
+    }
+    for (NamedExpression namedArgument in arguments.skip(1)) {
+      var name = namedArgument.name.label.name;
+      var exp = namedArgument.expression;
+      var string = exp is SimpleStringLiteral ? exp.value : exp.toString();
+      message[name] = string;
     }
     return message;
   }
-
-  void reportErrorLocation(ASTNode node) {
-    if (origin != null) print("    from $origin");
-    var info = root.lineInfo;
-    if (info != null) {
-      var line = info.getLocation(node.offset);
-      print("    line: ${line.lineNumber}, column: ${line.columnNumber}");
-    }
-  }
-}
-
-/**
- * Given a node that looks like an invocation of Intl.message, extract out
- * the message and the parameters and store them in [target].
- */
-class MessageVisitor extends GeneralizingASTVisitor {
-  IntlMessage target;
-
-  MessageVisitor(IntlMessage this.target);
-
-  /**
-   * Extract out the message string. If it's an interpolation, turn it into
-   * a single string with interpolation characters.
-   */
-  void visitArgumentList(ArgumentList node) {
-    var interpolation = new InterpolationVisitor(target);
-    node.arguments.elements.first.accept(interpolation);
-    target.messagePieces = interpolation.pieces;
-    super.visitArgumentList(node);
-  }
-
-  /**
-   * Find the values of all the named arguments, remove quotes, and save them
-   * into [target].
-   */
-  void visitNamedExpression(NamedExpression node) {
-    var name = node.name.label.name;
-    var exp = node.expression;
-    var string = exp is SimpleStringLiteral ? exp.value : exp.toString();
-    target[name] = string;
-    super.visitNamedExpression(node);
-  }
 }
 
 /**
  * Given an interpolation, find all of its chunks, validate that they are only
- * simple interpolations, and keep track of the chunks so that other parts
- * of the program can deal with the interpolations and the simple string
- * sections separately.
+ * simple variable substitutions or else Intl.plural/gender calls,
+ * and keep track of the pieces of text so that other parts
+ * of the program can deal with the simple string sections and the generated
+ * parts separately. Note that this is a SimpleASTVisitor, so it only
+ * traverses one level of children rather than automatically recursing. If we
+ * find a plural or gender, which requires recursion, we do it with a separate
+ * special-purpose visitor.
  */
-class InterpolationVisitor extends GeneralizingASTVisitor {
-  IntlMessage message;
+class InterpolationVisitor extends SimpleASTVisitor {
+  Message message;
 
   InterpolationVisitor(this.message);
 
   List pieces = [];
   String get extractedMessage => pieces.join();
 
+  void visitAdjacentStrings(AdjacentStrings node) {
+    node.visitChildren(this);
+    super.visitAdjacentStrings(node);
+  }
+
+  void visitStringInterpolation(StringInterpolation node) {
+    node.visitChildren(this);
+    super.visitStringInterpolation(node);
+  }
+
   void visitSimpleStringLiteral(SimpleStringLiteral node) {
     pieces.add(node.value);
     super.visitSimpleStringLiteral(node);
   }
 
   void visitInterpolationString(InterpolationString node) {
     pieces.add(node.value);
     super.visitInterpolationString(node);
   }
 
-  // TODO(alanknight): The limitation to simple identifiers is important
-  // to avoid letting translators write arbitrary code, but is a problem
-  // for plurals.
   void visitInterpolationExpression(InterpolationExpression node) {
-    if (node.expression is! SimpleIdentifier) {
+    if (node.expression is SimpleIdentifier) {
+      return handleSimpleInterpolation(node);
+    } else {
+      return lookForPluralOrGender(node);
+    }
+    // Note that we never end up calling super.
+  }
+
+  lookForPluralOrGender(InterpolationExpression node) {
+    var visitor = new PluralAndGenderVisitor(pieces, message);
+    node.accept(visitor);
+    if (!visitor.foundSomething) {
       throw new IntlMessageExtractionException(
-          "Only simple identifiers are allowed in message "
-          "interpolation expressions.\nError at $node");
+        "Only simple identifiers and Intl.plural/gender/select expressions "
+        "are allowed in message "
+        "interpolation expressions.\nError at $node");
     }
+  }
+
+  void handleSimpleInterpolation(InterpolationExpression node) {
     var index = arguments.indexOf(node.expression.toString());
     if (index == -1) {
       throw new IntlMessageExtractionException(
           "Cannot find argument ${node.expression}");
     }
     pieces.add(index);
-    super.visitInterpolationExpression(node);
   }
 
   List get arguments => message.arguments;
 }
 
 /**
+ * A visitor to extract information from Intl.plural/gender sends. Note that
+ * this is a SimpleASTVisitor, so it doesn't automatically recurse. So this
+ * needs to be called where we expect a plural or gender immediately below.
+ */
+class PluralAndGenderVisitor extends SimpleASTVisitor {
+  /**
+   * A plural or gender always exists in the context of a parent message,
+   * which could in turn also be a plural or gender.
+   */
+  ComplexMessage parent;
+
+  /**
+   * The pieces of the message. We are given an initial version of this
+   * from our parent and we add to it as we find additional information.
+   */
+  List pieces;
+
+  PluralAndGenderVisitor(this.pieces, this.parent) : super() {}
+
+  /** This will be set to true if we find a plural or gender. */
+  bool foundSomething = false;

[Emily Fortuna, 2013/07/03 17:52:33]

perhaps change name to foundPluralOrGender so it's a little more explicit what
you found? Unless you expect this to expand a lot to also signifying found a
number of other things.

[Alan Knight, 2013/07/03 18:41:07]

Done.

+
+  visitInterpolationExpression(InterpolationExpression node) {
+    // TODO(alanknight): Provide better errors for malformed expressions.
+   if (!looksLikePluralOrGender(node.expression)) return;

[Emily Fortuna, 2013/07/03 17:52:33]

+1 space for indentation

[Alan Knight, 2013/07/03 18:41:07]

Done.

+    var reason = checkValidity(node.expression);
+    if (reason != null) throw reason;
+    var message = messageFromMethodInvocation(node.expression);
+    foundSomething = true;
+    pieces.add(message);
+    super.visitInterpolationExpression(node);
+  }
+
+  /** Return true if [node] matches the pattern we expect for Intl.message() */
+  bool looksLikePluralOrGender(MethodInvocation node) {
+    if (!["plural", "gender"].contains(node.methodName.name)) return false;
+    if (!(node.target is SimpleIdentifier)) return false;
+    SimpleIdentifier target = node.target;
+    if (target.token.toString() != "Intl") return false;
+    return true;
+  }
+
+  /**
+   * Returns a String describing why the node is invalid, or null if no
+   * reason is found, so it's presumed valid.
+   */
+  String checkValidity(MethodInvocation node) {
+  // TODO(alanknight): Add reasonable validity checks.

[Emily Fortuna, 2013/07/03 17:52:33]

nit: indent two spaces :-P

[Alan Knight, 2013/07/03 18:41:07]

Done.

+  }
+
+  /**
+   * Create a MainMessage from [node] using the name and
+   * parameters of the last function/method declaration we encountered
+   * and the parameters to the Intl.message call.
+   */
+  messageFromMethodInvocation(MethodInvocation node) {
+    var message;
+    if (node.methodName.name == "gender") {
+      message = new Gender();
+    } else if (node.methodName.name == "plural") {
+      message = new Plural();
+    } else {
+      throw new IntlMessageExtractionException("Invalid plural/gender message");
+    }
+    message.parent = parent;
+
+    var arguments = node.argumentList.arguments.elements;
+    for (var arg in arguments.where((each) => each is NamedExpression)) {
+      try {
+        var interpolation = new InterpolationVisitor(message);
+        arg.expression.accept(interpolation);
+        message[arg.name.label.token.toString()] = interpolation.pieces;
+      } on IntlMessageExtractionException catch (e) {
+        message = null;
+        print("Error $e");
+        print("Processing <$node>");
+        _reportErrorLocation(node);
+      }
+    }
+    var mainArg = node.argumentList.arguments.elements.firstWhere(
+        (each) => each is! NamedExpression);
+    if (mainArg is SimpleStringLiteral) {
+      message.mainArgument = mainArg.toString();
+    } else {
+      message.mainArgument = mainArg.name;
+    }
+    return message;
+  }
+}
+
+/**
  * Exception thrown when we cannot process a message properly.
  */
 class IntlMessageExtractionException implements Exception {
   /**
    * A message describing the error.
    */
   final String message;
 
   /**
    * Creates a new exception with an optional error [message].
    */
   const IntlMessageExtractionException([this.message = ""]);
 
   String toString() => "IntlMessageExtractionException: $message";
 }