Add support for parsing comments with generic parameters /*<K, V>*/ with Fasta.

pkg/front_end/lib/src/fasta/parser/parser.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.parser.parser;
 
 import '../fasta_codes.dart'
     show
         FastaCode,
         FastaMessage,
@@ skipping 47 matching lines @@
 
 import '../scanner/recover.dart' show closeBraceFor, skipToEof;
 
 import '../scanner/keyword.dart' show Keyword;
 
 import '../scanner/precedence.dart'
     show
         ASSIGNMENT_PRECEDENCE,
         AS_INFO,
         CASCADE_PRECEDENCE,
+        EOF_INFO,
         EQUALITY_PRECEDENCE,
+        GENERIC_METHOD_TYPE_ASSIGN,
+        GENERIC_METHOD_TYPE_LIST,
         GT_INFO,
         IS_INFO,
         MINUS_MINUS_INFO,
         OPEN_PAREN_INFO,
         OPEN_SQUARE_BRACKET_INFO,
         PERIOD_INFO,
         PLUS_PLUS_INFO,
         POSTFIX_PRECEDENCE,
         PrecedenceInfo,
         QUESTION_INFO,
         QUESTION_PERIOD_INFO,
         RELATIONAL_PRECEDENCE,
         SCRIPT_INFO;
 
 import '../scanner/token.dart'
     show
         BeginGroupToken,
+        CommentToken,
         KeywordToken,
         SymbolToken,
         Token,
         isUserDefinableOperator;
 
 import '../scanner/token_constants.dart'
     show
         COMMA_TOKEN,
         DOUBLE_TOKEN,
         EOF_TOKEN,
@@ skipping 10 matching lines @@
         OPEN_CURLY_BRACKET_TOKEN,
         OPEN_PAREN_TOKEN,
         OPEN_SQUARE_BRACKET_TOKEN,
         PERIOD_TOKEN,
         SEMICOLON_TOKEN,
         STRING_INTERPOLATION_IDENTIFIER_TOKEN,
         STRING_INTERPOLATION_TOKEN,
         STRING_TOKEN;
 
 import '../scanner/characters.dart' show $CLOSE_CURLY_BRACKET;
+import '../scanner/string_scanner.dart';

[ahe, 2017/04/04 08:10:45]

The parser shouldn't be importing as specific scanner.

[scheglov, 2017/04/04 15:33:47]

Why?
The parser knows what it wants to parse and how.

 
 import '../util/link.dart' show Link;
 
 import 'async_modifier.dart' show AsyncModifier;
 
 import 'listener.dart' show Listener;
 
 import 'identifier_context.dart' show IdentifierContext;
 
 /// Returns true if [token] is the symbol or keyword [value].
@@ skipping 51 matching lines @@
 ///
 /// Historically, we over-used identical, and when identical is used on other
 /// objects than strings, it can often be replaced by `==`.
 class Parser {
   final Listener listener;
 
   Uri get uri => listener.uri;
 
   bool mayParseFunctionExpressions = true;
 
+  bool parseGenericMethodComments = false;
+
   /// Represents parser state: what asynchronous syntax is allowed in the
   /// function being currently parsed. In rare situations, this can be set by
   /// external clients, for example, to parse an expression outside a function.
   AsyncModifier asyncState = AsyncModifier.Sync;
 
   Parser(this.listener);
 
   bool get inGenerator {
     return asyncState == AsyncModifier.AsyncStar ||
         asyncState == AsyncModifier.SyncStar;
@@ skipping 429 matching lines @@
         token = expect('.', token.next);
         nameToken = token;
         token = parseIdentifier(token, IdentifierContext.fieldInitializer);
       } else {
         nameToken = token;
         token = parseIdentifier(
             token, IdentifierContext.formalParameterDeclaration);
       }
     }
 
+    token = _injectGenericCommentTypeList(token);

[ahe, 2017/04/04 08:10:45]

This method should be public and overridden in a subclass.

[scheglov, 2017/04/04 15:33:47]

In which subclass?

     if (optional('(', token)) {
       Token inlineFunctionTypeStart = token;
       listener.beginFunctionTypedFormalParameter(token);
       listener.handleNoTypeVariables(token);
       token = parseFormalParameters(token);
       listener.endFunctionTypedFormalParameter(
           covariantKeyword, thisKeyword, kind);
       // Generalized function types don't allow inline function types.
       // The following isn't allowed:
       //    int Function(int bar(String x)).
@@ skipping 454 matching lines @@
 
   Token parseTypeVariablesOpt(Token token) {
     return parseStuff(
         token,
         (t) => listener.beginTypeVariables(t),
         (t) => parseTypeVariable(t),
         (c, bt, et) => listener.endTypeVariables(c, bt, et),
         (t) => listener.handleNoTypeVariables(t));
   }
 
-  // TODO(ahe): Clean this up.
+  /// TODO(ahe): Clean this up.

[ahe, 2017/04/04 08:10:45]

This isn't a documentation comment.

[scheglov, 2017/04/04 15:33:47]

Well, I think any comment before a method, and which is related to the method,
should be a documentation comment.

   Token parseStuff(Token token, Function beginStuff, Function stuffParser,
       Function endStuff, Function handleNoStuff) {
+    token = _injectGenericCommentTypeList(token);
     if (optional('<', token)) {
       Token begin = token;
       beginStuff(begin);
       int count = 0;
       do {
         token = stuffParser(token.next);
         ++count;
       } while (optional(',', token));
       Token next = token.next;
       if (identical(token.stringValue, '>>')) {
@@ skipping 1658 matching lines @@
         token = parseArguments(token);
         listener.endSend(beginToken, token);
       } else {
         break;
       }
     }
     return token;
   }
 
   Token parsePrimary(Token token, IdentifierContext context) {
+    token = _injectGenericCommentTypeList(token);
     final kind = token.kind;
     if (kind == IDENTIFIER_TOKEN) {
       return parseSendOrFunctionLiteral(token, context);
     } else if (kind == INT_TOKEN || kind == HEXADECIMAL_TOKEN) {
       return parseLiteralInt(token);
     } else if (kind == DOUBLE_TOKEN) {
       return parseLiteralDouble(token);
     } else if (kind == STRING_TOKEN) {
       return parseLiteralString(token);
     } else if (kind == HASH_TOKEN) {
@@ skipping 272 matching lines @@
     listener.beginNewExpression(newKeyword);
     token = parseConstructorReference(token);
     token = parseRequiredArguments(token);
     listener.endNewExpression(newKeyword);
     return token;
   }
 
   Token parseConstExpression(Token token) {
     Token constKeyword = token;
     token = expect('const', token);
+    token = _injectGenericCommentTypeList(token);
     final String value = token.stringValue;
     if ((identical(value, '[')) || (identical(value, '[]'))) {
       listener.handleNoTypeArguments(token);
       return parseLiteralListSuffix(token, constKeyword);
     }
     if (identical(value, '{')) {
       listener.handleNoTypeArguments(token);
       return parseLiteralMapSuffix(token, constKeyword);
     }
     if (identical(value, '<')) {
@@ skipping 93 matching lines @@
 
   Token parseLiteralNull(Token token) {
     listener.handleLiteralNull(token);
     return token.next;
   }
 
   Token parseSend(Token token, IdentifierContext context) {
     Token beginToken = token;
     listener.beginSend(token);
     token = parseIdentifier(token, context);
+    token = _injectGenericCommentTypeList(token);
     if (isValidMethodTypeArguments(token)) {
       token = parseTypeArgumentsOpt(token);
     } else {
       listener.handleNoTypeArguments(token);
     }
     token = parseArgumentsOpt(token);
     listener.endSend(beginToken, token);
     return token;
   }
 
@@ skipping 576 matching lines @@
       Token token, FastaCode<TokenArgument> code) {
     return reportUnrecoverableError(
         token, () => code.format(uri, token.charOffset, token));
   }
 
   Token reportUnrecoverableErrorCodeWithString(
       Token token, FastaCode<StringArgument> code, String string) {
     return reportUnrecoverableError(
         token, () => code.format(uri, token.charOffset, string));
   }
+
+  /// Matches a generic comment type parameters or type arguments and injects
+  /// them into the token stream before the given [token].
+  Token _injectGenericCommentTypeList(Token token) {
+    return _injectGenericComment(token, GENERIC_METHOD_TYPE_LIST, 2);
+  }
+
+  /// Check if the given [token] has a comment token with the given [info],
+  /// which should be either [GENERIC_METHOD_TYPE_ASSIGN] or
+  /// [GENERIC_METHOD_TYPE_LIST].  If found, parse the comment into tokens and
+  /// inject into the token stream before the [token].
+  Token _injectGenericComment(Token token, PrecedenceInfo info, int prefixLen) {
+    if (parseGenericMethodComments) {
+      CommentToken t = token.precedingCommentTokens;
+      for (; t != null; t = t.next) {
+        if (t.info == info) {
+          String code = t.lexeme.substring(prefixLen, t.lexeme.length - 2);
+          Token tokens = _scanGenericMethodComment(code, t.offset + prefixLen);
+          if (tokens != null) {
+            // Remove the token from the comment stream.
+            t.remove();
+            // Insert the tokens into the stream.
+            _injectTokenList(token, tokens);
+            return tokens;
+          }
+        }
+      }
+    }
+    return token;
+  }
+
+  /// Scans the given [code], and returns the tokens, otherwise returns `null`.
+  Token _scanGenericMethodComment(String code, int offset) {
+    // TODO(scheglov) Let StringScanner to specify the string offset.
+    var scanner = new StringScanner(' ' * offset + code);

[ahe, 2017/04/04 08:10:45]

The listener should be asked how to create a scanner, otherwise you can't extend
it.

[scheglov, 2017/04/04 15:33:47]

What do we want to extend?
These comments are part of the language, and should be supported in the parser
for the language.

+    Token firstToken = scanner.tokenize();
+    if (scanner.hasErrors) {
+      return null;
+    }
+    return firstToken;
+  }
+
+  void _injectTokenList(Token beforeToken, Token firstToken) {
+    // Scanner creates a cyclic EOF token.
+    Token lastToken = firstToken;
+    while (lastToken.next.info != EOF_INFO) {
+      lastToken = lastToken.next;
+    }
+    // Inject these new tokens into the stream.
+    Token previous = beforeToken.previous;
+    lastToken.setNext(beforeToken);
+    previous.setNext(firstToken);
+    beforeToken = firstToken;
+  }
 }
 
 typedef FastaMessage NoArgument(Uri uri, int charOffset);
 
 typedef FastaMessage TokenArgument(Uri uri, int charOffset, Token token);
 
 typedef FastaMessage StringArgument(Uri uri, int charOffset, String string);