Parse type substitution comments /*=T*/ in variable declarations with Fasta.

pkg/front_end/lib/src/fasta/parser/parser.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.parser.parser;
 
 import '../fasta_codes.dart'
     show
         FastaCode,
         FastaMessage,
@@ skipping 47 matching lines @@
 
 import '../scanner/recover.dart' show closeBraceFor, skipToEof;
 
 import '../scanner/keyword.dart' show Keyword;
 
 import '../scanner/precedence.dart'
     show
         ASSIGNMENT_PRECEDENCE,
         AS_INFO,
         CASCADE_PRECEDENCE,
-        EOF_INFO,
         EQUALITY_PRECEDENCE,
-        GENERIC_METHOD_TYPE_ASSIGN,
-        GENERIC_METHOD_TYPE_LIST,
         GT_INFO,
         IS_INFO,
         MINUS_MINUS_INFO,
         OPEN_PAREN_INFO,
         OPEN_SQUARE_BRACKET_INFO,
         PERIOD_INFO,
         PLUS_PLUS_INFO,
         POSTFIX_PRECEDENCE,
         PrecedenceInfo,
         QUESTION_INFO,
         QUESTION_PERIOD_INFO,
         RELATIONAL_PRECEDENCE,
         SCRIPT_INFO;
 
 import '../scanner/token.dart'
     show
         BeginGroupToken,
-        CommentToken,
         KeywordToken,
         SymbolToken,
         Token,
         isUserDefinableOperator;
 
 import '../scanner/token_constants.dart'
     show
         COMMA_TOKEN,
         DOUBLE_TOKEN,
         EOF_TOKEN,
@@ skipping 10 matching lines @@
         OPEN_CURLY_BRACKET_TOKEN,
         OPEN_PAREN_TOKEN,
         OPEN_SQUARE_BRACKET_TOKEN,
         PERIOD_TOKEN,
         SEMICOLON_TOKEN,
         STRING_INTERPOLATION_IDENTIFIER_TOKEN,
         STRING_INTERPOLATION_TOKEN,
         STRING_TOKEN;
 
 import '../scanner/characters.dart' show $CLOSE_CURLY_BRACKET;
-import '../scanner/string_scanner.dart';
 
 import '../util/link.dart' show Link;
 
 import 'async_modifier.dart' show AsyncModifier;
 
 import 'listener.dart' show Listener;
 
 import 'identifier_context.dart' show IdentifierContext;
 
 /// Returns true if [token] is the symbol or keyword [value].
@@ skipping 51 matching lines @@
 ///
 /// Historically, we over-used identical, and when identical is used on other
 /// objects than strings, it can often be replaced by `==`.
 class Parser {
   final Listener listener;
 
   Uri get uri => listener.uri;
 
   bool mayParseFunctionExpressions = true;
 
-  bool parseGenericMethodComments = false;
-
   /// Represents parser state: what asynchronous syntax is allowed in the
   /// function being currently parsed. In rare situations, this can be set by
   /// external clients, for example, to parse an expression outside a function.
   AsyncModifier asyncState = AsyncModifier.Sync;
 
   Parser(this.listener);
 
   bool get inGenerator {
     return asyncState == AsyncModifier.AsyncStar ||
         asyncState == AsyncModifier.SyncStar;
@@ skipping 429 matching lines @@
         token = expect('.', token.next);
         nameToken = token;
         token = parseIdentifier(token, IdentifierContext.fieldInitializer);
       } else {
         nameToken = token;
         token = parseIdentifier(
             token, IdentifierContext.formalParameterDeclaration);
       }
     }
 
-    token = _injectGenericCommentTypeList(token);
+    token = listener.injectGenericCommentTypeList(token);
     if (optional('(', token)) {
       Token inlineFunctionTypeStart = token;
       listener.beginFunctionTypedFormalParameter(token);
       listener.handleNoTypeVariables(token);
       token = parseFormalParameters(token);
       listener.endFunctionTypedFormalParameter(
           covariantKeyword, thisKeyword, kind);
       // Generalized function types don't allow inline function types.
       // The following isn't allowed:
       //    int Function(int bar(String x)).
@@ skipping 65 matching lines @@
     } else {
       return expect(']', token);
     }
   }
 
   Token parseTypeOpt(Token token) {
     if (isGeneralizedFunctionType(token)) {
       // Function type without return type.
       return parseType(token);
     }
+    token = listener.injectGenericCommentTypeAssign(token);
     Token peek = peekAfterIfType(token);
     if (peek != null && (peek.isIdentifier() || optional('this', peek))) {
       return parseType(token);
     }
     listener.handleNoType(token);
     return token;
   }
 
   bool isValidTypeReference(Token token) {
     final kind = token.kind;
@@ skipping 372 matching lines @@
         token,
         (t) => listener.beginTypeVariables(t),
         (t) => parseTypeVariable(t),
         (c, bt, et) => listener.endTypeVariables(c, bt, et),
         (t) => listener.handleNoTypeVariables(t));
   }
 
   /// TODO(ahe): Clean this up.
   Token parseStuff(Token token, Function beginStuff, Function stuffParser,
       Function endStuff, Function handleNoStuff) {
-    token = _injectGenericCommentTypeList(token);
+    token = listener.injectGenericCommentTypeList(token);
     if (optional('<', token)) {
       Token begin = token;
       beginStuff(begin);
       int count = 0;
       do {
         token = stuffParser(token.next);
         ++count;
       } while (optional(',', token));
       Token next = token.next;
       if (identical(token.stringValue, '>>')) {
@@ skipping 1266 matching lines @@
   }
 
   Token parseExpressionStatementOrDeclaration(Token token) {
     if (!inPlainSync && optional("await", token)) {
       return parseExpressionStatement(token);
     }
     assert(token.isIdentifier() || identical(token.stringValue, 'void'));
     Token identifier = peekIdentifierAfterType(token);
     if (identifier != null) {
       assert(identifier.isIdentifier());
+
+      // If the identifier token has a type substitution comment /*=T*/,
+      // then the set of tokens type tokens should be replaced with the
+      // tokens parsed from the comment.
+      token =
+          listener.replaceTokenWithGenericCommentTypeAssign(token, identifier);
+
       Token afterId = identifier.next;
       int afterIdKind = afterId.kind;
       if (identical(afterIdKind, EQ_TOKEN) ||
           identical(afterIdKind, SEMICOLON_TOKEN) ||
           identical(afterIdKind, COMMA_TOKEN)) {
         // We are looking at "type identifier" followed by '=', ';', ','.
         return parseVariablesDeclaration(token);
       } else if (identical(afterIdKind, OPEN_PAREN_TOKEN)) {
         // We are looking at "type identifier '('".
         BeginGroupToken beginParen = afterId;
@@ skipping 64 matching lines @@
       }
     }
     return parseExpressionStatement(token);
   }
 
   Token parseExpressionStatementOrConstDeclaration(Token token) {
     assert(identical(token.stringValue, 'const'));
     if (isModifier(token.next)) {
       return parseVariablesDeclaration(token);
     }
+    listener.injectGenericCommentTypeAssign(token.next);
     Token identifier = peekIdentifierAfterOptionalType(token.next);
     if (identifier != null) {
       assert(identifier.isIdentifier());
       Token afterId = identifier.next;
       int afterIdKind = afterId.kind;
       if (identical(afterIdKind, EQ_TOKEN) ||
           identical(afterIdKind, SEMICOLON_TOKEN) ||
           identical(afterIdKind, COMMA_TOKEN)) {
         // We are looking at "const type identifier" followed by '=', ';', or
         // ','.
@@ skipping 288 matching lines @@
         token = parseArguments(token);
         listener.endSend(beginToken, token);
       } else {
         break;
       }
     }
     return token;
   }
 
   Token parsePrimary(Token token, IdentifierContext context) {
-    token = _injectGenericCommentTypeList(token);
+    token = listener.injectGenericCommentTypeList(token);
     final kind = token.kind;
     if (kind == IDENTIFIER_TOKEN) {
       return parseSendOrFunctionLiteral(token, context);
     } else if (kind == INT_TOKEN || kind == HEXADECIMAL_TOKEN) {
       return parseLiteralInt(token);
     } else if (kind == DOUBLE_TOKEN) {
       return parseLiteralDouble(token);
     } else if (kind == STRING_TOKEN) {
       return parseLiteralString(token);
     } else if (kind == HASH_TOKEN) {
@@ skipping 272 matching lines @@
     listener.beginNewExpression(newKeyword);
     token = parseConstructorReference(token);
     token = parseRequiredArguments(token);
     listener.endNewExpression(newKeyword);
     return token;
   }
 
   Token parseConstExpression(Token token) {
     Token constKeyword = token;
     token = expect('const', token);
-    token = _injectGenericCommentTypeList(token);
+    token = listener.injectGenericCommentTypeList(token);
     final String value = token.stringValue;
     if ((identical(value, '[')) || (identical(value, '[]'))) {
       listener.beginConstLiteral(token);
       listener.handleNoTypeArguments(token);
       token = parseLiteralListSuffix(token, constKeyword);
       listener.endConstLiteral(token);
       return token;
     }
     if (identical(value, '{')) {
       listener.beginConstLiteral(token);
@@ skipping 102 matching lines @@
 
   Token parseLiteralNull(Token token) {
     listener.handleLiteralNull(token);
     return token.next;
   }
 
   Token parseSend(Token token, IdentifierContext context) {
     Token beginToken = token;
     listener.beginSend(token);
     token = parseIdentifier(token, context);
-    token = _injectGenericCommentTypeList(token);
+    token = listener.injectGenericCommentTypeList(token);
     if (isValidMethodTypeArguments(token)) {
       token = parseTypeArgumentsOpt(token);
     } else {
       listener.handleNoTypeArguments(token);
     }
     token = parseArgumentsOpt(token);
     listener.endSend(beginToken, token);
     return token;
   }
 
@@ skipping 85 matching lines @@
 
   Token parseVariablesDeclarationNoSemicolon(Token token) {
     // Only called when parsing a for loop, so this is for parsing locals.
     return parseVariablesDeclarationMaybeSemicolon(token, false);
   }
 
   Token parseVariablesDeclarationMaybeSemicolon(
       Token token, bool endWithSemicolon) {
     int count = 1;
     token = parseMetadataStar(token);
+
+    // If the next token has a type substitution comment /*=T*/, then
+    // the current 'var' token should be repealed and replaced.
+    if (identical('var', token.stringValue)) {
+      token =
+          listener.replaceTokenWithGenericCommentTypeAssign(token, token.next);
+    }
+
     token = parseModifiers(token);
     token = parseTypeOpt(token);
     listener.beginVariablesDeclaration(token);
     token = parseOptionallyInitializedIdentifier(token);
     while (optional(',', token)) {
       token = parseOptionallyInitializedIdentifier(token.next);
       ++count;
     }
     if (endWithSemicolon) {
       Token semicolon = token;
@@ skipping 472 matching lines @@
       Token token, FastaCode<TokenArgument> code) {
     return reportUnrecoverableError(
         token, () => code.format(uri, token.charOffset, token));
   }
 
   Token reportUnrecoverableErrorCodeWithString(
       Token token, FastaCode<StringArgument> code, String string) {
     return reportUnrecoverableError(
         token, () => code.format(uri, token.charOffset, string));
   }
-
-  /// Matches a generic comment type parameters or type arguments and injects
-  /// them into the token stream before the given [token].
-  Token _injectGenericCommentTypeList(Token token) {
-    return _injectGenericComment(token, GENERIC_METHOD_TYPE_LIST, 2);
-  }
-
-  /// Check if the given [token] has a comment token with the given [info],
-  /// which should be either [GENERIC_METHOD_TYPE_ASSIGN] or
-  /// [GENERIC_METHOD_TYPE_LIST].  If found, parse the comment into tokens and
-  /// inject into the token stream before the [token].
-  Token _injectGenericComment(Token token, PrecedenceInfo info, int prefixLen) {
-    if (parseGenericMethodComments) {
-      CommentToken t = token.precedingCommentTokens;
-      for (; t != null; t = t.next) {
-        if (t.info == info) {
-          String code = t.lexeme.substring(prefixLen, t.lexeme.length - 2);
-          Token tokens = _scanGenericMethodComment(code, t.offset + prefixLen);
-          if (tokens != null) {
-            // Remove the token from the comment stream.
-            t.remove();
-            // Insert the tokens into the stream.
-            _injectTokenList(token, tokens);
-            return tokens;
-          }
-        }
-      }
-    }
-    return token;
-  }
-
-  /// Scans the given [code], and returns the tokens, otherwise returns `null`.
-  Token _scanGenericMethodComment(String code, int offset) {
-    var scanner = new SubStringScanner(offset, code);
-    Token firstToken = scanner.tokenize();
-    if (scanner.hasErrors) {
-      return null;
-    }
-    return firstToken;
-  }
-
-  void _injectTokenList(Token beforeToken, Token firstToken) {
-    // Scanner creates a cyclic EOF token.
-    Token lastToken = firstToken;
-    while (lastToken.next.info != EOF_INFO) {
-      lastToken = lastToken.next;
-    }
-    // Inject these new tokens into the stream.
-    Token previous = beforeToken.previous;
-    lastToken.setNext(beforeToken);
-    previous.setNext(firstToken);
-    beforeToken = firstToken;
-  }
 }
 
 typedef FastaMessage NoArgument(Uri uri, int charOffset);
 
 typedef FastaMessage TokenArgument(Uri uri, int charOffset, Token token);
 
 typedef FastaMessage StringArgument(Uri uri, int charOffset, String string);