fasta.CommentToken implement analyzer.CommentToken

pkg/analyzer/lib/src/fasta/token_utils.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.analyzer.token_utils;
 
 import 'package:front_end/src/fasta/parser/error_kind.dart' show ErrorKind;
 
 import 'package:front_end/src/fasta/scanner/error_token.dart' show ErrorToken;
 
 import 'package:front_end/src/fasta/scanner/keyword.dart' show Keyword;
 
 import 'package:front_end/src/fasta/scanner/precedence.dart';
 
 import 'package:front_end/src/fasta/scanner/token.dart'
-    show BeginGroupToken, KeywordToken, StringToken, SymbolToken, Token;
+    show
+        BeginGroupToken,
+        CommentToken,
+        DartDocToken,
+        KeywordToken,
+        StringToken,
+        SymbolToken,
+        Token;
 
 import 'package:front_end/src/fasta/scanner/token_constants.dart';
 
 import 'package:front_end/src/scanner/token.dart' as analyzer
     show
         BeginToken,
         BeginTokenWithComment,
         CommentToken,
         Keyword,
         KeywordToken,
         KeywordTokenWithComment,
         StringToken,
         StringTokenWithComment,
         Token,
         TokenWithComment;
 
-import 'package:front_end/src/scanner/errors.dart' as analyzer show ScannerErrorCode;
+import 'package:front_end/src/scanner/errors.dart' as analyzer
+    show ScannerErrorCode;
 
 import 'package:analyzer/dart/ast/token.dart' show TokenType;
 
 import 'package:front_end/src/fasta/errors.dart' show internalError;
 
 /// Class capable of converting a stream of Fasta tokens to a stream of analyzer
 /// tokens.
 ///
 /// This is a class rather than an ordinary method so that it can be subclassed
 /// in tests.
@@ skipping 36 matching lines @@
     _analyzerTokenTail = _analyzerTokenHead;
     _beginTokenStack = [null];
     _endTokenStack = <Token>[null];
 
     while (true) {
       if (token.info.kind == BAD_INPUT_TOKEN) {
         ErrorToken errorToken = token;
         _translateErrorToken(errorToken);
       } else {
         var translatedToken = translateToken(
-            token, translateCommentTokens(token.precedingComments));
+            token, translateCommentTokens(token.precedingCommentTokens));
         _matchGroups(token, translatedToken);
         translatedToken.setNext(translatedToken);
         _analyzerTokenTail.setNext(translatedToken);
         translatedToken.previous = _analyzerTokenTail;
         _analyzerTokenTail = translatedToken;
       }
       if (token.isEof) {
         return _analyzerTokenHead.next;
       }
       token = token.next;
@@ skipping 184 matching lines @@
       tail.next = fromAnalyzerToken(token);
       tail.next.previousToken = tail;
       tail = tail.next;
       token = token.next;
     }
     return head;
   }
 
   analyzer.Token translateAndAppend(analyzer.Token analyzerToken) {
     var token = fromAnalyzerToken(analyzerToken);
-    token.precedingComments =
+    token.precedingCommentTokens =
         translateComments(analyzerToken.precedingComments);
     tokenTail.next = token;
     tokenTail.next.previousToken = tokenTail;
     tokenTail = token;
     matchGroups(analyzerToken, token);
     return analyzerToken.next;
   }
 
   while (true) {
     // TODO(paulberry): join up begingroup/endgroup.
     if (analyzerToken.type == TokenType.EOF) {
       tokenTail.next = new SymbolToken(EOF_INFO, analyzerToken.offset);
       tokenTail.next.previousToken = tokenTail;
-      tokenTail.next.precedingComments =
+      tokenTail.next.precedingCommentTokens =
           translateComments(analyzerToken.precedingComments);
       return tokenHead.next;
     }
     analyzerToken = translateAndAppend(analyzerToken);
   }
 }
 
 /// Converts a single analyzer token into a Fasta token.
 Token fromAnalyzerToken(analyzer.Token token) {
   Token beginGroup(PrecedenceInfo info) =>
@@ skipping 23 matching lines @@
       return string(INT_INFO);
     case TokenType.KEYWORD:
       var keyword = Keyword.keywords[token.lexeme];
       if (keyword != null) {
         return new KeywordToken(keyword, token.offset);
       } else {
         return internalError("Unrecognized keyword: '${token.lexeme}'.");
       }
       break;
     case TokenType.MULTI_LINE_COMMENT:
-      return string(MULTI_LINE_COMMENT_INFO);
+      if (token.lexeme.startsWith('/**')) {
+        return new DartDocToken.fromSubstring(
+            MULTI_LINE_COMMENT_INFO, token.lexeme, 0, token.lexeme.length, 0);
+      }
+      return new CommentToken.fromSubstring(
+          MULTI_LINE_COMMENT_INFO, token.lexeme, 0, token.lexeme.length, 0);
     case TokenType.SCRIPT_TAG:
       return string(SCRIPT_INFO);
     case TokenType.SINGLE_LINE_COMMENT:
-      return string(SINGLE_LINE_COMMENT_INFO);
+      if (token.lexeme.startsWith('///')) {
+        return new DartDocToken.fromSubstring(
+            SINGLE_LINE_COMMENT_INFO, token.lexeme, 0, token.lexeme.length, 0);
+      }
+      return new CommentToken.fromSubstring(
+          SINGLE_LINE_COMMENT_INFO, token.lexeme, 0, token.lexeme.length, 0);
     case TokenType.STRING:
       return string(STRING_INFO);
     case TokenType.AMPERSAND:
       return symbol(AMPERSAND_INFO);
     case TokenType.AMPERSAND_AMPERSAND:
       return symbol(AMPERSAND_AMPERSAND_INFO);
     // case TokenType.AMPERSAND_AMPERSAND_EQ
     case TokenType.AMPERSAND_EQ:
       return symbol(AMPERSAND_EQ_INFO);
     case TokenType.AT:
@@ skipping 132 matching lines @@
     if (token.info.kind != BAD_INPUT_TOKEN) return false;
     // Otherwise keep looking.
   }
 }
 
 analyzer.Token toAnalyzerToken(Token token,
     [analyzer.CommentToken commentToken]) {
   if (token == null) return null;
   analyzer.Token makeStringToken(TokenType tokenType) {
     if (commentToken == null) {
-      return new analyzer.StringToken(tokenType, token.lexeme, token.charOffset);
+      return new analyzer.StringToken(
+          tokenType, token.lexeme, token.charOffset);
     } else {
       return new analyzer.StringTokenWithComment(
           tokenType, token.lexeme, token.charOffset, commentToken);
     }
   }
 
   analyzer.Token makeBeginToken(TokenType tokenType) {
     if (commentToken == null) {
       return new analyzer.BeginToken(tokenType, token.charOffset);
     } else {
@@ skipping 262 matching lines @@
     case PERIOD_PERIOD_PERIOD_TOKEN:
       return TokenType.PERIOD_PERIOD_PERIOD;
     // case GENERIC_METHOD_TYPE_LIST_TOKEN:
     //   return TokenType.GENERIC_METHOD_TYPE_LIST;
     // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN:
     //   return TokenType.GENERIC_METHOD_TYPE_ASSIGN;
     default:
       return internalError("Unhandled token ${token.info}");
   }
 }