flatten token hierarchy

pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.scanner.array_based_scanner;
 
 import 'error_token.dart' show ErrorToken, UnmatchedToken;
 
 import '../../scanner/token.dart'
-    show
-        BeginToken,
-        BeginTokenWithComment,
-        Keyword,
-        KeywordTokenWithComment,
-        SyntheticToken,
-        Token,
-        TokenType,
-        TokenWithComment;
+    show BeginToken, Keyword, KeywordToken, SyntheticToken, Token, TokenType;
 
 import '../../scanner/token.dart' as analyzer show StringToken;
 
 import 'token_constants.dart'
     show
         LT_TOKEN,
         OPEN_CURLY_BRACKET_TOKEN,
         OPEN_PAREN_TOKEN,
         STRING_INTERPOLATION_TOKEN;
 
@@ skipping 19 matching lines @@
   Link<BeginToken> groupingStack = const Link<BeginToken>();
 
   /**
    * Appends a fixed token whose kind and content is determined by [type].
    * Appends an *operator* token from [type].
    *
    * An operator token represent operators like ':', '.', ';', '&&', '==', '--',
    * '=>', etc.
    */
   void appendPrecedenceToken(TokenType type) {
-    appendToken(new TokenWithComment(type, tokenStart, comments));
+    appendToken(new Token(type, tokenStart, comments));
   }
 
   /**
    * Appends a fixed token based on whether the current char is [choice] or not.
    * If the current char is [choice] a fixed token whose kind and content
    * is determined by [yes] is appended, otherwise a fixed token whose kind
    * and content is determined by [no] is appended.
    */
   int select(int choice, TokenType yes, TokenType no) {
     int next = advance();
     if (identical(next, choice)) {
       appendPrecedenceToken(yes);
       return advance();
     } else {
       appendPrecedenceToken(no);
       return next;
     }
   }
 
   /**
    * Appends a keyword token whose kind is determined by [keyword].
    */
   void appendKeywordToken(Keyword keyword) {
     String syntax = keyword.lexeme;
     // Type parameters and arguments cannot contain 'this'.
     if (identical(syntax, 'this')) {
       discardOpenLt();
     }
-    appendToken(new KeywordTokenWithComment(keyword, tokenStart, comments));
+    appendToken(new KeywordToken(keyword, tokenStart, comments));
   }
 
   void appendEofToken() {
     beginToken();
     discardOpenLt();
     while (!groupingStack.isEmpty) {
       unmatchedBeginGroup(groupingStack.head);
       groupingStack = groupingStack.tail;
     }
     appendToken(new Token.eof(tokenStart, comments));
@@ skipping 20 matching lines @@
    */
   void lineFeedInMultiline() {
     lineStarts.add(stringOffset + 1);
   }
 
   /**
    * Appends a token that begins a new group, represented by [type].
    * Group begin tokens are '{', '(', '[', '<' and '${'.
    */
   void appendBeginGroup(TokenType type) {
-    Token token = new BeginTokenWithComment(type, tokenStart, comments);
+    Token token = new BeginToken(type, tokenStart, comments);
     appendToken(token);
 
     // { [ ${ cannot appear inside a type parameters / arguments.
     if (!identical(type.kind, LT_TOKEN) &&
         !identical(type.kind, OPEN_PAREN_TOKEN)) {
       discardOpenLt();
     }
     groupingStack = groupingStack.prepend(token);
   }
 
@@ skipping 228 matching lines @@
     //      |
     //     next
     //      v
     //     EOF
     TokenType type = closeBraceInfoFor(begin);
     appendToken(new SyntheticToken(type, tokenStart));
     begin.endGroup = tail;
     appendErrorToken(new UnmatchedToken(begin));
   }
 }