cleanup fasta token classes

pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.scanner.abstract_scanner;
 
 import 'dart:collection' show ListMixin;
 
 import 'dart:typed_data' show Uint16List, Uint32List;
 
-import '../../scanner/token.dart' show Token, TokenType;
+import '../../scanner/token.dart' show BeginToken, Token, TokenType;
 
 import '../scanner.dart'
     show ErrorToken, Keyword, Scanner, buildUnexpectedCharacterToken;
 
 import 'error_token.dart' show UnterminatedToken;
 
 import 'keyword_state.dart' show KeywordState;
 
-import 'token.dart'
-    show BeginGroupToken, CommentToken, DartDocToken, SymbolToken;
+import 'token.dart' show CommentToken, DartDocToken;
 
 import 'token_constants.dart';
 
 import 'characters.dart';
 
 abstract class AbstractScanner implements Scanner {
   final bool includeComments;
 
   /**
    * A flag indicating whether to parse generic method comments, of the form
@@ skipping 10 matching lines @@
    */
   int tokenStart = -1;
 
   /**
    * A pointer to the token stream created by this scanner. The first token
    * is a special token and not part of the source file. This is an
    * implementation detail to avoids special cases in the scanner. This token
    * is not exposed to clients of the scanner, which are expected to invoke
    * [firstToken] to access the token stream.
    */
-  final Token tokens = new SymbolToken.eof(-1);
+  final Token tokens = new Token.eof(-1);
 
   /**
    * A pointer to the last scanned token.
    */
   Token tail;
 
   /**
    * A pointer to the stream of comment tokens created by this scanner
    * before they are assigned to the [Token] precedingComments field
    * of a non-comment token. A value of `null` indicates no comment tokens.
@@ skipping 809 matching lines @@
     _appendToCommentStream(newComment);
   }
 
   /**
    * Append the given token to the [tail] of the current stream of tokens.
    */
   void appendToken(Token token) {
     tail.next = token;
     tail.next.previous = tail;
     tail = tail.next;
-    if (comments != null) {
+    if (comments != null && comments == token.precedingComments) {
+      comments = null;
+      commentsTail = null;
+    } else {
       // It is the responsibility of the caller to construct the token
       // being appended with preceeding comments if any
-      assert(identical(token.precedingComments, comments));
-      comments = null;
-      commentsTail = null;
+      assert(comments == null || token.isSynthetic);
     }
   }
 
   void _appendToCommentStream(Token newComment) {
     if (comments == null) {
       comments = newComment;
       commentsTail = comments;
     } else {
       commentsTail.next = newComment;
       commentsTail.next.previous = commentsTail;
@@ skipping 314 matching lines @@
   int advanceAfterError(bool shouldAdvance) {
     if (atEndOfFile()) return $EOF;
     if (shouldAdvance) {
       return advance(); // Ensure progress.
     } else {
       return -1;
     }
   }
 }
 
-TokenType closeBraceInfoFor(BeginGroupToken begin) {
+TokenType closeBraceInfoFor(BeginToken begin) {
   return const {
     '(': TokenType.CLOSE_PAREN,
     '[': TokenType.CLOSE_SQUARE_BRACKET,
     '{': TokenType.CLOSE_CURLY_BRACKET,
     '<': TokenType.GT,
     r'${': TokenType.CLOSE_CURLY_BRACKET,
   }[begin.lexeme];
 }
 
 class LineStarts extends Object with ListMixin<int> {
@@ skipping 65 matching lines @@
       switchToUint32(newLength);
     }
   }
 
   void switchToUint32(int newLength) {
     final newArray = new Uint32List(newLength);
     newArray.setRange(0, arrayLength, array);
     array = newArray;
   }
 }