cleanup fasta token classes

pkg/front_end/lib/src/scanner/token.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Defines the tokens that are produced by the scanner, used by the parser, and
  * referenced from the [AST structure](ast.dart).
  */
 import 'dart:collection';
 
@@ skipping 27 matching lines @@
   /**
    * The token that corresponds to this token.
    */
   Token endToken;
 
   /**
    * Initialize a newly created token to have the given [type] at the given
    * [offset].
    */
   BeginToken(TokenType type, int offset) : super(type, offset) {
-    assert(type == TokenType.OPEN_CURLY_BRACKET ||
+    assert(type == TokenType.LT ||
+        type == TokenType.OPEN_CURLY_BRACKET ||
         type == TokenType.OPEN_PAREN ||
         type == TokenType.OPEN_SQUARE_BRACKET ||
         type == TokenType.STRING_INTERPOLATION_EXPRESSION);
   }
 
   @override
   Token copy() => new BeginToken(type, offset);
+
+  /**
+   * The token that corresponds to this token.
+   */
+  Token get endGroup => endToken;
+
+  /**
+   * Set the token that corresponds to this token.
+   */
+  set endGroup(Token token) {
+    endToken = token;
+  }
 }
 
 /**
  * A begin token that is preceded by comments.
  */
 class BeginTokenWithComment extends BeginToken implements TokenWithComment {
   /**
    * The first comment in the list of comments that precede this token.
    */
   @override
@@ skipping 639 matching lines @@
 class SyntheticStringToken extends StringToken {
   /**
    * Initialize a newly created token to represent a token of the given [type]
    * with the given [value] at the given [offset].
    */
   SyntheticStringToken(TokenType type, String value, int offset)
       : super(type, value, offset);
 
   @override
   bool get isSynthetic => true;
+
+  @override
+  Token copy() => new SyntheticStringToken(type, _value, offset);
 }
 
 /**
+ * A synthetic token.
+ */
+class SyntheticToken extends SimpleToken {
+  SyntheticToken(TokenType type, int offset) : super(type, offset);
+
+  @override
+  int get length => 0;
+
+  @override
+  Token copy() => new SyntheticToken(type, offset);
+}
+
+/**
  * A token that was scanned from the input. Each token knows which tokens
  * precede and follow it, acting as a link in a doubly linked list of tokens.
  *
  * Clients may not extend, implement or mix-in this class.
  */
 abstract class Token implements SyntacticEntity {
   /**
    * Initialize a newly created token to have the given [type] and [offset].
    */
   factory Token(TokenType type, int offset) = SimpleToken;
 
   /**
+   * Initialize a newly created end-of-file token to have the given [offset].
+   */
+  factory Token.eof(int offset, [CommentToken precedingComments]) {
+    Token eof = precedingComments == null
+        ? new SimpleToken(TokenType.EOF, offset)
+        : new TokenWithComment(TokenType.EOF, offset, precedingComments);
+    // EOF points to itself so there's always infinite look-ahead.
+    eof.previous = eof;
+    eof.next = eof;
+    return eof;
+  }
+
+  /**
    * The number of characters parsed by this token.
    */
   int get charCount;
 
   /**
    * The character offset of the start of this token within the source text.
    */
   int get charOffset;
 
   /**
@@ skipping 898 matching lines @@
 
   void set precedingComments(CommentToken comment) {
     _precedingComment = comment;
     _setCommentParent(_precedingComment);
   }
 
   @override
   Token copy() =>
       new TokenWithComment(type, offset, copyComments(precedingComments));
 }