Move Token into the public API

pkg/analyzer/lib/src/generated/scanner.dart

 // Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library analyzer.src.generated.scanner;
 
-import 'dart:collection';
-
+import 'package:analyzer/dart/ast/token.dart';
+import 'package:analyzer/src/dart/ast/token.dart';
 import 'package:analyzer/src/generated/error.dart';
 import 'package:analyzer/src/generated/java_engine.dart';
 import 'package:analyzer/src/generated/source.dart';
 
-/**
- * The opening half of a grouping pair of tokens. This is used for curly
- * brackets ('{'), parentheses ('('), and square brackets ('[').
- */
-class BeginToken extends Token {
-  /**
-   * The token that corresponds to this token.
-   */
-  Token endToken;
-
-  /**
-   * Initialize a newly created token to have the given [type] at the given
-   * [offset].
-   */
-  BeginToken(TokenType type, int offset) : super(type, offset) {
-    assert(type == TokenType.OPEN_CURLY_BRACKET ||
-        type == TokenType.OPEN_PAREN ||
-        type == TokenType.OPEN_SQUARE_BRACKET ||
-        type == TokenType.STRING_INTERPOLATION_EXPRESSION);
-  }
-
-  @override
-  Token copy() => new BeginToken(type, offset);
-}
-
-/**
- * A begin token that is preceded by comments.
- */
-class BeginTokenWithComment extends BeginToken {
-  /**
-   * The first comment in the list of comments that precede this token.
-   */
-  CommentToken _precedingComment;
-
-  /**
-   * Initialize a newly created token to have the given [type] at the given
-   * [offset] and to be preceded by the comments reachable from the given
-   * [comment].
-   */
-  BeginTokenWithComment(TokenType type, int offset, this._precedingComment)
-      : super(type, offset) {
-    _setCommentParent(_precedingComment);
-  }
-
-  CommentToken get precedingComments => _precedingComment;
-
-  void set precedingComments(CommentToken comment) {
-    _precedingComment = comment;
-    _setCommentParent(_precedingComment);
-  }
-
-  @override
-  void applyDelta(int delta) {
-    super.applyDelta(delta);
-    Token token = precedingComments;
-    while (token != null) {
-      token.applyDelta(delta);
-      token = token.next;
-    }
-  }
-
-  @override
-  Token copy() =>
-      new BeginTokenWithComment(type, offset, copyComments(precedingComments));
-}
+export 'package:analyzer/dart/ast/token.dart';
+export 'package:analyzer/src/dart/ast/token.dart' hide SimpleToken;
 
 /**
  * A [CharacterReader] that reads a range of characters from another character
  * reader.
  */
 class CharacterRangeReader extends CharacterReader {
   /**
    * The reader from which the characters are actually being read.
    */
   final CharacterReader baseReader;
@@ skipping 132 matching lines @@
   @override
   int peek() {
     if (_charOffset + 1 >= _stringLength) {
       return -1;
     }
     return _sequence.codeUnitAt(_charOffset + 1);
   }
 }
 
 /**
- * A token representing a comment.
- */
-class CommentToken extends StringToken {
-  /**
-   * The [Token] that contains this comment.
-   */
-  Token parent;
-
-  /**
-   * Initialize a newly created token to represent a token of the given [type]
-   * with the given [value] at the given [offset].
-   */
-  CommentToken(TokenType type, String value, int offset)
-      : super(type, value, offset);
-
-  @override
-  CommentToken copy() => new CommentToken(type, _value, offset);
-}
-
-/**
- * A documentation comment token.
- */
-class DocumentationCommentToken extends CommentToken {
-  /**
-   * The references embedded within the documentation comment.
-   * This list will be empty unless this is a documentation comment that has
-   * references embedded within it.
-   */
-  final List<Token> references = <Token>[];
-
-  /**
-   * Initialize a newly created token to represent a token of the given [type]
-   * with the given [value] at the given [offset].
-   */
-  DocumentationCommentToken(TokenType type, String value, int offset)
-      : super(type, value, offset);
-
-  @override
-  CommentToken copy() {
-    DocumentationCommentToken copy =
-        new DocumentationCommentToken(type, _value, offset);
-    references.forEach((ref) => copy.references.add(ref.copy()));
-    return copy;
-  }
-}
-
-/**
- * The keywords in the Dart programming language.
- */
-class Keyword {
-  static const Keyword ASSERT = const Keyword('ASSERT', "assert");
-
-  static const Keyword BREAK = const Keyword('BREAK', "break");
-
-  static const Keyword CASE = const Keyword('CASE', "case");
-
-  static const Keyword CATCH = const Keyword('CATCH', "catch");
-
-  static const Keyword CLASS = const Keyword('CLASS', "class");
-
-  static const Keyword CONST = const Keyword('CONST', "const");
-
-  static const Keyword CONTINUE = const Keyword('CONTINUE', "continue");
-
-  static const Keyword DEFAULT = const Keyword('DEFAULT', "default");
-
-  static const Keyword DO = const Keyword('DO', "do");
-
-  static const Keyword ELSE = const Keyword('ELSE', "else");
-
-  static const Keyword ENUM = const Keyword('ENUM', "enum");
-
-  static const Keyword EXTENDS = const Keyword('EXTENDS', "extends");
-
-  static const Keyword FALSE = const Keyword('FALSE', "false");
-
-  static const Keyword FINAL = const Keyword('FINAL', "final");
-
-  static const Keyword FINALLY = const Keyword('FINALLY', "finally");
-
-  static const Keyword FOR = const Keyword('FOR', "for");
-
-  static const Keyword IF = const Keyword('IF', "if");
-
-  static const Keyword IN = const Keyword('IN', "in");
-
-  static const Keyword IS = const Keyword('IS', "is");
-
-  static const Keyword NEW = const Keyword('NEW', "new");
-
-  static const Keyword NULL = const Keyword('NULL', "null");
-
-  static const Keyword RETHROW = const Keyword('RETHROW', "rethrow");
-
-  static const Keyword RETURN = const Keyword('RETURN', "return");
-
-  static const Keyword SUPER = const Keyword('SUPER', "super");
-
-  static const Keyword SWITCH = const Keyword('SWITCH', "switch");
-
-  static const Keyword THIS = const Keyword('THIS', "this");
-
-  static const Keyword THROW = const Keyword('THROW', "throw");
-
-  static const Keyword TRUE = const Keyword('TRUE', "true");
-
-  static const Keyword TRY = const Keyword('TRY', "try");
-
-  static const Keyword VAR = const Keyword('VAR', "var");
-
-  static const Keyword VOID = const Keyword('VOID', "void");
-
-  static const Keyword WHILE = const Keyword('WHILE', "while");
-
-  static const Keyword WITH = const Keyword('WITH', "with");
-
-  static const Keyword ABSTRACT = const Keyword('ABSTRACT', "abstract", true);
-
-  static const Keyword AS = const Keyword('AS', "as", true);
-
-  static const Keyword DEFERRED = const Keyword('DEFERRED', "deferred", true);
-
-  static const Keyword DYNAMIC = const Keyword('DYNAMIC', "dynamic", true);
-
-  static const Keyword EXPORT = const Keyword('EXPORT', "export", true);
-
-  static const Keyword EXTERNAL = const Keyword('EXTERNAL', "external", true);
-
-  static const Keyword FACTORY = const Keyword('FACTORY', "factory", true);
-
-  static const Keyword GET = const Keyword('GET', "get", true);
-
-  static const Keyword IMPLEMENTS =
-      const Keyword('IMPLEMENTS', "implements", true);
-
-  static const Keyword IMPORT = const Keyword('IMPORT', "import", true);
-
-  static const Keyword LIBRARY = const Keyword('LIBRARY', "library", true);
-
-  static const Keyword OPERATOR = const Keyword('OPERATOR', "operator", true);
-
-  static const Keyword PART = const Keyword('PART', "part", true);
-
-  static const Keyword SET = const Keyword('SET', "set", true);
-
-  static const Keyword STATIC = const Keyword('STATIC', "static", true);
-
-  static const Keyword TYPEDEF = const Keyword('TYPEDEF', "typedef", true);
-
-  static const List<Keyword> values = const [
-    ASSERT,
-    BREAK,
-    CASE,
-    CATCH,
-    CLASS,
-    CONST,
-    CONTINUE,
-    DEFAULT,
-    DO,
-    ELSE,
-    ENUM,
-    EXTENDS,
-    FALSE,
-    FINAL,
-    FINALLY,
-    FOR,
-    IF,
-    IN,
-    IS,
-    NEW,
-    NULL,
-    RETHROW,
-    RETURN,
-    SUPER,
-    SWITCH,
-    THIS,
-    THROW,
-    TRUE,
-    TRY,
-    VAR,
-    VOID,
-    WHILE,
-    WITH,
-    ABSTRACT,
-    AS,
-    DEFERRED,
-    DYNAMIC,
-    EXPORT,
-    EXTERNAL,
-    FACTORY,
-    GET,
-    IMPLEMENTS,
-    IMPORT,
-    LIBRARY,
-    OPERATOR,
-    PART,
-    SET,
-    STATIC,
-    TYPEDEF
-  ];
-
-  /**
-   * A table mapping the lexemes of keywords to the corresponding keyword.
-   */
-  static final Map<String, Keyword> keywords = _createKeywordMap();
-
-  /**
-   * The name of the keyword type.
-   */
-  final String name;
-
-  /**
-   * The lexeme for the keyword.
-   */
-  final String syntax;
-
-  /**
-   * A flag indicating whether the keyword is a pseudo-keyword. Pseudo keywords
-   * can be used as identifiers.
-   */
-  final bool isPseudoKeyword;
-
-  /**
-   * Initialize a newly created keyword to have the given [name] and [syntax].
-   * The keyword is a pseudo-keyword if the [isPseudoKeyword] flag is `true`.
-   */
-  const Keyword(this.name, this.syntax, [this.isPseudoKeyword = false]);
-
-  @override
-  String toString() => name;
-
-  /**
-   * Create a table mapping the lexemes of keywords to the corresponding keyword
-   * and return the table that was created.
-   */
-  static Map<String, Keyword> _createKeywordMap() {
-    LinkedHashMap<String, Keyword> result =
-        new LinkedHashMap<String, Keyword>();
-    for (Keyword keyword in values) {
-      result[keyword.syntax] = keyword;
-    }
-    return result;
-  }
-}
-
-/**
  * A state in a state machine used to scan keywords.
  */
 class KeywordState {
   /**
    * An empty transition table used by leaf states.
    */
   static List<KeywordState> _EMPTY_TABLE = new List<KeywordState>(26);
 
   /**
    * The initial state in the state machine.
@@ skipping 86 matching lines @@
     List<String> strings = new List<String>(values.length);
     for (int i = 0; i < values.length; i++) {
       strings[i] = values[i].syntax;
     }
     strings.sort();
     return _computeKeywordStateTable(0, strings, 0, strings.length);
   }
 }
 
 /**
- * A token representing a keyword in the language.
- */
-class KeywordToken extends Token {
-  /**
-   * The keyword being represented by this token.
-   */
-  final Keyword keyword;
-
-  /**
-   * Initialize a newly created token to represent the given [keyword] at the
-   * given [offset].
-   */
-  KeywordToken(this.keyword, int offset) : super(TokenType.KEYWORD, offset);
-
-  @override
-  String get lexeme => keyword.syntax;
-
-  @override
-  Token copy() => new KeywordToken(keyword, offset);
-
-  @override
-  Keyword value() => keyword;
-}
-
-/**
- * A keyword token that is preceded by comments.
- */
-class KeywordTokenWithComment extends KeywordToken {
-  /**
-   * The first comment in the list of comments that precede this token.
-   */
-  CommentToken _precedingComment;
-
-  /**
-   * Initialize a newly created token to to represent the given [keyword] at the
-   * given [offset] and to be preceded by the comments reachable from the given
-   * [comment].
-   */
-  KeywordTokenWithComment(Keyword keyword, int offset, this._precedingComment)
-      : super(keyword, offset) {
-    _setCommentParent(_precedingComment);
-  }
-
-  CommentToken get precedingComments => _precedingComment;
-
-  void set precedingComments(CommentToken comment) {
-    _precedingComment = comment;
-    _setCommentParent(_precedingComment);
-  }
-
-  @override
-  void applyDelta(int delta) {
-    super.applyDelta(delta);
-    Token token = precedingComments;
-    while (token != null) {
-      token.applyDelta(delta);
-      token = token.next;
-    }
-  }
-
-  @override
-  Token copy() => new KeywordTokenWithComment(
-      keyword, offset, copyComments(precedingComments));
-}
-
-/**
  * The class `Scanner` implements a scanner for Dart code.
  *
  * The lexical structure of Dart is ambiguous without knowledge of the context
  * in which a token is being scanned. For example, without context we cannot
  * determine whether source of the form "<<" should be scanned as a single
  * left-shift operator or as two left angle brackets. This scanner does not have
  * any context, so it always resolves such conflicts by scanning the longest
  * possible token.
  */
 class Scanner {
@@ skipping 1237 matching lines @@
       : super(name, message, correction);
 
   @override
   ErrorSeverity get errorSeverity => ErrorSeverity.ERROR;
 
   @override
   ErrorType get type => ErrorType.SYNTACTIC_ERROR;
 }
 
 /**
- * A token whose value is independent of it's type.
- */
-class StringToken extends Token {
-  /**
-   * The lexeme represented by this token.
-   */
-  String _value;
-
-  /**
-   * Initialize a newly created token to represent a token of the given [type]
-   * with the given [value] at the given [offset].
-   */
-  StringToken(TokenType type, String value, int offset) : super(type, offset) {
-    this._value = StringUtilities.intern(value);
-  }
-
-  @override
-  String get lexeme => _value;
-
-  @override
-  Token copy() => new StringToken(type, _value, offset);
-
-  @override
-  String value() => _value;
-}
-
-/**
- * A string token that is preceded by comments.
- */
-class StringTokenWithComment extends StringToken {
-  /**
-   * The first comment in the list of comments that precede this token.
-   */
-  CommentToken _precedingComment;
-
-  /**
-   * Initialize a newly created token to have the given [type] at the given
-   * [offset] and to be preceded by the comments reachable from the given
-   * [comment].
-   */
-  StringTokenWithComment(
-      TokenType type, String value, int offset, this._precedingComment)
-      : super(type, value, offset) {
-    _setCommentParent(_precedingComment);
-  }
-
-  CommentToken get precedingComments => _precedingComment;
-
-  void set precedingComments(CommentToken comment) {
-    _precedingComment = comment;
-    _setCommentParent(_precedingComment);
-  }
-
-  @override
-  void applyDelta(int delta) {
-    super.applyDelta(delta);
-    Token token = precedingComments;
-    while (token != null) {
-      token.applyDelta(delta);
-      token = token.next;
-    }
-  }
-
-  @override
-  Token copy() => new StringTokenWithComment(
-      type, lexeme, offset, copyComments(precedingComments));
-}
-
-/**
  * A [CharacterReader] that reads characters from a character sequence, but adds
  * a delta when reporting the current character offset so that the character
  * sequence can be a subsequence from a larger sequence.
  */
 class SubSequenceReader extends CharSequenceReader {
   /**
    * The offset from the beginning of the file to the beginning of the source
    * being scanned.
    */
   final int _offsetDelta;
@@ skipping 10 matching lines @@
 
   @override
   void set offset(int offset) {
     super.offset = offset - _offsetDelta;
   }
 
   @override
   String getString(int start, int endDelta) =>
       super.getString(start - _offsetDelta, endDelta);
 }
-
-/**
- * A token whose value is independent of it's type.
- */
-class SyntheticStringToken extends StringToken {
-  /**
-   * Initialize a newly created token to represent a token of the given [type]
-   * with the given [value] at the given [offset].
-   */
-  SyntheticStringToken(TokenType type, String value, int offset)
-      : super(type, value, offset);
-
-  @override
-  bool get isSynthetic => true;
-}
-
-/**
- * A token that was scanned from the input. Each token knows which tokens
- * precede and follow it, acting as a link in a doubly linked list of tokens.
- */
-class Token {
-  /**
-   * The type of the token.
-   */
-  final TokenType type;
-
-  /**
-   * The offset from the beginning of the file to the first character in the
-   * token.
-   */
-  int offset = 0;
-
-  /**
-   * The previous token in the token stream.
-   */
-  Token previous;
-
-  /**
-   * The next token in the token stream.
-   */
-  Token _next;
-
-  /**
-   * Initialize a newly created token to have the given [type] and [offset].
-   */
-  Token(this.type, this.offset);
-
-  /**
-   * Return the offset from the beginning of the file to the character after the
-   * last character of the token.
-   */
-  int get end => offset + length;
-
-  /**
-   * Return `true` if this token represents an operator.
-   */
-  bool get isOperator => type.isOperator;
-
-  /**
-   * Return `true` if this token is a synthetic token. A synthetic token is a
-   * token that was introduced by the parser in order to recover from an error
-   * in the code.
-   */
-  bool get isSynthetic => length == 0;
-
-  /**
-   * Return `true` if this token represents an operator that can be defined by
-   * users.
-   */
-  bool get isUserDefinableOperator => type.isUserDefinableOperator;
-
-  /**
-   * Return the number of characters in the node's source range.
-   */
-  int get length => lexeme.length;
-
-  /**
-   * Return the lexeme that represents this token.
-   */
-  String get lexeme => type.lexeme;
-
-  /**
-   * Return the next token in the token stream.
-   */
-  Token get next => _next;
-
-  /**
-   * Return the first comment in the list of comments that precede this token,
-   * or `null` if there are no comments preceding this token. Additional
-   * comments can be reached by following the token stream using [next] until
-   * `null` is returned.
-   *
-   * For example, if the original contents were `/* one */ /* two */ id`, then
-   * the first preceding comment token will have a lexeme of `/* one */` and
-   * the next comment token will have a lexeme of `/* two */`.
-   */
-  CommentToken get precedingComments => null;
-
-  /**
-   * Apply (add) the given [delta] to this token's offset.
-   */
-  void applyDelta(int delta) {
-    offset += delta;
-  }
-
-  /**
-   * Return a newly created token that is a copy of this token but that is not a
-   * part of any token stream.
-   */
-  Token copy() => new Token(type, offset);
-
-  /**
-   * Copy a linked list of comment tokens identical to the given comment tokens.
-   */
-  Token copyComments(Token token) {
-    if (token == null) {
-      return null;
-    }
-    Token head = token.copy();
-    Token tail = head;
-    token = token.next;
-    while (token != null) {
-      tail = tail.setNext(token.copy());
-      token = token.next;
-    }
-    return head;
-  }
-
-  /**
-   * Return `true` if this token has any one of the given [types].
-   */
-  bool matchesAny(List<TokenType> types) {
-    for (TokenType type in types) {
-      if (this.type == type) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Set the next token in the token stream to the given [token]. This has the
-   * side-effect of setting this token to be the previous token for the given
-   * token. Return the token that was passed in.
-   */
-  Token setNext(Token token) {
-    _next = token;
-    token.previous = this;
-    return token;
-  }
-
-  /**
-   * Set the next token in the token stream to the given token without changing
-   * which token is the previous token for the given token. Return the token
-   * that was passed in.
-   */
-  Token setNextWithoutSettingPrevious(Token token) {
-    _next = token;
-    return token;
-  }
-
-  @override
-  String toString() => lexeme;
-
-  /**
-   * Return the value of this token. For keyword tokens, this is the keyword
-   * associated with the token, for other tokens it is the lexeme associated
-   * with the token.
-   */
-  Object value() => type.lexeme;
-
-  /**
-   * Sets the `parent` property to `this` for the given [comment] and all the
-   * next tokens.
-   */
-  void _setCommentParent(CommentToken comment) {
-    while (comment != null) {
-      comment.parent = this;
-      comment = comment.next;
-    }
-  }
-
-  /**
-   * Compare the given [tokens] to find the token that appears first in the
-   * source being parsed. That is, return the left-most of all of the tokens.
-   * The list must be non-`null`, but the elements of the list are allowed to be
-   * `null`. Return the token with the smallest offset, or `null` if the list is
-   * empty or if all of the elements of the list are `null`.
-   */
-  static Token lexicallyFirst(List<Token> tokens) {
-    Token first = null;
-    int offset = -1;
-    for (Token token in tokens) {
-      if (token != null && (offset < 0 || token.offset < offset)) {
-        first = token;
-        offset = token.offset;
-      }
-    }
-    return first;
-  }
-}
-
-/**
- * The classes (or groups) of tokens with a similar use.
- */
-class TokenClass {
-  /**
-   * A value used to indicate that the token type is not part of any specific
-   * class of token.
-   */
-  static const TokenClass NO_CLASS = const TokenClass('NO_CLASS');
-
-  /**
-   * A value used to indicate that the token type is an additive operator.
-   */
-  static const TokenClass ADDITIVE_OPERATOR =
-      const TokenClass('ADDITIVE_OPERATOR', 13);
-
-  /**
-   * A value used to indicate that the token type is an assignment operator.
-   */
-  static const TokenClass ASSIGNMENT_OPERATOR =
-      const TokenClass('ASSIGNMENT_OPERATOR', 1);
-
-  /**
-   * A value used to indicate that the token type is a bitwise-and operator.
-   */
-  static const TokenClass BITWISE_AND_OPERATOR =
-      const TokenClass('BITWISE_AND_OPERATOR', 11);
-
-  /**
-   * A value used to indicate that the token type is a bitwise-or operator.
-   */
-  static const TokenClass BITWISE_OR_OPERATOR =
-      const TokenClass('BITWISE_OR_OPERATOR', 9);
-
-  /**
-   * A value used to indicate that the token type is a bitwise-xor operator.
-   */
-  static const TokenClass BITWISE_XOR_OPERATOR =
-      const TokenClass('BITWISE_XOR_OPERATOR', 10);
-
-  /**
-   * A value used to indicate that the token type is a cascade operator.
-   */
-  static const TokenClass CASCADE_OPERATOR =
-      const TokenClass('CASCADE_OPERATOR', 2);
-
-  /**
-   * A value used to indicate that the token type is a conditional operator.
-   */
-  static const TokenClass CONDITIONAL_OPERATOR =
-      const TokenClass('CONDITIONAL_OPERATOR', 3);
-
-  /**
-   * A value used to indicate that the token type is an equality operator.
-   */
-  static const TokenClass EQUALITY_OPERATOR =
-      const TokenClass('EQUALITY_OPERATOR', 7);
-
-  /**
-   * A value used to indicate that the token type is an if-null operator.
-   */
-  static const TokenClass IF_NULL_OPERATOR =
-      const TokenClass('IF_NULL_OPERATOR', 4);
-
-  /**
-   * A value used to indicate that the token type is a logical-and operator.
-   */
-  static const TokenClass LOGICAL_AND_OPERATOR =
-      const TokenClass('LOGICAL_AND_OPERATOR', 6);
-
-  /**
-   * A value used to indicate that the token type is a logical-or operator.
-   */
-  static const TokenClass LOGICAL_OR_OPERATOR =
-      const TokenClass('LOGICAL_OR_OPERATOR', 5);
-
-  /**
-   * A value used to indicate that the token type is a multiplicative operator.
-   */
-  static const TokenClass MULTIPLICATIVE_OPERATOR =
-      const TokenClass('MULTIPLICATIVE_OPERATOR', 14);
-
-  /**
-   * A value used to indicate that the token type is a relational operator.
-   */
-  static const TokenClass RELATIONAL_OPERATOR =
-      const TokenClass('RELATIONAL_OPERATOR', 8);
-
-  /**
-   * A value used to indicate that the token type is a shift operator.
-   */
-  static const TokenClass SHIFT_OPERATOR =
-      const TokenClass('SHIFT_OPERATOR', 12);
-
-  /**
-   * A value used to indicate that the token type is a unary operator.
-   */
-  static const TokenClass UNARY_POSTFIX_OPERATOR =
-      const TokenClass('UNARY_POSTFIX_OPERATOR', 16);
-
-  /**
-   * A value used to indicate that the token type is a unary operator.
-   */
-  static const TokenClass UNARY_PREFIX_OPERATOR =
-      const TokenClass('UNARY_PREFIX_OPERATOR', 15);
-
-  /**
-   * The name of the token class.
-   */
-  final String name;
-
-  /**
-   * The precedence of tokens of this class, or `0` if the such tokens do not
-   * represent an operator.
-   */
-  final int precedence;
-
-  const TokenClass(this.name, [this.precedence = 0]);
-
-  @override
-  String toString() => name;
-}
-
-/**
- * The types of tokens that can be returned by the scanner.
- */
-class TokenType {
-  /**
-   * The type of the token that marks the end of the input.
-   */
-  static const TokenType EOF = const TokenType_EOF('EOF');
-
-  static const TokenType DOUBLE = const TokenType('DOUBLE');
-
-  static const TokenType HEXADECIMAL = const TokenType('HEXADECIMAL');
-
-  static const TokenType IDENTIFIER = const TokenType('IDENTIFIER');
-
-  static const TokenType INT = const TokenType('INT');
-
-  static const TokenType KEYWORD = const TokenType('KEYWORD');
-
-  static const TokenType MULTI_LINE_COMMENT =
-      const TokenType('MULTI_LINE_COMMENT');
-
-  static const TokenType SCRIPT_TAG = const TokenType('SCRIPT_TAG');
-
-  static const TokenType SINGLE_LINE_COMMENT =
-      const TokenType('SINGLE_LINE_COMMENT');
-
-  static const TokenType STRING = const TokenType('STRING');
-
-  static const TokenType AMPERSAND =
-      const TokenType('AMPERSAND', TokenClass.BITWISE_AND_OPERATOR, "&");
-
-  static const TokenType AMPERSAND_AMPERSAND = const TokenType(
-      'AMPERSAND_AMPERSAND', TokenClass.LOGICAL_AND_OPERATOR, "&&");
-
-  static const TokenType AMPERSAND_EQ =
-      const TokenType('AMPERSAND_EQ', TokenClass.ASSIGNMENT_OPERATOR, "&=");
-
-  static const TokenType AT = const TokenType('AT', TokenClass.NO_CLASS, "@");
-
-  static const TokenType BANG =
-      const TokenType('BANG', TokenClass.UNARY_PREFIX_OPERATOR, "!");
-
-  static const TokenType BANG_EQ =
-      const TokenType('BANG_EQ', TokenClass.EQUALITY_OPERATOR, "!=");
-
-  static const TokenType BAR =
-      const TokenType('BAR', TokenClass.BITWISE_OR_OPERATOR, "|");
-
-  static const TokenType BAR_BAR =
-      const TokenType('BAR_BAR', TokenClass.LOGICAL_OR_OPERATOR, "||");
-
-  static const TokenType BAR_EQ =
-      const TokenType('BAR_EQ', TokenClass.ASSIGNMENT_OPERATOR, "|=");
-
-  static const TokenType COLON =
-      const TokenType('COLON', TokenClass.NO_CLASS, ":");
-
-  static const TokenType COMMA =
-      const TokenType('COMMA', TokenClass.NO_CLASS, ",");
-
-  static const TokenType CARET =
-      const TokenType('CARET', TokenClass.BITWISE_XOR_OPERATOR, "^");
-
-  static const TokenType CARET_EQ =
-      const TokenType('CARET_EQ', TokenClass.ASSIGNMENT_OPERATOR, "^=");
-
-  static const TokenType CLOSE_CURLY_BRACKET =
-      const TokenType('CLOSE_CURLY_BRACKET', TokenClass.NO_CLASS, "}");
-
-  static const TokenType CLOSE_PAREN =
-      const TokenType('CLOSE_PAREN', TokenClass.NO_CLASS, ")");
-
-  static const TokenType CLOSE_SQUARE_BRACKET =
-      const TokenType('CLOSE_SQUARE_BRACKET', TokenClass.NO_CLASS, "]");
-
-  static const TokenType EQ =
-      const TokenType('EQ', TokenClass.ASSIGNMENT_OPERATOR, "=");
-
-  static const TokenType EQ_EQ =
-      const TokenType('EQ_EQ', TokenClass.EQUALITY_OPERATOR, "==");
-
-  static const TokenType FUNCTION =
-      const TokenType('FUNCTION', TokenClass.NO_CLASS, "=>");
-
-  static const TokenType GT =
-      const TokenType('GT', TokenClass.RELATIONAL_OPERATOR, ">");
-
-  static const TokenType GT_EQ =
-      const TokenType('GT_EQ', TokenClass.RELATIONAL_OPERATOR, ">=");
-
-  static const TokenType GT_GT =
-      const TokenType('GT_GT', TokenClass.SHIFT_OPERATOR, ">>");
-
-  static const TokenType GT_GT_EQ =
-      const TokenType('GT_GT_EQ', TokenClass.ASSIGNMENT_OPERATOR, ">>=");
-
-  static const TokenType HASH =
-      const TokenType('HASH', TokenClass.NO_CLASS, "#");
-
-  static const TokenType INDEX =
-      const TokenType('INDEX', TokenClass.UNARY_POSTFIX_OPERATOR, "[]");
-
-  static const TokenType INDEX_EQ =
-      const TokenType('INDEX_EQ', TokenClass.UNARY_POSTFIX_OPERATOR, "[]=");
-
-  static const TokenType IS =
-      const TokenType('IS', TokenClass.RELATIONAL_OPERATOR, "is");
-
-  static const TokenType LT =
-      const TokenType('LT', TokenClass.RELATIONAL_OPERATOR, "<");
-
-  static const TokenType LT_EQ =
-      const TokenType('LT_EQ', TokenClass.RELATIONAL_OPERATOR, "<=");
-
-  static const TokenType LT_LT =
-      const TokenType('LT_LT', TokenClass.SHIFT_OPERATOR, "<<");
-
-  static const TokenType LT_LT_EQ =
-      const TokenType('LT_LT_EQ', TokenClass.ASSIGNMENT_OPERATOR, "<<=");
-
-  static const TokenType MINUS =
-      const TokenType('MINUS', TokenClass.ADDITIVE_OPERATOR, "-");
-
-  static const TokenType MINUS_EQ =
-      const TokenType('MINUS_EQ', TokenClass.ASSIGNMENT_OPERATOR, "-=");
-
-  static const TokenType MINUS_MINUS =
-      const TokenType('MINUS_MINUS', TokenClass.UNARY_PREFIX_OPERATOR, "--");
-
-  static const TokenType OPEN_CURLY_BRACKET =
-      const TokenType('OPEN_CURLY_BRACKET', TokenClass.NO_CLASS, "{");
-
-  static const TokenType OPEN_PAREN =
-      const TokenType('OPEN_PAREN', TokenClass.UNARY_POSTFIX_OPERATOR, "(");
-
-  static const TokenType OPEN_SQUARE_BRACKET = const TokenType(
-      'OPEN_SQUARE_BRACKET', TokenClass.UNARY_POSTFIX_OPERATOR, "[");
-
-  static const TokenType PERCENT =
-      const TokenType('PERCENT', TokenClass.MULTIPLICATIVE_OPERATOR, "%");
-
-  static const TokenType PERCENT_EQ =
-      const TokenType('PERCENT_EQ', TokenClass.ASSIGNMENT_OPERATOR, "%=");
-
-  static const TokenType PERIOD =
-      const TokenType('PERIOD', TokenClass.UNARY_POSTFIX_OPERATOR, ".");
-
-  static const TokenType PERIOD_PERIOD =
-      const TokenType('PERIOD_PERIOD', TokenClass.CASCADE_OPERATOR, "..");
-
-  static const TokenType PLUS =
-      const TokenType('PLUS', TokenClass.ADDITIVE_OPERATOR, "+");
-
-  static const TokenType PLUS_EQ =
-      const TokenType('PLUS_EQ', TokenClass.ASSIGNMENT_OPERATOR, "+=");
-
-  static const TokenType PLUS_PLUS =
-      const TokenType('PLUS_PLUS', TokenClass.UNARY_PREFIX_OPERATOR, "++");
-
-  static const TokenType QUESTION =
-      const TokenType('QUESTION', TokenClass.CONDITIONAL_OPERATOR, "?");
-
-  static const TokenType QUESTION_PERIOD = const TokenType(
-      'QUESTION_PERIOD', TokenClass.UNARY_POSTFIX_OPERATOR, '?.');
-
-  static const TokenType QUESTION_QUESTION =
-      const TokenType('QUESTION_QUESTION', TokenClass.IF_NULL_OPERATOR, '??');
-
-  static const TokenType QUESTION_QUESTION_EQ = const TokenType(
-      'QUESTION_QUESTION_EQ', TokenClass.ASSIGNMENT_OPERATOR, '??=');
-
-  static const TokenType SEMICOLON =
-      const TokenType('SEMICOLON', TokenClass.NO_CLASS, ";");
-
-  static const TokenType SLASH =
-      const TokenType('SLASH', TokenClass.MULTIPLICATIVE_OPERATOR, "/");
-
-  static const TokenType SLASH_EQ =
-      const TokenType('SLASH_EQ', TokenClass.ASSIGNMENT_OPERATOR, "/=");
-
-  static const TokenType STAR =
-      const TokenType('STAR', TokenClass.MULTIPLICATIVE_OPERATOR, "*");
-
-  static const TokenType STAR_EQ =
-      const TokenType('STAR_EQ', TokenClass.ASSIGNMENT_OPERATOR, "*=");
-
-  static const TokenType STRING_INTERPOLATION_EXPRESSION = const TokenType(
-      'STRING_INTERPOLATION_EXPRESSION', TokenClass.NO_CLASS, "\${");
-
-  static const TokenType STRING_INTERPOLATION_IDENTIFIER = const TokenType(
-      'STRING_INTERPOLATION_IDENTIFIER', TokenClass.NO_CLASS, "\$");
-
-  static const TokenType TILDE =
-      const TokenType('TILDE', TokenClass.UNARY_PREFIX_OPERATOR, "~");
-
-  static const TokenType TILDE_SLASH =
-      const TokenType('TILDE_SLASH', TokenClass.MULTIPLICATIVE_OPERATOR, "~/");
-
-  static const TokenType TILDE_SLASH_EQ =
-      const TokenType('TILDE_SLASH_EQ', TokenClass.ASSIGNMENT_OPERATOR, "~/=");
-
-  static const TokenType BACKPING =
-      const TokenType('BACKPING', TokenClass.NO_CLASS, "`");
-
-  static const TokenType BACKSLASH =
-      const TokenType('BACKSLASH', TokenClass.NO_CLASS, "\\");
-
-  static const TokenType PERIOD_PERIOD_PERIOD =
-      const TokenType('PERIOD_PERIOD_PERIOD', TokenClass.NO_CLASS, "...");
-
-  static const TokenType GENERIC_METHOD_TYPE_LIST =
-      const TokenType('GENERIC_METHOD_TYPE_LIST');
-
-  static const TokenType GENERIC_METHOD_TYPE_ASSIGN =
-      const TokenType('GENERIC_METHOD_TYPE_ASSIGN');
-
-  /**
-   * The class of the token.
-   */
-  final TokenClass _tokenClass;
-
-  /**
-   * The name of the token type.
-   */
-  final String name;
-
-  /**
-   * The lexeme that defines this type of token, or `null` if there is more than
-   * one possible lexeme for this type of token.
-   */
-  final String lexeme;
-
-  const TokenType(this.name,
-      [this._tokenClass = TokenClass.NO_CLASS, this.lexeme = null]);
-
-  /**
-   * Return `true` if this type of token represents an additive operator.
-   */
-  bool get isAdditiveOperator => _tokenClass == TokenClass.ADDITIVE_OPERATOR;
-
-  /**
-   * Return `true` if this type of token represents an assignment operator.
-   */
-  bool get isAssignmentOperator =>
-      _tokenClass == TokenClass.ASSIGNMENT_OPERATOR;
-
-  /**
-   * Return `true` if this type of token represents an associative operator. An
-   * associative operator is an operator for which the following equality is
-   * true: `(a * b) * c == a * (b * c)`. In other words, if the result of
-   * applying the operator to multiple operands does not depend on the order in
-   * which those applications occur.
-   *
-   * Note: This method considers the logical-and and logical-or operators to be
-   * associative, even though the order in which the application of those
-   * operators can have an effect because evaluation of the right-hand operand
-   * is conditional.
-   */
-  bool get isAssociativeOperator =>
-      this == AMPERSAND ||
-      this == AMPERSAND_AMPERSAND ||
-      this == BAR ||
-      this == BAR_BAR ||
-      this == CARET ||
-      this == PLUS ||
-      this == STAR;
-
-  /**
-   * Return `true` if this type of token represents an equality operator.
-   */
-  bool get isEqualityOperator => _tokenClass == TokenClass.EQUALITY_OPERATOR;
-
-  /**
-   * Return `true` if this type of token represents an increment operator.
-   */
-  bool get isIncrementOperator =>
-      identical(lexeme, "++") || identical(lexeme, "--");
-
-  /**
-   * Return `true` if this type of token represents a multiplicative operator.
-   */
-  bool get isMultiplicativeOperator =>
-      _tokenClass == TokenClass.MULTIPLICATIVE_OPERATOR;
-
-  /**
-   * Return `true` if this token type represents an operator.
-   */
-  bool get isOperator =>
-      _tokenClass != TokenClass.NO_CLASS &&
-      this != OPEN_PAREN &&
-      this != OPEN_SQUARE_BRACKET &&
-      this != PERIOD;
-
-  /**
-   * Return `true` if this type of token represents a relational operator.
-   */
-  bool get isRelationalOperator =>
-      _tokenClass == TokenClass.RELATIONAL_OPERATOR;
-
-  /**
-   * Return `true` if this type of token represents a shift operator.
-   */
-  bool get isShiftOperator => _tokenClass == TokenClass.SHIFT_OPERATOR;
-
-  /**
-   * Return `true` if this type of token represents a unary postfix operator.
-   */
-  bool get isUnaryPostfixOperator =>
-      _tokenClass == TokenClass.UNARY_POSTFIX_OPERATOR;
-
-  /**
-   * Return `true` if this type of token represents a unary prefix operator.
-   */
-  bool get isUnaryPrefixOperator =>
-      _tokenClass == TokenClass.UNARY_PREFIX_OPERATOR;
-
-  /**
-   * Return `true` if this token type represents an operator that can be defined
-   * by users.
-   */
-  bool get isUserDefinableOperator =>
-      identical(lexeme, "==") ||
-      identical(lexeme, "~") ||
-      identical(lexeme, "[]") ||
-      identical(lexeme, "[]=") ||
-      identical(lexeme, "*") ||
-      identical(lexeme, "/") ||
-      identical(lexeme, "%") ||
-      identical(lexeme, "~/") ||
-      identical(lexeme, "+") ||
-      identical(lexeme, "-") ||
-      identical(lexeme, "<<") ||
-      identical(lexeme, ">>") ||
-      identical(lexeme, ">=") ||
-      identical(lexeme, ">") ||
-      identical(lexeme, "<=") ||
-      identical(lexeme, "<") ||
-      identical(lexeme, "&") ||
-      identical(lexeme, "^") ||
-      identical(lexeme, "|");
-
-  /**
-   * Return the precedence of the token, or `0` if the token does not represent
-   * an operator.
-   */
-  int get precedence => _tokenClass.precedence;
-
-  @override
-  String toString() => name;
-}
-
-class TokenType_EOF extends TokenType {
-  const TokenType_EOF(String name) : super(name, TokenClass.NO_CLASS, "");
-
-  @override
-  String toString() => "-eof-";
-}
-
-/**
- * A normal token that is preceded by comments.
- */
-class TokenWithComment extends Token {
-  /**
-   * The first comment in the list of comments that precede this token.
-   */
-  CommentToken _precedingComment;
-
-  /**
-   * Initialize a newly created token to have the given [type] at the given
-   * [offset] and to be preceded by the comments reachable from the given
-   * [comment].
-   */
-  TokenWithComment(TokenType type, int offset, this._precedingComment)
-      : super(type, offset) {
-    _setCommentParent(_precedingComment);
-  }
-
-  CommentToken get precedingComments => _precedingComment;
-
-  void set precedingComments(CommentToken comment) {
-    _precedingComment = comment;
-    _setCommentParent(_precedingComment);
-  }
-
-  @override
-  Token copy() => new TokenWithComment(type, offset, precedingComments);
-}