New analyzer snapshot. Sorted unit members.

pkg/analyzer/lib/src/generated/scanner.dart

 // Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 // This code was auto-generated, is not intended to be edited, and is subject to
 // significant change. Please see the README file for more information.
 
 library engine.scanner;
 
 import 'dart:collection';
 import 'java_core.dart';
 import 'java_engine.dart';
 import 'source.dart';
 import 'error.dart';
 import 'instrumentation.dart';
 import 'utilities_collection.dart' show TokenMap;
 
 /**
+ * Instances of the class `BeginToken` represent the opening half of a grouping pair of
+ * tokens. This is used for curly brackets ('{'), parentheses ('('), and square brackets ('[').
+ */
+class BeginToken extends Token {
+  /**
+   * The token that corresponds to this token.
+   */
+  Token endToken;
+
+  /**
+   * Initialize a newly created token representing the opening half of a grouping pair of tokens.
+   *
+   * @param type the type of the token
+   * @param offset the offset from the beginning of the file to the first character in the token
+   */
+  BeginToken(TokenType type, int offset) : super(type, offset) {
+    assert((type == TokenType.OPEN_CURLY_BRACKET || type == TokenType.OPEN_PAREN || type == TokenType.OPEN_SQUARE_BRACKET || type == TokenType.STRING_INTERPOLATION_EXPRESSION));
+  }
+
+  @override
+  Token copy() => new BeginToken(type, offset);
+}
+
+/**
+ * Instances of the class `BeginTokenWithComment` represent a begin token that is preceded by
+ * comments.
+ */
+class BeginTokenWithComment extends BeginToken {
+  /**
+   * The first comment in the list of comments that precede this token.
+   */
+  final Token _precedingComment;
+
+  /**
+   * Initialize a newly created token to have the given type and offset and to be preceded by the
+   * comments reachable from the given comment.
+   *
+   * @param type the type of the token
+   * @param offset the offset from the beginning of the file to the first character in the token
+   * @param precedingComment the first comment in the list of comments that precede this token
+   */
+  BeginTokenWithComment(TokenType type, int offset, this._precedingComment) : super(type, offset);
+
+  @override
+  Token copy() => new BeginTokenWithComment(type, offset, copyComments(_precedingComment));
+
+  @override
+  Token get precedingComments => _precedingComment;
+
+  @override
+  void applyDelta(int delta) {
+    super.applyDelta(delta);
+    Token token = _precedingComment;
+    while (token != null) {
+      token.applyDelta(delta);
+      token = token.next;
+    }
+  }
+}
+
+/**
+ * Instances of the class `CharSequenceReader` implement a [CharacterReader] that reads
+ * characters from a character sequence.
+ */
+class CharSequenceReader implements CharacterReader {
+  /**
+   * The sequence from which characters will be read.
+   */
+  final String _sequence;
+
+  /**
+   * The number of characters in the string.
+   */
+  int _stringLength = 0;
+
+  /**
+   * The index, relative to the string, of the last character that was read.
+   */
+  int _charOffset = 0;
+
+  /**
+   * Initialize a newly created reader to read the characters in the given sequence.
+   *
+   * @param sequence the sequence from which characters will be read
+   */
+  CharSequenceReader(this._sequence) {
+    this._stringLength = _sequence.length;
+    this._charOffset = -1;
+  }
+
+  @override
+  int advance() {
+    if (_charOffset + 1 >= _stringLength) {
+      return -1;
+    }
+    return _sequence.codeUnitAt(++_charOffset);
+  }
+
+  @override
+  int get offset => _charOffset;
+
+  @override
+  String getString(int start, int endDelta) => _sequence.substring(start, _charOffset + 1 + endDelta).toString();
+
+  @override
+  int peek() {
+    if (_charOffset + 1 >= _sequence.length) {
+      return -1;
+    }
+    return _sequence.codeUnitAt(_charOffset + 1);
+  }
+
+  @override
+  void set offset(int offset) {
+    _charOffset = offset;
+  }
+}
+
+/**
+ * The interface `CharacterReader`
+ */
+abstract class CharacterReader {
+  /**
+   * Advance the current position and return the character at the new current position.
+   *
+   * @return the character at the new current position
+   */
+  int advance();
+
+  /**
+   * Return the current offset relative to the beginning of the source. Return the initial offset if
+   * the scanner has not yet scanned the source code, and one (1) past the end of the source code if
+   * the entire source code has been scanned.
+   *
+   * @return the current offset of the scanner in the source
+   */
+  int get offset;
+
+  /**
+   * Return the substring of the source code between the start offset and the modified current
+   * position. The current position is modified by adding the end delta.
+   *
+   * @param start the offset to the beginning of the string, relative to the start of the file
+   * @param endDelta the number of characters after the current location to be included in the
+   *          string, or the number of characters before the current location to be excluded if the
+   *          offset is negative
+   * @return the specified substring of the source code
+   */
+  String getString(int start, int endDelta);
+
+  /**
+   * Return the character at the current position without changing the current position.
+   *
+   * @return the character at the current position
+   */
+  int peek();
+
+  /**
+   * Set the current offset relative to the beginning of the source. The new offset must be between
+   * the initial offset and one (1) past the end of the source code.
+   *
+   * @param offset the new offset in the source
+   */
+  void set offset(int offset);
+}
+
+/**
  * Instances of the class `IncrementalScanner` implement a scanner that scans a subset of a
  * string and inserts the resulting tokens into the middle of an existing token stream.
  */
 class IncrementalScanner extends Scanner {
   /**
    * The reader used to access the characters in the source.
    */
   CharacterReader _reader;
 
   /**
@@ skipping 200 matching lines @@
    * incremental scanner, two tokens are equal if they have the same type and lexeme.
    *
    * @param oldToken the token from the old stream that is being compared
    * @param newToken the token from the new stream that is being compared
    * @return `true` if the two tokens are equal to each other
    */
   bool _equalTokens(Token oldToken, Token newToken) => oldToken.type == newToken.type && oldToken.length == newToken.length && oldToken.lexeme == newToken.lexeme;
 }
 
 /**
- * The interface `CharacterReader`
+ * The enumeration `Keyword` defines the keywords in the Dart programming language.
  */
-abstract class CharacterReader {
-  /**
-   * Advance the current position and return the character at the new current position.
-   *
-   * @return the character at the new current position
-   */
-  int advance();
-
-  /**
-   * Return the current offset relative to the beginning of the source. Return the initial offset if
-   * the scanner has not yet scanned the source code, and one (1) past the end of the source code if
-   * the entire source code has been scanned.
-   *
-   * @return the current offset of the scanner in the source
-   */
-  int get offset;
-
-  /**
-   * Return the substring of the source code between the start offset and the modified current
-   * position. The current position is modified by adding the end delta.
-   *
-   * @param start the offset to the beginning of the string, relative to the start of the file
-   * @param endDelta the number of characters after the current location to be included in the
-   *          string, or the number of characters before the current location to be excluded if the
-   *          offset is negative
-   * @return the specified substring of the source code
-   */
-  String getString(int start, int endDelta);
-
-  /**
-   * Return the character at the current position without changing the current position.
-   *
-   * @return the character at the current position
-   */
-  int peek();
-
-  /**
-   * Set the current offset relative to the beginning of the source. The new offset must be between
-   * the initial offset and one (1) past the end of the source code.
-   *
-   * @param offset the new offset in the source
-   */
-  void set offset(int offset);
+class Keyword extends Enum<Keyword> {
+  static const Keyword ASSERT = const Keyword.con1('ASSERT', 0, "assert");
+
+  static const Keyword BREAK = const Keyword.con1('BREAK', 1, "break");
+
+  static const Keyword CASE = const Keyword.con1('CASE', 2, "case");
+
+  static const Keyword CATCH = const Keyword.con1('CATCH', 3, "catch");
+
+  static const Keyword CLASS = const Keyword.con1('CLASS', 4, "class");
+
+  static const Keyword CONST = const Keyword.con1('CONST', 5, "const");
+
+  static const Keyword CONTINUE = const Keyword.con1('CONTINUE', 6, "continue");
+
+  static const Keyword DEFAULT = const Keyword.con1('DEFAULT', 7, "default");
+
+  static const Keyword DO = const Keyword.con1('DO', 8, "do");
+
+  static const Keyword ELSE = const Keyword.con1('ELSE', 9, "else");
+
+  static const Keyword ENUM = const Keyword.con1('ENUM', 10, "enum");
+
+  static const Keyword EXTENDS = const Keyword.con1('EXTENDS', 11, "extends");
+
+  static const Keyword FALSE = const Keyword.con1('FALSE', 12, "false");
+
+  static const Keyword FINAL = const Keyword.con1('FINAL', 13, "final");
+
+  static const Keyword FINALLY = const Keyword.con1('FINALLY', 14, "finally");
+
+  static const Keyword FOR = const Keyword.con1('FOR', 15, "for");
+
+  static const Keyword IF = const Keyword.con1('IF', 16, "if");
+
+  static const Keyword IN = const Keyword.con1('IN', 17, "in");
+
+  static const Keyword IS = const Keyword.con1('IS', 18, "is");
+
+  static const Keyword NEW = const Keyword.con1('NEW', 19, "new");
+
+  static const Keyword NULL = const Keyword.con1('NULL', 20, "null");
+
+  static const Keyword RETHROW = const Keyword.con1('RETHROW', 21, "rethrow");
+
+  static const Keyword RETURN = const Keyword.con1('RETURN', 22, "return");
+
+  static const Keyword SUPER = const Keyword.con1('SUPER', 23, "super");
+
+  static const Keyword SWITCH = const Keyword.con1('SWITCH', 24, "switch");
+
+  static const Keyword THIS = const Keyword.con1('THIS', 25, "this");
+
+  static const Keyword THROW = const Keyword.con1('THROW', 26, "throw");
+
+  static const Keyword TRUE = const Keyword.con1('TRUE', 27, "true");
+
+  static const Keyword TRY = const Keyword.con1('TRY', 28, "try");
+
+  static const Keyword VAR = const Keyword.con1('VAR', 29, "var");
+
+  static const Keyword VOID = const Keyword.con1('VOID', 30, "void");
+
+  static const Keyword WHILE = const Keyword.con1('WHILE', 31, "while");
+
+  static const Keyword WITH = const Keyword.con1('WITH', 32, "with");
+
+  static const Keyword ABSTRACT = const Keyword.con2('ABSTRACT', 33, "abstract", true);
+
+  static const Keyword AS = const Keyword.con2('AS', 34, "as", true);
+
+  static const Keyword DEFERRED = const Keyword.con2('DEFERRED', 35, "deferred", true);
+
+  static const Keyword DYNAMIC = const Keyword.con2('DYNAMIC', 36, "dynamic", true);
+
+  static const Keyword EXPORT = const Keyword.con2('EXPORT', 37, "export", true);
+
+  static const Keyword EXTERNAL = const Keyword.con2('EXTERNAL', 38, "external", true);
+
+  static const Keyword FACTORY = const Keyword.con2('FACTORY', 39, "factory", true);
+
+  static const Keyword GET = const Keyword.con2('GET', 40, "get", true);
+
+  static const Keyword IMPLEMENTS = const Keyword.con2('IMPLEMENTS', 41, "implements", true);
+
+  static const Keyword IMPORT = const Keyword.con2('IMPORT', 42, "import", true);
+
+  static const Keyword LIBRARY = const Keyword.con2('LIBRARY', 43, "library", true);
+
+  static const Keyword OPERATOR = const Keyword.con2('OPERATOR', 44, "operator", true);
+
+  static const Keyword PART = const Keyword.con2('PART', 45, "part", true);
+
+  static const Keyword SET = const Keyword.con2('SET', 46, "set", true);
+
+  static const Keyword STATIC = const Keyword.con2('STATIC', 47, "static", true);
+
+  static const Keyword TYPEDEF = const Keyword.con2('TYPEDEF', 48, "typedef", true);
+
+  static const List<Keyword> values = const [
+      ASSERT,
+      BREAK,
+      CASE,
+      CATCH,
+      CLASS,
+      CONST,
+      CONTINUE,
+      DEFAULT,
+      DO,
+      ELSE,
+      ENUM,
+      EXTENDS,
+      FALSE,
+      FINAL,
+      FINALLY,
+      FOR,
+      IF,
+      IN,
+      IS,
+      NEW,
+      NULL,
+      RETHROW,
+      RETURN,
+      SUPER,
+      SWITCH,
+      THIS,
+      THROW,
+      TRUE,
+      TRY,
+      VAR,
+      VOID,
+      WHILE,
+      WITH,
+      ABSTRACT,
+      AS,
+      DEFERRED,
+      DYNAMIC,
+      EXPORT,
+      EXTERNAL,
+      FACTORY,
+      GET,
+      IMPLEMENTS,
+      IMPORT,
+      LIBRARY,
+      OPERATOR,
+      PART,
+      SET,
+      STATIC,
+      TYPEDEF];
+
+  /**
+   * The lexeme for the keyword.
+   */
+  final String syntax;
+
+  /**
+   * A flag indicating whether the keyword is a pseudo-keyword. Pseudo keywords can be used as
+   * identifiers.
+   */
+  final bool isPseudoKeyword;
+
+  /**
+   * A table mapping the lexemes of keywords to the corresponding keyword.
+   */
+  static Map<String, Keyword> keywords = _createKeywordMap();
+
+  /**
+   * Create a table mapping the lexemes of keywords to the corresponding keyword.
+   *
+   * @return the table that was created
+   */
+  static Map<String, Keyword> _createKeywordMap() {
+    LinkedHashMap<String, Keyword> result = new LinkedHashMap<String, Keyword>();
+    for (Keyword keyword in values) {
+      result[keyword.syntax] = keyword;
+    }
+    return result;
+  }
+
+  /**
+   * Initialize a newly created keyword to have the given syntax. The keyword is not a
+   * pseudo-keyword.
+   *
+   * @param syntax the lexeme for the keyword
+   */
+  const Keyword.con1(String name, int ordinal, String syntax) : this.con2(name, ordinal, syntax, false);
+
+  /**
+   * Initialize a newly created keyword to have the given syntax. The keyword is a pseudo-keyword if
+   * the given flag is `true`.
+   *
+   * @param syntax the lexeme for the keyword
+   * @param isPseudoKeyword `true` if this keyword is a pseudo-keyword
+   */
+  const Keyword.con2(String name, int ordinal, this.syntax, this.isPseudoKeyword) : super(name, ordinal);
 }
 
 /**
- * Instances of the class `TokenWithComment` represent a normal token that is preceded by
- * comments.
+ * Instances of the abstract class `KeywordState` represent a state in a state machine used to
+ * scan keywords.
  */
-class TokenWithComment extends Token {
+class KeywordState {
+  /**
+   * An empty transition table used by leaf states.
+   */
+  static List<KeywordState> _EMPTY_TABLE = new List<KeywordState>(26);
+
+  /**
+   * The initial state in the state machine.
+   */
+  static KeywordState KEYWORD_STATE = _createKeywordStateTable();
+
+  /**
+   * Create the next state in the state machine where we have already recognized the subset of
+   * strings in the given array of strings starting at the given offset and having the given length.
+   * All of these strings have a common prefix and the next character is at the given start index.
+   *
+   * @param start the index of the character in the strings used to transition to a new state
+   * @param strings an array containing all of the strings that will be recognized by the state
+   *          machine
+   * @param offset the offset of the first string in the array that has the prefix that is assumed
+   *          to have been recognized by the time we reach the state being built
+   * @param length the number of strings in the array that pass through the state being built
+   * @return the state that was created
+   */
+  static KeywordState _computeKeywordStateTable(int start, List<String> strings, int offset, int length) {
+    List<KeywordState> result = new List<KeywordState>(26);
+    assert(length != 0);
+    int chunk = 0x0;
+    int chunkStart = -1;
+    bool isLeaf = false;
+    for (int i = offset; i < offset + length; i++) {
+      if (strings[i].length == start) {
+        isLeaf = true;
+      }
+      if (strings[i].length > start) {
+        int c = strings[i].codeUnitAt(start);
+        if (chunk != c) {
+          if (chunkStart != -1) {
+            result[chunk - 0x61] = _computeKeywordStateTable(start + 1, strings, chunkStart, i - chunkStart);
+          }
+          chunkStart = i;
+          chunk = c;
+        }
+      }
+    }
+    if (chunkStart != -1) {
+      assert(result[chunk - 0x61] == null);
+      result[chunk - 0x61] = _computeKeywordStateTable(start + 1, strings, chunkStart, offset + length - chunkStart);
+    } else {
+      assert(length == 1);
+      return new KeywordState(_EMPTY_TABLE, strings[offset]);
+    }
+    if (isLeaf) {
+      return new KeywordState(result, strings[offset]);
+    } else {
+      return new KeywordState(result, null);
+    }
+  }
+
+  /**
+   * Create the initial state in the state machine.
+   *
+   * @return the state that was created
+   */
+  static KeywordState _createKeywordStateTable() {
+    List<Keyword> values = Keyword.values;
+    List<String> strings = new List<String>(values.length);
+    for (int i = 0; i < values.length; i++) {
+      strings[i] = values[i].syntax;
+    }
+    strings.sort();
+    return _computeKeywordStateTable(0, strings, 0, strings.length);
+  }
+
+  /**
+   * A table mapping characters to the states to which those characters will transition. (The index
+   * into the array is the offset from the character `'a'` to the transitioning character.)
+   */
+  final List<KeywordState> _table;
+
+  /**
+   * The keyword that is recognized by this state, or `null` if this state is not a terminal
+   * state.
+   */
+  Keyword _keyword;
+
+  /**
+   * Initialize a newly created state to have the given transitions and to recognize the keyword
+   * with the given syntax.
+   *
+   * @param table a table mapping characters to the states to which those characters will transition
+   * @param syntax the syntax of the keyword that is recognized by the state
+   */
+  KeywordState(this._table, String syntax) {
+    this._keyword = (syntax == null) ? null : Keyword.keywords[syntax];
+  }
+
+  /**
+   * Return the keyword that was recognized by this state, or `null` if this state does not
+   * recognized a keyword.
+   *
+   * @return the keyword that was matched by reaching this state
+   */
+  Keyword keyword() => _keyword;
+
+  /**
+   * Return the state that follows this state on a transition of the given character, or
+   * `null` if there is no valid state reachable from this state with such a transition.
+   *
+   * @param c the character used to transition from this state to another state
+   * @return the state that follows this state on a transition of the given character
+   */
+  KeywordState next(int c) => _table[c - 0x61];
+}
+
+/**
+ * Instances of the class `KeywordToken` represent a keyword in the language.
+ */
+class KeywordToken extends Token {
+  /**
+   * The keyword being represented by this token.
+   */
+  final Keyword keyword;
+
+  /**
+   * Initialize a newly created token to represent the given keyword.
+   *
+   * @param keyword the keyword being represented by this token
+   * @param offset the offset from the beginning of the file to the first character in the token
+   */
+  KeywordToken(this.keyword, int offset) : super(TokenType.KEYWORD, offset);
+
+  @override
+  Token copy() => new KeywordToken(keyword, offset);
+
+  @override
+  String get lexeme => keyword.syntax;
+
+  @override
+  Keyword value() => keyword;
+}
+
+/**
+ * Instances of the class `KeywordTokenWithComment` implement a keyword token that is preceded
+ * by comments.
+ */
+class KeywordTokenWithComment extends KeywordToken {
   /**
    * The first comment in the list of comments that precede this token.
    */
   final Token _precedingComment;
 
   /**
-   * Initialize a newly created token to have the given type and offset and to be preceded by the
+   * Initialize a newly created token to to represent the given keyword and to be preceded by the
    * comments reachable from the given comment.
    *
-   * @param type the type of the token
+   * @param keyword the keyword being represented by this token
    * @param offset the offset from the beginning of the file to the first character in the token
    * @param precedingComment the first comment in the list of comments that precede this token
    */
-  TokenWithComment(TokenType type, int offset, this._precedingComment) : super(type, offset);
+  KeywordTokenWithComment(Keyword keyword, int offset, this._precedingComment) : super(keyword, offset);
 
   @override
-  Token copy() => new TokenWithComment(type, offset, _precedingComment);
-
-  @override
-  Token get precedingComments => _precedingComment;
-}
-
-/**
- * Instances of the class `BeginTokenWithComment` represent a begin token that is preceded by
- * comments.
- */
-class BeginTokenWithComment extends BeginToken {
-  /**
-   * The first comment in the list of comments that precede this token.
-   */
-  final Token _precedingComment;
-
-  /**
-   * Initialize a newly created token to have the given type and offset and to be preceded by the
-   * comments reachable from the given comment.
-   *
-   * @param type the type of the token
-   * @param offset the offset from the beginning of the file to the first character in the token
-   * @param precedingComment the first comment in the list of comments that precede this token
-   */
-  BeginTokenWithComment(TokenType type, int offset, this._precedingComment) : super(type, offset);
-
-  @override
-  Token copy() => new BeginTokenWithComment(type, offset, copyComments(_precedingComment));
+  Token copy() => new KeywordTokenWithComment(keyword, offset, copyComments(_precedingComment));
 
   @override
   Token get precedingComments => _precedingComment;
 
   @override
   void applyDelta(int delta) {
     super.applyDelta(delta);
     Token token = _precedingComment;
     while (token != null) {
       token.applyDelta(delta);
       token = token.next;
     }
   }
 }
 
 /**
- * Instances of the class `SubSequenceReader` implement a [CharacterReader] that reads
- * characters from a character sequence, but adds a delta when reporting the current character
- * offset so that the character sequence can be a subsequence from a larger sequence.
+ * The class `Scanner` implements a scanner for Dart code.
+ *
+ * The lexical structure of Dart is ambiguous without knowledge of the context in which a token is
+ * being scanned. For example, without context we cannot determine whether source of the form "<<"
+ * should be scanned as a single left-shift operator or as two left angle brackets. This scanner
+ * does not have any context, so it always resolves such conflicts by scanning the longest possible
+ * token.
  */
-class SubSequenceReader extends CharSequenceReader {
-  /**
-   * The offset from the beginning of the file to the beginning of the source being scanned.
-   */
-  final int _offsetDelta;
-
-  /**
-   * Initialize a newly created reader to read the characters in the given sequence.
-   *
-   * @param sequence the sequence from which characters will be read
-   * @param offsetDelta the offset from the beginning of the file to the beginning of the source
-   *          being scanned
-   */
-  SubSequenceReader(String sequence, this._offsetDelta) : super(sequence);
-
-  @override
-  int get offset => _offsetDelta + super.offset;
-
-  @override
-  String getString(int start, int endDelta) => super.getString(start - _offsetDelta, endDelta);
-
-  @override
-  void set offset(int offset) {
-    super.offset = offset - _offsetDelta;
+class Scanner {
+  /**
+   * The source being scanned.
+   */
+  final Source source;
+
+  /**
+   * The reader used to access the characters in the source.
+   */
+  final CharacterReader _reader;
+
+  /**
+   * The error listener that will be informed of any errors that are found during the scan.
+   */
+  final AnalysisErrorListener _errorListener;
+
+  /**
+   * The flag specifying if documentation comments should be parsed.
+   */
+  bool _preserveComments = true;
+
+  /**
+   * The token pointing to the head of the linked list of tokens.
+   */
+  Token _tokens;
+
+  /**
+   * The last token that was scanned.
+   */
+  Token _tail;
+
+  /**
+   * The first token in the list of comment tokens found since the last non-comment token.
+   */
+  Token _firstComment;
+
+  /**
+   * The last token in the list of comment tokens found since the last non-comment token.
+   */
+  Token _lastComment;
+
+  /**
+   * The index of the first character of the current token.
+   */
+  int _tokenStart = 0;
+
+  /**
+   * A list containing the offsets of the first character of each line in the source code.
+   */
+  List<int> _lineStarts = new List<int>();
+
+  /**
+   * A list, treated something like a stack, of tokens representing the beginning of a matched pair.
+   * It is used to pair the end tokens with the begin tokens.
+   */
+  List<BeginToken> _groupingStack = new List<BeginToken>();
+
+  /**
+   * The index of the last item in the [groupingStack], or `-1` if the stack is empty.
+   */
+  int _stackEnd = -1;
+
+  /**
+   * A flag indicating whether any unmatched groups were found during the parse.
+   */
+  bool _hasUnmatchedGroups = false;
+
+  /**
+   * Initialize a newly created scanner.
+   *
+   * @param source the source being scanned
+   * @param reader the character reader used to read the characters in the source
+   * @param errorListener the error listener that will be informed of any errors that are found
+   */
+  Scanner(this.source, this._reader, this._errorListener) {
+    _tokens = new Token(TokenType.EOF, -1);
+    _tokens.setNext(_tokens);
+    _tail = _tokens;
+    _tokenStart = -1;
+    _lineStarts.add(0);
+  }
+
+  /**
+   * Return an array containing the offsets of the first character of each line in the source code.
+   *
+   * @return an array containing the offsets of the first character of each line in the source code
+   */
+  List<int> get lineStarts => _lineStarts;
+
+  /**
+   * Return `true` if any unmatched groups were found during the parse.
+   *
+   * @return `true` if any unmatched groups were found during the parse
+   */
+  bool get hasUnmatchedGroups => _hasUnmatchedGroups;
+
+  /**
+   * Set whether documentation tokens should be scanned.
+   *
+   * @param preserveComments `true` if documentation tokens should be scanned
+   */
+  void set preserveComments(bool preserveComments) {
+    this._preserveComments = preserveComments;
+  }
+
+  /**
+   * Record that the source begins on the given line and column at the current offset as given by
+   * the reader. The line starts for lines before the given line will not be correct.
+   *
+   * This method must be invoked at most one time and must be invoked before scanning begins. The
+   * values provided must be sensible. The results are undefined if these conditions are violated.
+   *
+   * @param line the one-based index of the line containing the first character of the source
+   * @param column the one-based index of the column in which the first character of the source
+   *          occurs
+   */
+  void setSourceStart(int line, int column) {
+    int offset = _reader.offset;
+    if (line < 1 || column < 1 || offset < 0 || (line + column - 2) >= offset) {
+      return;
+    }
+    for (int i = 2; i < line; i++) {
+      _lineStarts.add(1);
+    }
+    _lineStarts.add(offset - column + 1);
+  }
+
+  /**
+   * Scan the source code to produce a list of tokens representing the source.
+   *
+   * @return the first token in the list of tokens that were produced
+   */
+  Token tokenize() {
+    InstrumentationBuilder instrumentation = Instrumentation.builder2("dart.engine.AbstractScanner.tokenize");
+    int tokenCounter = 0;
+    try {
+      int next = _reader.advance();
+      while (next != -1) {
+        tokenCounter++;
+        next = bigSwitch(next);
+      }
+      _appendEofToken();
+      instrumentation.metric2("tokensCount", tokenCounter);
+      return firstToken;
+    } finally {
+      instrumentation.log2(2);
+    }
+  }
+
+  /**
+   * Append the given token to the end of the token stream being scanned. This method is intended to
+   * be used by subclasses that copy existing tokens and should not normally be used because it will
+   * fail to correctly associate any comments with the token being passed in.
+   *
+   * @param token the token to be appended
+   */
+  void appendToken(Token token) {
+    _tail = _tail.setNext(token);
+  }
+
+  int bigSwitch(int next) {
+    _beginToken();
+    if (next == 0xD) {
+      next = _reader.advance();
+      if (next == 0xA) {
+        next = _reader.advance();
+      }
+      recordStartOfLine();
+      return next;
+    } else if (next == 0xA) {
+      next = _reader.advance();
+      recordStartOfLine();
+      return next;
+    } else if (next == 0x9 || next == 0x20) {
+      return _reader.advance();
+    }
+    if (next == 0x72) {
+      int peek = _reader.peek();
+      if (peek == 0x22 || peek == 0x27) {
+        int start = _reader.offset;
+        return _tokenizeString(_reader.advance(), start, true);
+      }
+    }
+    if (0x61 <= next && next <= 0x7A) {
+      return _tokenizeKeywordOrIdentifier(next, true);
+    }
+    if ((0x41 <= next && next <= 0x5A) || next == 0x5F || next == 0x24) {
+      return _tokenizeIdentifier(next, _reader.offset, true);
+    }
+    if (next == 0x3C) {
+      return _tokenizeLessThan(next);
+    }
+    if (next == 0x3E) {
+      return _tokenizeGreaterThan(next);
+    }
+    if (next == 0x3D) {
+      return _tokenizeEquals(next);
+    }
+    if (next == 0x21) {
+      return _tokenizeExclamation(next);
+    }
+    if (next == 0x2B) {
+      return _tokenizePlus(next);
+    }
+    if (next == 0x2D) {
+      return _tokenizeMinus(next);
+    }
+    if (next == 0x2A) {
+      return _tokenizeMultiply(next);
+    }
+    if (next == 0x25) {
+      return _tokenizePercent(next);
+    }
+    if (next == 0x26) {
+      return _tokenizeAmpersand(next);
+    }
+    if (next == 0x7C) {
+      return _tokenizeBar(next);
+    }
+    if (next == 0x5E) {
+      return _tokenizeCaret(next);
+    }
+    if (next == 0x5B) {
+      return _tokenizeOpenSquareBracket(next);
+    }
+    if (next == 0x7E) {
+      return _tokenizeTilde(next);
+    }
+    if (next == 0x5C) {
+      _appendTokenOfType(TokenType.BACKSLASH);
+      return _reader.advance();
+    }
+    if (next == 0x23) {
+      return _tokenizeTag(next);
+    }
+    if (next == 0x28) {
+      _appendBeginToken(TokenType.OPEN_PAREN);
+      return _reader.advance();
+    }
+    if (next == 0x29) {
+      _appendEndToken(TokenType.CLOSE_PAREN, TokenType.OPEN_PAREN);
+      return _reader.advance();
+    }
+    if (next == 0x2C) {
+      _appendTokenOfType(TokenType.COMMA);
+      return _reader.advance();
+    }
+    if (next == 0x3A) {
+      _appendTokenOfType(TokenType.COLON);
+      return _reader.advance();
+    }
+    if (next == 0x3B) {
+      _appendTokenOfType(TokenType.SEMICOLON);
+      return _reader.advance();
+    }
+    if (next == 0x3F) {
+      _appendTokenOfType(TokenType.QUESTION);
+      return _reader.advance();
+    }
+    if (next == 0x5D) {
+      _appendEndToken(TokenType.CLOSE_SQUARE_BRACKET, TokenType.OPEN_SQUARE_BRACKET);
+      return _reader.advance();
+    }
+    if (next == 0x60) {
+      _appendTokenOfType(TokenType.BACKPING);
+      return _reader.advance();
+    }
+    if (next == 0x7B) {
+      _appendBeginToken(TokenType.OPEN_CURLY_BRACKET);
+      return _reader.advance();
+    }
+    if (next == 0x7D) {
+      _appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.OPEN_CURLY_BRACKET);
+      return _reader.advance();
+    }
+    if (next == 0x2F) {
+      return _tokenizeSlashOrComment(next);
+    }
+    if (next == 0x40) {
+      _appendTokenOfType(TokenType.AT);
+      return _reader.advance();
+    }
+    if (next == 0x22 || next == 0x27) {
+      return _tokenizeString(next, _reader.offset, false);
+    }
+    if (next == 0x2E) {
+      return _tokenizeDotOrNumber(next);
+    }
+    if (next == 0x30) {
+      return _tokenizeHexOrNumber(next);
+    }
+    if (0x31 <= next && next <= 0x39) {
+      return _tokenizeNumber(next);
+    }
+    if (next == -1) {
+      return -1;
+    }
+    _reportError(ScannerErrorCode.ILLEGAL_CHARACTER, [next]);
+    return _reader.advance();
+  }
+
+  /**
+   * Return the first token in the token stream that was scanned.
+   *
+   * @return the first token in the token stream that was scanned
+   */
+  Token get firstToken => _tokens.next;
+
+  /**
+   * Return the last token that was scanned.
+   *
+   * @return the last token that was scanned
+   */
+  Token get tail => _tail;
+
+  /**
+   * Record the fact that we are at the beginning of a new line in the source.
+   */
+  void recordStartOfLine() {
+    _lineStarts.add(_reader.offset);
+  }
+
+  void _appendBeginToken(TokenType type) {
+    BeginToken token;
+    if (_firstComment == null) {
+      token = new BeginToken(type, _tokenStart);
+    } else {
+      token = new BeginTokenWithComment(type, _tokenStart, _firstComment);
+      _firstComment = null;
+      _lastComment = null;
+    }
+    _tail = _tail.setNext(token);
+    _groupingStack.add(token);
+    _stackEnd++;
+  }
+
+  void _appendCommentToken(TokenType type, String value) {
+    // Ignore comment tokens if client specified that it doesn't need them.
+    if (!_preserveComments) {
+      return;
+    }
+    // OK, remember comment tokens.
+    if (_firstComment == null) {
+      _firstComment = new StringToken(type, value, _tokenStart);
+      _lastComment = _firstComment;
+    } else {
+      _lastComment = _lastComment.setNext(new StringToken(type, value, _tokenStart));
+    }
+  }
+
+  void _appendEndToken(TokenType type, TokenType beginType) {
+    Token token;
+    if (_firstComment == null) {
+      token = new Token(type, _tokenStart);
+    } else {
+      token = new TokenWithComment(type, _tokenStart, _firstComment);
+      _firstComment = null;
+      _lastComment = null;
+    }
+    _tail = _tail.setNext(token);
+    if (_stackEnd >= 0) {
+      BeginToken begin = _groupingStack[_stackEnd];
+      if (begin.type == beginType) {
+        begin.endToken = token;
+        _groupingStack.removeAt(_stackEnd--);
+      }
+    }
+  }
+
+  void _appendEofToken() {
+    Token eofToken;
+    if (_firstComment == null) {
+      eofToken = new Token(TokenType.EOF, _reader.offset + 1);
+    } else {
+      eofToken = new TokenWithComment(TokenType.EOF, _reader.offset + 1, _firstComment);
+      _firstComment = null;
+      _lastComment = null;
+    }
+    // The EOF token points to itself so that there is always infinite look-ahead.
+    eofToken.setNext(eofToken);
+    _tail = _tail.setNext(eofToken);
+    if (_stackEnd >= 0) {
+      _hasUnmatchedGroups = true;
+    }
+  }
+
+  void _appendKeywordToken(Keyword keyword) {
+    if (_firstComment == null) {
+      _tail = _tail.setNext(new KeywordToken(keyword, _tokenStart));
+    } else {
+      _tail = _tail.setNext(new KeywordTokenWithComment(keyword, _tokenStart, _firstComment));
+      _firstComment = null;
+      _lastComment = null;
+    }
+  }
+
+  void _appendStringToken(TokenType type, String value) {
+    if (_firstComment == null) {
+      _tail = _tail.setNext(new StringToken(type, value, _tokenStart));
+    } else {
+      _tail = _tail.setNext(new StringTokenWithComment(type, value, _tokenStart, _firstComment));
+      _firstComment = null;
+      _lastComment = null;
+    }
+  }
+
+  void _appendStringTokenWithOffset(TokenType type, String value, int offset) {
+    if (_firstComment == null) {
+      _tail = _tail.setNext(new StringToken(type, value, _tokenStart + offset));
+    } else {
+      _tail = _tail.setNext(new StringTokenWithComment(type, value, _tokenStart + offset, _firstComment));
+      _firstComment = null;
+      _lastComment = null;
+    }
+  }
+
+  void _appendTokenOfType(TokenType type) {
+    if (_firstComment == null) {
+      _tail = _tail.setNext(new Token(type, _tokenStart));
+    } else {
+      _tail = _tail.setNext(new TokenWithComment(type, _tokenStart, _firstComment));
+      _firstComment = null;
+      _lastComment = null;
+    }
+  }
+
+  void _appendTokenOfTypeWithOffset(TokenType type, int offset) {
+    if (_firstComment == null) {
+      _tail = _tail.setNext(new Token(type, offset));
+    } else {
+      _tail = _tail.setNext(new TokenWithComment(type, offset, _firstComment));
+      _firstComment = null;
+      _lastComment = null;
+    }
+  }
+
+  void _beginToken() {
+    _tokenStart = _reader.offset;
+  }
+
+  /**
+   * Return the beginning token corresponding to a closing brace that was found while scanning
+   * inside a string interpolation expression. Tokens that cannot be matched with the closing brace
+   * will be dropped from the stack.
+   *
+   * @return the token to be paired with the closing brace
+   */
+  BeginToken _findTokenMatchingClosingBraceInInterpolationExpression() {
+    while (_stackEnd >= 0) {
+      BeginToken begin = _groupingStack[_stackEnd];
+      if (begin.type == TokenType.OPEN_CURLY_BRACKET || begin.type == TokenType.STRING_INTERPOLATION_EXPRESSION) {
+        return begin;
+      }
+      _hasUnmatchedGroups = true;
+      _groupingStack.removeAt(_stackEnd--);
+    }
+    //
+    // We should never get to this point because we wouldn't be inside a string interpolation
+    // expression unless we had previously found the start of the expression.
+    //
+    return null;
+  }
+
+  /**
+   * Report an error at the current offset.
+   *
+   * @param errorCode the error code indicating the nature of the error
+   * @param arguments any arguments needed to complete the error message
+   */
+  void _reportError(ScannerErrorCode errorCode, List<Object> arguments) {
+    _errorListener.onError(new AnalysisError.con2(source, _reader.offset, 1, errorCode, arguments));
+  }
+
+  int _select(int choice, TokenType yesType, TokenType noType) {
+    int next = _reader.advance();
+    if (next == choice) {
+      _appendTokenOfType(yesType);
+      return _reader.advance();
+    } else {
+      _appendTokenOfType(noType);
+      return next;
+    }
+  }
+
+  int _selectWithOffset(int choice, TokenType yesType, TokenType noType, int offset) {
+    int next = _reader.advance();
+    if (next == choice) {
+      _appendTokenOfTypeWithOffset(yesType, offset);
+      return _reader.advance();
+    } else {
+      _appendTokenOfTypeWithOffset(noType, offset);
+      return next;
+    }
+  }
+
+  int _tokenizeAmpersand(int next) {
+    // && &= &
+    next = _reader.advance();
+    if (next == 0x26) {
+      _appendTokenOfType(TokenType.AMPERSAND_AMPERSAND);
+      return _reader.advance();
+    } else if (next == 0x3D) {
+      _appendTokenOfType(TokenType.AMPERSAND_EQ);
+      return _reader.advance();
+    } else {
+      _appendTokenOfType(TokenType.AMPERSAND);
+      return next;
+    }
+  }
+
+  int _tokenizeBar(int next) {
+    // | || |=
+    next = _reader.advance();
+    if (next == 0x7C) {
+      _appendTokenOfType(TokenType.BAR_BAR);
+      return _reader.advance();
+    } else if (next == 0x3D) {
+      _appendTokenOfType(TokenType.BAR_EQ);
+      return _reader.advance();
+    } else {
+      _appendTokenOfType(TokenType.BAR);
+      return next;
+    }
+  }
+
+  int _tokenizeCaret(int next) => _select(0x3D, TokenType.CARET_EQ, TokenType.CARET);
+
+  int _tokenizeDotOrNumber(int next) {
+    int start = _reader.offset;
+    next = _reader.advance();
+    if (0x30 <= next && next <= 0x39) {
+      return _tokenizeFractionPart(next, start);
+    } else if (0x2E == next) {
+      return _select(0x2E, TokenType.PERIOD_PERIOD_PERIOD, TokenType.PERIOD_PERIOD);
+    } else {
+      _appendTokenOfType(TokenType.PERIOD);
+      return next;
+    }
+  }
+
+  int _tokenizeEquals(int next) {
+    // = == =>
+    next = _reader.advance();
+    if (next == 0x3D) {
+      _appendTokenOfType(TokenType.EQ_EQ);
+      return _reader.advance();
+    } else if (next == 0x3E) {
+      _appendTokenOfType(TokenType.FUNCTION);
+      return _reader.advance();
+    }
+    _appendTokenOfType(TokenType.EQ);
+    return next;
+  }
+
+  int _tokenizeExclamation(int next) {
+    // ! !=
+    next = _reader.advance();
+    if (next == 0x3D) {
+      _appendTokenOfType(TokenType.BANG_EQ);
+      return _reader.advance();
+    }
+    _appendTokenOfType(TokenType.BANG);
+    return next;
+  }
+
+  int _tokenizeExponent(int next) {
+    if (next == 0x2B || next == 0x2D) {
+      next = _reader.advance();
+    }
+    bool hasDigits = false;
+    while (true) {
+      if (0x30 <= next && next <= 0x39) {
+        hasDigits = true;
+      } else {
+        if (!hasDigits) {
+          _reportError(ScannerErrorCode.MISSING_DIGIT, []);
+        }
+        return next;
+      }
+      next = _reader.advance();
+    }
+  }
+
+  int _tokenizeFractionPart(int next, int start) {
+    bool done = false;
+    bool hasDigit = false;
+    LOOP: while (!done) {
+      if (0x30 <= next && next <= 0x39) {
+        hasDigit = true;
+      } else if (0x65 == next || 0x45 == next) {
+        hasDigit = true;
+        next = _tokenizeExponent(_reader.advance());
+        done = true;
+        continue LOOP;
+      } else {
+        done = true;
+        continue LOOP;
+      }
+      next = _reader.advance();
+    }
+    if (!hasDigit) {
+      _appendStringToken(TokenType.INT, _reader.getString(start, -2));
+      if (0x2E == next) {
+        return _selectWithOffset(0x2E, TokenType.PERIOD_PERIOD_PERIOD, TokenType.PERIOD_PERIOD, _reader.offset - 1);
+      }
+      _appendTokenOfTypeWithOffset(TokenType.PERIOD, _reader.offset - 1);
+      return bigSwitch(next);
+    }
+    _appendStringToken(TokenType.DOUBLE, _reader.getString(start, next < 0 ? 0 : -1));
+    return next;
+  }
+
+  int _tokenizeGreaterThan(int next) {
+    // > >= >> >>=
+    next = _reader.advance();
+    if (0x3D == next) {
+      _appendTokenOfType(TokenType.GT_EQ);
+      return _reader.advance();
+    } else if (0x3E == next) {
+      next = _reader.advance();
+      if (0x3D == next) {
+        _appendTokenOfType(TokenType.GT_GT_EQ);
+        return _reader.advance();
+      } else {
+        _appendTokenOfType(TokenType.GT_GT);
+        return next;
+      }
+    } else {
+      _appendTokenOfType(TokenType.GT);
+      return next;
+    }
+  }
+
+  int _tokenizeHex(int next) {
+    int start = _reader.offset - 1;
+    bool hasDigits = false;
+    while (true) {
+      next = _reader.advance();
+      if ((0x30 <= next && next <= 0x39) || (0x41 <= next && next <= 0x46) || (0x61 <= next && next <= 0x66)) {
+        hasDigits = true;
+      } else {
+        if (!hasDigits) {
+          _reportError(ScannerErrorCode.MISSING_HEX_DIGIT, []);
+        }
+        _appendStringToken(TokenType.HEXADECIMAL, _reader.getString(start, next < 0 ? 0 : -1));
+        return next;
+      }
+    }
+  }
+
+  int _tokenizeHexOrNumber(int next) {
+    int x = _reader.peek();
+    if (x == 0x78 || x == 0x58) {
+      _reader.advance();
+      return _tokenizeHex(x);
+    }
+    return _tokenizeNumber(next);
+  }
+
+  int _tokenizeIdentifier(int next, int start, bool allowDollar) {
+    while ((0x61 <= next && next <= 0x7A) || (0x41 <= next && next <= 0x5A) || (0x30 <= next && next <= 0x39) || next == 0x5F || (next == 0x24 && allowDollar)) {
+      next = _reader.advance();
+    }
+    _appendStringToken(TokenType.IDENTIFIER, _reader.getString(start, next < 0 ? 0 : -1));
+    return next;
+  }
+
+  int _tokenizeInterpolatedExpression(int next, int start) {
+    _appendBeginToken(TokenType.STRING_INTERPOLATION_EXPRESSION);
+    next = _reader.advance();
+    while (next != -1) {
+      if (next == 0x7D) {
+        BeginToken begin = _findTokenMatchingClosingBraceInInterpolationExpression();
+        if (begin == null) {
+          _beginToken();
+          _appendTokenOfType(TokenType.CLOSE_CURLY_BRACKET);
+          next = _reader.advance();
+          _beginToken();
+          return next;
+        } else if (begin.type == TokenType.OPEN_CURLY_BRACKET) {
+          _beginToken();
+          _appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.OPEN_CURLY_BRACKET);
+          next = _reader.advance();
+          _beginToken();
+        } else if (begin.type == TokenType.STRING_INTERPOLATION_EXPRESSION) {
+          _beginToken();
+          _appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.STRING_INTERPOLATION_EXPRESSION);
+          next = _reader.advance();
+          _beginToken();
+          return next;
+        }
+      } else {
+        next = bigSwitch(next);
+      }
+    }
+    return next;
+  }
+
+  int _tokenizeInterpolatedIdentifier(int next, int start) {
+    _appendStringTokenWithOffset(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 0);
+    if ((0x41 <= next && next <= 0x5A) || (0x61 <= next && next <= 0x7A) || next == 0x5F) {
+      _beginToken();
+      next = _tokenizeKeywordOrIdentifier(next, false);
+    }
+    _beginToken();
+    return next;
+  }
+
+  int _tokenizeKeywordOrIdentifier(int next, bool allowDollar) {
+    KeywordState state = KeywordState.KEYWORD_STATE;
+    int start = _reader.offset;
+    while (state != null && 0x61 <= next && next <= 0x7A) {
+      state = state.next(next);
+      next = _reader.advance();
+    }
+    if (state == null || state.keyword() == null) {
+      return _tokenizeIdentifier(next, start, allowDollar);
+    }
+    if ((0x41 <= next && next <= 0x5A) || (0x30 <= next && next <= 0x39) || next == 0x5F || next == 0x24) {
+      return _tokenizeIdentifier(next, start, allowDollar);
+    } else if (next < 128) {
+      _appendKeywordToken(state.keyword());
+      return next;
+    } else {
+      return _tokenizeIdentifier(next, start, allowDollar);
+    }
+  }
+
+  int _tokenizeLessThan(int next) {
+    // < <= << <<=
+    next = _reader.advance();
+    if (0x3D == next) {
+      _appendTokenOfType(TokenType.LT_EQ);
+      return _reader.advance();
+    } else if (0x3C == next) {
+      return _select(0x3D, TokenType.LT_LT_EQ, TokenType.LT_LT);
+    } else {
+      _appendTokenOfType(TokenType.LT);
+      return next;
+    }
+  }
+
+  int _tokenizeMinus(int next) {
+    // - -- -=
+    next = _reader.advance();
+    if (next == 0x2D) {
+      _appendTokenOfType(TokenType.MINUS_MINUS);
+      return _reader.advance();
+    } else if (next == 0x3D) {
+      _appendTokenOfType(TokenType.MINUS_EQ);
+      return _reader.advance();
+    } else {
+      _appendTokenOfType(TokenType.MINUS);
+      return next;
+    }
+  }
+
+  int _tokenizeMultiLineComment(int next) {
+    int nesting = 1;
+    next = _reader.advance();
+    while (true) {
+      if (-1 == next) {
+        _reportError(ScannerErrorCode.UNTERMINATED_MULTI_LINE_COMMENT, []);
+        _appendCommentToken(TokenType.MULTI_LINE_COMMENT, _reader.getString(_tokenStart, 0));
+        return next;
+      } else if (0x2A == next) {
+        next = _reader.advance();
+        if (0x2F == next) {
+          --nesting;
+          if (0 == nesting) {
+            _appendCommentToken(TokenType.MULTI_LINE_COMMENT, _reader.getString(_tokenStart, 0));
+            return _reader.advance();
+          } else {
+            next = _reader.advance();
+          }
+        }
+      } else if (0x2F == next) {
+        next = _reader.advance();
+        if (0x2A == next) {
+          next = _reader.advance();
+          ++nesting;
+        }
+      } else if (next == 0xD) {
+        next = _reader.advance();
+        if (next == 0xA) {
+          next = _reader.advance();
+        }
+        recordStartOfLine();
+      } else if (next == 0xA) {
+        recordStartOfLine();
+        next = _reader.advance();
+      } else {
+        next = _reader.advance();
+      }
+    }
+  }
+
+  int _tokenizeMultiLineRawString(int quoteChar, int start) {
+    int next = _reader.advance();
+    outer: while (next != -1) {
+      while (next != quoteChar) {
+        next = _reader.advance();
+        if (next == -1) {
+          break outer;
+        } else if (next == 0xD) {
+          next = _reader.advance();
+          if (next == 0xA) {
+            next = _reader.advance();
+          }
+          recordStartOfLine();
+        } else if (next == 0xA) {
+          recordStartOfLine();
+          next = _reader.advance();
+        }
+      }
+      next = _reader.advance();
+      if (next == quoteChar) {
+        next = _reader.advance();
+        if (next == quoteChar) {
+          _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
+          return _reader.advance();
+        }
+      }
+    }
+    _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []);
+    _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
+    return _reader.advance();
+  }
+
+  int _tokenizeMultiLineString(int quoteChar, int start, bool raw) {
+    if (raw) {
+      return _tokenizeMultiLineRawString(quoteChar, start);
+    }
+    int next = _reader.advance();
+    while (next != -1) {
+      if (next == 0x24) {
+        _appendStringToken(TokenType.STRING, _reader.getString(start, -1));
+        _beginToken();
+        next = _tokenizeStringInterpolation(start);
+        start = _reader.offset;
+        continue;
+      }
+      if (next == quoteChar) {
+        next = _reader.advance();
+        if (next == quoteChar) {
+          next = _reader.advance();
+          if (next == quoteChar) {
+            _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
+            return _reader.advance();
+          }
+        }
+        continue;
+      }
+      if (next == 0x5C) {
+        next = _reader.advance();
+        if (next == -1) {
+          break;
+        }
+        if (next == 0xD) {
+          next = _reader.advance();
+          if (next == 0xA) {
+            next = _reader.advance();
+          }
+          recordStartOfLine();
+        } else if (next == 0xA) {
+          recordStartOfLine();
+          next = _reader.advance();
+        } else {
+          next = _reader.advance();
+        }
+      } else if (next == 0xD) {
+        next = _reader.advance();
+        if (next == 0xA) {
+          next = _reader.advance();
+        }
+        recordStartOfLine();
+      } else if (next == 0xA) {
+        recordStartOfLine();
+        next = _reader.advance();
+      } else {
+        next = _reader.advance();
+      }
+    }
+    _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []);
+    _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
+    return _reader.advance();
+  }
+
+  int _tokenizeMultiply(int next) => _select(0x3D, TokenType.STAR_EQ, TokenType.STAR);
+
+  int _tokenizeNumber(int next) {
+    int start = _reader.offset;
+    while (true) {
+      next = _reader.advance();
+      if (0x30 <= next && next <= 0x39) {
+        continue;
+      } else if (next == 0x2E) {
+        return _tokenizeFractionPart(_reader.advance(), start);
+      } else if (next == 0x65 || next == 0x45) {
+        return _tokenizeFractionPart(next, start);
+      } else {
+        _appendStringToken(TokenType.INT, _reader.getString(start, next < 0 ? 0 : -1));
+        return next;
+      }
+    }
+  }
+
+  int _tokenizeOpenSquareBracket(int next) {
+    // [ []  []=
+    next = _reader.advance();
+    if (next == 0x5D) {
+      return _select(0x3D, TokenType.INDEX_EQ, TokenType.INDEX);
+    } else {
+      _appendBeginToken(TokenType.OPEN_SQUARE_BRACKET);
+      return next;
+    }
+  }
+
+  int _tokenizePercent(int next) => _select(0x3D, TokenType.PERCENT_EQ, TokenType.PERCENT);
+
+  int _tokenizePlus(int next) {
+    // + ++ +=
+    next = _reader.advance();
+    if (0x2B == next) {
+      _appendTokenOfType(TokenType.PLUS_PLUS);
+      return _reader.advance();
+    } else if (0x3D == next) {
+      _appendTokenOfType(TokenType.PLUS_EQ);
+      return _reader.advance();
+    } else {
+      _appendTokenOfType(TokenType.PLUS);
+      return next;
+    }
+  }
+
+  int _tokenizeSingleLineComment(int next) {
+    while (true) {
+      next = _reader.advance();
+      if (-1 == next) {
+        _appendCommentToken(TokenType.SINGLE_LINE_COMMENT, _reader.getString(_tokenStart, 0));
+        return next;
+      } else if (0xA == next || 0xD == next) {
+        _appendCommentToken(TokenType.SINGLE_LINE_COMMENT, _reader.getString(_tokenStart, -1));
+        return next;
+      }
+    }
+  }
+
+  int _tokenizeSingleLineRawString(int next, int quoteChar, int start) {
+    next = _reader.advance();
+    while (next != -1) {
+      if (next == quoteChar) {
+        _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
+        return _reader.advance();
+      } else if (next == 0xD || next == 0xA) {
+        _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []);
+        _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
+        return _reader.advance();
+      }
+      next = _reader.advance();
+    }
+    _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []);
+    _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
+    return _reader.advance();
+  }
+
+  int _tokenizeSingleLineString(int next, int quoteChar, int start) {
+    while (next != quoteChar) {
+      if (next == 0x5C) {
+        next = _reader.advance();
+      } else if (next == 0x24) {
+        _appendStringToken(TokenType.STRING, _reader.getString(start, -1));
+        _beginToken();
+        next = _tokenizeStringInterpolation(start);
+        start = _reader.offset;
+        continue;
+      }
+      if (next <= 0xD && (next == 0xA || next == 0xD || next == -1)) {
+        _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []);
+        _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
+        return _reader.advance();
+      }
+      next = _reader.advance();
+    }
+    _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
+    return _reader.advance();
+  }
+
+  int _tokenizeSlashOrComment(int next) {
+    next = _reader.advance();
+    if (0x2A == next) {
+      return _tokenizeMultiLineComment(next);
+    } else if (0x2F == next) {
+      return _tokenizeSingleLineComment(next);
+    } else if (0x3D == next) {
+      _appendTokenOfType(TokenType.SLASH_EQ);
+      return _reader.advance();
+    } else {
+      _appendTokenOfType(TokenType.SLASH);
+      return next;
+    }
+  }
+
+  int _tokenizeString(int next, int start, bool raw) {
+    int quoteChar = next;
+    next = _reader.advance();
+    if (quoteChar == next) {
+      next = _reader.advance();
+      if (quoteChar == next) {
+        // Multiline string.
+        return _tokenizeMultiLineString(quoteChar, start, raw);
+      } else {
+        // Empty string.
+        _appendStringToken(TokenType.STRING, _reader.getString(start, -1));
+        return next;
+      }
+    }
+    if (raw) {
+      return _tokenizeSingleLineRawString(next, quoteChar, start);
+    } else {
+      return _tokenizeSingleLineString(next, quoteChar, start);
+    }
+  }
+
+  int _tokenizeStringInterpolation(int start) {
+    _beginToken();
+    int next = _reader.advance();
+    if (next == 0x7B) {
+      return _tokenizeInterpolatedExpression(next, start);
+    } else {
+      return _tokenizeInterpolatedIdentifier(next, start);
+    }
+  }
+
+  int _tokenizeTag(int next) {
+    // # or #!.*[\n\r]
+    if (_reader.offset == 0) {
+      if (_reader.peek() == 0x21) {
+        do {
+          next = _reader.advance();
+        } while (next != 0xA && next != 0xD && next > 0);
+        _appendStringToken(TokenType.SCRIPT_TAG, _reader.getString(_tokenStart, 0));
+        return next;
+      }
+    }
+    _appendTokenOfType(TokenType.HASH);
+    return _reader.advance();
+  }
+
+  int _tokenizeTilde(int next) {
+    // ~ ~/ ~/=
+    next = _reader.advance();
+    if (next == 0x2F) {
+      return _select(0x3D, TokenType.TILDE_SLASH_EQ, TokenType.TILDE_SLASH);
+    } else {
+      _appendTokenOfType(TokenType.TILDE);
+      return next;
+    }
   }
 }
 
 /**
  * The enumeration `ScannerErrorCode` defines the error codes used for errors detected by the
  * scanner.
  */
 class ScannerErrorCode extends Enum<ScannerErrorCode> implements ErrorCode {
   static const ScannerErrorCode ILLEGAL_CHARACTER = const ScannerErrorCode.con1('ILLEGAL_CHARACTER', 0, "Illegal character %x");
 
@@ skipping 42 matching lines @@
   const ScannerErrorCode.con2(String name, int ordinal, this.message, this.correction) : super(name, ordinal);
 
   @override
   ErrorSeverity get errorSeverity => ErrorSeverity.ERROR;
 
   @override
   ErrorType get type => ErrorType.SYNTACTIC_ERROR;
 }
 
 /**
- * Instances of the class `KeywordTokenWithComment` implement a keyword token that is preceded
- * by comments.
+ * Instances of the class `StringToken` represent a token whose value is independent of it's
+ * type.
  */
-class KeywordTokenWithComment extends KeywordToken {
+class StringToken extends Token {
+  /**
+   * The lexeme represented by this token.
+   */
+  String _value;
+
+  /**
+   * Initialize a newly created token to represent a token of the given type with the given value.
+   *
+   * @param type the type of the token
+   * @param value the lexeme represented by this token
+   * @param offset the offset from the beginning of the file to the first character in the token
+   */
+  StringToken(TokenType type, String value, int offset) : super(type, offset) {
+    this._value = StringUtilities.intern(value);
+  }
+
+  @override
+  Token copy() => new StringToken(type, _value, offset);
+
+  @override
+  String get lexeme => _value;
+
+  @override
+  String value() => _value;
+}
+
+/**
+ * Instances of the class `TokenWithComment` represent a string token that is preceded by
+ * comments.
+ */
+class StringTokenWithComment extends StringToken {
   /**
    * The first comment in the list of comments that precede this token.
    */
   final Token _precedingComment;
 
   /**
-   * Initialize a newly created token to to represent the given keyword and to be preceded by the
+   * Initialize a newly created token to have the given type and offset and to be preceded by the
    * comments reachable from the given comment.
    *
-   * @param keyword the keyword being represented by this token
+   * @param type the type of the token
    * @param offset the offset from the beginning of the file to the first character in the token
    * @param precedingComment the first comment in the list of comments that precede this token
    */
-  KeywordTokenWithComment(Keyword keyword, int offset, this._precedingComment) : super(keyword, offset);
+  StringTokenWithComment(TokenType type, String value, int offset, this._precedingComment) : super(type, value, offset);
 
   @override
-  Token copy() => new KeywordTokenWithComment(keyword, offset, copyComments(_precedingComment));
+  Token copy() => new StringTokenWithComment(type, lexeme, offset, copyComments(_precedingComment));
 
   @override
   Token get precedingComments => _precedingComment;
 
   @override
   void applyDelta(int delta) {
     super.applyDelta(delta);
     Token token = _precedingComment;
     while (token != null) {
       token.applyDelta(delta);
       token = token.next;
     }
   }
 }
 
 /**
+ * Instances of the class `SubSequenceReader` implement a [CharacterReader] that reads
+ * characters from a character sequence, but adds a delta when reporting the current character
+ * offset so that the character sequence can be a subsequence from a larger sequence.
+ */
+class SubSequenceReader extends CharSequenceReader {
+  /**
+   * The offset from the beginning of the file to the beginning of the source being scanned.
+   */
+  final int _offsetDelta;
+
+  /**
+   * Initialize a newly created reader to read the characters in the given sequence.
+   *
+   * @param sequence the sequence from which characters will be read
+   * @param offsetDelta the offset from the beginning of the file to the beginning of the source
+   *          being scanned
+   */
+  SubSequenceReader(String sequence, this._offsetDelta) : super(sequence);
+
+  @override
+  int get offset => _offsetDelta + super.offset;
+
+  @override
+  String getString(int start, int endDelta) => super.getString(start - _offsetDelta, endDelta);
+
+  @override
+  void set offset(int offset) {
+    super.offset = offset - _offsetDelta;
+  }
+}
+
+/**
+ * Synthetic `StringToken` represent a token whose value is independent of it's type.
+ */
+class SyntheticStringToken extends StringToken {
+  /**
+   * Initialize a newly created token to represent a token of the given type with the given value.
+   *
+   * @param type the type of the token
+   * @param value the lexeme represented by this token
+   * @param offset the offset from the beginning of the file to the first character in the token
+   */
+  SyntheticStringToken(TokenType type, String value, int offset) : super(type, value, offset);
+
+  @override
+  bool get isSynthetic => true;
+}
+
+/**
+ * Instances of the class `Token` represent a token that was scanned from the input. Each
+ * token knows which token follows it, acting as the head of a linked list of tokens.
+ */
+class Token {
+  /**
+   * The type of the token.
+   */
+  final TokenType type;
+
+  /**
+   * The offset from the beginning of the file to the first character in the token.
+   */
+  int offset = 0;
+
+  /**
+   * The previous token in the token stream.
+   */
+  Token previous;
+
+  /**
+   * The next token in the token stream.
+   */
+  Token _next;
+
+  /**
+   * Initialize a newly created token to have the given type and offset.
+   *
+   * @param type the type of the token
+   * @param offset the offset from the beginning of the file to the first character in the token
+   */
+  Token(this.type, int offset) {
+    this.offset = offset;
+  }
+
+  /**
+   * Return a newly created token that is a copy of this token but that is not a part of any token
+   * stream.
+   *
+   * @return a newly created token that is a copy of this token
+   */
+  Token copy() => new Token(type, offset);
+
+  /**
+   * Return the offset from the beginning of the file to the character after last character of the
+   * token.
+   *
+   * @return the offset from the beginning of the file to the first character after last character
+   *         of the token
+   */
+  int get end => offset + length;
+
+  /**
+   * Return the number of characters in the node's source range.
+   *
+   * @return the number of characters in the node's source range
+   */
+  int get length => lexeme.length;
+
+  /**
+   * Return the lexeme that represents this token.
+   *
+   * @return the lexeme that represents this token
+   */
+  String get lexeme => type.lexeme;
+
+  /**
+   * Return the next token in the token stream.
+   *
+   * @return the next token in the token stream
+   */
+  Token get next => _next;
+
+  /**
+   * Return the first comment in the list of comments that precede this token, or `null` if
+   * there are no comments preceding this token. Additional comments can be reached by following the
+   * token stream using [getNext] until `null` is returned.
+   *
+   * @return the first comment in the list of comments that precede this token
+   */
+  Token get precedingComments => null;
+
+  /**
+   * Return `true` if this token represents an operator.
+   *
+   * @return `true` if this token represents an operator
+   */
+  bool get isOperator => type.isOperator;
+
+  /**
+   * Return `true` if this token is a synthetic token. A synthetic token is a token that was
+   * introduced by the parser in order to recover from an error in the code.
+   *
+   * @return `true` if this token is a synthetic token
+   */
+  bool get isSynthetic => length == 0;
+
+  /**
+   * Return `true` if this token represents an operator that can be defined by users.
+   *
+   * @return `true` if this token represents an operator that can be defined by users
+   */
+  bool get isUserDefinableOperator => type.isUserDefinableOperator;
+
+  /**
+   * Return `true` if this token has any one of the given types.
+   *
+   * @param types the types of token that are being tested for
+   * @return `true` if this token has any of the given types
+   */
+  bool matchesAny(List<TokenType> types) {
+    for (TokenType type in types) {
+      if (this.type == type) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Set the next token in the token stream to the given token. This has the side-effect of setting
+   * this token to be the previous token for the given token.
+   *
+   * @param token the next token in the token stream
+   * @return the token that was passed in
+   */
+  Token setNext(Token token) {
+    _next = token;
+    token.previous = this;
+    return token;
+  }
+
+  /**
+   * Set the next token in the token stream to the given token without changing which token is the
+   * previous token for the given token.
+   *
+   * @param token the next token in the token stream
+   * @return the token that was passed in
+   */
+  Token setNextWithoutSettingPrevious(Token token) {
+    _next = token;
+    return token;
+  }
+
+  @override
+  String toString() => lexeme;
+
+  /**
+   * Return the value of this token. For keyword tokens, this is the keyword associated with the
+   * token, for other tokens it is the lexeme associated with the token.
+   *
+   * @return the value of this token
+   */
+  Object value() => type.lexeme;
+
+  /**
+   * Apply (add) the given delta to this token's offset.
+   *
+   * @param delta the amount by which the offset is to be adjusted
+   */
+  void applyDelta(int delta) {
+    offset += delta;
+  }
+
+  /**
+   * Copy a linked list of comment tokens identical to the given comment tokens.
+   *
+   * @param token the first token in the list, or `null` if there are no tokens to be copied
+   * @return the tokens that were created
+   */
+  Token copyComments(Token token) {
+    if (token == null) {
+      return null;
+    }
+    Token head = token.copy();
+    Token tail = head;
+    token = token.next;
+    while (token != null) {
+      tail = tail.setNext(token.copy());
+      token = token.next;
+    }
+    return head;
+  }
+}
+
+/**
+ * The enumeration `TokenClass` represents classes (or groups) of tokens with a similar use.
+ */
+class TokenClass extends Enum<TokenClass> {
+  /**
+   * A value used to indicate that the token type is not part of any specific class of token.
+   */
+  static const TokenClass NO_CLASS = const TokenClass.con1('NO_CLASS', 0);
+
+  /**
+   * A value used to indicate that the token type is an additive operator.
+   */
+  static const TokenClass ADDITIVE_OPERATOR = const TokenClass.con2('ADDITIVE_OPERATOR', 1, 12);
+
+  /**
+   * A value used to indicate that the token type is an assignment operator.
+   */
+  static const TokenClass ASSIGNMENT_OPERATOR = const TokenClass.con2('ASSIGNMENT_OPERATOR', 2, 1);
+
+  /**
+   * A value used to indicate that the token type is a bitwise-and operator.
+   */
+  static const TokenClass BITWISE_AND_OPERATOR = const TokenClass.con2('BITWISE_AND_OPERATOR', 3, 10);
+
+  /**
+   * A value used to indicate that the token type is a bitwise-or operator.
+   */
+  static const TokenClass BITWISE_OR_OPERATOR = const TokenClass.con2('BITWISE_OR_OPERATOR', 4, 8);
+
+  /**
+   * A value used to indicate that the token type is a bitwise-xor operator.
+   */
+  static const TokenClass BITWISE_XOR_OPERATOR = const TokenClass.con2('BITWISE_XOR_OPERATOR', 5, 9);
+
+  /**
+   * A value used to indicate that the token type is a cascade operator.
+   */
+  static const TokenClass CASCADE_OPERATOR = const TokenClass.con2('CASCADE_OPERATOR', 6, 2);
+
+  /**
+   * A value used to indicate that the token type is a conditional operator.
+   */
+  static const TokenClass CONDITIONAL_OPERATOR = const TokenClass.con2('CONDITIONAL_OPERATOR', 7, 3);
+
+  /**
+   * A value used to indicate that the token type is an equality operator.
+   */
+  static const TokenClass EQUALITY_OPERATOR = const TokenClass.con2('EQUALITY_OPERATOR', 8, 6);
+
+  /**
+   * A value used to indicate that the token type is a logical-and operator.
+   */
+  static const TokenClass LOGICAL_AND_OPERATOR = const TokenClass.con2('LOGICAL_AND_OPERATOR', 9, 5);
+
+  /**
+   * A value used to indicate that the token type is a logical-or operator.
+   */
+  static const TokenClass LOGICAL_OR_OPERATOR = const TokenClass.con2('LOGICAL_OR_OPERATOR', 10, 4);
+
+  /**
+   * A value used to indicate that the token type is a multiplicative operator.
+   */
+  static const TokenClass MULTIPLICATIVE_OPERATOR = const TokenClass.con2('MULTIPLICATIVE_OPERATOR', 11, 13);
+
+  /**
+   * A value used to indicate that the token type is a relational operator.
+   */
+  static const TokenClass RELATIONAL_OPERATOR = const TokenClass.con2('RELATIONAL_OPERATOR', 12, 7);
+
+  /**
+   * A value used to indicate that the token type is a shift operator.
+   */
+  static const TokenClass SHIFT_OPERATOR = const TokenClass.con2('SHIFT_OPERATOR', 13, 11);
+
+  /**
+   * A value used to indicate that the token type is a unary operator.
+   */
+  static const TokenClass UNARY_POSTFIX_OPERATOR = const TokenClass.con2('UNARY_POSTFIX_OPERATOR', 14, 15);
+
+  /**
+   * A value used to indicate that the token type is a unary operator.
+   */
+  static const TokenClass UNARY_PREFIX_OPERATOR = const TokenClass.con2('UNARY_PREFIX_OPERATOR', 15, 14);
+
+  static const List<TokenClass> values = const [
+      NO_CLASS,
+      ADDITIVE_OPERATOR,
+      ASSIGNMENT_OPERATOR,
+      BITWISE_AND_OPERATOR,
+      BITWISE_OR_OPERATOR,
+      BITWISE_XOR_OPERATOR,
+      CASCADE_OPERATOR,
+      CONDITIONAL_OPERATOR,
+      EQUALITY_OPERATOR,
+      LOGICAL_AND_OPERATOR,
+      LOGICAL_OR_OPERATOR,
+      MULTIPLICATIVE_OPERATOR,
+      RELATIONAL_OPERATOR,
+      SHIFT_OPERATOR,
+      UNARY_POSTFIX_OPERATOR,
+      UNARY_PREFIX_OPERATOR];
+
+  /**
+   * The precedence of tokens of this class, or `0` if the such tokens do not represent an
+   * operator.
+   */
+  final int precedence;
+
+  const TokenClass.con1(String name, int ordinal) : this.con2(name, ordinal, 0);
+
+  const TokenClass.con2(String name, int ordinal, this.precedence) : super(name, ordinal);
+}
+
+/**
  * The enumeration `TokenType` defines the types of tokens that can be returned by the
  * scanner.
  */
 class TokenType extends Enum<TokenType> {
   /**
    * The type of the token that marks the end of the input.
    */
   static const TokenType EOF = const TokenType_EOF('EOF', 0, TokenClass.NO_CLASS, "");
 
   static const TokenType DOUBLE = const TokenType.con1('DOUBLE', 1);
@@ skipping 315 matching lines @@
 }
 
 class TokenType_EOF extends TokenType {
   const TokenType_EOF(String name, int ordinal, TokenClass arg0, String arg1) : super.con2(name, ordinal, arg0, arg1);
 
   @override
   String toString() => "-eof-";
 }
 
 /**
- * Synthetic `StringToken` represent a token whose value is independent of it's type.
- */
-class SyntheticStringToken extends StringToken {
-  /**
-   * Initialize a newly created token to represent a token of the given type with the given value.
-   *
-   * @param type the type of the token
-   * @param value the lexeme represented by this token
-   * @param offset the offset from the beginning of the file to the first character in the token
-   */
-  SyntheticStringToken(TokenType type, String value, int offset) : super(type, value, offset);
-
-  @override
-  bool get isSynthetic => true;
-}
-
-/**
- * Instances of the class `CharSequenceReader` implement a [CharacterReader] that reads
- * characters from a character sequence.
- */
-class CharSequenceReader implements CharacterReader {
-  /**
-   * The sequence from which characters will be read.
-   */
-  final String _sequence;
-
-  /**
-   * The number of characters in the string.
-   */
-  int _stringLength = 0;
-
-  /**
-   * The index, relative to the string, of the last character that was read.
-   */
-  int _charOffset = 0;
-
-  /**
-   * Initialize a newly created reader to read the characters in the given sequence.
-   *
-   * @param sequence the sequence from which characters will be read
-   */
-  CharSequenceReader(this._sequence) {
-    this._stringLength = _sequence.length;
-    this._charOffset = -1;
-  }
-
-  @override
-  int advance() {
-    if (_charOffset + 1 >= _stringLength) {
-      return -1;
-    }
-    return _sequence.codeUnitAt(++_charOffset);
-  }
-
-  @override
-  int get offset => _charOffset;
-
-  @override
-  String getString(int start, int endDelta) => _sequence.substring(start, _charOffset + 1 + endDelta).toString();
-
-  @override
-  int peek() {
-    if (_charOffset + 1 >= _sequence.length) {
-      return -1;
-    }
-    return _sequence.codeUnitAt(_charOffset + 1);
-  }
-
-  @override
-  void set offset(int offset) {
-    _charOffset = offset;
-  }
-}
-
-/**
- * Instances of the class `Token` represent a token that was scanned from the input. Each
- * token knows which token follows it, acting as the head of a linked list of tokens.
- */
-class Token {
-  /**
-   * The type of the token.
-   */
-  final TokenType type;
-
-  /**
-   * The offset from the beginning of the file to the first character in the token.
-   */
-  int offset = 0;
-
-  /**
-   * The previous token in the token stream.
-   */
-  Token previous;
-
-  /**
-   * The next token in the token stream.
-   */
-  Token _next;
-
-  /**
-   * Initialize a newly created token to have the given type and offset.
-   *
-   * @param type the type of the token
-   * @param offset the offset from the beginning of the file to the first character in the token
-   */
-  Token(this.type, int offset) {
-    this.offset = offset;
-  }
-
-  /**
-   * Return a newly created token that is a copy of this token but that is not a part of any token
-   * stream.
-   *
-   * @return a newly created token that is a copy of this token
-   */
-  Token copy() => new Token(type, offset);
-
-  /**
-   * Return the offset from the beginning of the file to the character after last character of the
-   * token.
-   *
-   * @return the offset from the beginning of the file to the first character after last character
-   *         of the token
-   */
-  int get end => offset + length;
-
-  /**
-   * Return the number of characters in the node's source range.
-   *
-   * @return the number of characters in the node's source range
-   */
-  int get length => lexeme.length;
-
-  /**
-   * Return the lexeme that represents this token.
-   *
-   * @return the lexeme that represents this token
-   */
-  String get lexeme => type.lexeme;
-
-  /**
-   * Return the next token in the token stream.
-   *
-   * @return the next token in the token stream
-   */
-  Token get next => _next;
-
-  /**
-   * Return the first comment in the list of comments that precede this token, or `null` if
-   * there are no comments preceding this token. Additional comments can be reached by following the
-   * token stream using [getNext] until `null` is returned.
-   *
-   * @return the first comment in the list of comments that precede this token
-   */
-  Token get precedingComments => null;
-
-  /**
-   * Return `true` if this token represents an operator.
-   *
-   * @return `true` if this token represents an operator
-   */
-  bool get isOperator => type.isOperator;
-
-  /**
-   * Return `true` if this token is a synthetic token. A synthetic token is a token that was
-   * introduced by the parser in order to recover from an error in the code.
-   *
-   * @return `true` if this token is a synthetic token
-   */
-  bool get isSynthetic => length == 0;
-
-  /**
-   * Return `true` if this token represents an operator that can be defined by users.
-   *
-   * @return `true` if this token represents an operator that can be defined by users
-   */
-  bool get isUserDefinableOperator => type.isUserDefinableOperator;
-
-  /**
-   * Return `true` if this token has any one of the given types.
-   *
-   * @param types the types of token that are being tested for
-   * @return `true` if this token has any of the given types
-   */
-  bool matchesAny(List<TokenType> types) {
-    for (TokenType type in types) {
-      if (this.type == type) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Set the next token in the token stream to the given token. This has the side-effect of setting
-   * this token to be the previous token for the given token.
-   *
-   * @param token the next token in the token stream
-   * @return the token that was passed in
-   */
-  Token setNext(Token token) {
-    _next = token;
-    token.previous = this;
-    return token;
-  }
-
-  /**
-   * Set the next token in the token stream to the given token without changing which token is the
-   * previous token for the given token.
-   *
-   * @param token the next token in the token stream
-   * @return the token that was passed in
-   */
-  Token setNextWithoutSettingPrevious(Token token) {
-    _next = token;
-    return token;
-  }
-
-  @override
-  String toString() => lexeme;
-
-  /**
-   * Return the value of this token. For keyword tokens, this is the keyword associated with the
-   * token, for other tokens it is the lexeme associated with the token.
-   *
-   * @return the value of this token
-   */
-  Object value() => type.lexeme;
-
-  /**
-   * Apply (add) the given delta to this token's offset.
-   *
-   * @param delta the amount by which the offset is to be adjusted
-   */
-  void applyDelta(int delta) {
-    offset += delta;
-  }
-
-  /**
-   * Copy a linked list of comment tokens identical to the given comment tokens.
-   *
-   * @param token the first token in the list, or `null` if there are no tokens to be copied
-   * @return the tokens that were created
-   */
-  Token copyComments(Token token) {
-    if (token == null) {
-      return null;
-    }
-    Token head = token.copy();
-    Token tail = head;
-    token = token.next;
-    while (token != null) {
-      tail = tail.setNext(token.copy());
-      token = token.next;
-    }
-    return head;
-  }
-}
-
-/**
- * The enumeration `TokenClass` represents classes (or groups) of tokens with a similar use.
- */
-class TokenClass extends Enum<TokenClass> {
-  /**
-   * A value used to indicate that the token type is not part of any specific class of token.
-   */
-  static const TokenClass NO_CLASS = const TokenClass.con1('NO_CLASS', 0);
-
-  /**
-   * A value used to indicate that the token type is an additive operator.
-   */
-  static const TokenClass ADDITIVE_OPERATOR = const TokenClass.con2('ADDITIVE_OPERATOR', 1, 12);
-
-  /**
-   * A value used to indicate that the token type is an assignment operator.
-   */
-  static const TokenClass ASSIGNMENT_OPERATOR = const TokenClass.con2('ASSIGNMENT_OPERATOR', 2, 1);
-
-  /**
-   * A value used to indicate that the token type is a bitwise-and operator.
-   */
-  static const TokenClass BITWISE_AND_OPERATOR = const TokenClass.con2('BITWISE_AND_OPERATOR', 3, 10);
-
-  /**
-   * A value used to indicate that the token type is a bitwise-or operator.
-   */
-  static const TokenClass BITWISE_OR_OPERATOR = const TokenClass.con2('BITWISE_OR_OPERATOR', 4, 8);
-
-  /**
-   * A value used to indicate that the token type is a bitwise-xor operator.
-   */
-  static const TokenClass BITWISE_XOR_OPERATOR = const TokenClass.con2('BITWISE_XOR_OPERATOR', 5, 9);
-
-  /**
-   * A value used to indicate that the token type is a cascade operator.
-   */
-  static const TokenClass CASCADE_OPERATOR = const TokenClass.con2('CASCADE_OPERATOR', 6, 2);
-
-  /**
-   * A value used to indicate that the token type is a conditional operator.
-   */
-  static const TokenClass CONDITIONAL_OPERATOR = const TokenClass.con2('CONDITIONAL_OPERATOR', 7, 3);
-
-  /**
-   * A value used to indicate that the token type is an equality operator.
-   */
-  static const TokenClass EQUALITY_OPERATOR = const TokenClass.con2('EQUALITY_OPERATOR', 8, 6);
-
-  /**
-   * A value used to indicate that the token type is a logical-and operator.
-   */
-  static const TokenClass LOGICAL_AND_OPERATOR = const TokenClass.con2('LOGICAL_AND_OPERATOR', 9, 5);
-
-  /**
-   * A value used to indicate that the token type is a logical-or operator.
-   */
-  static const TokenClass LOGICAL_OR_OPERATOR = const TokenClass.con2('LOGICAL_OR_OPERATOR', 10, 4);
-
-  /**
-   * A value used to indicate that the token type is a multiplicative operator.
-   */
-  static const TokenClass MULTIPLICATIVE_OPERATOR = const TokenClass.con2('MULTIPLICATIVE_OPERATOR', 11, 13);
-
-  /**
-   * A value used to indicate that the token type is a relational operator.
-   */
-  static const TokenClass RELATIONAL_OPERATOR = const TokenClass.con2('RELATIONAL_OPERATOR', 12, 7);
-
-  /**
-   * A value used to indicate that the token type is a shift operator.
-   */
-  static const TokenClass SHIFT_OPERATOR = const TokenClass.con2('SHIFT_OPERATOR', 13, 11);
-
-  /**
-   * A value used to indicate that the token type is a unary operator.
-   */
-  static const TokenClass UNARY_POSTFIX_OPERATOR = const TokenClass.con2('UNARY_POSTFIX_OPERATOR', 14, 15);
-
-  /**
-   * A value used to indicate that the token type is a unary operator.
-   */
-  static const TokenClass UNARY_PREFIX_OPERATOR = const TokenClass.con2('UNARY_PREFIX_OPERATOR', 15, 14);
-
-  static const List<TokenClass> values = const [
-      NO_CLASS,
-      ADDITIVE_OPERATOR,
-      ASSIGNMENT_OPERATOR,
-      BITWISE_AND_OPERATOR,
-      BITWISE_OR_OPERATOR,
-      BITWISE_XOR_OPERATOR,
-      CASCADE_OPERATOR,
-      CONDITIONAL_OPERATOR,
-      EQUALITY_OPERATOR,
-      LOGICAL_AND_OPERATOR,
-      LOGICAL_OR_OPERATOR,
-      MULTIPLICATIVE_OPERATOR,
-      RELATIONAL_OPERATOR,
-      SHIFT_OPERATOR,
-      UNARY_POSTFIX_OPERATOR,
-      UNARY_PREFIX_OPERATOR];
-
-  /**
-   * The precedence of tokens of this class, or `0` if the such tokens do not represent an
-   * operator.
-   */
-  final int precedence;
-
-  const TokenClass.con1(String name, int ordinal) : this.con2(name, ordinal, 0);
-
-  const TokenClass.con2(String name, int ordinal, this.precedence) : super(name, ordinal);
-}
-
-/**
- * Instances of the abstract class `KeywordState` represent a state in a state machine used to
- * scan keywords.
- */
-class KeywordState {
-  /**
-   * An empty transition table used by leaf states.
-   */
-  static List<KeywordState> _EMPTY_TABLE = new List<KeywordState>(26);
-
-  /**
-   * The initial state in the state machine.
-   */
-  static KeywordState KEYWORD_STATE = _createKeywordStateTable();
-
-  /**
-   * Create the next state in the state machine where we have already recognized the subset of
-   * strings in the given array of strings starting at the given offset and having the given length.
-   * All of these strings have a common prefix and the next character is at the given start index.
-   *
-   * @param start the index of the character in the strings used to transition to a new state
-   * @param strings an array containing all of the strings that will be recognized by the state
-   *          machine
-   * @param offset the offset of the first string in the array that has the prefix that is assumed
-   *          to have been recognized by the time we reach the state being built
-   * @param length the number of strings in the array that pass through the state being built
-   * @return the state that was created
-   */
-  static KeywordState _computeKeywordStateTable(int start, List<String> strings, int offset, int length) {
-    List<KeywordState> result = new List<KeywordState>(26);
-    assert(length != 0);
-    int chunk = 0x0;
-    int chunkStart = -1;
-    bool isLeaf = false;
-    for (int i = offset; i < offset + length; i++) {
-      if (strings[i].length == start) {
-        isLeaf = true;
-      }
-      if (strings[i].length > start) {
-        int c = strings[i].codeUnitAt(start);
-        if (chunk != c) {
-          if (chunkStart != -1) {
-            result[chunk - 0x61] = _computeKeywordStateTable(start + 1, strings, chunkStart, i - chunkStart);
-          }
-          chunkStart = i;
-          chunk = c;
-        }
-      }
-    }
-    if (chunkStart != -1) {
-      assert(result[chunk - 0x61] == null);
-      result[chunk - 0x61] = _computeKeywordStateTable(start + 1, strings, chunkStart, offset + length - chunkStart);
-    } else {
-      assert(length == 1);
-      return new KeywordState(_EMPTY_TABLE, strings[offset]);
-    }
-    if (isLeaf) {
-      return new KeywordState(result, strings[offset]);
-    } else {
-      return new KeywordState(result, null);
-    }
-  }
-
-  /**
-   * Create the initial state in the state machine.
-   *
-   * @return the state that was created
-   */
-  static KeywordState _createKeywordStateTable() {
-    List<Keyword> values = Keyword.values;
-    List<String> strings = new List<String>(values.length);
-    for (int i = 0; i < values.length; i++) {
-      strings[i] = values[i].syntax;
-    }
-    strings.sort();
-    return _computeKeywordStateTable(0, strings, 0, strings.length);
-  }
-
-  /**
-   * A table mapping characters to the states to which those characters will transition. (The index
-   * into the array is the offset from the character `'a'` to the transitioning character.)
-   */
-  final List<KeywordState> _table;
-
-  /**
-   * The keyword that is recognized by this state, or `null` if this state is not a terminal
-   * state.
-   */
-  Keyword _keyword;
-
-  /**
-   * Initialize a newly created state to have the given transitions and to recognize the keyword
-   * with the given syntax.
-   *
-   * @param table a table mapping characters to the states to which those characters will transition
-   * @param syntax the syntax of the keyword that is recognized by the state
-   */
-  KeywordState(this._table, String syntax) {
-    this._keyword = (syntax == null) ? null : Keyword.keywords[syntax];
-  }
-
-  /**
-   * Return the keyword that was recognized by this state, or `null` if this state does not
-   * recognized a keyword.
-   *
-   * @return the keyword that was matched by reaching this state
-   */
-  Keyword keyword() => _keyword;
-
-  /**
-   * Return the state that follows this state on a transition of the given character, or
-   * `null` if there is no valid state reachable from this state with such a transition.
-   *
-   * @param c the character used to transition from this state to another state
-   * @return the state that follows this state on a transition of the given character
-   */
-  KeywordState next(int c) => _table[c - 0x61];
-}
-
-/**
- * The class `Scanner` implements a scanner for Dart code.
- *
- * The lexical structure of Dart is ambiguous without knowledge of the context in which a token is
- * being scanned. For example, without context we cannot determine whether source of the form "<<"
- * should be scanned as a single left-shift operator or as two left angle brackets. This scanner
- * does not have any context, so it always resolves such conflicts by scanning the longest possible
- * token.
- */
-class Scanner {
-  /**
-   * The source being scanned.
-   */
-  final Source source;
-
-  /**
-   * The reader used to access the characters in the source.
-   */
-  final CharacterReader _reader;
-
-  /**
-   * The error listener that will be informed of any errors that are found during the scan.
-   */
-  final AnalysisErrorListener _errorListener;
-
-  /**
-   * The flag specifying if documentation comments should be parsed.
-   */
-  bool _preserveComments = true;
-
-  /**
-   * The token pointing to the head of the linked list of tokens.
-   */
-  Token _tokens;
-
-  /**
-   * The last token that was scanned.
-   */
-  Token _tail;
-
-  /**
-   * The first token in the list of comment tokens found since the last non-comment token.
-   */
-  Token _firstComment;
-
-  /**
-   * The last token in the list of comment tokens found since the last non-comment token.
-   */
-  Token _lastComment;
-
-  /**
-   * The index of the first character of the current token.
-   */
-  int _tokenStart = 0;
-
-  /**
-   * A list containing the offsets of the first character of each line in the source code.
-   */
-  List<int> _lineStarts = new List<int>();
-
-  /**
-   * A list, treated something like a stack, of tokens representing the beginning of a matched pair.
-   * It is used to pair the end tokens with the begin tokens.
-   */
-  List<BeginToken> _groupingStack = new List<BeginToken>();
-
-  /**
-   * The index of the last item in the [groupingStack], or `-1` if the stack is empty.
-   */
-  int _stackEnd = -1;
-
-  /**
-   * A flag indicating whether any unmatched groups were found during the parse.
-   */
-  bool _hasUnmatchedGroups = false;
-
-  /**
-   * Initialize a newly created scanner.
-   *
-   * @param source the source being scanned
-   * @param reader the character reader used to read the characters in the source
-   * @param errorListener the error listener that will be informed of any errors that are found
-   */
-  Scanner(this.source, this._reader, this._errorListener) {
-    _tokens = new Token(TokenType.EOF, -1);
-    _tokens.setNext(_tokens);
-    _tail = _tokens;
-    _tokenStart = -1;
-    _lineStarts.add(0);
-  }
-
-  /**
-   * Return an array containing the offsets of the first character of each line in the source code.
-   *
-   * @return an array containing the offsets of the first character of each line in the source code
-   */
-  List<int> get lineStarts => _lineStarts;
-
-  /**
-   * Return `true` if any unmatched groups were found during the parse.
-   *
-   * @return `true` if any unmatched groups were found during the parse
-   */
-  bool get hasUnmatchedGroups => _hasUnmatchedGroups;
-
-  /**
-   * Set whether documentation tokens should be scanned.
-   *
-   * @param preserveComments `true` if documentation tokens should be scanned
-   */
-  void set preserveComments(bool preserveComments) {
-    this._preserveComments = preserveComments;
-  }
-
-  /**
-   * Record that the source begins on the given line and column at the current offset as given by
-   * the reader. The line starts for lines before the given line will not be correct.
-   *
-   * This method must be invoked at most one time and must be invoked before scanning begins. The
-   * values provided must be sensible. The results are undefined if these conditions are violated.
-   *
-   * @param line the one-based index of the line containing the first character of the source
-   * @param column the one-based index of the column in which the first character of the source
-   *          occurs
-   */
-  void setSourceStart(int line, int column) {
-    int offset = _reader.offset;
-    if (line < 1 || column < 1 || offset < 0 || (line + column - 2) >= offset) {
-      return;
-    }
-    for (int i = 2; i < line; i++) {
-      _lineStarts.add(1);
-    }
-    _lineStarts.add(offset - column + 1);
-  }
-
-  /**
-   * Scan the source code to produce a list of tokens representing the source.
-   *
-   * @return the first token in the list of tokens that were produced
-   */
-  Token tokenize() {
-    InstrumentationBuilder instrumentation = Instrumentation.builder2("dart.engine.AbstractScanner.tokenize");
-    int tokenCounter = 0;
-    try {
-      int next = _reader.advance();
-      while (next != -1) {
-        tokenCounter++;
-        next = bigSwitch(next);
-      }
-      _appendEofToken();
-      instrumentation.metric2("tokensCount", tokenCounter);
-      return firstToken;
-    } finally {
-      instrumentation.log2(2);
-    }
-  }
-
-  /**
-   * Append the given token to the end of the token stream being scanned. This method is intended to
-   * be used by subclasses that copy existing tokens and should not normally be used because it will
-   * fail to correctly associate any comments with the token being passed in.
-   *
-   * @param token the token to be appended
-   */
-  void appendToken(Token token) {
-    _tail = _tail.setNext(token);
-  }
-
-  int bigSwitch(int next) {
-    _beginToken();
-    if (next == 0xD) {
-      next = _reader.advance();
-      if (next == 0xA) {
-        next = _reader.advance();
-      }
-      recordStartOfLine();
-      return next;
-    } else if (next == 0xA) {
-      next = _reader.advance();
-      recordStartOfLine();
-      return next;
-    } else if (next == 0x9 || next == 0x20) {
-      return _reader.advance();
-    }
-    if (next == 0x72) {
-      int peek = _reader.peek();
-      if (peek == 0x22 || peek == 0x27) {
-        int start = _reader.offset;
-        return _tokenizeString(_reader.advance(), start, true);
-      }
-    }
-    if (0x61 <= next && next <= 0x7A) {
-      return _tokenizeKeywordOrIdentifier(next, true);
-    }
-    if ((0x41 <= next && next <= 0x5A) || next == 0x5F || next == 0x24) {
-      return _tokenizeIdentifier(next, _reader.offset, true);
-    }
-    if (next == 0x3C) {
-      return _tokenizeLessThan(next);
-    }
-    if (next == 0x3E) {
-      return _tokenizeGreaterThan(next);
-    }
-    if (next == 0x3D) {
-      return _tokenizeEquals(next);
-    }
-    if (next == 0x21) {
-      return _tokenizeExclamation(next);
-    }
-    if (next == 0x2B) {
-      return _tokenizePlus(next);
-    }
-    if (next == 0x2D) {
-      return _tokenizeMinus(next);
-    }
-    if (next == 0x2A) {
-      return _tokenizeMultiply(next);
-    }
-    if (next == 0x25) {
-      return _tokenizePercent(next);
-    }
-    if (next == 0x26) {
-      return _tokenizeAmpersand(next);
-    }
-    if (next == 0x7C) {
-      return _tokenizeBar(next);
-    }
-    if (next == 0x5E) {
-      return _tokenizeCaret(next);
-    }
-    if (next == 0x5B) {
-      return _tokenizeOpenSquareBracket(next);
-    }
-    if (next == 0x7E) {
-      return _tokenizeTilde(next);
-    }
-    if (next == 0x5C) {
-      _appendTokenOfType(TokenType.BACKSLASH);
-      return _reader.advance();
-    }
-    if (next == 0x23) {
-      return _tokenizeTag(next);
-    }
-    if (next == 0x28) {
-      _appendBeginToken(TokenType.OPEN_PAREN);
-      return _reader.advance();
-    }
-    if (next == 0x29) {
-      _appendEndToken(TokenType.CLOSE_PAREN, TokenType.OPEN_PAREN);
-      return _reader.advance();
-    }
-    if (next == 0x2C) {
-      _appendTokenOfType(TokenType.COMMA);
-      return _reader.advance();
-    }
-    if (next == 0x3A) {
-      _appendTokenOfType(TokenType.COLON);
-      return _reader.advance();
-    }
-    if (next == 0x3B) {
-      _appendTokenOfType(TokenType.SEMICOLON);
-      return _reader.advance();
-    }
-    if (next == 0x3F) {
-      _appendTokenOfType(TokenType.QUESTION);
-      return _reader.advance();
-    }
-    if (next == 0x5D) {
-      _appendEndToken(TokenType.CLOSE_SQUARE_BRACKET, TokenType.OPEN_SQUARE_BRACKET);
-      return _reader.advance();
-    }
-    if (next == 0x60) {
-      _appendTokenOfType(TokenType.BACKPING);
-      return _reader.advance();
-    }
-    if (next == 0x7B) {
-      _appendBeginToken(TokenType.OPEN_CURLY_BRACKET);
-      return _reader.advance();
-    }
-    if (next == 0x7D) {
-      _appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.OPEN_CURLY_BRACKET);
-      return _reader.advance();
-    }
-    if (next == 0x2F) {
-      return _tokenizeSlashOrComment(next);
-    }
-    if (next == 0x40) {
-      _appendTokenOfType(TokenType.AT);
-      return _reader.advance();
-    }
-    if (next == 0x22 || next == 0x27) {
-      return _tokenizeString(next, _reader.offset, false);
-    }
-    if (next == 0x2E) {
-      return _tokenizeDotOrNumber(next);
-    }
-    if (next == 0x30) {
-      return _tokenizeHexOrNumber(next);
-    }
-    if (0x31 <= next && next <= 0x39) {
-      return _tokenizeNumber(next);
-    }
-    if (next == -1) {
-      return -1;
-    }
-    _reportError(ScannerErrorCode.ILLEGAL_CHARACTER, [next]);
-    return _reader.advance();
-  }
-
-  /**
-   * Return the first token in the token stream that was scanned.
-   *
-   * @return the first token in the token stream that was scanned
-   */
-  Token get firstToken => _tokens.next;
-
-  /**
-   * Return the last token that was scanned.
-   *
-   * @return the last token that was scanned
-   */
-  Token get tail => _tail;
-
-  /**
-   * Record the fact that we are at the beginning of a new line in the source.
-   */
-  void recordStartOfLine() {
-    _lineStarts.add(_reader.offset);
-  }
-
-  void _appendBeginToken(TokenType type) {
-    BeginToken token;
-    if (_firstComment == null) {
-      token = new BeginToken(type, _tokenStart);
-    } else {
-      token = new BeginTokenWithComment(type, _tokenStart, _firstComment);
-      _firstComment = null;
-      _lastComment = null;
-    }
-    _tail = _tail.setNext(token);
-    _groupingStack.add(token);
-    _stackEnd++;
-  }
-
-  void _appendCommentToken(TokenType type, String value) {
-    // Ignore comment tokens if client specified that it doesn't need them.
-    if (!_preserveComments) {
-      return;
-    }
-    // OK, remember comment tokens.
-    if (_firstComment == null) {
-      _firstComment = new StringToken(type, value, _tokenStart);
-      _lastComment = _firstComment;
-    } else {
-      _lastComment = _lastComment.setNext(new StringToken(type, value, _tokenStart));
-    }
-  }
-
-  void _appendEndToken(TokenType type, TokenType beginType) {
-    Token token;
-    if (_firstComment == null) {
-      token = new Token(type, _tokenStart);
-    } else {
-      token = new TokenWithComment(type, _tokenStart, _firstComment);
-      _firstComment = null;
-      _lastComment = null;
-    }
-    _tail = _tail.setNext(token);
-    if (_stackEnd >= 0) {
-      BeginToken begin = _groupingStack[_stackEnd];
-      if (begin.type == beginType) {
-        begin.endToken = token;
-        _groupingStack.removeAt(_stackEnd--);
-      }
-    }
-  }
-
-  void _appendEofToken() {
-    Token eofToken;
-    if (_firstComment == null) {
-      eofToken = new Token(TokenType.EOF, _reader.offset + 1);
-    } else {
-      eofToken = new TokenWithComment(TokenType.EOF, _reader.offset + 1, _firstComment);
-      _firstComment = null;
-      _lastComment = null;
-    }
-    // The EOF token points to itself so that there is always infinite look-ahead.
-    eofToken.setNext(eofToken);
-    _tail = _tail.setNext(eofToken);
-    if (_stackEnd >= 0) {
-      _hasUnmatchedGroups = true;
-    }
-  }
-
-  void _appendKeywordToken(Keyword keyword) {
-    if (_firstComment == null) {
-      _tail = _tail.setNext(new KeywordToken(keyword, _tokenStart));
-    } else {
-      _tail = _tail.setNext(new KeywordTokenWithComment(keyword, _tokenStart, _firstComment));
-      _firstComment = null;
-      _lastComment = null;
-    }
-  }
-
-  void _appendStringToken(TokenType type, String value) {
-    if (_firstComment == null) {
-      _tail = _tail.setNext(new StringToken(type, value, _tokenStart));
-    } else {
-      _tail = _tail.setNext(new StringTokenWithComment(type, value, _tokenStart, _firstComment));
-      _firstComment = null;
-      _lastComment = null;
-    }
-  }
-
-  void _appendStringTokenWithOffset(TokenType type, String value, int offset) {
-    if (_firstComment == null) {
-      _tail = _tail.setNext(new StringToken(type, value, _tokenStart + offset));
-    } else {
-      _tail = _tail.setNext(new StringTokenWithComment(type, value, _tokenStart + offset, _firstComment));
-      _firstComment = null;
-      _lastComment = null;
-    }
-  }
-
-  void _appendTokenOfType(TokenType type) {
-    if (_firstComment == null) {
-      _tail = _tail.setNext(new Token(type, _tokenStart));
-    } else {
-      _tail = _tail.setNext(new TokenWithComment(type, _tokenStart, _firstComment));
-      _firstComment = null;
-      _lastComment = null;
-    }
-  }
-
-  void _appendTokenOfTypeWithOffset(TokenType type, int offset) {
-    if (_firstComment == null) {
-      _tail = _tail.setNext(new Token(type, offset));
-    } else {
-      _tail = _tail.setNext(new TokenWithComment(type, offset, _firstComment));
-      _firstComment = null;
-      _lastComment = null;
-    }
-  }
-
-  void _beginToken() {
-    _tokenStart = _reader.offset;
-  }
-
-  /**
-   * Return the beginning token corresponding to a closing brace that was found while scanning
-   * inside a string interpolation expression. Tokens that cannot be matched with the closing brace
-   * will be dropped from the stack.
-   *
-   * @return the token to be paired with the closing brace
-   */
-  BeginToken _findTokenMatchingClosingBraceInInterpolationExpression() {
-    while (_stackEnd >= 0) {
-      BeginToken begin = _groupingStack[_stackEnd];
-      if (begin.type == TokenType.OPEN_CURLY_BRACKET || begin.type == TokenType.STRING_INTERPOLATION_EXPRESSION) {
-        return begin;
-      }
-      _hasUnmatchedGroups = true;
-      _groupingStack.removeAt(_stackEnd--);
-    }
-    //
-    // We should never get to this point because we wouldn't be inside a string interpolation
-    // expression unless we had previously found the start of the expression.
-    //
-    return null;
-  }
-
-  /**
-   * Report an error at the current offset.
-   *
-   * @param errorCode the error code indicating the nature of the error
-   * @param arguments any arguments needed to complete the error message
-   */
-  void _reportError(ScannerErrorCode errorCode, List<Object> arguments) {
-    _errorListener.onError(new AnalysisError.con2(source, _reader.offset, 1, errorCode, arguments));
-  }
-
-  int _select(int choice, TokenType yesType, TokenType noType) {
-    int next = _reader.advance();
-    if (next == choice) {
-      _appendTokenOfType(yesType);
-      return _reader.advance();
-    } else {
-      _appendTokenOfType(noType);
-      return next;
-    }
-  }
-
-  int _selectWithOffset(int choice, TokenType yesType, TokenType noType, int offset) {
-    int next = _reader.advance();
-    if (next == choice) {
-      _appendTokenOfTypeWithOffset(yesType, offset);
-      return _reader.advance();
-    } else {
-      _appendTokenOfTypeWithOffset(noType, offset);
-      return next;
-    }
-  }
-
-  int _tokenizeAmpersand(int next) {
-    // && &= &
-    next = _reader.advance();
-    if (next == 0x26) {
-      _appendTokenOfType(TokenType.AMPERSAND_AMPERSAND);
-      return _reader.advance();
-    } else if (next == 0x3D) {
-      _appendTokenOfType(TokenType.AMPERSAND_EQ);
-      return _reader.advance();
-    } else {
-      _appendTokenOfType(TokenType.AMPERSAND);
-      return next;
-    }
-  }
-
-  int _tokenizeBar(int next) {
-    // | || |=
-    next = _reader.advance();
-    if (next == 0x7C) {
-      _appendTokenOfType(TokenType.BAR_BAR);
-      return _reader.advance();
-    } else if (next == 0x3D) {
-      _appendTokenOfType(TokenType.BAR_EQ);
-      return _reader.advance();
-    } else {
-      _appendTokenOfType(TokenType.BAR);
-      return next;
-    }
-  }
-
-  int _tokenizeCaret(int next) => _select(0x3D, TokenType.CARET_EQ, TokenType.CARET);
-
-  int _tokenizeDotOrNumber(int next) {
-    int start = _reader.offset;
-    next = _reader.advance();
-    if (0x30 <= next && next <= 0x39) {
-      return _tokenizeFractionPart(next, start);
-    } else if (0x2E == next) {
-      return _select(0x2E, TokenType.PERIOD_PERIOD_PERIOD, TokenType.PERIOD_PERIOD);
-    } else {
-      _appendTokenOfType(TokenType.PERIOD);
-      return next;
-    }
-  }
-
-  int _tokenizeEquals(int next) {
-    // = == =>
-    next = _reader.advance();
-    if (next == 0x3D) {
-      _appendTokenOfType(TokenType.EQ_EQ);
-      return _reader.advance();
-    } else if (next == 0x3E) {
-      _appendTokenOfType(TokenType.FUNCTION);
-      return _reader.advance();
-    }
-    _appendTokenOfType(TokenType.EQ);
-    return next;
-  }
-
-  int _tokenizeExclamation(int next) {
-    // ! !=
-    next = _reader.advance();
-    if (next == 0x3D) {
-      _appendTokenOfType(TokenType.BANG_EQ);
-      return _reader.advance();
-    }
-    _appendTokenOfType(TokenType.BANG);
-    return next;
-  }
-
-  int _tokenizeExponent(int next) {
-    if (next == 0x2B || next == 0x2D) {
-      next = _reader.advance();
-    }
-    bool hasDigits = false;
-    while (true) {
-      if (0x30 <= next && next <= 0x39) {
-        hasDigits = true;
-      } else {
-        if (!hasDigits) {
-          _reportError(ScannerErrorCode.MISSING_DIGIT, []);
-        }
-        return next;
-      }
-      next = _reader.advance();
-    }
-  }
-
-  int _tokenizeFractionPart(int next, int start) {
-    bool done = false;
-    bool hasDigit = false;
-    LOOP: while (!done) {
-      if (0x30 <= next && next <= 0x39) {
-        hasDigit = true;
-      } else if (0x65 == next || 0x45 == next) {
-        hasDigit = true;
-        next = _tokenizeExponent(_reader.advance());
-        done = true;
-        continue LOOP;
-      } else {
-        done = true;
-        continue LOOP;
-      }
-      next = _reader.advance();
-    }
-    if (!hasDigit) {
-      _appendStringToken(TokenType.INT, _reader.getString(start, -2));
-      if (0x2E == next) {
-        return _selectWithOffset(0x2E, TokenType.PERIOD_PERIOD_PERIOD, TokenType.PERIOD_PERIOD, _reader.offset - 1);
-      }
-      _appendTokenOfTypeWithOffset(TokenType.PERIOD, _reader.offset - 1);
-      return bigSwitch(next);
-    }
-    _appendStringToken(TokenType.DOUBLE, _reader.getString(start, next < 0 ? 0 : -1));
-    return next;
-  }
-
-  int _tokenizeGreaterThan(int next) {
-    // > >= >> >>=
-    next = _reader.advance();
-    if (0x3D == next) {
-      _appendTokenOfType(TokenType.GT_EQ);
-      return _reader.advance();
-    } else if (0x3E == next) {
-      next = _reader.advance();
-      if (0x3D == next) {
-        _appendTokenOfType(TokenType.GT_GT_EQ);
-        return _reader.advance();
-      } else {
-        _appendTokenOfType(TokenType.GT_GT);
-        return next;
-      }
-    } else {
-      _appendTokenOfType(TokenType.GT);
-      return next;
-    }
-  }
-
-  int _tokenizeHex(int next) {
-    int start = _reader.offset - 1;
-    bool hasDigits = false;
-    while (true) {
-      next = _reader.advance();
-      if ((0x30 <= next && next <= 0x39) || (0x41 <= next && next <= 0x46) || (0x61 <= next && next <= 0x66)) {
-        hasDigits = true;
-      } else {
-        if (!hasDigits) {
-          _reportError(ScannerErrorCode.MISSING_HEX_DIGIT, []);
-        }
-        _appendStringToken(TokenType.HEXADECIMAL, _reader.getString(start, next < 0 ? 0 : -1));
-        return next;
-      }
-    }
-  }
-
-  int _tokenizeHexOrNumber(int next) {
-    int x = _reader.peek();
-    if (x == 0x78 || x == 0x58) {
-      _reader.advance();
-      return _tokenizeHex(x);
-    }
-    return _tokenizeNumber(next);
-  }
-
-  int _tokenizeIdentifier(int next, int start, bool allowDollar) {
-    while ((0x61 <= next && next <= 0x7A) || (0x41 <= next && next <= 0x5A) || (0x30 <= next && next <= 0x39) || next == 0x5F || (next == 0x24 && allowDollar)) {
-      next = _reader.advance();
-    }
-    _appendStringToken(TokenType.IDENTIFIER, _reader.getString(start, next < 0 ? 0 : -1));
-    return next;
-  }
-
-  int _tokenizeInterpolatedExpression(int next, int start) {
-    _appendBeginToken(TokenType.STRING_INTERPOLATION_EXPRESSION);
-    next = _reader.advance();
-    while (next != -1) {
-      if (next == 0x7D) {
-        BeginToken begin = _findTokenMatchingClosingBraceInInterpolationExpression();
-        if (begin == null) {
-          _beginToken();
-          _appendTokenOfType(TokenType.CLOSE_CURLY_BRACKET);
-          next = _reader.advance();
-          _beginToken();
-          return next;
-        } else if (begin.type == TokenType.OPEN_CURLY_BRACKET) {
-          _beginToken();
-          _appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.OPEN_CURLY_BRACKET);
-          next = _reader.advance();
-          _beginToken();
-        } else if (begin.type == TokenType.STRING_INTERPOLATION_EXPRESSION) {
-          _beginToken();
-          _appendEndToken(TokenType.CLOSE_CURLY_BRACKET, TokenType.STRING_INTERPOLATION_EXPRESSION);
-          next = _reader.advance();
-          _beginToken();
-          return next;
-        }
-      } else {
-        next = bigSwitch(next);
-      }
-    }
-    return next;
-  }
-
-  int _tokenizeInterpolatedIdentifier(int next, int start) {
-    _appendStringTokenWithOffset(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 0);
-    if ((0x41 <= next && next <= 0x5A) || (0x61 <= next && next <= 0x7A) || next == 0x5F) {
-      _beginToken();
-      next = _tokenizeKeywordOrIdentifier(next, false);
-    }
-    _beginToken();
-    return next;
-  }
-
-  int _tokenizeKeywordOrIdentifier(int next, bool allowDollar) {
-    KeywordState state = KeywordState.KEYWORD_STATE;
-    int start = _reader.offset;
-    while (state != null && 0x61 <= next && next <= 0x7A) {
-      state = state.next(next);
-      next = _reader.advance();
-    }
-    if (state == null || state.keyword() == null) {
-      return _tokenizeIdentifier(next, start, allowDollar);
-    }
-    if ((0x41 <= next && next <= 0x5A) || (0x30 <= next && next <= 0x39) || next == 0x5F || next == 0x24) {
-      return _tokenizeIdentifier(next, start, allowDollar);
-    } else if (next < 128) {
-      _appendKeywordToken(state.keyword());
-      return next;
-    } else {
-      return _tokenizeIdentifier(next, start, allowDollar);
-    }
-  }
-
-  int _tokenizeLessThan(int next) {
-    // < <= << <<=
-    next = _reader.advance();
-    if (0x3D == next) {
-      _appendTokenOfType(TokenType.LT_EQ);
-      return _reader.advance();
-    } else if (0x3C == next) {
-      return _select(0x3D, TokenType.LT_LT_EQ, TokenType.LT_LT);
-    } else {
-      _appendTokenOfType(TokenType.LT);
-      return next;
-    }
-  }
-
-  int _tokenizeMinus(int next) {
-    // - -- -=
-    next = _reader.advance();
-    if (next == 0x2D) {
-      _appendTokenOfType(TokenType.MINUS_MINUS);
-      return _reader.advance();
-    } else if (next == 0x3D) {
-      _appendTokenOfType(TokenType.MINUS_EQ);
-      return _reader.advance();
-    } else {
-      _appendTokenOfType(TokenType.MINUS);
-      return next;
-    }
-  }
-
-  int _tokenizeMultiLineComment(int next) {
-    int nesting = 1;
-    next = _reader.advance();
-    while (true) {
-      if (-1 == next) {
-        _reportError(ScannerErrorCode.UNTERMINATED_MULTI_LINE_COMMENT, []);
-        _appendCommentToken(TokenType.MULTI_LINE_COMMENT, _reader.getString(_tokenStart, 0));
-        return next;
-      } else if (0x2A == next) {
-        next = _reader.advance();
-        if (0x2F == next) {
-          --nesting;
-          if (0 == nesting) {
-            _appendCommentToken(TokenType.MULTI_LINE_COMMENT, _reader.getString(_tokenStart, 0));
-            return _reader.advance();
-          } else {
-            next = _reader.advance();
-          }
-        }
-      } else if (0x2F == next) {
-        next = _reader.advance();
-        if (0x2A == next) {
-          next = _reader.advance();
-          ++nesting;
-        }
-      } else if (next == 0xD) {
-        next = _reader.advance();
-        if (next == 0xA) {
-          next = _reader.advance();
-        }
-        recordStartOfLine();
-      } else if (next == 0xA) {
-        recordStartOfLine();
-        next = _reader.advance();
-      } else {
-        next = _reader.advance();
-      }
-    }
-  }
-
-  int _tokenizeMultiLineRawString(int quoteChar, int start) {
-    int next = _reader.advance();
-    outer: while (next != -1) {
-      while (next != quoteChar) {
-        next = _reader.advance();
-        if (next == -1) {
-          break outer;
-        } else if (next == 0xD) {
-          next = _reader.advance();
-          if (next == 0xA) {
-            next = _reader.advance();
-          }
-          recordStartOfLine();
-        } else if (next == 0xA) {
-          recordStartOfLine();
-          next = _reader.advance();
-        }
-      }
-      next = _reader.advance();
-      if (next == quoteChar) {
-        next = _reader.advance();
-        if (next == quoteChar) {
-          _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
-          return _reader.advance();
-        }
-      }
-    }
-    _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []);
-    _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
-    return _reader.advance();
-  }
-
-  int _tokenizeMultiLineString(int quoteChar, int start, bool raw) {
-    if (raw) {
-      return _tokenizeMultiLineRawString(quoteChar, start);
-    }
-    int next = _reader.advance();
-    while (next != -1) {
-      if (next == 0x24) {
-        _appendStringToken(TokenType.STRING, _reader.getString(start, -1));
-        _beginToken();
-        next = _tokenizeStringInterpolation(start);
-        start = _reader.offset;
-        continue;
-      }
-      if (next == quoteChar) {
-        next = _reader.advance();
-        if (next == quoteChar) {
-          next = _reader.advance();
-          if (next == quoteChar) {
-            _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
-            return _reader.advance();
-          }
-        }
-        continue;
-      }
-      if (next == 0x5C) {
-        next = _reader.advance();
-        if (next == -1) {
-          break;
-        }
-        if (next == 0xD) {
-          next = _reader.advance();
-          if (next == 0xA) {
-            next = _reader.advance();
-          }
-          recordStartOfLine();
-        } else if (next == 0xA) {
-          recordStartOfLine();
-          next = _reader.advance();
-        } else {
-          next = _reader.advance();
-        }
-      } else if (next == 0xD) {
-        next = _reader.advance();
-        if (next == 0xA) {
-          next = _reader.advance();
-        }
-        recordStartOfLine();
-      } else if (next == 0xA) {
-        recordStartOfLine();
-        next = _reader.advance();
-      } else {
-        next = _reader.advance();
-      }
-    }
-    _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []);
-    _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
-    return _reader.advance();
-  }
-
-  int _tokenizeMultiply(int next) => _select(0x3D, TokenType.STAR_EQ, TokenType.STAR);
-
-  int _tokenizeNumber(int next) {
-    int start = _reader.offset;
-    while (true) {
-      next = _reader.advance();
-      if (0x30 <= next && next <= 0x39) {
-        continue;
-      } else if (next == 0x2E) {
-        return _tokenizeFractionPart(_reader.advance(), start);
-      } else if (next == 0x65 || next == 0x45) {
-        return _tokenizeFractionPart(next, start);
-      } else {
-        _appendStringToken(TokenType.INT, _reader.getString(start, next < 0 ? 0 : -1));
-        return next;
-      }
-    }
-  }
-
-  int _tokenizeOpenSquareBracket(int next) {
-    // [ []  []=
-    next = _reader.advance();
-    if (next == 0x5D) {
-      return _select(0x3D, TokenType.INDEX_EQ, TokenType.INDEX);
-    } else {
-      _appendBeginToken(TokenType.OPEN_SQUARE_BRACKET);
-      return next;
-    }
-  }
-
-  int _tokenizePercent(int next) => _select(0x3D, TokenType.PERCENT_EQ, TokenType.PERCENT);
-
-  int _tokenizePlus(int next) {
-    // + ++ +=
-    next = _reader.advance();
-    if (0x2B == next) {
-      _appendTokenOfType(TokenType.PLUS_PLUS);
-      return _reader.advance();
-    } else if (0x3D == next) {
-      _appendTokenOfType(TokenType.PLUS_EQ);
-      return _reader.advance();
-    } else {
-      _appendTokenOfType(TokenType.PLUS);
-      return next;
-    }
-  }
-
-  int _tokenizeSingleLineComment(int next) {
-    while (true) {
-      next = _reader.advance();
-      if (-1 == next) {
-        _appendCommentToken(TokenType.SINGLE_LINE_COMMENT, _reader.getString(_tokenStart, 0));
-        return next;
-      } else if (0xA == next || 0xD == next) {
-        _appendCommentToken(TokenType.SINGLE_LINE_COMMENT, _reader.getString(_tokenStart, -1));
-        return next;
-      }
-    }
-  }
-
-  int _tokenizeSingleLineRawString(int next, int quoteChar, int start) {
-    next = _reader.advance();
-    while (next != -1) {
-      if (next == quoteChar) {
-        _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
-        return _reader.advance();
-      } else if (next == 0xD || next == 0xA) {
-        _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []);
-        _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
-        return _reader.advance();
-      }
-      next = _reader.advance();
-    }
-    _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []);
-    _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
-    return _reader.advance();
-  }
-
-  int _tokenizeSingleLineString(int next, int quoteChar, int start) {
-    while (next != quoteChar) {
-      if (next == 0x5C) {
-        next = _reader.advance();
-      } else if (next == 0x24) {
-        _appendStringToken(TokenType.STRING, _reader.getString(start, -1));
-        _beginToken();
-        next = _tokenizeStringInterpolation(start);
-        start = _reader.offset;
-        continue;
-      }
-      if (next <= 0xD && (next == 0xA || next == 0xD || next == -1)) {
-        _reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, []);
-        _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
-        return _reader.advance();
-      }
-      next = _reader.advance();
-    }
-    _appendStringToken(TokenType.STRING, _reader.getString(start, 0));
-    return _reader.advance();
-  }
-
-  int _tokenizeSlashOrComment(int next) {
-    next = _reader.advance();
-    if (0x2A == next) {
-      return _tokenizeMultiLineComment(next);
-    } else if (0x2F == next) {
-      return _tokenizeSingleLineComment(next);
-    } else if (0x3D == next) {
-      _appendTokenOfType(TokenType.SLASH_EQ);
-      return _reader.advance();
-    } else {
-      _appendTokenOfType(TokenType.SLASH);
-      return next;
-    }
-  }
-
-  int _tokenizeString(int next, int start, bool raw) {
-    int quoteChar = next;
-    next = _reader.advance();
-    if (quoteChar == next) {
-      next = _reader.advance();
-      if (quoteChar == next) {
-        // Multiline string.
-        return _tokenizeMultiLineString(quoteChar, start, raw);
-      } else {
-        // Empty string.
-        _appendStringToken(TokenType.STRING, _reader.getString(start, -1));
-        return next;
-      }
-    }
-    if (raw) {
-      return _tokenizeSingleLineRawString(next, quoteChar, start);
-    } else {
-      return _tokenizeSingleLineString(next, quoteChar, start);
-    }
-  }
-
-  int _tokenizeStringInterpolation(int start) {
-    _beginToken();
-    int next = _reader.advance();
-    if (next == 0x7B) {
-      return _tokenizeInterpolatedExpression(next, start);
-    } else {
-      return _tokenizeInterpolatedIdentifier(next, start);
-    }
-  }
-
-  int _tokenizeTag(int next) {
-    // # or #!.*[\n\r]
-    if (_reader.offset == 0) {
-      if (_reader.peek() == 0x21) {
-        do {
-          next = _reader.advance();
-        } while (next != 0xA && next != 0xD && next > 0);
-        _appendStringToken(TokenType.SCRIPT_TAG, _reader.getString(_tokenStart, 0));
-        return next;
-      }
-    }
-    _appendTokenOfType(TokenType.HASH);
-    return _reader.advance();
-  }
-
-  int _tokenizeTilde(int next) {
-    // ~ ~/ ~/=
-    next = _reader.advance();
-    if (next == 0x2F) {
-      return _select(0x3D, TokenType.TILDE_SLASH_EQ, TokenType.TILDE_SLASH);
-    } else {
-      _appendTokenOfType(TokenType.TILDE);
-      return next;
-    }
-  }
-}
-
-/**
- * The enumeration `Keyword` defines the keywords in the Dart programming language.
- */
-class Keyword extends Enum<Keyword> {
-  static const Keyword ASSERT = const Keyword.con1('ASSERT', 0, "assert");
-
-  static const Keyword BREAK = const Keyword.con1('BREAK', 1, "break");
-
-  static const Keyword CASE = const Keyword.con1('CASE', 2, "case");
-
-  static const Keyword CATCH = const Keyword.con1('CATCH', 3, "catch");
-
-  static const Keyword CLASS = const Keyword.con1('CLASS', 4, "class");
-
-  static const Keyword CONST = const Keyword.con1('CONST', 5, "const");
-
-  static const Keyword CONTINUE = const Keyword.con1('CONTINUE', 6, "continue");
-
-  static const Keyword DEFAULT = const Keyword.con1('DEFAULT', 7, "default");
-
-  static const Keyword DO = const Keyword.con1('DO', 8, "do");
-
-  static const Keyword ELSE = const Keyword.con1('ELSE', 9, "else");
-
-  static const Keyword ENUM = const Keyword.con1('ENUM', 10, "enum");
-
-  static const Keyword EXTENDS = const Keyword.con1('EXTENDS', 11, "extends");
-
-  static const Keyword FALSE = const Keyword.con1('FALSE', 12, "false");
-
-  static const Keyword FINAL = const Keyword.con1('FINAL', 13, "final");
-
-  static const Keyword FINALLY = const Keyword.con1('FINALLY', 14, "finally");
-
-  static const Keyword FOR = const Keyword.con1('FOR', 15, "for");
-
-  static const Keyword IF = const Keyword.con1('IF', 16, "if");
-
-  static const Keyword IN = const Keyword.con1('IN', 17, "in");
-
-  static const Keyword IS = const Keyword.con1('IS', 18, "is");
-
-  static const Keyword NEW = const Keyword.con1('NEW', 19, "new");
-
-  static const Keyword NULL = const Keyword.con1('NULL', 20, "null");
-
-  static const Keyword RETHROW = const Keyword.con1('RETHROW', 21, "rethrow");
-
-  static const Keyword RETURN = const Keyword.con1('RETURN', 22, "return");
-
-  static const Keyword SUPER = const Keyword.con1('SUPER', 23, "super");
-
-  static const Keyword SWITCH = const Keyword.con1('SWITCH', 24, "switch");
-
-  static const Keyword THIS = const Keyword.con1('THIS', 25, "this");
-
-  static const Keyword THROW = const Keyword.con1('THROW', 26, "throw");
-
-  static const Keyword TRUE = const Keyword.con1('TRUE', 27, "true");
-
-  static const Keyword TRY = const Keyword.con1('TRY', 28, "try");
-
-  static const Keyword VAR = const Keyword.con1('VAR', 29, "var");
-
-  static const Keyword VOID = const Keyword.con1('VOID', 30, "void");
-
-  static const Keyword WHILE = const Keyword.con1('WHILE', 31, "while");
-
-  static const Keyword WITH = const Keyword.con1('WITH', 32, "with");
-
-  static const Keyword ABSTRACT = const Keyword.con2('ABSTRACT', 33, "abstract", true);
-
-  static const Keyword AS = const Keyword.con2('AS', 34, "as", true);
-
-  static const Keyword DEFERRED = const Keyword.con2('DEFERRED', 35, "deferred", true);
-
-  static const Keyword DYNAMIC = const Keyword.con2('DYNAMIC', 36, "dynamic", true);
-
-  static const Keyword EXPORT = const Keyword.con2('EXPORT', 37, "export", true);
-
-  static const Keyword EXTERNAL = const Keyword.con2('EXTERNAL', 38, "external", true);
-
-  static const Keyword FACTORY = const Keyword.con2('FACTORY', 39, "factory", true);
-
-  static const Keyword GET = const Keyword.con2('GET', 40, "get", true);
-
-  static const Keyword IMPLEMENTS = const Keyword.con2('IMPLEMENTS', 41, "implements", true);
-
-  static const Keyword IMPORT = const Keyword.con2('IMPORT', 42, "import", true);
-
-  static const Keyword LIBRARY = const Keyword.con2('LIBRARY', 43, "library", true);
-
-  static const Keyword OPERATOR = const Keyword.con2('OPERATOR', 44, "operator", true);
-
-  static const Keyword PART = const Keyword.con2('PART', 45, "part", true);
-
-  static const Keyword SET = const Keyword.con2('SET', 46, "set", true);
-
-  static const Keyword STATIC = const Keyword.con2('STATIC', 47, "static", true);
-
-  static const Keyword TYPEDEF = const Keyword.con2('TYPEDEF', 48, "typedef", true);
-
-  static const List<Keyword> values = const [
-      ASSERT,
-      BREAK,
-      CASE,
-      CATCH,
-      CLASS,
-      CONST,
-      CONTINUE,
-      DEFAULT,
-      DO,
-      ELSE,
-      ENUM,
-      EXTENDS,
-      FALSE,
-      FINAL,
-      FINALLY,
-      FOR,
-      IF,
-      IN,
-      IS,
-      NEW,
-      NULL,
-      RETHROW,
-      RETURN,
-      SUPER,
-      SWITCH,
-      THIS,
-      THROW,
-      TRUE,
-      TRY,
-      VAR,
-      VOID,
-      WHILE,
-      WITH,
-      ABSTRACT,
-      AS,
-      DEFERRED,
-      DYNAMIC,
-      EXPORT,
-      EXTERNAL,
-      FACTORY,
-      GET,
-      IMPLEMENTS,
-      IMPORT,
-      LIBRARY,
-      OPERATOR,
-      PART,
-      SET,
-      STATIC,
-      TYPEDEF];
-
-  /**
-   * The lexeme for the keyword.
-   */
-  final String syntax;
-
-  /**
-   * A flag indicating whether the keyword is a pseudo-keyword. Pseudo keywords can be used as
-   * identifiers.
-   */
-  final bool isPseudoKeyword;
-
-  /**
-   * A table mapping the lexemes of keywords to the corresponding keyword.
-   */
-  static Map<String, Keyword> keywords = _createKeywordMap();
-
-  /**
-   * Create a table mapping the lexemes of keywords to the corresponding keyword.
-   *
-   * @return the table that was created
-   */
-  static Map<String, Keyword> _createKeywordMap() {
-    LinkedHashMap<String, Keyword> result = new LinkedHashMap<String, Keyword>();
-    for (Keyword keyword in values) {
-      result[keyword.syntax] = keyword;
-    }
-    return result;
-  }
-
-  /**
-   * Initialize a newly created keyword to have the given syntax. The keyword is not a
-   * pseudo-keyword.
-   *
-   * @param syntax the lexeme for the keyword
-   */
-  const Keyword.con1(String name, int ordinal, String syntax) : this.con2(name, ordinal, syntax, false);
-
-  /**
-   * Initialize a newly created keyword to have the given syntax. The keyword is a pseudo-keyword if
-   * the given flag is `true`.
-   *
-   * @param syntax the lexeme for the keyword
-   * @param isPseudoKeyword `true` if this keyword is a pseudo-keyword
-   */
-  const Keyword.con2(String name, int ordinal, this.syntax, this.isPseudoKeyword) : super(name, ordinal);
-}
-
-/**
- * Instances of the class `TokenWithComment` represent a string token that is preceded by
+ * Instances of the class `TokenWithComment` represent a normal token that is preceded by
  * comments.
  */
-class StringTokenWithComment extends StringToken {
+class TokenWithComment extends Token {
   /**
    * The first comment in the list of comments that precede this token.
    */
   final Token _precedingComment;
 
   /**
    * Initialize a newly created token to have the given type and offset and to be preceded by the
    * comments reachable from the given comment.
    *
    * @param type the type of the token
    * @param offset the offset from the beginning of the file to the first character in the token
    * @param precedingComment the first comment in the list of comments that precede this token
    */
-  StringTokenWithComment(TokenType type, String value, int offset, this._precedingComment) : super(type, value, offset);
+  TokenWithComment(TokenType type, int offset, this._precedingComment) : super(type, offset);
 
   @override
-  Token copy() => new StringTokenWithComment(type, lexeme, offset, copyComments(_precedingComment));
+  Token copy() => new TokenWithComment(type, offset, _precedingComment);
 
   @override
   Token get precedingComments => _precedingComment;
-
-  @override
-  void applyDelta(int delta) {
-    super.applyDelta(delta);
-    Token token = _precedingComment;
-    while (token != null) {
-      token.applyDelta(delta);
-      token = token.next;
-    }
-  }
-}
-
-/**
- * Instances of the class `StringToken` represent a token whose value is independent of it's
- * type.
- */
-class StringToken extends Token {
-  /**
-   * The lexeme represented by this token.
-   */
-  String _value;
-
-  /**
-   * Initialize a newly created token to represent a token of the given type with the given value.
-   *
-   * @param type the type of the token
-   * @param value the lexeme represented by this token
-   * @param offset the offset from the beginning of the file to the first character in the token
-   */
-  StringToken(TokenType type, String value, int offset) : super(type, offset) {
-    this._value = StringUtilities.intern(value);
-  }
-
-  @override
-  Token copy() => new StringToken(type, _value, offset);
-
-  @override
-  String get lexeme => _value;
-
-  @override
-  String value() => _value;
-}
-
-/**
- * Instances of the class `BeginToken` represent the opening half of a grouping pair of
- * tokens. This is used for curly brackets ('{'), parentheses ('('), and square brackets ('[').
- */
-class BeginToken extends Token {
-  /**
-   * The token that corresponds to this token.
-   */
-  Token endToken;
-
-  /**
-   * Initialize a newly created token representing the opening half of a grouping pair of tokens.
-   *
-   * @param type the type of the token
-   * @param offset the offset from the beginning of the file to the first character in the token
-   */
-  BeginToken(TokenType type, int offset) : super(type, offset) {
-    assert((type == TokenType.OPEN_CURLY_BRACKET || type == TokenType.OPEN_PAREN || type == TokenType.OPEN_SQUARE_BRACKET || type == TokenType.STRING_INTERPOLATION_EXPRESSION));
-  }
-
-  @override
-  Token copy() => new BeginToken(type, offset);
-}
-
-/**
- * Instances of the class `KeywordToken` represent a keyword in the language.
- */
-class KeywordToken extends Token {
-  /**
-   * The keyword being represented by this token.
-   */
-  final Keyword keyword;
-
-  /**
-   * Initialize a newly created token to represent the given keyword.
-   *
-   * @param keyword the keyword being represented by this token
-   * @param offset the offset from the beginning of the file to the first character in the token
-   */
-  KeywordToken(this.keyword, int offset) : super(TokenType.KEYWORD, offset);
-
-  @override
-  Token copy() => new KeywordToken(keyword, offset);
-
-  @override
-  String get lexeme => keyword.syntax;
-
-  @override
-  Keyword value() => keyword;
 }