Port parser and scanner fixes from rasta branch.

pkg/dart_scanner/lib/src/token.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library dart_scanner.token;
 
 import 'dart:collection' show
     HashSet;
 
 import 'dart:convert' show
     UTF8;
 
 import 'keyword.dart' show
     Keyword;
 
 import 'precedence.dart' show
     BAD_INPUT_INFO,
+    EOF_INFO,
     PrecedenceInfo;
 
 import 'token_constants.dart' show
     IDENTIFIER_TOKEN;
 
 /**
  * A token that doubles as a linked list.
  */
 abstract class Token {
   /**
@@ skipping 77 matching lines @@
       // This is a token that wraps around an error message. Return 1
       // instead of the size of the length of the error message.
       return 1;
     } else {
       return value.length;
     }
   }
 
   /// The character offset of the end of this token within the source text.
   int get charEnd => charOffset + charCount;
+
+  bool get isEof => false;
 }
 
 /**
  * A [SymbolToken] represents the symbol in its precendence info.
  * Also used for end of file with EOF_INFO.
  */
 class SymbolToken extends Token {
   final PrecedenceInfo info;
 
   SymbolToken(this.info, int charOffset) : super(charOffset);
 
   String get value => info.value;
 
   String get stringValue => info.value;
 
   bool isIdentifier() => false;
 
   String toString() => "SymbolToken($value)";
+
+  bool get isEof => info == EOF_INFO;
 }
 
 /**
  * A [BeginGroupToken] represents a symbol that may be the beginning of
  * a pair of brackets, i.e., ( { [ < or ${
  * The [endGroup] token points to the matching closing bracked in case
  * it can be identified during scanning.
  */
 class BeginGroupToken extends SymbolToken {
   Token endGroup;
@@ skipping 14 matching lines @@
 
   String get value => keyword.syntax;
 
   String get stringValue => keyword.syntax;
 
   bool isIdentifier() => keyword.isPseudo || keyword.isBuiltIn;
 
   String toString() => "KeywordToken($value)";
 }
 
-abstract class ErrorToken extends Token {
-  ErrorToken(int charOffset) : super(charOffset);
-
-  PrecedenceInfo get info => BAD_INPUT_INFO;
-
-  String get value {
-    throw assertionMessage;
-  }
-
-  String get stringValue => null;
-
-  bool isIdentifier() => false;
-
-  String get assertionMessage;
-}
-
-class BadInputToken extends ErrorToken {
-  final int character;
-
-  BadInputToken(this.character, int charOffset) : super(charOffset);
-
-  String toString() => "BadInputToken($character)";
-
-  String get assertionMessage {
-    return 'Character U+${character.toRadixString(16)} not allowed here.';
-  }
-}
-
-class UnterminatedToken extends ErrorToken {
-  final String start;
-  final int endOffset;
-
-  UnterminatedToken(this.start, int charOffset, this.endOffset)
-      : super(charOffset);
-
-  String toString() => "UnterminatedToken($start)";
-
-  String get assertionMessage => "'$start' isn't terminated.";
-
-  int get charCount => endOffset - charOffset;
-}
-
-class UnmatchedToken extends ErrorToken {
-  final BeginGroupToken begin;
-
-  UnmatchedToken(BeginGroupToken begin)
-      : this.begin = begin,
-        super(begin.charOffset);
-
-  String toString() => "UnmatchedToken(${begin.value})";
-
-  String get assertionMessage => "'$begin' isn't closed.";
-}
-
 /**
  * A String-valued token. Represents identifiers, string literals,
  * number literals, comments, and error tokens, using the corresponding
  * precedence info.
  */
 class StringToken extends Token {
   /**
    * The length threshold above which substring tokens are computed lazily.
    *
    * For string tokens that are substrings of the program source, the actual
@@ skipping 185 matching lines @@
       value == '<=' ||
       value == '<' ||
       value == '&' ||
       value == '^' ||
       value == '|';
 }
 
 bool isTernaryOperator(String value) => value == '[]=';
 
 bool isMinusOperator(String value) => value == '-';