Port parser and scanner fixes from rasta branch.

pkg/dart_scanner/lib/src/token.dart

Index: pkg/dart_scanner/lib/src/token.dart
diff --git a/pkg/dart_scanner/lib/src/token.dart b/pkg/dart_scanner/lib/src/token.dart
index d094ae5226591f13a5686fb2525c0746ad4a4c2b..932557653ac2b01cef2f2e858b43ce005f15636a 100644
--- a/pkg/dart_scanner/lib/src/token.dart
+++ b/pkg/dart_scanner/lib/src/token.dart
@@ -15,6 +15,7 @@ import 'keyword.dart' show
 
 import 'precedence.dart' show
     BAD_INPUT_INFO,
+    EOF_INFO,
     PrecedenceInfo;
 
 import 'token_constants.dart' show
@@ -112,6 +113,8 @@ abstract class Token {
 
   /// The character offset of the end of this token within the source text.
   int get charEnd => charOffset + charCount;
+
+  bool get isEof => false;
 }
 
 /**
@@ -130,6 +133,8 @@ class SymbolToken extends Token {
   bool isIdentifier() => false;
 
   String toString() => "SymbolToken($value)";
+
+  bool get isEof => info == EOF_INFO;
 }
 
 /**
@@ -164,60 +169,6 @@ class KeywordToken extends Token {
   String toString() => "KeywordToken($value)";
 }
 
-abstract class ErrorToken extends Token {
-  ErrorToken(int charOffset) : super(charOffset);
-
-  PrecedenceInfo get info => BAD_INPUT_INFO;
-
-  String get value {
-    throw assertionMessage;
-  }
-
-  String get stringValue => null;
-
-  bool isIdentifier() => false;
-
-  String get assertionMessage;
-}
-
-class BadInputToken extends ErrorToken {
-  final int character;
-
-  BadInputToken(this.character, int charOffset) : super(charOffset);
-
-  String toString() => "BadInputToken($character)";
-
-  String get assertionMessage {
-    return 'Character U+${character.toRadixString(16)} not allowed here.';
-  }
-}
-
-class UnterminatedToken extends ErrorToken {
-  final String start;
-  final int endOffset;
-
-  UnterminatedToken(this.start, int charOffset, this.endOffset)
-      : super(charOffset);
-
-  String toString() => "UnterminatedToken($start)";
-
-  String get assertionMessage => "'$start' isn't terminated.";
-
-  int get charCount => endOffset - charOffset;
-}
-
-class UnmatchedToken extends ErrorToken {
-  final BeginGroupToken begin;
-
-  UnmatchedToken(BeginGroupToken begin)
-      : this.begin = begin,
-        super(begin.charOffset);
-
-  String toString() => "UnmatchedToken(${begin.value})";
-
-  String get assertionMessage => "'$begin' isn't closed.";
-}
-
 /**
  * A String-valued token. Represents identifiers, string literals,
  * number literals, comments, and error tokens, using the corresponding