Port parser and scanner fixes from rasta branch.

pkg/dart_scanner/lib/src/abstract_scanner.dart

Index: pkg/dart_scanner/lib/src/abstract_scanner.dart
diff --git a/pkg/dart_scanner/lib/src/abstract_scanner.dart b/pkg/dart_scanner/lib/src/abstract_scanner.dart
index 8bbf321e9a9cd11f500b3722aac9959229326b0b..222646eb7ab917ac635bc2730a72aa6a100fb786 100644
--- a/pkg/dart_scanner/lib/src/abstract_scanner.dart
+++ b/pkg/dart_scanner/lib/src/abstract_scanner.dart
@@ -5,7 +5,13 @@
 library dart_scanner.abstract_scanner;
 
 import '../dart_scanner.dart' show
-    Scanner;
+    ErrorToken,
+    Scanner,
+    buildUnexpectedCharacterToken;
+
+import 'error_token.dart' show
+    UnmatchedToken,
+    UnterminatedToken;
 
 import 'keyword.dart' show
     KeywordState,
@@ -14,14 +20,10 @@ import 'keyword.dart' show
 import 'precedence.dart';
 
 import 'token.dart' show
-    BadInputToken,
     BeginGroupToken,
-    ErrorToken,
     KeywordToken,
     SymbolToken,
-    Token,
-    UnmatchedToken,
-    UnterminatedToken;
+    Token;
 
 import 'token_constants.dart';
 
@@ -394,13 +396,6 @@ abstract class AbstractScanner implements Scanner {
 
     next = currentAsUnicode(next);
 
-    // The following are non-ASCII characters.
-
-    if (identical(next, $NBSP)) {
-      appendWhiteSpace(next);
-      return advance();
-    }
-
     return unexpected(next);
   }
 
@@ -1073,7 +1068,7 @@ abstract class AbstractScanner implements Scanner {
   }
 
   int unexpected(int character) {
-    appendErrorToken(new BadInputToken(character, tokenStart));
+    appendErrorToken(buildUnexpectedCharacterToken(character, tokenStart));
     return advanceAfterError(true);
   }