Port parser and scanner fixes from rasta branch.

pkg/fasta/lib/src/source/source_loader.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.source_loader;
 
 import 'dart:async' show
     Future;
 
 import 'dart:io' show
     FileSystemException;
 
 import 'package:dart_scanner/io.dart' show
     readBytesFromFile;
 
-import 'package:dart_scanner/src/token.dart' show
-    Token;
-
 import 'package:dart_scanner/dart_scanner.dart' show
+    ErrorToken,
+    Token,
     scan;
 
 import 'package:dart_parser/src/class_member_parser.dart' show
     ClassMemberParser;
 
 import 'package:kernel/ast.dart' show
     Program;
 
 import 'package:kernel/class_hierarchy.dart' show
     ClassHierarchy;
 
 import 'package:kernel/core_types.dart' show
     CoreTypes;
 
 import '../errors.dart' show
+    InputError,
     inputError;
 
 import '../export.dart' show
     Export;
 
 import '../analyzer/element_store.dart' show
     ElementStore;
 
 import '../builder/builder.dart' show
     Builder,
@@ skipping 25 matching lines @@
   // Used when building directly to kernel.
   ClassHierarchy hierarchy;
   CoreTypes coreTypes;
 
   // Used when building analyzer ASTs.
   ElementStore elementStore;
 
   SourceLoader(TargetImplementation target)
       : super(target);
 
-  Future<Token> tokenize(SourceLibraryBuilder library) async {
+  Future<Token> tokenize(SourceLibraryBuilder library,
+      {bool suppressLexicalErrors: false}) async {
     Uri uri = library.uri;
     if (uri.scheme != "file") {
       uri = target.translateUri(uri);
       if (uri == null) {
         print("Skipping ${library.uri}");
         return null;
       }
       library.fileUri = uri;
     }
     try {
       List<int> bytes = await readBytesFromFile(uri);
       byteCount += bytes.length - 1;
-      return scan(bytes).tokens;
+      Token token = scan(bytes).tokens;
+      while (token is ErrorToken) {
+        if (!suppressLexicalErrors) {
+          ErrorToken error = token;
+          String message = new InputError(
+              uri, token.charOffset, error.assertionMessage).format();
+          print(message);
+        }
+        token = token.next;
+      }
+      return token;
     } on FileSystemException catch (e) {
       String message = e.message;
       String osMessage = e.osError?.message;
       if (osMessage != null && osMessage.isNotEmpty) {
         message = osMessage;
       }
       return inputError(uri, -1, message);
     }
   }
 
   Future<Null> buildOutline(SourceLibraryBuilder library) async {
     Token tokens = await tokenize(library);
     if (tokens == null) return;
     OutlineBuilder listener = new OutlineBuilder(library);
     new ClassMemberParser(listener).parseUnit(tokens);
   }
 
   Future<Null> buildBody(LibraryBuilder library, AstKind astKind) async {
     if (library is SourceLibraryBuilder) {
-      Token tokens = await tokenize(library);
+      Token tokens = await tokenize(library, suppressLexicalErrors: true);

[Johnni Winther, 2017/01/30 09:04:39]

Why suppress errors here?

[ahe, 2017/01/30 13:26:22]

Added:

      // We tokenize source files twice to keep memory usage low. This is the
      // second time, and the first time was in [buildOutline] above. So this
      // time we suppress lexical errors.

       if (tokens == null) return;
       DietListener listener = new DietListener(
           library, elementStore, hierarchy, coreTypes, astKind);
       DietParser parser = new DietParser(listener);
       parser.parseUnit(tokens);
       for (SourceLibraryBuilder part in library.parts) {
         Token tokens = await tokenize(part);
         if (tokens != null) {
           parser.parseUnit(tokens);
         }
@@ skipping 213 matching lines @@
     ticker.logMs("Built analyzer element model.");
   }
 
   void computeHierarchy(Program program) {
     hierarchy = new ClassHierarchy(program);
     ticker.logMs("Computed class hierarchy");
     coreTypes = new CoreTypes(program);
     ticker.logMs("Computed core types");
   }
 }