fasta scanner recovery and error code translation improvements

pkg/front_end/lib/src/fasta/scanner.dart

Index: pkg/front_end/lib/src/fasta/scanner.dart
diff --git a/pkg/front_end/lib/src/fasta/scanner.dart b/pkg/front_end/lib/src/fasta/scanner.dart
index fe7a8025267ef6d50c70922282a1e9c96d7465dc..4c65e04bd19ec3266bdd04cef0429a6839e79487 100644
--- a/pkg/front_end/lib/src/fasta/scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner.dart
@@ -8,6 +8,8 @@ import 'dart:convert' show UNICODE_REPLACEMENT_CHARACTER_RUNE;
 
 import 'scanner/token.dart' show Token;
 
+import 'scanner/string_scanner.dart' show StringScanner;
+
 import 'scanner/utf8_bytes_scanner.dart' show Utf8BytesScanner;
 
 import 'scanner/recover.dart' show defaultRecoveryStrategy;
@@ -52,10 +54,13 @@ abstract class Scanner {
 class ScannerResult {
   final Token tokens;
   final List<int> lineStarts;
+  final bool hasErrors;
 
-  ScannerResult(this.tokens, this.lineStarts);
+  ScannerResult(this.tokens, this.lineStarts, this.hasErrors);
 }
 
+/// Scan/tokenize the given UTF8 [bytes].
+/// If [recover] is null, then the [defaultRecoveryStrategy] is used.
 ScannerResult scan(List<int> bytes,
     {bool includeComments: false, Recover recover}) {
   if (bytes.last != 0) {
@@ -63,10 +68,27 @@ ScannerResult scan(List<int> bytes,
   }
   Scanner scanner =
       new Utf8BytesScanner(bytes, includeComments: includeComments);
+  return _tokenizeAndRecover(scanner, recover, bytes);
+}
+
+/// Scan/tokenize the given [source].
+/// If [recover] is null, then the [defaultRecoveryStrategy] is used.
+ScannerResult scanString(String source,
+    {bool includeComments: false, Recover recover}) {
+  assert(source != null, 'source must not be null');
+  StringScanner scanner =
+      new StringScanner(source, includeComments: includeComments);
+  // TODO(danrubel): what should be passed for the "bytes" argument?
+  // does the Recover API need to be adjusted for a String source?

[ahe, 2017/03/23 11:52:36]

Since we only need the bytes when performing error recovery, we could pass in
the string source here and convert it to bytes if calling recover.

[danrubel, 2017/03/24 15:44:20]

Done.

+  return _tokenizeAndRecover(scanner, recover, null);
+}
+
+ScannerResult _tokenizeAndRecover(
+    Scanner scanner, Recover recover, List<int> bytes) {
   Token tokens = scanner.tokenize();
   if (scanner.hasErrors) {
     recover ??= defaultRecoveryStrategy;
     tokens = recover(bytes, tokens, scanner.lineStarts);
   }
-  return new ScannerResult(tokens, scanner.lineStarts);
+  return new ScannerResult(tokens, scanner.lineStarts, scanner.hasErrors);
 }