fasta scanner recovery and error code translation improvements

pkg/analyzer/lib/src/fasta/token_utils.dart

Index: pkg/analyzer/lib/src/fasta/token_utils.dart
diff --git a/pkg/analyzer/lib/src/fasta/token_utils.dart b/pkg/analyzer/lib/src/fasta/token_utils.dart
index 39b4c63c1bb17bb064da923e45193af20a65e5e6..1e26cbe725486b5963efe2c68343990a5b17e270 100644
--- a/pkg/analyzer/lib/src/fasta/token_utils.dart
+++ b/pkg/analyzer/lib/src/fasta/token_utils.dart
@@ -4,8 +4,6 @@
 
 library fasta.analyzer.token_utils;
 
-import 'package:front_end/src/fasta/parser/error_kind.dart' show ErrorKind;
-
 import 'package:front_end/src/fasta/scanner/error_token.dart' show ErrorToken;
 
 import 'package:front_end/src/fasta/scanner/keyword.dart' show Keyword;
@@ -24,6 +22,8 @@ import 'package:front_end/src/fasta/scanner/token.dart'
 
 import 'package:front_end/src/fasta/scanner/token_constants.dart';
 
+import 'package:front_end/src/scanner/errors.dart' show translateErrorToken;
+
 import 'package:front_end/src/scanner/token.dart' as analyzer
     show
         BeginToken,
@@ -92,7 +92,7 @@ class ToAnalyzerTokenStreamConverter {
     while (true) {
       if (token.info.kind == BAD_INPUT_TOKEN) {
         ErrorToken errorToken = token;
-        _translateErrorToken(errorToken);
+        translateErrorToken(errorToken, reportError);
       } else {
         var translatedToken = translateToken(
             token, translateCommentTokens(token.precedingCommentTokens));
@@ -163,61 +163,6 @@ class ToAnalyzerTokenStreamConverter {
       _endTokenStack.add(token.endGroup);
     }
   }
-
-  /// Translates the given error [token] into an analyzer error and reports it
-  /// using [reportError].
-  void _translateErrorToken(ErrorToken token) {
-    int charOffset = token.charOffset;
-    // TODO(paulberry,ahe): why is endOffset sometimes null?
-    int endOffset = token.endOffset ?? charOffset;
-    void _makeError(
-        analyzer.ScannerErrorCode errorCode, List<Object> arguments) {
-      if (_isAtEnd(token, charOffset)) {
-        // Analyzer never generates an error message past the end of the input,
-        // since such an error would not be visible in an editor.
-        // TODO(paulberry,ahe): would it make sense to replicate this behavior
-        // in fasta, or move it elsewhere in analyzer?
-        charOffset--;
-      }
-      reportError(errorCode, charOffset, arguments);
-    }
-
-    var errorCode = token.errorCode;
-    switch (errorCode) {
-      case ErrorKind.UnterminatedString:
-        // TODO(paulberry,ahe): Fasta reports the error location as the entire
-        // string; analyzer expects the end of the string.
-        charOffset = endOffset;
-        return _makeError(
-            analyzer.ScannerErrorCode.UNTERMINATED_STRING_LITERAL, null);
-      case ErrorKind.UnmatchedToken:
-        return null;
-      case ErrorKind.UnterminatedComment:
-        // TODO(paulberry,ahe): Fasta reports the error location as the entire
-        // comment; analyzer expects the end of the comment.
-        charOffset = endOffset;
-        return _makeError(
-            analyzer.ScannerErrorCode.UNTERMINATED_MULTI_LINE_COMMENT, null);
-      case ErrorKind.MissingExponent:
-        // TODO(paulberry,ahe): Fasta reports the error location as the entire
-        // number; analyzer expects the end of the number.
-        charOffset = endOffset;
-        return _makeError(analyzer.ScannerErrorCode.MISSING_DIGIT, null);
-      case ErrorKind.ExpectedHexDigit:
-        // TODO(paulberry,ahe): Fasta reports the error location as the entire
-        // number; analyzer expects the end of the number.
-        charOffset = endOffset;
-        return _makeError(analyzer.ScannerErrorCode.MISSING_HEX_DIGIT, null);
-      case ErrorKind.NonAsciiIdentifier:
-      case ErrorKind.NonAsciiWhitespace:
-        return _makeError(
-            analyzer.ScannerErrorCode.ILLEGAL_CHARACTER, [token.character]);
-      case ErrorKind.UnexpectedDollarInString:
-        return null;
-      default:
-        throw new UnimplementedError('$errorCode');
-    }
-  }
 }
 
 /// Converts a single Fasta comment token to an analyzer comment token.
@@ -237,8 +182,7 @@ analyzer.CommentToken toAnalyzerCommentToken(Token token) {
 /// trip through this function and [toAnalyzerTokenStream] will lose error
 /// information.
 Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) {
-  Token tokenHead = new SymbolToken(EOF_INFO, -1);
-  tokenHead.previous = tokenHead;
+  Token tokenHead = new SymbolToken.eof(-1);
   Token tokenTail = tokenHead;
 
   // Both fasta and analyzer have links from a "BeginToken" to its matching
@@ -313,7 +257,7 @@ Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) {
   while (true) {
     // TODO(paulberry): join up begingroup/endgroup.
     if (analyzerToken.type == TokenType.EOF) {
-      tokenTail.next = new SymbolToken(EOF_INFO, analyzerToken.offset);
+      tokenTail.next = new SymbolToken.eof(analyzerToken.offset);
       tokenTail.next.previousToken = tokenTail;
       tokenTail.next.precedingCommentTokens =
           translateComments(analyzerToken.precedingComments);
@@ -506,22 +450,6 @@ Token fromAnalyzerToken(analyzer.Token token) {
   }
 }
 
-/// Determines whether the given [charOffset], which came from the non-EOF token
-/// [token], represents the end of the input.
-bool _isAtEnd(Token token, int charOffset) {
-  while (true) {
-    // Skip to the next token.
-    token = token.next;
-    // If we've found an EOF token, its charOffset indicates where the end of
-    // the input is.
-    if (token.isEof) return token.charOffset == charOffset;
-    // If we've found a non-error token, then we know there is additional input
-    // text after [charOffset].
-    if (token.info.kind != BAD_INPUT_TOKEN) return false;
-    // Otherwise keep looking.
-  }
-}
-
 analyzer.Token toAnalyzerToken(Token token,
     [analyzer.CommentToken commentToken]) {
   if (token == null) return null;