fasta scanner recovery and error code translation improvements

pkg/analyzer/lib/src/fasta/token_utils.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.analyzer.token_utils;
 
-import 'package:front_end/src/fasta/parser/error_kind.dart' show ErrorKind;
-
 import 'package:front_end/src/fasta/scanner/error_token.dart' show ErrorToken;
 
 import 'package:front_end/src/fasta/scanner/keyword.dart' show Keyword;
 
 import 'package:front_end/src/fasta/scanner/precedence.dart';
 
 import 'package:front_end/src/fasta/scanner/token.dart'
     show
         BeginGroupToken,
         CommentToken,
         DartDocToken,
         KeywordToken,
         StringToken,
         SymbolToken,
         Token;
 
 import 'package:front_end/src/fasta/scanner/token_constants.dart';
 
+import 'package:front_end/src/scanner/errors.dart' show translateErrorToken;
+
 import 'package:front_end/src/scanner/token.dart' as analyzer
     show
         BeginToken,
         BeginTokenWithComment,
         CommentToken,
         Keyword,
         KeywordToken,
         KeywordTokenWithComment,
         StringToken,
         StringTokenWithComment,
@@ skipping 48 matching lines @@
   analyzer.Token convertTokens(Token token) {
     _analyzerTokenHead = new analyzer.Token(TokenType.EOF, -1);
     _analyzerTokenHead.previous = _analyzerTokenHead;
     _analyzerTokenTail = _analyzerTokenHead;
     _beginTokenStack = [null];
     _endTokenStack = <Token>[null];
 
     while (true) {
       if (token.info.kind == BAD_INPUT_TOKEN) {
         ErrorToken errorToken = token;
-        _translateErrorToken(errorToken);
+        translateErrorToken(errorToken, reportError);
       } else {
         var translatedToken = translateToken(
             token, translateCommentTokens(token.precedingCommentTokens));
         _matchGroups(token, translatedToken);
         translatedToken.setNext(translatedToken);
         _analyzerTokenTail.setNext(translatedToken);
         translatedToken.previous = _analyzerTokenTail;
         _analyzerTokenTail = translatedToken;
       }
       if (token.isEof) {
@@ skipping 50 matching lines @@
     }
     // Synthetic end tokens use the same offset as the begin token.
     if (translatedToken is analyzer.BeginToken &&
         token is BeginGroupToken &&
         token.endGroup != null &&
         token.endGroup.charOffset != token.charOffset) {
       _beginTokenStack.add(translatedToken);
       _endTokenStack.add(token.endGroup);
     }
   }
-
-  /// Translates the given error [token] into an analyzer error and reports it
-  /// using [reportError].
-  void _translateErrorToken(ErrorToken token) {
-    int charOffset = token.charOffset;
-    // TODO(paulberry,ahe): why is endOffset sometimes null?
-    int endOffset = token.endOffset ?? charOffset;
-    void _makeError(
-        analyzer.ScannerErrorCode errorCode, List<Object> arguments) {
-      if (_isAtEnd(token, charOffset)) {
-        // Analyzer never generates an error message past the end of the input,
-        // since such an error would not be visible in an editor.
-        // TODO(paulberry,ahe): would it make sense to replicate this behavior
-        // in fasta, or move it elsewhere in analyzer?
-        charOffset--;
-      }
-      reportError(errorCode, charOffset, arguments);
-    }
-
-    var errorCode = token.errorCode;
-    switch (errorCode) {
-      case ErrorKind.UnterminatedString:
-        // TODO(paulberry,ahe): Fasta reports the error location as the entire
-        // string; analyzer expects the end of the string.
-        charOffset = endOffset;
-        return _makeError(
-            analyzer.ScannerErrorCode.UNTERMINATED_STRING_LITERAL, null);
-      case ErrorKind.UnmatchedToken:
-        return null;
-      case ErrorKind.UnterminatedComment:
-        // TODO(paulberry,ahe): Fasta reports the error location as the entire
-        // comment; analyzer expects the end of the comment.
-        charOffset = endOffset;
-        return _makeError(
-            analyzer.ScannerErrorCode.UNTERMINATED_MULTI_LINE_COMMENT, null);
-      case ErrorKind.MissingExponent:
-        // TODO(paulberry,ahe): Fasta reports the error location as the entire
-        // number; analyzer expects the end of the number.
-        charOffset = endOffset;
-        return _makeError(analyzer.ScannerErrorCode.MISSING_DIGIT, null);
-      case ErrorKind.ExpectedHexDigit:
-        // TODO(paulberry,ahe): Fasta reports the error location as the entire
-        // number; analyzer expects the end of the number.
-        charOffset = endOffset;
-        return _makeError(analyzer.ScannerErrorCode.MISSING_HEX_DIGIT, null);
-      case ErrorKind.NonAsciiIdentifier:
-      case ErrorKind.NonAsciiWhitespace:
-        return _makeError(
-            analyzer.ScannerErrorCode.ILLEGAL_CHARACTER, [token.character]);
-      case ErrorKind.UnexpectedDollarInString:
-        return null;
-      default:
-        throw new UnimplementedError('$errorCode');
-    }
-  }
 }
 
 /// Converts a single Fasta comment token to an analyzer comment token.
 analyzer.CommentToken toAnalyzerCommentToken(Token token) {
   // TODO(paulberry,ahe): It would be nice if the scanner gave us an
   // easier way to distinguish between the two types of comment.
   var type = token.lexeme.startsWith('/*')
       ? TokenType.MULTI_LINE_COMMENT
       : TokenType.SINGLE_LINE_COMMENT;
   return new analyzer.CommentToken(type, token.lexeme, token.charOffset);
 }
 
 /// Converts a stream of Analyzer tokens (starting with [token] and continuing
 /// to EOF) to a stream of Fasta tokens.
 ///
 /// TODO(paulberry): Analyzer tokens do not record error conditions, so a round
 /// trip through this function and [toAnalyzerTokenStream] will lose error
 /// information.
 Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) {
-  Token tokenHead = new SymbolToken(EOF_INFO, -1);
-  tokenHead.previous = tokenHead;
+  Token tokenHead = new SymbolToken.eof(-1);
   Token tokenTail = tokenHead;
 
   // Both fasta and analyzer have links from a "BeginToken" to its matching
   // "EndToken" in a group (like parentheses and braces).  However, only fasta
   // makes these links for angle brackets.  We use these stacks to map the
   // links from the analyzer token stream into equivalent links in the fasta
   // token stream, and to create the links that fasta expects for angle
   // brackets.
 
   // Note: beginTokenStack and endTokenStack are seeded with a sentinel value
@@ skipping 54 matching lines @@
     tokenTail.next = token;
     tokenTail.next.previousToken = tokenTail;
     tokenTail = token;
     matchGroups(analyzerToken, token);
     return analyzerToken.next;
   }
 
   while (true) {
     // TODO(paulberry): join up begingroup/endgroup.
     if (analyzerToken.type == TokenType.EOF) {
-      tokenTail.next = new SymbolToken(EOF_INFO, analyzerToken.offset);
+      tokenTail.next = new SymbolToken.eof(analyzerToken.offset);
       tokenTail.next.previousToken = tokenTail;
       tokenTail.next.precedingCommentTokens =
           translateComments(analyzerToken.precedingComments);
       tokenTail.next.next = tokenTail.next;
       return tokenHead.next;
     }
     analyzerToken = translateAndAppend(analyzerToken);
   }
 }
 
@@ skipping 172 matching lines @@
       return symbol(BACKSLASH_INFO);
     case TokenType.PERIOD_PERIOD_PERIOD:
       return symbol(PERIOD_PERIOD_PERIOD_INFO);
     // case TokenType.GENERIC_METHOD_TYPE_ASSIGN
     // case TokenType.GENERIC_METHOD_TYPE_LIST
     default:
       return internalError('Unhandled token type ${token.type}');
   }
 }
 
-/// Determines whether the given [charOffset], which came from the non-EOF token
-/// [token], represents the end of the input.
-bool _isAtEnd(Token token, int charOffset) {
-  while (true) {
-    // Skip to the next token.
-    token = token.next;
-    // If we've found an EOF token, its charOffset indicates where the end of
-    // the input is.
-    if (token.isEof) return token.charOffset == charOffset;
-    // If we've found a non-error token, then we know there is additional input
-    // text after [charOffset].
-    if (token.info.kind != BAD_INPUT_TOKEN) return false;
-    // Otherwise keep looking.
-  }
-}
-
 analyzer.Token toAnalyzerToken(Token token,
     [analyzer.CommentToken commentToken]) {
   if (token == null) return null;
   analyzer.Token makeStringToken(TokenType tokenType) {
     if (commentToken == null) {
       return new analyzer.StringToken(
           tokenType, token.lexeme, token.charOffset);
     } else {
       return new analyzer.StringTokenWithComment(
           tokenType, token.lexeme, token.charOffset, commentToken);
@@ skipping 269 matching lines @@
     case PERIOD_PERIOD_PERIOD_TOKEN:
       return TokenType.PERIOD_PERIOD_PERIOD;
     // case GENERIC_METHOD_TYPE_LIST_TOKEN:
     //   return TokenType.GENERIC_METHOD_TYPE_LIST;
     // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN:
     //   return TokenType.GENERIC_METHOD_TYPE_ASSIGN;
     default:
       return internalError("Unhandled token ${token.info}");
   }
 }