Revert "improve fasta unterminated string recovery"

pkg/front_end/test/scanner_fasta_test.dart

 // Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 import 'package:analyzer/src/fasta/token_utils.dart';
 import 'package:front_end/src/fasta/fasta_codes.dart';
 import 'package:front_end/src/fasta/scanner/error_token.dart' as fasta;
 import 'package:front_end/src/fasta/scanner/string_scanner.dart' as fasta;
 import 'package:front_end/src/fasta/scanner/token.dart' as fasta;
 import 'package:front_end/src/fasta/scanner/token_constants.dart' as fasta;
@@ skipping 223 matching lines @@
           expect(commentToken.next.previous, commentToken);
         }
         commentToken = commentToken.next;
       }
       token = token.next;
     }
   }
 
   @override
   @failingTest
+  void test_string_multi_unterminated() {
+    // TODO(paulberry,ahe): bad error recovery.
+    super.test_string_multi_unterminated();
+  }
+
+  @override
+  @failingTest
   void test_string_multi_unterminated_interpolation_block() {
     // TODO(paulberry,ahe): bad error recovery.
     super.test_string_multi_unterminated_interpolation_block();
   }
 
   @override
   @failingTest
   void test_string_multi_unterminated_interpolation_identifier() {
     // TODO(paulberry,ahe): bad error recovery.
     super.test_string_multi_unterminated_interpolation_identifier();
   }
 
   @override
   @failingTest
+  void test_string_raw_multi_unterminated() {
+    // TODO(paulberry,ahe): bad error recovery.
+    super.test_string_raw_multi_unterminated();
+  }
+
+  @override
+  @failingTest
+  void test_string_raw_simple_unterminated_eof() {
+    // TODO(paulberry,ahe): bad error recovery.
+    super.test_string_raw_simple_unterminated_eof();
+  }
+
+  @override
+  @failingTest
+  void test_string_raw_simple_unterminated_eol() {
+    // TODO(paulberry,ahe): bad error recovery.
+    super.test_string_raw_simple_unterminated_eol();
+  }
+
+  @override
+  @failingTest
+  void test_string_simple_unterminated_eof() {
+    // TODO(paulberry,ahe): bad error recovery.
+    super.test_string_simple_unterminated_eof();
+  }
+
+  @override
+  @failingTest
+  void test_string_simple_unterminated_eol() {
+    // TODO(paulberry,ahe): bad error recovery.
+    super.test_string_simple_unterminated_eol();
+  }
+
+  @override
+  @failingTest
   void test_string_simple_unterminated_interpolation_block() {
     // TODO(paulberry,ahe): bad error recovery.
     super.test_string_simple_unterminated_interpolation_block();
   }
 
   @override
   @failingTest
   void test_string_simple_unterminated_interpolation_identifier() {
     // TODO(paulberry,ahe): bad error recovery.
     super.test_string_simple_unterminated_interpolation_identifier();
@@ skipping 22 matching lines @@
         lazyAssignmentOperators: lazyAssignmentOperators);
     listener.assertNoErrors();
     return token;
   }
 }
 
 /// Base class for scanner tests that examine the token stream in Fasta format.
 abstract class ScannerTest_Fasta_Base {
   Token scan(String source);
 
-  expectToken(Token token, TokenType type, int offset, int length,
-      {bool isSynthetic: false, String lexeme}) {
-    String description = '${token.type} $token';
-    expect(token.type, type, reason: description);
-    expect(token.offset, offset, reason: description);
-    expect(token.length, length, reason: description);
-    expect(token.isSynthetic, isSynthetic, reason: description);
-    if (lexeme != null) {
-      expect(token.lexeme, lexeme, reason: description);
-    }
-  }
-
-  void test_incomplete_string_interpolation() {
-    Token token = scan(r'"foo ${bar');
-    expectToken(token, TokenType.STRING, 0, 5, lexeme: '"foo ');
-
-    token = token.next;
-    expectToken(token, TokenType.STRING_INTERPOLATION_EXPRESSION, 5, 2);
-    BeginToken interpolationStart = token;
-
-    token = token.next;
-    expectToken(token, TokenType.IDENTIFIER, 7, 3, lexeme: 'bar');
-
-    // Expect interpolation to be terminated before string is closed
-    token = token.next;
-    expectToken(token, TokenType.CLOSE_CURLY_BRACKET, 10, 0,
-        isSynthetic: true, lexeme: '}');
-    expect(interpolationStart.endToken, same(token));
-
-    token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnmatchedToken));
-    expect((token as fasta.UnmatchedToken).begin, same(interpolationStart));
-
-    token = token.next;
-    expectToken(token, TokenType.STRING, 10, 0, isSynthetic: true, lexeme: '"');
-
-    token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-  }
-
-  void test_string_multi_unterminated() {
-    Token token = scan("'''string");
-    expectToken(token, TokenType.STRING, 0, 9, lexeme: "'''string");
-
-    token = token.next;
-    expectToken(token, TokenType.STRING, 9, 0,
-        isSynthetic: true, lexeme: "'''");
-
-    token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.ErrorToken).start, "'''");
-  }
-
-  void test_string_raw_multi_unterminated() {
-    Token token = scan("r'''string");
-    expectToken(token, TokenType.STRING, 0, 10, lexeme: "r'''string");
-
-    token = token.next;
-    expectToken(token, TokenType.STRING, 10, 0,
-        isSynthetic: true, lexeme: "'''");
-
-    token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.ErrorToken).start, "r'''");
-  }
-
-  void test_string_raw_simple_unterminated_eof() {
-    Token token = scan("r'string");
-    expectToken(token, TokenType.STRING, 0, 8, lexeme: "r'string");
-
-    token = token.next;
-    expectToken(token, TokenType.STRING, 8, 0, isSynthetic: true, lexeme: "'");
-
-    token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.ErrorToken).start, "r'");
-  }
-
-  void test_string_raw_simple_unterminated_eol() {
-    Token token = scan("r'string\n");
-    expectToken(token, TokenType.STRING, 0, 8, lexeme: "r'string");
-
-    token = token.next;
-    expectToken(token, TokenType.STRING, 8, 0, isSynthetic: true, lexeme: "'");
-
-    token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.ErrorToken).start, "r'");
-  }
-
-  void test_string_simple_unterminated_eof() {
-    Token token = scan("'string");
-    expectToken(token, TokenType.STRING, 0, 7, lexeme: "'string");
-
-    token = token.next;
-    expectToken(token, TokenType.STRING, 7, 0, isSynthetic: true, lexeme: "'");
-
-    token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.ErrorToken).start, "'");
-  }
-
-  void test_string_simple_unterminated_eol() {
-    Token token = scan("'string\n");
-    expectToken(token, TokenType.STRING, 0, 7, lexeme: "'string");
-
-    token = token.next;
-    expectToken(token, TokenType.STRING, 7, 0, isSynthetic: true, lexeme: "'");
-
-    token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.ErrorToken).start, "'");
-  }
-
   void test_match_angle_brackets() {
     var x = scan('x<y>');
     var lessThan = x.next as BeginToken;
     var y = lessThan.next;
     var greaterThan = y.next;
     expect(greaterThan.next.isEof, isTrue);
     expect(lessThan.endGroup, same(greaterThan));
   }
 
   void test_match_angle_brackets_gt_gt() {
@@ skipping 141 matching lines @@
 }
 
 /// Scanner tests that exercise the Fasta scanner directly.
 @reflectiveTest
 class ScannerTest_Fasta_Direct extends ScannerTest_Fasta_Base {
   @override
   Token scan(String source) {
     var scanner = new fasta.StringScanner(source, includeComments: true);
     return scanner.tokenize();
   }
+
+  test_unterminated_string_with_unterminated_interpolation() {
+    Token token = scan(r'"foo ${bar');
+    BeginToken interpolationStart = token.next;
+
+    Token previous;
+    while (token.kind != fasta.BAD_INPUT_TOKEN) {
+      expect(token.isEof, isFalse);
+      previous = token;
+      token = token.next;
+    }
+
+    // Expect interpolation to be terminated before string is closed
+
+    token = previous;
+    expect(token.isSynthetic, isTrue);
+    expect(token.length, 0);
+    expect(token.stringValue, '}');
+
+    token = token.next;
+    expect((token as fasta.ErrorToken).errorCode, same(codeUnmatchedToken));
+    expect((token as fasta.UnmatchedToken).begin, same(interpolationStart));
+
+    token = token.next;
+    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
+  }
 }
 
 /// Override of [ToAnalyzerTokenStreamConverter] that verifies that there are no
 /// errors.
 class ToAnalyzerTokenStreamConverter_NoErrors
     extends ToAnalyzerTokenStreamConverter {
   @override
   void reportError(
       ScannerErrorCode errorCode, int offset, List<Object> arguments) {
     fail('Unexpected error: $errorCode, $offset, $arguments');
   }
 }
 
 /// Override of [ToAnalyzerTokenStreamConverter] that records errors in an
 /// [ErrorListener].
 class ToAnalyzerTokenStreamConverter_WithListener
     extends ToAnalyzerTokenStreamConverter {
   final ErrorListener _listener;
 
   ToAnalyzerTokenStreamConverter_WithListener(this._listener);
 
   @override
   void reportError(
       ScannerErrorCode errorCode, int offset, List<Object> arguments) {
     _listener.errors.add(new TestError(offset, errorCode, arguments));
   }
 }