improve fasta unterminated string recovery

pkg/front_end/test/scanner_test.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 import 'package:front_end/src/base/errors.dart';
 import 'package:front_end/src/base/jenkins_smi_hash.dart';
 import 'package:front_end/src/scanner/errors.dart';
 import 'package:front_end/src/scanner/reader.dart';
 import 'package:front_end/src/scanner/scanner.dart';
 import 'package:front_end/src/scanner/token.dart';
@@ skipping 421 matching lines @@
     // https://code.google.com/p/dart/issues/detail?id=18073
     List<Token> expectedTokens = [
       new StringToken(TokenType.STRING, "\"foo ", 0),
       new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 5),
       new StringToken(TokenType.IDENTIFIER, "bar", 7),
     ];
     if (usingFasta) {
       // fasta inserts synthetic closers
       expectedTokens.addAll([
         new SyntheticToken(TokenType.CLOSE_CURLY_BRACKET, 10),
+        new SyntheticStringToken(TokenType.STRING, "\"", 10, 0),
       ]);
     } else {
       expectedTokens.addAll([
         new StringToken(TokenType.STRING, "", 10),
       ]);
     }
     _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 9,
         "\"foo \${bar", expectedTokens);
   }
 
@@ skipping 533 matching lines @@
 
   void test_string_multi_single() {
     _assertToken(TokenType.STRING, "'''string'''");
   }
 
   void test_string_multi_slashEnter() {
     _assertToken(TokenType.STRING, "'''\\\n'''");
   }
 
   void test_string_multi_unterminated() {
-    _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 8,
-        "'''string", [new StringToken(TokenType.STRING, "'''string", 0)]);
+    List<Token> expectedTokens = [];
+    if (usingFasta) {
+      // fasta inserts synthetic closers

[ahe, 2017/06/02 12:26:43]

fasta -> Fasta
Add period at end of sentence.

Here and several places below.

+      expectedTokens.addAll([
+        new SyntheticStringToken(TokenType.STRING, "'''string'''", 0, 9),
+      ]);
+    } else {
+      expectedTokens.addAll([
+        new StringToken(TokenType.STRING, "'''string", 0),
+      ]);
+    }
   }
 
   void test_string_multi_unterminated_interpolation_block() {
-    _assertErrorAndTokens(
-        ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 8, "'''\${name", [
+    List<Token> expectedTokens = [
       new StringToken(TokenType.STRING, "'''", 0),
       new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 3),
       new StringToken(TokenType.IDENTIFIER, "name", 5),
-      new StringToken(TokenType.STRING, "", 9)
-    ]);
+    ];
+    if (usingFasta) {
+      // fasta inserts synthetic closers
+      expectedTokens.addAll([
+        new SyntheticToken(TokenType.CLOSE_CURLY_BRACKET, 9),
+        new SyntheticStringToken(TokenType.STRING, "'''", 9, 0),
+      ]);
+    } else {
+      expectedTokens.addAll([
+        new StringToken(TokenType.STRING, "", 9),
+      ]);
+    }
+    _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 8,
+        "'''\${name", expectedTokens);
   }
 
   void test_string_multi_unterminated_interpolation_identifier() {
-    _assertErrorAndTokens(
-        ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 7, "'''\$name", [
+    List<Token> expectedTokens = [
       new StringToken(TokenType.STRING, "'''", 0),
       new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 3),
       new StringToken(TokenType.IDENTIFIER, "name", 4),
-      new StringToken(TokenType.STRING, "", 8)
-    ]);
+    ];
+    if (usingFasta) {
+      // fasta inserts synthetic closers
+      expectedTokens.addAll([
+        new SyntheticStringToken(TokenType.STRING, "'''", 8, 0),
+      ]);
+    } else {
+      expectedTokens.addAll([
+        new StringToken(TokenType.STRING, "", 8),
+      ]);
+    }
+    _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 7,
+        "'''\$name", expectedTokens);
   }
 
   void test_string_raw_multi_double() {
     _assertToken(TokenType.STRING, "r\"\"\"line1\nline2\"\"\"");
   }
 
   void test_string_raw_multi_single() {
     _assertToken(TokenType.STRING, "r'''string'''");
   }
 
   void test_string_raw_multi_unterminated() {
     String source = "r'''string";
+    List<Token> expectedTokens = [];
+    if (usingFasta) {
+      // fasta inserts synthetic closers
+      expectedTokens.addAll([
+        new SyntheticStringToken(TokenType.STRING, "r'''string'''", 0, 10),
+      ]);
+    } else {
+      expectedTokens.addAll([
+        new StringToken(TokenType.STRING, "r'''string", 0),
+      ]);
+    }
     _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 9,
-        source, [new StringToken(TokenType.STRING, source, 0)]);
+        source, expectedTokens);
   }
 
   void test_string_raw_simple_double() {
     _assertToken(TokenType.STRING, "r\"string\"");
   }
 
   void test_string_raw_simple_single() {
     _assertToken(TokenType.STRING, "r'string'");
   }
 
   void test_string_raw_simple_unterminated_eof() {
     String source = "r'string";
+    List<Token> expectedTokens = [];
+    if (usingFasta) {
+      // fasta inserts synthetic closers
+      expectedTokens.addAll([
+        new SyntheticStringToken(TokenType.STRING, "r'string'", 0, 8),
+      ]);
+    } else {
+      expectedTokens.addAll([
+        new StringToken(TokenType.STRING, "r'string", 0),
+      ]);
+    }
     _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 7,
-        source, [new StringToken(TokenType.STRING, source, 0)]);
+        source, expectedTokens);
   }
 
   void test_string_raw_simple_unterminated_eol() {
-    String source = "r'string";
-    _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 8,
-        "$source\n", [new StringToken(TokenType.STRING, source, 0)]);
+    String source = "r'string\n";
+    List<Token> expectedTokens = [];
+    if (usingFasta) {
+      // fasta inserts synthetic closers
+      expectedTokens.addAll([
+        new SyntheticStringToken(TokenType.STRING, "r'string'", 0, 8),
+      ]);
+    } else {
+      expectedTokens.addAll([
+        new StringToken(TokenType.STRING, "r'string", 0),
+      ]);
+    }
+    _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL,
+        usingFasta ? 7 : 8, source, expectedTokens);
   }
 
   void test_string_simple_double() {
     _assertToken(TokenType.STRING, "\"string\"");
   }
 
   void test_string_simple_escapedDollar() {
     _assertToken(TokenType.STRING, "'a\\\$b'");
   }
 
@@ skipping 75 matching lines @@
       new StringToken(TokenType.STRING, "1'", 2)
     ]);
   }
 
   void test_string_simple_single() {
     _assertToken(TokenType.STRING, "'string'");
   }
 
   void test_string_simple_unterminated_eof() {
     String source = "'string";
+    List<Token> expectedTokens = [];
+    if (usingFasta) {
+      // fasta inserts synthetic closers
+      expectedTokens.addAll([
+        new SyntheticStringToken(TokenType.STRING, "'string'", 0, 7),
+      ]);
+    } else {
+      expectedTokens.addAll([
+        new StringToken(TokenType.STRING, "'string", 0),
+      ]);
+    }
     _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 6,
-        source, [new StringToken(TokenType.STRING, source, 0)]);
+        source, expectedTokens);
   }
 
   void test_string_simple_unterminated_eol() {
-    String source = "'string";
-    _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 7,
-        "$source\r", [new StringToken(TokenType.STRING, source, 0)]);
+    String source = "'string\r";
+    List<Token> expectedTokens = [];
+    if (usingFasta) {
+      // fasta inserts synthetic closers
+      expectedTokens.addAll([
+        new SyntheticStringToken(TokenType.STRING, "'string'", 0, 7),
+      ]);
+    } else {
+      expectedTokens.addAll([
+        new StringToken(TokenType.STRING, "'string", 0),
+      ]);
+    }
+    _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL,
+        usingFasta ? 6 : 7, source, expectedTokens);
   }
 
   void test_string_simple_unterminated_interpolation_block() {
-    _assertErrorAndTokens(
-        ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 6, "'\${name", [
+    List<Token> expectedTokens = [
       new StringToken(TokenType.STRING, "'", 0),
       new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "\${", 1),
       new StringToken(TokenType.IDENTIFIER, "name", 3),
-      new StringToken(TokenType.STRING, "", 7)
-    ]);
+    ];
+    if (usingFasta) {
+      // fasta inserts synthetic closers
+      expectedTokens.addAll([
+        new SyntheticToken(TokenType.CLOSE_CURLY_BRACKET, 7),
+        new SyntheticStringToken(TokenType.STRING, "'", 7, 0),
+      ]);
+    } else {
+      expectedTokens.addAll([
+        new StringToken(TokenType.STRING, "", 7),
+      ]);
+    }
+    _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 6,
+        "'\${name", expectedTokens);
   }
 
   void test_string_simple_unterminated_interpolation_identifier() {
-    _assertErrorAndTokens(
-        ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 5, "'\$name", [
+    List<Token> expectedTokens = [
       new StringToken(TokenType.STRING, "'", 0),
       new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "\$", 1),
       new StringToken(TokenType.IDENTIFIER, "name", 2),
-      new StringToken(TokenType.STRING, "", 6)
-    ]);
+    ];
+    if (usingFasta) {
+      // fasta inserts synthetic closers
+      expectedTokens.addAll([
+        new SyntheticStringToken(TokenType.STRING, "'", 6, 0),
+      ]);
+    } else {
+      expectedTokens.addAll([
+        new StringToken(TokenType.STRING, "", 6),
+      ]);
+    }
+    _assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 5,
+        "'\$name", expectedTokens);
   }
 
   void test_sync_star() {
     Token token = _scan("sync*");
     expect(token.type.isKeyword, true);
     expect(token.lexeme, 'sync');
     expect(token.next.type, TokenType.STAR);
     expect(token.next.next.type, TokenType.EOF);
   }
 
@@ skipping 322 matching lines @@
   _TestScanner(CharacterReader reader, [this.listener]) : super.create(reader);
 
   @override
   void reportError(
       ScannerErrorCode errorCode, int offset, List<Object> arguments) {
     if (listener != null) {
       listener.errors.add(new TestError(offset, errorCode, arguments));
     }
   }
 }