move synthetic fasta closers into the token stream

pkg/front_end/test/scanner_replacement_test.dart

Index: pkg/front_end/test/scanner_replacement_test.dart
diff --git a/pkg/front_end/test/scanner_replacement_test.dart b/pkg/front_end/test/scanner_replacement_test.dart
index 171d943906d6aaf0ec019bb2d50ead2b5641ff37..63ea94f6bcfd3a64785afcd084da010d82f24886 100644
--- a/pkg/front_end/test/scanner_replacement_test.dart
+++ b/pkg/front_end/test/scanner_replacement_test.dart
@@ -9,6 +9,8 @@ import 'package:front_end/src/fasta/scanner/precedence.dart'
 import 'package:front_end/src/fasta/scanner/recover.dart'
     show defaultRecoveryStrategy;
 import 'package:front_end/src/fasta/scanner.dart' as fasta;
+import 'package:front_end/src/fasta/scanner/token.dart' as fasta;
+import 'package:front_end/src/fasta/scanner/error_token.dart' as fasta;
 import 'package:front_end/src/scanner/token.dart' as analyzer;
 import 'package:front_end/src/scanner/errors.dart'
     show ScannerErrorCode, translateErrorToken;
@@ -54,10 +56,11 @@ class ScannerTest_Replacement extends ScannerTest {
         }));
     fasta.Token tokens = result.tokens;
     assertValidTokenStream(tokens);
+    assertValidBeginTokens(tokens);
     if (result.hasErrors) {
       List<int> bytes = UTF8.encode(source);
       tokens = defaultRecoveryStrategy(bytes, tokens, result.lineStarts);
-      assertValidTokenStream(tokens);
+      assertValidTokenStream(tokens, errorsFirst: true);
     }
     return extractErrors(tokens, listener);
   }
@@ -94,25 +97,126 @@ class ScannerTest_Replacement extends ScannerTest {
     super.test_comment_generic_method_type_list();
   }
 
+  void _assertOpenClosePair(String source) {
+    fasta.BeginGroupToken open = _scan(source);
+    fasta.Token close = open.next;
+    expect(close.next.isEof, isTrue);
+    expect(open.endGroup, close);
+    expect(open.isSynthetic, isFalse);
+    expect(close.isSynthetic, isFalse);
+  }
+
+  void _assertOpenOnly(String source) {
+    fasta.BeginGroupToken open = _scan(source);
+    fasta.Token close = open.next;
+    expect(close.next.isEof, isTrue);
+    expect(open.endGroup, close);
+    expect(open.isSynthetic, isFalse);
+    expect(close.isSynthetic, isTrue);
+  }
+
+  void test_lt() {
+    // fasta does not automatically insert a closer for '<'
+    // because it could be part of an expression rather than an opener
+    fasta.BeginGroupToken lt = _scan('<');
+    expect(lt.next.isEof, isTrue);
+    expect(lt.isSynthetic, isFalse);
+  }
+
+  void test_lt_gt() {
+    _assertOpenClosePair('< >');
+  }
+
+  @override
+  void test_open_curly_bracket() {
+    _assertOpenOnly('{');
+  }
+
+  void test_open_curly_bracket_with_close() {
+    _assertOpenClosePair('{ }');
+  }
+
+  void test_open_paren() {
+    _assertOpenOnly('(');
+  }
+
+  void test_open_paren_with_close() {
+    _assertOpenClosePair('( )');
+  }
+
+  void test_open_square_bracket() {
+    _assertOpenOnly('[');
+  }
+
+  void test_open_square_bracket_with_close() {
+    _assertOpenClosePair('[ ]');
+  }
+
   @override
-  @failingTest
   void test_mismatched_closer() {
-    // TODO(danrubel): investigate and fix
-    super.test_mismatched_closer();
+    // When openers and closers are mismatched,
+    // fasta favors considering the opener to be mismatched,
+    // and inserts synthetic closers as needed.
+    // `(])` is parsed as `()])` where the first `)` is synthetic
+    // and the trailing `])` are unmatched.
+    fasta.BeginGroupToken openParen = _scan('(])');
+    fasta.Token closeParen = openParen.next;
+    fasta.Token closeBracket = closeParen.next;
+    fasta.Token closeParen2 = closeBracket.next;
+    fasta.Token eof = closeParen2.next;
+
+    expect(openParen.endToken, same(closeParen));
+    expect(closeParen.isSynthetic, isTrue);
+    expect(eof.isEof, isTrue);
   }
 
   @override
-  @failingTest
   void test_mismatched_opener() {
-    // TODO(danrubel): investigate and fix
-    super.test_mismatched_opener();
+    // When openers and closers are mismatched,
+    // fasta favors considering the opener to be mismatched
+    // and inserts synthetic closers as needed.
+    // `([)` is parsed as `([])` where `]` is synthetic.
+    fasta.BeginGroupToken openParen = _scan('([)');
+    fasta.BeginGroupToken openBracket = openParen.next;
+    fasta.Token closeBracket = openBracket.next; // <-- synthetic
+    fasta.Token closeParen = closeBracket.next;
+    fasta.Token eof = closeParen.next;
+
+    expect(openParen.endToken, same(closeParen));
+    expect(closeParen.isSynthetic, isFalse);
+    expect(openBracket.endToken, same(closeBracket));
+    expect(closeBracket.isSynthetic, isTrue);
+    expect(eof.isEof, isTrue);
   }
 
   @override
-  @failingTest
   void test_mismatched_opener_in_interpolation() {
-    // TODO(danrubel): investigate and fix
-    super.test_mismatched_opener_in_interpolation();
+    // When openers and closers are mismatched,
+    // fasta favors considering the opener to be mismatched
+    // and inserts synthetic closers as needed.
+    // r'"${({(}}"' is parsed as r'"${({()})}"'
+    // where both ')' are synthetic
+    var stringStart = _scan(r'"${({(}}"');
+    var interpolationStart = stringStart.next as fasta.BeginGroupToken;
+    var openParen1 = interpolationStart.next as fasta.BeginGroupToken;
+    var openBrace = openParen1.next as fasta.BeginGroupToken;
+    var openParen2 = openBrace.next as fasta.BeginGroupToken;
+    var closeParen2 = openParen2.next;
+    var closeBrace = closeParen2.next;
+    var closeParen1 = closeBrace.next;
+    var interpolationEnd = closeParen1.next;
+    var stringEnd = interpolationEnd.next;
+    var eof = stringEnd.next;
+
+    expect(interpolationStart.endToken, same(interpolationEnd));
+    expect(interpolationEnd.isSynthetic, isFalse);
+    expect(openParen1.endToken, same(closeParen1));
+    expect(closeParen1.isSynthetic, isTrue);
+    expect(openBrace.endToken, same(closeBrace));
+    expect(closeBrace.isSynthetic, isFalse);
+    expect(openParen2.endToken, same(closeParen2));
+    expect(closeParen2.isSynthetic, isTrue);
+    expect(eof.isEof, isTrue);
   }
 
   @override
@@ -199,24 +303,22 @@ class ScannerTest_Replacement extends ScannerTest {
     super.test_string_simple_unterminated_interpolation_identifier();
   }
 
-  @failingTest
   @override
   void test_unmatched_openers() {
-    // fasta recovery inserts closers
-    var openBrace = _scan('{[(<') as analyzer.BeginToken;
-    var openBracket = openBrace.next as analyzer.BeginToken;
-    var openParen = openBracket.next as analyzer.BeginToken;
-    var openLT = openParen.next as analyzer.BeginToken;
-    var closeGT = openLT.next;
-    var closeParen = closeGT.next;
+    // fasta inserts missing closers except for '<'
+    var openBrace = _scan('{[(<') as fasta.BeginGroupToken;
+    var openBracket = openBrace.next as fasta.BeginGroupToken;
+    var openParen = openBracket.next as fasta.BeginGroupToken;
+    var openLT = openParen.next as fasta.BeginGroupToken;
+    var closeParen = openLT.next;
     var closeBracket = closeParen.next;
     var closeBrace = closeBracket.next;
-    expect(closeBrace.next.type, analyzer.TokenType.EOF);
+    var eof = closeBrace.next;
 
-    expect(openBrace.endToken, closeBrace);
-    expect(openBracket.endToken, closeBracket);
-    expect(openParen.endToken, closeParen);
-    expect(openLT.endToken, closeGT);
+    expect(openBrace.endGroup, same(closeBrace));
+    expect(openBracket.endGroup, same(closeBracket));
+    expect(openParen.endGroup, same(closeParen));
+    expect(eof.isEof, true);
   }
 
   analyzer.Token _scan(String source,
@@ -250,13 +352,21 @@ class ScannerTest_Replacement extends ScannerTest {
   }
 
   /// Assert that the tokens in the stream are correctly connected prev/next.
-  void assertValidTokenStream(fasta.Token firstToken) {
+  void assertValidTokenStream(fasta.Token firstToken,
+      {bool errorsFirst: false}) {
     fasta.Token token = firstToken;
     fasta.Token previous = token.previousToken;
     expect(previous.isEof, isTrue, reason: 'Missing leading EOF');
     expect(previous.next, token, reason: 'Invalid leading EOF');
     expect(previous.previous, previous, reason: 'Invalid leading EOF');
+    if (errorsFirst) {
+      while (!token.isEof && token is fasta.ErrorToken) {
+        token = token.next;
+      }
+    }
+    var isNotErrorToken = isNot(new isInstanceOf<fasta.ErrorToken>());
     while (!token.isEof) {
+      if (errorsFirst) expect(token, isNotErrorToken);
       previous = token;
       token = token.next;
       expect(token, isNotNull, reason: previous.toString());
@@ -264,4 +374,29 @@ class ScannerTest_Replacement extends ScannerTest {
     }
     expect(token.next, token, reason: 'Invalid trailing EOF');
   }
+
+  /// Assert that all [fasta.BeginGroupToken] has a valid `endGroup`
+  /// that is in the stream.
+  void assertValidBeginTokens(fasta.Token firstToken) {
+    var openerStack = <fasta.BeginGroupToken>[];
+    fasta.BeginGroupToken lastClosedGroup;
+    fasta.Token token = firstToken;
+    while (!token.isEof) {
+      if (token is fasta.BeginGroupToken) {
+        if (token.lexeme != '<')
+          expect(token.endGroup, isNotNull, reason: token.lexeme);
+        if (token.endGroup != null) openerStack.add(token);
+      } else if (openerStack.isNotEmpty && openerStack.last.endGroup == token) {
+        lastClosedGroup = openerStack.removeLast();
+        expect(token.isSynthetic, token.next is fasta.UnmatchedToken,
+            reason: 'Expect synthetic closer then error token');
+      } else if (token is fasta.UnmatchedToken) {
+        expect(lastClosedGroup?.endGroup?.next, same(token),
+            reason: 'Unexpected error token for group: $lastClosedGroup');
+        expect(token.begin, lastClosedGroup);
+      }
+      token = token.next;
+    }
+    expect(openerStack, isEmpty, reason: 'Missing closers');
+  }
 }