move synthetic fasta closers into the token stream

pkg/front_end/test/scanner_replacement_test.dart

 // Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:convert' show UTF8;
 
 import 'package:front_end/src/fasta/scanner/precedence.dart'
     show BAD_INPUT_INFO, EOF_INFO;
 import 'package:front_end/src/fasta/scanner/recover.dart'
     show defaultRecoveryStrategy;
 import 'package:front_end/src/fasta/scanner.dart' as fasta;
+import 'package:front_end/src/fasta/scanner/token.dart' as fasta;
+import 'package:front_end/src/fasta/scanner/error_token.dart' as fasta;
 import 'package:front_end/src/scanner/token.dart' as analyzer;
 import 'package:front_end/src/scanner/errors.dart'
     show ScannerErrorCode, translateErrorToken;
 import 'package:test/test.dart';
 import 'package:test_reflective_loader/test_reflective_loader.dart';
 
 import 'scanner_test.dart';
 
 main() {
   defineReflectiveSuite(() {
@@ skipping 25 matching lines @@
     // Dart, remove this flag.
     fasta.ScannerResult result = fasta.scanString(source,
         includeComments: true,
         recover: ((List<int> bytes, fasta.Token tokens, List<int> lineStarts) {
           // perform recovery as a separate step
           // so that the token stream can be validated before and after recovery
           return tokens;
         }));
     fasta.Token tokens = result.tokens;
     assertValidTokenStream(tokens);
+    assertValidBeginTokens(tokens);
     if (result.hasErrors) {
       List<int> bytes = UTF8.encode(source);
       tokens = defaultRecoveryStrategy(bytes, tokens, result.lineStarts);
-      assertValidTokenStream(tokens);
+      assertValidTokenStream(tokens, errorsFirst: true);
     }
     return extractErrors(tokens, listener);
   }
 
   @override
   @failingTest
   void test_ampersand_ampersand_eq() {
     // TODO(paulberry,ahe): Fasta scanner doesn't support lazy assignment
     // operators.
     super.test_ampersand_ampersand_eq();
@@ skipping 16 matching lines @@
   }
 
   @override
   @failingTest
   void test_comment_generic_method_type_list() {
     // TODO(paulberry,ahe): Fasta scanner doesn't support generic comment
     // syntax.
     super.test_comment_generic_method_type_list();
   }
 
-  @override
-  @failingTest
-  void test_mismatched_closer() {
-    // TODO(danrubel): investigate and fix
-    super.test_mismatched_closer();
+  void _assertOpenClosePair(String source) {
+    fasta.BeginGroupToken open = _scan(source);
+    fasta.Token close = open.next;
+    expect(close.next.isEof, isTrue);
+    expect(open.endGroup, close);
+    expect(open.isSynthetic, isFalse);
+    expect(close.isSynthetic, isFalse);
+  }
+
+  void _assertOpenOnly(String source) {
+    fasta.BeginGroupToken open = _scan(source);
+    fasta.Token close = open.next;
+    expect(close.next.isEof, isTrue);
+    expect(open.endGroup, close);
+    expect(open.isSynthetic, isFalse);
+    expect(close.isSynthetic, isTrue);
+  }
+
+  void test_lt() {
+    // fasta does not automatically insert a closer for '<'
+    // because it could be part of an expression rather than an opener
+    fasta.BeginGroupToken lt = _scan('<');
+    expect(lt.next.isEof, isTrue);
+    expect(lt.isSynthetic, isFalse);
+  }
+
+  void test_lt_gt() {
+    _assertOpenClosePair('< >');
   }
 
   @override
-  @failingTest
-  void test_mismatched_opener() {
-    // TODO(danrubel): investigate and fix
-    super.test_mismatched_opener();
+  void test_open_curly_bracket() {
+    _assertOpenOnly('{');
+  }
+
+  void test_open_curly_bracket_with_close() {
+    _assertOpenClosePair('{ }');
+  }
+
+  void test_open_paren() {
+    _assertOpenOnly('(');
+  }
+
+  void test_open_paren_with_close() {
+    _assertOpenClosePair('( )');
+  }
+
+  void test_open_square_bracket() {
+    _assertOpenOnly('[');
+  }
+
+  void test_open_square_bracket_with_close() {
+    _assertOpenClosePair('[ ]');
   }
 
   @override
-  @failingTest
-  void test_mismatched_opener_in_interpolation() {
-    // TODO(danrubel): investigate and fix
-    super.test_mismatched_opener_in_interpolation();
+  void test_mismatched_closer() {
+    // When openers and closers are mismatched,
+    // fasta favors considering the opener to be mismatched,
+    // and inserts synthetic closers as needed.
+    // `(])` is parsed as `()])` where the first `)` is synthetic
+    // and the trailing `])` are unmatched.
+    fasta.BeginGroupToken openParen = _scan('(])');
+    fasta.Token closeParen = openParen.next;
+    fasta.Token closeBracket = closeParen.next;
+    fasta.Token closeParen2 = closeBracket.next;
+    fasta.Token eof = closeParen2.next;
+
+    expect(openParen.endToken, same(closeParen));
+    expect(closeParen.isSynthetic, isTrue);
+    expect(eof.isEof, isTrue);
   }
 
   @override
+  void test_mismatched_opener() {
+    // When openers and closers are mismatched,
+    // fasta favors considering the opener to be mismatched
+    // and inserts synthetic closers as needed.
+    // `([)` is parsed as `([])` where `]` is synthetic.
+    fasta.BeginGroupToken openParen = _scan('([)');
+    fasta.BeginGroupToken openBracket = openParen.next;
+    fasta.Token closeBracket = openBracket.next; // <-- synthetic
+    fasta.Token closeParen = closeBracket.next;
+    fasta.Token eof = closeParen.next;
+
+    expect(openParen.endToken, same(closeParen));
+    expect(closeParen.isSynthetic, isFalse);
+    expect(openBracket.endToken, same(closeBracket));
+    expect(closeBracket.isSynthetic, isTrue);
+    expect(eof.isEof, isTrue);
+  }
+
+  @override
+  void test_mismatched_opener_in_interpolation() {
+    // When openers and closers are mismatched,
+    // fasta favors considering the opener to be mismatched
+    // and inserts synthetic closers as needed.
+    // r'"${({(}}"' is parsed as r'"${({()})}"'
+    // where both ')' are synthetic
+    var stringStart = _scan(r'"${({(}}"');
+    var interpolationStart = stringStart.next as fasta.BeginGroupToken;
+    var openParen1 = interpolationStart.next as fasta.BeginGroupToken;
+    var openBrace = openParen1.next as fasta.BeginGroupToken;
+    var openParen2 = openBrace.next as fasta.BeginGroupToken;
+    var closeParen2 = openParen2.next;
+    var closeBrace = closeParen2.next;
+    var closeParen1 = closeBrace.next;
+    var interpolationEnd = closeParen1.next;
+    var stringEnd = interpolationEnd.next;
+    var eof = stringEnd.next;
+
+    expect(interpolationStart.endToken, same(interpolationEnd));
+    expect(interpolationEnd.isSynthetic, isFalse);
+    expect(openParen1.endToken, same(closeParen1));
+    expect(closeParen1.isSynthetic, isTrue);
+    expect(openBrace.endToken, same(closeBrace));
+    expect(closeBrace.isSynthetic, isFalse);
+    expect(openParen2.endToken, same(closeParen2));
+    expect(closeParen2.isSynthetic, isTrue);
+    expect(eof.isEof, isTrue);
+  }
+
+  @override
   @failingTest
   void test_string_multi_unterminated() {
     // See defaultRecoveryStrategy recoverString
     super.test_string_multi_unterminated();
   }
 
   @override
   @failingTest
   void test_string_multi_unterminated_interpolation_block() {
     // See defaultRecoveryStrategy recoverString
@@ skipping 63 matching lines @@
     super.test_string_simple_unterminated_interpolation_block();
   }
 
   @override
   @failingTest
   void test_string_simple_unterminated_interpolation_identifier() {
     // See defaultRecoveryStrategy recoverString
     super.test_string_simple_unterminated_interpolation_identifier();
   }
 
-  @failingTest
   @override
   void test_unmatched_openers() {
-    // fasta recovery inserts closers
-    var openBrace = _scan('{[(<') as analyzer.BeginToken;
-    var openBracket = openBrace.next as analyzer.BeginToken;
-    var openParen = openBracket.next as analyzer.BeginToken;
-    var openLT = openParen.next as analyzer.BeginToken;
-    var closeGT = openLT.next;
-    var closeParen = closeGT.next;
+    // fasta inserts missing closers except for '<'
+    var openBrace = _scan('{[(<') as fasta.BeginGroupToken;
+    var openBracket = openBrace.next as fasta.BeginGroupToken;
+    var openParen = openBracket.next as fasta.BeginGroupToken;
+    var openLT = openParen.next as fasta.BeginGroupToken;
+    var closeParen = openLT.next;
     var closeBracket = closeParen.next;
     var closeBrace = closeBracket.next;
-    expect(closeBrace.next.type, analyzer.TokenType.EOF);
+    var eof = closeBrace.next;
 
-    expect(openBrace.endToken, closeBrace);
-    expect(openBracket.endToken, closeBracket);
-    expect(openParen.endToken, closeParen);
-    expect(openLT.endToken, closeGT);
+    expect(openBrace.endGroup, same(closeBrace));
+    expect(openBracket.endGroup, same(closeBracket));
+    expect(openParen.endGroup, same(closeParen));
+    expect(eof.isEof, true);
   }
 
   analyzer.Token _scan(String source,
       {bool genericMethodComments: false,
       bool lazyAssignmentOperators: false}) {
     ErrorListener listener = new ErrorListener();
     analyzer.Token token = scanWithListener(source, listener,
         genericMethodComments: genericMethodComments,
         lazyAssignmentOperators: lazyAssignmentOperators);
     listener.assertNoErrors();
@@ skipping 13 matching lines @@
     }
     if (!token.previousToken.isEof) {
       var head = new fasta.SymbolToken(EOF_INFO, -1);
       token.previous = head;
       head.next = token;
     }
     return token;
   }
 
   /// Assert that the tokens in the stream are correctly connected prev/next.
-  void assertValidTokenStream(fasta.Token firstToken) {
+  void assertValidTokenStream(fasta.Token firstToken,
+      {bool errorsFirst: false}) {
     fasta.Token token = firstToken;
     fasta.Token previous = token.previousToken;
     expect(previous.isEof, isTrue, reason: 'Missing leading EOF');
     expect(previous.next, token, reason: 'Invalid leading EOF');
     expect(previous.previous, previous, reason: 'Invalid leading EOF');
+    if (errorsFirst) {
+      while (!token.isEof && token is fasta.ErrorToken) {
+        token = token.next;
+      }
+    }
+    var isNotErrorToken = isNot(new isInstanceOf<fasta.ErrorToken>());
     while (!token.isEof) {
+      if (errorsFirst) expect(token, isNotErrorToken);
       previous = token;
       token = token.next;
       expect(token, isNotNull, reason: previous.toString());
       expect(token.previous, previous, reason: token.toString());
     }
     expect(token.next, token, reason: 'Invalid trailing EOF');
   }
+
+  /// Assert that all [fasta.BeginGroupToken] has a valid `endGroup`
+  /// that is in the stream.
+  void assertValidBeginTokens(fasta.Token firstToken) {
+    var openerStack = <fasta.BeginGroupToken>[];
+    fasta.BeginGroupToken lastClosedGroup;
+    fasta.Token token = firstToken;
+    while (!token.isEof) {
+      if (token is fasta.BeginGroupToken) {
+        if (token.lexeme != '<')
+          expect(token.endGroup, isNotNull, reason: token.lexeme);
+        if (token.endGroup != null) openerStack.add(token);
+      } else if (openerStack.isNotEmpty && openerStack.last.endGroup == token) {
+        lastClosedGroup = openerStack.removeLast();
+        expect(token.isSynthetic, token.next is fasta.UnmatchedToken,
+            reason: 'Expect synthetic closer then error token');
+      } else if (token is fasta.UnmatchedToken) {
+        expect(lastClosedGroup?.endGroup?.next, same(token),
+            reason: 'Unexpected error token for group: $lastClosedGroup');
+        expect(token.begin, lastClosedGroup);
+      }
+      token = token.next;
+    }
+    expect(openerStack, isEmpty, reason: 'Missing closers');
+  }
 }