move synthetic fasta closers into the token stream

pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.scanner.array_based_scanner;
 
-import 'error_token.dart' show ErrorToken;
+import 'error_token.dart' show ErrorToken, UnmatchedToken;
 
 import 'keyword.dart' show Keyword;
 
 import 'precedence.dart' show PrecedenceInfo;
 
 import 'token.dart'
     show
         BeginGroupToken,
         CommentToken,
         DartDocToken,
         KeywordToken,
         StringToken,
         SymbolToken,
+        SyntheticSymbolToken,
         Token;
 
 import 'token_constants.dart'
     show
         LT_TOKEN,
         OPEN_CURLY_BRACKET_TOKEN,
         OPEN_PAREN_TOKEN,
         STRING_INTERPOLATION_TOKEN;
 
 import 'characters.dart' show $LF, $STX;
 
-import 'abstract_scanner.dart' show AbstractScanner;
+import 'abstract_scanner.dart' show AbstractScanner, closeBraceInfoFor;
 
 import '../util/link.dart' show Link;
 
 abstract class ArrayBasedScanner extends AbstractScanner {
   bool hasErrors = false;
 
   ArrayBasedScanner(bool includeComments, {int numberOfBytesHint})
       : super(includeComments, numberOfBytesHint: numberOfBytesHint);
 
   /**
@@ skipping 269 matching lines @@
    * In other words, this method is called when the scanner recognizes
    * something which cannot possibly be part of a type parameter/argument
    * list, like the '=' in the above example.
    */
   void discardOpenLt() {
     while (!groupingStack.isEmpty &&
         identical(groupingStack.head.kind, LT_TOKEN)) {
       groupingStack = groupingStack.tail;
     }
   }
+
+  void unmatchedBeginGroup(BeginGroupToken begin) {
+    // We want to ensure that unmatched BeginGroupTokens are reported as
+    // errors.  However, the diet parser assumes that groups are well-balanced
+    // and will never look at the endGroup token.  This is a nice property that
+    // allows us to skip quickly over correct code. By inserting an additional
+    // synthetic token in the stream, we can keep ignoring endGroup tokens.
+    //
+    // [begin] --next--> [tail]
+    // [begin] --endG--> [synthetic] --next--> [next] --next--> [tail]
+    //
+    // This allows the diet parser to skip from [begin] via endGroup to
+    // [synthetic] and ignore the [synthetic] token (assuming it's correct),
+    // then the error will be reported when parsing the [next] token.
+    //
+    // For example, tokenize("{[1};") produces:
+    //
+    // SymbolToken({) --endGroup------------------------+
+    //      |                                           |
+    //     next                                         |
+    //      v                                           |
+    // SymbolToken([) --endGroup--+                     |
+    //      |                     |                     |
+    //     next                   |                     |
+    //      v                     |                     |
+    // StringToken(1)             |                     |
+    //      |                     |                     |
+    //     next                   |                     |
+    //      v                     |                     |
+    // SymbolToken(])<------------+ <-- Synthetic token |
+    //      |                                           |
+    //     next                                         |
+    //      v                                           |
+    // UnmatchedToken([)                                |
+    //      |                                           |
+    //     next                                         |
+    //      v                                           |
+    // SymbolToken(})<----------------------------------+
+    //      |
+    //     next
+    //      v
+    // SymbolToken(;)
+    //      |
+    //     next
+    //      v
+    //     EOF
+    PrecedenceInfo info = closeBraceInfoFor(begin);
+    appendToken(new SyntheticSymbolToken(info, tokenStart));
+    begin.endGroup = tail;
+    appendErrorToken(new UnmatchedToken(begin));
+  }
 }