cleanup fasta token classes

pkg/front_end/test/scanner_replacement_test.dart

 // Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:convert' show UTF8;
 
 import 'package:front_end/src/fasta/scanner/recover.dart'
     show defaultRecoveryStrategy;
 import 'package:front_end/src/fasta/scanner.dart' as fasta;
 import 'package:front_end/src/fasta/scanner/token.dart' as fasta;
@@ skipping 38 matching lines @@
     assertValidBeginTokens(tokens);
     if (result.hasErrors) {
       List<int> bytes = UTF8.encode(source);
       tokens = defaultRecoveryStrategy(bytes, tokens, result.lineStarts);
       assertValidTokenStream(tokens, errorsFirst: true);
     }
     return extractErrors(tokens, listener);
   }
 
   void _assertOpenClosePair(String source) {
-    fasta.BeginGroupToken open = _scan(source);
+    analyzer.BeginToken open = _scan(source);
     fasta.Token close = open.next;
     expect(close.next.isEof, isTrue);
     expect(open.endGroup, close);
     expect(open.isSynthetic, isFalse);
     expect(close.isSynthetic, isFalse);
   }
 
   void _assertOpenOnly(String source) {
-    fasta.BeginGroupToken open = _scan(source);
+    analyzer.BeginToken open = _scan(source);
     fasta.Token close = open.next;
     expect(close.next.isEof, isTrue);
     expect(open.endGroup, close);
     expect(open.isSynthetic, isFalse);
     expect(close.isSynthetic, isTrue);
   }
 
   void test_lt() {
     // fasta does not automatically insert a closer for '<'
     // because it could be part of an expression rather than an opener
-    fasta.BeginGroupToken lt = _scan('<');
+    analyzer.BeginToken lt = _scan('<');
     expect(lt.next.isEof, isTrue);
     expect(lt.isSynthetic, isFalse);
   }
 
   void test_lt_gt() {
     _assertOpenClosePair('< >');
   }
 
   @override
   void test_open_curly_bracket() {
@@ skipping 20 matching lines @@
     _assertOpenClosePair('[ ]');
   }
 
   @override
   void test_mismatched_closer() {
     // When openers and closers are mismatched,
     // fasta favors considering the opener to be mismatched,
     // and inserts synthetic closers as needed.
     // `(])` is parsed as `()])` where the first `)` is synthetic
     // and the trailing `])` are unmatched.
-    fasta.BeginGroupToken openParen = _scan('(])');
+    analyzer.BeginToken openParen = _scan('(])');
     fasta.Token closeParen = openParen.next;
     fasta.Token closeBracket = closeParen.next;
     fasta.Token closeParen2 = closeBracket.next;
     fasta.Token eof = closeParen2.next;
 
     expect(openParen.endToken, same(closeParen));
     expect(closeParen.isSynthetic, isTrue);
     expect(eof.isEof, isTrue);
   }
 
   @override
   void test_mismatched_opener() {
     // When openers and closers are mismatched,
     // fasta favors considering the opener to be mismatched
     // and inserts synthetic closers as needed.
     // `([)` is parsed as `([])` where `]` is synthetic.
-    fasta.BeginGroupToken openParen = _scan('([)');
-    fasta.BeginGroupToken openBracket = openParen.next;
+    analyzer.BeginToken openParen = _scan('([)');
+    analyzer.BeginToken openBracket = openParen.next;
     fasta.Token closeBracket = openBracket.next; // <-- synthetic
     fasta.Token closeParen = closeBracket.next;
     fasta.Token eof = closeParen.next;
 
     expect(openParen.endToken, same(closeParen));
     expect(closeParen.isSynthetic, isFalse);
     expect(openBracket.endToken, same(closeBracket));
     expect(closeBracket.isSynthetic, isTrue);
     expect(eof.isEof, isTrue);
   }
 
   @override
   void test_mismatched_opener_in_interpolation() {
     // When openers and closers are mismatched,
     // fasta favors considering the opener to be mismatched
     // and inserts synthetic closers as needed.
     // r'"${({(}}"' is parsed as r'"${({()})}"'
     // where both ')' are synthetic
     var stringStart = _scan(r'"${({(}}"');
-    var interpolationStart = stringStart.next as fasta.BeginGroupToken;
-    var openParen1 = interpolationStart.next as fasta.BeginGroupToken;
-    var openBrace = openParen1.next as fasta.BeginGroupToken;
-    var openParen2 = openBrace.next as fasta.BeginGroupToken;
+    var interpolationStart = stringStart.next as analyzer.BeginToken;
+    var openParen1 = interpolationStart.next as analyzer.BeginToken;
+    var openBrace = openParen1.next as analyzer.BeginToken;
+    var openParen2 = openBrace.next as analyzer.BeginToken;
     var closeParen2 = openParen2.next;
     var closeBrace = closeParen2.next;
     var closeParen1 = closeBrace.next;
     var interpolationEnd = closeParen1.next;
     var stringEnd = interpolationEnd.next;
     var eof = stringEnd.next;
 
     expect(interpolationStart.endToken, same(interpolationEnd));
     expect(interpolationEnd.isSynthetic, isFalse);
     expect(openParen1.endToken, same(closeParen1));
@@ skipping 85 matching lines @@
   @override
   @failingTest
   void test_string_simple_unterminated_interpolation_identifier() {
     // See defaultRecoveryStrategy recoverString
     super.test_string_simple_unterminated_interpolation_identifier();
   }
 
   @override
   void test_unmatched_openers() {
     // fasta inserts missing closers except for '<'
-    var openBrace = _scan('{[(<') as fasta.BeginGroupToken;
-    var openBracket = openBrace.next as fasta.BeginGroupToken;
-    var openParen = openBracket.next as fasta.BeginGroupToken;
-    var openLT = openParen.next as fasta.BeginGroupToken;
+    var openBrace = _scan('{[(<') as analyzer.BeginToken;
+    var openBracket = openBrace.next as analyzer.BeginToken;
+    var openParen = openBracket.next as analyzer.BeginToken;
+    var openLT = openParen.next as analyzer.BeginToken;
     var closeParen = openLT.next;
     var closeBracket = closeParen.next;
     var closeBrace = closeBracket.next;
     var eof = closeBrace.next;
 
     expect(openBrace.endGroup, same(closeBrace));
     expect(openBracket.endGroup, same(closeBracket));
     expect(openParen.endGroup, same(closeParen));
     expect(eof.isEof, true);
   }
@@ skipping 14 matching lines @@
     // The default recovery strategy used by scanString
     // places all error tokens at the head of the stream.
     while (token.type == analyzer.TokenType.BAD_INPUT) {
       translateErrorToken(token,
           (ScannerErrorCode errorCode, int offset, List<Object> arguments) {
         listener.errors.add(new TestError(offset, errorCode, arguments));
       });
       token = token.next;
     }
     if (!token.previous.isEof) {
-      var head = new fasta.SymbolToken(analyzer.TokenType.EOF, -1);
+      var head = new analyzer.Token.eof(-1);
       token.previous = head;
       head.next = token;
     }
     return token;
   }
 
   /// Assert that the tokens in the stream are correctly connected prev/next.
   void assertValidTokenStream(fasta.Token firstToken,
       {bool errorsFirst: false}) {
     fasta.Token token = firstToken;
@@ skipping 10 matching lines @@
     while (!token.isEof) {
       if (errorsFirst) expect(token, isNotErrorToken);
       previous = token;
       token = token.next;
       expect(token, isNotNull, reason: previous.toString());
       expect(token.previous, previous, reason: token.toString());
     }
     expect(token.next, token, reason: 'Invalid trailing EOF');
   }
 
-  /// Assert that all [fasta.BeginGroupToken] has a valid `endGroup`
+  /// Assert that all [analyzer.BeginToken] has a valid `endGroup`
   /// that is in the stream.
   void assertValidBeginTokens(fasta.Token firstToken) {
-    var openerStack = <fasta.BeginGroupToken>[];
-    fasta.BeginGroupToken lastClosedGroup;
+    var openerStack = <analyzer.BeginToken>[];
+    analyzer.BeginToken lastClosedGroup;
     fasta.Token token = firstToken;
     while (!token.isEof) {
-      if (token is fasta.BeginGroupToken) {
+      if (token is analyzer.BeginToken) {
         if (token.lexeme != '<')
           expect(token.endGroup, isNotNull, reason: token.lexeme);
         if (token.endGroup != null) openerStack.add(token);
       } else if (openerStack.isNotEmpty && openerStack.last.endGroup == token) {
         lastClosedGroup = openerStack.removeLast();
         expect(token.isSynthetic, token.next is fasta.UnmatchedToken,
-            reason: 'Expect synthetic closer then error token');
+            reason: 'Expect synthetic closer then error token, '
+                'but found "$token" followed by "${token.next}"');
       } else if (token is fasta.UnmatchedToken) {
         expect(lastClosedGroup?.endGroup?.next, same(token),
             reason: 'Unexpected error token for group: $lastClosedGroup');
         expect(token.begin, lastClosedGroup);
       }
       token = token.next;
     }
     expect(openerStack, isEmpty, reason: 'Missing closers');
   }
 }