When translating analyzer/fasta token streams, match up begin/end tokens.

pkg/front_end/lib/src/fasta/analyzer/token_utils.dart

Index: pkg/front_end/lib/src/fasta/analyzer/token_utils.dart
diff --git a/pkg/front_end/lib/src/fasta/analyzer/token_utils.dart b/pkg/front_end/lib/src/fasta/analyzer/token_utils.dart
index 7907177338dffc53ff39de207d5669fae343a1a3..3bca4ece59856ff1eaada39876a1982d88d72876 100644
--- a/pkg/front_end/lib/src/fasta/analyzer/token_utils.dart
+++ b/pkg/front_end/lib/src/fasta/analyzer/token_utils.dart
@@ -8,7 +8,8 @@ import 'package:front_end/src/fasta/parser/error_kind.dart' show
     ErrorKind;
 
 import 'package:front_end/src/fasta/scanner/error_token.dart' show
-    ErrorToken;
+    ErrorToken,
+    UnmatchedToken;
 
 import 'package:front_end/src/fasta/scanner/keyword.dart' show
     Keyword;
@@ -25,6 +26,8 @@ import 'package:front_end/src/fasta/scanner/token.dart' show
 import 'package:front_end/src/fasta/scanner/token_constants.dart';
 
 import 'package:front_end/src/scanner/token.dart' as analyzer show
+    BeginToken,
+    BeginTokenWithComment,
     CommentToken,
     Keyword,
     KeywordToken,
@@ -61,6 +64,28 @@ analyzer.Token toAnalyzerTokenStream(
   // operations.
   analyzer.CommentToken currentCommentHead;
   analyzer.CommentToken currentCommentTail;
+  // Note: beginTokenStack and endTokenStack are seeded with a sentinel value
+  // so that we don't have to check if they're empty.
+  var beginTokenStack = <analyzer.BeginToken>[null];
+  var endTokenStack = <Token>[null];
+  void matchGroups(Token token, analyzer.Token translatedToken) {
+    // If this token closes a group, set the corresponding opener token

[Siggi Cherem (dart-lang), 2017/02/15 00:23:10]

super optional nit - (IMO not worth making a change for this).

I'd consider moving some of these algorithm comments out above beginTokenStack
and turn them into some general technique documentation as a whole. In other
words, right now I read this as a description of the implementation, but it's
not as obvious what's the underlying goal.

On a related note, we could also introduce some helper variables which may make
the code more self-explanatory further below. For example:

// Both fasta and analyzer have links from a `BeginToken` to 
// it's matching `EndToken` in a group (like parenthesis and braces).
// However, fasta may contain synthetic tokens from error-recovery 
// that are not mapped to the analyzer token stream. We use these
// stacks to create the appropriate links for non-synthetic tokens in
// the way analyzer expects them to be.

// Note: null is a sentinel value to avoid empty checks below.
var beginTokenStack = <analyzer.BeginToken>[null];
var endTokenStack = <Token>[null];

void matchGroups(Token token, analyzer.Token translatedToken) {
  if (identical(endTokenStack.last, token)) {
    beginTokenStack.last.endToken = translatedToken;
    beginTokenStack.removeLast();
    endTokenStack.removeLast();
  }

  if (translatedToken is analyzer.BeginToken && token is BeginGroupToken) {
    // Synthetic end-tokens use the same offset as the begin token.
    var isSynthetic = token.endGroup?.charOffset = token.charOffset;
    if (!isSynthetic) {
      beginStack.add(translatedToken);
      endStack.add(token.endGroup);
    }
  } 

+    // to point to it.
+    if (identical(endTokenStack.last, token)) {
+      beginTokenStack.last.endToken = translatedToken;
+      beginTokenStack.removeLast();
+      endTokenStack.removeLast();
+    }
+    // If this token opens a group, and there is a matching closer that's not

[Siggi Cherem (dart-lang), 2017/02/15 00:23:10]

minor terminology nit here: consider using closing-token, end-token,. or even
closer-token, instead of plain `closer`. For a flip-second I started thinking of
closer in the sense of distance (as opposed to farther), which was a possible
interpretation given that we are also looking into charOffsets

+    // synthetic, put it on the stack.  The easiest way to tell whether the
+    // closer is synthetic is to see if it has the same offset as the opener.
+    if (translatedToken is analyzer.BeginToken && token is BeginGroupToken &&
+        token.endGroup != null && token.endGroup.charOffset != token.charOffset) {

[ahe, 2017/02/14 11:28:32]

Long line.

[Paul Berry, 2017/02/14 12:40:39]

Sorry.  My usual workflow is hampered by not being able to run dartfmt on this
file (usually I edit files in an editor with wide margins, without regard to
line length, and then run dartfmt in bulk prior to sending out for review). 
Hopefully we will run dartfmt on the fasta codebase soon and then that won't be
an issue anymore.  Fixed.

+      beginTokenStack.add(translatedToken);
+      endTokenStack.add(token.endGroup);
+    }
+  }
+
   while (true) {
     if (token.info.kind == BAD_INPUT_TOKEN) {
       ErrorToken errorToken = token;
@@ -81,6 +106,7 @@ analyzer.Token toAnalyzerTokenStream(
       }
     } else {
       var translatedToken = toAnalyzerToken(token, currentCommentHead);
+      matchGroups(token, translatedToken);
       translatedToken.setNext(translatedToken);
       currentCommentHead = currentCommentTail = null;
       analyzerTokenTail.setNext(translatedToken);
@@ -103,10 +129,35 @@ analyzer.Token toAnalyzerTokenStream(
 Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) {
   Token tokenHead = new SymbolToken(EOF_INFO, -1);
   Token tokenTail = tokenHead;
+  // Note: beginTokenStack and endTokenStack are seeded with a sentinel value
+  // so that we don't have to check if they're empty.
+  var beginTokenStack = <BeginGroupToken>[null];
+  var endTokenStack = <analyzer.Token>[null];
+  void matchGroups(analyzer.Token analyzerToken, Token translatedToken) {
+    // If this token closes a group, set the corresponding opener token to point
+    // to it.
+    if (identical(endTokenStack.last, analyzerToken)) {
+      beginTokenStack.last.endGroup = translatedToken;
+      beginTokenStack.removeLast();
+      endTokenStack.removeLast();
+    }
+    // If this token opens a group, and there is a matching closer, put it on
+    // the stack.
+    // TODO(paulberry): generate synthetic closer tokens and "UnmatchedToken"
+    // tokens as appropriate.
+    // TODO(paulberry): match up "<" and ">"/">>" (analyzer doesn't match
+    // these).
+    if (translatedToken is BeginGroupToken && analyzerToken is analyzer.BeginToken &&

[ahe, 2017/02/14 11:28:32]

Long line.

[Paul Berry, 2017/02/14 12:40:39]

Done.

+        analyzerToken.endToken != null) {
+      beginTokenStack.add(translatedToken);
+      endTokenStack.add(analyzerToken.endToken);
+    }
+  }
   analyzer.Token translateAndAppend(analyzer.Token analyzerToken) {
     var token = fromAnalyzerToken(analyzerToken);
     tokenTail.next = token;
     tokenTail = token;
+    matchGroups(analyzerToken, token);
     return analyzerToken.next;
   }
 
@@ -380,6 +431,14 @@ analyzer.Token toAnalyzerToken(Token token,
     }
   }
 
+  analyzer.Token makeBeginToken(TokenType tokenType) {
+    if (commentToken == null) {
+      return new analyzer.BeginToken(tokenType, token.charOffset);
+    } else {
+      return new analyzer.BeginTokenWithComment(tokenType, token.charOffset, commentToken);

[ahe, 2017/02/14 11:28:32]

Long line.

[Paul Berry, 2017/02/14 12:40:39]

Done.

+    }
+  }
+
   switch (token.kind) {
     case DOUBLE_TOKEN:
       return makeStringToken(TokenType.DOUBLE);
@@ -423,6 +482,12 @@ analyzer.Token toAnalyzerToken(Token token,
     case STRING_TOKEN:
       return makeStringToken(TokenType.STRING);
 
+    case OPEN_CURLY_BRACKET_TOKEN:
+    case OPEN_SQUARE_BRACKET_TOKEN:
+    case OPEN_PAREN_TOKEN:
+    case STRING_INTERPOLATION_TOKEN:
+      return makeBeginToken(getTokenType(token));
+
     default:
       if (commentToken == null) {
         return new analyzer.Token(getTokenType(token), token.charOffset);