When translating analyzer/fasta token streams, match up begin/end tokens.

pkg/front_end/lib/src/fasta/analyzer/token_utils.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.analyzer.token_utils;
 
 import 'package:front_end/src/fasta/parser/error_kind.dart' show
     ErrorKind;
 
 import 'package:front_end/src/fasta/scanner/error_token.dart' show
-    ErrorToken;
+    ErrorToken,
+    UnmatchedToken;
 
 import 'package:front_end/src/fasta/scanner/keyword.dart' show
     Keyword;
 
 import 'package:front_end/src/fasta/scanner/precedence.dart';
 
 import 'package:front_end/src/fasta/scanner/token.dart' show
     BeginGroupToken,
     KeywordToken,
     StringToken,
     SymbolToken,
     Token;
 
 import 'package:front_end/src/fasta/scanner/token_constants.dart';
 
 import 'package:front_end/src/scanner/token.dart' as analyzer show
+    BeginToken,
+    BeginTokenWithComment,
     CommentToken,
     Keyword,
     KeywordToken,
     KeywordTokenWithComment,
     StringToken,
     StringTokenWithComment,
     Token,
     TokenWithComment;
 
 import 'package:front_end/src/scanner/errors.dart' as analyzer show
@@ skipping 16 matching lines @@
         List<Object> arguments)) {
   var analyzerTokenHead = new analyzer.Token(null, 0);
   analyzerTokenHead.previous = analyzerTokenHead;
   var analyzerTokenTail = analyzerTokenHead;
   // TODO(paulberry,ahe): Fasta includes comments directly in the token
   // stream, rather than pointing to them via a "precedingComment" pointer, as
   // analyzer does.  This seems like it will complicate parsing and other
   // operations.
   analyzer.CommentToken currentCommentHead;
   analyzer.CommentToken currentCommentTail;
+  // Note: beginTokenStack and endTokenStack are seeded with a sentinel value
+  // so that we don't have to check if they're empty.
+  var beginTokenStack = <analyzer.BeginToken>[null];
+  var endTokenStack = <Token>[null];
+  void matchGroups(Token token, analyzer.Token translatedToken) {
+    // If this token closes a group, set the corresponding opener token

[Siggi Cherem (dart-lang), 2017/02/15 00:23:10]

super optional nit - (IMO not worth making a change for this).

I'd consider moving some of these algorithm comments out above beginTokenStack
and turn them into some general technique documentation as a whole. In other
words, right now I read this as a description of the implementation, but it's
not as obvious what's the underlying goal.

On a related note, we could also introduce some helper variables which may make
the code more self-explanatory further below. For example:

// Both fasta and analyzer have links from a `BeginToken` to 
// it's matching `EndToken` in a group (like parenthesis and braces).
// However, fasta may contain synthetic tokens from error-recovery 
// that are not mapped to the analyzer token stream. We use these
// stacks to create the appropriate links for non-synthetic tokens in
// the way analyzer expects them to be.

// Note: null is a sentinel value to avoid empty checks below.
var beginTokenStack = <analyzer.BeginToken>[null];
var endTokenStack = <Token>[null];

void matchGroups(Token token, analyzer.Token translatedToken) {
  if (identical(endTokenStack.last, token)) {
    beginTokenStack.last.endToken = translatedToken;
    beginTokenStack.removeLast();
    endTokenStack.removeLast();
  }

  if (translatedToken is analyzer.BeginToken && token is BeginGroupToken) {
    // Synthetic end-tokens use the same offset as the begin token.
    var isSynthetic = token.endGroup?.charOffset = token.charOffset;
    if (!isSynthetic) {
      beginStack.add(translatedToken);
      endStack.add(token.endGroup);
    }
  } 

+    // to point to it.
+    if (identical(endTokenStack.last, token)) {
+      beginTokenStack.last.endToken = translatedToken;
+      beginTokenStack.removeLast();
+      endTokenStack.removeLast();
+    }
+    // If this token opens a group, and there is a matching closer that's not

[Siggi Cherem (dart-lang), 2017/02/15 00:23:10]

minor terminology nit here: consider using closing-token, end-token,. or even
closer-token, instead of plain `closer`. For a flip-second I started thinking of
closer in the sense of distance (as opposed to farther), which was a possible
interpretation given that we are also looking into charOffsets

+    // synthetic, put it on the stack.  The easiest way to tell whether the
+    // closer is synthetic is to see if it has the same offset as the opener.
+    if (translatedToken is analyzer.BeginToken && token is BeginGroupToken &&
+        token.endGroup != null && token.endGroup.charOffset != token.charOffset) {

[ahe, 2017/02/14 11:28:32]

Long line.

[Paul Berry, 2017/02/14 12:40:39]

Sorry.  My usual workflow is hampered by not being able to run dartfmt on this
file (usually I edit files in an editor with wide margins, without regard to
line length, and then run dartfmt in bulk prior to sending out for review). 
Hopefully we will run dartfmt on the fasta codebase soon and then that won't be
an issue anymore.  Fixed.

+      beginTokenStack.add(translatedToken);
+      endTokenStack.add(token.endGroup);
+    }
+  }
+
   while (true) {
     if (token.info.kind == BAD_INPUT_TOKEN) {
       ErrorToken errorToken = token;
       _translateErrorToken(errorToken, reportError);
     } else if (token.info.kind == COMMENT_TOKEN) {
       // TODO(paulberry,ahe): It would be nice if the scanner gave us an
       // easier way to distinguish between the two types of comment.
       var type = token.value.startsWith('/*')
           ? TokenType.MULTI_LINE_COMMENT
           : TokenType.SINGLE_LINE_COMMENT;
       var translatedToken =
           new analyzer.CommentToken(type, token.value, token.charOffset);
       if (currentCommentHead == null) {
         currentCommentHead = currentCommentTail = translatedToken;
       } else {
         currentCommentTail.setNext(translatedToken);
         currentCommentTail = translatedToken;
       }
     } else {
       var translatedToken = toAnalyzerToken(token, currentCommentHead);
+      matchGroups(token, translatedToken);
       translatedToken.setNext(translatedToken);
       currentCommentHead = currentCommentTail = null;
       analyzerTokenTail.setNext(translatedToken);
       translatedToken.previous = analyzerTokenTail;
       analyzerTokenTail = translatedToken;
     }
     if (token.isEof) {
       return analyzerTokenHead.next;
     }
     token = token.next;
   }
 }
 
 /// Converts a stream of Analyzer tokens (starting with [token] and continuing
 /// to EOF) to a stream of Fasta tokens.
 ///
 /// TODO(paulberry): Analyzer tokens do not record error conditions, so a round
 /// trip through this function and [toAnalyzerTokenStream] will lose error
 /// information.
 Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) {
   Token tokenHead = new SymbolToken(EOF_INFO, -1);
   Token tokenTail = tokenHead;
+  // Note: beginTokenStack and endTokenStack are seeded with a sentinel value
+  // so that we don't have to check if they're empty.
+  var beginTokenStack = <BeginGroupToken>[null];
+  var endTokenStack = <analyzer.Token>[null];
+  void matchGroups(analyzer.Token analyzerToken, Token translatedToken) {
+    // If this token closes a group, set the corresponding opener token to point
+    // to it.
+    if (identical(endTokenStack.last, analyzerToken)) {
+      beginTokenStack.last.endGroup = translatedToken;
+      beginTokenStack.removeLast();
+      endTokenStack.removeLast();
+    }
+    // If this token opens a group, and there is a matching closer, put it on
+    // the stack.
+    // TODO(paulberry): generate synthetic closer tokens and "UnmatchedToken"
+    // tokens as appropriate.
+    // TODO(paulberry): match up "<" and ">"/">>" (analyzer doesn't match
+    // these).
+    if (translatedToken is BeginGroupToken && analyzerToken is analyzer.BeginToken &&

[ahe, 2017/02/14 11:28:32]

Long line.

[Paul Berry, 2017/02/14 12:40:39]

Done.

+        analyzerToken.endToken != null) {
+      beginTokenStack.add(translatedToken);
+      endTokenStack.add(analyzerToken.endToken);
+    }
+  }
   analyzer.Token translateAndAppend(analyzer.Token analyzerToken) {
     var token = fromAnalyzerToken(analyzerToken);
     tokenTail.next = token;
     tokenTail = token;
+    matchGroups(analyzerToken, token);
     return analyzerToken.next;
   }
 
   while (true) {
     analyzer.Token commentToken = analyzerToken.precedingComments;
     while (commentToken != null) {
       commentToken = translateAndAppend(commentToken);
     }
     // TODO(paulberry): join up begingroup/endgroup.
     if (analyzerToken.type == TokenType.EOF) {
@@ skipping 253 matching lines @@
   if (token == null) return null;
   analyzer.Token makeStringToken(TokenType tokenType) {
     if (commentToken == null) {
       return new analyzer.StringToken(tokenType, token.value, token.charOffset);
     } else {
       return new analyzer.StringTokenWithComment(
           tokenType, token.value, token.charOffset, commentToken);
     }
   }
 
+  analyzer.Token makeBeginToken(TokenType tokenType) {
+    if (commentToken == null) {
+      return new analyzer.BeginToken(tokenType, token.charOffset);
+    } else {
+      return new analyzer.BeginTokenWithComment(tokenType, token.charOffset, commentToken);

[ahe, 2017/02/14 11:28:32]

Long line.

[Paul Berry, 2017/02/14 12:40:39]

Done.

+    }
+  }
+
   switch (token.kind) {
     case DOUBLE_TOKEN:
       return makeStringToken(TokenType.DOUBLE);
 
     case HEXADECIMAL_TOKEN:
       return makeStringToken(TokenType.HEXADECIMAL);
 
     case IDENTIFIER_TOKEN:
       return makeStringToken(TokenType.IDENTIFIER);
 
@@ skipping 23 matching lines @@
         return new analyzer.KeywordToken(keyword, token.charOffset);
       } else {
         return new analyzer.KeywordTokenWithComment(
             keyword, token.charOffset, commentToken);
       }
       break;
 
     case STRING_TOKEN:
       return makeStringToken(TokenType.STRING);
 
+    case OPEN_CURLY_BRACKET_TOKEN:
+    case OPEN_SQUARE_BRACKET_TOKEN:
+    case OPEN_PAREN_TOKEN:
+    case STRING_INTERPOLATION_TOKEN:
+      return makeBeginToken(getTokenType(token));
+
     default:
       if (commentToken == null) {
         return new analyzer.Token(getTokenType(token), token.charOffset);
       } else {
         return new analyzer.TokenWithComment(
             getTokenType(token), token.charOffset, commentToken);
       }
       break;
   }
 }
@@ skipping 130 matching lines @@
     case BACKSLASH_TOKEN: return TokenType.BACKSLASH;
     case PERIOD_PERIOD_PERIOD_TOKEN: return TokenType.PERIOD_PERIOD_PERIOD;
     // case GENERIC_METHOD_TYPE_LIST_TOKEN:
     //   return TokenType.GENERIC_METHOD_TYPE_LIST;
     // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN:
     //   return TokenType.GENERIC_METHOD_TYPE_ASSIGN;
     default:
       return internalError("Unhandled token ${token.info}");
   }
 }