Comment improvements

pkg/front_end/lib/src/fasta/analyzer/token_utils.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.analyzer.token_utils;
 
 import 'package:front_end/src/fasta/parser/error_kind.dart' show
     ErrorKind;
 
 import 'package:front_end/src/fasta/scanner/error_token.dart' show
@@ skipping 45 matching lines @@
         List<Object> arguments)) {
   var analyzerTokenHead = new analyzer.Token(null, 0);
   analyzerTokenHead.previous = analyzerTokenHead;
   var analyzerTokenTail = analyzerTokenHead;
   // TODO(paulberry,ahe): Fasta includes comments directly in the token
   // stream, rather than pointing to them via a "precedingComment" pointer, as
   // analyzer does.  This seems like it will complicate parsing and other
   // operations.
   analyzer.CommentToken currentCommentHead;
   analyzer.CommentToken currentCommentTail;
+
+  // Both fasta and analyzer have links from a "BeginToken" to its matching
+  // "EndToken" in a group (like parentheses and braces).  However, fasta may
+  // contain synthetic tokens from error recovery that are not mapped to the
+  // analyzer token stream.  We use these stacks to create the appropriate links
+  // for non-synthetic tokens in the way analyzer expects.
+
   // Note: beginTokenStack and endTokenStack are seeded with a sentinel value
   // so that we don't have to check if they're empty.
   var beginTokenStack = <analyzer.BeginToken>[null];
   var endTokenStack = <Token>[null];
+
   void matchGroups(Token token, analyzer.Token translatedToken) {
-    // If this token closes a group, set the corresponding opener token
-    // to point to it.
     if (identical(endTokenStack.last, token)) {
       beginTokenStack.last.endToken = translatedToken;
       beginTokenStack.removeLast();
       endTokenStack.removeLast();
     }
-    // If this token opens a group, and there is a matching closer that's not
-    // synthetic, put it on the stack.  The easiest way to tell whether the
-    // closer is synthetic is to see if it has the same offset as the opener.
+    // Synthetic end tokens use the same offset as the begin token.
     if (translatedToken is analyzer.BeginToken &&
         token is BeginGroupToken &&
         token.endGroup != null &&
         token.endGroup.charOffset != token.charOffset) {
       beginTokenStack.add(translatedToken);
       endTokenStack.add(token.endGroup);
     }
   }
 
   while (true) {
@@ skipping 32 matching lines @@
 
 /// Converts a stream of Analyzer tokens (starting with [token] and continuing
 /// to EOF) to a stream of Fasta tokens.
 ///
 /// TODO(paulberry): Analyzer tokens do not record error conditions, so a round
 /// trip through this function and [toAnalyzerTokenStream] will lose error
 /// information.
 Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) {
   Token tokenHead = new SymbolToken(EOF_INFO, -1);
   Token tokenTail = tokenHead;
+
+  // Both fasta and analyzer have links from a "BeginToken" to its matching
+  // "EndToken" in a group (like parentheses and braces).  However, only fasta
+  // makes these links for angle brackets.  We use these stacks to map the
+  // links from the analyzer token stream into equivalent links in the fasta
+  // token stream, and to create the links that fasta expects for angle
+  // brackets.
+
   // Note: beginTokenStack and endTokenStack are seeded with a sentinel value
   // so that we don't have to check if they're empty.
   var beginTokenStack = <BeginGroupToken>[null];
   var endTokenStack = <analyzer.Token>[null];
   var angleBracketStack = <BeginGroupToken>[];
   void matchGroups(analyzer.Token analyzerToken, Token translatedToken) {
-    // If this token closes a group, set the corresponding opener token to point
-    // to it.
     if (identical(endTokenStack.last, analyzerToken)) {
       angleBracketStack.clear();
       beginTokenStack.last.endGroup = translatedToken;
       beginTokenStack.removeLast();
       endTokenStack.removeLast();
     } else if (translatedToken.info.kind == LT_TOKEN) {
       BeginGroupToken beginGroupToken = translatedToken;
       angleBracketStack.add(beginGroupToken);
     } else if (translatedToken.info.kind == GT_TOKEN &&
         angleBracketStack.isNotEmpty) {
       angleBracketStack.removeLast().endGroup = translatedToken;
     } else if (translatedToken.info.kind == GT_GT_TOKEN &&
         angleBracketStack.isNotEmpty) {
       angleBracketStack.removeLast();
       if (angleBracketStack.isNotEmpty) {
         angleBracketStack.removeLast().endGroup = translatedToken;
       }
     }
-    // If this token opens a group, and there is a matching closer, put it on
-    // the stack.
     // TODO(paulberry): generate synthetic closer tokens and "UnmatchedToken"
     // tokens as appropriate.
     if (translatedToken is BeginGroupToken &&
         analyzerToken is analyzer.BeginToken &&
         analyzerToken.endToken != null) {
       angleBracketStack.clear();
       beginTokenStack.add(translatedToken);
       endTokenStack.add(analyzerToken.endToken);
     }
   }
@@ skipping 477 matching lines @@
     case BACKSLASH_TOKEN: return TokenType.BACKSLASH;
     case PERIOD_PERIOD_PERIOD_TOKEN: return TokenType.PERIOD_PERIOD_PERIOD;
     // case GENERIC_METHOD_TYPE_LIST_TOKEN:
     //   return TokenType.GENERIC_METHOD_TYPE_LIST;
     // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN:
     //   return TokenType.GENERIC_METHOD_TYPE_ASSIGN;
     default:
       return internalError("Unhandled token ${token.info}");
   }
 }