move comment tokens into preceedingComments field

pkg/front_end/lib/src/fasta/analyzer/token_utils.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.analyzer.token_utils;
 
 import '../parser/error_kind.dart' show ErrorKind;
 
 import '../scanner/error_token.dart' show ErrorToken;
 
@@ skipping 36 matching lines @@
 /// analyzer does.  This seems like it will complicate parsing and other
 /// operations.
 class ToAnalyzerTokenStreamConverter {
   /// Synthetic token pointing to the first token in the analyzer token stream.
   analyzer.Token _analyzerTokenHead;
 
   /// The most recently generated analyzer token, or [_analyzerTokenHead] if no
   /// tokens have been generated yet.
   analyzer.Token _analyzerTokenTail;
 
-  /// If a sequence of consecutive comment tokens is being processed, the first
-  /// translated analyzer comment token.  Otherwise `null`.
-  analyzer.CommentToken _currentCommentHead;
-
-  /// If a sequence of consecutive comment tokens is being processed, the last
-  /// translated analyzer comment token.  Otherwise `null`.
-  analyzer.CommentToken _currentCommentTail;
-
   /// Stack of analyzer "begin" tokens which need to be linked up to
   /// corresponding "end" tokens once those tokens are translated.
   ///
   /// The first element of this list is always a sentinel `null` value so that
   /// we don't have to check if it is empty.
   ///
   /// See additional documentation in [_matchGroups].
   List<analyzer.BeginToken> _beginTokenStack;
 
   /// Stack of fasta "end" tokens corresponding to the tokens in
   /// [_endTokenStack].
   ///
   /// The first element of this list is always a sentinel `null` value so that
   /// we don't have to check if it is empty.
   ///
   /// See additional documentation in [_matchGroups].
   List<Token> _endTokenStack;
 
   /// Converts a stream of Fasta tokens (starting with [token] and continuing to
   /// EOF) to a stream of analyzer tokens.
   analyzer.Token convertTokens(Token token) {
     _analyzerTokenHead = new analyzer.Token(TokenType.EOF, -1);
     _analyzerTokenHead.previous = _analyzerTokenHead;
     _analyzerTokenTail = _analyzerTokenHead;
-    _currentCommentHead = null;
-    _currentCommentTail = null;
     _beginTokenStack = [null];
     _endTokenStack = <Token>[null];
 
     while (true) {
       if (token.info.kind == BAD_INPUT_TOKEN) {
         ErrorToken errorToken = token;
         _translateErrorToken(errorToken);
-      } else if (token.info.kind == COMMENT_TOKEN) {
-        var translatedToken = translateCommentToken(token);
-        if (_currentCommentHead == null) {
-          _currentCommentHead = _currentCommentTail = translatedToken;
-        } else {
-          _currentCommentTail.setNext(translatedToken);
-          _currentCommentTail = translatedToken;
-        }
       } else {
-        var translatedToken = translateToken(token, _currentCommentHead);
+        var translatedToken = translateToken(
+            token, translateCommentTokens(token.precedingComments));
         _matchGroups(token, translatedToken);
         translatedToken.setNext(translatedToken);
-        _currentCommentHead = _currentCommentTail = null;
         _analyzerTokenTail.setNext(translatedToken);
         translatedToken.previous = _analyzerTokenTail;
         _analyzerTokenTail = translatedToken;
       }
       if (token.isEof) {
         return _analyzerTokenHead.next;
       }
       token = token.next;
     }
   }
 
   /// Handles an error found during [convertTokens].
   ///
   /// Intended to be overridden by derived classes; by default, does nothing.
   void reportError(analyzer.ScannerErrorCode errorCode, int offset,
       List<Object> arguments) {}
 
-  /// Translates a single fasta comment token to the corresponding analyzer
-  /// token.
-  analyzer.CommentToken translateCommentToken(Token token) {
-    // TODO(paulberry,ahe): It would be nice if the scanner gave us an
-    // easier way to distinguish between the two types of comment.
-    var type = token.value.startsWith('/*')
-        ? TokenType.MULTI_LINE_COMMENT
-        : TokenType.SINGLE_LINE_COMMENT;
-    return new analyzer.CommentToken(type, token.value, token.charOffset);
+  /// Translates a sequence of fasta comment tokens to the corresponding
+  /// analyzer tokens.
+  analyzer.CommentToken translateCommentTokens(Token token) {
+    analyzer.CommentToken translateOneComment(Token token) {
+      // TODO(paulberry,ahe): It would be nice if the scanner gave us an
+      // easier way to distinguish between the two types of comment.
+      var type = token.value.startsWith('/*')
+          ? TokenType.MULTI_LINE_COMMENT
+          : TokenType.SINGLE_LINE_COMMENT;
+      return new analyzer.CommentToken(type, token.value, token.charOffset);
+    }
+
+    analyzer.CommentToken head;
+    if (token != null) {
+      head = translateOneComment(token);
+      analyzer.CommentToken tail = head;
+      token = token.next;
+      while (token != null) {
+        tail = tail.setNext(translateOneComment(token));
+        token = token.next;
+      }
+    }
+    return head;
   }
 
   /// Translates a single fasta non-comment token to the corresponding analyzer
   /// token.
   ///
   /// [precedingComments] is not `null`, the translated token is pointed to it.
   analyzer.Token translateToken(
           Token token, analyzer.CommentToken precedingComments) =>
       toAnalyzerToken(token, precedingComments);
 
@@ skipping 123 matching lines @@
     // tokens as appropriate.
     if (translatedToken is BeginGroupToken &&
         analyzerToken is analyzer.BeginToken &&
         analyzerToken.endToken != null) {
       angleBracketStack.clear();
       beginTokenStack.add(translatedToken);
       endTokenStack.add(analyzerToken.endToken);
     }
   }
 
+  Token translateComments(analyzer.Token token) {
+    if (token == null) {
+      return null;
+    }
+    Token head = fromAnalyzerToken(token);
+    Token tail = head;
+    token = token.next;
+    while (token != null) {
+      tail.next = fromAnalyzerToken(token);
+      tail = tail.next;
+      token = token.next;
+    }
+    return head;
+  }
+
   analyzer.Token translateAndAppend(analyzer.Token analyzerToken) {
     var token = fromAnalyzerToken(analyzerToken);
+    token.precedingComments =
+        translateComments(analyzerToken.precedingComments);
     tokenTail.next = token;
     tokenTail = token;
     matchGroups(analyzerToken, token);
     return analyzerToken.next;
   }
 
   while (true) {
-    analyzer.Token commentToken = analyzerToken.precedingComments;
-    while (commentToken != null) {
-      commentToken = translateAndAppend(commentToken);
-    }
     // TODO(paulberry): join up begingroup/endgroup.
     if (analyzerToken.type == TokenType.EOF) {
       tokenTail.next = new SymbolToken(EOF_INFO, analyzerToken.offset);
+      tokenTail.next.precedingComments =
+          translateComments(analyzerToken.precedingComments);
       return tokenHead.next;
     }
     analyzerToken = translateAndAppend(analyzerToken);
   }
 }
 
 /// Converts a single analyzer token into a Fasta token.
 Token fromAnalyzerToken(analyzer.Token token) {
   Token beginGroup(PrecedenceInfo info) =>
       new BeginGroupToken(info, token.offset);
@@ skipping 458 matching lines @@
     case PERIOD_PERIOD_PERIOD_TOKEN:
       return TokenType.PERIOD_PERIOD_PERIOD;
     // case GENERIC_METHOD_TYPE_LIST_TOKEN:
     //   return TokenType.GENERIC_METHOD_TYPE_LIST;
     // case GENERIC_METHOD_TYPE_ASSIGN_TOKEN:
     //   return TokenType.GENERIC_METHOD_TYPE_ASSIGN;
     default:
       return internalError("Unhandled token ${token.info}");
   }
 }