Clone and link all tokens in AstCloner.

pkg/analyzer/lib/src/dart/ast/utilities.dart

Index: pkg/analyzer/lib/src/dart/ast/utilities.dart
diff --git a/pkg/analyzer/lib/src/dart/ast/utilities.dart b/pkg/analyzer/lib/src/dart/ast/utilities.dart
index 1a217462a9f09140ec52de78c13e86a84f3dbc86..33bc8213b9809dc10a2a4335f047ed09ae7ebe58 100644
--- a/pkg/analyzer/lib/src/dart/ast/utilities.dart
+++ b/pkg/analyzer/lib/src/dart/ast/utilities.dart
@@ -29,12 +29,32 @@ class AstCloner implements AstVisitor<AstNode> {
   final bool cloneTokens;
 
   /**
+   * Mapping from original tokes to cloned.
+   */
+  final Map<Token, Token> _clonedTokens = new Map<Token, Token>.identity();
+
+  /**
+   * The next original token to clone.
+   */
+  Token _nextToClone;
+
+  /**
+   * The last cloned token.
+   */
+  Token _lastCloned;
+
+  /**
+   * The offset of the last cloned token.
+   */
+  int _lastClonedOffset = -1;
+
+  /**
    * Initialize a newly created AST cloner to optionally clone tokens while
    * cloning AST nodes if [cloneTokens] is `true`.
+   *
+   * TODO(brianwilkerson) Change this to be a named parameter.
    */
-  AstCloner(
-      [this.cloneTokens =
-          false]); // TODO(brianwilkerson) Change this to be a named parameter.
+  AstCloner([this.cloneTokens = false]);
 
   /**
    * Return a clone of the given [node].
@@ -64,7 +84,15 @@ class AstCloner implements AstVisitor<AstNode> {
    */
   Token cloneToken(Token token) {
     if (cloneTokens) {
-      return (token == null ? null : token.copy());
+      if (token == null) {
+        return null;
+      }
+      if (_lastClonedOffset <= token.offset) {
+        _cloneTokens(_nextToClone ?? token, token.offset);
+      }
+      Token clone = _clonedTokens[token];
+      assert(clone != null);
+      return clone;
     } else {
       return token;
     }
@@ -75,7 +103,7 @@ class AstCloner implements AstVisitor<AstNode> {
    */
   List<Token> cloneTokenList(List<Token> tokens) {
     if (cloneTokens) {
-      return tokens.map((Token token) => token.copy()).toList();
+      return tokens.map(cloneToken).toList();
     }
     return tokens;
   }
@@ -185,7 +213,9 @@ class AstCloner implements AstVisitor<AstNode> {
   }
 
   @override
-  ClassTypeAlias visitClassTypeAlias(ClassTypeAlias node) => new ClassTypeAlias(
+  ClassTypeAlias visitClassTypeAlias(ClassTypeAlias node) {
+    cloneToken(node.abstractKeyword);
+    return new ClassTypeAlias(
       cloneNode(node.documentationComment),
       cloneNodeList(node.metadata),
       cloneToken(node.typedefKeyword),
@@ -197,6 +227,7 @@ class AstCloner implements AstVisitor<AstNode> {
       cloneNode(node.withClause),
       cloneNode(node.implementsClause),
       cloneToken(node.semicolon));
+  }
 
   @override
   Comment visitComment(Comment node) {
@@ -216,12 +247,16 @@ class AstCloner implements AstVisitor<AstNode> {
 
   @override
   CompilationUnit visitCompilationUnit(CompilationUnit node) {
+    ScriptTag scriptTag = cloneNode(node.scriptTag);
+    List<Directive> directives = cloneNodeList(node.directives);
+    List<CompilationUnitMember> declarations = cloneNodeList(node.declarations);
+    Token endToken = cloneToken(node.endToken);
+    Token beginToken = scriptTag?.beginToken ??
+        (directives.isEmpty ? null : directives.first.beginToken) ??
+        (declarations.isEmpty ? null : declarations.first.beginToken) ??
+        endToken;
     CompilationUnit clone = new CompilationUnit(
-        cloneToken(node.beginToken),
-        cloneNode(node.scriptTag),
-        cloneNodeList(node.directives),
-        cloneNodeList(node.declarations),
-        cloneToken(node.endToken));
+        beginToken, scriptTag, directives, declarations, endToken);
     clone.lineInfo = node.lineInfo;
     return clone;
   }
@@ -887,6 +922,54 @@ class AstCloner implements AstVisitor<AstNode> {
       cloneToken(node.semicolon));
 
   /**
+   * Clone all token starting from the given [token] up to a token that has
+   * offset greater then [stopAfter], and put mapping from originals to clones
+   * into [_clonedTokens].
+   *
+   * We cannot clone tokens as we visit nodes because not every token is a part
+   * of a node, E.g. commas in argument lists are not represented in AST. But
+   * we need to the sequence of tokens that is identical to the original one.
+   */
+  void _cloneTokens(Token token, int stopAfter) {
+    if (token == null) {
+      return;
+    }
+    if (token is CommentToken) {
+      token = (token as CommentToken).parent;
+    }
+    while (token != null) {
+      Token clone = token.copy();
+      {
+        CommentToken c1 = token.precedingComments;
+        CommentToken c2 = clone.precedingComments;
+        while (c1 != null && c2 != null) {
+          _clonedTokens[c1] = c2;
+          if (c1 is DocumentationCommentToken &&
+              c2 is DocumentationCommentToken) {
+            for (int i = 0; i < c1.references.length; i++) {
+              _clonedTokens[c1.references[i]] = c2.references[i];
+            }
+          }
+          c1 = c1.next;
+          c2 = c2.next;
+        }
+      }
+      _clonedTokens[token] = clone;
+      _lastCloned?.setNext(clone);
+      _lastCloned = clone;
+      if (token.type == TokenType.EOF) {
+        break;
+      }
+      if (token.offset > stopAfter) {
+        _nextToClone = token.next;
+        _lastClonedOffset = token.offset;
+        break;
+      }
+      token = token.next;
+    }
+  }
+
+  /**
    * Return a clone of the given [node].
    */
   static AstNode clone(AstNode node) {