Clone and link all tokens in AstCloner.

pkg/analyzer/lib/src/dart/ast/utilities.dart

 // Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library analyzer.src.dart.ast.utilities;
 
 import 'dart:collection';
 
 import 'package:analyzer/dart/ast/ast.dart';
 import 'package:analyzer/dart/ast/visitor.dart';
@@ skipping 11 matching lines @@
  * properties associated with the nodes.
  */
 class AstCloner implements AstVisitor<AstNode> {
   /**
    * A flag indicating whether tokens should be cloned while cloning an AST
    * structure.
    */
   final bool cloneTokens;
 
   /**
+   * Mapping from original tokes to cloned.
+   */
+  final Map<Token, Token> _clonedTokens = new Map<Token, Token>.identity();
+
+  /**
+   * The next original token to clone.
+   */
+  Token _nextToClone;
+
+  /**
+   * The last cloned token.
+   */
+  Token _lastCloned;
+
+  /**
+   * The offset of the last cloned token.
+   */
+  int _lastClonedOffset = -1;
+
+  /**
    * Initialize a newly created AST cloner to optionally clone tokens while
    * cloning AST nodes if [cloneTokens] is `true`.
+   *
+   * TODO(brianwilkerson) Change this to be a named parameter.
    */
-  AstCloner(
-      [this.cloneTokens =
-          false]); // TODO(brianwilkerson) Change this to be a named parameter.
+  AstCloner([this.cloneTokens = false]);
 
   /**
    * Return a clone of the given [node].
    */
   AstNode cloneNode(AstNode node) {
     if (node == null) {
       return null;
     }
     return node.accept(this) as AstNode;
   }
 
   /**
    * Return a list containing cloned versions of the nodes in the given list of
    * [nodes].
    */
   List<AstNode> cloneNodeList(NodeList nodes) {
     int count = nodes.length;
     List clonedNodes = new List();
     for (int i = 0; i < count; i++) {
       clonedNodes.add((nodes[i]).accept(this) as AstNode);
     }
     return clonedNodes;
   }
 
   /**
    * Clone the given [token] if tokens are supposed to be cloned.
    */
   Token cloneToken(Token token) {
     if (cloneTokens) {
-      return (token == null ? null : token.copy());
+      if (token == null) {
+        return null;
+      }
+      if (_lastClonedOffset <= token.offset) {
+        _cloneTokens(_nextToClone ?? token, token.offset);
+      }
+      Token clone = _clonedTokens[token];
+      assert(clone != null);
+      return clone;
     } else {
       return token;
     }
   }
 
   /**
    * Clone the given [tokens] if tokens are supposed to be cloned.
    */
   List<Token> cloneTokenList(List<Token> tokens) {
     if (cloneTokens) {
-      return tokens.map((Token token) => token.copy()).toList();
+      return tokens.map(cloneToken).toList();
     }
     return tokens;
   }
 
   @override
   AdjacentStrings visitAdjacentStrings(AdjacentStrings node) =>
       new AdjacentStrings(cloneNodeList(node.strings));
 
   @override
   Annotation visitAnnotation(Annotation node) => new Annotation(
@@ skipping 89 matching lines @@
         cloneNode(node.withClause),
         cloneNode(node.implementsClause),
         cloneToken(node.leftBracket),
         cloneNodeList(node.members),
         cloneToken(node.rightBracket));
     copy.nativeClause = cloneNode(node.nativeClause);
     return copy;
   }
 
   @override
-  ClassTypeAlias visitClassTypeAlias(ClassTypeAlias node) => new ClassTypeAlias(
+  ClassTypeAlias visitClassTypeAlias(ClassTypeAlias node) {
+    cloneToken(node.abstractKeyword);
+    return new ClassTypeAlias(
       cloneNode(node.documentationComment),
       cloneNodeList(node.metadata),
       cloneToken(node.typedefKeyword),
       cloneNode(node.name),
       cloneNode(node.typeParameters),
       cloneToken(node.equals),
       cloneToken(node.abstractKeyword),
       cloneNode(node.superclass),
       cloneNode(node.withClause),
       cloneNode(node.implementsClause),
       cloneToken(node.semicolon));
+  }
 
   @override
   Comment visitComment(Comment node) {
     if (node.isDocumentation) {
       return Comment.createDocumentationCommentWithReferences(
           cloneTokenList(node.tokens), cloneNodeList(node.references));
     } else if (node.isBlock) {
       return Comment.createBlockComment(cloneTokenList(node.tokens));
     }
     return Comment.createEndOfLineComment(cloneTokenList(node.tokens));
   }
 
   @override
   CommentReference visitCommentReference(CommentReference node) =>
       new CommentReference(
           cloneToken(node.newKeyword), cloneNode(node.identifier));
 
   @override
   CompilationUnit visitCompilationUnit(CompilationUnit node) {
+    ScriptTag scriptTag = cloneNode(node.scriptTag);
+    List<Directive> directives = cloneNodeList(node.directives);
+    List<CompilationUnitMember> declarations = cloneNodeList(node.declarations);
+    Token endToken = cloneToken(node.endToken);
+    Token beginToken = scriptTag?.beginToken ??
+        (directives.isEmpty ? null : directives.first.beginToken) ??
+        (declarations.isEmpty ? null : declarations.first.beginToken) ??
+        endToken;
     CompilationUnit clone = new CompilationUnit(
-        cloneToken(node.beginToken),
-        cloneNode(node.scriptTag),
-        cloneNodeList(node.directives),
-        cloneNodeList(node.declarations),
-        cloneToken(node.endToken));
+        beginToken, scriptTag, directives, declarations, endToken);
     clone.lineInfo = node.lineInfo;
     return clone;
   }
 
   @override
   ConditionalExpression visitConditionalExpression(
           ConditionalExpression node) =>
       new ConditionalExpression(
           cloneNode(node.condition),
           cloneToken(node.question),
@@ skipping 645 matching lines @@
       cloneToken(node.withKeyword), cloneNodeList(node.mixinTypes));
 
   @override
   YieldStatement visitYieldStatement(YieldStatement node) => new YieldStatement(
       cloneToken(node.yieldKeyword),
       cloneToken(node.star),
       cloneNode(node.expression),
       cloneToken(node.semicolon));
 
   /**
+   * Clone all token starting from the given [token] up to a token that has
+   * offset greater then [stopAfter], and put mapping from originals to clones
+   * into [_clonedTokens].
+   *
+   * We cannot clone tokens as we visit nodes because not every token is a part
+   * of a node, E.g. commas in argument lists are not represented in AST. But
+   * we need to the sequence of tokens that is identical to the original one.
+   */
+  void _cloneTokens(Token token, int stopAfter) {
+    if (token == null) {
+      return;
+    }
+    if (token is CommentToken) {
+      token = (token as CommentToken).parent;
+    }
+    while (token != null) {
+      Token clone = token.copy();
+      {
+        CommentToken c1 = token.precedingComments;
+        CommentToken c2 = clone.precedingComments;
+        while (c1 != null && c2 != null) {
+          _clonedTokens[c1] = c2;
+          if (c1 is DocumentationCommentToken &&
+              c2 is DocumentationCommentToken) {
+            for (int i = 0; i < c1.references.length; i++) {
+              _clonedTokens[c1.references[i]] = c2.references[i];
+            }
+          }
+          c1 = c1.next;
+          c2 = c2.next;
+        }
+      }
+      _clonedTokens[token] = clone;
+      _lastCloned?.setNext(clone);
+      _lastCloned = clone;
+      if (token.type == TokenType.EOF) {
+        break;
+      }
+      if (token.offset > stopAfter) {
+        _nextToClone = token.next;
+        _lastClonedOffset = token.offset;
+        break;
+      }
+      token = token.next;
+    }
+  }
+
+  /**
    * Return a clone of the given [node].
    */
   static AstNode clone(AstNode node) {
     return node.accept(new AstCloner());
   }
 }
 
 /**
  * An AstVisitor that compares the structure of two AstNodes to see whether they
  * are equal.
@@ skipping 5363 matching lines @@
    * Safely visit the given [token], printing the [suffix] after the token if it
    * is non-`null`.
    */
   void _visitTokenWithSuffix(Token token, String suffix) {
     if (token != null) {
       _writer.print(token.lexeme);
       _writer.print(suffix);
     }
   }
 }