Stop copying tokens in incremental scanner

pkg/analyzer/test/generated/incremental_scanner_test.dart

 // Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library engine.incremental_scanner_test;
 
 import 'package:analyzer/src/generated/incremental_scanner.dart';
 import 'package:analyzer/src/generated/scanner.dart';
 import 'package:analyzer/src/generated/source.dart';
 import 'package:unittest/unittest.dart';
 
 import '../reflective_tests.dart';
 import 'test_support.dart';
 
-
 main() {
   groupSep = ' | ';
   runReflectiveTests(IncrementalScannerTest);
 }
 
 class IncrementalScannerTest extends EngineTestCase {
   /**
    * The first token from the token stream resulting from parsing the original
    * source, or `null` if [scan] has not been invoked.
    */
@@ skipping 154 matching lines @@
     _scan("ca", "", "r", "t;");
     _assertTokens(-1, 1, ["cart", ";"]);
     expect(_incrementalScanner.hasNonWhitespaceChange, isTrue);
   }
 
   void test_insert_inIdentifier_right_firstToken() {
     // "a + b;"
     // "abs + b;"
     _scan("a", "", "bs", " + b;");
     _assertTokens(-1, 1, ["abs", "+", "b", ";"]);
-    _assertReplaced(1, "+");
     expect(_incrementalScanner.hasNonWhitespaceChange, isTrue);
   }
 
   void test_insert_inIdentifier_right_lastToken() {
     // "a + b"
     // "a + bc"
     _scan("a + b", "", "c", "");
     _assertTokens(1, 3, ["a", "+", "bc"]);
     expect(_incrementalScanner.hasNonWhitespaceChange, isTrue);
   }
 
   void test_insert_inIdentifier_right_middleToken() {
     // "a + b;"
     // "a + by;"
     _scan("a + b", "", "y", ";");
     _assertTokens(1, 3, ["a", "+", "by", ";"]);
     expect(_incrementalScanner.hasNonWhitespaceChange, isTrue);
   }
 
   void test_insert_newIdentifier_noSpaceBefore() {
     // "a; c;"
     // "a;b c;"
     _scan("a;", "", "b", " c;");
     _assertTokens(1, 3, ["a", ";", "b", "c", ";"]);
-    _assertReplaced(1, ";");
     expect(_incrementalScanner.hasNonWhitespaceChange, isTrue);
   }
 
   void test_insert_newIdentifier_spaceBefore() {
     // "a; c;"
     // "a; b c;"
     _scan("a; ", "", "b ", "c;");
     _assertTokens(1, 3, ["a", ";", "b", "c", ";"]);
     expect(_incrementalScanner.hasNonWhitespaceChange, isTrue);
   }
@@ skipping 77 matching lines @@
 
   void test_insert_whitespace_end_afterWhitespace() {
     // "a + b; "
     // "a + b;  "
     _scan("a + b; ", "", " ", "");
     _assertTokens(3, 4, ["a", "+", "b", ";"]);
     expect(_incrementalScanner.hasNonWhitespaceChange, isFalse);
   }
 
   void test_insert_whitespace_withMultipleComments() {
-    // "//comment", "//comment2", "a + b;"
-    // "//comment", "//comment2", "a  + b;"
+    // "//comment1", "//comment2", "a + b;"
+    // "//comment1", "//comment2", "a  + b;"
     _scan(r'''
-//comment
+//comment1
 //comment2
 a''', "", " ", " + b;");
     _assertTokens(1, 2, ["a", "+", "b", ";"]);
     expect(_incrementalScanner.hasNonWhitespaceChange, isFalse);
   }
 
   void test_replace_identifier_beginning() {
     // "bell + b;"
     // "fell + b;"
     _scan("", "b", "f", "ell + b;");
@@ skipping 70 matching lines @@
     if (comment != null) {
       while (comment != null) {
         count++;
         comment = comment.next;
       }
       fail("Expected ${lexemes.length} comments but found $count");
     }
   }
 
   /**
-   * Assert that the token at the given [offset] was replaced with a new token
-   * having the given [lexeme].
+   * Assert that the [expected] token is equal to the [actual] token.
    */
-  void _assertReplaced(int offset, String lexeme) {
-    Token oldToken = _originalTokens;
-    for (int i = 0; i < offset; i++) {
-      oldToken = oldToken.next;
-    }
-    expect(oldToken.lexeme, lexeme);
-    Token newToken = _incrementalScanner.tokenMap.get(oldToken);
-    expect(newToken, isNotNull);
-    expect(newToken.lexeme, lexeme);
-    expect(newToken, isNot(same(oldToken)));
+  void _assertEqualTokens(Token actual, Token expected) {
+    expect(actual.type, same(expected.type), reason: "Wrong type for token");
+    expect(actual.lexeme, expected.lexeme, reason: "Wrong lexeme for token");
+    expect(
+        actual.offset,
+        expected.offset,
+        reason: "Wrong offset for token ('${actual.lexeme}' != '${expected.lexeme}')");
+    expect(
+        actual.length,
+        expected.length,
+        reason: "Wrong length for token ('${actual.lexeme}' != '${expected.lexeme}')");
   }
 
   /**
    * Assert that the result of the incremental scan matches the given list of
    * [lexemes] and that the left and right tokens correspond to the tokens at
    * the [leftIndex] and [rightIndex].
    */
   void _assertTokens(int leftIndex, int rightIndex, List<String> lexemes) {
     int count = lexemes.length;
     expect(
@@ skipping 92 matching lines @@
         removed.length,
         added.length);
     //
     // Validate that the results of the incremental scan are the same as the
     // full scan of the modified source.
     //
     Token incrementalToken = _incrementalTokens;
     expect(incrementalToken, isNotNull);
     while (incrementalToken.type != TokenType.EOF &&
         modifiedTokens.type != TokenType.EOF) {
+      _assertEqualTokens(incrementalToken, modifiedTokens);
+      Token incrementalComment = incrementalToken.precedingComments;
+      Token modifiedComment = modifiedTokens.precedingComments;
+      while (incrementalComment != null && modifiedComment != null) {
+        _assertEqualTokens(incrementalComment, modifiedComment);
+        incrementalComment = incrementalComment.next;
+        modifiedComment = modifiedComment.next;
+      }
       expect(
-          incrementalToken.type,
-          same(modifiedTokens.type),
-          reason: "Wrong type for token");
+          incrementalComment,
+          isNull,
+          reason: "Too many comment tokens preceeding '${incrementalToken.lexeme}'");
       expect(
-          incrementalToken.offset,
-          modifiedTokens.offset,
-          reason:
-              "Wrong offset for token (${incrementalToken.lexeme} != ${modifiedTokens.lexeme})");
-      expect(
-          incrementalToken.length,
-          modifiedTokens.length,
-          reason:
-              "Wrong length for token (${incrementalToken.lexeme} != ${modifiedTokens.lexeme})");
-      expect(
-          incrementalToken.lexeme,
-          modifiedTokens.lexeme,
-          reason: "Wrong lexeme for token");
+          modifiedComment,
+          isNull,
+          reason: "Not enough comment tokens preceeding '${incrementalToken.lexeme}'");
       incrementalToken = incrementalToken.next;
       modifiedTokens = modifiedTokens.next;
     }
     expect(
         incrementalToken.type,
         same(TokenType.EOF),
         reason: "Too many tokens");
     expect(
         modifiedTokens.type,
         same(TokenType.EOF),
         reason: "Not enough tokens");
     // TODO(brianwilkerson) Verify that the errors are correct?
   }
 }