Introduce newline tokens into the token stream.

runtime/vm/object_test.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/assembler.h"
 #include "vm/bigint_operations.h"
 #include "vm/class_finalizer.h"
 #include "vm/dart_api_impl.h"
 #include "vm/dart_entry.h"
 #include "vm/isolate.h"
@@ skipping 2062 matching lines @@
 
   script.GetTokenLocation(4, &line, &col);  // Token 'return'
   EXPECT_EQ(4, line);  // 'return' is in line 4.
   EXPECT_EQ(5, col);   // Four spaces before 'return'.
 
   intptr_t first_idx, last_idx;
   script.TokenRangeAtLine(3, &first_idx, &last_idx);
   EXPECT_EQ(0, first_idx);  // Token 'main' is first token.
   EXPECT_EQ(3, last_idx);   // Token { is last token.
   script.TokenRangeAtLine(5, &first_idx, &last_idx);
-  EXPECT_EQ(7, first_idx);  // Token } is first and only token.
-  EXPECT_EQ(7, last_idx);
+  EXPECT_EQ(8, first_idx);  // Token } is first and only token.

[hausner, 2013/09/18 22:27:47]

Why has this token index changed?

[Michael Lippautz (Google), 2013/09/18 23:59:31]

The source text contains 2 NL, hence 7->9 (previous patchset still had a bug).
While we don't access the NL tokens (because the stream skips them), the index
still reflects the position in the stream.

+  EXPECT_EQ(8, last_idx);
 }
 
 
 TEST_CASE(Context) {
   const int kNumVariables = 5;
   const Context& parent_context = Context::Handle(Context::New(0));
   EXPECT_EQ(parent_context.isolate(), Isolate::Current());
   const Context& context = Context::Handle(Context::New(kNumVariables));
   context.set_parent(parent_context);
   const Context& check_parent_context = Context::Handle(context.parent());
@@ skipping 1355 matching lines @@
   cls = Object::dynamic_class();
   array = cls.fields();
   EXPECT(!array.IsNull());
   EXPECT(array.IsArray());
   array = cls.functions();
   EXPECT(!array.IsNull());
   EXPECT(array.IsArray());
 }
 
 }  // namespace dart