Introduce newline tokens into the token stream.

runtime/vm/object_test.cc

Index: runtime/vm/object_test.cc
diff --git a/runtime/vm/object_test.cc b/runtime/vm/object_test.cc
index 82a5d8d5c0ee029bcd9f6cdca26a02a422f05f20..0df5d3fa844705fd1e025b2dbda5e056868313f0 100644
--- a/runtime/vm/object_test.cc
+++ b/runtime/vm/object_test.cc
@@ -2080,8 +2080,8 @@ TEST_CASE(EmbeddedScript) {
   EXPECT_EQ(0, first_idx);  // Token 'main' is first token.
   EXPECT_EQ(3, last_idx);   // Token { is last token.
   script.TokenRangeAtLine(5, &first_idx, &last_idx);
-  EXPECT_EQ(7, first_idx);  // Token } is first and only token.
-  EXPECT_EQ(7, last_idx);
+  EXPECT_EQ(8, first_idx);  // Token } is first and only token.

[hausner, 2013/09/18 22:27:47]

Why has this token index changed?

[Michael Lippautz (Google), 2013/09/18 23:59:31]

The source text contains 2 NL, hence 7->9 (previous patchset still had a bug).
While we don't access the NL tokens (because the stream skips them), the index
still reflects the position in the stream.

+  EXPECT_EQ(8, last_idx);
 }