Introduce newline tokens into the token stream.

runtime/vm/scanner.cc

Index: runtime/vm/scanner.cc
diff --git a/runtime/vm/scanner.cc b/runtime/vm/scanner.cc
index ec87bd77fb11829e1571a731f8b859f734f3f7d1..483abed0a97c4fb949775a6377f06de098b7149b 100644
--- a/runtime/vm/scanner.cc
+++ b/runtime/vm/scanner.cc
@@ -42,10 +42,18 @@ void Scanner::InitKeywordTable() {
 
 
 void Scanner::Reset() {
+  // Non-chaning newline properties.
+  newline_token_.kind = Token::kNEWLINE;
+  newline_token_.literal = NULL;
+  newline_token_.length = 1;
+  // We don't preserve the column information.
+  newline_token_.position.column = 0;
+
   lookahead_pos_ = -1;
   token_start_ = 0;
   c0_ = '\0';
-  newline_seen_ = false;
+  newline_seen_ = 0;
+  newline_after_ = 0;
   while (saved_context_ != NULL) {
     ScanContext* ctx = saved_context_;
     saved_context_ = ctx->next;
@@ -117,12 +125,19 @@ void Scanner::PopContext() {
 
 void Scanner::BeginStringLiteral(const char delimiter) {
   string_delimiter_ = delimiter;
+  // Save the current newline counter, since newlines observed within a
+  // multi-line string literal should be appended after the actual token.
+  newline_save_ = newline_seen_;
 }
 
 
 void Scanner::EndStringLiteral() {
   string_delimiter_ = '\0';
   string_is_multiline_ = false;
+  // Update the counter for appending newlines after the actual token and
+  // restore the newline count.
+  newline_after_ = newline_seen_ - newline_save_;
+  newline_seen_ = newline_save_;
 }
 
 
@@ -207,7 +222,7 @@ bool Scanner::IsValidLiteral(const Scanner::GrowableTokenStream& tokens,
 void Scanner::ReadChar() {
   if (lookahead_pos_ < source_length_) {
     if (c0_ == '\n') {
-      newline_seen_ = true;
+      newline_seen_++;
       c0_pos_.line++;
       c0_pos_.column = 0;
       if (source_.CharAt(lookahead_pos_) == '\r') {
@@ -599,7 +614,8 @@ void Scanner::ScanLiteralStringChars(bool is_raw) {
 
 
 void Scanner::Scan() {
-  newline_seen_ = false;
+  newline_seen_ = 0;
+  newline_after_ = 0;
 
   do {
     if (!IsScanningString()) {
@@ -865,7 +881,26 @@ void Scanner::ScanAll(GrowableTokenStream* token_stream) {
   Reset();
   do {
     Scan();
+
+    // Account for all newlines that should be pushed to the stream before the
+    // actual token.
+    for (; newline_seen_ > 0; newline_seen_--) {
+      newline_token_.position.line =
+          current_token_.position.line - newline_seen_;
+      token_stream->Add(newline_token_);
+    }
+
     token_stream->Add(current_token_);
+
+    // Account for all newlines that should be pushed after the token, i.e., all

[hausner, 2013/09/18 22:27:47]

Why so complicated? Is it not enough to remember the line number of the previous
token and add the number of newline tokens to catch up with the line number of
the current token? No need even to remember the number of newlines before the
token, inside multiline strings, etc.

[Michael Lippautz (Google), 2013/09/18 23:59:31]

Done. Makes everything much simpler.

+    // newlines seen in multiline strings.
+    for (intptr_t max_after = newline_after_;
+         newline_after_ > 0;
+         newline_after_--) {
+      newline_token_.position.line =
+          current_token_.position.line + max_after - newline_after_;
+      token_stream->Add(newline_token_);
+    }
   } while (current_token_.kind != Token::kEOS);
 }