Introduce newline tokens into the token stream.

runtime/vm/scanner.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/scanner.h"
 
 #include "platform/assert.h"
 #include "vm/flags.h"
 #include "vm/object.h"
 #include "vm/object_store.h"
@@ skipping 24 matching lines @@
     Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i);
     keywords_[i].kind = token;
     keywords_[i].keyword_chars = Token::Str(token);
     keywords_[i].keyword_len = strlen(Token::Str(token));
     keywords_[i].keyword_symbol = NULL;
   }
 }
 
 
 void Scanner::Reset() {
+  // Non-chaning newline properties.
+  newline_token_.kind = Token::kNEWLINE;
+  newline_token_.literal = NULL;
+  newline_token_.length = 1;
+  // We don't preserve the column information.
+  newline_token_.position.column = 0;
+
   lookahead_pos_ = -1;
   token_start_ = 0;
   c0_ = '\0';
-  newline_seen_ = false;
+  newline_seen_ = 0;
+  newline_after_ = 0;
   while (saved_context_ != NULL) {
     ScanContext* ctx = saved_context_;
     saved_context_ = ctx->next;
     delete ctx;
   }
   string_delimiter_ = '\0';
   string_is_multiline_ = false;
   brace_level_ = 0;
   c0_pos_.line = 1;
   c0_pos_.column = 0;
@@ skipping 51 matching lines @@
   string_delimiter_ = ctx->string_delimiter;
   ASSERT(string_delimiter_ != '\0');
   string_is_multiline_ = ctx->string_is_multiline;
   brace_level_ = ctx->brace_level;
   delete ctx;
 }
 
 
 void Scanner::BeginStringLiteral(const char delimiter) {
   string_delimiter_ = delimiter;
+  // Save the current newline counter, since newlines observed within a
+  // multi-line string literal should be appended after the actual token.
+  newline_save_ = newline_seen_;
 }
 
 
 void Scanner::EndStringLiteral() {
   string_delimiter_ = '\0';
   string_is_multiline_ = false;
+  // Update the counter for appending newlines after the actual token and
+  // restore the newline count.
+  newline_after_ = newline_seen_ - newline_save_;
+  newline_seen_ = newline_save_;
 }
 
 
 bool Scanner::IsLetter(int32_t c) {
   return (('A' <= c) && (c <= 'Z')) || (('a' <= c) && (c <= 'z'));
 }
 
 
 bool Scanner::IsDecimalDigit(int32_t c) {
   return '0' <= c && c <= '9';
@@ skipping 64 matching lines @@
     *value = tokens[1].literal;
     return true;
   }
   return false;
 }
 
 
 void Scanner::ReadChar() {
   if (lookahead_pos_ < source_length_) {
     if (c0_ == '\n') {
-      newline_seen_ = true;
+      newline_seen_++;
       c0_pos_.line++;
       c0_pos_.column = 0;
       if (source_.CharAt(lookahead_pos_) == '\r') {
         // Replace a sequence of '\r' '\n' with a single '\n'.
         if (LookaheadChar(1) == '\n') {
           lookahead_pos_++;
         }
       }
     }
     lookahead_pos_++;
@@ skipping 371 matching lines @@
         }
       }
       string_chars.Add(ch1);
     }
     ReadChar();
   }
 }
 
 
 void Scanner::Scan() {
-  newline_seen_ = false;
+  newline_seen_ = 0;
+  newline_after_ = 0;
 
   do {
     if (!IsScanningString()) {
       ConsumeWhiteSpace();
     }
     token_start_ = lookahead_pos_;
     current_token_.offset = lookahead_pos_;
     current_token_.position = c0_pos_;
     current_token_.literal = NULL;
     current_token_.kind = Token::kILLEGAL;
@@ skipping 245 matching lines @@
     }
   } while (current_token_.kind == Token::kWHITESP);
   current_token_.length = lookahead_pos_ - token_start_;
 }
 
 
 void Scanner::ScanAll(GrowableTokenStream* token_stream) {
   Reset();
   do {
     Scan();
+
+    // Account for all newlines that should be pushed to the stream before the
+    // actual token.
+    for (; newline_seen_ > 0; newline_seen_--) {
+      newline_token_.position.line =
+          current_token_.position.line - newline_seen_;
+      token_stream->Add(newline_token_);
+    }
+
     token_stream->Add(current_token_);
+
+    // Account for all newlines that should be pushed after the token, i.e., all

[hausner, 2013/09/18 22:27:47]

Why so complicated? Is it not enough to remember the line number of the previous
token and add the number of newline tokens to catch up with the line number of
the current token? No need even to remember the number of newlines before the
token, inside multiline strings, etc.

[Michael Lippautz (Google), 2013/09/18 23:59:31]

Done. Makes everything much simpler.

+    // newlines seen in multiline strings.
+    for (intptr_t max_after = newline_after_;
+         newline_after_ > 0;
+         newline_after_--) {
+      newline_token_.position.line =
+          current_token_.position.line + max_after - newline_after_;
+      token_stream->Add(newline_token_);
+    }
   } while (current_token_.kind != Token::kEOS);
 }
 
 
 void Scanner::ScanTo(intptr_t token_index) {
   int index = 0;
   Reset();
   do {
     Scan();
     index++;
@@ skipping 71 matching lines @@
               "%c%#" Px "", kPrivateKeySeparator, key_value);
   const String& result = String::Handle(String::New(private_key, Heap::kOld));
   return result.raw();
 }
 
 
 void Scanner::InitOnce() {
 }
 
 }  // namespace dart