Introduce newline tokens into the token stream.

runtime/vm/scanner.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/scanner.h"
 
 #include "platform/assert.h"
 #include "vm/flags.h"
 #include "vm/object.h"
 #include "vm/object_store.h"
@@ skipping 24 matching lines @@
     Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i);
     keywords_[i].kind = token;
     keywords_[i].keyword_chars = Token::Str(token);
     keywords_[i].keyword_len = strlen(Token::Str(token));
     keywords_[i].keyword_symbol = NULL;
   }
 }
 
 
 void Scanner::Reset() {
+  // Non-chaning newline properties.
+  newline_token_.kind = Token::kNEWLINE;
+  newline_token_.literal = NULL;
+  newline_token_.length = 1;
+  // We don't preserve the column information.
+  newline_token_.position.column = 0;
+
   lookahead_pos_ = -1;
   token_start_ = 0;
   c0_ = '\0';
   newline_seen_ = false;
+  prev_token_line_ = 1;
   while (saved_context_ != NULL) {
     ScanContext* ctx = saved_context_;
     saved_context_ = ctx->next;
     delete ctx;
   }
   string_delimiter_ = '\0';
   string_is_multiline_ = false;
   brace_level_ = 0;
   c0_pos_.line = 1;
   c0_pos_.column = 0;
@@ skipping 797 matching lines @@
     }
   } while (current_token_.kind == Token::kWHITESP);
   current_token_.length = lookahead_pos_ - token_start_;
 }
 
 
 void Scanner::ScanAll(GrowableTokenStream* token_stream) {
   Reset();
   do {
     Scan();
+
+    for (intptr_t diff = current_token_.position.line - prev_token_line_;
+         diff > 0;
+         diff--) {
+      newline_token_.position.line = current_token_.position.line - diff;
+      token_stream->Add(newline_token_);
+    }
+
     token_stream->Add(current_token_);
+    prev_token_line_ = current_token_.position.line;
   } while (current_token_.kind != Token::kEOS);
 }
 
 
 void Scanner::ScanTo(intptr_t token_index) {
   int index = 0;
   Reset();
   do {
     Scan();
     index++;
@@ skipping 71 matching lines @@
               "%c%#" Px "", kPrivateKeySeparator, key_value);
   const String& result = String::Handle(String::New(private_key, Heap::kOld));
   return result.raw();
 }
 
 
 void Scanner::InitOnce() {
 }
 
 }  // namespace dart