Introduce newline tokens into the token stream.

runtime/vm/object.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/object.h"
 
 #include "include/dart_api.h"
 #include "platform/assert.h"
 #include "vm/assembler.h"
 #include "vm/cpu.h"
@@ skipping 6034 matching lines @@
 const char* TokenStream::ToCString() const {
   return "TokenStream";
 }
 
 
 void TokenStream::PrintToJSONStream(JSONStream* stream, bool ref) const {
   JSONObject jsobj(stream);
 }
 
 
-TokenStream::Iterator::Iterator(const TokenStream& tokens, intptr_t token_pos)
+TokenStream::Iterator::Iterator(const TokenStream& tokens,
+                                intptr_t token_pos,
+                                Iterator::StreamType stream_type)
     : tokens_(TokenStream::Handle(tokens.raw())),
       data_(ExternalTypedData::Handle(tokens.GetStream())),
       stream_(reinterpret_cast<uint8_t*>(data_.DataAddr(0)), data_.Length()),
       token_objects_(Array::Handle(tokens.TokenObjects())),
       obj_(Object::Handle()),
       cur_token_pos_(token_pos),
       cur_token_kind_(Token::kILLEGAL),
-      cur_token_obj_index_(-1) {
+      cur_token_obj_index_(-1),
+      stream_type_(stream_type) {
   SetCurrentPosition(token_pos);
 }
 
 
 void TokenStream::Iterator::SetStream(const TokenStream& tokens,
                                       intptr_t token_pos) {
   tokens_ = tokens.raw();
   data_ = tokens.GetStream();
   stream_.SetStream(reinterpret_cast<uint8_t*>(data_.DataAddr(0)),
                     data_.Length());
@@ skipping 11 matching lines @@
 }
 
 
 Token::Kind TokenStream::Iterator::LookaheadTokenKind(intptr_t num_tokens) {
   intptr_t saved_position = stream_.Position();
   Token::Kind kind = Token::kILLEGAL;
   intptr_t value = -1;
   intptr_t count = 0;
   while (count < num_tokens && value != Token::kEOS) {
     value = ReadToken();
-    count += 1;
+    if ((stream_type_ == kAllTokens) ||
+        (static_cast<Token::Kind>(value) != Token::kNEWLINE)) {
+      count += 1;
+    }
   }
   if (value < Token::kNumTokens) {
     kind = static_cast<Token::Kind>(value);
   } else {
     value = value - Token::kNumTokens;
     obj_ = token_objects_.At(value);
     if (obj_.IsLiteralToken()) {
       const LiteralToken& literal_token = LiteralToken::Cast(obj_);
       kind = literal_token.kind();
     } else {
@@ skipping 11 matching lines @@
 }
 
 
 void TokenStream::Iterator::SetCurrentPosition(intptr_t value) {
   stream_.SetPosition(value);
   Advance();
 }
 
 
 void TokenStream::Iterator::Advance() {
-  cur_token_pos_ = stream_.Position();
-  intptr_t value = ReadToken();
+  intptr_t value;
+  do {
+    cur_token_pos_ = stream_.Position();
+    value = ReadToken();
+  } while ((stream_type_ == kNoNewlines) &&
+           (static_cast<Token::Kind>(value) == Token::kNEWLINE));
   if (value < Token::kNumTokens) {
     cur_token_kind_ = static_cast<Token::Kind>(value);
     cur_token_obj_index_ = -1;
     return;
   }
   cur_token_obj_index_ = value - Token::kNumTokens;
   obj_ = token_objects_.At(cur_token_obj_index_);
   if (obj_.IsLiteralToken()) {
     const LiteralToken& literal_token = LiteralToken::Cast(obj_);
     cur_token_kind_ = literal_token.kind();
@@ skipping 8788 matching lines @@
   return "_MirrorReference";
 }
 
 
 void MirrorReference::PrintToJSONStream(JSONStream* stream, bool ref) const {
   JSONObject jsobj(stream);
 }
 
 
 }  // namespace dart