Introduce newline tokens into the token stream.

runtime/vm/scanner.h

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 //
 // Scanner class for the Dart language. The scanner reads source text
 // and produces a stream of tokens which is used by the parser.
 //
 
 #ifndef VM_SCANNER_H_
 #define VM_SCANNER_H_
@@ skipping 63 matching lines @@
                         intptr_t* last_token_index);
 
   // Scans entire source and returns a stream of tokens.
   // Should be called only once.
   const GrowableTokenStream& GetStream();
 
   // Info about most recently recognized token.
   const TokenDescriptor& current_token() const { return current_token_; }
 
   // Was there a line break before the current token?
-  bool NewlineBeforeToken() const { return newline_seen_; }
+  bool NewlineBeforeToken() const { return newline_seen_ > 0; }
 
   // Source code line number and column of current token.
   const SourcePosition& CurrentPosition() const {
     return current_token_.position;
   }
 
   static void InitOnce();
 
   // Allocated a private key which is used for name mangling.
   static RawString* AllocatePrivateKey(const Library& library);
@@ skipping 101 matching lines @@
   }
 
   // Reads a number literal.
   void ScanNumber(bool dec_point_seen);
 
   void ScanScriptTag();
 
   static void PrintTokens(const GrowableTokenStream& ts);
 
   TokenDescriptor current_token_;  // Current token.
+  TokenDescriptor newline_token_;  // Newline token.
   const String& source_;           // The source text being tokenized.
   intptr_t source_length_;     // The length of the source text.
   intptr_t lookahead_pos_;     // Position of lookahead character
                                // within source_.
   intptr_t token_start_;       // Begin of current token in src_.
   int32_t c0_;                 // Lookahead character.
-  bool newline_seen_;          // Newline before current token.
+  intptr_t newline_seen_;      // Number of newlines before current token.

[hausner, 2013/09/18 22:27:47]

num_newlines_before_ ?

[Michael Lippautz (Google), 2013/09/18 23:59:31]

Doesn't apply anymore. Reverted it to being a flag.

+  intptr_t newline_save_;      // Savepoint for newlines in StringLiterals.
+  intptr_t newline_after_;     // Number of newlines after current token.
 
   // The following fields keep track whether we are scanning a string literal
   // and its interpolated expressions.
   ScanContext* saved_context_;
   int32_t string_delimiter_;
   bool string_is_multiline_;
   int brace_level_;
 
   const String& private_key_;
 
   SourcePosition c0_pos_;      // Source position of lookahead character c0_.
   KeywordTable keywords_[Token::numKeywords];
   Array& keyword_symbol_table_;  // Access to keyword symbols in object store.
 };
 
 
 }  // namespace dart
 
 #endif  // VM_SCANNER_H_