Introduce newline tokens into the token stream.

runtime/vm/scanner.h

Index: runtime/vm/scanner.h
diff --git a/runtime/vm/scanner.h b/runtime/vm/scanner.h
index 316ac98339e475cbda70bc57a0f8960df29d3f81..bec1607bb75241ac1013e25879004f8e61c7876b 100644
--- a/runtime/vm/scanner.h
+++ b/runtime/vm/scanner.h
@@ -81,7 +81,7 @@ class Scanner : ValueObject {
   const TokenDescriptor& current_token() const { return current_token_; }
 
   // Was there a line break before the current token?
-  bool NewlineBeforeToken() const { return newline_seen_; }
+  bool NewlineBeforeToken() const { return newline_seen_ > 0; }
 
   // Source code line number and column of current token.
   const SourcePosition& CurrentPosition() const {
@@ -203,13 +203,16 @@ class Scanner : ValueObject {
   static void PrintTokens(const GrowableTokenStream& ts);
 
   TokenDescriptor current_token_;  // Current token.
+  TokenDescriptor newline_token_;  // Newline token.
   const String& source_;           // The source text being tokenized.
   intptr_t source_length_;     // The length of the source text.
   intptr_t lookahead_pos_;     // Position of lookahead character
                                // within source_.
   intptr_t token_start_;       // Begin of current token in src_.
   int32_t c0_;                 // Lookahead character.
-  bool newline_seen_;          // Newline before current token.
+  intptr_t newline_seen_;      // Number of newlines before current token.

[hausner, 2013/09/18 22:27:47]

num_newlines_before_ ?

[Michael Lippautz (Google), 2013/09/18 23:59:31]

Doesn't apply anymore. Reverted it to being a flag.

+  intptr_t newline_save_;      // Savepoint for newlines in StringLiterals.
+  intptr_t newline_after_;     // Number of newlines after current token.
 
   // The following fields keep track whether we are scanning a string literal
   // and its interpolated expressions.