- Ensure that the token stream from generated source matches the

runtime/vm/scanner.cc

Index: runtime/vm/scanner.cc
===================================================================
--- runtime/vm/scanner.cc	(revision 28974)
+++ runtime/vm/scanner.cc	(working copy)
@@ -42,13 +42,17 @@
 
 
 void Scanner::Reset() {
-  // Non-chaning newline properties.
+  // Non-changing newline properties.
   newline_token_.kind = Token::kNEWLINE;
   newline_token_.literal = NULL;
-  newline_token_.length = 1;
   // We don't preserve the column information.
   newline_token_.position.column = 0;
 
+  // Non-changing empty string token properties.
+  empty_string_token_.kind = Token::kSTRING;
+  empty_string_token_.literal = &Symbols::Empty();
+  empty_string_token_.position.column = 0;
+
   lookahead_pos_ = -1;
   token_start_ = 0;
   c0_ = '\0';
@@ -187,7 +191,7 @@
 bool Scanner::IsValidLiteral(const Scanner::GrowableTokenStream& tokens,
                              Token::Kind literal_kind,
                              bool* is_positive,
-                             String** value) {
+                             const String** value) {
   if ((tokens.length() == 2) &&
       (tokens[0].kind == literal_kind) &&
       (tokens[1].kind == Token::kEOS)) {
@@ -877,7 +881,6 @@
         }
     }
   } while (current_token_.kind == Token::kWHITESP);
-  current_token_.length = lookahead_pos_ - token_start_;
 }
 
 
@@ -886,13 +889,21 @@
   do {
     Scan();
 
+    bool inserted_new_lines = false;
     for (intptr_t diff = current_token_.position.line - prev_token_line_;
          diff > 0;
          diff--) {
       newline_token_.position.line = current_token_.position.line - diff;
       token_stream->Add(newline_token_);
+      inserted_new_lines = true;
     }
 
+    if (inserted_new_lines &&
+        ((current_token_.kind == Token::kINTERPOL_VAR) ||
+         (current_token_.kind == Token::kINTERPOL_START))) {
+      empty_string_token_.position.line = current_token_.position.line;
+      token_stream->Add(empty_string_token_);
+    }
     token_stream->Add(current_token_);
     prev_token_line_ = current_token_.position.line;
   } while (current_token_.kind != Token::kEOS);
@@ -904,7 +915,22 @@
   Reset();
   do {
     Scan();
+
+    bool inserted_new_lines = false;
+    for (intptr_t diff = current_token_.position.line - prev_token_line_;
+         diff > 0;
+         diff--) {
+      index++;  // We need to advance the index everytime a token is added.

[hausner, 2013/10/23 16:44:51]

The comment is a bit misleading, since no tokens are added. Maybe: Advance index
to account for newline tokens.

[Ivan Posva, 2013/10/23 18:20:00]

  // Advance the index to account for tokens added in ScanAll.

+      inserted_new_lines = true;
+    }
+
+    if (inserted_new_lines &&
+        ((current_token_.kind == Token::kINTERPOL_VAR) ||
+         (current_token_.kind == Token::kINTERPOL_START))) {
+          index++;  // We need to advance the index everytime a token is added.

[hausner, 2013/10/23 16:44:51]

ditto

[Ivan Posva, 2013/10/23 18:20:00]

ditto

+        }

[Florian Schneider, 2013/10/23 16:54:40]

Strange indentation.

[Ivan Posva, 2013/10/23 18:20:00]

Done.

     index++;
+    prev_token_line_ = current_token_.position.line;
   } while ((token_index >= index) && (current_token_.kind != Token::kEOS));
 }