Experimental parser: add ability to compare lexer shell outputs

src/lexer/lexer-shell.cc

Index: src/lexer/lexer-shell.cc
diff --git a/src/lexer/lexer-shell.cc b/src/lexer/lexer-shell.cc
index 8821030da1811be6f57a5d0aa2fd81ec6d304395..17d7276a16780283b9ce612522262d556c4b0d77 100644
--- a/src/lexer/lexer-shell.cc
+++ b/src/lexer/lexer-shell.cc
@@ -60,6 +60,7 @@ enum Encoding {
 struct LexerShellSettings {
   Encoding encoding;
   bool print_tokens;
+  bool print_tokens_for_compare;
   bool break_after_illegal;
   bool eos_test;
   int repeat;
@@ -69,6 +70,7 @@ struct LexerShellSettings {
   LexerShellSettings()
       : encoding(LATIN1),
         print_tokens(false),
+        print_tokens_for_compare(false),
         break_after_illegal(false),
         eos_test(false),
         repeat(1),
@@ -185,44 +187,6 @@ static uint16_t* ReadFile(const char* name,
 }
 
 
-struct TokenWithLocation {
-  Token::Value value;
-  size_t beg;
-  size_t end;
-  std::vector<int> literal;
-  bool is_ascii;
-  // The location of the latest octal position when the token was seen.
-  int octal_beg;
-  int octal_end;
-  TokenWithLocation() :
-      value(Token::ILLEGAL), beg(0), end(0), is_ascii(false) { }
-  TokenWithLocation(Token::Value value, size_t beg, size_t end,
-                    int octal_beg) :
-      value(value), beg(beg), end(end), is_ascii(false), octal_beg(octal_beg) {
-  }
-  bool operator==(const TokenWithLocation& other) {
-    return value == other.value && beg == other.beg && end == other.end &&
-           literal == other.literal && is_ascii == other.is_ascii &&
-        octal_beg == other.octal_beg;
-  }
-  bool operator!=(const TokenWithLocation& other) {
-    return !(*this == other);
-  }
-  void Print(const char* prefix) const {
-    printf("%s %11s at (%d, %d)",
-           prefix, Token::Name(value),
-           static_cast<int>(beg), static_cast<int>(end));
-    if (literal.size() > 0) {
-      for (size_t i = 0; i < literal.size(); i++) {
-        printf(is_ascii ? " %02x" : " %04x", literal[i]);
-      }
-      printf(" (is ascii: %d)", is_ascii);
-    }
-    printf(" (last octal start: %d)\n", octal_beg);
-  }
-};
-
-
 static bool HasLiteral(Token::Value token) {
   return token == Token::IDENTIFIER ||
          token == Token::STRING ||
@@ -231,37 +195,72 @@ static bool HasLiteral(Token::Value token) {
 
 
 template<typename Char>
-static std::vector<int> ToStdVector(const Vector<Char>& literal) {
-  std::vector<int> result;
+static void Copy(const Vector<Char>& literal,
+                 SmartArrayPointer<const uint16_t>* result,
+                 int* literal_length) {
+  uint16_t* data = new uint16_t[literal.length()];
+  result->Reset(data);
   for (int i = 0; i < literal.length(); i++) {
-    result.push_back(literal[i]);
+    data[i] = literal[i];
   }
-  return result;
+  *literal_length = literal.length();
 }
 
 
-template<typename Scanner>
-static TokenWithLocation GetTokenWithLocation(
-    Scanner *scanner, Token::Value token) {
-  int beg = scanner->location().beg_pos;
-  int end = scanner->location().end_pos;
-  TokenWithLocation result(token, beg, end, scanner->octal_position().beg_pos);
-  if (HasLiteral(token)) {
-    result.is_ascii = scanner->is_literal_ascii();
-    if (scanner->is_literal_ascii()) {
-      result.literal = ToStdVector(scanner->literal_ascii_string());
-    } else {
-      result.literal = ToStdVector(scanner->literal_utf16_string());
+class TokenWithLocation {
+ public:
+  Token::Value value;
+  int beg;
+  int end;
+  bool is_one_byte;
+  SmartArrayPointer<const uint16_t> literal;
+  int literal_length;
+  // The location of the latest octal position when the token was seen.
+  int octal_beg;
+  int octal_end;
+  TokenWithLocation(Token::Value token, Scanner* scanner) : value(token) {
+    beg = scanner->location().beg_pos;
+    end = scanner->location().end_pos;
+    octal_beg = scanner->octal_position().beg_pos;
+    octal_end = scanner->octal_position().end_pos;
+    is_one_byte = false;
+    literal_length = 0;
+    if (HasLiteral(token)) {
+      is_one_byte = scanner->is_literal_ascii();
+      if (scanner->is_literal_ascii()) {
+        Copy(scanner->literal_ascii_string(), &literal, &literal_length);
+      } else {
+        Copy(scanner->literal_utf16_string(), &literal, &literal_length);
+      }
     }
   }
-  return result;
-}
+  void Print(bool do_compare) const {
+    if (value == Token::ILLEGAL && do_compare) {
+      printf("%-15s (%d)\n", Token::Name(value), beg);
+      return;
+    }
+    printf("%-15s (%d, %d)", Token::Name(value), beg, end);
+    if (literal_length > 0) {
+      // TODO(dcarney): need some sort of checksum.
+      for (int i = 0; i < literal_length; i++) {
+        printf(is_one_byte ? " %02x" : " %04x", literal[i]);
+      }
+      printf(" (is_one_byte: %d)", is_one_byte);
+    }
+    if (octal_beg >= 0) {
+      printf(" (last octal start: %d)", octal_beg);
+    }
+    printf("\n");
+  }
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(TokenWithLocation);
+};
 
 
 static TimeDelta RunLexer(const uint16_t* source,
                           const uint8_t* source_end,
                           Isolate* isolate,
-                          std::vector<TokenWithLocation>* tokens,
                           const LexerShellSettings& settings) {
   SmartPointer<Utf16CharacterStream> stream;
   const uint8_t* one_byte_source = reinterpret_cast<const uint8_t*>(source);
@@ -293,13 +292,14 @@ static TimeDelta RunLexer(const uint16_t* source,
   scanner.SetHarmonyModules(settings.harmony_modules);
   scanner.SetHarmonyScoping(settings.harmony_scoping);
   ElapsedTimer timer;
+  std::vector<TokenWithLocation*> tokens;
   timer.Start();
   scanner.Initialize(stream.get());
   Token::Value token;
   do {
     token = scanner.Next();
     if (settings.print_tokens) {
-      tokens->push_back(GetTokenWithLocation(&scanner, token));
+      tokens.push_back(new TokenWithLocation(token, &scanner));
     } else if (HasLiteral(token)) {
       if (scanner.is_literal_ascii()) {
         scanner.literal_ascii_string();
@@ -307,7 +307,20 @@ static TimeDelta RunLexer(const uint16_t* source,
         scanner.literal_utf16_string();
       }
     }
+    if (token == Token::ILLEGAL && settings.break_after_illegal) break;
   } while (token != Token::EOS);
+  // Dump tokens.
+  if (settings.print_tokens) {
+    if (!settings.print_tokens_for_compare) {
+      printf("No of tokens:\t%d\n", static_cast<int>(tokens.size()));
+    }
+    for (size_t i = 0; i < tokens.size(); ++i) {
+      tokens[i]->Print(settings.print_tokens_for_compare);
+    }
+  }
+  for (size_t i = 0; i < tokens.size(); ++i) {
+    delete tokens[i];
+  }
   return timer.Elapsed();
 }
 
@@ -318,11 +331,10 @@ static TimeDelta ProcessFile(
     const LexerShellSettings& settings,
     int truncate_by,
     bool* can_truncate) {
-  if (settings.print_tokens) {
+  if (settings.print_tokens && !settings.print_tokens_for_compare) {
     printf("Processing file %s, truncating by %d bytes\n", fname, truncate_by);
   }
   HandleScope handle_scope(isolate);
-  std::vector<TokenWithLocation> tokens;
   TimeDelta time;
   {
     unsigned length_in_bytes;
@@ -333,20 +345,11 @@ static TimeDelta ProcessFile(
       *can_truncate = false;
     } else {
       buffer_end -= truncate_by;
-      time = RunLexer(buffer, buffer_end, isolate, &tokens, settings);
+      time = RunLexer(buffer, buffer_end, isolate, settings);
     }
     delete[] buffer;
   }
-  if (settings.print_tokens) {
-    printf("No of tokens:\t%d\n", static_cast<int>(tokens.size()));
-    for (size_t i = 0; i < tokens.size(); ++i) {
-      tokens[i].Print("=>");
-      if (tokens[i].value == Token::ILLEGAL) {
-        if (settings.break_after_illegal)
-          break;
-      }
-    }
-  }
+
   return time;
 }
 
@@ -371,6 +374,9 @@ int main(int argc, char* argv[]) {
 #endif
     } else if (strcmp(argv[i], "--print-tokens") == 0) {
       settings.print_tokens = true;
+    } else if (strcmp(argv[i], "--print-tokens-for-compare") == 0) {
+      settings.print_tokens = true;
+      settings.print_tokens_for_compare = true;
     } else if (strcmp(argv[i], "--no-baseline") == 0) {
       // Ignore.
     } else if (strcmp(argv[i], "--no-experimental") == 0) {
@@ -416,7 +422,9 @@ int main(int argc, char* argv[]) {
         ++truncate_by;
       } while (can_truncate);
     }
-    printf("RunTime: %.f ms\n", total_time);
+    if (!settings.print_tokens_for_compare) {
+      printf("RunTime: %.f ms\n", total_time);
+    }
   }
   v8::V8::Dispose();
   return 0;