Experimental parser: add ability to compare lexer shell outputs

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 42 matching lines @@
   UTF8,
   UTF16,
   UTF8TO16,  // Convert stream via scanner input stream
   UTF8TO16_PRECONVERT  // Convert stream during file read
 };
 
 
 struct LexerShellSettings {
   Encoding encoding;
   bool print_tokens;
+  bool print_tokens_for_compare;
   bool break_after_illegal;
   bool eos_test;
   int repeat;
   bool harmony_numeric_literals;
   bool harmony_modules;
   bool harmony_scoping;
   LexerShellSettings()
       : encoding(LATIN1),
         print_tokens(false),
+        print_tokens_for_compare(false),
         break_after_illegal(false),
         eos_test(false),
         repeat(1),
         harmony_numeric_literals(false),
         harmony_modules(false),
         harmony_scoping(false) {}
 };
 
 
 static uint16_t* ConvertUtf8ToUtf16(const uint16_t* const data_in,
@@ skipping 96 matching lines @@
   if (settings.repeat > 1) {
     uint16_t* new_data = Repeat(settings.repeat, data, length);
     delete data;
     data = new_data;
   }
 
   return data;
 }
 
 
-struct TokenWithLocation {
-  Token::Value value;
-  size_t beg;
-  size_t end;
-  std::vector<int> literal;
-  bool is_ascii;
-  // The location of the latest octal position when the token was seen.
-  int octal_beg;
-  int octal_end;
-  TokenWithLocation() :
-      value(Token::ILLEGAL), beg(0), end(0), is_ascii(false) { }
-  TokenWithLocation(Token::Value value, size_t beg, size_t end,
-                    int octal_beg) :
-      value(value), beg(beg), end(end), is_ascii(false), octal_beg(octal_beg) {
-  }
-  bool operator==(const TokenWithLocation& other) {
-    return value == other.value && beg == other.beg && end == other.end &&
-           literal == other.literal && is_ascii == other.is_ascii &&
-        octal_beg == other.octal_beg;
-  }
-  bool operator!=(const TokenWithLocation& other) {
-    return !(*this == other);
-  }
-  void Print(const char* prefix) const {
-    printf("%s %11s at (%d, %d)",
-           prefix, Token::Name(value),
-           static_cast<int>(beg), static_cast<int>(end));
-    if (literal.size() > 0) {
-      for (size_t i = 0; i < literal.size(); i++) {
-        printf(is_ascii ? " %02x" : " %04x", literal[i]);
-      }
-      printf(" (is ascii: %d)", is_ascii);
-    }
-    printf(" (last octal start: %d)\n", octal_beg);
-  }
-};
-
-
 static bool HasLiteral(Token::Value token) {
   return token == Token::IDENTIFIER ||
          token == Token::STRING ||
          token == Token::NUMBER;
 }
 
 
 template<typename Char>
-static std::vector<int> ToStdVector(const Vector<Char>& literal) {
-  std::vector<int> result;
+static void Copy(const Vector<Char>& literal,
+                 SmartArrayPointer<const uint16_t>* result,
+                 int* literal_length) {
+  uint16_t* data = new uint16_t[literal.length()];
+  result->Reset(data);
   for (int i = 0; i < literal.length(); i++) {
-    result.push_back(literal[i]);
+    data[i] = literal[i];
   }
-  return result;
+  *literal_length = literal.length();
 }
 
 
-template<typename Scanner>
-static TokenWithLocation GetTokenWithLocation(
-    Scanner *scanner, Token::Value token) {
-  int beg = scanner->location().beg_pos;
-  int end = scanner->location().end_pos;
-  TokenWithLocation result(token, beg, end, scanner->octal_position().beg_pos);
-  if (HasLiteral(token)) {
-    result.is_ascii = scanner->is_literal_ascii();
-    if (scanner->is_literal_ascii()) {
-      result.literal = ToStdVector(scanner->literal_ascii_string());
-    } else {
-      result.literal = ToStdVector(scanner->literal_utf16_string());
+class TokenWithLocation {
+ public:
+  Token::Value value;
+  int beg;
+  int end;
+  bool is_one_byte;
+  SmartArrayPointer<const uint16_t> literal;
+  int literal_length;
+  // The location of the latest octal position when the token was seen.
+  int octal_beg;
+  int octal_end;
+  TokenWithLocation(Token::Value token, Scanner* scanner) : value(token) {
+    beg = scanner->location().beg_pos;
+    end = scanner->location().end_pos;
+    octal_beg = scanner->octal_position().beg_pos;
+    octal_end = scanner->octal_position().end_pos;
+    is_one_byte = false;
+    literal_length = 0;
+    if (HasLiteral(token)) {
+      is_one_byte = scanner->is_literal_ascii();
+      if (scanner->is_literal_ascii()) {
+        Copy(scanner->literal_ascii_string(), &literal, &literal_length);
+      } else {
+        Copy(scanner->literal_utf16_string(), &literal, &literal_length);
+      }
     }
   }
-  return result;
-}
+  void Print(bool do_compare) const {
+    if (value == Token::ILLEGAL && do_compare) {
+      printf("%-15s (%d)\n", Token::Name(value), beg);
+      return;
+    }
+    printf("%-15s (%d, %d)", Token::Name(value), beg, end);
+    if (literal_length > 0) {
+      // TODO(dcarney): need some sort of checksum.
+      for (int i = 0; i < literal_length; i++) {
+        printf(is_one_byte ? " %02x" : " %04x", literal[i]);
+      }
+      printf(" (is_one_byte: %d)", is_one_byte);
+    }
+    if (octal_beg >= 0) {
+      printf(" (last octal start: %d)", octal_beg);
+    }
+    printf("\n");
+  }
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(TokenWithLocation);
+};
 
 
 static TimeDelta RunLexer(const uint16_t* source,
                           const uint8_t* source_end,
                           Isolate* isolate,
-                          std::vector<TokenWithLocation>* tokens,
                           const LexerShellSettings& settings) {
   SmartPointer<Utf16CharacterStream> stream;
   const uint8_t* one_byte_source = reinterpret_cast<const uint8_t*>(source);
   int bytes = source_end - one_byte_source;
   switch (settings.encoding) {
     case UTF8TO16:
     case UTF8:
       stream.Reset(new Utf8ToUtf16CharacterStream(one_byte_source, bytes));
       break;
     case UTF8TO16_PRECONVERT:
@@ skipping 11 matching lines @@
       stream.Reset(
           new GenericStringUtf16CharacterStream(result, 0, result->length()));
       break;
     }
   }
   Scanner scanner(isolate->unicode_cache());
   scanner.SetHarmonyNumericLiterals(settings.harmony_numeric_literals);
   scanner.SetHarmonyModules(settings.harmony_modules);
   scanner.SetHarmonyScoping(settings.harmony_scoping);
   ElapsedTimer timer;
+  std::vector<TokenWithLocation*> tokens;
   timer.Start();
   scanner.Initialize(stream.get());
   Token::Value token;
   do {
     token = scanner.Next();
     if (settings.print_tokens) {
-      tokens->push_back(GetTokenWithLocation(&scanner, token));
+      tokens.push_back(new TokenWithLocation(token, &scanner));
     } else if (HasLiteral(token)) {
       if (scanner.is_literal_ascii()) {
         scanner.literal_ascii_string();
       } else {
         scanner.literal_utf16_string();
       }
     }
+    if (token == Token::ILLEGAL && settings.break_after_illegal) break;
   } while (token != Token::EOS);
+  // Dump tokens.
+  if (settings.print_tokens) {
+    if (!settings.print_tokens_for_compare) {
+      printf("No of tokens:\t%d\n", static_cast<int>(tokens.size()));
+    }
+    for (size_t i = 0; i < tokens.size(); ++i) {
+      tokens[i]->Print(settings.print_tokens_for_compare);
+    }
+  }
+  for (size_t i = 0; i < tokens.size(); ++i) {
+    delete tokens[i];
+  }
   return timer.Elapsed();
 }
 
 
 static TimeDelta ProcessFile(
     const char* fname,
     Isolate* isolate,
     const LexerShellSettings& settings,
     int truncate_by,
     bool* can_truncate) {
-  if (settings.print_tokens) {
+  if (settings.print_tokens && !settings.print_tokens_for_compare) {
     printf("Processing file %s, truncating by %d bytes\n", fname, truncate_by);
   }
   HandleScope handle_scope(isolate);
-  std::vector<TokenWithLocation> tokens;
   TimeDelta time;
   {
     unsigned length_in_bytes;
     const uint16_t* buffer = ReadFile(fname, settings, &length_in_bytes);
     const uint8_t* char_data = reinterpret_cast<const uint8_t*>(buffer);
     const uint8_t* buffer_end = &char_data[length_in_bytes];
     if (truncate_by > buffer_end - char_data) {
       *can_truncate = false;
     } else {
       buffer_end -= truncate_by;
-      time = RunLexer(buffer, buffer_end, isolate, &tokens, settings);
+      time = RunLexer(buffer, buffer_end, isolate, settings);
     }
     delete[] buffer;
   }
-  if (settings.print_tokens) {
-    printf("No of tokens:\t%d\n", static_cast<int>(tokens.size()));
-    for (size_t i = 0; i < tokens.size(); ++i) {
-      tokens[i].Print("=>");
-      if (tokens[i].value == Token::ILLEGAL) {
-        if (settings.break_after_illegal)
-          break;
-      }
-    }
-  }
+
   return time;
 }
 
 
 int main(int argc, char* argv[]) {
   v8::V8::InitializeICU();
   v8::V8::SetFlagsFromCommandLine(&argc, argv, true);
   std::vector<std::string> fnames;
   LexerShellSettings settings;
   for (int i = 0; i < argc; ++i) {
     if (strcmp(argv[i], "--latin1") == 0) {
       settings.encoding = LATIN1;
     } else if (strcmp(argv[i], "--utf8") == 0) {
       settings.encoding = UTF8;
     } else if (strcmp(argv[i], "--utf16") == 0) {
       settings.encoding = UTF16;
     } else if (strcmp(argv[i], "--utf8to16") == 0) {
 #ifdef V8_USE_GENERATED_LEXER
       settings.encoding = UTF8TO16_PRECONVERT;
 #else
       settings.encoding = UTF8TO16;
 #endif
     } else if (strcmp(argv[i], "--print-tokens") == 0) {
       settings.print_tokens = true;
+    } else if (strcmp(argv[i], "--print-tokens-for-compare") == 0) {
+      settings.print_tokens = true;
+      settings.print_tokens_for_compare = true;
     } else if (strcmp(argv[i], "--no-baseline") == 0) {
       // Ignore.
     } else if (strcmp(argv[i], "--no-experimental") == 0) {
       // Ignore.
     } else if (strcmp(argv[i], "--no-check") == 0) {
       // Ignore.
     } else if (strcmp(argv[i], "--break-after-illegal") == 0) {
       settings.break_after_illegal = true;
     } else if (strcmp(argv[i], "--use-harmony") == 0) {
       settings.harmony_numeric_literals = true;
@@ skipping 25 matching lines @@
       do {
         TimeDelta t = ProcessFile(fnames[i].c_str(),
                                   internal_isolate,
                                   settings,
                                   truncate_by,
                                   &can_truncate);
         total_time += t.InMillisecondsF();
         ++truncate_by;
       } while (can_truncate);
     }
-    printf("RunTime: %.f ms\n", total_time);
+    if (!settings.print_tokens_for_compare) {
+      printf("RunTime: %.f ms\n", total_time);
+    }
   }
   v8::V8::Dispose();
   return 0;
 }