Experimental parser: eos test prints less stuff

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 263 matching lines @@
         printf(is_one_byte ? " %02x" : " %04x", literal[i]);
       }
       printf(" (is_one_byte: %d)", is_one_byte);
     }
     if (octal_beg >= 0) {
       printf(" (last octal start: %d)", octal_beg);
     }
     printf("\n");
   }
 
+  bool operator==(const TokenWithLocation& that) {
+    if (this->beg != that.beg ||
+        this->end != that.end ||
+        this->is_one_byte != that.is_one_byte ||
+        this->literal_length != that.literal_length ||
+        this->octal_beg != that.octal_beg ||
+        this->octal_end != that.octal_end
+      ) {
+      return false;
+    }
+    for (int i = 0; i < literal_length; i++) {
+      if (this->literal[i] != that.literal[i]) return false;
+    }
+    return true;
+  }
+
  private:
   Token::Value value;
   int beg;
   int end;
   bool is_one_byte;
   SmartArrayPointer<uint16_t> literal;
   int literal_length;
   // The location of the latest octal position when the token was seen.
   int octal_beg;
   int octal_end;
@@ skipping 54 matching lines @@
     }
     if (settings.print_tokens) {
       tokens->push_back(new TokenWithLocation(token, &scanner, literal));
     }
     if (token == Token::ILLEGAL && settings.break_after_illegal) break;
   } while (token != Token::EOS);
   return timer.Elapsed();
 }
 
 
+static void DumpTokens(const LexerShellSettings& settings,
+                       TokenVector* first_tokens,
+                       TokenVector* eos_tokens) {
+  if (!settings.print_tokens) return;
+  bool first_run = eos_tokens->size() == 0;
+  if (first_run && !settings.print_tokens_for_compare) {
+    printf("No of tokens:\t%d\n", static_cast<int>(first_tokens->size()));
+  }
+  if (first_run) {
+    for (size_t i = 0; i < first_tokens->size(); ++i) {
+      first_tokens->at(i)->Print(settings.print_tokens_for_compare);
+    }
+    return;
+  }
+  for (size_t i = 0; i < eos_tokens->size(); ++i) {
+    if (i <= first_tokens->size() &&
+        eos_tokens->at(i)->operator==(*first_tokens->at(i))) {
+      continue;
+    }
+    eos_tokens->at(i)->Print(settings.print_tokens_for_compare);
+  }
+}
+
+
+static void Clear(TokenVector* tokens) {
+  for (size_t i = 0; i < tokens->size(); ++i) delete tokens->at(i);
+  tokens->clear();
+}
+
+
 static void Run(const LexerShellSettings& settings,
                 const FileData& file_data) {
   Isolate* isolate = Isolate::Current();
   HandleScope handle_scope(isolate);
   v8::Context::Scope scope(v8::Context::New(v8::Isolate::GetCurrent()));
   double total_time = 0;
-  std::vector<TokenWithLocation*> tokens;
+  TokenVector first_tokens;
+  TokenVector eos_tokens;
   const uint16_t* const buffer = file_data.data;
   const uint8_t* const char_data = reinterpret_cast<const uint8_t*>(buffer);
   for (unsigned truncate_by = 0;
        truncate_by <= file_data.length_in_bytes;
        truncate_by += file_data.encoding == UTF16 ? 2 : 1) {
     if (settings.print_tokens && !settings.print_tokens_for_compare) {
       printf("Processing file %s, truncating by %d bytes\n",
              file_data.file_name, truncate_by);
     }
     HandleScope handle_scope(isolate);
     const uint8_t* buffer_end =
       &char_data[file_data.length_in_bytes] - truncate_by;
+    TokenVector* tokens = truncate_by == 0 ? &first_tokens : &eos_tokens;
     TimeDelta delta = RunLexer(
-        buffer, buffer_end, isolate, file_data.encoding, settings, &tokens);
+        buffer, buffer_end, isolate, file_data.encoding, settings, tokens);
     total_time += delta.InMillisecondsF();
     // Dump tokens.
-    if (settings.print_tokens) {
-      if (!settings.print_tokens_for_compare) {
-        printf("No of tokens:\t%d\n", static_cast<int>(tokens.size()));
-      }
-      for (size_t i = 0; i < tokens.size(); ++i) {
-        tokens[i]->Print(settings.print_tokens_for_compare);
-      }
-    }
+    DumpTokens(settings, &first_tokens, &eos_tokens);
+    if (!settings.eos_test) break;
     // Destroy tokens.
-    for (size_t i = 0; i < tokens.size(); ++i) {
-      delete tokens[i];
-    }
-    tokens.clear();
-    if (!settings.eos_test) break;
+    Clear(&eos_tokens);
   }
+  // Destroy tokens.
+  Clear(&first_tokens);
   if (!settings.print_tokens_for_compare) {
     printf("RunTime: %.f ms\n", total_time);
   }
 }
 
 
 int main(int argc, char* argv[]) {
   v8::V8::InitializeICU();
   v8::V8::SetFlagsFromCommandLine(&argc, argv, true);
   std::string file_name;
@@ skipping 37 matching lines @@
       file_name = std::string(argv[i]);
     }
   }
   CHECK_NE(0, file_name.size());
   FileData file_data = ReadFile(file_name.c_str(), settings);
   Run(settings, file_data);
   delete[] file_data.data;
   v8::V8::Dispose();
   return 0;
 }