Experimental parser: add a possibility to run lexing tests for truncated files.

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 31 matching lines @@
 #include "runtime.h"
 #include "scanner-character-streams.h"
 #include "scopeinfo.h"
 #include "string-stream.h"
 #include "scanner.h"
 
 #include "experimental-scanner.h"
 
 using namespace v8::internal;
 
-byte* ReadFile(const char* name, byte** end, int repeat,
+byte* ReadFile(const char* name, const byte** end, int repeat,
                bool convert_to_utf16) {
   FILE* file = fopen(name, "rb");
   if (file == NULL) return NULL;
 
   fseek(file, 0, SEEK_END);
   int file_size = ftell(file);
   rewind(file);
 
   int size = file_size * repeat;
 
@@ skipping 39 matching lines @@
 
 struct HarmonySettings {
   bool numeric_literals;
   bool modules;
   bool scoping;
   HarmonySettings() : numeric_literals(false), modules(false), scoping(false) {}
 };
 
 class BaselineScanner {
  public:
-  BaselineScanner(const char* fname,
+  BaselineScanner(const byte* source,
+                  const byte* source_end,
                   Isolate* isolate,
                   Encoding encoding,
                   ElapsedTimer* timer,
                   int repeat,
                   HarmonySettings harmony_settings)
-      : stream_(NULL) {
-    byte* end = 0;
-    source_ = ReadFile(fname, &end, repeat, false);
+      : source_(source), stream_(NULL) {
     unicode_cache_ = new UnicodeCache();
     scanner_ = new Scanner(unicode_cache_);
     scanner_->SetHarmonyNumericLiterals(harmony_settings.numeric_literals);
     scanner_->SetHarmonyModules(harmony_settings.modules);
     scanner_->SetHarmonyScoping(harmony_settings.scoping);
     switch (encoding) {
       case UTF8:
       case UTF8TO16:
-        stream_ = new Utf8ToUtf16CharacterStream(source_, end - source_);
+        stream_ = new Utf8ToUtf16CharacterStream(source_, source_end - source_);
         break;
       case UTF16: {
         Handle<String> result = isolate->factory()->NewStringFromTwoByte(
             Vector<const uint16_t>(
                 reinterpret_cast<const uint16_t*>(source_),
-                (end - source_) / 2));
+                (source_end - source_) / 2));
         stream_ =
             new GenericStringUtf16CharacterStream(result, 0, result->length());
         break;
       }
       case LATIN1: {
         Handle<String> result = isolate->factory()->NewStringFromOneByte(
-            Vector<const uint8_t>(source_, end - source_));
+            Vector<const uint8_t>(source_, source_end - source_));
         stream_ =
             new GenericStringUtf16CharacterStream(result, 0, result->length());
         break;
       }
     }
     timer->Start();
     scanner_->Initialize(stream_);
   }
 
   ~BaselineScanner() {
     delete scanner_;
     delete stream_;
     delete unicode_cache_;
-    delete[] source_;
   }
 
   Scanner* scanner_;
 
  private:
   UnicodeCache* unicode_cache_;
   const byte* source_;
   BufferedUtf16CharacterStream* stream_;
 };
 
@@ skipping 63 matching lines @@
     if (scanner->is_literal_ascii()) {
       result.literal = ToStdVector(scanner->literal_ascii_string());
     } else {
       result.literal = ToStdVector(scanner->literal_utf16_string());
     }
   }
   return result;
 }
 
 
-TimeDelta RunBaselineScanner(const char* fname,
+TimeDelta RunBaselineScanner(const byte* source,
+                             const byte* source_end,
                              Isolate* isolate,
                              Encoding encoding,
                              bool dump_tokens,
                              std::vector<TokenWithLocation>* tokens,
                              int repeat,
                              HarmonySettings harmony_settings) {
   ElapsedTimer timer;
-  BaselineScanner scanner(
-      fname, isolate, encoding, &timer, repeat, harmony_settings);
+  BaselineScanner scanner(source,
+                          source_end,
+                          isolate,
+                          encoding,
+                          &timer,
+                          repeat,
+                          harmony_settings);
   Token::Value token;
   do {
     token = scanner.scanner_->Next();
     if (dump_tokens) {
       tokens->push_back(GetTokenWithLocation(scanner.scanner_, token));
     } else if (HasLiteral(token)) {
       if (scanner.scanner_->is_literal_ascii()) {
         scanner.scanner_->literal_ascii_string();
       } else {
         scanner.scanner_->literal_utf16_string();
@@ skipping 51 matching lines @@
 std::pair<TimeDelta, TimeDelta> ProcessFile(
     const char* fname,
     Encoding encoding,
     Isolate* isolate,
     bool run_baseline,
     bool run_experimental,
     bool print_tokens,
     bool check_tokens,
     bool break_after_illegal,
     int repeat,
-    HarmonySettings harmony_settings) {
+    HarmonySettings harmony_settings,
+    int truncate_by,
+    bool* can_truncate) {
   if (print_tokens) {
-    printf("Processing file %s\n", fname);
+    printf("Processing file %s, truncating by %d bytes\n", fname, truncate_by);
   }
   HandleScope handle_scope(isolate);
   std::vector<TokenWithLocation> baseline_tokens, experimental_tokens;
   TimeDelta baseline_time, experimental_time;
   if (run_baseline) {
-    baseline_time = RunBaselineScanner(
-        fname, isolate, encoding, print_tokens || check_tokens,
-        &baseline_tokens, repeat, harmony_settings);
+    const byte* buffer_end = 0;
+    const byte* buffer = ReadFile(fname, &buffer_end, repeat, false);
+    if (truncate_by > buffer_end - buffer) {
+      *can_truncate = false;
+    } else {
+      buffer_end -= truncate_by;
+      baseline_time = RunBaselineScanner(
+          buffer, buffer_end, isolate, encoding, print_tokens || check_tokens,
+          &baseline_tokens, repeat, harmony_settings);
+    }
+    delete[] buffer;
   }
   if (run_experimental) {
     Handle<String> source;
-    byte* buffer_end = 0;
+    const byte* buffer_end = 0;
     const byte* buffer = ReadFile(fname, &buffer_end, repeat,
                                   encoding == UTF8TO16);
-    switch (encoding) {
-      case UTF8:
-      case LATIN1:
-        source = isolate->factory()->NewStringFromAscii(
-            Vector<const char>(reinterpret_cast<const char*>(buffer),
-                               buffer_end - buffer));
-        experimental_time = RunExperimentalScanner<uint8_t>(
-            source, isolate, encoding, print_tokens || check_tokens,
-            &experimental_tokens, repeat, harmony_settings);
-        break;
-      case UTF16:
-      case UTF8TO16: {
-        const uc16* buffer_16 = reinterpret_cast<const uc16*>(buffer);
-        const uc16* buffer_end_16 = reinterpret_cast<const uc16*>(buffer_end);
-        source = isolate->factory()->NewStringFromTwoByte(
-            Vector<const uc16>(buffer_16, buffer_end_16 - buffer_16));
-        // If the string was just an expaneded one byte string, V8 detects it
-        // and doesn't store it as two byte.
-        if (!source->IsTwoByteRepresentation()) {
+    if (truncate_by > buffer_end - buffer) {
+      *can_truncate = false;
+    } else {
+      buffer_end -= truncate_by;
+      switch (encoding) {
+        case UTF8:
+        case LATIN1:
+          source = isolate->factory()->NewStringFromAscii(
+              Vector<const char>(reinterpret_cast<const char*>(buffer),
+                                 buffer_end - buffer));
           experimental_time = RunExperimentalScanner<uint8_t>(
               source, isolate, encoding, print_tokens || check_tokens,
               &experimental_tokens, repeat, harmony_settings);
-        } else {
-          experimental_time = RunExperimentalScanner<uint16_t>(
-              source, isolate, encoding, print_tokens || check_tokens,
-              &experimental_tokens, repeat, harmony_settings);
+          break;
+        case UTF16:
+        case UTF8TO16: {
+          const uc16* buffer_16 = reinterpret_cast<const uc16*>(buffer);
+          const uc16* buffer_end_16 = reinterpret_cast<const uc16*>(buffer_end);
+          source = isolate->factory()->NewStringFromTwoByte(
+              Vector<const uc16>(buffer_16, buffer_end_16 - buffer_16));
+          // If the string was just an expaneded one byte string, V8 detects it
+          // and doesn't store it as two byte.
+          if (!source->IsTwoByteRepresentation()) {
+            experimental_time = RunExperimentalScanner<uint8_t>(
+                source, isolate, encoding, print_tokens || check_tokens,
+                &experimental_tokens, repeat, harmony_settings);
+          } else {
+            experimental_time = RunExperimentalScanner<uint16_t>(
+                source, isolate, encoding, print_tokens || check_tokens,
+                &experimental_tokens, repeat, harmony_settings);
+          }
+          break;
         }
-        break;
+        default:
+          printf("Encoding not supported by the experimental scanner\n");
+          exit(1);
+          break;
       }
-      default:
-        printf("Encoding not supported by the experimental scanner\n");
-        exit(1);
-        break;
     }
+    delete[] buffer;
   }
   if (print_tokens && !run_experimental) {
     PrintTokens("Baseline", baseline_tokens);
   }
   if (print_tokens && !run_baseline) {
     PrintTokens("Experimental", experimental_tokens);
   }
   if ((print_tokens || check_tokens) && run_baseline && run_experimental) {
     if (print_tokens) {
       printf("No of tokens in Baseline:     %d\n",
@@ skipping 30 matching lines @@
 
 int main(int argc, char* argv[]) {
   v8::V8::InitializeICU();
   v8::V8::SetFlagsFromCommandLine(&argc, argv, true);
   Encoding encoding = LATIN1;
   bool print_tokens = false;
   bool run_baseline = true;
   bool run_experimental = true;
   bool check_tokens = true;
   bool break_after_illegal = false;
+  bool eos_test = false;
   std::vector<std::string> fnames;
   std::string benchmark;
   int repeat = 1;
   HarmonySettings harmony_settings;
   for (int i = 0; i < argc; ++i) {
     if (strcmp(argv[i], "--latin1") == 0) {
       encoding = LATIN1;
     } else if (strcmp(argv[i], "--utf8") == 0) {
       encoding = UTF8;
     } else if (strcmp(argv[i], "--utf16") == 0) {
@@ skipping 12 matching lines @@
       break_after_illegal = true;
     } else if (strcmp(argv[i], "--use-harmony") == 0) {
       harmony_settings.numeric_literals = true;
       harmony_settings.modules = true;
       harmony_settings.scoping = true;
     } else if (strncmp(argv[i], "--benchmark=", 12) == 0) {
       benchmark = std::string(argv[i]).substr(12);
     } else if (strncmp(argv[i], "--repeat=", 9) == 0) {
       std::string repeat_str = std::string(argv[i]).substr(9);
       repeat = atoi(repeat_str.c_str());
+    } else if (strcmp(argv[i], "--eos-test") == 0) {
+      eos_test = true;
     } else if (i > 0 && argv[i][0] != '-') {
       fnames.push_back(std::string(argv[i]));
     }
   }
+  check_tokens = check_tokens && run_baseline && run_experimental;
   v8::Isolate* isolate = v8::Isolate::GetCurrent();
   {
     v8::HandleScope handle_scope(isolate);
     v8::Handle<v8::ObjectTemplate> global = v8::ObjectTemplate::New();
     v8::Local<v8::Context> context = v8::Context::New(isolate, NULL, global);
     ASSERT(!context.IsEmpty());
     {
       v8::Context::Scope scope(context);
       Isolate* internal_isolate = Isolate::Current();
       double baseline_total = 0, experimental_total = 0;
       for (size_t i = 0; i < fnames.size(); i++) {
         std::pair<TimeDelta, TimeDelta> times;
-        check_tokens = check_tokens && run_baseline && run_experimental;
-        times = ProcessFile(fnames[i].c_str(),
-                            encoding,
-                            internal_isolate,
-                            run_baseline,
-                            run_experimental,
-                            print_tokens,
-                            check_tokens,
-                            break_after_illegal,
-                            repeat,
-                            harmony_settings);
-        baseline_total += times.first.InMillisecondsF();
-        experimental_total += times.second.InMillisecondsF();
+        bool can_truncate = eos_test;
+        int truncate_by = 0;
+        do {

[ulan, 2014/01/24 12:57:39]

for loop would be also possible here.

[marja, 2014/01/24 14:10:18]

Done.

+          times = ProcessFile(fnames[i].c_str(),
+                              encoding,
+                              internal_isolate,
+                              run_baseline,
+                              run_experimental,
+                              print_tokens,
+                              check_tokens,
+                              break_after_illegal,
+                              repeat,
+                              harmony_settings,
+                              truncate_by,
+                              &can_truncate);
+          baseline_total += times.first.InMillisecondsF();
+          experimental_total += times.second.InMillisecondsF();
+          ++truncate_by;
+        } while (can_truncate);
       }
       if (run_baseline) {
         printf("Baseline%s(RunTime): %.f ms\n", benchmark.c_str(),
                baseline_total);
       }
       if (run_experimental) {
         if (benchmark.empty()) benchmark = "Experimental";
         printf("%s(RunTime): %.f ms\n", benchmark.c_str(),
                experimental_total);
       }
     }
   }
   v8::V8::Dispose();
   return 0;
 }