Experimental parser: Refactoring and timing.

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 12 matching lines @@
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <assert.h>
 #include <fcntl.h>
 #include <string.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <sys/time.h>
 #include "v8.h"
 
 #include "api.h"
 #include "ast.h"
 #include "char-predicates-inl.h"
 #include "messages.h"
 #include "platform.h"
 #include "runtime.h"
 #include "scanner-character-streams.h"
 #include "scopeinfo.h"
@@ skipping 31 matching lines @@
     unicode_cache_ = new UnicodeCache();
     scanner_ = new Scanner(unicode_cache_);
     stream_ = new Utf8ToUtf16CharacterStream(source_, length);
     scanner_->Initialize(stream_);
   }
 
   ~BaselineScanner() {
     delete scanner_;
     delete stream_;
     delete unicode_cache_;
-    delete source_;
+    delete[] source_;
   }
 
   Token::Value Next(int* beg_pos, int* end_pos) {
     Token::Value res = scanner_->Next();
     *beg_pos = scanner_->location().beg_pos;
     *end_pos = scanner_->location().end_pos;
     return res;
   }
 
  private:
   UnicodeCache* unicode_cache_;
   Scanner* scanner_;
   const byte* source_;
   Utf8ToUtf16CharacterStream* stream_;
 };
 
+ExperimentalScanner::ExperimentalScanner(const char* fname,
+                                         bool read_all_at_once)
+    : current_(0),
+      fetched_(0),
+      read_all_at_once_(read_all_at_once),
+      source_(0),
+      length_(0) {
+  file_ = fopen(fname, "rb");
+  scanner_ = new PushScanner(this);
+  if (read_all_at_once_) {
+    source_ = ReadFile(fname, NULL, &length_);
+  }
+}
+
+
+ExperimentalScanner::~ExperimentalScanner() {
+  fclose(file_);
+  delete[] source_;
+}
+
+
+void ExperimentalScanner::FillTokens() {
+  current_ = 0;
+  fetched_ = 0;
+  if (read_all_at_once_) {
+    scanner_->push(source_, length_ + 1);
+  } else {
+    uint8_t chars[BUFFER_SIZE];
+    int n = static_cast<int>(fread(&chars, 1, BUFFER_SIZE, file_));
+    for (int i = n; i < BUFFER_SIZE; i++) chars[i] = 0;
+    scanner_->push(chars, BUFFER_SIZE);
+  }
+}
+
+
+Token::Value ExperimentalScanner::Next(int* beg_pos, int* end_pos) {
+  while (current_ == fetched_) {
+    FillTokens();
+  }
+  *beg_pos = beg_[current_];
+  *end_pos = end_[current_];
+  Token::Value res = token_[current_];
+  if (token_[current_] != Token::Token::EOS &&
+      token_[current_] != Token::ILLEGAL) {
+    current_++;
+  }
+  return res;
+}
+
+
+void ExperimentalScanner::Record(Token::Value token, int beg, int end) {
+  if (token == Token::EOS) end--;
+  if (fetched_ >= token_.size()) {
+    token_.push_back(token);
+    beg_.push_back(beg);
+    end_.push_back(end);
+  } else {
+    token_[fetched_] = token;
+    beg_[fetched_] = beg;
+    end_[fetched_] = end;
+  }
+  fetched_++;
+}
+
 
 int main(int argc, char* argv[]) {
   v8::V8::InitializeICU();
   v8::V8::SetFlagsFromCommandLine(&argc, argv, true);
   v8::Isolate* isolate = v8::Isolate::GetCurrent();
   {
     v8::HandleScope handle_scope(isolate);
     v8::Handle<v8::ObjectTemplate> global = v8::ObjectTemplate::New();
     v8::Local<v8::Context> context = v8::Context::New(isolate, NULL, global);
     ASSERT(!context.IsEmpty());
     {
       v8::Context::Scope scope(context);
       Isolate* isolate = Isolate::Current();
       HandleScope handle_scope(isolate);
       BaselineScanner baseline(argv[1], isolate);
-      ExperimentalScanner experimental(argv[1]);
-      Token::Value expected_token, actual_token;
-      int expected_beg, expected_end, actual_beg, actual_end;
+      ExperimentalScanner experimental(argv[1], true);
+
+      std::vector<Token::Value> baseline_tokens, experimental_tokens;
+      std::vector<size_t> baseline_beg, baseline_end, experimental_beg,
+          experimental_end;
+      Token::Value token;
+      int beg, end;
+
+      struct timeval baseline_tv_start, baseline_tv_stop, experimental_tv_start,
+          experimental_tv_stop;
+      gettimeofday(&baseline_tv_start, NULL);

[ulan, 2013/10/17 13:43:32]

We can use ElapsedTimer of v8 here.

[marja, 2013/10/17 13:56:34]

Done.

       do {
-        expected_token = baseline.Next(&expected_beg, &expected_end);
-        actual_token = experimental.Next(&actual_beg, &actual_end);
+        token = baseline.Next(&beg, &end);
+        baseline_tokens.push_back(token);
+        baseline_beg.push_back(beg);
+        baseline_end.push_back(end);
+      } while (token != Token::EOS);
+      gettimeofday(&baseline_tv_stop, NULL);
+
+      gettimeofday(&experimental_tv_start, NULL);
+      do {
+        token = experimental.Next(&beg, &end);
+        experimental_tokens.push_back(token);
+        experimental_beg.push_back(beg);
+        experimental_end.push_back(end);
+      } while (token != Token::EOS);
+      gettimeofday(&experimental_tv_stop, NULL);
+
+      for (size_t i = 0; i < experimental_tokens.size(); ++i) {
         printf("=> %11s at (%d, %d)\n",
-               Token::Name(actual_token),
-               actual_beg, actual_end);
-        if (expected_token != actual_token ||
-            expected_beg != actual_beg ||
-            expected_end != actual_end) {
+               Token::Name(experimental_tokens[i]),
+               experimental_beg[i], experimental_end[i]);
+        if (experimental_tokens[i] != baseline_tokens[i] ||
+            experimental_beg[i] != baseline_beg[i] ||
+            experimental_end[i] != baseline_end[i]) {
           printf("MISMATCH:\n");
           printf("Expected: %s at (%d, %d)\n",
-                 Token::Name(expected_token),
-                 expected_beg, expected_end);
+                 Token::Name(baseline_tokens[i]),
+                 baseline_beg[i], baseline_end[i]);
           printf("Actual:   %s at (%d, %d)\n",
-                 Token::Name(actual_token),
-                 actual_beg, actual_end);
+                 Token::Name(experimental_tokens[i]),
+                 experimental_beg[i], experimental_end[i]);
           return 1;
         }
-      } while (actual_token != Token::EOS);
+      }
+      fprintf(stderr,
+              "Baseline: %ld\n",
+              1000000 * (baseline_tv_stop.tv_sec - baseline_tv_start.tv_sec) +
+                  baseline_tv_stop.tv_usec - baseline_tv_start.tv_usec);
+      fprintf(stderr,
+              "Experimental: %ld\n",
+              1000000 * (experimental_tv_stop.tv_sec -
+                         experimental_tv_start.tv_sec) +
+                  experimental_tv_stop.tv_usec - experimental_tv_start.tv_usec);
     }
   }
   v8::V8::Dispose();
   return 0;
 }