Experimental parser: Starting to unify the ExperimentalScanner iface with Scanner iface.

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 24 matching lines @@
 #include "api.h"
 #include "ast.h"
 #include "char-predicates-inl.h"
 #include "messages.h"
 #include "platform.h"
 #include "runtime.h"
 #include "scanner-character-streams.h"
 #include "scopeinfo.h"
 #include "string-stream.h"
 #include "scanner.h"
+
+#include "experimental-scanner.h"
 #include "lexer.h"
 
 using namespace v8::internal;
 
 const byte* ReadFile(const char* name, Isolate* isolate, int* size) {
   FILE* file = fopen(name, "rb");
   *size = 0;
   if (file == NULL) return NULL;
 
   fseek(file, 0, SEEK_END);
@@ skipping 36 matching lines @@
     return res;
   }
 
  private:
   UnicodeCache* unicode_cache_;
   Scanner* scanner_;
   const byte* source_;
   Utf8ToUtf16CharacterStream* stream_;
 };
 
-ExperimentalScanner::ExperimentalScanner(const char* fname,
-                                         bool read_all_at_once)
-    : current_(0),
-      fetched_(0),
-      read_all_at_once_(read_all_at_once),
-      source_(0),
-      length_(0) {
-  file_ = fopen(fname, "rb");
-  scanner_ = new PushScanner(this);
-  if (read_all_at_once_) {
-    source_ = ReadFile(fname, NULL, &length_);
-    token_.resize(1500);
-    beg_.resize(1500);
-    end_.resize(1500);
-  } else {
-    token_.resize(BUFFER_SIZE);
-    beg_.resize(BUFFER_SIZE);
-    end_.resize(BUFFER_SIZE);
-  }
-}
-
-
-ExperimentalScanner::~ExperimentalScanner() {
-  fclose(file_);
-  delete[] source_;
-}
-
-
-void ExperimentalScanner::FillTokens() {
-  current_ = 0;
-  fetched_ = 0;
-  if (read_all_at_once_) {
-    scanner_->push(source_, length_ + 1);
-  } else {
-    uint8_t chars[BUFFER_SIZE];
-    int n = static_cast<int>(fread(&chars, 1, BUFFER_SIZE, file_));
-    for (int i = n; i < BUFFER_SIZE; i++) chars[i] = 0;
-    scanner_->push(chars, BUFFER_SIZE);
-  }
-}
-
-
-Token::Value ExperimentalScanner::Next(int* beg_pos, int* end_pos) {
-  while (current_ == fetched_)
-    FillTokens();
-  *beg_pos = beg_[current_];
-  *end_pos = end_[current_];
-  Token::Value res = token_[current_];
-  if (res != Token::Token::EOS)
-    current_++;
-  return res;
-}
-
-
-void ExperimentalScanner::Record(Token::Value token, int beg, int end) {
-  if (token == Token::EOS) end--;
-  if (fetched_ >= token_.size()) {
-    token_.resize(token_.size() * 2);
-    beg_.resize(beg_.size() * 2);
-    end_.resize(end_.size() * 2);
-  }
-  token_[fetched_] = token;
-  beg_[fetched_] = beg;
-  end_[fetched_] = end;
-  fetched_++;
-}
-
 
 int main(int argc, char* argv[]) {
   v8::V8::InitializeICU();
   v8::V8::SetFlagsFromCommandLine(&argc, argv, true);
   v8::Isolate* isolate = v8::Isolate::GetCurrent();
   {
     v8::HandleScope handle_scope(isolate);
     v8::Handle<v8::ObjectTemplate> global = v8::ObjectTemplate::New();
     v8::Local<v8::Context> context = v8::Context::New(isolate, NULL, global);
     ASSERT(!context.IsEmpty());
@@ skipping 19 matching lines @@
           baseline_tokens.push_back(token);
           baseline_beg.push_back(beg);
           baseline_end.push_back(end);
         } while (token != Token::EOS);
         baseline_time = timer.Elapsed();
       }
 
       {
         timer.Start();
         do {
-          token = experimental.Next(&beg, &end);
+          token = experimental.Next();
           experimental_tokens.push_back(token);
-          experimental_beg.push_back(beg);
-          experimental_end.push_back(end);
+          ExperimentalScanner::Location location = experimental.location();
+          experimental_beg.push_back(location.beg_pos);
+          experimental_end.push_back(location.end_pos);
         } while (token != Token::EOS);
         experimental_time = timer.Elapsed();
       }
 
       for (size_t i = 0; i < experimental_tokens.size(); ++i) {
         printf("=> %11s at (%d, %d)\n",
                Token::Name(experimental_tokens[i]),
                experimental_beg[i], experimental_end[i]);
         if (experimental_tokens[i] != baseline_tokens[i] ||
             experimental_beg[i] != baseline_beg[i] ||
@@ skipping 10 matching lines @@
       }
       printf("No of tokens: %d\n", experimental_tokens.size());
       printf("Baseline: %f ms\nExperimental %f ms\n",
              baseline_time.InMillisecondsF(),
              experimental_time.InMillisecondsF());
     }
   }
   v8::V8::Dispose();
   return 0;
 }