Experimental parser: Small fixes.

src/lexer/lexer-shell.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 91 matching lines @@
                                          bool read_all_at_once)
     : current_(0),
       fetched_(0),
       read_all_at_once_(read_all_at_once),
       source_(0),
       length_(0) {
   file_ = fopen(fname, "rb");
   scanner_ = new PushScanner(this);
   if (read_all_at_once_) {
     source_ = ReadFile(fname, NULL, &length_);
+    token_.resize(1500);
+    beg_.resize(1500);
+    end_.resize(1500);
+  } else {
+    token_.resize(BUFFER_SIZE);
+    beg_.resize(BUFFER_SIZE);
+    end_.resize(BUFFER_SIZE);
   }
 }
 
 
 ExperimentalScanner::~ExperimentalScanner() {
   fclose(file_);
   delete[] source_;
 }
 
 
 void ExperimentalScanner::FillTokens() {
   current_ = 0;
   fetched_ = 0;
   if (read_all_at_once_) {
     scanner_->push(source_, length_ + 1);
   } else {
     uint8_t chars[BUFFER_SIZE];
     int n = static_cast<int>(fread(&chars, 1, BUFFER_SIZE, file_));
     for (int i = n; i < BUFFER_SIZE; i++) chars[i] = 0;
     scanner_->push(chars, BUFFER_SIZE);
   }
 }
 
 
 Token::Value ExperimentalScanner::Next(int* beg_pos, int* end_pos) {
-  while (current_ == fetched_) {
+  while (current_ == fetched_)
     FillTokens();
-  }
   *beg_pos = beg_[current_];
   *end_pos = end_[current_];
   Token::Value res = token_[current_];
-  if (token_[current_] != Token::Token::EOS &&
-      token_[current_] != Token::ILLEGAL) {
+  if (res != Token::Token::EOS)
     current_++;
-  }
   return res;
 }
 
 
 void ExperimentalScanner::Record(Token::Value token, int beg, int end) {
   if (token == Token::EOS) end--;
   if (fetched_ >= token_.size()) {
-    token_.push_back(token);
-    beg_.push_back(beg);
-    end_.push_back(end);
-  } else {
-    token_[fetched_] = token;
-    beg_[fetched_] = beg;
-    end_[fetched_] = end;
+    token_.resize(token_.size() * 2);
+    beg_.resize(beg_.size() * 2);
+    end_.resize(end_.size() * 2);
   }
+  token_[fetched_] = token;
+  beg_[fetched_] = beg;
+  end_[fetched_] = end;
   fetched_++;
 }
 
 
 int main(int argc, char* argv[]) {
   v8::V8::InitializeICU();
   v8::V8::SetFlagsFromCommandLine(&argc, argv, true);
   v8::Isolate* isolate = v8::Isolate::GetCurrent();
   {
     v8::HandleScope handle_scope(isolate);
@@ skipping 47 matching lines @@
           printf("MISMATCH:\n");
           printf("Expected: %s at (%d, %d)\n",
                  Token::Name(baseline_tokens[i]),
                  baseline_beg[i], baseline_end[i]);
           printf("Actual:   %s at (%d, %d)\n",
                  Token::Name(experimental_tokens[i]),
                  experimental_beg[i], experimental_end[i]);
           return 1;
         }
       }
+      printf("No of tokens: %d\n", experimental_tokens.size());
       printf("Baseline: %f ms\nExperimental %f ms\n",
              baseline_time.InMillisecondsF(),
              experimental_time.InMillisecondsF());
     }
   }
   v8::V8::Dispose();
   return 0;
 }