Experimental parser: Refactoring and timing.

src/lexer/lexer.h

+// Portions of this code based on re2c:
+//   (re2c/examples/push.re)
 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 10 matching lines @@
 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #ifndef V8_LEXER_LEXER_H
 #define V8_LEXER_LEXER_H
 
+#include <vector>
+
 #include "token.h"
 #include "flags.h"
 
-class PushScanner;
+class ExperimentalScanner;
+
+class PushScanner {
+ public:
+  explicit PushScanner(ExperimentalScanner* sink);
+
+  ~PushScanner();
+
+  void send(v8::internal::Token::Value token);
+  uint32_t push(const void *input, int input_size);
+
+ private:
+  bool eof_;
+  int32_t state_;
+  int32_t condition_;
+
+  uint8_t* limit_;
+  uint8_t* start_;
+  uint8_t* cursor_;
+  uint8_t* marker_;
+  int real_start_;
+
+  uint8_t* buffer_;
+  uint8_t* buffer_end_;
+
+  uint8_t yych;
+  uint32_t yyaccept;
+
+  ExperimentalScanner* sink_;
+};
 
 class ExperimentalScanner {
  public:
-  explicit ExperimentalScanner(const char* fname);
+  ExperimentalScanner(const char* fname, bool read_all_at_once);
   ~ExperimentalScanner();
   v8::internal::Token::Value Next(int* beg_pos, int* end_pos);
   void Record(v8::internal::Token::Value token, int beg_pos, int end_pos);
+
  private:
   void FillTokens();
   static const int BUFFER_SIZE = 256;
-  v8::internal::Token::Value token_[BUFFER_SIZE];
-  int beg_[BUFFER_SIZE];
-  int end_[BUFFER_SIZE];
-  int current_;
-  int fetched_;
+  std::vector<v8::internal::Token::Value> token_;
+  std::vector<int> beg_;
+  std::vector<int> end_;
+  size_t current_;
+  size_t fetched_;
   FILE* file_;
   PushScanner* scanner_;
+  bool read_all_at_once_;
+  const v8::internal::byte* source_;
+  int length_;
 };
 
 #endif  // V8_LEXER_LEXER_H