Experimental scanner: save all token data into one vector.

src/lexer/experimental-scanner.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 57 matching lines @@
     : current_(0),
       fetched_(0),
       read_all_at_once_(read_all_at_once),
       source_(0),
       length_(0) {
   file_ = fopen(fname, "rb");
   scanner_ = new PushScanner(this, isolate->unicode_cache());
   if (read_all_at_once_) {
     source_ = ReadFile(fname, isolate, &length_);
     token_.resize(1500);
-    beg_.resize(1500);
-    end_.resize(1500);
   } else {
     token_.resize(BUFFER_SIZE);
-    beg_.resize(BUFFER_SIZE);
-    end_.resize(BUFFER_SIZE);
   }
 }
 
 
 ExperimentalScanner::~ExperimentalScanner() {
   fclose(file_);
   delete[] source_;
 }
 
 
 void ExperimentalScanner::FillTokens() {
   current_ = 0;
   fetched_ = 0;
   if (read_all_at_once_) {
     scanner_->push(source_, length_ + 1);
   } else {
     uint8_t chars[BUFFER_SIZE];
     int n = static_cast<int>(fread(&chars, 1, BUFFER_SIZE, file_));
     for (int i = n; i < BUFFER_SIZE; i++) chars[i] = 0;
     scanner_->push(chars, BUFFER_SIZE);
   }
 }
 
 
 Token::Value ExperimentalScanner::Next() {
   while (current_ == fetched_)
     FillTokens();
-  return token_[current_++];
+  return token_[current_++].value;
 }
 
 
 Token::Value ExperimentalScanner::current_token() {
-  return token_[current_ - 1];
+  return token_[current_ - 1].value;
 }
 
 
 ExperimentalScanner::Location ExperimentalScanner::location() {
-  return Location(beg_[current_ - 1], end_[current_ - 1]);
+  return Location(token_[current_ - 1].beg, token_[current_ - 1].end);
 }
 
 
 void ExperimentalScanner::Record(Token::Value token, int beg, int end) {
   if (token == Token::EOS) end--;
   if (fetched_ >= token_.size()) {
     token_.resize(token_.size() * 2);
-    beg_.resize(beg_.size() * 2);
-    end_.resize(end_.size() * 2);
   }
-  token_[fetched_] = token;
-  beg_[fetched_] = beg;
-  end_[fetched_] = end;
+  token_[fetched_].value = token;
+  token_[fetched_].beg = beg;
+  token_[fetched_].end = end;
   fetched_++;
 }
 
 } }  // namespace v8::internal