Stand-alone parser template.

src/parser.cc

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 18 matching lines @@
 
 #include "api.h"
 #include "ast.h"
 #include "bootstrapper.h"
 #include "codegen.h"
 #include "compiler.h"
 #include "func-name-inferrer.h"
 #include "messages.h"
 #include "parser.h"
 #include "platform.h"
+#include "preparser.h"
 #include "runtime.h"
 #include "scopeinfo.h"
 #include "scopes.h"
 #include "string-stream.h"
 
 #include "ast-inl.h"
 #include "jump-target-inl.h"
 
 namespace v8 {
 namespace internal {
@@ skipping 334 matching lines @@
 
   template <typename T> ZoneListWrapper<T> NewList(int size) {
     return is_pre_parsing_ ? ZoneListWrapper<T>() : ZoneListWrapper<T>(size);
   }
 
  private:
   bool is_pre_parsing_;
 };
 
 
-class ParserLog BASE_EMBEDDED {
- public:
-  virtual ~ParserLog() { }
-
-  // Records the occurrence of a function.
-  virtual FunctionEntry LogFunction(int start) { return FunctionEntry(); }
-  virtual void LogSymbol(int start, Vector<const char> symbol) {}
-  virtual void LogError() { }
-  // Return the current position in the function entry log.
-  virtual int function_position() { return 0; }
-  virtual int symbol_position() { return 0; }
-  virtual int symbol_ids() { return 0; }
-  virtual void PauseRecording() {}
-  virtual void ResumeRecording() {}
-  virtual Vector<unsigned> ExtractData() {
-    return Vector<unsigned>();
-  };
-};
-
-
-
 class ConditionalLogPauseScope {
  public:
   ConditionalLogPauseScope(bool pause, ParserLog* log)
       : log_(log), pause_(pause) {
     if (pause) log->PauseRecording();
   }
   ~ConditionalLogPauseScope() {
     if (pause_) log_->ResumeRecording();
   }
  private:
@@ skipping 53 matching lines @@
                               int pos) {
     return new Call(expression, arguments, pos);
   }
 
   virtual Statement* EmptyStatement();
  private:
   List<Handle<String> > symbol_cache_;
 };
 
 
-// Record only functions.
-class PartialParserRecorder: public ParserLog {
- public:
-  PartialParserRecorder();
-  virtual FunctionEntry LogFunction(int start);
-
-  virtual int function_position() { return function_store_.size(); }
-
-  virtual void LogError() { }
-
-  virtual void LogMessage(Scanner::Location loc,
-                          const char* message,
-                          Vector<const char*> args);
-
-  virtual Vector<unsigned> ExtractData() {
-    int function_size = function_store_.size();
-    int total_size = ScriptDataImpl::kHeaderSize + function_size;
-    Vector<unsigned> data = Vector<unsigned>::New(total_size);
-    preamble_[ScriptDataImpl::kFunctionsSizeOffset] = function_size;
-    preamble_[ScriptDataImpl::kSymbolCountOffset] = 0;
-    memcpy(data.start(), preamble_, sizeof(preamble_));
-    int symbol_start = ScriptDataImpl::kHeaderSize + function_size;
-    if (function_size > 0) {
-      function_store_.WriteTo(data.SubVector(ScriptDataImpl::kHeaderSize,
-                                             symbol_start));
-    }
-    return data;
+Vector<unsigned> PartialParserRecorder::ExtractData() {
+  int function_size = function_store_.size();
+  int total_size = ScriptDataImpl::kHeaderSize + function_size;
+  Vector<unsigned> data = Vector<unsigned>::New(total_size);
+  preamble_[ScriptDataImpl::kFunctionsSizeOffset] = function_size;
+  preamble_[ScriptDataImpl::kSymbolCountOffset] = 0;
+  memcpy(data.start(), preamble_, sizeof(preamble_));
+  int symbol_start = ScriptDataImpl::kHeaderSize + function_size;
+  if (function_size > 0) {
+    function_store_.WriteTo(data.SubVector(ScriptDataImpl::kHeaderSize,
+                                           symbol_start));
   }
-
-  virtual void PauseRecording() {
-    pause_count_++;
-    is_recording_ = false;
-  }
-
-  virtual void ResumeRecording() {
-    ASSERT(pause_count_ > 0);
-    if (--pause_count_ == 0) is_recording_ = !has_error();
-  }
-
- protected:
-  bool has_error() {
-    return static_cast<bool>(preamble_[ScriptDataImpl::kHasErrorOffset]);
-  }
-  bool is_recording() {
-    return is_recording_;
-  }
-
-  void WriteString(Vector<const char> str);
-
-  Collector<unsigned> function_store_;
-  unsigned preamble_[ScriptDataImpl::kHeaderSize];
-  bool is_recording_;
-  int pause_count_;
-
-#ifdef DEBUG
-  int prev_start;
-#endif
-};
+  return data;
+}
 
 
-// Record both functions and symbols.
-class CompleteParserRecorder: public PartialParserRecorder {
- public:
-  CompleteParserRecorder();
+void CompleteParserRecorder::LogSymbol(int start, Vector<const char> literal) {
+  if (!is_recording_) return;
 
-  virtual void LogSymbol(int start, Vector<const char> literal) {
-    if (!is_recording_) return;
-    int hash = vector_hash(literal);
-    HashMap::Entry* entry = symbol_table_.Lookup(&literal, hash, true);
-    int id = static_cast<int>(reinterpret_cast<intptr_t>(entry->value));
-    if (id == 0) {
-      // Put (symbol_id_ + 1) into entry and increment it.
-      id = ++symbol_id_;
-      entry->value = reinterpret_cast<void*>(id);
-      Vector<Vector<const char> > symbol = symbol_entries_.AddBlock(1, literal);
-      entry->key = &symbol[0];
-    }
-    WriteNumber(id - 1);
+  int hash = vector_hash(literal);
+  HashMap::Entry* entry = symbol_table_.Lookup(&literal, hash, true);
+  int id = static_cast<int>(reinterpret_cast<intptr_t>(entry->value));
+  if (id == 0) {
+    // Put (symbol_id_ + 1) into entry and increment it.
+    id = ++symbol_id_;
+    entry->value = reinterpret_cast<void*>(id);
+    Vector<Vector<const char> > symbol = symbol_entries_.AddBlock(1, literal);
+    entry->key = &symbol[0];
   }
+  WriteNumber(id - 1);
+}
 
-  virtual Vector<unsigned> ExtractData() {
-    int function_size = function_store_.size();
-    // Add terminator to symbols, then pad to unsigned size.
-    int symbol_size = symbol_store_.size();
-    int padding = sizeof(unsigned) - (symbol_size % sizeof(unsigned));
-    symbol_store_.AddBlock(padding, ScriptDataImpl::kNumberTerminator);
-    symbol_size += padding;
-    int total_size = ScriptDataImpl::kHeaderSize + function_size
-        + (symbol_size / sizeof(unsigned));
-    Vector<unsigned> data = Vector<unsigned>::New(total_size);
-    preamble_[ScriptDataImpl::kFunctionsSizeOffset] = function_size;
-    preamble_[ScriptDataImpl::kSymbolCountOffset] = symbol_id_;
-    memcpy(data.start(), preamble_, sizeof(preamble_));
-    int symbol_start = ScriptDataImpl::kHeaderSize + function_size;
-    if (function_size > 0) {
-      function_store_.WriteTo(data.SubVector(ScriptDataImpl::kHeaderSize,
-                                             symbol_start));
-    }
-    if (!has_error()) {
-      symbol_store_.WriteTo(
-          Vector<byte>::cast(data.SubVector(symbol_start, total_size)));
-    }
-    return data;
+
+Vector<unsigned> CompleteParserRecorder::ExtractData() {
+  int function_size = function_store_.size();
+  // Add terminator to symbols, then pad to unsigned size.
+  int symbol_size = symbol_store_.size();
+  int padding = sizeof(unsigned) - (symbol_size % sizeof(unsigned));
+  symbol_store_.AddBlock(padding, ScriptDataImpl::kNumberTerminator);
+  symbol_size += padding;
+  int total_size = ScriptDataImpl::kHeaderSize + function_size
+      + (symbol_size / sizeof(unsigned));
+  Vector<unsigned> data = Vector<unsigned>::New(total_size);
+  preamble_[ScriptDataImpl::kFunctionsSizeOffset] = function_size;
+  preamble_[ScriptDataImpl::kSymbolCountOffset] = symbol_id_;
+  memcpy(data.start(), preamble_, sizeof(preamble_));
+  int symbol_start = ScriptDataImpl::kHeaderSize + function_size;
+  if (function_size > 0) {
+    function_store_.WriteTo(data.SubVector(ScriptDataImpl::kHeaderSize,
+                                           symbol_start));
   }
+  if (!has_error()) {
+    symbol_store_.WriteTo(
+        Vector<byte>::cast(data.SubVector(symbol_start, total_size)));
+  }
+  return data;
+}
 
-  virtual int symbol_position() { return symbol_store_.size(); }
-  virtual int symbol_ids() { return symbol_id_; }
- private:
-    static int vector_hash(Vector<const char> string) {
-    int hash = 0;
-    for (int i = 0; i < string.length(); i++) {
-      int c = string[i];
-      hash += c;
-      hash += (hash << 10);
-      hash ^= (hash >> 6);
-    }
-    return hash;
-  }
 
-  static bool vector_compare(void* a, void* b) {
-    Vector<const char>* string1 = reinterpret_cast<Vector<const char>* >(a);
-    Vector<const char>* string2 = reinterpret_cast<Vector<const char>* >(b);
-    int length = string1->length();
-    if (string2->length() != length) return false;
-    return memcmp(string1->start(), string2->start(), length) == 0;
-  }
-
-  // Write a non-negative number to the symbol store.
-  void WriteNumber(int number);
-
-  Collector<byte> symbol_store_;
-  Collector<Vector<const char> > symbol_entries_;
-  HashMap symbol_table_;
-  int symbol_id_;
-};
 
 
 FunctionEntry ScriptDataImpl::GetFunctionEntry(int start) {
   // The current pre-data entry must be a FunctionEntry with the given
   // start position.
   if ((function_index_ + FunctionEntry::kSize <= store_.length())
       && (static_cast<int>(store_[function_index_]) == start)) {
     int index = function_index_;
     function_index_ += FunctionEntry::kSize;
     return FunctionEntry(store_.SubVector(index,
@@ skipping 52 matching lines @@
       is_recording_(true),
       pause_count_(0) {
   preamble_[ScriptDataImpl::kMagicOffset] = ScriptDataImpl::kMagicNumber;
   preamble_[ScriptDataImpl::kVersionOffset] = ScriptDataImpl::kCurrentVersion;
   preamble_[ScriptDataImpl::kHasErrorOffset] = false;
   preamble_[ScriptDataImpl::kFunctionsSizeOffset] = 0;
   preamble_[ScriptDataImpl::kSymbolCountOffset] = 0;
   preamble_[ScriptDataImpl::kSizeOffset] = 0;
   ASSERT_EQ(6, ScriptDataImpl::kHeaderSize);
 #ifdef DEBUG
-  prev_start = -1;
+  prev_start_ = -1;
 #endif
 }
 
 
 CompleteParserRecorder::CompleteParserRecorder()
     : PartialParserRecorder(),
       symbol_store_(0),
       symbol_entries_(0),
       symbol_table_(vector_compare),
       symbol_id_(0) {
@@ skipping 30 matching lines @@
   for (int i = 0; i < length; i++) {
     result[i] = start[i + 1];
   }
   result[length] = '\0';
   if (chars != NULL) *chars = length;
   return result;
 }
 
 
 void PartialParserRecorder::LogMessage(Scanner::Location loc,
-                                      const char* message,
-                                      Vector<const char*> args) {
+                                       const char* message,
+                                       Vector<const char*> args) {
   if (has_error()) return;
   preamble_[ScriptDataImpl::kHasErrorOffset] = true;
   function_store_.Reset();
   STATIC_ASSERT(ScriptDataImpl::kMessageStartPos == 0);
   function_store_.Add(loc.beg_pos);
   STATIC_ASSERT(ScriptDataImpl::kMessageEndPos == 1);
   function_store_.Add(loc.end_pos);
   STATIC_ASSERT(ScriptDataImpl::kMessageArgCountPos == 2);
   function_store_.Add(args.length());
   STATIC_ASSERT(ScriptDataImpl::kMessageTextPos == 3);
@@ skipping 36 matching lines @@
 unsigned ScriptDataImpl::Read(int position) {
   return store_[ScriptDataImpl::kHeaderSize + position];
 }
 
 
 unsigned* ScriptDataImpl::ReadAddress(int position) {
   return &store_[ScriptDataImpl::kHeaderSize + position];
 }
 
 
-FunctionEntry PartialParserRecorder::LogFunction(int start) {
-#ifdef DEBUG
-  ASSERT(start > prev_start);
-  prev_start = start;
-#endif
-  if (!is_recording_) return FunctionEntry();
-  FunctionEntry result(function_store_.AddBlock(FunctionEntry::kSize, 0));
-  result.set_start_pos(start);
-  return result;
-}
-
-
 class AstBuildingParser : public Parser {
  public:
   AstBuildingParser(Handle<Script> script, bool allow_natives_syntax,
                     v8::Extension* extension, ScriptDataImpl* pre_data)
       : Parser(script,
                allow_natives_syntax,
                extension,
                PARSE,
                factory(),
                log(),
@@ skipping 204 matching lines @@
       allow_natives_syntax_(allow_natives_syntax),
       extension_(extension),
       factory_(factory),
       log_(log),
       is_pre_parsing_(is_pre_parsing == PREPARSE),
       pre_data_(pre_data),
       fni_(NULL) {
 }
 
 
-bool Parser::PreParseProgram(Handle<String> source,
-                             unibrow::CharacterStream* stream) {
-  HistogramTimerScope timer(&Counters::pre_parse);
-  AssertNoZoneAllocation assert_no_zone_allocation;
-  AssertNoAllocation assert_no_allocation;
-  NoHandleAllocation no_handle_allocation;
-  scanner_.Initialize(source, stream, JAVASCRIPT);
-  ASSERT(target_stack_ == NULL);
-  mode_ = FLAG_lazy ? PARSE_LAZILY : PARSE_EAGERLY;
-  if (allow_natives_syntax_ || extension_ != NULL) mode_ = PARSE_EAGERLY;
-  DummyScope top_scope;
-  LexicalScope scope(&this->top_scope_, &this->with_nesting_level_, &top_scope);
-  TemporaryScope temp_scope(&this->temp_scope_);
-  ZoneListWrapper<Statement> processor;
-  bool ok = true;
-  ParseSourceElements(&processor, Token::EOS, &ok);
-  return !scanner().stack_overflow();
-}
-
-
 FunctionLiteral* Parser::ParseProgram(Handle<String> source,
                                       bool in_global_context) {
   CompilationZoneScope zone_scope(DONT_DELETE_ON_EXIT);
 
   HistogramTimerScope timer(&Counters::parse);
   Counters::total_parse_size.Increment(source->length());
   fni_ = new FuncNameInferrer();
 
   // Initialize parser state.
   source->TryFlatten();
@@ skipping 664 matching lines @@
   }
 
   Expect(Token::NATIVE, CHECK_OK);
   Expect(Token::FUNCTION, CHECK_OK);
   Handle<String> name = ParseIdentifier(CHECK_OK);
   Expect(Token::LPAREN, CHECK_OK);
   bool done = (peek() == Token::RPAREN);
   while (!done) {
     ParseIdentifier(CHECK_OK);
     done = (peek() == Token::RPAREN);
-    if (!done) Expect(Token::COMMA, CHECK_OK);
+    if (!done) {
+      Expect(Token::COMMA, CHECK_OK);
+    }
   }
   Expect(Token::RPAREN, CHECK_OK);
   Expect(Token::SEMICOLON, CHECK_OK);
 
   if (is_pre_parsing_) return NULL;
 
   // Make sure that the function containing the native declaration
   // isn't lazily compiled. The extension structures are only
   // accessible while parsing the first time not when reparsing
   // because of lazy compilation.
@@ skipping 1959 matching lines @@
       }
       Counters::total_preparse_skipped.Increment(end_pos - function_block_pos);
       scanner_.SeekForward(end_pos);
       materialized_literal_count = entry.literal_count();
       expected_property_count = entry.property_count();
       only_simple_this_property_assignments = false;
       this_property_assignments = Factory::empty_fixed_array();
       Expect(Token::RBRACE, CHECK_OK);
     } else {
       FunctionEntry entry;
-      if (is_lazily_compiled) entry = log()->LogFunction(function_block_pos);
       {
         ConditionalLogPauseScope pause_if(is_lazily_compiled, log());
         ParseSourceElements(&body, Token::RBRACE, CHECK_OK);
       }
       materialized_literal_count = temp_scope.materialized_literal_count();
       expected_property_count = temp_scope.expected_property_count();
       only_simple_this_property_assignments =
           temp_scope.only_simple_this_property_assignments();
       this_property_assignments = temp_scope.this_property_assignments();
 
       Expect(Token::RBRACE, CHECK_OK);
       end_pos = scanner_.location().end_pos;
-      if (entry.is_valid()) {
-        ASSERT(is_lazily_compiled);
+      if (is_pre_parsing_ && is_lazily_compiled) {
         ASSERT(is_pre_parsing_);
-        entry.set_end_pos(end_pos);
-        entry.set_literal_count(materialized_literal_count);
-        entry.set_property_count(expected_property_count);
+        log()->LogFunction(function_block_pos, end_pos,
+                           materialized_literal_count,
+                           expected_property_count);
       }
     }
 
     FunctionLiteral* function_literal =
         NEW(FunctionLiteral(name,
                             top_scope_,
                             body.elements(),
                             materialized_literal_count,
                             expected_property_count,
                             only_simple_this_property_assignments,
@@ skipping 1244 matching lines @@
 const char* ScriptDataImpl::Data() {
   return reinterpret_cast<const char*>(store_.start());
 }
 
 
 bool ScriptDataImpl::HasError() {
   return has_error();
 }
 
 
-// Preparse, but only collect data that is immediately useful,
-// even if the preparser data is only used once.
-ScriptDataImpl* ParserApi::PartialPreParse(Handle<String> source,
-                                           unibrow::CharacterStream* stream,
-                                           v8::Extension* extension) {
-  Handle<Script> no_script;
-  bool allow_natives_syntax =
-      FLAG_allow_natives_syntax || Bootstrapper::IsActive();
-  PartialPreParser parser(no_script, allow_natives_syntax, extension);
-  if (!parser.PreParseProgram(source, stream)) return NULL;
-  // Extract the accumulated data from the recorder as a single
-  // contiguous vector that we are responsible for disposing.
-  Vector<unsigned> store = parser.recorder()->ExtractData();
-  return new ScriptDataImpl(store);
-}
-
-
 void ScriptDataImpl::Initialize() {
   // Prepares state for use.
   if (store_.length() >= kHeaderSize) {
     function_index_ = kHeaderSize;
     int symbol_data_offset = kHeaderSize + store_[kFunctionsSizeOffset];
     if (store_.length() > symbol_data_offset) {
       symbol_data_ = reinterpret_cast<byte*>(&store_[symbol_data_offset]);
     } else {
       // Partial preparse causes no symbol information.
       symbol_data_ = reinterpret_cast<byte*>(&store_[0] + store_.length());
@@ skipping 23 matching lines @@
     if (data >= symbol_data_end_) return -1;
     input = *data;
     result = (result << 7) | (input & 0x7f);
     data++;
   }
   *source = data;
   return result;
 }
 
 
+// Preparse, but only collect data that is immediately useful,
+// even if the preparser data is only used once.
+ScriptDataImpl* ParserApi::PartialPreParse(Handle<String> source,
+                                           unibrow::CharacterStream* stream,
+                                           v8::Extension* extension) {
+  Handle<Script> no_script;
+  preparser::PreParser<Scanner, PartialParserRecorder> parser;
+  Scanner scanner;
+  scanner.Initialize(source, stream, JAVASCRIPT);
+  bool allow_lazy = FLAG_lazy && (extension == NULL);
+  PartialParserRecorder recorder;
+  if (!parser.PreParseProgram(&scanner, &recorder, allow_lazy)) {
+    Top::StackOverflow();
+    return NULL;
+  }
+
+  // Extract the accumulated data from the recorder as a single
+  // contiguous vector that we are responsible for disposing.
+  Vector<unsigned> store = recorder.ExtractData();
+  return new ScriptDataImpl(store);
+}
+
+
 ScriptDataImpl* ParserApi::PreParse(Handle<String> source,
                                     unibrow::CharacterStream* stream,
                                     v8::Extension* extension) {
   Handle<Script> no_script;
-  bool allow_natives_syntax =
-      FLAG_allow_natives_syntax || Bootstrapper::IsActive();
-  CompletePreParser parser(no_script, allow_natives_syntax, extension);
-  if (!parser.PreParseProgram(source, stream)) return NULL;
+  preparser::PreParser<Scanner, CompleteParserRecorder> parser;
+  Scanner scanner;
+  scanner.Initialize(source, stream, JAVASCRIPT);
+  bool allow_lazy = FLAG_lazy && (extension == NULL);
+  CompleteParserRecorder recorder;
+  if (!parser.PreParseProgram(&scanner, &recorder, allow_lazy)) {
+    Top::StackOverflow();
+    return NULL;
+  }
   // Extract the accumulated data from the recorder as a single
   // contiguous vector that we are responsible for disposing.
-  Vector<unsigned> store = parser.recorder()->ExtractData();
+  Vector<unsigned> store = recorder.ExtractData();
   return new ScriptDataImpl(store);
 }
 
 
 bool RegExpParser::ParseRegExp(FlatStringReader* input,
                                bool multiline,
                                RegExpCompileData* result) {
   ASSERT(result != NULL);
   RegExpParser parser(input, &result->error, multiline);
   RegExpTree* tree = parser.ParsePattern();
@@ skipping 43 matching lines @@
     }
   }
 
   info->SetFunction(result);
   return (result != NULL);
 }
 
 #undef NEW
 
 } }  // namespace v8::internal