Add literal handling to experimental scanner.

src/parser.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 354 matching lines @@
   char* result = NewArray<char>(length + 1);
   for (int i = 0; i < length; i++) {
     result[i] = start[i + 1];
   }
   result[length] = '\0';
   if (chars != NULL) *chars = length;
   return result;
 }
 
 
-Scanner::Location ScriptDataImpl::MessageLocation() {
+ScannerBase::Location ScriptDataImpl::MessageLocation() {
   int beg_pos = Read(PreparseDataConstants::kMessageStartPos);
   int end_pos = Read(PreparseDataConstants::kMessageEndPos);
-  return Scanner::Location(beg_pos, end_pos);
+  return ScannerBase::Location(beg_pos, end_pos);
 }
 
 
 const char* ScriptDataImpl::BuildMessage() {
   unsigned* start = ReadAddress(PreparseDataConstants::kMessageTextPos);
   return ReadString(start, NULL);
 }
 
 
 Vector<const char*> ScriptDataImpl::BuildArgs() {
@@ skipping 140 matching lines @@
 #define CHECK_FAILED  /**/);   \
   if (failed_) return NULL; \
   ((void)0
 #define DUMMY )  // to make indentation work
 #undef DUMMY
 
 // ----------------------------------------------------------------------------
 // Implementation of Parser
 
 Parser::Parser(CompilationInfo* info)
-    : ParserBase(&scanner_, info->isolate()->stack_guard()->real_climit()),
+    : ParserBase(NULL, info->isolate()->stack_guard()->real_climit()),
       isolate_(info->isolate()),
       symbol_cache_(0, info->zone()),
       script_(info->script()),
-      scanner_(isolate_->unicode_cache()),
       reusable_preparser_(NULL),
       top_scope_(NULL),
       original_scope_(NULL),
       current_function_state_(NULL),
       target_stack_(NULL),
       extension_(info->extension()),
       pre_parse_data_(NULL),
       fni_(NULL),
       parenthesized_function_(false),
       zone_(info->zone()),
       info_(info) {
   ASSERT(!script_.is_null());
   isolate_->set_ast_node_id(0);
-  set_allow_harmony_scoping(!info->is_native() && FLAG_harmony_scoping);
-  set_allow_modules(!info->is_native() && FLAG_harmony_modules);
+  // FIXME: these can be done only when the ExperimentalScanner has been
+  // created, and they need to be redone when it's recreated.
+  // set_allow_harmony_scoping(!info->is_native() && FLAG_harmony_scoping);
+  // set_allow_modules(!info->is_native() && FLAG_harmony_modules);
   set_allow_natives_syntax(FLAG_allow_natives_syntax || info->is_native());
+  // set_allow_harmony_numeric_literals(FLAG_harmony_numeric_literals);
   set_allow_lazy(false);  // Must be explicitly enabled.
   set_allow_generators(FLAG_harmony_generators);
   set_allow_for_of(FLAG_harmony_iteration);
-  set_allow_harmony_numeric_literals(FLAG_harmony_numeric_literals);
 }
 
 
 FunctionLiteral* Parser::ParseProgram() {
   // TODO(bmeurer): We temporarily need to pass allow_nesting = true here,
   // see comment for HistogramTimerScope class.
   HistogramTimerScope timer_scope(isolate()->counters()->parse(), true);
   Handle<String> source(String::cast(script_->source()));
   isolate()->counters()->total_parse_size()->Increment(source->length());
   ElapsedTimer timer;
   if (FLAG_trace_parse) {
     timer.Start();
   }
   fni_ = new(zone()) FuncNameInferrer(isolate(), zone());
 
   // Initialize parser state.
   source->TryFlatten();
   FunctionLiteral* result;
   if (source->IsExternalTwoByteString()) {
     // Notice that the stream is destroyed at the end of the branch block.
     // The last line of the blocks can't be moved outside, even though they're
-    // identical calls.
-    ExternalTwoByteStringUtf16CharacterStream stream(
-        Handle<ExternalTwoByteString>::cast(source), 0, source->length());
-    scanner_.Initialize(&stream);
+    // identical calls. // FIXME
+    delete scanner_;
+    scanner_ = new ExperimentalScanner<uint16_t>(source, isolate());
+    // FIXME: set flags
     result = DoParseProgram(info(), source);
   } else {
-    GenericStringUtf16CharacterStream stream(source, 0, source->length());
-    scanner_.Initialize(&stream);
+    delete scanner_;
+    scanner_ = new ExperimentalScanner<uint8_t>(source, isolate());
+    // FIXME: set flags
     result = DoParseProgram(info(), source);
   }
 
   if (FLAG_trace_parse && result != NULL) {
     double ms = timer.Elapsed().InMillisecondsF();
     if (info()->is_eval()) {
       PrintF("[parsing eval");
     } else if (info()->script()->name()->IsString()) {
       String* name = String::cast(info()->script()->name());
       SmartArrayPointer<char> name_chars = name->ToCString();
@@ skipping 98 matching lines @@
 
 
 FunctionLiteral* Parser::ParseLazy() {
   HistogramTimerScope timer_scope(isolate()->counters()->parse_lazy());
   Handle<String> source(String::cast(script_->source()));
   isolate()->counters()->total_parse_size()->Increment(source->length());
   ElapsedTimer timer;
   if (FLAG_trace_parse) {
     timer.Start();
   }
-  Handle<SharedFunctionInfo> shared_info = info()->shared_info();
-
   // Initialize parser state.
   source->TryFlatten();
-  FunctionLiteral* result;
-  if (source->IsExternalTwoByteString()) {
-    ExternalTwoByteStringUtf16CharacterStream stream(
-        Handle<ExternalTwoByteString>::cast(source),
-        shared_info->start_position(),
-        shared_info->end_position());
-    result = ParseLazy(&stream);
-  } else {
-    GenericStringUtf16CharacterStream stream(source,
-                                             shared_info->start_position(),
-                                             shared_info->end_position());
-    result = ParseLazy(&stream);
-  }
-
+  Handle<SharedFunctionInfo> shared_info = info()->shared_info();
+  FunctionLiteral* result = ParseLazy(
+      source, shared_info->start_position(), shared_info->end_position());
   if (FLAG_trace_parse && result != NULL) {
     double ms = timer.Elapsed().InMillisecondsF();
     SmartArrayPointer<char> name_chars = result->debug_name()->ToCString();
     PrintF("[parsing function: %s - took %0.3f ms]\n", *name_chars, ms);
   }
   return result;
 }
 
 
-FunctionLiteral* Parser::ParseLazy(Utf16CharacterStream* source) {
+FunctionLiteral* Parser::ParseLazy(Handle<String> source, int start, int end) {
+  delete scanner_;
+  if (source->IsExternalTwoByteString()) {
+    scanner_ = new ExperimentalScanner<uint16_t>(source, isolate());
+  } else {
+    scanner_ = new ExperimentalScanner<uint8_t>(source, isolate());
+  }
+  scanner_->SeekForward(start);
+  scanner_->SetEnd(end);
+  // FIXME: set flags
+
   Handle<SharedFunctionInfo> shared_info = info()->shared_info();
-  scanner_.Initialize(source);
   ASSERT(top_scope_ == NULL);
   ASSERT(target_stack_ == NULL);
 
   Handle<String> name(String::cast(shared_info->name()));
   fni_ = new(zone()) FuncNameInferrer(isolate(), zone());
   fni_->PushEnclosingName(name);
 
   ParsingModeScope parsing_mode(this, PARSE_EAGERLY);
 
   // Place holder for the result.
@@ skipping 46 matching lines @@
 Handle<String> Parser::GetSymbol() {
   int symbol_id = -1;
   if (pre_parse_data() != NULL) {
     symbol_id = pre_parse_data()->GetSymbolIdentifier();
   }
   return LookupSymbol(symbol_id);
 }
 
 
 void Parser::ReportMessage(const char* message, Vector<const char*> args) {
-  Scanner::Location source_location = scanner().location();
+  ScannerBase::Location source_location = scanner().location();
   ReportMessageAt(source_location, message, args);
 }
 
 
 void Parser::ReportMessage(const char* message, Vector<Handle<String> > args) {
-  Scanner::Location source_location = scanner().location();
+  ScannerBase::Location source_location = scanner().location();
   ReportMessageAt(source_location, message, args);
 }
 
 
-void Parser::ReportMessageAt(Scanner::Location source_location,
+void Parser::ReportMessageAt(ScannerBase::Location source_location,
                              const char* message,
                              Vector<const char*> args) {
   MessageLocation location(script_,
                            source_location.beg_pos,
                            source_location.end_pos);
   Factory* factory = isolate()->factory();
   Handle<FixedArray> elements = factory->NewFixedArray(args.length());
   for (int i = 0; i < args.length(); i++) {
     Handle<String> arg_string = factory->NewStringFromUtf8(CStrVector(args[i]));
     elements->set(i, *arg_string);
   }
   Handle<JSArray> array = factory->NewJSArrayWithElements(elements);
   Handle<Object> result = factory->NewSyntaxError(message, array);
   isolate()->Throw(*result, &location);
 }
 
 
-void Parser::ReportMessageAt(Scanner::Location source_location,
+void Parser::ReportMessageAt(ScannerBase::Location source_location,
                              const char* message,
                              Vector<Handle<String> > args) {
   MessageLocation location(script_,
                            source_location.beg_pos,
                            source_location.end_pos);
   Factory* factory = isolate()->factory();
   Handle<FixedArray> elements = factory->NewFixedArray(args.length());
   for (int i = 0; i < args.length(); i++) {
     elements->set(i, *args[i]);
   }
@@ skipping 18 matching lines @@
   TargetScope scope(&this->target_stack_);
 
   ASSERT(processor != NULL);
   bool directive_prologue = true;     // Parsing directive prologue.
 
   while (peek() != end_token) {
     if (directive_prologue && peek() != Token::STRING) {
       directive_prologue = false;
     }
 
-    Scanner::Location token_loc = scanner().peek_location();
+    ScannerBase::Location token_loc = scanner().peek_location();
     Statement* stat;
     if (is_global && !is_eval) {
       stat = ParseModuleElement(NULL, CHECK_OK);
     } else {
       stat = ParseBlockElement(NULL, CHECK_OK);
     }
     if (stat == NULL || stat->IsEmpty()) {
       directive_prologue = false;   // End of directive prologue.
       continue;
     }
@@ skipping 3213 matching lines @@
       // FunctionState indicates that this function is a generator.
       Variable* temp = top_scope_->DeclarationScope()->NewTemporary(
           isolate()->factory()->dot_generator_object_string());
       function_state.set_generator_object_variable(temp);
     }
 
     //  FormalParameterList ::
     //    '(' (Identifier)*[','] ')'
     Expect(Token::LPAREN, CHECK_OK);
     scope->set_start_position(scanner().location().beg_pos);
-    Scanner::Location name_loc = Scanner::Location::invalid();
-    Scanner::Location dupe_loc = Scanner::Location::invalid();
-    Scanner::Location reserved_loc = Scanner::Location::invalid();
+    ScannerBase::Location name_loc = ScannerBase::Location::invalid();
+    ScannerBase::Location dupe_loc = ScannerBase::Location::invalid();
+    ScannerBase::Location reserved_loc = ScannerBase::Location::invalid();
 
     bool done = (peek() == Token::RPAREN);
     while (!done) {
       bool is_strict_reserved = false;
       Handle<String> param_name =
           ParseIdentifierOrStrictReservedWord(&is_strict_reserved, CHECK_OK);
 
       // Store locations for possible future error reports.
       if (!name_loc.IsValid() && IsEvalOrArguments(param_name)) {
         name_loc = scanner().location();
@@ skipping 94 matching lines @@
           set_stack_overflow();
           *ok = false;
           return NULL;
         }
         if (logger.has_error()) {
           const char* arg = logger.argument_opt();
           Vector<const char*> args;
           if (arg != NULL) {
             args = Vector<const char*>(&arg, 1);
           }
-          ReportMessageAt(Scanner::Location(logger.start(), logger.end()),
+          ReportMessageAt(ScannerBase::Location(logger.start(), logger.end()),
                           logger.message(), args);
           *ok = false;
           return NULL;
         }
         scope->set_end_position(logger.end());
         Expect(Token::RBRACE, CHECK_OK);
         isolate()->counters()->total_preparse_skipped()->Increment(
             scope->end_position() - function_block_pos);
         materialized_literal_count = logger.literals();
         expected_property_count = logger.properties();
@@ skipping 56 matching lines @@
       Expect(Token::RBRACE, CHECK_OK);
       scope->set_end_position(scanner().location().end_pos);
     }
 
     // Validate strict mode.
     if (!top_scope_->is_classic_mode()) {
       if (IsEvalOrArguments(function_name)) {
         int start_pos = scope->start_position();
         int position = function_token_pos != RelocInfo::kNoPosition
             ? function_token_pos : (start_pos > 0 ? start_pos - 1 : start_pos);
-        Scanner::Location location = Scanner::Location(position, start_pos);
+        ScannerBase::Location location =
+            ScannerBase::Location(position, start_pos);
         ReportMessageAt(location,
                         "strict_function_name", Vector<const char*>::empty());
         *ok = false;
         return NULL;
       }
       if (name_loc.IsValid()) {
         ReportMessageAt(name_loc, "strict_param_name",
                         Vector<const char*>::empty());
         *ok = false;
         return NULL;
       }
       if (dupe_loc.IsValid()) {
         ReportMessageAt(dupe_loc, "strict_param_dupe",
                         Vector<const char*>::empty());
         *ok = false;
         return NULL;
       }
       if (name_is_strict_reserved) {
         int start_pos = scope->start_position();
         int position = function_token_pos != RelocInfo::kNoPosition
             ? function_token_pos : (start_pos > 0 ? start_pos - 1 : start_pos);
-        Scanner::Location location = Scanner::Location(position, start_pos);
+        ScannerBase::Location location =
+            ScannerBase::Location(position, start_pos);
         ReportMessageAt(location, "strict_reserved_word",
                         Vector<const char*>::empty());
         *ok = false;
         return NULL;
       }
       if (reserved_loc.IsValid()) {
         ReportMessageAt(reserved_loc, "strict_reserved_word",
                         Vector<const char*>::empty());
         *ok = false;
         return NULL;
@@ skipping 33 matching lines @@
 }
 
 
 PreParser::PreParseResult Parser::LazyParseFunctionLiteral(
     SingletonLogger* logger) {
   HistogramTimerScope preparse_scope(isolate()->counters()->pre_parse());
   ASSERT_EQ(Token::LBRACE, scanner().current_token());
 
   if (reusable_preparser_ == NULL) {
     intptr_t stack_limit = isolate()->stack_guard()->real_climit();
-    reusable_preparser_ = new PreParser(&scanner_, NULL, stack_limit);
+    reusable_preparser_ = new PreParser(scanner_, NULL, stack_limit);
     reusable_preparser_->set_allow_harmony_scoping(allow_harmony_scoping());
     reusable_preparser_->set_allow_modules(allow_modules());
     reusable_preparser_->set_allow_natives_syntax(allow_natives_syntax());
     reusable_preparser_->set_allow_lazy(true);
     reusable_preparser_->set_allow_generators(allow_generators());
     reusable_preparser_->set_allow_for_of(allow_for_of());
     reusable_preparser_->set_allow_harmony_numeric_literals(
         allow_harmony_numeric_literals());
   }
   PreParser::PreParseResult result =
@@ skipping 188 matching lines @@
   if (lhs != NULL && !lhs->is_this() && IsEvalOrArguments(lhs->name())) {
     ReportMessage(error, Vector<const char*>::empty());
     *ok = false;
   }
 }
 
 
 // Checks whether an octal literal was last seen between beg_pos and end_pos.
 // If so, reports an error. Only called for strict mode.
 void ParserBase::CheckOctalLiteral(int beg_pos, int end_pos, bool* ok) {
-  Scanner::Location octal = scanner()->octal_position();
+  ScannerBase::Location octal = scanner()->octal_position();
   if (octal.IsValid() && beg_pos <= octal.beg_pos && octal.end_pos <= end_pos) {
     ReportMessageAt(octal, "strict_octal_literal");
     scanner()->clear_octal_position();
     *ok = false;
   }
 }
 
 
 void Parser::CheckConflictingVarDeclarations(Scope* scope, bool* ok) {
   Declaration* decl = scope->CheckConflictingVarDeclarations();
   if (decl != NULL) {
     // In harmony mode we treat conflicting variable bindinds as early
     // errors. See ES5 16 for a definition of early errors.
     Handle<String> name = decl->proxy()->name();
     SmartArrayPointer<char> c_string = name->ToCString(DISALLOW_NULLS);
     const char* elms[2] = { "Variable", *c_string };
     Vector<const char*> args(elms, 2);
     int position = decl->proxy()->position();
-    Scanner::Location location = position == RelocInfo::kNoPosition
-        ? Scanner::Location::invalid()
-        : Scanner::Location(position, position + 1);
+    ScannerBase::Location location = position == RelocInfo::kNoPosition
+        ? ScannerBase::Location::invalid()
+        : ScannerBase::Location(position, position + 1);
     ReportMessageAt(location, "redeclaration", args);
     *ok = false;
   }
 }
 
 
 // This function reads an identifier name and determines whether or not it
 // is 'get' or 'set'.
 Handle<String> Parser::ParseIdentifierNameOrGetOrSet(bool* is_get,
                                                      bool* is_set,
@@ skipping 983 matching lines @@
     if (data >= symbol_data_end_) return -1;
     input = *data;
     result = (result << 7) | (input & 0x7f);
     data++;
   }
   *source = data;
   return result;
 }
 
 
-// Create a Scanner for the preparser to use as input, and preparse the source.
+// Create a ScannerBase for the preparser to use as input, and preparse the
+// source.
 ScriptDataImpl* PreParserApi::PreParse(Isolate* isolate,
                                        Utf16CharacterStream* source) {
-  CompleteParserRecorder recorder;
-  HistogramTimerScope timer(isolate->counters()->pre_parse());
-  Scanner scanner(isolate->unicode_cache());
-  intptr_t stack_limit = isolate->stack_guard()->real_climit();
-  PreParser preparser(&scanner, &recorder, stack_limit);
-  preparser.set_allow_lazy(true);
-  preparser.set_allow_generators(FLAG_harmony_generators);
-  preparser.set_allow_for_of(FLAG_harmony_iteration);
-  preparser.set_allow_harmony_scoping(FLAG_harmony_scoping);
-  preparser.set_allow_harmony_numeric_literals(FLAG_harmony_numeric_literals);
-  scanner.Initialize(source);
-  PreParser::PreParseResult result = preparser.PreParseProgram();
-  if (result == PreParser::kPreParseStackOverflow) {
-    isolate->StackOverflow();
-    return NULL;
-  }
-
-  // Extract the accumulated data from the recorder as a single
-  // contiguous vector that we are responsible for disposing.
-  Vector<unsigned> store = recorder.ExtractData();
-  return new ScriptDataImpl(store);
+  // FIXME(experimental-scanner): implement
+  return NULL;
 }
 
 
 bool RegExpParser::ParseRegExp(FlatStringReader* input,
                                bool multiline,
                                RegExpCompileData* result,
                                Zone* zone) {
   ASSERT(result != NULL);
   RegExpParser parser(input, &result->error, multiline, zone);
   RegExpTree* tree = parser.ParsePattern();
@@ skipping 20 matching lines @@
     ASSERT(!info()->is_eval());
     if (info()->shared_info()->is_function()) {
       result = ParseLazy();
     } else {
       result = ParseProgram();
     }
   } else {
     ScriptDataImpl* pre_parse_data = info()->pre_parse_data();
     set_pre_parse_data(pre_parse_data);
     if (pre_parse_data != NULL && pre_parse_data->has_error()) {
-      Scanner::Location loc = pre_parse_data->MessageLocation();
+      ScannerBase::Location loc = pre_parse_data->MessageLocation();
       const char* message = pre_parse_data->BuildMessage();
       Vector<const char*> args = pre_parse_data->BuildArgs();
       ReportMessageAt(loc, message, args);
       DeleteArray(message);
       for (int i = 0; i < args.length(); i++) {
         DeleteArray(args[i]);
       }
       DeleteArray(args.start());
       ASSERT(info()->isolate()->has_pending_exception());
     } else {
       result = ParseProgram();
     }
   }
   info()->SetFunction(result);
   return (result != NULL);
 }
 
 } }  // namespace v8::internal