Add script streaming API

src/parser.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/v8.h"
 
 #include "src/api.h"
 #include "src/ast.h"
 #include "src/base/platform/platform.h"
 #include "src/bootstrapper.h"
@@ skipping 740 matching lines @@
       target_stack_(NULL),
       cached_parse_data_(NULL),
       ast_value_factory_(info->ast_value_factory()),
       info_(info),
       has_pending_error_(false),
       pending_error_message_(NULL),
       pending_error_arg_(NULL),
       pending_error_char_arg_(NULL),
       total_preparse_skipped_(0),
       pre_parse_timer_(NULL) {
-  DCHECK(!script_.is_null());
+  DCHECK(!info->script().is_null() || info->source_stream() != NULL);
   set_allow_harmony_scoping(!info->is_native() && FLAG_harmony_scoping);
   set_allow_modules(!info->is_native() && FLAG_harmony_modules);
   set_allow_natives_syntax(FLAG_allow_natives_syntax || info->is_native());
   set_allow_lazy(false);  // Must be explicitly enabled.
   set_allow_generators(FLAG_harmony_generators);
   set_allow_arrow_functions(FLAG_harmony_arrow_functions);
   set_allow_harmony_numeric_literals(FLAG_harmony_numeric_literals);
   set_allow_classes(FLAG_harmony_classes);
   for (int feature = 0; feature < v8::Isolate::kUseCounterFeatureCount;
        ++feature) {
@@ skipping 24 matching lines @@
   CompleteParserRecorder recorder;
 
   if (compile_options() == ScriptCompiler::kProduceParserCache) {
     log_ = &recorder;
   } else if (compile_options() == ScriptCompiler::kConsumeParserCache) {
     cached_parse_data_->Initialize();
   }
 
   source = String::Flatten(source);
   FunctionLiteral* result;
+
+  Scope* top_scope = NULL;
+  Scope* ad_hoc_eval_scope = NULL;
   if (source->IsExternalTwoByteString()) {
     // Notice that the stream is destroyed at the end of the branch block.
     // The last line of the blocks can't be moved outside, even though they're
     // identical calls.
     ExternalTwoByteStringUtf16CharacterStream stream(
         Handle<ExternalTwoByteString>::cast(source), 0, source->length());
     scanner_.Initialize(&stream);
-    result = DoParseProgram(info(), source);
+    result = DoParseProgram(info(), &top_scope, &ad_hoc_eval_scope);
   } else {
     GenericStringUtf16CharacterStream stream(source, 0, source->length());
     scanner_.Initialize(&stream);
-    result = DoParseProgram(info(), source);
+    result = DoParseProgram(info(), &top_scope, &ad_hoc_eval_scope);
+  }
+  top_scope->set_end_position(source->length());
+  if (ad_hoc_eval_scope != NULL) {
+    ad_hoc_eval_scope->set_end_position(source->length());
   }
   HandleSourceURLComments();
 
   if (FLAG_trace_parse && result != NULL) {
     double ms = timer.Elapsed().InMillisecondsF();
     if (info()->is_eval()) {
       PrintF("[parsing eval");
     } else if (info()->script()->name()->IsString()) {
       String* name = String::cast(info()->script()->name());
       SmartArrayPointer<char> name_chars = name->ToCString();
       PrintF("[parsing script: %s", name_chars.get());
     } else {
       PrintF("[parsing script");
     }
     PrintF(" - took %0.3f ms]\n", ms);
   }
   if (compile_options() == ScriptCompiler::kProduceParserCache) {
     if (result != NULL) *info_->cached_data() = recorder.GetScriptData();
     log_ = NULL;
   }
   return result;
 }
 
 
-FunctionLiteral* Parser::DoParseProgram(CompilationInfo* info,
-                                        Handle<String> source) {
+FunctionLiteral* Parser::DoParseProgram(CompilationInfo* info, Scope** scope,
+                                        Scope** ad_hod_eval_scope) {
   DCHECK(scope_ == NULL);
   DCHECK(target_stack_ == NULL);
 
   FunctionLiteral* result = NULL;
-  { Scope* scope = NewScope(scope_, GLOBAL_SCOPE);
-    info->SetGlobalScope(scope);
+  {
+    *scope = NewScope(scope_, GLOBAL_SCOPE);
+    info->SetGlobalScope(*scope);
     if (!info->context().is_null() && !info->context()->IsNativeContext()) {
-      scope = Scope::DeserializeScopeChain(*info->context(), scope, zone());
+      *scope = Scope::DeserializeScopeChain(*info->context(), *scope, zone());
       // The Scope is backed up by ScopeInfo (which is in the V8 heap); this
       // means the Parser cannot operate independent of the V8 heap. Tell the
       // string table to internalize strings and values right after they're
       // created.
       ast_value_factory_->Internalize(isolate());
     }
-    original_scope_ = scope;
+    original_scope_ = *scope;
     if (info->is_eval()) {
-      if (!scope->is_global_scope() || info->strict_mode() == STRICT) {
-        scope = NewScope(scope, EVAL_SCOPE);
+      if (!(*scope)->is_global_scope() || info->strict_mode() == STRICT) {
+        *scope = NewScope(*scope, EVAL_SCOPE);
       }
     } else if (info->is_global()) {
-      scope = NewScope(scope, GLOBAL_SCOPE);
+      *scope = NewScope(*scope, GLOBAL_SCOPE);
     }
-    scope->set_start_position(0);
-    scope->set_end_position(source->length());
+    (*scope)->set_start_position(0);
+    // End position will be set by the caller.
 
     // Compute the parsing mode.
     Mode mode = (FLAG_lazy && allow_lazy()) ? PARSE_LAZILY : PARSE_EAGERLY;
-    if (allow_natives_syntax() ||
-        extension_ != NULL ||
-        scope->is_eval_scope()) {
+    if (allow_natives_syntax() || extension_ != NULL ||
+        (*scope)->is_eval_scope()) {
       mode = PARSE_EAGERLY;
     }
     ParsingModeScope parsing_mode(this, mode);
 
     // Enters 'scope'.
-    FunctionState function_state(&function_state_, &scope_, scope, zone(),
+    FunctionState function_state(&function_state_, &scope_, *scope, zone(),
                                  ast_value_factory_, info->ast_node_id_gen());
 
     scope_->SetStrictMode(info->strict_mode());
     ZoneList<Statement*>* body = new(zone()) ZoneList<Statement*>(16, zone());
     bool ok = true;
     int beg_pos = scanner()->location().beg_pos;
-    ParseSourceElements(body, Token::EOS, info->is_eval(), true, &ok);
+    ParseSourceElements(body, Token::EOS, info->is_eval(), true,
+                        ad_hod_eval_scope, &ok);
 
     if (ok && strict_mode() == STRICT) {
       CheckOctalLiteral(beg_pos, scanner()->location().end_pos, &ok);
     }
 
     if (ok && allow_harmony_scoping() && strict_mode() == STRICT) {
       CheckConflictingVarDeclarations(scope_, &ok);
     }
 
     if (ok && info->parse_restriction() == ONLY_SINGLE_FUNCTION_LITERAL) {
@@ skipping 126 matching lines @@
 
   if (result != NULL) {
     Handle<String> inferred_name(shared_info->inferred_name());
     result->set_inferred_name(inferred_name);
   }
   return result;
 }
 
 
 void* Parser::ParseSourceElements(ZoneList<Statement*>* processor,
-                                  int end_token,
-                                  bool is_eval,
-                                  bool is_global,
-                                  bool* ok) {
+                                  int end_token, bool is_eval, bool is_global,
+                                  Scope** ad_hod_eval_scope, bool* ok) {
   // SourceElements ::
   //   (ModuleElement)* <end_token>
 
   // Allocate a target stack to use for this set of source
   // elements. This way, all scripts and functions get their own
   // target stack thus avoiding illegal breaks and continues across
   // functions.
   TargetScope scope(&this->target_stack_);
 
   DCHECK(processor != NULL);
@@ skipping 35 matching lines @@
           // as specified in ES5 10.4.2(3). The correct fix would be to always
           // add this scope in DoParseProgram(), but that requires adaptations
           // all over the code base, so we go with a quick-fix for now.
           // In the same manner, we have to patch the parsing mode.
           if (is_eval && !scope_->is_eval_scope()) {
             DCHECK(scope_->is_global_scope());
             Scope* scope = NewScope(scope_, EVAL_SCOPE);
             scope->set_start_position(scope_->start_position());
             scope->set_end_position(scope_->end_position());
             scope_ = scope;
+            if (ad_hod_eval_scope != NULL) {
+              // Caller will correct the positions of the ad hoc eval scope.
+              *ad_hod_eval_scope = scope;
+            }
             mode_ = PARSE_EAGERLY;
           }
           scope_->SetStrictMode(STRICT);
           // "use strict" is the only directive for now.
           directive_prologue = false;
         } else if (literal->raw_value()->AsString() ==
                        ast_value_factory_->use_asm_string() &&
                    token_loc.end_pos - token_loc.beg_pos ==
                        ast_value_factory_->use_asm_string()->length() + 2) {
           // Store the usage count; The actual use counter on the isolate is
@@ skipping 2626 matching lines @@
     Assignment* assignment = factory()->NewAssignment(
         Token::INIT_VAR, init_proxy, allocation, RelocInfo::kNoPosition);
     VariableProxy* get_proxy = factory()->NewVariableProxy(
         function_state_->generator_object_variable());
     Yield* yield = factory()->NewYield(
         get_proxy, assignment, Yield::kInitial, RelocInfo::kNoPosition);
     body->Add(factory()->NewExpressionStatement(
         yield, RelocInfo::kNoPosition), zone());
   }
 
-  ParseSourceElements(body, Token::RBRACE, false, false, CHECK_OK);
+  ParseSourceElements(body, Token::RBRACE, false, false, NULL, CHECK_OK);
 
   if (is_generator) {
     VariableProxy* get_proxy = factory()->NewVariableProxy(
         function_state_->generator_object_variable());
     Expression* undefined =
         factory()->NewUndefinedLiteral(RelocInfo::kNoPosition);
     Yield* yield = factory()->NewYield(get_proxy, undefined, Yield::kFinal,
                                        RelocInfo::kNoPosition);
     body->Add(factory()->NewExpressionStatement(
         yield, RelocInfo::kNoPosition), zone());
@@ skipping 181 matching lines @@
     Handle<String> source_mapping_url =
         scanner_.source_mapping_url()->Internalize(isolate());
     info_->script()->set_source_mapping_url(*source_mapping_url);
   }
 }
 
 
 void Parser::ThrowPendingError() {
   DCHECK(ast_value_factory_->IsInternalized());
   if (has_pending_error_) {
-    MessageLocation location(script_,
-                             pending_error_location_.beg_pos,
+    MessageLocation location(info()->script(), pending_error_location_.beg_pos,
                              pending_error_location_.end_pos);
     Factory* factory = isolate()->factory();
     bool has_arg =
         pending_error_arg_ != NULL || pending_error_char_arg_ != NULL;
     Handle<FixedArray> elements = factory->NewFixedArray(has_arg ? 1 : 0);
     if (pending_error_arg_ != NULL) {
       Handle<String> arg_string = pending_error_arg_->string();
       elements->set(0, *arg_string);
     } else if (pending_error_char_arg_ != NULL) {
       Handle<String> arg_string =
           factory->NewStringFromUtf8(CStrVector(pending_error_char_arg_))
           .ToHandleChecked();
       elements->set(0, *arg_string);
     }
-    isolate()->debug()->OnCompileError(script_);
+    isolate()->debug()->OnCompileError(info()->script());
 
     Handle<JSArray> array = factory->NewJSArrayWithElements(elements);
     Handle<Object> error;
     MaybeHandle<Object> maybe_error =
         pending_error_is_reference_error_
             ? factory->NewReferenceError(pending_error_message_, array)
             : factory->NewSyntaxError(pending_error_message_, array);
     if (maybe_error.ToHandle(&error)) isolate()->Throw(*error, &location);
   }
 }
@@ skipping 882 matching lines @@
   Internalize();
   DCHECK(ast_value_factory_->IsInternalized());
   // info takes ownership of ast_value_factory_.
   if (info()->ast_value_factory() == NULL) {
     info()->SetAstValueFactory(ast_value_factory_);
   }
   ast_value_factory_ = NULL;
   return (result != NULL);
 }
 
+
+void Parser::ParseOnBackground() {
+  DCHECK(info()->function() == NULL);
+  FunctionLiteral* result = NULL;
+  fni_ = new (zone()) FuncNameInferrer(ast_value_factory_, zone());
+
+  CompleteParserRecorder recorder;
+  if (compile_options() == ScriptCompiler::kProduceParserCache) {
+    log_ = &recorder;
+  }
+
+  DCHECK(info()->source_stream() != NULL);
+  ExternalStreamingStream stream(info()->source_stream(),
+                                 info()->source_stream_encoding());
+  scanner_.Initialize(&stream);
+  DCHECK(info()->context().is_null() || info()->context()->IsNativeContext());
+
+  // When streaming, we don't know the length of the source until we have parsed
+  // it. The raw data can be UTF-8, so we wouldn't know the source length until
+  // we have decoded it anyway even if we knew the raw data length (which we
+  // don't). We work around this by storing all the scopes which need their end
+  // position set at the end of the script (the top scope and possible eval
+  // scopes) and set their end position after we know the script length.
+  Scope* top_scope = NULL;
+  Scope* ad_hoc_eval_scope = NULL;
+  result = DoParseProgram(info(), &top_scope, &ad_hoc_eval_scope);
+
+  top_scope->set_end_position(scanner()->location().end_pos);
+  if (ad_hoc_eval_scope != NULL) {
+    ad_hoc_eval_scope->set_end_position(scanner()->location().end_pos);
+  }
+
+  if (info()->ast_value_factory() == NULL) {
+    info()->SetAstValueFactory(ast_value_factory_);
+  }
+  info()->SetFunction(result);
+
+  // We cannot internalize on a background thread; a foreground task will take
+  // care of calling Parser::Internalize just before compilation.
+
+  if (compile_options() == ScriptCompiler::kProduceParserCache) {
+    if (result != NULL) *info_->cached_data() = recorder.GetScriptData();
+    log_ = NULL;
+  }
+}
 } }  // namespace v8::internal