Inline Array.prototype.forEach in TurboFan

src/deoptimizer.cc

Index: src/deoptimizer.cc
diff --git a/src/deoptimizer.cc b/src/deoptimizer.cc
index 6ad0b76c07f39abadc7a9fd41f684b008a836196..f5c56243a0fabda2932411d22bb12735241b8695 100644
--- a/src/deoptimizer.cc
+++ b/src/deoptimizer.cc
@@ -827,6 +827,12 @@ void Deoptimizer::DoComputeOutputFrames() {
       case TranslatedFrame::kCompiledStub:
         DoComputeCompiledStubFrame(translated_frame, frame_index);
         break;
+      case TranslatedFrame::kBuiltinContinuation:
+        DoComputeBuiltinContinuation(translated_frame, frame_index, false);
+        break;
+      case TranslatedFrame::kJavaScriptBuiltinContinuation:
+        DoComputeBuiltinContinuation(translated_frame, frame_index, true);
+        break;
       case TranslatedFrame::kInvalid:
         FATAL("invalid frame");
         break;
@@ -2161,6 +2167,292 @@ void Deoptimizer::DoComputeCompiledStubFrame(TranslatedFrame* translated_frame,
       reinterpret_cast<intptr_t>(notify_failure->entry()));
 }
 
+// BuiltinContinuationFrames capture the machine state that is expected as input
+// to a builtin, including both input register values and stack parameters. When
+// the frame is reactivated (i.e. the frame below it returns), a
+// ContinueToBuiltin stub restores the register state from the frame and tail
+// calls to the actual target builtin, making it appear that the stub had been
+// directly called by the frame above it. The input values to populate the frame
+// are taken from the deopt's FrameState.
+//
+// Frame translation happens in two modes, EAGER and LAZY. In EAGER mode, all of
+// the parameters to the Builtin are explicitly specified in the TurboFan
+// FrameState node. In LAZY mode, there is always one fewer parameters specified
+// in the FrameState than expected by the Builtin. In that case, construction of
+// BuiltinContinuationFrame adds the final missing parameter during
+// deoptimization, and that parameter is always on the stack and contains the
+// value returned from the callee of the call site triggering the LAZY deopt
+// (e.g. rax on x64). This requires that continuation Builtins for LAZY deopts
+// must have at least one stack parameter.
+//
+//                TO
+//    |          ....           |
+//    +-------------------------+
+//    |     builtin param 0     |<- FrameState input value n becomes
+//    +-------------------------+
+//    |           ...           |
+//    +-------------------------+
+//    |     builtin param m     |<- FrameState input value n+m-1, or in
+//    +-------------------------+   the LAZY case, return LAZY result value
+//    | ContinueToBuiltin entry |
+//    +-------------------------+
+// |  |    saved frame (FP)     |
+// |  +=========================+<- fpreg
+// |  |constant pool (if ool_cp)|
+// v  +-------------------------+
+//    |BUILTIN_CONTINUATION mark|
+//    +-------------------------+
+//    |  JS Builtin code object |
+//    +-------------------------+
+//    | builtin input GPR reg0  |<- populated from deopt FrameState using
+//    +-------------------------+   the builtin's CallInterfaceDescriptor
+//    |          ...            |   to map a FrameState's 0..n-1 inputs to
+//    +-------------------------+   the builtin's n input register params.
+//    | builtin input GPR regn  |
+//    |-------------------------|<- spreg
+//
+void Deoptimizer::DoComputeBuiltinContinuation(
+    TranslatedFrame* translated_frame, int frame_index,
+    bool java_script_builtin) {
+  TranslatedFrame::iterator value_iterator = translated_frame->begin();
+  int input_index = 0;
+
+  // The output frame must have room for all of the parameters that need to be
+  // passed to the builtin continuation.
+  int height_in_words = translated_frame->height();
+
+  BailoutId bailout_id = translated_frame->node_id();
+  Builtins::Name builtin_name = Builtins::GetBuiltinFromBailoutId(bailout_id);
+  Code* builtin = isolate()->builtins()->builtin(builtin_name);
+  Callable continuation_callable =
+      Builtins::CallableFor(isolate(), builtin_name);
+  CallInterfaceDescriptor continuation_descriptor =
+      continuation_callable.descriptor();
+
+  bool is_bottommost = (0 == frame_index);
+  bool is_topmost = (output_count_ - 1 == frame_index);
+  bool must_handle_result = !is_topmost || bailout_type_ == LAZY;
+
+  const RegisterConfiguration* config(RegisterConfiguration::Turbofan());
+  int allocatable_register_count = config->num_allocatable_general_registers();
+  int register_parameter_count =
+      continuation_descriptor.GetRegisterParameterCount();
+  // Make sure to account for the context by removing it from the register
+  // parameter count.
+  int stack_param_count = height_in_words - register_parameter_count - 1;
+  if (must_handle_result) stack_param_count++;
+  int output_frame_size =
+      kPointerSize * (stack_param_count + allocatable_register_count) +
+      TYPED_FRAME_SIZE(2);  // For destination builtin code and registers
+
+  // Validate types of parameters. They must all be tagged except for argc for
+  // JS builtins.
+  bool has_argc = false;
+  for (int i = 0; i < register_parameter_count; ++i) {
+    MachineType type = continuation_descriptor.GetParameterType(i);
+    int code = continuation_descriptor.GetRegisterParameter(i).code();
+    // Only tagged and int32 arguments are supported, and int32 only for the
+    // arguments count on JavaScript builtins.
+    if (type == MachineType::Int32()) {
+      CHECK_EQ(code, kJavaScriptCallArgCountRegister.code());
+      has_argc = true;
+    } else {
+      // Any other argument must be a tagged value.
+      CHECK(IsAnyTagged(type.representation()));
+    }
+  }
+  CHECK_EQ(java_script_builtin, has_argc);
+
+  if (trace_scope_ != NULL) {
+    PrintF(trace_scope_->file(),
+           "  translating BuiltinContinuation to %s, stack param count %d\n",
+           Builtins::name(builtin_name), stack_param_count);
+  }
+
+  unsigned output_frame_offset = output_frame_size;
+  FrameDescription* output_frame =
+      new (output_frame_size) FrameDescription(output_frame_size);
+  output_[frame_index] = output_frame;
+
+  // The top address of the frame is computed from the previous frame's top and
+  // this frame's size.
+  intptr_t top_address;
+  if (is_bottommost) {
+    top_address = caller_frame_top_ - output_frame_size;
+  } else {
+    top_address = output_[frame_index - 1]->GetTop() - output_frame_size;
+  }
+  output_frame->SetTop(top_address);
+
+  output_frame->SetState(
+      Smi::FromInt(static_cast<int>(BailoutState::NO_REGISTERS)));
+
+  // Get the possible JSFunction for the case that
+  intptr_t maybe_function =
+      reinterpret_cast<intptr_t>(value_iterator->GetRawValue());
+  ++value_iterator;
+
+  std::vector<intptr_t> register_values;
+  int total_registers = config->num_general_registers();
+  register_values.resize(total_registers, 0);
+  for (int i = 0; i < total_registers; ++i) {
+    register_values[i] = 0;
+  }
+
+  intptr_t value;
+
+  Register result_reg = FullCodeGenerator::result_register();
+  if (must_handle_result) {
+    value = input_->GetRegister(result_reg.code());
+  } else {
+    value = reinterpret_cast<intptr_t>(isolate()->heap()->undefined_value());
+  }
+  output_frame->SetRegister(result_reg.code(), value);
+
+  int translated_stack_parameters =
+      must_handle_result ? stack_param_count - 1 : stack_param_count;
+
+  for (int i = 0; i < translated_stack_parameters; ++i) {
+    output_frame_offset -= kPointerSize;
+    WriteTranslatedValueToOutput(&value_iterator, &input_index, frame_index,
+                                 output_frame_offset);
+  }
+
+  if (must_handle_result) {
+    output_frame_offset -= kPointerSize;
+    WriteValueToOutput(isolate()->heap()->the_hole_value(), input_index,
+                       frame_index, output_frame_offset,
+                       "placeholder for return result on lazy deopt ");
+  }
+
+  for (int i = 0; i < register_parameter_count; ++i) {
+    value = reinterpret_cast<intptr_t>(value_iterator->GetRawValue());
+    int code = continuation_descriptor.GetRegisterParameter(i).code();
+    register_values[code] = value;
+    ++input_index;
+    ++value_iterator;
+  }
+
+  // The context register is always implicit in the CallInterfaceDescriptor but
+  // its register must be explicitly set when continuing to the builtin. Make
+  // sure that it's harvested from the translation and copied into the register
+  // set (it was automatically added at the end of the FrameState by the
+  // instruction selector).
+  value = reinterpret_cast<intptr_t>(value_iterator->GetRawValue());
+  register_values[kContextRegister.code()] = value;
+  output_frame->SetContext(value);
+  output_frame->SetRegister(kContextRegister.code(), value);
+  ++input_index;
+  ++value_iterator;
+
+  // Set caller's PC (JSFunction continuation).
+  output_frame_offset -= kPCOnStackSize;
+  if (is_bottommost) {
+    value = caller_pc_;
+  } else {
+    value = output_[frame_index - 1]->GetPc();
+  }
+  output_frame->SetCallerPc(output_frame_offset, value);
+  DebugPrintOutputSlot(value, frame_index, output_frame_offset,
+                       "caller's pc\n");
+
+  // Read caller's FP from the previous frame, and set this frame's FP.
+  output_frame_offset -= kFPOnStackSize;
+  if (is_bottommost) {
+    value = caller_fp_;
+  } else {
+    value = output_[frame_index - 1]->GetFp();
+  }
+  output_frame->SetCallerFp(output_frame_offset, value);
+  intptr_t fp_value = top_address + output_frame_offset;
+  output_frame->SetFp(fp_value);
+  DebugPrintOutputSlot(value, frame_index, output_frame_offset,
+                       "caller's fp\n");
+
+  if (FLAG_enable_embedded_constant_pool) {
+    // Read the caller's constant pool from the previous frame.
+    output_frame_offset -= kPointerSize;
+    if (is_bottommost) {
+      value = caller_constant_pool_;
+    } else {
+      value = output_[frame_index - 1]->GetConstantPool();
+    }
+    output_frame->SetCallerConstantPool(output_frame_offset, value);
+    DebugPrintOutputSlot(value, frame_index, output_frame_offset,
+                         "caller's constant_pool\n");
+  }
+
+  // A marker value is used in place of the context.
+  output_frame_offset -= kPointerSize;
+  intptr_t marker =
+      java_script_builtin
+          ? StackFrame::TypeToMarker(
+                StackFrame::JAVA_SCRIPT_BUILTIN_CONTINUATION)
+          : StackFrame::TypeToMarker(StackFrame::BUILTIN_CONTINUATION);
+  output_frame->SetFrameSlot(output_frame_offset, marker);
+  DebugPrintOutputSlot(marker, frame_index, output_frame_offset,
+                       "context (builtin continuation sentinel)\n");
+
+  output_frame_offset -= kPointerSize;
+  value = java_script_builtin ? maybe_function : 0;
+  output_frame->SetFrameSlot(output_frame_offset, value);
+  DebugPrintOutputSlot(value, frame_index, output_frame_offset,
+                       java_script_builtin ? "JSFunction\n" : "unused\n");
+
+  // The builtin to continue to
+  output_frame_offset -= kPointerSize;
+  value = reinterpret_cast<intptr_t>(builtin);
+  output_frame->SetFrameSlot(output_frame_offset, value);
+  DebugPrintOutputSlot(value, frame_index, output_frame_offset,
+                       "builtin address\n");
+
+  for (int i = 0; i < allocatable_register_count; ++i) {
+    output_frame_offset -= kPointerSize;
+    int code = config->GetAllocatableGeneralCode(i);
+    value = register_values[code];
+    output_frame->SetFrameSlot(output_frame_offset, value);
+    if (trace_scope_ != nullptr) {
+      ScopedVector<char> str(128);
+      if (java_script_builtin &&
+          code == kJavaScriptCallArgCountRegister.code()) {
+        SNPrintF(
+            str,
+            "tagged argument count %s (will be untagged by continuation)\n",
+            config->GetGeneralRegisterName(code));
+      } else {
+        SNPrintF(str, "builtin register argument %s\n",
+                 config->GetGeneralRegisterName(code));
+      }
+      DebugPrintOutputSlot(value, frame_index, output_frame_offset,
+                           str.start());
+    }
+  }
+
+  // Ensure the frame pointer register points to the callee's frame. The builtin
+  // will build its own frame once we continue to it.
+  Register fp_reg = JavaScriptFrame::fp_register();
+  output_frame->SetRegister(fp_reg.code(), output_[frame_index - 1]->GetFp());
+
+  Code* continue_to_builtin =
+      java_script_builtin
+          ? (must_handle_result
+                 ? isolate()->builtins()->builtin(
+                       Builtins::kContinueToJavaScriptBuiltinWithResult)
+                 : isolate()->builtins()->builtin(
+                       Builtins::kContinueToJavaScriptBuiltin))
+          : (must_handle_result
+                 ? isolate()->builtins()->builtin(
+                       Builtins::kContinueToCodeStubBuiltinWithResult)
+                 : isolate()->builtins()->builtin(
+                       Builtins::kContinueToCodeStubBuiltin));
+  output_frame->SetPc(
+      reinterpret_cast<intptr_t>(continue_to_builtin->instruction_start()));
+
+  Code* continuation =
+      isolate()->builtins()->builtin(Builtins::kNotifyBuiltinContinuation);
+  output_frame->SetContinuation(
+      reinterpret_cast<intptr_t>(continuation->entry()));
+}
 
 void Deoptimizer::MaterializeHeapObjects(JavaScriptFrameIterator* it) {
   // Walk to the last JavaScript output frame to find out if it has
@@ -2405,6 +2697,24 @@ Handle<ByteArray> TranslationBuffer::CreateByteArray(Factory* factory) {
   return result;
 }
 
+void Translation::BeginBuiltinContinuationFrame(BailoutId bailout_id,
+                                                int literal_id,
+                                                unsigned height) {
+  buffer_->Add(BUILTIN_CONTINUATION_FRAME);
+  buffer_->Add(bailout_id.ToInt());
+  buffer_->Add(literal_id);
+  buffer_->Add(height);
+}
+
+void Translation::BeginJavaScriptBuiltinContinuationFrame(BailoutId bailout_id,
+                                                          int literal_id,
+                                                          unsigned height) {
+  buffer_->Add(JAVA_SCRIPT_BUILTIN_CONTINUATION_FRAME);
+  buffer_->Add(bailout_id.ToInt());
+  buffer_->Add(literal_id);
+  buffer_->Add(height);
+}
+
 void Translation::BeginConstructStubFrame(BailoutId bailout_id, int literal_id,
                                           unsigned height) {
   buffer_->Add(CONSTRUCT_STUB_FRAME);
@@ -2607,6 +2917,8 @@ int Translation::NumberOfOperandsFor(Opcode opcode) {
     case JS_FRAME:
     case INTERPRETED_FRAME:
     case CONSTRUCT_STUB_FRAME:
+    case BUILTIN_CONTINUATION_FRAME:
+    case JAVA_SCRIPT_BUILTIN_CONTINUATION_FRAME:
       return 3;
     case ARGUMENTS_ELEMENTS:
     case ARGUMENTS_LENGTH:
@@ -3213,6 +3525,22 @@ TranslatedFrame TranslatedFrame::ConstructStubFrame(
   return frame;
 }
 
+TranslatedFrame TranslatedFrame::BuiltinContinuationFrame(
+    BailoutId bailout_id, SharedFunctionInfo* shared_info, int height) {
+  base::OS::DebugBreak();
+  TranslatedFrame frame(kBuiltinContinuation, shared_info->GetIsolate(),
+                        shared_info, height);
+  frame.node_id_ = bailout_id;
+  return frame;
+}
+
+TranslatedFrame TranslatedFrame::JavaScriptBuiltinContinuationFrame(
+    BailoutId bailout_id, SharedFunctionInfo* shared_info, int height) {
+  TranslatedFrame frame(kJavaScriptBuiltinContinuation,
+                        shared_info->GetIsolate(), shared_info, height);
+  frame.node_id_ = bailout_id;
+  return frame;
+}
 
 int TranslatedFrame::GetValueCount() {
   switch (kind()) {
@@ -3238,6 +3566,8 @@ int TranslatedFrame::GetValueCount() {
 
     case kArgumentsAdaptor:
     case kConstructStub:
+    case kBuiltinContinuation:
+    case kJavaScriptBuiltinContinuation:
       return 1 + height_;
 
     case kTailCallerFunction:
@@ -3341,6 +3671,38 @@ TranslatedFrame TranslatedState::CreateNextTranslatedFrame(
                                                  height);
     }
 
+    case Translation::BUILTIN_CONTINUATION_FRAME: {
+      BailoutId bailout_id = BailoutId(iterator->Next());
+      SharedFunctionInfo* shared_info =
+          SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
+      int height = iterator->Next();
+      if (trace_file != nullptr) {
+        std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
+        PrintF(trace_file, "  reading builtin continuation frame %s",
+               name.get());
+        PrintF(trace_file, " => bailout_id=%d, height=%d; inputs:\n",
+               bailout_id.ToInt(), height);
+      }
+      return TranslatedFrame::BuiltinContinuationFrame(bailout_id, shared_info,
+                                                       height);
+    }
+
+    case Translation::JAVA_SCRIPT_BUILTIN_CONTINUATION_FRAME: {
+      BailoutId bailout_id = BailoutId(iterator->Next());
+      SharedFunctionInfo* shared_info =
+          SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
+      int height = iterator->Next();
+      if (trace_file != nullptr) {
+        std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
+        PrintF(trace_file, "  reading JavaScript builtin continuation frame %s",
+               name.get());
+        PrintF(trace_file, " => bailout_id=%d, height=%d; inputs:\n",
+               bailout_id.ToInt(), height);
+      }
+      return TranslatedFrame::JavaScriptBuiltinContinuationFrame(
+          bailout_id, shared_info, height + 1);
+    }
+
     case Translation::GETTER_STUB_FRAME: {
       SharedFunctionInfo* shared_info =
           SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
@@ -3509,6 +3871,8 @@ int TranslatedState::CreateNextTranslatedValue(
     case Translation::GETTER_STUB_FRAME:
     case Translation::SETTER_STUB_FRAME:
     case Translation::COMPILED_STUB_FRAME:
+    case Translation::JAVA_SCRIPT_BUILTIN_CONTINUATION_FRAME:
+    case Translation::BUILTIN_CONTINUATION_FRAME:
       // Peeled off before getting here.
       break;