Inline Array.prototype.forEach in TurboFan

src/deoptimizer.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/deoptimizer.h"
 
 #include <memory>
 
 #include "src/accessors.h"
 #include "src/assembler-inl.h"
@@ skipping 809 matching lines @@
         break;
       case TranslatedFrame::kGetter:
         DoComputeAccessorStubFrame(translated_frame, frame_index, false);
         break;
       case TranslatedFrame::kSetter:
         DoComputeAccessorStubFrame(translated_frame, frame_index, true);
         break;
       case TranslatedFrame::kCompiledStub:
         DoComputeCompiledStubFrame(translated_frame, frame_index);
         break;
+      case TranslatedFrame::kBuiltinContinuation:
+        DoComputeBuiltinContinuation(translated_frame, frame_index);
+        break;
       case TranslatedFrame::kInvalid:
         FATAL("invalid frame");
         break;
     }
   }
 
   // Print some helpful diagnostic information.
   if (trace_scope_ != NULL) {
     double ms = timer.Elapsed().InMillisecondsF();
     int index = output_count_ - 1;  // Index of the topmost frame.
@@ skipping 1314 matching lines @@
     output_frame->SetRegister(constant_pool_reg.code(), constant_pool_value);
   }
   output_frame->SetState(
       Smi::FromInt(static_cast<int>(BailoutState::NO_REGISTERS)));
   Code* notify_failure =
       isolate_->builtins()->builtin(Builtins::kNotifyStubFailureSaveDoubles);
   output_frame->SetContinuation(
       reinterpret_cast<intptr_t>(notify_failure->entry()));
 }
 
+// BuiltinContinuationFrames capture the machine state that is expected as input
+// to a builtin, including both input register values and stack parameters. When
+// the frame is reactivated (i.e. the frame below it returns), a
+// ContinueToBuiltin stub restores the register state from the frame and tail
+// calls to the actual target builtin, making it appear that the stub had been
+// directly called by the frame above it. The input values to populate the frame
+// are taken from the deopt's FrameState.
+//
+// Frame translation happens in two modes, EAGER and LAZY. In EAGER mode, all of
+// the parameters to the Builtin are explicitly specified in the TurboFan
+// FrameState node. In LAZY mode, there is always one fewer parameters specified
+// in the FrameState than expected by the Builtin. In that case, construction of
+// BuiltinContinuationFrame adds the final missing parameter during
+// deoptimization, and that parameter is always on the stack and contains the
+// value returned from the callee of the call site triggering the LAZY deopt
+// (e.g. rax on x64). This requires that continuation Builtins for LAZY deopts
+// must have at least one stack parameter.
+//
+//                TO
+//    |          ....           |
+//    +-------------------------+
+//    |     builtin param 0     |<- FrameState input value n becomes
+//    +-------------------------+
+//    |           ...           |
+//    +-------------------------+
+//    |     builtin param m     |<- FrameState input value n+m-1, or in
+//    +-------------------------+   the LAZY case, return LAZY result value
+//    | ContinueToBuiltin entry |
+//    +-------------------------+
+// |  |    saved frame (FP)     |
+// |  +=========================+<- fpreg
+// |  |constant pool (if ool_cp)|
+// v  +-------------------------+
+//    |BUILTIN_CONTINUATION mark|
+//    +-------------------------+
+//    |  JS Builtin code object |
+//    +-------------------------+
+//    | builtin input GPR reg0  |<- populated from deopt FrameState using
+//    +-------------------------+   the builtin's CallInterfaceDescriptor
+//    |          ...            |   to map a FrameState's 0..n-1 inputs to
+//    +-------------------------+   the builtin's n input register params.
+//    | builtin input GPR regn  |
+//    |-------------------------|<- spreg
+//
+void Deoptimizer::DoComputeBuiltinContinuation(
+    TranslatedFrame* translated_frame, int frame_index) {
+  TranslatedFrame::iterator value_iterator = translated_frame->begin();
+  int input_index = 0;
+
+  // The output frame must have room for all of the parameters that need to be
+  // passed to the builtin continuation.
+  int height_in_words = translated_frame->height();
+
+  BailoutId bailout_id = translated_frame->node_id();
+  Builtins::Name builtin_name = Builtins::GetBuiltinFromBailoutId(bailout_id);
+  Code* builtin = isolate()->builtins()->builtin(builtin_name);
+  Callable continuation_callable =
+      Builtins::CallableFor(isolate(), builtin_name);
+  CallInterfaceDescriptor continuation_descriptor =
+      continuation_callable.descriptor();
+
+  bool is_bottommost = (0 == frame_index);
+  bool is_topmost = (output_count_ - 1 == frame_index);
+  bool must_handle_result = !is_topmost || bailout_type_ == LAZY;
+
+  const RegisterConfiguration* config(RegisterConfiguration::Turbofan());
+  int allocatable_register_count = config->num_allocatable_general_registers();
+  int register_parameter_count =
+      continuation_descriptor.GetRegisterParameterCount();
+  // Make sure to account for the context by removing it from the register
+  // parameter count.
+  int stack_param_count = height_in_words - register_parameter_count - 1;
+  if (must_handle_result) stack_param_count++;
+  int output_frame_size =
+      kPointerSize * stack_param_count +
+      TYPED_FRAME_SIZE(1 + allocatable_register_count);  // For destination
+                                                         // builtin code and
+                                                         // registers
+  if (trace_scope_ != NULL) {
+    PrintF(trace_scope_->file(),
+           "  translating BuiltinContinuation to %s, stack param count %d\n",
+           Builtins::name(builtin_name), stack_param_count);
+  }
+
+  unsigned output_frame_offset = output_frame_size;
+  FrameDescription* output_frame =
+      new (output_frame_size) FrameDescription(output_frame_size);
+  output_[frame_index] = output_frame;
+
+  // The top address of the frame is computed from the previous frame's top and
+  // this frame's size.
+  intptr_t top_address;
+  if (is_bottommost) {
+    top_address = caller_frame_top_ - output_frame_size;
+  } else {
+    top_address = output_[frame_index - 1]->GetTop() - output_frame_size;
+  }
+  output_frame->SetTop(top_address);
+
+  output_frame->SetState(
+      Smi::FromInt(static_cast<int>(BailoutState::NO_REGISTERS)));
+
+  // Get (and ignore) the JSFunction
+  (void)value_iterator->GetRawValue();
+  ++value_iterator;
+
+  std::vector<intptr_t> register_values;
+  int total_registers = config->num_general_registers();
+  register_values.resize(total_registers, 0);
+  for (int i = 0; i < total_registers; ++i) {
+    register_values[i] = 0;
+  }
+
+  bool java_script_builtin = false;
+  intptr_t value;
+  for (int i = 0; i < register_parameter_count; ++i) {
+    MachineType type = continuation_descriptor.GetParameterType(i);
+    int code = continuation_descriptor.GetRegisterParameter(i).code();
+    // Only tagged and int32 arguments are supported, and int32 only for the
+    // arguments count on JavaScript builtins.
+    if (type == MachineType::Int32()) {
+      CHECK_EQ(code, kJavaScriptCallArgCountRegister.code());
+      java_script_builtin = true;
+    } else {
+      // Any other argument must be a tagged value.
+      CHECK(IsAnyTagged(type.representation()));
+    }
+    value = reinterpret_cast<intptr_t>(value_iterator->GetRawValue());
+    register_values[code] = value;
+    ++input_index;
+    ++value_iterator;
+  }
+
+  value = reinterpret_cast<intptr_t>(value_iterator->GetRawValue());
+  register_values[kContextRegister.code()] = value;
+  output_frame->SetContext(value);
+  output_frame->SetRegister(kContextRegister.code(), value);
+  ++input_index;
+  ++value_iterator;
+
+  int translated_stack_parameters =
+      must_handle_result ? stack_param_count - 1 : stack_param_count;
+
+  for (int i = 0; i < translated_stack_parameters; ++i) {
+    output_frame_offset -= kPointerSize;
+    WriteTranslatedValueToOutput(&value_iterator, &input_index, frame_index,
+                                 output_frame_offset);
+  }
+
+  if (must_handle_result) {
+    output_frame_offset -= kPointerSize;
+    WriteValueToOutput(isolate()->heap()->the_hole_value(), input_index,
+                       frame_index, output_frame_offset,
+                       "placeholder for return result on lazy deopt ");
+  }
+
+  // Set caller's PC (JSFunction continuation).
+  output_frame_offset -= kPCOnStackSize;
+  if (is_bottommost) {
+    value = caller_pc_;
+  } else {
+    value = output_[frame_index - 1]->GetPc();
+  }
+  output_frame->SetCallerPc(output_frame_offset, value);
+  DebugPrintOutputSlot(value, frame_index, output_frame_offset,
+                       "caller's pc\n");
+
+  // Read caller's FP from the previous frame, and set this frame's FP.
+  output_frame_offset -= kFPOnStackSize;
+  if (is_bottommost) {
+    value = caller_fp_;
+  } else {
+    value = output_[frame_index - 1]->GetFp();
+  }
+  output_frame->SetCallerFp(output_frame_offset, value);
+  intptr_t fp_value = top_address + output_frame_offset;
+  output_frame->SetFp(fp_value);
+  DebugPrintOutputSlot(value, frame_index, output_frame_offset,
+                       "caller's fp\n");
+
+  if (FLAG_enable_embedded_constant_pool) {
+    // Read the caller's constant pool from the previous frame.
+    output_frame_offset -= kPointerSize;
+    if (is_bottommost) {
+      value = caller_constant_pool_;
+    } else {
+      value = output_[frame_index - 1]->GetConstantPool();
+    }
+    output_frame->SetCallerConstantPool(output_frame_offset, value);
+    DebugPrintOutputSlot(value, frame_index, output_frame_offset,
+                         "caller's constant_pool\n");
+  }
+
+  // A marker value is used in place of the context.
+  output_frame_offset -= kPointerSize;
+  intptr_t marker = StackFrame::TypeToMarker(StackFrame::BUILTIN_CONTINUATION);
+  output_frame->SetFrameSlot(output_frame_offset, marker);
+  DebugPrintOutputSlot(marker, frame_index, output_frame_offset,
+                       "context (builtin continuation sentinel)\n");
+
+  // The builtin to continue to
+  output_frame_offset -= kPointerSize;
+  value = reinterpret_cast<intptr_t>(builtin);
+  output_frame->SetFrameSlot(output_frame_offset, value);
+  DebugPrintOutputSlot(value, frame_index, output_frame_offset,
+                       "builtin address\n");
+
+  for (int i = 0; i < allocatable_register_count; ++i) {
+    output_frame_offset -= kPointerSize;
+    int code = config->GetAllocatableGeneralCode(i);
+    value = register_values[code];
+    output_frame->SetFrameSlot(output_frame_offset, value);
+    if (trace_scope_ != nullptr) {
+      ScopedVector<char> str(128);
+      if (java_script_builtin &&
+          code == kJavaScriptCallArgCountRegister.code()) {
+        SNPrintF(
+            str,
+            "tagged argument count %s (will be untagged by continuation)\n",
+            config->GetGeneralRegisterName(code));
+      } else {
+        SNPrintF(str, "builtin register argument %s\n",
+                 config->GetGeneralRegisterName(code));
+      }
+      DebugPrintOutputSlot(value, frame_index, output_frame_offset,
+                           str.start());
+    }
+  }
+
+  // Ensure the frame pointer register points to the callee's frame. The builtin
+  // will build its own frame once we continue to it.
+  Register fp_reg = JavaScriptFrame::fp_register();
+  output_frame->SetRegister(fp_reg.code(), output_[frame_index - 1]->GetFp());
+
+  Code* continue_to_builtin =
+      java_script_builtin
+          ? (must_handle_result
+                 ? isolate()->builtins()->builtin(
+                       Builtins::kContinueToJavaScriptBuiltinWithResult)
+                 : isolate()->builtins()->builtin(
+                       Builtins::kContinueToJavaScriptBuiltin))
+          : (must_handle_result
+                 ? isolate()->builtins()->builtin(
+                       Builtins::kContinueToCodeStubBuiltinWithResult)
+                 : isolate()->builtins()->builtin(
+                       Builtins::kContinueToCodeStubBuiltin));
+  output_frame->SetPc(
+      reinterpret_cast<intptr_t>(continue_to_builtin->instruction_start()));
+
+  Code* continuation =
+      isolate()->builtins()->builtin(Builtins::kNotifyBuiltinContinuation);
+  output_frame->SetContinuation(
+      reinterpret_cast<intptr_t>(continuation->entry()));
+}
 
 void Deoptimizer::MaterializeHeapObjects(JavaScriptFrameIterator* it) {
   // Walk to the last JavaScript output frame to find out if it has
   // adapted arguments.
   for (int frame_index = 0; frame_index < jsframe_count(); ++frame_index) {
     if (frame_index != 0) it->Advance();
   }
   translated_state_.Prepare(it->frame()->has_adapted_arguments(),
                             reinterpret_cast<Address>(stack_fp_));
 
@@ skipping 224 matching lines @@
   return is_negative ? -result : result;
 }
 
 
 Handle<ByteArray> TranslationBuffer::CreateByteArray(Factory* factory) {
   Handle<ByteArray> result = factory->NewByteArray(CurrentIndex(), TENURED);
   contents_.CopyTo(result->GetDataStartAddress());
   return result;
 }
 
+void Translation::BeginBuiltinContinuationFrame(BailoutId bailout_id,
+                                                int literal_id,
+                                                unsigned height) {
+  buffer_->Add(BUILTIN_CONTINUATION_FRAME);
+  buffer_->Add(bailout_id.ToInt());
+  buffer_->Add(literal_id);
+  buffer_->Add(height);
+}
+
 void Translation::BeginConstructStubFrame(BailoutId bailout_id, int literal_id,
                                           unsigned height) {
   buffer_->Add(CONSTRUCT_STUB_FRAME);
   buffer_->Add(bailout_id.ToInt());
   buffer_->Add(literal_id);
   buffer_->Add(height);
 }
 
 
 void Translation::BeginGetterStubFrame(int literal_id) {
@@ skipping 182 matching lines @@
     case LITERAL:
     case COMPILED_STUB_FRAME:
     case TAIL_CALLER_FRAME:
       return 1;
     case BEGIN:
     case ARGUMENTS_ADAPTOR_FRAME:
       return 2;
     case JS_FRAME:
     case INTERPRETED_FRAME:
     case CONSTRUCT_STUB_FRAME:
+    case BUILTIN_CONTINUATION_FRAME:
       return 3;
     case ARGUMENTS_ELEMENTS:
     case ARGUMENTS_LENGTH:
       return 1;
   }
   FATAL("Unexpected translation type");
   return -1;
 }
 
 
@@ skipping 587 matching lines @@
 }
 
 TranslatedFrame TranslatedFrame::ConstructStubFrame(
     BailoutId bailout_id, SharedFunctionInfo* shared_info, int height) {
   TranslatedFrame frame(kConstructStub, shared_info->GetIsolate(), shared_info,
                         height);
   frame.node_id_ = bailout_id;
   return frame;
 }
 
+TranslatedFrame TranslatedFrame::BuiltinContinuationFrame(
+    BailoutId bailout_id, SharedFunctionInfo* shared_info, int height) {
+  TranslatedFrame frame(kBuiltinContinuation, shared_info->GetIsolate(),
+                        shared_info, height);
+  frame.node_id_ = bailout_id;
+  return frame;
+}
 
 int TranslatedFrame::GetValueCount() {
   switch (kind()) {
     case kFunction: {
       int parameter_count =
           raw_shared_info_->internal_formal_parameter_count() + 1;
       // + 1 for function.
       return height_ + parameter_count + 1;
     }
 
     case kInterpretedFunction: {
       int parameter_count =
           raw_shared_info_->internal_formal_parameter_count() + 1;
       // + 2 for function and context.
       return height_ + parameter_count + 2;
     }
 
     case kGetter:
       return 2;  // Function and receiver.
 
     case kSetter:
       return 3;  // Function, receiver and the value to set.
 
     case kArgumentsAdaptor:
     case kConstructStub:
+    case kBuiltinContinuation:
       return 1 + height_;
 
     case kTailCallerFunction:
       return 1;  // Function.
 
     case kCompiledStub:
       return height_;
 
     case kInvalid:
       UNREACHABLE();
@@ skipping 84 matching lines @@
       if (trace_file != nullptr) {
         std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
         PrintF(trace_file, "  reading construct stub frame %s", name.get());
         PrintF(trace_file, " => bailout_id=%d, height=%d; inputs:\n",
                bailout_id.ToInt(), height);
       }
       return TranslatedFrame::ConstructStubFrame(bailout_id, shared_info,
                                                  height);
     }
 
+    case Translation::BUILTIN_CONTINUATION_FRAME: {
+      BailoutId bailout_id = BailoutId(iterator->Next());
+      SharedFunctionInfo* shared_info =
+          SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
+      int height = iterator->Next();
+      if (trace_file != nullptr) {
+        std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
+        PrintF(trace_file, "  reading builtin continuation frame %s",
+               name.get());
+        PrintF(trace_file, " => bailout_id=%d, height=%d; inputs:\n",
+               bailout_id.ToInt(), height);
+      }
+      return TranslatedFrame::BuiltinContinuationFrame(bailout_id, shared_info,
+                                                       height);
+    }
+
     case Translation::GETTER_STUB_FRAME: {
       SharedFunctionInfo* shared_info =
           SharedFunctionInfo::cast(literal_array->get(iterator->Next()));
       if (trace_file != nullptr) {
         std::unique_ptr<char[]> name = shared_info->DebugName()->ToCString();
         PrintF(trace_file, "  reading getter frame %s; inputs:\n", name.get());
       }
       return TranslatedFrame::AccessorFrame(TranslatedFrame::kGetter,
                                             shared_info);
     }
@@ skipping 148 matching lines @@
   switch (opcode) {
     case Translation::BEGIN:
     case Translation::JS_FRAME:
     case Translation::INTERPRETED_FRAME:
     case Translation::ARGUMENTS_ADAPTOR_FRAME:
     case Translation::TAIL_CALLER_FRAME:
     case Translation::CONSTRUCT_STUB_FRAME:
     case Translation::GETTER_STUB_FRAME:
     case Translation::SETTER_STUB_FRAME:
     case Translation::COMPILED_STUB_FRAME:
+    case Translation::BUILTIN_CONTINUATION_FRAME:
       // Peeled off before getting here.
       break;
 
     case Translation::DUPLICATED_OBJECT: {
       int object_id = iterator->Next();
       if (trace_file != nullptr) {
         PrintF(trace_file, "duplicated object #%d", object_id);
       }
       object_positions_.push_back(object_positions_[object_id]);
       TranslatedValue translated_value =
@@ skipping 934 matching lines @@
       CHECK(value_info->IsMaterializedObject());
 
       value_info->value_ =
           Handle<Object>(previously_materialized_objects->get(i), isolate_);
     }
   }
 }
 
 }  // namespace internal
 }  // namespace v8