Index: src/x64/stub-cache-x64.cc |
diff --git a/src/x64/stub-cache-x64.cc b/src/x64/stub-cache-x64.cc |
index dbfd419290c692b61758bacb58d9e76dcea55668..392c96be6c7759933bac9c72bbaf0deb3352fb15 100644 |
--- a/src/x64/stub-cache-x64.cc |
+++ b/src/x64/stub-cache-x64.cc |
@@ -392,57 +392,92 @@ static void CompileCallLoadPropertyWithInterceptor( |
static const int kFastApiCallArguments = FunctionCallbackArguments::kArgsLength; |
+// Reserves space for the extra arguments to API function in the |
+// caller's frame. |
+// |
+// These arguments are set by CheckPrototypes and GenerateFastApiCall. |
+static void ReserveSpaceForFastApiCall(MacroAssembler* masm, Register scratch) { |
+ // ----------- S t a t e ------------- |
+ // -- rsp[0] : return address |
+ // -- rsp[8] : last argument in the internal frame of the caller |
+ // ----------------------------------- |
+ __ movq(scratch, StackOperandForReturnAddress(0)); |
+ __ subq(rsp, Immediate(kFastApiCallArguments * kPointerSize)); |
+ __ movq(StackOperandForReturnAddress(0), scratch); |
+ __ Move(scratch, Smi::FromInt(0)); |
+ StackArgumentsAccessor args(rsp, kFastApiCallArguments, |
+ ARGUMENTS_DONT_CONTAIN_RECEIVER); |
+ for (int i = 0; i < kFastApiCallArguments; i++) { |
+ __ movp(args.GetArgumentOperand(i), scratch); |
+ } |
+} |
+ |
+ |
+// Undoes the effects of ReserveSpaceForFastApiCall. |
+static void FreeSpaceForFastApiCall(MacroAssembler* masm, Register scratch) { |
+ // ----------- S t a t e ------------- |
+ // -- rsp[0] : return address. |
+ // -- rsp[8] : last fast api call extra argument. |
+ // -- ... |
+ // -- rsp[kFastApiCallArguments * 8] : first fast api call extra |
+ // argument. |
+ // -- rsp[kFastApiCallArguments * 8 + 8] : last argument in the internal |
+ // frame. |
+ // ----------------------------------- |
+ __ movq(scratch, StackOperandForReturnAddress(0)); |
+ __ movq(StackOperandForReturnAddress(kFastApiCallArguments * kPointerSize), |
+ scratch); |
+ __ addq(rsp, Immediate(kPointerSize * kFastApiCallArguments)); |
+} |
+ |
+ |
static void GenerateFastApiCallBody(MacroAssembler* masm, |
const CallOptimization& optimization, |
int argc, |
- Register holder, |
- Register scratch1, |
- Register scratch2, |
- Register scratch3, |
bool restore_context); |
// Generates call to API function. |
static void GenerateFastApiCall(MacroAssembler* masm, |
const CallOptimization& optimization, |
- int argc, |
- Handle<Map> map_to_holder, |
- CallOptimization::HolderLookup holder_lookup) { |
- Counters* counters = masm->isolate()->counters(); |
- __ IncrementCounter(counters->call_const_fast_api(), 1); |
+ int argc) { |
+ typedef FunctionCallbackArguments FCA; |
+ StackArgumentsAccessor args(rsp, argc + kFastApiCallArguments); |
- // Move holder to a register |
- Register holder_reg = rax; |
- switch (holder_lookup) { |
- case CallOptimization::kHolderIsReceiver: |
- { |
- ASSERT(map_to_holder.is_null()); |
- StackArgumentsAccessor args(rsp, argc); |
- __ movp(holder_reg, args.GetReceiverOperand()); |
- } |
- break; |
- case CallOptimization::kHolderIsPrototypeOfMap: |
- { |
- Handle<JSObject> holder(JSObject::cast(map_to_holder->prototype())); |
- if (!masm->isolate()->heap()->InNewSpace(*holder)) { |
- __ Move(holder_reg, holder); |
- } else { |
- __ Move(holder_reg, map_to_holder); |
- __ movp(holder_reg, FieldOperand(holder_reg, Map::kPrototypeOffset)); |
- } |
- } |
- break; |
- case CallOptimization::kHolderNotFound: |
- UNREACHABLE(); |
+ // Save calling context. |
+ int offset = argc + kFastApiCallArguments; |
+ __ movp(args.GetArgumentOperand(offset - FCA::kContextSaveIndex), rsi); |
+ |
+ // Get the function and setup the context. |
+ Handle<JSFunction> function = optimization.constant_function(); |
+ __ Move(rdi, function); |
+ __ movp(rsi, FieldOperand(rdi, JSFunction::kContextOffset)); |
+ // Construct the FunctionCallbackInfo on the stack. |
+ __ movp(args.GetArgumentOperand(offset - FCA::kCalleeIndex), rdi); |
+ Handle<CallHandlerInfo> api_call_info = optimization.api_call_info(); |
+ Handle<Object> call_data(api_call_info->data(), masm->isolate()); |
+ if (masm->isolate()->heap()->InNewSpace(*call_data)) { |
+ __ Move(rcx, api_call_info); |
+ __ movp(rbx, FieldOperand(rcx, CallHandlerInfo::kDataOffset)); |
+ __ movp(args.GetArgumentOperand(offset - FCA::kDataIndex), rbx); |
+ } else { |
+ __ Move(args.GetArgumentOperand(offset - FCA::kDataIndex), call_data); |
} |
- GenerateFastApiCallBody(masm, |
- optimization, |
- argc, |
- holder_reg, |
- rbx, |
- rcx, |
- rdx, |
- false); |
+ __ Move(kScratchRegister, |
+ ExternalReference::isolate_address(masm->isolate())); |
+ __ movp(args.GetArgumentOperand(offset - FCA::kIsolateIndex), |
+ kScratchRegister); |
+ __ LoadRoot(kScratchRegister, Heap::kUndefinedValueRootIndex); |
+ __ movp(args.GetArgumentOperand(offset - FCA::kReturnValueDefaultValueIndex), |
+ kScratchRegister); |
+ __ movp(args.GetArgumentOperand(offset - FCA::kReturnValueOffset), |
+ kScratchRegister); |
+ |
+ // Prepare arguments. |
+ STATIC_ASSERT(kFastApiCallArguments == 7); |
+ __ lea(rax, args.GetArgumentOperand(offset - FCA::kHolderIndex)); |
+ |
+ GenerateFastApiCallBody(masm, optimization, argc, false); |
} |
@@ -458,9 +493,13 @@ static void GenerateFastApiCall(MacroAssembler* masm, |
Register scratch3, |
int argc, |
Register* values) { |
+ ASSERT(optimization.is_simple_api_call()); |
+ |
__ PopReturnAddressTo(scratch1); |
+ |
// receiver |
__ push(receiver); |
+ |
// Write the arguments to stack frame. |
for (int i = 0; i < argc; i++) { |
Register arg = values[argc-1-i]; |
@@ -470,35 +509,6 @@ static void GenerateFastApiCall(MacroAssembler* masm, |
ASSERT(!scratch3.is(arg)); |
__ push(arg); |
} |
- __ PushReturnAddressFrom(scratch1); |
- // Stack now matches JSFunction abi. |
- GenerateFastApiCallBody(masm, |
- optimization, |
- argc, |
- receiver, |
- scratch1, |
- scratch2, |
- scratch3, |
- true); |
-} |
- |
- |
-static void GenerateFastApiCallBody(MacroAssembler* masm, |
- const CallOptimization& optimization, |
- int argc, |
- Register holder, |
- Register scratch1, |
- Register scratch2, |
- Register scratch3, |
- bool restore_context) { |
- // ----------- S t a t e ------------- |
- // -- rsp[0] : return address |
- // -- rsp[8] : last argument |
- // -- ... |
- // -- rsp[argc * 8] : first argument |
- // -- rsp[(argc + 1) * 8] : receiver |
- // ----------------------------------- |
- ASSERT(optimization.is_simple_api_call()); |
typedef FunctionCallbackArguments FCA; |
@@ -511,9 +521,6 @@ static void GenerateFastApiCallBody(MacroAssembler* masm, |
STATIC_ASSERT(FCA::kContextSaveIndex == 6); |
STATIC_ASSERT(FCA::kArgsLength == 7); |
- __ PopReturnAddressTo(scratch1); |
- |
- ASSERT(!holder.is(rsi)); |
// context save |
__ push(rsi); |
@@ -550,13 +557,36 @@ static void GenerateFastApiCallBody(MacroAssembler* masm, |
ExternalReference::isolate_address(masm->isolate())); |
__ push(scratch3); |
// holder |
- __ push(holder); |
+ __ push(receiver); |
ASSERT(!scratch1.is(rax)); |
+ // store receiver address for GenerateFastApiCallBody |
__ movp(rax, rsp); |
- // Push return address back on stack. |
__ PushReturnAddressFrom(scratch1); |
+ GenerateFastApiCallBody(masm, optimization, argc, true); |
+} |
+ |
+ |
+static void GenerateFastApiCallBody(MacroAssembler* masm, |
+ const CallOptimization& optimization, |
+ int argc, |
+ bool restore_context) { |
+ // ----------- S t a t e ------------- |
+ // -- rsp[0] : return address |
+ // -- rsp[8] - rsp[56] : FunctionCallbackInfo, incl. |
+ // : object passing the type check |
+ // (set by CheckPrototypes) |
+ // -- rsp[64] : last argument |
+ // -- ... |
+ // -- rsp[(argc + 7) * 8] : first argument |
+ // -- rsp[(argc + 8) * 8] : receiver |
+ // |
+ // rax : receiver address |
+ // ----------------------------------- |
+ typedef FunctionCallbackArguments FCA; |
+ |
+ Handle<CallHandlerInfo> api_call_info = optimization.api_call_info(); |
// Function address is a foreign pointer outside V8's heap. |
Address function_address = v8::ToCData<Address>(api_call_info->callback()); |
@@ -652,17 +682,38 @@ class CallInterceptorCompiler BASE_EMBEDDED { |
ASSERT(optimization.is_constant_call()); |
ASSERT(!lookup->holder()->IsGlobalObject()); |
+ int depth1 = kInvalidProtoDepth; |
+ int depth2 = kInvalidProtoDepth; |
+ bool can_do_fast_api_call = false; |
+ if (optimization.is_simple_api_call() && |
+ !lookup->holder()->IsGlobalObject()) { |
+ depth1 = optimization.GetPrototypeDepthOfExpectedType( |
+ object, interceptor_holder); |
+ if (depth1 == kInvalidProtoDepth) { |
+ depth2 = optimization.GetPrototypeDepthOfExpectedType( |
+ interceptor_holder, Handle<JSObject>(lookup->holder())); |
+ } |
+ can_do_fast_api_call = |
+ depth1 != kInvalidProtoDepth || depth2 != kInvalidProtoDepth; |
+ } |
+ |
Counters* counters = masm->isolate()->counters(); |
__ IncrementCounter(counters->call_const_interceptor(), 1); |
+ if (can_do_fast_api_call) { |
+ __ IncrementCounter(counters->call_const_interceptor_fast_api(), 1); |
+ ReserveSpaceForFastApiCall(masm, scratch1); |
+ } |
+ |
// Check that the maps from receiver to interceptor's holder |
// haven't changed and thus we can invoke interceptor. |
Label miss_cleanup; |
+ Label* miss = can_do_fast_api_call ? &miss_cleanup : miss_label; |
Register holder = |
stub_compiler_->CheckPrototypes( |
IC::CurrentTypeOf(object, masm->isolate()), receiver, |
interceptor_holder, scratch1, scratch2, scratch3, |
- name, miss_label); |
+ name, depth1, miss); |
// Invoke an interceptor and if it provides a value, |
// branch to |regular_invoke|. |
@@ -679,41 +730,35 @@ class CallInterceptorCompiler BASE_EMBEDDED { |
stub_compiler_->CheckPrototypes( |
IC::CurrentTypeOf(interceptor_holder, masm->isolate()), holder, |
handle(lookup->holder()), scratch1, scratch2, scratch3, |
- name, miss_label); |
- } |
- |
- Handle<Map> lookup_map; |
- CallOptimization::HolderLookup holder_lookup = |
- CallOptimization::kHolderNotFound; |
- if (optimization.is_simple_api_call() && |
- !lookup->holder()->IsGlobalObject()) { |
- lookup_map = optimization.LookupHolderOfExpectedType( |
- object, object, interceptor_holder, &holder_lookup); |
- if (holder_lookup == CallOptimization::kHolderNotFound) { |
- lookup_map = |
- optimization.LookupHolderOfExpectedType( |
- object, |
- interceptor_holder, |
- Handle<JSObject>(lookup->holder()), |
- &holder_lookup); |
- } |
+ name, depth2, miss); |
+ } else { |
+ // CheckPrototypes has a side effect of fetching a 'holder' |
+ // for API (object which is instanceof for the signature). It's |
+ // safe to omit it here, as if present, it should be fetched |
+ // by the previous CheckPrototypes. |
+ ASSERT(depth2 == kInvalidProtoDepth); |
} |
// Invoke function. |
- if (holder_lookup != CallOptimization::kHolderNotFound) { |
- int argc = arguments_.immediate(); |
- GenerateFastApiCall(masm, |
- optimization, |
- argc, |
- lookup_map, |
- holder_lookup); |
+ if (can_do_fast_api_call) { |
+ GenerateFastApiCall(masm, optimization, arguments_.immediate()); |
} else { |
Handle<JSFunction> fun = optimization.constant_function(); |
stub_compiler_->GenerateJumpFunction(object, fun); |
} |
+ // Deferred code for fast API call case---clean preallocated space. |
+ if (can_do_fast_api_call) { |
+ __ bind(&miss_cleanup); |
+ FreeSpaceForFastApiCall(masm, scratch1); |
+ __ jmp(miss_label); |
+ } |
+ |
// Invoke a regular function. |
__ bind(®ular_invoke); |
+ if (can_do_fast_api_call) { |
+ FreeSpaceForFastApiCall(masm, scratch1); |
+ } |
} |
void CompileRegular(MacroAssembler* masm, |
@@ -1075,6 +1120,7 @@ Register StubCompiler::CheckPrototypes(Handle<HeapType> type, |
Register scratch1, |
Register scratch2, |
Handle<Name> name, |
+ int save_at_depth, |
Label* miss, |
PrototypeCheckType check) { |
Handle<Map> receiver_map(IC::TypeToMap(*type, isolate())); |
@@ -1093,6 +1139,15 @@ Register StubCompiler::CheckPrototypes(Handle<HeapType> type, |
Register reg = object_reg; |
int depth = 0; |
+ StackArgumentsAccessor args(rsp, kFastApiCallArguments, |
+ ARGUMENTS_DONT_CONTAIN_RECEIVER); |
+ const int kHolderIndex = kFastApiCallArguments - 1 - |
+ FunctionCallbackArguments::kHolderIndex; |
+ |
+ if (save_at_depth == depth) { |
+ __ movp(args.GetArgumentOperand(kHolderIndex), object_reg); |
+ } |
+ |
Handle<JSObject> current = Handle<JSObject>::null(); |
if (type->IsConstant()) current = Handle<JSObject>::cast(type->AsConstant()); |
Handle<JSObject> prototype = Handle<JSObject>::null(); |
@@ -1158,6 +1213,10 @@ Register StubCompiler::CheckPrototypes(Handle<HeapType> type, |
} |
} |
+ if (save_at_depth == depth) { |
+ __ movp(args.GetArgumentOperand(kHolderIndex), reg); |
+ } |
+ |
// Go to the next object in the prototype chain. |
current = prototype; |
current_map = handle(current->map()); |
@@ -1547,35 +1606,43 @@ Handle<Code> CallStubCompiler::CompileFastApiCall( |
if (object->IsGlobalObject()) return Handle<Code>::null(); |
if (!cell.is_null()) return Handle<Code>::null(); |
if (!object->IsJSObject()) return Handle<Code>::null(); |
- Handle<JSObject> receiver = Handle<JSObject>::cast(object); |
- CallOptimization::HolderLookup holder_lookup = |
- CallOptimization::kHolderNotFound; |
- Handle<Map> lookup_map = optimization.LookupHolderOfExpectedType( |
- receiver, receiver, holder, &holder_lookup); |
- if (holder_lookup == CallOptimization::kHolderNotFound) { |
- return Handle<Code>::null(); |
- } |
+ int depth = optimization.GetPrototypeDepthOfExpectedType( |
+ Handle<JSObject>::cast(object), holder); |
+ if (depth == kInvalidProtoDepth) return Handle<Code>::null(); |
- Label miss; |
- GenerateNameCheck(name, &miss); |
+ Label miss, miss_before_stack_reserved; |
+ GenerateNameCheck(name, &miss_before_stack_reserved); |
const int argc = arguments().immediate(); |
StackArgumentsAccessor args(rsp, argc); |
__ movp(rdx, args.GetReceiverOperand()); |
// Check that the receiver isn't a smi. |
- __ JumpIfSmi(rdx, &miss); |
+ __ JumpIfSmi(rdx, &miss_before_stack_reserved); |
Counters* counters = isolate()->counters(); |
__ IncrementCounter(counters->call_const(), 1); |
+ __ IncrementCounter(counters->call_const_fast_api(), 1); |
+ |
+ // Allocate space for v8::Arguments implicit values. Must be initialized |
+ // before calling any runtime function. |
+ __ subq(rsp, Immediate(kFastApiCallArguments * kPointerSize)); |
// Check that the maps haven't changed and find a Holder as a side effect. |
CheckPrototypes(IC::CurrentTypeOf(object, isolate()), rdx, holder, |
- rbx, rax, rdi, name, &miss); |
+ rbx, rax, rdi, name, depth, &miss); |
- GenerateFastApiCall(masm(), optimization, argc, lookup_map, holder_lookup); |
+ // Move the return address on top of the stack. |
+ __ movq(rax, |
+ StackOperandForReturnAddress(kFastApiCallArguments * kPointerSize)); |
+ __ movq(StackOperandForReturnAddress(0), rax); |
- HandlerFrontendFooter(&miss); |
+ GenerateFastApiCall(masm(), optimization, argc); |
+ |
+ __ bind(&miss); |
+ __ addq(rsp, Immediate(kFastApiCallArguments * kPointerSize)); |
+ |
+ HandlerFrontendFooter(&miss_before_stack_reserved); |
// Return the generated code. |
return GetCode(function); |