Port inlining of type checks in call ICs for API functions to x64 and arm (is...

src/arm/stub-cache-arm.cc

 // Copyright 2006-2009 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 582 matching lines @@
     compiler->CompileRegular(masm,
                              receiver,
                              reg,
                              scratch2,
                              holder,
                              miss);
   }
 }
 
 
+// Reserves space for the extra arguments to FastHandleApiCall in the
+// caller's frame.
+//
+// These arguments are set by CheckPrototypes and GenerateFastApiCall.
+static void ReserveSpaceForFastApiCall(MacroAssembler* masm,
+                                       Register scratch) {
+  __ mov(scratch, Operand(Smi::FromInt(0)));
+  __ push(scratch);
+  __ push(scratch);
+  __ push(scratch);
+  __ push(scratch);
+}
+
+
+// Undoes the effects of ReserveSpaceForFastApiCall.
+static void FreeSpaceForFastApiCall(MacroAssembler* masm) {
+  __ Drop(4);
+}
+
+
+// Generates call to FastHandleApiCall builtin.
+static void GenerateFastApiCall(MacroAssembler* masm,
+                                const CallOptimization& optimization,
+                                int argc) {
+  // Get the function and setup the context.
+  JSFunction* function = optimization.constant_function();
+  __ mov(r7, Operand(Handle<JSFunction>(function)));
+  __ ldr(cp, FieldMemOperand(r7, JSFunction::kContextOffset));
+
+  // Pass the additional arguments FastHandleApiCall expects.
+  bool info_loaded = false;
+  Object* callback = optimization.api_call_info()->callback();
+  if (Heap::InNewSpace(callback)) {
+    info_loaded = true;
+    __ Move(r0, Handle<CallHandlerInfo>(optimization.api_call_info()));
+    __ ldr(r6, FieldMemOperand(r0, CallHandlerInfo::kCallbackOffset));
+  } else {
+    __ Move(r6, Handle<Object>(callback));
+  }
+  Object* call_data = optimization.api_call_info()->data();
+  if (Heap::InNewSpace(call_data)) {
+    if (!info_loaded) {
+      __ Move(r0, Handle<CallHandlerInfo>(optimization.api_call_info()));
+    }
+    __ ldr(r5, FieldMemOperand(r0, CallHandlerInfo::kDataOffset));
+  } else {
+    __ Move(r5, Handle<Object>(call_data));
+  }
+
+  __ add(sp, sp, Operand(1 * kPointerSize));
+  __ stm(ia, sp, r5.bit() | r6.bit() | r7.bit());
+  __ sub(sp, sp, Operand(1 * kPointerSize));
+
+  // Set the number of arguments.
+  __ mov(r0, Operand(argc + 4));
+
+  // Jump to the fast api call builtin (tail call).
+  Handle<Code> code = Handle<Code>(
+      Builtins::builtin(Builtins::FastHandleApiCall));
+  ParameterCount expected(0);
+  __ InvokeCode(code, expected, expected,
+                RelocInfo::CODE_TARGET, JUMP_FUNCTION);
+}
+
+
+class CallInterceptorCompiler BASE_EMBEDDED {
+ public:
+  CallInterceptorCompiler(StubCompiler* stub_compiler,
+                          const ParameterCount& arguments,
+                          Register name)
+      : stub_compiler_(stub_compiler),
+        arguments_(arguments),
+        name_(name) {}
+
+  void Compile(MacroAssembler* masm,
+               JSObject* object,
+               JSObject* holder,
+               String* name,
+               LookupResult* lookup,
+               Register receiver,
+               Register scratch1,
+               Register scratch2,
+               Label* miss) {
+    ASSERT(holder->HasNamedInterceptor());
+    ASSERT(!holder->GetNamedInterceptor()->getter()->IsUndefined());
+
+    // Check that the receiver isn't a smi.
+    __ BranchOnSmi(receiver, miss);
+
+    CallOptimization optimization(lookup);
+
+    if (optimization.is_constant_call()) {
+      CompileCacheable(masm,
+                       object,
+                       receiver,
+                       scratch1,
+                       scratch2,
+                       holder,
+                       lookup,
+                       name,
+                       optimization,
+                       miss);
+    } else {
+      CompileRegular(masm,
+                     object,
+                     receiver,
+                     scratch1,
+                     scratch2,
+                     name,
+                     holder,
+                     miss);
+    }
+  }
+
+ private:
+  void CompileCacheable(MacroAssembler* masm,
+                       JSObject* object,
+                       Register receiver,
+                       Register scratch1,
+                       Register scratch2,
+                       JSObject* holder_obj,
+                       LookupResult* lookup,
+                       String* name,
+                       const CallOptimization& optimization,
+                       Label* miss_label) {
+    ASSERT(optimization.is_constant_call());
+    ASSERT(!lookup->holder()->IsGlobalObject());
+
+    int depth1 = kInvalidProtoDepth;
+    int depth2 = kInvalidProtoDepth;
+    bool can_do_fast_api_call = false;
+    if (optimization.is_simple_api_call() &&
+       !lookup->holder()->IsGlobalObject()) {
+     depth1 = optimization.GetPrototypeDepthOfExpectedType(object, holder_obj);
+     if (depth1 == kInvalidProtoDepth) {
+       depth2 = optimization.GetPrototypeDepthOfExpectedType(holder_obj,
+                                                             lookup->holder());
+     }
+     can_do_fast_api_call = (depth1 != kInvalidProtoDepth) ||
+                            (depth2 != kInvalidProtoDepth);
+    }
+
+    __ IncrementCounter(&Counters::call_const_interceptor, 1,
+                      scratch1, scratch2);
+
+    if (can_do_fast_api_call) {
+      __ IncrementCounter(&Counters::call_const_interceptor_fast_api, 1,
+                          scratch1, scratch2);
+      ReserveSpaceForFastApiCall(masm, scratch1);
+    }
+
+    Label miss_cleanup;
+    Label* miss = can_do_fast_api_call ? &miss_cleanup : miss_label;
+    Register holder =
+        stub_compiler_->CheckPrototypes(object, receiver, holder_obj, scratch1,
+                                        scratch2, name, depth1, miss);
+
+    Label regular_invoke;
+    LoadWithInterceptor(masm, receiver, holder, holder_obj, scratch2,
+                        &regular_invoke);
+
+    // Generate code for the failed interceptor case.
+
+    // Check the lookup is still valid.
+    stub_compiler_->CheckPrototypes(holder_obj, receiver,
+                                    lookup->holder(), scratch1,
+                                    scratch2, name, depth2, miss);
+
+    if (can_do_fast_api_call) {
+      GenerateFastApiCall(masm, optimization, arguments_.immediate());
+    } else {
+      __ InvokeFunction(optimization.constant_function(), arguments_,
+                        JUMP_FUNCTION);
+    }
+
+    if (can_do_fast_api_call) {
+      __ bind(&miss_cleanup);
+      FreeSpaceForFastApiCall(masm);
+      __ b(miss_label);
+    }
+
+    __ bind(&regular_invoke);
+    if (can_do_fast_api_call) {
+      FreeSpaceForFastApiCall(masm);
+    }
+  }
+
+  void CompileRegular(MacroAssembler* masm,
+                      JSObject* object,
+                      Register receiver,
+                      Register scratch1,
+                      Register scratch2,
+                      String* name,
+                      JSObject* holder_obj,
+                      Label* miss_label) {
+    Register holder =
+        stub_compiler_->CheckPrototypes(object, receiver, holder_obj,
+                                        scratch1, scratch2, name,
+                                        miss_label);
+
+    // Call a runtime function to load the interceptor property.
+    __ EnterInternalFrame();
+    // Save the name_ register across the call.
+    __ push(name_);
+
+    PushInterceptorArguments(masm,
+                             receiver,
+                             holder,
+                             name_,
+                             holder_obj);
+
+    __ CallExternalReference(
+          ExternalReference(
+              IC_Utility(IC::kLoadPropertyWithInterceptorForCall)),
+          5);
+
+    // Restore the name_ register.
+    __ pop(name_);
+    __ LeaveInternalFrame();
+  }
+
+  void LoadWithInterceptor(MacroAssembler* masm,
+                           Register receiver,
+                           Register holder,
+                           JSObject* holder_obj,
+                           Register scratch,
+                           Label* interceptor_succeeded) {
+    __ EnterInternalFrame();
+    __ push(holder);  // Save the holder.
+    __ push(name_);  // Save the name.
+
+    CompileCallLoadPropertyWithInterceptor(masm,
+                                           receiver,
+                                           holder,
+                                           name_,
+                                           holder_obj);
+
+    __ pop(name_);  // Restore the name.
+    __ pop(receiver);  // Restore the holder.
+    __ LeaveInternalFrame();
+
+    // If interceptor returns no-result sentinel, call the constant function.
+    __ LoadRoot(scratch, Heap::kNoInterceptorResultSentinelRootIndex);
+    __ cmp(r0, scratch);
+    __ b(ne, interceptor_succeeded);
+  }
+
+  StubCompiler* stub_compiler_;
+  const ParameterCount& arguments_;
+  Register name_;
+};
+
+
 // Generate code to check that a global property cell is empty. Create
 // the property cell at compilation time if no cell exists for the
 // property.
 static Object* GenerateCheckPropertyCell(MacroAssembler* masm,
                                          GlobalObject* global,
                                          String* name,
                                          Register scratch,
                                          Label* miss) {
   Object* probe = global->EnsurePropertyCell(name);
   if (probe->IsFailure()) return probe;
@@ skipping 14 matching lines @@
 
 
 Register StubCompiler::CheckPrototypes(JSObject* object,
                                        Register object_reg,
                                        JSObject* holder,
                                        Register holder_reg,
                                        Register scratch,
                                        String* name,
                                        int save_at_depth,
                                        Label* miss) {
-  // TODO(602): support object saving.
-  ASSERT(save_at_depth == kInvalidProtoDepth);
-
   // Check that the maps haven't changed.
   Register result =
-      masm()->CheckMaps(object, object_reg, holder, holder_reg, scratch, miss);
+      masm()->CheckMaps(object, object_reg, holder, holder_reg, scratch,
+                        save_at_depth, miss);
 
   // If we've skipped any global objects, it's not enough to verify
   // that their maps haven't changed.  We also need to check that the
   // property cell for the property is still empty.
   while (object != holder) {
     if (object->IsGlobalObject()) {
       Object* cell = GenerateCheckPropertyCell(masm(),
                                                GlobalObject::cast(object),
                                                name,
                                                scratch,
@@ skipping 286 matching lines @@
   //  -- r2    : name
   //  -- lr    : return address
   // -----------------------------------
   SharedFunctionInfo* function_info = function->shared();
   if (function_info->HasCustomCallGenerator()) {
     CustomCallGenerator generator =
         ToCData<CustomCallGenerator>(function_info->function_data());
     return generator(this, object, holder, function, name, check);
   }
 
-  Label miss;
+  Label miss_in_smi_check;
 
   // Get the receiver from the stack
   const int argc = arguments().immediate();
   __ ldr(r1, MemOperand(sp, argc * kPointerSize));
 
   // Check that the receiver isn't a smi.
   if (check != NUMBER_CHECK) {
     __ tst(r1, Operand(kSmiTagMask));
-    __ b(eq, &miss);
+    __ b(eq, &miss_in_smi_check);
   }
 
   // Make sure that it's okay not to patch the on stack receiver
   // unless we're doing a receiver map check.
   ASSERT(!object->IsGlobalObject() || check == RECEIVER_MAP_CHECK);
 
+  CallOptimization optimization(function);
+  int depth = kInvalidProtoDepth;
+  Label miss;
+
   switch (check) {
     case RECEIVER_MAP_CHECK:
+      __ IncrementCounter(&Counters::call_const, 1, r0, r3);
+
+      if (optimization.is_simple_api_call() && !object->IsGlobalObject()) {
+        depth = optimization.GetPrototypeDepthOfExpectedType(
+            JSObject::cast(object), holder);
+      }
+
+      if (depth != kInvalidProtoDepth) {
+        __ IncrementCounter(&Counters::call_const_fast_api, 1, r0, r3);
+        ReserveSpaceForFastApiCall(masm(), r0);
+      }
+
       // Check that the maps haven't changed.
-      CheckPrototypes(JSObject::cast(object), r1, holder, r3, r0, name, &miss);
+      CheckPrototypes(JSObject::cast(object), r1, holder, r0, r3, name,
+                      depth, &miss);
 
       // Patch the receiver on the stack with the global proxy if
       // necessary.
       if (object->IsGlobalObject()) {
+        ASSERT(depth == kInvalidProtoDepth);
         __ ldr(r3, FieldMemOperand(r1, GlobalObject::kGlobalReceiverOffset));
         __ str(r3, MemOperand(sp, argc * kPointerSize));
       }
       break;
 
     case STRING_CHECK:
       if (!function->IsBuiltin()) {
         // Calling non-builtins with a value as receiver requires boxing.
         __ jmp(&miss);
       } else {
@@ skipping 52 matching lines @@
         CheckPrototypes(JSObject::cast(object->GetPrototype()), r0, holder, r3,
                         r1, name, &miss);
       }
       break;
     }
 
     default:
       UNREACHABLE();
   }
 
-  __ InvokeFunction(function, arguments(), JUMP_FUNCTION);
+  if (depth != kInvalidProtoDepth) {
+    GenerateFastApiCall(masm(), optimization, argc);
+  } else {
+    __ InvokeFunction(function, arguments(), JUMP_FUNCTION);
+  }
 
   // Handle call cache miss.
   __ bind(&miss);
+  if (depth != kInvalidProtoDepth) {
+    FreeSpaceForFastApiCall(masm());
+  }
+
+  __ bind(&miss_in_smi_check);
   Handle<Code> ic = ComputeCallMiss(arguments().immediate());
   __ Jump(ic, RelocInfo::CODE_TARGET);
 
   // Return the generated code.
   String* function_name = NULL;
   if (function->shared()->name()->IsString()) {
     function_name = String::cast(function->shared()->name());
   }
   return GetCode(CONSTANT_FUNCTION, function_name);
 }
 
 
 Object* CallStubCompiler::CompileCallInterceptor(JSObject* object,
                                                  JSObject* holder,
                                                  String* name) {
   // ----------- S t a t e -------------
   //  -- r2    : name
   //  -- lr    : return address
   // -----------------------------------
-  ASSERT(holder->HasNamedInterceptor());
-  ASSERT(!holder->GetNamedInterceptor()->getter()->IsUndefined());
+
   Label miss;
 
-  const Register receiver = r0;
-  const Register holder_reg = r1;
-  const Register name_reg = r2;
-  const Register scratch = r3;
-
   // Get the number of arguments.
   const int argc = arguments().immediate();
 
   LookupResult lookup;
   LookupPostInterceptor(holder, name, &lookup);
 
-  // Get the receiver from the stack into r0.
-  __ ldr(r0, MemOperand(sp, argc * kPointerSize));
+  // Get the receiver from the stack.
+  __ ldr(r1, MemOperand(sp, argc * kPointerSize));
 
-  // Check that the receiver isn't a smi.
-  __ BranchOnSmi(receiver, &miss);
-
-  // Check that the maps haven't changed.
-  Register reg = CheckPrototypes(object, receiver, holder, holder_reg,
-                                 scratch, name, &miss);
-  if (!reg.is(holder_reg)) {
-    __ mov(holder_reg, reg);
-  }
-
-  // If we call a constant function when the interceptor returns
-  // the no-result sentinel, generate code that optimizes this case.
-  if (lookup.IsProperty() &&
-      lookup.IsCacheable() &&
-      lookup.type() == CONSTANT_FUNCTION &&
-      lookup.GetConstantFunction()->is_compiled() &&
-      !holder->IsJSArray()) {
-    // Constant functions cannot sit on global object.
-    ASSERT(!lookup.holder()->IsGlobalObject());
-
-    // Call the interceptor.
-    __ EnterInternalFrame();
-    __ push(holder_reg);
-    __ push(name_reg);
-    CompileCallLoadPropertyWithInterceptor(masm(),
-                                           receiver,
-                                           holder_reg,
-                                           name_reg,
-                                           holder);
-    __ pop(name_reg);
-    __ pop(holder_reg);
-    __ LeaveInternalFrame();
-    // r0 no longer contains the receiver.
-
-    // If interceptor returns no-result sentinal, call the constant function.
-    __ LoadRoot(scratch, Heap::kNoInterceptorResultSentinelRootIndex);
-    __ cmp(r0, scratch);
-    Label invoke;
-    __ b(ne, &invoke);
-    // Check the prototypes between the interceptor's holder and the
-    // constant function's holder.
-    CheckPrototypes(holder, holder_reg,
-                    lookup.holder(), r0,
-                    scratch,
-                    name,
-                    &miss);
-
-    __ InvokeFunction(lookup.GetConstantFunction(),
-                      arguments(),
-                      JUMP_FUNCTION);
-
-    __ bind(&invoke);
-
-  } else {
-    // Call a runtime function to load the interceptor property.
-    __ EnterInternalFrame();
-    __ push(name_reg);
-
-    PushInterceptorArguments(masm(), receiver, holder_reg, name_reg, holder);
-
-    __ CallExternalReference(
-        ExternalReference(IC_Utility(IC::kLoadPropertyWithInterceptorForCall)),
-        5);
-
-    __ pop(name_reg);
-    __ LeaveInternalFrame();
-  }
+  CallInterceptorCompiler compiler(this, arguments(), r2);
+  compiler.Compile(masm(),
+                   object,
+                   holder,
+                   name,
+                   &lookup,
+                   r1,
+                   r3,
+                   r4,
+                   &miss);
 
   // Move returned value, the function to call, to r1.
   __ mov(r1, r0);
   // Restore receiver.
-  __ ldr(receiver, MemOperand(sp, argc * kPointerSize));
+  __ ldr(r0, MemOperand(sp, argc * kPointerSize));
 
   GenerateCallFunction(masm(), object, arguments(), &miss);
 
   // Handle call cache miss.
   __ bind(&miss);
   Handle<Code> ic = ComputeCallMiss(arguments().immediate());
   __ Jump(ic, RelocInfo::CODE_TARGET);
 
   // Return the generated code.
   return GetCode(INTERCEPTOR, name);
@@ skipping 783 matching lines @@
   __ Jump(generic_construct_stub, RelocInfo::CODE_TARGET);
 
   // Return the generated code.
   return GetCode();
 }
 
 
 #undef __
 
 } }  // namespace v8::internal