Use platform specific stubs for vector-based Load/KeyedLoad.

src/x64/code-stubs-x64.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/v8.h"
 
 #if V8_TARGET_ARCH_X64
 
 #include "src/bootstrapper.h"
 #include "src/code-stubs.h"
 #include "src/codegen.h"
 #include "src/ic/handler-compiler.h"
 #include "src/ic/ic.h"
+#include "src/ic/stub-cache.h"
 #include "src/isolate.h"
 #include "src/jsregexp.h"
 #include "src/regexp-macro-assembler.h"
 #include "src/runtime/runtime.h"
 
 namespace v8 {
 namespace internal {
 
 
 static void InitializeArrayConstructorDescriptor(
@@ skipping 4306 matching lines @@
   __ PopReturnAddressTo(rcx);
   int additional_offset =
       function_mode() == JS_FUNCTION_STUB_MODE ? kPointerSize : 0;
   __ leap(rsp, MemOperand(rsp, rbx, times_pointer_size, additional_offset));
   __ jmp(rcx);  // Return to IC Miss stub, continuation still on stack.
 }
 
 
 void LoadICTrampolineStub::Generate(MacroAssembler* masm) {
   EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister());
-  VectorLoadStub stub(isolate(), state());
-  __ jmp(stub.GetCode(), RelocInfo::CODE_TARGET);
+  VectorRawLoadStub stub(isolate(), state());
+  stub.GenerateForTrampoline(masm);
 }
 
 
 void KeyedLoadICTrampolineStub::Generate(MacroAssembler* masm) {
   EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister());
-  VectorKeyedLoadStub stub(isolate());
-  __ jmp(stub.GetCode(), RelocInfo::CODE_TARGET);
-}
-
-
+  VectorRawKeyedLoadStub stub(isolate());
+  stub.GenerateForTrampoline(masm);
+}
+
+
+static void HandlePolymorphic(MacroAssembler* masm, Register receiver,
+                              Register key, Register vector, Register slot,
+                              Register feedback, Register receiver_map,
+                              Register length, Register element0,
+                              Register scratch1, Label* miss) {
+  Label next, next_loop, prepare_next;
+
+  // feedback initially contains vector[slot] and is a FixedArray.
+  Register counter = scratch1;
+  Register array_map = element0;
+
+  // length is in integer32 form.
+  // We prefetched the first element. Examine that outside of the loop.
+  __ cmpp(receiver_map, FieldOperand(array_map, WeakCell::kValueOffset));
+  __ j(not_equal, &next);
+  __ movp(feedback, FieldOperand(feedback, FixedArray::OffsetOfElementAt(1)));
+  __ leap(feedback, FieldOperand(feedback, Code::kHeaderSize));
+  __ jmp(feedback);
+
+  __ bind(&next);
+
+  // Polymorphic, we have to loop from 2 to length by twos.
+  __ movp(counter, Immediate(2));
+  __ cmpl(counter, length);
+  __ j(greater_equal, miss);
+
+  __ bind(&next_loop);
+  __ movp(array_map, FieldOperand(feedback, counter, times_pointer_size,
+                                  FixedArray::kHeaderSize));
+  __ cmpp(receiver_map, FieldOperand(array_map, WeakCell::kValueOffset));
+  __ j(not_equal, &prepare_next);
+  __ movp(feedback, FieldOperand(feedback, counter, times_pointer_size,
+                                 FixedArray::kHeaderSize + kPointerSize));
+  __ leap(feedback, FieldOperand(feedback, Code::kHeaderSize));
+  __ jmp(feedback);
+
+  __ bind(&prepare_next);
+  __ addl(counter, Immediate(2));
+  __ cmpl(counter, length);
+  __ j(less, &next_loop);
+
+  // We exhausted our array of map handler pairs.
+  __ jmp(miss);
+}
+
+
+static void HandleMonoOrPolyName(MacroAssembler* masm, Register receiver,
+                                 Register key, Register vector, Register slot,
+                                 Register feedback, Register receiver_map,
+                                 Register scratch2, Register scratch3,
+                                 Register scratch4, Label* miss) {
+  Label next, next_loop, prepare_next;
+
+  // feedback initially contains vector[slot] and is a FixedArray.
+  Register length = scratch2;
+  Register counter = scratch3;
+  Register array_map = scratch4;
+
+  // Try monomorphic case without loading array length.
+  __ movp(array_map, FieldOperand(feedback, FixedArray::OffsetOfElementAt(0)));
+  __ cmpp(receiver_map, FieldOperand(array_map, WeakCell::kValueOffset));
+  __ j(not_equal, &next);
+  __ movp(feedback, FieldOperand(feedback, FixedArray::OffsetOfElementAt(1)));
+  __ leap(feedback, FieldOperand(feedback, Code::kHeaderSize));
+  __ jmp(feedback);
+
+  // Polymorphic, we have to loop from 2 to length by twos.
+  __ bind(&next);
+  __ movp(length, FieldOperand(feedback, FixedArray::kLengthOffset));
+  __ SmiToInteger32(length, length);
+  __ cmpl(length, Immediate(2));
+  __ j(equal, miss);
+  __ movp(counter, Immediate(2));
+
+  __ bind(&next_loop);
+  __ movp(array_map, FieldOperand(feedback, counter, times_pointer_size,
+                                  FixedArray::kHeaderSize));
+  __ cmpp(receiver_map, FieldOperand(array_map, WeakCell::kValueOffset));
+  __ j(not_equal, &prepare_next);
+  __ movp(feedback, FieldOperand(feedback, counter, times_pointer_size,
+                                 FixedArray::kHeaderSize + kPointerSize));
+  __ leap(feedback, FieldOperand(feedback, Code::kHeaderSize));
+  __ jmp(feedback);
+
+  __ bind(&prepare_next);
+  __ addl(counter, Immediate(2));
+  __ cmpl(counter, length);
+  __ j(less, &next_loop);
+
+  // We exhausted our array of map handler pairs.
+  __ jmp(miss);
+}
+
+
+void VectorRawLoadStub::Generate(MacroAssembler* masm) {
+  GenerateImpl(masm, false);
+}
+
+
+void VectorRawLoadStub::GenerateForTrampoline(MacroAssembler* masm) {
+  GenerateImpl(masm, true);
+}
+
+
+void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
+  Register receiver = VectorLoadICDescriptor::ReceiverRegister();  // rdx
+  Register name = VectorLoadICDescriptor::NameRegister();          // rcx
+  Register vector = VectorLoadICDescriptor::VectorRegister();      // rbx
+  Register slot = VectorLoadICDescriptor::SlotRegister();          // rax
+  Factory* factory = masm->isolate()->factory();
+  Register scratch = rdi;
+  Register integer_slot = r8;
+
+  __ SmiToInteger32(integer_slot, slot);
+  __ movp(scratch, FieldOperand(vector, integer_slot, times_pointer_size,
+                                FixedArray::kHeaderSize));
+
+  // Is it a weak cell?
+  Label try_array;
+  Label megamorphic, smi_key, key_okay, miss;
+  Register receiver_map = r9;
+  Register weak_value_or_length = r11;
+  Label load_smi_map, weak_compare;
+
+  __ JumpIfSmi(receiver, &load_smi_map);
+  __ movp(receiver_map, FieldOperand(receiver, 0));
+  __ bind(&weak_compare);
+
+  STATIC_ASSERT(FixedArray::kLengthOffset == WeakCell::kValueOffset);
+  __ movp(weak_value_or_length, FieldOperand(scratch, WeakCell::kValueOffset));
+  __ cmpp(receiver_map, weak_value_or_length);
+  __ j(not_equal, &try_array);
+  __ movp(scratch, FieldOperand(vector, integer_slot, times_pointer_size,
+                                FixedArray::kHeaderSize + kPointerSize));
+  __ leap(scratch, FieldOperand(scratch, Code::kHeaderSize));
+  __ jmp(scratch);
+  __ int3();  // We shouldn't get here.

[Toon Verwaest, 2015/03/11 17:55:02]

Same here

[mvstanton, 2015/03/12 17:05:34]

Done.

+
+  // Is it a fixed array?
+  __ bind(&try_array);
+  __ Cmp(scratch, factory->megamorphic_symbol());
+  __ j(equal, &megamorphic);
+
+  __ movp(r15, FieldOperand(scratch, FixedArray::OffsetOfElementAt(0)));
+  __ JumpIfSmi(r15, &miss);
+  // Is it an array or a weak cell? The only way to know is to make
+  // sure that weak_value_or_length is a smi. If we have a weak cell then
+  // weak_value_or_length will be undefined.
+  __ JumpIfNotSmi(weak_value_or_length, &miss);
+  __ SmiToInteger32(weak_value_or_length, weak_value_or_length);
+  __ cmpl(weak_value_or_length, Immediate(2));
+  __ j(less, &miss);
+  // We have a polymorphic handler, and we know the length, and have
+  // prefetched the first weak cell that holds a map.
+  HandlePolymorphic(masm, receiver, name, vector, slot, scratch, receiver_map,
+                    weak_value_or_length, r15, integer_slot, &miss);
+  __ int3();  // We shouldn't get here.
+
+  __ bind(&megamorphic);
+  Code::Flags code_flags = Code::RemoveTypeAndHolderFromFlags(
+      Code::ComputeHandlerFlags(Code::LOAD_IC));
+  masm->isolate()->stub_cache()->GenerateProbe(
+      masm, Code::LOAD_IC, code_flags, false, receiver, name, scratch, no_reg);
+
+  __ bind(&miss);
+  LoadIC::GenerateMiss(masm);
+
+  __ bind(&load_smi_map);
+  __ Move(receiver_map, masm->isolate()->factory()->heap_number_map());
+  __ jmp(&weak_compare);
+}
+
+
+void VectorRawKeyedLoadStub::Generate(MacroAssembler* masm) {
+  GenerateImpl(masm, false);
+}
+
+
+void VectorRawKeyedLoadStub::GenerateForTrampoline(MacroAssembler* masm) {
+  GenerateImpl(masm, true);
+}
+
+
+void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
+  Register receiver = VectorLoadICDescriptor::ReceiverRegister();  // rdx
+  Register key = VectorLoadICDescriptor::NameRegister();           // rcx
+  Register vector = VectorLoadICDescriptor::VectorRegister();      // rbx
+  Register slot = VectorLoadICDescriptor::SlotRegister();          // rax
+  Factory* factory = masm->isolate()->factory();
+  Register scratch = rdi;
+  Register integer_slot = r8;
+
+  __ SmiToInteger32(integer_slot, slot);
+  __ movp(scratch, FieldOperand(vector, integer_slot, times_pointer_size,
+                                FixedArray::kHeaderSize));
+
+  // Is it a weak cell?
+  Label try_name, try_array, try_megamorphic;
+  Label not_array, smi_key, key_okay, miss;
+  Register receiver_map = r9;
+  Register weak_value_or_length = r11;
+  Label load_smi_map, weak_compare;
+
+  __ JumpIfSmi(receiver, &load_smi_map);
+  __ movp(receiver_map, FieldOperand(receiver, 0));
+  __ bind(&weak_compare);
+
+  STATIC_ASSERT(FixedArray::kLengthOffset == WeakCell::kValueOffset);
+  __ movp(weak_value_or_length, FieldOperand(scratch, WeakCell::kValueOffset));
+  __ cmpp(receiver_map, weak_value_or_length);
+  __ j(not_equal, &try_name);
+  __ JumpIfNotSmi(key, &miss);
+  __ movp(scratch, FieldOperand(vector, integer_slot, times_pointer_size,
+                                FixedArray::kHeaderSize + kPointerSize));
+  __ leap(scratch, FieldOperand(scratch, Code::kHeaderSize));
+  __ jmp(scratch);
+  __ int3();  // We shouldn't get here.
+
+  __ bind(&try_name);
+  // We might have a name in scratch, and a fixed array in the next slot.
+  __ cmpp(key, scratch);
+  __ j(not_equal, &try_megamorphic);
+  // If the name comparison succeeded, we know we have a fixed array with
+  // at least one map/handler pair.
+  __ movp(scratch, FieldOperand(vector, integer_slot, times_pointer_size,
+                                FixedArray::kHeaderSize + kPointerSize));
+  HandleMonoOrPolyName(masm, receiver, key, vector, slot, scratch, receiver_map,
+                       weak_value_or_length, integer_slot, r15, &miss);
+  __ int3();
+
+  __ bind(&try_megamorphic);
+  __ Cmp(scratch, factory->megamorphic_symbol());
+  __ j(not_equal, &try_array);
+  Handle<Code> megamorphic_stub =
+      KeyedLoadIC::ChooseMegamorphicStub(masm->isolate());
+  __ jmp(megamorphic_stub, RelocInfo::CODE_TARGET);
+  __ int3();  // We shouldn't get here.
+
+  // Is it a fixed array?
+  __ bind(&try_array);
+  __ movp(r15, FieldOperand(scratch, FixedArray::OffsetOfElementAt(0)));
+  __ JumpIfSmi(r15, &miss);
+  // Is it an array or a weak cell? The only way to know is to make
+  // sure that weak_value_or_length is a smi > 0 (because our vector arrays
+  // have a minimum size of 2).
+  __ JumpIfNotSmi(weak_value_or_length, &miss);
+  __ SmiToInteger32(weak_value_or_length, weak_value_or_length);
+  __ cmpl(weak_value_or_length, Immediate(2));
+  __ j(less, &miss);
+  // We have a polymorphic element handler, and we know the length.
+  __ JumpIfNotSmi(key, &miss);
+  // We have also prefetched the first weak cell that holds a map in the
+  // array.
+  HandlePolymorphic(masm, receiver, key, vector, slot, scratch, receiver_map,
+                    weak_value_or_length, r15, integer_slot, &miss);
+  __ int3();  // We shouldn't get here.
+
+  __ bind(&miss);
+  KeyedLoadIC::GenerateMiss(masm);
+
+  __ bind(&load_smi_map);
+  __ Move(receiver_map, masm->isolate()->factory()->heap_number_map());
+  __ jmp(&weak_compare);
+}
+
+
 void CallICTrampolineStub::Generate(MacroAssembler* masm) {
   EmitLoadTypeFeedbackVector(masm, rbx);
   CallICStub stub(isolate(), state());
   __ jmp(stub.GetCode(), RelocInfo::CODE_TARGET);
 }
 
 
 void CallIC_ArrayTrampolineStub::Generate(MacroAssembler* masm) {
   EmitLoadTypeFeedbackVector(masm, rbx);
   CallIC_ArrayStub stub(isolate(), state());
@@ skipping 786 matching lines @@
   CallApiFunctionAndReturn(masm, api_function_address, thunk_ref, getter_arg,
                            kStackSpace, nullptr, return_value_operand, NULL);
 }
 
 
 #undef __
 
 } }  // namespace v8::internal
 
 #endif  // V8_TARGET_ARCH_X64