Vector-ICs - speed towards the monomorphic exit as quickly as possible.

src/x64/code-stubs-x64.cc

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/v8.h"
 
 #if V8_TARGET_ARCH_X64
 
 #include "src/bootstrapper.h"
 #include "src/code-stubs.h"
@@ skipping 4333 matching lines @@
 
 void KeyedLoadICTrampolineStub::Generate(MacroAssembler* masm) {
   EmitLoadTypeFeedbackVector(masm, VectorLoadICDescriptor::VectorRegister());
   VectorRawKeyedLoadStub stub(isolate());
   stub.GenerateForTrampoline(masm);
 }
 
 
 static void HandleArrayCases(MacroAssembler* masm, Register receiver,
                              Register key, Register vector, Register slot,
-                             Register feedback, Register scratch1,
-                             Register scratch2, Register scratch3,
-                             Register scratch4, bool is_polymorphic,
+                             Register feedback, Register receiver_map,
+                             Register scratch1, Register scratch2,
+                             Register scratch3, bool is_polymorphic,
                              Label* miss) {
   // feedback initially contains the feedback array
   Label next_loop, prepare_next;
-  Label load_smi_map, compare_map;
   Label start_polymorphic;
 
-  Register receiver_map = scratch1;
-  Register counter = scratch2;
-  Register length = scratch3;
-  Register cached_map = scratch4;
+  Register counter = scratch1;
+  Register length = scratch2;
+  Register cached_map = scratch3;
 
-  // Receiver might not be a heap object.
-  __ JumpIfSmi(receiver, &load_smi_map);
-  __ movp(receiver_map, FieldOperand(receiver, 0));
-  __ bind(&compare_map);
   __ movp(cached_map, FieldOperand(feedback, FixedArray::OffsetOfElementAt(0)));
   __ cmpp(receiver_map, FieldOperand(cached_map, WeakCell::kValueOffset));
   __ j(not_equal, &start_polymorphic);
 
   // found, now call handler.
   Register handler = feedback;
   __ movp(handler, FieldOperand(feedback, FixedArray::OffsetOfElementAt(1)));
   __ leap(handler, FieldOperand(handler, Code::kHeaderSize));
   __ jmp(handler);
 
@@ skipping 18 matching lines @@
   __ leap(handler, FieldOperand(handler, Code::kHeaderSize));
   __ jmp(handler);
 
   __ bind(&prepare_next);
   __ addl(counter, Immediate(2));
   __ cmpl(counter, length);
   __ j(less, &next_loop);
 
   // We exhausted our array of map handler pairs.
   __ jmp(miss);
-
-  __ bind(&load_smi_map);
-  __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex);
-  __ jmp(&compare_map);
 }
 
 
 static void HandleMonomorphicCase(MacroAssembler* masm, Register receiver,
-                                  Register key, Register vector, Register slot,
-                                  Register weak_cell, Register integer_slot,
-                                  Label* miss) {
-  // feedback initially contains the feedback array
-  Label compare_smi_map;
+                                  Register receiver_map, Register feedback,
+                                  Register vector, Register integer_slot,
+                                  Label* compare_map, Label* load_smi_map,
+                                  Label* try_array) {
+  __ JumpIfSmi(receiver, load_smi_map);
+  __ movp(receiver_map, FieldOperand(receiver, 0));
 
-  // Move the weak map into the weak_cell register.
-  Register ic_map = weak_cell;
-  __ movp(ic_map, FieldOperand(weak_cell, WeakCell::kValueOffset));
-
-  // Receiver might not be a heap object.
-  __ JumpIfSmi(receiver, &compare_smi_map);
-  __ cmpp(ic_map, FieldOperand(receiver, 0));
-  __ j(not_equal, miss);
-  Register handler = weak_cell;
+  __ bind(compare_map);
+  __ cmpp(receiver_map, FieldOperand(feedback, WeakCell::kValueOffset));
+  __ j(not_equal, try_array);
+  Register handler = feedback;
   __ movp(handler, FieldOperand(vector, integer_slot, times_pointer_size,
                                 FixedArray::kHeaderSize + kPointerSize));
   __ leap(handler, FieldOperand(handler, Code::kHeaderSize));
-  __ jmp(handler);
-
-  // In microbenchmarks, it made sense to unroll this code so that the call to
-  // the handler is duplicated for a HeapObject receiver and a Smi receiver.
-  __ bind(&compare_smi_map);
-  __ CompareRoot(ic_map, Heap::kHeapNumberMapRootIndex);
-  __ j(not_equal, miss);
-  __ movp(handler, FieldOperand(vector, integer_slot, times_pointer_size,
-                                FixedArray::kHeaderSize + kPointerSize));
-  __ leap(handler, FieldOperand(handler, Code::kHeaderSize));
   __ jmp(handler);
 }
 
 
 void VectorRawLoadStub::Generate(MacroAssembler* masm) {
   GenerateImpl(masm, false);
 }
 
 
 void VectorRawLoadStub::GenerateForTrampoline(MacroAssembler* masm) {
   GenerateImpl(masm, true);
 }
 
 
 void VectorRawLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
   Register receiver = VectorLoadICDescriptor::ReceiverRegister();  // rdx
   Register name = VectorLoadICDescriptor::NameRegister();          // rcx
   Register vector = VectorLoadICDescriptor::VectorRegister();      // rbx
   Register slot = VectorLoadICDescriptor::SlotRegister();          // rax
   Register feedback = rdi;
   Register integer_slot = r8;
+  Register receiver_map = r9;
 
   __ SmiToInteger32(integer_slot, slot);
   __ movp(feedback, FieldOperand(vector, integer_slot, times_pointer_size,
                                  FixedArray::kHeaderSize));
 
-  // Is it a weak cell?
-  Label try_array;
-  Label not_array, smi_key, key_okay, miss;
-  __ CompareRoot(FieldOperand(feedback, 0), Heap::kWeakCellMapRootIndex);
-  __ j(not_equal, &try_array);
-  HandleMonomorphicCase(masm, receiver, name, vector, slot, feedback,
-                        integer_slot, &miss);
+  // Try to quickly handle the monomorphic case without knowing for sure
+  // if we have a weak cell in feedback. We do know it's safe to look
+  // at WeakCell::kValueOffset.
+  Label try_array, load_smi_map, compare_map;
+  Label not_array, miss;
+  HandleMonomorphicCase(masm, receiver, receiver_map, feedback, vector,
+                        integer_slot, &compare_map, &load_smi_map, &try_array);
 
   // Is it a fixed array?
   __ bind(&try_array);
   __ CompareRoot(FieldOperand(feedback, 0), Heap::kFixedArrayMapRootIndex);
   __ j(not_equal, &not_array);
-  HandleArrayCases(masm, receiver, name, vector, slot, feedback, integer_slot,
-                   r9, r11, r15, true, &miss);
+  HandleArrayCases(masm, receiver, name, vector, slot, feedback, receiver_map,
+                   integer_slot, r11, r15, true, &miss);
 
   __ bind(&not_array);
   __ CompareRoot(feedback, Heap::kmegamorphic_symbolRootIndex);
   __ j(not_equal, &miss);
   Code::Flags code_flags = Code::RemoveTypeAndHolderFromFlags(
       Code::ComputeHandlerFlags(Code::LOAD_IC));
   masm->isolate()->stub_cache()->GenerateProbe(
       masm, Code::LOAD_IC, code_flags, false, receiver, name, feedback, no_reg);
 
   __ bind(&miss);
   LoadIC::GenerateMiss(masm);
+
+  __ bind(&load_smi_map);
+  __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex);
+  __ jmp(&compare_map);
 }
 
 
 void VectorRawKeyedLoadStub::Generate(MacroAssembler* masm) {
   GenerateImpl(masm, false);
 }
 
 
 void VectorRawKeyedLoadStub::GenerateForTrampoline(MacroAssembler* masm) {
   GenerateImpl(masm, true);
 }
 
 
 void VectorRawKeyedLoadStub::GenerateImpl(MacroAssembler* masm, bool in_frame) {
   Register receiver = VectorLoadICDescriptor::ReceiverRegister();  // rdx
   Register key = VectorLoadICDescriptor::NameRegister();           // rcx
   Register vector = VectorLoadICDescriptor::VectorRegister();      // rbx
   Register slot = VectorLoadICDescriptor::SlotRegister();          // rax
   Register feedback = rdi;
   Register integer_slot = r8;
+  Register receiver_map = r9;
 
   __ SmiToInteger32(integer_slot, slot);
   __ movp(feedback, FieldOperand(vector, integer_slot, times_pointer_size,
                                  FixedArray::kHeaderSize));
 
-  // Is it a weak cell?
-  Label try_array;
-  Label not_array, smi_key, key_okay, miss;
-  __ CompareRoot(FieldOperand(feedback, 0), Heap::kWeakCellMapRootIndex);
-  __ j(not_equal, &try_array);
-  HandleMonomorphicCase(masm, receiver, key, vector, slot, feedback,
-                        integer_slot, &miss);
+  // Try to quickly handle the monomorphic case without knowing for sure
+  // if we have a weak cell in feedback. We do know it's safe to look
+  // at WeakCell::kValueOffset.
+  Label try_array, load_smi_map, compare_map;
+  Label not_array, miss;
+  HandleMonomorphicCase(masm, receiver, receiver_map, feedback, vector,
+                        integer_slot, &compare_map, &load_smi_map, &try_array);
 
   __ bind(&try_array);
   // Is it a fixed array?
   __ CompareRoot(FieldOperand(feedback, 0), Heap::kFixedArrayMapRootIndex);
   __ j(not_equal, &not_array);
 
   // We have a polymorphic element handler.
   Label polymorphic, try_poly_name;
   __ bind(&polymorphic);
-  HandleArrayCases(masm, receiver, key, vector, slot, feedback, integer_slot,
-                   r9, r11, r15, true, &miss);
+  HandleArrayCases(masm, receiver, key, vector, slot, feedback, receiver_map,
+                   integer_slot, r11, r15, true, &miss);
 
   __ bind(&not_array);
   // Is it generic?
   __ CompareRoot(feedback, Heap::kmegamorphic_symbolRootIndex);
   __ j(not_equal, &try_poly_name);
   Handle<Code> megamorphic_stub =
       KeyedLoadIC::ChooseMegamorphicStub(masm->isolate());
   __ jmp(megamorphic_stub, RelocInfo::CODE_TARGET);
 
   __ bind(&try_poly_name);
   // We might have a name in feedback, and a fixed array in the next slot.
   __ cmpp(key, feedback);
   __ j(not_equal, &miss);
   // If the name comparison succeeded, we know we have a fixed array with
   // at least one map/handler pair.
   __ movp(feedback, FieldOperand(vector, integer_slot, times_pointer_size,
                                  FixedArray::kHeaderSize + kPointerSize));
-  HandleArrayCases(masm, receiver, key, vector, slot, feedback, integer_slot,
-                   r9, r11, r15, false, &miss);
+  HandleArrayCases(masm, receiver, key, vector, slot, feedback, receiver_map,
+                   integer_slot, r11, r15, false, &miss);
 
   __ bind(&miss);
   KeyedLoadIC::GenerateMiss(masm);
+
+  __ bind(&load_smi_map);
+  __ LoadRoot(receiver_map, Heap::kHeapNumberMapRootIndex);
+  __ jmp(&compare_map);
 }
 
 
 void CallICTrampolineStub::Generate(MacroAssembler* masm) {
   EmitLoadTypeFeedbackVector(masm, rbx);
   CallICStub stub(isolate(), state());
   __ jmp(stub.GetCode(), RelocInfo::CODE_TARGET);
 }
 
 
@@ skipping 786 matching lines @@
   CallApiFunctionAndReturn(masm, api_function_address, thunk_ref, getter_arg,
                            kStackSpace, nullptr, return_value_operand, NULL);
 }
 
 
 #undef __
 
 } }  // namespace v8::internal
 
 #endif  // V8_TARGET_ARCH_X64