Optimizations to IC stub for unoptimized code performance on x64.

runtime/vm/stub_code_x64.cc

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/globals.h"
 #if defined(TARGET_ARCH_X64)
 
 #include "vm/assembler.h"
 #include "vm/compiler.h"
 #include "vm/dart_entry.h"
@@ skipping 1291 matching lines @@
   __ cmpq(Address(R13, 1 * kWordSize), imm_smi_cid);
   __ j(EQUAL, &ok, Assembler::kNearJump);
   __ Bind(&error);
   __ Stop("Incorrect IC data");
   __ Bind(&ok);
 #endif
 
   if (FLAG_optimization_counter_threshold >= 0) {
     const intptr_t count_offset = ICData::CountIndexFor(num_args) * kWordSize;
     // Update counter.
-    __ movq(R8, Address(R13, count_offset));
-    __ addq(R8, Immediate(Smi::RawValue(1)));
-    __ movq(R9, Immediate(Smi::RawValue(Smi::kMaxValue)));
-    __ cmovnoq(R9, R8);
-    __ StoreIntoSmiField(Address(R13, count_offset), R9);
+    __ addq(Address(R13, count_offset), Immediate(Smi::RawValue(1)));
   }
 
   __ ret();
 }
 
 
 // Generate inline cache check for 'num_args'.
 //  RBX: Inline cache data object.
 //  TOS(0): return address
 // Control flow:
 // - If receiver is null -> jump to IC miss.
 // - If receiver is Smi -> load Smi class.
 // - If receiver is not-Smi -> load receiver's class.
 // - Check if 'num_args' (including receiver) match any IC data group.
 // - Match found -> jump to target.
 // - Match not found -> jump to IC miss.
 void StubCode::GenerateNArgsCheckInlineCacheStub(
     Assembler* assembler,
     intptr_t num_args,
     const RuntimeEntry& handle_ic_miss,
     Token::Kind kind,
     bool optimized) {
-  ASSERT(num_args > 0);
+  ASSERT(num_args == 1 || num_args == 2);
 #if defined(DEBUG)
   {
     Label ok;
     // Check that the IC data array has NumArgsTested() == num_args.
     // 'NumArgsTested' is stored in the least significant bits of 'state_bits'.
     __ movl(RCX, FieldAddress(RBX, ICData::state_bits_offset()));
     ASSERT(ICData::NumArgsTestedShift() == 0);  // No shift needed.
     __ andq(RCX, Immediate(ICData::NumArgsTestedMask()));
     __ cmpq(RCX, Immediate(num_args));
     __ j(EQUAL, &ok, Assembler::kNearJump);
@@ skipping 14 matching lines @@
   Label not_smi_or_overflow;
   if (kind != Token::kILLEGAL) {
     EmitFastSmiOp(assembler, kind, num_args, &not_smi_or_overflow);
   }
   __ Bind(&not_smi_or_overflow);
 
   __ Comment("Extract ICData initial values and receiver cid");
   // Load arguments descriptor into R10.
   __ movq(R10, FieldAddress(RBX, ICData::arguments_descriptor_offset()));
   // Loop that checks if there is an IC data match.
-  Label loop, update, test, found;
+  Label loop, found, miss;
   // RBX: IC data object (preserved).
   __ movq(R13, FieldAddress(RBX, ICData::ic_data_offset()));
   // R13: ic_data_array with check entries: classes and target functions.
   __ leaq(R13, FieldAddress(R13, Array::data_offset()));
   // R13: points directly to the first ic data array element.
 
-  // Get the receiver's class ID (first read number of arguments from
-  // arguments descriptor array and then access the receiver from the stack).
-  __ movq(RAX, FieldAddress(R10, ArgumentsDescriptor::count_offset()));
-  __ movq(R9, Address(RSP, RAX, TIMES_4, 0));  // RAX (argument count) is Smi.
-  __ LoadTaggedClassIdMayBeSmi(RAX, R9);
-  // RAX: receiver's class ID as smi.
-  __ movq(R9, Address(R13, 0));  // First class ID (Smi) to check.
-  __ jmp(&test);
+  __ Comment("ICData loop");
 
-  __ Comment("ICData loop");
+  // We only unroll the generic one that is generated once.
+  bool optimize = kind == Token::kILLEGAL;
+  const intptr_t target_offset = ICData::TargetIndexFor(num_args) * kWordSize;
+  const intptr_t count_offset = ICData::CountIndexFor(num_args) * kWordSize;
+
+  bool receiver_loaded = false;
+
   __ Bind(&loop);
-  for (int i = 0; i < num_args; i++) {
-    if (i > 0) {
-      // If not the first, load the next argument's class ID.
-      __ movq(RAX, FieldAddress(R10, ArgumentsDescriptor::count_offset()));
-      __ movq(R9, Address(RSP, RAX, TIMES_4, -i * kWordSize));
+  for (int unroll = optimize ? 4 : 2; unroll >= 0; unroll--) {
+    Label update;
+    if (!receiver_loaded) {
+      receiver_loaded = true;
+      // Get argument descriptor into RCX.
+      __ movq(RCX, FieldAddress(R10, ArgumentsDescriptor::count_offset()));
+      // Load first argument into R9.
+      __ movq(R9, Address(RSP, RCX, TIMES_4, 0));
       __ LoadTaggedClassIdMayBeSmi(RAX, R9);
-      // RAX: next argument class ID (smi).
+      // RAX: first argument class ID (smi).
+      if (num_args == 2) {
+        // Load second argument into R9.
+        __ movq(R9, Address(RSP, RCX, TIMES_4, -kWordSize));
+        __ LoadTaggedClassIdMayBeSmi(RCX, R9);
+        // RCX: second argument class ID (smi).
+      }
+    }
+    for (int i = 0; i < num_args; i++) {
       __ movq(R9, Address(R13, i * kWordSize));
       // R9: next class ID to check (smi).
+      __ cmpq(i == 0 ? RAX : RCX, R9);  // Class id match?
+      if (i < (num_args - 1)) {
+        __ j(NOT_EQUAL, &update);  // Continue.
+      } else {
+        // Last check, all checks before matched.
+        __ j(EQUAL, &found);  // Break.
+      }
     }
-    __ cmpq(RAX, R9);  // Class id match?
-    if (i < (num_args - 1)) {
-      __ j(NOT_EQUAL, &update);  // Continue.
+    __ Bind(&update);
+
+    const intptr_t entry_size =
+        ICData::TestEntryLengthFor(num_args) * kWordSize;
+    __ addq(R13, Immediate(entry_size));  // Next entry.
+
+    __ cmpq(R9, Immediate(Smi::RawValue(kIllegalCid)));  // Done?
+    if (unroll == 0) {
+      __ j(NOT_EQUAL, &loop);
     } else {
-      // Last check, all checks before matched.
-      __ j(EQUAL, &found);  // Break.
+      __ j(EQUAL, &miss);
     }
   }
-  __ Bind(&update);
-  // Reload receiver class ID.  It has not been destroyed when num_args == 1.
-  if (num_args > 1) {
-    __ movq(RAX, FieldAddress(R10, ArgumentsDescriptor::count_offset()));
-    __ movq(R9, Address(RSP, RAX, TIMES_4, 0));
-    __ LoadTaggedClassIdMayBeSmi(RAX, R9);
-  }
 
-  const intptr_t entry_size = ICData::TestEntryLengthFor(num_args) * kWordSize;
-  __ addq(R13, Immediate(entry_size));  // Next entry.
-  __ movq(R9, Address(R13, 0));         // Next class ID.
-
-  __ Bind(&test);
-  __ cmpq(R9, Immediate(Smi::RawValue(kIllegalCid)));  // Done?
-  __ j(NOT_EQUAL, &loop, Assembler::kNearJump);
-
+  __ Bind(&miss);
   __ Comment("IC miss");
   // Compute address of arguments (first read number of arguments from
   // arguments descriptor array and then compute address on the stack).
   __ movq(RAX, FieldAddress(R10, ArgumentsDescriptor::count_offset()));
   __ leaq(RAX, Address(RSP, RAX, TIMES_4, 0));  // RAX is Smi.
   __ EnterStubFrame();
   __ pushq(R10);           // Preserve arguments descriptor array.
   __ pushq(RBX);           // Preserve IC data object.
   __ pushq(Immediate(0));  // Result slot.
   // Push call arguments.
@@ skipping 14 matching lines @@
   __ LeaveStubFrame();
   Label call_target_function;
   if (!FLAG_lazy_dispatchers) {
     GenerateDispatcherCode(assembler, &call_target_function);
   } else {
     __ jmp(&call_target_function);
   }
 
   __ Bind(&found);
   // R13: Pointer to an IC data check group.
-  const intptr_t target_offset = ICData::TargetIndexFor(num_args) * kWordSize;
-  const intptr_t count_offset = ICData::CountIndexFor(num_args) * kWordSize;
   __ movq(RAX, Address(R13, target_offset));
 
   if (FLAG_optimization_counter_threshold >= 0) {
-    // Update counter.
     __ Comment("Update caller's counter");
-    __ movq(R8, Address(R13, count_offset));
-    __ addq(R8, Immediate(Smi::RawValue(1)));
-    __ movq(R9, Immediate(Smi::RawValue(Smi::kMaxValue)));
-    __ cmovnoq(R9, R8);
-    __ StoreIntoSmiField(Address(R13, count_offset), R9);
+    __ addq(Address(R13, count_offset), Immediate(Smi::RawValue(1)));
   }
 
   __ Comment("Call target");
   __ Bind(&call_target_function);
   // RAX: Target function.
-  Label is_compiled;
   __ movq(CODE_REG, FieldAddress(RAX, Function::code_offset()));
   __ movq(RCX, FieldAddress(RAX, Function::entry_point_offset()));
   __ jmp(RCX);
 
   if (FLAG_support_debugger && !optimized) {
     __ Bind(&stepping);
     __ EnterStubFrame();
     __ pushq(RBX);
     __ CallRuntime(kSingleStepHandlerRuntimeEntry, 0);
     __ popq(RBX);
@@ skipping 117 matching lines @@
   // RBX: IC data object (preserved).
   __ movq(R12, FieldAddress(RBX, ICData::ic_data_offset()));
   // R12: ic_data_array with entries: target functions and count.
   __ leaq(R12, FieldAddress(R12, Array::data_offset()));
   // R12: points directly to the first ic data array element.
   const intptr_t target_offset = ICData::TargetIndexFor(0) * kWordSize;
   const intptr_t count_offset = ICData::CountIndexFor(0) * kWordSize;
 
   if (FLAG_optimization_counter_threshold >= 0) {
     // Increment count for this call.
-    __ movq(R8, Address(R12, count_offset));
-    __ addq(R8, Immediate(Smi::RawValue(1)));
-    __ movq(R13, Immediate(Smi::RawValue(Smi::kMaxValue)));
-    __ cmovnoq(R13, R8);
-    __ StoreIntoSmiField(Address(R12, count_offset), R13);
+    __ addq(Address(R12, count_offset), Immediate(Smi::RawValue(1)));

[Florian Schneider, 2017/01/20 17:26:35]

I think this should be safe: We already use INT_MIN to mark functions should not
be optimized in the near future. Wrapping around here would only mean missing
optimization of that function, but does not affect correct execution.

[erikcorry, 2017/01/24 15:35:41]

I'll make this change on 32 bit too, then.  It is 64 bit here, btw (that's what
the 'q' means).

[regis, 2017/01/24 18:27:21]

Definitely a 64-bit overflow check. My bad. I got confused by the reset value
being in the 32-bit range.

   }
 
   // Load arguments descriptor into R10.
   __ movq(R10, FieldAddress(RBX, ICData::arguments_descriptor_offset()));
 
   // Get function and call it, if possible.
   __ movq(RAX, Address(R12, target_offset));
   __ movq(CODE_REG, FieldAddress(RAX, Function::code_offset()));
   __ movq(RCX, FieldAddress(RAX, Function::entry_point_offset()));
   __ jmp(RCX);
@@ skipping 646 matching lines @@
 }
 
 
 void StubCode::GenerateFrameAwaitingMaterializationStub(Assembler* assembler) {
   __ int3();
 }
 
 }  // namespace dart
 
 #endif  // defined TARGET_ARCH_X64