Optimizations to IC stub for unoptimized code performance on x64.

runtime/vm/stub_code_arm64.cc

 // Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/globals.h"
 #if defined(TARGET_ARCH_ARM64)
 
 #include "vm/assembler.h"
 #include "vm/code_generator.h"
 #include "vm/compiler.h"
@@ skipping 1362 matching lines @@
   __ b(&error, NE);
   __ ldr(R1, Address(R6, kWordSize));
   __ CompareImmediate(R1, imm_smi_cid);
   __ b(&ok, EQ);
   __ Bind(&error);
   __ Stop("Incorrect IC data");
   __ Bind(&ok);
 #endif
   if (FLAG_optimization_counter_threshold >= 0) {
     const intptr_t count_offset = ICData::CountIndexFor(num_args) * kWordSize;
-    // Update counter.
+    // Update counter, ignore overflow.
     __ LoadFromOffset(R1, R6, count_offset);
     __ adds(R1, R1, Operand(Smi::RawValue(1)));
-    __ LoadImmediate(R2, Smi::RawValue(Smi::kMaxValue));
-    __ csel(R1, R2, R1, VS);  // Overflow.
     __ StoreToOffset(R1, R6, count_offset);
   }
 
   __ ret();
 }
 
 
 // Generate inline cache check for 'num_args'.
 //  LR: return address.
 //  R5: inline cache data object.
 // Control flow:
 // - If receiver is null -> jump to IC miss.
 // - If receiver is Smi -> load Smi class.
 // - If receiver is not-Smi -> load receiver's class.
 // - Check if 'num_args' (including receiver) match any IC data group.
 // - Match found -> jump to target.
 // - Match not found -> jump to IC miss.
 void StubCode::GenerateNArgsCheckInlineCacheStub(
     Assembler* assembler,
     intptr_t num_args,
     const RuntimeEntry& handle_ic_miss,
     Token::Kind kind,
     bool optimized) {
-  ASSERT(num_args > 0);
+  ASSERT(num_args == 1 || num_args == 2);
 #if defined(DEBUG)
   {
     Label ok;
     // Check that the IC data array has NumArgsTested() == num_args.
     // 'NumArgsTested' is stored in the least significant bits of 'state_bits'.
     __ LoadFromOffset(R6, R5, ICData::state_bits_offset() - kHeapObjectTag,
                       kUnsignedWord);
     ASSERT(ICData::NumArgsTestedShift() == 0);  // No shift needed.
     __ andi(R6, R6, Immediate(ICData::NumArgsTestedMask()));
     __ CompareImmediate(R6, num_args);
@@ skipping 16 matching lines @@
   Label not_smi_or_overflow;
   if (kind != Token::kILLEGAL) {
     EmitFastSmiOp(assembler, kind, num_args, &not_smi_or_overflow);
   }
   __ Bind(&not_smi_or_overflow);
 
   __ Comment("Extract ICData initial values and receiver cid");
   // Load arguments descriptor into R4.
   __ LoadFieldFromOffset(R4, R5, ICData::arguments_descriptor_offset());
   // Loop that checks if there is an IC data match.
-  Label loop, update, test, found;
+  Label loop, found, miss;
   // R5: IC data object (preserved).
   __ LoadFieldFromOffset(R6, R5, ICData::ic_data_offset());
   // R6: ic_data_array with check entries: classes and target functions.
   __ AddImmediate(R6, R6, Array::data_offset() - kHeapObjectTag);
   // R6: points directly to the first ic data array element.
 
   // Get the receiver's class ID (first read number of arguments from
   // arguments descriptor array and then access the receiver from the stack).
   __ LoadFieldFromOffset(R7, R4, ArgumentsDescriptor::count_offset());
   __ SmiUntag(R7);  // Untag so we can use the LSL 3 addressing mode.
   __ sub(R7, R7, Operand(1));
 
   // R0 <- [SP + (R7 << 3)]
   __ ldr(R0, Address(SP, R7, UXTX, Address::Scaled));
   __ LoadTaggedClassIdMayBeSmi(R0, R0);
 
-  // R7: argument_count - 1 (untagged).
-  // R0: receiver's class ID (smi).
-  __ ldr(R1, Address(R6));  // First class id (smi) to check.
-  __ b(&test);
+  if (num_args == 2) {
+    __ AddImmediate(R1, R7, -1);
+    // R1 <- [SP + (R1 << 3)]
+    __ ldr(R1, Address(SP, R1, UXTX, Address::Scaled));
+    __ LoadTaggedClassIdMayBeSmi(R1, R1);
+  }
+
+  // We unroll the generic one that is generated once more than the others.
+  bool optimize = kind == Token::kILLEGAL;
 
   __ Comment("ICData loop");
   __ Bind(&loop);
-  for (int i = 0; i < num_args; i++) {
-    if (i > 0) {
-      // If not the first, load the next argument's class ID.
-      __ AddImmediate(R0, R7, -i);
-      // R0 <- [SP + (R0 << 3)]
-      __ ldr(R0, Address(SP, R0, UXTX, Address::Scaled));
-      __ LoadTaggedClassIdMayBeSmi(R0, R0);
-      // R0: next argument class ID (smi).
-      __ LoadFromOffset(R1, R6, i * kWordSize);
-      // R1: next class ID to check (smi).
+  for (int unroll = optimize ? 4 : 2; unroll >= 0; unroll--) {
+    Label update;
+
+    __ LoadFromOffset(R2, R6, 0);
+    __ CompareRegisters(R0, R2);  // Class id match?
+    if (num_args == 2) {
+      __ b(&update, NE);  // Continue.
+      __ LoadFromOffset(R2, R6, kWordSize);
+      __ CompareRegisters(R1, R2);  // Class id match?
     }
-    __ CompareRegisters(R0, R1);  // Class id match?
-    if (i < (num_args - 1)) {
-      __ b(&update, NE);  // Continue.
+    __ b(&found, EQ);  // Break.
+
+    __ Bind(&update);
+
+    const intptr_t entry_size =
+        ICData::TestEntryLengthFor(num_args) * kWordSize;
+    __ AddImmediate(R6, R6, entry_size);  // Next entry.
+
+    __ CompareImmediate(R2, Smi::RawValue(kIllegalCid));  // Done?
+    if (unroll == 0) {
+      __ b(&loop, NE);
     } else {
-      // Last check, all checks before matched.
-      __ b(&found, EQ);  // Break.
+      __ b(&miss, EQ);
     }
   }
-  __ Bind(&update);
-  // Reload receiver class ID.  It has not been destroyed when num_args == 1.
-  if (num_args > 1) {
-    __ ldr(R0, Address(SP, R7, UXTX, Address::Scaled));
-    __ LoadTaggedClassIdMayBeSmi(R0, R0);
-  }
 
-  const intptr_t entry_size = ICData::TestEntryLengthFor(num_args) * kWordSize;
-  __ AddImmediate(R6, R6, entry_size);  // Next entry.
-  __ ldr(R1, Address(R6));              // Next class ID.
-
-  __ Bind(&test);
-  __ CompareImmediate(R1, Smi::RawValue(kIllegalCid));  // Done?
-  __ b(&loop, NE);
-
+  __ Bind(&miss);
   __ Comment("IC miss");
   // Compute address of arguments.
   // R7: argument_count - 1 (untagged).
   // R7 <- SP + (R7 << 3)
   __ add(R7, SP, Operand(R7, UXTX, 3));  // R7 is Untagged.
   // R7: address of receiver.
   // Create a stub frame as we are pushing some objects on the stack before
   // calling into the runtime.
   __ EnterStubFrame();
   // Preserve IC data object and arguments descriptor array and
@@ skipping 27 matching lines @@
   }
 
   __ Bind(&found);
   __ Comment("Update caller's counter");
   // R6: pointer to an IC data check group.
   const intptr_t target_offset = ICData::TargetIndexFor(num_args) * kWordSize;
   const intptr_t count_offset = ICData::CountIndexFor(num_args) * kWordSize;
   __ LoadFromOffset(R0, R6, target_offset);
 
   if (FLAG_optimization_counter_threshold >= 0) {
-    // Update counter.
+    // Update counter, ignore overflow.
     __ LoadFromOffset(R1, R6, count_offset);
     __ adds(R1, R1, Operand(Smi::RawValue(1)));
-    __ LoadImmediate(R2, Smi::RawValue(Smi::kMaxValue));
-    __ csel(R1, R2, R1, VS);  // Overflow.
     __ StoreToOffset(R1, R6, count_offset);
   }
 
   __ Comment("Call target");
   __ Bind(&call_target_function);
   // R0: target function.
   __ LoadFieldFromOffset(CODE_REG, R0, Function::code_offset());
   __ LoadFieldFromOffset(R2, R0, Function::entry_point_offset());
   __ br(R2);
 
@@ skipping 104 matching lines @@
 
   // R5: IC data object (preserved).
   __ LoadFieldFromOffset(R6, R5, ICData::ic_data_offset());
   // R6: ic_data_array with entries: target functions and count.
   __ AddImmediate(R6, R6, Array::data_offset() - kHeapObjectTag);
   // R6: points directly to the first ic data array element.
   const intptr_t target_offset = ICData::TargetIndexFor(0) * kWordSize;
   const intptr_t count_offset = ICData::CountIndexFor(0) * kWordSize;
 
   if (FLAG_optimization_counter_threshold >= 0) {
-    // Increment count for this call.
+    // Increment count for this call, ignore overflow.
     __ LoadFromOffset(R1, R6, count_offset);
     __ adds(R1, R1, Operand(Smi::RawValue(1)));
-    __ LoadImmediate(R2, Smi::RawValue(Smi::kMaxValue));
-    __ csel(R1, R2, R1, VS);  // Overflow.
     __ StoreToOffset(R1, R6, count_offset);
   }
 
   // Load arguments descriptor into R4.
   __ LoadFieldFromOffset(R4, R5, ICData::arguments_descriptor_offset());
 
   // Get function and call it, if possible.
   __ LoadFromOffset(R0, R6, target_offset);
   __ LoadFieldFromOffset(CODE_REG, R0, Function::code_offset());
   __ LoadFieldFromOffset(R2, R0, Function::entry_point_offset());
@@ skipping 640 matching lines @@
 }
 
 
 void StubCode::GenerateFrameAwaitingMaterializationStub(Assembler* assembler) {
   __ brk(0);
 }
 
 }  // namespace dart
 
 #endif  // defined TARGET_ARCH_ARM64