Faster IC stubs by specializing them for Binary Smi operations

runtime/vm/stub_code_arm64.cc

Index: runtime/vm/stub_code_arm64.cc
===================================================================
--- runtime/vm/stub_code_arm64.cc	(revision 39307)
+++ runtime/vm/stub_code_arm64.cc	(working copy)
@@ -1314,6 +1314,79 @@
 }
 
 
+// Note: xxx must be preserved if 'not_smi_or_overflow' is jumped to.

[Cutch, 2014/08/15 21:23:28]

xxx?

[zra, 2014/08/15 21:45:08]

Should be R5 as on ARM.

[srdjan, 2014/08/15 21:49:00]

R6.

[zra, 2014/08/15 22:12:33]

Ah, right, sorry.

[srdjan, 2014/08/18 20:40:27]

You are right: R5 :-).

+// Attempt a quick Smi operation for known operations ('kind'). The ICData
+// must have been primed with a Smi/Smi check that will be used for counting
+// the invocations.
+static void EmitFastSmiOp(Assembler* assembler,
+                          Token::Kind kind,
+                          intptr_t num_args,
+                          Label* not_smi_or_overflow) {
+  if (FLAG_throw_on_javascript_int_overflow) {
+    // The overflow check is more complex than implemented below.
+    return;
+  }
+  __ ldr(R0, Address(SP, + 0 * kWordSize));  // Right.
+  __ ldr(R1, Address(SP, + 1 * kWordSize));  // Left.
+  __ orr(TMP, R0, Operand(R1));
+  __ tsti(TMP, kSmiTagMask);
+  __ b(not_smi_or_overflow, NE);
+  switch (kind) {
+    case Token::kADD: {
+      __ adds(R0, R1, Operand(R0));  // Adds.
+      __ b(not_smi_or_overflow, VS);  // Branch if overflow.
+      break;
+    }
+    case Token::kSUB: {
+      __ subs(R0, R1, Operand(R0));  // Subtract.
+      __ b(not_smi_or_overflow, VS);  // Branch if overflow.
+      break;
+    }
+    case Token::kEQ: {
+      Label done, is_true;

[zra, 2014/08/15 21:45:08]

Using conditional select:

__ CompareRegisters(R0, R1);
__ LoadObject(R0, Bool::True(), PP);
__ LoadObject(R1, Bool::False(), PP);
__ csel(R0, R1, R0, NE);

[srdjan, 2014/08/18 20:40:27]

Done.

+      __ cmp(R0, Operand(R1));
+      __ b(&is_true, EQ);
+      __ LoadObject(R0, Bool::False(), PP);
+      __ b(&done);
+      __ Bind(&is_true);
+      __ LoadObject(R0, Bool::True(), PP);
+      __ Bind(&done);
+      break;
+    }
+    default: UNIMPLEMENTED();
+  }
+
+  // R5: IC data object (preserved).
+  __ LoadFieldFromOffset(R6, R5, ICData::ic_data_offset(), kNoPP);
+  // R6: ic_data_array with check entries: classes and target functions.
+  __ AddImmediate(R6, R6, Array::data_offset() - kHeapObjectTag, kNoPP);
+  // R6: points directly to the first ic data array element.
+#ifdef DEBUG
+  // Check that first entry is for Smi/Smi.
+  Label error, ok;
+  const intptr_t imm_smi_cid = reinterpret_cast<intptr_t>(Smi::New(kSmiCid));
+  __ ldr(R1, Address(R6, 0));
+  __ CompareImmediate(R1, imm_smi_cid, kNoPP);
+  __ b(&error, NE);
+  __ ldr(R1, Address(R6, kWordSize));
+  __ CompareImmediate(R1, imm_smi_cid, kNoPP);
+  __ b(&ok, EQ);
+  __ Bind(&error);

[zra, 2014/08/15 21:45:08]

Same comment as on ARM.

[srdjan, 2014/08/18 20:40:27]

Ditto.

+  __ Stop("Incorrect IC data");
+  __ Bind(&ok);
+#endif
+  const intptr_t count_offset = ICData::CountIndexFor(num_args) * kWordSize;
+  // Update counter.
+  __ LoadFromOffset(R1, R6, count_offset, kNoPP);
+  __ adds(R1, R1, Operand(Smi::RawValue(1)));
+  __ LoadImmediate(R2, Smi::RawValue(Smi::kMaxValue), kNoPP);
+  __ csel(R1, R2, R1, VS);  // Overflow.
+  __ StoreToOffset(R1, R6, count_offset, kNoPP);
+
+  __ ret();
+}
+
+
 // Generate inline cache check for 'num_args'.
 //  LR: return address.
 //  R5: inline cache data object.
@@ -1327,7 +1400,8 @@
 void StubCode::GenerateNArgsCheckInlineCacheStub(
     Assembler* assembler,
     intptr_t num_args,
-    const RuntimeEntry& handle_ic_miss) {
+    const RuntimeEntry& handle_ic_miss,
+    Token::Kind kind) {
   ASSERT(num_args > 0);
 #if defined(DEBUG)
   { Label ok;
@@ -1353,6 +1427,12 @@
   __ b(&stepping, NE);
   __ Bind(&done_stepping);
 
+  if (kind != Token::kILLEGAL) {
+    Label not_smi_or_overflow;
+    EmitFastSmiOp(assembler, kind, num_args, &not_smi_or_overflow);
+    __ Bind(&not_smi_or_overflow);
+  }
+
   // Load arguments descriptor into R4.
   __ LoadFieldFromOffset(R4, R5, ICData::arguments_descriptor_offset(), kNoPP);
   // Loop that checks if there is an IC data match.
@@ -1489,52 +1569,73 @@
 //   - 1 target function.
 void StubCode::GenerateOneArgCheckInlineCacheStub(Assembler* assembler) {
   GenerateUsageCounterIncrement(assembler, R6);
-  GenerateNArgsCheckInlineCacheStub(
-      assembler, 1, kInlineCacheMissHandlerOneArgRuntimeEntry);
+  GenerateNArgsCheckInlineCacheStub(assembler, 1,
+      kInlineCacheMissHandlerOneArgRuntimeEntry, Token::kILLEGAL);
 }
 
 
 void StubCode::GenerateTwoArgsCheckInlineCacheStub(Assembler* assembler) {
   GenerateUsageCounterIncrement(assembler, R6);
-  GenerateNArgsCheckInlineCacheStub(
-      assembler, 2, kInlineCacheMissHandlerTwoArgsRuntimeEntry);
+  GenerateNArgsCheckInlineCacheStub(assembler, 2,
+      kInlineCacheMissHandlerTwoArgsRuntimeEntry, Token::kILLEGAL);
 }
 
 
 void StubCode::GenerateThreeArgsCheckInlineCacheStub(Assembler* assembler) {
   GenerateUsageCounterIncrement(assembler, R6);
-  GenerateNArgsCheckInlineCacheStub(
-      assembler, 3, kInlineCacheMissHandlerThreeArgsRuntimeEntry);
+  GenerateNArgsCheckInlineCacheStub(assembler, 3,
+      kInlineCacheMissHandlerThreeArgsRuntimeEntry, Token::kILLEGAL);
 }
 
 
+void StubCode::GenerateSmiAddInlineCacheStub(Assembler* assembler) {
+  GenerateUsageCounterIncrement(assembler, R6);
+  GenerateNArgsCheckInlineCacheStub(assembler, 2,
+      kInlineCacheMissHandlerTwoArgsRuntimeEntry, Token::kADD);
+}
+
+
+void StubCode::GenerateSmiSubInlineCacheStub(Assembler* assembler) {
+  GenerateUsageCounterIncrement(assembler, R6);
+  GenerateNArgsCheckInlineCacheStub(assembler, 2,
+      kInlineCacheMissHandlerTwoArgsRuntimeEntry, Token::kSUB);
+}
+
+
+void StubCode::GenerateSmiEqualInlineCacheStub(Assembler* assembler) {
+  GenerateUsageCounterIncrement(assembler, R6);
+  GenerateNArgsCheckInlineCacheStub(assembler, 2,
+      kInlineCacheMissHandlerTwoArgsRuntimeEntry, Token::kEQ);
+}
+
+
 void StubCode::GenerateOneArgOptimizedCheckInlineCacheStub(
     Assembler* assembler) {
   GenerateOptimizedUsageCounterIncrement(assembler);
-  GenerateNArgsCheckInlineCacheStub(
-      assembler, 1, kInlineCacheMissHandlerOneArgRuntimeEntry);
+  GenerateNArgsCheckInlineCacheStub(assembler, 1,
+      kInlineCacheMissHandlerOneArgRuntimeEntry, Token::kILLEGAL);
 }
 
 
 void StubCode::GenerateTwoArgsOptimizedCheckInlineCacheStub(
     Assembler* assembler) {
   GenerateOptimizedUsageCounterIncrement(assembler);
-  GenerateNArgsCheckInlineCacheStub(
-      assembler, 2, kInlineCacheMissHandlerTwoArgsRuntimeEntry);
+  GenerateNArgsCheckInlineCacheStub(assembler, 2,
+      kInlineCacheMissHandlerTwoArgsRuntimeEntry, Token::kILLEGAL);
 }
 
 
 void StubCode::GenerateThreeArgsOptimizedCheckInlineCacheStub(
     Assembler* assembler) {
   GenerateOptimizedUsageCounterIncrement(assembler);
-  GenerateNArgsCheckInlineCacheStub(
-      assembler, 3, kInlineCacheMissHandlerThreeArgsRuntimeEntry);
+  GenerateNArgsCheckInlineCacheStub(assembler, 3,
+      kInlineCacheMissHandlerThreeArgsRuntimeEntry, Token::kILLEGAL);
 }
 
 
 void StubCode::GenerateClosureCallInlineCacheStub(Assembler* assembler) {
-  GenerateNArgsCheckInlineCacheStub(
-      assembler, 1, kInlineCacheMissHandlerOneArgRuntimeEntry);
+  GenerateNArgsCheckInlineCacheStub(assembler, 1,
+      kInlineCacheMissHandlerOneArgRuntimeEntry, Token::kILLEGAL);
 }
 
 
@@ -1604,8 +1705,8 @@
 
 void StubCode::GenerateTwoArgsUnoptimizedStaticCallStub(Assembler* assembler) {
   GenerateUsageCounterIncrement(assembler, R6);
-  GenerateNArgsCheckInlineCacheStub(
-      assembler, 2, kStaticCallMissHandlerTwoArgsRuntimeEntry);
+  GenerateNArgsCheckInlineCacheStub(assembler, 2,
+      kStaticCallMissHandlerTwoArgsRuntimeEntry, Token::kILLEGAL);
 }