Eliminates more large immediates from x64 code.

runtime/vm/assembler_x64.cc

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/globals.h"
 #if defined(TARGET_ARCH_X64)
 
 #include "vm/assembler.h"
 #include "vm/heap.h"
 #include "vm/memory_region.h"
@@ skipping 186 matching lines @@
     AssemblerBuffer::EnsureCapacity ensured(&buffer_);
     EmitUint8(0x68);
     EmitImmediate(imm);
   } else {
     movq(TMP, imm);
     pushq(TMP);
   }
 }
 
 
+void Assembler::PushImmediate(const Immediate& imm, Register pp) {
+  if (imm.is_int32()) {
+    pushq(imm);
+  } else {
+    LoadImmediate(TMP, imm, pp);
+    pushq(TMP);
+  }
+}
+
+
 void Assembler::popq(Register reg) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   EmitRegisterREX(reg, REX_NONE);
   EmitUint8(0x58 | (reg & 7));
 }
 
 
 void Assembler::popq(const Address& address) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   EmitOperandREX(0, address, REX_NONE);
@@ skipping 589 matching lines @@
 }
 
 void Assembler::notps(XmmRegister dst) {
   static const struct ALIGN16 {
     uint32_t a;
     uint32_t b;
     uint32_t c;
     uint32_t d;
   } float_not_constant =
       { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF };
-  movq(TMP, Immediate(reinterpret_cast<intptr_t>(&float_not_constant)));
+  LoadImmediate(
+      TMP, Immediate(reinterpret_cast<intptr_t>(&float_not_constant)), PP);
   xorps(dst, Address(TMP, 0));
 }
 
 
 void Assembler::negateps(XmmRegister dst) {
   static const struct ALIGN16 {
     uint32_t a;
     uint32_t b;
     uint32_t c;
     uint32_t d;
   } float_negate_constant =
       { 0x80000000, 0x80000000, 0x80000000, 0x80000000 };
-  movq(TMP, Immediate(reinterpret_cast<intptr_t>(&float_negate_constant)));
+  LoadImmediate(
+      TMP, Immediate(reinterpret_cast<intptr_t>(&float_negate_constant)), PP);
   xorps(dst, Address(TMP, 0));
 }
 
 
 void Assembler::absps(XmmRegister dst) {
   static const struct ALIGN16 {
     uint32_t a;
     uint32_t b;
     uint32_t c;
     uint32_t d;
   } float_absolute_constant =
       { 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF };
-  movq(TMP, Immediate(reinterpret_cast<intptr_t>(&float_absolute_constant)));
+  LoadImmediate(
+      TMP, Immediate(reinterpret_cast<intptr_t>(&float_absolute_constant)), PP);
   andps(dst, Address(TMP, 0));
 }
 
 
 void Assembler::zerowps(XmmRegister dst) {
   static const struct ALIGN16 {
     uint32_t a;
     uint32_t b;
     uint32_t c;
     uint32_t d;
   } float_zerow_constant =
       { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000 };
-  movq(TMP, Immediate(reinterpret_cast<intptr_t>(&float_zerow_constant)));
+  LoadImmediate(
+      TMP, Immediate(reinterpret_cast<intptr_t>(&float_zerow_constant)), PP);
   andps(dst, Address(TMP, 0));
 }
 
 
 void Assembler::cmppseq(XmmRegister dst, XmmRegister src) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   EmitREX_RB(dst, src);
   EmitUint8(0x0F);
   EmitUint8(0xC2);
   EmitXmmRegisterOperand(dst & 7, src);
@@ skipping 469 matching lines @@
 
 
 void Assembler::cmpq(Register reg, const Address& address) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   EmitOperandREX(reg, address, REX_W);
   EmitUint8(0x3B);
   EmitOperand(reg & 7, address);
 }
 
 
+void Assembler::CompareImmediate(Register reg, const Immediate& imm,
+                                 Register pp) {
+  if (imm.is_int32()) {
+    cmpq(reg, imm);
+  } else {
+    LoadImmediate(TMP, imm, pp);
+    cmpq(reg, TMP);
+  }
+}
+
+
+void Assembler::CompareImmediate(const Address& address, const Immediate& imm,
+                                 Register pp) {
+  if (imm.is_int32()) {
+    cmpq(address, imm);
+  } else {
+    LoadImmediate(TMP, imm, pp);
+    cmpq(address, TMP);
+  }
+}
+
+
 void Assembler::testl(Register reg1, Register reg2) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   Operand operand(reg2);
   EmitOperandREX(reg1, operand, REX_NONE);
   EmitUint8(0x85);
   EmitOperand(reg1 & 7, operand);
 }
 
 
 void Assembler::testl(Register reg, const Immediate& imm) {
@@ skipping 130 matching lines @@
     AssemblerBuffer::EnsureCapacity ensured(&buffer_);
     EmitRegisterREX(dst, REX_W);
     EmitComplex(4, Operand(dst), imm);
   } else {
     movq(TMP, imm);
     andq(dst, TMP);
   }
 }
 
 
+void Assembler::AndImmediate(Register dst, const Immediate& imm, Register pp) {
+  if (imm.is_int32()) {
+    andq(dst, imm);
+  } else {
+    ASSERT(dst != TMP);
+    LoadImmediate(TMP, imm, pp);
+    andq(dst, TMP);
+  }
+}
+
+
 void Assembler::orq(Register dst, Register src) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   Operand operand(src);
   EmitOperandREX(dst, operand, REX_W);
   EmitUint8(0x0B);
   EmitOperand(dst & 7, operand);
 }
 
 
 void Assembler::orq(Register dst, const Address& address) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   EmitOperandREX(dst, address, REX_W);
   EmitUint8(0x0B);
   EmitOperand(dst & 7, address);
 }
 
 
 void Assembler::orq(Register dst, const Immediate& imm) {
   if (imm.is_int32()) {
     AssemblerBuffer::EnsureCapacity ensured(&buffer_);
     EmitRegisterREX(dst, REX_W);
     EmitComplex(1, Operand(dst), imm);
   } else {
     movq(TMP, imm);
     orq(dst, TMP);
   }
 }
 
 
+void Assembler::OrImmediate(Register dst, const Immediate& imm, Register pp) {
+  if (imm.is_int32()) {
+    orq(dst, imm);
+  } else {
+    ASSERT(dst != TMP);
+    LoadImmediate(TMP, imm, pp);
+    orq(dst, TMP);
+  }
+}
+
+
 void Assembler::xorq(Register dst, Register src) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   Operand operand(src);
   EmitOperandREX(dst, operand, REX_W);
   EmitUint8(0x33);
   EmitOperand(dst & 7, operand);
 }
 
 
 void Assembler::xorq(Register dst, const Address& address) {
@@ skipping 17 matching lines @@
     AssemblerBuffer::EnsureCapacity ensured(&buffer_);
     EmitRegisterREX(dst, REX_W);
     EmitComplex(6, Operand(dst), imm);
   } else {
     movq(TMP, imm);
     xorq(dst, TMP);
   }
 }
 
 
+void Assembler::XorImmediate(Register dst, const Immediate& imm, Register pp) {
+  if (imm.is_int32()) {
+    xorq(dst, imm);
+  } else {
+    ASSERT(dst != TMP);
+    LoadImmediate(TMP, imm, pp);
+    xorq(dst, TMP);
+  }
+}
+
+
 void Assembler::addl(Register dst, Register src) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   Operand operand(src);
   EmitOperandREX(dst, operand, REX_NONE);
   EmitUint8(0x03);
   EmitOperand(dst & 7, operand);
 }
 
 
 void Assembler::addq(Register dst, Register src) {
@@ skipping 130 matching lines @@
     EmitUint8(0x69);
     EmitOperand(reg & 7, Operand(reg));
     EmitImmediate(imm);
   } else {
     movq(TMP, imm);
     imulq(reg, TMP);
   }
 }
 
 
+void Assembler::MulImmediate(Register reg, const Immediate& imm, Register pp) {
+  if (imm.is_int32()) {
+    imulq(reg, imm);
+  } else {
+    ASSERT(reg != TMP);
+    LoadImmediate(TMP, imm, pp);
+    imulq(reg, TMP);
+  }
+}
+
+
 void Assembler::imulq(Register dst, const Address& address) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   EmitOperandREX(dst, address, REX_W);
   EmitUint8(0x0F);
   EmitUint8(0xAF);
   EmitOperand(dst & 7, address);
 }
 
 
 void Assembler::subq(Register dst, Register src) {
@@ skipping 18 matching lines @@
 
 
 void Assembler::subq(Register reg, const Address& address) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   EmitOperandREX(reg, address, REX_W);
   EmitUint8(0x2B);
   EmitOperand(reg & 7, address);
 }
 
 
+void Assembler::subq(const Address& address, Register reg) {
+  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
+  EmitOperandREX(reg, address, REX_W);
+  EmitUint8(0x29);
+  EmitOperand(reg & 7, address);
+}
+
+
+void Assembler::subq(const Address& address, const Immediate& imm) {
+  // TODO(srdjan): Implement shorter version for imm32.
+  movq(TMP, imm);
+  subq(address, TMP);
+}
+
+
 void Assembler::shll(Register reg, const Immediate& imm) {
   EmitGenericShift(false, 4, reg, imm);
 }
 
 
 void Assembler::shll(Register operand, Register shifter) {
   EmitGenericShift(false, 4, operand, shifter);
 }
 
 
@@ skipping 372 matching lines @@
     movq(to, from);
   }
 }
 
 
 void Assembler::PopRegister(Register r) {
   popq(r);
 }
 
 
-void Assembler::AddImmediate(Register reg, const Immediate& imm) {
+void Assembler::AddImmediate(Register reg, const Immediate& imm, Register pp) {
   int64_t value = imm.value();
   if (value > 0) {
     if (value == 1) {
       incq(reg);
     } else if (value != 0) {
-      addq(reg, imm);
+      if (imm.is_int32()) {
+        addq(reg, imm);
+      } else {
+        ASSERT(reg != TMP);
+        LoadImmediate(TMP, imm, pp);
+        addq(reg, TMP);
+      }
     }
   } else if (value < 0) {
     value = -value;
     if (value == 1) {
       decq(reg);
     } else if (value != 0) {
-      subq(reg, Immediate(value));
+      const Immediate& s = Immediate(value);
+      if (s.is_int32()) {
+        subq(reg, Immediate(value));
+      } else {
+        ASSERT(reg != TMP);
+        LoadImmediate(TMP, s, pp);
+        subq(reg, TMP);
+      }
+    }
+  }
+}
+
+
+void Assembler::AddImmediate(const Address& address, const Immediate& imm,
+                             Register pp) {
+  int64_t value = imm.value();
+  if (value > 0) {
+    if (value == 1) {
+      incq(address);
+    } else if (value != 0) {
+      if (imm.is_int32()) {
+        addq(address, imm);
+      } else {
+        LoadImmediate(TMP, imm, pp);
+        addq(address, TMP);
+      }
+    }
+  } else if (value < 0) {
+    value = -value;
+    if (value == 1) {
+      decq(address);
+    } else if (value != 0) {
+      const Immediate& s = Immediate(value);
+      if (s.is_int32()) {
+        subq(address, s);
+      } else {
+        LoadImmediate(TMP, s, pp);
+        subq(address, TMP);
+      }
     }
   }
 }
 
 
 void Assembler::Drop(intptr_t stack_elements) {
   ASSERT(stack_elements >= 0);
   if (stack_elements <= 4) {
     for (intptr_t i = 0; i < stack_elements; i++) {
       popq(TMP);
@@ skipping 89 matching lines @@
 
 void Assembler::LoadObject(Register dst, const Object& object, Register pp) {
   if (CanLoadFromObjectPool(object)) {
     const int32_t offset =
         Array::element_offset(FindObject(object, kNotPatchable));
     LoadWordFromPoolOffset(dst, pp, offset - kHeapObjectTag);
   } else {
     ASSERT((Isolate::Current() == Dart::vm_isolate()) ||
            object.IsSmi() ||
            object.InVMHeap());
-    movq(dst, Immediate(reinterpret_cast<int64_t>(object.raw())));
+    LoadImmediate(dst, Immediate(reinterpret_cast<int64_t>(object.raw())), pp);
   }
 }
 
 
-void Assembler::StoreObject(const Address& dst, const Object& object) {
+void Assembler::StoreObject(const Address& dst, const Object& object,
+                            Register pp) {
   if (CanLoadFromObjectPool(object)) {
-    LoadObject(TMP, object, PP);
+    LoadObject(TMP, object, pp);
     movq(dst, TMP);
   } else {
-    movq(dst, Immediate(reinterpret_cast<int64_t>(object.raw())));
+    LoadImmediate(dst, Immediate(reinterpret_cast<int64_t>(object.raw())), pp);
   }
 }
 
 
-void Assembler::PushObject(const Object& object) {
+void Assembler::PushObject(const Object& object, Register pp) {
   if (CanLoadFromObjectPool(object)) {
-    LoadObject(TMP, object, PP);
+    LoadObject(TMP, object, pp);
     pushq(TMP);
   } else {
-    pushq(Immediate(reinterpret_cast<int64_t>(object.raw())));
+    PushImmediate(Immediate(reinterpret_cast<int64_t>(object.raw())), pp);
   }
 }
 
 
-void Assembler::CompareObject(Register reg, const Object& object) {
+void Assembler::CompareObject(Register reg, const Object& object, Register pp) {
   if (CanLoadFromObjectPool(object)) {
     ASSERT(reg != TMP);
-    LoadObject(TMP, object, PP);
+    LoadObject(TMP, object, pp);
     cmpq(reg, TMP);
   } else {
-    cmpq(reg, Immediate(reinterpret_cast<int64_t>(object.raw())));
+    CompareImmediate(
+        reg, Immediate(reinterpret_cast<int64_t>(object.raw())), pp);
+  }
+}
+
+
+intptr_t Assembler::FindImmediate(int64_t imm) {
+  ASSERT(Isolate::Current() != Dart::vm_isolate());
+  ASSERT(!object_pool_.IsNull());
+  const Smi& smi = Smi::Handle(reinterpret_cast<RawSmi*>(imm));
+  return FindObject(smi, kNotPatchable);
+}
+
+
+void Assembler::LoadImmediate(Register reg, const Immediate& imm, Register pp) {
+  if (imm.is_int32() ||
+      (pp == kNoRegister) ||
+      (Isolate::Current() == Dart::vm_isolate())) {
+    movq(reg, imm);
+  } else {
+    // It's a 64-bit constant and we're not in the VM isolate, so load from
+    // object pool.
+    int64_t val = imm.value();
+    // Save the bits that must be masked-off for the SmiTag
+    int64_t val_smi_tag = val & kSmiTagMask;
+    val &= ~kSmiTagMask;  // Mask off the tag bits.
+    const int32_t offset = Array::element_offset(FindImmediate(val));
+    LoadWordFromPoolOffset(reg, pp, offset - kHeapObjectTag);
+    if (val_smi_tag != 0) {
+      // Add back the tag bits.
+      orq(reg, Immediate(val_smi_tag));
+    }
+  }
+}
+
+
+void Assembler::LoadImmediate(const Address& dst, const Immediate& imm,
+                              Register pp) {
+  if (imm.is_int32() ||
+      (pp == kNoRegister) ||
+      (Isolate::Current() == Dart::vm_isolate())) {

[Florian Schneider, 2013/09/30 18:43:35]

These 3 conditions seem to be repeated a couple of times. Would it be worth to
encapsulate them in a helper function?

This would make it easy to use the same test for all the functions like
AddImmediate, CompareImmediate, etc.

[zra, 2013/09/30 19:59:31]

Done.

+    movq(dst, imm);
+  } else {
+    LoadImmediate(TMP, imm, pp);
+    movq(dst, TMP);
   }
 }
 
 
 // Destroys the value register.
 void Assembler::StoreIntoObjectFilterNoSmi(Register object,
                                            Register value,
                                            Label* no_update) {
   COMPILE_ASSERT((kNewObjectAlignmentOffset == kWordSize) &&
                  (kOldObjectAlignmentOffset == 0), young_alignment);
@@ skipping 70 matching lines @@
   // No store buffer update.
 }
 
 
 void Assembler::DoubleNegate(XmmRegister d) {
   static const struct ALIGN16 {
     uint64_t a;
     uint64_t b;
   } double_negate_constant =
       {0x8000000000000000LL, 0x8000000000000000LL};
-  movq(TMP, Immediate(reinterpret_cast<intptr_t>(&double_negate_constant)));
+  LoadImmediate(
+      TMP, Immediate(reinterpret_cast<intptr_t>(&double_negate_constant)), PP);
   xorpd(d, Address(TMP, 0));
 }
 
 
 void Assembler::DoubleAbs(XmmRegister reg) {
   static const struct ALIGN16 {
     uint64_t a;
     uint64_t b;
   } double_abs_constant =
       {0x7FFFFFFFFFFFFFFFLL, 0x7FFFFFFFFFFFFFFFLL};
-  movq(TMP, Immediate(reinterpret_cast<intptr_t>(&double_abs_constant)));
+  LoadImmediate(TMP,
+      Immediate(reinterpret_cast<intptr_t>(&double_abs_constant)), PP);
   andpd(reg, Address(TMP, 0));
 }
 
 
 void Assembler::Stop(const char* message) {
   int64_t message_address = reinterpret_cast<int64_t>(message);
   if (FLAG_print_stop_message) {
     pushq(TMP);  // Preserve TMP register.
     pushq(RDI);  // Preserve RDI register.
-    movq(RDI, Immediate(message_address));
+    LoadImmediate(RDI, Immediate(message_address), PP);
     call(&StubCode::PrintStopMessageLabel());
     popq(RDI);  // Restore RDI register.
     popq(TMP);  // Restore TMP register.
   } else {
     // Emit the lower half and the higher half of the message address as
     // immediate operands in the test rax instructions.
     testl(RAX, Immediate(Utils::Low32Bits(message_address)));
     testl(RAX, Immediate(Utils::High32Bits(message_address)));
   }
   // Emit the int3 instruction.
@@ skipping 52 matching lines @@
   LeaveFrameWithPP();
   ret();
   nop(4);
   ASSERT((buffer_.GetPosition() - start) == 13);
 }
 
 
 void Assembler::ReserveAlignedFrameSpace(intptr_t frame_space) {
   // Reserve space for arguments and align frame before entering
   // the C++ world.
-  AddImmediate(RSP, Immediate(-frame_space));
+  if (frame_space != 0) {
+    subq(RSP, Immediate(frame_space));
+  }
   if (OS::ActivationFrameAlignment() > 1) {
     andq(RSP, Immediate(~(OS::ActivationFrameAlignment() - 1)));
   }
 }
 
 
 // TODO(srdjan): Add XMM registers once they are used by the compiler.
 // Based on http://x86-64.org/documentation/abi.pdf Fig. 3.4
 static const intptr_t kNumberOfVolatileCpuRegisters = 9;
 static const Register volatile_cpu_registers[kNumberOfVolatileCpuRegisters] = {
@@ skipping 169 matching lines @@
   EnterFrame(0);
   pushq(Immediate(0));  // Push 0 in the saved PC area for stub frames.
   pushq(PP);  // Save caller's pool pointer
   LoadPoolPointer(PP);
 }
 
 
 void Assembler::TryAllocate(const Class& cls,
                             Label* failure,
                             bool near_jump,
-                            Register instance_reg) {
+                            Register instance_reg,
+                            Register pp) {
   ASSERT(failure != NULL);
   if (FLAG_inline_alloc) {
     Heap* heap = Isolate::Current()->heap();
     const intptr_t instance_size = cls.instance_size();
-    movq(TMP, Immediate(heap->TopAddress()));
+    LoadImmediate(TMP, Immediate(heap->TopAddress()), pp);
     movq(instance_reg, Address(TMP, 0));
-    addq(instance_reg, Immediate(instance_size));
+    AddImmediate(instance_reg, Immediate(instance_size), pp);
     // instance_reg: potential next object start.
-    movq(TMP, Immediate(heap->EndAddress()));
+    LoadImmediate(TMP, Immediate(heap->EndAddress()), pp);
     cmpq(instance_reg, Address(TMP, 0));
     j(ABOVE_EQUAL, failure, near_jump);
     // Successfully allocated the object, now update top to point to
     // next object start and store the class in the class field of object.
-    movq(TMP, Immediate(heap->TopAddress()));
+    LoadImmediate(TMP, Immediate(heap->TopAddress()), pp);
     movq(Address(TMP, 0), instance_reg);
     ASSERT(instance_size >= kHeapObjectTag);
-    subq(instance_reg, Immediate(instance_size - kHeapObjectTag));
+    AddImmediate(instance_reg, Immediate(kHeapObjectTag - instance_size), pp);
     uword tags = 0;
     tags = RawObject::SizeTag::update(instance_size, tags);
     ASSERT(cls.id() != kIllegalCid);
     tags = RawObject::ClassIdTag::update(cls.id(), tags);
-    movq(FieldAddress(instance_reg, Object::tags_offset()), Immediate(tags));
+    LoadImmediate(FieldAddress(instance_reg, Object::tags_offset()),
+                  Immediate(tags), pp);
   } else {
     jmp(failure);
   }
 }
 
 
 void Assembler::Align(int alignment, int offset) {
   ASSERT(Utils::IsPowerOfTwo(alignment));
   int pos = offset + buffer_.GetPosition();
   int mod = pos & (alignment - 1);
@@ skipping 179 matching lines @@
 
 
 const char* Assembler::FpuRegisterName(FpuRegister reg) {
   ASSERT((0 <= reg) && (reg < kNumberOfXmmRegisters));
   return xmm_reg_names[reg];
 }
 
 }  // namespace dart
 
 #endif  // defined TARGET_ARCH_X64