Support verified heap pointer writes on x64.

runtime/vm/assembler_x64.cc

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/globals.h"
 #if defined(TARGET_ARCH_X64)
 
 #include "vm/assembler.h"
 #include "vm/cpu.h"
 #include "vm/heap.h"
@@ skipping 2366 matching lines @@
 
 
 void Assembler::hlt() {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   EmitUint8(0xF4);
 }
 
 
 void Assembler::j(Condition condition, Label* label, bool near) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
+  if (VerifiedMemory::enabled()) {
+    near = Assembler::kFarJump;
+  }
   if (label->IsBound()) {
     static const int kShortSize = 2;
     static const int kLongSize = 6;
     intptr_t offset = label->Position() - buffer_.Size();
     ASSERT(offset <= 0);
     if (Utils::IsInt(8, offset - kShortSize)) {
       EmitUint8(0x70 + condition);
       EmitUint8((offset - kShortSize) & 0xFF);
     } else {
       EmitUint8(0x0F);
@@ skipping 32 matching lines @@
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   Operand operand(reg);
   EmitOperandREX(4, operand, REX_NONE);
   EmitUint8(0xFF);
   EmitOperand(4, operand);
 }
 
 
 void Assembler::jmp(Label* label, bool near) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
+  if (VerifiedMemory::enabled()) {
+    near = Assembler::kFarJump;
+  }
   if (label->IsBound()) {
     static const int kShortSize = 2;
     static const int kLongSize = 5;
     intptr_t offset = label->Position() - buffer_.Size();
     ASSERT(offset <= 0);
     if (Utils::IsInt(8, offset - kShortSize)) {
       EmitUint8(0xEB);
       EmitUint8((offset - kShortSize) & 0xFF);
     } else {
       EmitUint8(0xE9);
@@ skipping 415 matching lines @@
   // Mask out higher, uninteresting bits which were polluted by dest.
   andl(value, Immediate(kObjectAlignment - 1));
   // Compare with the expected bit pattern.
   cmpl(value, Immediate(
       (kNewObjectAlignmentOffset >> 1) + kHeapObjectTag +
       kOldObjectAlignmentOffset + kHeapObjectTag));
   j(NOT_ZERO, no_update, Assembler::kNearJump);
 }
 
 
+void Assembler::Verify(const Address& dest) {

[Ivan Posva, 2014/11/14 22:16:06]

ditto about the name.

[koda, 2014/11/17 17:08:59]

Done, matching ia32.

+  if (VerifiedMemory::enabled()) {
+    Register dest_addr = RDX;
+    Register old_value = RBX;
+    pushq(dest_addr);
+    pushq(old_value);
+    leaq(dest_addr, dest);
+    movq(old_value, Address(dest_addr, 0));
+    cmpq(old_value, Address(dest_addr, VerifiedMemory::offset()));
+    Label ok;
+    j(EQUAL, &ok, Assembler::kNearJump);

[Ivan Posva, 2014/11/14 22:16:07]

Will be a forced far jump.

[koda, 2014/11/17 17:09:00]

Good point; removed hint.

+    Stop("Write barrier verification failed");
+    Bind(&ok);
+    popq(old_value);
+    popq(dest_addr);
+  }
+}
+
+
+void Assembler::VerifiedWrite(const Address& dest, Register value) {
+  Verify(dest);
+  movq(dest, value);
+  if (VerifiedMemory::enabled()) {
+    Register temp = (value == RDX) ? RCX : RDX;
+    pushq(temp);
+    leaq(temp, dest);
+    movq(Address(temp, VerifiedMemory::offset()), value);
+    popq(temp);
+  }
+}
+
+
 void Assembler::StoreIntoObject(Register object,
                                 const Address& dest,
                                 Register value,
                                 bool can_value_be_smi) {
   ASSERT(object != value);
-  movq(dest, value);
+  VerifiedWrite(dest, value);
   Label done;
   if (can_value_be_smi) {
     StoreIntoObjectFilter(object, value, &done);
   } else {
     StoreIntoObjectFilterNoSmi(object, value, &done);
   }
   // A store buffer update is required.
   if (value != RAX) pushq(RAX);
   if (object != RAX) {
     movq(RAX, object);
   }
   StubCode* stub_code = Isolate::Current()->stub_code();
   Call(&stub_code->UpdateStoreBufferLabel(), PP);
   if (value != RAX) popq(RAX);
   Bind(&done);
 }
 
 
 void Assembler::StoreIntoObjectNoBarrier(Register object,
                                          const Address& dest,
                                          Register value) {
-  movq(dest, value);
+  VerifiedWrite(dest, value);
 #if defined(DEBUG)
   Label done;
   pushq(value);
   StoreIntoObjectFilter(object, value, &done);
   Stop("Store buffer update is required");
   Bind(&done);
   popq(value);
 #endif  // defined(DEBUG)
   // No store buffer update.
 }
 
 
+void Assembler::StoreIntoSmiField(const Address& dest, Register value) {
+  VerifiedWrite(dest, value);
+#if defined(DEBUG)
+  Label done;
+  testq(value, Immediate(kHeapObjectTag));
+  j(ZERO, &done);
+  Stop("Smi expected");
+  Bind(&done);
+#endif  // defined(DEBUG)
+}
+
+
+void Assembler::ZeroSmiField(const Address& dest) {
+  Verify(dest);  // TODO(koda): Add VerifySmi once we distinguish initalization.
+  Immediate zero(Smi::RawValue(0));
+  movq(dest, zero);
+  if (VerifiedMemory::enabled()) {
+    Register temp = RCX;
+    pushq(temp);
+    leaq(temp, dest);
+    movq(Address(temp, VerifiedMemory::offset()), zero);
+    popq(temp);
+  }
+}
+
+
+void Assembler::IncrementSmiField(const Address& dest, int64_t increment) {
+  // Note: FlowGraphCompiler::EdgeCounterIncrementSizeInBytes depends on
+  // the length of this instruction sequence.
+  Verify(dest);  // TODO(koda): Add VerifySmi once we distinguish initalization.
+  Immediate inc_imm(Smi::RawValue(increment));
+  addq(dest, inc_imm);
+  if (VerifiedMemory::enabled()) {
+    Register temp = RCX;
+    pushq(temp);
+    leaq(temp, dest);
+    addq(Address(temp, VerifiedMemory::offset()), inc_imm);
+    popq(temp);
+  }
+}
+
+
 void Assembler::DoubleNegate(XmmRegister d) {
   static const struct ALIGN16 {
     uint64_t a;
     uint64_t b;
   } double_negate_constant =
       {0x8000000000000000LL, 0x8000000000000000LL};
   LoadImmediate(
       TMP, Immediate(reinterpret_cast<intptr_t>(&double_negate_constant)), PP);
   xorpd(d, Address(TMP, 0));
 }
@@ skipping 739 matching lines @@
 
 
 const char* Assembler::FpuRegisterName(FpuRegister reg) {
   ASSERT((0 <= reg) && (reg < kNumberOfXmmRegisters));
   return xmm_reg_names[reg];
 }
 
 }  // namespace dart
 
 #endif  // defined TARGET_ARCH_X64