x64 port of r42717.

runtime/vm/assembler_x64.cc

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/globals.h"  // NOLINT
 #if defined(TARGET_ARCH_X64)
 
 #include "vm/assembler.h"
 #include "vm/cpu.h"
 #include "vm/heap.h"
@@ skipping 1583 matching lines @@
     } else {
       EmitRegisterREX(reg, REX_NONE);
       EmitUint8(0xF7);
       EmitUint8(0xC0 | (reg & 7));
     }
     EmitImmediate(imm);
   }
 }
 
 
+void Assembler::testb(const Address& address, const Immediate& imm) {
+  AssemblerBuffer::EnsureCapacity ensured(&buffer_);
+  EmitOperandREX(0, address, REX_NONE);
+  EmitUint8(0xF6);
+  EmitOperand(0, address);
+  ASSERT(imm.is_int8());
+  EmitUint8(imm.value() & 0xFF);
+}
+
+
 void Assembler::testq(Register reg1, Register reg2) {
   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
   Operand operand(reg2);
   EmitOperandREX(reg1, operand, REX_W);
   EmitUint8(0x85);
   EmitOperand(reg1 & 7, operand);
 }
 
 
 void Assembler::testq(Register reg, const Immediate& imm) {
@@ skipping 1368 matching lines @@
   // Mask out higher, uninteresting bits which were polluted by dest.
   andl(value, Immediate(kObjectAlignment - 1));
   // Compare with the expected bit pattern.
   cmpl(value, Immediate(
       (kNewObjectAlignmentOffset >> 1) + kHeapObjectTag +
       kOldObjectAlignmentOffset + kHeapObjectTag));
   j(NOT_ZERO, no_update, Assembler::kNearJump);
 }
 
 
-void Assembler::VerifyHeapWord(const Address& address) {
+void Assembler::VerifyHeapWord(const Address& address,
+                               FieldContent old_content) {
+#if defined(DEBUG)
+  switch (old_content) {
+    case kEmptyOrSmiOrNull:
+      VerifyUninitialized(address);
+      break;
+    case kHeapObjectOrSmi:
+      VerifyObjectOrSmi(address);
+      break;
+    case kOnlySmi:
+      VerifySmi(address);
+      break;
+  }
+#endif  // DEBUG
   if (VerifiedMemory::enabled()) {
     Register addr_reg = RDX;
     Register value = RBX;
     // Preserve registers.
     pushq(addr_reg);
     pushq(value);
     leaq(addr_reg, address);
     // ASSERT(*address == *(address + offset))
     movq(value, Address(addr_reg, 0));
     cmpq(value, Address(addr_reg, VerifiedMemory::offset()));
     Label ok;
     j(EQUAL, &ok);
     static const bool kFixedLengthEncoding = true;
     Stop("Write barrier verification failed", kFixedLengthEncoding);
     Bind(&ok);
     popq(value);
     popq(addr_reg);
   }
 }
 
 
-void Assembler::VerifiedWrite(const Address& dest, Register value) {
-  VerifyHeapWord(dest);
+void Assembler::VerifiedWrite(const Address& dest,
+                              Register value,
+                              FieldContent old_content) {
+  VerifyHeapWord(dest, old_content);
   movq(dest, value);
   if (VerifiedMemory::enabled()) {
     Register temp = (value == RDX) ? RCX : RDX;
     pushq(temp);
     leaq(temp, dest);
     movq(Address(temp, VerifiedMemory::offset()), value);
     popq(temp);
   }
 }
 
 
+#if defined(DEBUG)
+void Assembler::VerifyObjectOrSmi(const Address& dest) {
+  Label ok;
+  testb(dest, Immediate(kHeapObjectTag));
+  j(ZERO, &ok, Assembler::kNearJump);
+  // Non-smi case: Verify object pointer is word-aligned when untagged.
+  COMPILE_ASSERT(kHeapObjectTag == 1);
+  testb(dest, Immediate((kWordSize - 1) - kHeapObjectTag));
+  j(ZERO, &ok, Assembler::kNearJump);
+  Stop("Expected heap object or Smi");
+  Bind(&ok);
+}
+
+
+void Assembler::VerifyUninitialized(const Address& dest) {

[Ivan Posva, 2015/01/09 00:16:56]

Missed this in the original review:

#if defined(DEBUG)
  ...
#else
#error Only supported in DEBUG.
#endif  // defined(DEBUG)

Since this code will not work unless it is called in DEBUG due to unzapped
values.

[koda, 2015/01/09 02:02:34]

Done.

+  Label ok;
+  testb(dest, Immediate(kHeapObjectTag));
+  j(ZERO, &ok, Assembler::kNearJump);
+  // Non-smi case: Check for the special zap word or null.
+  cmpq(dest, Immediate(Heap::kZap64Bits));
+  j(EQUAL, &ok, Assembler::kNearJump);
+  cmpq(dest, Immediate(reinterpret_cast<uint64_t>(Object::null())));
+  j(EQUAL, &ok, Assembler::kNearJump);
+  Stop("Expected zapped, Smi or null");
+  Bind(&ok);
+}
+
+
+void Assembler::VerifySmi(const Address& dest, const char* stop_msg) {
+  Label done;
+  testb(dest, Immediate(kHeapObjectTag));
+  j(ZERO, &done, Assembler::kNearJump);
+  Stop(stop_msg);
+  Bind(&done);
+}
+#endif  // defined(DEBUG)
+
+
 void Assembler::StoreIntoObject(Register object,
                                 const Address& dest,
                                 Register value,
                                 bool can_value_be_smi) {
   ASSERT(object != value);
-  VerifiedWrite(dest, value);
+  VerifiedWrite(dest, value, kHeapObjectOrSmi);
   Label done;
   if (can_value_be_smi) {
     StoreIntoObjectFilter(object, value, &done);
   } else {
     StoreIntoObjectFilterNoSmi(object, value, &done);
   }
   // A store buffer update is required.
   if (value != RDX) pushq(RDX);
   if (object != RDX) {
     movq(RDX, object);
   }
   StubCode* stub_code = Isolate::Current()->stub_code();
   Call(&stub_code->UpdateStoreBufferLabel(), PP);
   if (value != RDX) popq(RDX);
   Bind(&done);
 }
 
 
 void Assembler::StoreIntoObjectNoBarrier(Register object,
                                          const Address& dest,
-                                         Register value) {
-  VerifiedWrite(dest, value);
+                                         Register value,
+                                         FieldContent old_content) {
+  VerifiedWrite(dest, value, old_content);
 #if defined(DEBUG)
   Label done;
   pushq(value);
   StoreIntoObjectFilter(object, value, &done);
   Stop("Store buffer update is required");
   Bind(&done);
   popq(value);
 #endif  // defined(DEBUG)
   // No store buffer update.
 }
 
 
 void Assembler::StoreIntoObjectNoBarrier(Register object,
                                          const Address& dest,
                                          const Object& value,
-                                         Register pp) {
+                                         Register pp,
+                                         FieldContent old_content) {
+  VerifyHeapWord(dest, old_content);
   if (VerifiedMemory::enabled()) {
-    VerifyHeapWord(dest);
     Register temp = (pp == RCX) ? RDX : RCX;
     pushq(temp);
     leaq(temp, dest);
     StoreObject(Address(temp, 0), value, pp);
     StoreObject(Address(temp, VerifiedMemory::offset()), value, pp);
     popq(temp);
   } else {
     StoreObject(dest, value, pp);
   }
   // TODO(koda): Use 'object', verify that generational barrier's not needed.
 }
 
 
 void Assembler::StoreIntoSmiField(const Address& dest, Register value) {
-  VerifiedWrite(dest, value);
 #if defined(DEBUG)
   Label done;
   testq(value, Immediate(kHeapObjectTag));
   j(ZERO, &done);
-  Stop("Smi expected");
+  Stop("New value must be Smi.");
   Bind(&done);
 #endif  // defined(DEBUG)
+  VerifiedWrite(dest, value, kOnlySmi);
 }
 
 
 void Assembler::ZeroInitSmiField(const Address& dest) {
   // TODO(koda): Add VerifySmi once we distinguish initalization.
-  VerifyHeapWord(dest);
+  VerifyHeapWord(dest, kEmptyOrSmiOrNull);
   Immediate zero(Smi::RawValue(0));
   movq(dest, zero);
   if (VerifiedMemory::enabled()) {
     Register temp = RCX;
     pushq(temp);
     leaq(temp, dest);
     movq(Address(temp, VerifiedMemory::offset()), zero);
     popq(temp);
   }
 }
 
 
 void Assembler::IncrementSmiField(const Address& dest, int64_t increment) {
   // Note: FlowGraphCompiler::EdgeCounterIncrementSizeInBytes depends on
   // the length of this instruction sequence.
   // TODO(koda): Add VerifySmi once we distinguish initalization.
-  VerifyHeapWord(dest);
+  VerifyHeapWord(dest, kOnlySmi);
   Immediate inc_imm(Smi::RawValue(increment));
   addq(dest, inc_imm);
   if (VerifiedMemory::enabled()) {
     Register temp = RCX;
     pushq(temp);
     leaq(temp, dest);
     addq(Address(temp, VerifiedMemory::offset()), inc_imm);
     popq(temp);
   }
 }
@@ skipping 801 matching lines @@
 
 
 const char* Assembler::FpuRegisterName(FpuRegister reg) {
   ASSERT((0 <= reg) && (reg < kNumberOfXmmRegisters));
   return xmm_reg_names[reg];
 }
 
 }  // namespace dart
 
 #endif  // defined TARGET_ARCH_X64