X64: Reenabled RSet.

src/x64/macro-assembler-x64.cc

Index: src/x64/macro-assembler-x64.cc
diff --git a/src/x64/macro-assembler-x64.cc b/src/x64/macro-assembler-x64.cc
index f58e1cdf9b6a95f702ace91a61db154a0ddcc061..93134362ddc6915ec472ab82382f5c2d39a507de 100644
--- a/src/x64/macro-assembler-x64.cc
+++ b/src/x64/macro-assembler-x64.cc
@@ -31,6 +31,7 @@
 #include "codegen-inl.h"
 #include "assembler-x64.h"
 #include "macro-assembler-x64.h"
+#include "serialize.h"
 #include "debug.h"
 
 namespace v8 {
@@ -45,11 +46,190 @@ MacroAssembler::MacroAssembler(void* buffer, int size)
 }
 
 
-// TODO(x64): For now, the write barrier is disabled on x64 and we
-// therefore generate no code.  This should be fixed when the write
-// barrier is enabled.
-void MacroAssembler::RecordWrite(Register object, int offset,
-                                 Register value, Register scratch) {
+
+static void RecordWriteHelper(MacroAssembler* masm,
+                              Register object,
+                              Register addr,
+                              Register scratch) {
+  Label fast;
+
+  // Compute the page address from the heap object pointer, leave it
+  // in 'object'.
+  ASSERT(is_int32(~Page::kPageAlignmentMask));
+  masm->and_(object,
+             Immediate(static_cast<int32_t>(~Page::kPageAlignmentMask)));
+
+  // Compute the bit addr in the remembered set, leave it in "addr".
+  masm->subq(addr, object);
+  masm->shr(addr, Immediate(kPointerSizeLog2));
+
+  // If the bit offset lies beyond the normal remembered set range, it is in
+  // the extra remembered set area of a large object.
+  masm->cmpq(addr, Immediate(Page::kPageSize / kPointerSize));
+  masm->j(less, &fast);
+
+  // Adjust 'addr' to be relative to the start of the extra remembered set
+  // and the page address in 'object' to be the address of the extra
+  // remembered set.
+  masm->subq(addr, Immediate(Page::kPageSize / kPointerSize));
+  // Load the array length into 'scratch'.
+  masm->movl(scratch,
+             Operand(object,
+                     Page::kObjectStartOffset + FixedArray::kLengthOffset));
+  // Extra remembered set starts right after FixedArray.
+  // Add the page header, array header, and array body size
+  // (length * pointer size) to the page address to find the extra remembered
+  // set start.
+  masm->lea(object,
+            Operand(object, scratch, times_pointer_size,
+                    Page::kObjectStartOffset + FixedArray::kHeaderSize));
+
+  // NOTE: For now, we use the bit-test-and-set (bts) x86 instruction
+  // to limit code size. We should probably evaluate this decision by
+  // measuring the performance of an equivalent implementation using
+  // "simpler" instructions
+  masm->bind(&fast);
+  masm->bts(Operand(object, Page::kRSetOffset), addr);
+}
+
+
+class RecordWriteStub : public CodeStub {
+ public:
+  RecordWriteStub(Register object, Register addr, Register scratch)
+      : object_(object), addr_(addr), scratch_(scratch) { }
+
+  void Generate(MacroAssembler* masm);
+
+ private:
+  Register object_;
+  Register addr_;
+  Register scratch_;
+
+#ifdef DEBUG
+  void Print() {
+    PrintF("RecordWriteStub (object reg %d), (addr reg %d), (scratch reg %d)\n",
+           object_.code(), addr_.code(), scratch_.code());
+  }
+#endif
+
+  // Minor key encoding in 12 bits of three registers (object, address and
+  // scratch) OOOOAAAASSSS.
+  class ScratchBits: public BitField<uint32_t, 0, 4> {};
+  class AddressBits: public BitField<uint32_t, 4, 4> {};
+  class ObjectBits: public BitField<uint32_t, 8, 4> {};
+
+  Major MajorKey() { return RecordWrite; }
+
+  int MinorKey() {
+    // Encode the registers.
+    return ObjectBits::encode(object_.code()) |
+           AddressBits::encode(addr_.code()) |
+           ScratchBits::encode(scratch_.code());
+  }
+};
+
+
+void RecordWriteStub::Generate(MacroAssembler* masm) {
+  RecordWriteHelper(masm, object_, addr_, scratch_);
+  masm->ret(0);
+}
+
+
+// Set the remembered set bit for [object+offset].
+// object is the object being stored into, value is the object being stored.
+// If offset is zero, then the scratch register contains the array index into
+// the elements array represented as a Smi.
+// All registers are clobbered by the operation.
+void MacroAssembler::RecordWrite(Register object,
+                                 int offset,
+                                 Register value,
+                                 Register scratch) {
+  // First, check if a remembered set write is even needed. The tests below
+  // catch stores of Smis and stores into young gen (which does not have space
+  // for the remembered set bits.
+  Label done;
+
+  // This optimization cannot survive serialization and deserialization,
+  // so we disable as long as serialization can take place.
+  intptr_t new_space_start =
+      reinterpret_cast<intptr_t>(
+          ExternalReference::new_space_start().address());
+  if (Serializer::enabled() || new_space_start < 0) {
+    // Cannot do smart bit-twiddling. Need to do two consecutive checks.
+    // Check for Smi first.
+    testl(value, Immediate(kSmiTagMask));
+    j(zero, &done);

[William Hesse, 2009/08/03 09:03:28]

Remove this smi test - we think it isn't worth it.

[Lasse Reichstein, 2009/08/03 10:45:45]

Done.

+    // Test that the object address is not in the new space.  We cannot
+    // set remembered set bits in the new space.
+    movq(value, object);
+    ASSERT(is_int32(static_cast<int64_t>(Heap::NewSpaceMask())));
+    and_(value, Immediate(static_cast<int32_t>(Heap::NewSpaceMask())));
+    movq(kScratchRegister, ExternalReference::new_space_start());
+    cmpq(value, kScratchRegister);
+    j(equal, &done);
+  } else {

[William Hesse, 2009/08/03 09:03:28]

This optimization is unsafe if object can be in the upper half of the address
space.  Remove it.

[Lasse Reichstein, 2009/08/03 10:45:45]

Done.

+    // move the value SmiTag into the sign bit
+    ASSERT(kSmiTagSize == 1);
+    ASSERT(kSmiTag == 0);
+    shl(value, Immediate(63));
+
+    // remove the uninteresting bits inside the page
+    movq(kScratchRegister, object);
+    ASSERT(is_int32(static_cast<int64_t>(Heap::NewSpaceMask())));
+    and_(kScratchRegister,
+         Immediate(static_cast<int32_t>(Heap::NewSpaceMask())));
+    // combine the object with value SmiTag
+    or_(value, kScratchRegister);
+    // xor has two effects:
+    // - if the value was a smi, then the result will be negative
+    // - if the object is pointing into new space area the page bits will
+    //   all be zero.
+    movq(kScratchRegister, new_space_start | (static_cast<int64_t>(1) << 63),
+         RelocInfo::NONE);
+    xor_(value, kScratchRegister);
+    // Check for both conditions in one branch
+    j(less_equal, &done);  // Jump if either zero or sign flag set.
+  }
+
+  if ((offset > 0) && (offset < Page::kMaxHeapObjectSize)) {
+    // Compute the bit offset in the remembered set, leave it in 'value'.
+    movq(value, object);
+    ASSERT(is_uint32(Page::kPageAlignmentMask));
+    and_(value, Immediate(static_cast<uint32_t>(Page::kPageAlignmentMask)));
+    addq(value, Immediate(offset));
+    shr(value, Immediate(kObjectAlignmentBits));
+
+    // Compute the page address from the heap object pointer, leave it in
+    // 'object' (immediate value is sign extended).
+    and_(object, Immediate(~Page::kPageAlignmentMask));
+
+    // NOTE: For now, we use the bit-test-and-set (bts) x86 instruction
+    // to limit code size. We should probably evaluate this decision by
+    // measuring the performance of an equivalent implementation using
+    // "simpler" instructions
+    bts(Operand(object, Page::kRSetOffset), value);
+  } else {
+    Register dst = scratch;
+    if (offset != 0) {
+      lea(dst, Operand(object, offset));
+    } else {
+      // array access: calculate the destination address in the same manner as
+      // KeyedStoreIC::GenerateGeneric.  Multiply a smi by 4 to get an offset
+      // into an array of words.
+      lea(dst, Operand(object, dst, times_half_pointer_size,
+                       FixedArray::kHeaderSize - kHeapObjectTag));
+    }
+    // If we are already generating a shared stub, not inlining the
+    // record write code isn't going to save us any memory.
+    if (generating_stub()) {
+      RecordWriteHelper(this, object, dst, value);
+    } else {
+      RecordWriteStub stub(object, dst, value);
+      CallStub(&stub);
+    }
+  }
+
+  bind(&done);
 }