ARM: Optimize stubs for incremental marking.

src/arm/macro-assembler-arm.cc

Index: src/arm/macro-assembler-arm.cc
diff --git a/src/arm/macro-assembler-arm.cc b/src/arm/macro-assembler-arm.cc
index 0aac220a66e64553308af9561bceb9d198f92119..112b90e960ab59f853eed7ad83b63d0e1b631858 100644
--- a/src/arm/macro-assembler-arm.cc
+++ b/src/arm/macro-assembler-arm.cc
@@ -492,12 +492,12 @@ void MacroAssembler::RecordWrite(Register object,
 
   CheckPageFlag(value,
                 value,  // Used as scratch.
-                MemoryChunk::POINTERS_TO_HERE_ARE_INTERESTING,
+                MemoryChunk::kPointersToHereAreInterestingMask,
                 eq,
                 &done);
   CheckPageFlag(object,
                 value,  // Used as scratch.
-                MemoryChunk::POINTERS_FROM_HERE_ARE_INTERESTING,
+                MemoryChunk::kPointersFromHereAreInterestingMask,
                 eq,
                 &done);
 
@@ -3200,12 +3200,12 @@ void MacroAssembler::GetRelocatedValueLocation(Register ldr_location,
 void MacroAssembler::CheckPageFlag(
     Register object,
     Register scratch,
-    MemoryChunk::MemoryChunkFlags flag,
+    int mask,
     Condition cc,
     Label* condition_met) {
   and_(scratch, object, Operand(~Page::kPageAlignmentMask));
   ldr(scratch, MemOperand(scratch, MemoryChunk::kFlagsOffset));
-  tst(scratch, Operand(1 << flag));
+  tst(scratch, Operand(mask));
   b(cc, condition_met);
 }
 
@@ -3283,6 +3283,104 @@ void MacroAssembler::GetMarkBits(Register addr_reg,
 }
 
 
+void MacroAssembler::EnsureNotWhite(
+    Register value,
+    Register bitmap_scratch,
+    Register mask_scratch,
+    Register load_scratch,
+    Label* value_is_white_and_not_data) {
+  ASSERT(!AreAliased(value, bitmap_scratch, mask_scratch, no_reg));
+  GetMarkBits(value, bitmap_scratch, mask_scratch);
+
+  // If the value is black or grey we don't need to do anything.
+  ASSERT(strcmp(Marking::kWhiteBitPattern, "00") == 0);
+  ASSERT(strcmp(Marking::kBlackBitPattern, "10") == 0);
+  ASSERT(strcmp(Marking::kGreyBitPattern, "11") == 0);
+  ASSERT(strcmp(Marking::kImpossibleBitPattern, "01") == 0);
+
+  Label done;
+
+  // Since both black and grey have a 1 in the first position and white does
+  // not have a 1 there we only need to check one bit.
+  ldr(load_scratch, MemOperand(bitmap_scratch, MemoryChunk::kHeaderSize));
+  tst(mask_scratch, load_scratch);
+  b(ne, &done);
+
+  if (FLAG_debug_code) {
+    // Check for impossible bit pattern.
+    Label ok;
+    // LSL may overflow, making the check conservative.
+    tst(load_scratch, Operand(mask_scratch, LSL, 1));
+    b(eq, &ok);
+    stop("Impossible marking bit pattern");
+    bind(&ok);
+  }
+
+  // Value is white.  We check whether it is data that doesn't need scanning.
+  // Currently only checks for HeapNumber and non-cons strings.
+  Register map = load_scratch;  // Holds map while checking type.
+  Register length = load_scratch;  // Holds length of object after testing type.
+  Label is_data_object;
+
+  // Check for heap-number
+  ldr(map, FieldMemOperand(value, HeapObject::kMapOffset));
+  cmp(map, Operand(FACTORY->heap_number_map()));

[Erik Corry, 2011/08/16 09:29:23]

Use CompareRoot here.  It saves an entry in the constant pool and reduces the
amount of relocation info associated with the stub.

[Michael Starzinger, 2011/08/16 15:31:05]

Done.

+  mov(length, Operand(HeapNumber::kSize), LeaveCC, eq);
+  b(eq, &is_data_object);
+
+  // Check for strings.
+  ASSERT(kConsStringTag == 1 && kIsConsStringMask == 1);
+  ASSERT(kNotStringTag == 0x80 && kIsNotStringMask == 0x80);
+  // If it's a string and it's not a cons string then it's an object containing
+  // no GC pointers.
+  Register instance_type = load_scratch;
+  ldrb(instance_type, FieldMemOperand(map, Map::kInstanceTypeOffset));
+  tst(instance_type, Operand(kIsConsStringMask | kIsNotStringMask));
+  b(ne, value_is_white_and_not_data);
+  // It's a non-cons string.
+  // If it's external, the length is just ExternalString::kSize.
+  // Otherwise it's String::kHeaderSize + string->length() * (1 or 2).
+  // External strings are the only ones with the kExternalStringTag bit
+  // set.
+  ASSERT_EQ(0, kSeqStringTag & kExternalStringTag);
+  ASSERT_EQ(0, kConsStringTag & kExternalStringTag);
+  tst(instance_type, Operand(kExternalStringTag));
+  mov(length, Operand(ExternalString::kSize), LeaveCC, ne);
+  b(ne, &is_data_object);
+
+  // Sequential string, either ASCII or UC16.
+  ASSERT(kAsciiStringTag == 0x04);
+  and_(instance_type, instance_type, Operand(kStringEncodingMask));

[Erik Corry, 2011/08/16 09:29:23]

This code goes to great lengths to avoid a hard-to-predict branch on Intel.  I
think on ARM it would be simpler to do a tst on the relevant bit and then a
conditional shift instruction.

[Michael Starzinger, 2011/08/16 15:31:05]

Done. Simplified this part of the stub a lot. It now just checks for the string
encoding bit and then conditionally shifts the (char-)length value to get the
byte-size. Thanks for the great hint!

+  eor(instance_type, instance_type, Operand(kStringEncodingMask));
+  add(instance_type, instance_type, Operand(0x04));
+  // Value now either 4 (if ASCII) or 8 (if UC16), i.e., char-size shifted
+  // by 2. If we multiply the string length as smi by this, it still
+  // won't overflow a 32-bit value.
+  ASSERT_EQ(SeqAsciiString::kMaxSize, SeqTwoByteString::kMaxSize);
+  ASSERT(SeqAsciiString::kMaxSize <=
+         static_cast<int>(0xffffffffu >> (2 + kSmiTagSize)));
+  ldr(ip, FieldMemOperand(value, String::kLengthOffset));
+  mul(length, instance_type, ip);
+  mov(length, Operand(length, LSR, 2 + kSmiTagSize));

[Erik Corry, 2011/08/16 09:29:23]

Why are we dividing by 4 here?

[Michael Starzinger, 2011/08/16 15:31:05]

Done. This one is gone now due to the above simplification. The SHR by 2 was in
place because char size was SHL by 2 in the above code. For ASCII
(instance_type==4) holds, for UC16 (instance_type==8) holds. The IA32 code is
very complex at that point and I subconsciously duplicated that complexity.
Should be much simpler now.

+  add(length, length, Operand(SeqString::kHeaderSize + kObjectAlignmentMask));
+  and_(length, length, Operand(~kObjectAlignmentMask));
+
+  bind(&is_data_object);
+  // Value is a data object, and it is white.  Mark it black.  Since we know
+  // that the object is white we can make it black by flipping one bit.
+  ldr(ip, MemOperand(bitmap_scratch, MemoryChunk::kHeaderSize));
+  orr(ip, ip, Operand(mask_scratch));
+  str(ip, MemOperand(bitmap_scratch, MemoryChunk::kHeaderSize));
+
+  and_(bitmap_scratch, bitmap_scratch, Operand(~Page::kPageAlignmentMask));
+  ldr(ip, MemOperand(bitmap_scratch, MemoryChunk::kLiveBytesOffset));
+  add(ip, ip, Operand(length));
+  str(ip, MemOperand(bitmap_scratch, MemoryChunk::kLiveBytesOffset));
+
+  bind(&done);
+}
+
+
 void MacroAssembler::ClampUint8(Register output_reg, Register input_reg) {
   Usat(output_reg, 8, Operand(input_reg));
 }