[heap] Black allocation.

src/heap/incremental-marking.cc

Index: src/heap/incremental-marking.cc
diff --git a/src/heap/incremental-marking.cc b/src/heap/incremental-marking.cc
index 47e1e12c77b6fbcd47f362a337edd583ecccaca4..e1f439f8b49567dc3e12284de1f8510f5e1dc61c 100644
--- a/src/heap/incremental-marking.cc
+++ b/src/heap/incremental-marking.cc
@@ -23,7 +23,6 @@ IncrementalMarking::StepActions IncrementalMarking::IdleStepActions() {
                      IncrementalMarking::DO_NOT_FORCE_COMPLETION);
 }
 
-
 IncrementalMarking::IncrementalMarking(Heap* heap)
     : heap_(heap),
       observer_(*this, kAllocatedThreshold),
@@ -42,11 +41,11 @@ IncrementalMarking::IncrementalMarking(Heap* heap)
       no_marking_scope_depth_(0),
       unscanned_bytes_of_large_object_(0),
       was_activated_(false),
+      black_allocation_(false),
       finalize_marking_completed_(false),
       incremental_marking_finalization_rounds_(0),
       request_type_(COMPLETE_MARKING) {}
 
-
 bool IncrementalMarking::BaseRecordWrite(HeapObject* obj, Object* value) {
   HeapObject* value_heap_obj = HeapObject::cast(value);
   MarkBit value_bit = Marking::MarkBitFrom(value_heap_obj);
@@ -324,6 +323,9 @@ class IncrementalMarkingMarkingVisitor
   }
 };
 
+void IncrementalMarking::IterateBlackCode(Code* code) {
+  Code::BodyDescriptor::IterateBody<IncrementalMarkingMarkingVisitor>(code);
+}
 
 class IncrementalMarkingRootMarkingVisitor : public ObjectVisitor {
  public:
@@ -598,12 +600,29 @@ void IncrementalMarking::StartMarking() {
   IncrementalMarkingRootMarkingVisitor visitor(this);
   heap_->IterateStrongRoots(&visitor, VISIT_ONLY_STRONG);
 
+  if (FLAG_black_allocation) {
+    StartBlackAllocation();

[Michael Lippautz, 2016/02/04 17:59:50]

Did you try to enable it later than that?

[Hannes Payer (out of office), 2016/02/06 08:50:16]

Nope, but it is possible to enable from start of marking time until finalization
of marking. We can play with that heuristic later.

+  }
+
   // Ready to start incremental marking.
   if (FLAG_trace_incremental_marking) {
     PrintF("[IncrementalMarking] Running\n");
   }
 }
 
+void IncrementalMarking::StartBlackAllocation() {
+  DCHECK(FLAG_black_allocation);
+  DCHECK(IsMarking());
+  black_allocation_ = true;
+  PagedSpaces spaces(heap());
+  for (PagedSpace* space = spaces.next(); space != NULL;
+       space = spaces.next()) {
+    space->EmptyAllocationInfo();
+    space->free_list()->Reset();
+  }
+}
+
+void IncrementalMarking::FinishBlackAllocation() { black_allocation_ = false; }
 
 void IncrementalMarking::MarkRoots() {
   DCHECK(!finalize_marking_completed_);
@@ -800,6 +819,14 @@ void IncrementalMarking::UpdateMarkingDequeAfterScavenge() {
       MapWord map_word = obj->map_word();
       if (map_word.IsForwardingAddress()) {
         HeapObject* dest = map_word.ToForwardingAddress();
+        if (Page::FromAddress(dest->address())->IsFlagSet(Page::BLACK_PAGE))
+          continue;
+        // If the color of the source object was not transitioned, fix that
+        // now.
+        MarkBit mark_bit_dest = Marking::MarkBitFrom(dest);

[Michael Lippautz, 2016/02/04 17:59:50]

Please provide more context on this fixup.

[Hannes Payer (out of office), 2016/02/06 08:50:16]

When I follow the forwarding pointer while processing the promoted objects, I
may be able to avoid that case here.

+        if (Marking::IsWhite(mark_bit_dest)) {
+          Marking::WhiteToGrey(mark_bit_dest);
+        }
         array[new_top] = dest;
         new_top = ((new_top + 1) & mask);
         DCHECK(new_top != marking_deque->bottom());
@@ -894,7 +921,12 @@ void IncrementalMarking::ProcessMarkingDeque() {
 
 
 void IncrementalMarking::Hurry() {
-  if (state() == MARKING) {
+  // A scavenge may have pushed new object on the marking deque (due to black
+  // allocation) even in a completed state. In that case we have to process the
+  // marking deque.
+  if (state() == MARKING ||

[Michael Lippautz, 2016/02/04 17:59:51]

Scary...

[Hannes Payer (out of office), 2016/02/06 08:50:16]

Simplified the check. COMPLETE and non-empty marking deque may only happen if
scavenges are forced from cctests after completing incremental marking.

+      (state() == COMPLETE &&
+       !heap_->mark_compact_collector()->marking_deque()->IsEmpty())) {
     double start = 0.0;
     if (FLAG_trace_incremental_marking || FLAG_print_cumulative_gc_stat) {
       start = heap_->MonotonicallyIncreasingTimeInMs();
@@ -959,6 +991,7 @@ void IncrementalMarking::Stop() {
   heap_->isolate()->stack_guard()->ClearGC();
   state_ = STOPPED;
   is_compacting_ = false;
+  black_allocation_ = false;
 }