[heap] Black allocation.

src/heap/incremental-marking.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/incremental-marking.h"
 
 #include "src/code-stubs.h"
 #include "src/compilation-cache.h"
 #include "src/conversions.h"
 #include "src/heap/gc-idle-time-handler.h"
 #include "src/heap/gc-tracer.h"
 #include "src/heap/mark-compact-inl.h"
 #include "src/heap/objects-visiting.h"
 #include "src/heap/objects-visiting-inl.h"
 #include "src/v8.h"
 
 namespace v8 {
 namespace internal {
 
 IncrementalMarking::StepActions IncrementalMarking::IdleStepActions() {
   return StepActions(IncrementalMarking::NO_GC_VIA_STACK_GUARD,
                      IncrementalMarking::FORCE_MARKING,
                      IncrementalMarking::DO_NOT_FORCE_COMPLETION);
 }
 
-
 IncrementalMarking::IncrementalMarking(Heap* heap)
     : heap_(heap),
       observer_(*this, kAllocatedThreshold),
       state_(STOPPED),
       is_compacting_(false),
       steps_count_(0),
       old_generation_space_available_at_start_of_incremental_(0),
       old_generation_space_used_at_start_of_incremental_(0),
       bytes_rescanned_(0),
       should_hurry_(false),
       marking_speed_(0),
       bytes_scanned_(0),
       allocated_(0),
       write_barriers_invoked_since_last_step_(0),
       idle_marking_delay_counter_(0),
       no_marking_scope_depth_(0),
       unscanned_bytes_of_large_object_(0),
       was_activated_(false),
+      black_allocation_(false),
       finalize_marking_completed_(false),
       incremental_marking_finalization_rounds_(0),
       request_type_(COMPLETE_MARKING) {}
 
-
 bool IncrementalMarking::BaseRecordWrite(HeapObject* obj, Object* value) {
   HeapObject* value_heap_obj = HeapObject::cast(value);
   MarkBit value_bit = Marking::MarkBitFrom(value_heap_obj);
   DCHECK(!Marking::IsImpossible(value_bit));
 
   MarkBit obj_bit = Marking::MarkBitFrom(obj);
   DCHECK(!Marking::IsImpossible(obj_bit));
   bool is_black = Marking::IsBlack(obj_bit);
 
   if (is_black && Marking::IsWhite(value_bit)) {
@@ skipping 257 matching lines @@
     MarkBit mark_bit = Marking::MarkBitFrom(heap_object);
     if (Marking::IsWhite(mark_bit)) {
       Marking::MarkBlack(mark_bit);
       MemoryChunk::IncrementLiveBytesFromGC(heap_object, heap_object->Size());
       return true;
     }
     return false;
   }
 };
 
+void IncrementalMarking::IterateBlackObject(HeapObject* object) {
+  if (black_allocation() &&
+      Page::FromAddress(object->address())->IsFlagSet(Page::BLACK_PAGE)) {
+    IncrementalMarkingMarkingVisitor::IterateBody(object->map(), object);
+  }
+}
 
 class IncrementalMarkingRootMarkingVisitor : public ObjectVisitor {
  public:
   explicit IncrementalMarkingRootMarkingVisitor(
       IncrementalMarking* incremental_marking)
       : heap_(incremental_marking->heap()) {}
 
   void VisitPointer(Object** p) override { MarkObjectByPointer(p); }
 
   void VisitPointers(Object** start, Object** end) override {
@@ skipping 253 matching lines @@
   if (FLAG_cleanup_code_caches_at_gc) {
     // We will mark cache black with a separate pass
     // when we finish marking.
     MarkObjectGreyDoNotEnqueue(heap_->polymorphic_code_cache());
   }
 
   // Mark strong roots grey.
   IncrementalMarkingRootMarkingVisitor visitor(this);
   heap_->IterateStrongRoots(&visitor, VISIT_ONLY_STRONG);
 
+  if (FLAG_black_allocation) {
+    StartBlackAllocation();
+  }
+
   // Ready to start incremental marking.
   if (FLAG_trace_incremental_marking) {
     PrintF("[IncrementalMarking] Running\n");
   }
 }
 
+void IncrementalMarking::StartBlackAllocation() {
+  if (heap_->isolate()->serializer_enabled()) {
+    if (FLAG_trace_incremental_marking) {
+      PrintF("[IncrementalMarking] Black allocation delayed - serializer\n");
+    }
+    return;
+  }
+  DCHECK(FLAG_black_allocation);
+  DCHECK(IsMarking());
+  black_allocation_ = true;
+  PagedSpaces spaces(heap());
+  for (PagedSpace* space = spaces.next(); space != NULL;
+       space = spaces.next()) {
+    space->EmptyAllocationInfo();
+    space->free_list()->Reset();
+  }
+  if (FLAG_trace_incremental_marking) {
+    PrintF("[IncrementalMarking] Black allocation started\n");
+  }
+}
+
+void IncrementalMarking::FinishBlackAllocation() {
+  black_allocation_ = false;
+  if (FLAG_trace_incremental_marking) {
+    PrintF("[IncrementalMarking] Black allocation finished\n");
+  }
+}
 
 void IncrementalMarking::MarkRoots() {
   DCHECK(!finalize_marking_completed_);
   DCHECK(IsMarking());
 
   IncrementalMarkingRootMarkingVisitor visitor(this);
   heap_->IterateStrongRoots(&visitor, VISIT_ONLY_STRONG);
 }
 
 
@@ skipping 182 matching lines @@
     // Only pointers to from space have to be updated.
     if (heap_->InFromSpace(obj)) {
       MapWord map_word = obj->map_word();
       // There may be objects on the marking deque that do not exist anymore,
       // e.g. left trimmed objects or objects from the root set (frames).
       // If these object are dead at scavenging time, their marking deque
       // entries will not point to forwarding addresses. Hence, we can discard
       // them.
       if (map_word.IsForwardingAddress()) {
         HeapObject* dest = map_word.ToForwardingAddress();
+        if (Page::FromAddress(dest->address())->IsFlagSet(Page::BLACK_PAGE))
+          continue;
         array[new_top] = dest;
         new_top = ((new_top + 1) & mask);
         DCHECK(new_top != marking_deque->bottom());
 #ifdef DEBUG
         MarkBit mark_bit = Marking::MarkBitFrom(obj);
         DCHECK(Marking::IsGrey(mark_bit) ||
                (obj->IsFiller() && Marking::IsWhite(mark_bit)));
 #endif
       }
     } else if (obj->map() != filler_map) {
@@ skipping 74 matching lines @@
     // correct only for objects that occupy at least two words.
     Map* map = obj->map();
     if (map == filler_map) continue;
 
     VisitObject(map, obj, obj->SizeFromMap(map));
   }
 }
 
 
 void IncrementalMarking::Hurry() {
-  if (state() == MARKING) {
+  // A scavenge may have pushed new objects on the marking deque (due to black
+  // allocation) even in COMPLETE state. This may happen if scavenges are
+  // forced e.g. in tests. It should not happen when COMPLETE was set when
+  // incremental marking finished and a regular GC was triggered after that
+  // because should_hurry_ will force a full GC.
+  if (!heap_->mark_compact_collector()->marking_deque()->IsEmpty()) {
     double start = 0.0;
     if (FLAG_trace_incremental_marking || FLAG_print_cumulative_gc_stat) {
       start = heap_->MonotonicallyIncreasingTimeInMs();
       if (FLAG_trace_incremental_marking) {
         PrintF("[IncrementalMarking] Hurry\n");
       }
     }
     // TODO(gc) hurry can mark objects it encounters black as mutator
     // was stopped.
     ProcessMarkingDeque();
@@ skipping 44 matching lines @@
   IncrementalMarking::set_should_hurry(false);
   ResetStepCounters();
   if (IsMarking()) {
     PatchIncrementalMarkingRecordWriteStubs(heap_,
                                             RecordWriteStub::STORE_BUFFER_ONLY);
     DeactivateIncrementalWriteBarrier();
   }
   heap_->isolate()->stack_guard()->ClearGC();
   state_ = STOPPED;
   is_compacting_ = false;
+  FinishBlackAllocation();
 }
 
 
 void IncrementalMarking::Finalize() {
   Hurry();
   Stop();
 }
 
 
 void IncrementalMarking::FinalizeMarking(CompletionAction action) {
@@ skipping 198 matching lines @@
       if (heap_->mark_compact_collector()->sweeping_in_progress() &&
           (heap_->mark_compact_collector()->IsSweepingCompleted() ||
            !FLAG_concurrent_sweeping)) {
         heap_->mark_compact_collector()->EnsureSweepingCompleted();
       }
       if (!heap_->mark_compact_collector()->sweeping_in_progress()) {
         bytes_scanned_ = 0;
         StartMarking();
       }
     } else if (state_ == MARKING) {
+      if (FLAG_black_allocation && !black_allocation()) {
+        // If black allocation was not started when incremental marking started
+        // start it now.
+        StartBlackAllocation();
+      }
       bytes_processed = ProcessMarkingDeque(bytes_to_process);
       if (heap_->mark_compact_collector()->marking_deque()->IsEmpty()) {
         if (completion == FORCE_COMPLETION ||
             IsIdleMarkingDelayCounterLimitReached()) {
           if (!finalize_marking_completed_) {
             FinalizeMarking(action);
           } else {
             MarkingComplete(action);
           }
         } else {
@@ skipping 45 matching lines @@
 void IncrementalMarking::IncrementIdleMarkingDelayCounter() {
   idle_marking_delay_counter_++;
 }
 
 
 void IncrementalMarking::ClearIdleMarkingDelayCounter() {
   idle_marking_delay_counter_ = 0;
 }
 }  // namespace internal
 }  // namespace v8