[heap] Use atomic marking operations in incremental marking if

src/heap/incremental-marking.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/incremental-marking.h"
 
 #include "src/code-stubs.h"
 #include "src/compilation-cache.h"
 #include "src/conversions.h"
 #include "src/heap/concurrent-marking.h"
@@ skipping 25 matching lines @@
       was_activated_(false),
       black_allocation_(false),
       finalize_marking_completed_(false),
       trace_wrappers_toggle_(false),
       request_type_(NONE),
       new_generation_observer_(*this, kAllocatedThreshold),
       old_generation_observer_(*this, kAllocatedThreshold) {}
 
 bool IncrementalMarking::BaseRecordWrite(HeapObject* obj, Object* value) {
   HeapObject* value_heap_obj = HeapObject::cast(value);
-  DCHECK(!ObjectMarking::IsImpossible(value_heap_obj,
-                                      marking_state(value_heap_obj)));
-  DCHECK(!ObjectMarking::IsImpossible(obj, marking_state(obj)));
-  const bool is_black = ObjectMarking::IsBlack(obj, marking_state(obj));
+  DCHECK(!ObjectMarking::IsImpossible<kAtomicity>(
+      value_heap_obj, marking_state(value_heap_obj)));
+  DCHECK(!ObjectMarking::IsImpossible<kAtomicity>(obj, marking_state(obj)));

[Michael Lippautz, 2017/05/03 07:49:51]

Is IsImpossible safe for concurrent marking? I remember that there were issues
with it.

[ulan, 2017/05/03 09:00:28]

It is not safe indeed. I will remove bunch of dchecks that are invalidated by
concurrent marking in a separate CL.

[Hannes Payer (out of office), 2017/05/03 10:08:53]

Is the current implementation of the check not correct? Why?

[ulan, 2017/05/03 12:09:30]

In general IsImpossible can produce false positive with concurrent marking.
Scenario:
1. The object is white.
2. IsImpossible function reads the first bit as 0.
3. In the meantime the concurrent thread transitions the object from white to
black.
4. IsImpossible function reads the second bit as 1 and returns true.

But in the BaseRecordWrite, the object is not white. So IsImpossible should
work.

+  const bool is_black =
+      ObjectMarking::IsBlack<kAtomicity>(obj, marking_state(obj));
 
   if (is_black && WhiteToGreyAndPush(value_heap_obj)) {
     RestartIfNotMarking();
   }
   return is_compacting_ && is_black;
 }
 
 
 void IncrementalMarking::RecordWriteSlow(HeapObject* obj, Object** slot,
                                          Object* value) {
@@ skipping 53 matching lines @@
 
 void IncrementalMarking::RecordWriteIntoCodeSlow(Code* host, RelocInfo* rinfo,
                                                  Object* value) {
   if (BaseRecordWrite(host, value)) {
     // Object is not going to be rescanned.  We need to record the slot.
     heap_->mark_compact_collector()->RecordRelocSlot(host, rinfo, value);
   }
 }
 
 bool IncrementalMarking::WhiteToGreyAndPush(HeapObject* obj) {
-  if (ObjectMarking::WhiteToGrey(obj, marking_state(obj))) {
+  if (ObjectMarking::WhiteToGrey<kAtomicity>(obj, marking_state(obj))) {
     marking_deque()->Push(obj);
     return true;
   }
   return false;
 }
 
 void IncrementalMarking::TransferMark(Heap* heap, HeapObject* from,
                                       HeapObject* to) {
   DCHECK(MemoryChunk::FromAddress(from->address())->SweepingDone());
   // This is only used when resizing an object.
   DCHECK(MemoryChunk::FromAddress(from->address()) ==
          MemoryChunk::FromAddress(to->address()));
 
   if (!IsMarking()) return;
 
   // If the mark doesn't move, we don't check the color of the object.
   // It doesn't matter whether the object is black, since it hasn't changed
   // size, so the adjustment to the live data count will be zero anyway.
   if (from == to) return;
 
   MarkBit new_mark_bit = ObjectMarking::MarkBitFrom(to, marking_state(to));
   MarkBit old_mark_bit = ObjectMarking::MarkBitFrom(from, marking_state(from));
 
-  if (Marking::IsBlack(old_mark_bit)) {
-    bool success = Marking::WhiteToBlack(new_mark_bit);
+  if (Marking::IsBlack<kAtomicity>(old_mark_bit)) {
+    bool success = Marking::WhiteToBlack<kAtomicity>(new_mark_bit);
     DCHECK(success);
     USE(success);
-  } else if (Marking::IsGrey(old_mark_bit)) {
-    bool success = Marking::WhiteToGrey(new_mark_bit);
+  } else if (Marking::IsGrey<kAtomicity>(old_mark_bit)) {
+    bool success = Marking::WhiteToGrey<kAtomicity>(new_mark_bit);
     DCHECK(success);
     USE(success);
     marking_deque()->Push(to);
     RestartIfNotMarking();
   }
 }
 
 class IncrementalMarkingMarkingVisitor
     : public StaticMarkingVisitor<IncrementalMarkingMarkingVisitor> {
  public:
@@ skipping 24 matching lines @@
       bool scan_until_end = false;
       do {
         VisitPointers(heap, object, HeapObject::RawField(object, start_offset),
                       HeapObject::RawField(object, end_offset));
         start_offset = end_offset;
         end_offset = Min(object_size, end_offset + kProgressBarScanningChunk);
         scan_until_end = heap->incremental_marking()->marking_deque()->IsFull();
       } while (scan_until_end && start_offset < object_size);
       chunk->set_progress_bar(start_offset);
       if (start_offset < object_size) {
-        if (ObjectMarking::IsGrey(
+        if (ObjectMarking::IsGrey<IncrementalMarking::kAtomicity>(
                 object, heap->incremental_marking()->marking_state(object))) {
           heap->incremental_marking()->marking_deque()->Unshift(object);
         } else {
-          DCHECK(ObjectMarking::IsBlack(
+          DCHECK(ObjectMarking::IsBlack<IncrementalMarking::kAtomicity>(
               object, heap->incremental_marking()->marking_state(object)));
           heap->mark_compact_collector()->UnshiftBlack(object);
         }
         heap->incremental_marking()->NotifyIncompleteScanOfObject(
             object_size - (start_offset - already_scanned_offset));
       }
     } else {
       FixedArrayVisitor::Visit(map, object);
     }
   }
 
   static void VisitNativeContextIncremental(Map* map, HeapObject* object) {
     Context* context = Context::cast(object);
 
     // We will mark cache black with a separate pass when we finish marking.
     // Note that GC can happen when the context is not fully initialized,
     // so the cache can be undefined.
     Object* cache = context->get(Context::NORMALIZED_MAP_CACHE_INDEX);
     if (!cache->IsUndefined(map->GetIsolate())) {
       if (cache->IsHeapObject()) {
         HeapObject* heap_obj = HeapObject::cast(cache);
         // Mark the object grey if it is white, do not enque it into the marking
         // deque.
         Heap* heap = map->GetHeap();
-        bool ignored = ObjectMarking::WhiteToGrey(
-            heap_obj, heap->incremental_marking()->marking_state(heap_obj));
+        bool ignored =
+            ObjectMarking::WhiteToGrey<IncrementalMarking::kAtomicity>(
+                heap_obj, heap->incremental_marking()->marking_state(heap_obj));
         USE(ignored);
       }
     }
     VisitNativeContext(map, context);
   }
 
   INLINE(static void VisitPointer(Heap* heap, HeapObject* object, Object** p)) {
     Object* target = *p;
     if (target->IsHeapObject()) {
       heap->mark_compact_collector()->RecordSlot(object, p, target);
@@ skipping 14 matching lines @@
 
   // Marks the object grey and pushes it on the marking stack.
   INLINE(static void MarkObject(Heap* heap, Object* obj)) {
     heap->incremental_marking()->WhiteToGreyAndPush(HeapObject::cast(obj));
   }
 
   // Marks the object black without pushing it on the marking stack.
   // Returns true if object needed marking and false otherwise.
   INLINE(static bool MarkObjectWithoutPush(Heap* heap, Object* obj)) {
     HeapObject* heap_object = HeapObject::cast(obj);
-    return ObjectMarking::WhiteToBlack(
+    return ObjectMarking::WhiteToBlack<IncrementalMarking::kAtomicity>(
         heap_object, heap->incremental_marking()->marking_state(heap_object));
   }
 };
 
 void IncrementalMarking::IterateBlackObject(HeapObject* object) {
-  if (IsMarking() && ObjectMarking::IsBlack(object, marking_state(object))) {
+  if (IsMarking() &&
+      ObjectMarking::IsBlack<kAtomicity>(object, marking_state(object))) {
     Page* page = Page::FromAddress(object->address());
     if ((page->owner() != nullptr) && (page->owner()->identity() == LO_SPACE)) {
       // IterateBlackObject requires us to visit the whole object.
       page->ResetProgressBar();
     }
     Map* map = object->map();
     WhiteToGreyAndPush(map);
     IncrementalMarkingMarkingVisitor::IterateBody(map, object);
   }
 }
@@ skipping 323 matching lines @@
   Object* weak_cell_obj = heap()->encountered_weak_cells();
   Object* weak_cell_head = Smi::kZero;
   WeakCell* prev_weak_cell_obj = NULL;
   while (weak_cell_obj != Smi::kZero) {
     WeakCell* weak_cell = reinterpret_cast<WeakCell*>(weak_cell_obj);
     // We do not insert cleared weak cells into the list, so the value
     // cannot be a Smi here.
     HeapObject* value = HeapObject::cast(weak_cell->value());
     // Remove weak cells with live objects from the list, they do not need
     // clearing.
-    if (ObjectMarking::IsBlackOrGrey(value, marking_state(value))) {
+    if (ObjectMarking::IsBlackOrGrey<kAtomicity>(value, marking_state(value))) {
       // Record slot, if value is pointing to an evacuation candidate.
       Object** slot = HeapObject::RawField(weak_cell, WeakCell::kValueOffset);
       heap_->mark_compact_collector()->RecordSlot(weak_cell, slot, *slot);
       // Remove entry somewhere after top.
       if (prev_weak_cell_obj != NULL) {
         prev_weak_cell_obj->set_next(weak_cell->next());
       }
       weak_cell_obj = weak_cell->next();
       weak_cell->clear_next(the_hole_value);
     } else {
@@ skipping 10 matching lines @@
 
 
 bool ShouldRetainMap(Map* map, int age) {
   if (age == 0) {
     // The map has aged. Do not retain this map.
     return false;
   }
   Object* constructor = map->GetConstructor();
   Heap* heap = map->GetHeap();
   if (!constructor->IsHeapObject() ||
-      ObjectMarking::IsWhite(HeapObject::cast(constructor),
-                             heap->incremental_marking()->marking_state(
-                                 HeapObject::cast(constructor)))) {
+      ObjectMarking::IsWhite<IncrementalMarking::kAtomicity>(
+          HeapObject::cast(constructor),
+          heap->incremental_marking()->marking_state(
+              HeapObject::cast(constructor)))) {
     // The constructor is dead, no new objects with this map can
     // be created. Do not retain this map.
     return false;
   }
   return true;
 }
 
 
 void IncrementalMarking::RetainMaps() {
   // Do not retain dead maps if flag disables it or there is
   // - memory pressure (reduce_memory_footprint_),
   // - GC is requested by tests or dev-tools (abort_incremental_marking_).
   bool map_retaining_is_disabled = heap()->ShouldReduceMemory() ||
                                    heap()->ShouldAbortIncrementalMarking() ||
                                    FLAG_retain_maps_for_n_gc == 0;
   ArrayList* retained_maps = heap()->retained_maps();
   int length = retained_maps->Length();
   // The number_of_disposed_maps separates maps in the retained_maps
   // array that were created before and after context disposal.
   // We do not age and retain disposed maps to avoid memory leaks.
   int number_of_disposed_maps = heap()->number_of_disposed_maps_;
   for (int i = 0; i < length; i += 2) {
     DCHECK(retained_maps->Get(i)->IsWeakCell());
     WeakCell* cell = WeakCell::cast(retained_maps->Get(i));
     if (cell->cleared()) continue;
     int age = Smi::cast(retained_maps->Get(i + 1))->value();
     int new_age;
     Map* map = Map::cast(cell->value());
     if (i >= number_of_disposed_maps && !map_retaining_is_disabled &&
-        ObjectMarking::IsWhite(map, marking_state(map))) {
+        ObjectMarking::IsWhite<kAtomicity>(map, marking_state(map))) {
       if (ShouldRetainMap(map, age)) {
         WhiteToGreyAndPush(map);
       }
       Object* prototype = map->prototype();
       if (age > 0 && prototype->IsHeapObject() &&
-          ObjectMarking::IsWhite(HeapObject::cast(prototype),
-                                 marking_state(HeapObject::cast(prototype)))) {
+          ObjectMarking::IsWhite<kAtomicity>(
+              HeapObject::cast(prototype),
+              marking_state(HeapObject::cast(prototype)))) {
         // The prototype is not marked, age the map.
         new_age = age - 1;
       } else {
         // The prototype and the constructor are marked, this map keeps only
         // transition tree alive, not JSObjects. Do not age the map.
         new_age = age;
       }
     } else {
       new_age = FLAG_retain_maps_for_n_gc;
     }
@@ skipping 70 matching lines @@
       MapWord map_word = obj->map_word();
       if (!map_word.IsForwardingAddress()) {
         // There may be objects on the marking deque that do not exist anymore,
         // e.g. left trimmed objects or objects from the root set (frames).
         // If these object are dead at scavenging time, their marking deque
         // entries will not point to forwarding addresses. Hence, we can discard
         // them.
         return nullptr;
       }
       HeapObject* dest = map_word.ToForwardingAddress();
-      if (ObjectMarking::IsBlack(dest, marking_state(dest))) {
+      if (ObjectMarking::IsBlack<kAtomicity>(dest, marking_state(dest))) {
         // The object is already processed by the marker.
         return nullptr;
       }
-      DCHECK(
-          ObjectMarking::IsGrey(obj, marking_state(obj)) ||
-          (obj->IsFiller() && ObjectMarking::IsWhite(obj, marking_state(obj))));
+      DCHECK(ObjectMarking::IsGrey<kAtomicity>(obj, marking_state(obj)) ||
+             (obj->IsFiller() &&
+              ObjectMarking::IsWhite<kAtomicity>(obj, marking_state(obj))));
       return dest;
     } else {
-      DCHECK(ObjectMarking::IsGrey(obj, marking_state(obj)) ||
+      DCHECK(ObjectMarking::IsGrey<kAtomicity>(obj, marking_state(obj)) ||
              (obj->IsFiller() &&
-              ObjectMarking::IsWhite(obj, marking_state(obj))) ||
+              ObjectMarking::IsWhite<kAtomicity>(obj, marking_state(obj))) ||
              (MemoryChunk::FromAddress(obj->address())
                   ->IsFlagSet(MemoryChunk::HAS_PROGRESS_BAR) &&
-              ObjectMarking::IsBlack(obj, marking_state(obj))));
+              ObjectMarking::IsBlack<kAtomicity>(obj, marking_state(obj))));
       // Skip one word filler objects that appear on the
       // stack when we perform in place array shift.
       return (obj->map() == filler_map) ? nullptr : obj;
     }
   });
 }
 
 
 void IncrementalMarking::VisitObject(Map* map, HeapObject* obj, int size) {
   WhiteToGreyAndPush(map);
 
   IncrementalMarkingMarkingVisitor::IterateBody(map, obj);
 
 #if ENABLE_SLOW_DCHECKS
   MarkBit mark_bit = ObjectMarking::MarkBitFrom(obj, marking_state(obj));
   MemoryChunk* chunk = MemoryChunk::FromAddress(obj->address());
-  SLOW_DCHECK(Marking::IsGrey(mark_bit) ||
+  SLOW_DCHECK(Marking::IsGrey<kAtomicity>(mark_bit) ||
               (chunk->IsFlagSet(MemoryChunk::HAS_PROGRESS_BAR) &&
-               Marking::IsBlack(mark_bit)));
+               Marking::IsBlack<kAtomicity>(mark_bit)));
 #endif
-  ObjectMarking::GreyToBlack(obj, marking_state(obj));
+  ObjectMarking::GreyToBlack<kAtomicity>(obj, marking_state(obj));
 }
 
 intptr_t IncrementalMarking::ProcessMarkingDeque(
     intptr_t bytes_to_process, ForceCompletionAction completion) {
   intptr_t bytes_processed = 0;
   while (!marking_deque()->IsEmpty() && (bytes_processed < bytes_to_process ||
                                          completion == FORCE_COMPLETION)) {
     HeapObject* obj = marking_deque()->Pop();
 
     // Left trimming may result in white, grey, or black filler objects on the
     // marking deque. Ignore these objects.
     if (obj->IsFiller()) {
-      DCHECK(!ObjectMarking::IsImpossible(obj, marking_state(obj)));
+      DCHECK(!ObjectMarking::IsImpossible<kAtomicity>(obj, marking_state(obj)));
       continue;
     }
 
     Map* map = obj->map();
     int size = obj->SizeFromMap(map);
     unscanned_bytes_of_large_object_ = 0;
     VisitObject(map, obj, size);
     bytes_processed += size - unscanned_bytes_of_large_object_;
   }
   // Report all found wrappers to the embedder. This is necessary as the
@@ skipping 35 matching lines @@
   }
 
   Object* context = heap_->native_contexts_list();
   while (!context->IsUndefined(heap_->isolate())) {
     // GC can happen when the context is not fully initialized,
     // so the cache can be undefined.
     HeapObject* cache = HeapObject::cast(
         Context::cast(context)->get(Context::NORMALIZED_MAP_CACHE_INDEX));
     if (!cache->IsUndefined(heap_->isolate())) {
       // Mark the cache black if it is grey.
-      bool ignored = ObjectMarking::GreyToBlack(cache, marking_state(cache));
+      bool ignored =
+          ObjectMarking::GreyToBlack<kAtomicity>(cache, marking_state(cache));
       USE(ignored);
     }
     context = Context::cast(context)->next_context_link();
   }
 }
 
 
 void IncrementalMarking::Stop() {
   if (IsStopped()) return;
   if (FLAG_trace_incremental_marking) {
@@ skipping 260 matching lines @@
   idle_marking_delay_counter_++;
 }
 
 
 void IncrementalMarking::ClearIdleMarkingDelayCounter() {
   idle_marking_delay_counter_ = 0;
 }
 
 }  // namespace internal
 }  // namespace v8