[heap] Reland "Make non-atomic markbit operations consistent with atomic ones."

src/heap/incremental-marking.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/incremental-marking.h"
 
 #include "src/code-stubs.h"
 #include "src/compilation-cache.h"
 #include "src/conversions.h"
 #include "src/heap/concurrent-marking.h"
@@ skipping 39 matching lines @@
       new_generation_observer_(*this, kAllocatedThreshold),
       old_generation_observer_(*this, kAllocatedThreshold) {}
 
 bool IncrementalMarking::BaseRecordWrite(HeapObject* obj, Object* value) {
   HeapObject* value_heap_obj = HeapObject::cast(value);
   DCHECK(!ObjectMarking::IsImpossible(value_heap_obj,
                                       marking_state(value_heap_obj)));
   DCHECK(!ObjectMarking::IsImpossible(obj, marking_state(obj)));
   const bool is_black = ObjectMarking::IsBlack(obj, marking_state(obj));
 
-  if (is_black &&
-      ObjectMarking::IsWhite(value_heap_obj, marking_state(value_heap_obj))) {
-    WhiteToGreyAndPush(value_heap_obj);
+  if (is_black && WhiteToGreyAndPush(value_heap_obj)) {
     RestartIfNotMarking();
   }
   return is_compacting_ && is_black;
 }
 
 
 void IncrementalMarking::RecordWriteSlow(HeapObject* obj, Object** slot,
                                          Object* value) {
   if (BaseRecordWrite(obj, value) && slot != NULL) {
     // Object is not going to be rescanned we need to record the slot.
@@ skipping 50 matching lines @@
 }
 
 void IncrementalMarking::RecordWriteIntoCodeSlow(Code* host, RelocInfo* rinfo,
                                                  Object* value) {
   if (BaseRecordWrite(host, value)) {
     // Object is not going to be rescanned.  We need to record the slot.
     heap_->mark_compact_collector()->RecordRelocSlot(host, rinfo, value);
   }
 }
 
-void IncrementalMarking::WhiteToGreyAndPush(HeapObject* obj) {
-  ObjectMarking::WhiteToGrey(obj, marking_state(obj));
-  marking_deque()->Push(obj);
+bool IncrementalMarking::WhiteToGreyAndPush(HeapObject* obj) {
+  if (ObjectMarking::WhiteToGrey(obj, marking_state(obj))) {
+    marking_deque()->Push(obj);
+    return true;
+  }
+  return false;
 }
 
 void IncrementalMarking::TransferMark(Heap* heap, HeapObject* from,
                                       HeapObject* to) {
   DCHECK(MemoryChunk::FromAddress(from->address())->SweepingDone());
   // This is only used when resizing an object.
   DCHECK(MemoryChunk::FromAddress(from->address()) ==
          MemoryChunk::FromAddress(to->address()));
 
   if (!IsMarking()) return;
 
   // If the mark doesn't move, we don't check the color of the object.
   // It doesn't matter whether the object is black, since it hasn't changed
   // size, so the adjustment to the live data count will be zero anyway.
   if (from == to) return;
 
   MarkBit new_mark_bit = ObjectMarking::MarkBitFrom(to, marking_state(to));
   MarkBit old_mark_bit = ObjectMarking::MarkBitFrom(from, marking_state(from));
 
   if (Marking::IsBlack(old_mark_bit)) {
-    Marking::MarkBlack(new_mark_bit);
+    if (from->address() + kPointerSize == to->address()) {
+      // The old and the new markbits overlap. The |to| object has the
+      // grey color. To make it black, we need to set second bit.
+      DCHECK(new_mark_bit.Get());
+      new_mark_bit.Next().Set();
+    } else {
+      bool success = Marking::WhiteToBlack(new_mark_bit);
+      DCHECK(success);
+      USE(success);
+    }
   } else if (Marking::IsGrey(old_mark_bit)) {
-    Marking::WhiteToGrey(new_mark_bit);
+    if (from->address() + kPointerSize == to->address()) {
+      // The old and the new markbits overlap. The |to| object has the
+      // white color. To make it black, we need to set both bits.
+      // Note that Marking::WhiteToGrey does not work here because
+      // old_mark_bit.Next() can be set by the concurrent marker at any time.
+      new_mark_bit.Set();
+      new_mark_bit.Next().Set();
+    } else {
+      bool success = Marking::WhiteToGrey(new_mark_bit);
+      DCHECK(success);
+      USE(success);
+    }
     marking_deque()->Push(to);
     RestartIfNotMarking();
   }
 }
 
 class IncrementalMarkingMarkingVisitor
     : public StaticMarkingVisitor<IncrementalMarkingMarkingVisitor> {
  public:
   static void Initialize() {
     StaticMarkingVisitor<IncrementalMarkingMarkingVisitor>::Initialize();
@@ skipping 51 matching lines @@
     // We will mark cache black with a separate pass when we finish marking.
     // Note that GC can happen when the context is not fully initialized,
     // so the cache can be undefined.
     Object* cache = context->get(Context::NORMALIZED_MAP_CACHE_INDEX);
     if (!cache->IsUndefined(map->GetIsolate())) {
       if (cache->IsHeapObject()) {
         HeapObject* heap_obj = HeapObject::cast(cache);
         // Mark the object grey if it is white, do not enque it into the marking
         // deque.
         Heap* heap = map->GetHeap();
-        if (ObjectMarking::IsWhite(
-                heap_obj,
-                heap->incremental_marking()->marking_state(heap_obj))) {
-          ObjectMarking::WhiteToGrey(
-              heap_obj, heap->incremental_marking()->marking_state(heap_obj));
-        }
+        bool ignored = ObjectMarking::WhiteToGrey(
+            heap_obj, heap->incremental_marking()->marking_state(heap_obj));
+        USE(ignored);
       }
     }
     VisitNativeContext(map, context);
   }
 
   INLINE(static void VisitPointer(Heap* heap, HeapObject* object, Object** p)) {
     Object* target = *p;
     if (target->IsHeapObject()) {
       heap->mark_compact_collector()->RecordSlot(object, p, target);
       MarkObject(heap, target);
     }
   }
 
   INLINE(static void VisitPointers(Heap* heap, HeapObject* object,
                                    Object** start, Object** end)) {
     for (Object** p = start; p < end; p++) {
       Object* target = *p;
       if (target->IsHeapObject()) {
         heap->mark_compact_collector()->RecordSlot(object, p, target);
         MarkObject(heap, target);
       }
     }
   }
 
   // Marks the object grey and pushes it on the marking stack.
   INLINE(static void MarkObject(Heap* heap, Object* obj)) {
-    heap->incremental_marking()->MarkGrey(HeapObject::cast(obj));
+    heap->incremental_marking()->WhiteToGreyAndPush(HeapObject::cast(obj));
   }
 
   // Marks the object black without pushing it on the marking stack.
   // Returns true if object needed marking and false otherwise.
   INLINE(static bool MarkObjectWithoutPush(Heap* heap, Object* obj)) {
     HeapObject* heap_object = HeapObject::cast(obj);
-    if (ObjectMarking::IsWhite(
-            heap_object,
-            heap->incremental_marking()->marking_state(heap_object))) {
-      ObjectMarking::WhiteToBlack(
-          heap_object, heap->incremental_marking()->marking_state(heap_object));
-      return true;
-    }
-    return false;
+    return ObjectMarking::WhiteToBlack(
+        heap_object, heap->incremental_marking()->marking_state(heap_object));
   }
 };
 
 void IncrementalMarking::IterateBlackObject(HeapObject* object) {
   if (IsMarking() && ObjectMarking::IsBlack(object, marking_state(object))) {
     Page* page = Page::FromAddress(object->address());
     if ((page->owner() != nullptr) && (page->owner()->identity() == LO_SPACE)) {
       // IterateBlackObject requires us to visit the whole object.
       page->ResetProgressBar();
     }
     Map* map = object->map();
-    MarkGrey(map);
+    WhiteToGreyAndPush(map);
     IncrementalMarkingMarkingVisitor::IterateBody(map, object);
   }
 }
 
 class IncrementalMarkingRootMarkingVisitor : public RootVisitor {
  public:
   explicit IncrementalMarkingRootMarkingVisitor(
       IncrementalMarking* incremental_marking)
       : heap_(incremental_marking->heap()) {}
 
   void VisitRootPointer(Root root, Object** p) override {
     MarkObjectByPointer(p);
   }
 
   void VisitRootPointers(Root root, Object** start, Object** end) override {
     for (Object** p = start; p < end; p++) MarkObjectByPointer(p);
   }
 
  private:
   void MarkObjectByPointer(Object** p) {
     Object* obj = *p;
     if (!obj->IsHeapObject()) return;
 
-    heap_->incremental_marking()->MarkGrey(HeapObject::cast(obj));
+    heap_->incremental_marking()->WhiteToGreyAndPush(HeapObject::cast(obj));
   }
 
   Heap* heap_;
 };
 
 
 void IncrementalMarking::Initialize() {
   IncrementalMarkingMarkingVisitor::Initialize();
 }
 
@@ skipping 370 matching lines @@
   for (int i = 0; i < length; i += 2) {
     DCHECK(retained_maps->Get(i)->IsWeakCell());
     WeakCell* cell = WeakCell::cast(retained_maps->Get(i));
     if (cell->cleared()) continue;
     int age = Smi::cast(retained_maps->Get(i + 1))->value();
     int new_age;
     Map* map = Map::cast(cell->value());
     if (i >= number_of_disposed_maps && !map_retaining_is_disabled &&
         ObjectMarking::IsWhite(map, marking_state(map))) {
       if (ShouldRetainMap(map, age)) {
-        MarkGrey(map);
+        WhiteToGreyAndPush(map);
       }
       Object* prototype = map->prototype();
       if (age > 0 && prototype->IsHeapObject() &&
           ObjectMarking::IsWhite(HeapObject::cast(prototype),
                                  marking_state(HeapObject::cast(prototype)))) {
         // The prototype is not marked, age the map.
         new_age = age - 1;
       } else {
         // The prototype and the constructor are marked, this map keeps only
         // transition tree alive, not JSObjects. Do not age the map.
@@ skipping 99 matching lines @@
               ObjectMarking::IsBlack(obj, marking_state(obj))));
       // Skip one word filler objects that appear on the
       // stack when we perform in place array shift.
       return (obj->map() == filler_map) ? nullptr : obj;
     }
   });
 }
 
 
 void IncrementalMarking::VisitObject(Map* map, HeapObject* obj, int size) {
-  MarkGrey(map);
+  WhiteToGreyAndPush(map);
 
   IncrementalMarkingMarkingVisitor::IterateBody(map, obj);
 
 #if ENABLE_SLOW_DCHECKS
   MarkBit mark_bit = ObjectMarking::MarkBitFrom(obj, marking_state(obj));
   MemoryChunk* chunk = MemoryChunk::FromAddress(obj->address());
   SLOW_DCHECK(Marking::IsGrey(mark_bit) ||
               (chunk->IsFlagSet(MemoryChunk::HAS_PROGRESS_BAR) &&
                Marking::IsBlack(mark_bit)));
 #endif
-  MarkBlack(obj, size);
-}
-
-void IncrementalMarking::MarkGrey(HeapObject* object) {
-  if (ObjectMarking::IsWhite(object, marking_state(object))) {
-    WhiteToGreyAndPush(object);
-  }
-}
-
-void IncrementalMarking::MarkBlack(HeapObject* obj, int size) {
-  if (ObjectMarking::IsBlack(obj, marking_state(obj))) return;
   ObjectMarking::GreyToBlack(obj, marking_state(obj));
 }
 
 intptr_t IncrementalMarking::ProcessMarkingDeque(
     intptr_t bytes_to_process, ForceCompletionAction completion) {
   intptr_t bytes_processed = 0;
   while (!marking_deque()->IsEmpty() && (bytes_processed < bytes_to_process ||
                                          completion == FORCE_COMPLETION)) {
     HeapObject* obj = marking_deque()->Pop();
 
@@ skipping 48 matching lines @@
     }
   }
 
   Object* context = heap_->native_contexts_list();
   while (!context->IsUndefined(heap_->isolate())) {
     // GC can happen when the context is not fully initialized,
     // so the cache can be undefined.
     HeapObject* cache = HeapObject::cast(
         Context::cast(context)->get(Context::NORMALIZED_MAP_CACHE_INDEX));
     if (!cache->IsUndefined(heap_->isolate())) {
-      if (ObjectMarking::IsGrey(cache, marking_state(cache))) {
-        ObjectMarking::GreyToBlack(cache, marking_state(cache));
-      }
+      // Mark the cache black if it is grey.
+      bool ignored = ObjectMarking::GreyToBlack(cache, marking_state(cache));
+      USE(ignored);
     }
     context = Context::cast(context)->next_context_link();
   }
 }
 
 
 void IncrementalMarking::Stop() {
   if (IsStopped()) return;
   if (FLAG_trace_incremental_marking) {
     int old_generation_size_mb =
@@ skipping 259 matching lines @@
   idle_marking_delay_counter_++;
 }
 
 
 void IncrementalMarking::ClearIdleMarkingDelayCounter() {
   idle_marking_delay_counter_ = 0;
 }
 
 }  // namespace internal
 }  // namespace v8