Reland "[heap] Provide ObjectMarking with marking transitions"

src/heap/heap.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/heap.h"
 
 #include "src/accessors.h"
 #include "src/api.h"
 #include "src/ast/context-slot-cache.h"
 #include "src/base/bits.h"
@@ skipping 3165 matching lines @@
 
 void Heap::AdjustLiveBytes(HeapObject* object, int by) {
   // As long as the inspected object is black and we are currently not iterating
   // the heap using HeapIterator, we can update the live byte count. We cannot
   // update while using HeapIterator because the iterator is temporarily
   // marking the whole object graph, without updating live bytes.
   if (lo_space()->Contains(object)) {
     lo_space()->AdjustLiveBytes(by);
   } else if (!in_heap_iterator() &&
              !mark_compact_collector()->sweeping_in_progress() &&
-             Marking::IsBlack(ObjectMarking::MarkBitFrom(object))) {
+             ObjectMarking::IsBlack(object)) {
     DCHECK(MemoryChunk::FromAddress(object->address())->SweepingDone());
     MemoryChunk::IncrementLiveBytes(object, by);
   }
 }
 
 
 FixedArrayBase* Heap::LeftTrimFixedArray(FixedArrayBase* object,
                                          int elements_to_trim) {
   CHECK_NOT_NULL(object);
+  DCHECK(CanMoveObjectStart(object));
   DCHECK(!object->IsFixedTypedArrayBase());
   DCHECK(!object->IsByteArray());
   const int element_size = object->IsFixedArray() ? kPointerSize : kDoubleSize;
   const int bytes_to_trim = elements_to_trim * element_size;
   Map* map = object->map();
 
   // For now this trick is only applied to objects in new and paged space.
   // In large object space the object's start must coincide with chunk
   // and thus the trick is just not applicable.
   DCHECK(!lo_space()->Contains(object));
@@ skipping 30 matching lines @@
       Marking::IsBlackOrGrey(ObjectMarking::MarkBitFrom(object))) {
     Page* page = Page::FromAddress(old_start);
     page->markbits()->ClearRange(
         page->AddressToMarkbitIndex(old_start),
         page->AddressToMarkbitIndex(old_start + bytes_to_trim));
   }
 
   // Initialize header of the trimmed array. Since left trimming is only
   // performed on pages which are not concurrently swept creating a filler
   // object does not require synchronization.
-  DCHECK(CanMoveObjectStart(object));
   Object** former_start = HeapObject::RawField(object, 0);
   int new_start_index = elements_to_trim * (element_size / kPointerSize);
   former_start[new_start_index] = map;
   former_start[new_start_index + 1] = Smi::FromInt(len - elements_to_trim);
 
   FixedArrayBase* new_object =
       FixedArrayBase::cast(HeapObject::FromAddress(new_start));
 
   // Maintain consistency of live bytes during incremental marking
   AdjustLiveBytes(new_object, -bytes_to_trim);
@@ skipping 50 matching lines @@
   // We do not create a filler for objects in large object space.
   // TODO(hpayer): We should shrink the large object page if the size
   // of the object changed significantly.
   if (!lo_space()->Contains(object)) {
     HeapObject* filler =
         CreateFillerObjectAt(new_end, bytes_to_trim, ClearRecordedSlots::kYes);
     DCHECK_NOT_NULL(filler);
     // Clear the mark bits of the black area that belongs now to the filler.
     // This is an optimization. The sweeper will release black fillers anyway.
     if (incremental_marking()->black_allocation() &&
-        Marking::IsBlackOrGrey(ObjectMarking::MarkBitFrom(filler))) {
+        ObjectMarking::IsBlackOrGrey(filler)) {
       Page* page = Page::FromAddress(new_end);
       page->markbits()->ClearRange(
           page->AddressToMarkbitIndex(new_end),
           page->AddressToMarkbitIndex(new_end + bytes_to_trim));
     }
   }
 
   // Initialize header of the trimmed array. We are storing the new length
   // using release store after creating a filler for the left-over space to
   // avoid races with the sweeper thread.
@@ skipping 970 matching lines @@
     return true;
   }
   return false;
 }
 
 void Heap::RegisterReservationsForBlackAllocation(Reservation* reservations) {
   // TODO(hpayer): We do not have to iterate reservations on black objects
   // for marking. We just have to execute the special visiting side effect
   // code that adds objects to global data structures, e.g. for array buffers.
 
-  // Code space, map space, and large object space do not use black pages.
-  // Hence we have to color all objects of the reservation first black to avoid
-  // unnecessary marking deque load.
   if (incremental_marking()->black_allocation()) {
+    // Iterate black objects in old space, code space, map space, and large
+    // object space for side effects.
     for (int i = OLD_SPACE; i < Serializer::kNumberOfSpaces; i++) {
       const Heap::Reservation& res = reservations[i];
       for (auto& chunk : res) {
         Address addr = chunk.start;
         while (addr < chunk.end) {
           HeapObject* obj = HeapObject::FromAddress(addr);
-          Marking::MarkBlack(ObjectMarking::MarkBitFrom(obj));
+          // There might be grey objects due to black to grey transitions in
+          // incremental marking. E.g. see VisitNativeContextIncremental.
+          DCHECK(ObjectMarking::IsBlackOrGrey(obj));
+          if (ObjectMarking::IsBlack(obj)) {
+            incremental_marking()->IterateBlackObject(obj);
+          }
           addr += obj->Size();
         }
       }
-    }
-    for (int i = OLD_SPACE; i < Serializer::kNumberOfSpaces; i++) {
-      const Heap::Reservation& res = reservations[i];
-      for (auto& chunk : res) {
-        Address addr = chunk.start;
-        while (addr < chunk.end) {
-          HeapObject* obj = HeapObject::FromAddress(addr);
-          incremental_marking()->IterateBlackObject(obj);
-          addr += obj->Size();
-        }
-      }
     }
   }
 }
 
 GCIdleTimeHeapState Heap::ComputeHeapState() {
   GCIdleTimeHeapState heap_state;
   heap_state.contexts_disposed = contexts_disposed_;
   heap_state.contexts_disposal_rate =
       tracer()->ContextDisposalRateInMilliseconds();
   heap_state.size_of_objects = static_cast<size_t>(SizeOfObjects());
@@ skipping 548 matching lines @@
 void Heap::IterateAndScavengePromotedObject(HeapObject* target, int size,
                                             bool was_marked_black) {
   // We are not collecting slots on new space objects during mutation
   // thus we have to scan for pointers to evacuation candidates when we
   // promote objects. But we should not record any slots in non-black
   // objects. Grey object's slots would be rescanned.
   // White object might not survive until the end of collection
   // it would be a violation of the invariant to record it's slots.
   bool record_slots = false;
   if (incremental_marking()->IsCompacting()) {
-    MarkBit mark_bit = ObjectMarking::MarkBitFrom(target);
-    record_slots = Marking::IsBlack(mark_bit);
+    record_slots = ObjectMarking::IsBlack(target);
   }
 
   IterateAndScavengePromotedObjectsVisitor visitor(this, target, record_slots);
   if (target->IsJSFunction()) {
     // JSFunctions reachable through kNextFunctionLinkOffset are weak. Slots for
     // this links are recorded during processing of weak lists.
     JSFunction::BodyDescriptorWeakCode::IterateBody(target, size, &visitor);
   } else {
     target->IterateBody(target->map()->instance_type(), size, &visitor);
   }
@@ skipping 1217 matching lines @@
   explicit UnreachableObjectsFilter(Heap* heap) : heap_(heap) {
     MarkReachableObjects();
   }
 
   ~UnreachableObjectsFilter() {
     heap_->mark_compact_collector()->ClearMarkbits();
   }
 
   bool SkipObject(HeapObject* object) {
     if (object->IsFiller()) return true;
-    MarkBit mark_bit = ObjectMarking::MarkBitFrom(object);
-    return Marking::IsWhite(mark_bit);
+    return ObjectMarking::IsWhite(object);
   }
 
  private:
   class MarkingVisitor : public ObjectVisitor {
    public:
     MarkingVisitor() : marking_stack_(10) {}
 
     void VisitPointers(Object** start, Object** end) override {
       for (Object** p = start; p < end; p++) {
         if (!(*p)->IsHeapObject()) continue;
         HeapObject* obj = HeapObject::cast(*p);
+        // Use Marking instead of ObjectMarking to avoid adjusting live bytes
+        // counter.
         MarkBit mark_bit = ObjectMarking::MarkBitFrom(obj);
         if (Marking::IsWhite(mark_bit)) {
           Marking::WhiteToBlack(mark_bit);
           marking_stack_.Add(obj);
         }
       }
     }
 
     void TransitiveClosure() {
       while (!marking_stack_.is_empty()) {
@@ skipping 452 matching lines @@
 }
 
 
 // static
 int Heap::GetStaticVisitorIdForMap(Map* map) {
   return StaticVisitorBase::GetVisitorId(map);
 }
 
 }  // namespace internal
 }  // namespace v8