[heap] Remove LocalStoreBuffer and add slots in parallel

src/heap/mark-compact.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/mark-compact.h"
 
 #include "src/base/atomicops.h"
 #include "src/base/bits.h"
 #include "src/base/sys-info.h"
 #include "src/code-stubs.h"
@@ skipping 1493 matching lines @@
 class MarkCompactCollector::HeapObjectVisitor {
  public:
   virtual ~HeapObjectVisitor() {}
   virtual bool Visit(HeapObject* object) = 0;
 };
 
 
 class MarkCompactCollector::EvacuateVisitorBase
     : public MarkCompactCollector::HeapObjectVisitor {
  public:
-  EvacuateVisitorBase(Heap* heap, CompactionSpaceCollection* compaction_spaces,
-                      LocalSlotsBuffer* old_to_old_slots,
-                      LocalSlotsBuffer* old_to_new_slots)
-      : heap_(heap),
-        compaction_spaces_(compaction_spaces),
-        old_to_old_slots_(old_to_old_slots),
-        old_to_new_slots_(old_to_new_slots) {}
+  EvacuateVisitorBase(Heap* heap, CompactionSpaceCollection* compaction_spaces)
+      : heap_(heap), compaction_spaces_(compaction_spaces) {}
 
   bool TryEvacuateObject(PagedSpace* target_space, HeapObject* object,
                          HeapObject** target_object) {
     int size = object->Size();
     AllocationAlignment alignment = object->RequiredAlignment();
     AllocationResult allocation = target_space->AllocateRaw(size, alignment);
     if (allocation.To(target_object)) {
       heap_->mark_compact_collector()->MigrateObject(
-          *target_object, object, size, target_space->identity(),
-          old_to_old_slots_, old_to_new_slots_);
+          *target_object, object, size, target_space->identity());
       return true;
     }
     return false;
   }
 
  protected:
   Heap* heap_;
   CompactionSpaceCollection* compaction_spaces_;
-  LocalSlotsBuffer* old_to_old_slots_;
-  LocalSlotsBuffer* old_to_new_slots_;
 };
 
 
 class MarkCompactCollector::EvacuateNewSpaceVisitor final
     : public MarkCompactCollector::EvacuateVisitorBase {
  public:
   static const intptr_t kLabSize = 4 * KB;
   static const intptr_t kMaxLabObjectSize = 256;
 
   explicit EvacuateNewSpaceVisitor(Heap* heap,
                                    CompactionSpaceCollection* compaction_spaces,
-                                   LocalSlotsBuffer* old_to_old_slots,
-                                   LocalSlotsBuffer* old_to_new_slots,
                                    HashMap* local_pretenuring_feedback)
-      : EvacuateVisitorBase(heap, compaction_spaces, old_to_old_slots,
-                            old_to_new_slots),
+      : EvacuateVisitorBase(heap, compaction_spaces),
         buffer_(LocalAllocationBuffer::InvalidBuffer()),
         space_to_allocate_(NEW_SPACE),
         promoted_size_(0),
         semispace_copied_size_(0),
         local_pretenuring_feedback_(local_pretenuring_feedback) {}
 
   bool Visit(HeapObject* object) override {
     heap_->UpdateAllocationSite<Heap::kCached>(object,
                                                local_pretenuring_feedback_);
     int size = object->Size();
     HeapObject* target_object = nullptr;
     if (heap_->ShouldBePromoted(object->address(), size) &&
         TryEvacuateObject(compaction_spaces_->Get(OLD_SPACE), object,
                           &target_object)) {
       // If we end up needing more special cases, we should factor this out.
       if (V8_UNLIKELY(target_object->IsJSArrayBuffer())) {
         heap_->array_buffer_tracker()->Promote(
             JSArrayBuffer::cast(target_object));
       }
       promoted_size_ += size;
       return true;
     }
     HeapObject* target = nullptr;
     AllocationSpace space = AllocateTargetObject(object, &target);
-    heap_->mark_compact_collector()->MigrateObject(
-        HeapObject::cast(target), object, size, space,
-        (space == NEW_SPACE) ? nullptr : old_to_old_slots_,
-        (space == NEW_SPACE) ? nullptr : old_to_new_slots_);
+    heap_->mark_compact_collector()->MigrateObject(HeapObject::cast(target),
+                                                   object, size, space);
     if (V8_UNLIKELY(target->IsJSArrayBuffer())) {
       heap_->array_buffer_tracker()->MarkLive(JSArrayBuffer::cast(target));
     }
     semispace_copied_size_ += size;
     return true;
   }
 
   intptr_t promoted_size() { return promoted_size_; }
   intptr_t semispace_copied_size() { return semispace_copied_size_; }
 
@@ skipping 98 matching lines @@
   intptr_t promoted_size_;
   intptr_t semispace_copied_size_;
   HashMap* local_pretenuring_feedback_;
 };
 
 
 class MarkCompactCollector::EvacuateOldSpaceVisitor final
     : public MarkCompactCollector::EvacuateVisitorBase {
  public:
   EvacuateOldSpaceVisitor(Heap* heap,
-                          CompactionSpaceCollection* compaction_spaces,
-                          LocalSlotsBuffer* old_to_old_slots,
-                          LocalSlotsBuffer* old_to_new_slots)
-      : EvacuateVisitorBase(heap, compaction_spaces, old_to_old_slots,
-                            old_to_new_slots) {}
+                          CompactionSpaceCollection* compaction_spaces)
+      : EvacuateVisitorBase(heap, compaction_spaces) {}
 
   bool Visit(HeapObject* object) override {
     CompactionSpace* target_space = compaction_spaces_->Get(
         Page::FromAddress(object->address())->owner()->identity());
     HeapObject* target_object = nullptr;
     if (TryEvacuateObject(target_space, object, &target_object)) {
       DCHECK(object->map_word().IsForwardingAddress());
       return true;
     }
     return false;
@@ skipping 802 matching lines @@
   HeapObject* undefined = heap()->undefined_value();
   Object* obj = heap()->encountered_transition_arrays();
   while (obj != Smi::FromInt(0)) {
     TransitionArray* array = TransitionArray::cast(obj);
     obj = array->next_link();
     array->set_next_link(undefined, SKIP_WRITE_BARRIER);
   }
   heap()->set_encountered_transition_arrays(Smi::FromInt(0));
 }
 
-void MarkCompactCollector::RecordMigratedSlot(
-    Object* value, Address slot, LocalSlotsBuffer* old_to_old_slots,
-    LocalSlotsBuffer* old_to_new_slots) {
-  // When parallel compaction is in progress, store and slots buffer entries
-  // require synchronization.
-  if (heap_->InNewSpace(value)) {
-    old_to_new_slots->Record(slot);
-  } else if (value->IsHeapObject() && IsOnEvacuationCandidate(value)) {
-    old_to_old_slots->Record(slot);
-  }
-}
-
 static inline SlotType SlotTypeForRMode(RelocInfo::Mode rmode) {
   if (RelocInfo::IsCodeTarget(rmode)) {
     return CODE_TARGET_SLOT;
   } else if (RelocInfo::IsCell(rmode)) {
     return CELL_TARGET_SLOT;
   } else if (RelocInfo::IsEmbeddedObject(rmode)) {
     return EMBEDDED_OBJECT_SLOT;
   } else if (RelocInfo::IsDebugBreakSlot(rmode)) {
     return DEBUG_TARGET_SLOT;
   }
@@ skipping 20 matching lines @@
         slot_type = OBJECT_SLOT;
       }
     }
     RememberedSet<OLD_TO_OLD>::InsertTyped(source_page, slot_type, addr);
   }
 }
 
 
 class RecordMigratedSlotVisitor final : public ObjectVisitor {
  public:
-  RecordMigratedSlotVisitor(MarkCompactCollector* collector,
-                            LocalSlotsBuffer* old_to_old_slots,
-                            LocalSlotsBuffer* old_to_new_slots)
-      : collector_(collector),
-        old_to_old_slots_(old_to_old_slots),
-        old_to_new_slots_(old_to_new_slots) {}
+  explicit RecordMigratedSlotVisitor(MarkCompactCollector* collector)
+      : collector_(collector) {}
 
   V8_INLINE void VisitPointer(Object** p) override {
-    collector_->RecordMigratedSlot(*p, reinterpret_cast<Address>(p),
-                                   old_to_old_slots_, old_to_new_slots_);
+    RecordMigratedSlot(*p, reinterpret_cast<Address>(p));
   }
 
   V8_INLINE void VisitPointers(Object** start, Object** end) override {
     while (start < end) {
-      collector_->RecordMigratedSlot(*start, reinterpret_cast<Address>(start),
-                                     old_to_old_slots_, old_to_new_slots_);
+      RecordMigratedSlot(*start, reinterpret_cast<Address>(start));
       ++start;
     }
   }
 
   V8_INLINE void VisitCodeEntry(Address code_entry_slot) override {
     if (collector_->compacting_) {
       Address code_entry = Memory::Address_at(code_entry_slot);
       if (Page::FromAddress(code_entry)->IsEvacuationCandidate()) {
-        old_to_old_slots_->Record(CODE_ENTRY_SLOT, code_entry_slot);
+        RememberedSet<OLD_TO_OLD>::InsertTyped(
+            Page::FromAddress(code_entry_slot), CODE_ENTRY_SLOT,
+            code_entry_slot);
       }
     }
   }
 
  private:
+  inline void RecordMigratedSlot(Object* value, Address slot) {
+    if (collector_->heap()->InNewSpace(value)) {
+      RememberedSet<OLD_TO_NEW>::Insert(Page::FromAddress(slot), slot);
+    } else if (value->IsHeapObject() &&
+               Page::FromAddress(reinterpret_cast<Address>(value))
+                   ->IsEvacuationCandidate()) {
+      RememberedSet<OLD_TO_OLD>::Insert(Page::FromAddress(slot), slot);
+    }
+  }
+
   MarkCompactCollector* collector_;
-  LocalSlotsBuffer* old_to_old_slots_;
-  LocalSlotsBuffer* old_to_new_slots_;
 };
 
 
 // We scavenge new space simultaneously with sweeping. This is done in two
 // passes.
 //
 // The first pass migrates all alive objects from one semispace to another or
 // promotes them to old space.  Forwarding address is written directly into
 // first word of object without any encoding.  If object is dead we write
 // NULL as a forwarding address.
 //
 // The second pass updates pointers to new space in all spaces.  It is possible
 // to encounter pointers to dead new space objects during traversal of pointers
 // to new space.  We should clear them to avoid encountering them during next
 // pointer iteration.  This is an issue if the store buffer overflows and we
 // have to scan the entire old space, including dead objects, looking for
 // pointers to new space.
 void MarkCompactCollector::MigrateObject(HeapObject* dst, HeapObject* src,
-                                         int size, AllocationSpace dest,
-                                         LocalSlotsBuffer* old_to_old_slots,
-                                         LocalSlotsBuffer* old_to_new_slots) {
+                                         int size, AllocationSpace dest) {
   Address dst_addr = dst->address();
   Address src_addr = src->address();
   DCHECK(heap()->AllowedToBeMigrated(src, dest));
   DCHECK(dest != LO_SPACE);
   if (dest == OLD_SPACE) {
     DCHECK_OBJECT_SIZE(size);
     DCHECK(IsAligned(size, kPointerSize));
 
     heap()->MoveBlock(dst->address(), src->address(), size);
     if (FLAG_ignition && dst->IsBytecodeArray()) {
       PROFILE(isolate(), CodeMoveEvent(AbstractCode::cast(src), dst_addr));
     }
-    RecordMigratedSlotVisitor visitor(this, old_to_old_slots, old_to_new_slots);
+    RecordMigratedSlotVisitor visitor(this);
     dst->IterateBody(&visitor);
   } else if (dest == CODE_SPACE) {
     DCHECK_CODEOBJECT_SIZE(size, heap()->code_space());
     PROFILE(isolate(), CodeMoveEvent(AbstractCode::cast(src), dst_addr));
     heap()->MoveBlock(dst_addr, src_addr, size);
-    old_to_old_slots->Record(RELOCATED_CODE_OBJECT, dst_addr);
+    RememberedSet<OLD_TO_OLD>::InsertTyped(Page::FromAddress(dst_addr),
+                                           RELOCATED_CODE_OBJECT, dst_addr);
     Code::cast(dst)->Relocate(dst_addr - src_addr);
   } else {
     DCHECK_OBJECT_SIZE(size);
-    DCHECK(old_to_old_slots == nullptr);
     DCHECK(dest == NEW_SPACE);
     heap()->MoveBlock(dst_addr, src_addr, size);
   }
   heap()->OnMoveEvent(dst, src, size);
   Memory::Address_at(src_addr) = dst_addr;
 }
 
 static inline void UpdateTypedSlot(Isolate* isolate, ObjectVisitor* v,
                                    SlotType slot_type, Address addr) {
   switch (slot_type) {
@@ skipping 288 matching lines @@
 
 
 class MarkCompactCollector::Evacuator : public Malloced {
  public:
   explicit Evacuator(MarkCompactCollector* collector)
       : collector_(collector),
         compaction_spaces_(collector->heap()),
         local_pretenuring_feedback_(HashMap::PointersMatch,
                                     kInitialLocalPretenuringFeedbackCapacity),
         new_space_visitor_(collector->heap(), &compaction_spaces_,
-                           &old_to_old_slots_, &old_to_new_slots_,
                            &local_pretenuring_feedback_),
-        old_space_visitor_(collector->heap(), &compaction_spaces_,
-                           &old_to_old_slots_, &old_to_new_slots_),
+        old_space_visitor_(collector->heap(), &compaction_spaces_),
         duration_(0.0),
         bytes_compacted_(0) {}
 
   inline bool EvacuatePage(MemoryChunk* chunk);
 
   // Merge back locally cached info sequentially. Note that this method needs
   // to be called from the main thread.
   inline void Finalize();
 
   CompactionSpaceCollection* compaction_spaces() { return &compaction_spaces_; }
 
  private:
   static const int kInitialLocalPretenuringFeedbackCapacity = 256;
 
   Heap* heap() { return collector_->heap(); }
 
   void ReportCompactionProgress(double duration, intptr_t bytes_compacted) {
     duration_ += duration;
     bytes_compacted_ += bytes_compacted;
   }
 
   inline bool EvacuateSinglePage(MemoryChunk* p, HeapObjectVisitor* visitor);
 
   MarkCompactCollector* collector_;
 
   // Locally cached collector data.
   CompactionSpaceCollection compaction_spaces_;
-  LocalSlotsBuffer old_to_old_slots_;
-  LocalSlotsBuffer old_to_new_slots_;
   HashMap local_pretenuring_feedback_;
 
-  // Vistors for the corresponding spaces.
+  // Visitors for the corresponding spaces.
   EvacuateNewSpaceVisitor new_space_visitor_;
   EvacuateOldSpaceVisitor old_space_visitor_;
 
   // Book keeping info.
   double duration_;
   intptr_t bytes_compacted_;
 };
 
 bool MarkCompactCollector::Evacuator::EvacuateSinglePage(
     MemoryChunk* p, HeapObjectVisitor* visitor) {
@@ skipping 41 matching lines @@
   heap()->code_space()->MergeCompactionSpace(
       compaction_spaces_.Get(CODE_SPACE));
   heap()->tracer()->AddCompactionEvent(duration_, bytes_compacted_);
   heap()->IncrementPromotedObjectsSize(new_space_visitor_.promoted_size());
   heap()->IncrementSemiSpaceCopiedObjectSize(
       new_space_visitor_.semispace_copied_size());
   heap()->IncrementYoungSurvivorsCounter(
       new_space_visitor_.promoted_size() +
       new_space_visitor_.semispace_copied_size());
   heap()->MergeAllocationSitePretenuringFeedback(local_pretenuring_feedback_);
-  // Move locally recorded slots to the global remembered sets.
-  old_to_new_slots_.Iterate(
-      [](Address slot) {
-        Page* page = Page::FromAddress(slot);
-        RememberedSet<OLD_TO_NEW>::Insert(page, slot);
-      },
-      [](SlotType type, Address slot) { UNREACHABLE(); });
-  old_to_old_slots_.Iterate(
-      [](Address slot) {
-        Page* page = Page::FromAddress(slot);
-        RememberedSet<OLD_TO_OLD>::Insert(page, slot);
-      },
-      [](SlotType type, Address slot) {
-        Page* page = Page::FromAddress(slot);
-        RememberedSet<OLD_TO_OLD>::InsertTyped(page, type, slot);
-      });
 }
 
 int MarkCompactCollector::NumberOfParallelCompactionTasks(int pages,
                                                           intptr_t live_bytes) {
   if (!FLAG_parallel_compaction) return 1;
   // Compute the number of needed tasks based on a target compaction time, the
   // profiled compaction speed and marked live memory.
   //
   // The number of parallel compaction tasks is limited by:
   // - #evacuation pages
@@ skipping 770 matching lines @@
     MarkBit mark_bit = Marking::MarkBitFrom(host);
     if (Marking::IsBlack(mark_bit)) {
       RelocInfo rinfo(isolate(), pc, RelocInfo::CODE_TARGET, 0, host);
       RecordRelocSlot(host, &rinfo, target);
     }
   }
 }
 
 }  // namespace internal
 }  // namespace v8