Reland "[heap] Unify evacuating an object for new and old generation."

src/heap/mark-compact.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/mark-compact.h"
 
 #include "src/base/atomicops.h"
 #include "src/base/bits.h"
 #include "src/base/sys-info.h"
 #include "src/code-stubs.h"
@@ skipping 1515 matching lines @@
 }
 
 
 class MarkCompactCollector::HeapObjectVisitor {
  public:
   virtual ~HeapObjectVisitor() {}
   virtual bool Visit(HeapObject* object) = 0;
 };
 
 
+class MarkCompactCollector::EvacuateVisitorBase
+    : public MarkCompactCollector::HeapObjectVisitor {
+ public:
+  EvacuateVisitorBase(Heap* heap, SlotsBuffer** evacuation_slots_buffer)
+      : heap_(heap), evacuation_slots_buffer_(evacuation_slots_buffer) {}
+
+  bool TryEvacuateObject(PagedSpace* target_space, HeapObject* object,
+                         HeapObject** target_object) {
+    int size = object->Size();
+    AllocationAlignment alignment = object->RequiredAlignment();
+    AllocationResult allocation = target_space->AllocateRaw(size, alignment);
+    if (allocation.To(target_object)) {
+      heap_->mark_compact_collector()->MigrateObject(
+          *target_object, object, size, target_space->identity(),
+          evacuation_slots_buffer_);
+      return true;
+    }
+    return false;
+  }
+
+ protected:
+  Heap* heap_;
+  SlotsBuffer** evacuation_slots_buffer_;
+};
+
+
 class MarkCompactCollector::EvacuateNewSpaceVisitor
-    : public MarkCompactCollector::HeapObjectVisitor {
+    : public MarkCompactCollector::EvacuateVisitorBase {
  public:
-  explicit EvacuateNewSpaceVisitor(Heap* heap) : heap_(heap) {}
+  explicit EvacuateNewSpaceVisitor(Heap* heap,
+                                   SlotsBuffer** evacuation_slots_buffer)
+      : EvacuateVisitorBase(heap, evacuation_slots_buffer) {}
 
-  virtual bool Visit(HeapObject* object) {
+  bool Visit(HeapObject* object) override {
     Heap::UpdateAllocationSiteFeedback(object, Heap::RECORD_SCRATCHPAD_SLOT);
     int size = object->Size();
-
-    // TODO(hpayer): Refactor EvacuateObject and call this function instead.
+    HeapObject* target_object = nullptr;
     if (heap_->ShouldBePromoted(object->address(), size) &&
-        heap_->mark_compact_collector()->TryPromoteObject(object, size)) {
+        TryEvacuateObject(heap_->old_space(), object, &target_object)) {
+      // If we end up needing more special cases, we should factor this out.
+      if (V8_UNLIKELY(target_object->IsJSArrayBuffer())) {
+        heap_->array_buffer_tracker()->Promote(
+            JSArrayBuffer::cast(target_object));
+      }
+      heap_->IncrementPromotedObjectsSize(size);
       return true;
     }
 
     AllocationAlignment alignment = object->RequiredAlignment();
     AllocationResult allocation =
         heap_->new_space()->AllocateRaw(size, alignment);
     if (allocation.IsRetry()) {
       if (!heap_->new_space()->AddFreshPage()) {
         // Shouldn't happen. We are sweeping linearly, and to-space
         // has the same number of pages as from-space, so there is
         // always room unless we are in an OOM situation.
         FatalProcessOutOfMemory("MarkCompactCollector: semi-space copy\n");
       }
       allocation = heap_->new_space()->AllocateRaw(size, alignment);
       DCHECK(!allocation.IsRetry());
     }
     Object* target = allocation.ToObjectChecked();
 
     heap_->mark_compact_collector()->MigrateObject(
         HeapObject::cast(target), object, size, NEW_SPACE, nullptr);
     if (V8_UNLIKELY(target->IsJSArrayBuffer())) {
       heap_->array_buffer_tracker()->MarkLive(JSArrayBuffer::cast(target));
     }
     heap_->IncrementSemiSpaceCopiedObjectSize(size);
     return true;
   }
-
- private:
-  Heap* heap_;
 };
 
 
 class MarkCompactCollector::EvacuateOldSpaceVisitor
-    : public MarkCompactCollector::HeapObjectVisitor {
+    : public MarkCompactCollector::EvacuateVisitorBase {
  public:
   EvacuateOldSpaceVisitor(Heap* heap,
                           CompactionSpaceCollection* compaction_spaces,
                           SlotsBuffer** evacuation_slots_buffer)
-      : heap_(heap),
-        compaction_spaces_(compaction_spaces),
-        evacuation_slots_buffer_(evacuation_slots_buffer) {}
+      : EvacuateVisitorBase(heap, evacuation_slots_buffer),
+        compaction_spaces_(compaction_spaces) {}
 
-  virtual bool Visit(HeapObject* object) {
-    int size = object->Size();
-    AllocationAlignment alignment = object->RequiredAlignment();
+  bool Visit(HeapObject* object) override {
+    CompactionSpace* target_space = compaction_spaces_->Get(
+        Page::FromAddress(object->address())->owner()->identity());
     HeapObject* target_object = nullptr;
-    AllocationSpace id =
-        Page::FromAddress(object->address())->owner()->identity();
-    AllocationResult allocation =
-        compaction_spaces_->Get(id)->AllocateRaw(size, alignment);
-    if (!allocation.To(&target_object)) {
-      return false;
+    if (TryEvacuateObject(target_space, object, &target_object)) {
+      DCHECK(object->map_word().IsForwardingAddress());
+      return true;
     }
-    heap_->mark_compact_collector()->MigrateObject(
-        target_object, object, size, id, evacuation_slots_buffer_);
-    DCHECK(object->map_word().IsForwardingAddress());
-    return true;
+    return false;
   }
 
  private:
-  Heap* heap_;
   CompactionSpaceCollection* compaction_spaces_;
-  SlotsBuffer** evacuation_slots_buffer_;
 };
 
 
-bool MarkCompactCollector::IterateLiveObjectsOnPage(MemoryChunk* page,
-                                                    HeapObjectVisitor* visitor,
-                                                    IterationMode mode) {
-  Address offsets[16];
-  for (MarkBitCellIterator it(page); !it.Done(); it.Advance()) {
-    Address cell_base = it.CurrentCellBase();
-    MarkBit::CellType* cell = it.CurrentCell();
-
-    if (*cell == 0) continue;
-
-    int live_objects = MarkWordToObjectStarts(*cell, cell_base, offsets);
-    for (int i = 0; i < live_objects; i++) {
-      HeapObject* object = HeapObject::FromAddress(offsets[i]);
-      DCHECK(Marking::IsBlack(Marking::MarkBitFrom(object)));
-      if (!visitor->Visit(object)) {
-        if ((mode == kClearMarkbits) && (i > 0)) {
-          page->markbits()->ClearRange(
-              page->AddressToMarkbitIndex(page->area_start()),
-              page->AddressToMarkbitIndex(offsets[i]));
-        }
-        return false;
-      }
-    }
-    if (mode == kClearMarkbits) {
-      *cell = 0;
-    }
-  }
-  return true;
-}
-
-
 void MarkCompactCollector::DiscoverGreyObjectsInSpace(PagedSpace* space) {
   PageIterator it(space);
   while (it.has_next()) {
     Page* p = it.next();
     DiscoverGreyObjectsOnPage(p);
     if (marking_deque()->IsFull()) return;
   }
 }
 
 
@@ skipping 1332 matching lines @@
   MapWord map_word = HeapObject::cast(*p)->map_word();
 
   if (map_word.IsForwardingAddress()) {
     return String::cast(map_word.ToForwardingAddress());
   }
 
   return String::cast(*p);
 }
 
 
-bool MarkCompactCollector::TryPromoteObject(HeapObject* object,
-                                            int object_size) {
-  OldSpace* old_space = heap()->old_space();
-
-  HeapObject* target = nullptr;
-  AllocationAlignment alignment = object->RequiredAlignment();
-  AllocationResult allocation = old_space->AllocateRaw(object_size, alignment);
-  if (allocation.To(&target)) {
-    MigrateObject(target, object, object_size, old_space->identity(),
-                  &migration_slots_buffer_);
-    // If we end up needing more special cases, we should factor this out.
-    if (V8_UNLIKELY(target->IsJSArrayBuffer())) {
-      heap()->array_buffer_tracker()->Promote(JSArrayBuffer::cast(target));
-    }
-    heap()->IncrementPromotedObjectsSize(object_size);
-    return true;
-  }
-
-  return false;
-}
-
-
 bool MarkCompactCollector::IsSlotInBlackObject(Page* p, Address slot,
                                                HeapObject** out_object) {
   Space* owner = p->owner();
   if (owner == heap_->lo_space() || owner == NULL) {
     Object* large_object = heap_->lo_space()->FindObject(slot);
     // This object has to exist, otherwise we would not have recorded a slot
     // for it.
     CHECK(large_object->IsHeapObject());
     HeapObject* large_heap_object = HeapObject::cast(large_object);
     if (IsMarked(large_heap_object)) {
@@ skipping 152 matching lines @@
   new_space->Flip();
   new_space->ResetAllocationInfo();
 
   int survivors_size = 0;
 
   // First pass: traverse all objects in inactive semispace, remove marks,
   // migrate live objects and write forwarding addresses.  This stage puts
   // new entries in the store buffer and may cause some pages to be marked
   // scan-on-scavenge.
   NewSpacePageIterator it(from_bottom, from_top);
-  EvacuateNewSpaceVisitor new_space_visitor(heap());
+  EvacuateNewSpaceVisitor new_space_visitor(heap(), &migration_slots_buffer_);
   while (it.has_next()) {
     NewSpacePage* p = it.next();
     survivors_size += p->LiveBytes();
-    bool ok = IterateLiveObjectsOnPage(p, &new_space_visitor, kClearMarkbits);
+    bool ok = VisitLiveObjects(p, &new_space_visitor, kClearMarkbits);
     USE(ok);
     DCHECK(ok);
   }
 
   heap_->IncrementYoungSurvivorsCounter(survivors_size);
   new_space->set_age_mark(new_space->top());
 }
 
 
 void MarkCompactCollector::AddEvacuationSlotsBufferSynchronized(
@@ skipping 181 matching lines @@
     DCHECK(static_cast<int>(p->parallel_sweeping_state().Value()) ==
            MemoryChunk::kSweepingDone);
     if (p->parallel_compaction_state().TrySetValue(
             MemoryChunk::kCompactingDone, MemoryChunk::kCompactingInProgress)) {
       if (p->IsEvacuationCandidate()) {
         DCHECK_EQ(p->parallel_compaction_state().Value(),
                   MemoryChunk::kCompactingInProgress);
         double start = heap()->MonotonicallyIncreasingTimeInMs();
         intptr_t live_bytes = p->LiveBytes();
         AlwaysAllocateScope always_allocate(isolate());
-        if (IterateLiveObjectsOnPage(p, &visitor, kClearMarkbits)) {
+        if (VisitLiveObjects(p, &visitor, kClearMarkbits)) {
           p->ResetLiveBytes();
           p->parallel_compaction_state().SetValue(
               MemoryChunk::kCompactingFinalize);
           compaction_spaces->ReportCompactionProgress(
               heap()->MonotonicallyIncreasingTimeInMs() - start, live_bytes);
         } else {
           p->parallel_compaction_state().SetValue(
               MemoryChunk::kCompactingAborted);
         }
       } else {
@@ skipping 166 matching lines @@
     DCHECK(p->IsEvacuationCandidate() ||
            p->IsFlagSet(Page::RESCAN_ON_EVACUATION));
     if (p->IsEvacuationCandidate()) {
       SlotsBuffer::RemoveObjectSlots(heap_, p->slots_buffer(), start_slot,
                                      end_slot);
     }
   }
 }
 
 
-void MarkCompactCollector::VisitLiveObjects(Page* page,
-                                            ObjectVisitor* visitor) {
-  // First pass on aborted pages.
+bool MarkCompactCollector::VisitLiveObjects(MemoryChunk* page,
+                                            HeapObjectVisitor* visitor,
+                                            IterationMode mode) {
+  Address offsets[16];
+  for (MarkBitCellIterator it(page); !it.Done(); it.Advance()) {
+    Address cell_base = it.CurrentCellBase();
+    MarkBit::CellType* cell = it.CurrentCell();
+    if (*cell == 0) continue;
+    int live_objects = MarkWordToObjectStarts(*cell, cell_base, offsets);
+    for (int i = 0; i < live_objects; i++) {
+      HeapObject* object = HeapObject::FromAddress(offsets[i]);
+      DCHECK(Marking::IsBlack(Marking::MarkBitFrom(object)));
+      if (!visitor->Visit(object)) {
+        if ((mode == kClearMarkbits) && (i > 0)) {
+          page->markbits()->ClearRange(
+              page->AddressToMarkbitIndex(page->area_start()),
+              page->AddressToMarkbitIndex(offsets[i]));
+        }
+        return false;
+      }
+    }
+    if (mode == kClearMarkbits) {
+      *cell = 0;
+    }
+  }
+  return true;
+}
+
+
+void MarkCompactCollector::VisitLiveObjectsBody(Page* page,
+                                                ObjectVisitor* visitor) {
   Address starts[16];
   for (MarkBitCellIterator it(page); !it.Done(); it.Advance()) {
     Address cell_base = it.CurrentCellBase();
     MarkBit::CellType* cell = it.CurrentCell();
     if (*cell == 0) continue;
     int live_objects = MarkWordToObjectStarts(*cell, cell_base, starts);
     for (int i = 0; i < live_objects; i++) {
       HeapObject* live_object = HeapObject::FromAddress(starts[i]);
       DCHECK(Marking::IsBlack(Marking::MarkBitFrom(live_object)));
       Map* map = live_object->synchronized_map();
@@ skipping 124 matching lines @@
 
         // Important: skip list should be cleared only after roots were updated
         // because root iteration traverses the stack and might have to find
         // code objects from non-updated pc pointing into evacuation candidate.
         SkipList* list = p->skip_list();
         if (list != NULL) list->Clear();
 
         // First pass on aborted pages, fixing up all live objects.
         if (p->IsFlagSet(Page::COMPACTION_WAS_ABORTED)) {
           p->ClearEvacuationCandidate();
-          VisitLiveObjects(p, &updating_visitor);
+          VisitLiveObjectsBody(p, &updating_visitor);
         }
       }
 
       if (p->IsFlagSet(Page::RESCAN_ON_EVACUATION)) {
         if (FLAG_gc_verbose) {
           PrintF("Sweeping 0x%" V8PRIxPTR " during evacuation.\n",
                  reinterpret_cast<intptr_t>(p));
         }
         PagedSpace* space = static_cast<PagedSpace*>(p->owner());
         p->ClearFlag(MemoryChunk::RESCAN_ON_EVACUATION);
@@ skipping 415 matching lines @@
     MarkBit mark_bit = Marking::MarkBitFrom(host);
     if (Marking::IsBlack(mark_bit)) {
       RelocInfo rinfo(isolate(), pc, RelocInfo::CODE_TARGET, 0, host);
       RecordRelocSlot(&rinfo, target);
     }
   }
 }
 
 }  // namespace internal
 }  // namespace v8