Revert of [heap] Refactor evacuation for young and old gen into visitors.

src/heap/mark-compact.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/mark-compact.h"
 
 #include "src/base/atomicops.h"
 #include "src/base/bits.h"
 #include "src/base/sys-info.h"
 #include "src/code-stubs.h"
@@ skipping 218 matching lines @@
   VerifyEvacuation(heap, heap->map_space());
   VerifyEvacuation(heap->new_space());
 
   VerifyEvacuationVisitor visitor;
   heap->IterateStrongRoots(&visitor, VISIT_ALL);
 }
 #endif  // VERIFY_HEAP
 
 
 void MarkCompactCollector::SetUp() {
-  DCHECK(strcmp(Marking::kWhiteBitPattern, "00") == 0);
-  DCHECK(strcmp(Marking::kBlackBitPattern, "10") == 0);
-  DCHECK(strcmp(Marking::kGreyBitPattern, "11") == 0);
-  DCHECK(strcmp(Marking::kImpossibleBitPattern, "01") == 0);
-
   free_list_old_space_.Reset(new FreeList(heap_->old_space()));
   free_list_code_space_.Reset(new FreeList(heap_->code_space()));
   free_list_map_space_.Reset(new FreeList(heap_->map_space()));
   EnsureMarkingDequeIsReserved();
   EnsureMarkingDequeIsCommitted(kMinMarkingDequeSize);
   slots_buffer_allocator_ = new SlotsBufferAllocator();
 }
 
 
 void MarkCompactCollector::TearDown() {
@@ skipping 1243 matching lines @@
       if (marking_deque()->IsFull()) return;
       offset += 2;
       grey_objects >>= 2;
     }
 
     grey_objects >>= (Bitmap::kBitsPerCell - 1);
   }
 }
 
 
-class MarkCompactCollector::HeapObjectVisitor {
- public:
-  virtual ~HeapObjectVisitor() {}
-  virtual bool Visit(HeapObject* object) = 0;
-};
+int MarkCompactCollector::DiscoverAndEvacuateBlackObjectsOnPage(
+    NewSpace* new_space, NewSpacePage* p) {
+  DCHECK(strcmp(Marking::kWhiteBitPattern, "00") == 0);
+  DCHECK(strcmp(Marking::kBlackBitPattern, "10") == 0);
+  DCHECK(strcmp(Marking::kGreyBitPattern, "11") == 0);
+  DCHECK(strcmp(Marking::kImpossibleBitPattern, "01") == 0);
 
+  MarkBit::CellType* cells = p->markbits()->cells();
+  int survivors_size = 0;
 
-class MarkCompactCollector::EvacuateNewSpaceVisitor
-    : public MarkCompactCollector::HeapObjectVisitor {
- public:
-  explicit EvacuateNewSpaceVisitor(Heap* heap) : heap_(heap) {}
-
-  virtual bool Visit(HeapObject* object) {
-    Heap::UpdateAllocationSiteFeedback(object, Heap::RECORD_SCRATCHPAD_SLOT);
-    int size = object->Size();
-
-    // TODO(hpayer): Refactor EvacuateObject and call this function instead.
-    if (heap_->ShouldBePromoted(object->address(), size) &&
-        heap_->mark_compact_collector()->TryPromoteObject(object, size)) {
-      return true;
-    }
-
-    AllocationAlignment alignment = object->RequiredAlignment();
-    AllocationResult allocation =
-        heap_->new_space()->AllocateRaw(size, alignment);
-    if (allocation.IsRetry()) {
-      if (!heap_->new_space()->AddFreshPage()) {
-        // Shouldn't happen. We are sweeping linearly, and to-space
-        // has the same number of pages as from-space, so there is
-        // always room unless we are in an OOM situation.
-        FatalProcessOutOfMemory("MarkCompactCollector: semi-space copy\n");
-      }
-      allocation = heap_->new_space()->AllocateRaw(size, alignment);
-      DCHECK(!allocation.IsRetry());
-    }
-    Object* target = allocation.ToObjectChecked();
-
-    heap_->mark_compact_collector()->MigrateObject(
-        HeapObject::cast(target), object, size, NEW_SPACE, nullptr);
-    if (V8_UNLIKELY(target->IsJSArrayBuffer())) {
-      heap_->array_buffer_tracker()->MarkLive(JSArrayBuffer::cast(target));
-    }
-    heap_->IncrementSemiSpaceCopiedObjectSize(size);
-    return true;
-  }
-
- private:
-  Heap* heap_;
-};
-
-
-class MarkCompactCollector::EvacuateOldSpaceVisitor
-    : public MarkCompactCollector::HeapObjectVisitor {
- public:
-  EvacuateOldSpaceVisitor(Heap* heap,
-                          CompactionSpaceCollection* compaction_spaces,
-                          SlotsBuffer** evacuation_slots_buffer)
-      : heap_(heap),
-        compaction_spaces_(compaction_spaces),
-        evacuation_slots_buffer_(evacuation_slots_buffer) {}
-
-  virtual bool Visit(HeapObject* object) {
-    int size = object->Size();
-    AllocationAlignment alignment = object->RequiredAlignment();
-    HeapObject* target_object = nullptr;
-    AllocationSpace id =
-        Page::FromAddress(object->address())->owner()->identity();
-    AllocationResult allocation =
-        compaction_spaces_->Get(id)->AllocateRaw(size, alignment);
-    if (!allocation.To(&target_object)) {
-      return false;
-    }
-    heap_->mark_compact_collector()->MigrateObject(
-        target_object, object, size, id, evacuation_slots_buffer_);
-    DCHECK(object->map_word().IsForwardingAddress());
-    return true;
-  }
-
- private:
-  Heap* heap_;
-  CompactionSpaceCollection* compaction_spaces_;
-  SlotsBuffer** evacuation_slots_buffer_;
-};
-
-
-bool MarkCompactCollector::IterateLiveObjectsOnPage(MemoryChunk* page,
-                                                    HeapObjectVisitor* visitor,
-                                                    IterationMode mode) {
-  Address offsets[16];
-  for (MarkBitCellIterator it(page); !it.Done(); it.Advance()) {
+  for (MarkBitCellIterator it(p); !it.Done(); it.Advance()) {
     Address cell_base = it.CurrentCellBase();
     MarkBit::CellType* cell = it.CurrentCell();
 
-    if (*cell == 0) continue;
+    MarkBit::CellType current_cell = *cell;
+    if (current_cell == 0) continue;
 
-    int live_objects = MarkWordToObjectStarts(*cell, cell_base, offsets);
-    for (int i = 0; i < live_objects; i++) {
-      HeapObject* object = HeapObject::FromAddress(offsets[i]);
+    int offset = 0;
+    while (current_cell != 0) {
+      int trailing_zeros = base::bits::CountTrailingZeros32(current_cell);
+      current_cell >>= trailing_zeros;
+      offset += trailing_zeros;
+      Address address = cell_base + offset * kPointerSize;
+      HeapObject* object = HeapObject::FromAddress(address);
       DCHECK(Marking::IsBlack(Marking::MarkBitFrom(object)));
-      if (!visitor->Visit(object)) {
-        if ((mode == kClearMarkbits) && (i > 0)) {
-          page->markbits()->ClearRange(
-              page->AddressToMarkbitIndex(page->area_start()),
-              page->AddressToMarkbitIndex(offsets[i]));
+
+      int size = object->Size();
+      survivors_size += size;
+
+      Heap::UpdateAllocationSiteFeedback(object, Heap::RECORD_SCRATCHPAD_SLOT);
+
+      offset += 2;
+      current_cell >>= 2;
+
+      // TODO(hpayer): Refactor EvacuateObject and call this function instead.
+      if (heap()->ShouldBePromoted(object->address(), size) &&
+          TryPromoteObject(object, size)) {
+        continue;
+      }
+
+      AllocationAlignment alignment = object->RequiredAlignment();
+      AllocationResult allocation = new_space->AllocateRaw(size, alignment);
+      if (allocation.IsRetry()) {
+        if (!new_space->AddFreshPage()) {
+          // Shouldn't happen. We are sweeping linearly, and to-space
+          // has the same number of pages as from-space, so there is
+          // always room unless we are in an OOM situation.
+          FatalProcessOutOfMemory("MarkCompactCollector: semi-space copy\n");
         }
-        return false;
+        allocation = new_space->AllocateRaw(size, alignment);
+        DCHECK(!allocation.IsRetry());
       }
+      Object* target = allocation.ToObjectChecked();
+
+      MigrateObject(HeapObject::cast(target), object, size, NEW_SPACE, nullptr);
+      if (V8_UNLIKELY(target->IsJSArrayBuffer())) {
+        heap()->array_buffer_tracker()->MarkLive(JSArrayBuffer::cast(target));
+      }
+      heap()->IncrementSemiSpaceCopiedObjectSize(size);
     }
-    if (mode == kClearMarkbits) {
-      *cell = 0;
-    }
+    *cells = 0;
   }
-  return true;
+  return survivors_size;
 }
 
 
 void MarkCompactCollector::DiscoverGreyObjectsInSpace(PagedSpace* space) {
   PageIterator it(space);
   while (it.has_next()) {
     Page* p = it.next();
     DiscoverGreyObjectsOnPage(p);
     if (marking_deque()->IsFull()) return;
   }
@@ skipping 1500 matching lines @@
   new_space->Flip();
   new_space->ResetAllocationInfo();
 
   int survivors_size = 0;
 
   // First pass: traverse all objects in inactive semispace, remove marks,
   // migrate live objects and write forwarding addresses.  This stage puts
   // new entries in the store buffer and may cause some pages to be marked
   // scan-on-scavenge.
   NewSpacePageIterator it(from_bottom, from_top);
-  EvacuateNewSpaceVisitor new_space_visitor(heap());
   while (it.has_next()) {
     NewSpacePage* p = it.next();
-    survivors_size += p->LiveBytes();
-    bool ok = IterateLiveObjectsOnPage(p, &new_space_visitor, kClearMarkbits);
-    USE(ok);
-    DCHECK(ok);
+    survivors_size += DiscoverAndEvacuateBlackObjectsOnPage(new_space, p);
   }
 
   heap_->IncrementYoungSurvivorsCounter(survivors_size);
   new_space->set_age_mark(new_space->top());
 }
 
 
 void MarkCompactCollector::AddEvacuationSlotsBufferSynchronized(
     SlotsBuffer* evacuation_slots_buffer) {
   base::LockGuard<base::Mutex> lock_guard(&evacuation_slots_buffers_mutex_);
   evacuation_slots_buffers_.Add(evacuation_slots_buffer);
 }
 
 
+bool MarkCompactCollector::EvacuateLiveObjectsFromPage(
+    Page* p, PagedSpace* target_space, SlotsBuffer** evacuation_slots_buffer) {
+  AlwaysAllocateScope always_allocate(isolate());
+  DCHECK(p->IsEvacuationCandidate() && !p->WasSwept());
+
+  Address starts[16];
+  for (MarkBitCellIterator it(p); !it.Done(); it.Advance()) {
+    Address cell_base = it.CurrentCellBase();
+    MarkBit::CellType* cell = it.CurrentCell();
+
+    if (*cell == 0) continue;
+
+    int live_objects = MarkWordToObjectStarts(*cell, cell_base, starts);
+    for (int i = 0; i < live_objects; i++) {
+      HeapObject* object = HeapObject::FromAddress(starts[i]);
+      DCHECK(Marking::IsBlack(Marking::MarkBitFrom(object)));
+
+      int size = object->Size();
+      AllocationAlignment alignment = object->RequiredAlignment();
+      HeapObject* target_object = nullptr;
+      AllocationResult allocation = target_space->AllocateRaw(size, alignment);
+      if (!allocation.To(&target_object)) {
+        // We need to abort compaction for this page. Make sure that we reset
+        // the mark bits for objects that have already been migrated.
+        if (i > 0) {
+          p->markbits()->ClearRange(p->AddressToMarkbitIndex(p->area_start()),
+                                    p->AddressToMarkbitIndex(starts[i]));
+        }
+        return false;
+      }
+
+      MigrateObject(target_object, object, size, target_space->identity(),
+                    evacuation_slots_buffer);
+      DCHECK(object->map_word().IsForwardingAddress());
+    }
+
+    // Clear marking bits for current cell.
+    *cell = 0;
+  }
+  p->ResetLiveBytes();
+  return true;
+}
+
+
 int MarkCompactCollector::NumberOfParallelCompactionTasks() {
   if (!FLAG_parallel_compaction) return 1;
   // Compute the number of needed tasks based on a target compaction time, the
   // profiled compaction speed and marked live memory.
   //
   // The number of parallel compaction tasks is limited by:
   // - #evacuation pages
   // - (#cores - 1)
   // - a hard limit
   const double kTargetCompactionTimeInMs = 1;
@@ skipping 144 matching lines @@
       pending_compaction_tasks_semaphore_.Wait();
     }
   }
   compaction_in_progress_ = false;
 }
 
 
 void MarkCompactCollector::EvacuatePages(
     CompactionSpaceCollection* compaction_spaces,
     SlotsBuffer** evacuation_slots_buffer) {
-  EvacuateOldSpaceVisitor visitor(heap(), compaction_spaces,
-                                  evacuation_slots_buffer);
   for (int i = 0; i < evacuation_candidates_.length(); i++) {
     Page* p = evacuation_candidates_[i];
     DCHECK(p->IsEvacuationCandidate() ||
            p->IsFlagSet(Page::RESCAN_ON_EVACUATION));
     DCHECK(static_cast<int>(p->parallel_sweeping_state().Value()) ==
            MemoryChunk::kSweepingDone);
     if (p->parallel_compaction_state().TrySetValue(
             MemoryChunk::kCompactingDone, MemoryChunk::kCompactingInProgress)) {
       if (p->IsEvacuationCandidate()) {
         DCHECK_EQ(p->parallel_compaction_state().Value(),
                   MemoryChunk::kCompactingInProgress);
         double start = heap()->MonotonicallyIncreasingTimeInMs();
         intptr_t live_bytes = p->LiveBytes();
-        if (IterateLiveObjectsOnPage(p, &visitor, kClearMarkbits)) {
-          p->ResetLiveBytes();
+        if (EvacuateLiveObjectsFromPage(
+                p, compaction_spaces->Get(p->owner()->identity()),
+                evacuation_slots_buffer)) {
           p->parallel_compaction_state().SetValue(
               MemoryChunk::kCompactingFinalize);
           compaction_spaces->ReportCompactionProgress(
               heap()->MonotonicallyIncreasingTimeInMs() - start, live_bytes);
         } else {
           p->parallel_compaction_state().SetValue(
               MemoryChunk::kCompactingAborted);
         }
       } else {
         // There could be popular pages in the list of evacuation candidates
@@ skipping 770 matching lines @@
     MarkBit mark_bit = Marking::MarkBitFrom(host);
     if (Marking::IsBlack(mark_bit)) {
       RelocInfo rinfo(pc, RelocInfo::CODE_TARGET, 0, host);
       RecordRelocSlot(&rinfo, target);
     }
   }
 }
 
 }  // namespace internal
 }  // namespace v8