🏄 [heap] Add page evacuation mode for new->old

src/heap/mark-compact.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/mark-compact.h"
 
 #include "src/base/atomicops.h"
 #include "src/base/bits.h"
 #include "src/base/sys-info.h"
 #include "src/code-stubs.h"
@@ skipping 42 matching lines @@
       state_(IDLE),
 #endif
       marking_parity_(ODD_MARKING_PARITY),
       was_marked_incrementally_(false),
       evacuation_(false),
       heap_(heap),
       marking_deque_memory_(NULL),
       marking_deque_memory_committed_(0),
       code_flusher_(nullptr),
       have_code_to_deoptimize_(false),
+      sweeping_list_shared_(nullptr),
       compacting_(false),
       sweeping_in_progress_(false),
       pending_sweeper_tasks_semaphore_(0),
       pending_compaction_tasks_semaphore_(0) {
 }
 
 #ifdef VERIFY_HEAP
 class VerifyMarkingVisitor : public ObjectVisitor {
  public:
   explicit VerifyMarkingVisitor(Heap* heap) : heap_(heap) {}
@@ skipping 401 matching lines @@
     DCHECK_LE(space_to_start_, LAST_PAGED_SPACE);
     const int offset = space_to_start_ - FIRST_PAGED_SPACE;
     const int num_spaces = LAST_PAGED_SPACE - FIRST_PAGED_SPACE + 1;
     for (int i = 0; i < num_spaces; i++) {
       const int space_id = FIRST_PAGED_SPACE + ((i + offset) % num_spaces);
       DCHECK_GE(space_id, FIRST_PAGED_SPACE);
       DCHECK_LE(space_id, LAST_PAGED_SPACE);
       heap_->mark_compact_collector()->SweepInParallel(
           heap_->paged_space(space_id), 0);
     }
+    std::vector<Page*>* shared_sweeping_list = nullptr;
+    {
+      base::LockGuard<base::Mutex> guard(
+          heap_->mark_compact_collector()->swept_pages_mutex());
+      shared_sweeping_list =
+          heap_->mark_compact_collector()->sweeping_list_shared_;
+    }
+    if (shared_sweeping_list != nullptr) {
+      heap_->mark_compact_collector()->SweepInParallel(*shared_sweeping_list,
+                                                       heap_->old_space(), 0);
+    }
     heap_->mark_compact_collector()->pending_sweeper_tasks_semaphore_.Signal();
   }
 
   Heap* heap_;
   AllocationSpace space_to_start_;
 
   DISALLOW_COPY_AND_ASSIGN(SweeperTask);
 };
 
 
 void MarkCompactCollector::StartSweeperThreads() {
   V8::GetCurrentPlatform()->CallOnBackgroundThread(
       new SweeperTask(heap(), OLD_SPACE), v8::Platform::kShortRunningTask);
   V8::GetCurrentPlatform()->CallOnBackgroundThread(
       new SweeperTask(heap(), CODE_SPACE), v8::Platform::kShortRunningTask);
-  V8::GetCurrentPlatform()->CallOnBackgroundThread(
-      new SweeperTask(heap(), MAP_SPACE), v8::Platform::kShortRunningTask);
 }
 
 
 void MarkCompactCollector::SweepOrWaitUntilSweepingCompleted(Page* page) {
   PagedSpace* owner = reinterpret_cast<PagedSpace*>(page->owner());
   if (!page->SweepingDone()) {
     SweepInParallel(page, owner);
     if (!page->SweepingDone()) {
       // We were not able to sweep that page, i.e., a concurrent
       // sweeper thread currently owns this page. Wait for the sweeper
@@ skipping 1247 matching lines @@
     return allocation;
   }
 
   LocalAllocationBuffer buffer_;
   AllocationSpace space_to_allocate_;
   intptr_t promoted_size_;
   intptr_t semispace_copied_size_;
   HashMap* local_pretenuring_feedback_;
 };
 
+class MarkCompactCollector::EvacuateNewSpacePageVisitor final
+    : public MarkCompactCollector::HeapObjectVisitor {
+ public:
+  EvacuateNewSpacePageVisitor() : promoted_size_(0) {}
+
+  static void MoveToOldSpace(NewSpacePage* page, PagedSpace* owner) {
+    page->heap()->new_space()->ReplaceWithEmptyPage(page);

[Hannes Payer (out of office), 2016/04/08 10:42:40]

Why don't we take care of the new space size at the end of the GC? Maybe we
decide to shrink new space and don't need the extra page... WDYT?

[Michael Lippautz, 2016/04/08 11:30:00]

As discussed offline: Let's keep it for now. We can unify growing/shrinking in
another CL.

+    Page* new_page = Page::Convert(page, owner);
+    new_page->SetFlag(Page::FAST_EVACUATION);
+  }
+
+  bool Visit(HeapObject* object) {
+    promoted_size_ += object->Size();
+    if (V8_UNLIKELY(object->IsJSArrayBuffer())) {
+      object->GetHeap()->array_buffer_tracker()->Promote(
+          JSArrayBuffer::cast(object));
+    }
+    RecordMigratedSlotVisitor visitor;
+    object->IterateBodyFast(&visitor);
+    return true;
+  }
+
+  intptr_t promoted_size() { return promoted_size_; }
+
+ private:
+  intptr_t promoted_size_;
+};
 
 class MarkCompactCollector::EvacuateOldSpaceVisitor final
     : public MarkCompactCollector::EvacuateVisitorBase {
  public:
   EvacuateOldSpaceVisitor(Heap* heap,
                           CompactionSpaceCollection* compaction_spaces)
       : EvacuateVisitorBase(heap, compaction_spaces) {}
 
   bool Visit(HeapObject* object) override {
     CompactionSpace* target_space = compaction_spaces_->Get(
@@ skipping 1145 matching lines @@
     newspace_evacuation_candidates_.Add(it.next());
   }
   new_space->Flip();
   new_space->ResetAllocationInfo();
 }
 
 void MarkCompactCollector::EvacuateNewSpaceEpilogue() {
   newspace_evacuation_candidates_.Rewind(0);
 }
 
-
 class MarkCompactCollector::Evacuator : public Malloced {
  public:
+  // NewSpacePages with more live bytes than this threshold qualify for fast
+  // evacuation.
+  static int FastEvacuationThreshold() {
+    return FLAG_page_evacuation_threshold * NewSpacePage::kAllocatableMemory /
+           100;
+  }
+
   explicit Evacuator(MarkCompactCollector* collector)
       : collector_(collector),
         compaction_spaces_(collector->heap()),
         local_pretenuring_feedback_(HashMap::PointersMatch,
                                     kInitialLocalPretenuringFeedbackCapacity),
-        new_space_visitor_(collector->heap(), &compaction_spaces_,
-                           &local_pretenuring_feedback_),
-        old_space_visitor_(collector->heap(), &compaction_spaces_),
+        evac_new_space_visitor_(collector->heap(), &compaction_spaces_,
+                                &local_pretenuring_feedback_),
+        evac_new_space_page_visitor_(),
+        evac_old_space_visitor_(collector->heap(), &compaction_spaces_),
         duration_(0.0),
         bytes_compacted_(0) {}
 
   inline bool EvacuatePage(MemoryChunk* chunk);
 
   // Merge back locally cached info sequentially. Note that this method needs
   // to be called from the main thread.
   inline void Finalize();
 
   CompactionSpaceCollection* compaction_spaces() { return &compaction_spaces_; }
 
  private:
   static const int kInitialLocalPretenuringFeedbackCapacity = 256;
 
   Heap* heap() { return collector_->heap(); }
 
   void ReportCompactionProgress(double duration, intptr_t bytes_compacted) {
     duration_ += duration;
     bytes_compacted_ += bytes_compacted;
   }
 
+  template <IterationMode mode>
   inline bool EvacuateSinglePage(MemoryChunk* p, HeapObjectVisitor* visitor);
 
   MarkCompactCollector* collector_;
 
   // Locally cached collector data.
   CompactionSpaceCollection compaction_spaces_;
   HashMap local_pretenuring_feedback_;
 
   // Visitors for the corresponding spaces.
-  EvacuateNewSpaceVisitor new_space_visitor_;
-  EvacuateOldSpaceVisitor old_space_visitor_;
+  EvacuateNewSpaceVisitor evac_new_space_visitor_;
+  EvacuateNewSpacePageVisitor evac_new_space_page_visitor_;
+  EvacuateOldSpaceVisitor evac_old_space_visitor_;
 
   // Book keeping info.
   double duration_;
   intptr_t bytes_compacted_;
 };
 
+template <MarkCompactCollector::IterationMode mode>
 bool MarkCompactCollector::Evacuator::EvacuateSinglePage(
     MemoryChunk* p, HeapObjectVisitor* visitor) {
   bool success = false;
-  DCHECK(p->IsEvacuationCandidate() || p->InNewSpace());
+  DCHECK(p->IsEvacuationCandidate() || p->InNewSpace() ||
+         p->IsFlagSet(Page::FAST_EVACUATION));
   int saved_live_bytes = p->LiveBytes();
   double evacuation_time;
   {
     AlwaysAllocateScope always_allocate(heap()->isolate());
     TimedScope timed_scope(&evacuation_time);
-    success = collector_->VisitLiveObjects(p, visitor, kClearMarkbits);
+    success = collector_->VisitLiveObjects(p, visitor, mode);
   }
   if (FLAG_trace_evacuation) {
-    PrintIsolate(heap()->isolate(),
-                 "evacuation[%p]: page=%p new_space=%d executable=%d "
-                 "live_bytes=%d time=%f\n",
-                 this, p, p->InNewSpace(),
-                 p->IsFlagSet(MemoryChunk::IS_EXECUTABLE), saved_live_bytes,
-                 evacuation_time);
+    PrintIsolate(
+        heap()->isolate(),
+        "evacuation[%p]: page=%p page_evacuation=%d new_space=%d executable=%d "
+        "live_bytes=%d time=%f\n",
+        this, p, mode == kKeepMarking, p->InNewSpace(),
+        p->IsFlagSet(MemoryChunk::IS_EXECUTABLE), saved_live_bytes,
+        evacuation_time);
   }
   if (success) {
     ReportCompactionProgress(evacuation_time, saved_live_bytes);
   }
   return success;
 }
 
 bool MarkCompactCollector::Evacuator::EvacuatePage(MemoryChunk* chunk) {
   bool success = false;
   if (chunk->InNewSpace()) {
     DCHECK_EQ(chunk->concurrent_sweeping_state().Value(),
               NewSpacePage::kSweepingDone);
-    success = EvacuateSinglePage(chunk, &new_space_visitor_);
+    success =
+        EvacuateSinglePage<kClearMarkbits>(chunk, &evac_new_space_visitor_);
     DCHECK(success);
     USE(success);
   } else {
-    DCHECK(chunk->IsEvacuationCandidate());
     DCHECK_EQ(chunk->concurrent_sweeping_state().Value(), Page::kSweepingDone);
-    success = EvacuateSinglePage(chunk, &old_space_visitor_);
+    if (chunk->IsFlagSet(MemoryChunk::FAST_EVACUATION)) {
+      success = EvacuateSinglePage<kKeepMarking>(chunk,
+                                                 &evac_new_space_page_visitor_);
+    } else {
+      DCHECK(chunk->IsEvacuationCandidate());
+      success =
+          EvacuateSinglePage<kClearMarkbits>(chunk, &evac_old_space_visitor_);
+    }
   }
   return success;
 }
 
 void MarkCompactCollector::Evacuator::Finalize() {
   heap()->old_space()->MergeCompactionSpace(compaction_spaces_.Get(OLD_SPACE));
   heap()->code_space()->MergeCompactionSpace(
       compaction_spaces_.Get(CODE_SPACE));
   heap()->tracer()->AddCompactionEvent(duration_, bytes_compacted_);
-  heap()->IncrementPromotedObjectsSize(new_space_visitor_.promoted_size());
+  heap()->IncrementPromotedObjectsSize(
+      evac_new_space_visitor_.promoted_size() +
+      evac_new_space_page_visitor_.promoted_size());
   heap()->IncrementSemiSpaceCopiedObjectSize(
-      new_space_visitor_.semispace_copied_size());
+      evac_new_space_visitor_.semispace_copied_size());
   heap()->IncrementYoungSurvivorsCounter(
-      new_space_visitor_.promoted_size() +
-      new_space_visitor_.semispace_copied_size());
+      evac_new_space_visitor_.promoted_size() +
+      evac_new_space_visitor_.semispace_copied_size() +
+      evac_new_space_page_visitor_.promoted_size());
   heap()->MergeAllocationSitePretenuringFeedback(local_pretenuring_feedback_);
 }
 
 int MarkCompactCollector::NumberOfParallelCompactionTasks(int pages,
                                                           intptr_t live_bytes) {
   if (!FLAG_parallel_compaction) return 1;
   // Compute the number of needed tasks based on a target compaction time, the
   // profiled compaction speed and marked live memory.
   //
   // The number of parallel compaction tasks is limited by:
@@ skipping 29 matching lines @@
 
   static bool ProcessPageInParallel(Heap* heap, PerTaskData evacuator,
                                     MemoryChunk* chunk, PerPageData) {
     return evacuator->EvacuatePage(chunk);
   }
 
   static void FinalizePageSequentially(Heap*, MemoryChunk* chunk, bool success,
                                        PerPageData data) {
     if (chunk->InNewSpace()) {
       DCHECK(success);
+    } else if (chunk->IsFlagSet(Page::FAST_EVACUATION)) {
+      // Nothing to do here, as the page is still owned by the compaction space.
     } else {
       Page* p = static_cast<Page*>(chunk);
       if (success) {
         DCHECK(p->IsEvacuationCandidate());
         DCHECK(p->SweepingDone());
         p->Unlink();
       } else {
         // We have partially compacted the page, i.e., some objects may have
         // moved, others are still in place.
         // We need to:
@@ skipping 18 matching lines @@
       heap_, heap_->isolate()->cancelable_task_manager());
 
   int abandoned_pages = 0;
   intptr_t live_bytes = 0;
   for (Page* page : evacuation_candidates_) {
     live_bytes += page->LiveBytes();
     job.AddPage(page, &abandoned_pages);
   }
   for (NewSpacePage* page : newspace_evacuation_candidates_) {
     live_bytes += page->LiveBytes();
+    if (!page->NeverEvacuate() &&
+        (page->LiveBytes() > Evacuator::FastEvacuationThreshold()) &&
+        page->IsFlagSet(MemoryChunk::NEW_SPACE_BELOW_AGE_MARK)) {
+      EvacuateNewSpacePageVisitor::MoveToOldSpace(page, heap()->old_space());
+    }
     job.AddPage(page, &abandoned_pages);
   }
   DCHECK_GE(job.NumberOfPages(), 1);
 
   // Used for trace summary.
   double compaction_speed = 0;
   if (FLAG_trace_evacuation) {
     compaction_speed = heap()->tracer()->CompactionSpeedInBytesPerMillisecond();
   }
 
   const int wanted_num_tasks =
       NumberOfParallelCompactionTasks(job.NumberOfPages(), live_bytes);
   Evacuator** evacuators = new Evacuator*[wanted_num_tasks];
   for (int i = 0; i < wanted_num_tasks; i++) {
     evacuators[i] = new Evacuator(this);
   }
   job.Run(wanted_num_tasks, [evacuators](int i) { return evacuators[i]; });
   for (int i = 0; i < wanted_num_tasks; i++) {
     evacuators[i]->Finalize();
     delete evacuators[i];
   }
   delete[] evacuators;
 
+  bool fast_evac_pages = false;

[Hannes Payer (out of office), 2016/04/08 10:42:40]

Can you factor this block of code out into a method. This one is getting pretty
long.

[Michael Lippautz, 2016/04/08 11:30:00]

Done.

+  std::vector<Page*>* shared_sweep_list = nullptr;
+  for (MemoryChunk* chunk : newspace_evacuation_candidates_) {
+    if (chunk->IsFlagSet(Page::FAST_EVACUATION)) {
+      Page* page = reinterpret_cast<Page*>(chunk);
+      page->ClearFlag(Page::FAST_EVACUATION);
+      page->concurrent_sweeping_state().SetValue(Page::kSweepingPending);
+      PagedSpace* space = static_cast<PagedSpace*>(page->owner());
+      DCHECK_EQ(space, heap()->old_space());
+      int to_sweep = page->area_size() - page->LiveBytes();
+      space->accounting_stats_.ShrinkSpace(to_sweep);
+      space->UnlinkFreeListCategories(page);

[Hannes Payer (out of office), 2016/04/08 10:42:40]

New space pages never have free lists. Can you DCHECK that this is the case. The
code dealing with free lists here should not be necessary.

[Michael Lippautz, 2016/04/08 11:30:00]

Done.

+      page->ForAllFreeListCategories(
+          [](FreeListCategory* category) { category->Reset(); });
+      if (shared_sweep_list == nullptr) {
+        shared_sweep_list = new std::vector<Page*>();
+      }
+      shared_sweep_list->push_back(page);
+      fast_evac_pages = true;
+    }
+  }
+  if (fast_evac_pages) {
+    {
+      base::LockGuard<base::Mutex> guard(swept_pages_mutex());
+      sweeping_list_shared_ = shared_sweep_list;
+    }
+    heap()
+        ->external_string_table_
+        .CleanUp<Heap::ExternalStringTable::CleanupMode::kPromoteOnly>();
+  }
+  if (FLAG_concurrent_sweeping) {
+    V8::GetCurrentPlatform()->CallOnBackgroundThread(
+        new SweeperTask(heap(), MAP_SPACE), v8::Platform::kShortRunningTask);
+  } else {
+    if (fast_evac_pages) {
+      SweepInParallel(*sweeping_list_shared_, heap()->old_space(), 0);
+    }
+  }
+
   if (FLAG_trace_evacuation) {
     PrintIsolate(
         isolate(),
         "%8.0f ms: evacuation-summary: parallel=%s pages=%d aborted=%d "
         "wanted_tasks=%d tasks=%d cores=%d live_bytes=%" V8_PTR_PREFIX
         "d compaction_speed=%.f\n",
         isolate()->time_millis_since_init(),
         FLAG_parallel_compaction ? "yes" : "no", job.NumberOfPages(),
         abandoned_pages, wanted_num_tasks, job.NumberOfTasks(),
         V8::GetCurrentPlatform()->NumberOfAvailableBackgroundThreads(),
@@ skipping 91 matching lines @@
     if (free_space_mode == ZAP_FREE_SPACE) {
       memset(free_start, 0xcc, size);
     }
     freed_bytes = space->UnaccountedFree(free_start, size);
     max_freed_bytes = Max(freed_bytes, max_freed_bytes);
   }
   p->concurrent_sweeping_state().SetValue(Page::kSweepingDone);
   return FreeList::GuaranteedAllocatable(static_cast<int>(max_freed_bytes));
 }
 
-
 void MarkCompactCollector::InvalidateCode(Code* code) {
   if (heap_->incremental_marking()->IsCompacting() &&
       !ShouldSkipEvacuationSlotRecording(code)) {
     DCHECK(compacting_);
 
     // If the object is white than no slots were recorded on it yet.
     MarkBit mark_bit = Marking::MarkBitFrom(code);
     if (Marking::IsWhite(mark_bit)) return;
 
     // Ignore all slots that might have been recorded in the body of the
@@ skipping 199 matching lines @@
       RememberedSet<OLD_TO_OLD>::IterateTyped(
           chunk, [isolate, visitor](SlotType type, Address slot) {
             UpdateTypedSlot(isolate, visitor, type, slot);
             return REMOVE_SLOT;
           });
     }
   }
 
   static void UpdateOldToNewSlot(HeapObject** address, HeapObject* object) {
     MapWord map_word = object->map_word();
-    // Since we only filter invalid slots in old space, the store buffer can
-    // still contain stale pointers in large object and in map spaces. Ignore
-    // these pointers here.
-    DCHECK(map_word.IsForwardingAddress() ||
-           !object->GetHeap()->old_space()->Contains(
-               reinterpret_cast<Address>(address)));
+    // There could still be stale pointers in large object space, map space,
+    // and old space for pages that have been promoted.
     if (map_word.IsForwardingAddress()) {
       // Update the corresponding slot.
       *address = map_word.ToForwardingAddress();
     }
   }
 };
 
 int NumberOfPointerUpdateTasks(int pages) {
   if (!FLAG_parallel_pointer_update) return 1;
   const int kMaxTasks = 4;
@@ skipping 115 matching lines @@
   }
   evacuation_candidates_.Rewind(0);
   compacting_ = false;
   heap()->FreeQueuedChunks();
 }
 
 
 int MarkCompactCollector::SweepInParallel(PagedSpace* space,
                                           int required_freed_bytes,
                                           int max_pages) {
+  return SweepInParallel(sweeping_list(space), space, required_freed_bytes,
+                         max_pages);
+}
+
+int MarkCompactCollector::SweepInParallel(std::vector<Page*>& pages,
+                                          PagedSpace* space,
+                                          int required_freed_bytes,
+                                          int max_pages) {
   int max_freed = 0;
   int max_freed_overall = 0;
   int page_count = 0;
-  for (Page* p : sweeping_list(space)) {
+  for (Page* p : pages) {
     max_freed = SweepInParallel(p, space);
     DCHECK(max_freed >= 0);
     if (required_freed_bytes > 0 && max_freed >= required_freed_bytes) {
       return max_freed;
     }
     max_freed_overall = Max(max_freed, max_freed_overall);
     page_count++;
     if (max_pages > 0 && page_count >= max_pages) {
       break;
     }
   }
   return max_freed_overall;
 }
 
-
 int MarkCompactCollector::SweepInParallel(Page* page, PagedSpace* space) {
   int max_freed = 0;
   if (page->mutex()->TryLock()) {
     // If this page was already swept in the meantime, we can return here.
     if (page->concurrent_sweeping_state().Value() != Page::kSweepingPending) {
       page->mutex()->Unlock();
       return 0;
     }
     page->concurrent_sweeping_state().SetValue(Page::kSweepingInProgress);
     if (space->identity() == OLD_SPACE) {
@@ skipping 124 matching lines @@
     heap_->tracer()->AddSweepingTime(heap_->MonotonicallyIncreasingTimeInMs() -
                                      start_time);
   }
 }
 
 
 void MarkCompactCollector::ParallelSweepSpacesComplete() {
   sweeping_list(heap()->old_space()).clear();
   sweeping_list(heap()->code_space()).clear();
   sweeping_list(heap()->map_space()).clear();
+  if (sweeping_list_shared_ != nullptr) {
+    base::LockGuard<base::Mutex> guard(swept_pages_mutex());
+    delete sweeping_list_shared_;
+    sweeping_list_shared_ = nullptr;
+  }
 }
 
 Isolate* MarkCompactCollector::isolate() const { return heap_->isolate(); }
 
 
 void MarkCompactCollector::Initialize() {
   MarkCompactMarkingVisitor::Initialize();
   IncrementalMarking::Initialize();
 }
 
@@ skipping 17 matching lines @@
     MarkBit mark_bit = Marking::MarkBitFrom(host);
     if (Marking::IsBlack(mark_bit)) {
       RelocInfo rinfo(isolate(), pc, RelocInfo::CODE_TARGET, 0, host);
       RecordRelocSlot(host, &rinfo, target);
     }
   }
 }
 
 }  // namespace internal
 }  // namespace v8