[heap] Minor MC: Add evacuation phase

src/heap/mark-compact.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/mark-compact.h"
 
 #include "src/base/atomicops.h"
 #include "src/base/bits.h"
 #include "src/base/sys-info.h"
 #include "src/code-stubs.h"
@@ skipping 182 matching lines @@
     }
   }
 }
 
 
 static void VerifyEvacuation(NewSpace* space) {
   VerifyEvacuationVisitor visitor;
   NewSpacePageRange range(space->bottom(), space->top());
   for (auto it = range.begin(); it != range.end();) {
     Page* page = *(it++);
+    if (page->IsFlagSet(Page::CANNOT_BE_VERIFIED)) continue;
     Address current = page->area_start();
     Address limit = it != range.end() ? page->area_end() : space->top();
     CHECK(limit == space->top() || !page->Contains(space->top()));
     while (current < limit) {
       HeapObject* object = HeapObject::FromAddress(current);
       object->Iterate(&visitor);
       current += object->Size();
     }
   }
 }
 
 
 static void VerifyEvacuation(Heap* heap, PagedSpace* space) {
   if (FLAG_use_allocation_folding && (space == heap->old_space())) {
     return;
   }
   for (Page* p : *space) {
     if (p->IsEvacuationCandidate()) continue;
     VerifyEvacuation(p);
   }
 }
 
-
+template <MarkCompactMode mode>
 static void VerifyEvacuation(Heap* heap) {
-  VerifyEvacuation(heap, heap->old_space());
-  VerifyEvacuation(heap, heap->code_space());
-  VerifyEvacuation(heap, heap->map_space());
+  if (mode == MarkCompactMode::FULL) {
+    VerifyEvacuation(heap, heap->old_space());
+    VerifyEvacuation(heap, heap->code_space());
+    VerifyEvacuation(heap, heap->map_space());
+  }
   VerifyEvacuation(heap->new_space());
 
   VerifyEvacuationVisitor visitor;
   heap->IterateStrongRoots(&visitor, VISIT_ALL);
 }
 #endif  // VERIFY_HEAP
 
 
 void MarkCompactCollector::SetUp() {
   DCHECK(strcmp(Marking::kWhiteBitPattern, "00") == 0);
@@ skipping 49 matching lines @@
     if (FLAG_trace_fragmentation) {
       TraceFragmentation(heap()->map_space());
     }
 
     compacting_ = evacuation_candidates_.length() > 0;
   }
 
   return compacting_;
 }
 
+void MarkCompactCollector::CollectGarbageInYoungGeneration() {
+  AlwaysAllocateScope scope(isolate());
+  sweeper().EnsureNewSpaceCompleted();
+  MarkLiveObjectsInYoungGeneration();
+  isolate()
+      ->global_handles()
+      ->IterateNewSpaceWeakUnmodifiedRoots<GlobalHandles::HANDLE_PHANTOM_NODES>(
+          nullptr);
+  ClearNonLiveReferencesInYoungGeneration();
+  EvacuateNewSpaceAndCandidates<MarkCompactMode::YOUNG_GENERATION>();

[Hannes Payer (out of office), 2016/11/23 11:25:37]

How about restructuring EvacuateNewSpaceAndCandidates such that we can only call
EvacuateNewSpace here?

+}
+
 void MarkCompactCollector::CollectGarbage() {
   // Make sure that Prepare() has been called. The individual steps below will
   // update the state as they proceed.
   DCHECK(state_ == PREPARE_GC);
 
   MarkLiveObjects();
 
   DCHECK(heap_->incremental_marking()->IsStopped());
 
   ClearNonLiveReferences();
 
   RecordObjectStats();
 
 #ifdef VERIFY_HEAP
   if (FLAG_verify_heap) {
     VerifyMarking(heap_);
   }
 #endif
 
   StartSweepSpaces();
 
-  EvacuateNewSpaceAndCandidates();
+  EvacuateNewSpaceAndCandidates<MarkCompactMode::FULL>();
 
   Finish();
 }
 
 #ifdef VERIFY_HEAP
 void MarkCompactCollector::VerifyMarkbitsAreClean(PagedSpace* space) {
   for (Page* p : *space) {
     CHECK(p->markbits()->IsClean());
     CHECK_EQ(0, p->LiveBytes());
   }
@@ skipping 195 matching lines @@
 void MarkCompactCollector::EnsureSweepingCompleted() {
   if (!sweeper().sweeping_in_progress()) return;
 
   sweeper().EnsureCompleted();
   heap()->old_space()->RefillFreeList();
   heap()->code_space()->RefillFreeList();
   heap()->map_space()->RefillFreeList();
 
 #ifdef VERIFY_HEAP
   if (FLAG_verify_heap && !evacuation()) {
-    VerifyEvacuation(heap_);
+    VerifyEvacuation<MarkCompactMode::FULL>(heap_);
   }
 #endif
 }
 
 bool MarkCompactCollector::Sweeper::AreSweeperTasksRunning() {
   DCHECK(FLAG_concurrent_sweeping);
   while (pending_sweeper_tasks_semaphore_.WaitFor(
       base::TimeDelta::FromSeconds(0))) {
     num_sweeping_tasks_.Increment(-1);
   }
@@ skipping 1944 matching lines @@
       TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_WEAK_CLOSURE_HARMONY);
       ProcessEphemeralMarking(&root_visitor, true);
       if (heap_->UsingEmbedderHeapTracer()) {
         TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_WRAPPER_EPILOGUE);
         heap()->embedder_heap_tracer()->TraceEpilogue();
       }
     }
   }
 }
 
+void MarkCompactCollector::ClearNonLiveReferencesInYoungGeneration() {
+  TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_CLEAR);
+
+  {
+    TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_CLEAR_STRING_TABLE);
+
+    ExternalStringTableCleaner external_visitor(heap(), nullptr);
+    heap()->external_string_table_.IterateNewSpaceStrings(&external_visitor);
+    heap()->external_string_table_.CleanupNewSpaceStrings();
+  }
+}
 
 void MarkCompactCollector::ClearNonLiveReferences() {
   TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_CLEAR);
 
   {
     TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_CLEAR_STRING_TABLE);
 
     // Prune the string table removing all strings only pointed to by the
     // string table.  Cannot use string_table() here because the string
     // table is marked.
@@ skipping 514 matching lines @@
     return String::cast(map_word.ToForwardingAddress());
   }
 
   return String::cast(*p);
 }
 
 void MarkCompactCollector::EvacuateNewSpacePrologue() {
   NewSpace* new_space = heap()->new_space();
   // Append the list of new space pages to be processed.
   for (Page* p : NewSpacePageRange(new_space->bottom(), new_space->top())) {
+    p->ClearFlag(Page::CANNOT_BE_VERIFIED);
     newspace_evacuation_candidates_.Add(p);
   }
   new_space->Flip();
   new_space->ResetAllocationInfo();
 }
 
 class MarkCompactCollector::Evacuator : public Malloced {
  public:
   enum EvacuationMode {
     kObjectsNewToOld,
@@ skipping 230 matching lines @@
           // sweeper, which will happen after updating pointers.
           *data += 1;
         }
         break;
       default:
         UNREACHABLE();
     }
   }
 };
 
+template <MarkCompactMode mode>
 void MarkCompactCollector::EvacuatePagesInParallel() {
   PageParallelJob<EvacuationJobTraits> job(
       heap_, heap_->isolate()->cancelable_task_manager(),
       &page_parallel_job_semaphore_);
 
   int abandoned_pages = 0;
   intptr_t live_bytes = 0;
-  for (Page* page : evacuation_candidates_) {
-    live_bytes += page->LiveBytes();
-    job.AddPage(page, &abandoned_pages);
+  if (mode == MarkCompactMode::FULL) {
+    for (Page* page : evacuation_candidates_) {
+      live_bytes += page->LiveBytes();
+      job.AddPage(page, &abandoned_pages);
+    }
   }
 
   const bool reduce_memory = heap()->ShouldReduceMemory();
   const Address age_mark = heap()->new_space()->age_mark();
   for (Page* page : newspace_evacuation_candidates_) {
     live_bytes += page->LiveBytes();
     if (!reduce_memory && !page->NeverEvacuate() &&
         (page->LiveBytes() > Evacuator::PageEvacuationThreshold()) &&
         !page->Contains(age_mark)) {
       if (page->IsFlagSet(MemoryChunk::NEW_SPACE_BELOW_AGE_MARK)) {
@@ skipping 46 matching lines @@
       HeapObject* heap_object = HeapObject::cast(object);
       MapWord map_word = heap_object->map_word();
       if (map_word.IsForwardingAddress()) {
         return map_word.ToForwardingAddress();
       }
     }
     return object;
   }
 };
 
+class MinorMCWeakObjectRetainer : public WeakObjectRetainer {
+ public:
+  explicit MinorMCWeakObjectRetainer(Heap* heap) : heap_(heap) {}
+
+  virtual Object* RetainAs(Object* object) {
+    if (!heap_->InFromSpace(object)) {
+      return object;
+    }
+
+    MapWord map_word = HeapObject::cast(object)->map_word();
+    if (map_word.IsForwardingAddress()) {
+      return map_word.ToForwardingAddress();
+    }
+    return NULL;
+  }
+
+ private:
+  Heap* heap_;
+};
+
 MarkCompactCollector::Sweeper::ClearOldToNewSlotsMode
 MarkCompactCollector::Sweeper::GetClearOldToNewSlotsMode(Page* p) {
   AllocationSpace identity = p->owner()->identity();
   if (p->old_to_new_slots() &&
       (identity == OLD_SPACE || identity == MAP_SPACE)) {
     return MarkCompactCollector::Sweeper::CLEAR_REGULAR_SLOTS;
   } else if (p->typed_old_to_new_slots() && identity == CODE_SPACE) {
     return MarkCompactCollector::Sweeper::CLEAR_TYPED_SLOTS;
   }
   return MarkCompactCollector::Sweeper::DO_NOT_CLEAR;
@@ skipping 203 matching lines @@
   }
   page->SetLiveBytes(new_live_size);
 }
 
 void MarkCompactCollector::Sweeper::AddSweptPageSafe(PagedSpace* space,
                                                      Page* page) {
   base::LockGuard<base::Mutex> guard(&mutex_);
   swept_list_[space->identity()].Add(page);
 }
 
+template <MarkCompactMode mode>
 void MarkCompactCollector::EvacuateNewSpaceAndCandidates() {
   TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_EVACUATE);
   Heap::RelocationLock relocation_lock(heap());
 
   {
     TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_EVACUATE_COPY);
     EvacuationScope evacuation_scope(this);
 
     EvacuateNewSpacePrologue();
-    EvacuatePagesInParallel();
+    EvacuatePagesInParallel<mode>();
     heap()->new_space()->set_age_mark(heap()->new_space()->top());
   }
 
-  UpdatePointersAfterEvacuation();
+  UpdatePointersAfterEvacuation<mode>();
 
-  if (!heap()->new_space()->Rebalance()) {
-    FatalProcessOutOfMemory("NewSpace::Rebalance");
+  {
+    TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_EVACUATE_REBALANCE);
+    if (!heap()->new_space()->Rebalance()) {
+      FatalProcessOutOfMemory("NewSpace::Rebalance");
+    }
   }
 
   // Give pages that are queued to be freed back to the OS. Note that filtering
   // slots only handles old space (for unboxed doubles), and thus map space can
   // still contain stale pointers. We only free the chunks after pointer updates
   // to still have access to page headers.
   heap()->memory_allocator()->unmapper()->FreeQueuedChunks();
 
   {
     TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_EVACUATE_CLEAN_UP);
 
     for (Page* p : newspace_evacuation_candidates_) {
       if (p->IsFlagSet(Page::PAGE_NEW_NEW_PROMOTION)) {
         p->ClearFlag(Page::PAGE_NEW_NEW_PROMOTION);
-        sweeper().AddPage(p->owner()->identity(), p);
+        p->SetFlag(Page::CANNOT_BE_VERIFIED);
+        if (mode == MarkCompactMode::FULL)
+          sweeper().AddPage(p->owner()->identity(), p);
       } else if (p->IsFlagSet(Page::PAGE_NEW_OLD_PROMOTION)) {
         p->ClearFlag(Page::PAGE_NEW_OLD_PROMOTION);
+        p->SetFlag(Page::CANNOT_BE_VERIFIED);
         p->ForAllFreeListCategories(
             [](FreeListCategory* category) { DCHECK(!category->is_linked()); });
-        sweeper().AddPage(p->owner()->identity(), p);
+        if (mode == MarkCompactMode::FULL)
+          sweeper().AddPage(p->owner()->identity(), p);
+      }
+      if (mode == MarkCompactMode::YOUNG_GENERATION) {
+        p->ClearLiveness();
       }
     }
     newspace_evacuation_candidates_.Rewind(0);
 
-    for (Page* p : evacuation_candidates_) {
-      // Important: skip list should be cleared only after roots were updated
-      // because root iteration traverses the stack and might have to find
-      // code objects from non-updated pc pointing into evacuation candidate.
-      SkipList* list = p->skip_list();
-      if (list != NULL) list->Clear();
-      if (p->IsFlagSet(Page::COMPACTION_WAS_ABORTED)) {
-        sweeper().AddPage(p->owner()->identity(), p);
-        p->ClearFlag(Page::COMPACTION_WAS_ABORTED);
+    if (mode == MarkCompactMode::FULL) {
+      for (Page* p : evacuation_candidates_) {
+        // Important: skip list should be cleared only after roots were updated
+        // because root iteration traverses the stack and might have to find
+        // code objects from non-updated pc pointing into evacuation candidate.
+        SkipList* list = p->skip_list();
+        if (list != NULL) list->Clear();
+        if (p->IsFlagSet(Page::COMPACTION_WAS_ABORTED)) {
+          sweeper().AddPage(p->owner()->identity(), p);
+          p->ClearFlag(Page::COMPACTION_WAS_ABORTED);
+        }
       }
+
+      // Deallocate evacuated candidate pages.
+      ReleaseEvacuationCandidates();
     }
-
-    // Deallocate evacuated candidate pages.
-    ReleaseEvacuationCandidates();
   }
 
 #ifdef VERIFY_HEAP
   if (FLAG_verify_heap && !sweeper().sweeping_in_progress()) {
-    VerifyEvacuation(heap());
+    VerifyEvacuation<mode>(heap());
   }
 #endif
 }
 
 template <PointerDirection direction>
 class PointerUpdateJobTraits {
  public:
   typedef int PerPageData;  // Per page data is not used in this job.
   typedef int PerTaskData;  // Per task data is not used in this job.
 
@@ skipping 167 matching lines @@
     Address start =
         page->Contains(space_start) ? space_start : page->area_start();
     Address end = page->Contains(space_end) ? space_end : page->area_end();
     job.AddPage(page, std::make_pair(start, end));
   }
   PointersUpdatingVisitor visitor;
   int num_tasks = FLAG_parallel_pointer_update ? job.NumberOfPages() : 1;
   job.Run(num_tasks, [&visitor](int i) { return &visitor; });
 }
 
+template <MarkCompactMode mode>
 void MarkCompactCollector::UpdatePointersAfterEvacuation() {
   TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS);
 
   PointersUpdatingVisitor updating_visitor;
 
   {
     TRACE_GC(heap()->tracer(),
              GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_TO_NEW);
     UpdateToSpacePointersInParallel(heap_, &page_parallel_job_semaphore_);
     // Update roots.
     heap_->IterateRoots(&updating_visitor, VISIT_ALL_IN_SWEEP_NEWSPACE);
     UpdatePointersInParallel<OLD_TO_NEW>(heap_, &page_parallel_job_semaphore_);
   }
 
-  {
+  if (mode == MarkCompactMode::FULL) {
     Heap* heap = this->heap();
     TRACE_GC(heap->tracer(),
              GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_TO_EVACUATED);
     UpdatePointersInParallel<OLD_TO_OLD>(heap_, &page_parallel_job_semaphore_);
   }
 
   {
     TRACE_GC(heap()->tracer(),
              GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_WEAK);
     // Update pointers from external string table.
-    heap_->UpdateReferencesInExternalStringTable(
-        &UpdateReferenceInExternalStringTableEntry);
-
-    EvacuationWeakObjectRetainer evacuation_object_retainer;
-    heap()->ProcessWeakListRoots(&evacuation_object_retainer);
+    if (mode == MarkCompactMode::FULL) {
+      heap_->UpdateReferencesInExternalStringTable(
+          &UpdateReferenceInExternalStringTableEntry);
+      EvacuationWeakObjectRetainer evacuation_object_retainer;
+      heap()->ProcessWeakListRoots(&evacuation_object_retainer);
+    } else {
+      heap_->UpdateNewSpaceReferencesInExternalStringTable(
+          &UpdateReferenceInExternalStringTableEntry);
+      MinorMCWeakObjectRetainer weak_object_retainer(heap());
+      heap()->ProcessYoungWeakReferences(&weak_object_retainer);
+    }
   }
 }
 
 
 void MarkCompactCollector::ReleaseEvacuationCandidates() {
   for (Page* p : evacuation_candidates_) {
     if (!p->IsEvacuationCandidate()) continue;
     PagedSpace* space = static_cast<PagedSpace*>(p->owner());
     p->ResetLiveBytes();
     CHECK(p->SweepingDone());
@@ skipping 205 matching lines @@
       // The target is always in old space, we don't have to record the slot in
       // the old-to-new remembered set.
       DCHECK(!heap()->InNewSpace(target));
       RecordRelocSlot(host, &rinfo, target);
     }
   }
 }
 
 }  // namespace internal
 }  // namespace v8