[heap] Minor MC: Add evacuation phase

src/heap/mark-compact.cc

Index: src/heap/mark-compact.cc
diff --git a/src/heap/mark-compact.cc b/src/heap/mark-compact.cc
index 8b1c7085c0b2fdf8190b3193b69a6059bd54e223..bfc2952d1ea051ca3efa4671f4995400c35b3347 100644
--- a/src/heap/mark-compact.cc
+++ b/src/heap/mark-compact.cc
@@ -200,6 +200,7 @@ static void VerifyEvacuation(NewSpace* space) {
   NewSpacePageRange range(space->bottom(), space->top());
   for (auto it = range.begin(); it != range.end();) {
     Page* page = *(it++);
+    if (page->IsFlagSet(Page::CANNOT_BE_VERIFIED)) continue;
     Address current = page->area_start();
     Address limit = it != range.end() ? page->area_end() : space->top();
     CHECK(limit == space->top() || !page->Contains(space->top()));
@@ -222,11 +223,13 @@ static void VerifyEvacuation(Heap* heap, PagedSpace* space) {
   }
 }
 
-
+template <MarkCompactMode mode>
 static void VerifyEvacuation(Heap* heap) {
-  VerifyEvacuation(heap, heap->old_space());
-  VerifyEvacuation(heap, heap->code_space());
-  VerifyEvacuation(heap, heap->map_space());
+  if (mode == MarkCompactMode::FULL) {
+    VerifyEvacuation(heap, heap->old_space());
+    VerifyEvacuation(heap, heap->code_space());
+    VerifyEvacuation(heap, heap->map_space());
+  }
   VerifyEvacuation(heap->new_space());
 
   VerifyEvacuationVisitor visitor;
@@ -296,6 +299,18 @@ bool MarkCompactCollector::StartCompaction() {
   return compacting_;
 }
 
+void MarkCompactCollector::CollectGarbageInYoungGeneration() {
+  AlwaysAllocateScope scope(isolate());
+  sweeper().EnsureNewSpaceCompleted();
+  MarkLiveObjectsInYoungGeneration();
+  isolate()
+      ->global_handles()
+      ->IterateNewSpaceWeakUnmodifiedRoots<GlobalHandles::HANDLE_PHANTOM_NODES>(
+          nullptr);
+  ClearNonLiveReferencesInYoungGeneration();
+  EvacuateNewSpaceAndCandidates<MarkCompactMode::YOUNG_GENERATION>();

[Hannes Payer (out of office), 2016/11/23 11:25:37]

How about restructuring EvacuateNewSpaceAndCandidates such that we can only call
EvacuateNewSpace here?

+}
+
 void MarkCompactCollector::CollectGarbage() {
   // Make sure that Prepare() has been called. The individual steps below will
   // update the state as they proceed.
@@ -317,7 +332,7 @@ void MarkCompactCollector::CollectGarbage() {
 
   StartSweepSpaces();
 
-  EvacuateNewSpaceAndCandidates();
+  EvacuateNewSpaceAndCandidates<MarkCompactMode::FULL>();
 
   Finish();
 }
@@ -533,7 +548,7 @@ void MarkCompactCollector::EnsureSweepingCompleted() {
 
 #ifdef VERIFY_HEAP
   if (FLAG_verify_heap && !evacuation()) {
-    VerifyEvacuation(heap_);
+    VerifyEvacuation<MarkCompactMode::FULL>(heap_);
   }
 #endif
 }
@@ -2498,6 +2513,17 @@ void MarkCompactCollector::MarkLiveObjects() {
   }
 }
 
+void MarkCompactCollector::ClearNonLiveReferencesInYoungGeneration() {
+  TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_CLEAR);
+
+  {
+    TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_CLEAR_STRING_TABLE);
+
+    ExternalStringTableCleaner external_visitor(heap(), nullptr);
+    heap()->external_string_table_.IterateNewSpaceStrings(&external_visitor);
+    heap()->external_string_table_.CleanupNewSpaceStrings();
+  }
+}
 
 void MarkCompactCollector::ClearNonLiveReferences() {
   TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_CLEAR);
@@ -3032,6 +3058,7 @@ void MarkCompactCollector::EvacuateNewSpacePrologue() {
   NewSpace* new_space = heap()->new_space();
   // Append the list of new space pages to be processed.
   for (Page* p : NewSpacePageRange(new_space->bottom(), new_space->top())) {
+    p->ClearFlag(Page::CANNOT_BE_VERIFIED);
     newspace_evacuation_candidates_.Add(p);
   }
   new_space->Flip();
@@ -3282,6 +3309,7 @@ class EvacuationJobTraits {
   }
 };
 
+template <MarkCompactMode mode>
 void MarkCompactCollector::EvacuatePagesInParallel() {
   PageParallelJob<EvacuationJobTraits> job(
       heap_, heap_->isolate()->cancelable_task_manager(),
@@ -3289,9 +3317,11 @@ void MarkCompactCollector::EvacuatePagesInParallel() {
 
   int abandoned_pages = 0;
   intptr_t live_bytes = 0;
-  for (Page* page : evacuation_candidates_) {
-    live_bytes += page->LiveBytes();
-    job.AddPage(page, &abandoned_pages);
+  if (mode == MarkCompactMode::FULL) {
+    for (Page* page : evacuation_candidates_) {
+      live_bytes += page->LiveBytes();
+      job.AddPage(page, &abandoned_pages);
+    }
   }
 
   const bool reduce_memory = heap()->ShouldReduceMemory();
@@ -3358,6 +3388,26 @@ class EvacuationWeakObjectRetainer : public WeakObjectRetainer {
   }
 };
 
+class MinorMCWeakObjectRetainer : public WeakObjectRetainer {
+ public:
+  explicit MinorMCWeakObjectRetainer(Heap* heap) : heap_(heap) {}
+
+  virtual Object* RetainAs(Object* object) {
+    if (!heap_->InFromSpace(object)) {
+      return object;
+    }
+
+    MapWord map_word = HeapObject::cast(object)->map_word();
+    if (map_word.IsForwardingAddress()) {
+      return map_word.ToForwardingAddress();
+    }
+    return NULL;
+  }
+
+ private:
+  Heap* heap_;
+};
+
 MarkCompactCollector::Sweeper::ClearOldToNewSlotsMode
 MarkCompactCollector::Sweeper::GetClearOldToNewSlotsMode(Page* p) {
   AllocationSpace identity = p->owner()->identity();
@@ -3581,6 +3631,7 @@ void MarkCompactCollector::Sweeper::AddSweptPageSafe(PagedSpace* space,
   swept_list_[space->identity()].Add(page);
 }
 
+template <MarkCompactMode mode>
 void MarkCompactCollector::EvacuateNewSpaceAndCandidates() {
   TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_EVACUATE);
   Heap::RelocationLock relocation_lock(heap());
@@ -3590,14 +3641,17 @@ void MarkCompactCollector::EvacuateNewSpaceAndCandidates() {
     EvacuationScope evacuation_scope(this);
 
     EvacuateNewSpacePrologue();
-    EvacuatePagesInParallel();
+    EvacuatePagesInParallel<mode>();
     heap()->new_space()->set_age_mark(heap()->new_space()->top());
   }
 
-  UpdatePointersAfterEvacuation();
+  UpdatePointersAfterEvacuation<mode>();
 
-  if (!heap()->new_space()->Rebalance()) {
-    FatalProcessOutOfMemory("NewSpace::Rebalance");
+  {
+    TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_EVACUATE_REBALANCE);
+    if (!heap()->new_space()->Rebalance()) {
+      FatalProcessOutOfMemory("NewSpace::Rebalance");
+    }
   }
 
   // Give pages that are queued to be freed back to the OS. Note that filtering
@@ -3612,35 +3666,44 @@ void MarkCompactCollector::EvacuateNewSpaceAndCandidates() {
     for (Page* p : newspace_evacuation_candidates_) {
       if (p->IsFlagSet(Page::PAGE_NEW_NEW_PROMOTION)) {
         p->ClearFlag(Page::PAGE_NEW_NEW_PROMOTION);
-        sweeper().AddPage(p->owner()->identity(), p);
+        p->SetFlag(Page::CANNOT_BE_VERIFIED);
+        if (mode == MarkCompactMode::FULL)
+          sweeper().AddPage(p->owner()->identity(), p);
       } else if (p->IsFlagSet(Page::PAGE_NEW_OLD_PROMOTION)) {
         p->ClearFlag(Page::PAGE_NEW_OLD_PROMOTION);
+        p->SetFlag(Page::CANNOT_BE_VERIFIED);
         p->ForAllFreeListCategories(
             [](FreeListCategory* category) { DCHECK(!category->is_linked()); });
-        sweeper().AddPage(p->owner()->identity(), p);
+        if (mode == MarkCompactMode::FULL)
+          sweeper().AddPage(p->owner()->identity(), p);
+      }
+      if (mode == MarkCompactMode::YOUNG_GENERATION) {
+        p->ClearLiveness();
       }
     }
     newspace_evacuation_candidates_.Rewind(0);
 
-    for (Page* p : evacuation_candidates_) {
-      // Important: skip list should be cleared only after roots were updated
-      // because root iteration traverses the stack and might have to find
-      // code objects from non-updated pc pointing into evacuation candidate.
-      SkipList* list = p->skip_list();
-      if (list != NULL) list->Clear();
-      if (p->IsFlagSet(Page::COMPACTION_WAS_ABORTED)) {
-        sweeper().AddPage(p->owner()->identity(), p);
-        p->ClearFlag(Page::COMPACTION_WAS_ABORTED);
+    if (mode == MarkCompactMode::FULL) {
+      for (Page* p : evacuation_candidates_) {
+        // Important: skip list should be cleared only after roots were updated
+        // because root iteration traverses the stack and might have to find
+        // code objects from non-updated pc pointing into evacuation candidate.
+        SkipList* list = p->skip_list();
+        if (list != NULL) list->Clear();
+        if (p->IsFlagSet(Page::COMPACTION_WAS_ABORTED)) {
+          sweeper().AddPage(p->owner()->identity(), p);
+          p->ClearFlag(Page::COMPACTION_WAS_ABORTED);
+        }
       }
-    }
 
-    // Deallocate evacuated candidate pages.
-    ReleaseEvacuationCandidates();
+      // Deallocate evacuated candidate pages.
+      ReleaseEvacuationCandidates();
+    }
   }
 
 #ifdef VERIFY_HEAP
   if (FLAG_verify_heap && !sweeper().sweeping_in_progress()) {
-    VerifyEvacuation(heap());
+    VerifyEvacuation<mode>(heap());
   }
 #endif
 }
@@ -3828,6 +3891,7 @@ void UpdateToSpacePointersInParallel(Heap* heap, base::Semaphore* semaphore) {
   job.Run(num_tasks, [&visitor](int i) { return &visitor; });
 }
 
+template <MarkCompactMode mode>
 void MarkCompactCollector::UpdatePointersAfterEvacuation() {
   TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS);
 
@@ -3842,7 +3906,7 @@ void MarkCompactCollector::UpdatePointersAfterEvacuation() {
     UpdatePointersInParallel<OLD_TO_NEW>(heap_, &page_parallel_job_semaphore_);
   }
 
-  {
+  if (mode == MarkCompactMode::FULL) {
     Heap* heap = this->heap();
     TRACE_GC(heap->tracer(),
              GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_TO_EVACUATED);
@@ -3853,11 +3917,17 @@ void MarkCompactCollector::UpdatePointersAfterEvacuation() {
     TRACE_GC(heap()->tracer(),
              GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_WEAK);
     // Update pointers from external string table.
-    heap_->UpdateReferencesInExternalStringTable(
-        &UpdateReferenceInExternalStringTableEntry);
-
-    EvacuationWeakObjectRetainer evacuation_object_retainer;
-    heap()->ProcessWeakListRoots(&evacuation_object_retainer);
+    if (mode == MarkCompactMode::FULL) {
+      heap_->UpdateReferencesInExternalStringTable(
+          &UpdateReferenceInExternalStringTableEntry);
+      EvacuationWeakObjectRetainer evacuation_object_retainer;
+      heap()->ProcessWeakListRoots(&evacuation_object_retainer);
+    } else {
+      heap_->UpdateNewSpaceReferencesInExternalStringTable(
+          &UpdateReferenceInExternalStringTableEntry);
+      MinorMCWeakObjectRetainer weak_object_retainer(heap());
+      heap()->ProcessYoungWeakReferences(&weak_object_retainer);
+    }
   }
 }