[heap] Sweeper refactoring round 2

src/heap/mark-compact.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/mark-compact.h"
 
 #include "src/base/atomicops.h"
 #include "src/base/bits.h"
 #include "src/base/sys-info.h"
 #include "src/code-stubs.h"
@@ skipping 527 matching lines @@
 Page* MarkCompactCollector::Sweeper::GetSweptPageSafe(PagedSpace* space) {
   base::LockGuard<base::Mutex> guard(&mutex_);
   SweptList& list = swept_list_[space->identity()];
   if (list.length() > 0) {
     return list.RemoveLast();
   }
   return nullptr;
 }
 
 void MarkCompactCollector::Sweeper::EnsureCompleted() {
-  DCHECK(sweeping_in_progress_ == true);
+  if (!sweeping_in_progress_) return;
 
   // If sweeping is not completed or not running at all, we try to complete it
   // here.
   if (!FLAG_concurrent_sweeping || !IsSweepingCompleted()) {
     ForAllSweepingSpaces(
         [this](AllocationSpace space) { ParallelSweepSpace(space, 0); });
   }
 
   if (FLAG_concurrent_sweeping) {
     while (num_sweeping_tasks_ > 0) {
       pending_sweeper_tasks_semaphore_.Wait();
       num_sweeping_tasks_--;
     }
   }
 
-  DCHECK(sweeping_list_[NEW_SPACE].empty());
-  ForAllSweepingSpaces([this](AllocationSpace space) {
-    DCHECK_NULL(tmp_late_sweeping_list_[space]);
-    sweeping_list_[space].clear();
-    if (late_sweeping_list_[space] != nullptr) {
-      delete late_sweeping_list_[space];
-    }
-  });
-
+  ForAllSweepingSpaces(
+      [this](AllocationSpace space) { DCHECK(sweeping_list_[space].empty()); });
+  late_pages_ = false;
   sweeping_in_progress_ = false;
 }
 
 void MarkCompactCollector::EnsureSweepingCompleted() {
-  DCHECK(sweeper().sweeping_in_progress() == true);
+  if (!sweeper().sweeping_in_progress()) return;
 
   sweeper().EnsureCompleted();
   heap()->old_space()->RefillFreeList();
   heap()->code_space()->RefillFreeList();
   heap()->map_space()->RefillFreeList();
 
 #ifdef VERIFY_HEAP
   if (FLAG_verify_heap && !evacuation()) {
     VerifyEvacuation(heap_);
   }
@@ skipping 295 matching lines @@
   if (!was_marked_incrementally_ && FLAG_verify_heap) {
     VerifyMarkbitsAreClean();
   }
 #endif
 }
 
 
 void MarkCompactCollector::Finish() {
   TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_FINISH);
 
+  if (sweeper().contains_late_pages() && FLAG_concurrent_sweeping) {
+    // If we added some more pages during MC, we need to start at least one
+    // more task as all other tasks might already be finished.
+    sweeper().StartSweepingHelper(OLD_SPACE);
+  }
+
   // The hashing of weak_object_to_code_table is no longer valid.
   heap()->weak_object_to_code_table()->Rehash(
       heap()->isolate()->factory()->undefined_value());
 
   // Clear the marking state of live large objects.
   heap_->lo_space()->ClearMarkingStateOfLiveObjects();
 
 #ifdef DEBUG
   DCHECK(state_ == SWEEP_SPACES || state_ == RELOCATE_OBJECTS);
   state_ = IDLE;
@@ skipping 2775 matching lines @@
   evacuation_candidates_.Rewind(0);
   compacting_ = false;
   heap()->FreeQueuedChunks();
 }
 
 int MarkCompactCollector::Sweeper::ParallelSweepSpace(AllocationSpace identity,
                                                       int required_freed_bytes,
                                                       int max_pages) {
   int max_freed = 0;
   int pages_freed = 0;
-  ParallelSweepList(sweeping_list_[identity], identity, required_freed_bytes,
-                    max_pages, &max_freed, &pages_freed);
-  if ((required_freed_bytes) > 0 && (max_freed >= required_freed_bytes))
-    return max_freed;
-  if ((max_pages > 0) && (pages_freed >= max_pages)) return max_freed;
-  SweepingList* late_list = GetLateSweepingListSafe(identity);
-  if (late_list != nullptr) {
-    ParallelSweepList(*late_list, identity, required_freed_bytes, max_pages,
-                      &max_freed, &pages_freed);
+  Page* page = nullptr;
+  while ((page = GetSweepingPageSafe(identity)) != nullptr) {
+    int freed = ParallelSweepPage(page, heap_->paged_space(identity));
+    pages_freed += 1;
+    DCHECK_GE(freed, 0);
+    max_freed = Max(max_freed, freed);
+    if ((required_freed_bytes) > 0 && (max_freed >= required_freed_bytes))
+      return max_freed;
+    if ((max_pages > 0) && (pages_freed >= max_pages)) return max_freed;
   }
   return max_freed;
 }
 
-void MarkCompactCollector::Sweeper::ParallelSweepList(
-    SweepingList& list, AllocationSpace out_space, int required_freed_bytes,
-    int max_pages, int* max_freed, int* pages_freed) {
-  for (Page* p : list) {
-    int freed = ParallelSweepPage(p, heap_->paged_space(out_space));
-    *pages_freed += 1;
-    DCHECK_GE(freed, 0);
-    *max_freed = Max(*max_freed, freed);
-    if ((required_freed_bytes) > 0 && (*max_freed >= required_freed_bytes))
-      return;
-    if ((max_pages > 0) && (*pages_freed >= max_pages)) return;
-  }
-}
-
 int MarkCompactCollector::Sweeper::ParallelSweepPage(Page* page,
                                                      PagedSpace* space) {
   int max_freed = 0;
   if (page->mutex()->TryLock()) {
     // If this page was already swept in the meantime, we can return here.
     if (page->concurrent_sweeping_state().Value() != Page::kSweepingPending) {
       page->mutex()->Unlock();
       return 0;
     }
     page->concurrent_sweeping_state().SetValue(Page::kSweepingInProgress);
@@ skipping 20 matching lines @@
 void MarkCompactCollector::Sweeper::AddPage(AllocationSpace space, Page* page) {
   DCHECK(!sweeping_in_progress_);
   PrepareToBeSweptPage(space, page);
   sweeping_list_[space].push_back(page);
 }
 
 void MarkCompactCollector::Sweeper::AddLatePage(AllocationSpace space,
                                                 Page* page) {
   DCHECK(sweeping_in_progress_);
   PrepareToBeSweptPage(space, page);
-  if (tmp_late_sweeping_list_[space] == nullptr) {
-    tmp_late_sweeping_list_[space] = new SweepingList();
-  }
-  tmp_late_sweeping_list_[space]->push_back(page);
+  late_pages_ = true;
+  AddSweepingPageSafe(space, page);
 }
 
 void MarkCompactCollector::Sweeper::PrepareToBeSweptPage(AllocationSpace space,
                                                          Page* page) {
   page->concurrent_sweeping_state().SetValue(Page::kSweepingPending);
   int to_sweep = page->area_size() - page->LiveBytes();
   heap_->paged_space(space)->accounting_stats_.ShrinkSpace(to_sweep);
 }
 
-void MarkCompactCollector::Sweeper::CommitLateList(AllocationSpace space) {
-  DCHECK(sweeping_in_progress_);
+Page* MarkCompactCollector::Sweeper::GetSweepingPageSafe(
+    AllocationSpace space) {
   base::LockGuard<base::Mutex> guard(&mutex_);
-  DCHECK_NULL(late_sweeping_list_[space]);
-  late_sweeping_list_[space] = tmp_late_sweeping_list_[space];
+  Page* page = nullptr;
+  if (!sweeping_list_[space].empty()) {
+    page = sweeping_list_[space].front();
+    sweeping_list_[space].pop_front();
+  }
+  return page;
 }
 
-MarkCompactCollector::Sweeper::SweepingList*
-MarkCompactCollector::Sweeper::GetLateSweepingListSafe(AllocationSpace space) {
+void MarkCompactCollector::Sweeper::AddSweepingPageSafe(AllocationSpace space,
+                                                        Page* page) {
   base::LockGuard<base::Mutex> guard(&mutex_);
-  return late_sweeping_list_[space];
+  sweeping_list_[space].push_back(page);
 }
 
 void MarkCompactCollector::StartSweepSpace(PagedSpace* space) {
   space->ClearStats();
 
   PageIterator it(space);
 
   int will_be_swept = 0;
   bool unused_page_present = false;
 
@@ skipping 119 matching lines @@
     MarkBit mark_bit = Marking::MarkBitFrom(host);
     if (Marking::IsBlack(mark_bit)) {
       RelocInfo rinfo(isolate(), pc, RelocInfo::CODE_TARGET, 0, host);
       RecordRelocSlot(host, &rinfo, target);
     }
   }
 }
 
 }  // namespace internal
 }  // namespace v8