[heap] Refactor Sweeper

src/heap/mark-compact.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef V8_HEAP_MARK_COMPACT_H_
 #define V8_HEAP_MARK_COMPACT_H_
 
 #include "src/base/bits.h"
 #include "src/heap/spaces.h"
 #include "src/heap/store-buffer.h"
@@ skipping 382 matching lines @@
   Address cell_base_;
   MarkBit::CellType current_cell_;
 };
 
 // -------------------------------------------------------------------------
 // Mark-Compact collector
 class MarkCompactCollector {
  public:
   class Evacuator;
 
+  class Sweeper {
+   public:
+    class SweeperTask;
+
+    enum SweepingMode { SWEEP_ONLY, SWEEP_AND_VISIT_LIVE_OBJECTS };
+    enum SkipListRebuildingMode { REBUILD_SKIP_LIST, IGNORE_SKIP_LIST };
+    enum FreeSpaceTreatmentMode { IGNORE_FREE_SPACE, ZAP_FREE_SPACE };
+    enum SweepingParallelism { SWEEP_ON_MAIN_THREAD, SWEEP_IN_PARALLEL };
+
+    typedef std::vector<Page*> SweepingList;
+    typedef List<Page*> SweptList;
+
+    template <SweepingMode sweeping_mode, SweepingParallelism parallelism,
+              SkipListRebuildingMode skip_list_mode,
+              FreeSpaceTreatmentMode free_space_mode>
+    static int RawSweep(PagedSpace* space, Page* p, ObjectVisitor* v);
+
+    explicit Sweeper(Heap* heap)
+        : heap_(heap),
+          pending_sweeper_tasks_semaphore_(0),
+          sweeping_in_progress_(false),
+          num_sweeping_tasks_(0) {
+      ForAllSweepingSpaces([this](AllocationSpace space) {
+        late_sweeping_list_[space] = nullptr;
+        tmp_late_sweeping_list_[space] = nullptr;
+      });
+    }
+
+    bool sweeping_in_progress() { return sweeping_in_progress_; }
+
+    void AddPage(AllocationSpace space, Page* page);
+    void AddLatePage(AllocationSpace space, Page* page);
+    void CommitLateList(AllocationSpace space);
+
+    int ParallelSweepSpace(AllocationSpace identity, int required_freed_bytes,
+                           int max_pages = 0);
+    int ParallelSweepPage(Page* page, PagedSpace* space);
+
+    void StartSweeping();
+    void StartSweepingHelper(AllocationSpace space_to_start);
+    void EnsureCompleted();
+    bool IsSweepingCompleted();
+    void SweepOrWaitUntilSweepingCompleted(Page* page);
+
+    void AddSweptPageSafe(PagedSpace* space, Page* page);
+    Page* GetSweptPageSafe(PagedSpace* space);
+
+   private:
+    static const int kAllocationSpaces = LAST_PAGED_SPACE + 1;
+
+    template <typename Callback>
+    void ForAllSweepingSpaces(Callback callback) {
+      for (int i = 0; i < kAllocationSpaces; i++) {
+        callback(static_cast<AllocationSpace>(i));
+      }
+    }
+
+    SweepingList* GetLateSweepingListSafe(AllocationSpace space);
+
+    void PrepareToBeSweptPage(AllocationSpace space, Page* page);
+    void ParallelSweepList(SweepingList& list, AllocationSpace out_space,
+                           int required_freed_bytes, int max_pages,
+                           int* max_freed, int* pages_freed);
+
+    Heap* heap_;
+    base::Mutex mutex_;
+    base::Semaphore pending_sweeper_tasks_semaphore_;
+    SweptList swept_list_[kAllocationSpaces];
+    SweepingList sweeping_list_[kAllocationSpaces];
+    SweepingList* late_sweeping_list_[kAllocationSpaces];
+    SweepingList* tmp_late_sweeping_list_[kAllocationSpaces];
+    bool sweeping_in_progress_;
+    int num_sweeping_tasks_;
+  };
+
   enum IterationMode {
     kKeepMarking,
     kClearMarkbits,
   };
 
   static void Initialize();
 
   void SetUp();
 
   void TearDown();
@@ skipping 31 matching lines @@
 
   static inline bool IsMarked(Object* obj);
   static bool IsUnmarkedHeapObjectWithHeap(Heap* heap, Object** p);
 
   inline Heap* heap() const { return heap_; }
   inline Isolate* isolate() const;
 
   CodeFlusher* code_flusher() { return code_flusher_; }
   inline bool is_code_flushing_enabled() const { return code_flusher_ != NULL; }
 
-  enum SweepingParallelism { SWEEP_ON_MAIN_THREAD, SWEEP_IN_PARALLEL };
-
 #ifdef VERIFY_HEAP
   void VerifyValidStoreAndSlotsBufferEntries();
   void VerifyMarkbitsAreClean();
   static void VerifyMarkbitsAreClean(PagedSpace* space);
   static void VerifyMarkbitsAreClean(NewSpace* space);
   void VerifyWeakEmbeddedObjectsInCode();
   void VerifyOmittedMapChecks();
 #endif
 
   INLINE(static bool ShouldSkipEvacuationSlotRecording(Object* host)) {
@@ skipping 17 matching lines @@
   void UpdateSlotsRecordedIn(SlotsBuffer* buffer);
 
   void InvalidateCode(Code* code);
 
   void ClearMarkbits();
 
   bool is_compacting() const { return compacting_; }
 
   MarkingParity marking_parity() { return marking_parity_; }
 
-  // Concurrent and parallel sweeping support. If required_freed_bytes was set
-  // to a value larger than 0, then sweeping returns after a block of at least
-  // required_freed_bytes was freed. If required_freed_bytes was set to zero
-  // then the whole given space is swept. It returns the size of the maximum
-  // continuous freed memory chunk.
-  int SweepInParallel(PagedSpace* space, int required_freed_bytes,
-                      int max_pages = 0);
-
-  // Sweeps a given page concurrently to the sweeper threads. It returns the
-  // size of the maximum continuous freed memory chunk.
-  int SweepInParallel(Page* page, PagedSpace* space);
-
   // Ensures that sweeping is finished.
   //
   // Note: Can only be called safely from main thread.
   void EnsureSweepingCompleted();
 
-  void SweepOrWaitUntilSweepingCompleted(Page* page);
-
   // Help out in sweeping the corresponding space and refill memory that has
   // been regained.
   //
   // Note: Thread-safe.
   void SweepAndRefill(CompactionSpace* space);
 
-  // If sweeper threads are not active this method will return true. If
-  // this is a latency issue we should be smarter here. Otherwise, it will
-  // return true if the sweeper threads are done processing the pages.
-  bool IsSweepingCompleted();
-
   // Checks if sweeping is in progress right now on any space.
-  bool sweeping_in_progress() { return sweeping_in_progress_; }
+  bool sweeping_in_progress() { return sweeper().sweeping_in_progress(); }
 
   void set_evacuation(bool evacuation) { evacuation_ = evacuation; }
 
   bool evacuation() const { return evacuation_; }
 
   // Special case for processing weak references in a full collection. We need
   // to artificially keep AllocationSites alive for a time.
   void MarkAllocationSite(AllocationSite* site);
 
   // Mark objects in implicit references groups if their parent object
@@ skipping 20 matching lines @@
   // The following two methods can just be called after marking, when the
   // whole transitive closure is known. They must be called before sweeping
   // when mark bits are still intact.
   bool IsSlotInBlackObject(MemoryChunk* p, Address slot);
   HeapObject* FindBlackObjectBySlotSlow(Address slot);
 
   // Removes all the slots in the slot buffers that are within the given
   // address range.
   void RemoveObjectSlots(Address start_slot, Address end_slot);
 
-  base::Mutex* swept_pages_mutex() { return &swept_pages_mutex_; }
-  List<Page*>* swept_pages(AllocationSpace id) {
-    switch (id) {
-      case OLD_SPACE:
-        return &swept_old_space_pages_;
-      case CODE_SPACE:
-        return &swept_code_space_pages_;
-      case MAP_SPACE:
-        return &swept_map_space_pages_;
-      default:
-        UNREACHABLE();
-    }
-    return nullptr;
-  }
+  Sweeper& sweeper() { return sweeper_; }
 
  private:
   class EvacuateNewSpaceVisitor;
   class EvacuateOldSpaceVisitor;
   class EvacuateVisitorBase;
   class HeapObjectVisitor;
-  class SweeperTask;
 
   typedef std::vector<Page*> SweepingList;
 
   explicit MarkCompactCollector(Heap* heap);
 
   bool WillBeDeoptimized(Code* code);
   void ClearInvalidRememberedSetSlots();
 
-  void StartSweeperThreads();
-
   void ComputeEvacuationHeuristics(int area_size,
                                    int* target_fragmentation_percent,
                                    int* max_evacuated_bytes);
 
 #ifdef DEBUG
   enum CollectorState {
     IDLE,
     PREPARE_GC,
     MARK_LIVE_OBJECTS,
     SWEEP_SPACES,
@@ skipping 148 matching lines @@
   // Phase 2: Sweeping to clear mark bits and free non-live objects for
   // a non-compacting collection.
   //
   //  Before: Live objects are marked and non-live objects are unmarked.
   //
   //   After: Live objects are unmarked, non-live regions have been added to
   //          their space's free list. Active eden semispace is compacted by
   //          evacuation.
   //
 
-  inline SweepingList& sweeping_list(Space* space);
-
   // If we are not compacting the heap, we simply sweep the spaces except
   // for the large object space, clearing mark bits and adding unmarked
   // regions to each space's free list.
   void SweepSpaces();
 
   void EvacuateNewSpacePrologue();
   void EvacuateNewSpaceEpilogue();
 
   void EvacuatePagesInParallel();
 
@@ skipping 14 matching lines @@
   void RecomputeLiveBytes(MemoryChunk* page);
 
   void SweepAbortedPages();
 
   void ReleaseEvacuationCandidates();
 
   // Starts sweeping of a space by contributing on the main thread and setting
   // up other pages for sweeping.
   void StartSweepSpace(PagedSpace* space);
 
-  // Finalizes the parallel sweeping phase. Marks all the pages that were
-  // swept in parallel.
-  void ParallelSweepSpacesComplete();
-
 #ifdef DEBUG
   friend class MarkObjectVisitor;
   static void VisitObject(HeapObject* obj);
 
   friend class UnmarkObjectVisitor;
   static void UnmarkObject(HeapObject* obj);
 #endif
 
   Heap* heap_;
   base::VirtualMemory* marking_deque_memory_;
   size_t marking_deque_memory_committed_;
   MarkingDeque marking_deque_;
   CodeFlusher* code_flusher_;
   bool have_code_to_deoptimize_;
 
   List<Page*> evacuation_candidates_;
   List<NewSpacePage*> newspace_evacuation_candidates_;
 
-  base::Mutex swept_pages_mutex_;
-  List<Page*> swept_old_space_pages_;
-  List<Page*> swept_code_space_pages_;
-  List<Page*> swept_map_space_pages_;
-
-  SweepingList sweeping_list_old_space_;
-  SweepingList sweeping_list_code_space_;
-  SweepingList sweeping_list_map_space_;
-
   // True if we are collecting slots to perform evacuation from evacuation
   // candidates.
   bool compacting_;
 
-  // True if concurrent or parallel sweeping is currently in progress.
-  bool sweeping_in_progress_;
-
-  // Semaphore used to synchronize sweeper tasks.
-  base::Semaphore pending_sweeper_tasks_semaphore_;
-
   // Semaphore used to synchronize compaction tasks.
   base::Semaphore pending_compaction_tasks_semaphore_;
 
   bool black_allocation_;
 
+  Sweeper sweeper_;
+
   friend class Heap;
   friend class StoreBuffer;
 };
 
 
 class EvacuationScope BASE_EMBEDDED {
  public:
   explicit EvacuationScope(MarkCompactCollector* collector)
       : collector_(collector) {
     collector_->set_evacuation(true);
   }
 
   ~EvacuationScope() { collector_->set_evacuation(false); }
 
  private:
   MarkCompactCollector* collector_;
 };
 
 
 const char* AllocationSpaceName(AllocationSpace space);
 }  // namespace internal
 }  // namespace v8
 
 #endif  // V8_HEAP_MARK_COMPACT_H_