Reland of "🏄 [heap] Add page evacuation mode for new->old"

src/heap/spaces.h

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef V8_HEAP_SPACES_H_
 #define V8_HEAP_SPACES_H_
 
 #include "src/allocation.h"
 #include "src/atomic-utils.h"
 #include "src/base/atomicops.h"
@@ skipping 401 matching lines @@
     NEW_SPACE_BELOW_AGE_MARK,
     EVACUATION_CANDIDATE,
     NEVER_EVACUATE,  // May contain immortal immutables.
 
     // Large objects can have a progress bar in their page header. These object
     // are scanned in increments and will be kept black while being scanned.
     // Even if the mutator writes to them they will be kept black and a white
     // to grey transition is performed in the value.
     HAS_PROGRESS_BAR,
 
+    // |PAGE_NEW_OLD_PROMOTION|: A page tagged with this flag has been promoted
+    // from new to old space during evacuation.
+    PAGE_NEW_OLD_PROMOTION,
+
     // A black page has all mark bits set to 1 (black). A black page currently
     // cannot be iterated because it is not swept. Moreover live bytes are also
     // not updated.
     BLACK_PAGE,
 
     // This flag is intended to be used for testing. Works only when both
     // FLAG_stress_compaction and FLAG_manual_evacuation_candidates_selection
     // are set. It forces the page to become an evacuation candidate at next
     // candidates selection cycle.
     FORCE_EVACUATION_CANDIDATE_FOR_TESTING,
@@ skipping 385 matching lines @@
 };
 
 // -----------------------------------------------------------------------------
 // A page is a memory chunk of a size 1MB. Large object pages may be larger.
 //
 // The only way to get a page pointer is by calling factory methods:
 //   Page* p = Page::FromAddress(addr); or
 //   Page* p = Page::FromAllocationTop(top);
 class Page : public MemoryChunk {
  public:
+  static inline Page* Convert(NewSpacePage* old_page, PagedSpace* new_owner);
+
   // Returns the page containing a given address. The address ranges
   // from [page_addr .. page_addr + kPageSize[
   // This only works if the object is in fact in a page.  See also MemoryChunk::
   // FromAddress() and FromAnyAddress().
   INLINE(static Page* FromAddress(Address a)) {
     return reinterpret_cast<Page*>(OffsetFrom(a) & ~kPageAlignmentMask);
   }
 
   // Only works for addresses in pointer spaces, not code space.
   inline static Page* FromAnyPointerAddress(Heap* heap, Address addr);
@@ skipping 94 matching lines @@
 
 #ifdef DEBUG
   void Print();
 #endif  // DEBUG
 
   inline void MarkNeverAllocateForTesting();
   inline void MarkEvacuationCandidate();
   inline void ClearEvacuationCandidate();
 
  private:
+  enum InitializationMode { kFreeMemory, kDoNotFreeMemory };
+
+  template <InitializationMode mode = kFreeMemory>
   static inline Page* Initialize(Heap* heap, MemoryChunk* chunk,
                                  Executability executable, PagedSpace* owner);
 
   inline void InitializeFreeListCategories();
 
   friend class MemoryAllocator;
 };
 
 
 class LargePage : public MemoryChunk {
@@ skipping 83 matching lines @@
   virtual intptr_t Available() = 0;
 
   virtual int RoundSizeDownToObjectAlignment(int size) {
     if (id_ == CODE_SPACE) {
       return RoundDown(size, kCodeAlignment);
     } else {
       return RoundDown(size, kPointerSize);
     }
   }
 
-#ifdef DEBUG
-  virtual void Print() = 0;
-#endif
-
- protected:
   void AccountCommitted(intptr_t bytes) {
     DCHECK_GE(bytes, 0);
     committed_ += bytes;
     if (committed_ > max_committed_) {
       max_committed_ = committed_;
     }
   }
 
   void AccountUncommitted(intptr_t bytes) {
     DCHECK_GE(bytes, 0);
     committed_ -= bytes;
     DCHECK_GE(committed_, 0);
   }
 
+#ifdef DEBUG
+  virtual void Print() = 0;
+#endif
+
+ protected:
   v8::base::SmartPointer<List<AllocationObserver*>> allocation_observers_;
   bool allocation_observers_paused_;
 
  private:
   Heap* heap_;
   AllocationSpace id_;
   Executability executable_;
 
   // Keeps track of committed memory in a space.
   intptr_t committed_;
@@ skipping 1275 matching lines @@
   static inline NewSpacePage* Initialize(Heap* heap, MemoryChunk* chunk,
                                          Executability executable,
                                          SemiSpace* owner);
 
   // GC related flags copied from from-space to to-space when
   // flipping semispaces.
   static const intptr_t kCopyOnFlipFlagsMask =
       (1 << MemoryChunk::POINTERS_TO_HERE_ARE_INTERESTING) |
       (1 << MemoryChunk::POINTERS_FROM_HERE_ARE_INTERESTING);
 
+  static const intptr_t kCopyAllFlags = ~0;
+
   // Create a NewSpacePage object that is only used as anchor
   // for the doubly-linked list of real pages.
   explicit NewSpacePage(SemiSpace* owner) { InitializeAsAnchor(owner); }
 
   // Intialize a fake NewSpacePage used as sentinel at the ends
   // of a doubly-linked list of real NewSpacePages.
   // Only uses the prev/next links, and sets flags to not be in new-space.
   void InitializeAsAnchor(SemiSpace* owner);
 
   friend class MemoryAllocator;
@@ skipping 65 matching lines @@
   bool AdvancePage() {
     NewSpacePage* next_page = current_page_->next_page();
     if (next_page == anchor()) return false;
     current_page_ = next_page;
     return true;
   }
 
   // Resets the space to using the first page.
   void Reset();
 
+  void ReplaceWithEmptyPage(NewSpacePage* page);
+
   // Age mark accessors.
   Address age_mark() { return age_mark_; }
   void set_age_mark(Address mark);
 
   // Returns the current capacity of the semispace.
   int current_capacity() { return current_capacity_; }
 
   // Returns the maximum capacity of the semispace.
   int maximum_capacity() { return maximum_capacity_; }
 
@@ skipping 200 matching lines @@
   }
 
   // Approximate amount of physical memory committed for this space.
   size_t CommittedPhysicalMemory() override;
 
   // Return the available bytes without growing.
   intptr_t Available() override { return Capacity() - Size(); }
 
   inline size_t AllocatedSinceLastGC();
 
+  void ReplaceWithEmptyPage(NewSpacePage* page) {
+    // This method is called after flipping the semispace.
+    DCHECK(page->InFromSpace());
+    from_space_.ReplaceWithEmptyPage(page);
+  }
+
   // Return the maximum capacity of a semispace.
   int MaximumCapacity() {
     DCHECK(to_space_.maximum_capacity() == from_space_.maximum_capacity());
     return to_space_.maximum_capacity();
   }
 
   bool IsAtMaximumCapacity() { return TotalCapacity() == MaximumCapacity(); }
 
   // Returns the initial capacity of a semispace.
   int InitialTotalCapacity() {
@@ skipping 416 matching lines @@
     count = 0;
   }
   // Must be small, since an iteration is used for lookup.
   static const int kMaxComments = 64;
 };
 #endif
 }  // namespace internal
 }  // namespace v8
 
 #endif  // V8_HEAP_SPACES_H_