[heap] Evacuation for young generation

src/heap/spaces.h

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef V8_HEAP_SPACES_H_
 #define V8_HEAP_SPACES_H_
 
 #include <list>
 #include <memory>
 #include <unordered_set>
@@ skipping 506 matching lines @@
 
   bool CanAllocate() {
     return !IsEvacuationCandidate() && !IsFlagSet(NEVER_ALLOCATE_ON_PAGE);
   }
 
   bool ShouldSkipEvacuationSlotRecording() {
     return ((flags_ & kSkipEvacuationSlotsRecordingMask) != 0) &&
            !IsFlagSet(COMPACTION_WAS_ABORTED);
   }
 
+  bool CanUseForAllocation() { return CanAllocate() && !NeverEvacuate(); }
+
   Executability executable() {
     return IsFlagSet(IS_EXECUTABLE) ? EXECUTABLE : NOT_EXECUTABLE;
   }
 
   bool InNewSpace() { return (flags_ & kIsInNewSpaceMask) != 0; }
 
   bool InToSpace() { return IsFlagSet(IN_TO_SPACE); }
 
   bool InFromSpace() { return IsFlagSet(IN_FROM_SPACE); }
 
@@ skipping 126 matching lines @@
 
   static MarkingState Internal(MemoryChunk* chunk) {
     return MarkingState(
         Bitmap::FromAddress(chunk->address() + MemoryChunk::kHeaderSize),
         &chunk->live_byte_count_);
   }
 
   MarkingState(Bitmap* bitmap, intptr_t* live_bytes)
       : bitmap_(bitmap), live_bytes_(live_bytes) {}
 
-  void IncrementLiveBytes(intptr_t by) const {
+  template <MarkBit::AccessMode mode = MarkBit::NON_ATOMIC>
+  inline void IncrementLiveBytes(intptr_t by) const {
     *live_bytes_ += static_cast<int>(by);
   }
+
   void SetLiveBytes(intptr_t value) const {
     *live_bytes_ = static_cast<int>(value);
   }
 
   void ClearLiveness() const {
     bitmap_->Clear();
     *live_bytes_ = 0;
   }
 
   Bitmap* bitmap() const { return bitmap_; }
   intptr_t live_bytes() const { return *live_bytes_; }
 
  private:
   Bitmap* bitmap_;
   intptr_t* live_bytes_;
 };
 
+template <>
+inline void MarkingState::IncrementLiveBytes<MarkBit::NON_ATOMIC>(
+    intptr_t by) const {
+  *live_bytes_ += static_cast<int>(by);
+}
+
+template <>
+inline void MarkingState::IncrementLiveBytes<MarkBit::ATOMIC>(
+    intptr_t by) const {
+  reinterpret_cast<base::AtomicNumber<intptr_t>*>(live_bytes_)->Increment(by);
+}
+
 // -----------------------------------------------------------------------------
 // A page is a memory chunk of a size 1MB. Large object pages may be larger.
 //
 // The only way to get a page pointer is by calling factory methods:
 //   Page* p = Page::FromAddress(addr); or
 //   Page* p = Page::FromTopOrLimit(top);
 class Page : public MemoryChunk {
  public:
   static const intptr_t kCopyAllFlags = ~0;
 
@@ skipping 882 matching lines @@
   Address limit_;
 };
 
 
 // An abstraction of the accounting statistics of a page-structured space.
 //
 // The stats are only set by functions that ensure they stay balanced. These
 // functions increase or decrease one of the non-capacity stats in conjunction
 // with capacity, or else they always balance increases and decreases to the
 // non-capacity stats.
 class AllocationStats BASE_EMBEDDED {

[Michael Lippautz, 2017/04/21 07:05:52]

We might borrow pages for evacuation (using the free lists) while at the same
time expanding the space as the main thread also participates in evacuation.
This is the reason these counters need to be atomic.

[Hannes Payer (out of office), 2017/04/21 14:46:27]

Acknowledged.

  public:
   AllocationStats() { Clear(); }
 
   // Zero out all the allocation statistics (i.e., no capacity).
   void Clear() {
-    capacity_ = 0;
-    max_capacity_ = 0;
-    size_ = 0;
+    capacity_.SetValue(0);
+    max_capacity_.SetValue(0);
+    size_.SetValue(0);
   }
 
-  void ClearSize() { size_ = capacity_; }
+  void ClearSize() { size_.SetValue(capacity_.Value()); }
 
   // Accessors for the allocation statistics.
-  size_t Capacity() { return capacity_; }
-  size_t MaxCapacity() { return max_capacity_; }
-  size_t Size() { return size_; }
+  size_t Capacity() { return capacity_.Value(); }
+  size_t MaxCapacity() { return max_capacity_.Value(); }
+  size_t Size() { return size_.Value(); }
 
   // Grow the space by adding available bytes.  They are initially marked as
   // being in use (part of the size), but will normally be immediately freed,
   // putting them on the free list and removing them from size_.
   void ExpandSpace(size_t bytes) {
-    DCHECK_GE(size_ + bytes, size_);
-    DCHECK_GE(capacity_ + bytes, capacity_);
-    capacity_ += bytes;
-    size_ += bytes;
-    if (capacity_ > max_capacity_) {
-      max_capacity_ = capacity_;
+    DCHECK_GE(size_.Value() + bytes, size_.Value());
+    DCHECK_GE(capacity_.Value() + bytes, capacity_.Value());
+    capacity_.Increment(bytes);

[Michael Lippautz, 2017/04/21 07:05:52]

This should be save even though there are 2 counters involved. We only
ExpandSpace and ShrinkSpace non-concurrently.

[Hannes Payer (out of office), 2017/04/21 14:46:27]

Acknowledged.

+    size_.Increment(bytes);
+    if (capacity_.Value() > max_capacity_.Value()) {
+      max_capacity_.SetValue(capacity_.Value());
     }
   }
 
   // Shrink the space by removing available bytes.  Since shrinking is done
   // during sweeping, bytes have been marked as being in use (part of the size)
   // and are hereby freed.
   void ShrinkSpace(size_t bytes) {
-    DCHECK_GE(capacity_, bytes);
-    DCHECK_GE(size_, bytes);
-    capacity_ -= bytes;
-    size_ -= bytes;
+    DCHECK_GE(capacity_.Value(), bytes);
+    DCHECK_GE(size_.Value(), bytes);
+    capacity_.Decrement(bytes);
+    size_.Decrement(bytes);
   }
 
   void AllocateBytes(size_t bytes) {
-    DCHECK_GE(size_ + bytes, size_);
-    size_ += bytes;
+    DCHECK_GE(size_.Value() + bytes, size_.Value());
+    size_.Increment(bytes);
   }
 
   void DeallocateBytes(size_t bytes) {
-    DCHECK_GE(size_, bytes);
-    size_ -= bytes;
+    DCHECK_GE(size_.Value(), bytes);
+    size_.Decrement(bytes);
   }
 
   void DecreaseCapacity(size_t bytes) {
-    DCHECK_GE(capacity_, bytes);
-    DCHECK_GE(capacity_ - bytes, size_);
-    capacity_ -= bytes;
+    DCHECK_GE(capacity_.Value(), bytes);
+    DCHECK_GE(capacity_.Value() - bytes, size_.Value());
+    capacity_.Decrement(bytes);
   }
 
   void IncreaseCapacity(size_t bytes) {
-    DCHECK_GE(capacity_ + bytes, capacity_);
-    capacity_ += bytes;
+    DCHECK_GE(capacity_.Value() + bytes, capacity_.Value());
+    capacity_.Increment(bytes);
   }
 
   // Merge |other| into |this|.
   void Merge(const AllocationStats& other) {
-    DCHECK_GE(capacity_ + other.capacity_, capacity_);
-    DCHECK_GE(size_ + other.size_, size_);
-    capacity_ += other.capacity_;
-    size_ += other.size_;
-    if (other.max_capacity_ > max_capacity_) {
-      max_capacity_ = other.max_capacity_;
+    DCHECK_GE(capacity_.Value() + other.capacity_.Value(), capacity_.Value());
+    DCHECK_GE(size_.Value() + other.size_.Value(), size_.Value());
+    capacity_.Increment(other.capacity_.Value());
+    size_.Increment(other.size_.Value());
+    if (other.max_capacity_.Value() > max_capacity_.Value()) {
+      max_capacity_.SetValue(other.max_capacity_.Value());
     }
   }
 
  private:
   // |capacity_|: The number of object-area bytes (i.e., not including page
   // bookkeeping structures) currently in the space.
-  size_t capacity_;
+  base::AtomicNumber<size_t> capacity_;
 
   // |max_capacity_|: The maximum capacity ever observed.
-  size_t max_capacity_;
+  base::AtomicNumber<size_t> max_capacity_;
 
   // |size_|: The number of allocated bytes.
-  size_t size_;
+  base::AtomicNumber<size_t> size_;
 };
 
 // A free list maintaining free blocks of memory. The free list is organized in
 // a way to encourage objects allocated around the same time to be near each
 // other. The normal way to allocate is intended to be by bumping a 'top'
 // pointer until it hits a 'limit' pointer.  When the limit is hit we need to
 // find a new space to allocate from. This is done with the free list, which is
 // divided up into rough categories to cut down on waste. Having finer
 // categories would scatter allocation more.
 
@@ skipping 245 matching lines @@
   LocalAllocationBuffer(Heap* heap, AllocationInfo allocation_info);
 
   Heap* heap_;
   AllocationInfo allocation_info_;
 };
 
 class V8_EXPORT_PRIVATE PagedSpace : NON_EXPORTED_BASE(public Space) {
  public:
   typedef PageIterator iterator;
 
+  // Reuse a page for allocation only if it has at least {kPageReuseThreshold}
+  // memory available in its FreeList.
+  static const size_t kPageReuseThreshold = 4 * KB;
+
   static const intptr_t kCompactionMemoryWanted = 500 * KB;
 
   // Creates a space with an id.
   PagedSpace(Heap* heap, AllocationSpace id, Executability executable);
 
   ~PagedSpace() override { TearDown(); }
 
   // Set up the space using the given address range of virtual memory (from
   // the memory allocator's initial chunk) if possible.  If the block of
   // addresses is not big enough to contain a single page-aligned page, a
@@ skipping 186 matching lines @@
 
   iterator begin() { return iterator(anchor_.next_page()); }
   iterator end() { return iterator(&anchor_); }
 
   // Shrink immortal immovable pages of the space to be exactly the size needed
   // using the high water mark.
   void ShrinkImmortalImmovablePages();
 
   std::unique_ptr<ObjectIterator> GetObjectIterator() override;
 
+  Page* RemovePageSafe();
+  void AddPage(Page* page);
+
  protected:
   // PagedSpaces that should be included in snapshots have different, i.e.,
   // smaller, initial pages.
   virtual bool snapshotable() { return true; }
 
   bool HasPages() { return anchor_.next_page() != &anchor_; }
 
   // Cleans up the space, frees all pages in this space except those belonging
   // to the initial chunk, uncommits addresses in the initial chunk.
   void TearDown();
@@ skipping 792 matching lines @@
   PageIterator old_iterator_;
   PageIterator code_iterator_;
   PageIterator map_iterator_;
   LargePageIterator lo_iterator_;
 };
 
 }  // namespace internal
 }  // namespace v8
 
 #endif  // V8_HEAP_SPACES_H_