[heap] Minor MC: Add evacuation phase

src/heap/spaces.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/spaces.h"
 
 #include <utility>
 
 #include "src/base/bits.h"
 #include "src/base/platform/platform.h"
@@ skipping 1208 matching lines @@
     while ((p = collector->sweeper().GetSweptPageSafe(this)) != nullptr) {
       // Only during compaction pages can actually change ownership. This is
       // safe because there exists no other competing action on the page links
       // during compaction.
       if (is_local() && (p->owner() != this)) {
         base::LockGuard<base::Mutex> guard(
             reinterpret_cast<PagedSpace*>(p->owner())->mutex());
         p->Unlink();
         p->set_owner(this);
         p->InsertAfter(anchor_.prev_page());
+      } else {
+        CHECK_EQ(this, p->owner());
+        // Regular refill on main thread.
+        if (p->available_in_free_list() < kPageReuseThreshold) {
+          // Relink categories with only little memory left previous to anchor.
+          p->Unlink();
+          p->InsertAfter(anchor()->prev_page());
+        }
       }
       added += RelinkFreeListCategories(p);
       added += p->wasted_memory();
       if (is_local() && (added > kCompactionMemoryWanted)) break;
     }
   }
   accounting_stats_.IncreaseCapacity(added);
 }
 
 void PagedSpace::MergeCompactionSpace(CompactionSpace* other) {
@@ skipping 60 matching lines @@
   for (HeapObject* obj = it.Next(); obj != NULL; obj = it.Next()) {
     Address cur = obj->address();
     Address next = cur + obj->Size();
     if ((cur <= addr) && (addr < next)) return obj;
   }
 
   UNREACHABLE();
   return Smi::kZero;
 }
 
+Page* PagedSpace::RemovePageSafe() {
+  base::LockGuard<base::Mutex> guard(mutex());
+
+  if (anchor()->next_page() == anchor() ||
+      anchor()->next_page()->available_in_free_list() < kPageReuseThreshold)
+    return nullptr;
+
+  Page* page = anchor()->next_page();
+  AccountUncommitted(page->size());
+  accounting_stats_.DeallocateBytes(page->LiveBytesFromFreeList());
+  accounting_stats_.DecreaseCapacity(page->size());
+  page->Unlink();
+  UnlinkFreeListCategories(page);
+  return page;
+}
+
+void PagedSpace::AddPage(Page* page) {
+  AccountCommitted(page->size());
+  accounting_stats_.IncreaseCapacity(page->size());
+  accounting_stats_.AllocateBytes(page->LiveBytesFromFreeList());
+  page->set_owner(this);
+  RelinkFreeListCategories(page);
+  page->InsertAfter(anchor()->prev_page());
+}
+
 void PagedSpace::ShrinkImmortalImmovablePages() {
   DCHECK(!heap()->deserialization_complete());
   MemoryChunk::UpdateHighWaterMark(allocation_info_.top());
   EmptyAllocationInfo();
   ResetFreeList();
 
   for (Page* page : *this) {
     DCHECK(page->IsFlagSet(Page::NEVER_EVACUATE));
     size_t unused = page->ShrinkToHighWaterMark();
     accounting_stats_.DecreaseCapacity(static_cast<intptr_t>(unused));
@@ skipping 120 matching lines @@
 
 #ifdef DEBUG
 void PagedSpace::Print() {}
 #endif
 
 #ifdef VERIFY_HEAP
 void PagedSpace::Verify(ObjectVisitor* visitor) {
   bool allocation_pointer_found_in_space =
       (allocation_info_.top() == allocation_info_.limit());
   for (Page* page : *this) {
+    if (page->IsFlagSet(Page::CANNOT_BE_VERIFIED)) continue;
     CHECK(page->owner() == this);
     if (page == Page::FromAllocationAreaAddress(allocation_info_.top())) {
       allocation_pointer_found_in_space = true;
     }
     CHECK(page->SweepingDone());
     HeapObjectIterator it(page);
     Address end_of_previous_object = page->area_start();
     Address top = page->area_end();
     int black_size = 0;
     for (HeapObject* object = it.Next(); object != NULL; object = it.Next()) {
@@ skipping 381 matching lines @@
   // allocation pointer.
   Address current = to_space_.first_page()->area_start();
   CHECK_EQ(current, to_space_.space_start());
 
   while (current != top()) {
     if (!Page::IsAlignedToPageSize(current)) {
       // The allocation pointer should not be in the middle of an object.
       CHECK(!Page::FromAllocationAreaAddress(current)->ContainsLimit(top()) ||
             current < top());
 
+      if (Page::FromAddress(current)->IsFlagSet(Page::CANNOT_BE_VERIFIED)) {
+        Page* page = Page::FromAddress(current);
+        if (page->next_page()->is_anchor()) {
+          current = top();
+        } else {
+          current = Page::FromAddress(current)->area_end();
+        }
+        continue;
+      }
+
       HeapObject* object = HeapObject::FromAddress(current);
 
       // The first word should be a map, and we expect all map pointers to
       // be in map space.
       Map* map = object->map();
       CHECK(map->IsMap());
       CHECK(heap()->map_space()->Contains(map));
 
       // The object should not be code or a map.
       CHECK(!object->IsMap());
@@ skipping 968 matching lines @@
     return free_list_.Allocate(size_in_bytes);
   }
   return nullptr;
 }
 
 HeapObject* PagedSpace::SlowAllocateRaw(int size_in_bytes) {
   DCHECK_GE(size_in_bytes, 0);
   const int kMaxPagesToSweep = 1;
 
   // Allocation in this space has failed.
-
   MarkCompactCollector* collector = heap()->mark_compact_collector();
   // Sweeping is still in progress.
   if (collector->sweeping_in_progress()) {
     // First try to refill the free-list, concurrent sweeper threads
     // may have freed some objects in the meantime.
     RefillFreeList();
 
     // Retry the free list allocation.
     HeapObject* object =
         free_list_.Allocate(static_cast<size_t>(size_in_bytes));
     if (object != NULL) return object;
 
     // If sweeping is still in progress try to sweep pages on the main thread.
     int max_freed = collector->sweeper().ParallelSweepSpace(
         identity(), size_in_bytes, kMaxPagesToSweep);
     RefillFreeList();
     if (max_freed >= size_in_bytes) {
       object = free_list_.Allocate(static_cast<size_t>(size_in_bytes));
       if (object != nullptr) return object;
     }
+  } else if (is_local()) {
+    // Sweeping not in progress and we are on a {CompactionSpace}.
+    Page* page = heap()->old_space()->RemovePageSafe();
+    if (page != nullptr) {
+      PrintF("Reusing page: available free list memory: %" PRIuS "\n",
+             page->available_in_free_list());
+      AddPage(page);
+      HeapObject* object =
+          free_list_.Allocate(static_cast<size_t>(size_in_bytes));
+      if (object != nullptr) return object;
+    }
   }
 
   if (heap()->ShouldExpandOldGenerationOnAllocationFailure() && Expand()) {
     DCHECK((CountTotalPages() > 1) ||
            (static_cast<size_t>(size_in_bytes) <= free_list_.Available()));
     return free_list_.Allocate(static_cast<size_t>(size_in_bytes));
   }
 
   // If sweeper threads are active, wait for them at that point and steal
   // elements form their free-lists. Allocation may still fail their which
@@ skipping 378 matching lines @@
     object->ShortPrint();
     PrintF("\n");
   }
   printf(" --------------------------------------\n");
   printf(" Marked: %x, LiveCount: %x\n", mark_size, LiveBytes());
 }
 
 #endif  // DEBUG
 }  // namespace internal
 }  // namespace v8