Add some support for promptly purging pages.

third_party/WebKit/Source/platform/heap/HeapPage.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 428 matching lines @@
   LOG_HEAP_FREELIST_VERBOSE("Heap size: %zu (%d)\n", size, arenaIndex());
   return size;
 }
 
 size_t NormalPageArena::FreeListSize() {
   size_t free_size = free_list_.FreeListSize();
   LOG_HEAP_FREELIST_VERBOSE("Free size: %zu (%d)\n", freeSize, arenaIndex());
   return free_size;
 }
 
+void NormalPage::CompactionContext::AddAvailable(BasePage* page) {
+  if (!available_pages_) {
+    DCHECK(!last_available_);
+    available_pages_ = last_available_ = page;
+    return;
+  }
+  DCHECK(last_available_);
+  BasePage* next_available_page = page;
+  last_available_->Link(&next_available_page);
+  last_available_ = page;
+}
+
+BasePage* NormalPage::CompactionContext::TakeAvailable() {
+  DCHECK(available_pages_);
+  BasePage* page = available_pages_;
+  page->Unlink(&available_pages_);
+  if (page == last_available_)
+    last_available_ = nullptr;
+  return page;
+}
+
 void NormalPageArena::SweepAndCompact() {
   ThreadHeap& heap = GetThreadState()->Heap();
   if (!heap.Compaction()->IsCompactingArena(ArenaIndex()))
     return;
 
   if (!first_unswept_page_) {
     heap.Compaction()->FinishedArenaCompaction(this, 0, 0);
     return;
   }
 
@@ skipping 34 matching lines @@
     }
     // Large objects do not belong to this arena.
     DCHECK(!page->IsLargeObjectPage());
     NormalPage* normal_page = static_cast<NormalPage*>(page);
     normal_page->Unlink(&first_unswept_page_);
     normal_page->MarkAsSwept();
     // If not the first page, add |normalPage| onto the available pages chain.
     if (!context.current_page_)
       context.current_page_ = normal_page;
     else
-      normal_page->Link(&context.available_pages_);
+      context.AddAvailable(normal_page);
     normal_page->SweepAndCompact(context);
   }
 
   // All pages were empty; nothing to compact.
   if (!context.current_page_) {
     heap.Compaction()->FinishedArenaCompaction(this, 0, 0);
     return;
   }
 
   size_t freed_size = 0;
@@ skipping 38 matching lines @@
     // decommits it. Recommitting the page must find a zeroed page later.
     // We cannot assume that the OS will hand back a zeroed page across
     // its "decommit" operation.
     //
     // If in a debug setting, the unused page contents will have been
     // zapped already; leave it in that state.
     DCHECK(!available_pages->IsLargeObjectPage());
     NormalPage* unused_page = reinterpret_cast<NormalPage*>(available_pages);
     memset(unused_page->Payload(), 0, unused_page->PayloadSize());
 #endif
-    available_pages->RemoveFromHeap();
+    // Try to cycle out the pages right away; done to try get
+    // immediately observable memory reduction across platforms.
+    available_pages->RemoveFromHeap(DecommitMemoryTiming::DecommitPromptly);
     available_pages = static_cast<NormalPage*>(next_page);
   }
   if (freed_page_count)
     LOG_HEAP_COMPACTION("\n");
   heap.Compaction()->FinishedArenaCompaction(this, freed_page_count,
                                              freed_size);
 }
 
 #if DCHECK_IS_ON()
 bool NormalPageArena::IsConsistentForGC() {
@@ skipping 78 matching lines @@
   // to the free list.
   ASAN_UNPOISON_MEMORY_REGION(page->Payload(), page->PayloadSize());
   Address address = page->Payload();
   for (size_t i = 0; i < page->PayloadSize(); i++)
     address[i] = kReuseAllowedZapValue;
   ASAN_POISON_MEMORY_REGION(page->Payload(), page->PayloadSize());
 #endif
   AddToFreeList(page->Payload(), page->PayloadSize());
 }
 
-void NormalPageArena::FreePage(NormalPage* page) {
+void NormalPageArena::FreePage(NormalPage* page,
+                               DecommitMemoryTiming decommit_hint) {
   GetThreadState()->Heap().HeapStats().DecreaseAllocatedSpace(page->size());
 
   PageMemory* memory = page->Storage();
   page->~NormalPage();
-  GetThreadState()->Heap().GetFreePagePool()->Add(ArenaIndex(), memory);
+  GetThreadState()->Heap().GetFreePagePool()->Add(ArenaIndex(), memory,
+                                                  decommit_hint);
 }
 
 bool NormalPageArena::Coalesce() {
   // Don't coalesce arenas if there are not enough promptly freed entries
   // to be coalesced.
   //
   // FIXME: This threshold is determined just to optimize blink_perf
   // benchmarks. Coalescing is very sensitive to the threashold and
   // we need further investigations on the coalescing scheme.
   if (promptly_freed_size_ < 1024 * 1024)
@@ skipping 607 matching lines @@
     DCHECK_LE(header_address, PayloadEnd());
   } while (header_address < PayloadEnd());
   return object_payload_size;
 }
 
 bool NormalPage::IsEmpty() {
   HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(Payload());
   return header->IsFree() && header->size() == PayloadSize();
 }
 
-void NormalPage::RemoveFromHeap() {
-  ArenaForNormalPage()->FreePage(this);
+void NormalPage::RemoveFromHeap(DecommitMemoryTiming decommit_hint) {
+  ArenaForNormalPage()->FreePage(this, decommit_hint);
 }
 
 #if !DCHECK_IS_ON() && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER)
 static void DiscardPages(Address begin, Address end) {
   uintptr_t begin_address =
       WTF::RoundUpToSystemPage(reinterpret_cast<uintptr_t>(begin));
   uintptr_t end_address =
       WTF::RoundDownToSystemPage(reinterpret_cast<uintptr_t>(end));
   if (begin_address < end_address)
     WTF::DiscardSystemPages(reinterpret_cast<void*>(begin_address),
@@ skipping 128 matching lines @@
       // TODO(sof): be more clever & compact later objects into
       // |currentPage|'s unused slop.
       current_page->Link(context.compacted_pages_);
       size_t free_size = current_page->PayloadSize() - allocation_point;
       if (free_size) {
         SET_MEMORY_INACCESSIBLE(compact_frontier, free_size);
         current_page->ArenaForNormalPage()->AddToFreeList(compact_frontier,
                                                           free_size);
       }
 
-      BasePage* next_available_page;
-      context.available_pages_->Unlink(&next_available_page);
-      current_page = reinterpret_cast<NormalPage*>(context.available_pages_);
-      context.available_pages_ = next_available_page;
+      current_page = static_cast<NormalPage*>(context.TakeAvailable());
       allocation_point = 0;
       compact_frontier = current_page->Payload();
     }
     if (LIKELY(compact_frontier != header_address)) {
 #if defined(ADDRESS_SANITIZER)
       // Unpoison the header + if it is a vector backing
       // store object, let go of the container annotations.
       // Do that by unpoisoning the payload entirely.
       ASAN_UNPOISON_MEMORY_REGION(header, sizeof(HeapObjectHeader));
       if (is_vector_arena)
@@ skipping 264 matching lines @@
 
 size_t LargeObjectPage::ObjectPayloadSizeForTesting() {
   MarkAsSwept();
   return PayloadSize();
 }
 
 bool LargeObjectPage::IsEmpty() {
   return !GetHeapObjectHeader()->IsMarked();
 }
 
-void LargeObjectPage::RemoveFromHeap() {
+void LargeObjectPage::RemoveFromHeap(DecommitMemoryTiming) {
   static_cast<LargeObjectArena*>(Arena())->FreeLargeObjectPage(this);
 }
 
 void LargeObjectPage::Sweep() {
   GetHeapObjectHeader()->Unmark();
   Arena()->GetThreadState()->IncreaseMarkedObjectSize(size());
 }
 
 void LargeObjectPage::MakeConsistentForMutator() {
   HeapObjectHeader* header = GetHeapObjectHeader();
@@ skipping 101 matching lines @@
 
   has_entries_ = true;
   size_t index = GetHash(address);
   DCHECK(!(index & 1));
   Address cache_page = RoundToBlinkPageStart(address);
   entries_[index + 1] = entries_[index];
   entries_[index] = cache_page;
 }
 
 }  // namespace blink