Refactor the Heap into ThreadHeap to prepare for per thread heaps

third_party/WebKit/Source/platform/heap/HeapPage.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 81 matching lines @@
 void HeapObjectHeader::zapMagic()
 {
     ASSERT(checkHeader());
     m_magic = zappedMagic;
 }
 #endif
 
 void HeapObjectHeader::finalize(Address object, size_t objectSize)
 {
     HeapAllocHooks::freeHookIfEnabled(object);
-    const GCInfo* gcInfo = Heap::gcInfo(gcInfoIndex());
+    const GCInfo* gcInfo = ThreadHeap::gcInfo(gcInfoIndex());
     if (gcInfo->hasFinalizer())
         gcInfo->m_finalize(object);
 
     ASAN_RETIRE_CONTAINER_ANNOTATION(object, objectSize);
 }
 
 BaseArena::BaseArena(ThreadState* state, int index)
     : m_firstPage(nullptr)
     , m_firstUnsweptPage(nullptr)
     , m_threadState(state)
     , m_index(index)
 {
 }
 
 BaseArena::~BaseArena()
 {
     ASSERT(!m_firstPage);
     ASSERT(!m_firstUnsweptPage);
 }
 
 void BaseArena::cleanupPages()
 {
     clearFreeLists();
 
     ASSERT(!m_firstUnsweptPage);
     // Add the BaseArena's pages to the orphanedPagePool.
     for (BasePage* page = m_firstPage; page; page = page->next()) {
-        Heap::decreaseAllocatedSpace(page->size());
-        Heap::getOrphanedPagePool()->addOrphanedPage(arenaIndex(), page);
+        getThreadState()->heap().heapStats().decreaseAllocatedSpace(page->size());
+        getThreadState()->heap().getOrphanedPagePool()->addOrphanedPage(arenaIndex(), page);
     }
     m_firstPage = nullptr;
 }
 
 void BaseArena::takeSnapshot(const String& dumpBaseName, ThreadState::GCSnapshotInfo& info)
 {
     // |dumpBaseName| at this point is "blink_gc/thread_X/heaps/HeapName"
     WebMemoryAllocatorDump* allocatorDump = BlinkGCMemoryDumpProvider::instance()->createMemoryAllocatorDumpForCurrentGC(dumpBaseName);
     size_t pageCount = 0;
     BasePage::HeapSnapshotInfo heapInfo;
@@ skipping 134 matching lines @@
     if (getThreadState()->sweepForbidden())
         return nullptr;
 
     TRACE_EVENT0("blink_gc", "BaseArena::lazySweepPages");
     ThreadState::SweepForbiddenScope sweepForbidden(getThreadState());
     ScriptForbiddenIfMainThreadScope scriptForbidden;
 
     double startTime = WTF::currentTimeMS();
     Address result = lazySweepPages(allocationSize, gcInfoIndex);
     getThreadState()->accumulateSweepingTime(WTF::currentTimeMS() - startTime);
-    Heap::reportMemoryUsageForTracing();
+    ThreadHeap::reportMemoryUsageForTracing();
 
     return result;
 }
 
 void BaseArena::sweepUnsweptPage()
 {
     BasePage* page = m_firstUnsweptPage;
     if (page->isEmpty()) {
         page->unlink(&m_firstUnsweptPage);
         page->removeFromHeap();
@@ skipping 17 matching lines @@
     RELEASE_ASSERT(getThreadState()->isSweepingInProgress());
     ASSERT(getThreadState()->sweepForbidden());
     ASSERT(!getThreadState()->isMainThread() || ScriptForbiddenScope::isScriptForbidden());
 
     int pageCount = 1;
     while (m_firstUnsweptPage) {
         sweepUnsweptPage();
         if (pageCount % deadlineCheckInterval == 0) {
             if (deadlineSeconds <= monotonicallyIncreasingTime()) {
                 // Deadline has come.
-                Heap::reportMemoryUsageForTracing();
+                ThreadHeap::reportMemoryUsageForTracing();
                 return !m_firstUnsweptPage;
             }
         }
         pageCount++;
     }
-    Heap::reportMemoryUsageForTracing();
+    ThreadHeap::reportMemoryUsageForTracing();
     return true;
 }
 
 void BaseArena::completeSweep()
 {
     RELEASE_ASSERT(getThreadState()->isSweepingInProgress());
     ASSERT(getThreadState()->sweepForbidden());
     ASSERT(!getThreadState()->isMainThread() || ScriptForbiddenScope::isScriptForbidden());
 
     while (m_firstUnsweptPage) {
         sweepUnsweptPage();
     }
-    Heap::reportMemoryUsageForTracing();
+    ThreadHeap::reportMemoryUsageForTracing();
 }
 
 NormalPageArena::NormalPageArena(ThreadState* state, int index)
     : BaseArena(state, index)
     , m_currentAllocationPoint(nullptr)
     , m_remainingAllocationSize(0)
     , m_lastRemainingAllocationSize(0)
     , m_promptlyFreedSize(0)
 {
     clearFreeLists();
@@ skipping 38 matching lines @@
     if (m_freeList.takeSnapshot(dumpName)) {
         WebMemoryAllocatorDump* bucketsDump = BlinkGCMemoryDumpProvider::instance()->createMemoryAllocatorDumpForCurrentGC(dumpName + "/buckets");
         WebMemoryAllocatorDump* pagesDump = BlinkGCMemoryDumpProvider::instance()->createMemoryAllocatorDumpForCurrentGC(dumpName + "/pages");
         BlinkGCMemoryDumpProvider::instance()->currentProcessMemoryDump()->addOwnershipEdge(pagesDump->guid(), bucketsDump->guid());
     }
 }
 
 void NormalPageArena::allocatePage()
 {
     getThreadState()->shouldFlushHeapDoesNotContainCache();
-    PageMemory* pageMemory = Heap::getFreePagePool()->takeFreePage(arenaIndex());
+    PageMemory* pageMemory = getThreadState()->heap().getFreePagePool()->takeFreePage(arenaIndex());
 
     if (!pageMemory) {
         // Allocate a memory region for blinkPagesPerRegion pages that
         // will each have the following layout.
         //
         //    [ guard os page | ... payload ... | guard os page ]
         //    ^---{ aligned to blink page size }
-        PageMemoryRegion* region = PageMemoryRegion::allocateNormalPages(Heap::getRegionTree());
+        PageMemoryRegion* region = PageMemoryRegion::allocateNormalPages(getThreadState()->heap().getRegionTree());
 
         // Setup the PageMemory object for each of the pages in the region.
         for (size_t i = 0; i < blinkPagesPerRegion; ++i) {
             PageMemory* memory = PageMemory::setupPageMemoryInRegion(region, i * blinkPageSize, blinkPagePayloadSize());
             // Take the first possible page ensuring that this thread actually
             // gets a page and add the rest to the page pool.
             if (!pageMemory) {
                 bool result = memory->commit();
                 // If you hit the ASSERT, it will mean that you're hitting
                 // the limit of the number of mmapped regions OS can support
                 // (e.g., /proc/sys/vm/max_map_count in Linux).
                 RELEASE_ASSERT(result);
                 pageMemory = memory;
             } else {
-                Heap::getFreePagePool()->addFreePage(arenaIndex(), memory);
+                getThreadState()->heap().getFreePagePool()->addFreePage(arenaIndex(), memory);
             }
         }
     }
 
     NormalPage* page = new (pageMemory->writableStart()) NormalPage(pageMemory, this);
     page->link(&m_firstPage);
 
-    Heap::increaseAllocatedSpace(page->size());
+    getThreadState()->heap().heapStats().increaseAllocatedSpace(page->size());
 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER)
     // Allow the following addToFreeList() to add the newly allocated memory
     // to the free list.
     ASAN_UNPOISON_MEMORY_REGION(page->payload(), page->payloadSize());
     Address address = page->payload();
     for (size_t i = 0; i < page->payloadSize(); i++)
         address[i] = reuseAllowedZapValue;
     ASAN_POISON_MEMORY_REGION(page->payload(), page->payloadSize());
 #endif
     addToFreeList(page->payload(), page->payloadSize());
 }
 
 void NormalPageArena::freePage(NormalPage* page)
 {
-    Heap::decreaseAllocatedSpace(page->size());
+    getThreadState()->heap().heapStats().decreaseAllocatedSpace(page->size());
 
     if (page->terminating()) {
         // The thread is shutting down and this page is being removed as a part
         // of the thread local GC.  In that case the object could be traced in
         // the next global GC if there is a dangling pointer from a live thread
         // heap to this dead thread heap.  To guard against this, we put the
         // page into the orphaned page pool and zap the page memory.  This
         // ensures that tracing the dangling pointer in the next global GC just
         // crashes instead of causing use-after-frees.  After the next global
         // GC, the orphaned pages are removed.
-        Heap::getOrphanedPagePool()->addOrphanedPage(arenaIndex(), page);
+        getThreadState()->heap().getOrphanedPagePool()->addOrphanedPage(arenaIndex(), page);
     } else {
         PageMemory* memory = page->storage();
         page->~NormalPage();
-        Heap::getFreePagePool()->addFreePage(arenaIndex(), memory);
+        getThreadState()->heap().getFreePagePool()->addFreePage(arenaIndex(), memory);
     }
 }
 
 bool NormalPageArena::coalesce()
 {
     // Don't coalesce arenas if there are not enough promptly freed entries
     // to be coalesced.
     //
     // FIXME: This threshold is determined just to optimize blink_perf
     // benchmarks. Coalescing is very sensitive to the threashold and
@@ skipping 85 matching lines @@
 }
 
 bool NormalPageArena::expandObject(HeapObjectHeader* header, size_t newSize)
 {
     // It's possible that Vector requests a smaller expanded size because
     // Vector::shrinkCapacity can set a capacity smaller than the actual payload
     // size.
     ASSERT(header->checkHeader());
     if (header->payloadSize() >= newSize)
         return true;
-    size_t allocationSize = Heap::allocationSizeFromSize(newSize);
+    size_t allocationSize = ThreadHeap::allocationSizeFromSize(newSize);
     ASSERT(allocationSize > header->size());
     size_t expandSize = allocationSize - header->size();
     if (isObjectAllocatedAtAllocationPoint(header) && expandSize <= m_remainingAllocationSize) {
         m_currentAllocationPoint += expandSize;
         ASSERT(m_remainingAllocationSize >= expandSize);
         setRemainingAllocationSize(m_remainingAllocationSize - expandSize);
         // Unpoison the memory used for the object (payload).
         SET_MEMORY_ACCESSIBLE(header->payloadEnd(), expandSize);
         header->setSize(allocationSize);
         ASSERT(findPageFromAddress(header->payloadEnd() - 1));
         return true;
     }
     return false;
 }
 
 bool NormalPageArena::shrinkObject(HeapObjectHeader* header, size_t newSize)
 {
     ASSERT(header->checkHeader());
     ASSERT(header->payloadSize() > newSize);
-    size_t allocationSize = Heap::allocationSizeFromSize(newSize);
+    size_t allocationSize = ThreadHeap::allocationSizeFromSize(newSize);
     ASSERT(header->size() > allocationSize);
     size_t shrinkSize = header->size() - allocationSize;
     if (isObjectAllocatedAtAllocationPoint(header)) {
         m_currentAllocationPoint -= shrinkSize;
         setRemainingAllocationSize(m_remainingAllocationSize + shrinkSize);
         SET_MEMORY_INACCESSIBLE(m_currentAllocationPoint, shrinkSize);
         header->setSize(allocationSize);
         return true;
     }
     ASSERT(shrinkSize >= sizeof(HeapObjectHeader));
@@ skipping 190 matching lines @@
 Address LargeObjectArena::doAllocateLargeObjectPage(size_t allocationSize, size_t gcInfoIndex)
 {
     size_t largeObjectSize = LargeObjectPage::pageHeaderSize() + allocationSize;
     // If ASan is supported we add allocationGranularity bytes to the allocated
     // space and poison that to detect overflows
 #if defined(ADDRESS_SANITIZER)
     largeObjectSize += allocationGranularity;
 #endif
 
     getThreadState()->shouldFlushHeapDoesNotContainCache();
-    PageMemory* pageMemory = PageMemory::allocate(largeObjectSize, Heap::getRegionTree());
+    PageMemory* pageMemory = PageMemory::allocate(largeObjectSize, getThreadState()->heap().getRegionTree());
     Address largeObjectAddress = pageMemory->writableStart();
     Address headerAddress = largeObjectAddress + LargeObjectPage::pageHeaderSize();
 #if ENABLE(ASSERT)
     // Verify that the allocated PageMemory is expectedly zeroed.
     for (size_t i = 0; i < largeObjectSize; ++i)
         ASSERT(!largeObjectAddress[i]);
 #endif
     ASSERT(gcInfoIndex > 0);
     HeapObjectHeader* header = new (NotNull, headerAddress) HeapObjectHeader(largeObjectSizeInHeader, gcInfoIndex);
     Address result = headerAddress + sizeof(*header);
     ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask));
     LargeObjectPage* largeObject = new (largeObjectAddress) LargeObjectPage(pageMemory, this, allocationSize);
     ASSERT(header->checkHeader());
 
     // Poison the object header and allocationGranularity bytes after the object
     ASAN_POISON_MEMORY_REGION(header, sizeof(*header));
     ASAN_POISON_MEMORY_REGION(largeObject->getAddress() + largeObject->size(), allocationGranularity);
 
     largeObject->link(&m_firstPage);
 
-    Heap::increaseAllocatedSpace(largeObject->size());
+    getThreadState()->heap().heapStats().increaseAllocatedSpace(largeObject->size());
     getThreadState()->increaseAllocatedObjectSize(largeObject->size());
     return result;
 }
 
 void LargeObjectArena::freeLargeObjectPage(LargeObjectPage* object)
 {
     ASAN_UNPOISON_MEMORY_REGION(object->payload(), object->payloadSize());
     object->heapObjectHeader()->finalize(object->payload(), object->payloadSize());
-    Heap::decreaseAllocatedSpace(object->size());
+    getThreadState()->heap().heapStats().decreaseAllocatedSpace(object->size());
 
     // Unpoison the object header and allocationGranularity bytes after the
     // object before freeing.
     ASAN_UNPOISON_MEMORY_REGION(object->heapObjectHeader(), sizeof(HeapObjectHeader));
     ASAN_UNPOISON_MEMORY_REGION(object->getAddress() + object->size(), allocationGranularity);
 
     if (object->terminating()) {
         ASSERT(ThreadState::current()->isTerminating());
         // The thread is shutting down and this page is being removed as a part
         // of the thread local GC.  In that case the object could be traced in
         // the next global GC if there is a dangling pointer from a live thread
         // heap to this dead thread heap.  To guard against this, we put the
         // page into the orphaned page pool and zap the page memory.  This
         // ensures that tracing the dangling pointer in the next global GC just
         // crashes instead of causing use-after-frees.  After the next global
         // GC, the orphaned pages are removed.
-        Heap::getOrphanedPagePool()->addOrphanedPage(arenaIndex(), object);
+        getThreadState()->heap().getOrphanedPagePool()->addOrphanedPage(arenaIndex(), object);
     } else {
         ASSERT(!ThreadState::current()->isTerminating());
         PageMemory* memory = object->storage();
         object->~LargeObjectPage();
         delete memory;
     }
 }
 
 Address LargeObjectArena::lazySweepPages(size_t allocationSize, size_t gcInfoIndex)
 {
@@ skipping 276 matching lines @@
             // that memory on the freelist is zero filled.
             SET_MEMORY_INACCESSIBLE(headerAddress, size);
             headerAddress += size;
             continue;
         }
         if (startOfGap != headerAddress) {
             pageArena->addToFreeList(startOfGap, headerAddress - startOfGap);
 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER)
             // Discarding pages increases page faults and may regress performance.
             // So we enable this only on low-RAM devices.
-            if (Heap::isLowEndDevice())
+            if (ProcessHeap::isLowEndDevice())
                 discardPages(startOfGap + sizeof(FreeListEntry), headerAddress);
 #endif
         }
         header->unmark();
         headerAddress += size;
         markedObjectSize += size;
         startOfGap = headerAddress;
     }
     if (startOfGap != payloadEnd()) {
         pageArena->addToFreeList(startOfGap, payloadEnd() - startOfGap);
 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER)
-        if (Heap::isLowEndDevice())
+        if (ProcessHeap::isLowEndDevice())
             discardPages(startOfGap + sizeof(FreeListEntry), payloadEnd());
 #endif
     }
 
     if (markedObjectSize)
         pageArena->getThreadState()->increaseMarkedObjectSize(markedObjectSize);
 }
 
 void NormalPage::makeConsistentForGC()
 {
@@ skipping 145 matching lines @@
         if (objectFields[i] != 0)
             return false;
     }
     return true;
 }
 #endif
 
 static void markPointer(Visitor* visitor, HeapObjectHeader* header)
 {
     ASSERT(header->checkHeader());
-    const GCInfo* gcInfo = Heap::gcInfo(header->gcInfoIndex());
+    const GCInfo* gcInfo = ThreadHeap::gcInfo(header->gcInfoIndex());
     if (gcInfo->hasVTable() && !vTableInitialized(header->payload())) {
         // We hit this branch when a GC strikes before GarbageCollected<>'s
         // constructor runs.
         //
         // class A : public GarbageCollected<A> { virtual void f() = 0; };
         // class B : public A {
         //   B() : A(foo()) { };
         // };
         //
         // If foo() allocates something and triggers a GC, the vtable of A
@@ skipping 229 matching lines @@
 
     m_hasEntries = true;
     size_t index = hash(address);
     ASSERT(!(index & 1));
     Address cachePage = roundToBlinkPageStart(address);
     m_entries[index + 1] = m_entries[index];
     m_entries[index] = cachePage;
 }
 
 } // namespace blink