Refactor the Heap into ThreadHeap to prepare for per thread heaps

third_party/WebKit/Source/platform/heap/HeapPage.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 108 matching lines @@
     ASSERT(!m_firstUnsweptPage);
 }
 
 void BaseHeap::cleanupPages()
 {
     clearFreeLists();
 
     ASSERT(!m_firstUnsweptPage);
     // Add the BaseHeap's pages to the orphanedPagePool.
     for (BasePage* page = m_firstPage; page; page = page->next()) {
-        Heap::decreaseAllocatedSpace(page->size());
-        Heap::orphanedPagePool()->addOrphanedPage(heapIndex(), page);
+        threadState()->gcGroup()->heapStats().decreaseAllocatedSpace(page->size());
+        threadState()->gcGroup()->orphanedPagePool()->addOrphanedPage(heapIndex(), page);
     }
     m_firstPage = nullptr;
 }
 
 void BaseHeap::takeSnapshot(const String& dumpBaseName, ThreadState::GCSnapshotInfo& info)
 {
     // |dumpBaseName| at this point is "blink_gc/thread_X/heaps/HeapName"
     WebMemoryAllocatorDump* allocatorDump = BlinkGCMemoryDumpProvider::instance()->createMemoryAllocatorDumpForCurrentGC(dumpBaseName);
     size_t pageIndex = 0;
     size_t heapTotalFreeSize = 0;
@@ skipping 153 matching lines @@
     // lazySweepPages(). This check prevents the sweeping from being executed
     // recursively.
     if (threadState()->sweepForbidden())
         return nullptr;
 
     TRACE_EVENT0("blink_gc", "BaseHeap::lazySweepPages");
     ThreadState::SweepForbiddenScope sweepForbidden(threadState());
     ScriptForbiddenIfMainThreadScope scriptForbidden;
 
     double startTime = WTF::currentTimeMS();
-    Address result = lazySweepPages(allocationSize, gcInfoIndex);
+    Address result = lazySweepPages(allocationSize, gcInfoIndex, threadState()->gcGroup());
     threadState()->accumulateSweepingTime(WTF::currentTimeMS() - startTime);
     Heap::reportMemoryUsageForTracing();
 
     return result;
 }
 
 void BaseHeap::sweepUnsweptPage()
 {
     BasePage* page = m_firstUnsweptPage;
     if (page->isEmpty()) {
@@ skipping 97 matching lines @@
     if (m_freeList.takeSnapshot(dumpName)) {
         WebMemoryAllocatorDump* bucketsDump = BlinkGCMemoryDumpProvider::instance()->createMemoryAllocatorDumpForCurrentGC(dumpName + "/buckets");
         WebMemoryAllocatorDump* pagesDump = BlinkGCMemoryDumpProvider::instance()->createMemoryAllocatorDumpForCurrentGC(dumpName + "/pages");
         BlinkGCMemoryDumpProvider::instance()->currentProcessMemoryDump()->addOwnershipEdge(pagesDump->guid(), bucketsDump->guid());
     }
 }
 
 void NormalPageHeap::allocatePage()
 {
     threadState()->shouldFlushHeapDoesNotContainCache();
-    PageMemory* pageMemory = Heap::freePagePool()->takeFreePage(heapIndex());
+    PageMemory* pageMemory = threadState()->gcGroup()->freePagePool()->takeFreePage(heapIndex());
 
     if (!pageMemory) {
         // Allocate a memory region for blinkPagesPerRegion pages that
         // will each have the following layout.
         //
         //    [ guard os page | ... payload ... | guard os page ]
         //    ^---{ aligned to blink page size }
-        PageMemoryRegion* region = PageMemoryRegion::allocateNormalPages();
+        PageMemoryRegion* region = PageMemoryRegion::allocateNormalPages(threadState()->gcGroup());
 
         // Setup the PageMemory object for each of the pages in the region.
         for (size_t i = 0; i < blinkPagesPerRegion; ++i) {
             PageMemory* memory = PageMemory::setupPageMemoryInRegion(region, i * blinkPageSize, blinkPagePayloadSize());
             // Take the first possible page ensuring that this thread actually
             // gets a page and add the rest to the page pool.
             if (!pageMemory) {
                 bool result = memory->commit();
                 // If you hit the ASSERT, it will mean that you're hitting
                 // the limit of the number of mmapped regions OS can support
                 // (e.g., /proc/sys/vm/max_map_count in Linux).
                 RELEASE_ASSERT(result);
                 pageMemory = memory;
             } else {
-                Heap::freePagePool()->addFreePage(heapIndex(), memory);
+                threadState()->gcGroup()->freePagePool()->addFreePage(heapIndex(), memory);
             }
         }
     }
 
     NormalPage* page = new (pageMemory->writableStart()) NormalPage(pageMemory, this);
     page->link(&m_firstPage);
 
-    Heap::increaseAllocatedSpace(page->size());
+    threadState()->gcGroup()->heapStats().increaseAllocatedSpace(page->size());
 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER)
     // Allow the following addToFreeList() to add the newly allocated memory
     // to the free list.
     ASAN_UNPOISON_MEMORY_REGION(page->payload(), page->payloadSize());
     Address address = page->payload();
     for (size_t i = 0; i < page->payloadSize(); i++)
         address[i] = reuseAllowedZapValue;
     ASAN_POISON_MEMORY_REGION(page->payload(), page->payloadSize());
 #endif
     addToFreeList(page->payload(), page->payloadSize());
 }
 
 void NormalPageHeap::freePage(NormalPage* page)
 {
-    Heap::decreaseAllocatedSpace(page->size());
+    threadState()->gcGroup()->heapStats().decreaseAllocatedSpace(page->size());
 
     if (page->terminating()) {
         // The thread is shutting down and this page is being removed as a part
         // of the thread local GC.  In that case the object could be traced in
         // the next global GC if there is a dangling pointer from a live thread
         // heap to this dead thread heap.  To guard against this, we put the
         // page into the orphaned page pool and zap the page memory.  This
         // ensures that tracing the dangling pointer in the next global GC just
         // crashes instead of causing use-after-frees.  After the next global
         // GC, the orphaned pages are removed.
-        Heap::orphanedPagePool()->addOrphanedPage(heapIndex(), page);
+        threadState()->gcGroup()->orphanedPagePool()->addOrphanedPage(heapIndex(), page);
     } else {
         PageMemory* memory = page->storage();
         page->~NormalPage();
-        Heap::freePagePool()->addFreePage(heapIndex(), memory);
+        threadState()->gcGroup()->freePagePool()->addFreePage(heapIndex(), memory);
     }
 }
 
 bool NormalPageHeap::coalesce()
 {
     // Don't coalesce heaps if there are not enough promptly freed entries
     // to be coalesced.
     //
     // FIXME: This threshold is determined just to optimize blink_perf
     // benchmarks. Coalescing is very sensitive to the threashold and
@@ skipping 44 matching lines @@
             if (startOfGap != headerAddress)
                 addToFreeList(startOfGap, headerAddress - startOfGap);
 
             headerAddress += size;
             startOfGap = headerAddress;
         }
 
         if (startOfGap != page->payloadEnd())
             addToFreeList(startOfGap, page->payloadEnd() - startOfGap);
     }
-    Heap::decreaseAllocatedObjectSize(freedSize);
+    threadState()->gcGroup()->heapStats().decreaseAllocatedObjectSize(freedSize);
     ASSERT(m_promptlyFreedSize == freedSize);
     m_promptlyFreedSize = 0;
     return true;
 }
 
 void NormalPageHeap::promptlyFreeObject(HeapObjectHeader* header)
 {
     ASSERT(!threadState()->sweepForbidden());
     ASSERT(header->checkHeader());
     Address address = reinterpret_cast<Address>(header);
@@ skipping 62 matching lines @@
     Address shrinkAddress = header->payloadEnd() - shrinkSize;
     HeapObjectHeader* freedHeader = new (NotNull, shrinkAddress) HeapObjectHeader(shrinkSize, header->gcInfoIndex());
     freedHeader->markPromptlyFreed();
     ASSERT(pageFromObject(reinterpret_cast<Address>(header)) == findPageFromAddress(reinterpret_cast<Address>(header)));
     m_promptlyFreedSize += shrinkSize;
     header->setSize(allocationSize);
     SET_MEMORY_INACCESSIBLE(shrinkAddress + sizeof(HeapObjectHeader), shrinkSize - sizeof(HeapObjectHeader));
     return false;
 }
 
-Address NormalPageHeap::lazySweepPages(size_t allocationSize, size_t gcInfoIndex)
+Address NormalPageHeap::lazySweepPages(size_t allocationSize, size_t gcInfoIndex, GCGroup*)

[haraken, 2016/01/28 15:52:49]

The GCGroup parameter looks unnecessary.

[keishi, 2016/02/29 06:02:33]

Done.

 {
     ASSERT(!hasCurrentAllocationArea());
     Address result = nullptr;
     while (m_firstUnsweptPage) {
         BasePage* page = m_firstUnsweptPage;
         if (page->isEmpty()) {
             page->unlink(&m_firstUnsweptPage);
             page->removeFromHeap();
         } else {
             // Sweep a page and move the page from m_firstUnsweptPages to
@@ skipping 14 matching lines @@
 }
 
 void NormalPageHeap::setRemainingAllocationSize(size_t newRemainingAllocationSize)
 {
     m_remainingAllocationSize = newRemainingAllocationSize;
 
     // Sync recorded allocated-object size:
     //  - if previous alloc checkpoint is larger, allocation size has increased.
     //  - if smaller, a net reduction in size since last call to updateRemainingAllocationSize().
     if (m_lastRemainingAllocationSize > m_remainingAllocationSize)
-        Heap::increaseAllocatedObjectSize(m_lastRemainingAllocationSize - m_remainingAllocationSize);
+        threadState()->gcGroup()->heapStats().increaseAllocatedObjectSize(m_lastRemainingAllocationSize - m_remainingAllocationSize);
     else if (m_lastRemainingAllocationSize != m_remainingAllocationSize)
-        Heap::decreaseAllocatedObjectSize(m_remainingAllocationSize - m_lastRemainingAllocationSize);
+        threadState()->gcGroup()->heapStats().decreaseAllocatedObjectSize(m_remainingAllocationSize - m_lastRemainingAllocationSize);
     m_lastRemainingAllocationSize = m_remainingAllocationSize;
 }
 
 void NormalPageHeap::updateRemainingAllocationSize()
 {
     if (m_lastRemainingAllocationSize > remainingAllocationSize()) {
-        Heap::increaseAllocatedObjectSize(m_lastRemainingAllocationSize - remainingAllocationSize());
+        threadState()->gcGroup()->heapStats().increaseAllocatedObjectSize(m_lastRemainingAllocationSize - remainingAllocationSize());
         m_lastRemainingAllocationSize = remainingAllocationSize();
     }
     ASSERT(m_lastRemainingAllocationSize == remainingAllocationSize());
 }
 
 void NormalPageHeap::setAllocationPoint(Address point, size_t size)
 {
 #if ENABLE(ASSERT)
     if (point) {
         ASSERT(size);
@@ skipping 13 matching lines @@
 Address NormalPageHeap::outOfLineAllocate(size_t allocationSize, size_t gcInfoIndex)
 {
     ASSERT(allocationSize > remainingAllocationSize());
     ASSERT(allocationSize >= allocationGranularity);
 
     // 1. If this allocation is big enough, allocate a large object.
     if (allocationSize >= largeObjectSizeThreshold) {
         // TODO(sof): support eagerly finalized large objects, if ever needed.
         RELEASE_ASSERT(heapIndex() != BlinkGC::EagerSweepHeapIndex);
         LargeObjectHeap* largeObjectHeap = static_cast<LargeObjectHeap*>(threadState()->heap(BlinkGC::LargeObjectHeapIndex));
-        Address largeObject = largeObjectHeap->allocateLargeObjectPage(allocationSize, gcInfoIndex);
+        Address largeObject = largeObjectHeap->allocateLargeObjectPage(allocationSize, gcInfoIndex, threadState()->gcGroup());
         ASAN_MARK_LARGE_VECTOR_CONTAINER(this, largeObject);
         return largeObject;
     }
 
     // 2. Try to allocate from a free list.
     updateRemainingAllocationSize();
     Address result = allocateFromFreeList(allocationSize, gcInfoIndex);
     if (result)
         return result;
 
@@ skipping 58 matching lines @@
     }
     m_freeList.m_biggestFreeListIndex = index;
     return nullptr;
 }
 
 LargeObjectHeap::LargeObjectHeap(ThreadState* state, int index)
     : BaseHeap(state, index)
 {
 }
 
-Address LargeObjectHeap::allocateLargeObjectPage(size_t allocationSize, size_t gcInfoIndex)
+Address LargeObjectHeap::allocateLargeObjectPage(size_t allocationSize, size_t gcInfoIndex, GCGroup* gcGroup)

[haraken, 2016/01/28 15:52:50]

Remove the GCGroup parameter.

[keishi, 2016/02/29 06:02:33]

Done.

 {
     // Caller already added space for object header and rounded up to allocation
     // alignment
     ASSERT(!(allocationSize & allocationMask));
 
     // 1. Try to sweep large objects more than allocationSize bytes
     // before allocating a new large object.
     Address result = lazySweep(allocationSize, gcInfoIndex);
     if (result)
         return result;
 
     // 2. If we have failed in sweeping allocationSize bytes,
     // we complete sweeping before allocating this large object.
     threadState()->completeSweep();
 
     // 3. Check if we should trigger a GC.
     threadState()->scheduleGCIfNeeded();
 
-    return doAllocateLargeObjectPage(allocationSize, gcInfoIndex);
+    return doAllocateLargeObjectPage(allocationSize, gcInfoIndex, gcGroup);
 }
 
-Address LargeObjectHeap::doAllocateLargeObjectPage(size_t allocationSize, size_t gcInfoIndex)
+Address LargeObjectHeap::doAllocateLargeObjectPage(size_t allocationSize, size_t gcInfoIndex, GCGroup* gcGroup)

[haraken, 2016/01/28 15:52:49]

Remove the GCGroup parameter.

[keishi, 2016/02/29 06:02:33]

Done.

 {
     size_t largeObjectSize = LargeObjectPage::pageHeaderSize() + allocationSize;
     // If ASan is supported we add allocationGranularity bytes to the allocated
     // space and poison that to detect overflows
 #if defined(ADDRESS_SANITIZER)
     largeObjectSize += allocationGranularity;
 #endif
 
     threadState()->shouldFlushHeapDoesNotContainCache();
-    PageMemory* pageMemory = PageMemory::allocate(largeObjectSize);
+    PageMemory* pageMemory = PageMemory::allocate(largeObjectSize, gcGroup);

[haraken, 2016/01/28 15:52:49]

And use threadState()->gcGroup().

[keishi, 2016/02/29 06:02:33]

Done.

     Address largeObjectAddress = pageMemory->writableStart();
     Address headerAddress = largeObjectAddress + LargeObjectPage::pageHeaderSize();
 #if ENABLE(ASSERT)
     // Verify that the allocated PageMemory is expectedly zeroed.
     for (size_t i = 0; i < largeObjectSize; ++i)
         ASSERT(!largeObjectAddress[i]);
 #endif
     ASSERT(gcInfoIndex > 0);
     HeapObjectHeader* header = new (NotNull, headerAddress) HeapObjectHeader(largeObjectSizeInHeader, gcInfoIndex);
     Address result = headerAddress + sizeof(*header);
     ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask));
     LargeObjectPage* largeObject = new (largeObjectAddress) LargeObjectPage(pageMemory, this, allocationSize);
     ASSERT(header->checkHeader());
 
     // Poison the object header and allocationGranularity bytes after the object
     ASAN_POISON_MEMORY_REGION(header, sizeof(*header));
     ASAN_POISON_MEMORY_REGION(largeObject->address() + largeObject->size(), allocationGranularity);
 
     largeObject->link(&m_firstPage);
 
-    Heap::increaseAllocatedSpace(largeObject->size());
-    Heap::increaseAllocatedObjectSize(largeObject->size());
+    threadState()->gcGroup()->heapStats().increaseAllocatedSpace(largeObject->size());
+    threadState()->gcGroup()->heapStats().increaseAllocatedObjectSize(largeObject->size());
     return result;
 }
 
 void LargeObjectHeap::freeLargeObjectPage(LargeObjectPage* object)
 {
     ASAN_UNPOISON_MEMORY_REGION(object->payload(), object->payloadSize());
     object->heapObjectHeader()->finalize(object->payload(), object->payloadSize());
-    Heap::decreaseAllocatedSpace(object->size());
+    threadState()->gcGroup()->heapStats().decreaseAllocatedSpace(object->size());
 
     // Unpoison the object header and allocationGranularity bytes after the
     // object before freeing.
     ASAN_UNPOISON_MEMORY_REGION(object->heapObjectHeader(), sizeof(HeapObjectHeader));
     ASAN_UNPOISON_MEMORY_REGION(object->address() + object->size(), allocationGranularity);
 
     if (object->terminating()) {
         ASSERT(ThreadState::current()->isTerminating());
         // The thread is shutting down and this page is being removed as a part
         // of the thread local GC.  In that case the object could be traced in
         // the next global GC if there is a dangling pointer from a live thread
         // heap to this dead thread heap.  To guard against this, we put the
         // page into the orphaned page pool and zap the page memory.  This
         // ensures that tracing the dangling pointer in the next global GC just
         // crashes instead of causing use-after-frees.  After the next global
         // GC, the orphaned pages are removed.
-        Heap::orphanedPagePool()->addOrphanedPage(heapIndex(), object);
+        threadState()->gcGroup()->orphanedPagePool()->addOrphanedPage(heapIndex(), object);
     } else {
         ASSERT(!ThreadState::current()->isTerminating());
         PageMemory* memory = object->storage();
         object->~LargeObjectPage();
         delete memory;
     }
 }
 
-Address LargeObjectHeap::lazySweepPages(size_t allocationSize, size_t gcInfoIndex)
+Address LargeObjectHeap::lazySweepPages(size_t allocationSize, size_t gcInfoIndex, GCGroup* gcGroup)

[haraken, 2016/01/28 15:52:49]

Remove the GCGroup parameter.

[keishi, 2016/02/29 06:02:33]

Done.

 {
     Address result = nullptr;
     size_t sweptSize = 0;
     while (m_firstUnsweptPage) {
         BasePage* page = m_firstUnsweptPage;
         if (page->isEmpty()) {
             sweptSize += static_cast<LargeObjectPage*>(page)->payloadSize() + sizeof(HeapObjectHeader);
             page->unlink(&m_firstUnsweptPage);
             page->removeFromHeap();
             // For LargeObjectPage, stop lazy sweeping once we have swept
             // more than allocationSize bytes.
             if (sweptSize >= allocationSize) {
-                result = doAllocateLargeObjectPage(allocationSize, gcInfoIndex);
+                result = doAllocateLargeObjectPage(allocationSize, gcInfoIndex, gcGroup);
                 ASSERT(result);
                 break;
             }
         } else {
             // Sweep a page and move the page from m_firstUnsweptPages to
             // m_firstPages.
             page->sweep();
             page->unlink(&m_firstUnsweptPage);
             page->link(&m_firstPage);
             page->markAsSwept();
@@ skipping 250 matching lines @@
             heapForNormalPage()->addToFreeList(startOfGap, headerAddress - startOfGap);
         header->unmark();
         headerAddress += header->size();
         markedObjectSize += header->size();
         startOfGap = headerAddress;
     }
     if (startOfGap != payloadEnd())
         heapForNormalPage()->addToFreeList(startOfGap, payloadEnd() - startOfGap);
 
     if (markedObjectSize)
-        Heap::increaseMarkedObjectSize(markedObjectSize);
+        ThreadState::current()->gcGroup()->heapStats().increaseMarkedObjectSize(markedObjectSize);

[haraken, 2016/01/28 15:52:50]

heap()->threadState()

[keishi, 2016/02/29 06:02:33]

Done.

 }
 
 void NormalPage::makeConsistentForGC()
 {
     size_t markedObjectSize = 0;
     for (Address headerAddress = payload(); headerAddress < payloadEnd();) {
         HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAddress);
         ASSERT(header->size() < blinkPagePayloadSize());
         // Check if a free list entry first since we cannot call
         // isMarked on a free list entry.
         if (header->isFree()) {
             headerAddress += header->size();
             continue;
         }
         ASSERT(header->checkHeader());
         if (header->isMarked()) {
             header->unmark();
             markedObjectSize += header->size();
         } else {
             header->markDead();
         }
         headerAddress += header->size();
     }
     if (markedObjectSize)
-        Heap::increaseMarkedObjectSize(markedObjectSize);
+        ThreadState::current()->gcGroup()->heapStats().increaseMarkedObjectSize(markedObjectSize);

[haraken, 2016/01/28 15:52:49]

heap()->threadState()

[keishi, 2016/02/29 06:02:33]

Done.

 }
 
 void NormalPage::makeConsistentForMutator()
 {
     Address startOfGap = payload();
     for (Address headerAddress = payload(); headerAddress < payloadEnd();) {
         HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAddress);
         size_t size = header->size();
         ASSERT(size < blinkPagePayloadSize());
         if (header->isPromptlyFreed())
@@ skipping 246 matching lines @@
 }
 
 void LargeObjectPage::removeFromHeap()
 {
     static_cast<LargeObjectHeap*>(heap())->freeLargeObjectPage(this);
 }
 
 void LargeObjectPage::sweep()
 {
     heapObjectHeader()->unmark();
-    Heap::increaseMarkedObjectSize(size());
+    ThreadState::current()->gcGroup()->heapStats().increaseMarkedObjectSize(size());

[haraken, 2016/01/28 15:52:49]

heap()->threadState()

[keishi, 2016/02/29 06:02:33]

Done.

 }
 
 void LargeObjectPage::makeConsistentForGC()
 {
     HeapObjectHeader* header = heapObjectHeader();
     if (header->isMarked()) {
         header->unmark();
-        Heap::increaseMarkedObjectSize(size());
+        ThreadState::current()->gcGroup()->heapStats().increaseMarkedObjectSize(size());

[haraken, 2016/01/28 15:52:49]

heap()->threadState()

[keishi, 2016/02/29 06:02:33]

Done.

     } else {
         header->markDead();
     }
 }
 
 void LargeObjectPage::makeConsistentForMutator()
 {
     HeapObjectHeader* header = heapObjectHeader();
     if (header->isMarked())
         header->unmark();
@@ skipping 102 matching lines @@
 
     m_hasEntries = true;
     size_t index = hash(address);
     ASSERT(!(index & 1));
     Address cachePage = roundToBlinkPageStart(address);
     m_entries[index + 1] = m_entries[index];
     m_entries[index] = cachePage;
 }
 
 } // namespace blink