Oilpan: Refactor the way we calculate heap statistics

Source/platform/heap/Heap.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 663 matching lines @@
     ASSERT(!m_firstPage);
     ASSERT(!m_firstLargeHeapObject);
 }
 
 template<typename Header>
 void ThreadHeap<Header>::cleanupPages()
 {
     clearFreeLists();
 
     // Add the ThreadHeap's pages to the orphanedPagePool.
-    for (HeapPage<Header>* page = m_firstPage; page; page = page->m_next)
+    for (HeapPage<Header>* page = m_firstPage; page; page = page->m_next) {
+        Heap::decreaseAllocatedSpace(blinkPageSize);
         Heap::orphanedPagePool()->addOrphanedPage(m_index, page);
+    }
     m_firstPage = 0;
 
-    for (LargeHeapObject<Header>* largeObject = m_firstLargeHeapObject; largeObject; largeObject = largeObject->m_next)
+    for (LargeHeapObject<Header>* largeObject = m_firstLargeHeapObject; largeObject; largeObject = largeObject->m_next) {
+        Heap::decreaseAllocatedSpace(largeObject->size());
         Heap::orphanedPagePool()->addOrphanedPage(m_index, largeObject);
+    }
     m_firstLargeHeapObject = 0;
 }
 
 template<typename Header>
 void ThreadHeap<Header>::updateRemainingAllocationSize()
 {
     if (m_lastRemainingAllocationSize > remainingAllocationSize()) {
-        stats().increaseObjectSpace(m_lastRemainingAllocationSize - remainingAllocationSize());
+        Heap::increaseAllocatedObjectSize(m_lastRemainingAllocationSize - remainingAllocationSize());
         m_lastRemainingAllocationSize = remainingAllocationSize();
     }
     ASSERT(m_lastRemainingAllocationSize == remainingAllocationSize());
 }
 
 template<typename Header>
 Address ThreadHeap<Header>::outOfLineAllocate(size_t payloadSize, size_t allocationSize, const GCInfo* gcInfo)
 {
     ASSERT(allocationSize > remainingAllocationSize());
     if (allocationSize > blinkPageSize / 2)
@@ skipping 247 matching lines @@
         page->clearObjectStartBitMap();
         page->resetPromptlyFreedSize();
         size_t freedCount = 0;
         Address startOfGap = page->payload();
         for (Address headerAddress = startOfGap; headerAddress < page->end(); ) {
             BasicObjectHeader* basicHeader = reinterpret_cast<BasicObjectHeader*>(headerAddress);
             ASSERT(basicHeader->size() > 0);
             ASSERT(basicHeader->size() < blinkPagePayloadSize());
 
             if (basicHeader->isPromptlyFreed()) {
-                stats().decreaseObjectSpace(reinterpret_cast<Header*>(basicHeader)->size());
+                Heap::decreaseAllocatedObjectSize(reinterpret_cast<Header*>(basicHeader)->size());
                 size_t size = basicHeader->size();
                 ASSERT(size >= sizeof(Header));
 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER)
                 memset(headerAddress, 0, sizeof(Header));
 #endif
                 ++freedCount;
                 headerAddress += size;
                 continue;
             }
 
@@ skipping 63 matching lines @@
     memset(headerAddress, 0, size);
     Header* header = new (NotNull, headerAddress) Header(size, gcInfo);
     Address result = headerAddress + sizeof(*header);
     ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask));
     LargeHeapObject<Header>* largeObject = new (largeObjectAddress) LargeHeapObject<Header>(pageMemory, gcInfo, threadState());
 
     // Poison the object header and allocationGranularity bytes after the object
     ASAN_POISON_MEMORY_REGION(header, sizeof(*header));
     ASAN_POISON_MEMORY_REGION(largeObject->address() + largeObject->size(), allocationGranularity);
     largeObject->link(&m_firstLargeHeapObject);
-    stats().increaseAllocatedSpace(largeObject->size());
-    stats().increaseObjectSpace(largeObject->size());
+    Heap::increaseAllocatedSpace(largeObject->size());
+    Heap::increaseAllocatedObjectSize(largeObject->size());
     return result;
 }
 
 template<typename Header>
 void ThreadHeap<Header>::freeLargeObject(LargeHeapObject<Header>* object, LargeHeapObject<Header>** previousNext)
 {
     object->unlink(previousNext);
     object->finalize();
+    Heap::decreaseAllocatedSpace(object->size());
 
     // Unpoison the object header and allocationGranularity bytes after the
     // object before freeing.
     ASAN_UNPOISON_MEMORY_REGION(object->heapObjectHeader(), sizeof(Header));
     ASAN_UNPOISON_MEMORY_REGION(object->address() + object->size(), allocationGranularity);
 
     if (object->terminating()) {
         ASSERT(ThreadState::current()->isTerminating());
         // The thread is shutting down so this object is being removed as part
         // of a thread local GC. In that case the object could be traced in the
@@ skipping 201 matching lines @@
 {
     // When adding a page to the ThreadHeap using HeapObjectHeaders store the GCInfo on the heap
     // since it is the same for all objects
     ASSERT(gcInfo);
     allocatePage(gcInfo);
 }
 
 template <typename Header>
 void ThreadHeap<Header>::removePageFromHeap(HeapPage<Header>* page)
 {
+    Heap::decreaseAllocatedSpace(blinkPageSize);
+
     MutexLocker locker(m_threadState->sweepMutex());
     if (page->terminating()) {
         // The thread is shutting down so this page is being removed as part
         // of a thread local GC. In that case the page could be accessed in the
         // next global GC either due to a dead object being traced via a
         // conservative pointer or due to a programming error where an object
         // in another thread heap keeps a dangling pointer to this object.
         // To guard against this we put the page in the orphanedPagePool to
         // ensure it is still reachable. After the next global GC it can be
         // decommitted and moved to the page pool assuming no rogue/dangling
@@ skipping 33 matching lines @@
                     pageMemory = memory;
                 else
                     delete memory;
             } else {
                 Heap::freePagePool()->addFreePage(m_index, memory);
             }
             offset += blinkPageSize;
         }
     }
     HeapPage<Header>* page = new (pageMemory->writableStart()) HeapPage<Header>(pageMemory, this, gcInfo);
+    Heap::increaseAllocatedSpace(blinkPageSize);
     // Use a separate list for pages allocated during sweeping to make
     // sure that we do not accidentally sweep objects that have been
     // allocated during sweeping.
     if (m_threadState->isSweepInProgress()) {
         if (!m_lastPageAllocatedDuringSweeping)
             m_lastPageAllocatedDuringSweeping = page;
         page->link(&m_firstPageAllocatedDuringSweeping);
     } else {
         page->link(&m_firstPage);
     }
@@ skipping 17 matching lines @@
 {
     for (HeapPage<Header>* page = m_firstPageAllocatedDuringSweeping; page; page = page->next()) {
         if (page->contains(address))
             return true;
     }
     return false;
 }
 #endif
 
 template<typename Header>
-void ThreadHeap<Header>::getStatsForTesting(HeapStats& stats)
+size_t ThreadHeap<Header>::objectPayloadSizeForTesting()
 {
     ASSERT(!m_firstPageAllocatedDuringSweeping);
+    size_t objectPayloadSize = 0;
     for (HeapPage<Header>* page = m_firstPage; page; page = page->next())
-        page->getStatsForTesting(stats);
+        objectPayloadSize += page->objectPayloadSizeForTesting();
     for (LargeHeapObject<Header>* current = m_firstLargeHeapObject; current; current = current->next())
-        current->getStatsForTesting(stats);
+        objectPayloadSize += current->objectPayloadSizeForTesting();
+    return objectPayloadSize;
 }
 
 template<typename Header>
-void ThreadHeap<Header>::sweepNormalPages(HeapStats* stats)
+void ThreadHeap<Header>::sweepNormalPages()
 {
     TRACE_EVENT0("blink_gc", "ThreadHeap::sweepNormalPages");
     HeapPage<Header>* page = m_firstPage;
     HeapPage<Header>** previousNext = &m_firstPage;
     HeapPage<Header>* previous = 0;
     while (page) {
         page->resetPromptlyFreedSize();
         if (page->isEmpty()) {
             HeapPage<Header>* unused = page;
             if (unused == m_mergePoint)
                 m_mergePoint = previous;
             page = page->next();
             HeapPage<Header>::unlink(this, unused, previousNext);
             --m_numberOfNormalPages;
         } else {
-            page->sweep(stats, this);
+            page->sweep(this);
             previousNext = &page->m_next;
             previous = page;
             page = page->next();
         }
     }
 }
 
 template<typename Header>
-void ThreadHeap<Header>::sweepLargePages(HeapStats* stats)
+void ThreadHeap<Header>::sweepLargePages()
 {
     TRACE_EVENT0("blink_gc", "ThreadHeap::sweepLargePages");
     LargeHeapObject<Header>** previousNext = &m_firstLargeHeapObject;
     for (LargeHeapObject<Header>* current = m_firstLargeHeapObject; current;) {
         if (current->isMarked()) {
-            stats->increaseAllocatedSpace(current->size());
-            stats->increaseObjectSpace(current->size());
+            Heap::increaseMarkedObjectSize(current->size());
             current->unmark();
             previousNext = &current->m_next;
             current = current->next();
         } else {
             LargeHeapObject<Header>* next = current->next();
             freeLargeObject(current, previousNext);
             current = next;
         }
     }
 }
 
 
 // STRICT_ASAN_FINALIZATION_CHECKING turns on poisoning of all objects during
 // sweeping to catch cases where dead objects touch each other. This is not
 // turned on by default because it also triggers for cases that are safe.
 // Examples of such safe cases are context life cycle observers and timers
 // embedded in garbage collected objects.
 #define STRICT_ASAN_FINALIZATION_CHECKING 0
 
 template<typename Header>
-void ThreadHeap<Header>::sweep(HeapStats* stats)
+void ThreadHeap<Header>::sweep()
 {
     ASSERT(isConsistentForSweeping());
 #if defined(ADDRESS_SANITIZER) && STRICT_ASAN_FINALIZATION_CHECKING
     // When using ASan do a pre-sweep where all unmarked objects are
     // poisoned before calling their finalizer methods. This can catch
     // the case where the finalizer of an object tries to modify
     // another object as part of finalization.
     for (HeapPage<Header>* page = m_firstPage; page; page = page->next())
         page->poisonUnmarkedObjects();
 #endif
-    sweepNormalPages(stats);
-    sweepLargePages(stats);
+    sweepNormalPages();
+    sweepLargePages();
 }
 
 template<typename Header>
 void ThreadHeap<Header>::postSweepProcessing()
 {
     // If pages have been allocated during sweeping, link them into
     // the list of pages.
     if (m_firstPageAllocatedDuringSweeping) {
         m_lastPageAllocatedDuringSweeping->m_next = m_firstPage;
         m_firstPage = m_firstPageAllocatedDuringSweeping;
@@ skipping 79 matching lines @@
 }
 
 template<typename Header>
 HeapPage<Header>::HeapPage(PageMemory* storage, ThreadHeap<Header>* heap, const GCInfo* gcInfo)
     : BaseHeapPage(storage, gcInfo, heap->threadState())
     , m_next(0)
 {
     COMPILE_ASSERT(!(sizeof(HeapPage<Header>) & allocationMask), page_header_incorrectly_aligned);
     m_objectStartBitMapComputed = false;
     ASSERT(isPageHeaderAddress(reinterpret_cast<Address>(this)));
-    heap->stats().increaseAllocatedSpace(blinkPageSize);
 }
 
 template<typename Header>
 void HeapPage<Header>::link(HeapPage** prevNext)
 {
     m_next = *prevNext;
     *prevNext = this;
 }
 
 template<typename Header>
 void HeapPage<Header>::unlink(ThreadHeap<Header>* heap, HeapPage* unused, HeapPage** prevNext)
 {
     *prevNext = unused->m_next;
     heap->removePageFromHeap(unused);
 }
 
 template<typename Header>
-void HeapPage<Header>::getStatsForTesting(HeapStats& stats)
+size_t HeapPage<Header>::objectPayloadSizeForTesting()
 {
-    stats.increaseAllocatedSpace(blinkPageSize);
+    size_t objectPayloadSize = 0;
     Address headerAddress = payload();
     ASSERT(headerAddress != end());
     do {
         Header* header = reinterpret_cast<Header*>(headerAddress);
         if (!header->isFree()) {
-            stats.increaseObjectSpace(header->payloadSize());
+            objectPayloadSize += header->payloadSize();
         }
         ASSERT(header->size() < blinkPagePayloadSize());
         headerAddress += header->size();
         ASSERT(headerAddress <= end());
     } while (headerAddress < end());
+    return objectPayloadSize;
 }
 
 template<typename Header>
 bool HeapPage<Header>::isEmpty()
 {
     BasicObjectHeader* header = reinterpret_cast<BasicObjectHeader*>(payload());
     return header->isFree() && (header->size() == payloadSize());
 }
 
 template<typename Header>
-void HeapPage<Header>::sweep(HeapStats* stats, ThreadHeap<Header>* heap)
+void HeapPage<Header>::sweep(ThreadHeap<Header>* heap)
 {
     clearObjectStartBitMap();
-    stats->increaseAllocatedSpace(blinkPageSize);
     Address startOfGap = payload();
     for (Address headerAddress = startOfGap; headerAddress < end(); ) {
         BasicObjectHeader* basicHeader = reinterpret_cast<BasicObjectHeader*>(headerAddress);
         ASSERT(basicHeader->size() > 0);
         ASSERT(basicHeader->size() < blinkPagePayloadSize());
 
         if (basicHeader->isFree()) {
             size_t size = basicHeader->size();
 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER)
             // Zero the memory in the free list header to maintain the
@@ skipping 25 matching lines @@
 #endif
             ASAN_POISON_MEMORY_REGION(header->payload(), header->payloadSize());
             headerAddress += size;
             continue;
         }
 
         if (startOfGap != headerAddress)
             heap->addToFreeList(startOfGap, headerAddress - startOfGap);
         header->unmark();
         headerAddress += header->size();
-        stats->increaseObjectSpace(header->size());
+        Heap::increaseMarkedObjectSize(header->size());
         startOfGap = headerAddress;
     }
     if (startOfGap != end())
         heap->addToFreeList(startOfGap, end() - startOfGap);
 }
 
 template<typename Header>
 void HeapPage<Header>::clearLiveAndMarkDead()
 {
     for (Address headerAddress = payload(); headerAddress < end();) {
@@ skipping 205 matching lines @@
     return gcInfo()->hasVTable();
 }
 
 template<>
 inline bool HeapPage<FinalizedHeapObjectHeader>::hasVTable(FinalizedHeapObjectHeader* header)
 {
     return header->hasVTable();
 }
 
 template<typename Header>
-void LargeHeapObject<Header>::getStatsForTesting(HeapStats& stats)
+size_t LargeHeapObject<Header>::objectPayloadSizeForTesting()
 {
-    stats.increaseAllocatedSpace(size());
-    stats.increaseObjectSpace(payloadSize());
+    return payloadSize();
 }
 
 #if ENABLE(GC_PROFILE_HEAP)
 template<typename Header>
 void LargeHeapObject<Header>::snapshot(TracedValue* json, ThreadState::SnapshotInfo* info)
 {
     Header* header = heapObjectHeader();
     size_t tag = info->getClassTag(header->gcInfo());
     size_t age = header->age();
     if (isMarked()) {
@@ skipping 319 matching lines @@
     ThreadState::init();
     s_markingStack = new CallbackStack();
     s_postMarkingCallbackStack = new CallbackStack();
     s_weakCallbackStack = new CallbackStack();
     s_ephemeronStack = new CallbackStack();
     s_heapDoesNotContainCache = new HeapDoesNotContainCache();
     s_markingVisitor = new MarkingVisitor(s_markingStack);
     s_freePagePool = new FreePagePool();
     s_orphanedPagePool = new OrphanedPagePool();
     s_markingThreads = new Vector<OwnPtr<WebThread>>();
+    s_allocatedObjectSize = 0;
+    s_allocatedSpace = 0;
+    s_markedObjectSize = 0;
     if (Platform::current()) {
         int processors = Platform::current()->numberOfProcessors();
         int numberOfMarkingThreads = std::min(processors, maxNumberOfMarkingThreads);
         for (int i = 0; i < numberOfMarkingThreads; i++)
             s_markingThreads->append(adoptPtr(Platform::current()->createThread("Blink GC Marking Thread")));
     }
 }
 
 void Heap::shutdown()
 {
@@ skipping 23 matching lines @@
     s_weakCallbackStack = 0;
     delete s_postMarkingCallbackStack;
     s_postMarkingCallbackStack = 0;
     delete s_markingStack;
     s_markingStack = 0;
     delete s_ephemeronStack;
     s_ephemeronStack = 0;
     delete s_regionTree;
     s_regionTree = 0;
     ThreadState::shutdown();
+    ASSERT(Heap::allocatedSpace() == 0);
 }
 
 BaseHeapPage* Heap::contains(Address address)
 {
     ASSERT(ThreadState::isAnyThreadInGC());
     ThreadState::AttachedThreadStateSet& threads = ThreadState::attachedThreads();
     for (ThreadState::AttachedThreadStateSet::iterator it = threads.begin(), end = threads.end(); it != end; ++it) {
         BaseHeapPage* page = (*it)->contains(address);
         if (page)
             return page;
@@ skipping 232 matching lines @@
     if (!gcScope.allThreadsParked()) {
         ThreadState::current()->setGCRequested();
         return;
     }
 
     if (state->isMainThread())
         ScriptForbiddenScope::enter();
 
     s_lastGCWasConservative = false;
 
+    Heap::resetMarkedObjectSize();
+    Heap::resetAllocatedObjectSize();
+
     TRACE_EVENT2("blink_gc", "Heap::collectGarbage",
         "precise", stackState == ThreadState::NoHeapPointersOnStack,
         "forced", cause == ThreadState::ForcedGC);
     TRACE_EVENT_SCOPED_SAMPLING_STATE("blink_gc", "BlinkGC");
     double timeStamp = WTF::currentTimeMS();
 #if ENABLE(GC_PROFILE_MARKING)
     static_cast<MarkingVisitor*>(s_markingVisitor)->objectGraph().clear();
 #endif
 
     // Disallow allocation during garbage collection (but not
@@ skipping 38 matching lines @@
     // marking we check that any object marked as dead is not traced. E.g. via a
     // conservatively found pointer or a programming error with an object containing
     // a dangling pointer.
     orphanedPagePool()->decommitOrphanedPages();
 
 #if ENABLE(GC_PROFILE_MARKING)
     static_cast<MarkingVisitor*>(s_markingVisitor)->reportStats();
 #endif
 
     if (Platform::current()) {
-        uint64_t objectSpaceSize;
-        uint64_t allocatedSpaceSize;
-        getHeapSpaceSize(&objectSpaceSize, &allocatedSpaceSize);
         Platform::current()->histogramCustomCounts("BlinkGC.CollectGarbage", WTF::currentTimeMS() - timeStamp, 0, 10 * 1000, 50);
-        Platform::current()->histogramCustomCounts("BlinkGC.TotalObjectSpace", objectSpaceSize / 1024, 0, 4 * 1024 * 1024, 50);
-        Platform::current()->histogramCustomCounts("BlinkGC.TotalAllocatedSpace", allocatedSpaceSize / 1024, 0, 4 * 1024 * 1024, 50);
+        Platform::current()->histogramCustomCounts("BlinkGC.TotalObjectSpace", Heap::allocatedObjectSize() / 1024, 0, 4 * 1024 * 1024, 50);
+        Platform::current()->histogramCustomCounts("BlinkGC.TotalAllocatedSpace", Heap::allocatedSpace() / 1024, 0, 4 * 1024 * 1024, 50);
     }
 
     if (state->isMainThread())
         ScriptForbiddenScope::exit();
 }
 
 void Heap::collectGarbageForTerminatingThread(ThreadState* state)
 {
     // We explicitly do not enter a safepoint while doing thread specific
     // garbage collection since we don't want to allow a global GC at the
@@ skipping 218 matching lines @@
             } else if (splitOff->m_freeList.m_freeLists[i]) {
                 m_freeList.m_lastFreeListEntries[i]->append(splitOff->m_freeList.m_freeLists[i]);
                 m_freeList.m_lastFreeListEntries[i] = splitOff->m_freeList.m_lastFreeListEntries[i];
             }
         }
         if (m_freeList.m_biggestFreeListIndex < splitOff->m_freeList.m_biggestFreeListIndex)
             m_freeList.m_biggestFreeListIndex = splitOff->m_freeList.m_biggestFreeListIndex;
     }
 }
 
-void Heap::getHeapSpaceSize(uint64_t* objectSpaceSize, uint64_t* allocatedSpaceSize)
+size_t Heap::objectPayloadSizeForTesting()
 {
-    *objectSpaceSize = 0;
-    *allocatedSpaceSize = 0;
-    ASSERT(ThreadState::isAnyThreadInGC());
-    ThreadState::AttachedThreadStateSet& threads = ThreadState::attachedThreads();
-    typedef ThreadState::AttachedThreadStateSet::iterator ThreadStateIterator;
-    for (ThreadStateIterator it = threads.begin(), end = threads.end(); it != end; ++it) {
-        *objectSpaceSize += (*it)->stats().totalObjectSpace();
-        *allocatedSpaceSize += (*it)->stats().totalAllocatedSpace();
-    }
-}
-
-void Heap::getStatsForTesting(HeapStats* stats)
-{
-    stats->clear();
+    size_t objectPayloadSize = 0;
     ASSERT(ThreadState::isAnyThreadInGC());
     makeConsistentForSweeping();
     ThreadState::AttachedThreadStateSet& threads = ThreadState::attachedThreads();
     typedef ThreadState::AttachedThreadStateSet::iterator ThreadStateIterator;
     for (ThreadStateIterator it = threads.begin(), end = threads.end(); it != end; ++it) {
-        HeapStats temp;
-        (*it)->getStatsForTesting(temp);
-        stats->add(&temp);
+        objectPayloadSize += (*it)->objectPayloadSizeForTesting();
     }
+    return objectPayloadSize;
 }
 
 #if ENABLE(ASSERT)
 bool Heap::isConsistentForSweeping()
 {
     ASSERT(ThreadState::isAnyThreadInGC());
     ThreadState::AttachedThreadStateSet& threads = ThreadState::attachedThreads();
     for (ThreadState::AttachedThreadStateSet::iterator it = threads.begin(), end = threads.end(); it != end; ++it) {
         if (!(*it)->isConsistentForSweeping())
             return false;
@@ skipping 165 matching lines @@
 CallbackStack* Heap::s_markingStack;
 CallbackStack* Heap::s_postMarkingCallbackStack;
 CallbackStack* Heap::s_weakCallbackStack;
 CallbackStack* Heap::s_ephemeronStack;
 HeapDoesNotContainCache* Heap::s_heapDoesNotContainCache;
 bool Heap::s_shutdownCalled = false;
 bool Heap::s_lastGCWasConservative = false;
 FreePagePool* Heap::s_freePagePool;
 OrphanedPagePool* Heap::s_orphanedPagePool;
 Heap::RegionTree* Heap::s_regionTree = 0;
+size_t Heap::s_allocatedObjectSize = 0;
+size_t Heap::s_allocatedSpace = 0;
+size_t Heap::s_markedObjectSize = 0;
 
 } // namespace blink