Oilpan: Simplify Heap::allocate

Source/platform/heap/Heap.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 624 matching lines @@
     Address addr = reinterpret_cast<Address>(const_cast<void*>(payload));
     FinalizedHeapObjectHeader* header =
         reinterpret_cast<FinalizedHeapObjectHeader*>(addr - finalizedHeaderSize);
     return header;
 }
 
 template<typename Header>
 ThreadHeap<Header>::ThreadHeap(ThreadState* state, int index)
     : m_currentAllocationPoint(0)
     , m_remainingAllocationSize(0)
+    , m_lastRemainingAllocationSize(0)
     , m_firstPage(0)
     , m_firstLargeHeapObject(0)
     , m_firstPageAllocatedDuringSweeping(0)
     , m_lastPageAllocatedDuringSweeping(0)
     , m_mergePoint(0)
     , m_biggestFreeListIndex(0)
     , m_threadState(state)
     , m_index(index)
     , m_numberOfNormalPages(0)
     , m_promptlyFreedCount(0)
@@ skipping 17 matching lines @@
     for (HeapPage<Header>* page = m_firstPage; page; page = page->m_next)
         Heap::orphanedPagePool()->addOrphanedPage(m_index, page);
     m_firstPage = 0;
 
     for (LargeHeapObject<Header>* largeObject = m_firstLargeHeapObject; largeObject; largeObject = largeObject->m_next)
         Heap::orphanedPagePool()->addOrphanedPage(m_index, largeObject);
     m_firstLargeHeapObject = 0;
 }
 
 template<typename Header>
+void ThreadHeap<Header>::updateRemainingAllocationSize()
+{
+    if (m_lastRemainingAllocationSize > remainingAllocationSize()) {
+        stats().increaseObjectSpace(m_lastRemainingAllocationSize - remainingAllocationSize());
+        m_lastRemainingAllocationSize = remainingAllocationSize();
+    }
+    ASSERT(m_lastRemainingAllocationSize == remainingAllocationSize());
+}
+
+template<typename Header>
 Address ThreadHeap<Header>::outOfLineAllocate(size_t size, const GCInfo* gcInfo)
 {
     size_t allocationSize = allocationSizeFromSize(size);
+    ASSERT(allocationSize > remainingAllocationSize());
+    if (allocationSize > blinkPageSize / 2)
+        return allocateLargeObject(allocationSize, gcInfo);
+
+    updateRemainingAllocationSize();
     if (threadState()->shouldGC()) {
         if (threadState()->shouldForceConservativeGC())
             Heap::collectGarbage(ThreadState::HeapPointersOnStack);
         else
             threadState()->setGCRequested();
     }
+    if (remainingAllocationSize() > 0) {
+        addToFreeList(currentAllocationPoint(), remainingAllocationSize());
+        setAllocationPoint(0, 0);
+    }
     ensureCurrentAllocation(allocationSize, gcInfo);
     return allocate(size, gcInfo);
 }
 
 template<typename Header>
 bool ThreadHeap<Header>::allocateFromFreeList(size_t minSize)
 {
     size_t bucketSize = 1 << m_biggestFreeListIndex;
     int i = m_biggestFreeListIndex;
     for (; i > 0; i--, bucketSize >>= 1) {
         if (bucketSize < minSize)
             break;
         FreeListEntry* entry = m_freeLists[i];
         if (entry) {
             m_biggestFreeListIndex = i;
             entry->unlink(&m_freeLists[i]);
             setAllocationPoint(entry->address(), entry->size());
             ASSERT(currentAllocationPoint() && remainingAllocationSize() >= minSize);
             return true;
         }
     }
     m_biggestFreeListIndex = i;
     return false;
 }
 
 template<typename Header>
 void ThreadHeap<Header>::ensureCurrentAllocation(size_t minSize, const GCInfo* gcInfo)
 {
     ASSERT(minSize >= allocationGranularity);
-    if (remainingAllocationSize() >= minSize)
-        return;
-
-    if (remainingAllocationSize() > 0) {
-        addToFreeList(currentAllocationPoint(), remainingAllocationSize());
-        setAllocationPoint(0, 0);
-    }
     if (allocateFromFreeList(minSize))
         return;
     if (coalesce(minSize) && allocateFromFreeList(minSize))
         return;
     addPageToHeap(gcInfo);
     bool success = allocateFromFreeList(minSize);
     RELEASE_ASSERT(success);
 }
 
 template<typename Header>
@@ skipping 169 matching lines @@
         page->clearObjectStartBitMap();
         page->resetPromptlyFreedSize();
         size_t freedCount = 0;
         Address startOfGap = page->payload();
         for (Address headerAddress = startOfGap; headerAddress < page->end(); ) {
             BasicObjectHeader* basicHeader = reinterpret_cast<BasicObjectHeader*>(headerAddress);
             ASSERT(basicHeader->size() > 0);
             ASSERT(basicHeader->size() < blinkPagePayloadSize());
 
             if (basicHeader->isPromptlyFreed()) {
-                stats().decreaseObjectSpace(reinterpret_cast<Header*>(basicHeader)->payloadSize());
+                stats().decreaseObjectSpace(reinterpret_cast<Header*>(basicHeader)->size());
                 size_t size = basicHeader->size();
                 ASSERT(size >= sizeof(Header));
 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER)
                 memset(headerAddress, 0, sizeof(Header));
 #endif
                 ++freedCount;
                 headerAddress += size;
                 continue;
             }
 
@@ skipping 44 matching lines @@
     // Ensure that there is enough space for alignment. If the header
     // is not a multiple of 8 bytes we will allocate an extra
     // headerPadding<Header> bytes to ensure it 8 byte aligned.
     allocationSize += headerPadding<Header>();
 
     // If ASan is supported we add allocationGranularity bytes to the allocated space and
     // poison that to detect overflows
 #if defined(ADDRESS_SANITIZER)
     allocationSize += allocationGranularity;
 #endif
+
+    updateRemainingAllocationSize();
     if (m_threadState->shouldGC())
         m_threadState->setGCRequested();
     m_threadState->shouldFlushHeapDoesNotContainCache();
     PageMemory* pageMemory = PageMemory::allocate(allocationSize);
     m_threadState->allocatedRegionsSinceLastGC().append(pageMemory->region());
     Address largeObjectAddress = pageMemory->writableStart();
     Address headerAddress = largeObjectAddress + sizeof(LargeHeapObject<Header>) + headerPadding<Header>();
     memset(headerAddress, 0, size);
     Header* header = new (NotNull, headerAddress) Header(size, gcInfo);
     Address result = headerAddress + sizeof(*header);
     ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask));
     LargeHeapObject<Header>* largeObject = new (largeObjectAddress) LargeHeapObject<Header>(pageMemory, gcInfo, threadState());
 
     // Poison the object header and allocationGranularity bytes after the object
     ASAN_POISON_MEMORY_REGION(header, sizeof(*header));
     ASAN_POISON_MEMORY_REGION(largeObject->address() + largeObject->size(), allocationGranularity);
     largeObject->link(&m_firstLargeHeapObject);
     stats().increaseAllocatedSpace(largeObject->size());
-    stats().increaseObjectSpace(largeObject->payloadSize());
+    stats().increaseObjectSpace(largeObject->size());
     return result;
 }
 
 template<typename Header>
 void ThreadHeap<Header>::freeLargeObject(LargeHeapObject<Header>* object, LargeHeapObject<Header>** previousNext)
 {
     object->unlink(previousNext);
     object->finalize();
 
     // Unpoison the object header and allocationGranularity bytes after the
@@ skipping 296 matching lines @@
 
 template<typename Header>
 bool ThreadHeap<Header>::pagesAllocatedDuringSweepingContains(Address address)
 {
     for (HeapPage<Header>* page = m_firstPageAllocatedDuringSweeping; page; page = page->next()) {
         if (page->contains(address))
             return true;
     }
     return false;
 }
-
-template<typename Header>
-void ThreadHeap<Header>::getScannedStats(HeapStats& scannedStats)
-{
-    ASSERT(!m_firstPageAllocatedDuringSweeping);
-    for (HeapPage<Header>* page = m_firstPage; page; page = page->next())
-        page->getStats(scannedStats);
-    for (LargeHeapObject<Header>* current = m_firstLargeHeapObject; current; current = current->next())
-        current->getStats(scannedStats);
-}
 #endif
 
 template<typename Header>
+void ThreadHeap<Header>::getStatsForTesting(HeapStats& stats)
+{
+    ASSERT(!m_firstPageAllocatedDuringSweeping);
+    for (HeapPage<Header>* page = m_firstPage; page; page = page->next())
+        page->getStatsForTesting(stats);
+    for (LargeHeapObject<Header>* current = m_firstLargeHeapObject; current; current = current->next())
+        current->getStatsForTesting(stats);
+}
+
+template<typename Header>
 void ThreadHeap<Header>::sweepNormalPages(HeapStats* stats)
 {
     TRACE_EVENT0("blink_gc", "ThreadHeap::sweepNormalPages");
     HeapPage<Header>* page = m_firstPage;
     HeapPage<Header>** previousNext = &m_firstPage;
     HeapPage<Header>* previous = 0;
     while (page) {
         page->resetPromptlyFreedSize();
         if (page->isEmpty()) {
             HeapPage<Header>* unused = page;
@@ skipping 12 matching lines @@
 }
 
 template<typename Header>
 void ThreadHeap<Header>::sweepLargePages(HeapStats* stats)
 {
     TRACE_EVENT0("blink_gc", "ThreadHeap::sweepLargePages");
     LargeHeapObject<Header>** previousNext = &m_firstLargeHeapObject;
     for (LargeHeapObject<Header>* current = m_firstLargeHeapObject; current;) {
         if (current->isMarked()) {
             stats->increaseAllocatedSpace(current->size());
-            stats->increaseObjectSpace(current->payloadSize());
+            stats->increaseObjectSpace(current->size());
             current->unmark();
             previousNext = &current->m_next;
             current = current->next();
         } else {
             LargeHeapObject<Header>* next = current->next();
             freeLargeObject(current, previousNext);
             current = next;
         }
     }
 }
@@ skipping 122 matching lines @@
 }
 
 template<typename Header>
 void HeapPage<Header>::unlink(ThreadHeap<Header>* heap, HeapPage* unused, HeapPage** prevNext)
 {
     *prevNext = unused->m_next;
     heap->removePageFromHeap(unused);
 }
 
 template<typename Header>
-void HeapPage<Header>::getStats(HeapStats& stats)
+void HeapPage<Header>::getStatsForTesting(HeapStats& stats)
 {
     stats.increaseAllocatedSpace(blinkPageSize);
     Address headerAddress = payload();
     ASSERT(headerAddress != end());
     do {
         Header* header = reinterpret_cast<Header*>(headerAddress);
-        if (!header->isFree())
+        if (!header->isFree()) {
             stats.increaseObjectSpace(header->payloadSize());
+        }
         ASSERT(header->size() < blinkPagePayloadSize());
         headerAddress += header->size();
         ASSERT(headerAddress <= end());
     } while (headerAddress < end());
 }
 
 template<typename Header>
 bool HeapPage<Header>::isEmpty()
 {
     BasicObjectHeader* header = reinterpret_cast<BasicObjectHeader*>(payload());
@@ skipping 43 matching lines @@
 #endif
             ASAN_POISON_MEMORY_REGION(header->payload(), header->payloadSize());
             headerAddress += size;
             continue;
         }
 
         if (startOfGap != headerAddress)
             heap->addToFreeList(startOfGap, headerAddress - startOfGap);
         header->unmark();
         headerAddress += header->size();
-        stats->increaseObjectSpace(header->payloadSize());
+        stats->increaseObjectSpace(header->size());
         startOfGap = headerAddress;
     }
     if (startOfGap != end())
         heap->addToFreeList(startOfGap, end() - startOfGap);
 }
 
 template<typename Header>
 void HeapPage<Header>::clearLiveAndMarkDead()
 {
     for (Address headerAddress = payload(); headerAddress < end();) {
@@ skipping 205 matching lines @@
     return gcInfo()->hasVTable();
 }
 
 template<>
 inline bool HeapPage<FinalizedHeapObjectHeader>::hasVTable(FinalizedHeapObjectHeader* header)
 {
     return header->hasVTable();
 }
 
 template<typename Header>
-void LargeHeapObject<Header>::getStats(HeapStats& stats)
+void LargeHeapObject<Header>::getStatsForTesting(HeapStats& stats)
 {
     stats.increaseAllocatedSpace(size());
     stats.increaseObjectSpace(payloadSize());
 }
 
 #if ENABLE(GC_PROFILE_HEAP)
 template<typename Header>
 void LargeHeapObject<Header>::snapshot(TracedValue* json, ThreadState::SnapshotInfo* info)
 {
     Header* header = heapObjectHeader();
@@ skipping 944 matching lines @@
     ASSERT(ThreadState::isAnyThreadInGC());
     ThreadState::AttachedThreadStateSet& threads = ThreadState::attachedThreads();
     typedef ThreadState::AttachedThreadStateSet::iterator ThreadStateIterator;
     for (ThreadStateIterator it = threads.begin(), end = threads.end(); it != end; ++it) {
         HeapStats temp;
         (*it)->getStats(temp);
         stats->add(&temp);
     }
 }
 
+void Heap::getStatsForTesting(HeapStats* stats)
+{
+    stats->clear();
+    ASSERT(ThreadState::isAnyThreadInGC());
+    makeConsistentForSweeping();
+    ThreadState::AttachedThreadStateSet& threads = ThreadState::attachedThreads();
+    typedef ThreadState::AttachedThreadStateSet::iterator ThreadStateIterator;
+    for (ThreadStateIterator it = threads.begin(), end = threads.end(); it != end; ++it) {
+        HeapStats temp;
+        (*it)->getStatsForTesting(temp);
+        stats->add(&temp);
+    }
+}
+
 #if ENABLE(ASSERT)
 bool Heap::isConsistentForSweeping()
 {
     ASSERT(ThreadState::isAnyThreadInGC());
     ThreadState::AttachedThreadStateSet& threads = ThreadState::attachedThreads();
     for (ThreadState::AttachedThreadStateSet::iterator it = threads.begin(), end = threads.end(); it != end; ++it) {
         if (!(*it)->isConsistentForSweeping())
             return false;
     }
     return true;
@@ skipping 138 matching lines @@
 CallbackStack* Heap::s_weakCallbackStack;
 CallbackStack* Heap::s_ephemeronStack;
 HeapDoesNotContainCache* Heap::s_heapDoesNotContainCache;
 bool Heap::s_shutdownCalled = false;
 bool Heap::s_lastGCWasConservative = false;
 FreePagePool* Heap::s_freePagePool;
 OrphanedPagePool* Heap::s_orphanedPagePool;
 Heap::RegionTree* Heap::s_regionTree = 0;
 
 }