Oilpan: Remove duplicated code between HeapPage and LargeObject

Source/platform/heap/Heap.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 514 matching lines @@
 
 void LargeObject::markUnmarkedObjectsDead()
 {
     HeapObjectHeader* header = heapObjectHeader();
     if (header->isMarked())
         header->unmark();
     else
         header->markDead();
 }
 
-void LargeObject::removeFromHeap(ThreadHeap* heap)
+void LargeObject::removeFromHeap()
 {
-    heap->freeLargeObject(this);
-}
-
-ThreadHeap::ThreadHeap(ThreadState* state, int index)
-    : m_currentAllocationPoint(nullptr)
-    , m_remainingAllocationSize(0)
-    , m_lastRemainingAllocationSize(0)
-    , m_firstPage(nullptr)
-    , m_firstLargeObject(nullptr)
-    , m_firstUnsweptPage(nullptr)
-    , m_firstUnsweptLargeObject(nullptr)
-    , m_threadState(state)
-    , m_index(index)
-    , m_promptlyFreedSize(0)
-#if ENABLE(GC_PROFILING)
-    , m_cumulativeAllocationSize(0)
-    , m_allocationCount(0)
-    , m_inlineAllocationCount(0)
-#endif
-{
-    clearFreeLists();
+    static_cast<ThreadHeapForLargeObject*>(heap())->freeLargeObject(this);
 }
 
 FreeList::FreeList()
     : m_biggestFreeListIndex(0)
 {
 }
 
+ThreadHeap::ThreadHeap(ThreadState* state, int index)
+    : m_firstPage(nullptr)
+    , m_firstUnsweptPage(nullptr)
+    , m_threadState(state)
+    , m_index(index)
+#if ENABLE(GC_PROFILING)
+    , m_cumulativeAllocationSize(0)
+    , m_allocationCount(0)
+    , m_inlineAllocationCount(0)
+#endif
+{
+}
+
+ThreadHeapForHeapPage::ThreadHeapForHeapPage(ThreadState* state, int index)
+    : ThreadHeap(state, index)
+    , m_currentAllocationPoint(nullptr)
+    , m_remainingAllocationSize(0)
+    , m_lastRemainingAllocationSize(0)
+    , m_promptlyFreedSize(0)
+{
+    clearFreeLists();
+}
+
+ThreadHeapForLargeObject::ThreadHeapForLargeObject(ThreadState* state, int index)
+    : ThreadHeap(state, index)
+{
+}
+
 ThreadHeap::~ThreadHeap()
 {
     ASSERT(!m_firstPage);
-    ASSERT(!m_firstLargeObject);
     ASSERT(!m_firstUnsweptPage);
-    ASSERT(!m_firstUnsweptLargeObject);
 }
 
 void ThreadHeap::cleanupPages()
 {
     clearFreeLists();
 
     ASSERT(!m_firstUnsweptPage);
-    ASSERT(!m_firstUnsweptLargeObject);
     // Add the ThreadHeap's pages to the orphanedPagePool.
-    for (HeapPage* page = m_firstPage; page; page = page->m_next) {
-        Heap::decreaseAllocatedSpace(blinkPageSize);
-        Heap::orphanedPagePool()->addOrphanedPage(m_index, page);
+    for (BaseHeapPage* page = m_firstPage; page; page = page->next()) {
+        Heap::decreaseAllocatedSpace(page->size());
+        Heap::orphanedPagePool()->addOrphanedPage(heapIndex(), page);
     }
     m_firstPage = nullptr;
-
-    for (LargeObject* largeObject = m_firstLargeObject; largeObject; largeObject = largeObject->m_next) {
-        Heap::decreaseAllocatedSpace(largeObject->size());
-        Heap::orphanedPagePool()->addOrphanedPage(m_index, largeObject);
-    }
-    m_firstLargeObject = nullptr;
 }
 
-void ThreadHeap::updateRemainingAllocationSize()
+void ThreadHeapForHeapPage::updateRemainingAllocationSize()
 {
     if (m_lastRemainingAllocationSize > remainingAllocationSize()) {
         Heap::increaseAllocatedObjectSize(m_lastRemainingAllocationSize - remainingAllocationSize());
         m_lastRemainingAllocationSize = remainingAllocationSize();
     }
     ASSERT(m_lastRemainingAllocationSize == remainingAllocationSize());
 }
 
-void ThreadHeap::setAllocationPoint(Address point, size_t size)
+void ThreadHeapForHeapPage::setAllocationPoint(Address point, size_t size)
 {
 #if ENABLE(ASSERT)
     if (point) {
         ASSERT(size);
         BaseHeapPage* page = pageFromObject(point);
         ASSERT(!page->isLargeObject());
         ASSERT(size <= static_cast<HeapPage*>(page)->payloadSize());
     }
 #endif
-    if (hasCurrentAllocationArea())
+    if (hasCurrentAllocationArea()) {
         addToFreeList(currentAllocationPoint(), remainingAllocationSize());
+    }
     updateRemainingAllocationSize();
     m_currentAllocationPoint = point;
     m_lastRemainingAllocationSize = m_remainingAllocationSize = size;
 }
 
-Address ThreadHeap::outOfLineAllocate(size_t allocationSize, size_t gcInfoIndex)
+Address ThreadHeapForHeapPage::outOfLineAllocate(size_t allocationSize, size_t gcInfoIndex)
 {
     ASSERT(allocationSize > remainingAllocationSize());
     ASSERT(allocationSize >= allocationGranularity);
 
 #if ENABLE(GC_PROFILING)
     m_threadState->snapshotFreeListIfNecessary();
 #endif
 
     // 1. If this allocation is big enough, allocate a large object.
     if (allocationSize >= largeObjectSizeThreshold)
-        return allocateLargeObject(allocationSize, gcInfoIndex);
+        return static_cast<ThreadHeapForLargeObject*>(threadState()->heap(LargeObjectHeap))->allocateLargeObject(allocationSize, gcInfoIndex);
 
     // 2. Check if we should trigger a GC.
     updateRemainingAllocationSize();
     threadState()->scheduleGCIfNeeded();
 
     // 3. Try to allocate from a free list.
     Address result = allocateFromFreeList(allocationSize, gcInfoIndex);
     if (result)
         return result;
 
     // 4. Reset the allocation point.
     setAllocationPoint(nullptr, 0);
 
     // 5. Lazily sweep pages of this heap until we find a freed area for
     // this allocation or we finish sweeping all pages of this heap.
-    result = lazySweepPages(allocationSize, gcInfoIndex);
+    result = lazySweep(allocationSize, gcInfoIndex);
     if (result)
         return result;
 
     // 6. Coalesce promptly freed areas and then try to allocate from a free
     // list.
     if (coalesce()) {
         result = allocateFromFreeList(allocationSize, gcInfoIndex);
         if (result)
             return result;
     }
 
     // 7. Complete sweeping.
     threadState()->completeSweep();
 
     // 8. Add a new page to this heap.
     allocatePage();
 
     // 9. Try to allocate from a free list. This allocation must succeed.
     result = allocateFromFreeList(allocationSize, gcInfoIndex);
     RELEASE_ASSERT(result);
     return result;
 }
 
-Address ThreadHeap::allocateFromFreeList(size_t allocationSize, size_t gcInfoIndex)
+Address ThreadHeapForHeapPage::allocateFromFreeList(size_t allocationSize, size_t gcInfoIndex)
 {
     // Try reusing a block from the largest bin. The underlying reasoning
     // being that we want to amortize this slow allocation call by carving
     // off as a large a free block as possible in one go; a block that will
     // service this block and let following allocations be serviced quickly
     // by bump allocation.
     size_t bucketSize = 1 << m_freeList.m_biggestFreeListIndex;
     int index = m_freeList.m_biggestFreeListIndex;
     for (; index > 0; --index, bucketSize >>= 1) {
         FreeListEntry* entry = m_freeList.m_freeLists[index];
@@ skipping 14 matching lines @@
         }
     }
     m_freeList.m_biggestFreeListIndex = index;
     return nullptr;
 }
 
 void ThreadHeap::prepareForSweep()
 {
     ASSERT(!threadState()->isInGC());
     ASSERT(!m_firstUnsweptPage);
-    ASSERT(!m_firstUnsweptLargeObject);
 
     // Move all pages to a list of unswept pages.
     m_firstUnsweptPage = m_firstPage;
-    m_firstUnsweptLargeObject = m_firstLargeObject;
     m_firstPage = nullptr;
-    m_firstLargeObject = nullptr;
 }
 
-Address ThreadHeap::lazySweepPages(size_t allocationSize, size_t gcInfoIndex)
+Address ThreadHeap::lazySweep(size_t allocationSize, size_t gcInfoIndex)
 {
-    ASSERT(!hasCurrentAllocationArea());
-    ASSERT(allocationSize < largeObjectSizeThreshold);
-
     // If there are no pages to be swept, return immediately.
     if (!m_firstUnsweptPage)
         return nullptr;
 
     RELEASE_ASSERT(threadState()->isSweepingInProgress());
 
     // lazySweepPages() can be called recursively if finalizers invoked in
     // page->sweep() allocate memory and the allocation triggers
     // lazySweepPages(). This check prevents the sweeping from being executed
     // recursively.
     if (threadState()->sweepForbidden())
         return nullptr;
 
     TRACE_EVENT0("blink_gc", "ThreadHeap::lazySweepPages");
-    ThreadState::SweepForbiddenScope scope(m_threadState);
+    ThreadState::SweepForbiddenScope scope(threadState());
 
     if (threadState()->isMainThread())
         ScriptForbiddenScope::enter();
 
+    Address result = lazySweepPages(allocationSize, gcInfoIndex);
+
+    if (threadState()->isMainThread())
+        ScriptForbiddenScope::exit();
+    return result;
+}
+
+Address ThreadHeapForHeapPage::lazySweepPages(size_t allocationSize, size_t gcInfoIndex)
+{
+    ASSERT(!hasCurrentAllocationArea());
     Address result = nullptr;
     while (m_firstUnsweptPage) {
-        HeapPage* page = m_firstUnsweptPage;
+        BaseHeapPage* page = m_firstUnsweptPage;
         if (page->isEmpty()) {
             page->unlink(&m_firstUnsweptPage);
-            page->removeFromHeap(this);
+            page->removeFromHeap();
         } else {
             // Sweep a page and move the page from m_firstUnsweptPages to
             // m_firstPages.
             page->sweep();
             page->unlink(&m_firstUnsweptPage);
             page->link(&m_firstPage);
             page->markAsSwept();
 
+            // For HeapPage, stop lazy sweeping once we find a slot to
+            // allocate a new object.
             result = allocateFromFreeList(allocationSize, gcInfoIndex);
             if (result)
                 break;
         }
     }
-
-    if (threadState()->isMainThread())
-        ScriptForbiddenScope::exit();
     return result;
 }
 
-bool ThreadHeap::lazySweepLargeObjects(size_t allocationSize)
+Address ThreadHeapForLargeObject::lazySweepPages(size_t allocationSize, size_t gcInfoIndex)
 {
-    ASSERT(allocationSize >= largeObjectSizeThreshold);
-
-    // If there are no large objects to be swept, return immediately.
-    if (!m_firstUnsweptLargeObject)
-        return false;
-
-    RELEASE_ASSERT(threadState()->isSweepingInProgress());
-
-    // lazySweepLargeObjects() can be called recursively if finalizers invoked
-    // in page->sweep() allocate memory and the allocation triggers
-    // lazySweepLargeObjects(). This check prevents the sweeping from being
-    // executed recursively.
-    if (threadState()->sweepForbidden())
-        return false;
-
-    TRACE_EVENT0("blink_gc", "ThreadHeap::lazySweepLargeObjects");
-    ThreadState::SweepForbiddenScope scope(m_threadState);
-
-    if (threadState()->isMainThread())
-        ScriptForbiddenScope::enter();
-
-    bool result = false;
+    Address result = nullptr;
     size_t sweptSize = 0;
-    while (m_firstUnsweptLargeObject) {
-        LargeObject* largeObject = m_firstUnsweptLargeObject;
-        if (largeObject->isEmpty()) {
-            sweptSize += largeObject->size();
-            largeObject->unlink(&m_firstUnsweptLargeObject);
-            largeObject->removeFromHeap(this);
-
-            // If we have swept large objects more than allocationSize,
-            // we stop the lazy sweeping.
+    while (m_firstUnsweptPage) {
+        BaseHeapPage* page = m_firstUnsweptPage;
+        if (page->isEmpty()) {
+            sweptSize += static_cast<LargeObject*>(page)->payloadSize() + sizeof(HeapObjectHeader);
+            page->unlink(&m_firstUnsweptPage);
+            page->removeFromHeap();
+            // For LargeObject, stop lazy sweeping once we have swept
+            // more than allocationSize bytes.
             if (sweptSize >= allocationSize) {
-                result = true;
+                result = doAllocateLargeObject(allocationSize, gcInfoIndex);
+                ASSERT(result);
                 break;
             }
         } else {
-            // Sweep a large object and move the large object from
-            // m_firstUnsweptLargeObjects to m_firstLargeObjects.
-            largeObject->sweep();
-            largeObject->unlink(&m_firstUnsweptLargeObject);
-            largeObject->link(&m_firstLargeObject);
-            largeObject->markAsSwept();
+            // Sweep a page and move the page from m_firstUnsweptPages to
+            // m_firstPages.
+            page->sweep();
+            page->unlink(&m_firstUnsweptPage);
+            page->link(&m_firstPage);
+            page->markAsSwept();
         }
     }
-
-    if (threadState()->isMainThread())
-        ScriptForbiddenScope::exit();
     return result;
 }
 
 void ThreadHeap::completeSweep()
 {
     RELEASE_ASSERT(threadState()->isSweepingInProgress());
     ASSERT(threadState()->sweepForbidden());
 
     if (threadState()->isMainThread())
         ScriptForbiddenScope::enter();
 
-    // Sweep normal pages.
     while (m_firstUnsweptPage) {
-        HeapPage* page = m_firstUnsweptPage;
+        BaseHeapPage* page = m_firstUnsweptPage;
         if (page->isEmpty()) {
             page->unlink(&m_firstUnsweptPage);
-            page->removeFromHeap(this);
+            page->removeFromHeap();
         } else {
             // Sweep a page and move the page from m_firstUnsweptPages to
             // m_firstPages.
             page->sweep();
             page->unlink(&m_firstUnsweptPage);
             page->link(&m_firstPage);
             page->markAsSwept();
         }
     }
 
-    // Sweep large objects.
-    while (m_firstUnsweptLargeObject) {
-        LargeObject* largeObject = m_firstUnsweptLargeObject;
-        if (largeObject->isEmpty()) {
-            largeObject->unlink(&m_firstUnsweptLargeObject);
-            largeObject->removeFromHeap(this);
-        } else {
-            // Sweep a large object and move the large object from
-            // m_firstUnsweptLargeObjects to m_firstLargeObjects.
-            largeObject->sweep();
-            largeObject->unlink(&m_firstUnsweptLargeObject);
-            largeObject->link(&m_firstLargeObject);
-            largeObject->markAsSwept();
-        }
-    }
-
     if (threadState()->isMainThread())
         ScriptForbiddenScope::exit();
 }
 
 #if ENABLE(ASSERT)
-static bool isLargeObjectAligned(LargeObject* largeObject, Address address)
-{
-    // Check that a large object is blinkPageSize aligned (modulo the osPageSize
-    // for the guard page).
-    return reinterpret_cast<Address>(largeObject) - WTF::kSystemPageSize == roundToBlinkPageStart(reinterpret_cast<Address>(largeObject));
-}
-#endif
-
-#if ENABLE(ASSERT) || ENABLE(GC_PROFILING)
 BaseHeapPage* ThreadHeap::findPageFromAddress(Address address)
 {
-    for (HeapPage* page = m_firstPage; page; page = page->next()) {
+    for (BaseHeapPage* page = m_firstPage; page; page = page->next()) {
         if (page->contains(address))
             return page;
     }
-    for (HeapPage* page = m_firstUnsweptPage; page; page = page->next()) {
+    for (BaseHeapPage* page = m_firstUnsweptPage; page; page = page->next()) {
         if (page->contains(address))
             return page;
     }
-    for (LargeObject* largeObject = m_firstLargeObject; largeObject; largeObject = largeObject->next()) {
-        ASSERT(isLargeObjectAligned(largeObject, address));
-        if (largeObject->contains(address))
-            return largeObject;
-    }
-    for (LargeObject* largeObject = m_firstUnsweptLargeObject; largeObject; largeObject = largeObject->next()) {
-        ASSERT(isLargeObjectAligned(largeObject, address));
-        if (largeObject->contains(address))
-            return largeObject;
-    }
     return nullptr;
 }
 #endif
 
 #if ENABLE(GC_PROFILING)
 #define GC_PROFILE_HEAP_PAGE_SNAPSHOT_THRESHOLD 0
 void ThreadHeap::snapshot(TracedValue* json, ThreadState::SnapshotInfo* info)
 {
     ASSERT(isConsistentForSweeping());
     size_t previousPageCount = info->pageCount;
 
     json->beginArray("pages");
-    for (HeapPage* page = m_firstPage; page; page = page->next(), ++info->pageCount) {
+    for (BaseHeapPage* page = m_firstPage; page; page = page->next(), ++info->pageCount) {
         // FIXME: To limit the size of the snapshot we only output "threshold" many page snapshots.
         if (info->pageCount < GC_PROFILE_HEAP_PAGE_SNAPSHOT_THRESHOLD) {
             json->beginArray();
             json->pushInteger(reinterpret_cast<intptr_t>(page));
             page->snapshot(json, info);
             json->endArray();
         } else {
             page->snapshot(0, info);
         }
     }
     json->endArray();
 
-    json->beginArray("largeObjects");
-    for (LargeObject* largeObject = m_firstLargeObject; largeObject; largeObject = largeObject->next()) {
-        json->beginDictionary();
-        largeObject->snapshot(json, info);
-        json->endDictionary();
-    }
-    json->endArray();
-
     json->setInteger("pageCount", info->pageCount - previousPageCount);
 }
 
 void ThreadHeap::incrementMarkedObjectsAge()
 {
     for (HeapPage* page = m_firstPage; page; page = page->next())
         page->incrementMarkedObjectsAge();
     for (LargeObject* largeObject = m_firstLargeObject; largeObject; largeObject = largeObject->next())
         largeObject->incrementMarkedObjectsAge();
 }
@@ skipping 26 matching lines @@
     // space.
     if (static_cast<HeapPage*>(page)->payloadSize() != size && !entry->shouldAddToFreeList())
         return;
 #endif
     int index = bucketIndexForSize(size);
     entry->link(&m_freeLists[index]);
     if (index > m_biggestFreeListIndex)
         m_biggestFreeListIndex = index;
 }
 
-bool ThreadHeap::expandObject(HeapObjectHeader* header, size_t newSize)
+bool ThreadHeapForHeapPage::expandObject(HeapObjectHeader* header, size_t newSize)
 {
     // It's possible that Vector requests a smaller expanded size because
     // Vector::shrinkCapacity can set a capacity smaller than the actual payload
     // size.
     if (header->payloadSize() >= newSize)
         return true;
     size_t allocationSize = allocationSizeFromSize(newSize);
     ASSERT(allocationSize > header->size());
     size_t expandSize = allocationSize - header->size();
     if (header->payloadEnd() == m_currentAllocationPoint && expandSize <= m_remainingAllocationSize) {
         m_currentAllocationPoint += expandSize;
         m_remainingAllocationSize -= expandSize;
 
         // Unpoison the memory used for the object (payload).
         ASAN_UNPOISON_MEMORY_REGION(header->payloadEnd(), expandSize);
         FILL_ZERO_IF_NOT_PRODUCTION(header->payloadEnd(), expandSize);
         header->setSize(allocationSize);
         ASSERT(findPageFromAddress(header->payloadEnd() - 1));
         return true;
     }
     return false;
 }
 
-void ThreadHeap::shrinkObject(HeapObjectHeader* header, size_t newSize)
+void ThreadHeapForHeapPage::shrinkObject(HeapObjectHeader* header, size_t newSize)
 {
     ASSERT(header->payloadSize() > newSize);
     size_t allocationSize = allocationSizeFromSize(newSize);
     ASSERT(header->size() > allocationSize);
     size_t shrinkSize = header->size() - allocationSize;
     if (header->payloadEnd() == m_currentAllocationPoint) {
         m_currentAllocationPoint -= shrinkSize;
         m_remainingAllocationSize += shrinkSize;
         FILL_ZERO_IF_PRODUCTION(m_currentAllocationPoint, shrinkSize);
         ASAN_POISON_MEMORY_REGION(m_currentAllocationPoint, shrinkSize);
         header->setSize(allocationSize);
     } else {
         ASSERT(shrinkSize >= sizeof(HeapObjectHeader));
         ASSERT(header->gcInfoIndex() > 0);
         HeapObjectHeader* freedHeader = new (NotNull, header->payloadEnd() - shrinkSize) HeapObjectHeader(shrinkSize, header->gcInfoIndex());
         freedHeader->markPromptlyFreed();
         ASSERT(pageFromObject(reinterpret_cast<Address>(header)) == findPageFromAddress(reinterpret_cast<Address>(header)));
         m_promptlyFreedSize += shrinkSize;
         header->setSize(allocationSize);
     }
 }
 
-void ThreadHeap::promptlyFreeObject(HeapObjectHeader* header)
+void ThreadHeapForHeapPage::promptlyFreeObject(HeapObjectHeader* header)
 {
-    ASSERT(!m_threadState->sweepForbidden());
+    ASSERT(!threadState()->sweepForbidden());
     header->checkHeader();
     Address address = reinterpret_cast<Address>(header);
     Address payload = header->payload();
     size_t size = header->size();
     size_t payloadSize = header->payloadSize();
     ASSERT(size > 0);
     ASSERT(pageFromObject(address) == findPageFromAddress(address));
 
     {
-        ThreadState::SweepForbiddenScope forbiddenScope(m_threadState);
+        ThreadState::SweepForbiddenScope forbiddenScope(threadState());
         header->finalize(payload, payloadSize);
         if (address + size == m_currentAllocationPoint) {
             m_currentAllocationPoint = address;
             if (m_lastRemainingAllocationSize == m_remainingAllocationSize) {
                 Heap::decreaseAllocatedObjectSize(size);
                 m_lastRemainingAllocationSize += size;
             }
             m_remainingAllocationSize += size;
             FILL_ZERO_IF_PRODUCTION(address, size);
             ASAN_POISON_MEMORY_REGION(address, size);
             return;
         }
         FILL_ZERO_IF_PRODUCTION(payload, payloadSize);
         header->markPromptlyFreed();
     }
 
     m_promptlyFreedSize += size;
 }
 
-bool ThreadHeap::coalesce()
+bool ThreadHeapForHeapPage::coalesce()
 {
     // Don't coalesce heaps if there are not enough promptly freed entries
     // to be coalesced.
     //
     // FIXME: This threshold is determined just to optimize blink_perf
     // benchmarks. Coalescing is very sensitive to the threashold and
     // we need further investigations on the coalescing scheme.
     if (m_promptlyFreedSize < 1024 * 1024)
         return false;
 
-    if (m_threadState->sweepForbidden())
+    if (threadState()->sweepForbidden())
         return false;
 
     ASSERT(!hasCurrentAllocationArea());
     TRACE_EVENT0("blink_gc", "ThreadHeap::coalesce");
 
     // Rebuild free lists.
     m_freeList.clear();
     size_t freedSize = 0;
-    for (HeapPage* page = m_firstPage; page; page = page->next()) {
+    for (HeapPage* page = static_cast<HeapPage*>(m_firstPage); page; page = static_cast<HeapPage*>(page->next())) {
         page->clearObjectStartBitMap();
         Address startOfGap = page->payload();
         for (Address headerAddress = startOfGap; headerAddress < page->payloadEnd(); ) {
             HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAddress);
             size_t size = header->size();
             ASSERT(size > 0);
             ASSERT(size < blinkPagePayloadSize());
 
             if (header->isPromptlyFreed()) {
                 ASSERT(size >= sizeof(HeapObjectHeader));
@@ skipping 20 matching lines @@
 
         if (startOfGap != page->payloadEnd())
             addToFreeList(startOfGap, page->payloadEnd() - startOfGap);
     }
     Heap::decreaseAllocatedObjectSize(freedSize);
     ASSERT(m_promptlyFreedSize == freedSize);
     m_promptlyFreedSize = 0;
     return true;
 }
 
-Address ThreadHeap::allocateLargeObject(size_t size, size_t gcInfoIndex)
+Address ThreadHeapForLargeObject::allocateLargeObject(size_t allocationSize, size_t gcInfoIndex)
 {
     // Caller already added space for object header and rounded up to allocation
     // alignment
-    ASSERT(!(size & allocationMask));
+    ASSERT(!(allocationSize & allocationMask));
 
-    size_t allocationSize = sizeof(LargeObject) + size;
+    // 1. Check if we should trigger a GC.
+    threadState()->scheduleGCIfNeeded();
 
-    // Ensure that there is enough space for alignment.  If the header
-    // is not a multiple of 8 bytes we will allocate an extra
-    // headerPadding bytes to ensure it 8 byte aligned.
-    allocationSize += headerPadding();
+    // 2. Try to sweep large objects more than allocationSize bytes
+    // before allocating a new large object.
+    Address result = lazySweep(allocationSize, gcInfoIndex);
+    if (result)
+        return result;
 
+    // 3. If we have failed in sweeping allocationSize bytes,
+    // we complete sweeping before allocating this large object.
+    threadState()->completeSweep();
+    return doAllocateLargeObject(allocationSize, gcInfoIndex);
+}
+
+Address ThreadHeapForLargeObject::doAllocateLargeObject(size_t allocationSize, size_t gcInfoIndex)
+{
+    size_t largeObjectSize = sizeof(LargeObject) + LargeObject::headerPadding() + allocationSize;
     // If ASan is supported we add allocationGranularity bytes to the allocated
     // space and poison that to detect overflows
 #if defined(ADDRESS_SANITIZER)
-    allocationSize += allocationGranularity;
+    largeObjectSize += allocationGranularity;
 #endif
 
-    // 1. Check if we should trigger a GC.
-    updateRemainingAllocationSize();
-    m_threadState->scheduleGCIfNeeded();
-
-    // 2. Try to sweep large objects more than allocationSize bytes
-    // before allocating a new large object.
-    if (!lazySweepLargeObjects(allocationSize)) {
-        // 3. If we have failed in sweeping allocationSize bytes,
-        // we complete sweeping before allocating this large object.
-        m_threadState->completeSweep();
-    }
-
-    m_threadState->shouldFlushHeapDoesNotContainCache();
-    PageMemory* pageMemory = PageMemory::allocate(allocationSize);
-    m_threadState->allocatedRegionsSinceLastGC().append(pageMemory->region());
+    threadState()->shouldFlushHeapDoesNotContainCache();
+    PageMemory* pageMemory = PageMemory::allocate(largeObjectSize);
+    threadState()->allocatedRegionsSinceLastGC().append(pageMemory->region());
     Address largeObjectAddress = pageMemory->writableStart();
-    Address headerAddress = largeObjectAddress + sizeof(LargeObject) + headerPadding();
+    Address headerAddress = largeObjectAddress + sizeof(LargeObject) + LargeObject::headerPadding();
 #if ENABLE(ASSERT)
     // Verify that the allocated PageMemory is expectedly zeroed.
-    for (size_t i = 0; i < size; ++i)
+    for (size_t i = 0; i < largeObjectSize; ++i)
         ASSERT(!headerAddress[i]);
 #endif
     ASSERT(gcInfoIndex > 0);
     HeapObjectHeader* header = new (NotNull, headerAddress) HeapObjectHeader(largeObjectSizeInHeader, gcInfoIndex);
     Address result = headerAddress + sizeof(*header);
     ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask));
-    LargeObject* largeObject = new (largeObjectAddress) LargeObject(pageMemory, this, size);
+    LargeObject* largeObject = new (largeObjectAddress) LargeObject(pageMemory, this, allocationSize);
     header->checkHeader();
 
     // Poison the object header and allocationGranularity bytes after the object
     ASAN_POISON_MEMORY_REGION(header, sizeof(*header));
     ASAN_POISON_MEMORY_REGION(largeObject->address() + largeObject->size(), allocationGranularity);
 
-    largeObject->link(&m_firstLargeObject);
+    largeObject->link(&m_firstPage);
 
     Heap::increaseAllocatedSpace(largeObject->size());
     Heap::increaseAllocatedObjectSize(largeObject->size());
     return result;
 }
 
-void ThreadHeap::freeLargeObject(LargeObject* object)
+void ThreadHeapForLargeObject::freeLargeObject(LargeObject* object)
 {
     object->heapObjectHeader()->finalize(object->payload(), object->payloadSize());
     Heap::decreaseAllocatedSpace(object->size());
 
     // Unpoison the object header and allocationGranularity bytes after the
     // object before freeing.
     ASAN_UNPOISON_MEMORY_REGION(object->heapObjectHeader(), sizeof(HeapObjectHeader));
     ASAN_UNPOISON_MEMORY_REGION(object->address() + object->size(), allocationGranularity);
 
     if (object->terminating()) {
         ASSERT(ThreadState::current()->isTerminating());
         // The thread is shutting down and this page is being removed as a part
         // of the thread local GC.  In that case the object could be traced in
         // the next global GC if there is a dangling pointer from a live thread
         // heap to this dead thread heap.  To guard against this, we put the
         // page into the orphaned page pool and zap the page memory.  This
         // ensures that tracing the dangling pointer in the next global GC just
         // crashes instead of causing use-after-frees.  After the next global
         // GC, the orphaned pages are removed.
-        Heap::orphanedPagePool()->addOrphanedPage(m_index, object);
+        Heap::orphanedPagePool()->addOrphanedPage(heapIndex(), object);
     } else {
         ASSERT(!ThreadState::current()->isTerminating());
         PageMemory* memory = object->storage();
         object->~LargeObject();
         delete memory;
     }
 }
 
 template<typename DataType>
 PagePool<DataType>::PagePool()
@@ skipping 40 matching lines @@
 
         // We got some memory, but failed to commit it, try again.
         delete memory;
     }
     return nullptr;
 }
 
 BaseHeapPage::BaseHeapPage(PageMemory* storage, ThreadHeap* heap)
     : m_storage(storage)
     , m_heap(heap)
+    , m_next(nullptr)
     , m_terminating(false)
     , m_swept(true)
 {
     ASSERT(isPageHeaderAddress(reinterpret_cast<Address>(this)));
 }
 
 void BaseHeapPage::markOrphaned()
 {
     m_heap = nullptr;
     m_terminating = false;
@@ skipping 85 matching lines @@
         for (PoolEntry* entry = m_pool[index]; entry; entry = entry->next) {
             BaseHeapPage* page = entry->data;
             if (page->contains(reinterpret_cast<Address>(object)))
                 return true;
         }
     }
     return false;
 }
 #endif
 
-void ThreadHeap::freePage(HeapPage* page)
+void ThreadHeapForHeapPage::freePage(HeapPage* page)
 {
-    Heap::decreaseAllocatedSpace(blinkPageSize);
+    Heap::decreaseAllocatedSpace(page->size());
 
     if (page->terminating()) {
         // The thread is shutting down and this page is being removed as a part
         // of the thread local GC.  In that case the object could be traced in
         // the next global GC if there is a dangling pointer from a live thread
         // heap to this dead thread heap.  To guard against this, we put the
         // page into the orphaned page pool and zap the page memory.  This
         // ensures that tracing the dangling pointer in the next global GC just
         // crashes instead of causing use-after-frees.  After the next global
         // GC, the orphaned pages are removed.
-        Heap::orphanedPagePool()->addOrphanedPage(m_index, page);
+        Heap::orphanedPagePool()->addOrphanedPage(heapIndex(), page);
     } else {
         PageMemory* memory = page->storage();
         page->~HeapPage();
-        Heap::freePagePool()->addFreePage(m_index, memory);
+        Heap::freePagePool()->addFreePage(heapIndex(), memory);
     }
 }
 
-void ThreadHeap::allocatePage()
+void ThreadHeapForHeapPage::allocatePage()
 {
-    m_threadState->shouldFlushHeapDoesNotContainCache();
-    PageMemory* pageMemory = Heap::freePagePool()->takeFreePage(m_index);
+    threadState()->shouldFlushHeapDoesNotContainCache();
+    PageMemory* pageMemory = Heap::freePagePool()->takeFreePage(heapIndex());
     // We continue allocating page memory until we succeed in committing one.
     while (!pageMemory) {
         // Allocate a memory region for blinkPagesPerRegion pages that
         // will each have the following layout.
         //
         //    [ guard os page | ... payload ... | guard os page ]
         //    ^---{ aligned to blink page size }
         PageMemoryRegion* region = PageMemoryRegion::allocateNormalPages();
-        m_threadState->allocatedRegionsSinceLastGC().append(region);
+        threadState()->allocatedRegionsSinceLastGC().append(region);
 
         // Setup the PageMemory object for each of the pages in the region.
         size_t offset = 0;
         for (size_t i = 0; i < blinkPagesPerRegion; ++i) {
             PageMemory* memory = PageMemory::setupPageMemoryInRegion(region, offset, blinkPagePayloadSize());
             // Take the first possible page ensuring that this thread actually
             // gets a page and add the rest to the page pool.
             if (!pageMemory) {
                 if (memory->commit())
                     pageMemory = memory;
                 else
                     delete memory;
             } else {
-                Heap::freePagePool()->addFreePage(m_index, memory);
+                Heap::freePagePool()->addFreePage(heapIndex(), memory);
             }
             offset += blinkPageSize;
         }
     }
     HeapPage* page = new (pageMemory->writableStart()) HeapPage(pageMemory, this);
-
     page->link(&m_firstPage);
 
-    Heap::increaseAllocatedSpace(blinkPageSize);
+    Heap::increaseAllocatedSpace(page->size());
     addToFreeList(page->payload(), page->payloadSize());
 }
 
 #if ENABLE(ASSERT)
-bool ThreadHeap::pagesToBeSweptContains(Address address)
+bool ThreadHeapForHeapPage::pagesToBeSweptContains(Address address)
 {
-    for (HeapPage* page = m_firstUnsweptPage; page; page = page->next()) {
+    for (BaseHeapPage* page = m_firstUnsweptPage; page; page = page->next()) {
         if (page->contains(address))
             return true;
     }
     return false;
 }
 #endif
 
 size_t ThreadHeap::objectPayloadSizeForTesting()
 {
     ASSERT(isConsistentForSweeping());
     ASSERT(!m_firstUnsweptPage);
-    ASSERT(!m_firstUnsweptLargeObject);
 
     size_t objectPayloadSize = 0;
-    for (HeapPage* page = m_firstPage; page; page = page->next())
+    for (BaseHeapPage* page = m_firstPage; page; page = page->next())
         objectPayloadSize += page->objectPayloadSizeForTesting();
-    for (LargeObject* largeObject = m_firstLargeObject; largeObject; largeObject = largeObject->next())
-        objectPayloadSize += largeObject->objectPayloadSizeForTesting();
     return objectPayloadSize;
 }
 
 #if ENABLE(ASSERT)
-bool ThreadHeap::isConsistentForSweeping()
+bool ThreadHeapForHeapPage::isConsistentForSweeping()
 {
     // A thread heap is consistent for sweeping if none of the pages to be swept
     // contain a freelist block or the current allocation point.
     for (size_t i = 0; i < blinkPageSizeLog2; ++i) {
         for (FreeListEntry* freeListEntry = m_freeList.m_freeLists[i]; freeListEntry; freeListEntry = freeListEntry->next()) {
             if (pagesToBeSweptContains(freeListEntry->address()))
                 return false;
         }
     }
     if (hasCurrentAllocationArea()) {
         if (pagesToBeSweptContains(currentAllocationPoint()))
             return false;
     }
     return true;
 }
 #endif
 
 void ThreadHeap::makeConsistentForSweeping()
 {
-    preparePagesForSweeping();
-    setAllocationPoint(nullptr, 0);
     clearFreeLists();
-}
-
-void ThreadHeap::preparePagesForSweeping()
-{
     ASSERT(isConsistentForSweeping());
-    for (HeapPage* page = m_firstPage; page; page = page->next())
+    for (BaseHeapPage* page = m_firstPage; page; page = page->next())
         page->markAsUnswept();
 
     // If a new GC is requested before this thread got around to sweep,
     // ie. due to the thread doing a long running operation, we clear
     // the mark bits and mark any of the dead objects as dead. The latter
     // is used to ensure the next GC marking does not trace already dead
     // objects. If we trace a dead object we could end up tracing into
     // garbage or the middle of another object via the newly conservatively
     // found object.
-    HeapPage* previousPage = nullptr;
-    for (HeapPage* page = m_firstUnsweptPage; page; previousPage = page, page = page->next()) {
+    BaseHeapPage* previousPage = nullptr;
+    for (BaseHeapPage* page = m_firstUnsweptPage; page; previousPage = page, page = page->next()) {
         page->markUnmarkedObjectsDead();
         ASSERT(!page->hasBeenSwept());
     }
     if (previousPage) {
         ASSERT(m_firstUnsweptPage);
         previousPage->m_next = m_firstPage;
         m_firstPage = m_firstUnsweptPage;
         m_firstUnsweptPage = nullptr;
     }
     ASSERT(!m_firstUnsweptPage);
-
-    for (LargeObject* largeObject = m_firstLargeObject; largeObject; largeObject = largeObject->next())
-        largeObject->markAsUnswept();
-
-    LargeObject* previousLargeObject = nullptr;
-    for (LargeObject* largeObject = m_firstUnsweptLargeObject; largeObject; previousLargeObject = largeObject, largeObject = largeObject->next()) {
-        largeObject->markUnmarkedObjectsDead();
-        ASSERT(!largeObject->hasBeenSwept());
-    }
-    if (previousLargeObject) {
-        ASSERT(m_firstUnsweptLargeObject);
-        previousLargeObject->m_next = m_firstLargeObject;
-        m_firstLargeObject = m_firstUnsweptLargeObject;
-        m_firstUnsweptLargeObject = nullptr;
-    }
-    ASSERT(!m_firstUnsweptLargeObject);
 }
 
-void ThreadHeap::clearFreeLists()
+void ThreadHeapForHeapPage::clearFreeLists()
 {
+    setAllocationPoint(nullptr, 0);
     m_freeList.clear();
 }
 
 #if ENABLE(GC_PROFILING)
 void ThreadHeap::snapshotFreeList(TracedValue& json)
 {
     json.setInteger("cumulativeAllocationSize", m_cumulativeAllocationSize);
     json.setDouble("inlineAllocationRate", static_cast<double>(m_inlineAllocationCount) / m_allocationCount);
     json.setInteger("inlineAllocationCount", m_inlineAllocationCount);
     json.setInteger("allocationCount", m_allocationCount);
@@ skipping 68 matching lines @@
             ++entryCount;
             freeSize += entry->size();
         }
         totalFreeSize += freeSize;
     }
 }
 #endif
 
 HeapPage::HeapPage(PageMemory* storage, ThreadHeap* heap)
     : BaseHeapPage(storage, heap)
-    , m_next(nullptr)
 {
     m_objectStartBitMapComputed = false;
     ASSERT(isPageHeaderAddress(reinterpret_cast<Address>(this)));
 }
 
 size_t HeapPage::objectPayloadSizeForTesting()
 {
     size_t objectPayloadSize = 0;
     Address headerAddress = payload();
     markAsSwept();
@@ skipping 22 matching lines @@
     clearObjectStartBitMap();
 
     size_t markedObjectSize = 0;
     Address startOfGap = payload();
     for (Address headerAddress = startOfGap; headerAddress < payloadEnd(); ) {
         HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAddress);
         ASSERT(header->size() > 0);
         ASSERT(header->size() < blinkPagePayloadSize());
 
         if (header->isPromptlyFreed())
-            heap()->decreasePromptlyFreedSize(header->size());
+            heapForHeapPage()->decreasePromptlyFreedSize(header->size());
         if (header->isFree()) {
             size_t size = header->size();
             // Zero the memory in the free list header to maintain the
             // invariant that memory on the free list is zero filled.
             // The rest of the memory is already on the free list and is
             // therefore already zero filled.
             FILL_ZERO_IF_PRODUCTION(headerAddress, size < sizeof(FreeListEntry) ? size : sizeof(FreeListEntry));
             headerAddress += size;
             continue;
         }
@@ skipping 12 matching lines @@
             header->finalize(payload, payloadSize);
             // This memory will be added to the freelist. Maintain the invariant
             // that memory on the freelist is zero filled.
             FILL_ZERO_IF_PRODUCTION(headerAddress, size);
             ASAN_POISON_MEMORY_REGION(payload, payloadSize);
             headerAddress += size;
             continue;
         }
 
         if (startOfGap != headerAddress)
-            heap()->addToFreeList(startOfGap, headerAddress - startOfGap);
+            heapForHeapPage()->addToFreeList(startOfGap, headerAddress - startOfGap);
         header->unmark();
         headerAddress += header->size();
         markedObjectSize += header->size();
         startOfGap = headerAddress;
     }
     if (startOfGap != payloadEnd())
-        heap()->addToFreeList(startOfGap, payloadEnd() - startOfGap);
+        heapForHeapPage()->addToFreeList(startOfGap, payloadEnd() - startOfGap);
 
     if (markedObjectSize)
         Heap::increaseMarkedObjectSize(markedObjectSize);
 }
 
 void HeapPage::markUnmarkedObjectsDead()
 {
     for (Address headerAddress = payload(); headerAddress < payloadEnd();) {
         HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAddress);
         ASSERT(header->size() < blinkPagePayloadSize());
         // Check if a free list entry first since we cannot call
         // isMarked on a free list entry.
         if (header->isFree()) {
             headerAddress += header->size();
             continue;
         }
         header->checkHeader();
         if (header->isMarked())
             header->unmark();
         else
             header->markDead();
         headerAddress += header->size();
     }
 }
 
-void HeapPage::removeFromHeap(ThreadHeap* heap)
+void HeapPage::removeFromHeap()
 {
-    heap->freePage(this);
+    heapForHeapPage()->freePage(this);
+}
+
+ThreadHeapForHeapPage* HeapPage::heapForHeapPage()
+{
+    return static_cast<ThreadHeapForHeapPage*>(heap());
 }
 
 void HeapPage::populateObjectStartBitMap()
 {
     memset(&m_objectStartBitMap, 0, objectStartBitMapSize);
     Address start = payload();
     for (Address headerAddress = start; headerAddress < payloadEnd();) {
         HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAddress);
         size_t objectOffset = headerAddress - start;
         ASSERT(!(objectOffset & allocationMask));
@@ skipping 898 matching lines @@
 
 double Heap::estimatedMarkingTime()
 {
     // FIXME: Implement heuristics
     return 0.0;
 }
 
 void ThreadHeap::prepareHeapForTermination()
 {
     ASSERT(!m_firstUnsweptPage);
-    ASSERT(!m_firstUnsweptLargeObject);
-    for (HeapPage* page = m_firstPage; page; page = page->next()) {
+    for (BaseHeapPage* page = m_firstPage; page; page = page->next()) {
         page->setTerminating();
     }
-    for (LargeObject* largeObject = m_firstLargeObject; largeObject; largeObject = largeObject->next()) {
-        largeObject->setTerminating();
-    }
 }
 
 size_t Heap::objectPayloadSizeForTesting()
 {
     size_t objectPayloadSize = 0;
     for (ThreadState* state : ThreadState::attachedThreads()) {
         state->setGCState(ThreadState::GCRunning);
         state->makeConsistentForSweeping();
         objectPayloadSize += state->objectPayloadSizeForTesting();
         state->setGCState(ThreadState::EagerSweepScheduled);
@@ skipping 14 matching lines @@
     ASSERT(!state->isInGC());
 
     // Don't promptly free large objects because their page is never reused.
     // Don't free backings allocated on other threads.
     BaseHeapPage* page = pageFromObject(address);
     if (page->isLargeObject() || page->heap()->threadState() != state)
         return;
 
     HeapObjectHeader* header = HeapObjectHeader::fromPayload(address);
     header->checkHeader();
-    static_cast<HeapPage*>(page)->heap()->promptlyFreeObject(header);
+    static_cast<HeapPage*>(page)->heapForHeapPage()->promptlyFreeObject(header);
 }
 
 void HeapAllocator::freeVectorBacking(void* address)
 {
     backingFree(address);
 }
 
 void HeapAllocator::freeInlineVectorBacking(void* address)
 {
     backingFree(address);
@@ skipping 16 matching lines @@
     ASSERT(state->isAllocationAllowed());
 
     // FIXME: Support expand for large objects.
     // Don't expand backings allocated on other threads.
     BaseHeapPage* page = pageFromObject(address);
     if (page->isLargeObject() || page->heap()->threadState() != state)
         return false;
 
     HeapObjectHeader* header = HeapObjectHeader::fromPayload(address);
     header->checkHeader();
-    return static_cast<HeapPage*>(page)->heap()->expandObject(header, newSize);
+    return static_cast<HeapPage*>(page)->heapForHeapPage()->expandObject(header, newSize);
 }
 
 bool HeapAllocator::expandVectorBacking(void* address, size_t newSize)
 {
     return backingExpand(address, newSize);
 }
 
 bool HeapAllocator::expandInlineVectorBacking(void* address, size_t newSize)
 {
     return backingExpand(address, newSize);
@@ skipping 22 matching lines @@
     ASSERT(state->isAllocationAllowed());
 
     // FIXME: Support shrink for large objects.
     // Don't shrink backings allocated on other threads.
     BaseHeapPage* page = pageFromObject(address);
     if (page->isLargeObject() || page->heap()->threadState() != state)
         return;
 
     HeapObjectHeader* header = HeapObjectHeader::fromPayload(address);
     header->checkHeader();
-    static_cast<HeapPage*>(page)->heap()->shrinkObject(header, quantizedShrunkSize);
+    static_cast<HeapPage*>(page)->heapForHeapPage()->shrinkObject(header, quantizedShrunkSize);
 }
 
 void HeapAllocator::shrinkVectorBackingInternal(void* address, size_t quantizedCurrentSize, size_t quantizedShrunkSize)
 {
     backingShrink(address, quantizedCurrentSize, quantizedShrunkSize);
 }
 
 void HeapAllocator::shrinkInlineVectorBackingInternal(void* address, size_t quantizedCurrentSize, size_t quantizedShrunkSize)
 {
     backingShrink(address, quantizedCurrentSize, quantizedShrunkSize);
@@ skipping 99 matching lines @@
 bool Heap::s_shutdownCalled = false;
 bool Heap::s_lastGCWasConservative = false;
 FreePagePool* Heap::s_freePagePool;
 OrphanedPagePool* Heap::s_orphanedPagePool;
 Heap::RegionTree* Heap::s_regionTree = nullptr;
 size_t Heap::s_allocatedObjectSize = 0;
 size_t Heap::s_allocatedSpace = 0;
 size_t Heap::s_markedObjectSize = 0;
 
 } // namespace blink