Prepare for multiple ThreadHeaps

third_party/WebKit/Source/platform/heap/ThreadState.cpp

Index: third_party/WebKit/Source/platform/heap/ThreadState.cpp
diff --git a/third_party/WebKit/Source/platform/heap/ThreadState.cpp b/third_party/WebKit/Source/platform/heap/ThreadState.cpp
index 119e5e6f2b90327caa62c37aabf3e0f4a29db958..137daa4aa50d7907021466e8f70c0a75b474b94b 100644
--- a/third_party/WebKit/Source/platform/heap/ThreadState.cpp
+++ b/third_party/WebKit/Source/platform/heap/ThreadState.cpp
@@ -72,13 +72,6 @@ WTF::ThreadSpecific<ThreadState*>* ThreadState::s_threadSpecific = nullptr;
 uintptr_t ThreadState::s_mainThreadStackStart = 0;
 uintptr_t ThreadState::s_mainThreadUnderestimatedStackSize = 0;
 uint8_t ThreadState::s_mainThreadStateStorage[sizeof(ThreadState)];
-SafePointBarrier* ThreadState::s_safePointBarrier = nullptr;
-
-RecursiveMutex& ThreadState::threadAttachMutex()
-{
-    DEFINE_THREAD_SAFE_STATIC_LOCAL(RecursiveMutex, mutex, (new RecursiveMutex));
-    return mutex;
-}
 
 ThreadState::ThreadState()
     : m_thread(currentThread())
@@ -122,7 +115,12 @@ ThreadState::ThreadState()
         size_t underestimatedStackSize = StackFrameDepth::getUnderestimatedStackSize();
         if (underestimatedStackSize > sizeof(void*))
             s_mainThreadUnderestimatedStackSize = underestimatedStackSize - sizeof(void*);
+        m_heap = new ThreadHeap();
+    } else {
+        m_heap = &ThreadState::mainThreadState()->heap();
     }
+    ASSERT(m_heap);
+    m_heap->attach(this);
 
     for (int arenaIndex = 0; arenaIndex < BlinkGC::LargeObjectArenaIndex; arenaIndex++)
         m_arenas[arenaIndex] = new NormalPageArena(this, arenaIndex);
@@ -152,20 +150,6 @@ ThreadState::~ThreadState()
     }
 }
 
-void ThreadState::init()
-{
-    s_threadSpecific = new WTF::ThreadSpecific<ThreadState*>();
-    s_safePointBarrier = new SafePointBarrier;
-}
-
-void ThreadState::shutdown()
-{
-    delete s_safePointBarrier;
-    s_safePointBarrier = nullptr;
-
-    // Thread-local storage shouldn't be disposed, so we don't call ~ThreadSpecific().
-}
-
 #if OS(WIN) && COMPILER(MSVC)
 size_t ThreadState::threadStackSize()
 {
@@ -203,11 +187,69 @@ size_t ThreadState::threadStackSize()
 
 void ThreadState::attachMainThread()
 {
-    MutexLocker locker(threadAttachMutex());
-    ThreadState* state = new (s_mainThreadStateStorage) ThreadState();
-    attachedThreads().add(state);
+    RELEASE_ASSERT(!ProcessHeap::s_shutdownComplete);
+    s_threadSpecific = new WTF::ThreadSpecific<ThreadState*>();
+    new (s_mainThreadStateStorage) ThreadState();
+}
+
+void ThreadState::attachCurrentThread()
+{
+    RELEASE_ASSERT(!ProcessHeap::s_shutdownComplete);
+    new ThreadState();
+}
+
+void ThreadState::cleanupPages()
+{
+    ASSERT(checkThread());
+    for (int i = 0; i < BlinkGC::NumberOfArenas; ++i)
+        m_arenas[i]->cleanupPages();
 }
 
+void ThreadState::runTerminationGC()
+{
+    if (isMainThread()) {
+        cleanupPages();
+        return;
+    }
+    ASSERT(checkThread());
+
+    // Finish sweeping.
+    completeSweep();
+
+    // From here on ignore all conservatively discovered
+    // pointers into the heap owned by this thread.
+    m_isTerminating = true;
+
+    releaseStaticPersistentNodes();
+
+    // Set the terminate flag on all heap pages of this thread. This is used to
+    // ensure we don't trace pages on other threads that are not part of the
+    // thread local GC.
+    prepareForThreadStateTermination();
+
+    ProcessHeap::crossThreadPersistentRegion().prepareForThreadStateTermination(this);
+
+    // Do thread local GC's as long as the count of thread local Persistents
+    // changes and is above zero.
+    int oldCount = -1;
+    int currentCount = getPersistentRegion()->numberOfPersistents();
+    ASSERT(currentCount >= 0);
+    while (currentCount != oldCount) {
+        ThreadHeap::collectGarbageForTerminatingThread(this);
+        oldCount = currentCount;
+        currentCount = getPersistentRegion()->numberOfPersistents();
+    }
+    // We should not have any persistents left when getting to this point,
+    // if we have it is probably a bug so adding a debug ASSERT to catch this.
+    ASSERT(!currentCount);
+    // All of pre-finalizers should be consumed.
+    ASSERT(m_orderedPreFinalizers.isEmpty());
+    RELEASE_ASSERT(gcState() == NoGCScheduled);
+
+    // Add pages to the orphaned page pool to ensure any global GCs from this point
+    // on will not trace objects on this thread's arenas.
+    cleanupPages();
+}
 
 void ThreadState::cleanupMainThread()
 {
@@ -244,115 +286,20 @@ void ThreadState::detachMainThread()
     // threadAttachMutex and waiting for other threads to pause or reach a
     // safepoint.
     ThreadState* state = mainThreadState();
-    ASSERT(state == ThreadState::current());
-    ASSERT(state->checkThread());
     ASSERT(!state->isSweepingInProgress());
 
-    // The main thread must be the last thread that gets detached.
-    RELEASE_ASSERT(ThreadState::attachedThreads().size() == 1);
-
-    // Add the main thread's heap pages to the orphaned pool.
-    state->cleanupPages();
-
-    // Detach the main thread. We don't need to grab a lock because
-    // the main thread should be the last thread that gets detached.
-    ASSERT(attachedThreads().contains(state));
-    attachedThreads().remove(state);
+    state->heap().detach(state);
     state->~ThreadState();
 }
 
-void ThreadState::attach()
-{
-    MutexLocker locker(threadAttachMutex());
-    ThreadState* state = new ThreadState();
-    attachedThreads().add(state);
-}
-
-void ThreadState::cleanupPages()
-{
-    ASSERT(checkThread());
-    for (int i = 0; i < BlinkGC::NumberOfArenas; ++i)
-        m_arenas[i]->cleanupPages();
-}
-
-void ThreadState::cleanup()
-{
-    ASSERT(checkThread());
-    {
-        // Grab the threadAttachMutex to ensure only one thread can shutdown at
-        // a time and that no other thread can do a global GC. It also allows
-        // safe iteration of the attachedThreads set which happens as part of
-        // thread local GC asserts. We enter a safepoint while waiting for the
-        // lock to avoid a dead-lock where another thread has already requested
-        // GC.
-        SafePointAwareMutexLocker locker(threadAttachMutex(), BlinkGC::NoHeapPointersOnStack);
-
-        // Finish sweeping.
-        completeSweep();
-
-        // From here on ignore all conservatively discovered
-        // pointers into the heap owned by this thread.
-        m_isTerminating = true;
-
-        releaseStaticPersistentNodes();
-
-        // Set the terminate flag on all heap pages of this thread. This is used to
-        // ensure we don't trace pages on other threads that are not part of the
-        // thread local GC.
-        prepareForThreadStateTermination();
-
-        ProcessHeap::crossThreadPersistentRegion().prepareForThreadStateTermination(this);
-
-        // Do thread local GC's as long as the count of thread local Persistents
-        // changes and is above zero.
-        int oldCount = -1;
-        int currentCount = getPersistentRegion()->numberOfPersistents();
-        ASSERT(currentCount >= 0);
-        while (currentCount != oldCount) {
-            ThreadHeap::collectGarbageForTerminatingThread(this);
-            oldCount = currentCount;
-            currentCount = getPersistentRegion()->numberOfPersistents();
-        }
-        // We should not have any persistents left when getting to this point,
-        // if we have it is probably a bug so adding a debug ASSERT to catch this.
-        ASSERT(!currentCount);
-        // All of pre-finalizers should be consumed.
-        ASSERT(m_orderedPreFinalizers.isEmpty());
-        RELEASE_ASSERT(gcState() == NoGCScheduled);
-
-        // Add pages to the orphaned page pool to ensure any global GCs from this point
-        // on will not trace objects on this thread's arenas.
-        cleanupPages();
-
-        ASSERT(attachedThreads().contains(this));
-        attachedThreads().remove(this);
-    }
-}
-
-void ThreadState::detach()
+void ThreadState::detachCurrentThread()
 {
     ThreadState* state = current();
-    state->cleanup();
+    state->heap().detach(state);
     RELEASE_ASSERT(state->gcState() == ThreadState::NoGCScheduled);
     delete state;
 }
 
-void ThreadState::visitPersistentRoots(Visitor* visitor)
-{
-    TRACE_EVENT0("blink_gc", "ThreadState::visitPersistentRoots");
-    ProcessHeap::crossThreadPersistentRegion().tracePersistentNodes(visitor);
-
-    for (ThreadState* state : attachedThreads())
-        state->visitPersistents(visitor);
-}
-
-void ThreadState::visitStackRoots(Visitor* visitor)
-{
-    TRACE_EVENT0("blink_gc", "ThreadState::visitStackRoots");
-    for (ThreadState* state : attachedThreads())
-        state->visitStack(visitor);
-}
-
 NO_SANITIZE_ADDRESS
 void ThreadState::visitAsanFakeStackForPointer(Visitor* visitor, Address ptr)
 {
@@ -375,13 +322,16 @@ void ThreadState::visitAsanFakeStackForPointer(Visitor* visitor, Address ptr)
             // within the stack range that we need to scan so we need
             // to visit the values in the fake frame.
             for (Address* p = fakeFrameStart; p < fakeFrameEnd; ++p)
-                ThreadHeap::checkAndMarkPointer(visitor, *p);
+                m_heap->checkAndMarkPointer(visitor, *p);
         }
     }
 #endif
 }
 
+// Stack scanning may overrun the bounds of local objects and/or race with
+// other threads that use this stack.
 NO_SANITIZE_ADDRESS
+NO_SANITIZE_THREAD
 void ThreadState::visitStack(Visitor* visitor)
 {
     if (m_stackState == BlinkGC::NoHeapPointersOnStack)
@@ -411,7 +361,7 @@ void ThreadState::visitStack(Visitor* visitor)
         // variable because we don't want to unpoison the original stack.
         __msan_unpoison(&ptr, sizeof(ptr));
 #endif
-        ThreadHeap::checkAndMarkPointer(visitor, ptr);
+        m_heap->checkAndMarkPointer(visitor, ptr);
         visitAsanFakeStackForPointer(visitor, ptr);
     }
 
@@ -420,7 +370,7 @@ void ThreadState::visitStack(Visitor* visitor)
         // See the comment above.
         __msan_unpoison(&ptr, sizeof(ptr));
 #endif
-        ThreadHeap::checkAndMarkPointer(visitor, ptr);
+        m_heap->checkAndMarkPointer(visitor, ptr);
         visitAsanFakeStackForPointer(visitor, ptr);
     }
 }
@@ -521,18 +471,18 @@ void ThreadState::threadLocalWeakProcessing()
 
 size_t ThreadState::totalMemorySize()
 {
-    return ThreadHeap::heapStats().allocatedObjectSize() + ThreadHeap::heapStats().markedObjectSize() + WTF::Partitions::totalSizeOfCommittedPages();
+    return m_heap->heapStats().allocatedObjectSize() + m_heap->heapStats().markedObjectSize() + WTF::Partitions::totalSizeOfCommittedPages();
 }
 
 size_t ThreadState::estimatedLiveSize(size_t estimationBaseSize, size_t sizeAtLastGC)
 {
-    if (ThreadHeap::heapStats().wrapperCountAtLastGC() == 0) {
+    if (m_heap->heapStats().wrapperCountAtLastGC() == 0) {
         // We'll reach here only before hitting the first GC.
         return 0;
     }
 
     // (estimated size) = (estimation base size) - (heap size at the last GC) / (# of persistent handles at the last GC) * (# of persistent handles collected since the last GC);
-    size_t sizeRetainedByCollectedPersistents = static_cast<size_t>(1.0 * sizeAtLastGC / ThreadHeap::heapStats().wrapperCountAtLastGC() * ThreadHeap::heapStats().collectedWrapperCount());
+    size_t sizeRetainedByCollectedPersistents = static_cast<size_t>(1.0 * sizeAtLastGC / m_heap->heapStats().wrapperCountAtLastGC() * m_heap->heapStats().collectedWrapperCount());
     if (estimationBaseSize < sizeRetainedByCollectedPersistents)
         return 0;
     return estimationBaseSize - sizeRetainedByCollectedPersistents;
@@ -540,8 +490,8 @@ size_t ThreadState::estimatedLiveSize(size_t estimationBaseSize, size_t sizeAtLa
 
 double ThreadState::heapGrowingRate()
 {
-    size_t currentSize = ThreadHeap::heapStats().allocatedObjectSize() + ThreadHeap::heapStats().markedObjectSize();
-    size_t estimatedSize = estimatedLiveSize(ThreadHeap::heapStats().markedObjectSizeAtLastCompleteSweep(), ThreadHeap::heapStats().markedObjectSizeAtLastCompleteSweep());
+    size_t currentSize = m_heap->heapStats().allocatedObjectSize() + m_heap->heapStats().markedObjectSize();
+    size_t estimatedSize = estimatedLiveSize(m_heap->heapStats().markedObjectSizeAtLastCompleteSweep(), m_heap->heapStats().markedObjectSizeAtLastCompleteSweep());
 
     // If the estimatedSize is 0, we set a high growing rate to trigger a GC.
     double growingRate = estimatedSize > 0 ? 1.0 * currentSize / estimatedSize : 100;
@@ -553,7 +503,7 @@ double ThreadState::heapGrowingRate()
 double ThreadState::partitionAllocGrowingRate()
 {
     size_t currentSize = WTF::Partitions::totalSizeOfCommittedPages();
-    size_t estimatedSize = estimatedLiveSize(currentSize, ThreadHeap::heapStats().partitionAllocSizeAtLastGC());
+    size_t estimatedSize = estimatedLiveSize(currentSize, m_heap->heapStats().partitionAllocSizeAtLastGC());
 
     // If the estimatedSize is 0, we set a high growing rate to trigger a GC.
     double growingRate = estimatedSize > 0 ? 1.0 * currentSize / estimatedSize : 100;
@@ -567,7 +517,7 @@ double ThreadState::partitionAllocGrowingRate()
 bool ThreadState::judgeGCThreshold(size_t totalMemorySizeThreshold, double heapGrowingRateThreshold)
 {
     // If the allocated object size or the total memory size is small, don't trigger a GC.
-    if (ThreadHeap::heapStats().allocatedObjectSize() < 100 * 1024 || totalMemorySize() < totalMemorySizeThreshold)
+    if (m_heap->heapStats().allocatedObjectSize() < 100 * 1024 || totalMemorySize() < totalMemorySizeThreshold)
         return false;
     // If the growing rate of Oilpan's heap or PartitionAlloc is high enough,
     // trigger a GC.
@@ -751,6 +701,15 @@ void ThreadState::scheduleGCIfNeeded()
     }
 }
 
+ThreadState* ThreadState::fromObject(const void* object)
+{
+    ASSERT(object);
+    BasePage* page = pageFromObject(object);
+    ASSERT(page);
+    ASSERT(page->arena());
+    return page->arena()->getThreadState();
+}
+
 void ThreadState::performIdleGC(double deadlineSeconds)
 {
     ASSERT(checkThread());
@@ -761,8 +720,8 @@ void ThreadState::performIdleGC(double deadlineSeconds)
         return;
 
     double idleDeltaInSeconds = deadlineSeconds - monotonicallyIncreasingTime();
-    TRACE_EVENT2("blink_gc", "ThreadState::performIdleGC", "idleDeltaInSeconds", idleDeltaInSeconds, "estimatedMarkingTime", ThreadHeap::heapStats().estimatedMarkingTime());
-    if (idleDeltaInSeconds <= ThreadHeap::heapStats().estimatedMarkingTime() && !Platform::current()->currentThread()->scheduler()->canExceedIdleDeadlineIfRequired()) {
+    TRACE_EVENT2("blink_gc", "ThreadState::performIdleGC", "idleDeltaInSeconds", idleDeltaInSeconds, "estimatedMarkingTime", m_heap->heapStats().estimatedMarkingTime());
+    if (idleDeltaInSeconds <= m_heap->heapStats().estimatedMarkingTime() && !Platform::current()->currentThread()->scheduler()->canExceedIdleDeadlineIfRequired()) {
         // If marking is estimated to take longer than the deadline and we can't
         // exceed the deadline, then reschedule for the next idle period.
         scheduleIdleGC();
@@ -964,7 +923,7 @@ void ThreadState::runScheduledGC(BlinkGC::StackState stackState)
 void ThreadState::flushHeapDoesNotContainCacheIfNeeded()
 {
     if (m_shouldFlushHeapDoesNotContainCache) {
-        ThreadHeap::flushHeapDoesNotContainCache();
+        m_heap->flushHeapDoesNotContainCache();
         m_shouldFlushHeapDoesNotContainCache = false;
     }
 }
@@ -1131,8 +1090,8 @@ void ThreadState::postSweep()
 
     if (isMainThread()) {
         double collectionRate = 0;
-        if (ThreadHeap::heapStats().objectSizeAtLastGC() > 0)
-            collectionRate = 1 - 1.0 * ThreadHeap::heapStats().markedObjectSize() / ThreadHeap::heapStats().objectSizeAtLastGC();
+        if (m_heap->heapStats().objectSizeAtLastGC() > 0)
+            collectionRate = 1 - 1.0 * m_heap->heapStats().markedObjectSize() / m_heap->heapStats().objectSizeAtLastGC();
         TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "ThreadState::collectionRate", static_cast<int>(100 * collectionRate));
 
 #if PRINT_HEAP_STATS
@@ -1141,12 +1100,12 @@ void ThreadState::postSweep()
 
         // ThreadHeap::markedObjectSize() may be underestimated here if any other
         // thread has not yet finished lazy sweeping.
-        ThreadHeap::heapStats().setMarkedObjectSizeAtLastCompleteSweep(ThreadHeap::heapStats().markedObjectSize());
+        m_heap->heapStats().setMarkedObjectSizeAtLastCompleteSweep(m_heap->heapStats().markedObjectSize());
 
         DEFINE_STATIC_LOCAL(CustomCountHistogram, objectSizeBeforeGCHistogram, ("BlinkGC.ObjectSizeBeforeGC", 1, 4 * 1024 * 1024, 50));
-        objectSizeBeforeGCHistogram.count(ThreadHeap::heapStats().objectSizeAtLastGC() / 1024);
+        objectSizeBeforeGCHistogram.count(m_heap->heapStats().objectSizeAtLastGC() / 1024);
         DEFINE_STATIC_LOCAL(CustomCountHistogram, objectSizeAfterGCHistogram, ("BlinkGC.ObjectSizeAfterGC", 1, 4 * 1024 * 1024, 50));
-        objectSizeAfterGCHistogram.count(ThreadHeap::heapStats().markedObjectSize() / 1024);
+        objectSizeAfterGCHistogram.count(m_heap->heapStats().markedObjectSize() / 1024);
         DEFINE_STATIC_LOCAL(CustomCountHistogram, collectionRateHistogram, ("BlinkGC.CollectionRate", 1, 100, 20));
         collectionRateHistogram.count(static_cast<int>(100 * collectionRate));
         DEFINE_STATIC_LOCAL(CustomCountHistogram, timeForSweepHistogram, ("BlinkGC.TimeForSweepingAllObjects", 1, 10 * 1000, 50));
@@ -1161,7 +1120,7 @@ void ThreadState::postSweep()
         break;                                                                \
     }
 
-        switch (ThreadHeap::lastGCReason()) {
+        switch (m_heap->lastGCReason()) {
             COUNT_COLLECTION_RATE_HISTOGRAM_BY_GC_REASON(IdleGC)
             COUNT_COLLECTION_RATE_HISTOGRAM_BY_GC_REASON(PreciseGC)
             COUNT_COLLECTION_RATE_HISTOGRAM_BY_GC_REASON(ConservativeGC)
@@ -1215,16 +1174,6 @@ size_t ThreadState::objectPayloadSizeForTesting()
     return objectPayloadSize;
 }
 
-bool ThreadState::stopThreads()
-{
-    return s_safePointBarrier->parkOthers();
-}
-
-void ThreadState::resumeThreads()
-{
-    s_safePointBarrier->resumeOthers();
-}
-
 void ThreadState::safePoint(BlinkGC::StackState stackState)
 {
     ASSERT(checkThread());
@@ -1234,7 +1183,7 @@ void ThreadState::safePoint(BlinkGC::StackState stackState)
     ASSERT(!m_atSafePoint);
     m_stackState = stackState;
     m_atSafePoint = true;
-    s_safePointBarrier->checkAndPark(this);
+    m_heap->checkAndPark(this, nullptr);
     m_atSafePoint = false;
     m_stackState = BlinkGC::HeapPointersOnStack;
     preSweep();
@@ -1278,14 +1227,14 @@ void ThreadState::enterSafePoint(BlinkGC::StackState stackState, void* scopeMark
     m_atSafePoint = true;
     m_stackState = stackState;
     m_safePointScopeMarker = scopeMarker;
-    s_safePointBarrier->enterSafePoint(this);
+    m_heap->enterSafePoint(this);
 }
 
 void ThreadState::leaveSafePoint(SafePointAwareMutexLocker* locker)
 {
     ASSERT(checkThread());
     ASSERT(m_atSafePoint);
-    s_safePointBarrier->leaveSafePoint(this, locker);
+    m_heap->leaveSafePoint(this, locker);
     m_atSafePoint = false;
     m_stackState = BlinkGC::HeapPointersOnStack;
     clearSafePointScopeMarker();
@@ -1312,19 +1261,19 @@ void ThreadState::resetHeapCounters()
 void ThreadState::increaseAllocatedObjectSize(size_t delta)
 {
     m_allocatedObjectSize += delta;
-    ThreadHeap::heapStats().increaseAllocatedObjectSize(delta);
+    m_heap->heapStats().increaseAllocatedObjectSize(delta);
 }
 
 void ThreadState::decreaseAllocatedObjectSize(size_t delta)
 {
     m_allocatedObjectSize -= delta;
-    ThreadHeap::heapStats().decreaseAllocatedObjectSize(delta);
+    m_heap->heapStats().decreaseAllocatedObjectSize(delta);
 }
 
 void ThreadState::increaseMarkedObjectSize(size_t delta)
 {
     m_markedObjectSize += delta;
-    ThreadHeap::heapStats().increaseMarkedObjectSize(delta);
+    m_heap->heapStats().increaseMarkedObjectSize(delta);
 }
 
 void ThreadState::copyStackUntilSafePointScope()
@@ -1358,7 +1307,7 @@ void ThreadState::addInterruptor(PassOwnPtr<BlinkGCInterruptor> interruptor)
     ASSERT(checkThread());
     SafePointScope scope(BlinkGC::HeapPointersOnStack);
     {
-        MutexLocker locker(threadAttachMutex());
+        MutexLocker locker(m_heap->threadAttachMutex());
         m_interruptors.append(std::move(interruptor));
     }
 }
@@ -1409,20 +1358,14 @@ void ThreadState::leaveStaticReferenceRegistrationDisabledScope()
 }
 #endif
 
-ThreadState::AttachedThreadStateSet& ThreadState::attachedThreads()
-{
-    DEFINE_STATIC_LOCAL(AttachedThreadStateSet, threads, ());
-    return threads;
-}
-
 void ThreadState::lockThreadAttachMutex()
 {
-    threadAttachMutex().lock();
+    m_heap->threadAttachMutex().lock();
 }
 
 void ThreadState::unlockThreadAttachMutex()
 {
-    threadAttachMutex().unlock();
+    m_heap->threadAttachMutex().unlock();
 }
 
 void ThreadState::invokePreFinalizers()