Refactor the Heap into ThreadHeap to prepare for per thread heaps

third_party/WebKit/Source/platform/heap/ThreadState.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 18 matching lines @@
  */
 
 #include "platform/heap/ThreadState.h"
 
 #include "platform/ScriptForbiddenScope.h"
 #include "platform/TraceEvent.h"
 #include "platform/heap/BlinkGCMemoryDumpProvider.h"
 #include "platform/heap/CallbackStack.h"
 #include "platform/heap/Handle.h"
 #include "platform/heap/Heap.h"
+#include "platform/heap/HeapPage.h"
 #include "platform/heap/MarkingVisitor.h"
+#include "platform/heap/PagePool.h"
 #include "platform/heap/SafePoint.h"
 #include "public/platform/Platform.h"
 #include "public/platform/WebMemoryAllocatorDump.h"
 #include "public/platform/WebProcessMemoryDump.h"
 #include "public/platform/WebScheduler.h"
 #include "public/platform/WebThread.h"
 #include "public/platform/WebTraceLocation.h"
 #include "wtf/DataLog.h"
 #include "wtf/Partitions.h"
 #include "wtf/ThreadingPrimitives.h"
 
 #if OS(WIN)
 #include <stddef.h>
 #include <windows.h>
 #include <winnt.h>
 #elif defined(__GLIBC__)
 extern "C" void* __libc_stack_end;  // NOLINT
 #endif
 
 #if defined(MEMORY_SANITIZER)
 #include <sanitizer/msan_interface.h>
 #endif
 
 #if OS(FREEBSD)
 #include <pthread_np.h>
 #endif
 
 namespace blink {
 
+GCHeapStats::GCHeapStats()
+    : m_allocatedSpace(0)
+    , m_allocatedObjectSize(0)
+    , m_objectSizeAtLastGC(0)
+    , m_markedObjectSize(0)
+    , m_markedObjectSizeAtLastCompleteSweep(0)
+    , m_wrapperCount(0)
+    , m_wrapperCountAtLastGC(0)
+    , m_collectedWrapperCount(0)
+    , m_partitionAllocSizeAtLastGC(WTF::Partitions::totalSizeOfCommittedPages())
+    , m_estimatedMarkingTimePerByte(0.0)
+{
+}
+
+double GCHeapStats::estimatedMarkingTime()
+{
+    // Use 8 ms as initial estimated marking time.
+    // 8 ms is long enough for low-end mobile devices to mark common
+    // real-world object graphs.
+    if (m_estimatedMarkingTimePerByte == 0)
+        return 0.008;
+
+    // Assuming that the collection rate of this GC will be mostly equal to
+    // the collection rate of the last GC, estimate the marking time of this GC.
+    return m_estimatedMarkingTimePerByte * (allocatedObjectSize() + markedObjectSize());
+}
+
+void GCHeapStats::reset()
+{
+    m_objectSizeAtLastGC = m_allocatedObjectSize + m_markedObjectSize;
+    m_partitionAllocSizeAtLastGC = WTF::Partitions::totalSizeOfCommittedPages();

[haraken, 2016/02/12 11:28:52]

It would be problematic to simply use PartitionAlloc's size for the threshold to
trigger per-thread GCs. If we do that, a per-thread GC of a worker thread may be
triggered because the main thread allocates a lot of objects on PartitionAlloc.

(Let's worry about this issue after resolving all other issues in this CL.)

+    m_allocatedObjectSize = 0;
+    m_markedObjectSize = 0;
+    m_wrapperCountAtLastGC = m_wrapperCount;
+    m_collectedWrapperCount = 0;
+}
+
+void GCHeapStats::increaseAllocatedObjectSize(size_t delta)

[haraken, 2016/02/12 11:28:52]

These changes will be made by peria-san's CL:

https://codereview.chromium.org/1670463002/

+{
+    atomicAdd(&m_allocatedObjectSize, static_cast<long>(delta));
+    Heap::increaseTotalAllocatedObjectSize(delta);
+}
+
+void GCHeapStats::decreaseAllocatedObjectSize(size_t delta)
+{
+    atomicSubtract(&m_allocatedObjectSize, static_cast<long>(delta));
+    Heap::decreaseTotalAllocatedObjectSize(delta);
+}
+
+void GCHeapStats::increaseMarkedObjectSize(size_t delta)
+{
+    atomicAdd(&m_markedObjectSize, static_cast<long>(delta));
+    Heap::increaseTotalMarkedObjectSize(delta);
+}
+
+void GCHeapStats::increaseAllocatedSpace(size_t delta)
+{
+    atomicAdd(&m_allocatedSpace, static_cast<long>(delta));
+    Heap::increaseTotalAllocatedSpace(delta);
+}
+
+void GCHeapStats::decreaseAllocatedSpace(size_t delta)
+{
+    atomicSubtract(&m_allocatedSpace, static_cast<long>(delta));
+    Heap::decreaseTotalAllocatedSpace(delta);
+}
+
+HashSet<GCGroup*>& GCGroup::all()
+{
+    DEFINE_STATIC_LOCAL(HashSet<GCGroup*>, groups, ());
+    return groups;
+}
+
+GCGroup::GCGroup()
+    : m_regionTree(nullptr)
+    , m_heapDoesNotContainCache(adoptPtr(new HeapDoesNotContainCache))
+{
+}
+
+GCGroup::~GCGroup()
+{
+}
+
+void GCGroup::flushHeapDoesNotContainCache()
+{
+    m_heapDoesNotContainCache->flush();
+}
+
+MultiThreadGCGroup::MultiThreadGCGroup()
+    : m_safePointBarrier(adoptPtr(new SafePointBarrier(this)))
+    , m_freePagePool(adoptPtr(new FreePagePool))

[haraken, 2016/02/12 11:28:52]

Maybe we should share the FreePagePool among all threads to save memory
consumption.

(Let's revisit this issue after resolving all other issues in this CL.)

+    , m_orphanedPagePool(adoptPtr(new OrphanedPagePool))
+{
+    GCGroup::all().add(this);
+}
+
+MultiThreadGCGroup::~MultiThreadGCGroup()
+{
+    GCGroup::all().remove(this);
+}
+
+void MultiThreadGCGroup::attach(ThreadState* thread)
+{
+    MutexLocker locker(m_threadAttachMutex);
+    m_threads.add(thread);
+}
+
+void MultiThreadGCGroup::detach(ThreadState* thread)
+{
+    SafePointAwareMutexLocker locker(m_threadAttachMutex, BlinkGC::NoHeapPointersOnStack);
+    thread->cleanupCallback();
+    ASSERT(m_threads.contains(thread));
+    m_threads.remove(thread);
+}
+
+bool MultiThreadGCGroup::park()
+{
+    return m_safePointBarrier->parkOthers();
+}
+
+void MultiThreadGCGroup::resume()
+{
+    m_safePointBarrier->resumeOthers();
+}
+
+bool MultiThreadGCGroup::isParked() const

[haraken, 2016/02/12 11:28:52]

Add #if ENABLE(ASSERT).

[haraken, 2016/02/12 11:28:52]

isParked => isAtSafePoint

[keishi, 2016/02/29 06:02:33]

Done.

[keishi, 2016/02/29 06:02:33]

Done.

+{
+    for (ThreadState* state : m_threads) {
+        if (!state->isAtSafePoint())
+            return false;
+    }
+    return true;
+}
+
+void MultiThreadGCGroup::lockThreadAttachMutex()
+{
+    m_threadAttachMutex.lock();
+}
+
+void MultiThreadGCGroup::unlockThreadAttachMutex()
+{
+    m_threadAttachMutex.unlock();
+}
+
+#if ENABLE(ASSERT)
+BasePage* MultiThreadGCGroup::findPageFromAddress(Address address)
+{
+    for (ThreadState* state : m_threads) {
+        if (BasePage* page = state->findPageFromAddress(address))
+            return page;
+    }
+    return nullptr;
+}
+#endif
+
+void MultiThreadGCGroup::preGC()
+{
+    for (ThreadState* state : m_threads) {
+        state->preGC();
+    }
+}
+
+void MultiThreadGCGroup::postGC(BlinkGC::GCType gcType)
+{
+    for (ThreadState* state : m_threads) {
+        state->postGC(gcType);
+    }
+}
+
+size_t MultiThreadGCGroup::objectPayloadSizeForTesting()
+{
+    size_t objectPayloadSize = 0;
+    for (ThreadState* state : m_threads) {
+        state->setGCState(ThreadState::GCRunning);
+        state->makeConsistentForGC();
+        objectPayloadSize += state->objectPayloadSizeForTesting();
+        state->setGCState(ThreadState::EagerSweepScheduled);
+        state->setGCState(ThreadState::Sweeping);
+        state->setGCState(ThreadState::NoGCScheduled);
+    }
+    return objectPayloadSize;
+}
+
+size_t MultiThreadGCGroup::size() const

[haraken, 2016/02/12 11:28:52]

size => threadCount ?

[keishi, 2016/02/29 06:02:34]

Done.

+{
+    return m_threads.size();
+}
+
+void MultiThreadGCGroup::visitPersistentRoots(Visitor* visitor)
+{
+    TRACE_EVENT0("blink_gc", "ThreadState::visitPersistentRoots");
+    Heap::crossThreadPersistentRegion().tracePersistentNodes(visitor);
+
+    for (ThreadState* state : m_threads) {
+        state->visitPersistents(visitor);
+    }
+}
+
+void MultiThreadGCGroup::visitStackRoots(Visitor* visitor)
+{
+    TRACE_EVENT0("blink_gc", "ThreadState::visitStackRoots");
+    for (ThreadState* state : m_threads) {
+        state->visitStack(visitor);
+    }
+}
+
+void MultiThreadGCGroup::checkAndPark(ThreadState* threadState, SafePointAwareMutexLocker* locker)
+{
+    m_safePointBarrier->checkAndPark(threadState, locker);
+}
+
+void MultiThreadGCGroup::enterSafePoint(ThreadState* threadState)
+{
+    m_safePointBarrier->enterSafePoint(threadState);
+}
+
+void MultiThreadGCGroup::leaveSafePoint(ThreadState* threadState, SafePointAwareMutexLocker* locker)
+{
+    m_safePointBarrier->leaveSafePoint(threadState, locker);
+}
+
+void MultiThreadGCGroup::shutdown()
+{
+    Heap::shutdown();
+}
+
+void MultiThreadGCGroup::shutdownIfNecessary()
+{
+    MutexLocker locker(m_threadAttachMutex);
+    if (m_threads.size() == 0 && Heap::s_shutdownCalled) {
+        shutdown();

[haraken, 2016/02/12 11:28:52]

As commented in Heap::shutdown(), I think we need to clean up the shutdown logic
a bit more. For example, Heap::shutdown() must be called only once. However, in
the current code, Heap::shutdown() will be called per MultiThreadGroup. Which is
wrong.

Let's visit this issue after resolving other issues.

+        delete this;
+    }
+}
+
+BasePage* GCGroup::lookupPageForAddress(Address address)
+{
+    ASSERT(ThreadState::current()->isInGC());
+    if (!m_regionTree)
+        return nullptr;
+    if (PageMemoryRegion* region = m_regionTree->lookup(address)) {
+        BasePage* page = region->pageFromAddress(address);
+        return page && !page->orphaned() ? page : nullptr;
+    }
+    return nullptr;
+}
+
+void GCGroup::addPageMemoryRegion(PageMemoryRegion* region)
+{
+    MutexLocker locker(m_regionTreeMutex);
+    RegionTree::add(new RegionTree(region), &m_regionTree);
+}
+
+void GCGroup::removePageMemoryRegion(PageMemoryRegion* region)
+{
+    // Deletion of large objects (and thus their regions) can happen
+    // concurrently on sweeper threads.  Removal can also happen during thread
+    // shutdown, but that case is safe.  Regardless, we make all removals
+    // mutually exclusive.
+    MutexLocker locker(m_regionTreeMutex);
+    RegionTree::remove(region, &m_regionTree);
+}
+
 WTF::ThreadSpecific<ThreadState*>* ThreadState::s_threadSpecific = nullptr;
 uintptr_t ThreadState::s_mainThreadStackStart = 0;
 uintptr_t ThreadState::s_mainThreadUnderestimatedStackSize = 0;
 uint8_t ThreadState::s_mainThreadStateStorage[sizeof(ThreadState)];
-SafePointBarrier* ThreadState::s_safePointBarrier = nullptr;
-
-RecursiveMutex& ThreadState::threadAttachMutex()
-{
-    DEFINE_THREAD_SAFE_STATIC_LOCAL(RecursiveMutex, mutex, (new RecursiveMutex));
-    return mutex;
-}
-
-ThreadState::ThreadState()
+
+ThreadState::ThreadState(bool perThreadHeapEnabled)
     : m_thread(currentThread())
     , m_persistentRegion(adoptPtr(new PersistentRegion()))
 #if OS(WIN) && COMPILER(MSVC)
     , m_threadStackSize(0)
 #endif
+    , m_perThreadHeapEnabled(perThreadHeapEnabled)
     , m_startOfStack(reinterpret_cast<intptr_t*>(StackFrameDepth::getStackStart()))
     , m_endOfStack(reinterpret_cast<intptr_t*>(StackFrameDepth::getStackStart()))
     , m_safePointScopeMarker(nullptr)
     , m_atSafePoint(false)
     , m_interruptors()
     , m_sweepForbidden(false)
     , m_noAllocationCount(0)
     , m_gcForbiddenCount(0)
     , m_accumulatedSweepingTime(0)
     , m_vectorBackingHeapIndex(BlinkGC::Vector1HeapIndex)
     , m_currentHeapAges(0)
     , m_isTerminating(false)
     , m_gcMixinMarker(nullptr)
     , m_shouldFlushHeapDoesNotContainCache(false)
     , m_gcState(NoGCScheduled)
     , m_traceDOMWrappers(nullptr)
+    , m_gcGroup(nullptr)
 #if defined(ADDRESS_SANITIZER)
     , m_asanFakeStack(__asan_get_current_fake_stack())
 #endif
 #if defined(LEAK_SANITIZER)
     , m_disabledStaticPersistentsRegistration(0)
 #endif
 {
     ASSERT(checkThread());
     ASSERT(!**s_threadSpecific);
     **s_threadSpecific = this;
 
     if (isMainThread()) {
         s_mainThreadStackStart = reinterpret_cast<uintptr_t>(m_startOfStack) - sizeof(void*);
         size_t underestimatedStackSize = StackFrameDepth::getUnderestimatedStackSize();
         if (underestimatedStackSize > sizeof(void*))
             s_mainThreadUnderestimatedStackSize = underestimatedStackSize - sizeof(void*);
+        m_gcGroup = new MultiThreadGCGroup();
+    } else if (m_perThreadHeapEnabled) {
+        m_gcGroup = new MultiThreadGCGroup();
+    } else {
+        m_gcGroup = ThreadState::mainThreadState()->gcGroup();
     }
+    ASSERT(m_gcGroup);
+    m_gcGroup->attach(this);
 
     for (int heapIndex = 0; heapIndex < BlinkGC::LargeObjectHeapIndex; heapIndex++)
         m_heaps[heapIndex] = new NormalPageHeap(this, heapIndex);
     m_heaps[BlinkGC::LargeObjectHeapIndex] = new LargeObjectHeap(this, BlinkGC::LargeObjectHeapIndex);
 
     m_likelyToBePromptlyFreed = adoptArrayPtr(new int[likelyToBePromptlyFreedArraySize]);
     clearHeapAges();
 
     m_threadLocalWeakCallbackStack = new CallbackStack();
 }
 
 ThreadState::~ThreadState()
 {
     ASSERT(checkThread());
     delete m_threadLocalWeakCallbackStack;
     m_threadLocalWeakCallbackStack = nullptr;
     for (int i = 0; i < BlinkGC::NumberOfHeaps; ++i)
         delete m_heaps[i];
 
     **s_threadSpecific = nullptr;
     if (isMainThread()) {
         s_mainThreadStackStart = 0;
         s_mainThreadUnderestimatedStackSize = 0;
     }
 }
 
 void ThreadState::init()
 {
     s_threadSpecific = new WTF::ThreadSpecific<ThreadState*>();
-    s_safePointBarrier = new SafePointBarrier;
 }
 
 void ThreadState::shutdown()
 {
-    delete s_safePointBarrier;
-    s_safePointBarrier = nullptr;
-
     // Thread-local storage shouldn't be disposed, so we don't call ~ThreadSpecific().
 }
 
 #if OS(WIN) && COMPILER(MSVC)
 size_t ThreadState::threadStackSize()
 {
     if (m_threadStackSize)
         return m_threadStackSize;
 
     // Notice that we cannot use the TIB's StackLimit for the stack end, as it
@@ skipping 18 matching lines @@
     // as off-limits and adjust the reported stack size accordingly.
     //
     // http://blogs.msdn.com/b/satyem/archive/2012/08/13/thread-s-stack-memory-management.aspx
     // explains the details.
     RELEASE_ASSERT(m_threadStackSize > 4 * 0x1000);
     m_threadStackSize -= 4 * 0x1000;
     return m_threadStackSize;
 }
 #endif
 
-void ThreadState::attachMainThread()
+void ThreadState::prepareForMainThread()
 {
     RELEASE_ASSERT(!Heap::s_shutdownCalled);
-    MutexLocker locker(threadAttachMutex());
-    ThreadState* state = new(s_mainThreadStateStorage) ThreadState();
-    attachedThreads().add(state);
+    new(s_mainThreadStateStorage) ThreadState(false);
 }
 
 void ThreadState::detachMainThread()
 {
     // Enter a safe point before trying to acquire threadAttachMutex
     // to avoid dead lock if another thread is preparing for GC, has acquired
     // threadAttachMutex and waiting for other threads to pause or reach a
     // safepoint.
     ThreadState* state = mainThreadState();
+    GCGroup* gcGroup = state->gcGroup();
 
     // 1. Finish sweeping.
     state->completeSweep();
-    {
-        SafePointAwareMutexLocker locker(threadAttachMutex(), BlinkGC::NoHeapPointersOnStack);
-
-        // 2. Add the main thread's heap pages to the orphaned pool.
-        state->cleanupPages();
-
-        // 3. Detach the main thread.
-        ASSERT(attachedThreads().contains(state));
-        attachedThreads().remove(state);
-        state->~ThreadState();
-    }
-    shutdownHeapIfNecessary();
+    gcGroup->detach(state);
+    state->~ThreadState();
+    gcGroup->shutdownIfNecessary();
 }
 
-void ThreadState::shutdownHeapIfNecessary()
-{
-    // We don't need to enter a safe point before acquiring threadAttachMutex
-    // because this thread is already detached.
-
-    MutexLocker locker(threadAttachMutex());
-    // We start shutting down the heap if there is no running thread
-    // and Heap::shutdown() is already called.
-    if (!attachedThreads().size() && Heap::s_shutdownCalled)
-        Heap::doShutdown();
-}
-
-void ThreadState::attach()
+void ThreadState::prepareForCurrentThread(bool perThreadHeap)
 {
     RELEASE_ASSERT(!Heap::s_shutdownCalled);
-    MutexLocker locker(threadAttachMutex());
-    ThreadState* state = new ThreadState();
-    attachedThreads().add(state);
+    new ThreadState(perThreadHeap);
 }
 
 void ThreadState::cleanupPages()
 {
     ASSERT(checkThread());
     for (int i = 0; i < BlinkGC::NumberOfHeaps; ++i)
         m_heaps[i]->cleanupPages();
 }
 
-void ThreadState::cleanup()
+void ThreadState::cleanupCallback()

[haraken, 2016/02/12 11:28:52]

cleanupCallback => cleanupThreadHeap ?

[keishi, 2016/02/29 06:02:33]

Done.

 {
-    ASSERT(checkThread());
-    {
-        // Grab the threadAttachMutex to ensure only one thread can shutdown at
-        // a time and that no other thread can do a global GC. It also allows
-        // safe iteration of the attachedThreads set which happens as part of
-        // thread local GC asserts. We enter a safepoint while waiting for the
-        // lock to avoid a dead-lock where another thread has already requested
-        // GC.
-        SafePointAwareMutexLocker locker(threadAttachMutex(), BlinkGC::NoHeapPointersOnStack);
+    if (isMainThread()) {

[haraken, 2016/02/12 11:28:52]

This should be:

  if (isMainThread() || m_perThreadHeapEnabled)

If this is a per-thread heap, you don't need to run a termination GC.

[keishi, 2016/02/29 06:02:33]

Done.

+        cleanupPages();
+        return;
+    }
 
-        // Finish sweeping.
-        completeSweep();
+    // Finish sweeping.
+    completeSweep();
 
-        // From here on ignore all conservatively discovered
-        // pointers into the heap owned by this thread.
-        m_isTerminating = true;
+    // From here on ignore all conservatively discovered
+    // pointers into the heap owned by this thread.
+    m_isTerminating = true;
 
-        // Set the terminate flag on all heap pages of this thread. This is used to
-        // ensure we don't trace pages on other threads that are not part of the
-        // thread local GC.
-        prepareForThreadStateTermination();
+    // Set the terminate flag on all heap pages of this thread. This is used to
+    // ensure we don't trace pages on other threads that are not part of the
+    // thread local GC.
+    prepareForThreadStateTermination();
 
-        Heap::crossThreadPersistentRegion().prepareForThreadStateTermination(this);
+    Heap::crossThreadPersistentRegion().prepareForThreadStateTermination(this);
 
-        // Do thread local GC's as long as the count of thread local Persistents
-        // changes and is above zero.
-        int oldCount = -1;
-        int currentCount = persistentRegion()->numberOfPersistents();
-        ASSERT(currentCount >= 0);
-        while (currentCount != oldCount) {
-            Heap::collectGarbageForTerminatingThread(this);
-            oldCount = currentCount;
-            currentCount = persistentRegion()->numberOfPersistents();
-        }
-        // We should not have any persistents left when getting to this point,
-        // if we have it is probably a bug so adding a debug ASSERT to catch this.
-        ASSERT(!currentCount);
-        // All of pre-finalizers should be consumed.
-        ASSERT(m_orderedPreFinalizers.isEmpty());
-        RELEASE_ASSERT(gcState() == NoGCScheduled);
+    // Do thread local GC's as long as the count of thread local Persistents
+    // changes and is above zero.
+    int oldCount = -1;
+    int currentCount = persistentRegion()->numberOfPersistents();
+    ASSERT(currentCount >= 0);
+    while (currentCount != oldCount) {
+        Heap::collectGarbageForTerminatingThread(this);
+        oldCount = currentCount;
+        currentCount = persistentRegion()->numberOfPersistents();
+    }

[haraken, 2016/02/12 11:28:52]

If m_perThreadHeapEnabled is true, the termination GC won't be executed. This
means that the above code doesn't run. This means that the per-thread heap
enables us to get rid of the restriction that all persistent handles must be
gone before the thread shuts down. This is great :)

+    // We should not have any persistents left when getting to this point,
+    // if we have it is probably a bug so adding a debug ASSERT to catch this.
+    ASSERT(!currentCount);
+    // All of pre-finalizers should be consumed.
+    ASSERT(m_orderedPreFinalizers.isEmpty());
+    RELEASE_ASSERT(gcState() == NoGCScheduled);
 
-        // Add pages to the orphaned page pool to ensure any global GCs from this point
-        // on will not trace objects on this thread's heaps.
-        cleanupPages();
-
-        ASSERT(attachedThreads().contains(this));
-        attachedThreads().remove(this);
-    }
+    // Add pages to the orphaned page pool to ensure any global GCs from this point
+    // on will not trace objects on this thread's heaps.
+    cleanupPages();
 }
 
 void ThreadState::detach()
 {
     ThreadState* state = current();
-    state->cleanup();
+    GCGroup* gcGroup = state->gcGroup();
+    gcGroup->detach(state);
     RELEASE_ASSERT(state->gcState() == ThreadState::NoGCScheduled);
     delete state;
-    shutdownHeapIfNecessary();
-}
-
-void ThreadState::visitPersistentRoots(Visitor* visitor)
-{
-    TRACE_EVENT0("blink_gc", "ThreadState::visitPersistentRoots");
-    Heap::crossThreadPersistentRegion().tracePersistentNodes(visitor);
-
-    for (ThreadState* state : attachedThreads())
-        state->visitPersistents(visitor);
-}
-
-void ThreadState::visitStackRoots(Visitor* visitor)
-{
-    TRACE_EVENT0("blink_gc", "ThreadState::visitStackRoots");
-    for (ThreadState* state : attachedThreads())
-        state->visitStack(visitor);
+    gcGroup->shutdownIfNecessary();
 }
 
 NO_SANITIZE_ADDRESS
 void ThreadState::visitAsanFakeStackForPointer(Visitor* visitor, Address ptr)
 {
 #if defined(ADDRESS_SANITIZER)
     Address* start = reinterpret_cast<Address*>(m_startOfStack);
     Address* end = reinterpret_cast<Address*>(m_endOfStack);
     Address* fakeFrameStart = nullptr;
     Address* fakeFrameEnd = nullptr;
@@ skipping 134 matching lines @@
     ScriptForbiddenIfMainThreadScope scriptForbiddenScope;
 
     // Disallow allocation during weak processing.
     // It would be technically safe to allow allocations, but it is unsafe
     // to mutate an object graph in a way in which a dead object gets
     // resurrected or mutate a HashTable (because HashTable's weak processing
     // assumes that the HashTable hasn't been mutated since the latest marking).
     // Due to the complexity, we just forbid allocations.
     NoAllocationScope noAllocationScope(this);
 
-    MarkingVisitor<Visitor::WeakProcessing> weakProcessingVisitor;
+    MarkingVisitor<Visitor::WeakProcessing> weakProcessingVisitor(nullptr);
 
     // Perform thread-specific weak processing.
     while (popAndInvokeThreadLocalWeakCallback(&weakProcessingVisitor)) { }
 
     if (isMainThread()) {
         double timeForThreadLocalWeakProcessing = WTF::currentTimeMS() - startTime;
         Platform::current()->histogramCustomCounts("BlinkGC.timeForThreadLocalWeakProcessing", timeForThreadLocalWeakProcessing, 1, 10 * 1000, 50);
     }
 }
 
 size_t ThreadState::totalMemorySize()
 {
-    return Heap::allocatedObjectSize() + Heap::markedObjectSize() + WTF::Partitions::totalSizeOfCommittedPages();
+    return gcGroup()->heapStats().allocatedObjectSize() + gcGroup()->heapStats().markedObjectSize() + WTF::Partitions::totalSizeOfCommittedPages();
 }
 
 size_t ThreadState::estimatedLiveSize(size_t estimationBaseSize, size_t sizeAtLastGC)
 {
-    if (Heap::wrapperCountAtLastGC() == 0) {
+    if (gcGroup()->heapStats().wrapperCountAtLastGC() == 0) {
         // We'll reach here only before hitting the first GC.
         return 0;
     }
 
     // (estimated size) = (estimation base size) - (heap size at the last GC) / (# of persistent handles at the last GC) * (# of persistent handles collected since the last GC);
-    size_t sizeRetainedByCollectedPersistents = static_cast<size_t>(1.0 * sizeAtLastGC / Heap::wrapperCountAtLastGC() * Heap::collectedWrapperCount());
+    size_t sizeRetainedByCollectedPersistents = static_cast<size_t>(1.0 * sizeAtLastGC / gcGroup()->heapStats().wrapperCountAtLastGC() * gcGroup()->heapStats().collectedWrapperCount());
     if (estimationBaseSize < sizeRetainedByCollectedPersistents)
         return 0;
     return estimationBaseSize - sizeRetainedByCollectedPersistents;
 }
 
 double ThreadState::heapGrowingRate()
 {
-    size_t currentSize = Heap::allocatedObjectSize() + Heap::markedObjectSize();
-    size_t estimatedSize = estimatedLiveSize(Heap::markedObjectSizeAtLastCompleteSweep(), Heap::markedObjectSizeAtLastCompleteSweep());
+    size_t currentSize = gcGroup()->heapStats().allocatedObjectSize() + gcGroup()->heapStats().markedObjectSize();
+    size_t estimatedSize = estimatedLiveSize(gcGroup()->heapStats().markedObjectSizeAtLastCompleteSweep(), gcGroup()->heapStats().markedObjectSizeAtLastCompleteSweep());
 
     // If the estimatedSize is 0, we set a high growing rate to trigger a GC.
     double growingRate = estimatedSize > 0 ? 1.0 * currentSize / estimatedSize : 100;
     TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "ThreadState::heapEstimatedSizeKB", std::min(estimatedSize / 1024, static_cast<size_t>(INT_MAX)));
     TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "ThreadState::heapGrowingRate", static_cast<int>(100 * growingRate));
     return growingRate;
 }
 
 double ThreadState::partitionAllocGrowingRate()
 {
     size_t currentSize = WTF::Partitions::totalSizeOfCommittedPages();
-    size_t estimatedSize = estimatedLiveSize(currentSize, Heap::partitionAllocSizeAtLastGC());
+    size_t estimatedSize = estimatedLiveSize(currentSize, gcGroup()->heapStats().partitionAllocSizeAtLastGC());
 
     // If the estimatedSize is 0, we set a high growing rate to trigger a GC.
     double growingRate = estimatedSize > 0 ? 1.0 * currentSize / estimatedSize : 100;
     TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "ThreadState::partitionAllocEstimatedSizeKB", std::min(estimatedSize / 1024, static_cast<size_t>(INT_MAX)));
     TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "ThreadState::partitionAllocGrowingRate", static_cast<int>(100 * growingRate));
     return growingRate;
 }
 
 // TODO(haraken): We should improve the GC heuristics. The heuristics affect
 // performance significantly.
 bool ThreadState::judgeGCThreshold(size_t totalMemorySizeThreshold, double heapGrowingRateThreshold)
 {
     // If the allocated object size or the total memory size is small, don't trigger a GC.
-    if (Heap::allocatedObjectSize() < 100 * 1024 || totalMemorySize() < totalMemorySizeThreshold)
+    if (gcGroup()->heapStats().allocatedObjectSize() < 100 * 1024 || totalMemorySize() < totalMemorySizeThreshold)
         return false;
     // If the growing rate of Oilpan's heap or PartitionAlloc is high enough,
     // trigger a GC.
 #if PRINT_HEAP_STATS
     dataLogF("heapGrowingRate=%.1lf, partitionAllocGrowingRate=%.1lf\n", heapGrowingRate(), partitionAllocGrowingRate());
 #endif
     return heapGrowingRate() >= heapGrowingRateThreshold || partitionAllocGrowingRate() >= heapGrowingRateThreshold;
 }
 
 bool ThreadState::shouldScheduleIdleGC()
@@ skipping 187 matching lines @@
 void ThreadState::performIdleGC(double deadlineSeconds)
 {
     ASSERT(checkThread());
     ASSERT(isMainThread());
     ASSERT(Platform::current()->currentThread()->scheduler());
 
     if (gcState() != IdleGCScheduled)
         return;
 
     double idleDeltaInSeconds = deadlineSeconds - Platform::current()->monotonicallyIncreasingTimeSeconds();
-    TRACE_EVENT2("blink_gc", "ThreadState::performIdleGC", "idleDeltaInSeconds", idleDeltaInSeconds, "estimatedMarkingTime", Heap::estimatedMarkingTime());
-    if (idleDeltaInSeconds <= Heap::estimatedMarkingTime() && !Platform::current()->currentThread()->scheduler()->canExceedIdleDeadlineIfRequired()) {
+    TRACE_EVENT2("blink_gc", "ThreadState::performIdleGC", "idleDeltaInSeconds", idleDeltaInSeconds, "estimatedMarkingTime", ThreadState::current()->gcGroup()->heapStats().estimatedMarkingTime());
+    if (idleDeltaInSeconds <= ThreadState::current()->gcGroup()->heapStats().estimatedMarkingTime() && !Platform::current()->currentThread()->scheduler()->canExceedIdleDeadlineIfRequired()) {
         // If marking is estimated to take longer than the deadline and we can't
         // exceed the deadline, then reschedule for the next idle period.
         scheduleIdleGC();
         return;
     }
 
     Heap::collectGarbage(BlinkGC::NoHeapPointersOnStack, BlinkGC::GCWithoutSweep, BlinkGC::IdleGC);
 }
 
 void ThreadState::performIdleLazySweep(double deadlineSeconds)
@@ skipping 181 matching lines @@
         // Idle time GC will be scheduled by Blink Scheduler.
         break;
     default:
         break;
     }
 }
 
 void ThreadState::flushHeapDoesNotContainCacheIfNeeded()
 {
     if (m_shouldFlushHeapDoesNotContainCache) {
-        Heap::flushHeapDoesNotContainCache();
+        gcGroup()->flushHeapDoesNotContainCache();
         m_shouldFlushHeapDoesNotContainCache = false;
     }
 }
 
 void ThreadState::makeConsistentForGC()
 {
     ASSERT(isInGC());
     TRACE_EVENT0("blink_gc", "ThreadState::makeConsistentForGC");
     for (int i = 0; i < BlinkGC::NumberOfHeaps; ++i)
         m_heaps[i]->makeConsistentForGC();
@@ skipping 152 matching lines @@
     postSweep();
 }
 
 void ThreadState::postSweep()
 {
     ASSERT(checkThread());
     Heap::reportMemoryUsageForTracing();
 
     if (isMainThread()) {
         double collectionRate = 0;
-        if (Heap::objectSizeAtLastGC() > 0)
-            collectionRate = 1 - 1.0 * Heap::markedObjectSize() / Heap::objectSizeAtLastGC();
+        if (gcGroup()->heapStats().objectSizeAtLastGC() > 0)
+            collectionRate = 1 - 1.0 * gcGroup()->heapStats().markedObjectSize() / gcGroup()->heapStats().objectSizeAtLastGC();
         TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "ThreadState::collectionRate", static_cast<int>(100 * collectionRate));
 
 #if PRINT_HEAP_STATS
         dataLogF("ThreadState::postSweep (collectionRate=%d%%)\n", static_cast<int>(100 * collectionRate));
 #endif
 
         // Heap::markedObjectSize() may be underestimated here if any other
         // thread has not yet finished lazy sweeping.
-        Heap::setMarkedObjectSizeAtLastCompleteSweep(Heap::markedObjectSize());
+        gcGroup()->heapStats().setMarkedObjectSizeAtLastCompleteSweep(gcGroup()->heapStats().markedObjectSize());
 
-        Platform::current()->histogramCustomCounts("BlinkGC.ObjectSizeBeforeGC", Heap::objectSizeAtLastGC() / 1024, 1, 4 * 1024 * 1024, 50);
-        Platform::current()->histogramCustomCounts("BlinkGC.ObjectSizeAfterGC", Heap::markedObjectSize() / 1024, 1, 4 * 1024 * 1024, 50);
+        Platform::current()->histogramCustomCounts("BlinkGC.ObjectSizeBeforeGC", gcGroup()->heapStats().objectSizeAtLastGC() / 1024, 1, 4 * 1024 * 1024, 50);
+        Platform::current()->histogramCustomCounts("BlinkGC.ObjectSizeAfterGC", gcGroup()->heapStats().markedObjectSize() / 1024, 1, 4 * 1024 * 1024, 50);
         Platform::current()->histogramCustomCounts("BlinkGC.CollectionRate", static_cast<int>(100 * collectionRate), 1, 100, 20);
         Platform::current()->histogramCustomCounts("BlinkGC.TimeForSweepingAllObjects", m_accumulatedSweepingTime, 1, 10 * 1000, 50);
     }
 
     switch (gcState()) {
     case Sweeping:
         setGCState(NoGCScheduled);
         break;
     case SweepingAndPreciseGCScheduled:
         setGCState(PreciseGCScheduled);
@@ skipping 26 matching lines @@
 #endif
 
 size_t ThreadState::objectPayloadSizeForTesting()
 {
     size_t objectPayloadSize = 0;
     for (int i = 0; i < BlinkGC::NumberOfHeaps; ++i)
         objectPayloadSize += m_heaps[i]->objectPayloadSizeForTesting();
     return objectPayloadSize;
 }
 
-bool ThreadState::stopThreads()
-{
-    return s_safePointBarrier->parkOthers();
-}
-
-void ThreadState::resumeThreads()
-{
-    s_safePointBarrier->resumeOthers();
-}
-
 void ThreadState::safePoint(BlinkGC::StackState stackState)
 {
     ASSERT(checkThread());
     Heap::reportMemoryUsageForTracing();
 
     runScheduledGC(stackState);
     ASSERT(!m_atSafePoint);
     m_stackState = stackState;
     m_atSafePoint = true;
-    s_safePointBarrier->checkAndPark(this);
+    m_gcGroup->checkAndPark(this, nullptr);
     m_atSafePoint = false;
     m_stackState = BlinkGC::HeapPointersOnStack;
     preSweep();
 }
 
 #ifdef ADDRESS_SANITIZER
 // When we are running under AddressSanitizer with detect_stack_use_after_return=1
 // then stack marker obtained from SafePointScope will point into a fake stack.
 // Detect this case by checking if it falls in between current stack frame
 // and stack start and use an arbitrary high enough value for it.
@@ skipping 23 matching lines @@
 #ifdef ADDRESS_SANITIZER
     if (stackState == BlinkGC::HeapPointersOnStack)
         scopeMarker = adjustScopeMarkerForAdressSanitizer(scopeMarker);
 #endif
     ASSERT(stackState == BlinkGC::NoHeapPointersOnStack || scopeMarker);
     runScheduledGC(stackState);
     ASSERT(!m_atSafePoint);
     m_atSafePoint = true;
     m_stackState = stackState;
     m_safePointScopeMarker = scopeMarker;
-    s_safePointBarrier->enterSafePoint(this);
+    m_gcGroup->enterSafePoint(this);
 }
 
 void ThreadState::leaveSafePoint(SafePointAwareMutexLocker* locker)
 {
     ASSERT(checkThread());
     ASSERT(m_atSafePoint);
-    s_safePointBarrier->leaveSafePoint(this, locker);
+    m_gcGroup->leaveSafePoint(this, locker);
     m_atSafePoint = false;
     m_stackState = BlinkGC::HeapPointersOnStack;
     clearSafePointScopeMarker();
     preSweep();
 }
 
 void ThreadState::copyStackUntilSafePointScope()
 {
     if (!m_safePointScopeMarker || m_stackState == BlinkGC::NoHeapPointersOnStack)
         return;
@@ skipping 17 matching lines @@
     for (size_t i = 0; i < slotCount; ++i) {
         m_safePointStackCopy[i] = from[i];
     }
 }
 
 void ThreadState::addInterruptor(PassOwnPtr<BlinkGCInterruptor> interruptor)
 {
     ASSERT(checkThread());
     SafePointScope scope(BlinkGC::HeapPointersOnStack);
     {
-        MutexLocker locker(threadAttachMutex());
+        MutexLocker locker(gcGroup()->threadAttachMutex());
         m_interruptors.append(interruptor);
     }
 }
 
 void ThreadState::removeInterruptor(BlinkGCInterruptor* interruptor)
 {
     ASSERT(checkThread());
     SafePointScope scope(BlinkGC::HeapPointersOnStack);
     {
-        MutexLocker locker(threadAttachMutex());
+        MutexLocker locker(gcGroup()->threadAttachMutex());
         size_t index = m_interruptors.find(interruptor);
         RELEASE_ASSERT(index != kNotFound);
         m_interruptors.remove(index);
     }
 }
 
 #if defined(LEAK_SANITIZER)
 void ThreadState::registerStaticPersistentNode(PersistentNode* node)
 {
     if (m_disabledStaticPersistentsRegistration)
@@ skipping 22 matching lines @@
     m_disabledStaticPersistentsRegistration--;
 }
 #endif
 
 ThreadState::AttachedThreadStateSet& ThreadState::attachedThreads()
 {
     DEFINE_STATIC_LOCAL(AttachedThreadStateSet, threads, ());
     return threads;
 }
 
-void ThreadState::lockThreadAttachMutex()
-{
-    threadAttachMutex().lock();
-}
-
-void ThreadState::unlockThreadAttachMutex()
-{
-    threadAttachMutex().unlock();
-}
-
 void ThreadState::invokePreFinalizers()
 {
     ASSERT(checkThread());
     ASSERT(!sweepForbidden());
     TRACE_EVENT0("blink_gc", "ThreadState::invokePreFinalizers");
 
     double startTime = WTF::currentTimeMS();
     if (!m_orderedPreFinalizers.isEmpty()) {
         SweepForbiddenScope sweepForbidden(this);
         ScriptForbiddenIfMainThreadScope scriptForbidden;
@@ skipping 141 matching lines @@
     threadDump->addScalar("live_count", "objects", totalLiveCount);
     threadDump->addScalar("dead_count", "objects", totalDeadCount);
     threadDump->addScalar("live_size", "bytes", totalLiveSize);
     threadDump->addScalar("dead_size", "bytes", totalDeadSize);
 
     WebMemoryAllocatorDump* heapsDump = BlinkGCMemoryDumpProvider::instance()->createMemoryAllocatorDumpForCurrentGC(heapsDumpName);
     WebMemoryAllocatorDump* classesDump = BlinkGCMemoryDumpProvider::instance()->createMemoryAllocatorDumpForCurrentGC(classesDumpName);
     BlinkGCMemoryDumpProvider::instance()->currentProcessMemoryDump()->addOwnershipEdge(classesDump->guid(), heapsDump->guid());
 }
 
+ThreadState* ThreadState::forObject(const void* object)

[haraken, 2016/02/12 11:28:52]

forObject => fromObject (for consistency with fromAddress etc)

[keishi, 2016/02/29 06:02:34]

Done.

+{
+    if (!object)
+        return nullptr;
+    BasePage* page = pageFromObject(object);
+    ASSERT(page);
+    ASSERT(page->heap());
+    return page->heap()->threadState();
+}
+
 } // namespace blink