Refactor the Heap into ThreadHeap to prepare for per thread heaps

third_party/WebKit/Source/platform/heap/Heap.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 50 matching lines @@
 
     ~GCForbiddenScope()
     {
         m_state->leaveGCForbiddenScope();
     }
 
 private:
     ThreadState* m_state;
 };
 
-class GCScope final {
+GCData::GCData(ThreadState* state, BlinkGC::StackState stackState, BlinkGC::GCType gcType)
+    : m_state(state)
+    , m_markingStack(adoptPtr(new CallbackStack()))
+    , m_postMarkingCallbackStack(adoptPtr(new CallbackStack()))
+    , m_globalWeakCallbackStack(adoptPtr(new CallbackStack()))
+    , m_ephemeronStack(adoptPtr(new CallbackStack()))
+{
+    ASSERT(m_state->checkThread());
+    switch (gcType) {
+    case BlinkGC::GCWithSweep:
+    case BlinkGC::GCWithoutSweep:
+        m_visitor = adoptPtr(new MarkingVisitor<Visitor::GlobalMarking>(this));
+        break;
+    case BlinkGC::TakeSnapshot:
+        m_visitor = adoptPtr(new MarkingVisitor<Visitor::SnapshotMarking>(this));
+        break;
+    case BlinkGC::ThreadTerminationGC:
+        m_visitor = adoptPtr(new MarkingVisitor<Visitor::ThreadLocalMarking>(this));
+        break;
+    default:
+        ASSERT_NOT_REACHED();
+    }
+}
+
+bool GCData::parkAllThreads(BlinkGC::StackState stackState, BlinkGC::GCType gcType)
+{
+    TRACE_EVENT0("blink_gc", "Heap::GCScope");
+    const char* samplingState = TRACE_EVENT_GET_SAMPLING_STATE();
+    if (m_state->isMainThread())
+        TRACE_EVENT_SET_SAMPLING_STATE("blink_gc", "BlinkGCWaiting");
+
+    // TODO(haraken): In an unlikely coincidence that two threads decide
+    // to collect garbage at the same time, avoid doing two GCs in
+    // a row and return false.
+    double startTime = WTF::currentTimeMS();
+    bool allParked = gcType != BlinkGC::ThreadTerminationGC && m_state->gcGroup()->park();
+    double timeForStoppingThreads = WTF::currentTimeMS() - startTime;
+    Platform::current()->histogramCustomCounts("BlinkGC.TimeForStoppingThreads", timeForStoppingThreads, 1, 1000, 50);
+
+    if (m_state->isMainThread())
+        TRACE_EVENT_SET_NONCONST_SAMPLING_STATE(samplingState);
+
+    return allParked;
+}
+
+class GCScope final : public GCData {

[haraken, 2016/01/28 15:52:49]

Is there any reason you want to distinguish GCScope from GCData? i.e., would it
make more sense to unify the two classes?

 public:
     GCScope(ThreadState* state, BlinkGC::StackState stackState, BlinkGC::GCType gcType)
-        : m_state(state)
+        : GCData(state, stackState, gcType)
         , m_gcForbiddenScope(state)
     {
-        ASSERT(m_state->checkThread());
-
-        switch (gcType) {
-        case BlinkGC::GCWithSweep:
-        case BlinkGC::GCWithoutSweep:
-            m_visitor = adoptPtr(new MarkingVisitor<Visitor::GlobalMarking>());
-            break;
-        case BlinkGC::TakeSnapshot:
-            m_visitor = adoptPtr(new MarkingVisitor<Visitor::SnapshotMarking>());
-            break;
-        case BlinkGC::ThreadTerminationGC:
-            m_visitor = adoptPtr(new MarkingVisitor<Visitor::ThreadLocalMarking>());
-            break;
-        default:
-            ASSERT_NOT_REACHED();
-        }
-    }
-
-    ~GCScope()
-    {
     }
 
-    bool parkAllThreads(BlinkGC::StackState stackState, BlinkGC::GCType gcType)
-    {
-        TRACE_EVENT0("blink_gc", "Heap::GCScope");
-        const char* samplingState = TRACE_EVENT_GET_SAMPLING_STATE();
-        if (m_state->isMainThread())
-            TRACE_EVENT_SET_SAMPLING_STATE("blink_gc", "BlinkGCWaiting");
-
-        // TODO(haraken): In an unlikely coincidence that two threads decide
-        // to collect garbage at the same time, avoid doing two GCs in
-        // a row and return false.
-        double startTime = WTF::currentTimeMS();
-        bool allParked = gcType != BlinkGC::ThreadTerminationGC && ThreadState::stopThreads();
-        double timeForStoppingThreads = WTF::currentTimeMS() - startTime;
-        Platform::current()->histogramCustomCounts("BlinkGC.TimeForStoppingThreads", timeForStoppingThreads, 1, 1000, 50);
-
-        if (m_state->isMainThread())
-            TRACE_EVENT_SET_NONCONST_SAMPLING_STATE(samplingState);
-
-        return allParked;
-    }
-
-    Visitor* visitor() const { return m_visitor.get(); }
-
 private:
-    ThreadState* m_state;
     // See ThreadState::runScheduledGC() why we need to already be in a
     // GCForbiddenScope before any safe point is entered.
     GCForbiddenScope m_gcForbiddenScope;
-    OwnPtr<Visitor> m_visitor;
 };
 
 class ResumeThreadScope {
 public:
-    explicit ResumeThreadScope(BlinkGC::GCType gcType)
+    explicit ResumeThreadScope(BlinkGC::GCType gcType, GCGroup* gcGroup)

[haraken, 2016/01/28 15:52:49]

Remove explicit.

         : m_resumeThreads(gcType != BlinkGC::ThreadTerminationGC)
+        , m_gcGroup(gcGroup)
     {
     }
     ~ResumeThreadScope()
     {
         // Only cleanup if we parked all threads in which case the GC happened
         // and we need to resume the other threads.
         if (m_resumeThreads)
-            ThreadState::resumeThreads();
+            m_gcGroup->resume();
     }
 private:
     bool m_resumeThreads;
+    GCGroup* m_gcGroup;
 };
 
-void Heap::flushHeapDoesNotContainCache()
-{
-    s_heapDoesNotContainCache->flush();
-}
-
 void Heap::init()
 {
     ThreadState::init();
-    s_markingStack = new CallbackStack();
-    s_postMarkingCallbackStack = new CallbackStack();
-    s_globalWeakCallbackStack = new CallbackStack();
-    s_ephemeronStack = new CallbackStack();
-    s_heapDoesNotContainCache = new HeapDoesNotContainCache();
-    s_freePagePool = new FreePagePool();
-    s_orphanedPagePool = new OrphanedPagePool();
-    s_allocatedSpace = 0;
-    s_allocatedObjectSize = 0;
-    s_objectSizeAtLastGC = 0;
-    s_markedObjectSize = 0;
-    s_markedObjectSizeAtLastCompleteSweep = 0;
-    s_wrapperCount = 0;
-    s_wrapperCountAtLastGC = 0;
-    s_collectedWrapperCount = 0;
-    s_partitionAllocSizeAtLastGC = WTF::Partitions::totalSizeOfCommittedPages();
-    s_estimatedMarkingTimePerByte = 0.0;
-#if ENABLE(ASSERT)
-    s_gcGeneration = 1;
-#endif
 
     GCInfoTable::init();
 
     if (Platform::current() && Platform::current()->currentThread())
         Platform::current()->registerMemoryDumpProvider(BlinkGCMemoryDumpProvider::instance(), "BlinkGC");
 }
 
 void Heap::shutdown()
 {
     if (Platform::current() && Platform::current()->currentThread())
         Platform::current()->unregisterMemoryDumpProvider(BlinkGCMemoryDumpProvider::instance());
     s_shutdownCalled = true;
-    ThreadState::shutdownHeapIfNecessary();
+    if (ThreadState::current() && ThreadState::current()->gcGroup())
+        ThreadState::current()->gcGroup()->shutdownIfNecessary();

[haraken, 2016/01/28 15:52:49]

I guess this is doing something wrong. Heap::shutdown() is a function to shut
down the entire system. It must be called after all GCGroups are shut down.

[keishi, 2016/02/29 06:02:32]

I had some mistakes in the new CL but
Heap::shutdown calls GCGroup::shutdownIfNecessary which calls GCGroup::shutdown
when all threads are detached. MultipleThreadGCGroup::shutdown will call
Heap::doShutdown because MultipleThreadGCGroup is always the main thread.
SingleThreadGCGroup which will be added later will not call Heap::doShutdown.

 }
 
 void Heap::doShutdown()
 {
     // We don't want to call doShutdown() twice.
-    if (!s_markingStack)
+    if (!s_doShutdownDone)
         return;
+    s_doShutdownDone = true;
 
-    ASSERT(!ThreadState::attachedThreads().size());
-    delete s_heapDoesNotContainCache;
-    s_heapDoesNotContainCache = nullptr;
-    delete s_freePagePool;
-    s_freePagePool = nullptr;
-    delete s_orphanedPagePool;
-    s_orphanedPagePool = nullptr;
-    delete s_globalWeakCallbackStack;
-    s_globalWeakCallbackStack = nullptr;
-    delete s_postMarkingCallbackStack;
-    s_postMarkingCallbackStack = nullptr;
-    delete s_markingStack;
-    s_markingStack = nullptr;
-    delete s_ephemeronStack;
-    s_ephemeronStack = nullptr;
-    delete s_regionTree;
-    s_regionTree = nullptr;
+    // ASSERT(!ThreadState::attachedThreads().size());
     GCInfoTable::shutdown();
     ThreadState::shutdown();
-    ASSERT(Heap::allocatedSpace() == 0);
+    // ASSERT(Heap::allocatedSpace() == 0);

[haraken, 2016/01/28 15:52:49]

We want to keep this ASSERT somehow. This ASSERT is important to make sure that
underlying memory doesn't leak.

[keishi, 2016/02/29 06:02:32]

Done.

 }
 
 CrossThreadPersistentRegion& Heap::crossThreadPersistentRegion()
 {
     DEFINE_THREAD_SAFE_STATIC_LOCAL(CrossThreadPersistentRegion, persistentRegion, new CrossThreadPersistentRegion());
     return persistentRegion;
 }
 
 #if ENABLE(ASSERT)
 BasePage* Heap::findPageFromAddress(Address address)

[haraken, 2016/01/28 15:52:49]

This method should be moved to GCGroup. I think what we want to do here is to
find the given address from the given GCGroup, not from all threads in the heap.

[keishi, 2016/02/29 06:02:32]

Removed

 {
-    MutexLocker lock(ThreadState::threadAttachMutex());
-    for (ThreadState* state : ThreadState::attachedThreads()) {
-        if (BasePage* page = state->findPageFromAddress(address))
+    for (GCGroup* gcGroup : GCGroup::all()) {

[haraken, 2016/01/28 15:52:49]

In general, it is wrong to replace:

  for (ThreadState* state : ThreadState::attachedThreads())

with:

  for (GCGroup* gcGroup : GCGroup::all())

It must be replaced with:

  for (ThreadState* state : gcGroup->threads())

+        if (BasePage* page = gcGroup->findPageFromAddress(address))
             return page;
     }
     return nullptr;
 }
 #endif
 
 Address Heap::checkAndMarkPointer(Visitor* visitor, Address address)
 {
     ASSERT(ThreadState::current()->isInGC());
 
 #if !ENABLE(ASSERT)
-    if (s_heapDoesNotContainCache->lookup(address))
+    if (visitor->gcData()->threadState()->gcGroup()->heapDoesNotContainCache()->lookup(address))

[haraken, 2016/01/28 15:52:49]

It's nasty that we have to write visitor->gcData()->threadState()->gcGroup()->
every time but is okay for now. (Ideally I want to unify Visitor and GCData into
one class.)

[keishi, 2016/02/29 06:02:32]

Done.

         return nullptr;
 #endif
 
-    if (BasePage* page = lookup(address)) {
+    // TODO: use Visitor::threadState
+    if (BasePage* page = visitor->gcData()->threadState()->gcGroup()->lookupPageForAddress(address)) {
         ASSERT(page->contains(address));
         ASSERT(!page->orphaned());
-        ASSERT(!s_heapDoesNotContainCache->lookup(address));
+        ASSERT(!visitor->gcData()->threadState()->gcGroup()->heapDoesNotContainCache()->lookup(address));
         page->checkAndMarkPointer(visitor, address);
         return address;
     }
 
 #if !ENABLE(ASSERT)
-    s_heapDoesNotContainCache->addEntry(address);
+    visitor->gcData()->threadState()->gcGroup()->heapDoesNotContainCache()->addEntry(address);
 #else
-    if (!s_heapDoesNotContainCache->lookup(address))
-        s_heapDoesNotContainCache->addEntry(address);
+    if (!visitor->gcData()->threadState()->gcGroup()->heapDoesNotContainCache()->lookup(address))
+        visitor->gcData()->threadState()->gcGroup()->heapDoesNotContainCache()->addEntry(address);

[haraken, 2016/01/28 15:52:49]

Cache the gcGroup into a local variable and avoid the code duplication.

 #endif
     return nullptr;
 }
 
-void Heap::pushTraceCallback(void* object, TraceCallback callback)
+void Heap::pushTraceCallback(void* object, TraceCallback callback, GCData* gcData)
 {
     ASSERT(ThreadState::current()->isInGC());
 
     // Trace should never reach an orphaned page.
-    ASSERT(!Heap::orphanedPagePool()->contains(object));
-    CallbackStack::Item* slot = s_markingStack->allocateEntry();
+    ASSERT(!gcData->threadState()->gcGroup()->orphanedPagePool()->contains(object));
+    CallbackStack::Item* slot = gcData->markingStack()->allocateEntry();
     *slot = CallbackStack::Item(object, callback);
 }
 
 bool Heap::popAndInvokeTraceCallback(Visitor* visitor)

[haraken, 2016/01/28 15:52:49]

It's inconsistent that you sometimes pass Visitor but sometimes GCData. Shall we
unify them into Visitor? (In long term, I want to unify Visitor and GCData into
one class.)

 {
-    CallbackStack::Item* item = s_markingStack->pop();
+    CallbackStack::Item* item = visitor->gcData()->markingStack()->pop();
     if (!item)
         return false;
     item->call(visitor);
     return true;
 }
 
-void Heap::pushPostMarkingCallback(void* object, TraceCallback callback)
+void Heap::pushPostMarkingCallback(void* object, TraceCallback callback, GCData* gcData)
 {
     ASSERT(ThreadState::current()->isInGC());
 
     // Trace should never reach an orphaned page.
-    ASSERT(!Heap::orphanedPagePool()->contains(object));
-    CallbackStack::Item* slot = s_postMarkingCallbackStack->allocateEntry();
+    ASSERT(!gcData->threadState()->gcGroup()->orphanedPagePool()->contains(object));
+    CallbackStack::Item* slot = gcData->postMarkingCallbackStack()->allocateEntry();
     *slot = CallbackStack::Item(object, callback);
 }
 
 bool Heap::popAndInvokePostMarkingCallback(Visitor* visitor)
 {
-    if (CallbackStack::Item* item = s_postMarkingCallbackStack->pop()) {
+    if (CallbackStack::Item* item = visitor->gcData()->postMarkingCallbackStack()->pop()) {
         item->call(visitor);
         return true;
     }
     return false;
 }
 
-void Heap::pushGlobalWeakCallback(void** cell, WeakCallback callback)
+void Heap::pushGlobalWeakCallback(void** cell, WeakCallback callback, GCData* gcData)
 {
     ASSERT(ThreadState::current()->isInGC());
 
     // Trace should never reach an orphaned page.
-    ASSERT(!Heap::orphanedPagePool()->contains(cell));
-    CallbackStack::Item* slot = s_globalWeakCallbackStack->allocateEntry();
+    ASSERT(!gcData->threadState()->gcGroup()->orphanedPagePool()->contains(cell));
+    CallbackStack::Item* slot = gcData->globalWeakCallbackStack()->allocateEntry();
     *slot = CallbackStack::Item(cell, callback);
 }
 
 void Heap::pushThreadLocalWeakCallback(void* closure, void* object, WeakCallback callback)

[haraken, 2016/01/28 15:52:49]

We should pass in Visitor.

[keishi, 2016/02/29 06:02:32]

Why? I don't think we can't use Visitor for anything

 {
     ASSERT(ThreadState::current()->isInGC());
 
+    ThreadState* state = pageFromObject(object)->heap()->threadState();
     // Trace should never reach an orphaned page.
-    ASSERT(!Heap::orphanedPagePool()->contains(object));
-    ThreadState* state = pageFromObject(object)->heap()->threadState();
+    ASSERT(!state->gcGroup()->orphanedPagePool()->contains(object));
     state->pushThreadLocalWeakCallback(closure, callback);
 }
 
 bool Heap::popAndInvokeGlobalWeakCallback(Visitor* visitor)
 {
-    if (CallbackStack::Item* item = s_globalWeakCallbackStack->pop()) {
+    if (CallbackStack::Item* item = visitor->gcData()->globalWeakCallbackStack()->pop()) {
         item->call(visitor);
         return true;
     }
     return false;
 }
 
-void Heap::registerWeakTable(void* table, EphemeronCallback iterationCallback, EphemeronCallback iterationDoneCallback)
+void Heap::registerWeakTable(void* table, EphemeronCallback iterationCallback, EphemeronCallback iterationDoneCallback, GCData* gcData)
 {
     ASSERT(ThreadState::current()->isInGC());
 
     // Trace should never reach an orphaned page.
-    ASSERT(!Heap::orphanedPagePool()->contains(table));
-    CallbackStack::Item* slot = s_ephemeronStack->allocateEntry();
+    ASSERT(!gcData->threadState()->gcGroup()->orphanedPagePool()->contains(table));
+    CallbackStack::Item* slot = gcData->ephemeronStack()->allocateEntry();
     *slot = CallbackStack::Item(table, iterationCallback);
 
     // Register a post-marking callback to tell the tables that
     // ephemeron iteration is complete.
-    pushPostMarkingCallback(table, iterationDoneCallback);
+    pushPostMarkingCallback(table, iterationDoneCallback, gcData);
 }
 
 #if ENABLE(ASSERT)
-bool Heap::weakTableRegistered(const void* table)
+bool Heap::weakTableRegistered(const void* table, GCData* gcData)
 {
-    ASSERT(s_ephemeronStack);
-    return s_ephemeronStack->hasCallbackForObject(table);
+    ASSERT(gcData->ephemeronStack());
+    return gcData->ephemeronStack()->hasCallbackForObject(table);
 }
 #endif
 
 void Heap::preGC()
 {

[haraken, 2016/01/28 15:52:49]

Is this helper function still useful?

[keishi, 2016/02/29 06:02:32]

Remvoed

     ASSERT(!ThreadState::current()->isInGC());
-    for (ThreadState* state : ThreadState::attachedThreads())
-        state->preGC();
+    ThreadState::current()->gcGroup()->preGC();
 }
 
 void Heap::postGC(BlinkGC::GCType gcType)

[haraken, 2016/01/28 15:52:49]

Ditto.

[keishi, 2016/02/29 06:02:32]

Removed

 {
     ASSERT(ThreadState::current()->isInGC());
-    for (ThreadState* state : ThreadState::attachedThreads())
-        state->postGC(gcType);
+    ThreadState::current()->gcGroup()->postGC(gcType);
 }
 
 const char* Heap::gcReasonString(BlinkGC::GCReason reason)
 {
     switch (reason) {
     case BlinkGC::IdleGC:
         return "IdleGC";
     case BlinkGC::PreciseGC:
         return "PreciseGC";
     case BlinkGC::ConservativeGC:
@@ skipping 20 matching lines @@
     GCScope gcScope(state, stackState, gcType);
     // See collectGarbageForTerminatingThread() comment on why a
     // safepoint scope isn't entered for it.
     SafePointScope safePointScope(stackState, gcType != BlinkGC::ThreadTerminationGC ? state : nullptr);
 
     // Try to park the other threads. If we're unable to, bail out of the GC.
     if (!gcScope.parkAllThreads(stackState, gcType))
         return;
 
     // Resume all parked threads upon leaving this scope.
-    ResumeThreadScope resumeThreads(gcType);
+    ResumeThreadScope resumeThreads(gcType, state->gcGroup());
     ScriptForbiddenIfMainThreadScope scriptForbidden;
 
     TRACE_EVENT2("blink_gc", "Heap::collectGarbage",
         "lazySweeping", gcType == BlinkGC::GCWithoutSweep,
         "gcReason", gcReasonString(reason));
     TRACE_EVENT_SCOPED_SAMPLING_STATE("blink_gc", "BlinkGC");
     double startTime = WTF::currentTimeMS();
 
     if (gcType == BlinkGC::TakeSnapshot)
         BlinkGCMemoryDumpProvider::instance()->clearProcessDumpForCurrentGC();
 
     // Disallow allocation during garbage collection (but not during the
     // finalization that happens when the gcScope is torn down).
     ThreadState::NoAllocationScope noAllocationScope(state);
 
     preGC();
 
     StackFrameDepthScope stackDepthScope;
 
-    size_t totalObjectSize = Heap::allocatedObjectSize() + Heap::markedObjectSize();
-    if (gcType != BlinkGC::TakeSnapshot)
-        Heap::resetHeapCounters();
+    size_t totalObjectSize = state->gcGroup()->heapStats().allocatedObjectSize() + ThreadState::current()->gcGroup()->heapStats().markedObjectSize();
+    if (gcType != BlinkGC::TakeSnapshot) {
+        Heap::reportMemoryUsageForTracing();
+        state->gcGroup()->heapStats().reset();
+    }
 
     // 1. Trace persistent roots.
-    ThreadState::visitPersistentRoots(gcScope.visitor());
+    state->gcGroup()->visitPersistentRoots(gcScope.visitor());
 
     // 2. Trace objects reachable from the stack.  We do this independent of the
     // given stackState since other threads might have a different stack state.
-    ThreadState::visitStackRoots(gcScope.visitor());
+    state->gcGroup()->visitStackRoots(gcScope.visitor());
 
     // 3. Transitive closure to trace objects including ephemerons.
     processMarkingStack(gcScope.visitor());
 
     postMarkingProcessing(gcScope.visitor());
     globalWeakProcessing(gcScope.visitor());
 
     // Now we can delete all orphaned pages because there are no dangling
     // pointers to the orphaned pages.  (If we have such dangling pointers,
     // we should have crashed during marking before getting here.)
-    orphanedPagePool()->decommitOrphanedPages();
+    state->gcGroup()->orphanedPagePool()->decommitOrphanedPages();
 
     double markingTimeInMilliseconds = WTF::currentTimeMS() - startTime;
-    s_estimatedMarkingTimePerByte = totalObjectSize ? (markingTimeInMilliseconds / 1000 / totalObjectSize) : 0;
+    state->gcGroup()->heapStats().setEstimatedMarkingTimePerByte(totalObjectSize ? (markingTimeInMilliseconds / 1000 / totalObjectSize) : 0);
 
 #if PRINT_HEAP_STATS
     dataLogF("Heap::collectGarbage (gcReason=%s, lazySweeping=%d, time=%.1lfms)\n", gcReasonString(reason), gcType == BlinkGC::GCWithoutSweep, markingTimeInMilliseconds);
 #endif
 
     Platform::current()->histogramCustomCounts("BlinkGC.CollectGarbage", markingTimeInMilliseconds, 0, 10 * 1000, 50);
-    Platform::current()->histogramCustomCounts("BlinkGC.TotalObjectSpace", Heap::allocatedObjectSize() / 1024, 0, 4 * 1024 * 1024, 50);
-    Platform::current()->histogramCustomCounts("BlinkGC.TotalAllocatedSpace", Heap::allocatedSpace() / 1024, 0, 4 * 1024 * 1024, 50);
+    Platform::current()->histogramCustomCounts("BlinkGC.TotalObjectSpace", Heap::totalAllocatedObjectSize() / 1024, 0, 4 * 1024 * 1024, 50);
+    Platform::current()->histogramCustomCounts("BlinkGC.TotalAllocatedSpace", Heap::totalAllocatedSpace() / 1024, 0, 4 * 1024 * 1024, 50);
     Platform::current()->histogramEnumeration("BlinkGC.GCReason", reason, BlinkGC::NumberOfGCReason);
     Heap::reportMemoryUsageHistogram();
     WTF::Partitions::reportMemoryUsageHistogram();
 
     postGC(gcType);
-
-#if ENABLE(ASSERT)
-    // 0 is used to figure non-assigned area, so avoid to use 0 in s_gcGeneration.
-    if (++s_gcGeneration == 0) {
-        s_gcGeneration = 1;
-    }
-#endif
 }
 
 void Heap::collectGarbageForTerminatingThread(ThreadState* state)
 {
     {
         // A thread-specific termination GC must not allow other global GCs to go
         // ahead while it is running, hence the termination GC does not enter a
         // safepoint. GCScope will not enter also a safepoint scope for
         // ThreadTerminationGC.
         GCScope gcScope(state, BlinkGC::NoHeapPointersOnStack, BlinkGC::ThreadTerminationGC);
@@ skipping 34 matching lines @@
             // Iteratively mark all objects that are reachable from the objects
             // currently pushed onto the marking stack.
             TRACE_EVENT0("blink_gc", "Heap::processMarkingStackSingleThreaded");
             while (popAndInvokeTraceCallback(visitor)) { }
         }
 
         {
             // Mark any strong pointers that have now become reachable in
             // ephemeron maps.
             TRACE_EVENT0("blink_gc", "Heap::processEphemeronStack");
-            s_ephemeronStack->invokeEphemeronCallbacks(visitor);
+            visitor->gcData()->ephemeronStack()->invokeEphemeronCallbacks(visitor);
         }
 
         // Rerun loop if ephemeron processing queued more objects for tracing.
-    } while (!s_markingStack->isEmpty());
+    } while (!visitor->gcData()->markingStack()->isEmpty());
 }
 
 void Heap::postMarkingProcessing(Visitor* visitor)
 {
     TRACE_EVENT0("blink_gc", "Heap::postMarkingProcessing");
     // Call post-marking callbacks including:
     // 1. the ephemeronIterationDone callbacks on weak tables to do cleanup
     //    (specifically to clear the queued bits for weak hash tables), and
     // 2. the markNoTracing callbacks on collection backings to mark them
     //    if they are only reachable from their front objects.
     while (popAndInvokePostMarkingCallback(visitor)) { }
 
-    s_ephemeronStack->clear();
+    visitor->gcData()->ephemeronStack()->clear();
 
     // Post-marking callbacks should not trace any objects and
     // therefore the marking stack should be empty after the
     // post-marking callbacks.
-    ASSERT(s_markingStack->isEmpty());
+    ASSERT(visitor->gcData()->markingStack()->isEmpty());
 }
 
 void Heap::globalWeakProcessing(Visitor* visitor)
 {
     TRACE_EVENT0("blink_gc", "Heap::globalWeakProcessing");
     double startTime = WTF::currentTimeMS();
 
     // Call weak callbacks on objects that may now be pointing to dead objects.
     while (popAndInvokeGlobalWeakCallback(visitor)) { }
 
     // It is not permitted to trace pointers of live objects in the weak
     // callback phase, so the marking stack should still be empty here.
-    ASSERT(s_markingStack->isEmpty());
+    ASSERT(visitor->gcData()->markingStack()->isEmpty());
 
     double timeForGlobalWeakProcessing = WTF::currentTimeMS() - startTime;
     Platform::current()->histogramCustomCounts("BlinkGC.TimeForGlobalWeakPrcessing", timeForGlobalWeakProcessing, 1, 10 * 1000, 50);
 }
 
 void Heap::collectAllGarbage()
 {
     // We need to run multiple GCs to collect a chain of persistent handles.
     size_t previousLiveObjects = 0;
     for (int i = 0; i < 5; ++i) {
         collectGarbage(BlinkGC::NoHeapPointersOnStack, BlinkGC::GCWithSweep, BlinkGC::ForcedGC);
-        size_t liveObjects = Heap::markedObjectSize();
+        size_t liveObjects = ThreadState::current()->gcGroup()->heapStats().markedObjectSize();
         if (liveObjects == previousLiveObjects)
             break;
         previousLiveObjects = liveObjects;
     }
 }
 
-double Heap::estimatedMarkingTime()
-{
-    ASSERT(ThreadState::current()->isMainThread());
-
-    // Use 8 ms as initial estimated marking time.
-    // 8 ms is long enough for low-end mobile devices to mark common
-    // real-world object graphs.
-    if (s_estimatedMarkingTimePerByte == 0)
-        return 0.008;
-
-    // Assuming that the collection rate of this GC will be mostly equal to
-    // the collection rate of the last GC, estimate the marking time of this GC.
-    return s_estimatedMarkingTimePerByte * (Heap::allocatedObjectSize() + Heap::markedObjectSize());
-}
-
 void Heap::reportMemoryUsageHistogram()
 {
     static size_t supportedMaxSizeInMB = 4 * 1024;
     static size_t observedMaxSizeInMB = 0;
 
     // We only report the memory in the main thread.
     if (!isMainThread())
         return;
     // +1 is for rounding up the sizeInMB.
-    size_t sizeInMB = Heap::allocatedSpace() / 1024 / 1024 + 1;
+    size_t sizeInMB = ThreadState::current()->gcGroup()->heapStats().allocatedSpace() / 1024 / 1024 + 1;
     if (sizeInMB >= supportedMaxSizeInMB)
         sizeInMB = supportedMaxSizeInMB - 1;
     if (sizeInMB > observedMaxSizeInMB) {
         // Send a UseCounter only when we see the highest memory usage
         // we've ever seen.
         Platform::current()->histogramEnumeration("BlinkGC.CommittedSize", sizeInMB, supportedMaxSizeInMB);
         observedMaxSizeInMB = sizeInMB;
     }
 }
 
 void Heap::reportMemoryUsageForTracing()
 {
 #if PRINT_HEAP_STATS
     // dataLogF("allocatedSpace=%ldMB, allocatedObjectSize=%ldMB, markedObjectSize=%ldMB, partitionAllocSize=%ldMB, wrapperCount=%ld, collectedWrapperCount=%ld\n", Heap::allocatedSpace() / 1024 / 1024, Heap::allocatedObjectSize() / 1024 / 1024, Heap::markedObjectSize() / 1024 / 1024, WTF::Partitions::totalSizeOfCommittedPages() / 1024 / 1024, Heap::wrapperCount(), Heap::collectedWrapperCount());
 #endif
 
     bool gcTracingEnabled;
     TRACE_EVENT_CATEGORY_GROUP_ENABLED("blink_gc", &gcTracingEnabled);
     if (!gcTracingEnabled)
         return;
 
+    GCGroup* gcGroup = ThreadState::current()->gcGroup();
+
     // These values are divided by 1024 to avoid overflow in practical cases (TRACE_COUNTER values are 32-bit ints).
     // They are capped to INT_MAX just in case.
-    TRACE_COUNTER1("blink_gc", "Heap::allocatedObjectSizeKB", std::min(Heap::allocatedObjectSize() / 1024, static_cast<size_t>(INT_MAX)));
-    TRACE_COUNTER1("blink_gc", "Heap::markedObjectSizeKB", std::min(Heap::markedObjectSize() / 1024, static_cast<size_t>(INT_MAX)));
-    TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "Heap::markedObjectSizeAtLastCompleteSweepKB", std::min(Heap::markedObjectSizeAtLastCompleteSweep() / 1024, static_cast<size_t>(INT_MAX)));
-    TRACE_COUNTER1("blink_gc", "Heap::allocatedSpaceKB", std::min(Heap::allocatedSpace() / 1024, static_cast<size_t>(INT_MAX)));
-    TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "Heap::objectSizeAtLastGCKB", std::min(Heap::objectSizeAtLastGC() / 1024, static_cast<size_t>(INT_MAX)));
-    TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "Heap::wrapperCount", std::min(Heap::wrapperCount(), static_cast<size_t>(INT_MAX)));
-    TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "Heap::wrapperCountAtLastGC", std::min(Heap::wrapperCountAtLastGC(), static_cast<size_t>(INT_MAX)));
-    TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "Heap::collectedWrapperCount", std::min(Heap::collectedWrapperCount(), static_cast<size_t>(INT_MAX)));
-    TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "Heap::partitionAllocSizeAtLastGCKB", std::min(Heap::partitionAllocSizeAtLastGC() / 1024, static_cast<size_t>(INT_MAX)));
+    TRACE_COUNTER1("blink_gc", "Heap::allocatedObjectSizeKB", std::min(gcGroup->heapStats().allocatedObjectSize() / 1024, static_cast<size_t>(INT_MAX)));
+    TRACE_COUNTER1("blink_gc", "Heap::markedObjectSizeKB", std::min(gcGroup->heapStats().markedObjectSize() / 1024, static_cast<size_t>(INT_MAX)));
+    TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "Heap::markedObjectSizeAtLastCompleteSweepKB", std::min(gcGroup->heapStats().markedObjectSizeAtLastCompleteSweep() / 1024, static_cast<size_t>(INT_MAX)));
+    TRACE_COUNTER1("blink_gc", "Heap::allocatedSpaceKB", std::min(Heap::totalAllocatedSpace() / 1024, static_cast<size_t>(INT_MAX)));
+    TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "Heap::objectSizeAtLastGCKB", std::min(gcGroup->heapStats().objectSizeAtLastGC() / 1024, static_cast<size_t>(INT_MAX)));
+    TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "Heap::wrapperCount", std::min(gcGroup->heapStats().wrapperCount(), static_cast<size_t>(INT_MAX)));
+    TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "Heap::wrapperCountAtLastGC", std::min(gcGroup->heapStats().wrapperCountAtLastGC(), static_cast<size_t>(INT_MAX)));
+    TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "Heap::collectedWrapperCount", std::min(gcGroup->heapStats().collectedWrapperCount(), static_cast<size_t>(INT_MAX)));
+    TRACE_COUNTER1(TRACE_DISABLED_BY_DEFAULT("blink_gc"), "Heap::partitionAllocSizeAtLastGCKB", std::min(gcGroup->heapStats().partitionAllocSizeAtLastGC() / 1024, static_cast<size_t>(INT_MAX)));
     TRACE_COUNTER1("blink_gc", "Partitions::totalSizeOfCommittedPagesKB", std::min(WTF::Partitions::totalSizeOfCommittedPages() / 1024, static_cast<size_t>(INT_MAX)));
 }
 
 size_t Heap::objectPayloadSizeForTesting()
 {
     size_t objectPayloadSize = 0;
-    for (ThreadState* state : ThreadState::attachedThreads()) {
-        state->setGCState(ThreadState::GCRunning);
-        state->makeConsistentForGC();
-        objectPayloadSize += state->objectPayloadSizeForTesting();
-        state->setGCState(ThreadState::EagerSweepScheduled);
-        state->setGCState(ThreadState::Sweeping);
-        state->setGCState(ThreadState::NoGCScheduled);
+    for (GCGroup* gcGroup : GCGroup::all()) {

[haraken, 2016/01/28 15:52:49]

This should not iterate all GCGroups. Just return:

  gcGroup->objectPayloadSizeForTesting();

[keishi, 2016/02/29 06:02:32]

Removed and used GCGroup::objectPayloadSizeForTesting directly.

+        objectPayloadSize += gcGroup->objectPayloadSizeForTesting();
     }
     return objectPayloadSize;
 }
 
-BasePage* Heap::lookup(Address address)
-{
-    ASSERT(ThreadState::current()->isInGC());
-    if (!s_regionTree)
-        return nullptr;
-    if (PageMemoryRegion* region = s_regionTree->lookup(address)) {
-        BasePage* page = region->pageFromAddress(address);
-        return page && !page->orphaned() ? page : nullptr;
-    }
-    return nullptr;
-}
-
-static Mutex& regionTreeMutex()
-{
-    DEFINE_THREAD_SAFE_STATIC_LOCAL(Mutex, mutex, new Mutex);
-    return mutex;
-}
-
-void Heap::removePageMemoryRegion(PageMemoryRegion* region)
-{
-    // Deletion of large objects (and thus their regions) can happen
-    // concurrently on sweeper threads.  Removal can also happen during thread
-    // shutdown, but that case is safe.  Regardless, we make all removals
-    // mutually exclusive.
-    MutexLocker locker(regionTreeMutex());
-    RegionTree::remove(region, &s_regionTree);
-}
-
-void Heap::addPageMemoryRegion(PageMemoryRegion* region)
-{
-    MutexLocker locker(regionTreeMutex());
-    RegionTree::add(new RegionTree(region), &s_regionTree);
-}
-
 void Heap::resetHeapCounters()
 {
     ASSERT(ThreadState::current()->isInGC());
 
     Heap::reportMemoryUsageForTracing();
 
-    s_objectSizeAtLastGC = s_allocatedObjectSize + s_markedObjectSize;
-    s_partitionAllocSizeAtLastGC = WTF::Partitions::totalSizeOfCommittedPages();
-    s_allocatedObjectSize = 0;
-    s_markedObjectSize = 0;
-    s_wrapperCountAtLastGC = s_wrapperCount;
-    s_collectedWrapperCount = 0;
+    ThreadState::current()->gcGroup()->heapStats().reset();
 }
 
-CallbackStack* Heap::s_markingStack;
-CallbackStack* Heap::s_postMarkingCallbackStack;
-CallbackStack* Heap::s_globalWeakCallbackStack;
-CallbackStack* Heap::s_ephemeronStack;
-HeapDoesNotContainCache* Heap::s_heapDoesNotContainCache;
 bool Heap::s_shutdownCalled = false;
-FreePagePool* Heap::s_freePagePool;
-OrphanedPagePool* Heap::s_orphanedPagePool;
-RegionTree* Heap::s_regionTree = nullptr;
-size_t Heap::s_allocatedSpace = 0;
-size_t Heap::s_allocatedObjectSize = 0;
-size_t Heap::s_objectSizeAtLastGC = 0;
-size_t Heap::s_markedObjectSize = 0;
-size_t Heap::s_markedObjectSizeAtLastCompleteSweep = 0;
-size_t Heap::s_wrapperCount = 0;
-size_t Heap::s_wrapperCountAtLastGC = 0;
-size_t Heap::s_collectedWrapperCount = 0;
-size_t Heap::s_partitionAllocSizeAtLastGC = 0;
-double Heap::s_estimatedMarkingTimePerByte = 0.0;
-#if ENABLE(ASSERT)
-uint16_t Heap::s_gcGeneration = 0;
-#endif
+bool Heap::s_doShutdownDone = false;
+size_t Heap::s_totalAllocatedSpace = 0;
+size_t Heap::s_totalAllocatedObjectSize = 0;
+size_t Heap::s_totalMarkedObjectSize = 0;
 
 } // namespace blink