Revert "Revert "[oilpan]: Make thread shutdown more robust.""

Source/platform/heap/Heap.h

Index: Source/platform/heap/Heap.h
diff --git a/Source/platform/heap/Heap.h b/Source/platform/heap/Heap.h
index 81bf4ad0cf68eb27e6c72f982c2e110a0707790b..0dc7eabf77e7419765710d3c1bfb54d40679007a 100644
--- a/Source/platform/heap/Heap.h
+++ b/Source/platform/heap/Heap.h
@@ -69,10 +69,25 @@ const size_t maxHeapObjectSize = 1 << 27;
 
 const size_t markBitMask = 1;
 const size_t freeListMask = 2;
-const size_t debugBitMask = 4;
+// The dead bit is used for objects that have gone through a GC marking, but did
+// not get swept before a new GC started. In that case we set the dead bit on
+// objects that were not marked in the previous GC to ensure we are not tracing
+// them via a conservatively found pointer. Tracing dead objects could lead to
+// tracing of already finalized objects in another thread's heap which is a
+// use-after-free situation.
+const size_t deadBitMask = 4;
 const size_t sizeMask = ~7;
 const uint8_t freelistZapValue = 42;
 const uint8_t finalizedZapValue = 24;
+// The orphaned zap value must be zero in the lowest bits to allow for using
+// the mark bit when tracing.
+const uint8_t orphanedZapValue = 240;
+
+enum CallbackInvocationMode {
+    GlobalMarking,
+    ThreadLocalMarking,
+    WeaknessProcessing,
+};
 
 class HeapStats;
 class PageMemory;
@@ -127,59 +142,30 @@ inline bool isPageHeaderAddress(Address address)
 }
 #endif
 
-// Mask an address down to the enclosing oilpan heap page base address.
+// Mask an address down to the enclosing oilpan heap base page.
 // All oilpan heap pages are aligned at blinkPageBase plus an OS page size.
 // FIXME: Remove PLATFORM_EXPORT once we get a proper public interface to our typed heaps.
 // This is only exported to enable tests in HeapTest.cpp.
-PLATFORM_EXPORT inline Address pageHeaderAddress(Address address)
+PLATFORM_EXPORT inline BaseHeapPage* pageHeaderFromObject(const void* object)
 {
-    return blinkPageAddress(address) + osPageSize();
+    Address address = reinterpret_cast<Address>(const_cast<void*>(object));
+    return reinterpret_cast<BaseHeapPage*>(blinkPageAddress(address) + osPageSize());
 }
 
-// Common header for heap pages.
-class BaseHeapPage {
-public:
-    BaseHeapPage(PageMemory* storage, const GCInfo* gcInfo, ThreadState* state)
-        : m_storage(storage)
-        , m_gcInfo(gcInfo)
-        , m_threadState(state)
-        , m_padding(0)
-    {
-        ASSERT(isPageHeaderAddress(reinterpret_cast<Address>(this)));
-    }
-
-    // Check if the given address points to an object in this
-    // heap page. If so, find the start of that object and mark it
-    // using the given Visitor. Otherwise do nothing. The pointer must
-    // be within the same aligned blinkPageSize as the this-pointer.
-    //
-    // This is used during conservative stack scanning to
-    // conservatively mark all objects that could be referenced from
-    // the stack.
-    virtual void checkAndMarkPointer(Visitor*, Address) = 0;
-
-#if ENABLE(GC_TRACING)
-    virtual const GCInfo* findGCInfo(Address) = 0;
+NO_SANITIZE_ADDRESS
+inline void asanMemset(Address base, uint8_t zapValue, size_t size)
+{
+#if defined(ADDRESS_SANITIZER)
+        // Don't use memset when running with ASan since this needs to zap

[Mads Ager (chromium), 2014/07/16 05:19:29]

I think we should move the comment to before the method. We should make it very
clear that this method should only be used if you need to memset poisoned
memory. Normally you should just use memset.

[wibling-chromium, 2014/07/16 08:45:33]

Done.

+        // poisoned memory as well and the NO_SANITIZE_ADDRESS annotation
+        // only works for code in this method and not for calls to memset.
+        for (Address current = base; current < base + size; ++current) {
+            *current = zapValue;
+        }
+#else
+        memset(base, zapValue, size);
 #endif
-
-    Address address() { return reinterpret_cast<Address>(this); }
-    PageMemory* storage() const { return m_storage; }
-    ThreadState* threadState() const { return m_threadState; }
-    const GCInfo* gcInfo() { return m_gcInfo; }
-    virtual bool isLargeObject() { return false; }
-
-private:
-    // Accessor to silence unused warnings for the m_padding field.
-    intptr_t padding() const { return m_padding; }
-
-    PageMemory* m_storage;
-    const GCInfo* m_gcInfo;
-    ThreadState* m_threadState;
-    // Pointer sized integer to ensure proper alignment of the
-    // HeapPage header. This can be used as a bit field if we need
-    // to associate more information with pages.
-    intptr_t m_padding;
-};
+}
 
 // Large allocations are allocated as separate objects and linked in a
 // list.
@@ -232,7 +218,7 @@ public:
     // Returns true for any address that is on one of the pages that this
     // large object uses. That ensures that we can use a negative result to
     // populate the negative page cache.
-    bool contains(Address object)
+    virtual bool contains(Address object) OVERRIDE
     {
         return roundToBlinkPageStart(address()) <= object && object < roundToBlinkPageEnd(address() + size());
     }
@@ -261,6 +247,14 @@ public:
     void getStats(HeapStats&);
     void mark(Visitor*);
     void finalize();
+    void setDeadMark();
+    virtual void markOrphaned()
+    {
+        // Zap the payload with a recognizable value to detect any incorrect
+        // cross thread pointer usage.
+        memset(payload(), orphanedZapValue, payloadSize());
+        BaseHeapPage::markOrphaned();
+    }
 
 private:
     friend class ThreadHeap<Header>;
@@ -329,9 +323,9 @@ public:
     inline size_t payloadSize();
     inline Address payloadEnd();
 
-    inline void setDebugMark();
-    inline void clearDebugMark();
-    inline bool hasDebugMark() const;
+    inline void setDeadMark();
+    inline void clearDeadMark();
+    inline bool hasDeadMark() const;
 
     // Zap magic number with a new magic number that means there was once an
     // object allocated here, but it was freed because nobody marked it during
@@ -468,7 +462,7 @@ public:
     // Returns true for the whole blinkPageSize page that the page is on, even
     // for the header, and the unmapped guard page at the start. That ensures
     // the result can be used to populate the negative page cache.
-    bool contains(Address addr)
+    virtual bool contains(Address addr) OVERRIDE
     {
         Address blinkPageStart = roundToBlinkPageStart(address());
         ASSERT(blinkPageStart == address() - osPageSize()); // Page is at aligned address plus guard page size.
@@ -490,7 +484,7 @@ public:
     Address end() { return payload() + payloadSize(); }
 
     void getStats(HeapStats&);
-    void clearMarks();
+    void clearLiveAndMarkDead();
     void sweep();
     void clearObjectStartBitMap();
     void finalize(Header*);
@@ -502,6 +496,13 @@ public:
 #if defined(ADDRESS_SANITIZER)
     void poisonUnmarkedObjects();
 #endif
+    virtual void markOrphaned()
+    {
+        // Zap the payload with a recognizable value to detect any incorrect
+        // cross thread pointer usage.
+        asanMemset(payload(), orphanedZapValue, payloadSize());
+        BaseHeapPage::markOrphaned();
+    }
 
 protected:
     Header* findHeaderFromAddress(Address);
@@ -677,6 +678,53 @@ private:
     mutable Mutex m_mutex;
 };
 
+template<typename DataType>
+class PagePool {
+protected:
+    PagePool();
+
+    class PoolEntry {
+    public:
+        PoolEntry(DataType* data, PoolEntry* next)
+            : data(data)
+            , next(next)
+        { }
+
+        DataType* data;
+        PoolEntry* next;
+    };
+
+    PoolEntry* m_pool[NumberOfHeaps];
+};
+
+// Once pages have been used for one type of thread heap they will never be
+// reused for another type of thread heap. Instead of unmapping, we add the
+// pages to a pool of pages to be reused later by a thread heap of the same
+// type. This is done as a security feature to avoid type confusion. The
+// heaps are type segregated by having separate thread heaps for different
+// types of objects. Holding on to pages ensures that the same virtual address
+// space cannot be used for objects of another type than the type contained
+// in this page to begin with.
+class FreePagePool : public PagePool<PageMemory> {
+public:
+    ~FreePagePool();
+    void addFreePage(int, PageMemory*);
+    PageMemory* takeFreePage(int);
+
+private:
+    Mutex m_mutex[NumberOfHeaps];
+};
+
+class OrphanedPagePool : public PagePool<BaseHeapPage> {
+public:
+    ~OrphanedPagePool();
+    void addOrphanedPage(int, BaseHeapPage*);
+    void decommitOrphanedPages();
+#ifndef NDEBUG
+    bool contains(void*);
+#endif
+};
+
 // The CallbackStack contains all the visitor callbacks used to trace and mark
 // objects. A specific CallbackStack instance contains at most bufferSize elements.
 // If more space is needed a new CallbackStack instance is created and chained
@@ -727,7 +775,7 @@ public:
             init(first);
         }
     }
-    bool popAndInvokeCallback(CallbackStack** first, Visitor*);
+    template<CallbackInvocationMode Mode> bool popAndInvokeCallback(CallbackStack** first, Visitor*);
     static void invokeCallbacks(CallbackStack** first, Visitor*);
 
     Item* allocateEntry(CallbackStack** first)
@@ -755,6 +803,7 @@ private:
 class BaseHeap {
 public:
     virtual ~BaseHeap() { }
+    virtual void cleanupPages() = 0;
 
     // Find the page in this thread heap containing the given
     // address. Returns 0 if the address is not contained in any
@@ -769,14 +818,8 @@ public:
     // and builds freelists for all the unused memory.
     virtual void sweep() = 0;
 
-    // Forcefully finalize all objects in this part of the Blink heap
-    // (potentially with the exception of one object). This is used
-    // during thread termination to make sure that all objects for the
-    // dying thread are finalized.
-    virtual void assertEmpty() = 0;
-
     virtual void clearFreeLists() = 0;
-    virtual void clearMarks() = 0;
+    virtual void clearLiveAndMarkDead() = 0;
 #ifndef NDEBUG
     virtual void getScannedStats(HeapStats&) = 0;
 #endif
@@ -784,6 +827,8 @@ public:
     virtual void makeConsistentForGC() = 0;
     virtual bool isConsistentForGC() = 0;
 
+    virtual void prepareHeapForTermination() = 0;
+
     // Returns a bucket number for inserting a FreeListEntry of a
     // given size. All FreeListEntries in the given bucket, n, have
     // size >= 2^n.
@@ -803,17 +848,17 @@ public:
 template<typename Header>
 class ThreadHeap : public BaseHeap {
 public:
-    ThreadHeap(ThreadState*);
+    ThreadHeap(ThreadState*, int);
     virtual ~ThreadHeap();
+    virtual void cleanupPages();
 
     virtual BaseHeapPage* heapPageFromAddress(Address);
 #if ENABLE(GC_TRACING)
     virtual const GCInfo* findGCInfoOfLargeHeapObject(Address);
 #endif
     virtual void sweep();
-    virtual void assertEmpty();
     virtual void clearFreeLists();
-    virtual void clearMarks();
+    virtual void clearLiveAndMarkDead();
 #ifndef NDEBUG
     virtual void getScannedStats(HeapStats&);
 #endif
@@ -830,41 +875,18 @@ public:
 
     inline Address allocate(size_t, const GCInfo*);
     void addToFreeList(Address, size_t);
-    void addPageMemoryToPool(PageMemory*);
-    void addPageToPool(HeapPage<Header>*);
     inline static size_t roundedAllocationSize(size_t size)
     {
         return allocationSizeFromSize(size) - sizeof(Header);
     }
 
-private:
-    // Once pages have been used for one thread heap they will never
-    // be reused for another thread heap. Instead of unmapping, we add
-    // the pages to a pool of pages to be reused later by this thread
-    // heap. This is done as a security feature to avoid type
-    // confusion. The heap is type segregated by having separate
-    // thread heaps for various types of objects. Holding on to pages
-    // ensures that the same virtual address space cannot be used for
-    // objects of another type than the type contained in this thread
-    // heap.
-    class PagePoolEntry {
-    public:
-        PagePoolEntry(PageMemory* storage, PagePoolEntry* next)
-            : m_storage(storage)
-            , m_next(next)
-        { }
-
-        PageMemory* storage() { return m_storage; }
-        PagePoolEntry* next() { return m_next; }
-
-    private:
-        PageMemory* m_storage;
-        PagePoolEntry* m_next;
-    };
+    void prepareHeapForTermination();
+    void removePageFromHeap(HeapPage<Header>*);
 
+private:
+    void addPageToHeap(const GCInfo*);
     PLATFORM_EXPORT Address outOfLineAllocate(size_t, const GCInfo*);
     static size_t allocationSizeFromSize(size_t);
-    void addPageToHeap(const GCInfo*);
     PLATFORM_EXPORT Address allocateLargeObject(size_t, const GCInfo*);
     Address currentAllocationPoint() const { return m_currentAllocationPoint; }
     size_t remainingAllocationSize() const { return m_remainingAllocationSize; }
@@ -880,11 +902,7 @@ private:
     bool allocateFromFreeList(size_t);
 
     void freeLargeObject(LargeHeapObject<Header>*, LargeHeapObject<Header>**);
-
     void allocatePage(const GCInfo*);
-    PageMemory* takePageFromPool();
-    void clearPagePool();
-    void deletePages();
 
     Address m_currentAllocationPoint;
     size_t m_remainingAllocationSize;
@@ -898,9 +916,9 @@ private:
     // All FreeListEntries in the nth list have size >= 2^n.
     FreeListEntry* m_freeLists[blinkPageSizeLog2];
 
-    // List of pages that have been previously allocated, but are now
-    // unused.
-    PagePoolEntry* m_pagePool;
+    // Index into the page pools. This is used to ensure that the pages of the
+    // same type go into the correct page pool and thus avoid type confusion.
+    int m_index;
 };
 
 class PLATFORM_EXPORT Heap {
@@ -912,6 +930,9 @@ public:
     static BaseHeapPage* contains(Address);
     static BaseHeapPage* contains(void* pointer) { return contains(reinterpret_cast<Address>(pointer)); }
     static BaseHeapPage* contains(const void* pointer) { return contains(const_cast<void*>(pointer)); }
+#ifndef NDEBUG
+    static bool containedInHeapOrOrphanedPage(void*);
+#endif
 
     // Push a trace callback on the marking stack.
     static void pushTraceCallback(void* containerObject, TraceCallback);
@@ -933,7 +954,7 @@ public:
 
     // Pop the top of the marking stack and call the callback with the visitor
     // and the object. Returns false when there is nothing more to do.
-    static bool popAndInvokeTraceCallback(Visitor*);
+    template<CallbackInvocationMode Mode> static bool popAndInvokeTraceCallback(Visitor*);
 
     // Remove an item from the weak callback work list and call the callback
     // with the visitor and the closure pointer. Returns false when there is
@@ -950,7 +971,9 @@ public:
     template<typename T> static Address reallocate(void* previous, size_t);
 
     static void collectGarbage(ThreadState::StackState);
+    static void collectGarbageForTerminatingThread(ThreadState*);
     static void collectAllGarbage();
+    template<CallbackInvocationMode Mode> static void traceRootsAndPerformGlobalWeakProcessing();
     static void setForcePreciseGCForTesting();
 
     static void prepareForGC();
@@ -988,6 +1011,9 @@ public:
     // during conservative scanning.
     static bool lastGCWasConservative() { return s_lastGCWasConservative; }
 
+    static FreePagePool* freePagePool() { return s_freePagePool; }
+    static OrphanedPagePool* orphanedPagePool() { return s_orphanedPagePool; }
+
 private:
     static Visitor* s_markingVisitor;
 
@@ -997,6 +1023,8 @@ private:
     static HeapDoesNotContainCache* s_heapDoesNotContainCache;
     static bool s_shutdownCalled;
     static bool s_lastGCWasConservative;
+    static FreePagePool* s_freePagePool;
+    static OrphanedPagePool* s_orphanedPagePool;
     friend class ThreadState;
 };
 
@@ -1303,7 +1331,10 @@ T* adoptRefCountedGarbageCollected(T* ptr)
 NO_SANITIZE_ADDRESS
 void HeapObjectHeader::checkHeader() const
 {
-    ASSERT(m_magic == magic);
+#ifndef NDEBUG
+    BaseHeapPage* page = pageHeaderFromObject(this);
+    ASSERT(page->orphaned() || m_magic == magic);
+#endif
 }
 
 Address HeapObjectHeader::payload()