Revert of Oilpan: Remove duplicated code between HeapPage and LargeObject

Source/platform/heap/Heap.h

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 71 matching lines @@
 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER)
 #define FILL_ZERO_IF_PRODUCTION(address, size) do { } while (false)
 #define FILL_ZERO_IF_NOT_PRODUCTION(address, size) memset((address), 0, (size))
 #else
 #define FILL_ZERO_IF_PRODUCTION(address, size) memset((address), 0, (size))
 #define FILL_ZERO_IF_NOT_PRODUCTION(address, size) do { } while (false)
 #endif
 
 class CallbackStack;
 class PageMemory;
-class ThreadHeapForHeapPage;
 template<ThreadAffinity affinity> class ThreadLocalPersistents;
 template<typename T, typename RootsAccessor = ThreadLocalPersistents<ThreadingTrait<T>::Affinity>> class Persistent;
 
 #if ENABLE(GC_PROFILING)
 class TracedValue;
 #endif
 
 // HeapObjectHeader is 4 byte (32 bit) that has the following layout:
 //
 // | gcInfoIndex (15 bit) | size (14 bit) | dead bit (1 bit) | freed bit (1 bit) | mark bit (1 bit) |
@@ skipping 221 matching lines @@
 inline Address roundToBlinkPageStart(Address address)
 {
     return reinterpret_cast<Address>(reinterpret_cast<uintptr_t>(address) & blinkPageBaseMask);
 }
 
 inline Address roundToBlinkPageEnd(Address address)
 {
     return reinterpret_cast<Address>(reinterpret_cast<uintptr_t>(address - 1) & blinkPageBaseMask) + blinkPageSize;
 }
 
+// Compute the amount of padding we have to add to a header to make
+// the size of the header plus the padding a multiple of 8 bytes.
+inline size_t headerPadding()
+{
+    return (allocationGranularity - (sizeof(HeapObjectHeader) % allocationGranularity)) % allocationGranularity;
+}
+
 // Masks an address down to the enclosing blink page base address.
 inline Address blinkPageAddress(Address address)
 {
     return reinterpret_cast<Address>(reinterpret_cast<uintptr_t>(address) & blinkPageBaseMask);
 }
 
 #if ENABLE(ASSERT)
 
 // Sanity check for a page header address: the address of the page
 // header should be OS page size away from being Blink page size
 // aligned.
 inline bool isPageHeaderAddress(Address address)
 {
     return !((reinterpret_cast<uintptr_t>(address) & blinkPageOffsetMask) - WTF::kSystemPageSize);
 }
 #endif
 
 // FIXME: Add a good comment about the heap layout once heap relayout work
 // is done.
 class BaseHeapPage {
 public:
     BaseHeapPage(PageMemory*, ThreadHeap*);
     virtual ~BaseHeapPage() { }
 
-    void link(BaseHeapPage** previousNext)
-    {
-        m_next = *previousNext;
-        *previousNext = this;
-    }
-    void unlink(BaseHeapPage** previousNext)
-    {
-        *previousNext = m_next;
-        m_next = nullptr;
-    }
-    BaseHeapPage* next() const { return m_next; }
-
     // virtual methods are slow. So performance-sensitive methods
     // should be defined as non-virtual methods on HeapPage and LargeObject.
     // The following methods are not performance-sensitive.
     virtual size_t objectPayloadSizeForTesting() = 0;
     virtual bool isEmpty() = 0;
-    virtual void removeFromHeap() = 0;
+    virtual void removeFromHeap(ThreadHeap*) = 0;
     virtual void sweep() = 0;
     virtual void markUnmarkedObjectsDead() = 0;
     // Check if the given address points to an object in this
     // heap page. If so, find the start of that object and mark it
     // using the given Visitor. Otherwise do nothing. The pointer must
     // be within the same aligned blinkPageSize as the this-pointer.
     //
     // This is used during conservative stack scanning to
     // conservatively mark all objects that could be referenced from
     // the stack.
@@ skipping 27 matching lines @@
 
     void markAsUnswept()
     {
         ASSERT(m_swept);
         m_swept = false;
     }
 
 private:
     PageMemory* m_storage;
     ThreadHeap* m_heap;
-    BaseHeapPage* m_next;
     // Whether the page is part of a terminating thread or not.
     bool m_terminating;
 
     // Track the sweeping state of a page. Set to true once
     // the lazy sweep completes has processed it.
     //
     // Set to false at the start of a sweep, true  upon completion
     // of lazy sweeping.
     bool m_swept;
-    friend class ThreadHeap;
 };
 
 class HeapPage final : public BaseHeapPage {
 public:
     HeapPage(PageMemory*, ThreadHeap*);
 
     Address payload()
     {
         return address() + sizeof(HeapPage) + headerPadding();
     }
     size_t payloadSize()
     {
         return (blinkPagePayloadSize() - sizeof(HeapPage) - headerPadding()) & ~allocationMask;
     }
     Address payloadEnd() { return payload() + payloadSize(); }
     bool containedInObjectPayload(Address address) { return payload() <= address && address < payloadEnd(); }
 
+    void link(HeapPage** previousNext)
+    {
+        m_next = *previousNext;
+        *previousNext = this;
+    }
+
+    void unlink(HeapPage** previousNext)
+    {
+        *previousNext = m_next;
+        m_next = nullptr;
+    }
+
     virtual size_t objectPayloadSizeForTesting() override;
     virtual bool isEmpty() override;
-    virtual void removeFromHeap() override;
+    virtual void removeFromHeap(ThreadHeap*) override;
     virtual void sweep() override;
     virtual void markUnmarkedObjectsDead() override;
     virtual void checkAndMarkPointer(Visitor*, Address) override;
     virtual void markOrphaned() override
     {
         // Zap the payload with a recognizable value to detect any incorrect
         // cross thread pointer usage.
 #if defined(ADDRESS_SANITIZER)
         // This needs to zap poisoned memory as well.
         // Force unpoison memory before memset.
@@ skipping 14 matching lines @@
     // for the header, and the unmapped guard page at the start. That ensures
     // the result can be used to populate the negative page cache.
     virtual bool contains(Address addr) override
     {
         Address blinkPageStart = roundToBlinkPageStart(address());
         ASSERT(blinkPageStart == address() - WTF::kSystemPageSize); // Page is at aligned address plus guard page size.
         return blinkPageStart <= addr && addr < blinkPageStart + blinkPageSize;
     }
 #endif
     virtual size_t size() override { return blinkPageSize; }
-    // Compute the amount of padding we have to add to a header to make
-    // the size of the header plus the padding a multiple of 8 bytes.
-    static size_t headerPadding()
-    {
-        return (sizeof(HeapPage) + allocationGranularity - (sizeof(HeapObjectHeader) % allocationGranularity)) % allocationGranularity;
-    }
 
+    HeapPage* next() { return m_next; }
 
-    ThreadHeapForHeapPage* heapForHeapPage();
     void clearObjectStartBitMap();
 
 #if defined(ADDRESS_SANITIZER)
     void poisonUnmarkedObjects();
 #endif
 
+    // This method is needed just to avoid compilers from removing m_padding.
+    uint64_t unusedMethod() const { return m_padding; }
+
 private:
     HeapObjectHeader* findHeaderFromAddress(Address);
     void populateObjectStartBitMap();
     bool isObjectStartBitMapComputed() { return m_objectStartBitMapComputed; }
 
+    HeapPage* m_next;
     bool m_objectStartBitMapComputed;
     uint8_t m_objectStartBitMap[reservedForObjectBitMap];
+    uint64_t m_padding; // Preserve 8-byte alignment on 32-bit systems.
+
+    friend class ThreadHeap;
 };
 
 // Large allocations are allocated as separate objects and linked in a list.
 //
 // In order to use the same memory allocation routines for everything allocated
 // in the heap, large objects are considered heap pages containing only one
 // object.
 class LargeObject final : public BaseHeapPage {
 public:
     LargeObject(PageMemory* storage, ThreadHeap* heap, size_t payloadSize)
         : BaseHeapPage(storage, heap)
         , m_payloadSize(payloadSize)
     {
     }
 
     Address payload() { return heapObjectHeader()->payload(); }
     size_t payloadSize() { return m_payloadSize; }
     Address payloadEnd() { return payload() + payloadSize(); }
     bool containedInObjectPayload(Address address) { return payload() <= address && address < payloadEnd(); }
 
     virtual size_t objectPayloadSizeForTesting() override;
     virtual bool isEmpty() override;
-    virtual void removeFromHeap() override;
+    virtual void removeFromHeap(ThreadHeap*) override;
     virtual void sweep() override;
     virtual void markUnmarkedObjectsDead() override;
     virtual void checkAndMarkPointer(Visitor*, Address) override;
     virtual void markOrphaned() override
     {
         // Zap the payload with a recognizable value to detect any incorrect
         // cross thread pointer usage.
         memset(payload(), orphanedZapValue, payloadSize());
         BaseHeapPage::markOrphaned();
     }
@@ skipping 12 matching lines @@
     // populate the negative page cache.
     virtual bool contains(Address object) override
     {
         return roundToBlinkPageStart(address()) <= object && object < roundToBlinkPageEnd(address() + size());
     }
 #endif
     virtual size_t size()
     {
         return sizeof(LargeObject) + headerPadding() +  sizeof(HeapObjectHeader) + m_payloadSize;
     }
-    // Compute the amount of padding we have to add to a header to make
-    // the size of the header plus the padding a multiple of 8 bytes.
-    static size_t headerPadding()
+    virtual bool isLargeObject() override { return true; }
+
+    void link(LargeObject** previousNext)
     {
-        return (sizeof(LargeObject) + allocationGranularity - (sizeof(HeapObjectHeader) % allocationGranularity)) % allocationGranularity;
+        m_next = *previousNext;
+        *previousNext = this;
     }
-    virtual bool isLargeObject() override { return true; }
+
+    void unlink(LargeObject** previousNext)
+    {
+        *previousNext = m_next;
+        m_next = nullptr;
+    }
+
+    LargeObject* next()
+    {
+        return m_next;
+    }
 
     HeapObjectHeader* heapObjectHeader()
     {
         Address headerAddress = address() + sizeof(LargeObject) + headerPadding();
         return reinterpret_cast<HeapObjectHeader*>(headerAddress);
     }
 
+    // This method is needed just to avoid compilers from removing m_padding.
+    uint64_t unusedMethod() const { return m_padding; }
+
 private:
-
+    friend class ThreadHeap;
+    LargeObject* m_next;
     size_t m_payloadSize;
+    uint64_t m_padding; // Preserve 8-byte alignment on 32-bit systems.
 };
 
 // A HeapDoesNotContainCache provides a fast way of taking an arbitrary
 // pointer-sized word, and determining whether it cannot be interpreted as a
 // pointer to an area that is managed by the garbage collected Blink heap.  This
 // is a cache of 'pages' that have previously been determined to be wholly
 // outside of the heap.  The size of these pages must be smaller than the
 // allocation alignment of the heap pages.  We determine off-heap-ness by
 // rounding down the pointer to the nearest page and looking up the page in the
 // cache.  If there is a miss in the cache we can determine the status of the
@@ skipping 108 matching lines @@
 
     void getFreeSizeStats(PerBucketFreeListStats bucketStats[], size_t& totalSize) const;
 #endif
 
 private:
     int m_biggestFreeListIndex;
 
     // All FreeListEntries in the nth list have size >= 2^n.
     FreeListEntry* m_freeLists[blinkPageSizeLog2];
 
-    friend class ThreadHeapForHeapPage;
+    friend class ThreadHeap;
 };
 
 // Thread heaps represent a part of the per-thread Blink heap.
 //
 // Each Blink thread has a number of thread heaps: one general heap
 // that contains any type of object and a number of heaps specialized
 // for specific object types (such as Node).
 //
 // Each thread heap contains the functionality to allocate new objects
 // (potentially adding new pages to the heap), to find and mark
 // objects during conservative stack scanning and to sweep the set of
 // pages after a GC.
-class PLATFORM_EXPORT ThreadHeap {
+class PLATFORM_EXPORT ThreadHeap final {
 public:
     ThreadHeap(ThreadState*, int);
-    virtual ~ThreadHeap();
+    ~ThreadHeap();
     void cleanupPages();
 
 #if ENABLE(ASSERT) || ENABLE(GC_PROFILING)
     BaseHeapPage* findPageFromAddress(Address);
 #endif
 #if ENABLE(GC_PROFILING)
     void snapshot(TracedValue*, ThreadState::SnapshotInfo*);
     void incrementMarkedObjectsAge();
 #endif
 
-    virtual void clearFreeLists() { }
+    void clearFreeLists();
     void makeConsistentForSweeping();
 #if ENABLE(ASSERT)
-    virtual bool isConsistentForSweeping() = 0;
+    bool isConsistentForSweeping();
 #endif
     size_t objectPayloadSizeForTesting();
-    void prepareHeapForTermination();
-    void prepareForSweep();
-    Address lazySweep(size_t, size_t gcInfoIndex);
-    void completeSweep();
 
     ThreadState* threadState() { return m_threadState; }
-    int heapIndex() const { return m_index; }
-    inline static size_t allocationSizeFromSize(size_t);
-    inline static size_t roundedAllocationSize(size_t size)
-    {
-        return allocationSizeFromSize(size) - sizeof(HeapObjectHeader);
-    }
 
-protected:
-    BaseHeapPage* m_firstPage;
-    BaseHeapPage* m_firstUnsweptPage;
-
-private:
-    virtual Address lazySweepPages(size_t, size_t gcInfoIndex) = 0;
-
-    ThreadState* m_threadState;
-
-    // Index into the page pools.  This is used to ensure that the pages of the
-    // same type go into the correct page pool and thus avoid type confusion.
-    int m_index;
-};
-
-class PLATFORM_EXPORT ThreadHeapForHeapPage final : public ThreadHeap {
-public:
-    ThreadHeapForHeapPage(ThreadState*, int);
     void addToFreeList(Address address, size_t size)
     {
         ASSERT(findPageFromAddress(address));
         ASSERT(findPageFromAddress(address + size - 1));
         m_freeList.addToFreeList(address, size);
     }
-    virtual void clearFreeLists() override;
-#if ENABLE(ASSERT)
-    virtual bool isConsistentForSweeping() override;
-    bool pagesToBeSweptContains(Address);
-#endif
 
     inline Address allocate(size_t payloadSize, size_t gcInfoIndex);
-    inline Address allocateObject(size_t allocationSize, size_t gcInfoIndex);
+    inline static size_t roundedAllocationSize(size_t size)
+    {
+        return allocationSizeFromSize(size) - sizeof(HeapObjectHeader);
+    }
+    inline static size_t allocationSizeFromSize(size_t);
+
+    void prepareHeapForTermination();
+    void prepareForSweep();
+    void completeSweep();
 
     void freePage(HeapPage*);
+    void freeLargeObject(LargeObject*);
 
-    bool coalesce();
     void promptlyFreeObject(HeapObjectHeader*);
     bool expandObject(HeapObjectHeader*, size_t);
     void shrinkObject(HeapObjectHeader*, size_t);
     void decreasePromptlyFreedSize(size_t size) { m_promptlyFreedSize -= size; }
 
 #if ENABLE(GC_PROFILING)
     void snapshotFreeList(TracedValue&);
 
     void countMarkedObjects(ClassAgeCountsMap&) const;
     void countObjectsToSweep(ClassAgeCountsMap&) const;
 #endif
 
 private:
-    void allocatePage();
-    virtual Address lazySweepPages(size_t, size_t gcInfoIndex) override;
     Address outOfLineAllocate(size_t allocationSize, size_t gcInfoIndex);
     Address currentAllocationPoint() const { return m_currentAllocationPoint; }
     size_t remainingAllocationSize() const { return m_remainingAllocationSize; }
     bool hasCurrentAllocationArea() const { return currentAllocationPoint() && remainingAllocationSize(); }
     inline void setAllocationPoint(Address, size_t);
     void updateRemainingAllocationSize();
     Address allocateFromFreeList(size_t, size_t gcInfoIndex);
+    Address lazySweepPages(size_t, size_t gcInfoIndex);
+    bool lazySweepLargeObjects(size_t);
 
-    FreeList m_freeList;
+    void allocatePage();
+    Address allocateLargeObject(size_t, size_t gcInfoIndex);
+
+    inline Address allocateObject(size_t allocationSize, size_t gcInfoIndex);
+
+#if ENABLE(ASSERT)
+    bool pagesToBeSweptContains(Address);
+#endif
+
+    bool coalesce();
+    void preparePagesForSweeping();
+
     Address m_currentAllocationPoint;
     size_t m_remainingAllocationSize;
     size_t m_lastRemainingAllocationSize;
 
+    HeapPage* m_firstPage;
+    LargeObject* m_firstLargeObject;
+    HeapPage* m_firstUnsweptPage;
+    LargeObject* m_firstUnsweptLargeObject;
+
+    ThreadState* m_threadState;
+
+    FreeList m_freeList;
+
+    // Index into the page pools.  This is used to ensure that the pages of the
+    // same type go into the correct page pool and thus avoid type confusion.
+    int m_index;
+
     // The size of promptly freed objects in the heap.
     size_t m_promptlyFreedSize;
 
 #if ENABLE(GC_PROFILING)
     size_t m_cumulativeAllocationSize;
     size_t m_allocationCount;
     size_t m_inlineAllocationCount;
 #endif
 };
 
-class ThreadHeapForLargeObject final : public ThreadHeap {
-public:
-    ThreadHeapForLargeObject(ThreadState*, int);
-    Address allocateLargeObject(size_t, size_t gcInfoIndex);
-    void freeLargeObject(LargeObject*);
-#if ENABLE(ASSERT)
-    virtual bool isConsistentForSweeping() override { return true; }
-#endif
-private:
-    Address doAllocateLargeObject(size_t, size_t gcInfoIndex);
-    virtual Address lazySweepPages(size_t, size_t gcInfoIndex) override;
-};
-
 // Mask an address down to the enclosing oilpan heap base page.  All oilpan heap
 // pages are aligned at blinkPageBase plus an OS page size.
 // FIXME: Remove PLATFORM_EXPORT once we get a proper public interface to our
 // typed heaps.  This is only exported to enable tests in HeapTest.cpp.
 PLATFORM_EXPORT inline BaseHeapPage* pageFromObject(const void* object)
 {
     Address address = reinterpret_cast<Address>(const_cast<void*>(object));
     BaseHeapPage* page = reinterpret_cast<BaseHeapPage*>(blinkPageAddress(address) + WTF::kSystemPageSize);
     ASSERT(page->contains(address));
     return page;
@@ skipping 495 matching lines @@
     // therefore has to happen before any calculation on the size.
     RELEASE_ASSERT(size < maxHeapObjectSize);
 
     // Add space for header.
     size_t allocationSize = size + sizeof(HeapObjectHeader);
     // Align size with allocation granularity.
     allocationSize = (allocationSize + allocationMask) & ~allocationMask;
     return allocationSize;
 }
 
-Address ThreadHeapForHeapPage::allocateObject(size_t allocationSize, size_t gcInfoIndex)
+Address ThreadHeap::allocateObject(size_t allocationSize, size_t gcInfoIndex)
 {
 #if ENABLE(GC_PROFILING)
     m_cumulativeAllocationSize += allocationSize;
     ++m_allocationCount;
 #endif
 
     if (LIKELY(allocationSize <= m_remainingAllocationSize)) {
 #if ENABLE(GC_PROFILING)
         ++m_inlineAllocationCount;
 #endif
         Address headerAddress = m_currentAllocationPoint;
         m_currentAllocationPoint += allocationSize;
         m_remainingAllocationSize -= allocationSize;
         ASSERT(gcInfoIndex > 0);
         new (NotNull, headerAddress) HeapObjectHeader(allocationSize, gcInfoIndex);
         Address result = headerAddress + sizeof(HeapObjectHeader);
         ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask));
 
         // Unpoison the memory used for the object (payload).
         ASAN_UNPOISON_MEMORY_REGION(result, allocationSize - sizeof(HeapObjectHeader));
         FILL_ZERO_IF_NOT_PRODUCTION(result, allocationSize - sizeof(HeapObjectHeader));
         ASSERT(findPageFromAddress(headerAddress + allocationSize - 1));
         return result;
     }
     return outOfLineAllocate(allocationSize, gcInfoIndex);
 }
 
-Address ThreadHeapForHeapPage::allocate(size_t size, size_t gcInfoIndex)
+Address ThreadHeap::allocate(size_t size, size_t gcInfoIndex)
 {
     return allocateObject(allocationSizeFromSize(size), gcInfoIndex);
 }
 
 template<typename T>
 struct HeapIndexTrait {
     static int index() { return GeneralHeap; };
 };
 
 // FIXME: The forward declaration is layering violation.
 #define DEFINE_TYPED_HEAP_TRAIT(Type)                \
     class Type;                                      \
     template<>                                       \
     struct HeapIndexTrait<class Type> {              \
     static int index() { return Type##Heap; }; \
     };
 FOR_EACH_TYPED_HEAP(DEFINE_TYPED_HEAP_TRAIT)
 #undef DEFINE_TYPED_HEAP_TRAIT
 
 template<typename T>
 Address Heap::allocateOnHeapIndex(size_t size, int heapIndex, size_t gcInfoIndex)
 {
     ThreadState* state = ThreadStateFor<ThreadingTrait<T>::Affinity>::state();
     ASSERT(state->isAllocationAllowed());
-    return static_cast<ThreadHeapForHeapPage*>(state->heap(heapIndex))->allocate(size, gcInfoIndex);
+    return state->heap(heapIndex)->allocate(size, gcInfoIndex);
 }
 
 template<typename T>
 Address Heap::allocate(size_t size)
 {
     return allocateOnHeapIndex<T>(size, HeapIndexTrait<T>::index(), GCInfoTrait<T>::index());
 }
 
 template<typename T>
 Address Heap::reallocate(void* previous, size_t size)
@@ skipping 1033 matching lines @@
 template<typename T, size_t inlineCapacity>
 struct GCInfoTrait<HeapVector<T, inlineCapacity>> : public GCInfoTrait<Vector<T, inlineCapacity, HeapAllocator>> { };
 template<typename T, size_t inlineCapacity>
 struct GCInfoTrait<HeapDeque<T, inlineCapacity>> : public GCInfoTrait<Deque<T, inlineCapacity, HeapAllocator>> { };
 template<typename T, typename U, typename V>
 struct GCInfoTrait<HeapHashCountedSet<T, U, V>> : public GCInfoTrait<HashCountedSet<T, U, V, HeapAllocator>> { };
 
 } // namespace blink
 
 #endif // Heap_h