Oilpan: introduce eager finalization.

Source/platform/heap/Heap.h

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 368 matching lines @@
     BasePage* next() const { return m_next; }
 
     // virtual methods are slow. So performance-sensitive methods
     // should be defined as non-virtual methods on NormalPage and LargeObjectPage.
     // The following methods are not performance-sensitive.
     virtual size_t objectPayloadSizeForTesting() = 0;
     virtual bool isEmpty() = 0;
     virtual void removeFromHeap() = 0;
     virtual void sweep() = 0;
     virtual void markUnmarkedObjectsDead() = 0;
+
 #if defined(ADDRESS_SANITIZER)
-    virtual void poisonUnmarkedObjects() = 0;
+    enum ObjectsToPoison {
+        UnmarkedOnly,
+        UnmarkedOrMarked,
+    };
+    enum Poisoning {
+        SetPoison,
+        ClearPoison,
+    };
+    virtual void poisonObjects(ObjectsToPoison, Poisoning) = 0;
 #endif
     // Check if the given address points to an object in this
     // heap page. If so, find the start of that object and mark it
     // using the given Visitor. Otherwise do nothing. The pointer must
     // be within the same aligned blinkPageSize as the this-pointer.
     //
     // This is used during conservative stack scanning to
     // conservatively mark all objects that could be referenced from
     // the stack.
     virtual void checkAndMarkPointer(Visitor*, Address) = 0;
@@ skipping 66 matching lines @@
     {
         return payload() <= address && address < payloadEnd();
     }
 
     virtual size_t objectPayloadSizeForTesting() override;
     virtual bool isEmpty() override;
     virtual void removeFromHeap() override;
     virtual void sweep() override;
     virtual void markUnmarkedObjectsDead() override;
 #if defined(ADDRESS_SANITIZER)
-    virtual void poisonUnmarkedObjects() override;
+    virtual void poisonObjects(ObjectsToPoison, Poisoning) override;
 #endif
     virtual void checkAndMarkPointer(Visitor*, Address) override;
     virtual void markOrphaned() override;
 #if ENABLE(GC_PROFILING)
     const GCInfo* findGCInfo(Address) override;
     void snapshot(TracedValue*, ThreadState::SnapshotInfo*) override;
     void incrementMarkedObjectsAge() override;
     void countMarkedObjects(ClassAgeCountsMap&) override;
     void countObjectsToSweep(ClassAgeCountsMap&) override;
 #endif
@@ skipping 41 matching lines @@
     {
         return payload() <= address && address < payloadEnd();
     }
 
     virtual size_t objectPayloadSizeForTesting() override;
     virtual bool isEmpty() override;
     virtual void removeFromHeap() override;
     virtual void sweep() override;
     virtual void markUnmarkedObjectsDead() override;
 #if defined(ADDRESS_SANITIZER)
-    virtual void poisonUnmarkedObjects() override;
+    virtual void poisonObjects(ObjectsToPoison, Poisoning) override;
 #endif
     virtual void checkAndMarkPointer(Visitor*, Address) override;
     virtual void markOrphaned() override;
 
 #if ENABLE(GC_PROFILING)
     const GCInfo* findGCInfo(Address) override;
     void snapshot(TracedValue*, ThreadState::SnapshotInfo*) override;
     void incrementMarkedObjectsAge() override;
     void countMarkedObjects(ClassAgeCountsMap&) override;
     void countObjectsToSweep(ClassAgeCountsMap&) override;
@@ skipping 144 matching lines @@
     virtual void clearFreeLists() { }
     void makeConsistentForSweeping();
 #if ENABLE(ASSERT)
     virtual bool isConsistentForSweeping() = 0;
 #endif
     size_t objectPayloadSizeForTesting();
     void prepareHeapForTermination();
     void prepareForSweep();
 #if defined(ADDRESS_SANITIZER)
     void poisonUnmarkedObjects();
+    void poisonHeap(bool poisonOrNot);
 #endif
     Address lazySweep(size_t, size_t gcInfoIndex);
     void sweepUnsweptPage();
     // Returns true if we have swept all pages within the deadline.
     // Returns false otherwise.
     bool lazySweepWithDeadline(double deadlineSeconds);
     void completeSweep();
 
     ThreadState* threadState() { return m_threadState; }
     int heapIndex() const { return m_index; }
@@ skipping 183 matching lines @@
         size_t allocationSize = size + sizeof(HeapObjectHeader);
         // Align size with allocation granularity.
         allocationSize = (allocationSize + allocationMask) & ~allocationMask;
         return allocationSize;
     }
     static inline size_t roundedAllocationSize(size_t size)
     {
         return allocationSizeFromSize(size) - sizeof(HeapObjectHeader);
     }
     static Address allocateOnHeapIndex(ThreadState*, size_t, int heapIndex, size_t gcInfoIndex);
-    template<typename T> static Address allocate(size_t);
+    template<typename T> static Address allocate(size_t, bool eagerlySweep = false);
     template<typename T> static Address reallocate(void* previous, size_t);
 
     enum GCReason {
         IdleGC,
         PreciseGC,
         ConservativeGC,
         ForcedGC,
         NumberOfGCReason
     };
     static const char* gcReasonString(GCReason);
@@ skipping 85 matching lines @@
         static void remove(PageMemoryRegion*, RegionTree**);
     private:
         PageMemoryRegion* m_region;
         RegionTree* m_left;
         RegionTree* m_right;
     };
 
     // Reset counters that track live and allocated-since-last-GC sizes.
     static void resetHeapCounters();
 
+    static int heapIndexForObjectSize(size_t);
+    static bool isNormalHeapIndex(int);
+
     static Visitor* s_markingVisitor;
     static CallbackStack* s_markingStack;
     static CallbackStack* s_postMarkingCallbackStack;
     static CallbackStack* s_globalWeakCallbackStack;
     static CallbackStack* s_ephemeronStack;
     static HeapDoesNotContainCache* s_heapDoesNotContainCache;
     static bool s_shutdownCalled;
     static bool s_lastGCWasConservative;
     static FreePagePool* s_freePagePool;
     static OrphanedPagePool* s_orphanedPagePool;
     static RegionTree* s_regionTree;
     static size_t s_allocatedSpace;
     static size_t s_allocatedObjectSize;
     static size_t s_markedObjectSize;
     static size_t s_estimatedLiveObjectSize;
     static size_t s_externalObjectSizeAtLastGC;
     static double s_estimatedMarkingTimePerByte;
 
     friend class ThreadState;
 };
 
+template<typename T>
+struct IsEagerlyFinalizedType {
+private:
+    typedef char YesType;
+    struct NoType {
+        char padding[8];
+    };
+
+    template <typename U> static YesType checkMarker(typename U::IsEagerlyFinalizedMarker*);
+    template <typename U> static NoType checkMarker(...);
+
+public:
+    static const bool value = sizeof(checkMarker<T>(nullptr)) == sizeof(YesType);
+};
+
 template<typename T> class GarbageCollected {
     WTF_MAKE_NONCOPYABLE(GarbageCollected);
 
     // For now direct allocation of arrays on the heap is not allowed.
     void* operator new[](size_t size);
 
 #if OS(WIN) && COMPILER(MSVC)
     // Due to some quirkiness in the MSVC compiler we have to provide
     // the delete[] operator in the GarbageCollected subclasses as it
     // is called when a class is exported in a DLL.
 protected:
     void operator delete[](void* p)
     {
         ASSERT_NOT_REACHED();
     }
 #else
     void operator delete[](void* p);
 #endif
 
 public:
     using GarbageCollectedBase = T;
 
     void* operator new(size_t size)
     {
-        return allocateObject(size);
+        return allocateObject(size, IsEagerlyFinalizedType<T>::value);
     }
 
-    static void* allocateObject(size_t size)
+    static void* allocateObject(size_t size, bool eagerlySweep)
     {
-        return Heap::allocate<T>(size);
+        return Heap::allocate<T>(size, eagerlySweep);
     }
 
     void operator delete(void* p)
     {
         ASSERT_NOT_REACHED();
     }
 
 protected:
     GarbageCollected()
     {
     }
 };
 
 // Assigning class types to their heaps.
 //
-// We use sized heaps for most 'normal' objcts to improve memory locality.
+// We use sized heaps for most 'normal' objects to improve memory locality.
 // It seems that the same type of objects are likely to be accessed together,
 // which means that we want to group objects by type. That's one reason
 // why we provide dedicated heaps for popular types (e.g., Node, CSSValue),
 // but it's not practical to prepare dedicated heaps for all types.
 // Thus we group objects by their sizes, hoping that this will approximately
 // group objects by their types.
 //
 // An exception to the use of sized heaps is made for class types that
 // require prompt finalization after a garbage collection. That is, their
 // instances have to be finalized early and cannot be delayed until lazy
 // sweeping kicks in for their heap and page. The EAGERLY_FINALIZE()
 // macro is used to declare a class (and its derived classes) as being
-// in need of eagerly finalized. Must be defined with 'public' visibility
+// in need of eager finalization. Must be defined with 'public' visibility
 // for a class.
 //
-template<typename T, typename Enabled = void>
-class HeapIndexTrait {
-public:
-    static int heapIndexForObject(size_t size)
-    {
-        if (size < 64) {
-            if (size < 32)
-                return NormalPage1HeapIndex;
-            return NormalPage2HeapIndex;
-        }
-        if (size < 128)
-            return NormalPage3HeapIndex;
-        return NormalPage4HeapIndex;
+
+inline int Heap::heapIndexForObjectSize(size_t size)
+{
+    if (size < 64) {
+        if (size < 32)
+            return NormalPage1HeapIndex;
+        return NormalPage2HeapIndex;
     }
-};
+    if (size < 128)
+        return NormalPage3HeapIndex;
+    return NormalPage4HeapIndex;
+}
+
+inline bool Heap::isNormalHeapIndex(int index)
+{
+    return index >= NormalPage1HeapIndex && index <= NormalPage4HeapIndex;
+}
 
 // TODO(Oilpan): enable this macro when enabling lazy sweeping, non-Oilpan.
 #if ENABLE(OILPAN)
 #define EAGERLY_FINALIZE() typedef int IsEagerlyFinalizedMarker
 #define EAGERLY_FINALIZE_WILL_BE_REMOVED()
 #else
 #define EAGERLY_FINALIZE()
 // TODO(Oilpan): define in terms of Oilpan's EAGERLY_FINALIZE() once lazy
 // sweeping is enabled non-Oilpan.
 #define EAGERLY_FINALIZE_WILL_BE_REMOVED()
 #endif
 
-template<typename T>
-struct IsEagerlyFinalizedType {
-private:
-    typedef char YesType;
-    struct NoType {
-        char padding[8];
-    };
-
-    template <typename U> static YesType checkMarker(typename U::IsEagerlyFinalizedMarker*);
-    template <typename U> static NoType checkMarker(...);
-
-public:
-    static const bool value = sizeof(checkMarker<T>(nullptr)) == sizeof(YesType);
-};
-
-template<typename T>
-class HeapIndexTrait<T, typename WTF::EnableIf<IsEagerlyFinalizedType<T>::value>::Type> {
-public:
-    static int heapIndexForObject(size_t)
-    {
-        return EagerSweepHeapIndex;
-    }
-};
-
 NO_SANITIZE_ADDRESS inline
 size_t HeapObjectHeader::size() const
 {
     size_t result = m_encoded & headerSizeMask;
     // Large objects should not refer to header->size().
     // The actual size of a large object is stored in
     // LargeObjectPage::m_payloadSize.
     ASSERT(result != largeObjectSizeInHeader);
     ASSERT(!pageFromObject(this)->isLargeObjectPage());
     return result;
@@ skipping 104 matching lines @@
 
 inline Address Heap::allocateOnHeapIndex(ThreadState* state, size_t size, int heapIndex, size_t gcInfoIndex)
 {
     ASSERT(state->isAllocationAllowed());
     ASSERT(heapIndex != LargeObjectHeapIndex);
     NormalPageHeap* heap = static_cast<NormalPageHeap*>(state->heap(heapIndex));
     return heap->allocateObject(allocationSizeFromSize(size), gcInfoIndex);
 }
 
 template<typename T>
-Address Heap::allocate(size_t size)
+Address Heap::allocate(size_t size, bool eagerlySweep)
 {
     ThreadState* state = ThreadStateFor<ThreadingTrait<T>::Affinity>::state();
-    return Heap::allocateOnHeapIndex(state, size, HeapIndexTrait<T>::heapIndexForObject(size), GCInfoTrait<T>::index());
+    return Heap::allocateOnHeapIndex(state, size, eagerlySweep ? EagerSweepHeapIndex : Heap::heapIndexForObjectSize(size), GCInfoTrait<T>::index());
 }
 
 template<typename T>
 Address Heap::reallocate(void* previous, size_t size)
 {
+    // Not intended to be a full C realloc() substitute;
+    // realloc(nullptr, size) is not a supported alias for malloc(size).
+
+    // TODO(sof): promptly free the previous object.
     if (!size) {
-        // If the new size is 0 this is equivalent to either free(previous) or
-        // malloc(0).  In both cases we do nothing and return nullptr.
+        // If the new size is 0 this is considered equivalent to free(previous).
         return nullptr;
     }
+
     ThreadState* state = ThreadStateFor<ThreadingTrait<T>::Affinity>::state();
-    // TODO(haraken): reallocate() should use the heap that the original object
-    // is using. This won't be a big deal since reallocate() is rarely used.
-    Address address = Heap::allocateOnHeapIndex(state, size, HeapIndexTrait<T>::heapIndexForObject(size), GCInfoTrait<T>::index());
-    if (!previous) {
-        // This is equivalent to malloc(size).
-        return address;
-    }
     HeapObjectHeader* previousHeader = HeapObjectHeader::fromPayload(previous);
+    BasePage* page = pageFromObject(previousHeader);
+    ASSERT(page);
+    int heapIndex = page->heap()->heapIndex();

[haraken, 2015/05/28 12:30:04]

I'd add ASSERT(heapIndex != EagerSweepHeapIndex) // Don't yet support.

reallocate is used by a very limited number of classes, so we won't need to
support it.

[sof, 2015/05/28 12:51:39]

That seems like a random restriction to make; what if you embed one in a
HeapTerminatedArray at some point?

(This also fixes e.g., Node/CSSValue reallocation.)

+    // Recompute the effective heap index if previous allocation
+    // was on the normal heaps or a large object.
+    if (isNormalHeapIndex(heapIndex) || heapIndex == LargeObjectHeapIndex)
+        heapIndex = heapIndexForObjectSize(size);
+
     // TODO(haraken): We don't support reallocate() for finalizable objects.
     ASSERT(!Heap::gcInfo(previousHeader->gcInfoIndex())->hasFinalizer());
     ASSERT(previousHeader->gcInfoIndex() == GCInfoTrait<T>::index());
+    Address address = Heap::allocateOnHeapIndex(state, size, heapIndex, GCInfoTrait<T>::index());
     size_t copySize = previousHeader->payloadSize();
     if (copySize > size)
         copySize = size;
     memcpy(address, previous, copySize);
     return address;
 }
 
 } // namespace blink
 
 #endif // Heap_h