Profile FreeList

Source/platform/heap/Heap.h

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 74 matching lines @@
 #else
 #define FILL_ZERO_IF_PRODUCTION(address, size) memset((address), 0, (size))
 #define FILL_ZERO_IF_NOT_PRODUCTION(address, size) do { } while (false)
 #endif
 
 class CallbackStack;
 class PageMemory;
 template<ThreadAffinity affinity> class ThreadLocalPersistents;
 template<typename T, typename RootsAccessor = ThreadLocalPersistents<ThreadingTrait<T>::Affinity>> class Persistent;
 
-#if ENABLE(GC_PROFILE_HEAP)
+#if ENABLE(GC_PROFILE_HEAP) || ENABLE(GC_PROFILE_FREE_LIST)
 class TracedValue;
 #endif
 
 // HeapObjectHeader is 4 byte (32 bit) that has the following layout:
 //
 // | gcInfoIndex (15 bit) | size (14 bit) | dead bit (1 bit) | freed bit (1 bit) | mark bit (1 bit) |
 //
 // - For non-large objects, 14 bit is enough for |size| because the blink
 //   page size is 2^17 byte and each object is guaranteed to be aligned with
 //   2^3 byte.
@@ skipping 12 matching lines @@
 // them via a conservatively found pointer. Tracing dead objects could lead to
 // tracing of already finalized objects in another thread's heap which is a
 // use-after-free situation.
 const size_t headerDeadBitMask = 4;
 // On free-list entries we reuse the dead bit to distinguish a normal free-list
 // entry from one that has been promptly freed.
 const size_t headerPromptlyFreedBitMask = headerFreedBitMask | headerDeadBitMask;
 const size_t largeObjectSizeInHeader = 0;
 const size_t gcInfoIndexForFreeListHeader = 0;
 const size_t nonLargeObjectSizeMax = 1 << 17;
-#if ENABLE(GC_PROFILE_HEAP)
+//#if ENABLE(GC_PROFILE_HEAP)
 const size_t maxHeapObjectAge = 7;
-#endif
+//#endif
 
 static_assert(nonLargeObjectSizeMax >= blinkPageSize, "max size supported by HeapObjectHeader must at least be blinkPageSize");
 
 class PLATFORM_EXPORT HeapObjectHeader {
 public:
     // If gcInfoIndex is 0, this header is interpreted as a free list header.
     NO_SANITIZE_ADDRESS
     HeapObjectHeader(size_t size, size_t gcInfoIndex)
     {
 #if ENABLE(ASSERT)
@@ skipping 46 matching lines @@
     // GC.
     void zapMagic();
 #endif
 
     void finalize(Address, size_t);
     static HeapObjectHeader* fromPayload(const void*);
 
     static const uint16_t magic = 0xfff1;
     static const uint16_t zappedMagic = 0x4321;
 
-#if ENABLE(GC_PROFILE_HEAP)
+    //#if ENABLE(GC_PROFILE_HEAP)
     NO_SANITIZE_ADDRESS
     size_t encodedSize() const { return m_encoded; }
 
     NO_SANITIZE_ADDRESS
     size_t age() const { return m_age; }
 
     NO_SANITIZE_ADDRESS
     void incAge()
     {
         if (m_age < maxHeapObjectAge)
             m_age++;
     }
-#endif
+    //#endif
 
 #if !ENABLE(ASSERT) && !ENABLE(GC_PROFILE_HEAP) && CPU(64BIT)
     // This method is needed just to avoid compilers from removing m_padding.
     uint64_t unusedMethod() const { return m_padding; }
 #endif
 
 private:
     uint32_t m_encoded;
 #if ENABLE(ASSERT)
     uint16_t m_magic;
 #endif
-#if ENABLE(GC_PROFILE_HEAP)
+    //#if ENABLE(GC_PROFILE_HEAP)
     uint8_t m_age;
-#endif
+    //#endif
 
     // In 64 bit architectures, we intentionally add 4 byte padding immediately
     // after the HeapHeaderObject. This is because:
     //
     // | HeapHeaderObject (4 byte) | padding (4 byte) | object payload (8 * n byte) |
     // ^8 byte aligned                                ^8 byte aligned
     //
     // is better than:
     //
     // | HeapHeaderObject (4 byte) | object payload (8 * n byte) | padding (4 byte) |
@@ skipping 140 matching lines @@
     // Check if the given address points to an object in this
     // heap page. If so, find the start of that object and mark it
     // using the given Visitor. Otherwise do nothing. The pointer must
     // be within the same aligned blinkPageSize as the this-pointer.
     //
     // This is used during conservative stack scanning to
     // conservatively mark all objects that could be referenced from
     // the stack.
     virtual void checkAndMarkPointer(Visitor*, Address) = 0;
     virtual void markOrphaned();
+
 #if ENABLE(GC_PROFILE_MARKING)
     virtual const GCInfo* findGCInfo(Address) = 0;
 #endif
 #if ENABLE(GC_PROFILE_HEAP)
     virtual void snapshot(TracedValue*, ThreadState::SnapshotInfo*) = 0;
 #endif
-#if ENABLE(ASSERT)
+#if ENABLE(ASSERT) || ENABLE(GC_PROFILE_MARKING)
     virtual bool contains(Address) = 0;
 #endif
     virtual size_t size() = 0;
     virtual bool isLargeObject() { return false; }
 
     Address address() { return reinterpret_cast<Address>(this); }
     PageMemory* storage() const { return m_storage; }
     ThreadHeap* heap() const { return m_heap; }
     bool orphaned() { return !m_heap; }
     bool terminating() { return m_terminating; }
@@ skipping 72 matching lines @@
 #endif
         memset(payload(), orphanedZapValue, payloadSize());
         BaseHeapPage::markOrphaned();
     }
 #if ENABLE(GC_PROFILE_MARKING)
     const GCInfo* findGCInfo(Address) override;
 #endif
 #if ENABLE(GC_PROFILE_HEAP)
     virtual void snapshot(TracedValue*, ThreadState::SnapshotInfo*);
 #endif
-#if ENABLE(ASSERT)
+#if ENABLE(ASSERT) || ENABLE(GC_PROFILE_MARKING)
     // Returns true for the whole blinkPageSize page that the page is on, even
     // for the header, and the unmapped guard page at the start. That ensures
     // the result can be used to populate the negative page cache.
     virtual bool contains(Address addr) override
     {
         Address blinkPageStart = roundToBlinkPageStart(address());
         ASSERT(blinkPageStart == address() - WTF::kSystemPageSize); // Page is at aligned address plus guard page size.
         return blinkPageStart <= addr && addr < blinkPageStart + blinkPageSize;
     }
 #endif
     virtual size_t size() override { return blinkPageSize; }
 
     HeapPage* next() { return m_next; }
 
     void clearObjectStartBitMap();
 
+    void countUnmarkedObjects();
+
 #if defined(ADDRESS_SANITIZER)
     void poisonUnmarkedObjects();
 #endif
 
     // This method is needed just to avoid compilers from removing m_padding.
     uint64_t unusedMethod() const { return m_padding; }
 
 private:
     HeapObjectHeader* findHeaderFromAddress(Address);
     void populateObjectStartBitMap();
@@ skipping 32 matching lines @@
     virtual void markUnmarkedObjectsDead() override;
     virtual void checkAndMarkPointer(Visitor*, Address) override;
     virtual void markOrphaned() override
     {
         // Zap the payload with a recognizable value to detect any incorrect
         // cross thread pointer usage.
         memset(payload(), orphanedZapValue, payloadSize());
         BaseHeapPage::markOrphaned();
     }
 #if ENABLE(GC_PROFILE_MARKING)
-    virtual const GCInfo* findGCInfo(Address address) override
-    {
-        if (!objectContains(address))
-            return nullptr;
-        return gcInfo();
-    }
+    virtual const GCInfo* findGCInfo(Address);
 #endif
 #if ENABLE(GC_PROFILE_HEAP)
     virtual void snapshot(TracedValue*, ThreadState::SnapshotInfo*) override;
 #endif
-#if ENABLE(ASSERT)
+#if ENABLE(ASSERT) || ENABLE(GC_PROFILE_MARKING)
     // Returns true for any address that is on one of the pages that this
     // large object uses. That ensures that we can use a negative result to
     // populate the negative page cache.
     virtual bool contains(Address object) override
     {
         return roundToBlinkPageStart(address()) <= object && object < roundToBlinkPageEnd(address() + size());
     }
 #endif
     virtual size_t size()
     {
@@ skipping 131 matching lines @@
 private:
     void clearMemory(PageMemory*);
 };
 
 class FreeList {
 public:
     FreeList();
 
     void addToFreeList(Address, size_t);
     void clear();
+    FreeListEntry* takeEntry(size_t allocationSize);
+
+#if ENABLE(GC_PROFILE_FREE_LIST)
+    void countBucketSizes(size_t[], size_t[], size_t* freeSize) const;
+#endif
 
     // Returns a bucket number for inserting a FreeListEntry of a given size.
     // All FreeListEntries in the given bucket, n, have size >= 2^n.
     static int bucketIndexForSize(size_t);
 
 private:
     int m_biggestFreeListIndex;
 
     // All FreeListEntries in the nth list have size >= 2^n.
     FreeListEntry* m_freeLists[blinkPageSizeLog2];
@@ skipping 10 matching lines @@
 // Each thread heap contains the functionality to allocate new objects
 // (potentially adding new pages to the heap), to find and mark
 // objects during conservative stack scanning and to sweep the set of
 // pages after a GC.
 class PLATFORM_EXPORT ThreadHeap final {
 public:
     ThreadHeap(ThreadState*, int);
     ~ThreadHeap();
     void cleanupPages();
 
-#if ENABLE(ASSERT)
+#if ENABLE(ASSERT) || ENABLE(GC_PROFILE_MARKING)
     BaseHeapPage* findPageFromAddress(Address);
 #endif
+#if ENABLE(GC_PROFILE_FREE_LIST)
+    virtual void snapshotFreeList(TracedValue*);
+#endif
 #if ENABLE(GC_PROFILE_HEAP)
     void snapshot(TracedValue*, ThreadState::SnapshotInfo*);
 #endif
 
     void clearFreeLists();
     void makeConsistentForSweeping();
 #if ENABLE(ASSERT)
     bool isConsistentForSweeping();
 #endif
     size_t objectPayloadSizeForTesting();
@@ skipping 46 matching lines @@
     bool pagesToBeSweptContains(Address);
 #endif
 
     bool coalesce();
     void preparePagesForSweeping();
 
     Address m_currentAllocationPoint;
     size_t m_remainingAllocationSize;
     size_t m_lastRemainingAllocationSize;
 
+    double m_totalAllocationSize;
+    size_t m_allocationCount;
+    size_t m_inlineAllocationCount;
+
     HeapPage* m_firstPage;
     LargeObject* m_firstLargeObject;
     HeapPage* m_firstUnsweptPage;
     LargeObject* m_firstUnsweptLargeObject;
 
     ThreadState* m_threadState;
 
     FreeList m_freeList;
 
     // Index into the page pools.  This is used to ensure that the pages of the
     // same type go into the correct page pool and thus avoid type confusion.
     int m_index;
 
     // The size of promptly freed objects in the heap.
     size_t m_promptlyFreedSize;
+
+#if ENABLE(GC_PROFILE_FREE_LIST)
+    size_t m_allocationPointSizeSum = 0;
+    size_t m_setAllocationPointCount = 0;
+#endif
 };
 
 // Mask an address down to the enclosing oilpan heap base page.  All oilpan heap
 // pages are aligned at blinkPageBase plus an OS page size.
 // FIXME: Remove PLATFORM_EXPORT once we get a proper public interface to our
 // typed heaps.  This is only exported to enable tests in HeapTest.cpp.
 PLATFORM_EXPORT inline BaseHeapPage* pageFromObject(const void* object)
 {
     Address address = reinterpret_cast<Address>(const_cast<void*>(object));
     BaseHeapPage* page = reinterpret_cast<BaseHeapPage*>(blinkPageAddress(address) + WTF::kSystemPageSize);
     ASSERT(page->contains(address));
     return page;
 }
 
 class PLATFORM_EXPORT Heap {
 public:
     static void init();
     static void shutdown();
     static void doShutdown();
 
-#if ENABLE(ASSERT)
+#if ENABLE(ASSERT) || ENABLE(GC_PROFILE_MARKING)
     static BaseHeapPage* findPageFromAddress(Address);
     static BaseHeapPage* findPageFromAddress(void* pointer) { return findPageFromAddress(reinterpret_cast<Address>(pointer)); }
     static bool containedInHeapOrOrphanedPage(void*);
 #endif
 
     // Is the finalizable GC object still alive, but slated for lazy sweeping?
     // If a lazy sweep is in progress, returns true if the object was found
     // to be not reachable during the marking phase, but it has yet to be swept
     // and finalized. The predicate returns false in all other cases.
     //
@@ skipping 70 matching lines @@
     static void collectAllGarbage();
 
     static void processMarkingStack(Visitor*);
     static void postMarkingProcessing(Visitor*);
     static void globalWeakProcessing(Visitor*);
     static void setForcePreciseGCForTesting();
 
     static void preGC();
     static void postGC(ThreadState::GCType);
 
+    static void reportSweepingStats();
+
     // Conservatively checks whether an address is a pointer in any of the
     // thread heaps.  If so marks the object pointed to as live.
     static Address checkAndMarkPointer(Visitor*, Address);
 
 #if ENABLE(GC_PROFILE_MARKING)
     // Dump the path to specified object on the next GC.  This method is to be
     // invoked from GDB.
     static void dumpPathToObjectOnNextGC(void* p);
 
     // Forcibly find GCInfo of the object at Address.  This is slow and should
@@ skipping 387 matching lines @@
 
     // Add space for header.
     size_t allocationSize = size + sizeof(HeapObjectHeader);
     // Align size with allocation granularity.
     allocationSize = (allocationSize + allocationMask) & ~allocationMask;
     return allocationSize;
 }
 
 Address ThreadHeap::allocateObject(size_t allocationSize, size_t gcInfoIndex)
 {
+    m_totalAllocationSize += allocationSize;
+    m_allocationCount++;
     if (LIKELY(allocationSize <= m_remainingAllocationSize)) {
+        m_inlineAllocationCount++;
         Address headerAddress = m_currentAllocationPoint;
         m_currentAllocationPoint += allocationSize;
         m_remainingAllocationSize -= allocationSize;
         ASSERT(gcInfoIndex > 0);
         new (NotNull, headerAddress) HeapObjectHeader(allocationSize, gcInfoIndex);
         Address result = headerAddress + sizeof(HeapObjectHeader);
         ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask));
 
         // Unpoison the memory used for the object (payload).
         ASAN_UNPOISON_MEMORY_REGION(result, allocationSize - sizeof(HeapObjectHeader));
@@ skipping 1076 matching lines @@
 template<typename T, size_t inlineCapacity>
 struct GCInfoTrait<HeapVector<T, inlineCapacity>> : public GCInfoTrait<Vector<T, inlineCapacity, HeapAllocator>> { };
 template<typename T, size_t inlineCapacity>
 struct GCInfoTrait<HeapDeque<T, inlineCapacity>> : public GCInfoTrait<Deque<T, inlineCapacity, HeapAllocator>> { };
 template<typename T, typename U, typename V>
 struct GCInfoTrait<HeapHashCountedSet<T, U, V>> : public GCInfoTrait<HashCountedSet<T, U, V, HeapAllocator>> { };
 
 } // namespace blink
 
 #endif // Heap_h