PartitionAlloc: fixes and improvements to large-chunk size tracking.

Source/wtf/PartitionAlloc.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 418 matching lines @@
         latestExtent->superPagesEnd = superPage + kSuperPageSize;
     } else {
         // We allocated next to an existing extent so just nudge the size up a little.
         ASSERT(currentExtent->superPagesEnd);
         currentExtent->superPagesEnd += kSuperPageSize;
         ASSERT(ret >= currentExtent->superPageBase && ret < currentExtent->superPagesEnd);
     }
     return ret;
 }
 
-static ALWAYS_INLINE size_t partitionBucketBytes(const PartitionBucket* bucket)
-{
-    return bucket->numSystemPagesPerSlotSpan * kSystemPageSize;
-}
-
-static ALWAYS_INLINE uint16_t partitionBucketSlots(const PartitionBucket* bucket)
-{
-    return static_cast<uint16_t>(partitionBucketBytes(bucket) / bucket->slotSize);
-}
-
 static ALWAYS_INLINE uint16_t partitionBucketPartitionPages(const PartitionBucket* bucket)
 {
     return (bucket->numSystemPagesPerSlotSpan + (kNumSystemPagesPerPartitionPage - 1)) / kNumSystemPagesPerPartitionPage;
 }
 
 static ALWAYS_INLINE void partitionPageReset(PartitionPage* page, PartitionBucket* bucket)
 {
     ASSERT(page != &PartitionRootGeneric::gSeedPage);
     ASSERT(page->bucket == bucket);
     ASSERT(!page->freelistHead);
@@ skipping 153 matching lines @@
         }
     }
 
     bucket->activePagesHead = 0;
     return false;
 }
 
 static ALWAYS_INLINE PartitionDirectMapExtent* partitionPageToDirectMapExtent(PartitionPage* page)
 {
     ASSERT(partitionBucketIsDirectMapped(page->bucket));
-    return reinterpret_cast<PartitionDirectMapExtent*>(reinterpret_cast<char*>(page) + 2 * kPageMetadataSize);
+    return reinterpret_cast<PartitionDirectMapExtent*>(reinterpret_cast<char*>(page) + 3 * kPageMetadataSize);
 }
 
-static ALWAYS_INLINE void* partitionDirectMap(PartitionRootBase* root, int flags, size_t size)
+static ALWAYS_INLINE void partitionPageSetRawSize(PartitionPage* page, size_t size)
 {
-    size = partitionDirectMapSize(size);
+    size_t* rawSizePtr = partitionPageGetRawSizePtr(page);
+    if (UNLIKELY(rawSizePtr != nullptr))
+        *rawSizePtr = size;
+}
+
+static ALWAYS_INLINE void* partitionDirectMap(PartitionRootBase* root, int flags, size_t rawSize)
+{
+    size_t size = partitionDirectMapSize(rawSize);
 
     // Because we need to fake looking like a super page, we need to allocate
     // a bunch of system pages more than "size":
     // - The first few system pages are the partition page in which the super
     // page metadata is stored. We fault just one system page out of a partition
     // page sized clump.
     // - We add a trailing guard page on 32-bit (on 64-bit we rely on the
     // massive address space plus randomization instead).
     size_t mapSize = size + kPartitionPageSize;
 #if !CPU(64BIT)
@@ skipping 22 matching lines @@
 
     PartitionSuperPageExtentEntry* extent = reinterpret_cast<PartitionSuperPageExtentEntry*>(partitionSuperPageToMetadataArea(ptr));
     extent->root = root;
     // Most new extents will be part of a larger extent, and these three fields
     // are unused, but we initialize them to 0 so that we get a clear signal
     // in case they are accidentally used.
     extent->superPageBase = 0;
     extent->superPagesEnd = 0;
     extent->next = 0;
     PartitionPage* page = partitionPointerToPageNoAlignmentCheck(ret);
-    PartitionBucket* bucket = reinterpret_cast<PartitionBucket*>(reinterpret_cast<char*>(page) + kPageMetadataSize);
+    PartitionBucket* bucket = reinterpret_cast<PartitionBucket*>(reinterpret_cast<char*>(page) + (kPageMetadataSize * 2));
     page->freelistHead = 0;
     page->nextPage = 0;
     page->bucket = bucket;
     page->numAllocatedSlots = 1;
     page->numUnprovisionedSlots = 0;
     page->pageOffset = 0;
     page->emptyCacheIndex = 0;
 
     bucket->activePagesHead = 0;
     bucket->emptyPagesHead = 0;
     bucket->slotSize = size;
     bucket->numSystemPagesPerSlotSpan = 0;
     bucket->numFullPages = 0;
 
+    partitionPageSetRawSize(page, rawSize);
+    ASSERT(partitionPageGetRawSize(page) == rawSize);
+
     PartitionDirectMapExtent* mapExtent = partitionPageToDirectMapExtent(page);
     mapExtent->mapSize = mapSize - kPartitionPageSize - kSystemPageSize;
     mapExtent->bucket = bucket;
 
     // Maintain the doubly-linked list of all direct mappings.
     mapExtent->nextExtent = root->directMapList;
     if (mapExtent->nextExtent)
         mapExtent->nextExtent->prevExtent = mapExtent;
     mapExtent->prevExtent = nullptr;
     root->directMapList = mapExtent;
@@ skipping 31 matching lines @@
     ASSERT(!(unmapSize & kPageAllocationGranularityOffsetMask));
 
     char* ptr = reinterpret_cast<char*>(partitionPageToPointer(page));
     // Account for the mapping starting a partition page before the actual
     // allocation address.
     ptr -= kPartitionPageSize;
 
     freePages(ptr, unmapSize);
 }
 
-static ALWAYS_INLINE size_t* partitionPageGetRawSizePtr(PartitionPage* page)
-{
-    // For single-slot buckets which span more than one partition page, we
-    // have some spare metadata space to store the raw allocation size. We
-    // can use this to report better statistics.
-    PartitionBucket* bucket = page->bucket;
-    if (bucket->slotSize <= kMaxSystemPagesPerSlotSpan * kSystemPageSize)
-        return nullptr;
-
-    ASSERT(partitionBucketSlots(bucket) == 1);
-    page++;
-    return reinterpret_cast<size_t*>(&page->freelistHead);
-}
-
-static ALWAYS_INLINE void partitionPageSetRawSize(PartitionPage* page, size_t size)
-{
-    size_t* rawSizePtr = partitionPageGetRawSizePtr(page);
-    if (UNLIKELY(rawSizePtr != nullptr))
-        *rawSizePtr = size;
-}
-
-static size_t partitionPageGetRawSize(PartitionPage* page)
-{
-    size_t* rawSizePtr = partitionPageGetRawSizePtr(page);
-    if (UNLIKELY(rawSizePtr != nullptr))
-        return *rawSizePtr;
-    return 0;
-}
-
 void* partitionAllocSlowPath(PartitionRootBase* root, int flags, size_t size, PartitionBucket* bucket)
 {
     // The slow path is called when the freelist is empty.
     ASSERT(!bucket->activePagesHead->freelistHead);
 
     PartitionPage* newPage = nullptr;
 
     // For the partitionAllocGeneric API, we have a bunch of buckets marked
     // as special cases. We bounce them through to the slow path so that we
     // can still have a blazing fast hot path due to lack of corner-case
@@ skipping 148 matching lines @@
 void partitionFreeSlowPath(PartitionPage* page)
 {
     PartitionBucket* bucket = page->bucket;
     ASSERT(page != &PartitionRootGeneric::gSeedPage);
     if (LIKELY(page->numAllocatedSlots == 0)) {
         // Page became fully unused.
         if (UNLIKELY(partitionBucketIsDirectMapped(bucket))) {
             partitionDirectUnmap(page);
             return;
         }
-        // If it's the current active page, attempt to change it. We'd prefer to leave
-        // the page empty as a gentle force towards defragmentation.
+        // Make sure all large allocations always bounce through the slow path,
+        // so that we correctly update size metadata.
+        // TODO(cevans): remove this special case when we start bouncing empty
+        // pages straight to the empty list. This will happen soon.
+        if (UNLIKELY(partitionPageGetRawSize(page))) {
+            // We can make that allocations from this empty page bounce
+            // through the slow path by marking the freelist head as null. To
+            // get the single slot filled correctly, we also have to tag the
+            // page as having a single unprovisioned slot.

[haraken, 2015/06/20 00:03:08]

Thanks, now I understand :)

+            ASSERT(partitionBucketSlots(bucket) == 1);
+            page->numUnprovisionedSlots = 1;
+            page->freelistHead = nullptr;
+        }
+        // If it's the current active page, attempt to change it. We'd prefer to
+        // leave the page empty as a gentle force towards defragmentation.
         if (LIKELY(page == bucket->activePagesHead) && page->nextPage) {
             if (partitionSetNewActivePage(page->nextPage)) {
                 ASSERT(bucket->activePagesHead != page);
                 // Link the empty page back in after the new current page, to
                 // avoid losing a reference to it.
                 // TODO: consider walking the list to link the empty page after
                 // all non-empty pages?
                 PartitionPage* currentPage = bucket->activePagesHead;
                 page->nextPage = currentPage->nextPage;
                 currentPage->nextPage = page;
@@ skipping 23 matching lines @@
             page->nextPage = bucket->activePagesHead;
         bucket->activePagesHead = page;
         --bucket->numFullPages;
         // Special case: for a partition page with just a single slot, it may
         // now be empty and we want to run it through the empty logic.
         if (UNLIKELY(page->numAllocatedSlots == 0))
             partitionFreeSlowPath(page);
     }
 }
 
-bool partitionReallocDirectMappedInPlace(PartitionRootGeneric* root, PartitionPage* page, size_t newSize)
+bool partitionReallocDirectMappedInPlace(PartitionRootGeneric* root, PartitionPage* page, size_t rawSize)
 {
     ASSERT(partitionBucketIsDirectMapped(page->bucket));
 
-    newSize = partitionCookieSizeAdjustAdd(newSize);
+    rawSize = partitionCookieSizeAdjustAdd(rawSize);
 
     // Note that the new size might be a bucketed size; this function is called
     // whenever we're reallocating a direct mapped allocation.
-    newSize = partitionDirectMapSize(newSize);
+    size_t newSize = partitionDirectMapSize(rawSize);
     if (newSize < kGenericMinDirectMappedDownsize)
         return false;
 
     // bucket->slotSize is the current size of the allocation.
     size_t currentSize = page->bucket->slotSize;
     if (newSize == currentSize)
         return true;
 
     char* charPtr = static_cast<char*>(partitionPageToPointer(page));
 
@@ skipping 21 matching lines @@
         memset(charPtr + currentSize, kUninitializedByte, recommitSize);
 #endif
     } else {
         // We can't perform the realloc in-place.
         // TODO: support this too when possible.
         return false;
     }
 
 #if ENABLE(ASSERT)
     // Write a new trailing cookie.
-    partitionCookieWriteValue(charPtr + newSize - kCookieSize);
+    partitionCookieWriteValue(charPtr + rawSize - kCookieSize);
 #endif
 
+    partitionPageSetRawSize(page, rawSize);
+    ASSERT(partitionPageGetRawSize(page) == rawSize);
+
     page->bucket->slotSize = newSize;
     return true;
 }
 
 void* partitionReallocGeneric(PartitionRootGeneric* root, void* ptr, size_t newSize)
 {
 #if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
     return realloc(ptr, newSize);
 #else
     if (UNLIKELY(!ptr))
@@ skipping 173 matching lines @@
     // partitionsDumpBucketStats is called after collecting stats because it
     // can use PartitionAlloc to allocate and this can affect the statistics.
     for (size_t i = 0; i < partitionNumBuckets; ++i) {
         if (memoryStats[i].isValid)
             partitionStatsDumper->partitionsDumpBucketStats(partitionName, &memoryStats[i]);
     }
 }
 
 } // namespace WTF