PartitionAlloc: support in-place resize of directly mapped allocation

Source/wtf/PartitionAlloc.cpp

Index: Source/wtf/PartitionAlloc.cpp
diff --git a/Source/wtf/PartitionAlloc.cpp b/Source/wtf/PartitionAlloc.cpp
index e62a9a66235e23719cd2c5eb30ae29ed76cfdc52..43f2f126492720dc28d138713316b9fc5acd89b5 100644
--- a/Source/wtf/PartitionAlloc.cpp
+++ b/Source/wtf/PartitionAlloc.cpp
@@ -47,6 +47,7 @@ COMPILE_ASSERT(WTF::kSystemPageSize * 4 <= WTF::kPartitionPageSize, ok_partition
 COMPILE_ASSERT(!(WTF::kPartitionPageSize % WTF::kSystemPageSize), ok_partition_page_multiple);
 COMPILE_ASSERT(sizeof(WTF::PartitionPage) <= WTF::kPageMetadataSize, PartitionPage_not_too_big);
 COMPILE_ASSERT(sizeof(WTF::PartitionSuperPageExtentEntry) <= WTF::kPageMetadataSize, PartitionSuperPageExtentEntry_not_too_big);
+COMPILE_ASSERT(sizeof(WTF::PartitionBucket) <= WTF::kPageMetadataSize, PartitionBucket_not_too_big);
 COMPILE_ASSERT(WTF::kPageMetadataSize * WTF::kNumPartitionPagesPerSuperPage <= WTF::kSystemPageSize, page_metadata_fits_in_hole);
 // Check that some of our zanier calculations worked out as expected.
 COMPILE_ASSERT(WTF::kGenericSmallestBucket == 8, generic_smallest_bucket);
@@ -524,6 +525,16 @@ static ALWAYS_INLINE bool partitionSetNewActivePage(PartitionPage* page)
     return false;
 }
 
+struct PartitionDirectMapExtent {
+    size_t mapSize; // Mapped size, not including guard pages and meta-data.
+};
+
+static ALWAYS_INLINE PartitionDirectMapExtent* partitionPageToDirectMapExtent(PartitionPage* page)
+{
+    ASSERT(partitionBucketIsDirectMapped(page->bucket));
+    return reinterpret_cast<PartitionDirectMapExtent*>(reinterpret_cast<char*>(page) + 2 * kPageMetadataSize);
+}
+
 static ALWAYS_INLINE void* partitionDirectMap(PartitionRootBase* root, int flags, size_t size)
 {
     size = partitionDirectMapSize(size);
@@ -581,17 +592,21 @@ static ALWAYS_INLINE void* partitionDirectMap(PartitionRootBase* root, int flags
     bucket->numSystemPagesPerSlotSpan = 0;
     bucket->numFullPages = 0;
 
+    PartitionDirectMapExtent* mapExtent = partitionPageToDirectMapExtent(page);
+    mapExtent->mapSize = mapSize - kPartitionPageSize - kSystemPageSize;
+
     return ret;
 }
 
 static ALWAYS_INLINE void partitionDirectUnmap(PartitionPage* page)
 {
-    size_t unmapSize = page->bucket->slotSize;
+    size_t unmapSize = partitionPageToDirectMapExtent(page)->mapSize;
+
     // Add on the size of the trailing guard page and preceeding partition
-    // page, then round up to allocation granularity.
+    // page.
     unmapSize += kPartitionPageSize + kSystemPageSize;
-    unmapSize += kPageAllocationGranularityOffsetMask;
-    unmapSize &= kPageAllocationGranularityBaseMask;
+
+    ASSERT(!(unmapSize & kPageAllocationGranularityOffsetMask));
 
     char* ptr = reinterpret_cast<char*>(partitionPageToPointer(page));
     // Account for the mapping starting a partition page before the actual
@@ -763,6 +778,56 @@ void partitionFreeSlowPath(PartitionPage* page)
     }
 }
 
+bool partitionReallocDirectMappedInPlace(PartitionRootGeneric* root, void* ptr, size_t newSize)
+{
+    newSize = partitionCookieSizeAdjustAdd(newSize);
+
+    // Note that the new size might be a bucketed size; this function is called
+    // whenever we're reallocating a direct mapped allocation.
+    newSize = partitionDirectMapSize(newSize);
+    if (newSize < kGenericMinDirectMapped)
+        return false;
+
+    PartitionPage* page = partitionPointerToPage(ptr);

[Chris Evans, 2014/03/06 05:55:13]

Pass in page instead of ptr to this function to avoid this small piece of work.

+    PartitionBucket* bucket = page->bucket;
+
+    ASSERT(partitionBucketIsDirectMapped(bucket));
+
+    // bucket->slotSize is the current size of the allocation.
+    if (newSize == bucket->slotSize)

[Chris Evans, 2014/03/06 05:55:13]

This check looks duplicated with a check in the parent function. Can we remove
this check by re-ordering the parent function slightly? We could even ASSERT
that this is then _not_ equal.

[Jens Widell, 2014/03/06 06:03:04]

If you mean the actualOldSize/actualNewSize comparison in the parent function, I
actually reordered and introduced this check in the latest version of the patch.
The values in the comparison in the parent function are rounded to bucket sizes
(when below gGenericMaxBucketed) which doesn't really make sense if we're
downsizing a direct mapping.

+        return true;
+
+    char* charPtr = static_cast<char*>(ptr);
+
+    if (newSize < bucket->slotSize) {
+        // Shrink by decommitting unneeded pages and making them inaccessible.
+        size_t decommitSize = bucket->slotSize - newSize;
+        decommitSystemPages(charPtr + newSize, decommitSize);
+        setSystemPagesInaccessible(charPtr + newSize, decommitSize);
+    } else if (newSize <= partitionPageToDirectMapExtent(page)->mapSize) {
+        // Grow within the actually allocated memory. Just need to make the
+        // pages accessible again.
+        size_t recommitSize = newSize - bucket->slotSize;
+        setSystemPagesAccessible(charPtr + bucket->slotSize, recommitSize);
+
+#ifndef NDEBUG
+        memset(charPtr + bucket->slotSize, kUninitializedByte, recommitSize);
+#endif
+    } else {
+        // We can't perform the realloc in-place.
+        // TODO: support this too when possible.
+        return false;
+    }
+
+#ifndef NDEBUG
+    // Write a new trailing cookie.
+    partitionCookieWriteValue(charPtr + newSize - kCookieSize);
+#endif
+
+    bucket->slotSize = newSize;
+    return true;
+}
+
 void* partitionReallocGeneric(PartitionRootGeneric* root, void* ptr, size_t newSize)
 {
 #if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
@@ -775,8 +840,21 @@ void* partitionReallocGeneric(PartitionRootGeneric* root, void* ptr, size_t newS
         return 0;
     }
 
+    RELEASE_ASSERT(newSize <= INT_MAX - kSystemPageSize);

[Chris Evans, 2014/03/06 05:55:13]

The RHS is just kGenericMaxDirectMapped, I think.

+
     ASSERT(partitionPointerIsValid(partitionCookieFreePointerAdjust(ptr)));
 
+    void* realPtr = partitionCookieFreePointerAdjust(ptr);
+    PartitionPage* page = partitionPointerToPage(realPtr);
+
+    if (partitionBucketIsDirectMapped(page->bucket)) {

[Chris Evans, 2014/03/06 05:55:13]

Might as well wrap that condition in an UNLIKELY ?

+        // We may be able to perform the realloc in place by changing the
+        // accessibility of memory pages and, if reducing the size, decommitting
+        // them.
+        if (partitionReallocDirectMappedInPlace(root, realPtr, newSize))

[Chris Evans, 2014/03/06 05:55:13]

Pass in the page and not the pointer to do a little less work.

+            return ptr;
+    }
+
     size_t actualNewSize = partitionAllocActualSize(root, newSize);
     size_t actualOldSize = partitionAllocGetSize(ptr);
 
@@ -787,8 +865,6 @@ void* partitionReallocGeneric(PartitionRootGeneric* root, void* ptr, size_t newS
         // Trying to allocate a block of size newSize would give us a block of
         // the same size as the one we've already got, so no point in doing
         // anything here.
-        // TODO: for an upsize or downsize on a direct mapped allocation, we
-        // should really try and resize it in-place.
         return ptr;
     }