PartitionAlloc: support in-place resize of directly mapped allocation

Source/wtf/PartitionAlloc.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 558 matching lines @@
     PartitionBucket* bucket = reinterpret_cast<PartitionBucket*>(reinterpret_cast<char*>(page) + kPageMetadataSize);
     page->freelistHead = 0;
     page->nextPage = 0;
     page->bucket = bucket;
     page->numAllocatedSlots = 1;
     page->numUnprovisionedSlots = 0;
     page->pageOffset = 0;
     page->freeCacheIndex = 0;
 
     bucket->activePagesHead = 0;
-    bucket->freePagesHead = 0;
+    bucket->freePagesHead = reinterpret_cast<PartitionPage*>(ret + size);
     bucket->slotSize = size;
     bucket->numSystemPagesPerSlotSpan = 0;
     bucket->numFullPages = 0;
 
     return ret;
 }
 
 static ALWAYS_INLINE void partitionDirectUnmap(PartitionPage* page)
 {
     size_t unmapSize = page->bucket->slotSize;

[Jens Widell, 2014/02/27 17:02:56]

This would need adjusting to use page->bucket->freePagesHead instead;
page->bucket->slotSize is the size of current allocation and doesn't necessarily
represent what we actually mapped and should unmap.

     // Add on the size of the trailing guard page and preceeding partition
     // page, then round up to allocation granularity.
     unmapSize += kPartitionPageSize + kSystemPageSize;
     unmapSize += kPageAllocationGranularityOffsetMask;
     unmapSize &= kPageAllocationGranularityBaseMask;
 
     char* ptr = reinterpret_cast<char*>(partitionPageToPointer(page));
     // Account for the mapping starting a partition page before the actual
     // allocation address.
     ptr -= kPartitionPageSize;
@@ skipping 156 matching lines @@
         page->nextPage = bucket->activePagesHead;
         bucket->activePagesHead = page;
         --bucket->numFullPages;
         // Special case: for a partition page with just a single slot, it may
         // now be empty and we want to run it through the empty logic.
         if (UNLIKELY(page->numAllocatedSlots == 0))
             partitionFreeSlowPath(page);
     }
 }
 
+bool partitionReallocDirectMappedInPlace(PartitionRootGeneric* root, void* ptr, size_t newSize)
+{
+    newSize = partitionCookieSizeAdjustAdd(newSize);
+    ASSERT(partitionDirectMapSize(newSize) == newSize);
+
+    char* realPtr = static_cast<char*>(partitionCookieFreePointerAdjust(ptr));
+    PartitionBucket* bucket = partitionPointerToPage(realPtr)->bucket;
+
+    // bucket->slotSize is the current size of the allocation.
+    // bucket->freePagesHead points to the first system page beyond what we
+    // first allocated; we can grow the allocation up to that point.
+
+    ASSERT(partitionBucketIsDirectMapped(bucket));
+
+    if (newSize < bucket->slotSize) {
+        // Shrink by decommitting unneeded pages and making them inaccessible.
+        // TODO: maybe refuse if the new size is too much smaller?
+        decommitSystemPages(realPtr + newSize, bucket->slotSize - newSize);
+        setSystemPagesInaccessible(realPtr + newSize, kSystemPageSize);

[Jens Widell, 2014/02/27 17:10:53]

Ought to make all decommitted pages inaccessible (as the comment states,) not
just one of them.

+    } else if (realPtr + newSize <= reinterpret_cast<char*>(bucket->freePagesHead)) {
+        // Grow within the actually allocated memory. Just need to make the
+        // pages accessible again.
+        setSystemPagesAccessible(realPtr + bucket->slotSize, newSize - bucket->slotSize);
+
+#ifndef NDEBUG
+        memset(realPtr + bucket->slotSize, kUninitializedByte, newSize - bucket->slotSize);
+#endif
+    } else {
+        // We can't perform the realloc in-place.
+        return false;
+    }
+
+#ifndef NDEBUG
+    // Write a new trailing cookie.
+    partitionCookieWriteValue(realPtr + newSize - kCookieSize);
+#endif
+
+    bucket->slotSize = newSize;
+    return true;
+}
+
 void* partitionReallocGeneric(PartitionRootGeneric* root, void* ptr, size_t newSize)
 {
 #if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
     return realloc(ptr, newSize);
 #else
     if (UNLIKELY(!ptr))
         return partitionAllocGeneric(root, newSize);
     if (UNLIKELY(!newSize)) {
         partitionFreeGeneric(root, ptr);
         return 0;
     }
 
     ASSERT(partitionPointerIsValid(partitionCookieFreePointerAdjust(ptr)));
 
     size_t actualNewSize = partitionAllocActualSize(root, newSize);
     size_t actualOldSize = partitionAllocGetSize(ptr);
 
     // TODO: note that tcmalloc will "ignore" a downsizing realloc() unless the
     // new size is a significant percentage smaller. We could do the same if we
     // determine it is a win.
     if (actualNewSize == actualOldSize) {
         // Trying to allocate a block of size newSize would give us a block of
         // the same size as the one we've already got, so no point in doing
         // anything here.
-        // TODO: for an upsize or downsize on a direct mapped allocation, we
-        // should really try and resize it in-place.
         return ptr;
     }
 
+    if (partitionSizeIsDirectMapped(actualOldSize) && partitionSizeIsDirectMapped(actualNewSize)) {
+        // We may be able to perform the realloc in place by changing the
+        // accessibility of memory pages and, if reducing the size, decommitting
+        // them.
+        if (partitionReallocDirectMappedInPlace(root, ptr, actualNewSize))
+            return ptr;
+    }
+
     // This realloc cannot be resized in-place. Sadness.
     void* ret = partitionAllocGeneric(root, newSize);
     size_t copySize = actualOldSize;
     if (newSize < copySize)
         copySize = newSize;
 
     memcpy(ret, ptr, copySize);
     partitionFreeGeneric(root, ptr);
     return ret;
 #endif
@@ skipping 50 matching lines @@
     printf("total live: %zu bytes\n", totalLive);
     printf("total resident: %zu bytes\n", totalResident);
     printf("total freeable: %zu bytes\n", totalFreeable);
     fflush(stdout);
 }
 
 #endif // !NDEBUG
 
 } // namespace WTF