PartitionAlloc: fixes and improvements to large-chunk size tracking.

Source/wtf/PartitionAlloc.h

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 200 matching lines @@
 // value (probably it is a "out of virtual address space" crash),
 // a special crash stack trace is generated at |partitionOutOfMemory|.
 // This is to distinguish "out of virtual address space" from
 // "out of physical memory" in crash reports.
 static const size_t kReasonableSizeOfUnusedPages = 1024 * 1024 * 1024; // 1GiB
 
 #if ENABLE(ASSERT)
 // These two byte values match tcmalloc.
 static const unsigned char kUninitializedByte = 0xAB;
 static const unsigned char kFreedByte = 0xCD;
-static const uint32_t kCookieValue = 0xDEADBEEFu;
 static const size_t kCookieSize = 16; // Handles alignment up to XMM instructions on Intel.
+static const unsigned char kCookieValue[kCookieSize] = { 0xDE, 0xAD, 0xBE, 0xEF, 0xCA, 0xFE, 0xD0, 0x0D, 0x13, 0x37, 0xF0, 0x05, 0xBA, 0x11, 0xAB, 0x1E };
 #endif
 
 struct PartitionBucket;
 struct PartitionRootBase;
 
 struct PartitionFreelistEntry {
     PartitionFreelistEntry* next;
 };
 
 // Some notes on page states. A page can be in one of four major states:
@@ skipping 197 matching lines @@
 #if ENABLE(ASSERT)
     // The value given to the application is actually just after the cookie.
     ptr = static_cast<char*>(ptr) - kCookieSize;
 #endif
     return ptr;
 }
 
 ALWAYS_INLINE void partitionCookieWriteValue(void* ptr)
 {
 #if ENABLE(ASSERT)
-    uint32_t* cookiePtr = reinterpret_cast<uint32_t*>(ptr);
-    for (size_t i = 0; i < kCookieSize / sizeof(kCookieValue); ++i, ++cookiePtr)
-        *cookiePtr = kCookieValue;
+    unsigned char* cookiePtr = reinterpret_cast<unsigned char*>(ptr);
+    for (size_t i = 0; i < kCookieSize; ++i, ++cookiePtr)
+        *cookiePtr = kCookieValue[i];
 #endif
 }
 
 ALWAYS_INLINE void partitionCookieCheckValue(void* ptr)
 {
 #if ENABLE(ASSERT)
-    uint32_t* cookiePtr = reinterpret_cast<uint32_t*>(ptr);
-    for (size_t i = 0; i < kCookieSize / sizeof(kCookieValue); ++i, ++cookiePtr)
-        ASSERT(*cookiePtr == kCookieValue);
+    unsigned char* cookiePtr = reinterpret_cast<unsigned char*>(ptr);
+    for (size_t i = 0; i < kCookieSize; ++i, ++cookiePtr)
+        ASSERT(*cookiePtr == kCookieValue[i]);
 #endif
 }
 
 ALWAYS_INLINE char* partitionSuperPageToMetadataArea(char* ptr)
 {
     uintptr_t pointerAsUint = reinterpret_cast<uintptr_t>(ptr);
     ASSERT(!(pointerAsUint & kSuperPageOffsetMask));
     // The metadata area is exactly one system page (the guard page) into the
     // super page.
     return reinterpret_cast<char*>(pointerAsUint + kSystemPageSize);
@@ skipping 31 matching lines @@
 }
 
 ALWAYS_INLINE PartitionPage* partitionPointerToPage(void* ptr)
 {
     PartitionPage* page = partitionPointerToPageNoAlignmentCheck(ptr);
     // Checks that the pointer is a multiple of bucket size.
     ASSERT(!((reinterpret_cast<uintptr_t>(ptr) - reinterpret_cast<uintptr_t>(partitionPageToPointer(page))) % page->bucket->slotSize));
     return page;
 }
 
+ALWAYS_INLINE bool partitionBucketIsDirectMapped(const PartitionBucket* bucket)
+{
+    return !bucket->numSystemPagesPerSlotSpan;
+}
+
+ALWAYS_INLINE size_t partitionBucketBytes(const PartitionBucket* bucket)
+{
+    return bucket->numSystemPagesPerSlotSpan * kSystemPageSize;
+}
+
+ALWAYS_INLINE uint16_t partitionBucketSlots(const PartitionBucket* bucket)
+{
+    return static_cast<uint16_t>(partitionBucketBytes(bucket) / bucket->slotSize);
+}
+
+ALWAYS_INLINE size_t* partitionPageGetRawSizePtr(PartitionPage* page)
+{
+    // For single-slot buckets which span more than one partition page, we
+    // have some spare metadata space to store the raw allocation size. We
+    // can use this to report better statistics.
+    PartitionBucket* bucket = page->bucket;
+    if (bucket->slotSize <= kMaxSystemPagesPerSlotSpan * kSystemPageSize)
+        return nullptr;
+
+    ASSERT(partitionBucketIsDirectMapped(bucket) || partitionBucketSlots(bucket) == 1);
+    page++;
+    return reinterpret_cast<size_t*>(&page->freelistHead);
+}
+
+ALWAYS_INLINE size_t partitionPageGetRawSize(PartitionPage* page)
+{
+    size_t* rawSizePtr = partitionPageGetRawSizePtr(page);
+    if (UNLIKELY(rawSizePtr != nullptr))
+        return *rawSizePtr;
+    return 0;
+}
+
 ALWAYS_INLINE PartitionRootBase* partitionPageToRoot(PartitionPage* page)
 {
     PartitionSuperPageExtentEntry* extentEntry = reinterpret_cast<PartitionSuperPageExtentEntry*>(reinterpret_cast<uintptr_t>(page) & kSystemPageBaseMask);
     return extentEntry->root;
 }
 
 ALWAYS_INLINE bool partitionPointerIsValid(void* ptr)
 {
     PartitionPage* page = partitionPointerToPage(ptr);
     PartitionRootBase* root = partitionPageToRoot(page);
     return root->invertedSelf == ~reinterpret_cast<uintptr_t>(root);
 }
 
 ALWAYS_INLINE void* partitionBucketAlloc(PartitionRootBase* root, int flags, size_t size, PartitionBucket* bucket)
 {
     PartitionPage* page = bucket->activePagesHead;
     // Check that this page is neither full nor freed.
     ASSERT(page->numAllocatedSlots >= 0);
     void* ret = page->freelistHead;
     if (LIKELY(ret != 0)) {
         // If these asserts fire, you probably corrupted memory.
         ASSERT(partitionPointerIsValid(ret));
+        // All large allocations must go through the slow path to correctly
+        // update the size metadata.
+        ASSERT(partitionPageGetRawSize(page) == 0);
         PartitionFreelistEntry* newHead = partitionFreelistMask(static_cast<PartitionFreelistEntry*>(ret)->next);
         page->freelistHead = newHead;
         page->numAllocatedSlots++;
     } else {
         ret = partitionAllocSlowPath(root, flags, size, bucket);
         ASSERT(!ret || partitionPointerIsValid(ret));
     }
 #if ENABLE(ASSERT)
     if (!ret)
         return 0;
-    // Fill the uninitialized pattern. and write the cookies.
+    // Fill the uninitialized pattern, and write the cookies.
     page = partitionPointerToPage(ret);
-    size_t bucketSize = page->bucket->slotSize;
-    memset(ret, kUninitializedByte, bucketSize);
-    partitionCookieWriteValue(ret);
-    partitionCookieWriteValue(reinterpret_cast<char*>(ret) + bucketSize - kCookieSize);
+    size_t slotSize = page->bucket->slotSize;
+    size_t rawSize = partitionPageGetRawSize(page);
+    if (rawSize) {
+        ASSERT(rawSize == size);
+        slotSize = rawSize;
+    }
+    size_t noCookieSize = partitionCookieSizeAdjustSubtract(slotSize);
+    char* charRet = static_cast<char*>(ret);
     // The value given to the application is actually just after the cookie.
-    ret = static_cast<char*>(ret) + kCookieSize;
+    ret = charRet + kCookieSize;
+    memset(ret, kUninitializedByte, noCookieSize);
+    partitionCookieWriteValue(charRet);
+    partitionCookieWriteValue(charRet + kCookieSize + noCookieSize);
 #endif
     return ret;
 }
 
 ALWAYS_INLINE void* partitionAlloc(PartitionRoot* root, size_t size)
 {
 #if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
     void* result = malloc(size);
     RELEASE_ASSERT(result);
     return result;
 #else
     size = partitionCookieSizeAdjustAdd(size);
     ASSERT(root->initialized);
     size_t index = size >> kBucketShift;
     ASSERT(index < root->numBuckets);
     ASSERT(size == index << kBucketShift);
     PartitionBucket* bucket = &root->buckets()[index];
     return partitionBucketAlloc(root, 0, size, bucket);
 #endif // defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
 }
 
 ALWAYS_INLINE void partitionFreeWithPage(void* ptr, PartitionPage* page)
 {
     // If these asserts fire, you probably corrupted memory.
 #if ENABLE(ASSERT)
-    size_t bucketSize = page->bucket->slotSize;
+    size_t slotSize = page->bucket->slotSize;
+    size_t rawSize = partitionPageGetRawSize(page);
+    if (rawSize)
+        slotSize = rawSize;
     partitionCookieCheckValue(ptr);
-    partitionCookieCheckValue(reinterpret_cast<char*>(ptr) + bucketSize - kCookieSize);
-    memset(ptr, kFreedByte, bucketSize);
+    partitionCookieCheckValue(reinterpret_cast<char*>(ptr) + slotSize - kCookieSize);
+    memset(ptr, kFreedByte, slotSize);
 #endif
     ASSERT(page->numAllocatedSlots);
     PartitionFreelistEntry* freelistHead = page->freelistHead;
     ASSERT(!freelistHead || partitionPointerIsValid(freelistHead));
     RELEASE_ASSERT_WITH_SECURITY_IMPLICATION(ptr != freelistHead); // Catches an immediate double free.
     ASSERT_WITH_SECURITY_IMPLICATION(!freelistHead || ptr != partitionFreelistMask(freelistHead->next)); // Look for double free one level deeper in debug.
     PartitionFreelistEntry* entry = static_cast<PartitionFreelistEntry*>(ptr);
     entry->next = partitionFreelistMask(freelistHead);
     page->freelistHead = entry;
     --page->numAllocatedSlots;
@@ skipping 60 matching lines @@
 
     ptr = partitionCookieFreePointerAdjust(ptr);
     ASSERT(partitionPointerIsValid(ptr));
     PartitionPage* page = partitionPointerToPage(ptr);
     spinLockLock(&root->lock);
     partitionFreeWithPage(ptr, page);
     spinLockUnlock(&root->lock);
 #endif
 }
 
-ALWAYS_INLINE bool partitionBucketIsDirectMapped(const PartitionBucket* bucket)
-{
-    return !bucket->numSystemPagesPerSlotSpan;
-}
-
 ALWAYS_INLINE size_t partitionDirectMapSize(size_t size)
 {
     // Caller must check that the size is not above the kGenericMaxDirectMapped
     // limit before calling. This also guards against integer overflow in the
     // calculation here.
     ASSERT(size <= kGenericMaxDirectMapped);
     return (size + kSystemPageOffsetMask) & kSystemPageBaseMask;
 }
 
 ALWAYS_INLINE size_t partitionAllocActualSize(PartitionRootGeneric* root, size_t size)
@@ skipping 72 matching lines @@
 using WTF::partitionAlloc;
 using WTF::partitionFree;
 using WTF::partitionAllocGeneric;
 using WTF::partitionFreeGeneric;
 using WTF::partitionReallocGeneric;
 using WTF::partitionAllocActualSize;
 using WTF::partitionAllocSupportsGetSize;
 using WTF::partitionAllocGetSize;
 
 #endif // WTF_PartitionAlloc_h