For large (but not direct mapped) allocations, track the real allocation size.

Source/wtf/PartitionAlloc.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 472 matching lines @@
 
     uint16_t numPartitionPages = partitionBucketPartitionPages(bucket);
     char* pageCharPtr = reinterpret_cast<char*>(page);
     for (uint16_t i = 1; i < numPartitionPages; ++i) {
         pageCharPtr += kPageMetadataSize;
         PartitionPage* secondaryPage = reinterpret_cast<PartitionPage*>(pageCharPtr);
         secondaryPage->pageOffset = i;
     }
 }
 
+static ALWAYS_INLINE size_t partitionRoundUpToSystemPage(size_t size)
+{
+    return (size + kSystemPageOffsetMask) & kSystemPageBaseMask;
+}
+
 static ALWAYS_INLINE char* partitionPageAllocAndFillFreelist(PartitionPage* page)
 {
     ASSERT(page != &PartitionRootGeneric::gSeedPage);
     uint16_t numSlots = page->numUnprovisionedSlots;
     ASSERT(numSlots);
     PartitionBucket* bucket = page->bucket;
     // We should only get here when _every_ slot is either used or unprovisioned.
     // (The third state is "on the freelist". If we have a non-empty freelist, we should not get here.)
     ASSERT(numSlots + page->numAllocatedSlots == partitionBucketSlots(bucket));
     // Similarly, make explicitly sure that the freelist is empty.
     ASSERT(!page->freelistHead);
     ASSERT(page->numAllocatedSlots >= 0);
 
     size_t size = bucket->slotSize;
     char* base = reinterpret_cast<char*>(partitionPageToPointer(page));
     char* returnObject = base + (size * page->numAllocatedSlots);
     char* firstFreelistPointer = returnObject + size;
     char* firstFreelistPointerExtent = firstFreelistPointer + sizeof(PartitionFreelistEntry*);
     // Our goal is to fault as few system pages as possible. We calculate the
     // page containing the "end" of the returned slot, and then allow freelist
     // pointers to be written up to the end of that page.
-    char* subPageLimit = reinterpret_cast<char*>((reinterpret_cast<uintptr_t>(firstFreelistPointer) + kSystemPageOffsetMask) & kSystemPageBaseMask);
+    char* subPageLimit = reinterpret_cast<char*>(partitionRoundUpToSystemPage(reinterpret_cast<size_t>(firstFreelistPointer)));
     char* slotsLimit = returnObject + (size * numSlots);
     char* freelistLimit = subPageLimit;
     if (UNLIKELY(slotsLimit < freelistLimit))
         freelistLimit = slotsLimit;
 
     uint16_t numNewFreelistEntries = 0;
     if (LIKELY(firstFreelistPointerExtent <= freelistLimit)) {
         // Only consider used space in the slot span. If we consider wasted
         // space, we may get an off-by-one when a freelist pointer fits in the
         // wasted space, but a slot does not.
@@ skipping 68 matching lines @@
             bucket->emptyPagesHead = page;
         } else {
             // If we get here, we found a full page. Skip over it too, and also
             // tag it as full (via a negative value). We need it tagged so that
             // free'ing can tell, and move it back into the active page list.
             ASSERT(page->numAllocatedSlots == partitionBucketSlots(bucket));
             page->numAllocatedSlots = -page->numAllocatedSlots;
             ++bucket->numFullPages;
             // numFullPages is a uint16_t for efficient packing so guard against
             // overflow to be safe.
-            if (!bucket->numFullPages)
+            if (UNLIKELY(!bucket->numFullPages))
                 partitionBucketFull();
             // Not necessary but might help stop accidents.
             page->nextPage = 0;
         }
     }
 
     bucket->activePagesHead = 0;
     return false;
 }
 
@@ skipping 107 matching lines @@
     ASSERT(!(unmapSize & kPageAllocationGranularityOffsetMask));
 
     char* ptr = reinterpret_cast<char*>(partitionPageToPointer(page));
     // Account for the mapping starting a partition page before the actual
     // allocation address.
     ptr -= kPartitionPageSize;
 
     freePages(ptr, unmapSize);
 }
 
+static ALWAYS_INLINE size_t* partitionPageGetRawSizePtr(PartitionPage* page)
+{
+    // For single-slot buckets which span more than one partition page, we
+    // have some spare metadata space to store the raw allocation size. We
+    // can use this to report better statistics.
+    PartitionBucket* bucket = page->bucket;
+    if (bucket->slotSize <= kMaxSystemPagesPerSlotSpan * kSystemPageSize)
+        return nullptr;
+
+    ASSERT(partitionBucketSlots(bucket) == 1);
+    page++;
+    return reinterpret_cast<size_t*>(&page->freelistHead);
+}
+
+static ALWAYS_INLINE void partitionPageSetRawSize(PartitionPage* page, size_t size)
+{
+    size_t* rawSizePtr = partitionPageGetRawSizePtr(page);
+    if (UNLIKELY(rawSizePtr != nullptr))
+        *rawSizePtr = size;
+}
+
 void* partitionAllocSlowPath(PartitionRootBase* root, int flags, size_t size, PartitionBucket* bucket)
 {
     // The slow path is called when the freelist is empty.
     ASSERT(!bucket->activePagesHead->freelistHead);
 
     PartitionPage* newPage = nullptr;
 
     // For the partitionAllocGeneric API, we have a bunch of buckets marked
     // as special cases. We bounce them through to the slow path so that we
     // can still have a blazing fast hot path due to lack of corner-case
     // branches.
     bool returnNull = flags & PartitionAllocReturnNull;
     if (UNLIKELY(partitionBucketIsDirectMapped(bucket))) {
         ASSERT(size > kGenericMaxBucketed);
         ASSERT(bucket == &PartitionRootBase::gPagedBucket);
         if (size > kGenericMaxDirectMapped) {
             if (returnNull)
                 return 0;
             partitionExcessiveAllocationSize();
         }
-        void* ptr = partitionDirectMap(root, flags, size);
-        if (ptr)
-            return ptr;
+        void* ret = partitionDirectMap(root, flags, size);
+        if (ret)
+            return ret;
         goto partitionAllocSlowPathFailed;
     }
 
-    // First, look for a usable page in the existing active pages list.
-    // Change active page, accepting the current page as a candidate.
     if (LIKELY(partitionSetNewActivePage(bucket->activePagesHead))) {
+        // First, look for a usable page in the existing active pages list.
+        // Change active page, accepting the current page as a candidate.
         newPage = bucket->activePagesHead;
         if (LIKELY(newPage->freelistHead != 0)) {
-            PartitionFreelistEntry* ret = newPage->freelistHead;
-            newPage->freelistHead = partitionFreelistMask(ret->next);
+            PartitionFreelistEntry* entry = newPage->freelistHead;
+            newPage->freelistHead = partitionFreelistMask(entry->next);
             newPage->numAllocatedSlots++;
-            return ret;
+            partitionPageSetRawSize(newPage, size);
+            return entry;
         }
-        ASSERT(newPage->numUnprovisionedSlots);
-        return partitionPageAllocAndFillFreelist(newPage);
-    }
-
-    // Second, look in our list of freed but reserved pages.
-    newPage = bucket->emptyPagesHead;
-    if (LIKELY(newPage != 0)) {
+    } else if (LIKELY(bucket->emptyPagesHead != nullptr)) {
+        // Second, look in our list of freed but reserved pages.
+        newPage = bucket->emptyPagesHead;
         bucket->emptyPagesHead = newPage->nextPage;
         void* addr = partitionPageToPointer(newPage);
         partitionRecommitSystemPages(root, addr, partitionBucketBytes(newPage->bucket));
         partitionPageReset(newPage, bucket);
     } else {
         // Third. If we get here, we need a brand new page.
         uint16_t numPartitionPages = partitionBucketPartitionPages(bucket);
         void* rawNewPage = partitionAllocPartitionPages(root, flags, numPartitionPages);
         if (UNLIKELY(!rawNewPage))
             goto partitionAllocSlowPathFailed;
         // Skip the alignment check because it depends on page->bucket, which is not yet set.
         newPage = partitionPointerToPageNoAlignmentCheck(rawNewPage);
         partitionPageSetup(newPage, bucket);
     }
 
     bucket->activePagesHead = newPage;
+    partitionPageSetRawSize(newPage, size);
+
     return partitionPageAllocAndFillFreelist(newPage);
 
 partitionAllocSlowPathFailed:
     if (returnNull) {
         // If we get here, we will set the active page to null, which is an
         // invalid state. To support continued use of this bucket, we need to
         // restore a valid state, by setting the active page to the seed page.
         bucket->activePagesHead = &PartitionRootGeneric::gSeedPage;
         return nullptr;
     }
@@ skipping 238 matching lines @@
 }
 
 static void partitionDumpPageStats(PartitionBucketMemoryStats* statsOut, const PartitionPage* page)
 {
     uint16_t bucketNumSlots = partitionBucketSlots(page->bucket);
 
     if (!page->freelistHead && page->numAllocatedSlots == 0) {
         ASSERT(!page->numUnprovisionedSlots);
         ++statsOut->numDecommittedPages;
     } else {
-        statsOut->activeBytes += (page->numAllocatedSlots * statsOut->bucketSlotSize);
+        size_t* rawSizePtr = partitionPageGetRawSizePtr(const_cast<PartitionPage*>(page));

[haraken, 2015/06/10 00:54:01]

Nit: It looks a bit tidier to introduce partitionPageGetRawSize(page) (which
returns 0 if the raw size is not available).

+        if (rawSizePtr)
+            statsOut->activeBytes += static_cast<uint32_t>(partitionRoundUpToSystemPage(*rawSizePtr));
+        else
+            statsOut->activeBytes += (page->numAllocatedSlots * statsOut->bucketSlotSize);
         size_t pageBytesResident = (bucketNumSlots - page->numUnprovisionedSlots) * statsOut->bucketSlotSize;
         // Round up to system page size.
-        size_t pageBytesResidentRounded = (pageBytesResident + kSystemPageOffsetMask) & kSystemPageBaseMask;
+        size_t pageBytesResidentRounded = partitionRoundUpToSystemPage(pageBytesResident);
         statsOut->residentBytes += pageBytesResidentRounded;
         if (!page->numAllocatedSlots) {
             statsOut->freeableBytes += pageBytesResidentRounded;
             ++statsOut->numEmptyPages;
         } else if (page->numAllocatedSlots == bucketNumSlots) {
             ++statsOut->numFullPages;
         } else {
             ++statsOut->numActivePages;
         }
     }
@@ skipping 103 matching lines @@
     // partitionsDumpBucketStats is called after collecting stats because it
     // can use PartitionAlloc to allocate and this can affect the statistics.
     for (size_t i = 0; i < partitionNumBuckets; ++i) {
         if (memoryStats[i].isValid)
             partitionStatsDumper->partitionsDumpBucketStats(partitionName, &memoryStats[i]);
     }
 }
 
 } // namespace WTF