Reapply the non-MathExtras parts of

Source/wtf/PartitionAlloc.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 42 matching lines @@
 COMPILE_ASSERT(WTF::kGenericSmallestBucket == 8, generic_smallest_bucket);
 COMPILE_ASSERT(WTF::kGenericMaxBucketed == 983040, generic_max_bucketed);
 
 namespace WTF {
 
 int PartitionRootBase::gInitializedLock = 0;
 bool PartitionRootBase::gInitialized = false;
 PartitionPage PartitionRootBase::gSeedPage;
 PartitionBucket PartitionRootBase::gPagedBucket;
 
-static size_t partitionBucketNumSystemPages(size_t size)
+static uint16_t partitionBucketNumSystemPages(size_t size)
 {
     // This works out reasonably for the current bucket sizes of the generic
     // allocator, and the current values of partition page size and constants.
     // Specifically, we have enough room to always pack the slots perfectly into
     // some number of system pages. The only waste is the waste associated with
     // unfaulted pages (i.e. wasted address space).
     // TODO: we end up using a lot of system pages for very small sizes. For
     // example, we'll use 12 system pages for slot size 24. The slot size is
     // so small that the waste would be tiny with just 4, or 1, system pages.
     // Later, we can investigate whether there are anti-fragmentation benefits
     // to using fewer system pages.
     double bestWasteRatio = 1.0f;
-    size_t bestPages = 0;
+    uint16_t bestPages = 0;
     if (size > kMaxSystemPagesPerSlotSpan * kSystemPageSize) {
         ASSERT(!(size % kSystemPageSize));
-        return size / kSystemPageSize;
+        return static_cast<uint16_t>(size / kSystemPageSize);
     }
     ASSERT(size <= kMaxSystemPagesPerSlotSpan * kSystemPageSize);
-    for (size_t i = kNumSystemPagesPerPartitionPage - 1; i <= kMaxSystemPagesPerSlotSpan; ++i) {
+    for (uint16_t i = kNumSystemPagesPerPartitionPage - 1; i <= kMaxSystemPagesPerSlotSpan; ++i) {
         size_t pageSize = kSystemPageSize * i;
         size_t numSlots = pageSize / size;
         size_t waste = pageSize - (numSlots * size);
         // Leaving a page unfaulted is not free; the page will occupy an empty page table entry.
         // Make a simple attempt to account for that.
         size_t numRemainderPages = i & (kNumSystemPagesPerPartitionPage - 1);
         size_t numUnfaultedPages = numRemainderPages ? (kNumSystemPagesPerPartitionPage - numRemainderPages) : 0;
         waste += sizeof(void*) * numUnfaultedPages;
         double wasteRatio = (double) waste / (double) pageSize;
         if (wasteRatio < bestWasteRatio) {
@@ skipping 222 matching lines @@
     ASSERT(root->totalSizeOfCommittedPages > len);
     root->totalSizeOfCommittedPages -= len;
 }
 
 static ALWAYS_INLINE void partitionRecommitSystemPages(PartitionRootBase* root, void* addr, size_t len)
 {
     recommitSystemPages(addr, len);
     root->totalSizeOfCommittedPages += len;
 }
 
-static ALWAYS_INLINE void* partitionAllocPartitionPages(PartitionRootBase* root, int flags, size_t numPartitionPages)
+static ALWAYS_INLINE void* partitionAllocPartitionPages(PartitionRootBase* root, int flags, uint16_t numPartitionPages)
 {
     ASSERT(!(reinterpret_cast<uintptr_t>(root->nextPartitionPage) % kPartitionPageSize));
     ASSERT(!(reinterpret_cast<uintptr_t>(root->nextPartitionPageEnd) % kPartitionPageSize));
     RELEASE_ASSERT(numPartitionPages <= kNumPartitionPagesPerSuperPage);
     size_t totalSize = kPartitionPageSize * numPartitionPages;
     root->totalSizeOfCommittedPages += totalSize;
     size_t numPartitionPagesLeft = (root->nextPartitionPageEnd - root->nextPartitionPage) >> kPartitionPageShift;
     if (LIKELY(numPartitionPagesLeft >= numPartitionPages)) {
         // In this case, we can still hand out pages from the current super page
         // allocation.
@@ skipping 61 matching lines @@
     return ret;
 }
 
 static ALWAYS_INLINE void partitionUnusePage(PartitionRootBase* root, PartitionPage* page)
 {
     ASSERT(page->bucket->numSystemPagesPerSlotSpan);
     void* addr = partitionPageToPointer(page);
     partitionDecommitSystemPages(root, addr, page->bucket->numSystemPagesPerSlotSpan * kSystemPageSize);
 }
 
-static ALWAYS_INLINE size_t partitionBucketSlots(const PartitionBucket* bucket)
+static ALWAYS_INLINE uint16_t partitionBucketSlots(const PartitionBucket* bucket)
 {
-    return (bucket->numSystemPagesPerSlotSpan * kSystemPageSize) / bucket->slotSize;
+    return static_cast<uint16_t>((bucket->numSystemPagesPerSlotSpan * kSystemPageSize) / bucket->slotSize);
 }
 
-static ALWAYS_INLINE size_t partitionBucketPartitionPages(const PartitionBucket* bucket)
+static ALWAYS_INLINE uint16_t partitionBucketPartitionPages(const PartitionBucket* bucket)
 {
     return (bucket->numSystemPagesPerSlotSpan + (kNumSystemPagesPerPartitionPage - 1)) / kNumSystemPagesPerPartitionPage;
 }
 
 static ALWAYS_INLINE void partitionPageReset(PartitionPage* page, PartitionBucket* bucket)
 {
     ASSERT(page != &PartitionRootGeneric::gSeedPage);
     page->numAllocatedSlots = 0;
     page->numUnprovisionedSlots = partitionBucketSlots(bucket);
     ASSERT(page->numUnprovisionedSlots);
     page->bucket = bucket;
     page->nextPage = 0;
     // NULLing the freelist is not strictly necessary but it makes an ASSERT in partitionPageFillFreelist simpler.
     page->freelistHead = 0;
     page->pageOffset = 0;
     page->freeCacheIndex = -1;
-    size_t numPartitionPages = partitionBucketPartitionPages(bucket);
-    size_t i;
+    uint16_t numPartitionPages = partitionBucketPartitionPages(bucket);
     char* pageCharPtr = reinterpret_cast<char*>(page);
-    for (i = 1; i < numPartitionPages; ++i) {
+    for (uint16_t i = 1; i < numPartitionPages; ++i) {
         pageCharPtr += kPageMetadataSize;
         PartitionPage* secondaryPage = reinterpret_cast<PartitionPage*>(pageCharPtr);
         secondaryPage->pageOffset = i;
     }
 }
 
 static ALWAYS_INLINE char* partitionPageAllocAndFillFreelist(PartitionPage* page)
 {
     ASSERT(page != &PartitionRootGeneric::gSeedPage);
-    size_t numSlots = page->numUnprovisionedSlots;
+    uint16_t numSlots = page->numUnprovisionedSlots;
     ASSERT(numSlots);
     PartitionBucket* bucket = page->bucket;
     // We should only get here when _every_ slot is either used or unprovisioned.
     // (The third state is "on the freelist". If we have a non-empty freelist, we should not get here.)
     ASSERT(numSlots + page->numAllocatedSlots == partitionBucketSlots(bucket));
     // Similarly, make explicitly sure that the freelist is empty.
     ASSERT(!page->freelistHead);
     ASSERT(page->numAllocatedSlots >= 0);
 
     size_t size = bucket->slotSize;
     char* base = reinterpret_cast<char*>(partitionPageToPointer(page));
     char* returnObject = base + (size * page->numAllocatedSlots);
     char* firstFreelistPointer = returnObject + size;
     char* firstFreelistPointerExtent = firstFreelistPointer + sizeof(PartitionFreelistEntry*);
     // Our goal is to fault as few system pages as possible. We calculate the
     // page containing the "end" of the returned slot, and then allow freelist
     // pointers to be written up to the end of that page.
     char* subPageLimit = reinterpret_cast<char*>((reinterpret_cast<uintptr_t>(firstFreelistPointer) + kSystemPageOffsetMask) & kSystemPageBaseMask);
     char* slotsLimit = returnObject + (size * page->numUnprovisionedSlots);
     char* freelistLimit = subPageLimit;
     if (UNLIKELY(slotsLimit < freelistLimit))
         freelistLimit = slotsLimit;
 
-    size_t numNewFreelistEntries = 0;
+    uint16_t numNewFreelistEntries = 0;
     if (LIKELY(firstFreelistPointerExtent <= freelistLimit)) {
         // Only consider used space in the slot span. If we consider wasted
         // space, we may get an off-by-one when a freelist pointer fits in the
         // wasted space, but a slot does not.
         // We know we can fit at least one freelist pointer.
         numNewFreelistEntries = 1;
         // Any further entries require space for the whole slot span.
-        numNewFreelistEntries += (freelistLimit - firstFreelistPointerExtent) / size;
+        numNewFreelistEntries += static_cast<uint16_t>((freelistLimit - firstFreelistPointerExtent) / size);
     }
 
     // We always return an object slot -- that's the +1 below.
     // We do not neccessarily create any new freelist entries, because we cross sub page boundaries frequently for large bucket sizes.
     ASSERT(numNewFreelistEntries + 1 <= numSlots);
     numSlots -= (numNewFreelistEntries + 1);
     page->numUnprovisionedSlots = numSlots;
     page->numAllocatedSlots++;
 
     if (LIKELY(numNewFreelistEntries)) {
@@ skipping 47 matching lines @@
         ASSERT(page->numAllocatedSlots >= 0);
         if (LIKELY(page->numAllocatedSlots == 0)) {
             ASSERT(page->freeCacheIndex == -1);
             // We hit a free page, so shepherd it on to the free page list.
             page->nextPage = bucket->freePagesHead;
             bucket->freePagesHead = page;
         } else {
             // If we get here, we found a full page. Skip over it too, and also
             // tag it as full (via a negative value). We need it tagged so that
             // free'ing can tell, and move it back into the active page list.
-            ASSERT(page->numAllocatedSlots == static_cast<int>(partitionBucketSlots(bucket)));
+            ASSERT(page->numAllocatedSlots == partitionBucketSlots(bucket));
             page->numAllocatedSlots = -page->numAllocatedSlots;
             ++bucket->numFullPages;
             // numFullPages is a uint16_t for efficient packing so guard against
             // overflow to be safe.
             RELEASE_ASSERT(bucket->numFullPages);
             // Not necessary but might help stop accidents.
             page->nextPage = 0;
         }
     }
 
@@ skipping 133 matching lines @@
         ASSERT(newPage != &PartitionRootGeneric::gSeedPage);
         ASSERT(!newPage->freelistHead);
         ASSERT(!newPage->numAllocatedSlots);
         ASSERT(!newPage->numUnprovisionedSlots);
         ASSERT(newPage->freeCacheIndex == -1);
         bucket->freePagesHead = newPage->nextPage;
         void* addr = partitionPageToPointer(newPage);
         partitionRecommitSystemPages(root, addr, newPage->bucket->numSystemPagesPerSlotSpan * kSystemPageSize);
     } else {
         // Third. If we get here, we need a brand new page.
-        size_t numPartitionPages = partitionBucketPartitionPages(bucket);
+        uint16_t numPartitionPages = partitionBucketPartitionPages(bucket);
         void* rawNewPage = partitionAllocPartitionPages(root, flags, numPartitionPages);
         if (UNLIKELY(!rawNewPage)) {
             ASSERT(returnNull);
             return 0;
         }
         // Skip the alignment check because it depends on page->bucket, which is not yet set.
         newPage = partitionPointerToPageNoAlignmentCheck(rawNewPage);
     }
 
     partitionPageReset(newPage, bucket);
@@ skipping 20 matching lines @@
     PartitionRootBase* root = partitionPageToRoot(page);
 
     // If the page is already registered as empty, give it another life.
     if (page->freeCacheIndex != -1) {
         ASSERT(page->freeCacheIndex >= 0);
         ASSERT(static_cast<unsigned>(page->freeCacheIndex) < kMaxFreeableSpans);
         ASSERT(root->globalEmptyPageRing[page->freeCacheIndex] == page);
         root->globalEmptyPageRing[page->freeCacheIndex] = 0;
     }
 
-    size_t currentIndex = root->globalEmptyPageRingIndex;
+    int16_t currentIndex = root->globalEmptyPageRingIndex;
     PartitionPage* pageToFree = root->globalEmptyPageRing[currentIndex];
     // The page might well have been re-activated, filled up, etc. before we get
     // around to looking at it here.
     if (pageToFree) {
         ASSERT(pageToFree != &PartitionRootBase::gSeedPage);
         ASSERT(pageToFree->freeCacheIndex >= 0);
         ASSERT(static_cast<unsigned>(pageToFree->freeCacheIndex) < kMaxFreeableSpans);
         ASSERT(pageToFree == root->globalEmptyPageRing[pageToFree->freeCacheIndex]);
         if (!pageToFree->numAllocatedSlots && pageToFree->freelistHead) {
             // The page is still empty, and not freed, so _really_ free it.
@@ skipping 44 matching lines @@
         }
         partitionRegisterEmptyPage(page);
     } else {
         // Ensure that the page is full. That's the only valid case if we
         // arrive here.
         ASSERT(page->numAllocatedSlots < 0);
         // A transition of numAllocatedSlots from 0 to -1 is not legal, and
         // likely indicates a double-free.
         RELEASE_ASSERT(page->numAllocatedSlots != -1);
         page->numAllocatedSlots = -page->numAllocatedSlots - 2;
-        ASSERT(page->numAllocatedSlots == static_cast<int>(partitionBucketSlots(bucket) - 1));
+        ASSERT(page->numAllocatedSlots == partitionBucketSlots(bucket) - 1);
         // Fully used page became partially used. It must be put back on the
         // non-full page list. Also make it the current page to increase the
         // chances of it being filled up again. The old current page will be
         // the next page.
         page->nextPage = bucket->activePagesHead;
         bucket->activePagesHead = page;
         --bucket->numFullPages;
         // Special case: for a partition page with just a single slot, it may
         // now be empty and we want to run it through the empty logic.
         if (UNLIKELY(page->numAllocatedSlots == 0))
@@ skipping 122 matching lines @@
             // Empty bucket with no freelist or full pages. Skip reporting it.
             continue;
         }
         size_t numFreePages = 0;
         PartitionPage* freePages = bucket.freePagesHead;
         while (freePages) {
             ++numFreePages;
             freePages = freePages->nextPage;
         }
         size_t bucketSlotSize = bucket.slotSize;
-        size_t bucketNumSlots = partitionBucketSlots(&bucket);
+        uint16_t bucketNumSlots = partitionBucketSlots(&bucket);
         size_t bucketUsefulStorage = bucketSlotSize * bucketNumSlots;
         size_t bucketPageSize = bucket.numSystemPagesPerSlotSpan * kSystemPageSize;
         size_t bucketWaste = bucketPageSize - bucketUsefulStorage;
         size_t numActiveBytes = bucket.numFullPages * bucketUsefulStorage;
         size_t numResidentBytes = bucket.numFullPages * bucketPageSize;
         size_t numFreeableBytes = 0;
         size_t numActivePages = 0;
         const PartitionPage* page = bucket.activePagesHead;
         while (page) {
             ASSERT(page != &PartitionRootGeneric::gSeedPage);
@@ skipping 20 matching lines @@
     printf("total live: %zu bytes\n", totalLive);
     printf("total resident: %zu bytes\n", totalResident);
     printf("total freeable: %zu bytes\n", totalFreeable);
     fflush(stdout);
 }
 
 #endif // !NDEBUG
 
 } // namespace WTF