PartitionAlloc: simplify PartitionPage structure some more.

Source/wtf/PartitionAllocTest.cpp

 /*
  * Copyright (C) 2013 Google Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *     * Redistributions of source code must retain the above copyright
  * notice, this list of conditions and the following disclaimer.
  *     * Redistributions in binary form must reproduce the above
@@ skipping 82 matching lines @@
         EXPECT_TRUE(ptr);
         if (!i)
             first = WTF::partitionCookieFreePointerAdjust(ptr);
         else if (i == numSlots - 1)
             last = WTF::partitionCookieFreePointerAdjust(ptr);
     }
     EXPECT_EQ(WTF::partitionPointerToPage(first), WTF::partitionPointerToPage(last));
     if (bucket->numSystemPagesPerSlotSpan == WTF::kNumSystemPagesPerPartitionPage)
         EXPECT_EQ(reinterpret_cast<size_t>(first) & WTF::kPartitionPageBaseMask, reinterpret_cast<size_t>(last) & WTF::kPartitionPageBaseMask);
     EXPECT_EQ(numSlots, static_cast<size_t>(bucket->activePagesHead->numAllocatedSlots));
-    EXPECT_EQ(0, partitionPageFreelistHead(bucket->activePagesHead));
+    EXPECT_EQ(0, bucket->activePagesHead->freelistHead);
     EXPECT_TRUE(bucket->activePagesHead);
     EXPECT_TRUE(bucket->activePagesHead != &WTF::PartitionRootGeneric::gSeedPage);
     return bucket->activePagesHead;
 }
 
 static void FreeFullPage(WTF::PartitionPage* page)
 {
     size_t size = page->bucket->slotSize;
     size_t numSlots = (page->bucket->numSystemPagesPerSlotSpan * WTF::kSystemPageSize) / size;
     EXPECT_EQ(numSlots, static_cast<size_t>(abs(page->numAllocatedSlots)));
     char* ptr = reinterpret_cast<char*>(partitionPageToPointer(page));
     size_t i;
     for (i = 0; i < numSlots; ++i) {
         partitionFree(ptr + kPointerOffset);
         ptr += size;
     }
 }
 
 // Check that the most basic of allocate / free pairs work.
 TEST(WTF_PartitionAlloc, Basic)
 {
     TestSetup();
     WTF::PartitionBucket* bucket = &allocator.root()->buckets()[kTestBucketIndex];
     WTF::PartitionPage* seedPage = &WTF::PartitionRootGeneric::gSeedPage;
 
     EXPECT_FALSE(bucket->freePagesHead);
     EXPECT_EQ(seedPage, bucket->activePagesHead);
-    EXPECT_EQ(0, bucket->activePagesHead->activePageNext);
+    EXPECT_EQ(0, bucket->activePagesHead->nextPage);
 
     void* ptr = partitionAlloc(allocator.root(), kTestAllocSize);
     EXPECT_TRUE(ptr);
     EXPECT_EQ(kPointerOffset, reinterpret_cast<size_t>(ptr) & WTF::kPartitionPageOffsetMask);
     // Check that the offset appears to include a guard page.
     EXPECT_EQ(WTF::kPartitionPageSize + kPointerOffset, reinterpret_cast<size_t>(ptr) & WTF::kSuperPageOffsetMask);
 
     partitionFree(ptr);
     // Expect that the last active page does not get tossed to the freelist.
     EXPECT_FALSE(bucket->freePagesHead);
@@ skipping 52 matching lines @@
 // Test a bucket with multiple pages.
 TEST(WTF_PartitionAlloc, MultiPages)
 {
     TestSetup();
     WTF::PartitionBucket* bucket = &allocator.root()->buckets()[kTestBucketIndex];
 
     WTF::PartitionPage* page = GetFullPage(kTestAllocSize);
     FreeFullPage(page);
     EXPECT_FALSE(bucket->freePagesHead);
     EXPECT_EQ(page, bucket->activePagesHead);
-    EXPECT_EQ(0, page->activePageNext);
+    EXPECT_EQ(0, page->nextPage);
     EXPECT_EQ(0, page->numAllocatedSlots);
 
     page = GetFullPage(kTestAllocSize);
     WTF::PartitionPage* page2 = GetFullPage(kTestAllocSize);
 
     EXPECT_EQ(page2, bucket->activePagesHead);
-    EXPECT_EQ(0, page2->activePageNext);
+    EXPECT_EQ(0, page2->nextPage);
     EXPECT_EQ(reinterpret_cast<uintptr_t>(partitionPageToPointer(page)) & WTF::kSuperPageBaseMask, reinterpret_cast<uintptr_t>(partitionPageToPointer(page2)) & WTF::kSuperPageBaseMask);
 
     // Fully free the non-current page. It should not be freelisted because
     // there is no other immediately useable page. The other page is full.
     FreeFullPage(page);
     EXPECT_EQ(0, page->numAllocatedSlots);
     EXPECT_FALSE(bucket->freePagesHead);
     EXPECT_EQ(page, bucket->activePagesHead);
 
     // Allocate a new page, it should pull from the freelist.
     page = GetFullPage(kTestAllocSize);
     EXPECT_FALSE(bucket->freePagesHead);
     EXPECT_EQ(page, bucket->activePagesHead);
 
     FreeFullPage(page);
     FreeFullPage(page2);
     EXPECT_EQ(0, page->numAllocatedSlots);
     EXPECT_EQ(0, page2->numAllocatedSlots);
-    EXPECT_TRUE(partitionPageIsFree(page2));
+    EXPECT_EQ(0, page2->numUnprovisionedSlots);
+    EXPECT_EQ(0, page2->freelistHead);
 
     TestShutdown();
 }
 
 // Test some finer aspects of internal page transitions.
 TEST(WTF_PartitionAlloc, PageTransitions)
 {
     TestSetup();
     WTF::PartitionBucket* bucket = &allocator.root()->buckets()[kTestBucketIndex];
 
     WTF::PartitionPage* page1 = GetFullPage(kTestAllocSize);
     EXPECT_EQ(page1, bucket->activePagesHead);
-    EXPECT_EQ(0, page1->activePageNext);
+    EXPECT_EQ(0, page1->nextPage);
     WTF::PartitionPage* page2 = GetFullPage(kTestAllocSize);
     EXPECT_EQ(page2, bucket->activePagesHead);
-    EXPECT_EQ(0, page2->activePageNext);
+    EXPECT_EQ(0, page2->nextPage);
 
     // Bounce page1 back into the non-full list then fill it up again.
     char* ptr = reinterpret_cast<char*>(partitionPageToPointer(page1)) + kPointerOffset;
     partitionFree(ptr);
     EXPECT_EQ(page1, bucket->activePagesHead);
     (void) partitionAlloc(allocator.root(), kTestAllocSize);
     EXPECT_EQ(page1, bucket->activePagesHead);
-    EXPECT_EQ(page2, bucket->activePagesHead->activePageNext);
+    EXPECT_EQ(page2, bucket->activePagesHead->nextPage);
 
     // Allocating another page at this point should cause us to scan over page1
     // (which is both full and NOT our current page), and evict it from the
     // freelist. Older code had a O(n^2) condition due to failure to do this.
     WTF::PartitionPage* page3 = GetFullPage(kTestAllocSize);
     EXPECT_EQ(page3, bucket->activePagesHead);
-    EXPECT_EQ(0, page3->activePageNext);
+    EXPECT_EQ(0, page3->nextPage);
 
     // Work out a pointer into page2 and free it.
     ptr = reinterpret_cast<char*>(partitionPageToPointer(page2)) + kPointerOffset;
     partitionFree(ptr);
     // Trying to allocate at this time should cause us to cycle around to page2
     // and find the recently freed slot.
     char* newPtr = reinterpret_cast<char*>(partitionAlloc(allocator.root(), kTestAllocSize));
     EXPECT_EQ(ptr, newPtr);
     EXPECT_EQ(page2, bucket->activePagesHead);
-    EXPECT_EQ(page3, page2->activePageNext);
+    EXPECT_EQ(page3, page2->nextPage);
 
     // Work out a pointer into page1 and free it. This should pull the page
     // back into the list of available pages.
     ptr = reinterpret_cast<char*>(partitionPageToPointer(page1)) + kPointerOffset;
     partitionFree(ptr);
     // This allocation should be satisfied by page1.
     newPtr = reinterpret_cast<char*>(partitionAlloc(allocator.root(), kTestAllocSize));
     EXPECT_EQ(ptr, newPtr);
     EXPECT_EQ(page1, bucket->activePagesHead);
-    EXPECT_EQ(page2, page1->activePageNext);
+    EXPECT_EQ(page2, page1->nextPage);
 
     FreeFullPage(page3);
     FreeFullPage(page2);
     FreeFullPage(page1);
 
     // Allocating whilst in this state exposed a bug, so keep the test.
     ptr = reinterpret_cast<char*>(partitionAlloc(allocator.root(), kTestAllocSize));
     partitionFree(ptr);
 
     TestShutdown();
@@ skipping 13 matching lines @@
     OwnPtr<WTF::PartitionPage*[]> pages = adoptArrayPtr(new WTF::PartitionPage*[numToFillFreeListPage]);
 
     size_t i;
     for (i = 0; i < numToFillFreeListPage; ++i) {
         pages[i] = GetFullPage(kTestAllocSize);
     }
     EXPECT_EQ(pages[numToFillFreeListPage - 1], bucket->activePagesHead);
     for (i = 0; i < numToFillFreeListPage; ++i)
         FreeFullPage(pages[i]);
     EXPECT_EQ(0, bucket->activePagesHead->numAllocatedSlots);
-    EXPECT_EQ(0, bucket->activePagesHead->activePageNext);
+    EXPECT_EQ(0, bucket->activePagesHead->nextPage->freelistHead);
+    EXPECT_EQ(0, bucket->activePagesHead->nextPage->numAllocatedSlots);
+    EXPECT_EQ(0, bucket->activePagesHead->nextPage->numUnprovisionedSlots);
 
     // Allocate / free in a different bucket size so we get control of a
     // different free page list. We need two pages because one will be the last
     // active page and not get freed.
     WTF::PartitionPage* page1 = GetFullPage(kTestAllocSize * 2);
     WTF::PartitionPage* page2 = GetFullPage(kTestAllocSize * 2);
     FreeFullPage(page1);
     FreeFullPage(page2);
 
     // If we re-allocate all kTestAllocSize allocations, we'll pull all the
     // free pages and end up freeing the first page for free page objects.
     // It's getting a bit tricky but a nice re-entrancy is going on:
     // alloc(kTestAllocSize) -> pulls page from free page list ->
     // free(PartitionFreepagelistEntry) -> last entry in page freed ->
     // alloc(PartitionFreepagelistEntry).
     for (i = 0; i < numToFillFreeListPage; ++i) {
         pages[i] = GetFullPage(kTestAllocSize);
     }
     EXPECT_EQ(pages[numToFillFreeListPage - 1], bucket->activePagesHead);
 
     // As part of the final free-up, we'll test another re-entrancy:
     // free(kTestAllocSize) -> last entry in page freed ->
     // alloc(PartitionFreepagelistEntry) -> pulls page from free page list ->
     // free(PartitionFreepagelistEntry)
     for (i = 0; i < numToFillFreeListPage; ++i)
         FreeFullPage(pages[i]);
     EXPECT_EQ(0, bucket->activePagesHead->numAllocatedSlots);
-    EXPECT_EQ(0, bucket->activePagesHead->activePageNext);
+    EXPECT_EQ(0, bucket->activePagesHead->nextPage->freelistHead);
+    EXPECT_EQ(0, bucket->activePagesHead->nextPage->numAllocatedSlots);
+    EXPECT_EQ(0, bucket->activePagesHead->nextPage->numUnprovisionedSlots);
 
     TestShutdown();
 }
 
 // Test a large series of allocations that cross more than one underlying
 // 64KB super page allocation.
 TEST(WTF_PartitionAlloc, MultiPageAllocs)
 {
     TestSetup();
     // This is guaranteed to cross a super page boundary because the first
@@ skipping 144 matching lines @@
     EXPECT_TRUE(ptr);
     memset(ptr, 'A', size);
     ptr2 = partitionAllocGeneric(genericAllocator.root(), size);
     EXPECT_TRUE(ptr2);
     void* ptr3 = partitionAllocGeneric(genericAllocator.root(), size);
     EXPECT_TRUE(ptr3);
     void* ptr4 = partitionAllocGeneric(genericAllocator.root(), size);
     EXPECT_TRUE(ptr4);
 
     page = WTF::partitionPointerToPage(WTF::partitionCookieFreePointerAdjust(ptr));
-    EXPECT_NE(page, WTF::partitionPointerToPage(WTF::partitionCookieFreePointerAdjust(ptr3)));
+    WTF::PartitionPage* page2 = WTF::partitionPointerToPage(WTF::partitionCookieFreePointerAdjust(ptr3));
+    EXPECT_NE(page, page2);
 
     partitionFreeGeneric(genericAllocator.root(), ptr);
     partitionFreeGeneric(genericAllocator.root(), ptr3);
     partitionFreeGeneric(genericAllocator.root(), ptr2);
-    // Should have freelisted at this point.
-    EXPECT_TRUE(page->bucket->freePagesHead);
-    ptr3 = partitionAllocGeneric(genericAllocator.root(), size);
-    EXPECT_TRUE(ptr3);
+    // Should have been freed at this point.
+    EXPECT_FALSE(page->freelistHead);
+    EXPECT_EQ(0, page->numAllocatedSlots);
+    EXPECT_EQ(0, page->numUnprovisionedSlots);
     void* newPtr = partitionAllocGeneric(genericAllocator.root(), size);
+    EXPECT_EQ(ptr3, newPtr);
+    newPtr = partitionAllocGeneric(genericAllocator.root(), size);
     EXPECT_EQ(ptr, newPtr);
 #if OS(LINUX) && defined(NDEBUG)
     // On Linux, we have a guarantee that freelisting a page should cause its
     // contents to be nulled out. We check for null here to detect an bug we
     // had where a large slot size was causing us to not properly free all
     // resources back to the system.
     // We only run the check in optimized builds because the debug build
     // writes over the allocated area with an "uninitialized" byte pattern.
     EXPECT_EQ(0, *(reinterpret_cast<char*>(newPtr) + (size - 1)));
 #endif
@@ skipping 17 matching lines @@
 
     void* ptr = partitionAlloc(allocator.root(), bigSize);
     EXPECT_TRUE(ptr);
 
     WTF::PartitionPage* page = WTF::partitionPointerToPage(WTF::partitionCookieFreePointerAdjust(ptr));
     size_t totalSlots = (page->bucket->numSystemPagesPerSlotSpan * WTF::kSystemPageSize) / (bigSize + kExtraAllocSize);
     EXPECT_EQ(4u, totalSlots);
     // The freelist should have one entry, because we were able to exactly fit
     // one object slot and one freelist pointer (the null that the head points
     // to) into a system page.
-    EXPECT_TRUE(partitionPageFreelistHead(page));
+    EXPECT_TRUE(page->freelistHead);
     EXPECT_EQ(1, page->numAllocatedSlots);
     EXPECT_EQ(2, page->numUnprovisionedSlots);
 
     void* ptr2 = partitionAlloc(allocator.root(), bigSize);
     EXPECT_TRUE(ptr2);
-    EXPECT_FALSE(partitionPageFreelistHead(page));
+    EXPECT_FALSE(page->freelistHead);
     EXPECT_EQ(2, page->numAllocatedSlots);
     EXPECT_EQ(2, page->numUnprovisionedSlots);
 
     void* ptr3 = partitionAlloc(allocator.root(), bigSize);
     EXPECT_TRUE(ptr3);
-    EXPECT_TRUE(partitionPageFreelistHead(page));
+    EXPECT_TRUE(page->freelistHead);
     EXPECT_EQ(3, page->numAllocatedSlots);
     EXPECT_EQ(0, page->numUnprovisionedSlots);
 
     void* ptr4 = partitionAlloc(allocator.root(), bigSize);
     EXPECT_TRUE(ptr4);
-    EXPECT_FALSE(partitionPageFreelistHead(page));
+    EXPECT_FALSE(page->freelistHead);
     EXPECT_EQ(4, page->numAllocatedSlots);
     EXPECT_EQ(0, page->numUnprovisionedSlots);
 
     void* ptr5 = partitionAlloc(allocator.root(), bigSize);
     EXPECT_TRUE(ptr5);
 
     WTF::PartitionPage* page2 = WTF::partitionPointerToPage(WTF::partitionCookieFreePointerAdjust(ptr5));
     EXPECT_EQ(1, page2->numAllocatedSlots);
 
     // Churn things a little whilst there's a partial page freelist.
     partitionFree(ptr);
     ptr = partitionAlloc(allocator.root(), bigSize);
     void* ptr6 = partitionAlloc(allocator.root(), bigSize);
 
     partitionFree(ptr);
     partitionFree(ptr2);
     partitionFree(ptr3);
     partitionFree(ptr4);
     partitionFree(ptr5);
     partitionFree(ptr6);
     EXPECT_TRUE(bucket->freePagesHead);
     EXPECT_EQ(page, bucket->freePagesHead);
-    EXPECT_TRUE(partitionPageFreelistHead(page2));
+    EXPECT_TRUE(page2->freelistHead);
     EXPECT_EQ(0, page2->numAllocatedSlots);
 
     // And test a couple of sizes that do not cross kSystemPageSize with a single allocation.
     size_t mediumSize = (WTF::kSystemPageSize / 2) - kExtraAllocSize;
     bucketIdx = (mediumSize + kExtraAllocSize) >> WTF::kBucketShift;
     bucket = &allocator.root()->buckets()[bucketIdx];
     EXPECT_EQ(0, bucket->freePagesHead);
 
     ptr = partitionAlloc(allocator.root(), mediumSize);
     EXPECT_TRUE(ptr);
@@ skipping 13 matching lines @@
 
     ptr = partitionAlloc(allocator.root(), smallSize);
     EXPECT_TRUE(ptr);
     page = WTF::partitionPointerToPage(WTF::partitionCookieFreePointerAdjust(ptr));
     EXPECT_EQ(1, page->numAllocatedSlots);
     totalSlots = (page->bucket->numSystemPagesPerSlotSpan * WTF::kSystemPageSize) / (smallSize + kExtraAllocSize);
     firstPageSlots = WTF::kSystemPageSize / (smallSize + kExtraAllocSize);
     EXPECT_EQ(totalSlots - firstPageSlots, page->numUnprovisionedSlots);
 
     partitionFree(ptr);
-    EXPECT_TRUE(partitionPageFreelistHead(page));
+    EXPECT_TRUE(page->freelistHead);
     EXPECT_EQ(0, page->numAllocatedSlots);
 
     size_t verySmallSize = (WTF::kAllocationGranularity * 2) - kExtraAllocSize;
     bucketIdx = (verySmallSize + kExtraAllocSize) >> WTF::kBucketShift;
     bucket = &allocator.root()->buckets()[bucketIdx];
     EXPECT_EQ(0, bucket->freePagesHead);
 
     ptr = partitionAlloc(allocator.root(), verySmallSize);
     EXPECT_TRUE(ptr);
     page = WTF::partitionPointerToPage(WTF::partitionCookieFreePointerAdjust(ptr));
     EXPECT_EQ(1, page->numAllocatedSlots);
     totalSlots = (page->bucket->numSystemPagesPerSlotSpan * WTF::kSystemPageSize) / (verySmallSize + kExtraAllocSize);
     firstPageSlots = WTF::kSystemPageSize / (verySmallSize + kExtraAllocSize);
     EXPECT_EQ(totalSlots - firstPageSlots, page->numUnprovisionedSlots);
 
     partitionFree(ptr);
-    EXPECT_TRUE(partitionPageFreelistHead(page));
+    EXPECT_TRUE(page->freelistHead);
     EXPECT_EQ(0, page->numAllocatedSlots);
 
     // And try an allocation size (against the generic allocator) that is
     // larger than a system page.
     size_t pageAndAHalfSize = (WTF::kSystemPageSize + (WTF::kSystemPageSize / 2)) - kExtraAllocSize;
     ptr = partitionAllocGeneric(genericAllocator.root(), pageAndAHalfSize);
     EXPECT_TRUE(ptr);
     page = WTF::partitionPointerToPage(WTF::partitionCookieFreePointerAdjust(ptr));
     EXPECT_EQ(1, page->numAllocatedSlots);
-    EXPECT_TRUE(partitionPageFreelistHead(page));
+    EXPECT_TRUE(page->freelistHead);
     totalSlots = (page->bucket->numSystemPagesPerSlotSpan * WTF::kSystemPageSize) / (pageAndAHalfSize + kExtraAllocSize);
     EXPECT_EQ(totalSlots - 2, page->numUnprovisionedSlots);
     partitionFreeGeneric(genericAllocator.root(), ptr);
 
     // And then make sure than exactly the page size only faults one page.
     size_t pageSize = WTF::kSystemPageSize - kExtraAllocSize;
     ptr = partitionAllocGeneric(genericAllocator.root(), pageSize);
     EXPECT_TRUE(ptr);
     page = WTF::partitionPointerToPage(WTF::partitionCookieFreePointerAdjust(ptr));
     EXPECT_EQ(1, page->numAllocatedSlots);
-    EXPECT_FALSE(partitionPageFreelistHead(page));
+    EXPECT_FALSE(page->freelistHead);
     totalSlots = (page->bucket->numSystemPagesPerSlotSpan * WTF::kSystemPageSize) / (pageSize + kExtraAllocSize);
     EXPECT_EQ(totalSlots - 1, page->numUnprovisionedSlots);
     partitionFreeGeneric(genericAllocator.root(), ptr);
 
     TestShutdown();
 }
 
 // Test some of the fragmentation-resistant properties of the allocator.
 TEST(WTF_PartitionAlloc, PageRefilling)
 {
@@ skipping 165 matching lines @@
     EXPECT_EQ(32u, WTF::countLeadingZerosSizet(0));
     EXPECT_EQ(31u, WTF::countLeadingZerosSizet(1));
     EXPECT_EQ(1u, WTF::countLeadingZerosSizet(1 << 30));
     EXPECT_EQ(0u, WTF::countLeadingZerosSizet(1 << 31));
 #endif
 }
 
 } // namespace
 
 #endif // !defined(MEMORY_TOOL_REPLACES_ALLOCATOR)