Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(580)

Side by Side Diff: base/allocator/partition_allocator/partition_alloc.cc

Issue 2518253002: Move Partition Allocator into Chromium base. (Closed)
Patch Set: Move OOM_CRASH into its own, more specific header. Fixes Windows build. Created 4 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 /* 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 * Copyright (C) 2013 Google Inc. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be
3 * 3 // found in the LICENSE file.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are
6 * met:
7 *
8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above
11 * copyright notice, this list of conditions and the following disclaimer
12 * in the documentation and/or other materials provided with the
13 * distribution.
14 * * Neither the name of Google Inc. nor the names of its
15 * contributors may be used to endorse or promote products derived from
16 * this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30 4
31 #include "wtf/allocator/PartitionAlloc.h" 5 #include "base/allocator/partition_allocator/partition_alloc.h"
32 6
33 #include <string.h> 7 #include <string.h>
34 8
35 #ifndef NDEBUG 9 #include "base/allocator/oom.h"
36 #include <stdio.h> 10 #include "base/compiler_specific.h"
37 #endif 11 #include "base/synchronization/spin_lock.h"
38 12
39 // Two partition pages are used as guard / metadata page so make sure the super 13 // Two partition pages are used as guard / metadata page so make sure the super
40 // page size is bigger. 14 // page size is bigger.
41 static_assert(WTF::kPartitionPageSize * 4 <= WTF::kSuperPageSize, 15 static_assert(base::kPartitionPageSize * 4 <= base::kSuperPageSize,
42 "ok super page size"); 16 "ok super page size");
43 static_assert(!(WTF::kSuperPageSize % WTF::kPartitionPageSize), 17 static_assert(!(base::kSuperPageSize % base::kPartitionPageSize),
44 "ok super page multiple"); 18 "ok super page multiple");
45 // Four system pages gives us room to hack out a still-guard-paged piece 19 // Four system pages gives us room to hack out a still-guard-paged piece
46 // of metadata in the middle of a guard partition page. 20 // of metadata in the middle of a guard partition page.
47 static_assert(WTF::kSystemPageSize * 4 <= WTF::kPartitionPageSize, 21 static_assert(base::kSystemPageSize * 4 <= base::kPartitionPageSize,
48 "ok partition page size"); 22 "ok partition page size");
49 static_assert(!(WTF::kPartitionPageSize % WTF::kSystemPageSize), 23 static_assert(!(base::kPartitionPageSize % base::kSystemPageSize),
50 "ok partition page multiple"); 24 "ok partition page multiple");
51 static_assert(sizeof(WTF::PartitionPage) <= WTF::kPageMetadataSize, 25 static_assert(sizeof(base::PartitionPage) <= base::kPageMetadataSize,
52 "PartitionPage should not be too big"); 26 "PartitionPage should not be too big");
53 static_assert(sizeof(WTF::PartitionBucket) <= WTF::kPageMetadataSize, 27 static_assert(sizeof(base::PartitionBucket) <= base::kPageMetadataSize,
54 "PartitionBucket should not be too big"); 28 "PartitionBucket should not be too big");
55 static_assert(sizeof(WTF::PartitionSuperPageExtentEntry) <= 29 static_assert(sizeof(base::PartitionSuperPageExtentEntry) <=
56 WTF::kPageMetadataSize, 30 base::kPageMetadataSize,
57 "PartitionSuperPageExtentEntry should not be too big"); 31 "PartitionSuperPageExtentEntry should not be too big");
58 static_assert(WTF::kPageMetadataSize * WTF::kNumPartitionPagesPerSuperPage <= 32 static_assert(base::kPageMetadataSize * base::kNumPartitionPagesPerSuperPage <=
59 WTF::kSystemPageSize, 33 base::kSystemPageSize,
60 "page metadata fits in hole"); 34 "page metadata fits in hole");
61 // Check that some of our zanier calculations worked out as expected. 35 // Check that some of our zanier calculations worked out as expected.
62 static_assert(WTF::kGenericSmallestBucket == 8, "generic smallest bucket"); 36 static_assert(base::kGenericSmallestBucket == 8, "generic smallest bucket");
63 static_assert(WTF::kGenericMaxBucketed == 983040, "generic max bucketed"); 37 static_assert(base::kGenericMaxBucketed == 983040, "generic max bucketed");
64 static_assert(WTF::kMaxSystemPagesPerSlotSpan < (1 << 8), 38 static_assert(base::kMaxSystemPagesPerSlotSpan < (1 << 8),
65 "System pages per slot span must be less than 128."); 39 "System pages per slot span must be less than 128.");
66 40
67 namespace WTF { 41 namespace base {
68 42
69 SpinLock PartitionRootBase::gInitializedLock; 43 subtle::SpinLock PartitionRootBase::gInitializedLock;
70 bool PartitionRootBase::gInitialized = false; 44 bool PartitionRootBase::gInitialized = false;
71 PartitionPage PartitionRootBase::gSeedPage; 45 PartitionPage PartitionRootBase::gSeedPage;
72 PartitionBucket PartitionRootBase::gPagedBucket; 46 PartitionBucket PartitionRootBase::gPagedBucket;
73 void (*PartitionRootBase::gOomHandlingFunction)() = nullptr; 47 void (*PartitionRootBase::gOomHandlingFunction)() = nullptr;
74 PartitionAllocHooks::AllocationHook* PartitionAllocHooks::m_allocationHook = 48 PartitionAllocHooks::AllocationHook* PartitionAllocHooks::m_allocationHook =
75 nullptr; 49 nullptr;
76 PartitionAllocHooks::FreeHook* PartitionAllocHooks::m_freeHook = nullptr; 50 PartitionAllocHooks::FreeHook* PartitionAllocHooks::m_freeHook = nullptr;
77 51
78 static uint8_t partitionBucketNumSystemPages(size_t size) { 52 static uint8_t partitionBucketNumSystemPages(size_t size) {
79 // This works out reasonably for the current bucket sizes of the generic 53 // This works out reasonably for the current bucket sizes of the generic
80 // allocator, and the current values of partition page size and constants. 54 // allocator, and the current values of partition page size and constants.
81 // Specifically, we have enough room to always pack the slots perfectly into 55 // Specifically, we have enough room to always pack the slots perfectly into
82 // some number of system pages. The only waste is the waste associated with 56 // some number of system pages. The only waste is the waste associated with
83 // unfaulted pages (i.e. wasted address space). 57 // unfaulted pages (i.e. wasted address space).
84 // TODO: we end up using a lot of system pages for very small sizes. For 58 // TODO: we end up using a lot of system pages for very small sizes. For
85 // example, we'll use 12 system pages for slot size 24. The slot size is 59 // example, we'll use 12 system pages for slot size 24. The slot size is
86 // so small that the waste would be tiny with just 4, or 1, system pages. 60 // so small that the waste would be tiny with just 4, or 1, system pages.
87 // Later, we can investigate whether there are anti-fragmentation benefits 61 // Later, we can investigate whether there are anti-fragmentation benefits
88 // to using fewer system pages. 62 // to using fewer system pages.
89 double bestWasteRatio = 1.0f; 63 double bestWasteRatio = 1.0f;
90 uint16_t bestPages = 0; 64 uint16_t bestPages = 0;
91 if (size > kMaxSystemPagesPerSlotSpan * kSystemPageSize) { 65 if (size > kMaxSystemPagesPerSlotSpan * kSystemPageSize) {
92 ASSERT(!(size % kSystemPageSize)); 66 DCHECK(!(size % kSystemPageSize));
93 bestPages = static_cast<uint16_t>(size / kSystemPageSize); 67 bestPages = static_cast<uint16_t>(size / kSystemPageSize);
94 RELEASE_ASSERT(bestPages < (1 << 8)); 68 CHECK(bestPages < (1 << 8));
95 return static_cast<uint8_t>(bestPages); 69 return static_cast<uint8_t>(bestPages);
96 } 70 }
97 ASSERT(size <= kMaxSystemPagesPerSlotSpan * kSystemPageSize); 71 DCHECK(size <= kMaxSystemPagesPerSlotSpan * kSystemPageSize);
98 for (uint16_t i = kNumSystemPagesPerPartitionPage - 1; 72 for (uint16_t i = kNumSystemPagesPerPartitionPage - 1;
99 i <= kMaxSystemPagesPerSlotSpan; ++i) { 73 i <= kMaxSystemPagesPerSlotSpan; ++i) {
100 size_t pageSize = kSystemPageSize * i; 74 size_t pageSize = kSystemPageSize * i;
101 size_t numSlots = pageSize / size; 75 size_t numSlots = pageSize / size;
102 size_t waste = pageSize - (numSlots * size); 76 size_t waste = pageSize - (numSlots * size);
103 // Leaving a page unfaulted is not free; the page will occupy an empty page 77 // Leaving a page unfaulted is not free; the page will occupy an empty page
104 // table entry. Make a simple attempt to account for that. 78 // table entry. Make a simple attempt to account for that.
105 size_t numRemainderPages = i & (kNumSystemPagesPerPartitionPage - 1); 79 size_t numRemainderPages = i & (kNumSystemPagesPerPartitionPage - 1);
106 size_t numUnfaultedPages = 80 size_t numUnfaultedPages =
107 numRemainderPages 81 numRemainderPages
108 ? (kNumSystemPagesPerPartitionPage - numRemainderPages) 82 ? (kNumSystemPagesPerPartitionPage - numRemainderPages)
109 : 0; 83 : 0;
110 waste += sizeof(void*) * numUnfaultedPages; 84 waste += sizeof(void*) * numUnfaultedPages;
111 double wasteRatio = (double)waste / (double)pageSize; 85 double wasteRatio = (double)waste / (double)pageSize;
112 if (wasteRatio < bestWasteRatio) { 86 if (wasteRatio < bestWasteRatio) {
113 bestWasteRatio = wasteRatio; 87 bestWasteRatio = wasteRatio;
114 bestPages = i; 88 bestPages = i;
115 } 89 }
116 } 90 }
117 ASSERT(bestPages > 0); 91 DCHECK(bestPages > 0);
118 RELEASE_ASSERT(bestPages <= kMaxSystemPagesPerSlotSpan); 92 CHECK(bestPages <= kMaxSystemPagesPerSlotSpan);
119 return static_cast<uint8_t>(bestPages); 93 return static_cast<uint8_t>(bestPages);
120 } 94 }
121 95
122 static void partitionAllocBaseInit(PartitionRootBase* root) { 96 static void partitionAllocBaseInit(PartitionRootBase* root) {
123 ASSERT(!root->initialized); 97 DCHECK(!root->initialized);
124 { 98 {
125 SpinLock::Guard guard(PartitionRootBase::gInitializedLock); 99 subtle::SpinLock::Guard guard(PartitionRootBase::gInitializedLock);
126 if (!PartitionRootBase::gInitialized) { 100 if (!PartitionRootBase::gInitialized) {
127 PartitionRootBase::gInitialized = true; 101 PartitionRootBase::gInitialized = true;
128 // We mark the seed page as free to make sure it is skipped by our 102 // We mark the seed page as free to make sure it is skipped by our
129 // logic to find a new active page. 103 // logic to find a new active page.
130 PartitionRootBase::gPagedBucket.activePagesHead = 104 PartitionRootBase::gPagedBucket.activePagesHead =
131 &PartitionRootGeneric::gSeedPage; 105 &PartitionRootGeneric::gSeedPage;
132 } 106 }
133 } 107 }
134 108
135 root->initialized = true; 109 root->initialized = true;
(...skipping 18 matching lines...) Expand all
154 PartitionRootBase* root) { 128 PartitionRootBase* root) {
155 bucket->activePagesHead = &PartitionRootGeneric::gSeedPage; 129 bucket->activePagesHead = &PartitionRootGeneric::gSeedPage;
156 bucket->emptyPagesHead = 0; 130 bucket->emptyPagesHead = 0;
157 bucket->decommittedPagesHead = 0; 131 bucket->decommittedPagesHead = 0;
158 bucket->numFullPages = 0; 132 bucket->numFullPages = 0;
159 bucket->numSystemPagesPerSlotSpan = 133 bucket->numSystemPagesPerSlotSpan =
160 partitionBucketNumSystemPages(bucket->slotSize); 134 partitionBucketNumSystemPages(bucket->slotSize);
161 } 135 }
162 136
163 void partitionAllocGlobalInit(void (*oomHandlingFunction)()) { 137 void partitionAllocGlobalInit(void (*oomHandlingFunction)()) {
164 ASSERT(oomHandlingFunction); 138 DCHECK(oomHandlingFunction);
165 PartitionRootBase::gOomHandlingFunction = oomHandlingFunction; 139 PartitionRootBase::gOomHandlingFunction = oomHandlingFunction;
166 } 140 }
167 141
168 void partitionAllocInit(PartitionRoot* root, 142 void partitionAllocInit(PartitionRoot* root,
169 size_t numBuckets, 143 size_t numBuckets,
170 size_t maxAllocation) { 144 size_t maxAllocation) {
171 partitionAllocBaseInit(root); 145 partitionAllocBaseInit(root);
172 146
173 root->numBuckets = numBuckets; 147 root->numBuckets = numBuckets;
174 root->maxAllocation = maxAllocation; 148 root->maxAllocation = maxAllocation;
175 size_t i; 149 size_t i;
176 for (i = 0; i < root->numBuckets; ++i) { 150 for (i = 0; i < root->numBuckets; ++i) {
177 PartitionBucket* bucket = &root->buckets()[i]; 151 PartitionBucket* bucket = &root->buckets()[i];
178 if (!i) 152 if (!i)
179 bucket->slotSize = kAllocationGranularity; 153 bucket->slotSize = kAllocationGranularity;
180 else 154 else
181 bucket->slotSize = i << kBucketShift; 155 bucket->slotSize = i << kBucketShift;
182 partitionBucketInitBase(bucket, root); 156 partitionBucketInitBase(bucket, root);
183 } 157 }
184 } 158 }
185 159
186 void partitionAllocGenericInit(PartitionRootGeneric* root) { 160 void partitionAllocGenericInit(PartitionRootGeneric* root) {
187 SpinLock::Guard guard(root->lock); 161 subtle::SpinLock::Guard guard(root->lock);
188 162
189 partitionAllocBaseInit(root); 163 partitionAllocBaseInit(root);
190 164
191 // Precalculate some shift and mask constants used in the hot path. 165 // Precalculate some shift and mask constants used in the hot path.
192 // Example: malloc(41) == 101001 binary. 166 // Example: malloc(41) == 101001 binary.
193 // Order is 6 (1 << 6-1)==32 is highest bit set. 167 // Order is 6 (1 << 6-1)==32 is highest bit set.
194 // orderIndex is the next three MSB == 010 == 2. 168 // orderIndex is the next three MSB == 010 == 2.
195 // subOrderIndexMask is a mask for the remaining bits == 11 (masking to 01 for 169 // subOrderIndexMask is a mask for the remaining bits == 11 (masking to 01 for
196 // the subOrderIndex). 170 // the subOrderIndex).
197 size_t order; 171 size_t order;
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after
230 bucket->slotSize = currentSize; 204 bucket->slotSize = currentSize;
231 partitionBucketInitBase(bucket, root); 205 partitionBucketInitBase(bucket, root);
232 // Disable psuedo buckets so that touching them faults. 206 // Disable psuedo buckets so that touching them faults.
233 if (currentSize % kGenericSmallestBucket) 207 if (currentSize % kGenericSmallestBucket)
234 bucket->activePagesHead = 0; 208 bucket->activePagesHead = 0;
235 currentSize += currentIncrement; 209 currentSize += currentIncrement;
236 ++bucket; 210 ++bucket;
237 } 211 }
238 currentIncrement <<= 1; 212 currentIncrement <<= 1;
239 } 213 }
240 ASSERT(currentSize == 1 << kGenericMaxBucketedOrder); 214 DCHECK(currentSize == 1 << kGenericMaxBucketedOrder);
241 ASSERT(bucket == &root->buckets[0] + kGenericNumBuckets); 215 DCHECK(bucket == &root->buckets[0] + kGenericNumBuckets);
242 216
243 // Then set up the fast size -> bucket lookup table. 217 // Then set up the fast size -> bucket lookup table.
244 bucket = &root->buckets[0]; 218 bucket = &root->buckets[0];
245 PartitionBucket** bucketPtr = &root->bucketLookups[0]; 219 PartitionBucket** bucketPtr = &root->bucketLookups[0];
246 for (order = 0; order <= kBitsPerSizet; ++order) { 220 for (order = 0; order <= kBitsPerSizet; ++order) {
247 for (j = 0; j < kGenericNumBucketsPerOrder; ++j) { 221 for (j = 0; j < kGenericNumBucketsPerOrder; ++j) {
248 if (order < kGenericMinBucketedOrder) { 222 if (order < kGenericMinBucketedOrder) {
249 // Use the bucket of the finest granularity for malloc(0) etc. 223 // Use the bucket of the finest granularity for malloc(0) etc.
250 *bucketPtr++ = &root->buckets[0]; 224 *bucketPtr++ = &root->buckets[0];
251 } else if (order > kGenericMaxBucketedOrder) { 225 } else if (order > kGenericMaxBucketedOrder) {
252 *bucketPtr++ = &PartitionRootGeneric::gPagedBucket; 226 *bucketPtr++ = &PartitionRootGeneric::gPagedBucket;
253 } else { 227 } else {
254 PartitionBucket* validBucket = bucket; 228 PartitionBucket* validBucket = bucket;
255 // Skip over invalid buckets. 229 // Skip over invalid buckets.
256 while (validBucket->slotSize % kGenericSmallestBucket) 230 while (validBucket->slotSize % kGenericSmallestBucket)
257 validBucket++; 231 validBucket++;
258 *bucketPtr++ = validBucket; 232 *bucketPtr++ = validBucket;
259 bucket++; 233 bucket++;
260 } 234 }
261 } 235 }
262 } 236 }
263 ASSERT(bucket == &root->buckets[0] + kGenericNumBuckets); 237 DCHECK(bucket == &root->buckets[0] + kGenericNumBuckets);
264 ASSERT(bucketPtr == 238 DCHECK(bucketPtr ==
265 &root->bucketLookups[0] + 239 &root->bucketLookups[0] +
266 ((kBitsPerSizet + 1) * kGenericNumBucketsPerOrder)); 240 ((kBitsPerSizet + 1) * kGenericNumBucketsPerOrder));
267 // And there's one last bucket lookup that will be hit for e.g. malloc(-1), 241 // And there's one last bucket lookup that will be hit for e.g. malloc(-1),
268 // which tries to overflow to a non-existant order. 242 // which tries to overflow to a non-existant order.
269 *bucketPtr = &PartitionRootGeneric::gPagedBucket; 243 *bucketPtr = &PartitionRootGeneric::gPagedBucket;
270 } 244 }
271 245
272 static bool partitionAllocShutdownBucket(PartitionBucket* bucket) { 246 static bool partitionAllocShutdownBucket(PartitionBucket* bucket) {
273 // Failure here indicates a memory leak. 247 // Failure here indicates a memory leak.
274 bool foundLeak = bucket->numFullPages; 248 bool foundLeak = bucket->numFullPages != 0;
275 for (PartitionPage* page = bucket->activePagesHead; page; 249 for (PartitionPage* page = bucket->activePagesHead; page;
276 page = page->nextPage) 250 page = page->nextPage)
277 foundLeak |= (page->numAllocatedSlots > 0); 251 foundLeak |= (page->numAllocatedSlots > 0);
278 return foundLeak; 252 return foundLeak;
279 } 253 }
280 254
281 static bool partitionAllocBaseShutdown(PartitionRootBase* root) { 255 static bool partitionAllocBaseShutdown(PartitionRootBase* root) {
282 ASSERT(root->initialized); 256 DCHECK(root->initialized);
283 root->initialized = false; 257 root->initialized = false;
284 258
285 // Now that we've examined all partition pages in all buckets, it's safe 259 // Now that we've examined all partition pages in all buckets, it's safe
286 // to free all our super pages. Since the super page extent entries are 260 // to free all our super pages. Since the super page extent entries are
287 // stored in the super pages, we need to be careful not to access them 261 // stored in the super pages, we need to be careful not to access them
288 // after we've released the corresponding super page. 262 // after we've released the corresponding super page.
289 PartitionSuperPageExtentEntry* entry = root->firstExtent; 263 PartitionSuperPageExtentEntry* entry = root->firstExtent;
290 while (entry) { 264 while (entry) {
291 PartitionSuperPageExtentEntry* nextEntry = entry->next; 265 PartitionSuperPageExtentEntry* nextEntry = entry->next;
292 char* superPage = entry->superPageBase; 266 char* superPage = entry->superPageBase;
293 char* superPagesEnd = entry->superPagesEnd; 267 char* superPagesEnd = entry->superPagesEnd;
294 while (superPage < superPagesEnd) { 268 while (superPage < superPagesEnd) {
295 freePages(superPage, kSuperPageSize); 269 freePages(superPage, kSuperPageSize);
296 superPage += kSuperPageSize; 270 superPage += kSuperPageSize;
297 } 271 }
298 entry = nextEntry; 272 entry = nextEntry;
299 } 273 }
300 return root->directMapList; 274 return root->directMapList != nullptr;
301 } 275 }
302 276
303 bool partitionAllocShutdown(PartitionRoot* root) { 277 bool partitionAllocShutdown(PartitionRoot* root) {
304 bool foundLeak = false; 278 bool foundLeak = false;
305 size_t i; 279 size_t i;
306 for (i = 0; i < root->numBuckets; ++i) { 280 for (i = 0; i < root->numBuckets; ++i) {
307 PartitionBucket* bucket = &root->buckets()[i]; 281 PartitionBucket* bucket = &root->buckets()[i];
308 foundLeak |= partitionAllocShutdownBucket(bucket); 282 foundLeak |= partitionAllocShutdownBucket(bucket);
309 } 283 }
310 foundLeak |= partitionAllocBaseShutdown(root); 284 foundLeak |= partitionAllocBaseShutdown(root);
311 return !foundLeak; 285 return !foundLeak;
312 } 286 }
313 287
314 bool partitionAllocGenericShutdown(PartitionRootGeneric* root) { 288 bool partitionAllocGenericShutdown(PartitionRootGeneric* root) {
315 SpinLock::Guard guard(root->lock); 289 subtle::SpinLock::Guard guard(root->lock);
316 bool foundLeak = false; 290 bool foundLeak = false;
317 size_t i; 291 size_t i;
318 for (i = 0; i < kGenericNumBuckets; ++i) { 292 for (i = 0; i < kGenericNumBuckets; ++i) {
319 PartitionBucket* bucket = &root->buckets[i]; 293 PartitionBucket* bucket = &root->buckets[i];
320 foundLeak |= partitionAllocShutdownBucket(bucket); 294 foundLeak |= partitionAllocShutdownBucket(bucket);
321 } 295 }
322 foundLeak |= partitionAllocBaseShutdown(root); 296 foundLeak |= partitionAllocBaseShutdown(root);
323 return !foundLeak; 297 return !foundLeak;
324 } 298 }
325 299
326 #if !CPU(64BIT) 300 #if !defined(ARCH_CPU_64_BITS)
327 static NEVER_INLINE void partitionOutOfMemoryWithLotsOfUncommitedPages() { 301 static NOINLINE void partitionOutOfMemoryWithLotsOfUncommitedPages() {
328 OOM_CRASH(); 302 OOM_CRASH();
329 } 303 }
330 #endif 304 #endif
331 305
332 static NEVER_INLINE void partitionOutOfMemory(const PartitionRootBase* root) { 306 static NOINLINE void partitionOutOfMemory(const PartitionRootBase* root) {
333 #if !CPU(64BIT) 307 #if !defined(ARCH_CPU_64_BITS)
334 // Check whether this OOM is due to a lot of super pages that are allocated 308 // Check whether this OOM is due to a lot of super pages that are allocated
335 // but not committed, probably due to http://crbug.com/421387. 309 // but not committed, probably due to http://crbug.com/421387.
336 if (root->totalSizeOfSuperPages + root->totalSizeOfDirectMappedPages - 310 if (root->totalSizeOfSuperPages + root->totalSizeOfDirectMappedPages -
337 root->totalSizeOfCommittedPages > 311 root->totalSizeOfCommittedPages >
338 kReasonableSizeOfUnusedPages) { 312 kReasonableSizeOfUnusedPages) {
339 partitionOutOfMemoryWithLotsOfUncommitedPages(); 313 partitionOutOfMemoryWithLotsOfUncommitedPages();
340 } 314 }
341 #endif 315 #endif
342 if (PartitionRootBase::gOomHandlingFunction) 316 if (PartitionRootBase::gOomHandlingFunction)
343 (*PartitionRootBase::gOomHandlingFunction)(); 317 (*PartitionRootBase::gOomHandlingFunction)();
344 OOM_CRASH(); 318 OOM_CRASH();
345 } 319 }
346 320
347 static NEVER_INLINE void partitionExcessiveAllocationSize() { 321 static NOINLINE void partitionExcessiveAllocationSize() {
348 OOM_CRASH(); 322 OOM_CRASH();
349 } 323 }
350 324
351 static NEVER_INLINE void partitionBucketFull() { 325 static NOINLINE void partitionBucketFull() {
352 OOM_CRASH(); 326 OOM_CRASH();
353 } 327 }
354 328
355 // partitionPageStateIs* 329 // partitionPageStateIs*
356 // Note that it's only valid to call these functions on pages found on one of 330 // Note that it's only valid to call these functions on pages found on one of
357 // the page lists. Specifically, you can't call these functions on full pages 331 // the page lists. Specifically, you can't call these functions on full pages
358 // that were detached from the active list. 332 // that were detached from the active list.
359 static bool ALWAYS_INLINE 333 static bool ALWAYS_INLINE
360 partitionPageStateIsActive(const PartitionPage* page) { 334 partitionPageStateIsActive(const PartitionPage* page) {
361 ASSERT(page != &PartitionRootGeneric::gSeedPage); 335 DCHECK(page != &PartitionRootGeneric::gSeedPage);
362 ASSERT(!page->pageOffset); 336 DCHECK(!page->pageOffset);
363 return (page->numAllocatedSlots > 0 && 337 return (page->numAllocatedSlots > 0 &&
364 (page->freelistHead || page->numUnprovisionedSlots)); 338 (page->freelistHead || page->numUnprovisionedSlots));
365 } 339 }
366 340
367 static bool ALWAYS_INLINE partitionPageStateIsFull(const PartitionPage* page) { 341 static bool ALWAYS_INLINE partitionPageStateIsFull(const PartitionPage* page) {
368 ASSERT(page != &PartitionRootGeneric::gSeedPage); 342 DCHECK(page != &PartitionRootGeneric::gSeedPage);
369 ASSERT(!page->pageOffset); 343 DCHECK(!page->pageOffset);
370 bool ret = (page->numAllocatedSlots == partitionBucketSlots(page->bucket)); 344 bool ret = (page->numAllocatedSlots == partitionBucketSlots(page->bucket));
371 if (ret) { 345 if (ret) {
372 ASSERT(!page->freelistHead); 346 DCHECK(!page->freelistHead);
373 ASSERT(!page->numUnprovisionedSlots); 347 DCHECK(!page->numUnprovisionedSlots);
374 } 348 }
375 return ret; 349 return ret;
376 } 350 }
377 351
378 static bool ALWAYS_INLINE partitionPageStateIsEmpty(const PartitionPage* page) { 352 static bool ALWAYS_INLINE partitionPageStateIsEmpty(const PartitionPage* page) {
379 ASSERT(page != &PartitionRootGeneric::gSeedPage); 353 DCHECK(page != &PartitionRootGeneric::gSeedPage);
380 ASSERT(!page->pageOffset); 354 DCHECK(!page->pageOffset);
381 return (!page->numAllocatedSlots && page->freelistHead); 355 return (!page->numAllocatedSlots && page->freelistHead);
382 } 356 }
383 357
384 static bool ALWAYS_INLINE 358 static bool ALWAYS_INLINE
385 partitionPageStateIsDecommitted(const PartitionPage* page) { 359 partitionPageStateIsDecommitted(const PartitionPage* page) {
386 ASSERT(page != &PartitionRootGeneric::gSeedPage); 360 DCHECK(page != &PartitionRootGeneric::gSeedPage);
387 ASSERT(!page->pageOffset); 361 DCHECK(!page->pageOffset);
388 bool ret = (!page->numAllocatedSlots && !page->freelistHead); 362 bool ret = (!page->numAllocatedSlots && !page->freelistHead);
389 if (ret) { 363 if (ret) {
390 ASSERT(!page->numUnprovisionedSlots); 364 DCHECK(!page->numUnprovisionedSlots);
391 ASSERT(page->emptyCacheIndex == -1); 365 DCHECK(page->emptyCacheIndex == -1);
392 } 366 }
393 return ret; 367 return ret;
394 } 368 }
395 369
396 static void partitionIncreaseCommittedPages(PartitionRootBase* root, 370 static void partitionIncreaseCommittedPages(PartitionRootBase* root,
397 size_t len) { 371 size_t len) {
398 root->totalSizeOfCommittedPages += len; 372 root->totalSizeOfCommittedPages += len;
399 ASSERT(root->totalSizeOfCommittedPages <= 373 DCHECK(root->totalSizeOfCommittedPages <=
400 root->totalSizeOfSuperPages + root->totalSizeOfDirectMappedPages); 374 root->totalSizeOfSuperPages + root->totalSizeOfDirectMappedPages);
401 } 375 }
402 376
403 static void partitionDecreaseCommittedPages(PartitionRootBase* root, 377 static void partitionDecreaseCommittedPages(PartitionRootBase* root,
404 size_t len) { 378 size_t len) {
405 root->totalSizeOfCommittedPages -= len; 379 root->totalSizeOfCommittedPages -= len;
406 ASSERT(root->totalSizeOfCommittedPages <= 380 DCHECK(root->totalSizeOfCommittedPages <=
407 root->totalSizeOfSuperPages + root->totalSizeOfDirectMappedPages); 381 root->totalSizeOfSuperPages + root->totalSizeOfDirectMappedPages);
408 } 382 }
409 383
410 static ALWAYS_INLINE void partitionDecommitSystemPages(PartitionRootBase* root, 384 static ALWAYS_INLINE void partitionDecommitSystemPages(PartitionRootBase* root,
411 void* addr, 385 void* addr,
412 size_t len) { 386 size_t len) {
413 decommitSystemPages(addr, len); 387 decommitSystemPages(addr, len);
414 partitionDecreaseCommittedPages(root, len); 388 partitionDecreaseCommittedPages(root, len);
415 } 389 }
416 390
417 static ALWAYS_INLINE void partitionRecommitSystemPages(PartitionRootBase* root, 391 static ALWAYS_INLINE void partitionRecommitSystemPages(PartitionRootBase* root,
418 void* addr, 392 void* addr,
419 size_t len) { 393 size_t len) {
420 recommitSystemPages(addr, len); 394 recommitSystemPages(addr, len);
421 partitionIncreaseCommittedPages(root, len); 395 partitionIncreaseCommittedPages(root, len);
422 } 396 }
423 397
424 static ALWAYS_INLINE void* partitionAllocPartitionPages( 398 static ALWAYS_INLINE void* partitionAllocPartitionPages(
425 PartitionRootBase* root, 399 PartitionRootBase* root,
426 int flags, 400 int flags,
427 uint16_t numPartitionPages) { 401 uint16_t numPartitionPages) {
428 ASSERT(!(reinterpret_cast<uintptr_t>(root->nextPartitionPage) % 402 DCHECK(!(reinterpret_cast<uintptr_t>(root->nextPartitionPage) %
429 kPartitionPageSize)); 403 kPartitionPageSize));
430 ASSERT(!(reinterpret_cast<uintptr_t>(root->nextPartitionPageEnd) % 404 DCHECK(!(reinterpret_cast<uintptr_t>(root->nextPartitionPageEnd) %
431 kPartitionPageSize)); 405 kPartitionPageSize));
432 ASSERT(numPartitionPages <= kNumPartitionPagesPerSuperPage); 406 DCHECK(numPartitionPages <= kNumPartitionPagesPerSuperPage);
433 size_t totalSize = kPartitionPageSize * numPartitionPages; 407 size_t totalSize = kPartitionPageSize * numPartitionPages;
434 size_t numPartitionPagesLeft = 408 size_t numPartitionPagesLeft =
435 (root->nextPartitionPageEnd - root->nextPartitionPage) >> 409 (root->nextPartitionPageEnd - root->nextPartitionPage) >>
436 kPartitionPageShift; 410 kPartitionPageShift;
437 if (LIKELY(numPartitionPagesLeft >= numPartitionPages)) { 411 if (LIKELY(numPartitionPagesLeft >= numPartitionPages)) {
438 // In this case, we can still hand out pages from the current super page 412 // In this case, we can still hand out pages from the current super page
439 // allocation. 413 // allocation.
440 char* ret = root->nextPartitionPage; 414 char* ret = root->nextPartitionPage;
441 root->nextPartitionPage += totalSize; 415 root->nextPartitionPage += totalSize;
442 partitionIncreaseCommittedPages(root, totalSize); 416 partitionIncreaseCommittedPages(root, totalSize);
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after
492 // are unused, but we initialize them to 0 so that we get a clear signal 466 // are unused, but we initialize them to 0 so that we get a clear signal
493 // in case they are accidentally used. 467 // in case they are accidentally used.
494 latestExtent->superPageBase = 0; 468 latestExtent->superPageBase = 0;
495 latestExtent->superPagesEnd = 0; 469 latestExtent->superPagesEnd = 0;
496 latestExtent->next = 0; 470 latestExtent->next = 0;
497 471
498 PartitionSuperPageExtentEntry* currentExtent = root->currentExtent; 472 PartitionSuperPageExtentEntry* currentExtent = root->currentExtent;
499 bool isNewExtent = (superPage != requestedAddress); 473 bool isNewExtent = (superPage != requestedAddress);
500 if (UNLIKELY(isNewExtent)) { 474 if (UNLIKELY(isNewExtent)) {
501 if (UNLIKELY(!currentExtent)) { 475 if (UNLIKELY(!currentExtent)) {
502 ASSERT(!root->firstExtent); 476 DCHECK(!root->firstExtent);
503 root->firstExtent = latestExtent; 477 root->firstExtent = latestExtent;
504 } else { 478 } else {
505 ASSERT(currentExtent->superPageBase); 479 DCHECK(currentExtent->superPageBase);
506 currentExtent->next = latestExtent; 480 currentExtent->next = latestExtent;
507 } 481 }
508 root->currentExtent = latestExtent; 482 root->currentExtent = latestExtent;
509 latestExtent->superPageBase = superPage; 483 latestExtent->superPageBase = superPage;
510 latestExtent->superPagesEnd = superPage + kSuperPageSize; 484 latestExtent->superPagesEnd = superPage + kSuperPageSize;
511 } else { 485 } else {
512 // We allocated next to an existing extent so just nudge the size up a 486 // We allocated next to an existing extent so just nudge the size up a
513 // little. 487 // little.
514 ASSERT(currentExtent->superPagesEnd); 488 DCHECK(currentExtent->superPagesEnd);
515 currentExtent->superPagesEnd += kSuperPageSize; 489 currentExtent->superPagesEnd += kSuperPageSize;
516 ASSERT(ret >= currentExtent->superPageBase && 490 DCHECK(ret >= currentExtent->superPageBase &&
517 ret < currentExtent->superPagesEnd); 491 ret < currentExtent->superPagesEnd);
518 } 492 }
519 return ret; 493 return ret;
520 } 494 }
521 495
522 static ALWAYS_INLINE uint16_t 496 static ALWAYS_INLINE uint16_t
523 partitionBucketPartitionPages(const PartitionBucket* bucket) { 497 partitionBucketPartitionPages(const PartitionBucket* bucket) {
524 return (bucket->numSystemPagesPerSlotSpan + 498 return (bucket->numSystemPagesPerSlotSpan +
525 (kNumSystemPagesPerPartitionPage - 1)) / 499 (kNumSystemPagesPerPartitionPage - 1)) /
526 kNumSystemPagesPerPartitionPage; 500 kNumSystemPagesPerPartitionPage;
527 } 501 }
528 502
529 static ALWAYS_INLINE void partitionPageReset(PartitionPage* page) { 503 static ALWAYS_INLINE void partitionPageReset(PartitionPage* page) {
530 ASSERT(partitionPageStateIsDecommitted(page)); 504 DCHECK(partitionPageStateIsDecommitted(page));
531 505
532 page->numUnprovisionedSlots = partitionBucketSlots(page->bucket); 506 page->numUnprovisionedSlots = partitionBucketSlots(page->bucket);
533 ASSERT(page->numUnprovisionedSlots); 507 DCHECK(page->numUnprovisionedSlots);
534 508
535 page->nextPage = nullptr; 509 page->nextPage = nullptr;
536 } 510 }
537 511
538 static ALWAYS_INLINE void partitionPageSetup(PartitionPage* page, 512 static ALWAYS_INLINE void partitionPageSetup(PartitionPage* page,
539 PartitionBucket* bucket) { 513 PartitionBucket* bucket) {
540 // The bucket never changes. We set it up once. 514 // The bucket never changes. We set it up once.
541 page->bucket = bucket; 515 page->bucket = bucket;
542 page->emptyCacheIndex = -1; 516 page->emptyCacheIndex = -1;
543 517
(...skipping 10 matching lines...) Expand all
554 for (uint16_t i = 1; i < numPartitionPages; ++i) { 528 for (uint16_t i = 1; i < numPartitionPages; ++i) {
555 pageCharPtr += kPageMetadataSize; 529 pageCharPtr += kPageMetadataSize;
556 PartitionPage* secondaryPage = 530 PartitionPage* secondaryPage =
557 reinterpret_cast<PartitionPage*>(pageCharPtr); 531 reinterpret_cast<PartitionPage*>(pageCharPtr);
558 secondaryPage->pageOffset = i; 532 secondaryPage->pageOffset = i;
559 } 533 }
560 } 534 }
561 535
562 static ALWAYS_INLINE char* partitionPageAllocAndFillFreelist( 536 static ALWAYS_INLINE char* partitionPageAllocAndFillFreelist(
563 PartitionPage* page) { 537 PartitionPage* page) {
564 ASSERT(page != &PartitionRootGeneric::gSeedPage); 538 DCHECK(page != &PartitionRootGeneric::gSeedPage);
565 uint16_t numSlots = page->numUnprovisionedSlots; 539 uint16_t numSlots = page->numUnprovisionedSlots;
566 ASSERT(numSlots); 540 DCHECK(numSlots);
567 PartitionBucket* bucket = page->bucket; 541 PartitionBucket* bucket = page->bucket;
568 // We should only get here when _every_ slot is either used or unprovisioned. 542 // We should only get here when _every_ slot is either used or unprovisioned.
569 // (The third state is "on the freelist". If we have a non-empty freelist, we 543 // (The third state is "on the freelist". If we have a non-empty freelist, we
570 // should not get here.) 544 // should not get here.)
571 ASSERT(numSlots + page->numAllocatedSlots == partitionBucketSlots(bucket)); 545 DCHECK(numSlots + page->numAllocatedSlots == partitionBucketSlots(bucket));
572 // Similarly, make explicitly sure that the freelist is empty. 546 // Similarly, make explicitly sure that the freelist is empty.
573 ASSERT(!page->freelistHead); 547 DCHECK(!page->freelistHead);
574 ASSERT(page->numAllocatedSlots >= 0); 548 DCHECK(page->numAllocatedSlots >= 0);
575 549
576 size_t size = bucket->slotSize; 550 size_t size = bucket->slotSize;
577 char* base = reinterpret_cast<char*>(partitionPageToPointer(page)); 551 char* base = reinterpret_cast<char*>(partitionPageToPointer(page));
578 char* returnObject = base + (size * page->numAllocatedSlots); 552 char* returnObject = base + (size * page->numAllocatedSlots);
579 char* firstFreelistPointer = returnObject + size; 553 char* firstFreelistPointer = returnObject + size;
580 char* firstFreelistPointerExtent = 554 char* firstFreelistPointerExtent =
581 firstFreelistPointer + sizeof(PartitionFreelistEntry*); 555 firstFreelistPointer + sizeof(PartitionFreelistEntry*);
582 // Our goal is to fault as few system pages as possible. We calculate the 556 // Our goal is to fault as few system pages as possible. We calculate the
583 // page containing the "end" of the returned slot, and then allow freelist 557 // page containing the "end" of the returned slot, and then allow freelist
584 // pointers to be written up to the end of that page. 558 // pointers to be written up to the end of that page.
585 char* subPageLimit = reinterpret_cast<char*>( 559 char* subPageLimit = reinterpret_cast<char*>(
586 WTF::roundUpToSystemPage(reinterpret_cast<size_t>(firstFreelistPointer))); 560 roundUpToSystemPage(reinterpret_cast<size_t>(firstFreelistPointer)));
587 char* slotsLimit = returnObject + (size * numSlots); 561 char* slotsLimit = returnObject + (size * numSlots);
588 char* freelistLimit = subPageLimit; 562 char* freelistLimit = subPageLimit;
589 if (UNLIKELY(slotsLimit < freelistLimit)) 563 if (UNLIKELY(slotsLimit < freelistLimit))
590 freelistLimit = slotsLimit; 564 freelistLimit = slotsLimit;
591 565
592 uint16_t numNewFreelistEntries = 0; 566 uint16_t numNewFreelistEntries = 0;
593 if (LIKELY(firstFreelistPointerExtent <= freelistLimit)) { 567 if (LIKELY(firstFreelistPointerExtent <= freelistLimit)) {
594 // Only consider used space in the slot span. If we consider wasted 568 // Only consider used space in the slot span. If we consider wasted
595 // space, we may get an off-by-one when a freelist pointer fits in the 569 // space, we may get an off-by-one when a freelist pointer fits in the
596 // wasted space, but a slot does not. 570 // wasted space, but a slot does not.
597 // We know we can fit at least one freelist pointer. 571 // We know we can fit at least one freelist pointer.
598 numNewFreelistEntries = 1; 572 numNewFreelistEntries = 1;
599 // Any further entries require space for the whole slot span. 573 // Any further entries require space for the whole slot span.
600 numNewFreelistEntries += static_cast<uint16_t>( 574 numNewFreelistEntries += static_cast<uint16_t>(
601 (freelistLimit - firstFreelistPointerExtent) / size); 575 (freelistLimit - firstFreelistPointerExtent) / size);
602 } 576 }
603 577
604 // We always return an object slot -- that's the +1 below. 578 // We always return an object slot -- that's the +1 below.
605 // We do not neccessarily create any new freelist entries, because we cross 579 // We do not neccessarily create any new freelist entries, because we cross
606 // sub page boundaries frequently for large bucket sizes. 580 // sub page boundaries frequently for large bucket sizes.
607 ASSERT(numNewFreelistEntries + 1 <= numSlots); 581 DCHECK(numNewFreelistEntries + 1 <= numSlots);
608 numSlots -= (numNewFreelistEntries + 1); 582 numSlots -= (numNewFreelistEntries + 1);
609 page->numUnprovisionedSlots = numSlots; 583 page->numUnprovisionedSlots = numSlots;
610 page->numAllocatedSlots++; 584 page->numAllocatedSlots++;
611 585
612 if (LIKELY(numNewFreelistEntries)) { 586 if (LIKELY(numNewFreelistEntries)) {
613 char* freelistPointer = firstFreelistPointer; 587 char* freelistPointer = firstFreelistPointer;
614 PartitionFreelistEntry* entry = 588 PartitionFreelistEntry* entry =
615 reinterpret_cast<PartitionFreelistEntry*>(freelistPointer); 589 reinterpret_cast<PartitionFreelistEntry*>(freelistPointer);
616 page->freelistHead = entry; 590 page->freelistHead = entry;
617 while (--numNewFreelistEntries) { 591 while (--numNewFreelistEntries) {
(...skipping 20 matching lines...) Expand all
638 // decommitted page list and full pages are unlinked from any list. 612 // decommitted page list and full pages are unlinked from any list.
639 static bool partitionSetNewActivePage(PartitionBucket* bucket) { 613 static bool partitionSetNewActivePage(PartitionBucket* bucket) {
640 PartitionPage* page = bucket->activePagesHead; 614 PartitionPage* page = bucket->activePagesHead;
641 if (page == &PartitionRootBase::gSeedPage) 615 if (page == &PartitionRootBase::gSeedPage)
642 return false; 616 return false;
643 617
644 PartitionPage* nextPage; 618 PartitionPage* nextPage;
645 619
646 for (; page; page = nextPage) { 620 for (; page; page = nextPage) {
647 nextPage = page->nextPage; 621 nextPage = page->nextPage;
648 ASSERT(page->bucket == bucket); 622 DCHECK(page->bucket == bucket);
649 ASSERT(page != bucket->emptyPagesHead); 623 DCHECK(page != bucket->emptyPagesHead);
650 ASSERT(page != bucket->decommittedPagesHead); 624 DCHECK(page != bucket->decommittedPagesHead);
651 625
652 // Deal with empty and decommitted pages. 626 // Deal with empty and decommitted pages.
653 if (LIKELY(partitionPageStateIsActive(page))) { 627 if (LIKELY(partitionPageStateIsActive(page))) {
654 // This page is usable because it has freelist entries, or has 628 // This page is usable because it has freelist entries, or has
655 // unprovisioned slots we can create freelist entries from. 629 // unprovisioned slots we can create freelist entries from.
656 bucket->activePagesHead = page; 630 bucket->activePagesHead = page;
657 return true; 631 return true;
658 } 632 }
659 if (LIKELY(partitionPageStateIsEmpty(page))) { 633 if (LIKELY(partitionPageStateIsEmpty(page))) {
660 page->nextPage = bucket->emptyPagesHead; 634 page->nextPage = bucket->emptyPagesHead;
661 bucket->emptyPagesHead = page; 635 bucket->emptyPagesHead = page;
662 } else if (LIKELY(partitionPageStateIsDecommitted(page))) { 636 } else if (LIKELY(partitionPageStateIsDecommitted(page))) {
663 page->nextPage = bucket->decommittedPagesHead; 637 page->nextPage = bucket->decommittedPagesHead;
664 bucket->decommittedPagesHead = page; 638 bucket->decommittedPagesHead = page;
665 } else { 639 } else {
666 ASSERT(partitionPageStateIsFull(page)); 640 DCHECK(partitionPageStateIsFull(page));
667 // If we get here, we found a full page. Skip over it too, and also 641 // If we get here, we found a full page. Skip over it too, and also
668 // tag it as full (via a negative value). We need it tagged so that 642 // tag it as full (via a negative value). We need it tagged so that
669 // free'ing can tell, and move it back into the active page list. 643 // free'ing can tell, and move it back into the active page list.
670 page->numAllocatedSlots = -page->numAllocatedSlots; 644 page->numAllocatedSlots = -page->numAllocatedSlots;
671 ++bucket->numFullPages; 645 ++bucket->numFullPages;
672 // numFullPages is a uint16_t for efficient packing so guard against 646 // numFullPages is a uint16_t for efficient packing so guard against
673 // overflow to be safe. 647 // overflow to be safe.
674 if (UNLIKELY(!bucket->numFullPages)) 648 if (UNLIKELY(!bucket->numFullPages))
675 partitionBucketFull(); 649 partitionBucketFull();
676 // Not necessary but might help stop accidents. 650 // Not necessary but might help stop accidents.
677 page->nextPage = 0; 651 page->nextPage = 0;
678 } 652 }
679 } 653 }
680 654
681 bucket->activePagesHead = &PartitionRootGeneric::gSeedPage; 655 bucket->activePagesHead = &PartitionRootGeneric::gSeedPage;
682 return false; 656 return false;
683 } 657 }
684 658
685 static ALWAYS_INLINE PartitionDirectMapExtent* partitionPageToDirectMapExtent( 659 static ALWAYS_INLINE PartitionDirectMapExtent* partitionPageToDirectMapExtent(
686 PartitionPage* page) { 660 PartitionPage* page) {
687 ASSERT(partitionBucketIsDirectMapped(page->bucket)); 661 DCHECK(partitionBucketIsDirectMapped(page->bucket));
688 return reinterpret_cast<PartitionDirectMapExtent*>( 662 return reinterpret_cast<PartitionDirectMapExtent*>(
689 reinterpret_cast<char*>(page) + 3 * kPageMetadataSize); 663 reinterpret_cast<char*>(page) + 3 * kPageMetadataSize);
690 } 664 }
691 665
692 static ALWAYS_INLINE void partitionPageSetRawSize(PartitionPage* page, 666 static ALWAYS_INLINE void partitionPageSetRawSize(PartitionPage* page,
693 size_t size) { 667 size_t size) {
694 size_t* rawSizePtr = partitionPageGetRawSizePtr(page); 668 size_t* rawSizePtr = partitionPageGetRawSizePtr(page);
695 if (UNLIKELY(rawSizePtr != nullptr)) 669 if (UNLIKELY(rawSizePtr != nullptr))
696 *rawSizePtr = size; 670 *rawSizePtr = size;
697 } 671 }
698 672
699 static ALWAYS_INLINE PartitionPage* partitionDirectMap(PartitionRootBase* root, 673 static ALWAYS_INLINE PartitionPage* partitionDirectMap(PartitionRootBase* root,
700 int flags, 674 int flags,
701 size_t rawSize) { 675 size_t rawSize) {
702 size_t size = partitionDirectMapSize(rawSize); 676 size_t size = partitionDirectMapSize(rawSize);
703 677
704 // Because we need to fake looking like a super page, we need to allocate 678 // Because we need to fake looking like a super page, we need to allocate
705 // a bunch of system pages more than "size": 679 // a bunch of system pages more than "size":
706 // - The first few system pages are the partition page in which the super 680 // - The first few system pages are the partition page in which the super
707 // page metadata is stored. We fault just one system page out of a partition 681 // page metadata is stored. We fault just one system page out of a partition
708 // page sized clump. 682 // page sized clump.
709 // - We add a trailing guard page on 32-bit (on 64-bit we rely on the 683 // - We add a trailing guard page on 32-bit (on 64-bit we rely on the
710 // massive address space plus randomization instead). 684 // massive address space plus randomization instead).
711 size_t mapSize = size + kPartitionPageSize; 685 size_t mapSize = size + kPartitionPageSize;
712 #if !CPU(64BIT) 686 #if !defined(ARCH_CPU_64_BITS)
713 mapSize += kSystemPageSize; 687 mapSize += kSystemPageSize;
714 #endif 688 #endif
715 // Round up to the allocation granularity. 689 // Round up to the allocation granularity.
716 mapSize += kPageAllocationGranularityOffsetMask; 690 mapSize += kPageAllocationGranularityOffsetMask;
717 mapSize &= kPageAllocationGranularityBaseMask; 691 mapSize &= kPageAllocationGranularityBaseMask;
718 692
719 // TODO: these pages will be zero-filled. Consider internalizing an 693 // TODO: these pages will be zero-filled. Consider internalizing an
720 // allocZeroed() API so we can avoid a memset() entirely in this case. 694 // allocZeroed() API so we can avoid a memset() entirely in this case.
721 char* ptr = reinterpret_cast<char*>( 695 char* ptr = reinterpret_cast<char*>(
722 allocPages(0, mapSize, kSuperPageSize, PageAccessible)); 696 allocPages(0, mapSize, kSuperPageSize, PageAccessible));
723 if (UNLIKELY(!ptr)) 697 if (UNLIKELY(!ptr))
724 return nullptr; 698 return nullptr;
725 699
726 size_t committedPageSize = size + kSystemPageSize; 700 size_t committedPageSize = size + kSystemPageSize;
727 root->totalSizeOfDirectMappedPages += committedPageSize; 701 root->totalSizeOfDirectMappedPages += committedPageSize;
728 partitionIncreaseCommittedPages(root, committedPageSize); 702 partitionIncreaseCommittedPages(root, committedPageSize);
729 703
730 char* slot = ptr + kPartitionPageSize; 704 char* slot = ptr + kPartitionPageSize;
731 setSystemPagesInaccessible(ptr + (kSystemPageSize * 2), 705 setSystemPagesInaccessible(ptr + (kSystemPageSize * 2),
732 kPartitionPageSize - (kSystemPageSize * 2)); 706 kPartitionPageSize - (kSystemPageSize * 2));
733 #if !CPU(64BIT) 707 #if !defined(ARCH_CPU_64_BITS)
734 setSystemPagesInaccessible(ptr, kSystemPageSize); 708 setSystemPagesInaccessible(ptr, kSystemPageSize);
735 setSystemPagesInaccessible(slot + size, kSystemPageSize); 709 setSystemPagesInaccessible(slot + size, kSystemPageSize);
736 #endif 710 #endif
737 711
738 PartitionSuperPageExtentEntry* extent = 712 PartitionSuperPageExtentEntry* extent =
739 reinterpret_cast<PartitionSuperPageExtentEntry*>( 713 reinterpret_cast<PartitionSuperPageExtentEntry*>(
740 partitionSuperPageToMetadataArea(ptr)); 714 partitionSuperPageToMetadataArea(ptr));
741 extent->root = root; 715 extent->root = root;
742 // The new structures are all located inside a fresh system page so they 716 // The new structures are all located inside a fresh system page so they
743 // will all be zeroed out. These ASSERTs are for documentation. 717 // will all be zeroed out. These DCHECKs are for documentation.
744 ASSERT(!extent->superPageBase); 718 DCHECK(!extent->superPageBase);
745 ASSERT(!extent->superPagesEnd); 719 DCHECK(!extent->superPagesEnd);
746 ASSERT(!extent->next); 720 DCHECK(!extent->next);
747 PartitionPage* page = partitionPointerToPageNoAlignmentCheck(slot); 721 PartitionPage* page = partitionPointerToPageNoAlignmentCheck(slot);
748 PartitionBucket* bucket = reinterpret_cast<PartitionBucket*>( 722 PartitionBucket* bucket = reinterpret_cast<PartitionBucket*>(
749 reinterpret_cast<char*>(page) + (kPageMetadataSize * 2)); 723 reinterpret_cast<char*>(page) + (kPageMetadataSize * 2));
750 ASSERT(!page->nextPage); 724 DCHECK(!page->nextPage);
751 ASSERT(!page->numAllocatedSlots); 725 DCHECK(!page->numAllocatedSlots);
752 ASSERT(!page->numUnprovisionedSlots); 726 DCHECK(!page->numUnprovisionedSlots);
753 ASSERT(!page->pageOffset); 727 DCHECK(!page->pageOffset);
754 ASSERT(!page->emptyCacheIndex); 728 DCHECK(!page->emptyCacheIndex);
755 page->bucket = bucket; 729 page->bucket = bucket;
756 page->freelistHead = reinterpret_cast<PartitionFreelistEntry*>(slot); 730 page->freelistHead = reinterpret_cast<PartitionFreelistEntry*>(slot);
757 PartitionFreelistEntry* nextEntry = 731 PartitionFreelistEntry* nextEntry =
758 reinterpret_cast<PartitionFreelistEntry*>(slot); 732 reinterpret_cast<PartitionFreelistEntry*>(slot);
759 nextEntry->next = partitionFreelistMask(0); 733 nextEntry->next = partitionFreelistMask(0);
760 734
761 ASSERT(!bucket->activePagesHead); 735 DCHECK(!bucket->activePagesHead);
762 ASSERT(!bucket->emptyPagesHead); 736 DCHECK(!bucket->emptyPagesHead);
763 ASSERT(!bucket->decommittedPagesHead); 737 DCHECK(!bucket->decommittedPagesHead);
764 ASSERT(!bucket->numSystemPagesPerSlotSpan); 738 DCHECK(!bucket->numSystemPagesPerSlotSpan);
765 ASSERT(!bucket->numFullPages); 739 DCHECK(!bucket->numFullPages);
766 bucket->slotSize = size; 740 bucket->slotSize = size;
767 741
768 PartitionDirectMapExtent* mapExtent = partitionPageToDirectMapExtent(page); 742 PartitionDirectMapExtent* mapExtent = partitionPageToDirectMapExtent(page);
769 mapExtent->mapSize = mapSize - kPartitionPageSize - kSystemPageSize; 743 mapExtent->mapSize = mapSize - kPartitionPageSize - kSystemPageSize;
770 mapExtent->bucket = bucket; 744 mapExtent->bucket = bucket;
771 745
772 // Maintain the doubly-linked list of all direct mappings. 746 // Maintain the doubly-linked list of all direct mappings.
773 mapExtent->nextExtent = root->directMapList; 747 mapExtent->nextExtent = root->directMapList;
774 if (mapExtent->nextExtent) 748 if (mapExtent->nextExtent)
775 mapExtent->nextExtent->prevExtent = mapExtent; 749 mapExtent->nextExtent->prevExtent = mapExtent;
776 mapExtent->prevExtent = nullptr; 750 mapExtent->prevExtent = nullptr;
777 root->directMapList = mapExtent; 751 root->directMapList = mapExtent;
778 752
779 return page; 753 return page;
780 } 754 }
781 755
782 static ALWAYS_INLINE void partitionDirectUnmap(PartitionPage* page) { 756 static ALWAYS_INLINE void partitionDirectUnmap(PartitionPage* page) {
783 PartitionRootBase* root = partitionPageToRoot(page); 757 PartitionRootBase* root = partitionPageToRoot(page);
784 const PartitionDirectMapExtent* extent = partitionPageToDirectMapExtent(page); 758 const PartitionDirectMapExtent* extent = partitionPageToDirectMapExtent(page);
785 size_t unmapSize = extent->mapSize; 759 size_t unmapSize = extent->mapSize;
786 760
787 // Maintain the doubly-linked list of all direct mappings. 761 // Maintain the doubly-linked list of all direct mappings.
788 if (extent->prevExtent) { 762 if (extent->prevExtent) {
789 ASSERT(extent->prevExtent->nextExtent == extent); 763 DCHECK(extent->prevExtent->nextExtent == extent);
790 extent->prevExtent->nextExtent = extent->nextExtent; 764 extent->prevExtent->nextExtent = extent->nextExtent;
791 } else { 765 } else {
792 root->directMapList = extent->nextExtent; 766 root->directMapList = extent->nextExtent;
793 } 767 }
794 if (extent->nextExtent) { 768 if (extent->nextExtent) {
795 ASSERT(extent->nextExtent->prevExtent == extent); 769 DCHECK(extent->nextExtent->prevExtent == extent);
796 extent->nextExtent->prevExtent = extent->prevExtent; 770 extent->nextExtent->prevExtent = extent->prevExtent;
797 } 771 }
798 772
799 // Add on the size of the trailing guard page and preceeding partition 773 // Add on the size of the trailing guard page and preceeding partition
800 // page. 774 // page.
801 unmapSize += kPartitionPageSize + kSystemPageSize; 775 unmapSize += kPartitionPageSize + kSystemPageSize;
802 776
803 size_t uncommittedPageSize = page->bucket->slotSize + kSystemPageSize; 777 size_t uncommittedPageSize = page->bucket->slotSize + kSystemPageSize;
804 partitionDecreaseCommittedPages(root, uncommittedPageSize); 778 partitionDecreaseCommittedPages(root, uncommittedPageSize);
805 ASSERT(root->totalSizeOfDirectMappedPages >= uncommittedPageSize); 779 DCHECK(root->totalSizeOfDirectMappedPages >= uncommittedPageSize);
806 root->totalSizeOfDirectMappedPages -= uncommittedPageSize; 780 root->totalSizeOfDirectMappedPages -= uncommittedPageSize;
807 781
808 ASSERT(!(unmapSize & kPageAllocationGranularityOffsetMask)); 782 DCHECK(!(unmapSize & kPageAllocationGranularityOffsetMask));
809 783
810 char* ptr = reinterpret_cast<char*>(partitionPageToPointer(page)); 784 char* ptr = reinterpret_cast<char*>(partitionPageToPointer(page));
811 // Account for the mapping starting a partition page before the actual 785 // Account for the mapping starting a partition page before the actual
812 // allocation address. 786 // allocation address.
813 ptr -= kPartitionPageSize; 787 ptr -= kPartitionPageSize;
814 788
815 freePages(ptr, unmapSize); 789 freePages(ptr, unmapSize);
816 } 790 }
817 791
818 void* partitionAllocSlowPath(PartitionRootBase* root, 792 void* partitionAllocSlowPath(PartitionRootBase* root,
819 int flags, 793 int flags,
820 size_t size, 794 size_t size,
821 PartitionBucket* bucket) { 795 PartitionBucket* bucket) {
822 // The slow path is called when the freelist is empty. 796 // The slow path is called when the freelist is empty.
823 ASSERT(!bucket->activePagesHead->freelistHead); 797 DCHECK(!bucket->activePagesHead->freelistHead);
824 798
825 PartitionPage* newPage = nullptr; 799 PartitionPage* newPage = nullptr;
826 800
827 // For the partitionAllocGeneric API, we have a bunch of buckets marked 801 // For the partitionAllocGeneric API, we have a bunch of buckets marked
828 // as special cases. We bounce them through to the slow path so that we 802 // as special cases. We bounce them through to the slow path so that we
829 // can still have a blazing fast hot path due to lack of corner-case 803 // can still have a blazing fast hot path due to lack of corner-case
830 // branches. 804 // branches.
831 bool returnNull = flags & PartitionAllocReturnNull; 805 bool returnNull = flags & PartitionAllocReturnNull;
832 if (UNLIKELY(partitionBucketIsDirectMapped(bucket))) { 806 if (UNLIKELY(partitionBucketIsDirectMapped(bucket))) {
833 ASSERT(size > kGenericMaxBucketed); 807 DCHECK(size > kGenericMaxBucketed);
834 ASSERT(bucket == &PartitionRootBase::gPagedBucket); 808 DCHECK(bucket == &PartitionRootBase::gPagedBucket);
835 ASSERT(bucket->activePagesHead == &PartitionRootGeneric::gSeedPage); 809 DCHECK(bucket->activePagesHead == &PartitionRootGeneric::gSeedPage);
836 if (size > kGenericMaxDirectMapped) { 810 if (size > kGenericMaxDirectMapped) {
837 if (returnNull) 811 if (returnNull)
838 return nullptr; 812 return nullptr;
839 partitionExcessiveAllocationSize(); 813 partitionExcessiveAllocationSize();
840 } 814 }
841 newPage = partitionDirectMap(root, flags, size); 815 newPage = partitionDirectMap(root, flags, size);
842 } else if (LIKELY(partitionSetNewActivePage(bucket))) { 816 } else if (LIKELY(partitionSetNewActivePage(bucket))) {
843 // First, did we find an active page in the active pages list? 817 // First, did we find an active page in the active pages list?
844 newPage = bucket->activePagesHead; 818 newPage = bucket->activePagesHead;
845 ASSERT(partitionPageStateIsActive(newPage)); 819 DCHECK(partitionPageStateIsActive(newPage));
846 } else if (LIKELY(bucket->emptyPagesHead != nullptr) || 820 } else if (LIKELY(bucket->emptyPagesHead != nullptr) ||
847 LIKELY(bucket->decommittedPagesHead != nullptr)) { 821 LIKELY(bucket->decommittedPagesHead != nullptr)) {
848 // Second, look in our lists of empty and decommitted pages. 822 // Second, look in our lists of empty and decommitted pages.
849 // Check empty pages first, which are preferred, but beware that an 823 // Check empty pages first, which are preferred, but beware that an
850 // empty page might have been decommitted. 824 // empty page might have been decommitted.
851 while (LIKELY((newPage = bucket->emptyPagesHead) != nullptr)) { 825 while (LIKELY((newPage = bucket->emptyPagesHead) != nullptr)) {
852 ASSERT(newPage->bucket == bucket); 826 DCHECK(newPage->bucket == bucket);
853 ASSERT(partitionPageStateIsEmpty(newPage) || 827 DCHECK(partitionPageStateIsEmpty(newPage) ||
854 partitionPageStateIsDecommitted(newPage)); 828 partitionPageStateIsDecommitted(newPage));
855 bucket->emptyPagesHead = newPage->nextPage; 829 bucket->emptyPagesHead = newPage->nextPage;
856 // Accept the empty page unless it got decommitted. 830 // Accept the empty page unless it got decommitted.
857 if (newPage->freelistHead) { 831 if (newPage->freelistHead) {
858 newPage->nextPage = nullptr; 832 newPage->nextPage = nullptr;
859 break; 833 break;
860 } 834 }
861 ASSERT(partitionPageStateIsDecommitted(newPage)); 835 DCHECK(partitionPageStateIsDecommitted(newPage));
862 newPage->nextPage = bucket->decommittedPagesHead; 836 newPage->nextPage = bucket->decommittedPagesHead;
863 bucket->decommittedPagesHead = newPage; 837 bucket->decommittedPagesHead = newPage;
864 } 838 }
865 if (UNLIKELY(!newPage) && LIKELY(bucket->decommittedPagesHead != nullptr)) { 839 if (UNLIKELY(!newPage) && LIKELY(bucket->decommittedPagesHead != nullptr)) {
866 newPage = bucket->decommittedPagesHead; 840 newPage = bucket->decommittedPagesHead;
867 ASSERT(newPage->bucket == bucket); 841 DCHECK(newPage->bucket == bucket);
868 ASSERT(partitionPageStateIsDecommitted(newPage)); 842 DCHECK(partitionPageStateIsDecommitted(newPage));
869 bucket->decommittedPagesHead = newPage->nextPage; 843 bucket->decommittedPagesHead = newPage->nextPage;
870 void* addr = partitionPageToPointer(newPage); 844 void* addr = partitionPageToPointer(newPage);
871 partitionRecommitSystemPages(root, addr, 845 partitionRecommitSystemPages(root, addr,
872 partitionBucketBytes(newPage->bucket)); 846 partitionBucketBytes(newPage->bucket));
873 partitionPageReset(newPage); 847 partitionPageReset(newPage);
874 } 848 }
875 ASSERT(newPage); 849 DCHECK(newPage);
876 } else { 850 } else {
877 // Third. If we get here, we need a brand new page. 851 // Third. If we get here, we need a brand new page.
878 uint16_t numPartitionPages = partitionBucketPartitionPages(bucket); 852 uint16_t numPartitionPages = partitionBucketPartitionPages(bucket);
879 void* rawPages = 853 void* rawPages =
880 partitionAllocPartitionPages(root, flags, numPartitionPages); 854 partitionAllocPartitionPages(root, flags, numPartitionPages);
881 if (LIKELY(rawPages != nullptr)) { 855 if (LIKELY(rawPages != nullptr)) {
882 newPage = partitionPointerToPageNoAlignmentCheck(rawPages); 856 newPage = partitionPointerToPageNoAlignmentCheck(rawPages);
883 partitionPageSetup(newPage, bucket); 857 partitionPageSetup(newPage, bucket);
884 } 858 }
885 } 859 }
886 860
887 // Bail if we had a memory allocation failure. 861 // Bail if we had a memory allocation failure.
888 if (UNLIKELY(!newPage)) { 862 if (UNLIKELY(!newPage)) {
889 ASSERT(bucket->activePagesHead == &PartitionRootGeneric::gSeedPage); 863 DCHECK(bucket->activePagesHead == &PartitionRootGeneric::gSeedPage);
890 if (returnNull) 864 if (returnNull)
891 return nullptr; 865 return nullptr;
892 partitionOutOfMemory(root); 866 partitionOutOfMemory(root);
893 } 867 }
894 868
895 bucket = newPage->bucket; 869 bucket = newPage->bucket;
896 ASSERT(bucket != &PartitionRootBase::gPagedBucket); 870 DCHECK(bucket != &PartitionRootBase::gPagedBucket);
897 bucket->activePagesHead = newPage; 871 bucket->activePagesHead = newPage;
898 partitionPageSetRawSize(newPage, size); 872 partitionPageSetRawSize(newPage, size);
899 873
900 // If we found an active page with free slots, or an empty page, we have a 874 // If we found an active page with free slots, or an empty page, we have a
901 // usable freelist head. 875 // usable freelist head.
902 if (LIKELY(newPage->freelistHead != nullptr)) { 876 if (LIKELY(newPage->freelistHead != nullptr)) {
903 PartitionFreelistEntry* entry = newPage->freelistHead; 877 PartitionFreelistEntry* entry = newPage->freelistHead;
904 PartitionFreelistEntry* newHead = partitionFreelistMask(entry->next); 878 PartitionFreelistEntry* newHead = partitionFreelistMask(entry->next);
905 newPage->freelistHead = newHead; 879 newPage->freelistHead = newHead;
906 newPage->numAllocatedSlots++; 880 newPage->numAllocatedSlots++;
907 return entry; 881 return entry;
908 } 882 }
909 // Otherwise, we need to build the freelist. 883 // Otherwise, we need to build the freelist.
910 ASSERT(newPage->numUnprovisionedSlots); 884 DCHECK(newPage->numUnprovisionedSlots);
911 return partitionPageAllocAndFillFreelist(newPage); 885 return partitionPageAllocAndFillFreelist(newPage);
912 } 886 }
913 887
914 static ALWAYS_INLINE void partitionDecommitPage(PartitionRootBase* root, 888 static ALWAYS_INLINE void partitionDecommitPage(PartitionRootBase* root,
915 PartitionPage* page) { 889 PartitionPage* page) {
916 ASSERT(partitionPageStateIsEmpty(page)); 890 DCHECK(partitionPageStateIsEmpty(page));
917 ASSERT(!partitionBucketIsDirectMapped(page->bucket)); 891 DCHECK(!partitionBucketIsDirectMapped(page->bucket));
918 void* addr = partitionPageToPointer(page); 892 void* addr = partitionPageToPointer(page);
919 partitionDecommitSystemPages(root, addr, partitionBucketBytes(page->bucket)); 893 partitionDecommitSystemPages(root, addr, partitionBucketBytes(page->bucket));
920 894
921 // We actually leave the decommitted page in the active list. We'll sweep 895 // We actually leave the decommitted page in the active list. We'll sweep
922 // it on to the decommitted page list when we next walk the active page 896 // it on to the decommitted page list when we next walk the active page
923 // list. 897 // list.
924 // Pulling this trick enables us to use a singly-linked page list for all 898 // Pulling this trick enables us to use a singly-linked page list for all
925 // cases, which is critical in keeping the page metadata structure down to 899 // cases, which is critical in keeping the page metadata structure down to
926 // 32 bytes in size. 900 // 32 bytes in size.
927 page->freelistHead = 0; 901 page->freelistHead = 0;
928 page->numUnprovisionedSlots = 0; 902 page->numUnprovisionedSlots = 0;
929 ASSERT(partitionPageStateIsDecommitted(page)); 903 DCHECK(partitionPageStateIsDecommitted(page));
930 } 904 }
931 905
932 static void partitionDecommitPageIfPossible(PartitionRootBase* root, 906 static void partitionDecommitPageIfPossible(PartitionRootBase* root,
933 PartitionPage* page) { 907 PartitionPage* page) {
934 ASSERT(page->emptyCacheIndex >= 0); 908 DCHECK(page->emptyCacheIndex >= 0);
935 ASSERT(static_cast<unsigned>(page->emptyCacheIndex) < kMaxFreeableSpans); 909 DCHECK(static_cast<unsigned>(page->emptyCacheIndex) < kMaxFreeableSpans);
936 ASSERT(page == root->globalEmptyPageRing[page->emptyCacheIndex]); 910 DCHECK(page == root->globalEmptyPageRing[page->emptyCacheIndex]);
937 page->emptyCacheIndex = -1; 911 page->emptyCacheIndex = -1;
938 if (partitionPageStateIsEmpty(page)) 912 if (partitionPageStateIsEmpty(page))
939 partitionDecommitPage(root, page); 913 partitionDecommitPage(root, page);
940 } 914 }
941 915
942 static ALWAYS_INLINE void partitionRegisterEmptyPage(PartitionPage* page) { 916 static ALWAYS_INLINE void partitionRegisterEmptyPage(PartitionPage* page) {
943 ASSERT(partitionPageStateIsEmpty(page)); 917 DCHECK(partitionPageStateIsEmpty(page));
944 PartitionRootBase* root = partitionPageToRoot(page); 918 PartitionRootBase* root = partitionPageToRoot(page);
945 919
946 // If the page is already registered as empty, give it another life. 920 // If the page is already registered as empty, give it another life.
947 if (page->emptyCacheIndex != -1) { 921 if (page->emptyCacheIndex != -1) {
948 ASSERT(page->emptyCacheIndex >= 0); 922 DCHECK(page->emptyCacheIndex >= 0);
949 ASSERT(static_cast<unsigned>(page->emptyCacheIndex) < kMaxFreeableSpans); 923 DCHECK(static_cast<unsigned>(page->emptyCacheIndex) < kMaxFreeableSpans);
950 ASSERT(root->globalEmptyPageRing[page->emptyCacheIndex] == page); 924 DCHECK(root->globalEmptyPageRing[page->emptyCacheIndex] == page);
951 root->globalEmptyPageRing[page->emptyCacheIndex] = 0; 925 root->globalEmptyPageRing[page->emptyCacheIndex] = 0;
952 } 926 }
953 927
954 int16_t currentIndex = root->globalEmptyPageRingIndex; 928 int16_t currentIndex = root->globalEmptyPageRingIndex;
955 PartitionPage* pageToDecommit = root->globalEmptyPageRing[currentIndex]; 929 PartitionPage* pageToDecommit = root->globalEmptyPageRing[currentIndex];
956 // The page might well have been re-activated, filled up, etc. before we get 930 // The page might well have been re-activated, filled up, etc. before we get
957 // around to looking at it here. 931 // around to looking at it here.
958 if (pageToDecommit) 932 if (pageToDecommit)
959 partitionDecommitPageIfPossible(root, pageToDecommit); 933 partitionDecommitPageIfPossible(root, pageToDecommit);
960 934
(...skipping 13 matching lines...) Expand all
974 for (size_t i = 0; i < kMaxFreeableSpans; ++i) { 948 for (size_t i = 0; i < kMaxFreeableSpans; ++i) {
975 PartitionPage* page = root->globalEmptyPageRing[i]; 949 PartitionPage* page = root->globalEmptyPageRing[i];
976 if (page) 950 if (page)
977 partitionDecommitPageIfPossible(root, page); 951 partitionDecommitPageIfPossible(root, page);
978 root->globalEmptyPageRing[i] = nullptr; 952 root->globalEmptyPageRing[i] = nullptr;
979 } 953 }
980 } 954 }
981 955
982 void partitionFreeSlowPath(PartitionPage* page) { 956 void partitionFreeSlowPath(PartitionPage* page) {
983 PartitionBucket* bucket = page->bucket; 957 PartitionBucket* bucket = page->bucket;
984 ASSERT(page != &PartitionRootGeneric::gSeedPage); 958 DCHECK(page != &PartitionRootGeneric::gSeedPage);
985 if (LIKELY(page->numAllocatedSlots == 0)) { 959 if (LIKELY(page->numAllocatedSlots == 0)) {
986 // Page became fully unused. 960 // Page became fully unused.
987 if (UNLIKELY(partitionBucketIsDirectMapped(bucket))) { 961 if (UNLIKELY(partitionBucketIsDirectMapped(bucket))) {
988 partitionDirectUnmap(page); 962 partitionDirectUnmap(page);
989 return; 963 return;
990 } 964 }
991 // If it's the current active page, change it. We bounce the page to 965 // If it's the current active page, change it. We bounce the page to
992 // the empty list as a force towards defragmentation. 966 // the empty list as a force towards defragmentation.
993 if (LIKELY(page == bucket->activePagesHead)) 967 if (LIKELY(page == bucket->activePagesHead))
994 (void)partitionSetNewActivePage(bucket); 968 (void)partitionSetNewActivePage(bucket);
995 ASSERT(bucket->activePagesHead != page); 969 DCHECK(bucket->activePagesHead != page);
996 970
997 partitionPageSetRawSize(page, 0); 971 partitionPageSetRawSize(page, 0);
998 ASSERT(!partitionPageGetRawSize(page)); 972 DCHECK(!partitionPageGetRawSize(page));
999 973
1000 partitionRegisterEmptyPage(page); 974 partitionRegisterEmptyPage(page);
1001 } else { 975 } else {
1002 ASSERT(!partitionBucketIsDirectMapped(bucket)); 976 DCHECK(!partitionBucketIsDirectMapped(bucket));
1003 // Ensure that the page is full. That's the only valid case if we 977 // Ensure that the page is full. That's the only valid case if we
1004 // arrive here. 978 // arrive here.
1005 ASSERT(page->numAllocatedSlots < 0); 979 DCHECK(page->numAllocatedSlots < 0);
1006 // A transition of numAllocatedSlots from 0 to -1 is not legal, and 980 // A transition of numAllocatedSlots from 0 to -1 is not legal, and
1007 // likely indicates a double-free. 981 // likely indicates a double-free.
1008 SECURITY_CHECK(page->numAllocatedSlots != -1); 982 CHECK(page->numAllocatedSlots != -1);
1009 page->numAllocatedSlots = -page->numAllocatedSlots - 2; 983 page->numAllocatedSlots = -page->numAllocatedSlots - 2;
1010 ASSERT(page->numAllocatedSlots == partitionBucketSlots(bucket) - 1); 984 DCHECK(page->numAllocatedSlots == partitionBucketSlots(bucket) - 1);
1011 // Fully used page became partially used. It must be put back on the 985 // Fully used page became partially used. It must be put back on the
1012 // non-full page list. Also make it the current page to increase the 986 // non-full page list. Also make it the current page to increase the
1013 // chances of it being filled up again. The old current page will be 987 // chances of it being filled up again. The old current page will be
1014 // the next page. 988 // the next page.
1015 ASSERT(!page->nextPage); 989 DCHECK(!page->nextPage);
1016 if (LIKELY(bucket->activePagesHead != &PartitionRootGeneric::gSeedPage)) 990 if (LIKELY(bucket->activePagesHead != &PartitionRootGeneric::gSeedPage))
1017 page->nextPage = bucket->activePagesHead; 991 page->nextPage = bucket->activePagesHead;
1018 bucket->activePagesHead = page; 992 bucket->activePagesHead = page;
1019 --bucket->numFullPages; 993 --bucket->numFullPages;
1020 // Special case: for a partition page with just a single slot, it may 994 // Special case: for a partition page with just a single slot, it may
1021 // now be empty and we want to run it through the empty logic. 995 // now be empty and we want to run it through the empty logic.
1022 if (UNLIKELY(page->numAllocatedSlots == 0)) 996 if (UNLIKELY(page->numAllocatedSlots == 0))
1023 partitionFreeSlowPath(page); 997 partitionFreeSlowPath(page);
1024 } 998 }
1025 } 999 }
1026 1000
1027 bool partitionReallocDirectMappedInPlace(PartitionRootGeneric* root, 1001 bool partitionReallocDirectMappedInPlace(PartitionRootGeneric* root,
1028 PartitionPage* page, 1002 PartitionPage* page,
1029 size_t rawSize) { 1003 size_t rawSize) {
1030 ASSERT(partitionBucketIsDirectMapped(page->bucket)); 1004 DCHECK(partitionBucketIsDirectMapped(page->bucket));
1031 1005
1032 rawSize = partitionCookieSizeAdjustAdd(rawSize); 1006 rawSize = partitionCookieSizeAdjustAdd(rawSize);
1033 1007
1034 // Note that the new size might be a bucketed size; this function is called 1008 // Note that the new size might be a bucketed size; this function is called
1035 // whenever we're reallocating a direct mapped allocation. 1009 // whenever we're reallocating a direct mapped allocation.
1036 size_t newSize = partitionDirectMapSize(rawSize); 1010 size_t newSize = partitionDirectMapSize(rawSize);
1037 if (newSize < kGenericMinDirectMappedDownsize) 1011 if (newSize < kGenericMinDirectMappedDownsize)
1038 return false; 1012 return false;
1039 1013
1040 // bucket->slotSize is the current size of the allocation. 1014 // bucket->slotSize is the current size of the allocation.
(...skipping 13 matching lines...) Expand all
1054 1028
1055 // Shrink by decommitting unneeded pages and making them inaccessible. 1029 // Shrink by decommitting unneeded pages and making them inaccessible.
1056 size_t decommitSize = currentSize - newSize; 1030 size_t decommitSize = currentSize - newSize;
1057 partitionDecommitSystemPages(root, charPtr + newSize, decommitSize); 1031 partitionDecommitSystemPages(root, charPtr + newSize, decommitSize);
1058 setSystemPagesInaccessible(charPtr + newSize, decommitSize); 1032 setSystemPagesInaccessible(charPtr + newSize, decommitSize);
1059 } else if (newSize <= partitionPageToDirectMapExtent(page)->mapSize) { 1033 } else if (newSize <= partitionPageToDirectMapExtent(page)->mapSize) {
1060 // Grow within the actually allocated memory. Just need to make the 1034 // Grow within the actually allocated memory. Just need to make the
1061 // pages accessible again. 1035 // pages accessible again.
1062 size_t recommitSize = newSize - currentSize; 1036 size_t recommitSize = newSize - currentSize;
1063 bool ret = setSystemPagesAccessible(charPtr + currentSize, recommitSize); 1037 bool ret = setSystemPagesAccessible(charPtr + currentSize, recommitSize);
1064 RELEASE_ASSERT(ret); 1038 CHECK(ret);
1065 partitionRecommitSystemPages(root, charPtr + currentSize, recommitSize); 1039 partitionRecommitSystemPages(root, charPtr + currentSize, recommitSize);
1066 1040
1067 #if ENABLE(ASSERT) 1041 #if DCHECK_IS_ON()
1068 memset(charPtr + currentSize, kUninitializedByte, recommitSize); 1042 memset(charPtr + currentSize, kUninitializedByte, recommitSize);
1069 #endif 1043 #endif
1070 } else { 1044 } else {
1071 // We can't perform the realloc in-place. 1045 // We can't perform the realloc in-place.
1072 // TODO: support this too when possible. 1046 // TODO: support this too when possible.
1073 return false; 1047 return false;
1074 } 1048 }
1075 1049
1076 #if ENABLE(ASSERT) 1050 #if DCHECK_IS_ON()
1077 // Write a new trailing cookie. 1051 // Write a new trailing cookie.
1078 partitionCookieWriteValue(charPtr + rawSize - kCookieSize); 1052 partitionCookieWriteValue(charPtr + rawSize - kCookieSize);
1079 #endif 1053 #endif
1080 1054
1081 partitionPageSetRawSize(page, rawSize); 1055 partitionPageSetRawSize(page, rawSize);
1082 ASSERT(partitionPageGetRawSize(page) == rawSize); 1056 DCHECK(partitionPageGetRawSize(page) == rawSize);
1083 1057
1084 page->bucket->slotSize = newSize; 1058 page->bucket->slotSize = newSize;
1085 return true; 1059 return true;
1086 } 1060 }
1087 1061
1088 void* partitionReallocGeneric(PartitionRootGeneric* root, 1062 void* partitionReallocGeneric(PartitionRootGeneric* root,
1089 void* ptr, 1063 void* ptr,
1090 size_t newSize, 1064 size_t newSize,
1091 const char* typeName) { 1065 const char* typeName) {
1092 #if defined(MEMORY_TOOL_REPLACES_ALLOCATOR) 1066 #if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
1093 return realloc(ptr, newSize); 1067 return realloc(ptr, newSize);
1094 #else 1068 #else
1095 if (UNLIKELY(!ptr)) 1069 if (UNLIKELY(!ptr))
1096 return partitionAllocGeneric(root, newSize, typeName); 1070 return partitionAllocGeneric(root, newSize, typeName);
1097 if (UNLIKELY(!newSize)) { 1071 if (UNLIKELY(!newSize)) {
1098 partitionFreeGeneric(root, ptr); 1072 partitionFreeGeneric(root, ptr);
1099 return 0; 1073 return 0;
1100 } 1074 }
1101 1075
1102 if (newSize > kGenericMaxDirectMapped) 1076 if (newSize > kGenericMaxDirectMapped)
1103 partitionExcessiveAllocationSize(); 1077 partitionExcessiveAllocationSize();
1104 1078
1105 ASSERT(partitionPointerIsValid(partitionCookieFreePointerAdjust(ptr))); 1079 DCHECK(partitionPointerIsValid(partitionCookieFreePointerAdjust(ptr)));
1106 1080
1107 PartitionPage* page = 1081 PartitionPage* page =
1108 partitionPointerToPage(partitionCookieFreePointerAdjust(ptr)); 1082 partitionPointerToPage(partitionCookieFreePointerAdjust(ptr));
1109 1083
1110 if (UNLIKELY(partitionBucketIsDirectMapped(page->bucket))) { 1084 if (UNLIKELY(partitionBucketIsDirectMapped(page->bucket))) {
1111 // We may be able to perform the realloc in place by changing the 1085 // We may be able to perform the realloc in place by changing the
1112 // accessibility of memory pages and, if reducing the size, decommitting 1086 // accessibility of memory pages and, if reducing the size, decommitting
1113 // them. 1087 // them.
1114 if (partitionReallocDirectMappedInPlace(root, page, newSize)) { 1088 if (partitionReallocDirectMappedInPlace(root, page, newSize)) {
1115 PartitionAllocHooks::reallocHookIfEnabled(ptr, ptr, newSize, typeName); 1089 PartitionAllocHooks::reallocHookIfEnabled(ptr, ptr, newSize, typeName);
(...skipping 30 matching lines...) Expand all
1146 const PartitionBucket* bucket = page->bucket; 1120 const PartitionBucket* bucket = page->bucket;
1147 size_t slotSize = bucket->slotSize; 1121 size_t slotSize = bucket->slotSize;
1148 if (slotSize < kSystemPageSize || !page->numAllocatedSlots) 1122 if (slotSize < kSystemPageSize || !page->numAllocatedSlots)
1149 return 0; 1123 return 0;
1150 1124
1151 size_t bucketNumSlots = partitionBucketSlots(bucket); 1125 size_t bucketNumSlots = partitionBucketSlots(bucket);
1152 size_t discardableBytes = 0; 1126 size_t discardableBytes = 0;
1153 1127
1154 size_t rawSize = partitionPageGetRawSize(const_cast<PartitionPage*>(page)); 1128 size_t rawSize = partitionPageGetRawSize(const_cast<PartitionPage*>(page));
1155 if (rawSize) { 1129 if (rawSize) {
1156 uint32_t usedBytes = 1130 uint32_t usedBytes = static_cast<uint32_t>(roundUpToSystemPage(rawSize));
1157 static_cast<uint32_t>(WTF::roundUpToSystemPage(rawSize));
1158 discardableBytes = bucket->slotSize - usedBytes; 1131 discardableBytes = bucket->slotSize - usedBytes;
1159 if (discardableBytes && discard) { 1132 if (discardableBytes && discard) {
1160 char* ptr = reinterpret_cast<char*>(partitionPageToPointer(page)); 1133 char* ptr = reinterpret_cast<char*>(partitionPageToPointer(page));
1161 ptr += usedBytes; 1134 ptr += usedBytes;
1162 discardSystemPages(ptr, discardableBytes); 1135 discardSystemPages(ptr, discardableBytes);
1163 } 1136 }
1164 return discardableBytes; 1137 return discardableBytes;
1165 } 1138 }
1166 1139
1167 const size_t maxSlotCount = 1140 const size_t maxSlotCount =
1168 (kPartitionPageSize * kMaxPartitionPagesPerSlotSpan) / kSystemPageSize; 1141 (kPartitionPageSize * kMaxPartitionPagesPerSlotSpan) / kSystemPageSize;
1169 ASSERT(bucketNumSlots <= maxSlotCount); 1142 DCHECK(bucketNumSlots <= maxSlotCount);
1170 ASSERT(page->numUnprovisionedSlots < bucketNumSlots); 1143 DCHECK(page->numUnprovisionedSlots < bucketNumSlots);
1171 size_t numSlots = bucketNumSlots - page->numUnprovisionedSlots; 1144 size_t numSlots = bucketNumSlots - page->numUnprovisionedSlots;
1172 char slotUsage[maxSlotCount]; 1145 char slotUsage[maxSlotCount];
1173 size_t lastSlot = static_cast<size_t>(-1); 1146 size_t lastSlot = static_cast<size_t>(-1);
1174 memset(slotUsage, 1, numSlots); 1147 memset(slotUsage, 1, numSlots);
1175 char* ptr = reinterpret_cast<char*>(partitionPageToPointer(page)); 1148 char* ptr = reinterpret_cast<char*>(partitionPageToPointer(page));
1176 PartitionFreelistEntry* entry = page->freelistHead; 1149 PartitionFreelistEntry* entry = page->freelistHead;
1177 // First, walk the freelist for this page and make a bitmap of which slots 1150 // First, walk the freelist for this page and make a bitmap of which slots
1178 // are not in use. 1151 // are not in use.
1179 while (entry) { 1152 while (entry) {
1180 size_t slotIndex = (reinterpret_cast<char*>(entry) - ptr) / slotSize; 1153 size_t slotIndex = (reinterpret_cast<char*>(entry) - ptr) / slotSize;
1181 ASSERT(slotIndex < numSlots); 1154 DCHECK(slotIndex < numSlots);
1182 slotUsage[slotIndex] = 0; 1155 slotUsage[slotIndex] = 0;
1183 entry = partitionFreelistMask(entry->next); 1156 entry = partitionFreelistMask(entry->next);
1184 // If we have a slot where the masked freelist entry is 0, we can 1157 // If we have a slot where the masked freelist entry is 0, we can
1185 // actually discard that freelist entry because touching a discarded 1158 // actually discard that freelist entry because touching a discarded
1186 // page is guaranteed to return original content or 0. 1159 // page is guaranteed to return original content or 0.
1187 // (Note that this optimization won't fire on big endian machines 1160 // (Note that this optimization won't fire on big endian machines
1188 // because the masking function is negation.) 1161 // because the masking function is negation.)
1189 if (!partitionFreelistMask(entry)) 1162 if (!partitionFreelistMask(entry))
1190 lastSlot = slotIndex; 1163 lastSlot = slotIndex;
1191 } 1164 }
1192 1165
1193 // If the slot(s) at the end of the slot span are not in used, we can 1166 // If the slot(s) at the end of the slot span are not in used, we can
1194 // truncate them entirely and rewrite the freelist. 1167 // truncate them entirely and rewrite the freelist.
1195 size_t truncatedSlots = 0; 1168 size_t truncatedSlots = 0;
1196 while (!slotUsage[numSlots - 1]) { 1169 while (!slotUsage[numSlots - 1]) {
1197 truncatedSlots++; 1170 truncatedSlots++;
1198 numSlots--; 1171 numSlots--;
1199 ASSERT(numSlots); 1172 DCHECK(numSlots);
1200 } 1173 }
1201 // First, do the work of calculating the discardable bytes. Don't actually 1174 // First, do the work of calculating the discardable bytes. Don't actually
1202 // discard anything unless the discard flag was passed in. 1175 // discard anything unless the discard flag was passed in.
1203 char* beginPtr = nullptr; 1176 char* beginPtr = nullptr;
1204 char* endPtr = nullptr; 1177 char* endPtr = nullptr;
1205 size_t unprovisionedBytes = 0; 1178 size_t unprovisionedBytes = 0;
1206 if (truncatedSlots) { 1179 if (truncatedSlots) {
1207 beginPtr = ptr + (numSlots * slotSize); 1180 beginPtr = ptr + (numSlots * slotSize);
1208 endPtr = beginPtr + (slotSize * truncatedSlots); 1181 endPtr = beginPtr + (slotSize * truncatedSlots);
1209 beginPtr = reinterpret_cast<char*>( 1182 beginPtr = reinterpret_cast<char*>(
1210 WTF::roundUpToSystemPage(reinterpret_cast<size_t>(beginPtr))); 1183 roundUpToSystemPage(reinterpret_cast<size_t>(beginPtr)));
1211 // We round the end pointer here up and not down because we're at the 1184 // We round the end pointer here up and not down because we're at the
1212 // end of a slot span, so we "own" all the way up the page boundary. 1185 // end of a slot span, so we "own" all the way up the page boundary.
1213 endPtr = reinterpret_cast<char*>( 1186 endPtr = reinterpret_cast<char*>(
1214 WTF::roundUpToSystemPage(reinterpret_cast<size_t>(endPtr))); 1187 roundUpToSystemPage(reinterpret_cast<size_t>(endPtr)));
1215 ASSERT(endPtr <= ptr + partitionBucketBytes(bucket)); 1188 DCHECK(endPtr <= ptr + partitionBucketBytes(bucket));
1216 if (beginPtr < endPtr) { 1189 if (beginPtr < endPtr) {
1217 unprovisionedBytes = endPtr - beginPtr; 1190 unprovisionedBytes = endPtr - beginPtr;
1218 discardableBytes += unprovisionedBytes; 1191 discardableBytes += unprovisionedBytes;
1219 } 1192 }
1220 } 1193 }
1221 if (unprovisionedBytes && discard) { 1194 if (unprovisionedBytes && discard) {
1222 ASSERT(truncatedSlots > 0); 1195 DCHECK(truncatedSlots > 0);
1223 size_t numNewEntries = 0; 1196 size_t numNewEntries = 0;
1224 page->numUnprovisionedSlots += static_cast<uint16_t>(truncatedSlots); 1197 page->numUnprovisionedSlots += static_cast<uint16_t>(truncatedSlots);
1225 // Rewrite the freelist. 1198 // Rewrite the freelist.
1226 PartitionFreelistEntry** entryPtr = &page->freelistHead; 1199 PartitionFreelistEntry** entryPtr = &page->freelistHead;
1227 for (size_t slotIndex = 0; slotIndex < numSlots; ++slotIndex) { 1200 for (size_t slotIndex = 0; slotIndex < numSlots; ++slotIndex) {
1228 if (slotUsage[slotIndex]) 1201 if (slotUsage[slotIndex])
1229 continue; 1202 continue;
1230 PartitionFreelistEntry* entry = reinterpret_cast<PartitionFreelistEntry*>( 1203 PartitionFreelistEntry* entry = reinterpret_cast<PartitionFreelistEntry*>(
1231 ptr + (slotSize * slotIndex)); 1204 ptr + (slotSize * slotIndex));
1232 *entryPtr = partitionFreelistMask(entry); 1205 *entryPtr = partitionFreelistMask(entry);
1233 entryPtr = reinterpret_cast<PartitionFreelistEntry**>(entry); 1206 entryPtr = reinterpret_cast<PartitionFreelistEntry**>(entry);
1234 numNewEntries++; 1207 numNewEntries++;
1235 } 1208 }
1236 // Terminate the freelist chain. 1209 // Terminate the freelist chain.
1237 *entryPtr = nullptr; 1210 *entryPtr = nullptr;
1238 // The freelist head is stored unmasked. 1211 // The freelist head is stored unmasked.
1239 page->freelistHead = partitionFreelistMask(page->freelistHead); 1212 page->freelistHead = partitionFreelistMask(page->freelistHead);
1240 ASSERT(numNewEntries == numSlots - page->numAllocatedSlots); 1213 DCHECK(numNewEntries == numSlots - page->numAllocatedSlots);
1241 // Discard the memory. 1214 // Discard the memory.
1242 discardSystemPages(beginPtr, unprovisionedBytes); 1215 discardSystemPages(beginPtr, unprovisionedBytes);
1243 } 1216 }
1244 1217
1245 // Next, walk the slots and for any not in use, consider where the system 1218 // Next, walk the slots and for any not in use, consider where the system
1246 // page boundaries occur. We can release any system pages back to the 1219 // page boundaries occur. We can release any system pages back to the
1247 // system as long as we don't interfere with a freelist pointer or an 1220 // system as long as we don't interfere with a freelist pointer or an
1248 // adjacent slot. 1221 // adjacent slot.
1249 for (size_t i = 0; i < numSlots; ++i) { 1222 for (size_t i = 0; i < numSlots; ++i) {
1250 if (slotUsage[i]) 1223 if (slotUsage[i])
1251 continue; 1224 continue;
1252 // The first address we can safely discard is just after the freelist 1225 // The first address we can safely discard is just after the freelist
1253 // pointer. There's one quirk: if the freelist pointer is actually a 1226 // pointer. There's one quirk: if the freelist pointer is actually a
1254 // null, we can discard that pointer value too. 1227 // null, we can discard that pointer value too.
1255 char* beginPtr = ptr + (i * slotSize); 1228 char* beginPtr = ptr + (i * slotSize);
1256 char* endPtr = beginPtr + slotSize; 1229 char* endPtr = beginPtr + slotSize;
1257 if (i != lastSlot) 1230 if (i != lastSlot)
1258 beginPtr += sizeof(PartitionFreelistEntry); 1231 beginPtr += sizeof(PartitionFreelistEntry);
1259 beginPtr = reinterpret_cast<char*>( 1232 beginPtr = reinterpret_cast<char*>(
1260 WTF::roundUpToSystemPage(reinterpret_cast<size_t>(beginPtr))); 1233 roundUpToSystemPage(reinterpret_cast<size_t>(beginPtr)));
1261 endPtr = reinterpret_cast<char*>( 1234 endPtr = reinterpret_cast<char*>(
1262 WTF::roundDownToSystemPage(reinterpret_cast<size_t>(endPtr))); 1235 roundDownToSystemPage(reinterpret_cast<size_t>(endPtr)));
1263 if (beginPtr < endPtr) { 1236 if (beginPtr < endPtr) {
1264 size_t partialSlotBytes = endPtr - beginPtr; 1237 size_t partialSlotBytes = endPtr - beginPtr;
1265 discardableBytes += partialSlotBytes; 1238 discardableBytes += partialSlotBytes;
1266 if (discard) 1239 if (discard)
1267 discardSystemPages(beginPtr, partialSlotBytes); 1240 discardSystemPages(beginPtr, partialSlotBytes);
1268 } 1241 }
1269 } 1242 }
1270 return discardableBytes; 1243 return discardableBytes;
1271 } 1244 }
1272 1245
1273 static void partitionPurgeBucket(PartitionBucket* bucket) { 1246 static void partitionPurgeBucket(PartitionBucket* bucket) {
1274 if (bucket->activePagesHead != &PartitionRootGeneric::gSeedPage) { 1247 if (bucket->activePagesHead != &PartitionRootGeneric::gSeedPage) {
1275 for (PartitionPage* page = bucket->activePagesHead; page; 1248 for (PartitionPage* page = bucket->activePagesHead; page;
1276 page = page->nextPage) { 1249 page = page->nextPage) {
1277 ASSERT(page != &PartitionRootGeneric::gSeedPage); 1250 DCHECK(page != &PartitionRootGeneric::gSeedPage);
1278 (void)partitionPurgePage(page, true); 1251 (void)partitionPurgePage(page, true);
1279 } 1252 }
1280 } 1253 }
1281 } 1254 }
1282 1255
1283 void partitionPurgeMemory(PartitionRoot* root, int flags) { 1256 void partitionPurgeMemory(PartitionRoot* root, int flags) {
1284 if (flags & PartitionPurgeDecommitEmptyPages) 1257 if (flags & PartitionPurgeDecommitEmptyPages)
1285 partitionDecommitEmptyPages(root); 1258 partitionDecommitEmptyPages(root);
1286 // We don't currently do anything for PartitionPurgeDiscardUnusedSystemPages 1259 // We don't currently do anything for PartitionPurgeDiscardUnusedSystemPages
1287 // here because that flag is only useful for allocations >= system page 1260 // here because that flag is only useful for allocations >= system page
1288 // size. We only have allocations that large inside generic partitions 1261 // size. We only have allocations that large inside generic partitions
1289 // at the moment. 1262 // at the moment.
1290 } 1263 }
1291 1264
1292 void partitionPurgeMemoryGeneric(PartitionRootGeneric* root, int flags) { 1265 void partitionPurgeMemoryGeneric(PartitionRootGeneric* root, int flags) {
1293 SpinLock::Guard guard(root->lock); 1266 subtle::SpinLock::Guard guard(root->lock);
1294 if (flags & PartitionPurgeDecommitEmptyPages) 1267 if (flags & PartitionPurgeDecommitEmptyPages)
1295 partitionDecommitEmptyPages(root); 1268 partitionDecommitEmptyPages(root);
1296 if (flags & PartitionPurgeDiscardUnusedSystemPages) { 1269 if (flags & PartitionPurgeDiscardUnusedSystemPages) {
1297 for (size_t i = 0; i < kGenericNumBuckets; ++i) { 1270 for (size_t i = 0; i < kGenericNumBuckets; ++i) {
1298 PartitionBucket* bucket = &root->buckets[i]; 1271 PartitionBucket* bucket = &root->buckets[i];
1299 if (bucket->slotSize >= kSystemPageSize) 1272 if (bucket->slotSize >= kSystemPageSize)
1300 partitionPurgeBucket(bucket); 1273 partitionPurgeBucket(bucket);
1301 } 1274 }
1302 } 1275 }
1303 } 1276 }
(...skipping 11 matching lines...) Expand all
1315 partitionPurgePage(const_cast<PartitionPage*>(page), false); 1288 partitionPurgePage(const_cast<PartitionPage*>(page), false);
1316 1289
1317 size_t rawSize = partitionPageGetRawSize(const_cast<PartitionPage*>(page)); 1290 size_t rawSize = partitionPageGetRawSize(const_cast<PartitionPage*>(page));
1318 if (rawSize) 1291 if (rawSize)
1319 statsOut->activeBytes += static_cast<uint32_t>(rawSize); 1292 statsOut->activeBytes += static_cast<uint32_t>(rawSize);
1320 else 1293 else
1321 statsOut->activeBytes += 1294 statsOut->activeBytes +=
1322 (page->numAllocatedSlots * statsOut->bucketSlotSize); 1295 (page->numAllocatedSlots * statsOut->bucketSlotSize);
1323 1296
1324 size_t pageBytesResident = 1297 size_t pageBytesResident =
1325 WTF::roundUpToSystemPage((bucketNumSlots - page->numUnprovisionedSlots) * 1298 roundUpToSystemPage((bucketNumSlots - page->numUnprovisionedSlots) *
1326 statsOut->bucketSlotSize); 1299 statsOut->bucketSlotSize);
1327 statsOut->residentBytes += pageBytesResident; 1300 statsOut->residentBytes += pageBytesResident;
1328 if (partitionPageStateIsEmpty(page)) { 1301 if (partitionPageStateIsEmpty(page)) {
1329 statsOut->decommittableBytes += pageBytesResident; 1302 statsOut->decommittableBytes += pageBytesResident;
1330 ++statsOut->numEmptyPages; 1303 ++statsOut->numEmptyPages;
1331 } else if (partitionPageStateIsFull(page)) { 1304 } else if (partitionPageStateIsFull(page)) {
1332 ++statsOut->numFullPages; 1305 ++statsOut->numFullPages;
1333 } else { 1306 } else {
1334 ASSERT(partitionPageStateIsActive(page)); 1307 DCHECK(partitionPageStateIsActive(page));
1335 ++statsOut->numActivePages; 1308 ++statsOut->numActivePages;
1336 } 1309 }
1337 } 1310 }
1338 1311
1339 static void partitionDumpBucketStats(PartitionBucketMemoryStats* statsOut, 1312 static void partitionDumpBucketStats(PartitionBucketMemoryStats* statsOut,
1340 const PartitionBucket* bucket) { 1313 const PartitionBucket* bucket) {
1341 ASSERT(!partitionBucketIsDirectMapped(bucket)); 1314 DCHECK(!partitionBucketIsDirectMapped(bucket));
1342 statsOut->isValid = false; 1315 statsOut->isValid = false;
1343 // If the active page list is empty (== &PartitionRootGeneric::gSeedPage), 1316 // If the active page list is empty (== &PartitionRootGeneric::gSeedPage),
1344 // the bucket might still need to be reported if it has a list of empty, 1317 // the bucket might still need to be reported if it has a list of empty,
1345 // decommitted or full pages. 1318 // decommitted or full pages.
1346 if (bucket->activePagesHead == &PartitionRootGeneric::gSeedPage && 1319 if (bucket->activePagesHead == &PartitionRootGeneric::gSeedPage &&
1347 !bucket->emptyPagesHead && !bucket->decommittedPagesHead && 1320 !bucket->emptyPagesHead && !bucket->decommittedPagesHead &&
1348 !bucket->numFullPages) 1321 !bucket->numFullPages)
1349 return; 1322 return;
1350 1323
1351 memset(statsOut, '\0', sizeof(*statsOut)); 1324 memset(statsOut, '\0', sizeof(*statsOut));
1352 statsOut->isValid = true; 1325 statsOut->isValid = true;
1353 statsOut->isDirectMap = false; 1326 statsOut->isDirectMap = false;
1354 statsOut->numFullPages = static_cast<size_t>(bucket->numFullPages); 1327 statsOut->numFullPages = static_cast<size_t>(bucket->numFullPages);
1355 statsOut->bucketSlotSize = bucket->slotSize; 1328 statsOut->bucketSlotSize = bucket->slotSize;
1356 uint16_t bucketNumSlots = partitionBucketSlots(bucket); 1329 uint16_t bucketNumSlots = partitionBucketSlots(bucket);
1357 size_t bucketUsefulStorage = statsOut->bucketSlotSize * bucketNumSlots; 1330 size_t bucketUsefulStorage = statsOut->bucketSlotSize * bucketNumSlots;
1358 statsOut->allocatedPageSize = partitionBucketBytes(bucket); 1331 statsOut->allocatedPageSize = partitionBucketBytes(bucket);
1359 statsOut->activeBytes = bucket->numFullPages * bucketUsefulStorage; 1332 statsOut->activeBytes = bucket->numFullPages * bucketUsefulStorage;
1360 statsOut->residentBytes = bucket->numFullPages * statsOut->allocatedPageSize; 1333 statsOut->residentBytes = bucket->numFullPages * statsOut->allocatedPageSize;
1361 1334
1362 for (const PartitionPage* page = bucket->emptyPagesHead; page; 1335 for (const PartitionPage* page = bucket->emptyPagesHead; page;
1363 page = page->nextPage) { 1336 page = page->nextPage) {
1364 ASSERT(partitionPageStateIsEmpty(page) || 1337 DCHECK(partitionPageStateIsEmpty(page) ||
1365 partitionPageStateIsDecommitted(page)); 1338 partitionPageStateIsDecommitted(page));
1366 partitionDumpPageStats(statsOut, page); 1339 partitionDumpPageStats(statsOut, page);
1367 } 1340 }
1368 for (const PartitionPage* page = bucket->decommittedPagesHead; page; 1341 for (const PartitionPage* page = bucket->decommittedPagesHead; page;
1369 page = page->nextPage) { 1342 page = page->nextPage) {
1370 ASSERT(partitionPageStateIsDecommitted(page)); 1343 DCHECK(partitionPageStateIsDecommitted(page));
1371 partitionDumpPageStats(statsOut, page); 1344 partitionDumpPageStats(statsOut, page);
1372 } 1345 }
1373 1346
1374 if (bucket->activePagesHead != &PartitionRootGeneric::gSeedPage) { 1347 if (bucket->activePagesHead != &PartitionRootGeneric::gSeedPage) {
1375 for (const PartitionPage* page = bucket->activePagesHead; page; 1348 for (const PartitionPage* page = bucket->activePagesHead; page;
1376 page = page->nextPage) { 1349 page = page->nextPage) {
1377 ASSERT(page != &PartitionRootGeneric::gSeedPage); 1350 DCHECK(page != &PartitionRootGeneric::gSeedPage);
1378 partitionDumpPageStats(statsOut, page); 1351 partitionDumpPageStats(statsOut, page);
1379 } 1352 }
1380 } 1353 }
1381 } 1354 }
1382 1355
1383 void partitionDumpStatsGeneric(PartitionRootGeneric* partition, 1356 void partitionDumpStatsGeneric(PartitionRootGeneric* partition,
1384 const char* partitionName, 1357 const char* partitionName,
1385 bool isLightDump, 1358 bool isLightDump,
1386 PartitionStatsDumper* partitionStatsDumper) { 1359 PartitionStatsDumper* partitionStatsDumper) {
1387 PartitionBucketMemoryStats bucketStats[kGenericNumBuckets]; 1360 PartitionBucketMemoryStats bucketStats[kGenericNumBuckets];
1388 static const size_t kMaxReportableDirectMaps = 4096; 1361 static const size_t kMaxReportableDirectMaps = 4096;
1389 uint32_t directMapLengths[kMaxReportableDirectMaps]; 1362 uint32_t directMapLengths[kMaxReportableDirectMaps];
1390 size_t numDirectMappedAllocations = 0; 1363 size_t numDirectMappedAllocations = 0;
1391 1364
1392 { 1365 {
1393 SpinLock::Guard guard(partition->lock); 1366 subtle::SpinLock::Guard guard(partition->lock);
1394 1367
1395 for (size_t i = 0; i < kGenericNumBuckets; ++i) { 1368 for (size_t i = 0; i < kGenericNumBuckets; ++i) {
1396 const PartitionBucket* bucket = &partition->buckets[i]; 1369 const PartitionBucket* bucket = &partition->buckets[i];
1397 // Don't report the pseudo buckets that the generic allocator sets up in 1370 // Don't report the pseudo buckets that the generic allocator sets up in
1398 // order to preserve a fast size->bucket map (see 1371 // order to preserve a fast size->bucket map (see
1399 // partitionAllocGenericInit for details). 1372 // partitionAllocGenericInit for details).
1400 if (!bucket->activePagesHead) 1373 if (!bucket->activePagesHead)
1401 bucketStats[i].isValid = false; 1374 bucketStats[i].isValid = false;
1402 else 1375 else
1403 partitionDumpBucketStats(&bucketStats[i], bucket); 1376 partitionDumpBucketStats(&bucketStats[i], bucket);
1404 } 1377 }
1405 1378
1406 for (PartitionDirectMapExtent* extent = partition->directMapList; extent; 1379 for (PartitionDirectMapExtent* extent = partition->directMapList; extent;
1407 extent = extent->nextExtent) { 1380 extent = extent->nextExtent) {
1408 ASSERT(!extent->nextExtent || extent->nextExtent->prevExtent == extent); 1381 DCHECK(!extent->nextExtent || extent->nextExtent->prevExtent == extent);
1409 directMapLengths[numDirectMappedAllocations] = extent->bucket->slotSize; 1382 directMapLengths[numDirectMappedAllocations] = extent->bucket->slotSize;
1410 ++numDirectMappedAllocations; 1383 ++numDirectMappedAllocations;
1411 if (numDirectMappedAllocations == kMaxReportableDirectMaps) 1384 if (numDirectMappedAllocations == kMaxReportableDirectMaps)
1412 break; 1385 break;
1413 } 1386 }
1414 } 1387 }
1415 1388
1416 // partitionsDumpBucketStats is called after collecting stats because it 1389 // partitionsDumpBucketStats is called after collecting stats because it
1417 // can try to allocate using PartitionAllocGeneric and it can't obtain the 1390 // can try to allocate using PartitionAllocGeneric and it can't obtain the
1418 // lock. 1391 // lock.
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after
1456 partitionStatsDumper->partitionDumpTotals(partitionName, &partitionStats); 1429 partitionStatsDumper->partitionDumpTotals(partitionName, &partitionStats);
1457 } 1430 }
1458 1431
1459 void partitionDumpStats(PartitionRoot* partition, 1432 void partitionDumpStats(PartitionRoot* partition,
1460 const char* partitionName, 1433 const char* partitionName,
1461 bool isLightDump, 1434 bool isLightDump,
1462 PartitionStatsDumper* partitionStatsDumper) { 1435 PartitionStatsDumper* partitionStatsDumper) {
1463 static const size_t kMaxReportableBuckets = 4096 / sizeof(void*); 1436 static const size_t kMaxReportableBuckets = 4096 / sizeof(void*);
1464 PartitionBucketMemoryStats memoryStats[kMaxReportableBuckets]; 1437 PartitionBucketMemoryStats memoryStats[kMaxReportableBuckets];
1465 const size_t partitionNumBuckets = partition->numBuckets; 1438 const size_t partitionNumBuckets = partition->numBuckets;
1466 ASSERT(partitionNumBuckets <= kMaxReportableBuckets); 1439 DCHECK(partitionNumBuckets <= kMaxReportableBuckets);
1467 1440
1468 for (size_t i = 0; i < partitionNumBuckets; ++i) 1441 for (size_t i = 0; i < partitionNumBuckets; ++i)
1469 partitionDumpBucketStats(&memoryStats[i], &partition->buckets()[i]); 1442 partitionDumpBucketStats(&memoryStats[i], &partition->buckets()[i]);
1470 1443
1471 // partitionsDumpBucketStats is called after collecting stats because it 1444 // partitionsDumpBucketStats is called after collecting stats because it
1472 // can use PartitionAlloc to allocate and this can affect the statistics. 1445 // can use PartitionAlloc to allocate and this can affect the statistics.
1473 PartitionMemoryStats partitionStats = {0}; 1446 PartitionMemoryStats partitionStats = {0};
1474 partitionStats.totalMmappedBytes = partition->totalSizeOfSuperPages; 1447 partitionStats.totalMmappedBytes = partition->totalSizeOfSuperPages;
1475 partitionStats.totalCommittedBytes = partition->totalSizeOfCommittedPages; 1448 partitionStats.totalCommittedBytes = partition->totalSizeOfCommittedPages;
1476 ASSERT(!partition->totalSizeOfDirectMappedPages); 1449 DCHECK(!partition->totalSizeOfDirectMappedPages);
1477 for (size_t i = 0; i < partitionNumBuckets; ++i) { 1450 for (size_t i = 0; i < partitionNumBuckets; ++i) {
1478 if (memoryStats[i].isValid) { 1451 if (memoryStats[i].isValid) {
1479 partitionStats.totalResidentBytes += memoryStats[i].residentBytes; 1452 partitionStats.totalResidentBytes += memoryStats[i].residentBytes;
1480 partitionStats.totalActiveBytes += memoryStats[i].activeBytes; 1453 partitionStats.totalActiveBytes += memoryStats[i].activeBytes;
1481 partitionStats.totalDecommittableBytes += 1454 partitionStats.totalDecommittableBytes +=
1482 memoryStats[i].decommittableBytes; 1455 memoryStats[i].decommittableBytes;
1483 partitionStats.totalDiscardableBytes += memoryStats[i].discardableBytes; 1456 partitionStats.totalDiscardableBytes += memoryStats[i].discardableBytes;
1484 if (!isLightDump) 1457 if (!isLightDump)
1485 partitionStatsDumper->partitionsDumpBucketStats(partitionName, 1458 partitionStatsDumper->partitionsDumpBucketStats(partitionName,
1486 &memoryStats[i]); 1459 &memoryStats[i]);
1487 } 1460 }
1488 } 1461 }
1489 partitionStatsDumper->partitionDumpTotals(partitionName, &partitionStats); 1462 partitionStatsDumper->partitionDumpTotals(partitionName, &partitionStats);
1490 } 1463 }
1491 1464
1492 } // namespace WTF 1465 } // namespace base
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698