Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(17)

Side by Side Diff: third_party/WebKit/Source/platform/heap/HeapPage.cpp

Issue 2619493003: Replace ASSERTs in platform/heap/ with DCHECKs
Patch Set: temp Created 3 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 /* 1 /*
2 * Copyright (C) 2013 Google Inc. All rights reserved. 2 * Copyright (C) 2013 Google Inc. All rights reserved.
3 * 3 *
4 * Redistribution and use in source and binary forms, with or without 4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are 5 * modification, are permitted provided that the following conditions are
6 * met: 6 * met:
7 * 7 *
8 * * Redistributions of source code must retain the above copyright 8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer. 9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above 10 * * Redistributions in binary form must reproduce the above
(...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after
55 #ifdef ANNOTATE_CONTIGUOUS_CONTAINER 55 #ifdef ANNOTATE_CONTIGUOUS_CONTAINER
56 // FIXME: have ContainerAnnotations.h define an ENABLE_-style name instead. 56 // FIXME: have ContainerAnnotations.h define an ENABLE_-style name instead.
57 #define ENABLE_ASAN_CONTAINER_ANNOTATIONS 1 57 #define ENABLE_ASAN_CONTAINER_ANNOTATIONS 1
58 58
59 // When finalizing a non-inlined vector backing store/container, remove 59 // When finalizing a non-inlined vector backing store/container, remove
60 // its contiguous container annotation. Required as it will not be destructed 60 // its contiguous container annotation. Required as it will not be destructed
61 // from its Vector. 61 // from its Vector.
62 #define ASAN_RETIRE_CONTAINER_ANNOTATION(object, objectSize) \ 62 #define ASAN_RETIRE_CONTAINER_ANNOTATION(object, objectSize) \
63 do { \ 63 do { \
64 BasePage* page = pageFromObject(object); \ 64 BasePage* page = pageFromObject(object); \
65 ASSERT(page); \ 65 DCHECK(page); \
66 bool isContainer = \ 66 bool isContainer = \
67 ThreadState::isVectorArenaIndex(page->arena()->arenaIndex()); \ 67 ThreadState::isVectorArenaIndex(page->arena()->arenaIndex()); \
68 if (!isContainer && page->isLargeObjectPage()) \ 68 if (!isContainer && page->isLargeObjectPage()) \
69 isContainer = \ 69 isContainer = \
70 static_cast<LargeObjectPage*>(page)->isVectorBackingPage(); \ 70 static_cast<LargeObjectPage*>(page)->isVectorBackingPage(); \
71 if (isContainer) \ 71 if (isContainer) \
72 ANNOTATE_DELETE_BUFFER(object, objectSize, 0); \ 72 ANNOTATE_DELETE_BUFFER(object, objectSize, 0); \
73 } while (0) 73 } while (0)
74 74
75 // A vector backing store represented by a large object is marked 75 // A vector backing store represented by a large object is marked
76 // so that when it is finalized, its ASan annotation will be 76 // so that when it is finalized, its ASan annotation will be
77 // correctly retired. 77 // correctly retired.
78 #define ASAN_MARK_LARGE_VECTOR_CONTAINER(arena, largeObject) \ 78 #define ASAN_MARK_LARGE_VECTOR_CONTAINER(arena, largeObject) \
79 if (ThreadState::isVectorArenaIndex(arena->arenaIndex())) { \ 79 if (ThreadState::isVectorArenaIndex(arena->arenaIndex())) { \
80 BasePage* largePage = pageFromObject(largeObject); \ 80 BasePage* largePage = pageFromObject(largeObject); \
81 ASSERT(largePage->isLargeObjectPage()); \ 81 DCHECK(largePage->isLargeObjectPage()); \
82 static_cast<LargeObjectPage*>(largePage)->setIsVectorBackingPage(); \ 82 static_cast<LargeObjectPage*>(largePage)->setIsVectorBackingPage(); \
83 } 83 }
84 #else 84 #else
85 #define ENABLE_ASAN_CONTAINER_ANNOTATIONS 0 85 #define ENABLE_ASAN_CONTAINER_ANNOTATIONS 0
86 #define ASAN_RETIRE_CONTAINER_ANNOTATION(payload, payloadSize) 86 #define ASAN_RETIRE_CONTAINER_ANNOTATION(payload, payloadSize)
87 #define ASAN_MARK_LARGE_VECTOR_CONTAINER(arena, largeObject) 87 #define ASAN_MARK_LARGE_VECTOR_CONTAINER(arena, largeObject)
88 #endif 88 #endif
89 89
90 namespace blink { 90 namespace blink {
91 91
92 #if ENABLE(ASSERT) 92 #if DCHECK_IS_ON()
93 NO_SANITIZE_ADDRESS 93 NO_SANITIZE_ADDRESS
94 void HeapObjectHeader::zapMagic() { 94 void HeapObjectHeader::zapMagic() {
95 ASSERT(checkHeader()); 95 DCHECK(checkHeader());
96 m_magic = zappedMagic; 96 m_magic = zappedMagic;
97 } 97 }
98 #endif 98 #endif
99 99
100 void HeapObjectHeader::finalize(Address object, size_t objectSize) { 100 void HeapObjectHeader::finalize(Address object, size_t objectSize) {
101 HeapAllocHooks::freeHookIfEnabled(object); 101 HeapAllocHooks::freeHookIfEnabled(object);
102 const GCInfo* gcInfo = ThreadHeap::gcInfo(gcInfoIndex()); 102 const GCInfo* gcInfo = ThreadHeap::gcInfo(gcInfoIndex());
103 if (gcInfo->hasFinalizer()) 103 if (gcInfo->hasFinalizer())
104 gcInfo->m_finalize(object); 104 gcInfo->m_finalize(object);
105 105
106 ASAN_RETIRE_CONTAINER_ANNOTATION(object, objectSize); 106 ASAN_RETIRE_CONTAINER_ANNOTATION(object, objectSize);
107 } 107 }
108 108
109 BaseArena::BaseArena(ThreadState* state, int index) 109 BaseArena::BaseArena(ThreadState* state, int index)
110 : m_firstPage(nullptr), 110 : m_firstPage(nullptr),
111 m_firstUnsweptPage(nullptr), 111 m_firstUnsweptPage(nullptr),
112 m_threadState(state), 112 m_threadState(state),
113 m_index(index) {} 113 m_index(index) {}
114 114
115 BaseArena::~BaseArena() { 115 BaseArena::~BaseArena() {
116 ASSERT(!m_firstPage); 116 DCHECK(!m_firstPage);
117 ASSERT(!m_firstUnsweptPage); 117 DCHECK(!m_firstUnsweptPage);
118 } 118 }
119 119
120 void BaseArena::cleanupPages() { 120 void BaseArena::cleanupPages() {
121 clearFreeLists(); 121 clearFreeLists();
122 122
123 ASSERT(!m_firstUnsweptPage); 123 DCHECK(!m_firstUnsweptPage);
124 // Add the BaseArena's pages to the orphanedPagePool. 124 // Add the BaseArena's pages to the orphanedPagePool.
125 for (BasePage* page = m_firstPage; page; page = page->next()) { 125 for (BasePage* page = m_firstPage; page; page = page->next()) {
126 getThreadState()->heap().heapStats().decreaseAllocatedSpace(page->size()); 126 getThreadState()->heap().heapStats().decreaseAllocatedSpace(page->size());
127 getThreadState()->heap().getOrphanedPagePool()->addOrphanedPage( 127 getThreadState()->heap().getOrphanedPagePool()->addOrphanedPage(
128 arenaIndex(), page); 128 arenaIndex(), page);
129 } 129 }
130 m_firstPage = nullptr; 130 m_firstPage = nullptr;
131 } 131 }
132 132
133 void BaseArena::takeSnapshot(const String& dumpBaseName, 133 void BaseArena::takeSnapshot(const String& dumpBaseName,
(...skipping 17 matching lines...) Expand all
151 allocatorDump->AddScalar("blink_page_count", "objects", pageCount); 151 allocatorDump->AddScalar("blink_page_count", "objects", pageCount);
152 152
153 // When taking a full dump (w/ freelist), both the /buckets and /pages 153 // When taking a full dump (w/ freelist), both the /buckets and /pages
154 // report their free size but they are not meant to be added together. 154 // report their free size but they are not meant to be added together.
155 // Therefore, here we override the free_size of the parent heap to be 155 // Therefore, here we override the free_size of the parent heap to be
156 // equal to the free_size of the sum of its heap pages. 156 // equal to the free_size of the sum of its heap pages.
157 allocatorDump->AddScalar("free_size", "bytes", heapInfo.freeSize); 157 allocatorDump->AddScalar("free_size", "bytes", heapInfo.freeSize);
158 allocatorDump->AddScalar("free_count", "objects", heapInfo.freeCount); 158 allocatorDump->AddScalar("free_count", "objects", heapInfo.freeCount);
159 } 159 }
160 160
161 #if ENABLE(ASSERT) 161 #if DCHECK_IS_ON()
162 BasePage* BaseArena::findPageFromAddress(Address address) { 162 BasePage* BaseArena::findPageFromAddress(Address address) {
163 for (BasePage* page = m_firstPage; page; page = page->next()) { 163 for (BasePage* page = m_firstPage; page; page = page->next()) {
164 if (page->contains(address)) 164 if (page->contains(address))
165 return page; 165 return page;
166 } 166 }
167 for (BasePage* page = m_firstUnsweptPage; page; page = page->next()) { 167 for (BasePage* page = m_firstUnsweptPage; page; page = page->next()) {
168 if (page->contains(address)) 168 if (page->contains(address))
169 return page; 169 return page;
170 } 170 }
171 return nullptr; 171 return nullptr;
172 } 172 }
173 #endif 173 #endif
174 174
175 void BaseArena::makeConsistentForGC() { 175 void BaseArena::makeConsistentForGC() {
176 clearFreeLists(); 176 clearFreeLists();
177 ASSERT(isConsistentForGC()); 177 DCHECK(isConsistentForGC());
178 for (BasePage* page = m_firstPage; page; page = page->next()) { 178 for (BasePage* page = m_firstPage; page; page = page->next()) {
179 page->markAsUnswept(); 179 page->markAsUnswept();
180 page->invalidateObjectStartBitmap(); 180 page->invalidateObjectStartBitmap();
181 } 181 }
182 182
183 // If a new GC is requested before this thread got around to sweep, 183 // If a new GC is requested before this thread got around to sweep,
184 // ie. due to the thread doing a long running operation, we clear 184 // ie. due to the thread doing a long running operation, we clear
185 // the mark bits and mark any of the dead objects as dead. The latter 185 // the mark bits and mark any of the dead objects as dead. The latter
186 // is used to ensure the next GC marking does not trace already dead 186 // is used to ensure the next GC marking does not trace already dead
187 // objects. If we trace a dead object we could end up tracing into 187 // objects. If we trace a dead object we could end up tracing into
188 // garbage or the middle of another object via the newly conservatively 188 // garbage or the middle of another object via the newly conservatively
189 // found object. 189 // found object.
190 BasePage* previousPage = nullptr; 190 BasePage* previousPage = nullptr;
191 for (BasePage *page = m_firstUnsweptPage; page; 191 for (BasePage *page = m_firstUnsweptPage; page;
192 previousPage = page, page = page->next()) { 192 previousPage = page, page = page->next()) {
193 page->makeConsistentForGC(); 193 page->makeConsistentForGC();
194 ASSERT(!page->hasBeenSwept()); 194 DCHECK(!page->hasBeenSwept());
195 page->invalidateObjectStartBitmap(); 195 page->invalidateObjectStartBitmap();
196 } 196 }
197 if (previousPage) { 197 if (previousPage) {
198 ASSERT(m_firstUnsweptPage); 198 DCHECK(m_firstUnsweptPage);
199 previousPage->m_next = m_firstPage; 199 previousPage->m_next = m_firstPage;
200 m_firstPage = m_firstUnsweptPage; 200 m_firstPage = m_firstUnsweptPage;
201 m_firstUnsweptPage = nullptr; 201 m_firstUnsweptPage = nullptr;
202 } 202 }
203 ASSERT(!m_firstUnsweptPage); 203 DCHECK(!m_firstUnsweptPage);
204 204
205 HeapCompact* heapCompactor = getThreadState()->heap().compaction(); 205 HeapCompact* heapCompactor = getThreadState()->heap().compaction();
206 if (!heapCompactor->isCompactingArena(arenaIndex())) 206 if (!heapCompactor->isCompactingArena(arenaIndex()))
207 return; 207 return;
208 208
209 BasePage* nextPage = m_firstPage; 209 BasePage* nextPage = m_firstPage;
210 while (nextPage) { 210 while (nextPage) {
211 if (!nextPage->isLargeObjectPage()) 211 if (!nextPage->isLargeObjectPage())
212 heapCompactor->addCompactingPage(nextPage); 212 heapCompactor->addCompactingPage(nextPage);
213 nextPage = nextPage->next(); 213 nextPage = nextPage->next();
214 } 214 }
215 } 215 }
216 216
217 void BaseArena::makeConsistentForMutator() { 217 void BaseArena::makeConsistentForMutator() {
218 clearFreeLists(); 218 clearFreeLists();
219 ASSERT(isConsistentForGC()); 219 DCHECK(isConsistentForGC());
220 ASSERT(!m_firstPage); 220 DCHECK(!m_firstPage);
221 221
222 // Drop marks from marked objects and rebuild free lists in preparation for 222 // Drop marks from marked objects and rebuild free lists in preparation for
223 // resuming the executions of mutators. 223 // resuming the executions of mutators.
224 BasePage* previousPage = nullptr; 224 BasePage* previousPage = nullptr;
225 for (BasePage *page = m_firstUnsweptPage; page; 225 for (BasePage *page = m_firstUnsweptPage; page;
226 previousPage = page, page = page->next()) { 226 previousPage = page, page = page->next()) {
227 page->makeConsistentForMutator(); 227 page->makeConsistentForMutator();
228 page->markAsSwept(); 228 page->markAsSwept();
229 page->invalidateObjectStartBitmap(); 229 page->invalidateObjectStartBitmap();
230 } 230 }
231 if (previousPage) { 231 if (previousPage) {
232 ASSERT(m_firstUnsweptPage); 232 DCHECK(m_firstUnsweptPage);
233 previousPage->m_next = m_firstPage; 233 previousPage->m_next = m_firstPage;
234 m_firstPage = m_firstUnsweptPage; 234 m_firstPage = m_firstUnsweptPage;
235 m_firstUnsweptPage = nullptr; 235 m_firstUnsweptPage = nullptr;
236 } 236 }
237 ASSERT(!m_firstUnsweptPage); 237 DCHECK(!m_firstUnsweptPage);
238 } 238 }
239 239
240 size_t BaseArena::objectPayloadSizeForTesting() { 240 size_t BaseArena::objectPayloadSizeForTesting() {
241 ASSERT(isConsistentForGC()); 241 DCHECK(isConsistentForGC());
242 ASSERT(!m_firstUnsweptPage); 242 DCHECK(!m_firstUnsweptPage);
243 243
244 size_t objectPayloadSize = 0; 244 size_t objectPayloadSize = 0;
245 for (BasePage* page = m_firstPage; page; page = page->next()) 245 for (BasePage* page = m_firstPage; page; page = page->next())
246 objectPayloadSize += page->objectPayloadSizeForTesting(); 246 objectPayloadSize += page->objectPayloadSizeForTesting();
247 return objectPayloadSize; 247 return objectPayloadSize;
248 } 248 }
249 249
250 void BaseArena::prepareHeapForTermination() { 250 void BaseArena::prepareHeapForTermination() {
251 ASSERT(!m_firstUnsweptPage); 251 DCHECK(!m_firstUnsweptPage);
252 for (BasePage* page = m_firstPage; page; page = page->next()) { 252 for (BasePage* page = m_firstPage; page; page = page->next()) {
253 page->setTerminating(); 253 page->setTerminating();
254 } 254 }
255 } 255 }
256 256
257 void BaseArena::prepareForSweep() { 257 void BaseArena::prepareForSweep() {
258 ASSERT(getThreadState()->isInGC()); 258 DCHECK(getThreadState()->isInGC());
259 ASSERT(!m_firstUnsweptPage); 259 DCHECK(!m_firstUnsweptPage);
260 260
261 // Move all pages to a list of unswept pages. 261 // Move all pages to a list of unswept pages.
262 m_firstUnsweptPage = m_firstPage; 262 m_firstUnsweptPage = m_firstPage;
263 m_firstPage = nullptr; 263 m_firstPage = nullptr;
264 } 264 }
265 265
266 #if defined(ADDRESS_SANITIZER) 266 #if defined(ADDRESS_SANITIZER)
267 void BaseArena::poisonArena() { 267 void BaseArena::poisonArena() {
268 for (BasePage* page = m_firstUnsweptPage; page; page = page->next()) 268 for (BasePage* page = m_firstUnsweptPage; page; page = page->next())
269 page->poisonUnmarkedObjects(); 269 page->poisonUnmarkedObjects();
270 } 270 }
271 #endif 271 #endif
272 272
273 Address BaseArena::lazySweep(size_t allocationSize, size_t gcInfoIndex) { 273 Address BaseArena::lazySweep(size_t allocationSize, size_t gcInfoIndex) {
274 // If there are no pages to be swept, return immediately. 274 // If there are no pages to be swept, return immediately.
275 if (!m_firstUnsweptPage) 275 if (!m_firstUnsweptPage)
276 return nullptr; 276 return nullptr;
277 277
278 RELEASE_ASSERT(getThreadState()->isSweepingInProgress()); 278 CHECK(getThreadState()->isSweepingInProgress());
279 279
280 // lazySweepPages() can be called recursively if finalizers invoked in 280 // lazySweepPages() can be called recursively if finalizers invoked in
281 // page->sweep() allocate memory and the allocation triggers 281 // page->sweep() allocate memory and the allocation triggers
282 // lazySweepPages(). This check prevents the sweeping from being executed 282 // lazySweepPages(). This check prevents the sweeping from being executed
283 // recursively. 283 // recursively.
284 if (getThreadState()->sweepForbidden()) 284 if (getThreadState()->sweepForbidden())
285 return nullptr; 285 return nullptr;
286 286
287 TRACE_EVENT0("blink_gc", "BaseArena::lazySweepPages"); 287 TRACE_EVENT0("blink_gc", "BaseArena::lazySweepPages");
288 ThreadState::SweepForbiddenScope sweepForbidden(getThreadState()); 288 ThreadState::SweepForbiddenScope sweepForbidden(getThreadState());
(...skipping 22 matching lines...) Expand all
311 } 311 }
312 } 312 }
313 313
314 bool BaseArena::lazySweepWithDeadline(double deadlineSeconds) { 314 bool BaseArena::lazySweepWithDeadline(double deadlineSeconds) {
315 // It might be heavy to call 315 // It might be heavy to call
316 // Platform::current()->monotonicallyIncreasingTimeSeconds() per page (i.e., 316 // Platform::current()->monotonicallyIncreasingTimeSeconds() per page (i.e.,
317 // 128 KB sweep or one LargeObject sweep), so we check the deadline per 10 317 // 128 KB sweep or one LargeObject sweep), so we check the deadline per 10
318 // pages. 318 // pages.
319 static const int deadlineCheckInterval = 10; 319 static const int deadlineCheckInterval = 10;
320 320
321 RELEASE_ASSERT(getThreadState()->isSweepingInProgress()); 321 CHECK(getThreadState()->isSweepingInProgress());
322 ASSERT(getThreadState()->sweepForbidden()); 322 DCHECK(getThreadState()->sweepForbidden());
323 ASSERT(!getThreadState()->isMainThread() || 323 DCHECK(!getThreadState()->isMainThread() ||
324 ScriptForbiddenScope::isScriptForbidden()); 324 ScriptForbiddenScope::isScriptForbidden());
325 325
326 NormalPageArena* normalArena = nullptr; 326 NormalPageArena* normalArena = nullptr;
327 if (m_firstUnsweptPage && !m_firstUnsweptPage->isLargeObjectPage()) { 327 if (m_firstUnsweptPage && !m_firstUnsweptPage->isLargeObjectPage()) {
328 // Mark this NormalPageArena as being lazily swept. 328 // Mark this NormalPageArena as being lazily swept.
329 NormalPage* normalPage = reinterpret_cast<NormalPage*>(m_firstUnsweptPage); 329 NormalPage* normalPage = reinterpret_cast<NormalPage*>(m_firstUnsweptPage);
330 normalArena = normalPage->arenaForNormalPage(); 330 normalArena = normalPage->arenaForNormalPage();
331 normalArena->setIsLazySweeping(true); 331 normalArena->setIsLazySweeping(true);
332 } 332 }
333 int pageCount = 1; 333 int pageCount = 1;
(...skipping 10 matching lines...) Expand all
344 } 344 }
345 pageCount++; 345 pageCount++;
346 } 346 }
347 ThreadHeap::reportMemoryUsageForTracing(); 347 ThreadHeap::reportMemoryUsageForTracing();
348 if (normalArena) 348 if (normalArena)
349 normalArena->setIsLazySweeping(false); 349 normalArena->setIsLazySweeping(false);
350 return true; 350 return true;
351 } 351 }
352 352
353 void BaseArena::completeSweep() { 353 void BaseArena::completeSweep() {
354 RELEASE_ASSERT(getThreadState()->isSweepingInProgress()); 354 CHECK(getThreadState()->isSweepingInProgress());
355 ASSERT(getThreadState()->sweepForbidden()); 355 DCHECK(getThreadState()->sweepForbidden());
356 ASSERT(!getThreadState()->isMainThread() || 356 DCHECK(!getThreadState()->isMainThread() ||
357 ScriptForbiddenScope::isScriptForbidden()); 357 ScriptForbiddenScope::isScriptForbidden());
358 358
359 while (m_firstUnsweptPage) { 359 while (m_firstUnsweptPage) {
360 sweepUnsweptPage(); 360 sweepUnsweptPage();
361 } 361 }
362 ThreadHeap::reportMemoryUsageForTracing(); 362 ThreadHeap::reportMemoryUsageForTracing();
363 } 363 }
364 364
365 Address BaseArena::allocateLargeObject(size_t allocationSize, 365 Address BaseArena::allocateLargeObject(size_t allocationSize,
366 size_t gcInfoIndex) { 366 size_t gcInfoIndex) {
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after
420 // => |objectPointer| will not be lazily swept. 420 // => |objectPointer| will not be lazily swept.
421 // 421 //
422 // Notice that |objectPointer| might be pointer to a GarbageCollectedMixin, 422 // Notice that |objectPointer| might be pointer to a GarbageCollectedMixin,
423 // hence using fromPayload() to derive the HeapObjectHeader isn't possible 423 // hence using fromPayload() to derive the HeapObjectHeader isn't possible
424 // (and use its value to check if |headerAddress| is equal to it.) 424 // (and use its value to check if |headerAddress| is equal to it.)
425 if (headerAddress > objectPointer) 425 if (headerAddress > objectPointer)
426 return false; 426 return false;
427 if (!header->isFree() && header->isMarked()) { 427 if (!header->isFree() && header->isMarked()) {
428 // There must be a marked object on this page and the one located must 428 // There must be a marked object on this page and the one located must
429 // have room after it for the unmarked |objectPointer| object. 429 // have room after it for the unmarked |objectPointer| object.
430 DCHECK(headerAddress + size < pageEnd); 430 DCHECK_LT(headerAddress + size, pageEnd);
431 return true; 431 return true;
432 } 432 }
433 headerAddress += size; 433 headerAddress += size;
434 } 434 }
435 NOTREACHED(); 435 NOTREACHED();
436 return true; 436 return true;
437 } 437 }
438 438
439 NormalPageArena::NormalPageArena(ThreadState* state, int index) 439 NormalPageArena::NormalPageArena(ThreadState* state, int index)
440 : BaseArena(state, index), 440 : BaseArena(state, index),
(...skipping 114 matching lines...) Expand 10 before | Expand all | Expand 10 after
555 size_t pageSize = availablePages->size(); 555 size_t pageSize = availablePages->size();
556 #if DEBUG_HEAP_COMPACTION 556 #if DEBUG_HEAP_COMPACTION
557 if (!freedPageCount) 557 if (!freedPageCount)
558 LOG_HEAP_COMPACTION("Releasing:"); 558 LOG_HEAP_COMPACTION("Releasing:");
559 LOG_HEAP_COMPACTION(" [%p, %p]", availablePages, availablePages + pageSize); 559 LOG_HEAP_COMPACTION(" [%p, %p]", availablePages, availablePages + pageSize);
560 #endif 560 #endif
561 freedSize += pageSize; 561 freedSize += pageSize;
562 freedPageCount++; 562 freedPageCount++;
563 BasePage* nextPage; 563 BasePage* nextPage;
564 availablePages->unlink(&nextPage); 564 availablePages->unlink(&nextPage);
565 #if !(ENABLE(ASSERT) || defined(LEAK_SANITIZER) || \ 565 #if !(DCHECK_IS_ON() || defined(LEAK_SANITIZER) || \
566 defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER)) 566 defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER))
567 // Clear out the page before adding it to the free page pool, which 567 // Clear out the page before adding it to the free page pool, which
568 // decommits it. Recommitting the page must find a zeroed page later. 568 // decommits it. Recommitting the page must find a zeroed page later.
569 // We cannot assume that the OS will hand back a zeroed page across 569 // We cannot assume that the OS will hand back a zeroed page across
570 // its "decommit" operation. 570 // its "decommit" operation.
571 // 571 //
572 // If in a debug setting, the unused page contents will have been 572 // If in a debug setting, the unused page contents will have been
573 // zapped already; leave it in that state. 573 // zapped already; leave it in that state.
574 DCHECK(!availablePages->isLargeObjectPage()); 574 DCHECK(!availablePages->isLargeObjectPage());
575 NormalPage* unusedPage = reinterpret_cast<NormalPage*>(availablePages); 575 NormalPage* unusedPage = reinterpret_cast<NormalPage*>(availablePages);
576 memset(unusedPage->payload(), 0, unusedPage->payloadSize()); 576 memset(unusedPage->payload(), 0, unusedPage->payloadSize());
577 #endif 577 #endif
578 availablePages->removeFromHeap(); 578 availablePages->removeFromHeap();
579 availablePages = static_cast<NormalPage*>(nextPage); 579 availablePages = static_cast<NormalPage*>(nextPage);
580 } 580 }
581 if (freedPageCount) 581 if (freedPageCount)
582 LOG_HEAP_COMPACTION("\n"); 582 LOG_HEAP_COMPACTION("\n");
583 heap.compaction()->finishedArenaCompaction(this, freedPageCount, freedSize); 583 heap.compaction()->finishedArenaCompaction(this, freedPageCount, freedSize);
584 } 584 }
585 585
586 #if ENABLE(ASSERT) 586 #if DCHECK_IS_ON()
587 bool NormalPageArena::isConsistentForGC() { 587 bool NormalPageArena::isConsistentForGC() {
588 // A thread heap is consistent for sweeping if none of the pages to be swept 588 // A thread heap is consistent for sweeping if none of the pages to be swept
589 // contain a freelist block or the current allocation point. 589 // contain a freelist block or the current allocation point.
590 for (size_t i = 0; i < blinkPageSizeLog2; ++i) { 590 for (size_t i = 0; i < blinkPageSizeLog2; ++i) {
591 for (FreeListEntry* freeListEntry = m_freeList.m_freeLists[i]; 591 for (FreeListEntry* freeListEntry = m_freeList.m_freeLists[i];
592 freeListEntry; freeListEntry = freeListEntry->next()) { 592 freeListEntry; freeListEntry = freeListEntry->next()) {
593 if (pagesToBeSweptContains(freeListEntry->getAddress())) 593 if (pagesToBeSweptContains(freeListEntry->getAddress()))
594 return false; 594 return false;
595 } 595 }
596 } 596 }
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after
639 getThreadState()->heap().getRegionTree()); 639 getThreadState()->heap().getRegionTree());
640 640
641 // Setup the PageMemory object for each of the pages in the region. 641 // Setup the PageMemory object for each of the pages in the region.
642 for (size_t i = 0; i < blinkPagesPerRegion; ++i) { 642 for (size_t i = 0; i < blinkPagesPerRegion; ++i) {
643 PageMemory* memory = PageMemory::setupPageMemoryInRegion( 643 PageMemory* memory = PageMemory::setupPageMemoryInRegion(
644 region, i * blinkPageSize, blinkPagePayloadSize()); 644 region, i * blinkPageSize, blinkPagePayloadSize());
645 // Take the first possible page ensuring that this thread actually 645 // Take the first possible page ensuring that this thread actually
646 // gets a page and add the rest to the page pool. 646 // gets a page and add the rest to the page pool.
647 if (!pageMemory) { 647 if (!pageMemory) {
648 bool result = memory->commit(); 648 bool result = memory->commit();
649 // If you hit the ASSERT, it will mean that you're hitting 649 // If you hit the DCHECK, it will mean that you're hitting
650 // the limit of the number of mmapped regions OS can support 650 // the limit of the number of mmapped regions OS can support
651 // (e.g., /proc/sys/vm/max_map_count in Linux). 651 // (e.g., /proc/sys/vm/max_map_count in Linux).
652 RELEASE_ASSERT(result); 652 CHECK(result);
653 pageMemory = memory; 653 pageMemory = memory;
654 } else { 654 } else {
655 getThreadState()->heap().getFreePagePool()->addFreePage(arenaIndex(), 655 getThreadState()->heap().getFreePagePool()->addFreePage(arenaIndex(),
656 memory); 656 memory);
657 } 657 }
658 } 658 }
659 } 659 }
660 NormalPage* page = 660 NormalPage* page =
661 new (pageMemory->writableStart()) NormalPage(pageMemory, this); 661 new (pageMemory->writableStart()) NormalPage(pageMemory, this);
662 page->link(&m_firstPage); 662 page->link(&m_firstPage);
663 663
664 getThreadState()->heap().heapStats().increaseAllocatedSpace(page->size()); 664 getThreadState()->heap().heapStats().increaseAllocatedSpace(page->size());
665 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) 665 #if DCHECK_IS_ON() || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER)
666 // Allow the following addToFreeList() to add the newly allocated memory 666 // Allow the following addToFreeList() to add the newly allocated memory
667 // to the free list. 667 // to the free list.
668 ASAN_UNPOISON_MEMORY_REGION(page->payload(), page->payloadSize()); 668 ASAN_UNPOISON_MEMORY_REGION(page->payload(), page->payloadSize());
669 Address address = page->payload(); 669 Address address = page->payload();
670 for (size_t i = 0; i < page->payloadSize(); i++) 670 for (size_t i = 0; i < page->payloadSize(); i++)
671 address[i] = reuseAllowedZapValue; 671 address[i] = reuseAllowedZapValue;
672 ASAN_POISON_MEMORY_REGION(page->payload(), page->payloadSize()); 672 ASAN_POISON_MEMORY_REGION(page->payload(), page->payloadSize());
673 #endif 673 #endif
674 addToFreeList(page->payload(), page->payloadSize()); 674 addToFreeList(page->payload(), page->payloadSize());
675 } 675 }
(...skipping 26 matching lines...) Expand all
702 // 702 //
703 // FIXME: This threshold is determined just to optimize blink_perf 703 // FIXME: This threshold is determined just to optimize blink_perf
704 // benchmarks. Coalescing is very sensitive to the threashold and 704 // benchmarks. Coalescing is very sensitive to the threashold and
705 // we need further investigations on the coalescing scheme. 705 // we need further investigations on the coalescing scheme.
706 if (m_promptlyFreedSize < 1024 * 1024) 706 if (m_promptlyFreedSize < 1024 * 1024)
707 return false; 707 return false;
708 708
709 if (getThreadState()->sweepForbidden()) 709 if (getThreadState()->sweepForbidden())
710 return false; 710 return false;
711 711
712 ASSERT(!hasCurrentAllocationArea()); 712 DCHECK(!hasCurrentAllocationArea());
713 TRACE_EVENT0("blink_gc", "BaseArena::coalesce"); 713 TRACE_EVENT0("blink_gc", "BaseArena::coalesce");
714 714
715 // Rebuild free lists. 715 // Rebuild free lists.
716 m_freeList.clear(); 716 m_freeList.clear();
717 size_t freedSize = 0; 717 size_t freedSize = 0;
718 for (NormalPage* page = static_cast<NormalPage*>(m_firstPage); page; 718 for (NormalPage* page = static_cast<NormalPage*>(m_firstPage); page;
719 page = static_cast<NormalPage*>(page->next())) { 719 page = static_cast<NormalPage*>(page->next())) {
720 Address startOfGap = page->payload(); 720 Address startOfGap = page->payload();
721 for (Address headerAddress = startOfGap; 721 for (Address headerAddress = startOfGap;
722 headerAddress < page->payloadEnd();) { 722 headerAddress < page->payloadEnd();) {
723 HeapObjectHeader* header = 723 HeapObjectHeader* header =
724 reinterpret_cast<HeapObjectHeader*>(headerAddress); 724 reinterpret_cast<HeapObjectHeader*>(headerAddress);
725 size_t size = header->size(); 725 size_t size = header->size();
726 ASSERT(size > 0); 726 DCHECK_GT(size, 0UL);
727 ASSERT(size < blinkPagePayloadSize()); 727 DCHECK_LT(size, blinkPagePayloadSize());
728 728
729 if (header->isPromptlyFreed()) { 729 if (header->isPromptlyFreed()) {
730 ASSERT(size >= sizeof(HeapObjectHeader)); 730 DCHECK_GE(size, sizeof(HeapObjectHeader));
731 // Zero the memory in the free list header to maintain the 731 // Zero the memory in the free list header to maintain the
732 // invariant that memory on the free list is zero filled. 732 // invariant that memory on the free list is zero filled.
733 // The rest of the memory is already on the free list and is 733 // The rest of the memory is already on the free list and is
734 // therefore already zero filled. 734 // therefore already zero filled.
735 SET_MEMORY_INACCESSIBLE(headerAddress, sizeof(HeapObjectHeader)); 735 SET_MEMORY_INACCESSIBLE(headerAddress, sizeof(HeapObjectHeader));
736 CHECK_MEMORY_INACCESSIBLE(headerAddress, size); 736 CHECK_MEMORY_INACCESSIBLE(headerAddress, size);
737 freedSize += size; 737 freedSize += size;
738 headerAddress += size; 738 headerAddress += size;
739 continue; 739 continue;
740 } 740 }
741 if (header->isFree()) { 741 if (header->isFree()) {
742 // Zero the memory in the free list header to maintain the 742 // Zero the memory in the free list header to maintain the
743 // invariant that memory on the free list is zero filled. 743 // invariant that memory on the free list is zero filled.
744 // The rest of the memory is already on the free list and is 744 // The rest of the memory is already on the free list and is
745 // therefore already zero filled. 745 // therefore already zero filled.
746 SET_MEMORY_INACCESSIBLE(headerAddress, size < sizeof(FreeListEntry) 746 SET_MEMORY_INACCESSIBLE(headerAddress, size < sizeof(FreeListEntry)
747 ? size 747 ? size
748 : sizeof(FreeListEntry)); 748 : sizeof(FreeListEntry));
749 CHECK_MEMORY_INACCESSIBLE(headerAddress, size); 749 CHECK_MEMORY_INACCESSIBLE(headerAddress, size);
750 headerAddress += size; 750 headerAddress += size;
751 continue; 751 continue;
752 } 752 }
753 ASSERT(header->checkHeader()); 753 DCHECK(header->checkHeader());
754 if (startOfGap != headerAddress) 754 if (startOfGap != headerAddress)
755 addToFreeList(startOfGap, headerAddress - startOfGap); 755 addToFreeList(startOfGap, headerAddress - startOfGap);
756 756
757 headerAddress += size; 757 headerAddress += size;
758 startOfGap = headerAddress; 758 startOfGap = headerAddress;
759 } 759 }
760 760
761 if (startOfGap != page->payloadEnd()) 761 if (startOfGap != page->payloadEnd())
762 addToFreeList(startOfGap, page->payloadEnd() - startOfGap); 762 addToFreeList(startOfGap, page->payloadEnd() - startOfGap);
763 } 763 }
764 getThreadState()->decreaseAllocatedObjectSize(freedSize); 764 getThreadState()->decreaseAllocatedObjectSize(freedSize);
765 ASSERT(m_promptlyFreedSize == freedSize); 765 DCHECK_EQ(m_promptlyFreedSize, freedSize);
766 m_promptlyFreedSize = 0; 766 m_promptlyFreedSize = 0;
767 return true; 767 return true;
768 } 768 }
769 769
770 void NormalPageArena::promptlyFreeObject(HeapObjectHeader* header) { 770 void NormalPageArena::promptlyFreeObject(HeapObjectHeader* header) {
771 ASSERT(!getThreadState()->sweepForbidden()); 771 DCHECK(!getThreadState()->sweepForbidden());
772 ASSERT(header->checkHeader()); 772 DCHECK(header->checkHeader());
773 Address address = reinterpret_cast<Address>(header); 773 Address address = reinterpret_cast<Address>(header);
774 Address payload = header->payload(); 774 Address payload = header->payload();
775 size_t size = header->size(); 775 size_t size = header->size();
776 size_t payloadSize = header->payloadSize(); 776 size_t payloadSize = header->payloadSize();
777 ASSERT(size > 0); 777 DCHECK_GT(size, 0UL);
778 ASSERT(pageFromObject(address) == findPageFromAddress(address)); 778 DCHECK_EQ(pageFromObject(address), findPageFromAddress(address));
779 779
780 { 780 {
781 ThreadState::SweepForbiddenScope forbiddenScope(getThreadState()); 781 ThreadState::SweepForbiddenScope forbiddenScope(getThreadState());
782 header->finalize(payload, payloadSize); 782 header->finalize(payload, payloadSize);
783 if (address + size == m_currentAllocationPoint) { 783 if (address + size == m_currentAllocationPoint) {
784 m_currentAllocationPoint = address; 784 m_currentAllocationPoint = address;
785 setRemainingAllocationSize(m_remainingAllocationSize + size); 785 setRemainingAllocationSize(m_remainingAllocationSize + size);
786 SET_MEMORY_INACCESSIBLE(address, size); 786 SET_MEMORY_INACCESSIBLE(address, size);
787 return; 787 return;
788 } 788 }
789 SET_MEMORY_INACCESSIBLE(payload, payloadSize); 789 SET_MEMORY_INACCESSIBLE(payload, payloadSize);
790 header->markPromptlyFreed(); 790 header->markPromptlyFreed();
791 } 791 }
792 792
793 m_promptlyFreedSize += size; 793 m_promptlyFreedSize += size;
794 } 794 }
795 795
796 bool NormalPageArena::expandObject(HeapObjectHeader* header, size_t newSize) { 796 bool NormalPageArena::expandObject(HeapObjectHeader* header, size_t newSize) {
797 // It's possible that Vector requests a smaller expanded size because 797 // It's possible that Vector requests a smaller expanded size because
798 // Vector::shrinkCapacity can set a capacity smaller than the actual payload 798 // Vector::shrinkCapacity can set a capacity smaller than the actual payload
799 // size. 799 // size.
800 ASSERT(header->checkHeader()); 800 DCHECK(header->checkHeader());
801 if (header->payloadSize() >= newSize) 801 if (header->payloadSize() >= newSize)
802 return true; 802 return true;
803 size_t allocationSize = ThreadHeap::allocationSizeFromSize(newSize); 803 size_t allocationSize = ThreadHeap::allocationSizeFromSize(newSize);
804 ASSERT(allocationSize > header->size()); 804 DCHECK_GT(allocationSize, header->size());
805 size_t expandSize = allocationSize - header->size(); 805 size_t expandSize = allocationSize - header->size();
806 if (isObjectAllocatedAtAllocationPoint(header) && 806 if (isObjectAllocatedAtAllocationPoint(header) &&
807 expandSize <= m_remainingAllocationSize) { 807 expandSize <= m_remainingAllocationSize) {
808 m_currentAllocationPoint += expandSize; 808 m_currentAllocationPoint += expandSize;
809 ASSERT(m_remainingAllocationSize >= expandSize); 809 DCHECK_GE(m_remainingAllocationSize, expandSize);
810 setRemainingAllocationSize(m_remainingAllocationSize - expandSize); 810 setRemainingAllocationSize(m_remainingAllocationSize - expandSize);
811 // Unpoison the memory used for the object (payload). 811 // Unpoison the memory used for the object (payload).
812 SET_MEMORY_ACCESSIBLE(header->payloadEnd(), expandSize); 812 SET_MEMORY_ACCESSIBLE(header->payloadEnd(), expandSize);
813 header->setSize(allocationSize); 813 header->setSize(allocationSize);
814 ASSERT(findPageFromAddress(header->payloadEnd() - 1)); 814 DCHECK(findPageFromAddress(header->payloadEnd() - 1));
815 return true; 815 return true;
816 } 816 }
817 return false; 817 return false;
818 } 818 }
819 819
820 bool NormalPageArena::shrinkObject(HeapObjectHeader* header, size_t newSize) { 820 bool NormalPageArena::shrinkObject(HeapObjectHeader* header, size_t newSize) {
821 ASSERT(header->checkHeader()); 821 DCHECK(header->checkHeader());
822 ASSERT(header->payloadSize() > newSize); 822 DCHECK_GT(header->payloadSize(), newSize);
823 size_t allocationSize = ThreadHeap::allocationSizeFromSize(newSize); 823 size_t allocationSize = ThreadHeap::allocationSizeFromSize(newSize);
824 ASSERT(header->size() > allocationSize); 824 DCHECK_GT(header->size(), allocationSize);
825 size_t shrinkSize = header->size() - allocationSize; 825 size_t shrinkSize = header->size() - allocationSize;
826 if (isObjectAllocatedAtAllocationPoint(header)) { 826 if (isObjectAllocatedAtAllocationPoint(header)) {
827 m_currentAllocationPoint -= shrinkSize; 827 m_currentAllocationPoint -= shrinkSize;
828 setRemainingAllocationSize(m_remainingAllocationSize + shrinkSize); 828 setRemainingAllocationSize(m_remainingAllocationSize + shrinkSize);
829 SET_MEMORY_INACCESSIBLE(m_currentAllocationPoint, shrinkSize); 829 SET_MEMORY_INACCESSIBLE(m_currentAllocationPoint, shrinkSize);
830 header->setSize(allocationSize); 830 header->setSize(allocationSize);
831 return true; 831 return true;
832 } 832 }
833 ASSERT(shrinkSize >= sizeof(HeapObjectHeader)); 833 DCHECK_GE(shrinkSize, sizeof(HeapObjectHeader));
834 ASSERT(header->gcInfoIndex() > 0); 834 DCHECK_GT(header->gcInfoIndex(), 0UL);
835 Address shrinkAddress = header->payloadEnd() - shrinkSize; 835 Address shrinkAddress = header->payloadEnd() - shrinkSize;
836 HeapObjectHeader* freedHeader = new (NotNull, shrinkAddress) 836 HeapObjectHeader* freedHeader = new (NotNull, shrinkAddress)
837 HeapObjectHeader(shrinkSize, header->gcInfoIndex()); 837 HeapObjectHeader(shrinkSize, header->gcInfoIndex());
838 freedHeader->markPromptlyFreed(); 838 freedHeader->markPromptlyFreed();
839 ASSERT(pageFromObject(reinterpret_cast<Address>(header)) == 839 DCHECK_EQ(pageFromObject(reinterpret_cast<Address>(header)),
840 findPageFromAddress(reinterpret_cast<Address>(header))); 840 findPageFromAddress(reinterpret_cast<Address>(header)));
841 m_promptlyFreedSize += shrinkSize; 841 m_promptlyFreedSize += shrinkSize;
842 header->setSize(allocationSize); 842 header->setSize(allocationSize);
843 SET_MEMORY_INACCESSIBLE(shrinkAddress + sizeof(HeapObjectHeader), 843 SET_MEMORY_INACCESSIBLE(shrinkAddress + sizeof(HeapObjectHeader),
844 shrinkSize - sizeof(HeapObjectHeader)); 844 shrinkSize - sizeof(HeapObjectHeader));
845 return false; 845 return false;
846 } 846 }
847 847
848 Address NormalPageArena::lazySweepPages(size_t allocationSize, 848 Address NormalPageArena::lazySweepPages(size_t allocationSize,
849 size_t gcInfoIndex) { 849 size_t gcInfoIndex) {
850 ASSERT(!hasCurrentAllocationArea()); 850 DCHECK(!hasCurrentAllocationArea());
851 AutoReset<bool> isLazySweeping(&m_isLazySweeping, true); 851 AutoReset<bool> isLazySweeping(&m_isLazySweeping, true);
852 Address result = nullptr; 852 Address result = nullptr;
853 while (m_firstUnsweptPage) { 853 while (m_firstUnsweptPage) {
854 BasePage* page = m_firstUnsweptPage; 854 BasePage* page = m_firstUnsweptPage;
855 if (page->isEmpty()) { 855 if (page->isEmpty()) {
856 page->unlink(&m_firstUnsweptPage); 856 page->unlink(&m_firstUnsweptPage);
857 page->removeFromHeap(); 857 page->removeFromHeap();
858 } else { 858 } else {
859 // Sweep a page and move the page from m_firstUnsweptPages to 859 // Sweep a page and move the page from m_firstUnsweptPages to
860 // m_firstPages. 860 // m_firstPages.
(...skipping 28 matching lines...) Expand all
889 m_remainingAllocationSize - m_lastRemainingAllocationSize); 889 m_remainingAllocationSize - m_lastRemainingAllocationSize);
890 m_lastRemainingAllocationSize = m_remainingAllocationSize; 890 m_lastRemainingAllocationSize = m_remainingAllocationSize;
891 } 891 }
892 892
893 void NormalPageArena::updateRemainingAllocationSize() { 893 void NormalPageArena::updateRemainingAllocationSize() {
894 if (m_lastRemainingAllocationSize > remainingAllocationSize()) { 894 if (m_lastRemainingAllocationSize > remainingAllocationSize()) {
895 getThreadState()->increaseAllocatedObjectSize( 895 getThreadState()->increaseAllocatedObjectSize(
896 m_lastRemainingAllocationSize - remainingAllocationSize()); 896 m_lastRemainingAllocationSize - remainingAllocationSize());
897 m_lastRemainingAllocationSize = remainingAllocationSize(); 897 m_lastRemainingAllocationSize = remainingAllocationSize();
898 } 898 }
899 ASSERT(m_lastRemainingAllocationSize == remainingAllocationSize()); 899 DCHECK_EQ(m_lastRemainingAllocationSize, remainingAllocationSize());
900 } 900 }
901 901
902 void NormalPageArena::setAllocationPoint(Address point, size_t size) { 902 void NormalPageArena::setAllocationPoint(Address point, size_t size) {
903 #if ENABLE(ASSERT) 903 #if DCHECK_IS_ON()
904 if (point) { 904 if (point) {
905 ASSERT(size); 905 DCHECK(size);
906 BasePage* page = pageFromObject(point); 906 BasePage* page = pageFromObject(point);
907 ASSERT(!page->isLargeObjectPage()); 907 DCHECK(!page->isLargeObjectPage());
908 ASSERT(size <= static_cast<NormalPage*>(page)->payloadSize()); 908 DCHECK_LE(size, static_cast<NormalPage*>(page)->payloadSize());
909 } 909 }
910 #endif 910 #endif
911 if (hasCurrentAllocationArea()) { 911 if (hasCurrentAllocationArea()) {
912 addToFreeList(currentAllocationPoint(), remainingAllocationSize()); 912 addToFreeList(currentAllocationPoint(), remainingAllocationSize());
913 } 913 }
914 updateRemainingAllocationSize(); 914 updateRemainingAllocationSize();
915 m_currentAllocationPoint = point; 915 m_currentAllocationPoint = point;
916 m_lastRemainingAllocationSize = m_remainingAllocationSize = size; 916 m_lastRemainingAllocationSize = m_remainingAllocationSize = size;
917 } 917 }
918 918
919 Address NormalPageArena::outOfLineAllocate(size_t allocationSize, 919 Address NormalPageArena::outOfLineAllocate(size_t allocationSize,
920 size_t gcInfoIndex) { 920 size_t gcInfoIndex) {
921 ASSERT(allocationSize > remainingAllocationSize()); 921 DCHECK_GT(allocationSize, remainingAllocationSize());
922 ASSERT(allocationSize >= allocationGranularity); 922 DCHECK_GE(allocationSize, allocationGranularity);
923 923
924 // 1. If this allocation is big enough, allocate a large object. 924 // 1. If this allocation is big enough, allocate a large object.
925 if (allocationSize >= largeObjectSizeThreshold) 925 if (allocationSize >= largeObjectSizeThreshold)
926 return allocateLargeObject(allocationSize, gcInfoIndex); 926 return allocateLargeObject(allocationSize, gcInfoIndex);
927 927
928 // 2. Try to allocate from a free list. 928 // 2. Try to allocate from a free list.
929 updateRemainingAllocationSize(); 929 updateRemainingAllocationSize();
930 Address result = allocateFromFreeList(allocationSize, gcInfoIndex); 930 Address result = allocateFromFreeList(allocationSize, gcInfoIndex);
931 if (result) 931 if (result)
932 return result; 932 return result;
(...skipping 19 matching lines...) Expand all
952 getThreadState()->completeSweep(); 952 getThreadState()->completeSweep();
953 953
954 // 7. Check if we should trigger a GC. 954 // 7. Check if we should trigger a GC.
955 getThreadState()->scheduleGCIfNeeded(); 955 getThreadState()->scheduleGCIfNeeded();
956 956
957 // 8. Add a new page to this heap. 957 // 8. Add a new page to this heap.
958 allocatePage(); 958 allocatePage();
959 959
960 // 9. Try to allocate from a free list. This allocation must succeed. 960 // 9. Try to allocate from a free list. This allocation must succeed.
961 result = allocateFromFreeList(allocationSize, gcInfoIndex); 961 result = allocateFromFreeList(allocationSize, gcInfoIndex);
962 RELEASE_ASSERT(result); 962 CHECK(result);
963 return result; 963 return result;
964 } 964 }
965 965
966 Address NormalPageArena::allocateFromFreeList(size_t allocationSize, 966 Address NormalPageArena::allocateFromFreeList(size_t allocationSize,
967 size_t gcInfoIndex) { 967 size_t gcInfoIndex) {
968 // Try reusing a block from the largest bin. The underlying reasoning 968 // Try reusing a block from the largest bin. The underlying reasoning
969 // being that we want to amortize this slow allocation call by carving 969 // being that we want to amortize this slow allocation call by carving
970 // off as a large a free block as possible in one go; a block that will 970 // off as a large a free block as possible in one go; a block that will
971 // service this block and let following allocations be serviced quickly 971 // service this block and let following allocations be serviced quickly
972 // by bump allocation. 972 // by bump allocation.
973 size_t bucketSize = static_cast<size_t>(1) 973 size_t bucketSize = static_cast<size_t>(1)
974 << m_freeList.m_biggestFreeListIndex; 974 << m_freeList.m_biggestFreeListIndex;
975 int index = m_freeList.m_biggestFreeListIndex; 975 int index = m_freeList.m_biggestFreeListIndex;
976 for (; index > 0; --index, bucketSize >>= 1) { 976 for (; index > 0; --index, bucketSize >>= 1) {
977 FreeListEntry* entry = m_freeList.m_freeLists[index]; 977 FreeListEntry* entry = m_freeList.m_freeLists[index];
978 if (allocationSize > bucketSize) { 978 if (allocationSize > bucketSize) {
979 // Final bucket candidate; check initial entry if it is able 979 // Final bucket candidate; check initial entry if it is able
980 // to service this allocation. Do not perform a linear scan, 980 // to service this allocation. Do not perform a linear scan,
981 // as it is considered too costly. 981 // as it is considered too costly.
982 if (!entry || entry->size() < allocationSize) 982 if (!entry || entry->size() < allocationSize)
983 break; 983 break;
984 } 984 }
985 if (entry) { 985 if (entry) {
986 entry->unlink(&m_freeList.m_freeLists[index]); 986 entry->unlink(&m_freeList.m_freeLists[index]);
987 setAllocationPoint(entry->getAddress(), entry->size()); 987 setAllocationPoint(entry->getAddress(), entry->size());
988 ASSERT(hasCurrentAllocationArea()); 988 DCHECK(hasCurrentAllocationArea());
989 ASSERT(remainingAllocationSize() >= allocationSize); 989 DCHECK_GE(remainingAllocationSize(), allocationSize);
990 m_freeList.m_biggestFreeListIndex = index; 990 m_freeList.m_biggestFreeListIndex = index;
991 return allocateObject(allocationSize, gcInfoIndex); 991 return allocateObject(allocationSize, gcInfoIndex);
992 } 992 }
993 } 993 }
994 m_freeList.m_biggestFreeListIndex = index; 994 m_freeList.m_biggestFreeListIndex = index;
995 return nullptr; 995 return nullptr;
996 } 996 }
997 997
998 LargeObjectArena::LargeObjectArena(ThreadState* state, int index) 998 LargeObjectArena::LargeObjectArena(ThreadState* state, int index)
999 : BaseArena(state, index) {} 999 : BaseArena(state, index) {}
1000 1000
1001 Address LargeObjectArena::allocateLargeObjectPage(size_t allocationSize, 1001 Address LargeObjectArena::allocateLargeObjectPage(size_t allocationSize,
1002 size_t gcInfoIndex) { 1002 size_t gcInfoIndex) {
1003 // Caller already added space for object header and rounded up to allocation 1003 // Caller already added space for object header and rounded up to allocation
1004 // alignment 1004 // alignment
1005 ASSERT(!(allocationSize & allocationMask)); 1005 DCHECK(!(allocationSize & allocationMask));
1006 1006
1007 // 1. Try to sweep large objects more than allocationSize bytes 1007 // 1. Try to sweep large objects more than allocationSize bytes
1008 // before allocating a new large object. 1008 // before allocating a new large object.
1009 Address result = lazySweep(allocationSize, gcInfoIndex); 1009 Address result = lazySweep(allocationSize, gcInfoIndex);
1010 if (result) 1010 if (result)
1011 return result; 1011 return result;
1012 1012
1013 // 2. If we have failed in sweeping allocationSize bytes, 1013 // 2. If we have failed in sweeping allocationSize bytes,
1014 // we complete sweeping before allocating this large object. 1014 // we complete sweeping before allocating this large object.
1015 getThreadState()->completeSweep(); 1015 getThreadState()->completeSweep();
(...skipping 12 matching lines...) Expand all
1028 #if defined(ADDRESS_SANITIZER) 1028 #if defined(ADDRESS_SANITIZER)
1029 largeObjectSize += allocationGranularity; 1029 largeObjectSize += allocationGranularity;
1030 #endif 1030 #endif
1031 1031
1032 getThreadState()->shouldFlushHeapDoesNotContainCache(); 1032 getThreadState()->shouldFlushHeapDoesNotContainCache();
1033 PageMemory* pageMemory = PageMemory::allocate( 1033 PageMemory* pageMemory = PageMemory::allocate(
1034 largeObjectSize, getThreadState()->heap().getRegionTree()); 1034 largeObjectSize, getThreadState()->heap().getRegionTree());
1035 Address largeObjectAddress = pageMemory->writableStart(); 1035 Address largeObjectAddress = pageMemory->writableStart();
1036 Address headerAddress = 1036 Address headerAddress =
1037 largeObjectAddress + LargeObjectPage::pageHeaderSize(); 1037 largeObjectAddress + LargeObjectPage::pageHeaderSize();
1038 #if ENABLE(ASSERT) 1038 #if DCHECK_IS_ON()
1039 // Verify that the allocated PageMemory is expectedly zeroed. 1039 // Verify that the allocated PageMemory is expectedly zeroed.
1040 for (size_t i = 0; i < largeObjectSize; ++i) 1040 for (size_t i = 0; i < largeObjectSize; ++i)
1041 ASSERT(!largeObjectAddress[i]); 1041 DCHECK(!largeObjectAddress[i]);
1042 #endif 1042 #endif
1043 ASSERT(gcInfoIndex > 0); 1043 DCHECK_GT(gcInfoIndex, 0UL);
1044 HeapObjectHeader* header = new (NotNull, headerAddress) 1044 HeapObjectHeader* header = new (NotNull, headerAddress)
1045 HeapObjectHeader(largeObjectSizeInHeader, gcInfoIndex); 1045 HeapObjectHeader(largeObjectSizeInHeader, gcInfoIndex);
1046 Address result = headerAddress + sizeof(*header); 1046 Address result = headerAddress + sizeof(*header);
1047 ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask)); 1047 DCHECK(!(reinterpret_cast<uintptr_t>(result) & allocationMask));
1048 LargeObjectPage* largeObject = new (largeObjectAddress) 1048 LargeObjectPage* largeObject = new (largeObjectAddress)
1049 LargeObjectPage(pageMemory, this, allocationSize); 1049 LargeObjectPage(pageMemory, this, allocationSize);
1050 ASSERT(header->checkHeader()); 1050 DCHECK(header->checkHeader());
1051 1051
1052 // Poison the object header and allocationGranularity bytes after the object 1052 // Poison the object header and allocationGranularity bytes after the object
1053 ASAN_POISON_MEMORY_REGION(header, sizeof(*header)); 1053 ASAN_POISON_MEMORY_REGION(header, sizeof(*header));
1054 ASAN_POISON_MEMORY_REGION(largeObject->getAddress() + largeObject->size(), 1054 ASAN_POISON_MEMORY_REGION(largeObject->getAddress() + largeObject->size(),
1055 allocationGranularity); 1055 allocationGranularity);
1056 1056
1057 largeObject->link(&m_firstPage); 1057 largeObject->link(&m_firstPage);
1058 1058
1059 getThreadState()->heap().heapStats().increaseAllocatedSpace( 1059 getThreadState()->heap().heapStats().increaseAllocatedSpace(
1060 largeObject->size()); 1060 largeObject->size());
1061 getThreadState()->increaseAllocatedObjectSize(largeObject->size()); 1061 getThreadState()->increaseAllocatedObjectSize(largeObject->size());
1062 return result; 1062 return result;
1063 } 1063 }
1064 1064
1065 void LargeObjectArena::freeLargeObjectPage(LargeObjectPage* object) { 1065 void LargeObjectArena::freeLargeObjectPage(LargeObjectPage* object) {
1066 ASAN_UNPOISON_MEMORY_REGION(object->payload(), object->payloadSize()); 1066 ASAN_UNPOISON_MEMORY_REGION(object->payload(), object->payloadSize());
1067 object->heapObjectHeader()->finalize(object->payload(), 1067 object->heapObjectHeader()->finalize(object->payload(),
1068 object->payloadSize()); 1068 object->payloadSize());
1069 getThreadState()->heap().heapStats().decreaseAllocatedSpace(object->size()); 1069 getThreadState()->heap().heapStats().decreaseAllocatedSpace(object->size());
1070 1070
1071 // Unpoison the object header and allocationGranularity bytes after the 1071 // Unpoison the object header and allocationGranularity bytes after the
1072 // object before freeing. 1072 // object before freeing.
1073 ASAN_UNPOISON_MEMORY_REGION(object->heapObjectHeader(), 1073 ASAN_UNPOISON_MEMORY_REGION(object->heapObjectHeader(),
1074 sizeof(HeapObjectHeader)); 1074 sizeof(HeapObjectHeader));
1075 ASAN_UNPOISON_MEMORY_REGION(object->getAddress() + object->size(), 1075 ASAN_UNPOISON_MEMORY_REGION(object->getAddress() + object->size(),
1076 allocationGranularity); 1076 allocationGranularity);
1077 1077
1078 if (object->terminating()) { 1078 if (object->terminating()) {
1079 ASSERT(ThreadState::current()->isTerminating()); 1079 DCHECK(ThreadState::current()->isTerminating());
1080 // The thread is shutting down and this page is being removed as a part 1080 // The thread is shutting down and this page is being removed as a part
1081 // of the thread local GC. In that case the object could be traced in 1081 // of the thread local GC. In that case the object could be traced in
1082 // the next global GC if there is a dangling pointer from a live thread 1082 // the next global GC if there is a dangling pointer from a live thread
1083 // heap to this dead thread heap. To guard against this, we put the 1083 // heap to this dead thread heap. To guard against this, we put the
1084 // page into the orphaned page pool and zap the page memory. This 1084 // page into the orphaned page pool and zap the page memory. This
1085 // ensures that tracing the dangling pointer in the next global GC just 1085 // ensures that tracing the dangling pointer in the next global GC just
1086 // crashes instead of causing use-after-frees. After the next global 1086 // crashes instead of causing use-after-frees. After the next global
1087 // GC, the orphaned pages are removed. 1087 // GC, the orphaned pages are removed.
1088 getThreadState()->heap().getOrphanedPagePool()->addOrphanedPage( 1088 getThreadState()->heap().getOrphanedPagePool()->addOrphanedPage(
1089 arenaIndex(), object); 1089 arenaIndex(), object);
1090 } else { 1090 } else {
1091 ASSERT(!ThreadState::current()->isTerminating()); 1091 DCHECK(!ThreadState::current()->isTerminating());
1092 PageMemory* memory = object->storage(); 1092 PageMemory* memory = object->storage();
1093 object->~LargeObjectPage(); 1093 object->~LargeObjectPage();
1094 delete memory; 1094 delete memory;
1095 } 1095 }
1096 } 1096 }
1097 1097
1098 Address LargeObjectArena::lazySweepPages(size_t allocationSize, 1098 Address LargeObjectArena::lazySweepPages(size_t allocationSize,
1099 size_t gcInfoIndex) { 1099 size_t gcInfoIndex) {
1100 Address result = nullptr; 1100 Address result = nullptr;
1101 size_t sweptSize = 0; 1101 size_t sweptSize = 0;
1102 while (m_firstUnsweptPage) { 1102 while (m_firstUnsweptPage) {
1103 BasePage* page = m_firstUnsweptPage; 1103 BasePage* page = m_firstUnsweptPage;
1104 if (page->isEmpty()) { 1104 if (page->isEmpty()) {
1105 sweptSize += static_cast<LargeObjectPage*>(page)->payloadSize() + 1105 sweptSize += static_cast<LargeObjectPage*>(page)->payloadSize() +
1106 sizeof(HeapObjectHeader); 1106 sizeof(HeapObjectHeader);
1107 page->unlink(&m_firstUnsweptPage); 1107 page->unlink(&m_firstUnsweptPage);
1108 page->removeFromHeap(); 1108 page->removeFromHeap();
1109 // For LargeObjectPage, stop lazy sweeping once we have swept 1109 // For LargeObjectPage, stop lazy sweeping once we have swept
1110 // more than allocationSize bytes. 1110 // more than allocationSize bytes.
1111 if (sweptSize >= allocationSize) { 1111 if (sweptSize >= allocationSize) {
1112 result = doAllocateLargeObjectPage(allocationSize, gcInfoIndex); 1112 result = doAllocateLargeObjectPage(allocationSize, gcInfoIndex);
1113 ASSERT(result); 1113 DCHECK(result);
1114 break; 1114 break;
1115 } 1115 }
1116 } else { 1116 } else {
1117 // Sweep a page and move the page from m_firstUnsweptPages to 1117 // Sweep a page and move the page from m_firstUnsweptPages to
1118 // m_firstPages. 1118 // m_firstPages.
1119 page->sweep(); 1119 page->sweep();
1120 page->unlink(&m_firstUnsweptPage); 1120 page->unlink(&m_firstUnsweptPage);
1121 page->link(&m_firstPage); 1121 page->link(&m_firstPage);
1122 page->markAsSwept(); 1122 page->markAsSwept();
1123 } 1123 }
1124 } 1124 }
1125 return result; 1125 return result;
1126 } 1126 }
1127 1127
1128 FreeList::FreeList() : m_biggestFreeListIndex(0) {} 1128 FreeList::FreeList() : m_biggestFreeListIndex(0) {}
1129 1129
1130 void FreeList::addToFreeList(Address address, size_t size) { 1130 void FreeList::addToFreeList(Address address, size_t size) {
1131 ASSERT(size < blinkPagePayloadSize()); 1131 DCHECK_LT(size, blinkPagePayloadSize());
1132 // The free list entries are only pointer aligned (but when we allocate 1132 // The free list entries are only pointer aligned (but when we allocate
1133 // from them we are 8 byte aligned due to the header size). 1133 // from them we are 8 byte aligned due to the header size).
1134 ASSERT(!((reinterpret_cast<uintptr_t>(address) + sizeof(HeapObjectHeader)) & 1134 DCHECK(!((reinterpret_cast<uintptr_t>(address) + sizeof(HeapObjectHeader)) &
1135 allocationMask)); 1135 allocationMask));
1136 ASSERT(!(size & allocationMask)); 1136 DCHECK(!(size & allocationMask));
1137 ASAN_UNPOISON_MEMORY_REGION(address, size); 1137 ASAN_UNPOISON_MEMORY_REGION(address, size);
1138 FreeListEntry* entry; 1138 FreeListEntry* entry;
1139 if (size < sizeof(*entry)) { 1139 if (size < sizeof(*entry)) {
1140 // Create a dummy header with only a size and freelist bit set. 1140 // Create a dummy header with only a size and freelist bit set.
1141 ASSERT(size >= sizeof(HeapObjectHeader)); 1141 DCHECK_GE(size, sizeof(HeapObjectHeader));
1142 // Free list encode the size to mark the lost memory as freelist memory. 1142 // Free list encode the size to mark the lost memory as freelist memory.
1143 new (NotNull, address) HeapObjectHeader(size, gcInfoIndexForFreeListHeader); 1143 new (NotNull, address) HeapObjectHeader(size, gcInfoIndexForFreeListHeader);
1144 1144
1145 ASAN_POISON_MEMORY_REGION(address, size); 1145 ASAN_POISON_MEMORY_REGION(address, size);
1146 // This memory gets lost. Sweeping can reclaim it. 1146 // This memory gets lost. Sweeping can reclaim it.
1147 return; 1147 return;
1148 } 1148 }
1149 entry = new (NotNull, address) FreeListEntry(size); 1149 entry = new (NotNull, address) FreeListEntry(size);
1150 1150
1151 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) 1151 #if DCHECK_IS_ON() || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER)
1152 // The following logic delays reusing free lists for (at least) one GC 1152 // The following logic delays reusing free lists for (at least) one GC
1153 // cycle or coalescing. This is helpful to detect use-after-free errors 1153 // cycle or coalescing. This is helpful to detect use-after-free errors
1154 // that could be caused by lazy sweeping etc. 1154 // that could be caused by lazy sweeping etc.
1155 size_t allowedCount = 0; 1155 size_t allowedCount = 0;
1156 size_t forbiddenCount = 0; 1156 size_t forbiddenCount = 0;
1157 for (size_t i = sizeof(FreeListEntry); i < size; i++) { 1157 for (size_t i = sizeof(FreeListEntry); i < size; i++) {
1158 if (address[i] == reuseAllowedZapValue) 1158 if (address[i] == reuseAllowedZapValue)
1159 allowedCount++; 1159 allowedCount++;
1160 else if (address[i] == reuseForbiddenZapValue) 1160 else if (address[i] == reuseForbiddenZapValue)
1161 forbiddenCount++; 1161 forbiddenCount++;
1162 else 1162 else
1163 ASSERT_NOT_REACHED(); 1163 NOTREACHED();
1164 } 1164 }
1165 size_t entryCount = size - sizeof(FreeListEntry); 1165 size_t entryCount = size - sizeof(FreeListEntry);
1166 if (forbiddenCount == entryCount) { 1166 if (forbiddenCount == entryCount) {
1167 // If all values in the memory region are reuseForbiddenZapValue, 1167 // If all values in the memory region are reuseForbiddenZapValue,
1168 // we flip them to reuseAllowedZapValue. This allows the next 1168 // we flip them to reuseAllowedZapValue. This allows the next
1169 // addToFreeList() to add the memory region to the free list 1169 // addToFreeList() to add the memory region to the free list
1170 // (unless someone concatenates the memory region with another memory 1170 // (unless someone concatenates the memory region with another memory
1171 // region that contains reuseForbiddenZapValue.) 1171 // region that contains reuseForbiddenZapValue.)
1172 for (size_t i = sizeof(FreeListEntry); i < size; i++) 1172 for (size_t i = sizeof(FreeListEntry); i < size; i++)
1173 address[i] = reuseAllowedZapValue; 1173 address[i] = reuseAllowedZapValue;
(...skipping 17 matching lines...) Expand all
1191 // region to the free list and reuse it for another object. 1191 // region to the free list and reuse it for another object.
1192 #endif 1192 #endif
1193 ASAN_POISON_MEMORY_REGION(address, size); 1193 ASAN_POISON_MEMORY_REGION(address, size);
1194 1194
1195 int index = bucketIndexForSize(size); 1195 int index = bucketIndexForSize(size);
1196 entry->link(&m_freeLists[index]); 1196 entry->link(&m_freeLists[index]);
1197 if (index > m_biggestFreeListIndex) 1197 if (index > m_biggestFreeListIndex)
1198 m_biggestFreeListIndex = index; 1198 m_biggestFreeListIndex = index;
1199 } 1199 }
1200 1200
1201 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) || \ 1201 #if DCHECK_IS_ON() || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) || \
1202 defined(MEMORY_SANITIZER) 1202 defined(MEMORY_SANITIZER)
1203 NO_SANITIZE_ADDRESS 1203 NO_SANITIZE_ADDRESS
1204 NO_SANITIZE_MEMORY 1204 NO_SANITIZE_MEMORY
1205 void NEVER_INLINE FreeList::zapFreedMemory(Address address, size_t size) { 1205 void NEVER_INLINE FreeList::zapFreedMemory(Address address, size_t size) {
1206 for (size_t i = 0; i < size; i++) { 1206 for (size_t i = 0; i < size; i++) {
1207 // See the comment in addToFreeList(). 1207 // See the comment in addToFreeList().
1208 if (address[i] != reuseAllowedZapValue) 1208 if (address[i] != reuseAllowedZapValue)
1209 address[i] = reuseForbiddenZapValue; 1209 address[i] = reuseForbiddenZapValue;
1210 } 1210 }
1211 } 1211 }
1212 1212
1213 void NEVER_INLINE FreeList::checkFreedMemoryIsZapped(Address address, 1213 void NEVER_INLINE FreeList::checkFreedMemoryIsZapped(Address address,
1214 size_t size) { 1214 size_t size) {
1215 for (size_t i = 0; i < size; i++) { 1215 for (size_t i = 0; i < size; i++) {
1216 ASSERT(address[i] == reuseAllowedZapValue || 1216 DCHECK(address[i] == reuseAllowedZapValue ||
1217 address[i] == reuseForbiddenZapValue); 1217 address[i] == reuseForbiddenZapValue);
1218 } 1218 }
1219 } 1219 }
1220 #endif 1220 #endif
1221 1221
1222 size_t FreeList::freeListSize() const { 1222 size_t FreeList::freeListSize() const {
1223 size_t freeSize = 0; 1223 size_t freeSize = 0;
1224 for (unsigned i = 0; i < blinkPageSizeLog2; ++i) { 1224 for (unsigned i = 0; i < blinkPageSizeLog2; ++i) {
1225 FreeListEntry* entry = m_freeLists[i]; 1225 FreeListEntry* entry = m_freeLists[i];
1226 while (entry) { 1226 while (entry) {
(...skipping 23 matching lines...) Expand all
1250 return freeSize; 1250 return freeSize;
1251 } 1251 }
1252 1252
1253 void FreeList::clear() { 1253 void FreeList::clear() {
1254 m_biggestFreeListIndex = 0; 1254 m_biggestFreeListIndex = 0;
1255 for (size_t i = 0; i < blinkPageSizeLog2; ++i) 1255 for (size_t i = 0; i < blinkPageSizeLog2; ++i)
1256 m_freeLists[i] = nullptr; 1256 m_freeLists[i] = nullptr;
1257 } 1257 }
1258 1258
1259 int FreeList::bucketIndexForSize(size_t size) { 1259 int FreeList::bucketIndexForSize(size_t size) {
1260 ASSERT(size > 0); 1260 DCHECK_GT(size, 0UL);
1261 int index = -1; 1261 int index = -1;
1262 while (size) { 1262 while (size) {
1263 size >>= 1; 1263 size >>= 1;
1264 index++; 1264 index++;
1265 } 1265 }
1266 return index; 1266 return index;
1267 } 1267 }
1268 1268
1269 bool FreeList::takeSnapshot(const String& dumpBaseName) { 1269 bool FreeList::takeSnapshot(const String& dumpBaseName) {
1270 bool didDumpBucketStats = false; 1270 bool didDumpBucketStats = false;
(...skipping 17 matching lines...) Expand all
1288 } 1288 }
1289 return didDumpBucketStats; 1289 return didDumpBucketStats;
1290 } 1290 }
1291 1291
1292 BasePage::BasePage(PageMemory* storage, BaseArena* arena) 1292 BasePage::BasePage(PageMemory* storage, BaseArena* arena)
1293 : m_storage(storage), 1293 : m_storage(storage),
1294 m_arena(arena), 1294 m_arena(arena),
1295 m_next(nullptr), 1295 m_next(nullptr),
1296 m_terminating(false), 1296 m_terminating(false),
1297 m_swept(true) { 1297 m_swept(true) {
1298 ASSERT(isPageHeaderAddress(reinterpret_cast<Address>(this))); 1298 DCHECK(isPageHeaderAddress(reinterpret_cast<Address>(this)));
1299 } 1299 }
1300 1300
1301 void BasePage::markOrphaned() { 1301 void BasePage::markOrphaned() {
1302 m_arena = nullptr; 1302 m_arena = nullptr;
1303 m_terminating = false; 1303 m_terminating = false;
1304 // Since we zap the page payload for orphaned pages we need to mark it as 1304 // Since we zap the page payload for orphaned pages we need to mark it as
1305 // unused so a conservative pointer won't interpret the object headers. 1305 // unused so a conservative pointer won't interpret the object headers.
1306 storage()->markUnused(); 1306 storage()->markUnused();
1307 } 1307 }
1308 1308
1309 NormalPage::NormalPage(PageMemory* storage, BaseArena* arena) 1309 NormalPage::NormalPage(PageMemory* storage, BaseArena* arena)
1310 : BasePage(storage, arena), m_objectStartBitMapComputed(false) { 1310 : BasePage(storage, arena), m_objectStartBitMapComputed(false) {
1311 ASSERT(isPageHeaderAddress(reinterpret_cast<Address>(this))); 1311 DCHECK(isPageHeaderAddress(reinterpret_cast<Address>(this)));
1312 } 1312 }
1313 1313
1314 size_t NormalPage::objectPayloadSizeForTesting() { 1314 size_t NormalPage::objectPayloadSizeForTesting() {
1315 size_t objectPayloadSize = 0; 1315 size_t objectPayloadSize = 0;
1316 Address headerAddress = payload(); 1316 Address headerAddress = payload();
1317 markAsSwept(); 1317 markAsSwept();
1318 ASSERT(headerAddress != payloadEnd()); 1318 DCHECK_NE(headerAddress, payloadEnd());
1319 do { 1319 do {
1320 HeapObjectHeader* header = 1320 HeapObjectHeader* header =
1321 reinterpret_cast<HeapObjectHeader*>(headerAddress); 1321 reinterpret_cast<HeapObjectHeader*>(headerAddress);
1322 if (!header->isFree()) { 1322 if (!header->isFree()) {
1323 ASSERT(header->checkHeader()); 1323 DCHECK(header->checkHeader());
1324 objectPayloadSize += header->payloadSize(); 1324 objectPayloadSize += header->payloadSize();
1325 } 1325 }
1326 ASSERT(header->size() < blinkPagePayloadSize()); 1326 DCHECK_LT(header->size(), blinkPagePayloadSize());
1327 headerAddress += header->size(); 1327 headerAddress += header->size();
1328 ASSERT(headerAddress <= payloadEnd()); 1328 DCHECK_LE(headerAddress, payloadEnd());
1329 } while (headerAddress < payloadEnd()); 1329 } while (headerAddress < payloadEnd());
1330 return objectPayloadSize; 1330 return objectPayloadSize;
1331 } 1331 }
1332 1332
1333 bool NormalPage::isEmpty() { 1333 bool NormalPage::isEmpty() {
1334 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(payload()); 1334 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(payload());
1335 return header->isFree() && header->size() == payloadSize(); 1335 return header->isFree() && header->size() == payloadSize();
1336 } 1336 }
1337 1337
1338 void NormalPage::removeFromHeap() { 1338 void NormalPage::removeFromHeap() {
1339 arenaForNormalPage()->freePage(this); 1339 arenaForNormalPage()->freePage(this);
1340 } 1340 }
1341 1341
1342 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER) 1342 #if !DCHECK_IS_ON() && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER)
1343 static void discardPages(Address begin, Address end) { 1343 static void discardPages(Address begin, Address end) {
1344 uintptr_t beginAddress = 1344 uintptr_t beginAddress =
1345 WTF::roundUpToSystemPage(reinterpret_cast<uintptr_t>(begin)); 1345 WTF::roundUpToSystemPage(reinterpret_cast<uintptr_t>(begin));
1346 uintptr_t endAddress = 1346 uintptr_t endAddress =
1347 WTF::roundDownToSystemPage(reinterpret_cast<uintptr_t>(end)); 1347 WTF::roundDownToSystemPage(reinterpret_cast<uintptr_t>(end));
1348 if (beginAddress < endAddress) 1348 if (beginAddress < endAddress)
1349 WTF::discardSystemPages(reinterpret_cast<void*>(beginAddress), 1349 WTF::discardSystemPages(reinterpret_cast<void*>(beginAddress),
1350 endAddress - beginAddress); 1350 endAddress - beginAddress);
1351 } 1351 }
1352 #endif 1352 #endif
1353 1353
1354 void NormalPage::sweep() { 1354 void NormalPage::sweep() {
1355 size_t markedObjectSize = 0; 1355 size_t markedObjectSize = 0;
1356 Address startOfGap = payload(); 1356 Address startOfGap = payload();
1357 NormalPageArena* pageArena = arenaForNormalPage(); 1357 NormalPageArena* pageArena = arenaForNormalPage();
1358 for (Address headerAddress = startOfGap; headerAddress < payloadEnd();) { 1358 for (Address headerAddress = startOfGap; headerAddress < payloadEnd();) {
1359 HeapObjectHeader* header = 1359 HeapObjectHeader* header =
1360 reinterpret_cast<HeapObjectHeader*>(headerAddress); 1360 reinterpret_cast<HeapObjectHeader*>(headerAddress);
1361 size_t size = header->size(); 1361 size_t size = header->size();
1362 ASSERT(size > 0); 1362 DCHECK_GT(size, 0UL);
1363 ASSERT(size < blinkPagePayloadSize()); 1363 DCHECK_LT(size, blinkPagePayloadSize());
1364 1364
1365 if (header->isPromptlyFreed()) 1365 if (header->isPromptlyFreed())
1366 pageArena->decreasePromptlyFreedSize(size); 1366 pageArena->decreasePromptlyFreedSize(size);
1367 if (header->isFree()) { 1367 if (header->isFree()) {
1368 // Zero the memory in the free list header to maintain the 1368 // Zero the memory in the free list header to maintain the
1369 // invariant that memory on the free list is zero filled. 1369 // invariant that memory on the free list is zero filled.
1370 // The rest of the memory is already on the free list and is 1370 // The rest of the memory is already on the free list and is
1371 // therefore already zero filled. 1371 // therefore already zero filled.
1372 SET_MEMORY_INACCESSIBLE(headerAddress, size < sizeof(FreeListEntry) 1372 SET_MEMORY_INACCESSIBLE(headerAddress, size < sizeof(FreeListEntry)
1373 ? size 1373 ? size
(...skipping 14 matching lines...) Expand all
1388 ASAN_UNPOISON_MEMORY_REGION(payload, payloadSize); 1388 ASAN_UNPOISON_MEMORY_REGION(payload, payloadSize);
1389 header->finalize(payload, payloadSize); 1389 header->finalize(payload, payloadSize);
1390 // This memory will be added to the freelist. Maintain the invariant 1390 // This memory will be added to the freelist. Maintain the invariant
1391 // that memory on the freelist is zero filled. 1391 // that memory on the freelist is zero filled.
1392 SET_MEMORY_INACCESSIBLE(headerAddress, size); 1392 SET_MEMORY_INACCESSIBLE(headerAddress, size);
1393 headerAddress += size; 1393 headerAddress += size;
1394 continue; 1394 continue;
1395 } 1395 }
1396 if (startOfGap != headerAddress) { 1396 if (startOfGap != headerAddress) {
1397 pageArena->addToFreeList(startOfGap, headerAddress - startOfGap); 1397 pageArena->addToFreeList(startOfGap, headerAddress - startOfGap);
1398 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER) 1398 #if !DCHECK_IS_ON() && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER)
1399 // Discarding pages increases page faults and may regress performance. 1399 // Discarding pages increases page faults and may regress performance.
1400 // So we enable this only on low-RAM devices. 1400 // So we enable this only on low-RAM devices.
1401 if (MemoryCoordinator::isLowEndDevice()) 1401 if (MemoryCoordinator::isLowEndDevice())
1402 discardPages(startOfGap + sizeof(FreeListEntry), headerAddress); 1402 discardPages(startOfGap + sizeof(FreeListEntry), headerAddress);
1403 #endif 1403 #endif
1404 } 1404 }
1405 header->unmark(); 1405 header->unmark();
1406 headerAddress += size; 1406 headerAddress += size;
1407 markedObjectSize += size; 1407 markedObjectSize += size;
1408 startOfGap = headerAddress; 1408 startOfGap = headerAddress;
1409 } 1409 }
1410 if (startOfGap != payloadEnd()) { 1410 if (startOfGap != payloadEnd()) {
1411 pageArena->addToFreeList(startOfGap, payloadEnd() - startOfGap); 1411 pageArena->addToFreeList(startOfGap, payloadEnd() - startOfGap);
1412 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER) 1412 #if !DCHECK_IS_ON() && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER)
1413 if (MemoryCoordinator::isLowEndDevice()) 1413 if (MemoryCoordinator::isLowEndDevice())
1414 discardPages(startOfGap + sizeof(FreeListEntry), payloadEnd()); 1414 discardPages(startOfGap + sizeof(FreeListEntry), payloadEnd());
1415 #endif 1415 #endif
1416 } 1416 }
1417 1417
1418 if (markedObjectSize) 1418 if (markedObjectSize)
1419 pageArena->getThreadState()->increaseMarkedObjectSize(markedObjectSize); 1419 pageArena->getThreadState()->increaseMarkedObjectSize(markedObjectSize);
1420 } 1420 }
1421 1421
1422 void NormalPage::sweepAndCompact(CompactionContext& context) { 1422 void NormalPage::sweepAndCompact(CompactionContext& context) {
(...skipping 29 matching lines...) Expand all
1452 // finalized object will be zero-filled and poison'ed afterwards. 1452 // finalized object will be zero-filled and poison'ed afterwards.
1453 // Given all other unmarked objects are poisoned, ASan will detect 1453 // Given all other unmarked objects are poisoned, ASan will detect
1454 // an error if the finalizer touches any other on-heap object that 1454 // an error if the finalizer touches any other on-heap object that
1455 // die at the same GC cycle. 1455 // die at the same GC cycle.
1456 ASAN_UNPOISON_MEMORY_REGION(headerAddress, size); 1456 ASAN_UNPOISON_MEMORY_REGION(headerAddress, size);
1457 header->finalize(payload, payloadSize); 1457 header->finalize(payload, payloadSize);
1458 1458
1459 // As compaction is under way, leave the freed memory accessible 1459 // As compaction is under way, leave the freed memory accessible
1460 // while compacting the rest of the page. We just zap the payload 1460 // while compacting the rest of the page. We just zap the payload
1461 // to catch out other finalizers trying to access it. 1461 // to catch out other finalizers trying to access it.
1462 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) || \ 1462 #if DCHECK_IS_ON() || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) || \
1463 defined(MEMORY_SANITIZER) 1463 defined(MEMORY_SANITIZER)
1464 FreeList::zapFreedMemory(payload, payloadSize); 1464 FreeList::zapFreedMemory(payload, payloadSize);
1465 #endif 1465 #endif
1466 headerAddress += size; 1466 headerAddress += size;
1467 continue; 1467 continue;
1468 } 1468 }
1469 header->unmark(); 1469 header->unmark();
1470 // Allocate and copy over the live object. 1470 // Allocate and copy over the live object.
1471 Address compactFrontier = currentPage->payload() + allocationPoint; 1471 Address compactFrontier = currentPage->payload() + allocationPoint;
1472 if (compactFrontier + size > currentPage->payloadEnd()) { 1472 if (compactFrontier + size > currentPage->payloadEnd()) {
(...skipping 29 matching lines...) Expand all
1502 // Use a non-overlapping copy, if possible. 1502 // Use a non-overlapping copy, if possible.
1503 if (currentPage == this) 1503 if (currentPage == this)
1504 memmove(compactFrontier, headerAddress, size); 1504 memmove(compactFrontier, headerAddress, size);
1505 else 1505 else
1506 memcpy(compactFrontier, headerAddress, size); 1506 memcpy(compactFrontier, headerAddress, size);
1507 compact->relocate(payload, compactFrontier + sizeof(HeapObjectHeader)); 1507 compact->relocate(payload, compactFrontier + sizeof(HeapObjectHeader));
1508 } 1508 }
1509 headerAddress += size; 1509 headerAddress += size;
1510 markedObjectSize += size; 1510 markedObjectSize += size;
1511 allocationPoint += size; 1511 allocationPoint += size;
1512 DCHECK(allocationPoint <= currentPage->payloadSize()); 1512 DCHECK_LE(allocationPoint, currentPage->payloadSize());
1513 } 1513 }
1514 if (markedObjectSize) 1514 if (markedObjectSize)
1515 pageArena->getThreadState()->increaseMarkedObjectSize(markedObjectSize); 1515 pageArena->getThreadState()->increaseMarkedObjectSize(markedObjectSize);
1516 1516
1517 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) || \ 1517 #if DCHECK_IS_ON() || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) || \
1518 defined(MEMORY_SANITIZER) 1518 defined(MEMORY_SANITIZER)
1519 // Zap the unused portion, until it is either compacted into or freed. 1519 // Zap the unused portion, until it is either compacted into or freed.
1520 if (currentPage != this) { 1520 if (currentPage != this) {
1521 FreeList::zapFreedMemory(payload(), payloadSize()); 1521 FreeList::zapFreedMemory(payload(), payloadSize());
1522 } else { 1522 } else {
1523 FreeList::zapFreedMemory(payload() + allocationPoint, 1523 FreeList::zapFreedMemory(payload() + allocationPoint,
1524 payloadSize() - allocationPoint); 1524 payloadSize() - allocationPoint);
1525 } 1525 }
1526 #endif 1526 #endif
1527 } 1527 }
1528 1528
1529 void NormalPage::makeConsistentForGC() { 1529 void NormalPage::makeConsistentForGC() {
1530 size_t markedObjectSize = 0; 1530 size_t markedObjectSize = 0;
1531 for (Address headerAddress = payload(); headerAddress < payloadEnd();) { 1531 for (Address headerAddress = payload(); headerAddress < payloadEnd();) {
1532 HeapObjectHeader* header = 1532 HeapObjectHeader* header =
1533 reinterpret_cast<HeapObjectHeader*>(headerAddress); 1533 reinterpret_cast<HeapObjectHeader*>(headerAddress);
1534 ASSERT(header->size() < blinkPagePayloadSize()); 1534 DCHECK_LT(header->size(), blinkPagePayloadSize());
1535 // Check if a free list entry first since we cannot call 1535 // Check if a free list entry first since we cannot call
1536 // isMarked on a free list entry. 1536 // isMarked on a free list entry.
1537 if (header->isFree()) { 1537 if (header->isFree()) {
1538 headerAddress += header->size(); 1538 headerAddress += header->size();
1539 continue; 1539 continue;
1540 } 1540 }
1541 if (header->isMarked()) { 1541 if (header->isMarked()) {
1542 header->unmark(); 1542 header->unmark();
1543 markedObjectSize += header->size(); 1543 markedObjectSize += header->size();
1544 } else { 1544 } else {
1545 header->markDead(); 1545 header->markDead();
1546 } 1546 }
1547 headerAddress += header->size(); 1547 headerAddress += header->size();
1548 } 1548 }
1549 if (markedObjectSize) 1549 if (markedObjectSize)
1550 arenaForNormalPage()->getThreadState()->increaseMarkedObjectSize( 1550 arenaForNormalPage()->getThreadState()->increaseMarkedObjectSize(
1551 markedObjectSize); 1551 markedObjectSize);
1552 } 1552 }
1553 1553
1554 void NormalPage::makeConsistentForMutator() { 1554 void NormalPage::makeConsistentForMutator() {
1555 Address startOfGap = payload(); 1555 Address startOfGap = payload();
1556 NormalPageArena* normalArena = arenaForNormalPage(); 1556 NormalPageArena* normalArena = arenaForNormalPage();
1557 for (Address headerAddress = payload(); headerAddress < payloadEnd();) { 1557 for (Address headerAddress = payload(); headerAddress < payloadEnd();) {
1558 HeapObjectHeader* header = 1558 HeapObjectHeader* header =
1559 reinterpret_cast<HeapObjectHeader*>(headerAddress); 1559 reinterpret_cast<HeapObjectHeader*>(headerAddress);
1560 size_t size = header->size(); 1560 size_t size = header->size();
1561 ASSERT(size < blinkPagePayloadSize()); 1561 DCHECK_LT(size, blinkPagePayloadSize());
1562 if (header->isPromptlyFreed()) 1562 if (header->isPromptlyFreed())
1563 arenaForNormalPage()->decreasePromptlyFreedSize(size); 1563 arenaForNormalPage()->decreasePromptlyFreedSize(size);
1564 if (header->isFree()) { 1564 if (header->isFree()) {
1565 // Zero the memory in the free list header to maintain the 1565 // Zero the memory in the free list header to maintain the
1566 // invariant that memory on the free list is zero filled. 1566 // invariant that memory on the free list is zero filled.
1567 // The rest of the memory is already on the free list and is 1567 // The rest of the memory is already on the free list and is
1568 // therefore already zero filled. 1568 // therefore already zero filled.
1569 SET_MEMORY_INACCESSIBLE(headerAddress, size < sizeof(FreeListEntry) 1569 SET_MEMORY_INACCESSIBLE(headerAddress, size < sizeof(FreeListEntry)
1570 ? size 1570 ? size
1571 : sizeof(FreeListEntry)); 1571 : sizeof(FreeListEntry));
1572 CHECK_MEMORY_INACCESSIBLE(headerAddress, size); 1572 CHECK_MEMORY_INACCESSIBLE(headerAddress, size);
1573 headerAddress += size; 1573 headerAddress += size;
1574 continue; 1574 continue;
1575 } 1575 }
1576 if (startOfGap != headerAddress) 1576 if (startOfGap != headerAddress)
1577 normalArena->addToFreeList(startOfGap, headerAddress - startOfGap); 1577 normalArena->addToFreeList(startOfGap, headerAddress - startOfGap);
1578 if (header->isMarked()) 1578 if (header->isMarked())
1579 header->unmark(); 1579 header->unmark();
1580 headerAddress += size; 1580 headerAddress += size;
1581 startOfGap = headerAddress; 1581 startOfGap = headerAddress;
1582 ASSERT(headerAddress <= payloadEnd()); 1582 DCHECK_LE(headerAddress, payloadEnd());
1583 } 1583 }
1584 if (startOfGap != payloadEnd()) 1584 if (startOfGap != payloadEnd())
1585 normalArena->addToFreeList(startOfGap, payloadEnd() - startOfGap); 1585 normalArena->addToFreeList(startOfGap, payloadEnd() - startOfGap);
1586 } 1586 }
1587 1587
1588 #if defined(ADDRESS_SANITIZER) 1588 #if defined(ADDRESS_SANITIZER)
1589 void NormalPage::poisonUnmarkedObjects() { 1589 void NormalPage::poisonUnmarkedObjects() {
1590 for (Address headerAddress = payload(); headerAddress < payloadEnd();) { 1590 for (Address headerAddress = payload(); headerAddress < payloadEnd();) {
1591 HeapObjectHeader* header = 1591 HeapObjectHeader* header =
1592 reinterpret_cast<HeapObjectHeader*>(headerAddress); 1592 reinterpret_cast<HeapObjectHeader*>(headerAddress);
1593 ASSERT(header->size() < blinkPagePayloadSize()); 1593 DCHECK_LT(header->size(), blinkPagePayloadSize());
1594 // Check if a free list entry first since we cannot call 1594 // Check if a free list entry first since we cannot call
1595 // isMarked on a free list entry. 1595 // isMarked on a free list entry.
1596 if (header->isFree()) { 1596 if (header->isFree()) {
1597 headerAddress += header->size(); 1597 headerAddress += header->size();
1598 continue; 1598 continue;
1599 } 1599 }
1600 if (!header->isMarked()) 1600 if (!header->isMarked())
1601 ASAN_POISON_MEMORY_REGION(header->payload(), header->payloadSize()); 1601 ASAN_POISON_MEMORY_REGION(header->payload(), header->payloadSize());
1602 headerAddress += header->size(); 1602 headerAddress += header->size();
1603 } 1603 }
1604 } 1604 }
1605 #endif 1605 #endif
1606 1606
1607 void NormalPage::populateObjectStartBitMap() { 1607 void NormalPage::populateObjectStartBitMap() {
1608 memset(&m_objectStartBitMap, 0, objectStartBitMapSize); 1608 memset(&m_objectStartBitMap, 0, objectStartBitMapSize);
1609 Address start = payload(); 1609 Address start = payload();
1610 for (Address headerAddress = start; headerAddress < payloadEnd();) { 1610 for (Address headerAddress = start; headerAddress < payloadEnd();) {
1611 HeapObjectHeader* header = 1611 HeapObjectHeader* header =
1612 reinterpret_cast<HeapObjectHeader*>(headerAddress); 1612 reinterpret_cast<HeapObjectHeader*>(headerAddress);
1613 size_t objectOffset = headerAddress - start; 1613 size_t objectOffset = headerAddress - start;
1614 ASSERT(!(objectOffset & allocationMask)); 1614 DCHECK(!(objectOffset & allocationMask));
1615 size_t objectStartNumber = objectOffset / allocationGranularity; 1615 size_t objectStartNumber = objectOffset / allocationGranularity;
1616 size_t mapIndex = objectStartNumber / 8; 1616 size_t mapIndex = objectStartNumber / 8;
1617 ASSERT(mapIndex < objectStartBitMapSize); 1617 DCHECK_LT(mapIndex, objectStartBitMapSize);
1618 m_objectStartBitMap[mapIndex] |= (1 << (objectStartNumber & 7)); 1618 m_objectStartBitMap[mapIndex] |= (1 << (objectStartNumber & 7));
1619 headerAddress += header->size(); 1619 headerAddress += header->size();
1620 ASSERT(headerAddress <= payloadEnd()); 1620 DCHECK_LE(headerAddress, payloadEnd());
1621 } 1621 }
1622 m_objectStartBitMapComputed = true; 1622 m_objectStartBitMapComputed = true;
1623 } 1623 }
1624 1624
1625 static int numberOfLeadingZeroes(uint8_t byte) { 1625 static int numberOfLeadingZeroes(uint8_t byte) {
1626 if (!byte) 1626 if (!byte)
1627 return 8; 1627 return 8;
1628 int result = 0; 1628 int result = 0;
1629 if (byte <= 0x0F) { 1629 if (byte <= 0x0F) {
1630 result += 4; 1630 result += 4;
1631 byte = byte << 4; 1631 byte = byte << 4;
1632 } 1632 }
1633 if (byte <= 0x3F) { 1633 if (byte <= 0x3F) {
1634 result += 2; 1634 result += 2;
1635 byte = byte << 2; 1635 byte = byte << 2;
1636 } 1636 }
1637 if (byte <= 0x7F) 1637 if (byte <= 0x7F)
1638 result++; 1638 result++;
1639 return result; 1639 return result;
1640 } 1640 }
1641 1641
1642 HeapObjectHeader* NormalPage::findHeaderFromAddress(Address address) { 1642 HeapObjectHeader* NormalPage::findHeaderFromAddress(Address address) {
1643 if (address < payload()) 1643 if (address < payload())
1644 return nullptr; 1644 return nullptr;
1645 if (!m_objectStartBitMapComputed) 1645 if (!m_objectStartBitMapComputed)
1646 populateObjectStartBitMap(); 1646 populateObjectStartBitMap();
1647 size_t objectOffset = address - payload(); 1647 size_t objectOffset = address - payload();
1648 size_t objectStartNumber = objectOffset / allocationGranularity; 1648 size_t objectStartNumber = objectOffset / allocationGranularity;
1649 size_t mapIndex = objectStartNumber / 8; 1649 size_t mapIndex = objectStartNumber / 8;
1650 ASSERT(mapIndex < objectStartBitMapSize); 1650 DCHECK_LT(mapIndex, objectStartBitMapSize);
1651 size_t bit = objectStartNumber & 7; 1651 size_t bit = objectStartNumber & 7;
1652 uint8_t byte = m_objectStartBitMap[mapIndex] & ((1 << (bit + 1)) - 1); 1652 uint8_t byte = m_objectStartBitMap[mapIndex] & ((1 << (bit + 1)) - 1);
1653 while (!byte) { 1653 while (!byte) {
1654 ASSERT(mapIndex > 0); 1654 DCHECK_GT(mapIndex, 0UL);
1655 byte = m_objectStartBitMap[--mapIndex]; 1655 byte = m_objectStartBitMap[--mapIndex];
1656 } 1656 }
1657 int leadingZeroes = numberOfLeadingZeroes(byte); 1657 int leadingZeroes = numberOfLeadingZeroes(byte);
1658 objectStartNumber = (mapIndex * 8) + 7 - leadingZeroes; 1658 objectStartNumber = (mapIndex * 8) + 7 - leadingZeroes;
1659 objectOffset = objectStartNumber * allocationGranularity; 1659 objectOffset = objectStartNumber * allocationGranularity;
1660 Address objectAddress = objectOffset + payload(); 1660 Address objectAddress = objectOffset + payload();
1661 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(objectAddress); 1661 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(objectAddress);
1662 if (header->isFree()) 1662 if (header->isFree())
1663 return nullptr; 1663 return nullptr;
1664 ASSERT(header->checkHeader()); 1664 DCHECK(header->checkHeader());
1665 return header; 1665 return header;
1666 } 1666 }
1667 1667
1668 #if ENABLE(ASSERT) 1668 #if DCHECK_IS_ON()
1669 static bool isUninitializedMemory(void* objectPointer, size_t objectSize) { 1669 static bool isUninitializedMemory(void* objectPointer, size_t objectSize) {
1670 // Scan through the object's fields and check that they are all zero. 1670 // Scan through the object's fields and check that they are all zero.
1671 Address* objectFields = reinterpret_cast<Address*>(objectPointer); 1671 Address* objectFields = reinterpret_cast<Address*>(objectPointer);
1672 for (size_t i = 0; i < objectSize / sizeof(Address); ++i) { 1672 for (size_t i = 0; i < objectSize / sizeof(Address); ++i) {
1673 if (objectFields[i] != 0) 1673 if (objectFields[i] != 0)
1674 return false; 1674 return false;
1675 } 1675 }
1676 return true; 1676 return true;
1677 } 1677 }
1678 #endif 1678 #endif
1679 1679
1680 static void markPointer(Visitor* visitor, HeapObjectHeader* header) { 1680 static void markPointer(Visitor* visitor, HeapObjectHeader* header) {
1681 ASSERT(header->checkHeader()); 1681 DCHECK(header->checkHeader());
1682 const GCInfo* gcInfo = ThreadHeap::gcInfo(header->gcInfoIndex()); 1682 const GCInfo* gcInfo = ThreadHeap::gcInfo(header->gcInfoIndex());
1683 if (gcInfo->hasVTable() && !vTableInitialized(header->payload())) { 1683 if (gcInfo->hasVTable() && !vTableInitialized(header->payload())) {
1684 // We hit this branch when a GC strikes before GarbageCollected<>'s 1684 // We hit this branch when a GC strikes before GarbageCollected<>'s
1685 // constructor runs. 1685 // constructor runs.
1686 // 1686 //
1687 // class A : public GarbageCollected<A> { virtual void f() = 0; }; 1687 // class A : public GarbageCollected<A> { virtual void f() = 0; };
1688 // class B : public A { 1688 // class B : public A {
1689 // B() : A(foo()) { }; 1689 // B() : A(foo()) { };
1690 // }; 1690 // };
1691 // 1691 //
1692 // If foo() allocates something and triggers a GC, the vtable of A 1692 // If foo() allocates something and triggers a GC, the vtable of A
1693 // has not yet been initialized. In this case, we should mark the A 1693 // has not yet been initialized. In this case, we should mark the A
1694 // object without tracing any member of the A object. 1694 // object without tracing any member of the A object.
1695 visitor->markHeaderNoTracing(header); 1695 visitor->markHeaderNoTracing(header);
1696 ASSERT(isUninitializedMemory(header->payload(), header->payloadSize())); 1696 DCHECK(isUninitializedMemory(header->payload(), header->payloadSize()));
1697 } else { 1697 } else {
1698 visitor->markHeader(header, gcInfo->m_trace); 1698 visitor->markHeader(header, gcInfo->m_trace);
1699 } 1699 }
1700 } 1700 }
1701 1701
1702 void NormalPage::checkAndMarkPointer(Visitor* visitor, Address address) { 1702 void NormalPage::checkAndMarkPointer(Visitor* visitor, Address address) {
1703 ASSERT(contains(address)); 1703 DCHECK(contains(address));
1704 HeapObjectHeader* header = findHeaderFromAddress(address); 1704 HeapObjectHeader* header = findHeaderFromAddress(address);
1705 if (!header || header->isDead()) 1705 if (!header || header->isDead())
1706 return; 1706 return;
1707 markPointer(visitor, header); 1707 markPointer(visitor, header);
1708 } 1708 }
1709 1709
1710 void NormalPage::markOrphaned() { 1710 void NormalPage::markOrphaned() {
1711 // Zap the payload with a recognizable value to detect any incorrect 1711 // Zap the payload with a recognizable value to detect any incorrect
1712 // cross thread pointer usage. 1712 // cross thread pointer usage.
1713 #if defined(ADDRESS_SANITIZER) 1713 #if defined(ADDRESS_SANITIZER)
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after
1756 pageDump->AddScalar("live_count", "objects", liveCount); 1756 pageDump->AddScalar("live_count", "objects", liveCount);
1757 pageDump->AddScalar("dead_count", "objects", deadCount); 1757 pageDump->AddScalar("dead_count", "objects", deadCount);
1758 pageDump->AddScalar("free_count", "objects", freeCount); 1758 pageDump->AddScalar("free_count", "objects", freeCount);
1759 pageDump->AddScalar("live_size", "bytes", liveSize); 1759 pageDump->AddScalar("live_size", "bytes", liveSize);
1760 pageDump->AddScalar("dead_size", "bytes", deadSize); 1760 pageDump->AddScalar("dead_size", "bytes", deadSize);
1761 pageDump->AddScalar("free_size", "bytes", freeSize); 1761 pageDump->AddScalar("free_size", "bytes", freeSize);
1762 heapInfo.freeSize += freeSize; 1762 heapInfo.freeSize += freeSize;
1763 heapInfo.freeCount += freeCount; 1763 heapInfo.freeCount += freeCount;
1764 } 1764 }
1765 1765
1766 #if ENABLE(ASSERT) 1766 #if DCHECK_IS_ON()
1767 bool NormalPage::contains(Address addr) { 1767 bool NormalPage::contains(Address addr) {
1768 Address blinkPageStart = roundToBlinkPageStart(getAddress()); 1768 Address blinkPageStart = roundToBlinkPageStart(getAddress());
1769 // Page is at aligned address plus guard page size. 1769 // Page is at aligned address plus guard page size.
1770 ASSERT(blinkPageStart == getAddress() - blinkGuardPageSize); 1770 DCHECK_EQ(blinkPageStart, getAddress() - blinkGuardPageSize);
1771 return blinkPageStart <= addr && addr < blinkPageStart + blinkPageSize; 1771 return blinkPageStart <= addr && addr < blinkPageStart + blinkPageSize;
1772 } 1772 }
1773 #endif 1773 #endif
1774 1774
1775 LargeObjectPage::LargeObjectPage(PageMemory* storage, 1775 LargeObjectPage::LargeObjectPage(PageMemory* storage,
1776 BaseArena* arena, 1776 BaseArena* arena,
1777 size_t payloadSize) 1777 size_t payloadSize)
1778 : BasePage(storage, arena), 1778 : BasePage(storage, arena),
1779 m_payloadSize(payloadSize) 1779 m_payloadSize(payloadSize)
1780 #if ENABLE(ASAN_CONTAINER_ANNOTATIONS) 1780 #if ENABLE(ASAN_CONTAINER_ANNOTATIONS)
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after
1820 1820
1821 #if defined(ADDRESS_SANITIZER) 1821 #if defined(ADDRESS_SANITIZER)
1822 void LargeObjectPage::poisonUnmarkedObjects() { 1822 void LargeObjectPage::poisonUnmarkedObjects() {
1823 HeapObjectHeader* header = heapObjectHeader(); 1823 HeapObjectHeader* header = heapObjectHeader();
1824 if (!header->isMarked()) 1824 if (!header->isMarked())
1825 ASAN_POISON_MEMORY_REGION(header->payload(), header->payloadSize()); 1825 ASAN_POISON_MEMORY_REGION(header->payload(), header->payloadSize());
1826 } 1826 }
1827 #endif 1827 #endif
1828 1828
1829 void LargeObjectPage::checkAndMarkPointer(Visitor* visitor, Address address) { 1829 void LargeObjectPage::checkAndMarkPointer(Visitor* visitor, Address address) {
1830 ASSERT(contains(address)); 1830 DCHECK(contains(address));
1831 if (!containedInObjectPayload(address) || heapObjectHeader()->isDead()) 1831 if (!containedInObjectPayload(address) || heapObjectHeader()->isDead())
1832 return; 1832 return;
1833 markPointer(visitor, heapObjectHeader()); 1833 markPointer(visitor, heapObjectHeader());
1834 } 1834 }
1835 1835
1836 void LargeObjectPage::markOrphaned() { 1836 void LargeObjectPage::markOrphaned() {
1837 // Zap the payload with a recognizable value to detect any incorrect 1837 // Zap the payload with a recognizable value to detect any incorrect
1838 // cross thread pointer usage. 1838 // cross thread pointer usage.
1839 OrphanedPagePool::asanDisabledMemset( 1839 OrphanedPagePool::asanDisabledMemset(
1840 payload(), OrphanedPagePool::orphanedZapValue, payloadSize()); 1840 payload(), OrphanedPagePool::orphanedZapValue, payloadSize());
(...skipping 22 matching lines...) Expand all
1863 info.deadCount[gcInfoIndex]++; 1863 info.deadCount[gcInfoIndex]++;
1864 info.deadSize[gcInfoIndex] += payloadSize; 1864 info.deadSize[gcInfoIndex] += payloadSize;
1865 } 1865 }
1866 1866
1867 pageDump->AddScalar("live_count", "objects", liveCount); 1867 pageDump->AddScalar("live_count", "objects", liveCount);
1868 pageDump->AddScalar("dead_count", "objects", deadCount); 1868 pageDump->AddScalar("dead_count", "objects", deadCount);
1869 pageDump->AddScalar("live_size", "bytes", liveSize); 1869 pageDump->AddScalar("live_size", "bytes", liveSize);
1870 pageDump->AddScalar("dead_size", "bytes", deadSize); 1870 pageDump->AddScalar("dead_size", "bytes", deadSize);
1871 } 1871 }
1872 1872
1873 #if ENABLE(ASSERT) 1873 #if DCHECK_IS_ON()
1874 bool LargeObjectPage::contains(Address object) { 1874 bool LargeObjectPage::contains(Address object) {
1875 return roundToBlinkPageStart(getAddress()) <= object && 1875 return roundToBlinkPageStart(getAddress()) <= object &&
1876 object < roundToBlinkPageEnd(getAddress() + size()); 1876 object < roundToBlinkPageEnd(getAddress() + size());
1877 } 1877 }
1878 #endif 1878 #endif
1879 1879
1880 void HeapDoesNotContainCache::flush() { 1880 void HeapDoesNotContainCache::flush() {
1881 if (m_hasEntries) { 1881 if (m_hasEntries) {
1882 for (int i = 0; i < numberOfEntries; ++i) 1882 for (int i = 0; i < numberOfEntries; ++i)
1883 m_entries[i] = nullptr; 1883 m_entries[i] = nullptr;
1884 m_hasEntries = false; 1884 m_hasEntries = false;
1885 } 1885 }
1886 } 1886 }
1887 1887
1888 size_t HeapDoesNotContainCache::hash(Address address) { 1888 size_t HeapDoesNotContainCache::hash(Address address) {
1889 size_t value = (reinterpret_cast<size_t>(address) >> blinkPageSizeLog2); 1889 size_t value = (reinterpret_cast<size_t>(address) >> blinkPageSizeLog2);
1890 value ^= value >> numberOfEntriesLog2; 1890 value ^= value >> numberOfEntriesLog2;
1891 value ^= value >> (numberOfEntriesLog2 * 2); 1891 value ^= value >> (numberOfEntriesLog2 * 2);
1892 value &= numberOfEntries - 1; 1892 value &= numberOfEntries - 1;
1893 return value & ~1; // Returns only even number. 1893 return value & ~1; // Returns only even number.
1894 } 1894 }
1895 1895
1896 bool HeapDoesNotContainCache::lookup(Address address) { 1896 bool HeapDoesNotContainCache::lookup(Address address) {
1897 ASSERT(ThreadState::current()->isInGC()); 1897 DCHECK(ThreadState::current()->isInGC());
1898 1898
1899 size_t index = hash(address); 1899 size_t index = hash(address);
1900 ASSERT(!(index & 1)); 1900 DCHECK(!(index & 1));
1901 Address cachePage = roundToBlinkPageStart(address); 1901 Address cachePage = roundToBlinkPageStart(address);
1902 if (m_entries[index] == cachePage) 1902 if (m_entries[index] == cachePage)
1903 return m_entries[index]; 1903 return m_entries[index];
1904 if (m_entries[index + 1] == cachePage) 1904 if (m_entries[index + 1] == cachePage)
1905 return m_entries[index + 1]; 1905 return m_entries[index + 1];
1906 return false; 1906 return false;
1907 } 1907 }
1908 1908
1909 void HeapDoesNotContainCache::addEntry(Address address) { 1909 void HeapDoesNotContainCache::addEntry(Address address) {
1910 ASSERT(ThreadState::current()->isInGC()); 1910 DCHECK(ThreadState::current()->isInGC());
1911 1911
1912 m_hasEntries = true; 1912 m_hasEntries = true;
1913 size_t index = hash(address); 1913 size_t index = hash(address);
1914 ASSERT(!(index & 1)); 1914 DCHECK(!(index & 1));
1915 Address cachePage = roundToBlinkPageStart(address); 1915 Address cachePage = roundToBlinkPageStart(address);
1916 m_entries[index + 1] = m_entries[index]; 1916 m_entries[index + 1] = m_entries[index];
1917 m_entries[index] = cachePage; 1917 m_entries[index] = cachePage;
1918 } 1918 }
1919 1919
1920 } // namespace blink 1920 } // namespace blink
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698