| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (C) 2013 Google Inc. All rights reserved. | 2 * Copyright (C) 2013 Google Inc. All rights reserved. |
| 3 * | 3 * |
| 4 * Redistribution and use in source and binary forms, with or without | 4 * Redistribution and use in source and binary forms, with or without |
| 5 * modification, are permitted provided that the following conditions are | 5 * modification, are permitted provided that the following conditions are |
| 6 * met: | 6 * met: |
| 7 * | 7 * |
| 8 * * Redistributions of source code must retain the above copyright | 8 * * Redistributions of source code must retain the above copyright |
| 9 * notice, this list of conditions and the following disclaimer. | 9 * notice, this list of conditions and the following disclaimer. |
| 10 * * Redistributions in binary form must reproduce the above | 10 * * Redistributions in binary form must reproduce the above |
| (...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 82 static_cast<LargeObjectPage*>(largePage)->setIsVectorBackingPage(); \ | 82 static_cast<LargeObjectPage*>(largePage)->setIsVectorBackingPage(); \ |
| 83 } | 83 } |
| 84 #else | 84 #else |
| 85 #define ENABLE_ASAN_CONTAINER_ANNOTATIONS 0 | 85 #define ENABLE_ASAN_CONTAINER_ANNOTATIONS 0 |
| 86 #define ASAN_RETIRE_CONTAINER_ANNOTATION(payload, payloadSize) | 86 #define ASAN_RETIRE_CONTAINER_ANNOTATION(payload, payloadSize) |
| 87 #define ASAN_MARK_LARGE_VECTOR_CONTAINER(arena, largeObject) | 87 #define ASAN_MARK_LARGE_VECTOR_CONTAINER(arena, largeObject) |
| 88 #endif | 88 #endif |
| 89 | 89 |
| 90 namespace blink { | 90 namespace blink { |
| 91 | 91 |
| 92 #if ENABLE(ASSERT) | 92 #if DCHECK_IS_ON() |
| 93 NO_SANITIZE_ADDRESS | 93 NO_SANITIZE_ADDRESS |
| 94 void HeapObjectHeader::zapMagic() { | 94 void HeapObjectHeader::zapMagic() { |
| 95 ASSERT(checkHeader()); | 95 ASSERT(checkHeader()); |
| 96 m_magic = zappedMagic; | 96 m_magic = zappedMagic; |
| 97 } | 97 } |
| 98 #endif | 98 #endif |
| 99 | 99 |
| 100 void HeapObjectHeader::finalize(Address object, size_t objectSize) { | 100 void HeapObjectHeader::finalize(Address object, size_t objectSize) { |
| 101 HeapAllocHooks::freeHookIfEnabled(object); | 101 HeapAllocHooks::freeHookIfEnabled(object); |
| 102 const GCInfo* gcInfo = ThreadHeap::gcInfo(gcInfoIndex()); | 102 const GCInfo* gcInfo = ThreadHeap::gcInfo(gcInfoIndex()); |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 151 allocatorDump->AddScalar("blink_page_count", "objects", pageCount); | 151 allocatorDump->AddScalar("blink_page_count", "objects", pageCount); |
| 152 | 152 |
| 153 // When taking a full dump (w/ freelist), both the /buckets and /pages | 153 // When taking a full dump (w/ freelist), both the /buckets and /pages |
| 154 // report their free size but they are not meant to be added together. | 154 // report their free size but they are not meant to be added together. |
| 155 // Therefore, here we override the free_size of the parent heap to be | 155 // Therefore, here we override the free_size of the parent heap to be |
| 156 // equal to the free_size of the sum of its heap pages. | 156 // equal to the free_size of the sum of its heap pages. |
| 157 allocatorDump->AddScalar("free_size", "bytes", heapInfo.freeSize); | 157 allocatorDump->AddScalar("free_size", "bytes", heapInfo.freeSize); |
| 158 allocatorDump->AddScalar("free_count", "objects", heapInfo.freeCount); | 158 allocatorDump->AddScalar("free_count", "objects", heapInfo.freeCount); |
| 159 } | 159 } |
| 160 | 160 |
| 161 #if ENABLE(ASSERT) | 161 #if DCHECK_IS_ON() |
| 162 BasePage* BaseArena::findPageFromAddress(Address address) { | 162 BasePage* BaseArena::findPageFromAddress(Address address) { |
| 163 for (BasePage* page = m_firstPage; page; page = page->next()) { | 163 for (BasePage* page = m_firstPage; page; page = page->next()) { |
| 164 if (page->contains(address)) | 164 if (page->contains(address)) |
| 165 return page; | 165 return page; |
| 166 } | 166 } |
| 167 for (BasePage* page = m_firstUnsweptPage; page; page = page->next()) { | 167 for (BasePage* page = m_firstUnsweptPage; page; page = page->next()) { |
| 168 if (page->contains(address)) | 168 if (page->contains(address)) |
| 169 return page; | 169 return page; |
| 170 } | 170 } |
| 171 return nullptr; | 171 return nullptr; |
| (...skipping 383 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 555 size_t pageSize = availablePages->size(); | 555 size_t pageSize = availablePages->size(); |
| 556 #if DEBUG_HEAP_COMPACTION | 556 #if DEBUG_HEAP_COMPACTION |
| 557 if (!freedPageCount) | 557 if (!freedPageCount) |
| 558 LOG_HEAP_COMPACTION("Releasing:"); | 558 LOG_HEAP_COMPACTION("Releasing:"); |
| 559 LOG_HEAP_COMPACTION(" [%p, %p]", availablePages, availablePages + pageSize); | 559 LOG_HEAP_COMPACTION(" [%p, %p]", availablePages, availablePages + pageSize); |
| 560 #endif | 560 #endif |
| 561 freedSize += pageSize; | 561 freedSize += pageSize; |
| 562 freedPageCount++; | 562 freedPageCount++; |
| 563 BasePage* nextPage; | 563 BasePage* nextPage; |
| 564 availablePages->unlink(&nextPage); | 564 availablePages->unlink(&nextPage); |
| 565 #if !(ENABLE(ASSERT) || defined(LEAK_SANITIZER) || \ | 565 #if !(DCHECK_IS_ON() || defined(LEAK_SANITIZER) || \ |
| 566 defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER)) | 566 defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER)) |
| 567 // Clear out the page before adding it to the free page pool, which | 567 // Clear out the page before adding it to the free page pool, which |
| 568 // decommits it. Recommitting the page must find a zeroed page later. | 568 // decommits it. Recommitting the page must find a zeroed page later. |
| 569 // We cannot assume that the OS will hand back a zeroed page across | 569 // We cannot assume that the OS will hand back a zeroed page across |
| 570 // its "decommit" operation. | 570 // its "decommit" operation. |
| 571 // | 571 // |
| 572 // If in a debug setting, the unused page contents will have been | 572 // If in a debug setting, the unused page contents will have been |
| 573 // zapped already; leave it in that state. | 573 // zapped already; leave it in that state. |
| 574 DCHECK(!availablePages->isLargeObjectPage()); | 574 DCHECK(!availablePages->isLargeObjectPage()); |
| 575 NormalPage* unusedPage = reinterpret_cast<NormalPage*>(availablePages); | 575 NormalPage* unusedPage = reinterpret_cast<NormalPage*>(availablePages); |
| 576 memset(unusedPage->payload(), 0, unusedPage->payloadSize()); | 576 memset(unusedPage->payload(), 0, unusedPage->payloadSize()); |
| 577 #endif | 577 #endif |
| 578 availablePages->removeFromHeap(); | 578 availablePages->removeFromHeap(); |
| 579 availablePages = static_cast<NormalPage*>(nextPage); | 579 availablePages = static_cast<NormalPage*>(nextPage); |
| 580 } | 580 } |
| 581 if (freedPageCount) | 581 if (freedPageCount) |
| 582 LOG_HEAP_COMPACTION("\n"); | 582 LOG_HEAP_COMPACTION("\n"); |
| 583 heap.compaction()->finishedArenaCompaction(this, freedPageCount, freedSize); | 583 heap.compaction()->finishedArenaCompaction(this, freedPageCount, freedSize); |
| 584 } | 584 } |
| 585 | 585 |
| 586 #if ENABLE(ASSERT) | 586 #if DCHECK_IS_ON() |
| 587 bool NormalPageArena::isConsistentForGC() { | 587 bool NormalPageArena::isConsistentForGC() { |
| 588 // A thread heap is consistent for sweeping if none of the pages to be swept | 588 // A thread heap is consistent for sweeping if none of the pages to be swept |
| 589 // contain a freelist block or the current allocation point. | 589 // contain a freelist block or the current allocation point. |
| 590 for (size_t i = 0; i < blinkPageSizeLog2; ++i) { | 590 for (size_t i = 0; i < blinkPageSizeLog2; ++i) { |
| 591 for (FreeListEntry* freeListEntry = m_freeList.m_freeLists[i]; | 591 for (FreeListEntry* freeListEntry = m_freeList.m_freeLists[i]; |
| 592 freeListEntry; freeListEntry = freeListEntry->next()) { | 592 freeListEntry; freeListEntry = freeListEntry->next()) { |
| 593 if (pagesToBeSweptContains(freeListEntry->getAddress())) | 593 if (pagesToBeSweptContains(freeListEntry->getAddress())) |
| 594 return false; | 594 return false; |
| 595 } | 595 } |
| 596 } | 596 } |
| (...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 655 getThreadState()->heap().getFreePagePool()->addFreePage(arenaIndex(), | 655 getThreadState()->heap().getFreePagePool()->addFreePage(arenaIndex(), |
| 656 memory); | 656 memory); |
| 657 } | 657 } |
| 658 } | 658 } |
| 659 } | 659 } |
| 660 NormalPage* page = | 660 NormalPage* page = |
| 661 new (pageMemory->writableStart()) NormalPage(pageMemory, this); | 661 new (pageMemory->writableStart()) NormalPage(pageMemory, this); |
| 662 page->link(&m_firstPage); | 662 page->link(&m_firstPage); |
| 663 | 663 |
| 664 getThreadState()->heap().heapStats().increaseAllocatedSpace(page->size()); | 664 getThreadState()->heap().heapStats().increaseAllocatedSpace(page->size()); |
| 665 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) | 665 #if DCHECK_IS_ON() || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) |
| 666 // Allow the following addToFreeList() to add the newly allocated memory | 666 // Allow the following addToFreeList() to add the newly allocated memory |
| 667 // to the free list. | 667 // to the free list. |
| 668 ASAN_UNPOISON_MEMORY_REGION(page->payload(), page->payloadSize()); | 668 ASAN_UNPOISON_MEMORY_REGION(page->payload(), page->payloadSize()); |
| 669 Address address = page->payload(); | 669 Address address = page->payload(); |
| 670 for (size_t i = 0; i < page->payloadSize(); i++) | 670 for (size_t i = 0; i < page->payloadSize(); i++) |
| 671 address[i] = reuseAllowedZapValue; | 671 address[i] = reuseAllowedZapValue; |
| 672 ASAN_POISON_MEMORY_REGION(page->payload(), page->payloadSize()); | 672 ASAN_POISON_MEMORY_REGION(page->payload(), page->payloadSize()); |
| 673 #endif | 673 #endif |
| 674 addToFreeList(page->payload(), page->payloadSize()); | 674 addToFreeList(page->payload(), page->payloadSize()); |
| 675 } | 675 } |
| (...skipping 217 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 893 void NormalPageArena::updateRemainingAllocationSize() { | 893 void NormalPageArena::updateRemainingAllocationSize() { |
| 894 if (m_lastRemainingAllocationSize > remainingAllocationSize()) { | 894 if (m_lastRemainingAllocationSize > remainingAllocationSize()) { |
| 895 getThreadState()->increaseAllocatedObjectSize( | 895 getThreadState()->increaseAllocatedObjectSize( |
| 896 m_lastRemainingAllocationSize - remainingAllocationSize()); | 896 m_lastRemainingAllocationSize - remainingAllocationSize()); |
| 897 m_lastRemainingAllocationSize = remainingAllocationSize(); | 897 m_lastRemainingAllocationSize = remainingAllocationSize(); |
| 898 } | 898 } |
| 899 ASSERT(m_lastRemainingAllocationSize == remainingAllocationSize()); | 899 ASSERT(m_lastRemainingAllocationSize == remainingAllocationSize()); |
| 900 } | 900 } |
| 901 | 901 |
| 902 void NormalPageArena::setAllocationPoint(Address point, size_t size) { | 902 void NormalPageArena::setAllocationPoint(Address point, size_t size) { |
| 903 #if ENABLE(ASSERT) | 903 #if DCHECK_IS_ON() |
| 904 if (point) { | 904 if (point) { |
| 905 ASSERT(size); | 905 ASSERT(size); |
| 906 BasePage* page = pageFromObject(point); | 906 BasePage* page = pageFromObject(point); |
| 907 ASSERT(!page->isLargeObjectPage()); | 907 ASSERT(!page->isLargeObjectPage()); |
| 908 ASSERT(size <= static_cast<NormalPage*>(page)->payloadSize()); | 908 ASSERT(size <= static_cast<NormalPage*>(page)->payloadSize()); |
| 909 } | 909 } |
| 910 #endif | 910 #endif |
| 911 if (hasCurrentAllocationArea()) { | 911 if (hasCurrentAllocationArea()) { |
| 912 addToFreeList(currentAllocationPoint(), remainingAllocationSize()); | 912 addToFreeList(currentAllocationPoint(), remainingAllocationSize()); |
| 913 } | 913 } |
| (...skipping 114 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1028 #if defined(ADDRESS_SANITIZER) | 1028 #if defined(ADDRESS_SANITIZER) |
| 1029 largeObjectSize += allocationGranularity; | 1029 largeObjectSize += allocationGranularity; |
| 1030 #endif | 1030 #endif |
| 1031 | 1031 |
| 1032 getThreadState()->shouldFlushHeapDoesNotContainCache(); | 1032 getThreadState()->shouldFlushHeapDoesNotContainCache(); |
| 1033 PageMemory* pageMemory = PageMemory::allocate( | 1033 PageMemory* pageMemory = PageMemory::allocate( |
| 1034 largeObjectSize, getThreadState()->heap().getRegionTree()); | 1034 largeObjectSize, getThreadState()->heap().getRegionTree()); |
| 1035 Address largeObjectAddress = pageMemory->writableStart(); | 1035 Address largeObjectAddress = pageMemory->writableStart(); |
| 1036 Address headerAddress = | 1036 Address headerAddress = |
| 1037 largeObjectAddress + LargeObjectPage::pageHeaderSize(); | 1037 largeObjectAddress + LargeObjectPage::pageHeaderSize(); |
| 1038 #if ENABLE(ASSERT) | 1038 #if DCHECK_IS_ON() |
| 1039 // Verify that the allocated PageMemory is expectedly zeroed. | 1039 // Verify that the allocated PageMemory is expectedly zeroed. |
| 1040 for (size_t i = 0; i < largeObjectSize; ++i) | 1040 for (size_t i = 0; i < largeObjectSize; ++i) |
| 1041 ASSERT(!largeObjectAddress[i]); | 1041 ASSERT(!largeObjectAddress[i]); |
| 1042 #endif | 1042 #endif |
| 1043 ASSERT(gcInfoIndex > 0); | 1043 ASSERT(gcInfoIndex > 0); |
| 1044 HeapObjectHeader* header = new (NotNull, headerAddress) | 1044 HeapObjectHeader* header = new (NotNull, headerAddress) |
| 1045 HeapObjectHeader(largeObjectSizeInHeader, gcInfoIndex); | 1045 HeapObjectHeader(largeObjectSizeInHeader, gcInfoIndex); |
| 1046 Address result = headerAddress + sizeof(*header); | 1046 Address result = headerAddress + sizeof(*header); |
| 1047 ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask)); | 1047 ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask)); |
| 1048 LargeObjectPage* largeObject = new (largeObjectAddress) | 1048 LargeObjectPage* largeObject = new (largeObjectAddress) |
| (...skipping 92 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1141 ASSERT(size >= sizeof(HeapObjectHeader)); | 1141 ASSERT(size >= sizeof(HeapObjectHeader)); |
| 1142 // Free list encode the size to mark the lost memory as freelist memory. | 1142 // Free list encode the size to mark the lost memory as freelist memory. |
| 1143 new (NotNull, address) HeapObjectHeader(size, gcInfoIndexForFreeListHeader); | 1143 new (NotNull, address) HeapObjectHeader(size, gcInfoIndexForFreeListHeader); |
| 1144 | 1144 |
| 1145 ASAN_POISON_MEMORY_REGION(address, size); | 1145 ASAN_POISON_MEMORY_REGION(address, size); |
| 1146 // This memory gets lost. Sweeping can reclaim it. | 1146 // This memory gets lost. Sweeping can reclaim it. |
| 1147 return; | 1147 return; |
| 1148 } | 1148 } |
| 1149 entry = new (NotNull, address) FreeListEntry(size); | 1149 entry = new (NotNull, address) FreeListEntry(size); |
| 1150 | 1150 |
| 1151 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) | 1151 #if DCHECK_IS_ON() || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) |
| 1152 // The following logic delays reusing free lists for (at least) one GC | 1152 // The following logic delays reusing free lists for (at least) one GC |
| 1153 // cycle or coalescing. This is helpful to detect use-after-free errors | 1153 // cycle or coalescing. This is helpful to detect use-after-free errors |
| 1154 // that could be caused by lazy sweeping etc. | 1154 // that could be caused by lazy sweeping etc. |
| 1155 size_t allowedCount = 0; | 1155 size_t allowedCount = 0; |
| 1156 size_t forbiddenCount = 0; | 1156 size_t forbiddenCount = 0; |
| 1157 for (size_t i = sizeof(FreeListEntry); i < size; i++) { | 1157 for (size_t i = sizeof(FreeListEntry); i < size; i++) { |
| 1158 if (address[i] == reuseAllowedZapValue) | 1158 if (address[i] == reuseAllowedZapValue) |
| 1159 allowedCount++; | 1159 allowedCount++; |
| 1160 else if (address[i] == reuseForbiddenZapValue) | 1160 else if (address[i] == reuseForbiddenZapValue) |
| 1161 forbiddenCount++; | 1161 forbiddenCount++; |
| (...skipping 29 matching lines...) Expand all Loading... |
| 1191 // region to the free list and reuse it for another object. | 1191 // region to the free list and reuse it for another object. |
| 1192 #endif | 1192 #endif |
| 1193 ASAN_POISON_MEMORY_REGION(address, size); | 1193 ASAN_POISON_MEMORY_REGION(address, size); |
| 1194 | 1194 |
| 1195 int index = bucketIndexForSize(size); | 1195 int index = bucketIndexForSize(size); |
| 1196 entry->link(&m_freeLists[index]); | 1196 entry->link(&m_freeLists[index]); |
| 1197 if (index > m_biggestFreeListIndex) | 1197 if (index > m_biggestFreeListIndex) |
| 1198 m_biggestFreeListIndex = index; | 1198 m_biggestFreeListIndex = index; |
| 1199 } | 1199 } |
| 1200 | 1200 |
| 1201 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) || \ | 1201 #if DCHECK_IS_ON() || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) || \ |
| 1202 defined(MEMORY_SANITIZER) | 1202 defined(MEMORY_SANITIZER) |
| 1203 NO_SANITIZE_ADDRESS | 1203 NO_SANITIZE_ADDRESS |
| 1204 NO_SANITIZE_MEMORY | 1204 NO_SANITIZE_MEMORY |
| 1205 void NEVER_INLINE FreeList::zapFreedMemory(Address address, size_t size) { | 1205 void NEVER_INLINE FreeList::zapFreedMemory(Address address, size_t size) { |
| 1206 for (size_t i = 0; i < size; i++) { | 1206 for (size_t i = 0; i < size; i++) { |
| 1207 // See the comment in addToFreeList(). | 1207 // See the comment in addToFreeList(). |
| 1208 if (address[i] != reuseAllowedZapValue) | 1208 if (address[i] != reuseAllowedZapValue) |
| 1209 address[i] = reuseForbiddenZapValue; | 1209 address[i] = reuseForbiddenZapValue; |
| 1210 } | 1210 } |
| 1211 } | 1211 } |
| (...skipping 120 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1332 | 1332 |
| 1333 bool NormalPage::isEmpty() { | 1333 bool NormalPage::isEmpty() { |
| 1334 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(payload()); | 1334 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(payload()); |
| 1335 return header->isFree() && header->size() == payloadSize(); | 1335 return header->isFree() && header->size() == payloadSize(); |
| 1336 } | 1336 } |
| 1337 | 1337 |
| 1338 void NormalPage::removeFromHeap() { | 1338 void NormalPage::removeFromHeap() { |
| 1339 arenaForNormalPage()->freePage(this); | 1339 arenaForNormalPage()->freePage(this); |
| 1340 } | 1340 } |
| 1341 | 1341 |
| 1342 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER) | 1342 #if !DCHECK_IS_ON() && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER) |
| 1343 static void discardPages(Address begin, Address end) { | 1343 static void discardPages(Address begin, Address end) { |
| 1344 uintptr_t beginAddress = | 1344 uintptr_t beginAddress = |
| 1345 WTF::RoundUpToSystemPage(reinterpret_cast<uintptr_t>(begin)); | 1345 WTF::RoundUpToSystemPage(reinterpret_cast<uintptr_t>(begin)); |
| 1346 uintptr_t endAddress = | 1346 uintptr_t endAddress = |
| 1347 WTF::RoundDownToSystemPage(reinterpret_cast<uintptr_t>(end)); | 1347 WTF::RoundDownToSystemPage(reinterpret_cast<uintptr_t>(end)); |
| 1348 if (beginAddress < endAddress) | 1348 if (beginAddress < endAddress) |
| 1349 WTF::DiscardSystemPages(reinterpret_cast<void*>(beginAddress), | 1349 WTF::DiscardSystemPages(reinterpret_cast<void*>(beginAddress), |
| 1350 endAddress - beginAddress); | 1350 endAddress - beginAddress); |
| 1351 } | 1351 } |
| 1352 #endif | 1352 #endif |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1388 ASAN_UNPOISON_MEMORY_REGION(payload, payloadSize); | 1388 ASAN_UNPOISON_MEMORY_REGION(payload, payloadSize); |
| 1389 header->finalize(payload, payloadSize); | 1389 header->finalize(payload, payloadSize); |
| 1390 // This memory will be added to the freelist. Maintain the invariant | 1390 // This memory will be added to the freelist. Maintain the invariant |
| 1391 // that memory on the freelist is zero filled. | 1391 // that memory on the freelist is zero filled. |
| 1392 SET_MEMORY_INACCESSIBLE(headerAddress, size); | 1392 SET_MEMORY_INACCESSIBLE(headerAddress, size); |
| 1393 headerAddress += size; | 1393 headerAddress += size; |
| 1394 continue; | 1394 continue; |
| 1395 } | 1395 } |
| 1396 if (startOfGap != headerAddress) { | 1396 if (startOfGap != headerAddress) { |
| 1397 pageArena->addToFreeList(startOfGap, headerAddress - startOfGap); | 1397 pageArena->addToFreeList(startOfGap, headerAddress - startOfGap); |
| 1398 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER) | 1398 #if !DCHECK_IS_ON() && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER) |
| 1399 // Discarding pages increases page faults and may regress performance. | 1399 // Discarding pages increases page faults and may regress performance. |
| 1400 // So we enable this only on low-RAM devices. | 1400 // So we enable this only on low-RAM devices. |
| 1401 if (MemoryCoordinator::isLowEndDevice()) | 1401 if (MemoryCoordinator::isLowEndDevice()) |
| 1402 discardPages(startOfGap + sizeof(FreeListEntry), headerAddress); | 1402 discardPages(startOfGap + sizeof(FreeListEntry), headerAddress); |
| 1403 #endif | 1403 #endif |
| 1404 } | 1404 } |
| 1405 header->unmark(); | 1405 header->unmark(); |
| 1406 headerAddress += size; | 1406 headerAddress += size; |
| 1407 markedObjectSize += size; | 1407 markedObjectSize += size; |
| 1408 startOfGap = headerAddress; | 1408 startOfGap = headerAddress; |
| 1409 } | 1409 } |
| 1410 if (startOfGap != payloadEnd()) { | 1410 if (startOfGap != payloadEnd()) { |
| 1411 pageArena->addToFreeList(startOfGap, payloadEnd() - startOfGap); | 1411 pageArena->addToFreeList(startOfGap, payloadEnd() - startOfGap); |
| 1412 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER) | 1412 #if !DCHECK_IS_ON() && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER) |
| 1413 if (MemoryCoordinator::isLowEndDevice()) | 1413 if (MemoryCoordinator::isLowEndDevice()) |
| 1414 discardPages(startOfGap + sizeof(FreeListEntry), payloadEnd()); | 1414 discardPages(startOfGap + sizeof(FreeListEntry), payloadEnd()); |
| 1415 #endif | 1415 #endif |
| 1416 } | 1416 } |
| 1417 | 1417 |
| 1418 if (markedObjectSize) | 1418 if (markedObjectSize) |
| 1419 pageArena->getThreadState()->increaseMarkedObjectSize(markedObjectSize); | 1419 pageArena->getThreadState()->increaseMarkedObjectSize(markedObjectSize); |
| 1420 } | 1420 } |
| 1421 | 1421 |
| 1422 void NormalPage::sweepAndCompact(CompactionContext& context) { | 1422 void NormalPage::sweepAndCompact(CompactionContext& context) { |
| (...skipping 29 matching lines...) Expand all Loading... |
| 1452 // finalized object will be zero-filled and poison'ed afterwards. | 1452 // finalized object will be zero-filled and poison'ed afterwards. |
| 1453 // Given all other unmarked objects are poisoned, ASan will detect | 1453 // Given all other unmarked objects are poisoned, ASan will detect |
| 1454 // an error if the finalizer touches any other on-heap object that | 1454 // an error if the finalizer touches any other on-heap object that |
| 1455 // die at the same GC cycle. | 1455 // die at the same GC cycle. |
| 1456 ASAN_UNPOISON_MEMORY_REGION(headerAddress, size); | 1456 ASAN_UNPOISON_MEMORY_REGION(headerAddress, size); |
| 1457 header->finalize(payload, payloadSize); | 1457 header->finalize(payload, payloadSize); |
| 1458 | 1458 |
| 1459 // As compaction is under way, leave the freed memory accessible | 1459 // As compaction is under way, leave the freed memory accessible |
| 1460 // while compacting the rest of the page. We just zap the payload | 1460 // while compacting the rest of the page. We just zap the payload |
| 1461 // to catch out other finalizers trying to access it. | 1461 // to catch out other finalizers trying to access it. |
| 1462 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) || \ | 1462 #if DCHECK_IS_ON() || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) || \ |
| 1463 defined(MEMORY_SANITIZER) | 1463 defined(MEMORY_SANITIZER) |
| 1464 FreeList::zapFreedMemory(payload, payloadSize); | 1464 FreeList::zapFreedMemory(payload, payloadSize); |
| 1465 #endif | 1465 #endif |
| 1466 headerAddress += size; | 1466 headerAddress += size; |
| 1467 continue; | 1467 continue; |
| 1468 } | 1468 } |
| 1469 header->unmark(); | 1469 header->unmark(); |
| 1470 // Allocate and copy over the live object. | 1470 // Allocate and copy over the live object. |
| 1471 Address compactFrontier = currentPage->payload() + allocationPoint; | 1471 Address compactFrontier = currentPage->payload() + allocationPoint; |
| 1472 if (compactFrontier + size > currentPage->payloadEnd()) { | 1472 if (compactFrontier + size > currentPage->payloadEnd()) { |
| (...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1507 compact->relocate(payload, compactFrontier + sizeof(HeapObjectHeader)); | 1507 compact->relocate(payload, compactFrontier + sizeof(HeapObjectHeader)); |
| 1508 } | 1508 } |
| 1509 headerAddress += size; | 1509 headerAddress += size; |
| 1510 markedObjectSize += size; | 1510 markedObjectSize += size; |
| 1511 allocationPoint += size; | 1511 allocationPoint += size; |
| 1512 DCHECK(allocationPoint <= currentPage->payloadSize()); | 1512 DCHECK(allocationPoint <= currentPage->payloadSize()); |
| 1513 } | 1513 } |
| 1514 if (markedObjectSize) | 1514 if (markedObjectSize) |
| 1515 pageArena->getThreadState()->increaseMarkedObjectSize(markedObjectSize); | 1515 pageArena->getThreadState()->increaseMarkedObjectSize(markedObjectSize); |
| 1516 | 1516 |
| 1517 #if ENABLE(ASSERT) || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) || \ | 1517 #if DCHECK_IS_ON() || defined(LEAK_SANITIZER) || defined(ADDRESS_SANITIZER) || \ |
| 1518 defined(MEMORY_SANITIZER) | 1518 defined(MEMORY_SANITIZER) |
| 1519 // Zap the unused portion, until it is either compacted into or freed. | 1519 // Zap the unused portion, until it is either compacted into or freed. |
| 1520 if (currentPage != this) { | 1520 if (currentPage != this) { |
| 1521 FreeList::zapFreedMemory(payload(), payloadSize()); | 1521 FreeList::zapFreedMemory(payload(), payloadSize()); |
| 1522 } else { | 1522 } else { |
| 1523 FreeList::zapFreedMemory(payload() + allocationPoint, | 1523 FreeList::zapFreedMemory(payload() + allocationPoint, |
| 1524 payloadSize() - allocationPoint); | 1524 payloadSize() - allocationPoint); |
| 1525 } | 1525 } |
| 1526 #endif | 1526 #endif |
| 1527 } | 1527 } |
| (...skipping 130 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1658 objectStartNumber = (mapIndex * 8) + 7 - leadingZeroes; | 1658 objectStartNumber = (mapIndex * 8) + 7 - leadingZeroes; |
| 1659 objectOffset = objectStartNumber * allocationGranularity; | 1659 objectOffset = objectStartNumber * allocationGranularity; |
| 1660 Address objectAddress = objectOffset + payload(); | 1660 Address objectAddress = objectOffset + payload(); |
| 1661 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(objectAddress); | 1661 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(objectAddress); |
| 1662 if (header->isFree()) | 1662 if (header->isFree()) |
| 1663 return nullptr; | 1663 return nullptr; |
| 1664 ASSERT(header->checkHeader()); | 1664 ASSERT(header->checkHeader()); |
| 1665 return header; | 1665 return header; |
| 1666 } | 1666 } |
| 1667 | 1667 |
| 1668 #if ENABLE(ASSERT) | 1668 #if DCHECK_IS_ON() |
| 1669 static bool isUninitializedMemory(void* objectPointer, size_t objectSize) { | 1669 static bool isUninitializedMemory(void* objectPointer, size_t objectSize) { |
| 1670 // Scan through the object's fields and check that they are all zero. | 1670 // Scan through the object's fields and check that they are all zero. |
| 1671 Address* objectFields = reinterpret_cast<Address*>(objectPointer); | 1671 Address* objectFields = reinterpret_cast<Address*>(objectPointer); |
| 1672 for (size_t i = 0; i < objectSize / sizeof(Address); ++i) { | 1672 for (size_t i = 0; i < objectSize / sizeof(Address); ++i) { |
| 1673 if (objectFields[i] != 0) | 1673 if (objectFields[i] != 0) |
| 1674 return false; | 1674 return false; |
| 1675 } | 1675 } |
| 1676 return true; | 1676 return true; |
| 1677 } | 1677 } |
| 1678 #endif | 1678 #endif |
| (...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1756 pageDump->AddScalar("live_count", "objects", liveCount); | 1756 pageDump->AddScalar("live_count", "objects", liveCount); |
| 1757 pageDump->AddScalar("dead_count", "objects", deadCount); | 1757 pageDump->AddScalar("dead_count", "objects", deadCount); |
| 1758 pageDump->AddScalar("free_count", "objects", freeCount); | 1758 pageDump->AddScalar("free_count", "objects", freeCount); |
| 1759 pageDump->AddScalar("live_size", "bytes", liveSize); | 1759 pageDump->AddScalar("live_size", "bytes", liveSize); |
| 1760 pageDump->AddScalar("dead_size", "bytes", deadSize); | 1760 pageDump->AddScalar("dead_size", "bytes", deadSize); |
| 1761 pageDump->AddScalar("free_size", "bytes", freeSize); | 1761 pageDump->AddScalar("free_size", "bytes", freeSize); |
| 1762 heapInfo.freeSize += freeSize; | 1762 heapInfo.freeSize += freeSize; |
| 1763 heapInfo.freeCount += freeCount; | 1763 heapInfo.freeCount += freeCount; |
| 1764 } | 1764 } |
| 1765 | 1765 |
| 1766 #if ENABLE(ASSERT) | 1766 #if DCHECK_IS_ON() |
| 1767 bool NormalPage::contains(Address addr) { | 1767 bool NormalPage::contains(Address addr) { |
| 1768 Address blinkPageStart = roundToBlinkPageStart(getAddress()); | 1768 Address blinkPageStart = roundToBlinkPageStart(getAddress()); |
| 1769 // Page is at aligned address plus guard page size. | 1769 // Page is at aligned address plus guard page size. |
| 1770 ASSERT(blinkPageStart == getAddress() - blinkGuardPageSize); | 1770 ASSERT(blinkPageStart == getAddress() - blinkGuardPageSize); |
| 1771 return blinkPageStart <= addr && addr < blinkPageStart + blinkPageSize; | 1771 return blinkPageStart <= addr && addr < blinkPageStart + blinkPageSize; |
| 1772 } | 1772 } |
| 1773 #endif | 1773 #endif |
| 1774 | 1774 |
| 1775 LargeObjectPage::LargeObjectPage(PageMemory* storage, | 1775 LargeObjectPage::LargeObjectPage(PageMemory* storage, |
| 1776 BaseArena* arena, | 1776 BaseArena* arena, |
| (...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1863 info.deadCount[gcInfoIndex]++; | 1863 info.deadCount[gcInfoIndex]++; |
| 1864 info.deadSize[gcInfoIndex] += payloadSize; | 1864 info.deadSize[gcInfoIndex] += payloadSize; |
| 1865 } | 1865 } |
| 1866 | 1866 |
| 1867 pageDump->AddScalar("live_count", "objects", liveCount); | 1867 pageDump->AddScalar("live_count", "objects", liveCount); |
| 1868 pageDump->AddScalar("dead_count", "objects", deadCount); | 1868 pageDump->AddScalar("dead_count", "objects", deadCount); |
| 1869 pageDump->AddScalar("live_size", "bytes", liveSize); | 1869 pageDump->AddScalar("live_size", "bytes", liveSize); |
| 1870 pageDump->AddScalar("dead_size", "bytes", deadSize); | 1870 pageDump->AddScalar("dead_size", "bytes", deadSize); |
| 1871 } | 1871 } |
| 1872 | 1872 |
| 1873 #if ENABLE(ASSERT) | 1873 #if DCHECK_IS_ON() |
| 1874 bool LargeObjectPage::contains(Address object) { | 1874 bool LargeObjectPage::contains(Address object) { |
| 1875 return roundToBlinkPageStart(getAddress()) <= object && | 1875 return roundToBlinkPageStart(getAddress()) <= object && |
| 1876 object < roundToBlinkPageEnd(getAddress() + size()); | 1876 object < roundToBlinkPageEnd(getAddress() + size()); |
| 1877 } | 1877 } |
| 1878 #endif | 1878 #endif |
| 1879 | 1879 |
| 1880 void HeapDoesNotContainCache::flush() { | 1880 void HeapDoesNotContainCache::flush() { |
| 1881 if (m_hasEntries) { | 1881 if (m_hasEntries) { |
| 1882 for (int i = 0; i < numberOfEntries; ++i) | 1882 for (int i = 0; i < numberOfEntries; ++i) |
| 1883 m_entries[i] = nullptr; | 1883 m_entries[i] = nullptr; |
| (...skipping 27 matching lines...) Expand all Loading... |
| 1911 | 1911 |
| 1912 m_hasEntries = true; | 1912 m_hasEntries = true; |
| 1913 size_t index = hash(address); | 1913 size_t index = hash(address); |
| 1914 ASSERT(!(index & 1)); | 1914 ASSERT(!(index & 1)); |
| 1915 Address cachePage = roundToBlinkPageStart(address); | 1915 Address cachePage = roundToBlinkPageStart(address); |
| 1916 m_entries[index + 1] = m_entries[index]; | 1916 m_entries[index + 1] = m_entries[index]; |
| 1917 m_entries[index] = cachePage; | 1917 m_entries[index] = cachePage; |
| 1918 } | 1918 } |
| 1919 | 1919 |
| 1920 } // namespace blink | 1920 } // namespace blink |
| OLD | NEW |