| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (C) 2013 Google Inc. All rights reserved. | 2 * Copyright (C) 2013 Google Inc. All rights reserved. |
| 3 * | 3 * |
| 4 * Redistribution and use in source and binary forms, with or without | 4 * Redistribution and use in source and binary forms, with or without |
| 5 * modification, are permitted provided that the following conditions are | 5 * modification, are permitted provided that the following conditions are |
| 6 * met: | 6 * met: |
| 7 * | 7 * |
| 8 * * Redistributions of source code must retain the above copyright | 8 * * Redistributions of source code must retain the above copyright |
| 9 * notice, this list of conditions and the following disclaimer. | 9 * notice, this list of conditions and the following disclaimer. |
| 10 * * Redistributions in binary form must reproduce the above | 10 * * Redistributions in binary form must reproduce the above |
| (...skipping 236 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 247 { | 247 { |
| 248 ASSERT(threadState()->isInGC()); | 248 ASSERT(threadState()->isInGC()); |
| 249 ASSERT(!m_firstUnsweptPage); | 249 ASSERT(!m_firstUnsweptPage); |
| 250 | 250 |
| 251 // Move all pages to a list of unswept pages. | 251 // Move all pages to a list of unswept pages. |
| 252 m_firstUnsweptPage = m_firstPage; | 252 m_firstUnsweptPage = m_firstPage; |
| 253 m_firstPage = nullptr; | 253 m_firstPage = nullptr; |
| 254 } | 254 } |
| 255 | 255 |
| 256 #if defined(ADDRESS_SANITIZER) | 256 #if defined(ADDRESS_SANITIZER) |
| 257 void BaseArena::poisonHeap(BlinkGC::ObjectsToPoison objectsToPoison, BlinkGC::Po
isoning poisoning) | 257 void BaseArena::poisonArena(BlinkGC::ObjectsToPoison objectsToPoison, BlinkGC::P
oisoning poisoning) |
| 258 { | 258 { |
| 259 // TODO(sof): support complete poisoning of all arenas. | 259 // TODO(sof): support complete poisoning of all arenas. |
| 260 ASSERT(objectsToPoison != BlinkGC::MarkedAndUnmarked || arenaIndex() == Blin
kGC::EagerSweepArenaIndex); | 260 ASSERT(objectsToPoison != BlinkGC::MarkedAndUnmarked || arenaIndex() == Blin
kGC::EagerSweepArenaIndex); |
| 261 | 261 |
| 262 // This method may either be called to poison (SetPoison) heap | 262 // This method may either be called to poison (SetPoison) heap |
| 263 // object payloads prior to sweeping, or it may be called at | 263 // object payloads prior to sweeping, or it may be called at |
| 264 // the completion of a sweep to unpoison (ClearPoison) the | 264 // the completion of a sweep to unpoison (ClearPoison) the |
| 265 // objects remaining in the heap. Those will all be live and unmarked. | 265 // objects remaining in the heap. Those will all be live and unmarked. |
| 266 // | 266 // |
| 267 // Poisoning may be limited to unmarked objects only, or apply to all. | 267 // Poisoning may be limited to unmarked objects only, or apply to all. |
| (...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 354 RELEASE_ASSERT(threadState()->isSweepingInProgress()); | 354 RELEASE_ASSERT(threadState()->isSweepingInProgress()); |
| 355 ASSERT(threadState()->sweepForbidden()); | 355 ASSERT(threadState()->sweepForbidden()); |
| 356 ASSERT(!threadState()->isMainThread() || ScriptForbiddenScope::isScriptForbi
dden()); | 356 ASSERT(!threadState()->isMainThread() || ScriptForbiddenScope::isScriptForbi
dden()); |
| 357 | 357 |
| 358 while (m_firstUnsweptPage) { | 358 while (m_firstUnsweptPage) { |
| 359 sweepUnsweptPage(); | 359 sweepUnsweptPage(); |
| 360 } | 360 } |
| 361 Heap::reportMemoryUsageForTracing(); | 361 Heap::reportMemoryUsageForTracing(); |
| 362 } | 362 } |
| 363 | 363 |
| 364 NormalPageHeap::NormalPageHeap(ThreadState* state, int index) | 364 NormalPageArena::NormalPageArena(ThreadState* state, int index) |
| 365 : BaseArena(state, index) | 365 : BaseArena(state, index) |
| 366 , m_currentAllocationPoint(nullptr) | 366 , m_currentAllocationPoint(nullptr) |
| 367 , m_remainingAllocationSize(0) | 367 , m_remainingAllocationSize(0) |
| 368 , m_lastRemainingAllocationSize(0) | 368 , m_lastRemainingAllocationSize(0) |
| 369 , m_promptlyFreedSize(0) | 369 , m_promptlyFreedSize(0) |
| 370 { | 370 { |
| 371 clearFreeLists(); | 371 clearFreeLists(); |
| 372 } | 372 } |
| 373 | 373 |
| 374 void NormalPageHeap::clearFreeLists() | 374 void NormalPageArena::clearFreeLists() |
| 375 { | 375 { |
| 376 setAllocationPoint(nullptr, 0); | 376 setAllocationPoint(nullptr, 0); |
| 377 m_freeList.clear(); | 377 m_freeList.clear(); |
| 378 } | 378 } |
| 379 | 379 |
| 380 #if ENABLE(ASSERT) | 380 #if ENABLE(ASSERT) |
| 381 bool NormalPageHeap::isConsistentForGC() | 381 bool NormalPageArena::isConsistentForGC() |
| 382 { | 382 { |
| 383 // A thread heap is consistent for sweeping if none of the pages to be swept | 383 // A thread heap is consistent for sweeping if none of the pages to be swept |
| 384 // contain a freelist block or the current allocation point. | 384 // contain a freelist block or the current allocation point. |
| 385 for (size_t i = 0; i < blinkPageSizeLog2; ++i) { | 385 for (size_t i = 0; i < blinkPageSizeLog2; ++i) { |
| 386 for (FreeListEntry* freeListEntry = m_freeList.m_freeLists[i]; freeListE
ntry; freeListEntry = freeListEntry->next()) { | 386 for (FreeListEntry* freeListEntry = m_freeList.m_freeLists[i]; freeListE
ntry; freeListEntry = freeListEntry->next()) { |
| 387 if (pagesToBeSweptContains(freeListEntry->getAddress())) | 387 if (pagesToBeSweptContains(freeListEntry->getAddress())) |
| 388 return false; | 388 return false; |
| 389 } | 389 } |
| 390 } | 390 } |
| 391 if (hasCurrentAllocationArea()) { | 391 if (hasCurrentAllocationArea()) { |
| 392 if (pagesToBeSweptContains(currentAllocationPoint())) | 392 if (pagesToBeSweptContains(currentAllocationPoint())) |
| 393 return false; | 393 return false; |
| 394 } | 394 } |
| 395 return true; | 395 return true; |
| 396 } | 396 } |
| 397 | 397 |
| 398 bool NormalPageHeap::pagesToBeSweptContains(Address address) | 398 bool NormalPageArena::pagesToBeSweptContains(Address address) |
| 399 { | 399 { |
| 400 for (BasePage* page = m_firstUnsweptPage; page; page = page->next()) { | 400 for (BasePage* page = m_firstUnsweptPage; page; page = page->next()) { |
| 401 if (page->contains(address)) | 401 if (page->contains(address)) |
| 402 return true; | 402 return true; |
| 403 } | 403 } |
| 404 return false; | 404 return false; |
| 405 } | 405 } |
| 406 #endif | 406 #endif |
| 407 | 407 |
| 408 void NormalPageHeap::takeFreelistSnapshot(const String& dumpName) | 408 void NormalPageArena::takeFreelistSnapshot(const String& dumpName) |
| 409 { | 409 { |
| 410 if (m_freeList.takeSnapshot(dumpName)) { | 410 if (m_freeList.takeSnapshot(dumpName)) { |
| 411 WebMemoryAllocatorDump* bucketsDump = BlinkGCMemoryDumpProvider::instanc
e()->createMemoryAllocatorDumpForCurrentGC(dumpName + "/buckets"); | 411 WebMemoryAllocatorDump* bucketsDump = BlinkGCMemoryDumpProvider::instanc
e()->createMemoryAllocatorDumpForCurrentGC(dumpName + "/buckets"); |
| 412 WebMemoryAllocatorDump* pagesDump = BlinkGCMemoryDumpProvider::instance(
)->createMemoryAllocatorDumpForCurrentGC(dumpName + "/pages"); | 412 WebMemoryAllocatorDump* pagesDump = BlinkGCMemoryDumpProvider::instance(
)->createMemoryAllocatorDumpForCurrentGC(dumpName + "/pages"); |
| 413 BlinkGCMemoryDumpProvider::instance()->currentProcessMemoryDump()->addOw
nershipEdge(pagesDump->guid(), bucketsDump->guid()); | 413 BlinkGCMemoryDumpProvider::instance()->currentProcessMemoryDump()->addOw
nershipEdge(pagesDump->guid(), bucketsDump->guid()); |
| 414 } | 414 } |
| 415 } | 415 } |
| 416 | 416 |
| 417 void NormalPageHeap::allocatePage() | 417 void NormalPageArena::allocatePage() |
| 418 { | 418 { |
| 419 threadState()->shouldFlushHeapDoesNotContainCache(); | 419 threadState()->shouldFlushHeapDoesNotContainCache(); |
| 420 PageMemory* pageMemory = Heap::freePagePool()->takeFreePage(arenaIndex()); | 420 PageMemory* pageMemory = Heap::freePagePool()->takeFreePage(arenaIndex()); |
| 421 | 421 |
| 422 if (!pageMemory) { | 422 if (!pageMemory) { |
| 423 // Allocate a memory region for blinkPagesPerRegion pages that | 423 // Allocate a memory region for blinkPagesPerRegion pages that |
| 424 // will each have the following layout. | 424 // will each have the following layout. |
| 425 // | 425 // |
| 426 // [ guard os page | ... payload ... | guard os page ] | 426 // [ guard os page | ... payload ... | guard os page ] |
| 427 // ^---{ aligned to blink page size } | 427 // ^---{ aligned to blink page size } |
| (...skipping 26 matching lines...) Expand all Loading... |
| 454 // to the free list. | 454 // to the free list. |
| 455 ASAN_UNPOISON_MEMORY_REGION(page->payload(), page->payloadSize()); | 455 ASAN_UNPOISON_MEMORY_REGION(page->payload(), page->payloadSize()); |
| 456 Address address = page->payload(); | 456 Address address = page->payload(); |
| 457 for (size_t i = 0; i < page->payloadSize(); i++) | 457 for (size_t i = 0; i < page->payloadSize(); i++) |
| 458 address[i] = reuseAllowedZapValue; | 458 address[i] = reuseAllowedZapValue; |
| 459 ASAN_POISON_MEMORY_REGION(page->payload(), page->payloadSize()); | 459 ASAN_POISON_MEMORY_REGION(page->payload(), page->payloadSize()); |
| 460 #endif | 460 #endif |
| 461 addToFreeList(page->payload(), page->payloadSize()); | 461 addToFreeList(page->payload(), page->payloadSize()); |
| 462 } | 462 } |
| 463 | 463 |
| 464 void NormalPageHeap::freePage(NormalPage* page) | 464 void NormalPageArena::freePage(NormalPage* page) |
| 465 { | 465 { |
| 466 Heap::decreaseAllocatedSpace(page->size()); | 466 Heap::decreaseAllocatedSpace(page->size()); |
| 467 | 467 |
| 468 if (page->terminating()) { | 468 if (page->terminating()) { |
| 469 // The thread is shutting down and this page is being removed as a part | 469 // The thread is shutting down and this page is being removed as a part |
| 470 // of the thread local GC. In that case the object could be traced in | 470 // of the thread local GC. In that case the object could be traced in |
| 471 // the next global GC if there is a dangling pointer from a live thread | 471 // the next global GC if there is a dangling pointer from a live thread |
| 472 // heap to this dead thread heap. To guard against this, we put the | 472 // heap to this dead thread heap. To guard against this, we put the |
| 473 // page into the orphaned page pool and zap the page memory. This | 473 // page into the orphaned page pool and zap the page memory. This |
| 474 // ensures that tracing the dangling pointer in the next global GC just | 474 // ensures that tracing the dangling pointer in the next global GC just |
| 475 // crashes instead of causing use-after-frees. After the next global | 475 // crashes instead of causing use-after-frees. After the next global |
| 476 // GC, the orphaned pages are removed. | 476 // GC, the orphaned pages are removed. |
| 477 Heap::orphanedPagePool()->addOrphanedPage(arenaIndex(), page); | 477 Heap::orphanedPagePool()->addOrphanedPage(arenaIndex(), page); |
| 478 } else { | 478 } else { |
| 479 PageMemory* memory = page->storage(); | 479 PageMemory* memory = page->storage(); |
| 480 page->~NormalPage(); | 480 page->~NormalPage(); |
| 481 Heap::freePagePool()->addFreePage(arenaIndex(), memory); | 481 Heap::freePagePool()->addFreePage(arenaIndex(), memory); |
| 482 } | 482 } |
| 483 } | 483 } |
| 484 | 484 |
| 485 bool NormalPageHeap::coalesce() | 485 bool NormalPageArena::coalesce() |
| 486 { | 486 { |
| 487 // Don't coalesce arenas if there are not enough promptly freed entries | 487 // Don't coalesce arenas if there are not enough promptly freed entries |
| 488 // to be coalesced. | 488 // to be coalesced. |
| 489 // | 489 // |
| 490 // FIXME: This threshold is determined just to optimize blink_perf | 490 // FIXME: This threshold is determined just to optimize blink_perf |
| 491 // benchmarks. Coalescing is very sensitive to the threashold and | 491 // benchmarks. Coalescing is very sensitive to the threashold and |
| 492 // we need further investigations on the coalescing scheme. | 492 // we need further investigations on the coalescing scheme. |
| 493 if (m_promptlyFreedSize < 1024 * 1024) | 493 if (m_promptlyFreedSize < 1024 * 1024) |
| 494 return false; | 494 return false; |
| 495 | 495 |
| (...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 542 | 542 |
| 543 if (startOfGap != page->payloadEnd()) | 543 if (startOfGap != page->payloadEnd()) |
| 544 addToFreeList(startOfGap, page->payloadEnd() - startOfGap); | 544 addToFreeList(startOfGap, page->payloadEnd() - startOfGap); |
| 545 } | 545 } |
| 546 threadState()->decreaseAllocatedObjectSize(freedSize); | 546 threadState()->decreaseAllocatedObjectSize(freedSize); |
| 547 ASSERT(m_promptlyFreedSize == freedSize); | 547 ASSERT(m_promptlyFreedSize == freedSize); |
| 548 m_promptlyFreedSize = 0; | 548 m_promptlyFreedSize = 0; |
| 549 return true; | 549 return true; |
| 550 } | 550 } |
| 551 | 551 |
| 552 void NormalPageHeap::promptlyFreeObject(HeapObjectHeader* header) | 552 void NormalPageArena::promptlyFreeObject(HeapObjectHeader* header) |
| 553 { | 553 { |
| 554 ASSERT(!threadState()->sweepForbidden()); | 554 ASSERT(!threadState()->sweepForbidden()); |
| 555 ASSERT(header->checkHeader()); | 555 ASSERT(header->checkHeader()); |
| 556 Address address = reinterpret_cast<Address>(header); | 556 Address address = reinterpret_cast<Address>(header); |
| 557 Address payload = header->payload(); | 557 Address payload = header->payload(); |
| 558 size_t size = header->size(); | 558 size_t size = header->size(); |
| 559 size_t payloadSize = header->payloadSize(); | 559 size_t payloadSize = header->payloadSize(); |
| 560 ASSERT(size > 0); | 560 ASSERT(size > 0); |
| 561 ASSERT(pageFromObject(address) == findPageFromAddress(address)); | 561 ASSERT(pageFromObject(address) == findPageFromAddress(address)); |
| 562 | 562 |
| 563 { | 563 { |
| 564 ThreadState::SweepForbiddenScope forbiddenScope(threadState()); | 564 ThreadState::SweepForbiddenScope forbiddenScope(threadState()); |
| 565 header->finalize(payload, payloadSize); | 565 header->finalize(payload, payloadSize); |
| 566 if (address + size == m_currentAllocationPoint) { | 566 if (address + size == m_currentAllocationPoint) { |
| 567 m_currentAllocationPoint = address; | 567 m_currentAllocationPoint = address; |
| 568 setRemainingAllocationSize(m_remainingAllocationSize + size); | 568 setRemainingAllocationSize(m_remainingAllocationSize + size); |
| 569 SET_MEMORY_INACCESSIBLE(address, size); | 569 SET_MEMORY_INACCESSIBLE(address, size); |
| 570 return; | 570 return; |
| 571 } | 571 } |
| 572 SET_MEMORY_INACCESSIBLE(payload, payloadSize); | 572 SET_MEMORY_INACCESSIBLE(payload, payloadSize); |
| 573 header->markPromptlyFreed(); | 573 header->markPromptlyFreed(); |
| 574 } | 574 } |
| 575 | 575 |
| 576 m_promptlyFreedSize += size; | 576 m_promptlyFreedSize += size; |
| 577 } | 577 } |
| 578 | 578 |
| 579 bool NormalPageHeap::expandObject(HeapObjectHeader* header, size_t newSize) | 579 bool NormalPageArena::expandObject(HeapObjectHeader* header, size_t newSize) |
| 580 { | 580 { |
| 581 // It's possible that Vector requests a smaller expanded size because | 581 // It's possible that Vector requests a smaller expanded size because |
| 582 // Vector::shrinkCapacity can set a capacity smaller than the actual payload | 582 // Vector::shrinkCapacity can set a capacity smaller than the actual payload |
| 583 // size. | 583 // size. |
| 584 ASSERT(header->checkHeader()); | 584 ASSERT(header->checkHeader()); |
| 585 if (header->payloadSize() >= newSize) | 585 if (header->payloadSize() >= newSize) |
| 586 return true; | 586 return true; |
| 587 size_t allocationSize = Heap::allocationSizeFromSize(newSize); | 587 size_t allocationSize = Heap::allocationSizeFromSize(newSize); |
| 588 ASSERT(allocationSize > header->size()); | 588 ASSERT(allocationSize > header->size()); |
| 589 size_t expandSize = allocationSize - header->size(); | 589 size_t expandSize = allocationSize - header->size(); |
| 590 if (isObjectAllocatedAtAllocationPoint(header) && expandSize <= m_remainingA
llocationSize) { | 590 if (isObjectAllocatedAtAllocationPoint(header) && expandSize <= m_remainingA
llocationSize) { |
| 591 m_currentAllocationPoint += expandSize; | 591 m_currentAllocationPoint += expandSize; |
| 592 ASSERT(m_remainingAllocationSize >= expandSize); | 592 ASSERT(m_remainingAllocationSize >= expandSize); |
| 593 setRemainingAllocationSize(m_remainingAllocationSize - expandSize); | 593 setRemainingAllocationSize(m_remainingAllocationSize - expandSize); |
| 594 // Unpoison the memory used for the object (payload). | 594 // Unpoison the memory used for the object (payload). |
| 595 SET_MEMORY_ACCESSIBLE(header->payloadEnd(), expandSize); | 595 SET_MEMORY_ACCESSIBLE(header->payloadEnd(), expandSize); |
| 596 header->setSize(allocationSize); | 596 header->setSize(allocationSize); |
| 597 ASSERT(findPageFromAddress(header->payloadEnd() - 1)); | 597 ASSERT(findPageFromAddress(header->payloadEnd() - 1)); |
| 598 return true; | 598 return true; |
| 599 } | 599 } |
| 600 return false; | 600 return false; |
| 601 } | 601 } |
| 602 | 602 |
| 603 bool NormalPageHeap::shrinkObject(HeapObjectHeader* header, size_t newSize) | 603 bool NormalPageArena::shrinkObject(HeapObjectHeader* header, size_t newSize) |
| 604 { | 604 { |
| 605 ASSERT(header->checkHeader()); | 605 ASSERT(header->checkHeader()); |
| 606 ASSERT(header->payloadSize() > newSize); | 606 ASSERT(header->payloadSize() > newSize); |
| 607 size_t allocationSize = Heap::allocationSizeFromSize(newSize); | 607 size_t allocationSize = Heap::allocationSizeFromSize(newSize); |
| 608 ASSERT(header->size() > allocationSize); | 608 ASSERT(header->size() > allocationSize); |
| 609 size_t shrinkSize = header->size() - allocationSize; | 609 size_t shrinkSize = header->size() - allocationSize; |
| 610 if (isObjectAllocatedAtAllocationPoint(header)) { | 610 if (isObjectAllocatedAtAllocationPoint(header)) { |
| 611 m_currentAllocationPoint -= shrinkSize; | 611 m_currentAllocationPoint -= shrinkSize; |
| 612 setRemainingAllocationSize(m_remainingAllocationSize + shrinkSize); | 612 setRemainingAllocationSize(m_remainingAllocationSize + shrinkSize); |
| 613 SET_MEMORY_INACCESSIBLE(m_currentAllocationPoint, shrinkSize); | 613 SET_MEMORY_INACCESSIBLE(m_currentAllocationPoint, shrinkSize); |
| 614 header->setSize(allocationSize); | 614 header->setSize(allocationSize); |
| 615 return true; | 615 return true; |
| 616 } | 616 } |
| 617 ASSERT(shrinkSize >= sizeof(HeapObjectHeader)); | 617 ASSERT(shrinkSize >= sizeof(HeapObjectHeader)); |
| 618 ASSERT(header->gcInfoIndex() > 0); | 618 ASSERT(header->gcInfoIndex() > 0); |
| 619 Address shrinkAddress = header->payloadEnd() - shrinkSize; | 619 Address shrinkAddress = header->payloadEnd() - shrinkSize; |
| 620 HeapObjectHeader* freedHeader = new (NotNull, shrinkAddress) HeapObjectHeade
r(shrinkSize, header->gcInfoIndex()); | 620 HeapObjectHeader* freedHeader = new (NotNull, shrinkAddress) HeapObjectHeade
r(shrinkSize, header->gcInfoIndex()); |
| 621 freedHeader->markPromptlyFreed(); | 621 freedHeader->markPromptlyFreed(); |
| 622 ASSERT(pageFromObject(reinterpret_cast<Address>(header)) == findPageFromAddr
ess(reinterpret_cast<Address>(header))); | 622 ASSERT(pageFromObject(reinterpret_cast<Address>(header)) == findPageFromAddr
ess(reinterpret_cast<Address>(header))); |
| 623 m_promptlyFreedSize += shrinkSize; | 623 m_promptlyFreedSize += shrinkSize; |
| 624 header->setSize(allocationSize); | 624 header->setSize(allocationSize); |
| 625 SET_MEMORY_INACCESSIBLE(shrinkAddress + sizeof(HeapObjectHeader), shrinkSize
- sizeof(HeapObjectHeader)); | 625 SET_MEMORY_INACCESSIBLE(shrinkAddress + sizeof(HeapObjectHeader), shrinkSize
- sizeof(HeapObjectHeader)); |
| 626 return false; | 626 return false; |
| 627 } | 627 } |
| 628 | 628 |
| 629 Address NormalPageHeap::lazySweepPages(size_t allocationSize, size_t gcInfoIndex
) | 629 Address NormalPageArena::lazySweepPages(size_t allocationSize, size_t gcInfoInde
x) |
| 630 { | 630 { |
| 631 ASSERT(!hasCurrentAllocationArea()); | 631 ASSERT(!hasCurrentAllocationArea()); |
| 632 Address result = nullptr; | 632 Address result = nullptr; |
| 633 while (m_firstUnsweptPage) { | 633 while (m_firstUnsweptPage) { |
| 634 BasePage* page = m_firstUnsweptPage; | 634 BasePage* page = m_firstUnsweptPage; |
| 635 if (page->isEmpty()) { | 635 if (page->isEmpty()) { |
| 636 page->unlink(&m_firstUnsweptPage); | 636 page->unlink(&m_firstUnsweptPage); |
| 637 page->removeFromHeap(); | 637 page->removeFromHeap(); |
| 638 } else { | 638 } else { |
| 639 // Sweep a page and move the page from m_firstUnsweptPages to | 639 // Sweep a page and move the page from m_firstUnsweptPages to |
| 640 // m_firstPages. | 640 // m_firstPages. |
| 641 page->sweep(); | 641 page->sweep(); |
| 642 page->unlink(&m_firstUnsweptPage); | 642 page->unlink(&m_firstUnsweptPage); |
| 643 page->link(&m_firstPage); | 643 page->link(&m_firstPage); |
| 644 page->markAsSwept(); | 644 page->markAsSwept(); |
| 645 | 645 |
| 646 // For NormalPage, stop lazy sweeping once we find a slot to | 646 // For NormalPage, stop lazy sweeping once we find a slot to |
| 647 // allocate a new object. | 647 // allocate a new object. |
| 648 result = allocateFromFreeList(allocationSize, gcInfoIndex); | 648 result = allocateFromFreeList(allocationSize, gcInfoIndex); |
| 649 if (result) | 649 if (result) |
| 650 break; | 650 break; |
| 651 } | 651 } |
| 652 } | 652 } |
| 653 return result; | 653 return result; |
| 654 } | 654 } |
| 655 | 655 |
| 656 void NormalPageHeap::setRemainingAllocationSize(size_t newRemainingAllocationSiz
e) | 656 void NormalPageArena::setRemainingAllocationSize(size_t newRemainingAllocationSi
ze) |
| 657 { | 657 { |
| 658 m_remainingAllocationSize = newRemainingAllocationSize; | 658 m_remainingAllocationSize = newRemainingAllocationSize; |
| 659 | 659 |
| 660 // Sync recorded allocated-object size: | 660 // Sync recorded allocated-object size: |
| 661 // - if previous alloc checkpoint is larger, allocation size has increased. | 661 // - if previous alloc checkpoint is larger, allocation size has increased. |
| 662 // - if smaller, a net reduction in size since last call to updateRemaining
AllocationSize(). | 662 // - if smaller, a net reduction in size since last call to updateRemaining
AllocationSize(). |
| 663 if (m_lastRemainingAllocationSize > m_remainingAllocationSize) | 663 if (m_lastRemainingAllocationSize > m_remainingAllocationSize) |
| 664 threadState()->increaseAllocatedObjectSize(m_lastRemainingAllocationSize
- m_remainingAllocationSize); | 664 threadState()->increaseAllocatedObjectSize(m_lastRemainingAllocationSize
- m_remainingAllocationSize); |
| 665 else if (m_lastRemainingAllocationSize != m_remainingAllocationSize) | 665 else if (m_lastRemainingAllocationSize != m_remainingAllocationSize) |
| 666 threadState()->decreaseAllocatedObjectSize(m_remainingAllocationSize - m
_lastRemainingAllocationSize); | 666 threadState()->decreaseAllocatedObjectSize(m_remainingAllocationSize - m
_lastRemainingAllocationSize); |
| 667 m_lastRemainingAllocationSize = m_remainingAllocationSize; | 667 m_lastRemainingAllocationSize = m_remainingAllocationSize; |
| 668 } | 668 } |
| 669 | 669 |
| 670 void NormalPageHeap::updateRemainingAllocationSize() | 670 void NormalPageArena::updateRemainingAllocationSize() |
| 671 { | 671 { |
| 672 if (m_lastRemainingAllocationSize > remainingAllocationSize()) { | 672 if (m_lastRemainingAllocationSize > remainingAllocationSize()) { |
| 673 threadState()->increaseAllocatedObjectSize(m_lastRemainingAllocationSize
- remainingAllocationSize()); | 673 threadState()->increaseAllocatedObjectSize(m_lastRemainingAllocationSize
- remainingAllocationSize()); |
| 674 m_lastRemainingAllocationSize = remainingAllocationSize(); | 674 m_lastRemainingAllocationSize = remainingAllocationSize(); |
| 675 } | 675 } |
| 676 ASSERT(m_lastRemainingAllocationSize == remainingAllocationSize()); | 676 ASSERT(m_lastRemainingAllocationSize == remainingAllocationSize()); |
| 677 } | 677 } |
| 678 | 678 |
| 679 void NormalPageHeap::setAllocationPoint(Address point, size_t size) | 679 void NormalPageArena::setAllocationPoint(Address point, size_t size) |
| 680 { | 680 { |
| 681 #if ENABLE(ASSERT) | 681 #if ENABLE(ASSERT) |
| 682 if (point) { | 682 if (point) { |
| 683 ASSERT(size); | 683 ASSERT(size); |
| 684 BasePage* page = pageFromObject(point); | 684 BasePage* page = pageFromObject(point); |
| 685 ASSERT(!page->isLargeObjectPage()); | 685 ASSERT(!page->isLargeObjectPage()); |
| 686 ASSERT(size <= static_cast<NormalPage*>(page)->payloadSize()); | 686 ASSERT(size <= static_cast<NormalPage*>(page)->payloadSize()); |
| 687 } | 687 } |
| 688 #endif | 688 #endif |
| 689 if (hasCurrentAllocationArea()) { | 689 if (hasCurrentAllocationArea()) { |
| 690 addToFreeList(currentAllocationPoint(), remainingAllocationSize()); | 690 addToFreeList(currentAllocationPoint(), remainingAllocationSize()); |
| 691 } | 691 } |
| 692 updateRemainingAllocationSize(); | 692 updateRemainingAllocationSize(); |
| 693 m_currentAllocationPoint = point; | 693 m_currentAllocationPoint = point; |
| 694 m_lastRemainingAllocationSize = m_remainingAllocationSize = size; | 694 m_lastRemainingAllocationSize = m_remainingAllocationSize = size; |
| 695 } | 695 } |
| 696 | 696 |
| 697 Address NormalPageHeap::outOfLineAllocate(size_t allocationSize, size_t gcInfoIn
dex) | 697 Address NormalPageArena::outOfLineAllocate(size_t allocationSize, size_t gcInfoI
ndex) |
| 698 { | 698 { |
| 699 ASSERT(allocationSize > remainingAllocationSize()); | 699 ASSERT(allocationSize > remainingAllocationSize()); |
| 700 ASSERT(allocationSize >= allocationGranularity); | 700 ASSERT(allocationSize >= allocationGranularity); |
| 701 | 701 |
| 702 // 1. If this allocation is big enough, allocate a large object. | 702 // 1. If this allocation is big enough, allocate a large object. |
| 703 if (allocationSize >= largeObjectSizeThreshold) { | 703 if (allocationSize >= largeObjectSizeThreshold) { |
| 704 // TODO(sof): support eagerly finalized large objects, if ever needed. | 704 // TODO(sof): support eagerly finalized large objects, if ever needed. |
| 705 RELEASE_ASSERT(arenaIndex() != BlinkGC::EagerSweepArenaIndex); | 705 RELEASE_ASSERT(arenaIndex() != BlinkGC::EagerSweepArenaIndex); |
| 706 LargeObjectHeap* largeObjectHeap = static_cast<LargeObjectHeap*>(threadS
tate()->arena(BlinkGC::LargeObjectArenaIndex)); | 706 LargeObjectArena* largeObjectArena = static_cast<LargeObjectArena*>(thre
adState()->arena(BlinkGC::LargeObjectArenaIndex)); |
| 707 Address largeObject = largeObjectHeap->allocateLargeObjectPage(allocatio
nSize, gcInfoIndex); | 707 Address largeObject = largeObjectArena->allocateLargeObjectPage(allocati
onSize, gcInfoIndex); |
| 708 ASAN_MARK_LARGE_VECTOR_CONTAINER(this, largeObject); | 708 ASAN_MARK_LARGE_VECTOR_CONTAINER(this, largeObject); |
| 709 return largeObject; | 709 return largeObject; |
| 710 } | 710 } |
| 711 | 711 |
| 712 // 2. Try to allocate from a free list. | 712 // 2. Try to allocate from a free list. |
| 713 updateRemainingAllocationSize(); | 713 updateRemainingAllocationSize(); |
| 714 Address result = allocateFromFreeList(allocationSize, gcInfoIndex); | 714 Address result = allocateFromFreeList(allocationSize, gcInfoIndex); |
| 715 if (result) | 715 if (result) |
| 716 return result; | 716 return result; |
| 717 | 717 |
| (...skipping 22 matching lines...) Expand all Loading... |
| 740 | 740 |
| 741 // 8. Add a new page to this heap. | 741 // 8. Add a new page to this heap. |
| 742 allocatePage(); | 742 allocatePage(); |
| 743 | 743 |
| 744 // 9. Try to allocate from a free list. This allocation must succeed. | 744 // 9. Try to allocate from a free list. This allocation must succeed. |
| 745 result = allocateFromFreeList(allocationSize, gcInfoIndex); | 745 result = allocateFromFreeList(allocationSize, gcInfoIndex); |
| 746 RELEASE_ASSERT(result); | 746 RELEASE_ASSERT(result); |
| 747 return result; | 747 return result; |
| 748 } | 748 } |
| 749 | 749 |
| 750 Address NormalPageHeap::allocateFromFreeList(size_t allocationSize, size_t gcInf
oIndex) | 750 Address NormalPageArena::allocateFromFreeList(size_t allocationSize, size_t gcIn
foIndex) |
| 751 { | 751 { |
| 752 // Try reusing a block from the largest bin. The underlying reasoning | 752 // Try reusing a block from the largest bin. The underlying reasoning |
| 753 // being that we want to amortize this slow allocation call by carving | 753 // being that we want to amortize this slow allocation call by carving |
| 754 // off as a large a free block as possible in one go; a block that will | 754 // off as a large a free block as possible in one go; a block that will |
| 755 // service this block and let following allocations be serviced quickly | 755 // service this block and let following allocations be serviced quickly |
| 756 // by bump allocation. | 756 // by bump allocation. |
| 757 size_t bucketSize = 1 << m_freeList.m_biggestFreeListIndex; | 757 size_t bucketSize = 1 << m_freeList.m_biggestFreeListIndex; |
| 758 int index = m_freeList.m_biggestFreeListIndex; | 758 int index = m_freeList.m_biggestFreeListIndex; |
| 759 for (; index > 0; --index, bucketSize >>= 1) { | 759 for (; index > 0; --index, bucketSize >>= 1) { |
| 760 FreeListEntry* entry = m_freeList.m_freeLists[index]; | 760 FreeListEntry* entry = m_freeList.m_freeLists[index]; |
| (...skipping 10 matching lines...) Expand all Loading... |
| 771 ASSERT(hasCurrentAllocationArea()); | 771 ASSERT(hasCurrentAllocationArea()); |
| 772 ASSERT(remainingAllocationSize() >= allocationSize); | 772 ASSERT(remainingAllocationSize() >= allocationSize); |
| 773 m_freeList.m_biggestFreeListIndex = index; | 773 m_freeList.m_biggestFreeListIndex = index; |
| 774 return allocateObject(allocationSize, gcInfoIndex); | 774 return allocateObject(allocationSize, gcInfoIndex); |
| 775 } | 775 } |
| 776 } | 776 } |
| 777 m_freeList.m_biggestFreeListIndex = index; | 777 m_freeList.m_biggestFreeListIndex = index; |
| 778 return nullptr; | 778 return nullptr; |
| 779 } | 779 } |
| 780 | 780 |
| 781 LargeObjectHeap::LargeObjectHeap(ThreadState* state, int index) | 781 LargeObjectArena::LargeObjectArena(ThreadState* state, int index) |
| 782 : BaseArena(state, index) | 782 : BaseArena(state, index) |
| 783 { | 783 { |
| 784 } | 784 } |
| 785 | 785 |
| 786 Address LargeObjectHeap::allocateLargeObjectPage(size_t allocationSize, size_t g
cInfoIndex) | 786 Address LargeObjectArena::allocateLargeObjectPage(size_t allocationSize, size_t
gcInfoIndex) |
| 787 { | 787 { |
| 788 // Caller already added space for object header and rounded up to allocation | 788 // Caller already added space for object header and rounded up to allocation |
| 789 // alignment | 789 // alignment |
| 790 ASSERT(!(allocationSize & allocationMask)); | 790 ASSERT(!(allocationSize & allocationMask)); |
| 791 | 791 |
| 792 // 1. Try to sweep large objects more than allocationSize bytes | 792 // 1. Try to sweep large objects more than allocationSize bytes |
| 793 // before allocating a new large object. | 793 // before allocating a new large object. |
| 794 Address result = lazySweep(allocationSize, gcInfoIndex); | 794 Address result = lazySweep(allocationSize, gcInfoIndex); |
| 795 if (result) | 795 if (result) |
| 796 return result; | 796 return result; |
| 797 | 797 |
| 798 // 2. If we have failed in sweeping allocationSize bytes, | 798 // 2. If we have failed in sweeping allocationSize bytes, |
| 799 // we complete sweeping before allocating this large object. | 799 // we complete sweeping before allocating this large object. |
| 800 threadState()->completeSweep(); | 800 threadState()->completeSweep(); |
| 801 | 801 |
| 802 // 3. Check if we should trigger a GC. | 802 // 3. Check if we should trigger a GC. |
| 803 threadState()->scheduleGCIfNeeded(); | 803 threadState()->scheduleGCIfNeeded(); |
| 804 | 804 |
| 805 return doAllocateLargeObjectPage(allocationSize, gcInfoIndex); | 805 return doAllocateLargeObjectPage(allocationSize, gcInfoIndex); |
| 806 } | 806 } |
| 807 | 807 |
| 808 Address LargeObjectHeap::doAllocateLargeObjectPage(size_t allocationSize, size_t
gcInfoIndex) | 808 Address LargeObjectArena::doAllocateLargeObjectPage(size_t allocationSize, size_
t gcInfoIndex) |
| 809 { | 809 { |
| 810 size_t largeObjectSize = LargeObjectPage::pageHeaderSize() + allocationSize; | 810 size_t largeObjectSize = LargeObjectPage::pageHeaderSize() + allocationSize; |
| 811 // If ASan is supported we add allocationGranularity bytes to the allocated | 811 // If ASan is supported we add allocationGranularity bytes to the allocated |
| 812 // space and poison that to detect overflows | 812 // space and poison that to detect overflows |
| 813 #if defined(ADDRESS_SANITIZER) | 813 #if defined(ADDRESS_SANITIZER) |
| 814 largeObjectSize += allocationGranularity; | 814 largeObjectSize += allocationGranularity; |
| 815 #endif | 815 #endif |
| 816 | 816 |
| 817 threadState()->shouldFlushHeapDoesNotContainCache(); | 817 threadState()->shouldFlushHeapDoesNotContainCache(); |
| 818 PageMemory* pageMemory = PageMemory::allocate(largeObjectSize); | 818 PageMemory* pageMemory = PageMemory::allocate(largeObjectSize); |
| (...skipping 15 matching lines...) Expand all Loading... |
| 834 ASAN_POISON_MEMORY_REGION(header, sizeof(*header)); | 834 ASAN_POISON_MEMORY_REGION(header, sizeof(*header)); |
| 835 ASAN_POISON_MEMORY_REGION(largeObject->getAddress() + largeObject->size(), a
llocationGranularity); | 835 ASAN_POISON_MEMORY_REGION(largeObject->getAddress() + largeObject->size(), a
llocationGranularity); |
| 836 | 836 |
| 837 largeObject->link(&m_firstPage); | 837 largeObject->link(&m_firstPage); |
| 838 | 838 |
| 839 Heap::increaseAllocatedSpace(largeObject->size()); | 839 Heap::increaseAllocatedSpace(largeObject->size()); |
| 840 threadState()->increaseAllocatedObjectSize(largeObject->size()); | 840 threadState()->increaseAllocatedObjectSize(largeObject->size()); |
| 841 return result; | 841 return result; |
| 842 } | 842 } |
| 843 | 843 |
| 844 void LargeObjectHeap::freeLargeObjectPage(LargeObjectPage* object) | 844 void LargeObjectArena::freeLargeObjectPage(LargeObjectPage* object) |
| 845 { | 845 { |
| 846 ASAN_UNPOISON_MEMORY_REGION(object->payload(), object->payloadSize()); | 846 ASAN_UNPOISON_MEMORY_REGION(object->payload(), object->payloadSize()); |
| 847 object->heapObjectHeader()->finalize(object->payload(), object->payloadSize(
)); | 847 object->heapObjectHeader()->finalize(object->payload(), object->payloadSize(
)); |
| 848 Heap::decreaseAllocatedSpace(object->size()); | 848 Heap::decreaseAllocatedSpace(object->size()); |
| 849 | 849 |
| 850 // Unpoison the object header and allocationGranularity bytes after the | 850 // Unpoison the object header and allocationGranularity bytes after the |
| 851 // object before freeing. | 851 // object before freeing. |
| 852 ASAN_UNPOISON_MEMORY_REGION(object->heapObjectHeader(), sizeof(HeapObjectHea
der)); | 852 ASAN_UNPOISON_MEMORY_REGION(object->heapObjectHeader(), sizeof(HeapObjectHea
der)); |
| 853 ASAN_UNPOISON_MEMORY_REGION(object->getAddress() + object->size(), allocatio
nGranularity); | 853 ASAN_UNPOISON_MEMORY_REGION(object->getAddress() + object->size(), allocatio
nGranularity); |
| 854 | 854 |
| 855 if (object->terminating()) { | 855 if (object->terminating()) { |
| 856 ASSERT(ThreadState::current()->isTerminating()); | 856 ASSERT(ThreadState::current()->isTerminating()); |
| 857 // The thread is shutting down and this page is being removed as a part | 857 // The thread is shutting down and this page is being removed as a part |
| 858 // of the thread local GC. In that case the object could be traced in | 858 // of the thread local GC. In that case the object could be traced in |
| 859 // the next global GC if there is a dangling pointer from a live thread | 859 // the next global GC if there is a dangling pointer from a live thread |
| 860 // heap to this dead thread heap. To guard against this, we put the | 860 // heap to this dead thread heap. To guard against this, we put the |
| 861 // page into the orphaned page pool and zap the page memory. This | 861 // page into the orphaned page pool and zap the page memory. This |
| 862 // ensures that tracing the dangling pointer in the next global GC just | 862 // ensures that tracing the dangling pointer in the next global GC just |
| 863 // crashes instead of causing use-after-frees. After the next global | 863 // crashes instead of causing use-after-frees. After the next global |
| 864 // GC, the orphaned pages are removed. | 864 // GC, the orphaned pages are removed. |
| 865 Heap::orphanedPagePool()->addOrphanedPage(arenaIndex(), object); | 865 Heap::orphanedPagePool()->addOrphanedPage(arenaIndex(), object); |
| 866 } else { | 866 } else { |
| 867 ASSERT(!ThreadState::current()->isTerminating()); | 867 ASSERT(!ThreadState::current()->isTerminating()); |
| 868 PageMemory* memory = object->storage(); | 868 PageMemory* memory = object->storage(); |
| 869 object->~LargeObjectPage(); | 869 object->~LargeObjectPage(); |
| 870 delete memory; | 870 delete memory; |
| 871 } | 871 } |
| 872 } | 872 } |
| 873 | 873 |
| 874 Address LargeObjectHeap::lazySweepPages(size_t allocationSize, size_t gcInfoInde
x) | 874 Address LargeObjectArena::lazySweepPages(size_t allocationSize, size_t gcInfoInd
ex) |
| 875 { | 875 { |
| 876 Address result = nullptr; | 876 Address result = nullptr; |
| 877 size_t sweptSize = 0; | 877 size_t sweptSize = 0; |
| 878 while (m_firstUnsweptPage) { | 878 while (m_firstUnsweptPage) { |
| 879 BasePage* page = m_firstUnsweptPage; | 879 BasePage* page = m_firstUnsweptPage; |
| 880 if (page->isEmpty()) { | 880 if (page->isEmpty()) { |
| 881 sweptSize += static_cast<LargeObjectPage*>(page)->payloadSize() + si
zeof(HeapObjectHeader); | 881 sweptSize += static_cast<LargeObjectPage*>(page)->payloadSize() + si
zeof(HeapObjectHeader); |
| 882 page->unlink(&m_firstUnsweptPage); | 882 page->unlink(&m_firstUnsweptPage); |
| 883 page->removeFromHeap(); | 883 page->removeFromHeap(); |
| 884 // For LargeObjectPage, stop lazy sweeping once we have swept | 884 // For LargeObjectPage, stop lazy sweeping once we have swept |
| (...skipping 224 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1109 uintptr_t endAddress = WTF::roundDownToSystemPage(reinterpret_cast<uintptr_t
>(end)); | 1109 uintptr_t endAddress = WTF::roundDownToSystemPage(reinterpret_cast<uintptr_t
>(end)); |
| 1110 if (beginAddress < endAddress) | 1110 if (beginAddress < endAddress) |
| 1111 WTF::discardSystemPages(reinterpret_cast<void*>(beginAddress), endAddres
s - beginAddress); | 1111 WTF::discardSystemPages(reinterpret_cast<void*>(beginAddress), endAddres
s - beginAddress); |
| 1112 } | 1112 } |
| 1113 #endif | 1113 #endif |
| 1114 | 1114 |
| 1115 void NormalPage::sweep() | 1115 void NormalPage::sweep() |
| 1116 { | 1116 { |
| 1117 size_t markedObjectSize = 0; | 1117 size_t markedObjectSize = 0; |
| 1118 Address startOfGap = payload(); | 1118 Address startOfGap = payload(); |
| 1119 NormalPageHeap* pageHeap = arenaForNormalPage(); | 1119 NormalPageArena* pageArena = arenaForNormalPage(); |
| 1120 for (Address headerAddress = startOfGap; headerAddress < payloadEnd(); ) { | 1120 for (Address headerAddress = startOfGap; headerAddress < payloadEnd(); ) { |
| 1121 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAdd
ress); | 1121 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAdd
ress); |
| 1122 size_t size = header->size(); | 1122 size_t size = header->size(); |
| 1123 ASSERT(size > 0); | 1123 ASSERT(size > 0); |
| 1124 ASSERT(size < blinkPagePayloadSize()); | 1124 ASSERT(size < blinkPagePayloadSize()); |
| 1125 | 1125 |
| 1126 if (header->isPromptlyFreed()) | 1126 if (header->isPromptlyFreed()) |
| 1127 pageHeap->decreasePromptlyFreedSize(size); | 1127 pageArena->decreasePromptlyFreedSize(size); |
| 1128 if (header->isFree()) { | 1128 if (header->isFree()) { |
| 1129 // Zero the memory in the free list header to maintain the | 1129 // Zero the memory in the free list header to maintain the |
| 1130 // invariant that memory on the free list is zero filled. | 1130 // invariant that memory on the free list is zero filled. |
| 1131 // The rest of the memory is already on the free list and is | 1131 // The rest of the memory is already on the free list and is |
| 1132 // therefore already zero filled. | 1132 // therefore already zero filled. |
| 1133 SET_MEMORY_INACCESSIBLE(headerAddress, size < sizeof(FreeListEntry)
? size : sizeof(FreeListEntry)); | 1133 SET_MEMORY_INACCESSIBLE(headerAddress, size < sizeof(FreeListEntry)
? size : sizeof(FreeListEntry)); |
| 1134 CHECK_MEMORY_INACCESSIBLE(headerAddress, size); | 1134 CHECK_MEMORY_INACCESSIBLE(headerAddress, size); |
| 1135 headerAddress += size; | 1135 headerAddress += size; |
| 1136 continue; | 1136 continue; |
| 1137 } | 1137 } |
| (...skipping 10 matching lines...) Expand all Loading... |
| 1148 // die at the same GC cycle. | 1148 // die at the same GC cycle. |
| 1149 ASAN_UNPOISON_MEMORY_REGION(payload, payloadSize); | 1149 ASAN_UNPOISON_MEMORY_REGION(payload, payloadSize); |
| 1150 header->finalize(payload, payloadSize); | 1150 header->finalize(payload, payloadSize); |
| 1151 // This memory will be added to the freelist. Maintain the invariant | 1151 // This memory will be added to the freelist. Maintain the invariant |
| 1152 // that memory on the freelist is zero filled. | 1152 // that memory on the freelist is zero filled. |
| 1153 SET_MEMORY_INACCESSIBLE(headerAddress, size); | 1153 SET_MEMORY_INACCESSIBLE(headerAddress, size); |
| 1154 headerAddress += size; | 1154 headerAddress += size; |
| 1155 continue; | 1155 continue; |
| 1156 } | 1156 } |
| 1157 if (startOfGap != headerAddress) { | 1157 if (startOfGap != headerAddress) { |
| 1158 pageHeap->addToFreeList(startOfGap, headerAddress - startOfGap); | 1158 pageArena->addToFreeList(startOfGap, headerAddress - startOfGap); |
| 1159 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER) | 1159 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER) |
| 1160 // Discarding pages increases page faults and may regress performanc
e. | 1160 // Discarding pages increases page faults and may regress performanc
e. |
| 1161 // So we enable this only on low-RAM devices. | 1161 // So we enable this only on low-RAM devices. |
| 1162 if (Heap::isLowEndDevice()) | 1162 if (Heap::isLowEndDevice()) |
| 1163 discardPages(startOfGap + sizeof(FreeListEntry), headerAddress); | 1163 discardPages(startOfGap + sizeof(FreeListEntry), headerAddress); |
| 1164 #endif | 1164 #endif |
| 1165 } | 1165 } |
| 1166 header->unmark(); | 1166 header->unmark(); |
| 1167 headerAddress += size; | 1167 headerAddress += size; |
| 1168 markedObjectSize += size; | 1168 markedObjectSize += size; |
| 1169 startOfGap = headerAddress; | 1169 startOfGap = headerAddress; |
| 1170 } | 1170 } |
| 1171 if (startOfGap != payloadEnd()) { | 1171 if (startOfGap != payloadEnd()) { |
| 1172 pageHeap->addToFreeList(startOfGap, payloadEnd() - startOfGap); | 1172 pageArena->addToFreeList(startOfGap, payloadEnd() - startOfGap); |
| 1173 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER) | 1173 #if !ENABLE(ASSERT) && !defined(LEAK_SANITIZER) && !defined(ADDRESS_SANITIZER) |
| 1174 if (Heap::isLowEndDevice()) | 1174 if (Heap::isLowEndDevice()) |
| 1175 discardPages(startOfGap + sizeof(FreeListEntry), payloadEnd()); | 1175 discardPages(startOfGap + sizeof(FreeListEntry), payloadEnd()); |
| 1176 #endif | 1176 #endif |
| 1177 } | 1177 } |
| 1178 | 1178 |
| 1179 if (markedObjectSize) | 1179 if (markedObjectSize) |
| 1180 pageHeap->threadState()->increaseMarkedObjectSize(markedObjectSize); | 1180 pageArena->threadState()->increaseMarkedObjectSize(markedObjectSize); |
| 1181 } | 1181 } |
| 1182 | 1182 |
| 1183 void NormalPage::makeConsistentForGC() | 1183 void NormalPage::makeConsistentForGC() |
| 1184 { | 1184 { |
| 1185 size_t markedObjectSize = 0; | 1185 size_t markedObjectSize = 0; |
| 1186 for (Address headerAddress = payload(); headerAddress < payloadEnd();) { | 1186 for (Address headerAddress = payload(); headerAddress < payloadEnd();) { |
| 1187 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAdd
ress); | 1187 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAdd
ress); |
| 1188 ASSERT(header->size() < blinkPagePayloadSize()); | 1188 ASSERT(header->size() < blinkPagePayloadSize()); |
| 1189 // Check if a free list entry first since we cannot call | 1189 // Check if a free list entry first since we cannot call |
| 1190 // isMarked on a free list entry. | 1190 // isMarked on a free list entry. |
| (...skipping 235 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1426 | 1426 |
| 1427 #if ENABLE(ASSERT) | 1427 #if ENABLE(ASSERT) |
| 1428 bool NormalPage::contains(Address addr) | 1428 bool NormalPage::contains(Address addr) |
| 1429 { | 1429 { |
| 1430 Address blinkPageStart = roundToBlinkPageStart(getAddress()); | 1430 Address blinkPageStart = roundToBlinkPageStart(getAddress()); |
| 1431 ASSERT(blinkPageStart == getAddress() - blinkGuardPageSize); // Page is at a
ligned address plus guard page size. | 1431 ASSERT(blinkPageStart == getAddress() - blinkGuardPageSize); // Page is at a
ligned address plus guard page size. |
| 1432 return blinkPageStart <= addr && addr < blinkPageStart + blinkPageSize; | 1432 return blinkPageStart <= addr && addr < blinkPageStart + blinkPageSize; |
| 1433 } | 1433 } |
| 1434 #endif | 1434 #endif |
| 1435 | 1435 |
| 1436 NormalPageHeap* NormalPage::arenaForNormalPage() | 1436 NormalPageArena* NormalPage::arenaForNormalPage() |
| 1437 { | 1437 { |
| 1438 return static_cast<NormalPageHeap*>(arena()); | 1438 return static_cast<NormalPageArena*>(arena()); |
| 1439 } | 1439 } |
| 1440 | 1440 |
| 1441 LargeObjectPage::LargeObjectPage(PageMemory* storage, BaseArena* arena, size_t p
ayloadSize) | 1441 LargeObjectPage::LargeObjectPage(PageMemory* storage, BaseArena* arena, size_t p
ayloadSize) |
| 1442 : BasePage(storage, arena) | 1442 : BasePage(storage, arena) |
| 1443 , m_payloadSize(payloadSize) | 1443 , m_payloadSize(payloadSize) |
| 1444 #if ENABLE(ASAN_CONTAINER_ANNOTATIONS) | 1444 #if ENABLE(ASAN_CONTAINER_ANNOTATIONS) |
| 1445 , m_isVectorBackingPage(false) | 1445 , m_isVectorBackingPage(false) |
| 1446 #endif | 1446 #endif |
| 1447 { | 1447 { |
| 1448 } | 1448 } |
| 1449 | 1449 |
| 1450 size_t LargeObjectPage::objectPayloadSizeForTesting() | 1450 size_t LargeObjectPage::objectPayloadSizeForTesting() |
| 1451 { | 1451 { |
| 1452 markAsSwept(); | 1452 markAsSwept(); |
| 1453 return payloadSize(); | 1453 return payloadSize(); |
| 1454 } | 1454 } |
| 1455 | 1455 |
| 1456 bool LargeObjectPage::isEmpty() | 1456 bool LargeObjectPage::isEmpty() |
| 1457 { | 1457 { |
| 1458 return !heapObjectHeader()->isMarked(); | 1458 return !heapObjectHeader()->isMarked(); |
| 1459 } | 1459 } |
| 1460 | 1460 |
| 1461 void LargeObjectPage::removeFromHeap() | 1461 void LargeObjectPage::removeFromHeap() |
| 1462 { | 1462 { |
| 1463 static_cast<LargeObjectHeap*>(arena())->freeLargeObjectPage(this); | 1463 static_cast<LargeObjectArena*>(arena())->freeLargeObjectPage(this); |
| 1464 } | 1464 } |
| 1465 | 1465 |
| 1466 void LargeObjectPage::sweep() | 1466 void LargeObjectPage::sweep() |
| 1467 { | 1467 { |
| 1468 heapObjectHeader()->unmark(); | 1468 heapObjectHeader()->unmark(); |
| 1469 arena()->threadState()->increaseMarkedObjectSize(size()); | 1469 arena()->threadState()->increaseMarkedObjectSize(size()); |
| 1470 } | 1470 } |
| 1471 | 1471 |
| 1472 void LargeObjectPage::makeConsistentForGC() | 1472 void LargeObjectPage::makeConsistentForGC() |
| 1473 { | 1473 { |
| (...skipping 114 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1588 | 1588 |
| 1589 m_hasEntries = true; | 1589 m_hasEntries = true; |
| 1590 size_t index = hash(address); | 1590 size_t index = hash(address); |
| 1591 ASSERT(!(index & 1)); | 1591 ASSERT(!(index & 1)); |
| 1592 Address cachePage = roundToBlinkPageStart(address); | 1592 Address cachePage = roundToBlinkPageStart(address); |
| 1593 m_entries[index + 1] = m_entries[index]; | 1593 m_entries[index + 1] = m_entries[index]; |
| 1594 m_entries[index] = cachePage; | 1594 m_entries[index] = cachePage; |
| 1595 } | 1595 } |
| 1596 | 1596 |
| 1597 } // namespace blink | 1597 } // namespace blink |
| OLD | NEW |