Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(346)

Side by Side Diff: Source/platform/heap/Heap.cpp

Issue 906213002: Oilpan: Rename heap classes (Closed) Base URL: svn://svn.chromium.org/blink/trunk
Patch Set: Created 5 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « Source/platform/heap/Heap.h ('k') | Source/platform/heap/HeapTest.cpp » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 /* 1 /*
2 * Copyright (C) 2013 Google Inc. All rights reserved. 2 * Copyright (C) 2013 Google Inc. All rights reserved.
3 * 3 *
4 * Redistribution and use in source and binary forms, with or without 4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are 5 * modification, are permitted provided that the following conditions are
6 * met: 6 * met:
7 * 7 *
8 * * Redistributions of source code must retain the above copyright 8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer. 9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above 10 * * Redistributions in binary form must reproduce the above
(...skipping 171 matching lines...) Expand 10 before | Expand all | Expand 10 after
182 static PageMemoryRegion* allocateLargePage(size_t size) 182 static PageMemoryRegion* allocateLargePage(size_t size)
183 { 183 {
184 return allocate(size, 1); 184 return allocate(size, 1);
185 } 185 }
186 186
187 static PageMemoryRegion* allocateNormalPages() 187 static PageMemoryRegion* allocateNormalPages()
188 { 188 {
189 return allocate(blinkPageSize * blinkPagesPerRegion, blinkPagesPerRegion ); 189 return allocate(blinkPageSize * blinkPagesPerRegion, blinkPagesPerRegion );
190 } 190 }
191 191
192 BaseHeapPage* pageFromAddress(Address address) 192 BasePage* pageFromAddress(Address address)
193 { 193 {
194 ASSERT(contains(address)); 194 ASSERT(contains(address));
195 if (!m_inUse[index(address)]) 195 if (!m_inUse[index(address)])
196 return nullptr; 196 return nullptr;
197 if (m_isLargePage) 197 if (m_isLargePage)
198 return pageFromObject(base()); 198 return pageFromObject(base());
199 return pageFromObject(address); 199 return pageFromObject(address);
200 } 200 }
201 201
202 private: 202 private:
(...skipping 267 matching lines...) Expand 10 before | Expand all | Expand 10 after
470 // Zap the primary vTable entry (secondary vTable entries are not zapped). 470 // Zap the primary vTable entry (secondary vTable entries are not zapped).
471 if (gcInfo->hasVTable()) { 471 if (gcInfo->hasVTable()) {
472 *(reinterpret_cast<uintptr_t*>(object)) = zappedVTable; 472 *(reinterpret_cast<uintptr_t*>(object)) = zappedVTable;
473 } 473 }
474 #endif 474 #endif
475 // In Release builds, the entire object is zeroed out when it is added to 475 // In Release builds, the entire object is zeroed out when it is added to
476 // the free list. This happens right after sweeping the page and before the 476 // the free list. This happens right after sweeping the page and before the
477 // thread commences execution. 477 // thread commences execution.
478 } 478 }
479 479
480 void LargeObject::sweep() 480 void LargeObjectPage::sweep()
481 { 481 {
482 Heap::increaseMarkedObjectSize(size()); 482 Heap::increaseMarkedObjectSize(size());
483 heapObjectHeader()->unmark(); 483 heapObjectHeader()->unmark();
484 } 484 }
485 485
486 bool LargeObject::isEmpty() 486 bool LargeObjectPage::isEmpty()
487 { 487 {
488 return !heapObjectHeader()->isMarked(); 488 return !heapObjectHeader()->isMarked();
489 } 489 }
490 490
491 #if ENABLE(ASSERT) 491 #if ENABLE(ASSERT)
492 static bool isUninitializedMemory(void* objectPointer, size_t objectSize) 492 static bool isUninitializedMemory(void* objectPointer, size_t objectSize)
493 { 493 {
494 // Scan through the object's fields and check that they are all zero. 494 // Scan through the object's fields and check that they are all zero.
495 Address* objectFields = reinterpret_cast<Address*>(objectPointer); 495 Address* objectFields = reinterpret_cast<Address*>(objectPointer);
496 for (size_t i = 0; i < objectSize / sizeof(Address); ++i) { 496 for (size_t i = 0; i < objectSize / sizeof(Address); ++i) {
497 if (objectFields[i] != 0) 497 if (objectFields[i] != 0)
498 return false; 498 return false;
499 } 499 }
500 return true; 500 return true;
501 } 501 }
502 #endif 502 #endif
503 503
504 static void markPointer(Visitor* visitor, HeapObjectHeader* header) 504 static void markPointer(Visitor* visitor, HeapObjectHeader* header)
505 { 505 {
506 const GCInfo* gcInfo = Heap::gcInfo(header->gcInfoIndex()); 506 const GCInfo* gcInfo = Heap::gcInfo(header->gcInfoIndex());
507 if (gcInfo->hasVTable() && !vTableInitialized(header->payload())) { 507 if (gcInfo->hasVTable() && !vTableInitialized(header->payload())) {
508 visitor->markHeaderNoTracing(header); 508 visitor->markHeaderNoTracing(header);
509 ASSERT(isUninitializedMemory(header->payload(), header->payloadSize())); 509 ASSERT(isUninitializedMemory(header->payload(), header->payloadSize()));
510 } else { 510 } else {
511 visitor->markHeader(header, gcInfo->m_trace); 511 visitor->markHeader(header, gcInfo->m_trace);
512 } 512 }
513 } 513 }
514 514
515 void LargeObject::checkAndMarkPointer(Visitor* visitor, Address address) 515 void LargeObjectPage::checkAndMarkPointer(Visitor* visitor, Address address)
516 { 516 {
517 ASSERT(contains(address)); 517 ASSERT(contains(address));
518 if (!containedInObjectPayload(address) || heapObjectHeader()->isDead()) 518 if (!containedInObjectPayload(address) || heapObjectHeader()->isDead())
519 return; 519 return;
520 #if ENABLE(GC_PROFILING) 520 #if ENABLE(GC_PROFILING)
521 visitor->setHostInfo(&address, "stack"); 521 visitor->setHostInfo(&address, "stack");
522 #endif 522 #endif
523 markPointer(visitor, heapObjectHeader()); 523 markPointer(visitor, heapObjectHeader());
524 } 524 }
525 525
526 void LargeObject::markUnmarkedObjectsDead() 526 void LargeObjectPage::markUnmarkedObjectsDead()
527 { 527 {
528 HeapObjectHeader* header = heapObjectHeader(); 528 HeapObjectHeader* header = heapObjectHeader();
529 if (header->isMarked()) 529 if (header->isMarked())
530 header->unmark(); 530 header->unmark();
531 else 531 else
532 header->markDead(); 532 header->markDead();
533 } 533 }
534 534
535 void LargeObject::removeFromHeap() 535 void LargeObjectPage::removeFromHeap()
536 { 536 {
537 static_cast<ThreadHeapForLargeObject*>(heap())->freeLargeObject(this); 537 static_cast<LargeObjectHeap*>(heap())->freeLargeObjectPage(this);
538 } 538 }
539 539
540 FreeList::FreeList() 540 FreeList::FreeList()
541 : m_biggestFreeListIndex(0) 541 : m_biggestFreeListIndex(0)
542 { 542 {
543 } 543 }
544 544
545 ThreadHeap::ThreadHeap(ThreadState* state, int index) 545 BaseHeap::BaseHeap(ThreadState* state, int index)
546 : m_firstPage(nullptr) 546 : m_firstPage(nullptr)
547 , m_firstUnsweptPage(nullptr) 547 , m_firstUnsweptPage(nullptr)
548 , m_threadState(state) 548 , m_threadState(state)
549 , m_index(index) 549 , m_index(index)
550 #if ENABLE(GC_PROFILING) 550 #if ENABLE(GC_PROFILING)
551 , m_cumulativeAllocationSize(0) 551 , m_cumulativeAllocationSize(0)
552 , m_allocationCount(0) 552 , m_allocationCount(0)
553 , m_inlineAllocationCount(0) 553 , m_inlineAllocationCount(0)
554 #endif 554 #endif
555 { 555 {
556 } 556 }
557 557
558 ThreadHeapForHeapPage::ThreadHeapForHeapPage(ThreadState* state, int index) 558 NormalPageHeap::NormalPageHeap(ThreadState* state, int index)
559 : ThreadHeap(state, index) 559 : BaseHeap(state, index)
560 , m_currentAllocationPoint(nullptr) 560 , m_currentAllocationPoint(nullptr)
561 , m_remainingAllocationSize(0) 561 , m_remainingAllocationSize(0)
562 , m_lastRemainingAllocationSize(0) 562 , m_lastRemainingAllocationSize(0)
563 , m_promptlyFreedSize(0) 563 , m_promptlyFreedSize(0)
564 { 564 {
565 clearFreeLists(); 565 clearFreeLists();
566 } 566 }
567 567
568 ThreadHeapForLargeObject::ThreadHeapForLargeObject(ThreadState* state, int index ) 568 LargeObjectHeap::LargeObjectHeap(ThreadState* state, int index)
569 : ThreadHeap(state, index) 569 : BaseHeap(state, index)
570 { 570 {
571 } 571 }
572 572
573 ThreadHeap::~ThreadHeap() 573 BaseHeap::~BaseHeap()
574 { 574 {
575 ASSERT(!m_firstPage); 575 ASSERT(!m_firstPage);
576 ASSERT(!m_firstUnsweptPage); 576 ASSERT(!m_firstUnsweptPage);
577 } 577 }
578 578
579 void ThreadHeap::cleanupPages() 579 void BaseHeap::cleanupPages()
580 { 580 {
581 clearFreeLists(); 581 clearFreeLists();
582 582
583 ASSERT(!m_firstUnsweptPage); 583 ASSERT(!m_firstUnsweptPage);
584 // Add the ThreadHeap's pages to the orphanedPagePool. 584 // Add the BaseHeap's pages to the orphanedPagePool.
585 for (BaseHeapPage* page = m_firstPage; page; page = page->next()) { 585 for (BasePage* page = m_firstPage; page; page = page->next()) {
586 Heap::decreaseAllocatedSpace(page->size()); 586 Heap::decreaseAllocatedSpace(page->size());
587 Heap::orphanedPagePool()->addOrphanedPage(heapIndex(), page); 587 Heap::orphanedPagePool()->addOrphanedPage(heapIndex(), page);
588 } 588 }
589 m_firstPage = nullptr; 589 m_firstPage = nullptr;
590 } 590 }
591 591
592 void ThreadHeapForHeapPage::updateRemainingAllocationSize() 592 void NormalPageHeap::updateRemainingAllocationSize()
593 { 593 {
594 if (m_lastRemainingAllocationSize > remainingAllocationSize()) { 594 if (m_lastRemainingAllocationSize > remainingAllocationSize()) {
595 Heap::increaseAllocatedObjectSize(m_lastRemainingAllocationSize - remain ingAllocationSize()); 595 Heap::increaseAllocatedObjectSize(m_lastRemainingAllocationSize - remain ingAllocationSize());
596 m_lastRemainingAllocationSize = remainingAllocationSize(); 596 m_lastRemainingAllocationSize = remainingAllocationSize();
597 } 597 }
598 ASSERT(m_lastRemainingAllocationSize == remainingAllocationSize()); 598 ASSERT(m_lastRemainingAllocationSize == remainingAllocationSize());
599 } 599 }
600 600
601 void ThreadHeapForHeapPage::setAllocationPoint(Address point, size_t size) 601 void NormalPageHeap::setAllocationPoint(Address point, size_t size)
602 { 602 {
603 #if ENABLE(ASSERT) 603 #if ENABLE(ASSERT)
604 if (point) { 604 if (point) {
605 ASSERT(size); 605 ASSERT(size);
606 BaseHeapPage* page = pageFromObject(point); 606 BasePage* page = pageFromObject(point);
607 ASSERT(!page->isLargeObject()); 607 ASSERT(!page->isLargeObjectPage());
608 ASSERT(size <= static_cast<HeapPage*>(page)->payloadSize()); 608 ASSERT(size <= static_cast<NormalPage*>(page)->payloadSize());
609 } 609 }
610 #endif 610 #endif
611 if (hasCurrentAllocationArea()) { 611 if (hasCurrentAllocationArea()) {
612 addToFreeList(currentAllocationPoint(), remainingAllocationSize()); 612 addToFreeList(currentAllocationPoint(), remainingAllocationSize());
613 } 613 }
614 updateRemainingAllocationSize(); 614 updateRemainingAllocationSize();
615 m_currentAllocationPoint = point; 615 m_currentAllocationPoint = point;
616 m_lastRemainingAllocationSize = m_remainingAllocationSize = size; 616 m_lastRemainingAllocationSize = m_remainingAllocationSize = size;
617 } 617 }
618 618
619 Address ThreadHeapForHeapPage::outOfLineAllocate(size_t allocationSize, size_t g cInfoIndex) 619 Address NormalPageHeap::outOfLineAllocate(size_t allocationSize, size_t gcInfoIn dex)
620 { 620 {
621 ASSERT(allocationSize > remainingAllocationSize()); 621 ASSERT(allocationSize > remainingAllocationSize());
622 ASSERT(allocationSize >= allocationGranularity); 622 ASSERT(allocationSize >= allocationGranularity);
623 623
624 #if ENABLE(GC_PROFILING) 624 #if ENABLE(GC_PROFILING)
625 m_threadState->snapshotFreeListIfNecessary(); 625 m_threadState->snapshotFreeListIfNecessary();
626 #endif 626 #endif
627 627
628 // 1. If this allocation is big enough, allocate a large object. 628 // 1. If this allocation is big enough, allocate a large object.
629 if (allocationSize >= largeObjectSizeThreshold) 629 if (allocationSize >= largeObjectSizeThreshold)
630 return static_cast<ThreadHeapForLargeObject*>(threadState()->heap(LargeO bjectHeap))->allocateLargeObject(allocationSize, gcInfoIndex); 630 return static_cast<LargeObjectHeap*>(threadState()->heap(LargeObjectHeap Index))->allocateLargeObjectPage(allocationSize, gcInfoIndex);
631 631
632 // 2. Check if we should trigger a GC. 632 // 2. Check if we should trigger a GC.
633 updateRemainingAllocationSize(); 633 updateRemainingAllocationSize();
634 threadState()->scheduleGCIfNeeded(); 634 threadState()->scheduleGCIfNeeded();
635 635
636 // 3. Try to allocate from a free list. 636 // 3. Try to allocate from a free list.
637 Address result = allocateFromFreeList(allocationSize, gcInfoIndex); 637 Address result = allocateFromFreeList(allocationSize, gcInfoIndex);
638 if (result) 638 if (result)
639 return result; 639 return result;
640 640
(...skipping 19 matching lines...) Expand all
660 660
661 // 8. Add a new page to this heap. 661 // 8. Add a new page to this heap.
662 allocatePage(); 662 allocatePage();
663 663
664 // 9. Try to allocate from a free list. This allocation must succeed. 664 // 9. Try to allocate from a free list. This allocation must succeed.
665 result = allocateFromFreeList(allocationSize, gcInfoIndex); 665 result = allocateFromFreeList(allocationSize, gcInfoIndex);
666 RELEASE_ASSERT(result); 666 RELEASE_ASSERT(result);
667 return result; 667 return result;
668 } 668 }
669 669
670 Address ThreadHeapForHeapPage::allocateFromFreeList(size_t allocationSize, size_ t gcInfoIndex) 670 Address NormalPageHeap::allocateFromFreeList(size_t allocationSize, size_t gcInf oIndex)
671 { 671 {
672 // Try reusing a block from the largest bin. The underlying reasoning 672 // Try reusing a block from the largest bin. The underlying reasoning
673 // being that we want to amortize this slow allocation call by carving 673 // being that we want to amortize this slow allocation call by carving
674 // off as a large a free block as possible in one go; a block that will 674 // off as a large a free block as possible in one go; a block that will
675 // service this block and let following allocations be serviced quickly 675 // service this block and let following allocations be serviced quickly
676 // by bump allocation. 676 // by bump allocation.
677 size_t bucketSize = 1 << m_freeList.m_biggestFreeListIndex; 677 size_t bucketSize = 1 << m_freeList.m_biggestFreeListIndex;
678 int index = m_freeList.m_biggestFreeListIndex; 678 int index = m_freeList.m_biggestFreeListIndex;
679 for (; index > 0; --index, bucketSize >>= 1) { 679 for (; index > 0; --index, bucketSize >>= 1) {
680 FreeListEntry* entry = m_freeList.m_freeLists[index]; 680 FreeListEntry* entry = m_freeList.m_freeLists[index];
(...skipping 10 matching lines...) Expand all
691 ASSERT(hasCurrentAllocationArea()); 691 ASSERT(hasCurrentAllocationArea());
692 ASSERT(remainingAllocationSize() >= allocationSize); 692 ASSERT(remainingAllocationSize() >= allocationSize);
693 m_freeList.m_biggestFreeListIndex = index; 693 m_freeList.m_biggestFreeListIndex = index;
694 return allocateObject(allocationSize, gcInfoIndex); 694 return allocateObject(allocationSize, gcInfoIndex);
695 } 695 }
696 } 696 }
697 m_freeList.m_biggestFreeListIndex = index; 697 m_freeList.m_biggestFreeListIndex = index;
698 return nullptr; 698 return nullptr;
699 } 699 }
700 700
701 void ThreadHeap::prepareForSweep() 701 void BaseHeap::prepareForSweep()
702 { 702 {
703 ASSERT(!threadState()->isInGC()); 703 ASSERT(!threadState()->isInGC());
704 ASSERT(!m_firstUnsweptPage); 704 ASSERT(!m_firstUnsweptPage);
705 705
706 // Move all pages to a list of unswept pages. 706 // Move all pages to a list of unswept pages.
707 m_firstUnsweptPage = m_firstPage; 707 m_firstUnsweptPage = m_firstPage;
708 m_firstPage = nullptr; 708 m_firstPage = nullptr;
709 } 709 }
710 710
711 Address ThreadHeap::lazySweep(size_t allocationSize, size_t gcInfoIndex) 711 Address BaseHeap::lazySweep(size_t allocationSize, size_t gcInfoIndex)
712 { 712 {
713 // If there are no pages to be swept, return immediately. 713 // If there are no pages to be swept, return immediately.
714 if (!m_firstUnsweptPage) 714 if (!m_firstUnsweptPage)
715 return nullptr; 715 return nullptr;
716 716
717 RELEASE_ASSERT(threadState()->isSweepingInProgress()); 717 RELEASE_ASSERT(threadState()->isSweepingInProgress());
718 718
719 // lazySweepPages() can be called recursively if finalizers invoked in 719 // lazySweepPages() can be called recursively if finalizers invoked in
720 // page->sweep() allocate memory and the allocation triggers 720 // page->sweep() allocate memory and the allocation triggers
721 // lazySweepPages(). This check prevents the sweeping from being executed 721 // lazySweepPages(). This check prevents the sweeping from being executed
722 // recursively. 722 // recursively.
723 if (threadState()->sweepForbidden()) 723 if (threadState()->sweepForbidden())
724 return nullptr; 724 return nullptr;
725 725
726 TRACE_EVENT0("blink_gc", "ThreadHeap::lazySweepPages"); 726 TRACE_EVENT0("blink_gc", "BaseHeap::lazySweepPages");
727 ThreadState::SweepForbiddenScope scope(threadState()); 727 ThreadState::SweepForbiddenScope scope(threadState());
728 728
729 if (threadState()->isMainThread()) 729 if (threadState()->isMainThread())
730 ScriptForbiddenScope::enter(); 730 ScriptForbiddenScope::enter();
731 731
732 Address result = lazySweepPages(allocationSize, gcInfoIndex); 732 Address result = lazySweepPages(allocationSize, gcInfoIndex);
733 733
734 if (threadState()->isMainThread()) 734 if (threadState()->isMainThread())
735 ScriptForbiddenScope::exit(); 735 ScriptForbiddenScope::exit();
736 return result; 736 return result;
737 } 737 }
738 738
739 Address ThreadHeapForHeapPage::lazySweepPages(size_t allocationSize, size_t gcIn foIndex) 739 Address NormalPageHeap::lazySweepPages(size_t allocationSize, size_t gcInfoIndex )
740 { 740 {
741 ASSERT(!hasCurrentAllocationArea()); 741 ASSERT(!hasCurrentAllocationArea());
742 Address result = nullptr; 742 Address result = nullptr;
743 while (m_firstUnsweptPage) { 743 while (m_firstUnsweptPage) {
744 BaseHeapPage* page = m_firstUnsweptPage; 744 BasePage* page = m_firstUnsweptPage;
745 if (page->isEmpty()) { 745 if (page->isEmpty()) {
746 page->unlink(&m_firstUnsweptPage); 746 page->unlink(&m_firstUnsweptPage);
747 page->removeFromHeap(); 747 page->removeFromHeap();
748 } else { 748 } else {
749 // Sweep a page and move the page from m_firstUnsweptPages to 749 // Sweep a page and move the page from m_firstUnsweptPages to
750 // m_firstPages. 750 // m_firstPages.
751 page->sweep(); 751 page->sweep();
752 page->unlink(&m_firstUnsweptPage); 752 page->unlink(&m_firstUnsweptPage);
753 page->link(&m_firstPage); 753 page->link(&m_firstPage);
754 page->markAsSwept(); 754 page->markAsSwept();
755 755
756 // For HeapPage, stop lazy sweeping once we find a slot to 756 // For NormalPage, stop lazy sweeping once we find a slot to
757 // allocate a new object. 757 // allocate a new object.
758 result = allocateFromFreeList(allocationSize, gcInfoIndex); 758 result = allocateFromFreeList(allocationSize, gcInfoIndex);
759 if (result) 759 if (result)
760 break; 760 break;
761 } 761 }
762 } 762 }
763 return result; 763 return result;
764 } 764 }
765 765
766 Address ThreadHeapForLargeObject::lazySweepPages(size_t allocationSize, size_t g cInfoIndex) 766 Address LargeObjectHeap::lazySweepPages(size_t allocationSize, size_t gcInfoInde x)
767 { 767 {
768 Address result = nullptr; 768 Address result = nullptr;
769 size_t sweptSize = 0; 769 size_t sweptSize = 0;
770 while (m_firstUnsweptPage) { 770 while (m_firstUnsweptPage) {
771 BaseHeapPage* page = m_firstUnsweptPage; 771 BasePage* page = m_firstUnsweptPage;
772 if (page->isEmpty()) { 772 if (page->isEmpty()) {
773 sweptSize += static_cast<LargeObject*>(page)->payloadSize() + sizeof (HeapObjectHeader); 773 sweptSize += static_cast<LargeObjectPage*>(page)->payloadSize() + si zeof(HeapObjectHeader);
774 page->unlink(&m_firstUnsweptPage); 774 page->unlink(&m_firstUnsweptPage);
775 page->removeFromHeap(); 775 page->removeFromHeap();
776 // For LargeObject, stop lazy sweeping once we have swept 776 // For LargeObjectPage, stop lazy sweeping once we have swept
777 // more than allocationSize bytes. 777 // more than allocationSize bytes.
778 if (sweptSize >= allocationSize) { 778 if (sweptSize >= allocationSize) {
779 result = doAllocateLargeObject(allocationSize, gcInfoIndex); 779 result = doAllocateLargeObjectPage(allocationSize, gcInfoIndex);
780 ASSERT(result); 780 ASSERT(result);
781 break; 781 break;
782 } 782 }
783 } else { 783 } else {
784 // Sweep a page and move the page from m_firstUnsweptPages to 784 // Sweep a page and move the page from m_firstUnsweptPages to
785 // m_firstPages. 785 // m_firstPages.
786 page->sweep(); 786 page->sweep();
787 page->unlink(&m_firstUnsweptPage); 787 page->unlink(&m_firstUnsweptPage);
788 page->link(&m_firstPage); 788 page->link(&m_firstPage);
789 page->markAsSwept(); 789 page->markAsSwept();
790 } 790 }
791 } 791 }
792 return result; 792 return result;
793 } 793 }
794 794
795 void ThreadHeap::completeSweep() 795 void BaseHeap::completeSweep()
796 { 796 {
797 RELEASE_ASSERT(threadState()->isSweepingInProgress()); 797 RELEASE_ASSERT(threadState()->isSweepingInProgress());
798 ASSERT(threadState()->sweepForbidden()); 798 ASSERT(threadState()->sweepForbidden());
799 799
800 if (threadState()->isMainThread()) 800 if (threadState()->isMainThread())
801 ScriptForbiddenScope::enter(); 801 ScriptForbiddenScope::enter();
802 802
803 while (m_firstUnsweptPage) { 803 while (m_firstUnsweptPage) {
804 BaseHeapPage* page = m_firstUnsweptPage; 804 BasePage* page = m_firstUnsweptPage;
805 if (page->isEmpty()) { 805 if (page->isEmpty()) {
806 page->unlink(&m_firstUnsweptPage); 806 page->unlink(&m_firstUnsweptPage);
807 page->removeFromHeap(); 807 page->removeFromHeap();
808 } else { 808 } else {
809 // Sweep a page and move the page from m_firstUnsweptPages to 809 // Sweep a page and move the page from m_firstUnsweptPages to
810 // m_firstPages. 810 // m_firstPages.
811 page->sweep(); 811 page->sweep();
812 page->unlink(&m_firstUnsweptPage); 812 page->unlink(&m_firstUnsweptPage);
813 page->link(&m_firstPage); 813 page->link(&m_firstPage);
814 page->markAsSwept(); 814 page->markAsSwept();
815 } 815 }
816 } 816 }
817 817
818 if (threadState()->isMainThread()) 818 if (threadState()->isMainThread())
819 ScriptForbiddenScope::exit(); 819 ScriptForbiddenScope::exit();
820 } 820 }
821 821
822 #if ENABLE(ASSERT) 822 #if ENABLE(ASSERT)
823 BaseHeapPage* ThreadHeap::findPageFromAddress(Address address) 823 BasePage* BaseHeap::findPageFromAddress(Address address)
824 { 824 {
825 for (BaseHeapPage* page = m_firstPage; page; page = page->next()) { 825 for (BasePage* page = m_firstPage; page; page = page->next()) {
826 if (page->contains(address)) 826 if (page->contains(address))
827 return page; 827 return page;
828 } 828 }
829 for (BaseHeapPage* page = m_firstUnsweptPage; page; page = page->next()) { 829 for (BasePage* page = m_firstUnsweptPage; page; page = page->next()) {
830 if (page->contains(address)) 830 if (page->contains(address))
831 return page; 831 return page;
832 } 832 }
833 return nullptr; 833 return nullptr;
834 } 834 }
835 #endif 835 #endif
836 836
837 #if ENABLE(GC_PROFILING) 837 #if ENABLE(GC_PROFILING)
838 #define GC_PROFILE_HEAP_PAGE_SNAPSHOT_THRESHOLD 0 838 #define GC_PROFILE_HEAP_PAGE_SNAPSHOT_THRESHOLD 0
839 void ThreadHeap::snapshot(TracedValue* json, ThreadState::SnapshotInfo* info) 839 void BaseHeap::snapshot(TracedValue* json, ThreadState::SnapshotInfo* info)
840 { 840 {
841 ASSERT(isConsistentForSweeping()); 841 ASSERT(isConsistentForSweeping());
842 size_t previousPageCount = info->pageCount; 842 size_t previousPageCount = info->pageCount;
843 843
844 json->beginArray("pages"); 844 json->beginArray("pages");
845 for (BaseHeapPage* page = m_firstPage; page; page = page->next(), ++info->pa geCount) { 845 for (BasePage* page = m_firstPage; page; page = page->next(), ++info->pageCo unt) {
846 // FIXME: To limit the size of the snapshot we only output "threshold" m any page snapshots. 846 // FIXME: To limit the size of the snapshot we only output "threshold" m any page snapshots.
847 if (info->pageCount < GC_PROFILE_HEAP_PAGE_SNAPSHOT_THRESHOLD) { 847 if (info->pageCount < GC_PROFILE_HEAP_PAGE_SNAPSHOT_THRESHOLD) {
848 json->beginArray(); 848 json->beginArray();
849 json->pushInteger(reinterpret_cast<intptr_t>(page)); 849 json->pushInteger(reinterpret_cast<intptr_t>(page));
850 page->snapshot(json, info); 850 page->snapshot(json, info);
851 json->endArray(); 851 json->endArray();
852 } else { 852 } else {
853 page->snapshot(0, info); 853 page->snapshot(0, info);
854 } 854 }
855 } 855 }
856 json->endArray(); 856 json->endArray();
857 857
858 json->setInteger("pageCount", info->pageCount - previousPageCount); 858 json->setInteger("pageCount", info->pageCount - previousPageCount);
859 } 859 }
860 860
861 void ThreadHeap::incrementMarkedObjectsAge() 861 void BaseHeap::incrementMarkedObjectsAge()
862 { 862 {
863 for (HeapPage* page = m_firstPage; page; page = page->next()) 863 for (NormalPage* page = m_firstPage; page; page = page->next())
864 page->incrementMarkedObjectsAge(); 864 page->incrementMarkedObjectsAge();
865 for (LargeObject* largeObject = m_firstLargeObject; largeObject; largeObject = largeObject->next()) 865 for (LargeObjectPage* largeObject = m_firstLargeObjectPage; largeObject; lar geObject = largeObject->next())
866 largeObject->incrementMarkedObjectsAge(); 866 largeObject->incrementMarkedObjectsAge();
867 } 867 }
868 #endif 868 #endif
869 869
870 void FreeList::addToFreeList(Address address, size_t size) 870 void FreeList::addToFreeList(Address address, size_t size)
871 { 871 {
872 ASSERT(size < blinkPagePayloadSize()); 872 ASSERT(size < blinkPagePayloadSize());
873 // The free list entries are only pointer aligned (but when we allocate 873 // The free list entries are only pointer aligned (but when we allocate
874 // from them we are 8 byte aligned due to the header size). 874 // from them we are 8 byte aligned due to the header size).
875 ASSERT(!((reinterpret_cast<uintptr_t>(address) + sizeof(HeapObjectHeader)) & allocationMask)); 875 ASSERT(!((reinterpret_cast<uintptr_t>(address) + sizeof(HeapObjectHeader)) & allocationMask));
876 ASSERT(!(size & allocationMask)); 876 ASSERT(!(size & allocationMask));
877 ASAN_POISON_MEMORY_REGION(address, size); 877 ASAN_POISON_MEMORY_REGION(address, size);
878 FreeListEntry* entry; 878 FreeListEntry* entry;
879 if (size < sizeof(*entry)) { 879 if (size < sizeof(*entry)) {
880 // Create a dummy header with only a size and freelist bit set. 880 // Create a dummy header with only a size and freelist bit set.
881 ASSERT(size >= sizeof(HeapObjectHeader)); 881 ASSERT(size >= sizeof(HeapObjectHeader));
882 // Free list encode the size to mark the lost memory as freelist memory. 882 // Free list encode the size to mark the lost memory as freelist memory.
883 new (NotNull, address) HeapObjectHeader(size, gcInfoIndexForFreeListHead er); 883 new (NotNull, address) HeapObjectHeader(size, gcInfoIndexForFreeListHead er);
884 // This memory gets lost. Sweeping can reclaim it. 884 // This memory gets lost. Sweeping can reclaim it.
885 return; 885 return;
886 } 886 }
887 entry = new (NotNull, address) FreeListEntry(size); 887 entry = new (NotNull, address) FreeListEntry(size);
888 #if defined(ADDRESS_SANITIZER) 888 #if defined(ADDRESS_SANITIZER)
889 BaseHeapPage* page = pageFromObject(address); 889 BasePage* page = pageFromObject(address);
890 ASSERT(!page->isLargeObject()); 890 ASSERT(!page->isLargeObjectPage());
891 // For ASan we don't add the entry to the free lists until the 891 // For ASan we don't add the entry to the free lists until the
892 // asanDeferMemoryReuseCount reaches zero. However we always add entire 892 // asanDeferMemoryReuseCount reaches zero. However we always add entire
893 // pages to ensure that adding a new page will increase the allocation 893 // pages to ensure that adding a new page will increase the allocation
894 // space. 894 // space.
895 if (static_cast<HeapPage*>(page)->payloadSize() != size && !entry->shouldAdd ToFreeList()) 895 if (static_cast<NormalPage*>(page)->payloadSize() != size && !entry->shouldA ddToFreeList())
896 return; 896 return;
897 #endif 897 #endif
898 int index = bucketIndexForSize(size); 898 int index = bucketIndexForSize(size);
899 entry->link(&m_freeLists[index]); 899 entry->link(&m_freeLists[index]);
900 if (index > m_biggestFreeListIndex) 900 if (index > m_biggestFreeListIndex)
901 m_biggestFreeListIndex = index; 901 m_biggestFreeListIndex = index;
902 } 902 }
903 903
904 bool ThreadHeapForHeapPage::expandObject(HeapObjectHeader* header, size_t newSiz e) 904 bool NormalPageHeap::expandObject(HeapObjectHeader* header, size_t newSize)
905 { 905 {
906 // It's possible that Vector requests a smaller expanded size because 906 // It's possible that Vector requests a smaller expanded size because
907 // Vector::shrinkCapacity can set a capacity smaller than the actual payload 907 // Vector::shrinkCapacity can set a capacity smaller than the actual payload
908 // size. 908 // size.
909 if (header->payloadSize() >= newSize) 909 if (header->payloadSize() >= newSize)
910 return true; 910 return true;
911 size_t allocationSize = allocationSizeFromSize(newSize); 911 size_t allocationSize = allocationSizeFromSize(newSize);
912 ASSERT(allocationSize > header->size()); 912 ASSERT(allocationSize > header->size());
913 size_t expandSize = allocationSize - header->size(); 913 size_t expandSize = allocationSize - header->size();
914 if (header->payloadEnd() == m_currentAllocationPoint && expandSize <= m_rema iningAllocationSize) { 914 if (header->payloadEnd() == m_currentAllocationPoint && expandSize <= m_rema iningAllocationSize) {
915 m_currentAllocationPoint += expandSize; 915 m_currentAllocationPoint += expandSize;
916 m_remainingAllocationSize -= expandSize; 916 m_remainingAllocationSize -= expandSize;
917 917
918 // Unpoison the memory used for the object (payload). 918 // Unpoison the memory used for the object (payload).
919 ASAN_UNPOISON_MEMORY_REGION(header->payloadEnd(), expandSize); 919 ASAN_UNPOISON_MEMORY_REGION(header->payloadEnd(), expandSize);
920 FILL_ZERO_IF_NOT_PRODUCTION(header->payloadEnd(), expandSize); 920 FILL_ZERO_IF_NOT_PRODUCTION(header->payloadEnd(), expandSize);
921 header->setSize(allocationSize); 921 header->setSize(allocationSize);
922 ASSERT(findPageFromAddress(header->payloadEnd() - 1)); 922 ASSERT(findPageFromAddress(header->payloadEnd() - 1));
923 return true; 923 return true;
924 } 924 }
925 return false; 925 return false;
926 } 926 }
927 927
928 void ThreadHeapForHeapPage::shrinkObject(HeapObjectHeader* header, size_t newSiz e) 928 void NormalPageHeap::shrinkObject(HeapObjectHeader* header, size_t newSize)
929 { 929 {
930 ASSERT(header->payloadSize() > newSize); 930 ASSERT(header->payloadSize() > newSize);
931 size_t allocationSize = allocationSizeFromSize(newSize); 931 size_t allocationSize = allocationSizeFromSize(newSize);
932 ASSERT(header->size() > allocationSize); 932 ASSERT(header->size() > allocationSize);
933 size_t shrinkSize = header->size() - allocationSize; 933 size_t shrinkSize = header->size() - allocationSize;
934 if (header->payloadEnd() == m_currentAllocationPoint) { 934 if (header->payloadEnd() == m_currentAllocationPoint) {
935 m_currentAllocationPoint -= shrinkSize; 935 m_currentAllocationPoint -= shrinkSize;
936 m_remainingAllocationSize += shrinkSize; 936 m_remainingAllocationSize += shrinkSize;
937 FILL_ZERO_IF_PRODUCTION(m_currentAllocationPoint, shrinkSize); 937 FILL_ZERO_IF_PRODUCTION(m_currentAllocationPoint, shrinkSize);
938 ASAN_POISON_MEMORY_REGION(m_currentAllocationPoint, shrinkSize); 938 ASAN_POISON_MEMORY_REGION(m_currentAllocationPoint, shrinkSize);
939 header->setSize(allocationSize); 939 header->setSize(allocationSize);
940 } else { 940 } else {
941 ASSERT(shrinkSize >= sizeof(HeapObjectHeader)); 941 ASSERT(shrinkSize >= sizeof(HeapObjectHeader));
942 ASSERT(header->gcInfoIndex() > 0); 942 ASSERT(header->gcInfoIndex() > 0);
943 HeapObjectHeader* freedHeader = new (NotNull, header->payloadEnd() - shr inkSize) HeapObjectHeader(shrinkSize, header->gcInfoIndex()); 943 HeapObjectHeader* freedHeader = new (NotNull, header->payloadEnd() - shr inkSize) HeapObjectHeader(shrinkSize, header->gcInfoIndex());
944 freedHeader->markPromptlyFreed(); 944 freedHeader->markPromptlyFreed();
945 ASSERT(pageFromObject(reinterpret_cast<Address>(header)) == findPageFrom Address(reinterpret_cast<Address>(header))); 945 ASSERT(pageFromObject(reinterpret_cast<Address>(header)) == findPageFrom Address(reinterpret_cast<Address>(header)));
946 m_promptlyFreedSize += shrinkSize; 946 m_promptlyFreedSize += shrinkSize;
947 header->setSize(allocationSize); 947 header->setSize(allocationSize);
948 } 948 }
949 } 949 }
950 950
951 void ThreadHeapForHeapPage::promptlyFreeObject(HeapObjectHeader* header) 951 void NormalPageHeap::promptlyFreeObject(HeapObjectHeader* header)
952 { 952 {
953 ASSERT(!threadState()->sweepForbidden()); 953 ASSERT(!threadState()->sweepForbidden());
954 header->checkHeader(); 954 header->checkHeader();
955 Address address = reinterpret_cast<Address>(header); 955 Address address = reinterpret_cast<Address>(header);
956 Address payload = header->payload(); 956 Address payload = header->payload();
957 size_t size = header->size(); 957 size_t size = header->size();
958 size_t payloadSize = header->payloadSize(); 958 size_t payloadSize = header->payloadSize();
959 ASSERT(size > 0); 959 ASSERT(size > 0);
960 ASSERT(pageFromObject(address) == findPageFromAddress(address)); 960 ASSERT(pageFromObject(address) == findPageFromAddress(address));
961 961
(...skipping 11 matching lines...) Expand all
973 ASAN_POISON_MEMORY_REGION(address, size); 973 ASAN_POISON_MEMORY_REGION(address, size);
974 return; 974 return;
975 } 975 }
976 FILL_ZERO_IF_PRODUCTION(payload, payloadSize); 976 FILL_ZERO_IF_PRODUCTION(payload, payloadSize);
977 header->markPromptlyFreed(); 977 header->markPromptlyFreed();
978 } 978 }
979 979
980 m_promptlyFreedSize += size; 980 m_promptlyFreedSize += size;
981 } 981 }
982 982
983 bool ThreadHeapForHeapPage::coalesce() 983 bool NormalPageHeap::coalesce()
984 { 984 {
985 // Don't coalesce heaps if there are not enough promptly freed entries 985 // Don't coalesce heaps if there are not enough promptly freed entries
986 // to be coalesced. 986 // to be coalesced.
987 // 987 //
988 // FIXME: This threshold is determined just to optimize blink_perf 988 // FIXME: This threshold is determined just to optimize blink_perf
989 // benchmarks. Coalescing is very sensitive to the threashold and 989 // benchmarks. Coalescing is very sensitive to the threashold and
990 // we need further investigations on the coalescing scheme. 990 // we need further investigations on the coalescing scheme.
991 if (m_promptlyFreedSize < 1024 * 1024) 991 if (m_promptlyFreedSize < 1024 * 1024)
992 return false; 992 return false;
993 993
994 if (threadState()->sweepForbidden()) 994 if (threadState()->sweepForbidden())
995 return false; 995 return false;
996 996
997 ASSERT(!hasCurrentAllocationArea()); 997 ASSERT(!hasCurrentAllocationArea());
998 TRACE_EVENT0("blink_gc", "ThreadHeap::coalesce"); 998 TRACE_EVENT0("blink_gc", "BaseHeap::coalesce");
999 999
1000 // Rebuild free lists. 1000 // Rebuild free lists.
1001 m_freeList.clear(); 1001 m_freeList.clear();
1002 size_t freedSize = 0; 1002 size_t freedSize = 0;
1003 for (HeapPage* page = static_cast<HeapPage*>(m_firstPage); page; page = stat ic_cast<HeapPage*>(page->next())) { 1003 for (NormalPage* page = static_cast<NormalPage*>(m_firstPage); page; page = static_cast<NormalPage*>(page->next())) {
1004 page->clearObjectStartBitMap(); 1004 page->clearObjectStartBitMap();
1005 Address startOfGap = page->payload(); 1005 Address startOfGap = page->payload();
1006 for (Address headerAddress = startOfGap; headerAddress < page->payloadEn d(); ) { 1006 for (Address headerAddress = startOfGap; headerAddress < page->payloadEn d(); ) {
1007 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(heade rAddress); 1007 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(heade rAddress);
1008 size_t size = header->size(); 1008 size_t size = header->size();
1009 ASSERT(size > 0); 1009 ASSERT(size > 0);
1010 ASSERT(size < blinkPagePayloadSize()); 1010 ASSERT(size < blinkPagePayloadSize());
1011 1011
1012 if (header->isPromptlyFreed()) { 1012 if (header->isPromptlyFreed()) {
1013 ASSERT(size >= sizeof(HeapObjectHeader)); 1013 ASSERT(size >= sizeof(HeapObjectHeader));
(...skipping 20 matching lines...) Expand all
1034 1034
1035 if (startOfGap != page->payloadEnd()) 1035 if (startOfGap != page->payloadEnd())
1036 addToFreeList(startOfGap, page->payloadEnd() - startOfGap); 1036 addToFreeList(startOfGap, page->payloadEnd() - startOfGap);
1037 } 1037 }
1038 Heap::decreaseAllocatedObjectSize(freedSize); 1038 Heap::decreaseAllocatedObjectSize(freedSize);
1039 ASSERT(m_promptlyFreedSize == freedSize); 1039 ASSERT(m_promptlyFreedSize == freedSize);
1040 m_promptlyFreedSize = 0; 1040 m_promptlyFreedSize = 0;
1041 return true; 1041 return true;
1042 } 1042 }
1043 1043
1044 Address ThreadHeapForLargeObject::allocateLargeObject(size_t allocationSize, siz e_t gcInfoIndex) 1044 Address LargeObjectHeap::allocateLargeObjectPage(size_t allocationSize, size_t g cInfoIndex)
1045 { 1045 {
1046 // Caller already added space for object header and rounded up to allocation 1046 // Caller already added space for object header and rounded up to allocation
1047 // alignment 1047 // alignment
1048 ASSERT(!(allocationSize & allocationMask)); 1048 ASSERT(!(allocationSize & allocationMask));
1049 1049
1050 // 1. Check if we should trigger a GC. 1050 // 1. Check if we should trigger a GC.
1051 threadState()->scheduleGCIfNeeded(); 1051 threadState()->scheduleGCIfNeeded();
1052 1052
1053 // 2. Try to sweep large objects more than allocationSize bytes 1053 // 2. Try to sweep large objects more than allocationSize bytes
1054 // before allocating a new large object. 1054 // before allocating a new large object.
1055 Address result = lazySweep(allocationSize, gcInfoIndex); 1055 Address result = lazySweep(allocationSize, gcInfoIndex);
1056 if (result) 1056 if (result)
1057 return result; 1057 return result;
1058 1058
1059 // 3. If we have failed in sweeping allocationSize bytes, 1059 // 3. If we have failed in sweeping allocationSize bytes,
1060 // we complete sweeping before allocating this large object. 1060 // we complete sweeping before allocating this large object.
1061 threadState()->completeSweep(); 1061 threadState()->completeSweep();
1062 return doAllocateLargeObject(allocationSize, gcInfoIndex); 1062 return doAllocateLargeObjectPage(allocationSize, gcInfoIndex);
1063 } 1063 }
1064 1064
1065 Address ThreadHeapForLargeObject::doAllocateLargeObject(size_t allocationSize, s ize_t gcInfoIndex) 1065 Address LargeObjectHeap::doAllocateLargeObjectPage(size_t allocationSize, size_t gcInfoIndex)
1066 { 1066 {
1067 size_t largeObjectSize = sizeof(LargeObject) + LargeObject::headerPadding() + allocationSize; 1067 size_t largeObjectSize = sizeof(LargeObjectPage) + LargeObjectPage::headerPa dding() + allocationSize;
1068 // If ASan is supported we add allocationGranularity bytes to the allocated 1068 // If ASan is supported we add allocationGranularity bytes to the allocated
1069 // space and poison that to detect overflows 1069 // space and poison that to detect overflows
1070 #if defined(ADDRESS_SANITIZER) 1070 #if defined(ADDRESS_SANITIZER)
1071 largeObjectSize += allocationGranularity; 1071 largeObjectSize += allocationGranularity;
1072 #endif 1072 #endif
1073 1073
1074 threadState()->shouldFlushHeapDoesNotContainCache(); 1074 threadState()->shouldFlushHeapDoesNotContainCache();
1075 PageMemory* pageMemory = PageMemory::allocate(largeObjectSize); 1075 PageMemory* pageMemory = PageMemory::allocate(largeObjectSize);
1076 threadState()->allocatedRegionsSinceLastGC().append(pageMemory->region()); 1076 threadState()->allocatedRegionsSinceLastGC().append(pageMemory->region());
1077 Address largeObjectAddress = pageMemory->writableStart(); 1077 Address largeObjectAddress = pageMemory->writableStart();
1078 Address headerAddress = largeObjectAddress + sizeof(LargeObject) + LargeObje ct::headerPadding(); 1078 Address headerAddress = largeObjectAddress + sizeof(LargeObjectPage) + Large ObjectPage::headerPadding();
1079 #if ENABLE(ASSERT) 1079 #if ENABLE(ASSERT)
1080 // Verify that the allocated PageMemory is expectedly zeroed. 1080 // Verify that the allocated PageMemory is expectedly zeroed.
1081 for (size_t i = 0; i < largeObjectSize; ++i) 1081 for (size_t i = 0; i < largeObjectSize; ++i)
1082 ASSERT(!headerAddress[i]); 1082 ASSERT(!headerAddress[i]);
1083 #endif 1083 #endif
1084 ASSERT(gcInfoIndex > 0); 1084 ASSERT(gcInfoIndex > 0);
1085 HeapObjectHeader* header = new (NotNull, headerAddress) HeapObjectHeader(lar geObjectSizeInHeader, gcInfoIndex); 1085 HeapObjectHeader* header = new (NotNull, headerAddress) HeapObjectHeader(lar geObjectSizeInHeader, gcInfoIndex);
1086 Address result = headerAddress + sizeof(*header); 1086 Address result = headerAddress + sizeof(*header);
1087 ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask)); 1087 ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask));
1088 LargeObject* largeObject = new (largeObjectAddress) LargeObject(pageMemory, this, allocationSize); 1088 LargeObjectPage* largeObject = new (largeObjectAddress) LargeObjectPage(page Memory, this, allocationSize);
1089 header->checkHeader(); 1089 header->checkHeader();
1090 1090
1091 // Poison the object header and allocationGranularity bytes after the object 1091 // Poison the object header and allocationGranularity bytes after the object
1092 ASAN_POISON_MEMORY_REGION(header, sizeof(*header)); 1092 ASAN_POISON_MEMORY_REGION(header, sizeof(*header));
1093 ASAN_POISON_MEMORY_REGION(largeObject->address() + largeObject->size(), allo cationGranularity); 1093 ASAN_POISON_MEMORY_REGION(largeObject->address() + largeObject->size(), allo cationGranularity);
1094 1094
1095 largeObject->link(&m_firstPage); 1095 largeObject->link(&m_firstPage);
1096 1096
1097 Heap::increaseAllocatedSpace(largeObject->size()); 1097 Heap::increaseAllocatedSpace(largeObject->size());
1098 Heap::increaseAllocatedObjectSize(largeObject->size()); 1098 Heap::increaseAllocatedObjectSize(largeObject->size());
1099 return result; 1099 return result;
1100 } 1100 }
1101 1101
1102 void ThreadHeapForLargeObject::freeLargeObject(LargeObject* object) 1102 void LargeObjectHeap::freeLargeObjectPage(LargeObjectPage* object)
1103 { 1103 {
1104 object->heapObjectHeader()->finalize(object->payload(), object->payloadSize( )); 1104 object->heapObjectHeader()->finalize(object->payload(), object->payloadSize( ));
1105 Heap::decreaseAllocatedSpace(object->size()); 1105 Heap::decreaseAllocatedSpace(object->size());
1106 1106
1107 // Unpoison the object header and allocationGranularity bytes after the 1107 // Unpoison the object header and allocationGranularity bytes after the
1108 // object before freeing. 1108 // object before freeing.
1109 ASAN_UNPOISON_MEMORY_REGION(object->heapObjectHeader(), sizeof(HeapObjectHea der)); 1109 ASAN_UNPOISON_MEMORY_REGION(object->heapObjectHeader(), sizeof(HeapObjectHea der));
1110 ASAN_UNPOISON_MEMORY_REGION(object->address() + object->size(), allocationGr anularity); 1110 ASAN_UNPOISON_MEMORY_REGION(object->address() + object->size(), allocationGr anularity);
1111 1111
1112 if (object->terminating()) { 1112 if (object->terminating()) {
1113 ASSERT(ThreadState::current()->isTerminating()); 1113 ASSERT(ThreadState::current()->isTerminating());
1114 // The thread is shutting down and this page is being removed as a part 1114 // The thread is shutting down and this page is being removed as a part
1115 // of the thread local GC. In that case the object could be traced in 1115 // of the thread local GC. In that case the object could be traced in
1116 // the next global GC if there is a dangling pointer from a live thread 1116 // the next global GC if there is a dangling pointer from a live thread
1117 // heap to this dead thread heap. To guard against this, we put the 1117 // heap to this dead thread heap. To guard against this, we put the
1118 // page into the orphaned page pool and zap the page memory. This 1118 // page into the orphaned page pool and zap the page memory. This
1119 // ensures that tracing the dangling pointer in the next global GC just 1119 // ensures that tracing the dangling pointer in the next global GC just
1120 // crashes instead of causing use-after-frees. After the next global 1120 // crashes instead of causing use-after-frees. After the next global
1121 // GC, the orphaned pages are removed. 1121 // GC, the orphaned pages are removed.
1122 Heap::orphanedPagePool()->addOrphanedPage(heapIndex(), object); 1122 Heap::orphanedPagePool()->addOrphanedPage(heapIndex(), object);
1123 } else { 1123 } else {
1124 ASSERT(!ThreadState::current()->isTerminating()); 1124 ASSERT(!ThreadState::current()->isTerminating());
1125 PageMemory* memory = object->storage(); 1125 PageMemory* memory = object->storage();
1126 object->~LargeObject(); 1126 object->~LargeObjectPage();
1127 delete memory; 1127 delete memory;
1128 } 1128 }
1129 } 1129 }
1130 1130
1131 template<typename DataType> 1131 template<typename DataType>
1132 PagePool<DataType>::PagePool() 1132 PagePool<DataType>::PagePool()
1133 { 1133 {
1134 for (int i = 0; i < NumberOfHeaps; ++i) { 1134 for (int i = 0; i < NumberOfHeaps; ++i) {
1135 m_pool[i] = nullptr; 1135 m_pool[i] = nullptr;
1136 } 1136 }
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after
1170 delete entry; 1170 delete entry;
1171 if (memory->commit()) 1171 if (memory->commit())
1172 return memory; 1172 return memory;
1173 1173
1174 // We got some memory, but failed to commit it, try again. 1174 // We got some memory, but failed to commit it, try again.
1175 delete memory; 1175 delete memory;
1176 } 1176 }
1177 return nullptr; 1177 return nullptr;
1178 } 1178 }
1179 1179
1180 BaseHeapPage::BaseHeapPage(PageMemory* storage, ThreadHeap* heap) 1180 BasePage::BasePage(PageMemory* storage, BaseHeap* heap)
1181 : m_storage(storage) 1181 : m_storage(storage)
1182 , m_heap(heap) 1182 , m_heap(heap)
1183 , m_next(nullptr) 1183 , m_next(nullptr)
1184 , m_terminating(false) 1184 , m_terminating(false)
1185 , m_swept(true) 1185 , m_swept(true)
1186 { 1186 {
1187 ASSERT(isPageHeaderAddress(reinterpret_cast<Address>(this))); 1187 ASSERT(isPageHeaderAddress(reinterpret_cast<Address>(this)));
1188 } 1188 }
1189 1189
1190 void BaseHeapPage::markOrphaned() 1190 void BasePage::markOrphaned()
1191 { 1191 {
1192 m_heap = nullptr; 1192 m_heap = nullptr;
1193 m_terminating = false; 1193 m_terminating = false;
1194 // Since we zap the page payload for orphaned pages we need to mark it as 1194 // Since we zap the page payload for orphaned pages we need to mark it as
1195 // unused so a conservative pointer won't interpret the object headers. 1195 // unused so a conservative pointer won't interpret the object headers.
1196 storage()->markUnused(); 1196 storage()->markUnused();
1197 } 1197 }
1198 1198
1199 OrphanedPagePool::~OrphanedPagePool() 1199 OrphanedPagePool::~OrphanedPagePool()
1200 { 1200 {
1201 for (int index = 0; index < NumberOfHeaps; ++index) { 1201 for (int index = 0; index < NumberOfHeaps; ++index) {
1202 while (PoolEntry* entry = m_pool[index]) { 1202 while (PoolEntry* entry = m_pool[index]) {
1203 m_pool[index] = entry->next; 1203 m_pool[index] = entry->next;
1204 BaseHeapPage* page = entry->data; 1204 BasePage* page = entry->data;
1205 delete entry; 1205 delete entry;
1206 PageMemory* memory = page->storage(); 1206 PageMemory* memory = page->storage();
1207 ASSERT(memory); 1207 ASSERT(memory);
1208 page->~BaseHeapPage(); 1208 page->~BasePage();
1209 delete memory; 1209 delete memory;
1210 } 1210 }
1211 } 1211 }
1212 } 1212 }
1213 1213
1214 void OrphanedPagePool::addOrphanedPage(int index, BaseHeapPage* page) 1214 void OrphanedPagePool::addOrphanedPage(int index, BasePage* page)
1215 { 1215 {
1216 page->markOrphaned(); 1216 page->markOrphaned();
1217 PoolEntry* entry = new PoolEntry(page, m_pool[index]); 1217 PoolEntry* entry = new PoolEntry(page, m_pool[index]);
1218 m_pool[index] = entry; 1218 m_pool[index] = entry;
1219 } 1219 }
1220 1220
1221 NO_SANITIZE_ADDRESS 1221 NO_SANITIZE_ADDRESS
1222 void OrphanedPagePool::decommitOrphanedPages() 1222 void OrphanedPagePool::decommitOrphanedPages()
1223 { 1223 {
1224 ASSERT(ThreadState::current()->isInGC()); 1224 ASSERT(ThreadState::current()->isInGC());
1225 1225
1226 #if ENABLE(ASSERT) 1226 #if ENABLE(ASSERT)
1227 // No locking needed as all threads are at safepoints at this point in time. 1227 // No locking needed as all threads are at safepoints at this point in time.
1228 for (ThreadState* state : ThreadState::attachedThreads()) 1228 for (ThreadState* state : ThreadState::attachedThreads())
1229 ASSERT(state->isAtSafePoint()); 1229 ASSERT(state->isAtSafePoint());
1230 #endif 1230 #endif
1231 1231
1232 for (int index = 0; index < NumberOfHeaps; ++index) { 1232 for (int index = 0; index < NumberOfHeaps; ++index) {
1233 PoolEntry* entry = m_pool[index]; 1233 PoolEntry* entry = m_pool[index];
1234 PoolEntry** prevNext = &m_pool[index]; 1234 PoolEntry** prevNext = &m_pool[index];
1235 while (entry) { 1235 while (entry) {
1236 BaseHeapPage* page = entry->data; 1236 BasePage* page = entry->data;
1237 // Check if we should reuse the memory or just free it. 1237 // Check if we should reuse the memory or just free it.
1238 // Large object memory is not reused but freed, normal blink heap 1238 // Large object memory is not reused but freed, normal blink heap
1239 // pages are reused. 1239 // pages are reused.
1240 // NOTE: We call the destructor before freeing or adding to the 1240 // NOTE: We call the destructor before freeing or adding to the
1241 // free page pool. 1241 // free page pool.
1242 PageMemory* memory = page->storage(); 1242 PageMemory* memory = page->storage();
1243 if (page->isLargeObject()) { 1243 if (page->isLargeObjectPage()) {
1244 page->~BaseHeapPage(); 1244 page->~BasePage();
1245 delete memory; 1245 delete memory;
1246 } else { 1246 } else {
1247 page->~BaseHeapPage(); 1247 page->~BasePage();
1248 clearMemory(memory); 1248 clearMemory(memory);
1249 Heap::freePagePool()->addFreePage(index, memory); 1249 Heap::freePagePool()->addFreePage(index, memory);
1250 } 1250 }
1251 1251
1252 PoolEntry* deadEntry = entry; 1252 PoolEntry* deadEntry = entry;
1253 entry = entry->next; 1253 entry = entry->next;
1254 *prevNext = entry; 1254 *prevNext = entry;
1255 delete deadEntry; 1255 delete deadEntry;
1256 } 1256 }
1257 } 1257 }
(...skipping 12 matching lines...) Expand all
1270 #else 1270 #else
1271 memset(memory->writableStart(), 0, blinkPagePayloadSize()); 1271 memset(memory->writableStart(), 0, blinkPagePayloadSize());
1272 #endif 1272 #endif
1273 } 1273 }
1274 1274
1275 #if ENABLE(ASSERT) 1275 #if ENABLE(ASSERT)
1276 bool OrphanedPagePool::contains(void* object) 1276 bool OrphanedPagePool::contains(void* object)
1277 { 1277 {
1278 for (int index = 0; index < NumberOfHeaps; ++index) { 1278 for (int index = 0; index < NumberOfHeaps; ++index) {
1279 for (PoolEntry* entry = m_pool[index]; entry; entry = entry->next) { 1279 for (PoolEntry* entry = m_pool[index]; entry; entry = entry->next) {
1280 BaseHeapPage* page = entry->data; 1280 BasePage* page = entry->data;
1281 if (page->contains(reinterpret_cast<Address>(object))) 1281 if (page->contains(reinterpret_cast<Address>(object)))
1282 return true; 1282 return true;
1283 } 1283 }
1284 } 1284 }
1285 return false; 1285 return false;
1286 } 1286 }
1287 #endif 1287 #endif
1288 1288
1289 void ThreadHeapForHeapPage::freePage(HeapPage* page) 1289 void NormalPageHeap::freePage(NormalPage* page)
1290 { 1290 {
1291 Heap::decreaseAllocatedSpace(page->size()); 1291 Heap::decreaseAllocatedSpace(page->size());
1292 1292
1293 if (page->terminating()) { 1293 if (page->terminating()) {
1294 // The thread is shutting down and this page is being removed as a part 1294 // The thread is shutting down and this page is being removed as a part
1295 // of the thread local GC. In that case the object could be traced in 1295 // of the thread local GC. In that case the object could be traced in
1296 // the next global GC if there is a dangling pointer from a live thread 1296 // the next global GC if there is a dangling pointer from a live thread
1297 // heap to this dead thread heap. To guard against this, we put the 1297 // heap to this dead thread heap. To guard against this, we put the
1298 // page into the orphaned page pool and zap the page memory. This 1298 // page into the orphaned page pool and zap the page memory. This
1299 // ensures that tracing the dangling pointer in the next global GC just 1299 // ensures that tracing the dangling pointer in the next global GC just
1300 // crashes instead of causing use-after-frees. After the next global 1300 // crashes instead of causing use-after-frees. After the next global
1301 // GC, the orphaned pages are removed. 1301 // GC, the orphaned pages are removed.
1302 Heap::orphanedPagePool()->addOrphanedPage(heapIndex(), page); 1302 Heap::orphanedPagePool()->addOrphanedPage(heapIndex(), page);
1303 } else { 1303 } else {
1304 PageMemory* memory = page->storage(); 1304 PageMemory* memory = page->storage();
1305 page->~HeapPage(); 1305 page->~NormalPage();
1306 Heap::freePagePool()->addFreePage(heapIndex(), memory); 1306 Heap::freePagePool()->addFreePage(heapIndex(), memory);
1307 } 1307 }
1308 } 1308 }
1309 1309
1310 void ThreadHeapForHeapPage::allocatePage() 1310 void NormalPageHeap::allocatePage()
1311 { 1311 {
1312 threadState()->shouldFlushHeapDoesNotContainCache(); 1312 threadState()->shouldFlushHeapDoesNotContainCache();
1313 PageMemory* pageMemory = Heap::freePagePool()->takeFreePage(heapIndex()); 1313 PageMemory* pageMemory = Heap::freePagePool()->takeFreePage(heapIndex());
1314 // We continue allocating page memory until we succeed in committing one. 1314 // We continue allocating page memory until we succeed in committing one.
1315 while (!pageMemory) { 1315 while (!pageMemory) {
1316 // Allocate a memory region for blinkPagesPerRegion pages that 1316 // Allocate a memory region for blinkPagesPerRegion pages that
1317 // will each have the following layout. 1317 // will each have the following layout.
1318 // 1318 //
1319 // [ guard os page | ... payload ... | guard os page ] 1319 // [ guard os page | ... payload ... | guard os page ]
1320 // ^---{ aligned to blink page size } 1320 // ^---{ aligned to blink page size }
(...skipping 10 matching lines...) Expand all
1331 if (memory->commit()) 1331 if (memory->commit())
1332 pageMemory = memory; 1332 pageMemory = memory;
1333 else 1333 else
1334 delete memory; 1334 delete memory;
1335 } else { 1335 } else {
1336 Heap::freePagePool()->addFreePage(heapIndex(), memory); 1336 Heap::freePagePool()->addFreePage(heapIndex(), memory);
1337 } 1337 }
1338 offset += blinkPageSize; 1338 offset += blinkPageSize;
1339 } 1339 }
1340 } 1340 }
1341 HeapPage* page = new (pageMemory->writableStart()) HeapPage(pageMemory, this ); 1341 NormalPage* page = new (pageMemory->writableStart()) NormalPage(pageMemory, this);
1342 page->link(&m_firstPage); 1342 page->link(&m_firstPage);
1343 1343
1344 Heap::increaseAllocatedSpace(page->size()); 1344 Heap::increaseAllocatedSpace(page->size());
1345 addToFreeList(page->payload(), page->payloadSize()); 1345 addToFreeList(page->payload(), page->payloadSize());
1346 } 1346 }
1347 1347
1348 #if ENABLE(ASSERT) 1348 #if ENABLE(ASSERT)
1349 bool ThreadHeapForHeapPage::pagesToBeSweptContains(Address address) 1349 bool NormalPageHeap::pagesToBeSweptContains(Address address)
1350 { 1350 {
1351 for (BaseHeapPage* page = m_firstUnsweptPage; page; page = page->next()) { 1351 for (BasePage* page = m_firstUnsweptPage; page; page = page->next()) {
1352 if (page->contains(address)) 1352 if (page->contains(address))
1353 return true; 1353 return true;
1354 } 1354 }
1355 return false; 1355 return false;
1356 } 1356 }
1357 #endif 1357 #endif
1358 1358
1359 size_t ThreadHeap::objectPayloadSizeForTesting() 1359 size_t BaseHeap::objectPayloadSizeForTesting()
1360 { 1360 {
1361 ASSERT(isConsistentForSweeping()); 1361 ASSERT(isConsistentForSweeping());
1362 ASSERT(!m_firstUnsweptPage); 1362 ASSERT(!m_firstUnsweptPage);
1363 1363
1364 size_t objectPayloadSize = 0; 1364 size_t objectPayloadSize = 0;
1365 for (BaseHeapPage* page = m_firstPage; page; page = page->next()) 1365 for (BasePage* page = m_firstPage; page; page = page->next())
1366 objectPayloadSize += page->objectPayloadSizeForTesting(); 1366 objectPayloadSize += page->objectPayloadSizeForTesting();
1367 return objectPayloadSize; 1367 return objectPayloadSize;
1368 } 1368 }
1369 1369
1370 #if ENABLE(ASSERT) 1370 #if ENABLE(ASSERT)
1371 bool ThreadHeapForHeapPage::isConsistentForSweeping() 1371 bool NormalPageHeap::isConsistentForSweeping()
1372 { 1372 {
1373 // A thread heap is consistent for sweeping if none of the pages to be swept 1373 // A thread heap is consistent for sweeping if none of the pages to be swept
1374 // contain a freelist block or the current allocation point. 1374 // contain a freelist block or the current allocation point.
1375 for (size_t i = 0; i < blinkPageSizeLog2; ++i) { 1375 for (size_t i = 0; i < blinkPageSizeLog2; ++i) {
1376 for (FreeListEntry* freeListEntry = m_freeList.m_freeLists[i]; freeListE ntry; freeListEntry = freeListEntry->next()) { 1376 for (FreeListEntry* freeListEntry = m_freeList.m_freeLists[i]; freeListE ntry; freeListEntry = freeListEntry->next()) {
1377 if (pagesToBeSweptContains(freeListEntry->address())) 1377 if (pagesToBeSweptContains(freeListEntry->address()))
1378 return false; 1378 return false;
1379 } 1379 }
1380 } 1380 }
1381 if (hasCurrentAllocationArea()) { 1381 if (hasCurrentAllocationArea()) {
1382 if (pagesToBeSweptContains(currentAllocationPoint())) 1382 if (pagesToBeSweptContains(currentAllocationPoint()))
1383 return false; 1383 return false;
1384 } 1384 }
1385 return true; 1385 return true;
1386 } 1386 }
1387 #endif 1387 #endif
1388 1388
1389 void ThreadHeap::makeConsistentForSweeping() 1389 void BaseHeap::makeConsistentForSweeping()
1390 { 1390 {
1391 clearFreeLists(); 1391 clearFreeLists();
1392 ASSERT(isConsistentForSweeping()); 1392 ASSERT(isConsistentForSweeping());
1393 for (BaseHeapPage* page = m_firstPage; page; page = page->next()) 1393 for (BasePage* page = m_firstPage; page; page = page->next())
1394 page->markAsUnswept(); 1394 page->markAsUnswept();
1395 1395
1396 // If a new GC is requested before this thread got around to sweep, 1396 // If a new GC is requested before this thread got around to sweep,
1397 // ie. due to the thread doing a long running operation, we clear 1397 // ie. due to the thread doing a long running operation, we clear
1398 // the mark bits and mark any of the dead objects as dead. The latter 1398 // the mark bits and mark any of the dead objects as dead. The latter
1399 // is used to ensure the next GC marking does not trace already dead 1399 // is used to ensure the next GC marking does not trace already dead
1400 // objects. If we trace a dead object we could end up tracing into 1400 // objects. If we trace a dead object we could end up tracing into
1401 // garbage or the middle of another object via the newly conservatively 1401 // garbage or the middle of another object via the newly conservatively
1402 // found object. 1402 // found object.
1403 BaseHeapPage* previousPage = nullptr; 1403 BasePage* previousPage = nullptr;
1404 for (BaseHeapPage* page = m_firstUnsweptPage; page; previousPage = page, pag e = page->next()) { 1404 for (BasePage* page = m_firstUnsweptPage; page; previousPage = page, page = page->next()) {
1405 page->markUnmarkedObjectsDead(); 1405 page->markUnmarkedObjectsDead();
1406 ASSERT(!page->hasBeenSwept()); 1406 ASSERT(!page->hasBeenSwept());
1407 } 1407 }
1408 if (previousPage) { 1408 if (previousPage) {
1409 ASSERT(m_firstUnsweptPage); 1409 ASSERT(m_firstUnsweptPage);
1410 previousPage->m_next = m_firstPage; 1410 previousPage->m_next = m_firstPage;
1411 m_firstPage = m_firstUnsweptPage; 1411 m_firstPage = m_firstUnsweptPage;
1412 m_firstUnsweptPage = nullptr; 1412 m_firstUnsweptPage = nullptr;
1413 } 1413 }
1414 ASSERT(!m_firstUnsweptPage); 1414 ASSERT(!m_firstUnsweptPage);
1415 } 1415 }
1416 1416
1417 void ThreadHeapForHeapPage::clearFreeLists() 1417 void NormalPageHeap::clearFreeLists()
1418 { 1418 {
1419 setAllocationPoint(nullptr, 0); 1419 setAllocationPoint(nullptr, 0);
1420 m_freeList.clear(); 1420 m_freeList.clear();
1421 } 1421 }
1422 1422
1423 #if ENABLE(GC_PROFILING) 1423 #if ENABLE(GC_PROFILING)
1424 void ThreadHeap::snapshotFreeList(TracedValue& json) 1424 void BaseHeap::snapshotFreeList(TracedValue& json)
1425 { 1425 {
1426 json.setInteger("cumulativeAllocationSize", m_cumulativeAllocationSize); 1426 json.setInteger("cumulativeAllocationSize", m_cumulativeAllocationSize);
1427 json.setDouble("inlineAllocationRate", static_cast<double>(m_inlineAllocatio nCount) / m_allocationCount); 1427 json.setDouble("inlineAllocationRate", static_cast<double>(m_inlineAllocatio nCount) / m_allocationCount);
1428 json.setInteger("inlineAllocationCount", m_inlineAllocationCount); 1428 json.setInteger("inlineAllocationCount", m_inlineAllocationCount);
1429 json.setInteger("allocationCount", m_allocationCount); 1429 json.setInteger("allocationCount", m_allocationCount);
1430 size_t pageCount = 0; 1430 size_t pageCount = 0;
1431 size_t totalPageSize = 0; 1431 size_t totalPageSize = 0;
1432 for (HeapPage* page = m_firstPage; page; page = page->next()) { 1432 for (NormalPage* page = m_firstPage; page; page = page->next()) {
1433 ++pageCount; 1433 ++pageCount;
1434 totalPageSize += page->payloadSize(); 1434 totalPageSize += page->payloadSize();
1435 } 1435 }
1436 json.setInteger("pageCount", pageCount); 1436 json.setInteger("pageCount", pageCount);
1437 json.setInteger("totalPageSize", totalPageSize); 1437 json.setInteger("totalPageSize", totalPageSize);
1438 1438
1439 FreeList::PerBucketFreeListStats bucketStats[blinkPageSizeLog2]; 1439 FreeList::PerBucketFreeListStats bucketStats[blinkPageSizeLog2];
1440 size_t totalFreeSize; 1440 size_t totalFreeSize;
1441 m_freeList.getFreeSizeStats(bucketStats, totalFreeSize); 1441 m_freeList.getFreeSizeStats(bucketStats, totalFreeSize);
1442 json.setInteger("totalFreeSize", totalFreeSize); 1442 json.setInteger("totalFreeSize", totalFreeSize);
1443 1443
1444 json.beginArray("perBucketEntryCount"); 1444 json.beginArray("perBucketEntryCount");
1445 for (size_t i = 0; i < blinkPageSizeLog2; ++i) 1445 for (size_t i = 0; i < blinkPageSizeLog2; ++i)
1446 json.pushInteger(bucketStats[i].entryCount); 1446 json.pushInteger(bucketStats[i].entryCount);
1447 json.endArray(); 1447 json.endArray();
1448 1448
1449 json.beginArray("perBucketFreeSize"); 1449 json.beginArray("perBucketFreeSize");
1450 for (size_t i = 0; i < blinkPageSizeLog2; ++i) 1450 for (size_t i = 0; i < blinkPageSizeLog2; ++i)
1451 json.pushInteger(bucketStats[i].freeSize); 1451 json.pushInteger(bucketStats[i].freeSize);
1452 json.endArray(); 1452 json.endArray();
1453 } 1453 }
1454 1454
1455 void ThreadHeap::countMarkedObjects(ClassAgeCountsMap& classAgeCounts) const 1455 void BaseHeap::countMarkedObjects(ClassAgeCountsMap& classAgeCounts) const
1456 { 1456 {
1457 for (HeapPage* page = m_firstPage; page; page = page->next()) 1457 for (NormalPage* page = m_firstPage; page; page = page->next())
1458 page->countMarkedObjects(classAgeCounts); 1458 page->countMarkedObjects(classAgeCounts);
1459 for (LargeObject* largeObject = m_firstLargeObject; largeObject; largeObject = largeObject->next()) 1459 for (LargeObjectPage* largeObject = m_firstLargeObjectPage; largeObject; lar geObject = largeObject->next())
1460 largeObject->countMarkedObjects(classAgeCounts); 1460 largeObject->countMarkedObjects(classAgeCounts);
1461 } 1461 }
1462 1462
1463 void ThreadHeap::countObjectsToSweep(ClassAgeCountsMap& classAgeCounts) const 1463 void BaseHeap::countObjectsToSweep(ClassAgeCountsMap& classAgeCounts) const
1464 { 1464 {
1465 for (HeapPage* page = m_firstPage; page; page = page->next()) 1465 for (NormalPage* page = m_firstPage; page; page = page->next())
1466 page->countObjectsToSweep(classAgeCounts); 1466 page->countObjectsToSweep(classAgeCounts);
1467 for (LargeObject* largeObject = m_firstLargeObject; largeObject; largeObject = largeObject->next()) 1467 for (LargeObjectPage* largeObject = m_firstLargeObjectPage; largeObject; lar geObject = largeObject->next())
1468 largeObject->countObjectsToSweep(classAgeCounts); 1468 largeObject->countObjectsToSweep(classAgeCounts);
1469 } 1469 }
1470 #endif 1470 #endif
1471 1471
1472 void FreeList::clear() 1472 void FreeList::clear()
1473 { 1473 {
1474 m_biggestFreeListIndex = 0; 1474 m_biggestFreeListIndex = 0;
1475 for (size_t i = 0; i < blinkPageSizeLog2; ++i) 1475 for (size_t i = 0; i < blinkPageSizeLog2; ++i)
1476 m_freeLists[i] = nullptr; 1476 m_freeLists[i] = nullptr;
1477 } 1477 }
(...skipping 18 matching lines...) Expand all
1496 size_t& freeSize = bucketStats[i].freeSize; 1496 size_t& freeSize = bucketStats[i].freeSize;
1497 for (FreeListEntry* entry = m_freeLists[i]; entry; entry = entry->next() ) { 1497 for (FreeListEntry* entry = m_freeLists[i]; entry; entry = entry->next() ) {
1498 ++entryCount; 1498 ++entryCount;
1499 freeSize += entry->size(); 1499 freeSize += entry->size();
1500 } 1500 }
1501 totalFreeSize += freeSize; 1501 totalFreeSize += freeSize;
1502 } 1502 }
1503 } 1503 }
1504 #endif 1504 #endif
1505 1505
1506 HeapPage::HeapPage(PageMemory* storage, ThreadHeap* heap) 1506 NormalPage::NormalPage(PageMemory* storage, BaseHeap* heap)
1507 : BaseHeapPage(storage, heap) 1507 : BasePage(storage, heap)
1508 { 1508 {
1509 m_objectStartBitMapComputed = false; 1509 m_objectStartBitMapComputed = false;
1510 ASSERT(isPageHeaderAddress(reinterpret_cast<Address>(this))); 1510 ASSERT(isPageHeaderAddress(reinterpret_cast<Address>(this)));
1511 } 1511 }
1512 1512
1513 size_t HeapPage::objectPayloadSizeForTesting() 1513 size_t NormalPage::objectPayloadSizeForTesting()
1514 { 1514 {
1515 size_t objectPayloadSize = 0; 1515 size_t objectPayloadSize = 0;
1516 Address headerAddress = payload(); 1516 Address headerAddress = payload();
1517 markAsSwept(); 1517 markAsSwept();
1518 ASSERT(headerAddress != payloadEnd()); 1518 ASSERT(headerAddress != payloadEnd());
1519 do { 1519 do {
1520 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAdd ress); 1520 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAdd ress);
1521 if (!header->isFree()) { 1521 if (!header->isFree()) {
1522 header->checkHeader(); 1522 header->checkHeader();
1523 objectPayloadSize += header->payloadSize(); 1523 objectPayloadSize += header->payloadSize();
1524 } 1524 }
1525 ASSERT(header->size() < blinkPagePayloadSize()); 1525 ASSERT(header->size() < blinkPagePayloadSize());
1526 headerAddress += header->size(); 1526 headerAddress += header->size();
1527 ASSERT(headerAddress <= payloadEnd()); 1527 ASSERT(headerAddress <= payloadEnd());
1528 } while (headerAddress < payloadEnd()); 1528 } while (headerAddress < payloadEnd());
1529 return objectPayloadSize; 1529 return objectPayloadSize;
1530 } 1530 }
1531 1531
1532 bool HeapPage::isEmpty() 1532 bool NormalPage::isEmpty()
1533 { 1533 {
1534 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(payload()); 1534 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(payload());
1535 return header->isFree() && header->size() == payloadSize(); 1535 return header->isFree() && header->size() == payloadSize();
1536 } 1536 }
1537 1537
1538 void HeapPage::sweep() 1538 void NormalPage::sweep()
1539 { 1539 {
1540 clearObjectStartBitMap(); 1540 clearObjectStartBitMap();
1541 1541
1542 size_t markedObjectSize = 0; 1542 size_t markedObjectSize = 0;
1543 Address startOfGap = payload(); 1543 Address startOfGap = payload();
1544 for (Address headerAddress = startOfGap; headerAddress < payloadEnd(); ) { 1544 for (Address headerAddress = startOfGap; headerAddress < payloadEnd(); ) {
1545 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAdd ress); 1545 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAdd ress);
1546 ASSERT(header->size() > 0); 1546 ASSERT(header->size() > 0);
1547 ASSERT(header->size() < blinkPagePayloadSize()); 1547 ASSERT(header->size() < blinkPagePayloadSize());
1548 1548
1549 if (header->isPromptlyFreed()) 1549 if (header->isPromptlyFreed())
1550 heapForHeapPage()->decreasePromptlyFreedSize(header->size()); 1550 heapForNormalPage()->decreasePromptlyFreedSize(header->size());
1551 if (header->isFree()) { 1551 if (header->isFree()) {
1552 size_t size = header->size(); 1552 size_t size = header->size();
1553 // Zero the memory in the free list header to maintain the 1553 // Zero the memory in the free list header to maintain the
1554 // invariant that memory on the free list is zero filled. 1554 // invariant that memory on the free list is zero filled.
1555 // The rest of the memory is already on the free list and is 1555 // The rest of the memory is already on the free list and is
1556 // therefore already zero filled. 1556 // therefore already zero filled.
1557 FILL_ZERO_IF_PRODUCTION(headerAddress, size < sizeof(FreeListEntry) ? size : sizeof(FreeListEntry)); 1557 FILL_ZERO_IF_PRODUCTION(headerAddress, size < sizeof(FreeListEntry) ? size : sizeof(FreeListEntry));
1558 headerAddress += size; 1558 headerAddress += size;
1559 continue; 1559 continue;
1560 } 1560 }
(...skipping 12 matching lines...) Expand all
1573 header->finalize(payload, payloadSize); 1573 header->finalize(payload, payloadSize);
1574 // This memory will be added to the freelist. Maintain the invariant 1574 // This memory will be added to the freelist. Maintain the invariant
1575 // that memory on the freelist is zero filled. 1575 // that memory on the freelist is zero filled.
1576 FILL_ZERO_IF_PRODUCTION(headerAddress, size); 1576 FILL_ZERO_IF_PRODUCTION(headerAddress, size);
1577 ASAN_POISON_MEMORY_REGION(payload, payloadSize); 1577 ASAN_POISON_MEMORY_REGION(payload, payloadSize);
1578 headerAddress += size; 1578 headerAddress += size;
1579 continue; 1579 continue;
1580 } 1580 }
1581 1581
1582 if (startOfGap != headerAddress) 1582 if (startOfGap != headerAddress)
1583 heapForHeapPage()->addToFreeList(startOfGap, headerAddress - startOf Gap); 1583 heapForNormalPage()->addToFreeList(startOfGap, headerAddress - start OfGap);
1584 header->unmark(); 1584 header->unmark();
1585 headerAddress += header->size(); 1585 headerAddress += header->size();
1586 markedObjectSize += header->size(); 1586 markedObjectSize += header->size();
1587 startOfGap = headerAddress; 1587 startOfGap = headerAddress;
1588 } 1588 }
1589 if (startOfGap != payloadEnd()) 1589 if (startOfGap != payloadEnd())
1590 heapForHeapPage()->addToFreeList(startOfGap, payloadEnd() - startOfGap); 1590 heapForNormalPage()->addToFreeList(startOfGap, payloadEnd() - startOfGap );
1591 1591
1592 if (markedObjectSize) 1592 if (markedObjectSize)
1593 Heap::increaseMarkedObjectSize(markedObjectSize); 1593 Heap::increaseMarkedObjectSize(markedObjectSize);
1594 } 1594 }
1595 1595
1596 void HeapPage::markUnmarkedObjectsDead() 1596 void NormalPage::markUnmarkedObjectsDead()
1597 { 1597 {
1598 for (Address headerAddress = payload(); headerAddress < payloadEnd();) { 1598 for (Address headerAddress = payload(); headerAddress < payloadEnd();) {
1599 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAdd ress); 1599 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAdd ress);
1600 ASSERT(header->size() < blinkPagePayloadSize()); 1600 ASSERT(header->size() < blinkPagePayloadSize());
1601 // Check if a free list entry first since we cannot call 1601 // Check if a free list entry first since we cannot call
1602 // isMarked on a free list entry. 1602 // isMarked on a free list entry.
1603 if (header->isFree()) { 1603 if (header->isFree()) {
1604 headerAddress += header->size(); 1604 headerAddress += header->size();
1605 continue; 1605 continue;
1606 } 1606 }
1607 header->checkHeader(); 1607 header->checkHeader();
1608 if (header->isMarked()) 1608 if (header->isMarked())
1609 header->unmark(); 1609 header->unmark();
1610 else 1610 else
1611 header->markDead(); 1611 header->markDead();
1612 headerAddress += header->size(); 1612 headerAddress += header->size();
1613 } 1613 }
1614 } 1614 }
1615 1615
1616 void HeapPage::removeFromHeap() 1616 void NormalPage::removeFromHeap()
1617 { 1617 {
1618 heapForHeapPage()->freePage(this); 1618 heapForNormalPage()->freePage(this);
1619 } 1619 }
1620 1620
1621 ThreadHeapForHeapPage* HeapPage::heapForHeapPage() 1621 NormalPageHeap* NormalPage::heapForNormalPage()
1622 { 1622 {
1623 return static_cast<ThreadHeapForHeapPage*>(heap()); 1623 return static_cast<NormalPageHeap*>(heap());
1624 } 1624 }
1625 1625
1626 void HeapPage::populateObjectStartBitMap() 1626 void NormalPage::populateObjectStartBitMap()
1627 { 1627 {
1628 memset(&m_objectStartBitMap, 0, objectStartBitMapSize); 1628 memset(&m_objectStartBitMap, 0, objectStartBitMapSize);
1629 Address start = payload(); 1629 Address start = payload();
1630 for (Address headerAddress = start; headerAddress < payloadEnd();) { 1630 for (Address headerAddress = start; headerAddress < payloadEnd();) {
1631 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAdd ress); 1631 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(headerAdd ress);
1632 size_t objectOffset = headerAddress - start; 1632 size_t objectOffset = headerAddress - start;
1633 ASSERT(!(objectOffset & allocationMask)); 1633 ASSERT(!(objectOffset & allocationMask));
1634 size_t objectStartNumber = objectOffset / allocationGranularity; 1634 size_t objectStartNumber = objectOffset / allocationGranularity;
1635 size_t mapIndex = objectStartNumber / 8; 1635 size_t mapIndex = objectStartNumber / 8;
1636 ASSERT(mapIndex < objectStartBitMapSize); 1636 ASSERT(mapIndex < objectStartBitMapSize);
1637 m_objectStartBitMap[mapIndex] |= (1 << (objectStartNumber & 7)); 1637 m_objectStartBitMap[mapIndex] |= (1 << (objectStartNumber & 7));
1638 headerAddress += header->size(); 1638 headerAddress += header->size();
1639 ASSERT(headerAddress <= payloadEnd()); 1639 ASSERT(headerAddress <= payloadEnd());
1640 } 1640 }
1641 m_objectStartBitMapComputed = true; 1641 m_objectStartBitMapComputed = true;
1642 } 1642 }
1643 1643
1644 void HeapPage::clearObjectStartBitMap() 1644 void NormalPage::clearObjectStartBitMap()
1645 { 1645 {
1646 m_objectStartBitMapComputed = false; 1646 m_objectStartBitMapComputed = false;
1647 } 1647 }
1648 1648
1649 static int numberOfLeadingZeroes(uint8_t byte) 1649 static int numberOfLeadingZeroes(uint8_t byte)
1650 { 1650 {
1651 if (!byte) 1651 if (!byte)
1652 return 8; 1652 return 8;
1653 int result = 0; 1653 int result = 0;
1654 if (byte <= 0x0F) { 1654 if (byte <= 0x0F) {
1655 result += 4; 1655 result += 4;
1656 byte = byte << 4; 1656 byte = byte << 4;
1657 } 1657 }
1658 if (byte <= 0x3F) { 1658 if (byte <= 0x3F) {
1659 result += 2; 1659 result += 2;
1660 byte = byte << 2; 1660 byte = byte << 2;
1661 } 1661 }
1662 if (byte <= 0x7F) 1662 if (byte <= 0x7F)
1663 result++; 1663 result++;
1664 return result; 1664 return result;
1665 } 1665 }
1666 1666
1667 HeapObjectHeader* HeapPage::findHeaderFromAddress(Address address) 1667 HeapObjectHeader* NormalPage::findHeaderFromAddress(Address address)
1668 { 1668 {
1669 if (address < payload()) 1669 if (address < payload())
1670 return nullptr; 1670 return nullptr;
1671 if (!isObjectStartBitMapComputed()) 1671 if (!isObjectStartBitMapComputed())
1672 populateObjectStartBitMap(); 1672 populateObjectStartBitMap();
1673 size_t objectOffset = address - payload(); 1673 size_t objectOffset = address - payload();
1674 size_t objectStartNumber = objectOffset / allocationGranularity; 1674 size_t objectStartNumber = objectOffset / allocationGranularity;
1675 size_t mapIndex = objectStartNumber / 8; 1675 size_t mapIndex = objectStartNumber / 8;
1676 ASSERT(mapIndex < objectStartBitMapSize); 1676 ASSERT(mapIndex < objectStartBitMapSize);
1677 size_t bit = objectStartNumber & 7; 1677 size_t bit = objectStartNumber & 7;
1678 uint8_t byte = m_objectStartBitMap[mapIndex] & ((1 << (bit + 1)) - 1); 1678 uint8_t byte = m_objectStartBitMap[mapIndex] & ((1 << (bit + 1)) - 1);
1679 while (!byte) { 1679 while (!byte) {
1680 ASSERT(mapIndex > 0); 1680 ASSERT(mapIndex > 0);
1681 byte = m_objectStartBitMap[--mapIndex]; 1681 byte = m_objectStartBitMap[--mapIndex];
1682 } 1682 }
1683 int leadingZeroes = numberOfLeadingZeroes(byte); 1683 int leadingZeroes = numberOfLeadingZeroes(byte);
1684 objectStartNumber = (mapIndex * 8) + 7 - leadingZeroes; 1684 objectStartNumber = (mapIndex * 8) + 7 - leadingZeroes;
1685 objectOffset = objectStartNumber * allocationGranularity; 1685 objectOffset = objectStartNumber * allocationGranularity;
1686 Address objectAddress = objectOffset + payload(); 1686 Address objectAddress = objectOffset + payload();
1687 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(objectAddress ); 1687 HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(objectAddress );
1688 if (header->isFree()) 1688 if (header->isFree())
1689 return nullptr; 1689 return nullptr;
1690 header->checkHeader(); 1690 header->checkHeader();
1691 return header; 1691 return header;
1692 } 1692 }
1693 1693
1694 void HeapPage::checkAndMarkPointer(Visitor* visitor, Address address) 1694 void NormalPage::checkAndMarkPointer(Visitor* visitor, Address address)
1695 { 1695 {
1696 ASSERT(contains(address)); 1696 ASSERT(contains(address));
1697 HeapObjectHeader* header = findHeaderFromAddress(address); 1697 HeapObjectHeader* header = findHeaderFromAddress(address);
1698 if (!header || header->isDead()) 1698 if (!header || header->isDead())
1699 return; 1699 return;
1700 #if ENABLE(GC_PROFILING) 1700 #if ENABLE(GC_PROFILING)
1701 visitor->setHostInfo(&address, "stack"); 1701 visitor->setHostInfo(&address, "stack");
1702 #endif 1702 #endif
1703 markPointer(visitor, header); 1703 markPointer(visitor, header);
1704 } 1704 }
1705 1705
1706 #if ENABLE(GC_PROFILING) 1706 #if ENABLE(GC_PROFILING)
1707 const GCInfo* HeapPage::findGCInfo(Address address) 1707 const GCInfo* NormalPage::findGCInfo(Address address)
1708 { 1708 {
1709 if (address < payload()) 1709 if (address < payload())
1710 return nullptr; 1710 return nullptr;
1711 1711
1712 HeapObjectHeader* header = findHeaderFromAddress(address); 1712 HeapObjectHeader* header = findHeaderFromAddress(address);
1713 if (!header) 1713 if (!header)
1714 return nullptr; 1714 return nullptr;
1715 1715
1716 return Heap::gcInfo(header->gcInfoIndex()); 1716 return Heap::gcInfo(header->gcInfoIndex());
1717 } 1717 }
1718 #endif 1718 #endif
1719 1719
1720 #if ENABLE(GC_PROFILING) 1720 #if ENABLE(GC_PROFILING)
1721 void HeapPage::snapshot(TracedValue* json, ThreadState::SnapshotInfo* info) 1721 void NormalPage::snapshot(TracedValue* json, ThreadState::SnapshotInfo* info)
1722 { 1722 {
1723 HeapObjectHeader* header = nullptr; 1723 HeapObjectHeader* header = nullptr;
1724 for (Address addr = payload(); addr < payloadEnd(); addr += header->size()) { 1724 for (Address addr = payload(); addr < payloadEnd(); addr += header->size()) {
1725 header = reinterpret_cast<HeapObjectHeader*>(addr); 1725 header = reinterpret_cast<HeapObjectHeader*>(addr);
1726 if (json) 1726 if (json)
1727 json->pushInteger(header->encodedSize()); 1727 json->pushInteger(header->encodedSize());
1728 if (header->isFree()) { 1728 if (header->isFree()) {
1729 info->freeSize += header->size(); 1729 info->freeSize += header->size();
1730 continue; 1730 continue;
1731 } 1731 }
(...skipping 11 matching lines...) Expand all
1743 } else { 1743 } else {
1744 info->deadCount[tag] += 1; 1744 info->deadCount[tag] += 1;
1745 info->deadSize[tag] += header->size(); 1745 info->deadSize[tag] += header->size();
1746 // Count objects that are dead before the final generation. 1746 // Count objects that are dead before the final generation.
1747 if (age < maxHeapObjectAge) 1747 if (age < maxHeapObjectAge)
1748 info->generations[tag][age] += 1; 1748 info->generations[tag][age] += 1;
1749 } 1749 }
1750 } 1750 }
1751 } 1751 }
1752 1752
1753 void HeapPage::incrementMarkedObjectsAge() 1753 void NormalPage::incrementMarkedObjectsAge()
1754 { 1754 {
1755 HeapObjectHeader* header = nullptr; 1755 HeapObjectHeader* header = nullptr;
1756 for (Address address = payload(); address < payloadEnd(); address += header- >size()) { 1756 for (Address address = payload(); address < payloadEnd(); address += header- >size()) {
1757 header = reinterpret_cast<HeapObjectHeader*>(address); 1757 header = reinterpret_cast<HeapObjectHeader*>(address);
1758 if (header->isMarked()) 1758 if (header->isMarked())
1759 header->incrementAge(); 1759 header->incrementAge();
1760 } 1760 }
1761 } 1761 }
1762 1762
1763 void HeapPage::countMarkedObjects(ClassAgeCountsMap& classAgeCounts) 1763 void NormalPage::countMarkedObjects(ClassAgeCountsMap& classAgeCounts)
1764 { 1764 {
1765 HeapObjectHeader* header = nullptr; 1765 HeapObjectHeader* header = nullptr;
1766 for (Address address = payload(); address < payloadEnd(); address += header- >size()) { 1766 for (Address address = payload(); address < payloadEnd(); address += header- >size()) {
1767 header = reinterpret_cast<HeapObjectHeader*>(address); 1767 header = reinterpret_cast<HeapObjectHeader*>(address);
1768 if (header->isMarked()) { 1768 if (header->isMarked()) {
1769 String className(classOf(header->payload())); 1769 String className(classOf(header->payload()));
1770 ++(classAgeCounts.add(className, AgeCounts()).storedValue->value.age s[header->age()]); 1770 ++(classAgeCounts.add(className, AgeCounts()).storedValue->value.age s[header->age()]);
1771 } 1771 }
1772 } 1772 }
1773 } 1773 }
1774 1774
1775 void HeapPage::countObjectsToSweep(ClassAgeCountsMap& classAgeCounts) 1775 void NormalPage::countObjectsToSweep(ClassAgeCountsMap& classAgeCounts)
1776 { 1776 {
1777 HeapObjectHeader* header = nullptr; 1777 HeapObjectHeader* header = nullptr;
1778 for (Address address = payload(); address < payloadEnd(); address += header- >size()) { 1778 for (Address address = payload(); address < payloadEnd(); address += header- >size()) {
1779 header = reinterpret_cast<HeapObjectHeader*>(address); 1779 header = reinterpret_cast<HeapObjectHeader*>(address);
1780 if (!header->isFree() && !header->isMarked()) { 1780 if (!header->isFree() && !header->isMarked()) {
1781 String className(classOf(header->payload())); 1781 String className(classOf(header->payload()));
1782 ++(classAgeCounts.add(className, AgeCounts()).storedValue->value.age s[header->age()]); 1782 ++(classAgeCounts.add(className, AgeCounts()).storedValue->value.age s[header->age()]);
1783 } 1783 }
1784 } 1784 }
1785 } 1785 }
1786 #endif 1786 #endif
1787 1787
1788 size_t LargeObject::objectPayloadSizeForTesting() 1788 size_t LargeObjectPage::objectPayloadSizeForTesting()
1789 { 1789 {
1790 markAsSwept(); 1790 markAsSwept();
1791 return payloadSize(); 1791 return payloadSize();
1792 } 1792 }
1793 1793
1794 #if ENABLE(GC_PROFILING) 1794 #if ENABLE(GC_PROFILING)
1795 const GCInfo* LargeObject::findGCInfo(Address address) 1795 const GCInfo* LargeObjectPage::findGCInfo(Address address)
1796 { 1796 {
1797 if (!containedInObjectPayload(address)) 1797 if (!containedInObjectPayload(address))
1798 return nullptr; 1798 return nullptr;
1799 HeapObjectHeader* header = heapObjectHeader(); 1799 HeapObjectHeader* header = heapObjectHeader();
1800 return Heap::gcInfo(header->gcInfoIndex()); 1800 return Heap::gcInfo(header->gcInfoIndex());
1801 } 1801 }
1802 1802
1803 void LargeObject::snapshot(TracedValue* json, ThreadState::SnapshotInfo* info) 1803 void LargeObjectPage::snapshot(TracedValue* json, ThreadState::SnapshotInfo* inf o)
1804 { 1804 {
1805 HeapObjectHeader* header = heapObjectHeader(); 1805 HeapObjectHeader* header = heapObjectHeader();
1806 size_t tag = info->getClassTag(Heap::gcInfo(header->gcInfoIndex())); 1806 size_t tag = info->getClassTag(Heap::gcInfo(header->gcInfoIndex()));
1807 size_t age = header->age(); 1807 size_t age = header->age();
1808 if (header->isMarked()) { 1808 if (header->isMarked()) {
1809 info->liveCount[tag] += 1; 1809 info->liveCount[tag] += 1;
1810 info->liveSize[tag] += header->size(); 1810 info->liveSize[tag] += header->size();
1811 // Count objects that are live when promoted to the final generation. 1811 // Count objects that are live when promoted to the final generation.
1812 if (age == maxHeapObjectAge - 1) 1812 if (age == maxHeapObjectAge - 1)
1813 info->generations[tag][maxHeapObjectAge] += 1; 1813 info->generations[tag][maxHeapObjectAge] += 1;
1814 } else { 1814 } else {
1815 info->deadCount[tag] += 1; 1815 info->deadCount[tag] += 1;
1816 info->deadSize[tag] += header->size(); 1816 info->deadSize[tag] += header->size();
1817 // Count objects that are dead before the final generation. 1817 // Count objects that are dead before the final generation.
1818 if (age < maxHeapObjectAge) 1818 if (age < maxHeapObjectAge)
1819 info->generations[tag][age] += 1; 1819 info->generations[tag][age] += 1;
1820 } 1820 }
1821 1821
1822 if (json) { 1822 if (json) {
1823 json->setInteger("class", tag); 1823 json->setInteger("class", tag);
1824 json->setInteger("size", header->size()); 1824 json->setInteger("size", header->size());
1825 json->setInteger("isMarked", header->isMarked()); 1825 json->setInteger("isMarked", header->isMarked());
1826 } 1826 }
1827 } 1827 }
1828 1828
1829 void LargeObject::incrementMarkedObjectsAge() 1829 void LargeObjectPage::incrementMarkedObjectsAge()
1830 { 1830 {
1831 HeapObjectHeader* header = heapObjectHeader(); 1831 HeapObjectHeader* header = heapObjectHeader();
1832 if (header->isMarked()) 1832 if (header->isMarked())
1833 header->incrementAge(); 1833 header->incrementAge();
1834 } 1834 }
1835 1835
1836 void LargeObject::countMarkedObjects(ClassAgeCountsMap& classAgeCounts) 1836 void LargeObjectPage::countMarkedObjects(ClassAgeCountsMap& classAgeCounts)
1837 { 1837 {
1838 HeapObjectHeader* header = heapObjectHeader(); 1838 HeapObjectHeader* header = heapObjectHeader();
1839 if (header->isMarked()) { 1839 if (header->isMarked()) {
1840 String className(classOf(header->payload())); 1840 String className(classOf(header->payload()));
1841 ++(classAgeCounts.add(className, AgeCounts()).storedValue->value.ages[he ader->age()]); 1841 ++(classAgeCounts.add(className, AgeCounts()).storedValue->value.ages[he ader->age()]);
1842 } 1842 }
1843 } 1843 }
1844 1844
1845 void LargeObject::countObjectsToSweep(ClassAgeCountsMap& classAgeCounts) 1845 void LargeObjectPage::countObjectsToSweep(ClassAgeCountsMap& classAgeCounts)
1846 { 1846 {
1847 HeapObjectHeader* header = heapObjectHeader(); 1847 HeapObjectHeader* header = heapObjectHeader();
1848 if (!header->isFree() && !header->isMarked()) { 1848 if (!header->isFree() && !header->isMarked()) {
1849 String className(classOf(header->payload())); 1849 String className(classOf(header->payload()));
1850 ++(classAgeCounts.add(className, AgeCounts()).storedValue->value.ages[he ader->age()]); 1850 ++(classAgeCounts.add(className, AgeCounts()).storedValue->value.ages[he ader->age()]);
1851 } 1851 }
1852 } 1852 }
1853 #endif 1853 #endif
1854 1854
1855 void HeapDoesNotContainCache::flush() 1855 void HeapDoesNotContainCache::flush()
(...skipping 222 matching lines...) Expand 10 before | Expand all | Expand 10 after
2078 virtual void registerWeakCellWithCallback(void** cell, WeakPointerCallback c allback) override 2078 virtual void registerWeakCellWithCallback(void** cell, WeakPointerCallback c allback) override
2079 { 2079 {
2080 Impl::registerWeakCellWithCallback(cell, callback); 2080 Impl::registerWeakCellWithCallback(cell, callback);
2081 } 2081 }
2082 2082
2083 inline bool shouldMarkObject(const void* objectPointer) 2083 inline bool shouldMarkObject(const void* objectPointer)
2084 { 2084 {
2085 if (Mode != ThreadLocalMarking) 2085 if (Mode != ThreadLocalMarking)
2086 return true; 2086 return true;
2087 2087
2088 BaseHeapPage* page = pageFromObject(objectPointer); 2088 BasePage* page = pageFromObject(objectPointer);
2089 ASSERT(!page->orphaned()); 2089 ASSERT(!page->orphaned());
2090 // When doing a thread local GC, the marker checks if 2090 // When doing a thread local GC, the marker checks if
2091 // the object resides in another thread's heap. If it 2091 // the object resides in another thread's heap. If it
2092 // does, the object should not be marked & traced. 2092 // does, the object should not be marked & traced.
2093 return page->terminating(); 2093 return page->terminating();
2094 } 2094 }
2095 2095
2096 #if ENABLE(ASSERT) 2096 #if ENABLE(ASSERT)
2097 virtual void checkMarkingAllowed() override 2097 virtual void checkMarkingAllowed() override
2098 { 2098 {
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after
2149 delete s_ephemeronStack; 2149 delete s_ephemeronStack;
2150 s_ephemeronStack = nullptr; 2150 s_ephemeronStack = nullptr;
2151 delete s_regionTree; 2151 delete s_regionTree;
2152 s_regionTree = nullptr; 2152 s_regionTree = nullptr;
2153 GCInfoTable::shutdown(); 2153 GCInfoTable::shutdown();
2154 ThreadState::shutdown(); 2154 ThreadState::shutdown();
2155 ASSERT(Heap::allocatedSpace() == 0); 2155 ASSERT(Heap::allocatedSpace() == 0);
2156 } 2156 }
2157 2157
2158 #if ENABLE(ASSERT) 2158 #if ENABLE(ASSERT)
2159 BaseHeapPage* Heap::findPageFromAddress(Address address) 2159 BasePage* Heap::findPageFromAddress(Address address)
2160 { 2160 {
2161 ASSERT(ThreadState::current()->isInGC()); 2161 ASSERT(ThreadState::current()->isInGC());
2162 for (ThreadState* state : ThreadState::attachedThreads()) { 2162 for (ThreadState* state : ThreadState::attachedThreads()) {
2163 if (BaseHeapPage* page = state->findPageFromAddress(address)) 2163 if (BasePage* page = state->findPageFromAddress(address))
2164 return page; 2164 return page;
2165 } 2165 }
2166 return nullptr; 2166 return nullptr;
2167 } 2167 }
2168 2168
2169 bool Heap::containedInHeapOrOrphanedPage(void* object) 2169 bool Heap::containedInHeapOrOrphanedPage(void* object)
2170 { 2170 {
2171 return findPageFromAddress(object) || orphanedPagePool()->contains(object); 2171 return findPageFromAddress(object) || orphanedPagePool()->contains(object);
2172 } 2172 }
2173 #endif 2173 #endif
2174 2174
2175 Address Heap::checkAndMarkPointer(Visitor* visitor, Address address) 2175 Address Heap::checkAndMarkPointer(Visitor* visitor, Address address)
2176 { 2176 {
2177 ASSERT(ThreadState::current()->isInGC()); 2177 ASSERT(ThreadState::current()->isInGC());
2178 2178
2179 #if !ENABLE(ASSERT) 2179 #if !ENABLE(ASSERT)
2180 if (s_heapDoesNotContainCache->lookup(address)) 2180 if (s_heapDoesNotContainCache->lookup(address))
2181 return nullptr; 2181 return nullptr;
2182 #endif 2182 #endif
2183 2183
2184 if (BaseHeapPage* page = lookup(address)) { 2184 if (BasePage* page = lookup(address)) {
2185 ASSERT(page->contains(address)); 2185 ASSERT(page->contains(address));
2186 ASSERT(!page->orphaned()); 2186 ASSERT(!page->orphaned());
2187 ASSERT(!s_heapDoesNotContainCache->lookup(address)); 2187 ASSERT(!s_heapDoesNotContainCache->lookup(address));
2188 page->checkAndMarkPointer(visitor, address); 2188 page->checkAndMarkPointer(visitor, address);
2189 // FIXME: We only need to set the conservative flag if 2189 // FIXME: We only need to set the conservative flag if
2190 // checkAndMarkPointer actually marked the pointer. 2190 // checkAndMarkPointer actually marked the pointer.
2191 s_lastGCWasConservative = true; 2191 s_lastGCWasConservative = true;
2192 return address; 2192 return address;
2193 } 2193 }
2194 2194
(...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after
2288 2288
2289 void Heap::pushWeakCellPointerCallback(void** cell, WeakPointerCallback callback ) 2289 void Heap::pushWeakCellPointerCallback(void** cell, WeakPointerCallback callback )
2290 { 2290 {
2291 ASSERT(!Heap::orphanedPagePool()->contains(cell)); 2291 ASSERT(!Heap::orphanedPagePool()->contains(cell));
2292 CallbackStack::Item* slot = s_weakCallbackStack->allocateEntry(); 2292 CallbackStack::Item* slot = s_weakCallbackStack->allocateEntry();
2293 *slot = CallbackStack::Item(cell, callback); 2293 *slot = CallbackStack::Item(cell, callback);
2294 } 2294 }
2295 2295
2296 void Heap::pushWeakPointerCallback(void* closure, void* object, WeakPointerCallb ack callback) 2296 void Heap::pushWeakPointerCallback(void* closure, void* object, WeakPointerCallb ack callback)
2297 { 2297 {
2298 BaseHeapPage* page = pageFromObject(object); 2298 BasePage* page = pageFromObject(object);
2299 ASSERT(!page->orphaned()); 2299 ASSERT(!page->orphaned());
2300 ThreadState* state = page->heap()->threadState(); 2300 ThreadState* state = page->heap()->threadState();
2301 state->pushWeakPointerCallback(closure, callback); 2301 state->pushWeakPointerCallback(closure, callback);
2302 } 2302 }
2303 2303
2304 bool Heap::popAndInvokeWeakPointerCallback(Visitor* visitor) 2304 bool Heap::popAndInvokeWeakPointerCallback(Visitor* visitor)
2305 { 2305 {
2306 // For weak processing we should never reach orphaned pages since orphaned 2306 // For weak processing we should never reach orphaned pages since orphaned
2307 // pages are not traced and thus objects on those pages are never be 2307 // pages are not traced and thus objects on those pages are never be
2308 // registered as objects on orphaned pages. We cannot assert this here 2308 // registered as objects on orphaned pages. We cannot assert this here
(...skipping 222 matching lines...) Expand 10 before | Expand all | Expand 10 after
2531 for (int i = 0; i < 5; ++i) 2531 for (int i = 0; i < 5; ++i)
2532 collectGarbage(ThreadState::NoHeapPointersOnStack); 2532 collectGarbage(ThreadState::NoHeapPointersOnStack);
2533 } 2533 }
2534 2534
2535 double Heap::estimatedMarkingTime() 2535 double Heap::estimatedMarkingTime()
2536 { 2536 {
2537 // FIXME: Implement heuristics 2537 // FIXME: Implement heuristics
2538 return 0.0; 2538 return 0.0;
2539 } 2539 }
2540 2540
2541 void ThreadHeap::prepareHeapForTermination() 2541 void BaseHeap::prepareHeapForTermination()
2542 { 2542 {
2543 ASSERT(!m_firstUnsweptPage); 2543 ASSERT(!m_firstUnsweptPage);
2544 for (BaseHeapPage* page = m_firstPage; page; page = page->next()) { 2544 for (BasePage* page = m_firstPage; page; page = page->next()) {
2545 page->setTerminating(); 2545 page->setTerminating();
2546 } 2546 }
2547 } 2547 }
2548 2548
2549 size_t Heap::objectPayloadSizeForTesting() 2549 size_t Heap::objectPayloadSizeForTesting()
2550 { 2550 {
2551 size_t objectPayloadSize = 0; 2551 size_t objectPayloadSize = 0;
2552 for (ThreadState* state : ThreadState::attachedThreads()) { 2552 for (ThreadState* state : ThreadState::attachedThreads()) {
2553 state->setGCState(ThreadState::GCRunning); 2553 state->setGCState(ThreadState::GCRunning);
2554 state->makeConsistentForSweeping(); 2554 state->makeConsistentForSweeping();
(...skipping 10 matching lines...) Expand all
2565 if (!address) 2565 if (!address)
2566 return; 2566 return;
2567 2567
2568 ThreadState* state = ThreadState::current(); 2568 ThreadState* state = ThreadState::current();
2569 if (state->sweepForbidden()) 2569 if (state->sweepForbidden())
2570 return; 2570 return;
2571 ASSERT(!state->isInGC()); 2571 ASSERT(!state->isInGC());
2572 2572
2573 // Don't promptly free large objects because their page is never reused. 2573 // Don't promptly free large objects because their page is never reused.
2574 // Don't free backings allocated on other threads. 2574 // Don't free backings allocated on other threads.
2575 BaseHeapPage* page = pageFromObject(address); 2575 BasePage* page = pageFromObject(address);
2576 if (page->isLargeObject() || page->heap()->threadState() != state) 2576 if (page->isLargeObjectPage() || page->heap()->threadState() != state)
2577 return; 2577 return;
2578 2578
2579 HeapObjectHeader* header = HeapObjectHeader::fromPayload(address); 2579 HeapObjectHeader* header = HeapObjectHeader::fromPayload(address);
2580 header->checkHeader(); 2580 header->checkHeader();
2581 static_cast<HeapPage*>(page)->heapForHeapPage()->promptlyFreeObject(header); 2581 static_cast<NormalPage*>(page)->heapForNormalPage()->promptlyFreeObject(head er);
2582 } 2582 }
2583 2583
2584 void HeapAllocator::freeVectorBacking(void* address) 2584 void HeapAllocator::freeVectorBacking(void* address)
2585 { 2585 {
2586 backingFree(address); 2586 backingFree(address);
2587 } 2587 }
2588 2588
2589 void HeapAllocator::freeInlineVectorBacking(void* address) 2589 void HeapAllocator::freeInlineVectorBacking(void* address)
2590 { 2590 {
2591 backingFree(address); 2591 backingFree(address);
(...skipping 10 matching lines...) Expand all
2602 return false; 2602 return false;
2603 2603
2604 ThreadState* state = ThreadState::current(); 2604 ThreadState* state = ThreadState::current();
2605 if (state->sweepForbidden()) 2605 if (state->sweepForbidden())
2606 return false; 2606 return false;
2607 ASSERT(!state->isInGC()); 2607 ASSERT(!state->isInGC());
2608 ASSERT(state->isAllocationAllowed()); 2608 ASSERT(state->isAllocationAllowed());
2609 2609
2610 // FIXME: Support expand for large objects. 2610 // FIXME: Support expand for large objects.
2611 // Don't expand backings allocated on other threads. 2611 // Don't expand backings allocated on other threads.
2612 BaseHeapPage* page = pageFromObject(address); 2612 BasePage* page = pageFromObject(address);
2613 if (page->isLargeObject() || page->heap()->threadState() != state) 2613 if (page->isLargeObjectPage() || page->heap()->threadState() != state)
2614 return false; 2614 return false;
2615 2615
2616 HeapObjectHeader* header = HeapObjectHeader::fromPayload(address); 2616 HeapObjectHeader* header = HeapObjectHeader::fromPayload(address);
2617 header->checkHeader(); 2617 header->checkHeader();
2618 return static_cast<HeapPage*>(page)->heapForHeapPage()->expandObject(header, newSize); 2618 return static_cast<NormalPage*>(page)->heapForNormalPage()->expandObject(hea der, newSize);
2619 } 2619 }
2620 2620
2621 bool HeapAllocator::expandVectorBacking(void* address, size_t newSize) 2621 bool HeapAllocator::expandVectorBacking(void* address, size_t newSize)
2622 { 2622 {
2623 return backingExpand(address, newSize); 2623 return backingExpand(address, newSize);
2624 } 2624 }
2625 2625
2626 bool HeapAllocator::expandInlineVectorBacking(void* address, size_t newSize) 2626 bool HeapAllocator::expandInlineVectorBacking(void* address, size_t newSize)
2627 { 2627 {
2628 return backingExpand(address, newSize); 2628 return backingExpand(address, newSize);
(...skipping 16 matching lines...) Expand all
2645 return; 2645 return;
2646 2646
2647 ThreadState* state = ThreadState::current(); 2647 ThreadState* state = ThreadState::current();
2648 if (state->sweepForbidden()) 2648 if (state->sweepForbidden())
2649 return; 2649 return;
2650 ASSERT(!state->isInGC()); 2650 ASSERT(!state->isInGC());
2651 ASSERT(state->isAllocationAllowed()); 2651 ASSERT(state->isAllocationAllowed());
2652 2652
2653 // FIXME: Support shrink for large objects. 2653 // FIXME: Support shrink for large objects.
2654 // Don't shrink backings allocated on other threads. 2654 // Don't shrink backings allocated on other threads.
2655 BaseHeapPage* page = pageFromObject(address); 2655 BasePage* page = pageFromObject(address);
2656 if (page->isLargeObject() || page->heap()->threadState() != state) 2656 if (page->isLargeObjectPage() || page->heap()->threadState() != state)
2657 return; 2657 return;
2658 2658
2659 HeapObjectHeader* header = HeapObjectHeader::fromPayload(address); 2659 HeapObjectHeader* header = HeapObjectHeader::fromPayload(address);
2660 header->checkHeader(); 2660 header->checkHeader();
2661 static_cast<HeapPage*>(page)->heapForHeapPage()->shrinkObject(header, quanti zedShrunkSize); 2661 static_cast<NormalPage*>(page)->heapForNormalPage()->shrinkObject(header, qu antizedShrunkSize);
2662 } 2662 }
2663 2663
2664 void HeapAllocator::shrinkVectorBackingInternal(void* address, size_t quantizedC urrentSize, size_t quantizedShrunkSize) 2664 void HeapAllocator::shrinkVectorBackingInternal(void* address, size_t quantizedC urrentSize, size_t quantizedShrunkSize)
2665 { 2665 {
2666 backingShrink(address, quantizedCurrentSize, quantizedShrunkSize); 2666 backingShrink(address, quantizedCurrentSize, quantizedShrunkSize);
2667 } 2667 }
2668 2668
2669 void HeapAllocator::shrinkInlineVectorBackingInternal(void* address, size_t quan tizedCurrentSize, size_t quantizedShrunkSize) 2669 void HeapAllocator::shrinkInlineVectorBackingInternal(void* address, size_t quan tizedCurrentSize, size_t quantizedShrunkSize)
2670 { 2670 {
2671 backingShrink(address, quantizedCurrentSize, quantizedShrunkSize); 2671 backingShrink(address, quantizedCurrentSize, quantizedShrunkSize);
2672 } 2672 }
2673 2673
2674 BaseHeapPage* Heap::lookup(Address address) 2674 BasePage* Heap::lookup(Address address)
2675 { 2675 {
2676 ASSERT(ThreadState::current()->isInGC()); 2676 ASSERT(ThreadState::current()->isInGC());
2677 if (!s_regionTree) 2677 if (!s_regionTree)
2678 return nullptr; 2678 return nullptr;
2679 if (PageMemoryRegion* region = s_regionTree->lookup(address)) { 2679 if (PageMemoryRegion* region = s_regionTree->lookup(address)) {
2680 BaseHeapPage* page = region->pageFromAddress(address); 2680 BasePage* page = region->pageFromAddress(address);
2681 return page && !page->orphaned() ? page : nullptr; 2681 return page && !page->orphaned() ? page : nullptr;
2682 } 2682 }
2683 return nullptr; 2683 return nullptr;
2684 } 2684 }
2685 2685
2686 static Mutex& regionTreeMutex() 2686 static Mutex& regionTreeMutex()
2687 { 2687 {
2688 AtomicallyInitializedStaticReference(Mutex, mutex, new Mutex); 2688 AtomicallyInitializedStaticReference(Mutex, mutex, new Mutex);
2689 return mutex; 2689 return mutex;
2690 } 2690 }
(...skipping 80 matching lines...) Expand 10 before | Expand all | Expand 10 after
2771 bool Heap::s_shutdownCalled = false; 2771 bool Heap::s_shutdownCalled = false;
2772 bool Heap::s_lastGCWasConservative = false; 2772 bool Heap::s_lastGCWasConservative = false;
2773 FreePagePool* Heap::s_freePagePool; 2773 FreePagePool* Heap::s_freePagePool;
2774 OrphanedPagePool* Heap::s_orphanedPagePool; 2774 OrphanedPagePool* Heap::s_orphanedPagePool;
2775 Heap::RegionTree* Heap::s_regionTree = nullptr; 2775 Heap::RegionTree* Heap::s_regionTree = nullptr;
2776 size_t Heap::s_allocatedObjectSize = 0; 2776 size_t Heap::s_allocatedObjectSize = 0;
2777 size_t Heap::s_allocatedSpace = 0; 2777 size_t Heap::s_allocatedSpace = 0;
2778 size_t Heap::s_markedObjectSize = 0; 2778 size_t Heap::s_markedObjectSize = 0;
2779 2779
2780 } // namespace blink 2780 } // namespace blink
OLDNEW
« no previous file with comments | « Source/platform/heap/Heap.h ('k') | Source/platform/heap/HeapTest.cpp » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698