[heap] Enforce explicit MarkingState

src/heap/spaces.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/spaces.h"
 
 #include <utility>
 
 #include "src/base/bits.h"
 #include "src/base/platform/platform.h"
@@ skipping 520 matching lines @@
   chunk->old_to_old_slots_ = nullptr;
   chunk->typed_old_to_new_slots_.SetValue(nullptr);
   chunk->typed_old_to_old_slots_ = nullptr;
   chunk->skip_list_ = nullptr;
   chunk->progress_bar_ = 0;
   chunk->high_water_mark_.SetValue(static_cast<intptr_t>(area_start - base));
   chunk->concurrent_sweeping_state().SetValue(kSweepingDone);
   chunk->mutex_ = new base::Mutex();
   chunk->available_in_free_list_ = 0;
   chunk->wasted_memory_ = 0;
-  chunk->ClearLiveness();
   chunk->young_generation_bitmap_ = nullptr;
   chunk->set_next_chunk(nullptr);
   chunk->set_prev_chunk(nullptr);
   chunk->local_tracker_ = nullptr;
 
+  MarkingState::Internal(chunk).ClearLiveness();
+
   DCHECK(OFFSET_OF(MemoryChunk, flags_) == kFlagsOffset);
 
   if (executable == EXECUTABLE) {
     chunk->SetFlag(IS_EXECUTABLE);
   }
 
   if (reservation != nullptr) {
     chunk->reservation_.TakeControl(reservation);
   }
 
@@ skipping 290 matching lines @@
     CHECK_EQ(filler->address() + filler->Size(), area_end());
   }
   return unused;
 }
 
 void Page::CreateBlackArea(Address start, Address end) {
   DCHECK(heap()->incremental_marking()->black_allocation());
   DCHECK_EQ(Page::FromAddress(start), this);
   DCHECK_NE(start, end);
   DCHECK_EQ(Page::FromAddress(end - 1), this);
-  markbits()->SetRange(AddressToMarkbitIndex(start),
-                       AddressToMarkbitIndex(end));
-  IncrementLiveBytes(static_cast<int>(end - start));
+  MarkingState::Internal(this).bitmap()->SetRange(AddressToMarkbitIndex(start),
+                                                  AddressToMarkbitIndex(end));
+  MarkingState::Internal(this).IncrementLiveBytes(
+      static_cast<int>(end - start));
 }
 
 void MemoryAllocator::PartialFreeMemory(MemoryChunk* chunk,
                                         Address start_free) {
   // We do not allow partial shrink for code.
   DCHECK(chunk->executable() == NOT_EXECUTABLE);
 
   intptr_t size;
   base::VirtualMemory* reservation = chunk->reserved_memory();
   DCHECK(reservation->IsReserved());
@@ skipping 319 matching lines @@
   DCHECK_NULL(young_generation_bitmap_);
   young_generation_bitmap_ = static_cast<Bitmap*>(calloc(1, Bitmap::kSize));
 }
 
 void MemoryChunk::ReleaseYoungGenerationBitmap() {
   DCHECK_NOT_NULL(young_generation_bitmap_);
   free(young_generation_bitmap_);
   young_generation_bitmap_ = nullptr;
 }
 
-template <MarkingMode mode>
-void MemoryChunk::ClearLiveness() {
-  markbits<mode>()->Clear();
-  ResetLiveBytes<mode>();
-}
-
-template void MemoryChunk::ClearLiveness<MarkingMode::FULL>();
-template void MemoryChunk::ClearLiveness<MarkingMode::YOUNG_GENERATION>();
-
 // -----------------------------------------------------------------------------
 // PagedSpace implementation
 
 STATIC_ASSERT(static_cast<ObjectSpace>(1 << AllocationSpace::NEW_SPACE) ==
               ObjectSpace::kObjectSpaceNewSpace);
 STATIC_ASSERT(static_cast<ObjectSpace>(1 << AllocationSpace::OLD_SPACE) ==
               ObjectSpace::kObjectSpaceOldSpace);
 STATIC_ASSERT(static_cast<ObjectSpace>(1 << AllocationSpace::CODE_SPACE) ==
               ObjectSpace::kObjectSpaceCodeSpace);
 STATIC_ASSERT(static_cast<ObjectSpace>(1 << AllocationSpace::MAP_SPACE) ==
@@ skipping 195 matching lines @@
   if (current_top == nullptr) {
     DCHECK(current_limit == nullptr);
     return;
   }
 
   if (heap()->incremental_marking()->black_allocation()) {
     Page* page = Page::FromAllocationAreaAddress(current_top);
 
     // Clear the bits in the unused black area.
     if (current_top != current_limit) {
-      page->markbits()->ClearRange(page->AddressToMarkbitIndex(current_top),
-                                   page->AddressToMarkbitIndex(current_limit));
-      page->IncrementLiveBytes(-static_cast<int>(current_limit - current_top));
+      MarkingState::Internal(page).bitmap()->ClearRange(
+          page->AddressToMarkbitIndex(current_top),
+          page->AddressToMarkbitIndex(current_limit));
+      MarkingState::Internal(page).IncrementLiveBytes(
+          -static_cast<int>(current_limit - current_top));
     }
   }
 
   SetTopAndLimit(NULL, NULL);
   DCHECK_GE(current_limit, current_top);
   Free(current_top, current_limit - current_top);
 }
 
 void PagedSpace::IncreaseCapacity(size_t bytes) {
   accounting_stats_.ExpandSpace(bytes);
 }
 
 void PagedSpace::ReleasePage(Page* page) {
-  DCHECK_EQ(page->LiveBytes(), 0);
+  DCHECK_EQ(0, MarkingState::Internal(page).live_bytes());
   DCHECK_EQ(page->owner(), this);
 
   free_list_.EvictFreeListItems(page);
   DCHECK(!free_list_.ContainsPageFreeListItems(page));
 
   if (Page::FromAllocationAreaAddress(allocation_info_.top()) == page) {
     allocation_info_.Reset(nullptr, nullptr);
   }
 
   // If page is still in a list, unlink it from that list.
@@ skipping 40 matching lines @@
 
       // Perform space-specific object verification.
       VerifyObject(object);
 
       // The object itself should look OK.
       object->ObjectVerify();
 
       // All the interior pointers should be contained in the heap.
       int size = object->Size();
       object->IterateBody(map->instance_type(), size, visitor);
-      if (ObjectMarking::IsBlack(object)) {
+      if (ObjectMarking::IsBlack(object, MarkingState::Internal(object))) {
         black_size += size;
       }
 
       CHECK(object->address() + size <= top);
       end_of_previous_object = object->address() + size;
     }
-    CHECK_LE(black_size, page->LiveBytes());
+    CHECK_LE(black_size, MarkingState::Internal(page).live_bytes());
   }
   CHECK(allocation_pointer_found_in_space);
 }
 #endif  // VERIFY_HEAP
 
 // -----------------------------------------------------------------------------
 // NewSpace implementation
 
 bool NewSpace::SetUp(size_t initial_semispace_capacity,
                      size_t maximum_semispace_capacity) {
@@ skipping 112 matching lines @@
       }
     }
     while (actual_pages < expected_pages) {
       actual_pages++;
       current_page =
           heap()->memory_allocator()->AllocatePage<MemoryAllocator::kPooled>(
               Page::kAllocatableMemory, this, executable());
       if (current_page == nullptr) return false;
       DCHECK_NOT_NULL(current_page);
       current_page->InsertAfter(anchor());
-      current_page->ClearLiveness();
+      MarkingState::Internal(current_page).ClearLiveness();
       current_page->SetFlags(anchor()->prev_page()->GetFlags(),
                              static_cast<uintptr_t>(Page::kCopyAllFlags));
       heap()->CreateFillerObjectAt(current_page->area_start(),
                                    static_cast<int>(current_page->area_size()),
                                    ClearRecordedSlots::kNo);
     }
   }
   return true;
 }
 
@@ skipping 51 matching lines @@
   DCHECK_SEMISPACE_ALLOCATION_INFO(allocation_info_, to_space_);
 }
 
 
 void NewSpace::ResetAllocationInfo() {
   Address old_top = allocation_info_.top();
   to_space_.Reset();
   UpdateAllocationInfo();
   // Clear all mark-bits in the to-space.
   for (Page* p : to_space_) {
-    p->ClearLiveness();
+    MarkingState::Internal(p).ClearLiveness();
   }
   InlineAllocationStep(old_top, allocation_info_.top(), nullptr, 0);
 }
 
 
 void NewSpace::UpdateInlineAllocationLimit(int size_in_bytes) {
   if (heap()->inline_allocation_disabled()) {
     // Lowest limit when linear allocation was disabled.
     Address high = to_space_.page_high();
     Address new_top = allocation_info_.top() + size_in_bytes;
@@ skipping 280 matching lines @@
   DCHECK_NE(last_page, anchor());
   for (int pages_added = 0; pages_added < delta_pages; pages_added++) {
     Page* new_page =
         heap()->memory_allocator()->AllocatePage<MemoryAllocator::kPooled>(
             Page::kAllocatableMemory, this, executable());
     if (new_page == nullptr) {
       RewindPages(last_page, pages_added);
       return false;
     }
     new_page->InsertAfter(last_page);
-    new_page->ClearLiveness();
+    MarkingState::Internal(new_page).ClearLiveness();
     // Duplicate the flags that was set on the old page.
     new_page->SetFlags(last_page->GetFlags(), Page::kCopyOnFlipFlagsMask);
     last_page = new_page;
   }
   AccountCommitted(delta);
   current_capacity_ = new_capacity;
   return true;
 }
 
 void SemiSpace::RewindPages(Page* start, int num_pages) {
@@ skipping 40 matching lines @@
   anchor_.prev_page()->set_next_page(&anchor_);
   anchor_.next_page()->set_prev_page(&anchor_);
 
   for (Page* page : *this) {
     page->set_owner(this);
     page->SetFlags(flags, mask);
     if (id_ == kToSpace) {
       page->ClearFlag(MemoryChunk::IN_FROM_SPACE);
       page->SetFlag(MemoryChunk::IN_TO_SPACE);
       page->ClearFlag(MemoryChunk::NEW_SPACE_BELOW_AGE_MARK);
-      page->ResetLiveBytes();
+      MarkingState::Internal(page).SetLiveBytes(0);
     } else {
       page->SetFlag(MemoryChunk::IN_FROM_SPACE);
       page->ClearFlag(MemoryChunk::IN_TO_SPACE);
     }
     DCHECK(page->IsFlagSet(MemoryChunk::IN_TO_SPACE) ||
            page->IsFlagSet(MemoryChunk::IN_FROM_SPACE));
   }
 }
 
 
@@ skipping 952 matching lines @@
   }
 
   heap()->StartIncrementalMarkingIfAllocationLimitIsReached(Heap::kNoGCFlags,
                                                             kNoGCCallbackFlags);
   AllocationStep(object->address(), object_size);
 
   heap()->CreateFillerObjectAt(object->address(), object_size,
                                ClearRecordedSlots::kNo);
 
   if (heap()->incremental_marking()->black_allocation()) {
-    ObjectMarking::WhiteToBlack(object);
+    ObjectMarking::WhiteToBlack(object, MarkingState::Internal(object));
   }
   return object;
 }
 
 
 size_t LargeObjectSpace::CommittedPhysicalMemory() {
   // On a platform that provides lazy committing of memory, we over-account
   // the actually committed memory. There is no easy way right now to support
   // precise accounting of committed memory in large object space.
   return CommittedMemory();
@@ skipping 26 matching lines @@
       return page;
     }
   }
   return NULL;
 }
 
 
 void LargeObjectSpace::ClearMarkingStateOfLiveObjects() {
   LargeObjectIterator it(this);
   for (HeapObject* obj = it.Next(); obj != NULL; obj = it.Next()) {
-    if (ObjectMarking::IsBlackOrGrey(obj)) {
-      Marking::MarkWhite(ObjectMarking::MarkBitFrom(obj));
+    if (ObjectMarking::IsBlackOrGrey(obj, MarkingState::Internal(obj))) {
+      Marking::MarkWhite(
+          ObjectMarking::MarkBitFrom(obj, MarkingState::Internal(obj)));
       MemoryChunk* chunk = MemoryChunk::FromAddress(obj->address());
       chunk->ResetProgressBar();
-      chunk->ResetLiveBytes();
+      MarkingState::Internal(chunk).SetLiveBytes(0);
     }
-    DCHECK(ObjectMarking::IsWhite(obj));
+    DCHECK(ObjectMarking::IsWhite(obj, MarkingState::Internal(obj)));
   }
 }
 
 void LargeObjectSpace::InsertChunkMapEntries(LargePage* page) {
   // Register all MemoryChunk::kAlignment-aligned chunks covered by
   // this large page in the chunk map.
   uintptr_t start = reinterpret_cast<uintptr_t>(page) / MemoryChunk::kAlignment;
   uintptr_t limit = (reinterpret_cast<uintptr_t>(page) + (page->size() - 1)) /
                     MemoryChunk::kAlignment;
   // There may be concurrent access on the chunk map. We have to take the lock
@@ skipping 21 matching lines @@
   for (uintptr_t key = start; key <= limit; key++) {
     chunk_map_.Remove(reinterpret_cast<void*>(key), static_cast<uint32_t>(key));
   }
 }
 
 void LargeObjectSpace::FreeUnmarkedObjects() {
   LargePage* previous = NULL;
   LargePage* current = first_page_;
   while (current != NULL) {
     HeapObject* object = current->GetObject();
-    DCHECK(!ObjectMarking::IsGrey(object));
-    if (ObjectMarking::IsBlack(object)) {
+    DCHECK(!ObjectMarking::IsGrey(object, MarkingState::Internal(object)));
+    if (ObjectMarking::IsBlack(object, MarkingState::Internal(object))) {
       Address free_start;
       if ((free_start = current->GetAddressToShrink()) != 0) {
         // TODO(hpayer): Perform partial free concurrently.
         current->ClearOutOfLiveRangeSlots(free_start);
         RemoveChunkMapEntries(current, free_start);
         heap()->memory_allocator()->PartialFreeMemory(current, free_start);
       }
       previous = current;
       current = current->next_page();
     } else {
@@ skipping 115 matching lines @@
 
 void Page::Print() {
   // Make a best-effort to print the objects in the page.
   PrintF("Page@%p in %s\n", static_cast<void*>(this->address()),
          AllocationSpaceName(this->owner()->identity()));
   printf(" --------------------------------------\n");
   HeapObjectIterator objects(this);
   unsigned mark_size = 0;
   for (HeapObject* object = objects.Next(); object != NULL;
        object = objects.Next()) {
-    bool is_marked = ObjectMarking::IsBlackOrGrey(object);
+    bool is_marked =
+        ObjectMarking::IsBlackOrGrey(object, MarkingState::Internal(object));
     PrintF(" %c ", (is_marked ? '!' : ' '));  // Indent a little.
     if (is_marked) {
       mark_size += object->Size();
     }
     object->ShortPrint();
     PrintF("\n");
   }
   printf(" --------------------------------------\n");
-  printf(" Marked: %x, LiveCount: %x\n", mark_size, LiveBytes());
+  printf(" Marked: %x, LiveCount: %" V8PRIdPTR "\n", mark_size,
+         MarkingState::Internal(this).live_bytes());
 }
 
 #endif  // DEBUG
 }  // namespace internal
 }  // namespace v8