[heap] Reland "Remove black pages and use black areas instead."

src/heap/spaces.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/spaces.h"
 
 #include "src/base/bits.h"
 #include "src/base/platform/platform.h"
 #include "src/base/platform/semaphore.h"
 #include "src/full-codegen/full-codegen.h"
@@ skipping 1211 matching lines @@
   if (!heap()->CanExpandOldGeneration(size)) return false;
 
   Page* p = heap()->memory_allocator()->AllocatePage(size, this, executable());
   if (p == nullptr) return false;
 
   AccountCommitted(static_cast<intptr_t>(p->size()));
 
   // Pages created during bootstrapping may contain immortal immovable objects.
   if (!heap()->deserialization_complete()) p->MarkNeverEvacuate();
 
-  // When incremental marking was activated, old space pages are allocated
-  // black.
-  if (heap()->incremental_marking()->black_allocation() &&
-      identity() == OLD_SPACE) {
-    p->markbits()->SetAllBits();
-    p->SetFlag(Page::BLACK_PAGE);
-    if (FLAG_trace_incremental_marking) {
-      PrintIsolate(heap()->isolate(), "Added black page %p\n",
-                   static_cast<void*>(p));
-    }
-  }
-
   DCHECK(Capacity() <= heap()->MaxOldGenerationSize());
 
   p->InsertAfter(anchor_.prev_page());
 
   return true;
 }
 
 
 int PagedSpace::CountTotalPages() {
   int count = 0;
   for (Page* page : *this) {
     count++;
     USE(page);
   }
   return count;
 }
 
 
 void PagedSpace::ResetFreeListStatistics() {
   for (Page* page : *this) {
     page->ResetFreeListStatistics();
   }
 }
 
+void PagedSpace::SetAllocationInfo(Address top, Address limit) {
+  SetTopAndLimit(top, limit);
+  if (top != nullptr && top != limit &&
+      heap()->incremental_marking()->black_allocation()) {
+    Page* page = Page::FromAddress(top);
+    page->markbits()->SetRange(page->AddressToMarkbitIndex(top),
+                               page->AddressToMarkbitIndex(limit));
+    page->IncrementLiveBytes(static_cast<int>(limit - top));
+  }
+}
+
+void PagedSpace::MarkAllocationInfoBlack() {
+  DCHECK(heap()->incremental_marking()->black_allocation());
+  Address current_top = top();
+  Address current_limit = limit();
+  if (current_top != nullptr && current_top != current_limit) {
+    Page* page = Page::FromAddress(current_top);
+    page->markbits()->SetRange(page->AddressToMarkbitIndex(current_top),
+                               page->AddressToMarkbitIndex(current_limit));
+    page->IncrementLiveBytes(static_cast<int>(current_limit - current_top));
+  }
+}
+
+// Empty space allocation info, returning unused area to free list.
+void PagedSpace::EmptyAllocationInfo() {
+  // Mark the old linear allocation area with a free space map so it can be
+  // skipped when scanning the heap.
+  Address current_top = top();
+  Address current_limit = limit();
+  if (current_top == nullptr) {
+    DCHECK(current_limit == nullptr);
+    return;
+  }
+  int old_linear_size = static_cast<int>(current_limit - current_top);
+  SetTopAndLimit(NULL, NULL);
+  if (current_top != current_limit &&
+      heap()->incremental_marking()->black_allocation()) {
+    Page* page = Page::FromAddress(current_top);
+    page->markbits()->ClearRange(page->AddressToMarkbitIndex(current_top),
+                                 page->AddressToMarkbitIndex(current_limit));
+    page->IncrementLiveBytes(-static_cast<int>(current_limit - current_top));
+  }
+  Free(current_top, old_linear_size);
+}
 
 void PagedSpace::IncreaseCapacity(int size) {
   accounting_stats_.ExpandSpace(size);
 }
 
 void PagedSpace::ReleasePage(Page* page) {
   DCHECK_EQ(page->LiveBytes(), 0);
   DCHECK_EQ(AreaSize(), page->area_size());
   DCHECK_EQ(page->owner(), this);
 
@@ skipping 46 matching lines @@
 
       // Perform space-specific object verification.
       VerifyObject(object);
 
       // The object itself should look OK.
       object->ObjectVerify();
 
       // All the interior pointers should be contained in the heap.
       int size = object->Size();
       object->IterateBody(map->instance_type(), size, visitor);
-      if (!page->IsFlagSet(Page::BLACK_PAGE) &&
-          Marking::IsBlack(ObjectMarking::MarkBitFrom(object))) {
+      if (Marking::IsBlack(ObjectMarking::MarkBitFrom(object))) {
         black_size += size;
       }
 
       CHECK(object->address() + size <= top);
       end_of_previous_object = object->address() + size;
     }
     CHECK_LE(black_size, page->LiveBytes());
   }
   CHECK(allocation_pointer_found_in_space);
 }
@@ skipping 1076 matching lines @@
   DCHECK(0 < size_in_bytes);
   DCHECK(size_in_bytes <= kMaxBlockSize);
   DCHECK(IsAligned(size_in_bytes, kPointerSize));
   // Don't free list allocate if there is linear space available.
   DCHECK(owner_->limit() - owner_->top() < size_in_bytes);
 
   int old_linear_size = static_cast<int>(owner_->limit() - owner_->top());
   // Mark the old linear allocation area with a free space map so it can be
   // skipped when scanning the heap.  This also puts it back in the free list
   // if it is big enough.
-  owner_->Free(owner_->top(), old_linear_size);
-  owner_->SetTopAndLimit(nullptr, nullptr);
+  owner_->EmptyAllocationInfo();
 
   owner_->heap()->incremental_marking()->OldSpaceStep(size_in_bytes -
                                                       old_linear_size);
 
   int new_node_size = 0;
   FreeSpace* new_node = FindNodeFor(size_in_bytes, &new_node_size);
   if (new_node == nullptr) return nullptr;
 
   int bytes_left = new_node_size - size_in_bytes;
   DCHECK(bytes_left >= 0);
@@ skipping 13 matching lines @@
   const int kThreshold = IncrementalMarking::kAllocatedThreshold;
 
   // Memory in the linear allocation area is counted as allocated.  We may free
   // a little of this again immediately - see below.
   owner_->Allocate(new_node_size);
 
   if (owner_->heap()->inline_allocation_disabled()) {
     // Keep the linear allocation area empty if requested to do so, just
     // return area back to the free list instead.
     owner_->Free(new_node->address() + size_in_bytes, bytes_left);
-    owner_->SetTopAndLimit(new_node->address() + size_in_bytes,
-                           new_node->address() + size_in_bytes);
+    owner_->SetAllocationInfo(new_node->address() + size_in_bytes,
+                              new_node->address() + size_in_bytes);
   } else if (bytes_left > kThreshold &&
              owner_->heap()->incremental_marking()->IsMarkingIncomplete() &&
              FLAG_incremental_marking) {
     int linear_size = owner_->RoundSizeDownToObjectAlignment(kThreshold);
     // We don't want to give too large linear areas to the allocator while
     // incremental marking is going on, because we won't check again whether
     // we want to do another increment until the linear area is used up.
     owner_->Free(new_node->address() + size_in_bytes + linear_size,
                  new_node_size - size_in_bytes - linear_size);
-    owner_->SetTopAndLimit(new_node->address() + size_in_bytes,
-                           new_node->address() + size_in_bytes + linear_size);
+    owner_->SetAllocationInfo(
+        new_node->address() + size_in_bytes,
+        new_node->address() + size_in_bytes + linear_size);
   } else {
     DCHECK(bytes_left >= 0);
     // Normally we give the rest of the node to the allocator as its new
     // linear allocation area.
-    owner_->SetTopAndLimit(new_node->address() + size_in_bytes,
-                           new_node->address() + new_node_size);
+    owner_->SetAllocationInfo(new_node->address() + size_in_bytes,
+                              new_node->address() + new_node_size);
   }
 
   owner_->AllocationStep(new_node->address(), size_in_bytes);
 
   return new_node;
 }
 
 intptr_t FreeList::EvictFreeListItems(Page* page) {
   intptr_t sum = 0;
   page->ForAllFreeListCategories(
@@ skipping 369 matching lines @@
   if (Heap::ShouldZapGarbage()) {
     // Make the object consistent so the heap can be verified in OldSpaceStep.
     // We only need to do this in debug builds or if verify_heap is on.
     reinterpret_cast<Object**>(object->address())[0] =
         heap()->fixed_array_map();
     reinterpret_cast<Object**>(object->address())[1] = Smi::FromInt(0);
   }
 
   heap()->incremental_marking()->OldSpaceStep(object_size);
   AllocationStep(object->address(), object_size);
+
+  if (heap()->incremental_marking()->black_allocation()) {
+    Marking::MarkBlack(ObjectMarking::MarkBitFrom(object));
+    MemoryChunk::IncrementLiveBytesFromGC(object, object_size);
+  }
   return object;
 }
 
 
 size_t LargeObjectSpace::CommittedPhysicalMemory() {
   // On a platform that provides lazy committing of memory, we over-account
   // the actually committed memory. There is no easy way right now to support
   // precise accounting of committed memory in large object space.
   return CommittedMemory();
 }
@@ skipping 213 matching lines @@
     object->ShortPrint();
     PrintF("\n");
   }
   printf(" --------------------------------------\n");
   printf(" Marked: %x, LiveCount: %x\n", mark_size, LiveBytes());
 }
 
 #endif  // DEBUG
 }  // namespace internal
 }  // namespace v8