[heap] Register end of black areas to support faster filtering of invalid slots.

src/heap/spaces.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/spaces.h"
 
 #include "src/base/bits.h"
 #include "src/base/platform/platform.h"
 #include "src/base/platform/semaphore.h"
 #include "src/full-codegen/full-codegen.h"
@@ skipping 486 matching lines @@
   chunk->high_water_mark_.SetValue(static_cast<intptr_t>(area_start - base));
   chunk->concurrent_sweeping_state().SetValue(kSweepingDone);
   chunk->mutex_ = new base::Mutex();
   chunk->available_in_free_list_ = 0;
   chunk->wasted_memory_ = 0;
   chunk->ResetLiveBytes();
   chunk->ClearLiveness();
   chunk->set_next_chunk(nullptr);
   chunk->set_prev_chunk(nullptr);
   chunk->local_tracker_ = nullptr;
+  chunk->black_area_end_marker_map_ = nullptr;
 
   DCHECK(OFFSET_OF(MemoryChunk, flags_) == kFlagsOffset);
   DCHECK(OFFSET_OF(MemoryChunk, live_byte_count_) == kLiveBytesOffset);
 
   if (executable == EXECUTABLE) {
     chunk->SetFlag(IS_EXECUTABLE);
   }
 
   if (reservation != nullptr) {
     chunk->reservation_.TakeControl(reservation);
@@ skipping 762 matching lines @@
 // Empty space allocation info, returning unused area to free list.
 void PagedSpace::EmptyAllocationInfo() {
   // Mark the old linear allocation area with a free space map so it can be
   // skipped when scanning the heap.
   Address current_top = top();
   Address current_limit = limit();
   if (current_top == nullptr) {
     DCHECK(current_limit == nullptr);
     return;
   }
-  int old_linear_size = static_cast<int>(current_limit - current_top);
+
+  if (heap()->incremental_marking()->black_allocation()) {
+    Page* page = Page::FromAddress(current_top);
+    // We have to remember the end of the current black allocation area if
+    // something was allocated in the current bump pointer range.
+    if (allocation_info_.original_top() != current_top) {
+      Address end_black_area = current_top - kPointerSize;
+      page->AddBlackAreaEndMarker(end_black_area);
+    }
+
+    // Clear the bits in the unused black area.
+    if (current_top != current_limit) {
+      page->markbits()->ClearRange(page->AddressToMarkbitIndex(current_top),
+                                   page->AddressToMarkbitIndex(current_limit));
+      page->IncrementLiveBytes(-static_cast<int>(current_limit - current_top));
+    }
+  }
+
   SetTopAndLimit(NULL, NULL);
-  if (current_top != current_limit &&
-      heap()->incremental_marking()->black_allocation()) {
-    Page* page = Page::FromAddress(current_top);
-    page->markbits()->ClearRange(page->AddressToMarkbitIndex(current_top),
-                                 page->AddressToMarkbitIndex(current_limit));
-    page->IncrementLiveBytes(-static_cast<int>(current_limit - current_top));
-  }
-  Free(current_top, old_linear_size);
+  Free(current_top, static_cast<int>(current_limit - current_top));
 }
 
 void PagedSpace::IncreaseCapacity(int size) {
   accounting_stats_.ExpandSpace(size);
 }
 
 void PagedSpace::ReleasePage(Page* page) {
   DCHECK_EQ(page->LiveBytes(), 0);
   DCHECK_EQ(AreaSize(), page->area_size());
   DCHECK_EQ(page->owner(), this);
 
   free_list_.EvictFreeListItems(page);
   DCHECK(!free_list_.ContainsPageFreeListItems(page));
 
+  page->ReleaseBlackAreaEndMarkerMap();
+
   if (Page::FromAllocationAreaAddress(allocation_info_.top()) == page) {
     allocation_info_.Reset(nullptr, nullptr);
   }
 
   // If page is still in a list, unlink it from that list.
   if (page->next_chunk() != NULL) {
     DCHECK(page->prev_chunk() != NULL);
     page->Unlink();
   }
 
@@ skipping 1809 matching lines @@
     object->ShortPrint();
     PrintF("\n");
   }
   printf(" --------------------------------------\n");
   printf(" Marked: %x, LiveCount: %x\n", mark_size, LiveBytes());
 }
 
 #endif  // DEBUG
 }  // namespace internal
 }  // namespace v8