[heap] Black allocation.

src/heap/mark-compact.cc

Index: src/heap/mark-compact.cc
diff --git a/src/heap/mark-compact.cc b/src/heap/mark-compact.cc
index 65e126ab9b3bfa004144e7e4d3e3bc3d9230b51a..c141972149614ea23a2a68693bd298ab5b867c0d 100644
--- a/src/heap/mark-compact.cc
+++ b/src/heap/mark-compact.cc
@@ -122,6 +122,15 @@ static void VerifyMarking(Heap* heap, Address bottom, Address top) {
   }
 }
 
+static void VerifyMarkingBlackPage(Heap* heap, Page* page) {
+  CHECK(page->IsFlagSet(Page::BLACK_PAGE));
+  VerifyMarkingVisitor visitor(heap);
+  HeapObjectIterator it(page);
+  for (HeapObject* object = it.Next(); object != NULL; object = it.Next()) {
+    CHECK(Marking::IsBlack(Marking::MarkBitFrom(object)));
+    object->Iterate(&visitor);
+  }
+}
 
 static void VerifyMarking(NewSpace* space) {
   Address end = space->top();
@@ -144,7 +153,11 @@ static void VerifyMarking(PagedSpace* space) {
 
   while (it.has_next()) {
     Page* p = it.next();
-    VerifyMarking(space->heap(), p->area_start(), p->area_end());
+    if (p->IsFlagSet(Page::BLACK_PAGE)) {
+      VerifyMarkingBlackPage(space->heap(), p);
+    } else {
+      VerifyMarking(space->heap(), p->area_start(), p->area_end());
+    }
   }
 }
 
@@ -448,7 +461,11 @@ static void ClearMarkbitsInPagedSpace(PagedSpace* space) {
   PageIterator it(space);
 
   while (it.has_next()) {
-    Bitmap::Clear(it.next());
+    Page* p = it.next();
+    Bitmap::Clear(p);
+    if (p->IsFlagSet(Page::BLACK_PAGE)) {
+      p->ClearFlag(Page::BLACK_PAGE);
+    }
   }
 }
 
@@ -471,8 +488,12 @@ void MarkCompactCollector::ClearMarkbits() {
   LargeObjectIterator it(heap_->lo_space());
   for (HeapObject* obj = it.Next(); obj != NULL; obj = it.Next()) {
     Marking::MarkWhite(Marking::MarkBitFrom(obj));
-    Page::FromAddress(obj->address())->ResetProgressBar();
-    Page::FromAddress(obj->address())->ResetLiveBytes();
+    MemoryChunk* chunk = MemoryChunk::FromAddress(obj->address());
+    chunk->ResetProgressBar();
+    chunk->ResetLiveBytes();
+    if (chunk->IsFlagSet(Page::BLACK_PAGE)) {
+      chunk->ClearFlag(Page::BLACK_PAGE);
+    }
   }
 }
 
@@ -589,7 +610,9 @@ void Marking::TransferMark(Heap* heap, Address old_start, Address new_start) {
   DCHECK(MemoryChunk::FromAddress(old_start) ==
          MemoryChunk::FromAddress(new_start));
 
-  if (!heap->incremental_marking()->IsMarking()) return;
+  if (!heap->incremental_marking()->IsMarking() ||
+      Page::FromAddress(old_start)->IsFlagSet(Page::BLACK_PAGE))
+    return;
 
   // If the mark doesn't move, we don't check the color of the object.
   // It doesn't matter whether the object is black, since it hasn't changed
@@ -698,6 +721,7 @@ void MarkCompactCollector::CollectEvacuationCandidates(PagedSpace* space) {
   while (it.has_next()) {
     Page* p = it.next();
     if (p->NeverEvacuate()) continue;
+    if (p->IsFlagSet(Page::BLACK_PAGE)) continue;
     if (p->IsFlagSet(Page::POPULAR_PAGE)) {
       // This page had slots buffer overflow on previous GC, skip it.
       p->ClearFlag(Page::POPULAR_PAGE);
@@ -1741,8 +1765,10 @@ void MarkCompactCollector::DiscoverGreyObjectsInSpace(PagedSpace* space) {
   PageIterator it(space);
   while (it.has_next()) {
     Page* p = it.next();
-    DiscoverGreyObjectsOnPage(p);
-    if (marking_deque()->IsFull()) return;
+    if (!p->IsFlagSet(Page::BLACK_PAGE)) {
+      DiscoverGreyObjectsOnPage(p);
+      if (marking_deque()->IsFull()) return;
+    }
   }
 }
 
@@ -2922,6 +2948,12 @@ bool MarkCompactCollector::IsSlotInBlackObject(Page* p, Address slot,
     return false;
   }
 
+  // If we are on a black page, we cannot find the actual object start
+  // easiliy. We just return true but do not set the out_object.
+  if (p->IsFlagSet(Page::BLACK_PAGE)) {
+    return true;
+  }
+
   uint32_t mark_bit_index = p->AddressToMarkbitIndex(slot);
   unsigned int cell_index = mark_bit_index >> Bitmap::kBitsPerCellLog2;
   MarkBit::CellType index_mask = 1u << Bitmap::IndexInCell(mark_bit_index);
@@ -3002,7 +3034,6 @@ bool MarkCompactCollector::IsSlotInBlackObject(Page* p, Address slot,
 
 
 bool MarkCompactCollector::IsSlotInBlackObjectSlow(Page* p, Address slot) {
-  // This function does not support large objects right now.
   Space* owner = p->owner();
   if (owner == heap_->lo_space() || owner == NULL) {
     Object* large_object = heap_->lo_space()->FindObject(slot);
@@ -3016,6 +3047,8 @@ bool MarkCompactCollector::IsSlotInBlackObjectSlow(Page* p, Address slot) {
     return false;
   }
 
+  if (p->IsFlagSet(Page::BLACK_PAGE)) return true;
+
   LiveObjectIterator<kBlackObjects> it(p);
   HeapObject* object = NULL;
   while ((object = it.Next()) != NULL) {
@@ -3031,15 +3064,21 @@ bool MarkCompactCollector::IsSlotInBlackObjectSlow(Page* p, Address slot) {
 
 
 bool MarkCompactCollector::IsSlotInLiveObject(Address slot) {
-  HeapObject* object = NULL;
   // The target object is black but we don't know if the source slot is black.
   // The source object could have died and the slot could be part of a free
   // space. Find out based on mark bits if the slot is part of a live object.
-  if (!IsSlotInBlackObject(Page::FromAddress(slot), slot, &object)) {
+  Page* page = Page::FromAddress(slot);
+  HeapObject* object = NULL;
+  if (!IsSlotInBlackObject(page, slot, &object)) {
     return false;
   }
 
-  DCHECK(object != NULL);
+  // If the slot is on a black page, the object will be live.
+  DCHECK(object != NULL || page->IsFlagSet(Page::BLACK_PAGE));
+  if (page->IsFlagSet(Page::BLACK_PAGE)) {
+    return true;
+  }
+
   int offset = static_cast<int>(slot - object->address());
   return object->IsValidSlot(offset);
 }
@@ -3441,6 +3480,7 @@ template <SweepingMode sweeping_mode,
 static int Sweep(PagedSpace* space, FreeList* free_list, Page* p,
                  ObjectVisitor* v) {
   DCHECK(!p->IsEvacuationCandidate() && !p->SweepingDone());
+  DCHECK(!p->IsFlagSet(Page::BLACK_PAGE));
   DCHECK_EQ(skip_list_mode == REBUILD_SKIP_LIST,
             space->identity() == CODE_SPACE);
   DCHECK((p->skip_list() == NULL) || (skip_list_mode == REBUILD_SKIP_LIST));
@@ -3903,6 +3943,16 @@ void MarkCompactCollector::StartSweepSpace(PagedSpace* space) {
     Page* p = it.next();
     DCHECK(p->SweepingDone());
 
+    // We can not sweep black pages, since all mark bits are set for these
+    // pages.
+    if (p->IsFlagSet(Page::BLACK_PAGE)) {
+      Bitmap::Clear(p);
+      p->concurrent_sweeping_state().SetValue(Page::kSweepingDone);
+      p->ClearFlag(Page::BLACK_PAGE);
+      // TODO(hpayer): Free unused memory of last black page.
+      continue;
+    }
+
     if (p->IsFlagSet(Page::RESCAN_ON_EVACUATION) ||
         p->IsEvacuationCandidate()) {
       // Will be processed in EvacuateNewSpaceAndCandidates.