[heap] Black allocation.

src/heap/heap.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/heap.h"
 
 #include "src/accessors.h"
 #include "src/api.h"
 #include "src/ast/scopeinfo.h"
 #include "src/base/bits.h"
@@ skipping 1127 matching lines @@
 }
 #endif  // VERIFY_HEAP
 
 
 bool Heap::ReserveSpace(Reservation* reservations) {
   bool gc_performed = true;
   int counter = 0;
   static const int kThreshold = 20;
   while (gc_performed && counter++ < kThreshold) {
     gc_performed = false;
+    bool black_allocation = incremental_marking_->black_allocation();
     for (int space = NEW_SPACE; space < Serializer::kNumberOfSpaces; space++) {
       Reservation* reservation = &reservations[space];
       DCHECK_LE(1, reservation->length());
       if (reservation->at(0).size == 0) continue;
       bool perform_gc = false;
       if (space == LO_SPACE) {
         DCHECK_EQ(1, reservation->length());
         perform_gc = !CanExpandOldGeneration(reservation->at(0).size);
       } else {
         for (auto& chunk : *reservation) {
           AllocationResult allocation;
           int size = chunk.size;
           DCHECK_LE(size, MemoryAllocator::PageAreaSize(
                               static_cast<AllocationSpace>(space)));
           if (space == NEW_SPACE) {
             allocation = new_space()->AllocateRawUnaligned(size);
           } else {
             allocation = paged_space(space)->AllocateRawUnaligned(size);
           }
           HeapObject* free_space = nullptr;
           if (allocation.To(&free_space)) {
             // Mark with a free list node, in case we have a GC before
             // deserializing.
             Address free_space_address = free_space->address();
             CreateFillerObjectAt(free_space_address, size);
             DCHECK(space < Serializer::kNumberOfPreallocatedSpaces);
             chunk.start = free_space_address;
             chunk.end = free_space_address + size;
+            // We have to make sure that either all pages are black pages or
+            // none. If we turned on black allocation while allocating
+            // reservations for different spaces we have to abort.
+            if (black_allocation != incremental_marking_->black_allocation()) {
+              perform_gc = true;

[ulan, 2016/02/10 10:27:39]

How about not starting black allocation if deserializer is enabled?
Otherwise, we cannot guarantee progress.

[Hannes Payer (out of office), 2016/02/11 18:18:07]

I don't think it would be an issue since there should be enough slack between a
fresh GC and reaching the allocation limit. Anyhow, let's play safe. I changed
the black allocation start logic. When the serializer is enabled, we don't start
it. We may start it later when performing an incremental marking step.

+            }
           } else {
             perform_gc = true;
             break;
           }
         }
       }
       if (perform_gc) {
         if (space == NEW_SPACE) {
           CollectGarbage(NEW_SPACE, "failed to reserve space in the new space");
         } else {
@@ skipping 768 matching lines @@
         HeapObject* target;
         int size;
         promotion_queue()->remove(&target, &size);
 
         // Promoted object might be already partially visited
         // during old space pointer iteration. Thus we search specifically
         // for pointers to from semispace instead of looking for pointers
         // to new space.
         DCHECK(!target->IsMap());
 
-        IteratePointersToFromSpace(target, size, &Scavenger::ScavengeObject);
+        IteratePromotedObject(target, size, &Scavenger::ScavengeObject);
       }
     }
 
     // Take another spin if there are now unswept objects in new space
     // (there are currently no more unswept promoted objects).
   } while (new_space_front != new_space_.top());
 
   return new_space_front;
 }
 
@@ skipping 1335 matching lines @@
   return elements;
 }
 
 
 AllocationResult Heap::AllocateCode(int object_size, bool immovable) {
   DCHECK(IsAligned(static_cast<intptr_t>(object_size), kCodeAlignment));
   AllocationResult allocation = AllocateRaw(object_size, CODE_SPACE);
 
   HeapObject* result = nullptr;
   if (!allocation.To(&result)) return allocation;
-
   if (immovable) {
     Address address = result->address();
     // Code objects which should stay at a fixed address are allocated either
     // in the first page of code space (objects on the first page of each space
     // are never moved) or in large object space.
     if (!code_space_->FirstPage()->Contains(address) &&
         MemoryChunk::FromAddress(address)->owner()->identity() != LO_SPACE) {
       // Discard the first code allocation, which was on a page where it could
       // be moved.
       CreateFillerObjectAt(result->address(), object_size);
@@ skipping 29 matching lines @@
   Address new_addr = result->address();
   CopyBlock(new_addr, old_addr, obj_size);
   Code* new_code = Code::cast(result);
 
   // Relocate the copy.
   DCHECK(IsAligned(bit_cast<intptr_t>(new_code->address()), kCodeAlignment));
   DCHECK(isolate_->code_range() == NULL || !isolate_->code_range()->valid() ||
          isolate_->code_range()->contains(code->address()) ||
          obj_size <= code_space()->AreaSize());
   new_code->Relocate(new_addr - old_addr);
+  // We have to iterate over the object and process its pointers when black
+  // allocation is on.
+  if (incremental_marking()->black_allocation() &&
+      Page::FromAddress(new_code->address())->IsFlagSet(Page::BLACK_PAGE)) {
+    incremental_marking()->IterateBlackCode(new_code);

[ulan, 2016/02/10 10:27:39]

How about making this more generic "allocation barrier":
incremental_marking()->RecordUnsafeAllocation(new_code).

Unsafe allocation would be any allocation in old generation that skips write
barriers.

RecordUnsafeAllocation would check black_allocation() and BLACK_PAGE, invoke the
marking visitor and in debug mode check the marking invariant.

[Hannes Payer (out of office), 2016/02/11 18:18:07]

I like it. It will be also called when processing the promotion queue. I call it
IterateBlackObject.

+  }
   return new_code;
 }
 
 
 AllocationResult Heap::CopyCode(Code* code, Vector<byte> reloc_info) {
   // Allocate ByteArray before the Code object, so that we do not risk
   // leaving uninitialized Code object (and breaking the heap).
   ByteArray* reloc_info_array = nullptr;
   {
     AllocationResult allocation =
@@ skipping 27 matching lines @@
   CopyBytes(new_code->relocation_start(), reloc_info.start(),
             static_cast<size_t>(reloc_info.length()));
 
   // Relocate the copy.
   DCHECK(IsAligned(bit_cast<intptr_t>(new_code->address()), kCodeAlignment));
   DCHECK(isolate_->code_range() == NULL || !isolate_->code_range()->valid() ||
          isolate_->code_range()->contains(code->address()) ||
          new_obj_size <= code_space()->AreaSize());
 
   new_code->Relocate(new_addr - old_addr);
-
+  // We have to iterate over over the object and process its pointers when
+  // black allocation is on.
+  if (incremental_marking()->black_allocation() &&
+      Page::FromAddress(new_code->address())->IsFlagSet(Page::BLACK_PAGE)) {
+    incremental_marking()->IterateBlackCode(new_code);
+  }
 #ifdef VERIFY_HEAP
   if (FLAG_verify_heap) code->ObjectVerify();
 #endif
   return new_code;
 }
 
 
 void Heap::InitializeAllocationMemento(AllocationMemento* memento,
                                        AllocationSite* allocation_site) {
   memento->set_map_no_write_barrier(allocation_memento_map());
@@ skipping 1043 matching lines @@
                           new_space_.FromSpaceEnd());
   while (it.has_next()) {
     NewSpacePage* page = it.next();
     for (Address cursor = page->area_start(), limit = page->area_end();
          cursor < limit; cursor += kPointerSize) {
       Memory::Address_at(cursor) = kFromSpaceZapValue;
     }
   }
 }
 
-
-void Heap::IterateAndMarkPointersToFromSpace(HeapObject* object, Address start,
-                                             Address end, bool record_slots,
-                                             ObjectSlotCallback callback) {
+void Heap::IteratePromotedObjectPointers(HeapObject* object, Address start,
+                                         Address end, bool record_slots,
+                                         ObjectSlotCallback callback) {
   Address slot_address = start;
 
   while (slot_address < end) {
     Object** slot = reinterpret_cast<Object**>(slot_address);
     Object* target = *slot;
     // If the store buffer becomes overfull we mark pages as being exempt from
     // the store buffer.  These pages are scanned to find pointers that point
     // to the new space.  In that case we may hit newly promoted objects and
     // fix the pointers before the promotion queue gets to them.  Thus the 'if'.
     if (target->IsHeapObject()) {
       if (Heap::InFromSpace(target)) {
         callback(reinterpret_cast<HeapObject**>(slot),
                  HeapObject::cast(target));
         Object* new_target = *slot;
         if (InNewSpace(new_target)) {
           SLOW_DCHECK(Heap::InToSpace(new_target));
           SLOW_DCHECK(new_target->IsHeapObject());
           store_buffer_.EnterDirectlyIntoStoreBuffer(
               reinterpret_cast<Address>(slot));
         }
         SLOW_DCHECK(!MarkCompactCollector::IsOnEvacuationCandidate(new_target));
       } else if (record_slots &&
                  MarkCompactCollector::IsOnEvacuationCandidate(target)) {
         mark_compact_collector()->RecordSlot(object, slot, target);
       }
+      // Black allocations requires to mark objects referenced by promoted
+      // objects. If the object got forwarded to *slot
+      // UpdateMarkingDequeAfterScavenge will take care of the color
+      // transition.
+      if (incremental_marking()->black_allocation()) {
+        IncrementalMarking::MarkObject(this, HeapObject::cast(target));

[ulan, 2016/02/10 10:27:39]

This ignores all the special handling that the marking visitor might have for
this object. In particular weak slots and maintaining  global lists, e.g.
JSFunction list for code flushing.

We need to either make marking vistiors of all new space object trivial or
invoke the non-trivial marking visitor on promotion to black pages.

[Hannes Payer (out of office), 2016/02/11 18:18:07]

Weakness during black allocation is special, i.e. in the current GC a weak
references becomes strong (c.f. WeakCell::initialize). However, missing other
side effects of the visitors will be problematic. I decided to do another
iteration over the objects using the regular visitor. Ideally, we would just do
one pass over the object, not two. I left a TODO for a follow-up CL.

+      }
     }
     slot_address += kPointerSize;
   }
 }
 
-
-class IteratePointersToFromSpaceVisitor final : public ObjectVisitor {
+class IteratePromotedObjectsVisitor final : public ObjectVisitor {
  public:
-  IteratePointersToFromSpaceVisitor(Heap* heap, HeapObject* target,
-                                    bool record_slots,
-                                    ObjectSlotCallback callback)
+  IteratePromotedObjectsVisitor(Heap* heap, HeapObject* target,
+                                bool record_slots, ObjectSlotCallback callback)
       : heap_(heap),
         target_(target),
         record_slots_(record_slots),
         callback_(callback) {}
 
   V8_INLINE void VisitPointers(Object** start, Object** end) override {
-    heap_->IterateAndMarkPointersToFromSpace(
+    heap_->IteratePromotedObjectPointers(
         target_, reinterpret_cast<Address>(start),
         reinterpret_cast<Address>(end), record_slots_, callback_);
   }
 
-  V8_INLINE void VisitCodeEntry(Address code_entry_slot) override {}
+  V8_INLINE void VisitCodeEntry(Address code_entry_slot) override {
+    // Black allocation requires us to process objects referenced by
+    // promoted objects.
+    if (heap_->incremental_marking()->black_allocation()) {
+      Code* code = Code::cast(Code::GetObjectFromEntryAddress(code_entry_slot));
+      IncrementalMarking::MarkObject(heap_, code);
+    }
+  }
 
  private:
   Heap* heap_;
   HeapObject* target_;
   bool record_slots_;
   ObjectSlotCallback callback_;
 };
 
-
-void Heap::IteratePointersToFromSpace(HeapObject* target, int size,
-                                      ObjectSlotCallback callback) {
+void Heap::IteratePromotedObject(HeapObject* target, int size,
+                                 ObjectSlotCallback callback) {
   // We are not collecting slots on new space objects during mutation
   // thus we have to scan for pointers to evacuation candidates when we
   // promote objects. But we should not record any slots in non-black
   // objects. Grey object's slots would be rescanned.
   // White object might not survive until the end of collection
   // it would be a violation of the invariant to record it's slots.
   bool record_slots = false;
   if (incremental_marking()->IsCompacting()) {
     MarkBit mark_bit = Marking::MarkBitFrom(target);
     record_slots = Marking::IsBlack(mark_bit);
   }
 
-  IteratePointersToFromSpaceVisitor visitor(this, target, record_slots,
-                                            callback);
+  IteratePromotedObjectsVisitor visitor(this, target, record_slots, callback);
   target->IterateBody(target->map()->instance_type(), size, &visitor);
+
+  // Black allocation requires us to process objects referenced by
+  // promoted objects.
+  if (incremental_marking()->black_allocation()) {
+    Map* map = target->map();
+    IncrementalMarking::MarkObject(this, map);
+  }
 }
 
 
 void Heap::IterateRoots(ObjectVisitor* v, VisitMode mode) {
   IterateStrongRoots(v, mode);
   IterateWeakRoots(v, mode);
 }
 
 
 void Heap::IterateWeakRoots(ObjectVisitor* v, VisitMode mode) {
@@ skipping 1653 matching lines @@
 }
 
 
 // static
 int Heap::GetStaticVisitorIdForMap(Map* map) {
   return StaticVisitorBase::GetVisitorId(map);
 }
 
 }  // namespace internal
 }  // namespace v8