Immediatelly promote marked objects

src/heap/heap.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/heap.h"
 
 #include "src/accessors.h"
 #include "src/api.h"
 #include "src/ast/scopeinfo.h"
 #include "src/base/bits.h"
@@ skipping 1581 matching lines @@
     if (map_word.IsForwardingAddress()) {
       return map_word.ToForwardingAddress();
     }
     return NULL;
   }
 
  private:
   Heap* heap_;
 };
 
+void Heap::Scavenge() {
+  if (incremental_marking()->IsMarking()) {
+    Scavenge<PROMOTE_MARKED>();
+  } else {
+    Scavenge<DEFAULT_PROMOTION>();
+  }
+}
 
+template <PromotionMode promotion_mode>
 void Heap::Scavenge() {
   TRACE_GC(tracer(), GCTracer::Scope::SCAVENGER_SCAVENGE);
   RelocationLock relocation_lock(this);
   // There are soft limits in the allocation code, designed to trigger a mark
   // sweep collection by failing allocations. There is no sense in trying to
   // trigger one during scavenge: scavenges allocation should always succeed.
   AlwaysAllocateScope scope(isolate());
 
   // Bump-pointer allocations done during scavenge are not real allocations.
   // Pause the inline allocation steps.
@@ skipping 33 matching lines @@
   // objects lie between a 'front' mark and a 'rear' mark that is
   // updated as a side effect of promoting an object.
   //
   // There is guaranteed to be enough room at the top of the to space
   // for the addresses of promoted objects: every object promoted
   // frees up its size in bytes from the top of the new space, and
   // objects are at least one pointer in size.
   Address new_space_front = new_space_.ToSpaceStart();
   promotion_queue_.Initialize();
 
-  ScavengeVisitor scavenge_visitor(this);
+  ScavengeVisitor<promotion_mode> scavenge_visitor(this);
 
   if (FLAG_scavenge_reclaim_unmodified_objects) {
     isolate()->global_handles()->IdentifyWeakUnmodifiedObjects(
         &IsUnmodifiedHeapObject);
   }
 
   {
     // Copy roots.
     TRACE_GC(tracer(), GCTracer::Scope::SCAVENGER_ROOTS);
     IterateRoots(&scavenge_visitor, VISIT_ALL_IN_SCAVENGE);
   }
 
   {
     // Copy objects reachable from the old generation.
     TRACE_GC(tracer(), GCTracer::Scope::SCAVENGER_OLD_TO_NEW_POINTERS);
     RememberedSet<OLD_TO_NEW>::Iterate(this, [this](Address addr) {
-      return Scavenger::CheckAndScavengeObject(this, addr, DEFAULT_PROMOTION);
+      return Scavenger::CheckAndScavengeObject<promotion_mode>(this, addr);
     });
 
     RememberedSet<OLD_TO_NEW>::IterateTyped(
         this, [this](SlotType type, Address addr) {
           return UpdateTypedSlotHelper::UpdateTypedSlot(
               isolate(), type, addr, [this](Object** addr) {
                 // We expect that objects referenced by code are long living.
                 // If we do not force promotion, then we need to clear
                 // old_to_new slots in dead code objects after mark-compact.
-                return Scavenger::CheckAndScavengeObject(
-                    this, reinterpret_cast<Address>(addr), FORCE_PROMOTION);
+                return Scavenger::CheckAndScavengeObject<FORCE_PROMOTION>(
+                    this, reinterpret_cast<Address>(addr));
               });
         });
   }
 
   {
     TRACE_GC(tracer(), GCTracer::Scope::SCAVENGER_WEAK);
     // Copy objects reachable from the encountered weak collections list.
     scavenge_visitor.VisitPointer(&encountered_weak_collections_);
     // Copy objects reachable from the encountered weak cells.
     scavenge_visitor.VisitPointer(&encountered_weak_cells_);
   }
 
   {
     // Copy objects reachable from the code flushing candidates list.
     TRACE_GC(tracer(), GCTracer::Scope::SCAVENGER_CODE_FLUSH_CANDIDATES);
     MarkCompactCollector* collector = mark_compact_collector();
     if (collector->is_code_flushing_enabled()) {
       collector->code_flusher()->IteratePointersToFromSpace(&scavenge_visitor);
     }
   }
 
   {
     TRACE_GC(tracer(), GCTracer::Scope::SCAVENGER_SEMISPACE);
-    new_space_front = DoScavenge(&scavenge_visitor, new_space_front);
+    new_space_front =
+        DoScavenge<promotion_mode>(&scavenge_visitor, new_space_front);
   }
 
   if (FLAG_scavenge_reclaim_unmodified_objects) {
     isolate()->global_handles()->MarkNewSpaceWeakUnmodifiedObjectsPending(
         &IsUnscavengedHeapObject);
 
     isolate()->global_handles()->IterateNewSpaceWeakUnmodifiedRoots(
         &scavenge_visitor);
-    new_space_front = DoScavenge(&scavenge_visitor, new_space_front);
+    new_space_front =
+        DoScavenge<promotion_mode>(&scavenge_visitor, new_space_front);
   } else {
     TRACE_GC(tracer(), GCTracer::Scope::SCAVENGER_OBJECT_GROUPS);
     while (isolate()->global_handles()->IterateObjectGroups(
         &scavenge_visitor, &IsUnscavengedHeapObject)) {
-      new_space_front = DoScavenge(&scavenge_visitor, new_space_front);
+      new_space_front =
+          DoScavenge<promotion_mode>(&scavenge_visitor, new_space_front);
     }
     isolate()->global_handles()->RemoveObjectGroups();
     isolate()->global_handles()->RemoveImplicitRefGroups();
 
     isolate()->global_handles()->IdentifyNewSpaceWeakIndependentHandles(
         &IsUnscavengedHeapObject);
 
     isolate()->global_handles()->IterateNewSpaceWeakIndependentRoots(
         &scavenge_visitor);
-    new_space_front = DoScavenge(&scavenge_visitor, new_space_front);
+    new_space_front =
+        DoScavenge<promotion_mode>(&scavenge_visitor, new_space_front);
   }
 
   UpdateNewSpaceReferencesInExternalStringTable(
       &UpdateNewSpaceReferenceInExternalStringTableEntry);
 
   promotion_queue_.Destroy();
 
   incremental_marking()->UpdateMarkingDequeAfterScavenge();
 
   ScavengeWeakObjectRetainer weak_object_retainer(this);
@@ skipping 161 matching lines @@
       }
     }
 
    private:
     v8::ExternalResourceVisitor* visitor_;
   } external_string_table_visitor(visitor);
 
   external_string_table_.Iterate(&external_string_table_visitor);
 }
 
-
+template <PromotionMode promotion_mode>
 Address Heap::DoScavenge(ObjectVisitor* scavenge_visitor,
                          Address new_space_front) {
   do {
     SemiSpace::AssertValidRange(new_space_front, new_space_.top());
     // The addresses new_space_front and new_space_.top() define a
     // queue of unprocessed copied objects.  Process them until the
     // queue is empty.
     while (new_space_front != new_space_.top()) {
       if (!Page::IsAlignedToPageSize(new_space_front)) {
         HeapObject* object = HeapObject::FromAddress(new_space_front);
-        new_space_front +=
-            StaticScavengeVisitor::IterateBody(object->map(), object);
+        new_space_front += StaticScavengeVisitor<promotion_mode>::IterateBody(
+            object->map(), object);
       } else {
         new_space_front = Page::FromAllocationAreaAddress(new_space_front)
                               ->next_page()
                               ->area_start();
       }
     }
 
     // Promote and process all the to-be-promoted objects.
     {
       while (!promotion_queue()->is_empty()) {
         HeapObject* target;
         int32_t size;
         bool was_marked_black;
         promotion_queue()->remove(&target, &size, &was_marked_black);
 
         // Promoted object might be already partially visited
         // during old space pointer iteration. Thus we search specifically
         // for pointers to from semispace instead of looking for pointers
         // to new space.
         DCHECK(!target->IsMap());
 
         IteratePromotedObject(target, static_cast<int>(size), was_marked_black,
-                              &Scavenger::ScavengeObject);
+                              &Scavenger::ScavengeObject<promotion_mode>);
       }
     }
 
     // Take another spin if there are now unswept objects in new space
     // (there are currently no more unswept promoted objects).
   } while (new_space_front != new_space_.top());
 
   return new_space_front;
 }
 
@@ skipping 2722 matching lines @@
                                          Address end, bool record_slots,
                                          ObjectSlotCallback callback) {
   Address slot_address = start;
   Page* page = Page::FromAddress(start);
 
   while (slot_address < end) {
     Object** slot = reinterpret_cast<Object**>(slot_address);
     Object* target = *slot;
     if (target->IsHeapObject()) {
       if (Heap::InFromSpace(target)) {
-        callback(reinterpret_cast<HeapObject**>(slot), HeapObject::cast(target),
-                 DEFAULT_PROMOTION);
+        callback(reinterpret_cast<HeapObject**>(slot),
+                 HeapObject::cast(target));
         Object* new_target = *slot;
         if (InNewSpace(new_target)) {
           SLOW_DCHECK(Heap::InToSpace(new_target));
           SLOW_DCHECK(new_target->IsHeapObject());
           RememberedSet<OLD_TO_NEW>::Insert(page, slot_address);
         }
         SLOW_DCHECK(!MarkCompactCollector::IsOnEvacuationCandidate(new_target));
       } else if (record_slots &&
                  MarkCompactCollector::IsOnEvacuationCandidate(target)) {
         mark_compact_collector()->RecordSlot(object, slot, target);
@@ skipping 521 matching lines @@
        space = spaces.next()) {
     space->EmptyAllocationInfo();
   }
 }
 
 
 V8_DECLARE_ONCE(initialize_gc_once);
 
 static void InitializeGCOnce() {
   Scavenger::Initialize();
-  StaticScavengeVisitor::Initialize();
+  StaticScavengeVisitor<DEFAULT_PROMOTION>::Initialize();
+  StaticScavengeVisitor<PROMOTE_MARKED>::Initialize();
+  StaticScavengeVisitor<FORCE_PROMOTION>::Initialize();
   MarkCompactCollector::Initialize();
 }
 
 
 bool Heap::SetUp() {
 #ifdef DEBUG
   allocation_timeout_ = FLAG_gc_interval;
 #endif
 
   // Initialize heap spaces and initial maps and objects. Whenever something
@@ skipping 1145 matching lines @@
 }
 
 
 // static
 int Heap::GetStaticVisitorIdForMap(Map* map) {
   return StaticVisitorBase::GetVisitorId(map);
 }
 
 }  // namespace internal
 }  // namespace v8