Immediatelly promote marked objects

src/heap/scavenger.cc

 // Copyright 2015 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/scavenger.h"
 
 #include "src/contexts.h"
 #include "src/heap/heap.h"
 #include "src/heap/objects-visiting-inl.h"
 #include "src/heap/scavenger-inl.h"
 #include "src/isolate.h"
 #include "src/log.h"
 #include "src/profiler/cpu-profiler.h"
 
 namespace v8 {
 namespace internal {
 
 enum LoggingAndProfiling {
   LOGGING_AND_PROFILING_ENABLED,
   LOGGING_AND_PROFILING_DISABLED
 };
 
 
 enum MarksHandling { TRANSFER_MARKS, IGNORE_MARKS };
 
-
-template <MarksHandling marks_handling,
+template <MarksHandling marks_handling, PromotionMode promotion_mode,
           LoggingAndProfiling logging_and_profiling_mode>
 class ScavengingVisitor : public StaticVisitorBase {
  public:
   static void Initialize() {
     table_.Register(kVisitSeqOneByteString, &EvacuateSeqOneByteString);
     table_.Register(kVisitSeqTwoByteString, &EvacuateSeqTwoByteString);
     table_.Register(kVisitShortcutCandidate, &EvacuateShortcutCandidate);
     table_.Register(kVisitByteArray, &EvacuateByteArray);
     table_.Register(kVisitFixedArray, &EvacuateFixedArray);
     table_.Register(kVisitFixedDoubleArray, &EvacuateFixedDoubleArray);
@@ skipping 157 matching lines @@
             Marking::IsBlack(Marking::MarkBitFrom(object)));
       }
       heap->IncrementPromotedObjectsSize(object_size);
       return true;
     }
     return false;
   }
 
   template <ObjectContents object_contents, AllocationAlignment alignment>
   static inline void EvacuateObject(Map* map, HeapObject** slot,
-                                    HeapObject* object, int object_size,
-                                    PromotionMode promotion_mode) {
+                                    HeapObject* object, int object_size) {
     SLOW_DCHECK(object_size <= Page::kAllocatableMemory);
     SLOW_DCHECK(object->Size() == object_size);
     Heap* heap = map->GetHeap();
 
-    if (promotion_mode != FORCE_PROMOTION &&
-        !heap->ShouldBePromoted(object->address(), object_size)) {
+    if (!heap->ShouldBePromoted<promotion_mode>(object->address(),
+                                                object_size)) {
       // A semi-space copy may fail due to fragmentation. In that case, we
       // try to promote the object.
       if (SemiSpaceCopyObject<alignment>(map, slot, object, object_size)) {
         return;
       }
     }
 
     if (PromoteObject<object_contents, alignment>(map, slot, object,
                                                   object_size)) {
       return;
     }
     if (promotion_mode == FORCE_PROMOTION) {
       FatalProcessOutOfMemory("Scavenger: forced promotion\n");
     }
+    if (promotion_mode == PROMOTE_MARKED) {
+      FatalProcessOutOfMemory("Scavenger: promoting marked\n");
+    }
     // If promotion failed, we try to copy the object to the other semi-space
     if (SemiSpaceCopyObject<alignment>(map, slot, object, object_size)) return;
 
     FatalProcessOutOfMemory("Scavenger: semi-space copy\n");
   }
 
   static inline void EvacuateJSFunction(Map* map, HeapObject** slot,
-                                        HeapObject* object,
-                                        PromotionMode promotion_mode) {
-    ObjectEvacuationStrategy<POINTER_OBJECT>::Visit(map, slot, object,
-                                                    promotion_mode);
+                                        HeapObject* object) {
+    ObjectEvacuationStrategy<POINTER_OBJECT>::Visit(map, slot, object);
 
     if (marks_handling == IGNORE_MARKS) return;
 
     MapWord map_word = object->map_word();
     DCHECK(map_word.IsForwardingAddress());
     HeapObject* target = map_word.ToForwardingAddress();
 
     MarkBit mark_bit = Marking::MarkBitFrom(target);
     if (Marking::IsBlack(mark_bit)) {
       // This object is black and it might not be rescanned by marker.
       // We should explicitly record code entry slot for compaction because
       // promotion queue processing (IteratePromotedObjectPointers) will
       // miss it as it is not HeapObject-tagged.
       Address code_entry_slot =
           target->address() + JSFunction::kCodeEntryOffset;
       Code* code = Code::cast(Code::GetObjectFromEntryAddress(code_entry_slot));
       map->GetHeap()->mark_compact_collector()->RecordCodeEntrySlot(
           target, code_entry_slot, code);
     }
   }
 
   static inline void EvacuateFixedArray(Map* map, HeapObject** slot,
-                                        HeapObject* object,
-                                        PromotionMode promotion_mode) {
+                                        HeapObject* object) {
     int length = reinterpret_cast<FixedArray*>(object)->synchronized_length();
     int object_size = FixedArray::SizeFor(length);
-    EvacuateObject<POINTER_OBJECT, kWordAligned>(map, slot, object, object_size,
-                                                 promotion_mode);
+    EvacuateObject<POINTER_OBJECT, kWordAligned>(map, slot, object,
+                                                 object_size);
   }
 
   static inline void EvacuateFixedDoubleArray(Map* map, HeapObject** slot,
-                                              HeapObject* object,
-                                              PromotionMode promotion_mode) {
+                                              HeapObject* object) {
     int length = reinterpret_cast<FixedDoubleArray*>(object)->length();
     int object_size = FixedDoubleArray::SizeFor(length);
-    EvacuateObject<DATA_OBJECT, kDoubleAligned>(map, slot, object, object_size,
-                                                promotion_mode);
+    EvacuateObject<DATA_OBJECT, kDoubleAligned>(map, slot, object, object_size);
   }
 
   static inline void EvacuateFixedTypedArray(Map* map, HeapObject** slot,
-                                             HeapObject* object,
-                                             PromotionMode promotion_mode) {
+                                             HeapObject* object) {
     int object_size = reinterpret_cast<FixedTypedArrayBase*>(object)->size();
-    EvacuateObject<POINTER_OBJECT, kWordAligned>(map, slot, object, object_size,
-                                                 promotion_mode);
+    EvacuateObject<POINTER_OBJECT, kWordAligned>(map, slot, object,
+                                                 object_size);
   }
 
   static inline void EvacuateFixedFloat64Array(Map* map, HeapObject** slot,
-                                               HeapObject* object,
-                                               PromotionMode promotion_mode) {
+                                               HeapObject* object) {
     int object_size = reinterpret_cast<FixedFloat64Array*>(object)->size();
     EvacuateObject<POINTER_OBJECT, kDoubleAligned>(map, slot, object,
-                                                   object_size, promotion_mode);
+                                                   object_size);
   }
 
   static inline void EvacuateJSArrayBuffer(Map* map, HeapObject** slot,
-                                           HeapObject* object,
-                                           PromotionMode promotion_mode) {
-    ObjectEvacuationStrategy<POINTER_OBJECT>::Visit(map, slot, object,
-                                                    promotion_mode);
+                                           HeapObject* object) {
+    ObjectEvacuationStrategy<POINTER_OBJECT>::Visit(map, slot, object);
 
     Heap* heap = map->GetHeap();
     MapWord map_word = object->map_word();
     DCHECK(map_word.IsForwardingAddress());
     HeapObject* target = map_word.ToForwardingAddress();
     if (!heap->InNewSpace(target)) {
       heap->array_buffer_tracker()->Promote(JSArrayBuffer::cast(target));
     }
   }
 
   static inline void EvacuateByteArray(Map* map, HeapObject** slot,
-                                       HeapObject* object,
-                                       PromotionMode promotion_mode) {
+                                       HeapObject* object) {
     int object_size = reinterpret_cast<ByteArray*>(object)->ByteArraySize();
-    EvacuateObject<DATA_OBJECT, kWordAligned>(map, slot, object, object_size,
-                                              promotion_mode);
+    EvacuateObject<DATA_OBJECT, kWordAligned>(map, slot, object, object_size);
   }
 
   static inline void EvacuateSeqOneByteString(Map* map, HeapObject** slot,
-                                              HeapObject* object,
-                                              PromotionMode promotion_mode) {
+                                              HeapObject* object) {
     int object_size = SeqOneByteString::cast(object)
                           ->SeqOneByteStringSize(map->instance_type());
-    EvacuateObject<DATA_OBJECT, kWordAligned>(map, slot, object, object_size,
-                                              promotion_mode);
+    EvacuateObject<DATA_OBJECT, kWordAligned>(map, slot, object, object_size);
   }
 
   static inline void EvacuateSeqTwoByteString(Map* map, HeapObject** slot,
-                                              HeapObject* object,
-                                              PromotionMode promotion_mode) {
+                                              HeapObject* object) {
     int object_size = SeqTwoByteString::cast(object)
                           ->SeqTwoByteStringSize(map->instance_type());
-    EvacuateObject<DATA_OBJECT, kWordAligned>(map, slot, object, object_size,
-                                              promotion_mode);
+    EvacuateObject<DATA_OBJECT, kWordAligned>(map, slot, object, object_size);
   }
 
   static inline void EvacuateShortcutCandidate(Map* map, HeapObject** slot,
-                                               HeapObject* object,
-                                               PromotionMode promotion_mode) {
+                                               HeapObject* object) {
     DCHECK(IsShortcutCandidate(map->instance_type()));
 
     Heap* heap = map->GetHeap();
 
     if (marks_handling == IGNORE_MARKS &&
         ConsString::cast(object)->unchecked_second() == heap->empty_string()) {
       HeapObject* first =
           HeapObject::cast(ConsString::cast(object)->unchecked_first());
 
       *slot = first;
 
       if (!heap->InNewSpace(first)) {
         object->set_map_word(MapWord::FromForwardingAddress(first));
         return;
       }
 
       MapWord first_word = first->map_word();
       if (first_word.IsForwardingAddress()) {
         HeapObject* target = first_word.ToForwardingAddress();
 
         *slot = target;
         object->set_map_word(MapWord::FromForwardingAddress(target));
         return;
       }
 
-      Scavenger::ScavengeObjectSlow(slot, first, promotion_mode);
+      Scavenger::ScavengeObjectSlow<promotion_mode>(slot, first);
       object->set_map_word(MapWord::FromForwardingAddress(*slot));
       return;
     }
 
     int object_size = ConsString::kSize;
-    EvacuateObject<POINTER_OBJECT, kWordAligned>(map, slot, object, object_size,
-                                                 promotion_mode);
+    EvacuateObject<POINTER_OBJECT, kWordAligned>(map, slot, object,
+                                                 object_size);
   }
 
   template <ObjectContents object_contents>
   class ObjectEvacuationStrategy {
    public:
     template <int object_size>
     static inline void VisitSpecialized(Map* map, HeapObject** slot,
-                                        HeapObject* object,
-                                        PromotionMode promotion_mode) {
-      EvacuateObject<object_contents, kWordAligned>(
-          map, slot, object, object_size, promotion_mode);
+                                        HeapObject* object) {
+      EvacuateObject<object_contents, kWordAligned>(map, slot, object,
+                                                    object_size);
     }
 
-    static inline void Visit(Map* map, HeapObject** slot, HeapObject* object,
-                             PromotionMode promotion_mode) {
+    static inline void Visit(Map* map, HeapObject** slot, HeapObject* object) {
       int object_size = map->instance_size();
-      EvacuateObject<object_contents, kWordAligned>(
-          map, slot, object, object_size, promotion_mode);
+      EvacuateObject<object_contents, kWordAligned>(map, slot, object,
+                                                    object_size);
     }
   };
 
   static VisitorDispatchTable<ScavengingCallback> table_;
 };
 
-
-template <MarksHandling marks_handling,
+template <MarksHandling marks_handling, PromotionMode promotion_mode,
           LoggingAndProfiling logging_and_profiling_mode>
-VisitorDispatchTable<ScavengingCallback>
-    ScavengingVisitor<marks_handling, logging_and_profiling_mode>::table_;
-
+VisitorDispatchTable<ScavengingCallback> ScavengingVisitor<
+    marks_handling, promotion_mode, logging_and_profiling_mode>::table_;
 
 // static
 void Scavenger::Initialize() {
-  ScavengingVisitor<TRANSFER_MARKS,
+  ScavengingVisitor<TRANSFER_MARKS, PROMOTE_MARKED,
                     LOGGING_AND_PROFILING_DISABLED>::Initialize();
-  ScavengingVisitor<IGNORE_MARKS, LOGGING_AND_PROFILING_DISABLED>::Initialize();
-  ScavengingVisitor<TRANSFER_MARKS,
+  ScavengingVisitor<IGNORE_MARKS, DEFAULT_PROMOTION,
+                    LOGGING_AND_PROFILING_DISABLED>::Initialize();
+  ScavengingVisitor<TRANSFER_MARKS, PROMOTE_MARKED,
                     LOGGING_AND_PROFILING_ENABLED>::Initialize();
-  ScavengingVisitor<IGNORE_MARKS, LOGGING_AND_PROFILING_ENABLED>::Initialize();
+  ScavengingVisitor<IGNORE_MARKS, DEFAULT_PROMOTION,
+                    LOGGING_AND_PROFILING_ENABLED>::Initialize();
 }
 
 
-// static
-void Scavenger::ScavengeObjectSlow(HeapObject** p, HeapObject* object,
-                                   PromotionMode promotion_mode) {
-  SLOW_DCHECK(object->GetIsolate()->heap()->InFromSpace(object));
-  MapWord first_word = object->map_word();
-  SLOW_DCHECK(!first_word.IsForwardingAddress());
-  Map* map = first_word.ToMap();
-  Scavenger* scavenger = map->GetHeap()->scavenge_collector_;
-  scavenger->scavenging_visitors_table_.GetVisitor(map)(map, p, object,
-                                                        promotion_mode);
-}
-
-
 void Scavenger::SelectScavengingVisitorsTable() {
   bool logging_and_profiling =
       FLAG_verify_predictable || isolate()->logger()->is_logging() ||
       isolate()->cpu_profiler()->is_profiling() ||
       (isolate()->heap_profiler() != NULL &&
        isolate()->heap_profiler()->is_tracking_object_moves());
 
   if (!heap()->incremental_marking()->IsMarking()) {
     if (!logging_and_profiling) {
       scavenging_visitors_table_.CopyFrom(
-          ScavengingVisitor<IGNORE_MARKS,
+          ScavengingVisitor<IGNORE_MARKS, DEFAULT_PROMOTION,
                             LOGGING_AND_PROFILING_DISABLED>::GetTable());
     } else {
       scavenging_visitors_table_.CopyFrom(
-          ScavengingVisitor<IGNORE_MARKS,
+          ScavengingVisitor<IGNORE_MARKS, DEFAULT_PROMOTION,
                             LOGGING_AND_PROFILING_ENABLED>::GetTable());
     }
   } else {
     if (!logging_and_profiling) {
       scavenging_visitors_table_.CopyFrom(
-          ScavengingVisitor<TRANSFER_MARKS,
+          ScavengingVisitor<TRANSFER_MARKS, PROMOTE_MARKED,
                             LOGGING_AND_PROFILING_DISABLED>::GetTable());
     } else {
       scavenging_visitors_table_.CopyFrom(
-          ScavengingVisitor<TRANSFER_MARKS,
+          ScavengingVisitor<TRANSFER_MARKS, PROMOTE_MARKED,
                             LOGGING_AND_PROFILING_ENABLED>::GetTable());
     }
 
     if (heap()->incremental_marking()->IsCompacting()) {
       // When compacting forbid short-circuiting of cons-strings.
       // Scavenging code relies on the fact that new space object
       // can't be evacuated into evacuation candidate but
       // short-circuiting violates this assumption.
       scavenging_visitors_table_.Register(
           StaticVisitorBase::kVisitShortcutCandidate,
           scavenging_visitors_table_.GetVisitorById(
               StaticVisitorBase::kVisitConsString));
     }
   }
 }
 
 
 Isolate* Scavenger::isolate() { return heap()->isolate(); }
 
 
-void ScavengeVisitor::VisitPointer(Object** p) { ScavengePointer(p); }
-
-
-void ScavengeVisitor::VisitPointers(Object** start, Object** end) {
-  // Copy all HeapObject pointers in [start, end)
-  for (Object** p = start; p < end; p++) ScavengePointer(p);
-}
-
-
-void ScavengeVisitor::ScavengePointer(Object** p) {
-  Object* object = *p;
-  if (!heap_->InNewSpace(object)) return;
-  Scavenger::ScavengeObject(reinterpret_cast<HeapObject**>(p),
-                            reinterpret_cast<HeapObject*>(object));
-}
-
 }  // namespace internal
 }  // namespace v8