Preserve marking bits across scavenges if incremental marker is running.

src/heap.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 455 matching lines @@
   // allow at least a few allocations after a collection. The reason
   // for this is that we have a lot of allocation sequences and we
   // assume that a garbage collection will allow the subsequent
   // allocation attempts to go through.
   allocation_timeout_ = Max(6, FLAG_gc_interval);
 #endif
 
   if (collector == SCAVENGER &&
       IncrementalMarking::state() != IncrementalMarking::STOPPED) {
     if (FLAG_trace_incremental_marking) {
-      PrintF("[IncrementalMarking] SCAVENGE -> MARK-SWEEP\n");
+      PrintF("[IncrementalMarking] Scavenge during marking.\n");
     }
-    collector = MARK_COMPACTOR;
+  }
+
+  if (collector == MARK_COMPACTOR &&
+      !MarkCompactCollector::PreciseSweepingRequired() &&
+      IncrementalMarking::state() == IncrementalMarking::MARKING) {
+    if (FLAG_trace_incremental_marking) {
+      PrintF("[IncrementalMarking] Delaying MarkSweep.\n");
+    }
+    collector = SCAVENGER;
   }
 
   bool next_gc_likely_to_collect_more = false;
 
   { GCTracer tracer;
     GarbageCollectionPrologue();
     // The GC count was incremented in the prologue.  Tell the tracer about
     // it.
     tracer.set_gc_count(gc_count_);
 
     // Tell the tracer which collector we've selected.
     tracer.set_collector(collector);
 
     HistogramTimer* rate = (collector == SCAVENGER)
         ? &Counters::gc_scavenger
         : &Counters::gc_compactor;
     rate->Start();
     next_gc_likely_to_collect_more =
         PerformGarbageCollection(collector, &tracer);
     rate->Stop();
 
     GarbageCollectionEpilogue();
   }
 
-  ASSERT(IncrementalMarking::state() == IncrementalMarking::STOPPED);
-  if (IncrementalMarking::WorthActivating() && NextGCIsLikelyToBeFull()) {
-    IncrementalMarking::Start();
+
+  ASSERT(collector == SCAVENGER || IncrementalMarking::IsStopped());
+  if (IncrementalMarking::IsStopped()) {
+    if (IncrementalMarking::WorthActivating() && NextGCIsLikelyToBeFull()) {
+      IncrementalMarking::Start();
+    }
   }
 
 #ifdef ENABLE_LOGGING_AND_PROFILING
   if (FLAG_log_gc) HeapProfiler::WriteSample();
 #endif
 
   return next_gc_likely_to_collect_more;
 }
 
 
@@ skipping 434 matching lines @@
   LOG(ResourceEvent("scavenge", "begin"));
 
   // Clear descriptor cache.
   DescriptorLookupCache::Clear();
 
   // Used for updating survived_since_last_expansion_ at function end.
   intptr_t survived_watermark = PromotedSpaceSize();
 
   CheckNewSpaceExpansionCriteria();
 
+  SelectScavengingVisitorsTable();
+
+  IncrementalMarking::PrepareForScavenge();
+
   // Flip the semispaces.  After flipping, to space is empty, from space has
   // live objects.
   new_space_.Flip();
   new_space_.ResetAllocationInfo();
 
   // We need to sweep newly copied objects which can be either in the
   // to space or promoted to the old generation.  For to-space
   // objects, we treat the bottom of the to space as a queue.  Newly
   // copied and unswept objects lie between a 'front' mark and the
   // allocation pointer.
@@ skipping 41 matching lines @@
   // Scavenge object reachable from the global contexts list directly.
   scavenge_visitor.VisitPointer(BitCast<Object**>(&global_contexts_list_));
 
   new_space_front = DoScavenge(&scavenge_visitor, new_space_front);
 
   UpdateNewSpaceReferencesInExternalStringTable(
       &UpdateNewSpaceReferenceInExternalStringTableEntry);
 
   LiveObjectList::UpdateReferencesForScavengeGC();
   RuntimeProfiler::UpdateSamplesAfterScavenge();
+  IncrementalMarking::UpdateMarkingStackAfterScavenge();
 
   ASSERT(new_space_front == new_space_.top());
 
   // Set age mark.
   new_space_.set_age_mark(new_space_.top());
 
   // Update how much has survived scavenge.
   IncrementYoungSurvivorsCounter(static_cast<int>(
       (PromotedSpaceSize() - survived_watermark) + new_space_.Size()));
 
@@ skipping 175 matching lines @@
     }
 
     // Take another spin if there are now unswept objects in new space
     // (there are currently no more unswept promoted objects).
   } while (new_space_front < new_space_.top());
 
   return new_space_front;
 }
 
 
+enum MarksHandling { TRANSFER_MARKS, IGNORE_MARKS };
+
+typedef void (*ScavengingCallback)(Map* map,
+                                   HeapObject** slot,
+                                   HeapObject* object);
+
+static VisitorDispatchTable<ScavengingCallback> scavening_visitors_table_;
+
+static inline void DoScavengeObject(Map* map,
+                                    HeapObject** slot,
+                                    HeapObject* obj) {
+  scavening_visitors_table_.GetVisitor(map)(map, slot, obj);
+}
+
+
+template<MarksHandling marks_handling>
 class ScavengingVisitor : public StaticVisitorBase {
  public:
   static void Initialize() {
     table_.Register(kVisitSeqAsciiString, &EvacuateSeqAsciiString);
     table_.Register(kVisitSeqTwoByteString, &EvacuateSeqTwoByteString);
     table_.Register(kVisitShortcutCandidate, &EvacuateShortcutCandidate);
     table_.Register(kVisitByteArray, &EvacuateByteArray);
     table_.Register(kVisitFixedArray, &EvacuateFixedArray);
+
     table_.Register(kVisitGlobalContext,
                     &ObjectEvacuationStrategy<POINTER_OBJECT>::
-                        VisitSpecialized<Context::kSize>);
-
-    typedef ObjectEvacuationStrategy<POINTER_OBJECT> PointerObject;
+                        template VisitSpecialized<Context::kSize>);
 
     table_.Register(kVisitConsString,
                     &ObjectEvacuationStrategy<POINTER_OBJECT>::
-                        VisitSpecialized<ConsString::kSize>);
+                        template VisitSpecialized<ConsString::kSize>);
 
     table_.Register(kVisitSharedFunctionInfo,
                     &ObjectEvacuationStrategy<POINTER_OBJECT>::
-                        VisitSpecialized<SharedFunctionInfo::kSize>);
+                        template VisitSpecialized<SharedFunctionInfo::kSize>);
 
     table_.Register(kVisitJSFunction,
                     &ObjectEvacuationStrategy<POINTER_OBJECT>::
-                    VisitSpecialized<JSFunction::kSize>);
+                        template VisitSpecialized<JSFunction::kSize>);
 
     table_.RegisterSpecializations<ObjectEvacuationStrategy<DATA_OBJECT>,
                                    kVisitDataObject,
                                    kVisitDataObjectGeneric>();
 
     table_.RegisterSpecializations<ObjectEvacuationStrategy<POINTER_OBJECT>,
                                    kVisitJSObject,
                                    kVisitJSObjectGeneric>();
 
     table_.RegisterSpecializations<ObjectEvacuationStrategy<POINTER_OBJECT>,
                                    kVisitStruct,
                                    kVisitStructGeneric>();
   }
 
-
-  static inline void Scavenge(Map* map, HeapObject** slot, HeapObject* obj) {
-    table_.GetVisitor(map)(map, slot, obj);
+  static VisitorDispatchTable<ScavengingCallback>* GetTable() {
+    return &table_;
   }
 
-
  private:
   enum ObjectContents  { DATA_OBJECT, POINTER_OBJECT };
   enum SizeRestriction { SMALL, UNKNOWN_SIZE };
 
 #if defined(DEBUG) || defined(ENABLE_LOGGING_AND_PROFILING)
   static void RecordCopiedObject(HeapObject* obj) {
     bool should_record = false;
 #ifdef DEBUG
     should_record = FLAG_heap_stats;
 #endif
@@ skipping 28 matching lines @@
 #endif
     HEAP_PROFILE(ObjectMoveEvent(source->address(), target->address()));
 #if defined(ENABLE_LOGGING_AND_PROFILING)
     if (Logger::is_logging() || CpuProfiler::is_profiling()) {
       if (target->IsSharedFunctionInfo()) {
         PROFILE(SharedFunctionInfoMoveEvent(
             source->address(), target->address()));
       }
     }
 #endif
+
+    if (marks_handling == TRANSFER_MARKS) TransferMark(source, target);
+
     return target;
   }
 
 
+  INLINE(static void TransferMark(HeapObject* from, HeapObject* to)) {
+    MarkBit from_mark_bit = Marking::MarkBitFrom(from);
+    if (IncrementalMarking::IsBlack(from_mark_bit)) {
+      IncrementalMarking::MarkBlack(Marking::MarkBitFrom(to));
+    }
+  }
+
   template<ObjectContents object_contents, SizeRestriction size_restriction>
   static inline void EvacuateObject(Map* map,
                                     HeapObject** slot,
                                     HeapObject* object,
                                     int object_size) {
     ASSERT((size_restriction != SMALL) ||
            (object_size <= Page::kMaxHeapObjectSize));
     ASSERT(object->Size() == object_size);
 
     if (Heap::ShouldBePromoted(object->address(), object_size)) {
@@ skipping 73 matching lines @@
 
   static inline bool IsShortcutCandidate(int type) {
     return ((type & kShortcutTypeMask) == kShortcutTypeTag);
   }
 
   static inline void EvacuateShortcutCandidate(Map* map,
                                                HeapObject** slot,
                                                HeapObject* object) {
     ASSERT(IsShortcutCandidate(map->instance_type()));
 
-    if (ConsString::cast(object)->unchecked_second() == Heap::empty_string()) {
+    if (marks_handling == IGNORE_MARKS &&
+        ConsString::cast(object)->unchecked_second() == Heap::empty_string()) {
       HeapObject* first =
           HeapObject::cast(ConsString::cast(object)->unchecked_first());
 
       *slot = first;
 
       if (!Heap::InNewSpace(first)) {
         object->set_map_word(MapWord::FromForwardingAddress(first));
         return;
       }
 
       MapWord first_word = first->map_word();
       if (first_word.IsForwardingAddress()) {
         HeapObject* target = first_word.ToForwardingAddress();
 
         *slot = target;
         object->set_map_word(MapWord::FromForwardingAddress(target));
         return;
       }
 
-      Scavenge(first->map(), slot, first);
+      DoScavengeObject(first->map(), slot, first);
       object->set_map_word(MapWord::FromForwardingAddress(*slot));
       return;
     }
 
     int object_size = ConsString::kSize;
     EvacuateObject<POINTER_OBJECT, SMALL>(map, slot, object, object_size);
   }
 
   template<ObjectContents object_contents>
   class ObjectEvacuationStrategy {
    public:
     template<int object_size>
     static inline void VisitSpecialized(Map* map,
                                         HeapObject** slot,
                                         HeapObject* object) {
       EvacuateObject<object_contents, SMALL>(map, slot, object, object_size);
     }
 
     static inline void Visit(Map* map,
                              HeapObject** slot,
                              HeapObject* object) {
       int object_size = map->instance_size();
       EvacuateObject<object_contents, SMALL>(map, slot, object, object_size);
     }
   };
 
-  typedef void (*Callback)(Map* map, HeapObject** slot, HeapObject* object);
-
-  static VisitorDispatchTable<Callback> table_;
+  static VisitorDispatchTable<ScavengingCallback> table_;
 };
 
 
-VisitorDispatchTable<ScavengingVisitor::Callback> ScavengingVisitor::table_;
+template<MarksHandling marks_handling>
+VisitorDispatchTable<ScavengingCallback>
+    ScavengingVisitor<marks_handling>::table_;
+
+
+void Heap::SelectScavengingVisitorsTable() {
+  if (IncrementalMarking::IsStopped()) {
+    scavening_visitors_table_.CopyFrom(
+        ScavengingVisitor<IGNORE_MARKS>::GetTable());
+  } else {
+    scavening_visitors_table_.CopyFrom(
+        ScavengingVisitor<TRANSFER_MARKS>::GetTable());
+  }
+}
 
 
 void Heap::ScavengeObjectSlow(HeapObject** p, HeapObject* object) {
   ASSERT(InFromSpace(object));
   MapWord first_word = object->map_word();
   ASSERT(!first_word.IsForwardingAddress());
   Map* map = first_word.ToMap();
-  ScavengingVisitor::Scavenge(map, p, object);
+  DoScavengeObject(map, p, object);
 }
 
 
 void Heap::ScavengePointer(HeapObject** p) {
   ScavengeObject(p, *p);
 }
 
 
 MaybeObject* Heap::AllocatePartialMap(InstanceType instance_type,
                                       int instance_size) {
@@ skipping 2226 matching lines @@
   { MaybeObject* maybe_result = Heap::Allocate(map, space);
     if (!maybe_result->ToObject(&result)) return maybe_result;
   }
   Struct::cast(result)->InitializeBody(size);
   return result;
 }
 
 
 void Heap::EnsureHeapIsIterable() {
   ASSERT(IsAllocationAllowed());
-  if (IncrementalMarking::state() != IncrementalMarking::STOPPED ||
-      old_pointer_space()->was_swept_conservatively() ||
+  if (old_pointer_space()->was_swept_conservatively() ||
       old_data_space()->was_swept_conservatively()) {
     CollectAllGarbage(kMakeHeapIterableMask);
   }
   ASSERT(!old_pointer_space()->was_swept_conservatively());
   ASSERT(!old_data_space()->was_swept_conservatively());
 }
 
 
 bool Heap::IdleNotification() {
   static const int kIdlesBeforeScavenge = 4;
@@ skipping 63 matching lines @@
   } else if (number_idle_notifications > kIdlesBeforeMarkCompact) {
     // If we have received more than kIdlesBeforeMarkCompact idle
     // notifications we do not perform any cleanup because we don't
     // expect to gain much by doing so.
     finished = true;
   }
 
   // Make sure that we have no pending context disposals and
   // conditionally uncommit from space.
   ASSERT(contexts_disposed_ == 0);
-  if (uncommit && IncrementalMarking::IsStopped()) Heap::UncommitFromSpace();
+  if (uncommit) Heap::UncommitFromSpace();
   return finished;
 }
 
 
 #ifdef DEBUG
 
 void Heap::Print() {
   if (!HasBeenSetup()) return;
   Top::PrintStack();
   AllSpaces spaces;
@@ skipping 786 matching lines @@
   // call Heap::TearDown() to release allocated memory.
   //
   // If the heap is not yet configured (eg, through the API), configure it.
   // Configuration is based on the flags new-space-size (really the semispace
   // size) and old-space-size if set or the initial values of semispace_size_
   // and old_generation_size_ otherwise.
   if (!heap_configured) {
     if (!ConfigureHeapDefault()) return false;
   }
 
-  ScavengingVisitor::Initialize();
+  ScavengingVisitor<TRANSFER_MARKS>::Initialize();
+  ScavengingVisitor<IGNORE_MARKS>::Initialize();
   NewSpaceScavenger::Initialize();
   MarkCompactCollector::Initialize();
 
   MarkMapPointersAsEncoded(false);
 
   // Setup memory allocator.
   if (!MemoryAllocator::Setup(MaxReserved(), MaxExecutableSize())) return false;
 
   // Setup new space.
   if (!new_space_.Setup(reserved_semispace_size_)) {
@@ skipping 920 matching lines @@
 void ExternalStringTable::TearDown() {
   new_space_strings_.Free();
   old_space_strings_.Free();
 }
 
 
 List<Object*> ExternalStringTable::new_space_strings_;
 List<Object*> ExternalStringTable::old_space_strings_;
 
 } }  // namespace v8::internal