Reland of "[heap] Better integrate handling of aborted compaction pages"

src/heap/mark-compact.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/mark-compact.h"
 
 #include "src/base/atomicops.h"
 #include "src/base/bits.h"
 #include "src/base/sys-info.h"
 #include "src/code-stubs.h"
@@ skipping 1800 matching lines @@
         Page::FromAddress(object->address())->owner()->identity());
     HeapObject* target_object = nullptr;
     if (TryEvacuateObject(target_space, object, &target_object)) {
       DCHECK(object->map_word().IsForwardingAddress());
       return true;
     }
     return false;
   }
 };
 
+class MarkCompactCollector::EvacuateRecordOnlyVisitor final
+    : public MarkCompactCollector::HeapObjectVisitor {
+ public:
+  explicit EvacuateRecordOnlyVisitor(AllocationSpace space) : space_(space) {}
+
+  inline bool Visit(HeapObject* object) {
+    if (space_ == OLD_SPACE) {
+      RecordMigratedSlotVisitor visitor;
+      object->IterateBody(&visitor);
+    } else {
+      DCHECK_EQ(space_, CODE_SPACE);
+      // Add a typed slot for the whole code object.
+      RememberedSet<OLD_TO_OLD>::InsertTyped(

[Michael Lippautz, 2016/04/14 07:50:11]

We also need to record within CODE_SPACE. I am using the hammer
(RELOCATED_CODE_OBJECT) instead of beefing up RecordMigratedSlotVisitor. (Same
as we do in MigrateObject).

+          Page::FromAddress(object->address()), RELOCATED_CODE_OBJECT,
+          object->address());
+    }
+    return true;
+  }
+
+ private:
+  AllocationSpace space_;
+};
 
 void MarkCompactCollector::DiscoverGreyObjectsInSpace(PagedSpace* space) {
   PageIterator it(space);
   while (it.has_next()) {
     Page* p = it.next();
     if (!p->IsFlagSet(Page::BLACK_PAGE)) {
       DiscoverGreyObjectsOnPage(p);
     }
     if (marking_deque()->IsFull()) return;
   }
@@ skipping 1254 matching lines @@
   if (chunk->InNewSpace()) {
     DCHECK_EQ(chunk->concurrent_sweeping_state().Value(),
               NewSpacePage::kSweepingDone);
     success = EvacuateSinglePage<kClearMarkbits>(chunk, &new_space_visitor_);
     DCHECK(success);
     USE(success);
   } else {
     DCHECK(chunk->IsEvacuationCandidate());
     DCHECK_EQ(chunk->concurrent_sweeping_state().Value(), Page::kSweepingDone);
     success = EvacuateSinglePage<kClearMarkbits>(chunk, &old_space_visitor_);
+    if (!success) {
+      // Aborted compaction page. We can record slots here to have them
+      // processed in parallel later on.
+      EvacuateRecordOnlyVisitor record_visitor(chunk->owner()->identity());
+      success = EvacuateSinglePage<kKeepMarking>(chunk, &record_visitor);
+      DCHECK(success);
+      USE(success);
+      // We need to return failure here to indicate that we want this page added
+      // to the sweeper.
+      return false;
+    }
   }
   return success;
 }
 
 void MarkCompactCollector::Evacuator::Finalize() {
   heap()->old_space()->MergeCompactionSpace(compaction_spaces_.Get(OLD_SPACE));
   heap()->code_space()->MergeCompactionSpace(
       compaction_spaces_.Get(CODE_SPACE));
   heap()->tracer()->AddCompactionEvent(duration_, bytes_compacted_);
   heap()->IncrementPromotedObjectsSize(new_space_visitor_.promoted_size());
@@ skipping 40 matching lines @@
   typedef int* PerPageData;  // Pointer to number of aborted pages.
   typedef MarkCompactCollector::Evacuator* PerTaskData;
 
   static const bool NeedSequentialFinalization = true;
 
   static bool ProcessPageInParallel(Heap* heap, PerTaskData evacuator,
                                     MemoryChunk* chunk, PerPageData) {
     return evacuator->EvacuatePage(chunk);
   }
 
-  static void FinalizePageSequentially(Heap*, MemoryChunk* chunk, bool success,
-                                       PerPageData data) {
+  static void FinalizePageSequentially(Heap* heap, MemoryChunk* chunk,
+                                       bool success, PerPageData data) {
     if (chunk->InNewSpace()) {
       DCHECK(success);
     } else {
       Page* p = static_cast<Page*>(chunk);
       if (success) {
         DCHECK(p->IsEvacuationCandidate());
         DCHECK(p->SweepingDone());
         p->Unlink();
       } else {
         // We have partially compacted the page, i.e., some objects may have
         // moved, others are still in place.
-        // We need to:
-        // - Leave the evacuation candidate flag for later processing of slots
-        //   buffer entries.
-        // - Leave the slots buffer there for processing of entries added by
-        //   the write barrier.
-        // - Rescan the page as slot recording in the migration buffer only
-        //   happens upon moving (which we potentially didn't do).
-        // - Leave the page in the list of pages of a space since we could not
-        //   fully evacuate it.
-        DCHECK(p->IsEvacuationCandidate());
         p->SetFlag(Page::COMPACTION_WAS_ABORTED);
+        p->ClearEvacuationCandidate();
+        // Slots have already been recorded so we just need to add it to the
+        // sweeper.

[Michael Lippautz, 2016/04/14 07:50:11]

We cannot add the page to the sweeper here because we still require forward
pointers to be fixed during pointer updates (basically: update between
evacuated)

         *data += 1;
       }
     }
   }
 };
 
 void MarkCompactCollector::EvacuatePagesInParallel() {
   PageParallelJob<EvacuationJobTraits> job(
       heap_, heap_->isolate()->cancelable_task_manager());
 
@@ skipping 223 matching lines @@
     object->IterateBody(map->instance_type(), size, visitor);
   }
 }
 
 void MarkCompactCollector::Sweeper::AddSweptPageSafe(PagedSpace* space,
                                                      Page* page) {
   base::LockGuard<base::Mutex> guard(&mutex_);
   swept_list_[space->identity()].Add(page);
 }
 
-void MarkCompactCollector::SweepAbortedPages() {
-  // Second pass on aborted pages.
-  for (Page* p : evacuation_candidates_) {
-    if (p->IsFlagSet(Page::COMPACTION_WAS_ABORTED)) {
-      p->ClearFlag(MemoryChunk::COMPACTION_WAS_ABORTED);
-      p->concurrent_sweeping_state().SetValue(Page::kSweepingInProgress);
-      PagedSpace* space = static_cast<PagedSpace*>(p->owner());
-      switch (space->identity()) {
-        case OLD_SPACE:
-          Sweeper::RawSweep<Sweeper::SWEEP_ONLY, Sweeper::SWEEP_ON_MAIN_THREAD,
-                            Sweeper::IGNORE_SKIP_LIST,
-                            Sweeper::IGNORE_FREE_SPACE>(space, p, nullptr);
-          break;
-        case CODE_SPACE:
-          if (FLAG_zap_code_space) {
-            Sweeper::RawSweep<
-                Sweeper::SWEEP_ONLY, Sweeper::SWEEP_ON_MAIN_THREAD,
-                Sweeper::REBUILD_SKIP_LIST, Sweeper::ZAP_FREE_SPACE>(space, p,
-                                                                     nullptr);
-          } else {
-            Sweeper::RawSweep<
-                Sweeper::SWEEP_ONLY, Sweeper::SWEEP_ON_MAIN_THREAD,
-                Sweeper::REBUILD_SKIP_LIST, Sweeper::IGNORE_FREE_SPACE>(
-                space, p, nullptr);
-          }
-          break;
-        default:
-          UNREACHABLE();
-          break;
-      }
-      sweeper().AddSweptPageSafe(space, p);
-    }
-  }
-}
-
-
 void MarkCompactCollector::EvacuateNewSpaceAndCandidates() {
   TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_EVACUATE);
   Heap::RelocationLock relocation_lock(heap());
 
   {
     TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_EVACUATE_COPY);
     EvacuationScope evacuation_scope(this);
 
     EvacuateNewSpacePrologue();
     EvacuatePagesInParallel();
     EvacuateNewSpaceEpilogue();
     heap()->new_space()->set_age_mark(heap()->new_space()->top());
   }
 
   UpdatePointersAfterEvacuation();
 
   // Give pages that are queued to be freed back to the OS. Note that filtering
   // slots only handles old space (for unboxed doubles), and thus map space can
   // still contain stale pointers. We only free the chunks after pointer updates
   // to still have access to page headers.
   heap()->FreeQueuedChunks();
 
   {
     TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_EVACUATE_CLEAN_UP);
-    // After updating all pointers, we can finally sweep the aborted pages,
-    // effectively overriding any forward pointers.
-    SweepAbortedPages();
+
+    for (Page* p : evacuation_candidates_) {

[Michael Lippautz, 2016/04/14 07:50:11]

Moving this part to MC_EVACUATE_CLEAN_UP as it has nothing to do with the
previous phase anymore.

MC_EVACUATE_UPDATE_POINTERS_BETWEEN_EVACUATED does not exist anymore as we do
all necessary pointer updating in parallel in the preceding phases.

+      // Important: skip list should be cleared only after roots were updated
+      // because root iteration traverses the stack and might have to find
+      // code objects from non-updated pc pointing into evacuation candidate.
+      SkipList* list = p->skip_list();
+      if (list != NULL) list->Clear();
+      if (p->IsFlagSet(Page::COMPACTION_WAS_ABORTED)) {
+        sweeper().AddLatePage(p->owner()->identity(), p);
+        p->ClearFlag(Page::COMPACTION_WAS_ABORTED);
+      }
+    }
 
     // EvacuateNewSpaceAndCandidates iterates over new space objects and for
     // ArrayBuffers either re-registers them as live or promotes them. This is
     // needed to properly free them.
     heap()->array_buffer_tracker()->FreeDead(false);
 
     // Deallocate evacuated candidate pages.
     ReleaseEvacuationCandidates();
   }
 
@@ skipping 135 matching lines @@
 
   {
     Heap* heap = this->heap();
     TRACE_GC(heap->tracer(),
              GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_TO_EVACUATED);
     UpdatePointersInParallel<OLD_TO_OLD>(heap_);
   }
 
   {
     TRACE_GC(heap()->tracer(),
-             GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_BETWEEN_EVACUATED);
-    for (Page* p : evacuation_candidates_) {
-      DCHECK(p->IsEvacuationCandidate());
-      // Important: skip list should be cleared only after roots were updated
-      // because root iteration traverses the stack and might have to find
-      // code objects from non-updated pc pointing into evacuation candidate.
-      SkipList* list = p->skip_list();
-      if (list != NULL) list->Clear();
-
-      // First pass on aborted pages, fixing up all live objects.
-      if (p->IsFlagSet(Page::COMPACTION_WAS_ABORTED)) {
-        p->ClearEvacuationCandidate();
-        VisitLiveObjectsBody(p, &updating_visitor);
-      }
-    }
-  }
-
-  {
-    TRACE_GC(heap()->tracer(),
              GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_WEAK);
     // Update pointers from external string table.
     heap_->UpdateReferencesInExternalStringTable(
         &UpdateReferenceInExternalStringTableEntry);
 
     EvacuationWeakObjectRetainer evacuation_object_retainer;
     heap()->ProcessWeakListRoots(&evacuation_object_retainer);
   }
 }
 
@@ skipping 227 matching lines @@
     MarkBit mark_bit = Marking::MarkBitFrom(host);
     if (Marking::IsBlack(mark_bit)) {
       RelocInfo rinfo(isolate(), pc, RelocInfo::CODE_TARGET, 0, host);
       RecordRelocSlot(host, &rinfo, target);
     }
   }
 }
 
 }  // namespace internal
 }  // namespace v8