Implement parallel pointer updates after evacuation.

src/heap/mark-compact.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/mark-compact.h"
 
 #include "src/base/atomicops.h"
 #include "src/base/bits.h"
 #include "src/base/sys-info.h"
 #include "src/code-stubs.h"
 #include "src/compilation-cache.h"
 #include "src/deoptimizer.h"
 #include "src/execution.h"
 #include "src/frames-inl.h"
 #include "src/gdb-jit.h"
 #include "src/global-handles.h"
 #include "src/heap/array-buffer-tracker.h"
 #include "src/heap/gc-tracer.h"
 #include "src/heap/incremental-marking.h"
 #include "src/heap/mark-compact-inl.h"
 #include "src/heap/object-stats.h"
 #include "src/heap/objects-visiting-inl.h"
 #include "src/heap/objects-visiting.h"
+#include "src/heap/page-parallel-job.h"
 #include "src/heap/spaces-inl.h"
 #include "src/ic/ic.h"
 #include "src/ic/stub-cache.h"
 #include "src/profiler/cpu-profiler.h"
 #include "src/utils-inl.h"
 #include "src/v8.h"
 
 namespace v8 {
 namespace internal {
 
@@ skipping 2747 matching lines @@
              !MarkCompactCollector::IsOnEvacuationCandidate(target));
     }
   }
 
  private:
   inline void UpdatePointer(Object** p) { UpdateSlot(heap_, p); }
 
   Heap* heap_;
 };
 
-
-static void UpdatePointer(HeapObject** address, HeapObject* object) {
+static void UpdateOldToNewSlot(HeapObject** address, HeapObject* object) {

[Michael Lippautz, 2016/03/09 09:48:53]

If its only use is within PointerUpdateJobTraits, just move it there?

   MapWord map_word = object->map_word();
   // Since we only filter invalid slots in old space, the store buffer can
   // still contain stale pointers in large object and in map spaces. Ignore
   // these pointers here.
   DCHECK(map_word.IsForwardingAddress() ||
          !object->GetHeap()->old_space()->Contains(
              reinterpret_cast<Address>(address)));
   if (map_word.IsForwardingAddress()) {
     // Update the corresponding slot.
     *address = map_word.ToForwardingAddress();
@@ skipping 348 matching lines @@
   //
   // The number of parallel compaction tasks is limited by:
   // - #evacuation pages
   // - (#cores - 1)
   const double kTargetCompactionTimeInMs = 1;
   const int kNumSweepingTasks = 3;
 
   intptr_t compaction_speed =
       heap()->tracer()->CompactionSpeedInBytesPerMillisecond();
 
-  const int available_cores =
-      Max(1, base::SysInfo::NumberOfProcessors() - kNumSweepingTasks - 1);
+  const int available_cores = Max(
+      1, static_cast<int>(

[Michael Lippautz, 2016/03/09 09:48:53]

Thanks for the drive-by-fix ;)

[ulan, 2016/03/09 17:34:18]

Done.

+             V8::GetCurrentPlatform()->NumberOfAvailableBackgroundThreads()) -
+             kNumSweepingTasks - 1);
   int tasks;
   if (compaction_speed > 0) {
     tasks = 1 + static_cast<int>(static_cast<double>(live_bytes) /
                                  compaction_speed / kTargetCompactionTimeInMs);
   } else {
     tasks = pages;
   }
   const int tasks_capped_pages = Min(pages, tasks);
   return Min(available_cores, tasks_capped_pages);
 }
@@ skipping 405 matching lines @@
     ReleaseEvacuationCandidates();
   }
 
 #ifdef VERIFY_HEAP
   if (FLAG_verify_heap && !sweeping_in_progress_) {
     VerifyEvacuation(heap());
   }
 #endif
 }
 
+template <PointerDirection direction>
+class PointerUpdateJobTraits {
+ public:
+  typedef int PerPageData;

[Michael Lippautz, 2016/03/09 09:48:53]

Maybe add a comment that PerPageData is unused for this trait.

[ulan, 2016/03/09 17:34:18]

Done.

+  typedef PointersUpdatingVisitor* PerTaskData;
+
+  static bool ProcessPageInParallel(Heap* heap,
+                                    PointersUpdatingVisitor* visitor,

[Michael Lippautz, 2016/03/09 09:48:53]

Not sure if the equivalent version of the decl
  static bool ProcessPageInParallel(Heap*, __ PerTaskData __, MemoryChunk*,
PerPageData) 
is more readable

[ulan, 2016/03/09 17:34:18]

Done. It is shorter :)

+                                    MemoryChunk* chunk, PerPageData) {
+    UpdateUntypedPointers(heap, chunk);
+    UpdateTypedPointers(heap, chunk, visitor);
+    return true;
+  }
+  static const bool NeedSequentialFinalization = false;
+  static void FinalizePageSequentially(Heap*, MemoryChunk*, bool, PerPageData) {
+  }
+
+ private:
+  static void UpdateUntypedPointers(Heap* heap, MemoryChunk* chunk) {
+    if (direction == OLD_TO_NEW) {
+      RememberedSet<OLD_TO_NEW>::IterateWithWrapper(heap, chunk,
+                                                    UpdateOldToNewSlot);
+    } else {
+      RememberedSet<OLD_TO_OLD>::Iterate(chunk, [heap](Address slot) {
+        PointersUpdatingVisitor::UpdateSlot(heap,
+                                            reinterpret_cast<Object**>(slot));
+        return REMOVE_SLOT;
+      });
+    }
+  }
+
+  static void UpdateTypedPointers(Heap* heap, MemoryChunk* chunk,
+                                  PointersUpdatingVisitor* visitor) {
+    if (direction == OLD_TO_OLD) {
+      Isolate* isolate = heap->isolate();
+      RememberedSet<OLD_TO_OLD>::IterateTyped(
+          chunk, [isolate, visitor](SlotType type, Address slot) {
+            UpdateTypedSlot(isolate, visitor, type, slot);
+            return REMOVE_SLOT;
+          });
+    }
+  }
+};
+
+int NumberOfPointerUpdateTasks(int pages) {
+  if (!FLAG_parallel_pointer_update) return 1;
+  const int kMaxTasks = 4;
+  const int kPagesPerTask = 4;
+  return Min(kMaxTasks, (pages + kPagesPerTask - 1) / kPagesPerTask);
+}
+
+template <PointerDirection direction>
+void UpdatePointersInParallel(Heap* heap) {
+  PageParallelJob<PointerUpdateJobTraits<direction> > job(
+      heap, heap->isolate()->cancelable_task_manager());
+  RememberedSet<direction>::IterateMemoryChunks(
+      heap, [&job](MemoryChunk* chunk) { job.AddPage(chunk, 0); });
+  PointersUpdatingVisitor visitor(heap);
+  int num_pages = job.NumberOfPages();
+  int num_tasks = NumberOfPointerUpdateTasks(num_pages);
+  job.Run(num_tasks, [&visitor](int i) { return &visitor; });
+}
 
 void MarkCompactCollector::UpdatePointersAfterEvacuation() {
   GCTracer::Scope gc_scope(heap()->tracer(),
                            GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS);
 
   PointersUpdatingVisitor updating_visitor(heap());
 
   {
     GCTracer::Scope gc_scope(
         heap()->tracer(), GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_TO_NEW);
     // Update pointers in to space.
     SemiSpaceIterator to_it(heap()->new_space());
     for (HeapObject* object = to_it.Next(); object != NULL;
          object = to_it.Next()) {
       Map* map = object->map();
       object->IterateBody(map->instance_type(), object->SizeFromMap(map),
                           &updating_visitor);
     }
     // Update roots.
     heap_->IterateRoots(&updating_visitor, VISIT_ALL_IN_SWEEP_NEWSPACE);
 
-    RememberedSet<OLD_TO_NEW>::IterateWithWrapper(heap_, UpdatePointer);
+    UpdatePointersInParallel<OLD_TO_NEW>(heap_);
   }
 
   {
     Heap* heap = this->heap();
     GCTracer::Scope gc_scope(
         heap->tracer(),
         GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_TO_EVACUATED);
-
-    RememberedSet<OLD_TO_OLD>::Iterate(heap, [heap](Address slot) {
-      PointersUpdatingVisitor::UpdateSlot(heap,
-                                          reinterpret_cast<Object**>(slot));
-      return REMOVE_SLOT;
-    });
-    Isolate* isolate = heap->isolate();
-    PointersUpdatingVisitor* visitor = &updating_visitor;
-    RememberedSet<OLD_TO_OLD>::IterateTyped(
-        heap, [isolate, visitor](SlotType type, Address slot) {
-          UpdateTypedSlot(isolate, visitor, type, slot);
-          return REMOVE_SLOT;
-        });
+    UpdatePointersInParallel<OLD_TO_OLD>(heap_);
   }
 
   {
     GCTracer::Scope gc_scope(
         heap()->tracer(),
         GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_BETWEEN_EVACUATED);
     for (Page* p : evacuation_candidates_) {
       DCHECK(p->IsEvacuationCandidate() ||
              p->IsFlagSet(Page::RESCAN_ON_EVACUATION));
 
@@ skipping 267 matching lines @@
     MarkBit mark_bit = Marking::MarkBitFrom(host);
     if (Marking::IsBlack(mark_bit)) {
       RelocInfo rinfo(isolate(), pc, RelocInfo::CODE_TARGET, 0, host);
       RecordRelocSlot(host, &rinfo, target);
     }
   }
 }
 
 }  // namespace internal
 }  // namespace v8