[heap] Allow concurrently transferring colors

src/heap/incremental-marking.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef V8_HEAP_INCREMENTAL_MARKING_H_
 #define V8_HEAP_INCREMENTAL_MARKING_H_
 
 #include "src/cancelable-task.h"
 #include "src/execution.h"
 #include "src/heap/heap.h"
@@ skipping 14 matching lines @@
 class V8_EXPORT_PRIVATE IncrementalMarking {
  public:
   enum State { STOPPED, SWEEPING, MARKING, COMPLETE };
 
   enum CompletionAction { GC_VIA_STACK_GUARD, NO_GC_VIA_STACK_GUARD };
 
   enum ForceCompletionAction { FORCE_COMPLETION, DO_NOT_FORCE_COMPLETION };
 
   enum GCRequestType { NONE, COMPLETE_MARKING, FINALIZATION };
 
+  static void MarkGrey(Heap* heap, HeapObject* object);
+
+  static void MarkBlack(HeapObject* object, int size);
+
+  // Transfers mark bits without requiring proper object headers.
+  static void TransferMark(Heap* heap, HeapObject* from, HeapObject* to);
+
+  // Transfers color including live byte count, requiring properly set up
+  // objects.
+  template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
+  V8_INLINE static void TransferColor(HeapObject* from, HeapObject* to) {
+    if (ObjectMarking::IsBlack<access_mode>(to, MarkingState::Internal(to))) {
+      DCHECK(to->GetHeap()->incremental_marking()->black_allocation());
+      return;
+    }
+
+    DCHECK(ObjectMarking::IsWhite<access_mode>(to, MarkingState::Internal(to)));
+    if (ObjectMarking::IsGrey<access_mode>(from,
+                                           MarkingState::Internal(from))) {
+      ObjectMarking::WhiteToGrey<access_mode>(to, MarkingState::Internal(to));
+    } else if (ObjectMarking::IsBlack<access_mode>(
+                   from, MarkingState::Internal(from))) {
+      ObjectMarking::WhiteToBlack<access_mode>(to, MarkingState::Internal(to));
+    }
+  }
+
   explicit IncrementalMarking(Heap* heap);
 
   static void Initialize();
 
   State state() {
     DCHECK(state_ == STOPPED || FLAG_incremental_marking);
     return state_;
   }
 
   bool should_hurry() { return should_hurry_; }
@@ skipping 127 matching lines @@
   void ActivateGeneratedStub(Code* stub);
 
   void NotifyIncompleteScanOfObject(int unscanned_bytes) {
     unscanned_bytes_of_large_object_ = unscanned_bytes;
   }
 
   void ClearIdleMarkingDelayCounter();
 
   bool IsIdleMarkingDelayCounterLimitReached();
 
-  static void MarkGrey(Heap* heap, HeapObject* object);
-
-  static void MarkBlack(HeapObject* object, int size);
-
-  static void TransferMark(Heap* heap, HeapObject* from, HeapObject* to);
-
-  V8_INLINE static void TransferColor(HeapObject* from, HeapObject* to) {
-    if (ObjectMarking::IsBlack(to, MarkingState::Internal(to))) {
-      DCHECK(to->GetHeap()->incremental_marking()->black_allocation());
-      return;
-    }
-
-    DCHECK(ObjectMarking::IsWhite(to, MarkingState::Internal(to)));
-    if (ObjectMarking::IsGrey(from, MarkingState::Internal(from))) {
-      ObjectMarking::WhiteToGrey(to, MarkingState::Internal(to));
-    } else if (ObjectMarking::IsBlack(from, MarkingState::Internal(from))) {
-      ObjectMarking::WhiteToBlack(to, MarkingState::Internal(to));
-    }
-  }
-
   void IterateBlackObject(HeapObject* object);
 
   Heap* heap() const { return heap_; }
 
   IncrementalMarkingJob* incremental_marking_job() {
     return &incremental_marking_job_;
   }
 
   bool black_allocation() { return black_allocation_; }
 
@@ skipping 83 matching lines @@
 
   Observer new_generation_observer_;
   Observer old_generation_observer_;
 
   DISALLOW_IMPLICIT_CONSTRUCTORS(IncrementalMarking);
 };
 }  // namespace internal
 }  // namespace v8
 
 #endif  // V8_HEAP_INCREMENTAL_MARKING_H_