[heap] Reland: Make SlotSet allocation thread-safe and refactor code.

src/heap/remembered-set.h

 // Copyright 2016 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef V8_REMEMBERED_SET_H
 #define V8_REMEMBERED_SET_H
 
 #include "src/assembler.h"
 #include "src/heap/heap.h"
 #include "src/heap/slot-set.h"
 #include "src/heap/spaces.h"
 
 namespace v8 {
 namespace internal {
 
-enum PointerDirection { OLD_TO_OLD, OLD_TO_NEW };
-
 // TODO(ulan): Investigate performance of de-templatizing this class.
-template <PointerDirection direction>
+template <RememberedSetType type>
 class RememberedSet : public AllStatic {
  public:
   // Given a page and a slot in that page, this function adds the slot to the
   // remembered set.
   static void Insert(MemoryChunk* chunk, Address slot_addr) {
     DCHECK(chunk->Contains(slot_addr));
-    SlotSet* slot_set = GetSlotSet(chunk);
+    SlotSet* slot_set = chunk->slot_set<type>();
     if (slot_set == nullptr) {
-      slot_set = AllocateSlotSet(chunk);
+      slot_set = chunk->AllocateSlotSet<type>();
     }
     uintptr_t offset = slot_addr - chunk->address();
     slot_set[offset / Page::kPageSize].Insert(offset % Page::kPageSize);
   }
 
   // Given a page and a slot in that page, this function returns true if
   // the remembered set contains the slot.
   static bool Contains(MemoryChunk* chunk, Address slot_addr) {
     DCHECK(chunk->Contains(slot_addr));
-    SlotSet* slot_set = GetSlotSet(chunk);
+    SlotSet* slot_set = chunk->slot_set<type>();
     if (slot_set == nullptr) {
       return false;
     }
     uintptr_t offset = slot_addr - chunk->address();
     return slot_set[offset / Page::kPageSize].Contains(offset %
                                                        Page::kPageSize);
   }
 
   // Given a page and a slot in that page, this function removes the slot from
   // the remembered set.
   // If the slot was never added, then the function does nothing.
   static void Remove(MemoryChunk* chunk, Address slot_addr) {
     DCHECK(chunk->Contains(slot_addr));
-    SlotSet* slot_set = GetSlotSet(chunk);
+    SlotSet* slot_set = chunk->slot_set<type>();
     if (slot_set != nullptr) {
       uintptr_t offset = slot_addr - chunk->address();
       slot_set[offset / Page::kPageSize].Remove(offset % Page::kPageSize);
     }
   }
 
   // Given a page and a range of slots in that page, this function removes the
   // slots from the remembered set.
   static void RemoveRange(MemoryChunk* chunk, Address start, Address end,
                           SlotSet::EmptyBucketMode mode) {
-    SlotSet* slot_set = GetSlotSet(chunk);
+    SlotSet* slot_set = chunk->slot_set<type>();
     if (slot_set != nullptr) {
       uintptr_t start_offset = start - chunk->address();
       uintptr_t end_offset = end - chunk->address();
       DCHECK_LT(start_offset, end_offset);
       if (end_offset < static_cast<uintptr_t>(Page::kPageSize)) {
         slot_set->RemoveRange(static_cast<int>(start_offset),
                               static_cast<int>(end_offset), mode);
       } else {
         // The large page has multiple slot sets.
         // Compute slot set indicies for the range [start_offset, end_offset).
@@ skipping 31 matching lines @@
         heap, [callback](MemoryChunk* chunk) { Iterate(chunk, callback); });
   }
 
   // Iterates over all memory chunks that contains non-empty slot sets.
   // The callback should take (MemoryChunk* chunk) and return void.
   template <typename Callback>
   static void IterateMemoryChunks(Heap* heap, Callback callback) {
     MemoryChunkIterator it(heap);
     MemoryChunk* chunk;
     while ((chunk = it.next()) != nullptr) {
-      SlotSet* slots = GetSlotSet(chunk);
-      TypedSlotSet* typed_slots = GetTypedSlotSet(chunk);
+      SlotSet* slots = chunk->slot_set<type>();
+      TypedSlotSet* typed_slots = chunk->typed_slot_set<type>();
       if (slots != nullptr || typed_slots != nullptr) {
         callback(chunk);
       }
     }
   }
 
   // Iterates and filters the remembered set in the given memory chunk with
   // the given callback. The callback should take (Address slot) and return
   // SlotCallbackResult.
   template <typename Callback>
   static void Iterate(MemoryChunk* chunk, Callback callback) {
-    SlotSet* slots = GetSlotSet(chunk);
+    SlotSet* slots = chunk->slot_set<type>();
     if (slots != nullptr) {
       size_t pages = (chunk->size() + Page::kPageSize - 1) / Page::kPageSize;
       int new_count = 0;
       for (size_t page = 0; page < pages; page++) {
         new_count +=
             slots[page].Iterate(callback, SlotSet::PREFREE_EMPTY_BUCKETS);
       }
       // Only old-to-old slot sets are released eagerly. Old-new-slot sets are
       // released by the sweeper threads.
-      if (direction == OLD_TO_OLD && new_count == 0) {
-        chunk->ReleaseOldToOldSlots();
+      if (type == OLD_TO_OLD && new_count == 0) {
+        chunk->ReleaseSlotSet<OLD_TO_OLD>();
       }
     }
   }
 
   // Given a page and a typed slot in that page, this function adds the slot
   // to the remembered set.
   static void InsertTyped(Page* page, Address host_addr, SlotType slot_type,
                           Address slot_addr) {
-    TypedSlotSet* slot_set = GetTypedSlotSet(page);
+    TypedSlotSet* slot_set = page->typed_slot_set<type>();
     if (slot_set == nullptr) {
-      AllocateTypedSlotSet(page);
-      slot_set = GetTypedSlotSet(page);
+      slot_set = page->AllocateTypedSlotSet<type>();
     }
     if (host_addr == nullptr) {
       host_addr = page->address();
     }
     uintptr_t offset = slot_addr - page->address();
     uintptr_t host_offset = host_addr - page->address();
     DCHECK_LT(offset, static_cast<uintptr_t>(TypedSlotSet::kMaxOffset));
     DCHECK_LT(host_offset, static_cast<uintptr_t>(TypedSlotSet::kMaxOffset));
     slot_set->Insert(slot_type, static_cast<uint32_t>(host_offset),
                      static_cast<uint32_t>(offset));
   }
 
   // Given a page and a range of typed slots in that page, this function removes
   // the slots from the remembered set.
   static void RemoveRangeTyped(MemoryChunk* page, Address start, Address end) {
-    TypedSlotSet* slots = GetTypedSlotSet(page);
+    TypedSlotSet* slots = page->typed_slot_set<type>();
     if (slots != nullptr) {
       slots->Iterate(
           [start, end](SlotType slot_type, Address host_addr,
                        Address slot_addr) {
             return start <= slot_addr && slot_addr < end ? REMOVE_SLOT
                                                          : KEEP_SLOT;
           },
           TypedSlotSet::PREFREE_EMPTY_CHUNKS);
     }
   }
 
   // Iterates and filters the remembered set with the given callback.
   // The callback should take (SlotType slot_type, SlotAddress slot) and return
   // SlotCallbackResult.
   template <typename Callback>
   static void IterateTyped(Heap* heap, Callback callback) {
     IterateMemoryChunks(heap, [callback](MemoryChunk* chunk) {
       IterateTyped(chunk, callback);
     });
   }
 
   // Iterates and filters typed old to old pointers in the given memory chunk
   // with the given callback. The callback should take (SlotType slot_type,
   // Address slot_addr) and return SlotCallbackResult.
   template <typename Callback>
   static void IterateTyped(MemoryChunk* chunk, Callback callback) {
-    TypedSlotSet* slots = GetTypedSlotSet(chunk);
+    TypedSlotSet* slots = chunk->typed_slot_set<type>();
     if (slots != nullptr) {
       int new_count = slots->Iterate(callback, TypedSlotSet::KEEP_EMPTY_CHUNKS);
       if (new_count == 0) {
-        ReleaseTypedSlotSet(chunk);
+        chunk->ReleaseTypedSlotSet<type>();
       }
     }
   }
 
   // Clear all old to old slots from the remembered set.
   static void ClearAll(Heap* heap) {
-    STATIC_ASSERT(direction == OLD_TO_OLD);
+    STATIC_ASSERT(type == OLD_TO_OLD);
     MemoryChunkIterator it(heap);
     MemoryChunk* chunk;
     while ((chunk = it.next()) != nullptr) {
-      chunk->ReleaseOldToOldSlots();
-      chunk->ReleaseTypedOldToOldSlots();
+      chunk->ReleaseSlotSet<OLD_TO_OLD>();
+      chunk->ReleaseTypedSlotSet<OLD_TO_OLD>();
     }
   }
 
   // Eliminates all stale slots from the remembered set, i.e.
   // slots that are not part of live objects anymore. This method must be
   // called after marking, when the whole transitive closure is known and
   // must be called before sweeping when mark bits are still intact.
   static void ClearInvalidTypedSlots(Heap* heap, MemoryChunk* chunk);
 
  private:
-  static SlotSet* GetSlotSet(MemoryChunk* chunk) {
-    if (direction == OLD_TO_OLD) {
-      return chunk->old_to_old_slots();
-    } else {
-      return chunk->old_to_new_slots();
-    }
-  }
-
-  static TypedSlotSet* GetTypedSlotSet(MemoryChunk* chunk) {
-    if (direction == OLD_TO_OLD) {
-      return chunk->typed_old_to_old_slots();
-    } else {
-      return chunk->typed_old_to_new_slots();
-    }
-  }
-
-  static void ReleaseTypedSlotSet(MemoryChunk* chunk) {
-    if (direction == OLD_TO_OLD) {
-      chunk->ReleaseTypedOldToOldSlots();
-    }
-  }
-
-  static SlotSet* AllocateSlotSet(MemoryChunk* chunk) {
-    if (direction == OLD_TO_OLD) {
-      chunk->AllocateOldToOldSlots();
-      return chunk->old_to_old_slots();
-    } else {
-      chunk->AllocateOldToNewSlots();
-      return chunk->old_to_new_slots();
-    }
-  }
-
-  static TypedSlotSet* AllocateTypedSlotSet(MemoryChunk* chunk) {
-    if (direction == OLD_TO_OLD) {
-      chunk->AllocateTypedOldToOldSlots();
-      return chunk->typed_old_to_old_slots();
-    } else {
-      chunk->AllocateTypedOldToNewSlots();
-      return chunk->typed_old_to_new_slots();
-    }
-  }
-
   static bool IsValidSlot(Heap* heap, MemoryChunk* chunk, Object** slot);
 };
 
 class UpdateTypedSlotHelper {
  public:
   // Updates a cell slot using an untyped slot callback.
   // The callback accepts Object** and returns SlotCallbackResult.
   template <typename Callback>
   static SlotCallbackResult UpdateCell(RelocInfo* rinfo, Callback callback) {
     DCHECK(rinfo->rmode() == RelocInfo::CELL);
@@ skipping 119 matching lines @@
     return DEBUG_TARGET_SLOT;
   }
   UNREACHABLE();
   return CLEARED_SLOT;
 }
 
 }  // namespace internal
 }  // namespace v8
 
 #endif  // V8_REMEMBERED_SET_H