[heap] Parallel newspace evacuation, semispace copy, and compaction \o/

src/heap/heap-inl.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef V8_HEAP_HEAP_INL_H_
 #define V8_HEAP_HEAP_INL_H_
 
 #include <cmath>
 
 #include "src/base/platform/platform.h"
@@ skipping 449 matching lines @@
     Object** end_slot = src_slot + size_in_words;
 
     while (src_slot != end_slot) {
       *dst_slot++ = *src_slot++;
     }
   } else {
     MemMove(dst, src, static_cast<size_t>(byte_size));
   }
 }
 
-
+template <Heap::FindMementoMode mode>
 AllocationMemento* Heap::FindAllocationMemento(HeapObject* object) {
   // Check if there is potentially a memento behind the object. If
   // the last word of the memento is on another page we return
   // immediately.
   Address object_address = object->address();
   Address memento_address = object_address + object->Size();
   Address last_memento_word_address = memento_address + kPointerSize;
   if (!NewSpacePage::OnSamePage(object_address, last_memento_word_address)) {
-    return NULL;
+    return nullptr;
   }
-
   HeapObject* candidate = HeapObject::FromAddress(memento_address);
   Map* candidate_map = candidate->map();
   // This fast check may peek at an uninitialized word. However, the slow check
   // below (memento_address == top) ensures that this is safe. Mark the word as
   // initialized to silence MemorySanitizer warnings.
   MSAN_MEMORY_IS_INITIALIZED(&candidate_map, sizeof(candidate_map));
-  if (candidate_map != allocation_memento_map()) return NULL;
+  if (candidate_map != allocation_memento_map()) {
+    return nullptr;
+  }
+  AllocationMemento* memento_candidate = AllocationMemento::cast(candidate);
 
-  // Either the object is the last object in the new space, or there is another
-  // object of at least word size (the header map word) following it, so
-  // suffices to compare ptr and top here. Note that technically we do not have
-  // to compare with the current top pointer of the from space page during GC,
-  // since we always install filler objects above the top pointer of a from
-  // space page when performing a garbage collection. However, always performing
-  // the test makes it possible to have a single, unified version of
-  // FindAllocationMemento that is used both by the GC and the mutator.
-  Address top = NewSpaceTop();
-  DCHECK(memento_address == top ||
-         memento_address + HeapObject::kHeaderSize <= top ||
-         !NewSpacePage::OnSamePage(memento_address, top - 1));
-  if (memento_address == top) return NULL;
-
-  AllocationMemento* memento = AllocationMemento::cast(candidate);
-  if (!memento->IsValid()) return NULL;
-  return memento;
+  // Depending on what the memento is used for, we might need to perform
+  // additional checks.
+  Address top;
+  switch (mode) {
+    case Heap::kForGC:
+      return memento_candidate;
+    case Heap::kForRuntime:
+      if (memento_candidate == nullptr) return nullptr;
+      // Either the object is the last object in the new space, or there is
+      // another object of at least word size (the header map word) following
+      // it, so suffices to compare ptr and top here.
+      top = NewSpaceTop();
+      DCHECK(memento_address == top ||
+             memento_address + HeapObject::kHeaderSize <= top ||
+             !NewSpacePage::OnSamePage(memento_address, top - 1));
+      if ((memento_address != top) && memento_candidate->IsValid()) {
+        return memento_candidate;
+      }
+      return nullptr;
+    default:
+      UNREACHABLE();
+  }
+  UNREACHABLE();
+  return nullptr;
 }
 
 
 void Heap::UpdateAllocationSite(HeapObject* object,
                                 HashMap* pretenuring_feedback) {
   DCHECK(InFromSpace(object));
   if (!FLAG_allocation_site_pretenuring ||
       !AllocationSite::CanTrack(object->map()->instance_type()))
     return;
-  AllocationMemento* memento = FindAllocationMemento(object);
-  if (memento == nullptr) return;
-
-  AllocationSite* key = memento->GetAllocationSite();
-  DCHECK(!key->IsZombie());
+  AllocationMemento* memento_candidate = FindAllocationMemento<kForGC>(object);
+  if (memento_candidate == nullptr) return;
 
   if (pretenuring_feedback == global_pretenuring_feedback_) {
+    // Entering global pretenuring feedback is only used in the scavenger, where
+    // we are allowed to actually touch the allocation site.
+    if (!memento_candidate->IsValid()) return;
+    AllocationSite* site = memento_candidate->GetAllocationSite();
+    DCHECK(!site->IsZombie());
     // For inserting in the global pretenuring storage we need to first
     // increment the memento found count on the allocation site.
-    if (key->IncrementMementoFoundCount()) {
-      global_pretenuring_feedback_->LookupOrInsert(
-          key, static_cast<uint32_t>(bit_cast<uintptr_t>(key)));
+    if (site->IncrementMementoFoundCount()) {
+      global_pretenuring_feedback_->LookupOrInsert(site,
+                                                   ObjectHash(site->address()));
     }
   } else {
-    // Any other pretenuring storage than the global one is used as a cache,
-    // where the count is later on merge in the allocation site.
-    HashMap::Entry* e = pretenuring_feedback->LookupOrInsert(
-        key, static_cast<uint32_t>(bit_cast<uintptr_t>(key)));
+    // Entering cached feedback is used in the parallel case. We are not allowed
+    // to dereference the allocation site and rather have to postpone all checks
+    // till actually merging the data.
+    Address key = memento_candidate->GetAllocationSiteUnchecked();
+    HashMap::Entry* e =
+        pretenuring_feedback->LookupOrInsert(key, ObjectHash(key));
     DCHECK(e != nullptr);
     (*bit_cast<intptr_t*>(&e->value))++;
   }
 }
 
 
 void Heap::RemoveAllocationSitePretenuringFeedback(AllocationSite* site) {
   global_pretenuring_feedback_->Remove(
       site, static_cast<uint32_t>(bit_cast<uintptr_t>(site)));
 }
@@ skipping 175 matching lines @@
 
 void VerifySmisVisitor::VisitPointers(Object** start, Object** end) {
   for (Object** current = start; current < end; current++) {
     CHECK((*current)->IsSmi());
   }
 }
 }  // namespace internal
 }  // namespace v8
 
 #endif  // V8_HEAP_HEAP_INL_H_