[heap] Parallel newspace evacuation, semispace copy, and compaction \o/

src/heap/heap-inl.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef V8_HEAP_HEAP_INL_H_
 #define V8_HEAP_HEAP_INL_H_
 
 #include <cmath>
 
 #include "src/base/platform/platform.h"
@@ skipping 450 matching lines @@
 
     while (src_slot != end_slot) {
       *dst_slot++ = *src_slot++;
     }
   } else {
     MemMove(dst, src, static_cast<size_t>(byte_size));
   }
 }
 
 
 AllocationMemento* Heap::FindAllocationMemento(HeapObject* object) {

[Hannes Payer (out of office), 2016/01/18 11:46:33]

As discussed offline:
There was a reason why there is just version of that function: to not forget
which one is the right one to use. Let's templatize this function based on it's
caller, GC or Runtime, and perform the corresponding checks.

[Michael Lippautz, 2016/01/19 14:56:52]

Done.

+  AllocationMemento* memento_candidate = FindAllocationMementoCandidate(object);
+  if (memento_candidate == nullptr) return nullptr;
+  // Either the object is the last object in the new space, or there is another
+  // object of at least word size (the header map word) following it, so
+  // suffices to compare ptr and top here. Note that technically we do not have
+  // to compare with the current top pointer of the from space page during GC,
+  // since we always install filler objects above the top pointer of a from
+  // space page when performing a garbage collection. However, always performing
+  // the test makes it possible to have a single, unified version of
+  // FindAllocationMemento that is used both by the GC and the mutator.
+  Address top = NewSpaceTop();
+  Address memento_address = memento_candidate->address();
+  DCHECK(memento_address == top ||
+         memento_address + HeapObject::kHeaderSize <= top ||
+         !NewSpacePage::OnSamePage(memento_address, top - 1));
+  if (memento_address == top) {
+    return nullptr;
+  }
+  if (memento_candidate->IsValid()) {
+    return memento_candidate;
+  }
+
+  return nullptr;
+}
+
+
+AllocationMemento* Heap::FindAllocationMementoCandidate(HeapObject* object) {
   // Check if there is potentially a memento behind the object. If
   // the last word of the memento is on another page we return
   // immediately.
   Address object_address = object->address();
   Address memento_address = object_address + object->Size();
   Address last_memento_word_address = memento_address + kPointerSize;
   if (!NewSpacePage::OnSamePage(object_address, last_memento_word_address)) {
-    return NULL;
+    return nullptr;
   }
-
   HeapObject* candidate = HeapObject::FromAddress(memento_address);
   Map* candidate_map = candidate->map();
   // This fast check may peek at an uninitialized word. However, the slow check
   // below (memento_address == top) ensures that this is safe. Mark the word as
   // initialized to silence MemorySanitizer warnings.
   MSAN_MEMORY_IS_INITIALIZED(&candidate_map, sizeof(candidate_map));
-  if (candidate_map != allocation_memento_map()) return NULL;
-
-  // Either the object is the last object in the new space, or there is another
-  // object of at least word size (the header map word) following it, so
-  // suffices to compare ptr and top here. Note that technically we do not have
-  // to compare with the current top pointer of the from space page during GC,
-  // since we always install filler objects above the top pointer of a from
-  // space page when performing a garbage collection. However, always performing
-  // the test makes it possible to have a single, unified version of
-  // FindAllocationMemento that is used both by the GC and the mutator.
-  Address top = NewSpaceTop();
-  DCHECK(memento_address == top ||
-         memento_address + HeapObject::kHeaderSize <= top ||
-         !NewSpacePage::OnSamePage(memento_address, top - 1));
-  if (memento_address == top) return NULL;
-
-  AllocationMemento* memento = AllocationMemento::cast(candidate);
-  if (!memento->IsValid()) return NULL;
-  return memento;
+  if (candidate_map != allocation_memento_map()) {
+    return nullptr;
+  }
+  return AllocationMemento::cast(candidate);
 }
 
 
 void Heap::UpdateAllocationSite(HeapObject* object,
                                 HashMap* pretenuring_feedback) {
   DCHECK(InFromSpace(object));
   if (!FLAG_allocation_site_pretenuring ||
       !AllocationSite::CanTrack(object->map()->instance_type()))
     return;
-  AllocationMemento* memento = FindAllocationMemento(object);
-  if (memento == nullptr) return;
-
-  AllocationSite* key = memento->GetAllocationSite();
-  DCHECK(!key->IsZombie());
+  AllocationMemento* memento_candidate = FindAllocationMementoCandidate(object);
+  if (memento_candidate == nullptr) return;
 
   if (pretenuring_feedback == global_pretenuring_feedback_) {
+    // Entering global pretenuring feedback is only used in the scavenger, where
+    // we are allowed to actually touch the allocation site.
+    if (!memento_candidate->IsValid()) return;
+    AllocationSite* site = memento_candidate->GetAllocationSite();
+    DCHECK(!site->IsZombie());
     // For inserting in the global pretenuring storage we need to first
     // increment the memento found count on the allocation site.
-    if (key->IncrementMementoFoundCount()) {
+    if (site->IncrementMementoFoundCount()) {
       global_pretenuring_feedback_->LookupOrInsert(
-          key, static_cast<uint32_t>(bit_cast<uintptr_t>(key)));
+          site, static_cast<uint32_t>(bit_cast<uintptr_t>(site) >> 3));
     }
   } else {
-    // Any other pretenuring storage than the global one is used as a cache,
-    // where the count is later on merge in the allocation site.
+    // Entering cached feedback is used in the parallel case. We are not allowed
+    // to dereference the allocation site and rather have to postpone all checks
+    // till actually merging the data.

[Hannes Payer (out of office), 2016/01/18 11:46:33]

We could add a similar DCHECK as above:
low-level allocation site map check || or is forwarding pointer && high level
map check of forwarded object.
WDYT?

[Michael Lippautz, 2016/01/19 14:56:52]

I would like to avoid touching the address here. The complete check is done
during merging in Heap::MergeAllocationSitePretenuringFeedback.

+    Address key = memento_candidate->GetAllocationSiteUnchecked();
     HashMap::Entry* e = pretenuring_feedback->LookupOrInsert(
-        key, static_cast<uint32_t>(bit_cast<uintptr_t>(key)));
+        key, static_cast<uint32_t>(bit_cast<uintptr_t>(key) >> 3));
     DCHECK(e != nullptr);
     (*bit_cast<intptr_t*>(&e->value))++;
   }
 }
 
 
 void Heap::RemoveAllocationSitePretenuringFeedback(AllocationSite* site) {
   global_pretenuring_feedback_->Remove(
       site, static_cast<uint32_t>(bit_cast<uintptr_t>(site)));
 }
@@ skipping 175 matching lines @@
 
 void VerifySmisVisitor::VisitPointers(Object** start, Object** end) {
   for (Object** current = start; current < end; current++) {
     CHECK((*current)->IsSmi());
   }
 }
 }  // namespace internal
 }  // namespace v8
 
 #endif  // V8_HEAP_HEAP_INL_H_