[heap] Move to LAB-based allocation for newspace evacuation.

src/heap/mark-compact.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/heap/mark-compact.h"
 
 #include "src/base/atomicops.h"
 #include "src/base/bits.h"
 #include "src/base/sys-info.h"
 #include "src/code-stubs.h"
@@ skipping 1552 matching lines @@
     }
     return false;
   }
 
  protected:
   Heap* heap_;
   SlotsBuffer** evacuation_slots_buffer_;
 };
 
 
-class MarkCompactCollector::EvacuateNewSpaceVisitor
+class MarkCompactCollector::EvacuateNewSpaceVisitor final
     : public MarkCompactCollector::EvacuateVisitorBase {
  public:
+  static const intptr_t kLabSize = 2 * KB;
+  static const intptr_t kMaxLabObjectSize = 256;

[Hannes Payer (out of office), 2015/12/16 14:51:06]

How did you identify these parameters? Do you think 256 is small enough to be
the common case?

[Michael Lippautz, 2015/12/16 20:06:04]

These constant are not backed by real-world experiments, yet. Runs on our perf
bots show no regression using these values.

+
   explicit EvacuateNewSpaceVisitor(Heap* heap,
                                    SlotsBuffer** evacuation_slots_buffer)
-      : EvacuateVisitorBase(heap, evacuation_slots_buffer) {}
+      : EvacuateVisitorBase(heap, evacuation_slots_buffer),
+        buffer_(LocalAllocationBuffer::InvalidBuffer()),
+        space_to_allocate_(NEW_SPACE) {}
 
   bool Visit(HeapObject* object) override {
     Heap::UpdateAllocationSiteFeedback(object, Heap::RECORD_SCRATCHPAD_SLOT);
     int size = object->Size();
     HeapObject* target_object = nullptr;
     if (heap_->ShouldBePromoted(object->address(), size) &&
         TryEvacuateObject(heap_->old_space(), object, &target_object)) {
       // If we end up needing more special cases, we should factor this out.
       if (V8_UNLIKELY(target_object->IsJSArrayBuffer())) {
         heap_->array_buffer_tracker()->Promote(
             JSArrayBuffer::cast(target_object));
       }
       heap_->IncrementPromotedObjectsSize(size);
       return true;
     }
-
-    AllocationAlignment alignment = object->RequiredAlignment();
-    AllocationResult allocation =
-        heap_->new_space()->AllocateRaw(size, alignment);
-    if (allocation.IsRetry()) {
-      if (!heap_->new_space()->AddFreshPage()) {
-        // Shouldn't happen. We are sweeping linearly, and to-space
-        // has the same number of pages as from-space, so there is
-        // always room unless we are in an OOM situation.
-        FatalProcessOutOfMemory("MarkCompactCollector: semi-space copy\n");
-      }
-      allocation = heap_->new_space()->AllocateRaw(size, alignment);
-      DCHECK(!allocation.IsRetry());
-    }
-    Object* target = allocation.ToObjectChecked();
-
+    HeapObject* target = nullptr;
+    AllocationSpace space = AllocateTargetObject(object, &target);
     heap_->mark_compact_collector()->MigrateObject(
-        HeapObject::cast(target), object, size, NEW_SPACE, nullptr);
+        HeapObject::cast(target), object, size, space,
+        (space == NEW_SPACE) ? nullptr : evacuation_slots_buffer_);
     if (V8_UNLIKELY(target->IsJSArrayBuffer())) {
       heap_->array_buffer_tracker()->MarkLive(JSArrayBuffer::cast(target));
     }
     heap_->IncrementSemiSpaceCopiedObjectSize(size);
     return true;
   }
+
+ private:
+  inline AllocationSpace AllocateTargetObject(HeapObject* old_object,
+                                              HeapObject** target_object) {
+    const int size = old_object->Size();
+    AllocationAlignment alignment = old_object->RequiredAlignment();
+    AllocationResult allocation;
+    if (space_to_allocate_ == NEW_SPACE) {
+      if (size > kMaxLabObjectSize) {
+        allocation = AllocateInNewSpace(size, alignment);
+      } else {
+        allocation = AllocateInLab(size, alignment);
+      }
+    }
+    if (space_to_allocate_ == OLD_SPACE) {
+      allocation = AllocateInOldSpace(size, alignment);
+    }
+    bool ok = allocation.To(target_object);
+    DCHECK(ok);
+    USE(ok);
+    return space_to_allocate_;
+  }
+
+  inline bool NewLocalAllocationBuffer() {
+    AllocationResult result = AllocateInNewSpace(kLabSize, kWordAligned);
+    buffer_ = LocalAllocationBuffer::FromResult(heap_, result, kLabSize);
+    return buffer_.IsValid();
+  }
+
+  inline AllocationResult AllocateInNewSpace(int size_in_bytes,
+                                             AllocationAlignment alignment) {
+    AllocationResult allocation =
+        heap_->new_space()->AllocateRawSynchronized(size_in_bytes, alignment);
+    if (allocation.IsRetry()) {
+      if (!heap_->new_space()->AddFreshPageSynchronized()) {
+        space_to_allocate_ = OLD_SPACE;
+      } else {
+        allocation = heap_->new_space()->AllocateRawSynchronized(size_in_bytes,
+                                                                 alignment);
+        if (allocation.IsRetry()) {
+          space_to_allocate_ = OLD_SPACE;
+        }
+      }
+    }
+    return allocation;
+  }
+
+  inline AllocationResult AllocateInOldSpace(int size_in_bytes,

[Hannes Payer (out of office), 2015/12/16 14:51:06]

Are you going to provide LABs in old space in a separate CL?

[Michael Lippautz, 2015/12/16 20:06:04]

LABs in old space are not needed as we have compaction spaces. 

A future CL will pass them as parameters to the new space visitor, which will
use them for allocating in old space. This way, the same memory is used for
evacuating new space pages and old space pages. 

+                                             AllocationAlignment alignment) {
+    AllocationResult allocation =
+        heap_->old_space()->AllocateRaw(size_in_bytes, alignment);
+    if (allocation.IsRetry()) {
+      FatalProcessOutOfMemory(
+          "MarkCompactCollector: semi-space copy, fallback in old gen\n");
+    }
+    return allocation;
+  }
+
+  inline AllocationResult AllocateInLab(int size_in_bytes,
+                                        AllocationAlignment alignment) {
+    AllocationResult allocation;
+    if (!buffer_.IsValid()) {
+      if (!NewLocalAllocationBuffer()) {
+        space_to_allocate_ = OLD_SPACE;
+        return AllocationResult::Retry(OLD_SPACE);
+      }
+    }
+    allocation = buffer_.AllocateRawAligned(size_in_bytes, alignment);
+    if (allocation.IsRetry()) {
+      if (!NewLocalAllocationBuffer()) {

[Hannes Payer (out of office), 2015/12/16 14:51:06]

It would be nice to preserve the same behavior we have right now in the single
threaded case, i.e. no fragmentation. When allocating a new LAB, you could merge
the leftover from the old LAB with the new LAB if the are adjacent.

[Michael Lippautz, 2015/12/16 20:06:04]

Done, however there's still a slight mismatch in behavior which we have to live
with I think. Objects larger than kMaxLabObjectSize will get allocated between
LABs, even in single-threaded cases, making the LABs impossible to merge.

+        space_to_allocate_ = OLD_SPACE;
+        return AllocationResult::Retry(OLD_SPACE);
+      } else {
+        allocation = buffer_.AllocateRawAligned(size_in_bytes, alignment);
+        if (allocation.IsRetry()) {
+          space_to_allocate_ = OLD_SPACE;
+          return AllocationResult::Retry(OLD_SPACE);
+        }
+      }
+    }
+    return allocation;
+  }
+
+  LocalAllocationBuffer buffer_;
+  AllocationSpace space_to_allocate_;
 };
 
 
-class MarkCompactCollector::EvacuateOldSpaceVisitor
+class MarkCompactCollector::EvacuateOldSpaceVisitor final
     : public MarkCompactCollector::EvacuateVisitorBase {
  public:
   EvacuateOldSpaceVisitor(Heap* heap,
                           CompactionSpaceCollection* compaction_spaces,
                           SlotsBuffer** evacuation_slots_buffer)
       : EvacuateVisitorBase(heap, evacuation_slots_buffer),
         compaction_spaces_(compaction_spaces) {}
 
   bool Visit(HeapObject* object) override {
     CompactionSpace* target_space = compaction_spaces_->Get(
@@ skipping 2327 matching lines @@
     MarkBit mark_bit = Marking::MarkBitFrom(host);
     if (Marking::IsBlack(mark_bit)) {
       RelocInfo rinfo(isolate(), pc, RelocInfo::CODE_TARGET, 0, host);
       RecordRelocSlot(&rinfo, target);
     }
   }
 }
 
 }  // namespace internal
 }  // namespace v8