Makes new space iterable by filling up the mutator's TLAB

runtime/vm/heap.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/heap.h"
 
 #include "platform/assert.h"
 #include "platform/utils.h"
 #include "vm/flags.h"
 #include "vm/isolate.h"
@@ skipping 39 matching lines @@
 Heap::~Heap() {
   delete barrier_;
   delete barrier_done_;
 
   for (int sel = 0; sel < kNumWeakSelectors; sel++) {
     delete new_weak_tables_[sel];
     delete old_weak_tables_[sel];
   }
 }
 
+void Heap::FillRemainingTLAB(Thread* thread) {
+  ASSERT(Thread::Current()->no_safepoint_scope_depth() == 0);

[rmacnak, 2017/08/04 17:31:59]

Fails from e.g., Heap::IterateObjects.

I would leave this out until multiple threads have TLABs.

+  uword start = thread->top();
+  uword end = thread->end();
+  ASSERT(end >= start);
+  intptr_t size = end - start;
+  ASSERT(Utils::IsAligned(size, kObjectAlignment));
+  if (size >= kObjectAlignment) {
+    FreeListElement::AsElement(start, size);
+    ASSERT(RawObject::FromAddr(start)->Size() == size);
+    ASSERT((start + size) == new_space_.top());
+  }
+}
+
+void Heap::AbandonRemainingTLAB(Thread* thread) {
+  FillRemainingTLAB(thread);
+  thread->set_top(0);
+  thread->set_end(0);
+}
+
+intptr_t Heap::CalculateTLABSize() {
+  intptr_t size = new_space_.end() - new_space_.top();
+  return Utils::RoundDown(size, kObjectAlignment);
+}
+
 uword Heap::AllocateNew(intptr_t size) {
   ASSERT(Thread::Current()->no_safepoint_scope_depth() == 0);
   // Currently, only the Dart thread may allocate in new space.
   isolate()->AssertCurrentThreadIsMutator();
   Thread* thread = Thread::Current();
   uword addr = new_space_.TryAllocateInTLAB(thread, size);
-  if (addr == 0) {
-    // This call to CollectGarbage might end up "reusing" a collection spawned
-    // from a different thread and will be racing to allocate the requested
-    // memory with other threads being released after the collection.
-    CollectGarbage(kNew);
-    addr = new_space_.TryAllocateInTLAB(thread, size);
-    if (addr == 0) {
-      return AllocateOld(size, HeapPage::kData);
+  if (addr != 0) {
+    return addr;
+  }
+
+  intptr_t tlab_size = CalculateTLABSize();
+  if ((tlab_size > 0) && (size > tlab_size)) {
+    return AllocateOld(size, HeapPage::kData);
+  }
+
+  AbandonRemainingTLAB(thread);
+  if (tlab_size > 0) {
+    uword tlab_top = new_space_.TryAllocateNewTLAB(thread, tlab_size);
+    if (tlab_top != 0) {
+      addr = new_space_.TryAllocateInTLAB(thread, size);
+      ASSERT(addr != 0);
+      return addr;
     }
   }
-  return addr;
+
+  ASSERT(!thread->HasActiveTLAB());
+
+  // This call to CollectGarbage might end up "reusing" a collection spawned
+  // from a different thread and will be racing to allocate the requested
+  // memory with other threads being released after the collection.
+  CollectGarbage(kNew);
+  tlab_size = CalculateTLABSize();
+  uword tlab_top = new_space_.TryAllocateNewTLAB(thread, tlab_size);
+  if (tlab_top != 0) {
+    addr = new_space_.TryAllocateInTLAB(thread, size);
+    // It is possible a GC doesn't clear enough space.
+    // In that case, we must fall through and allocate into old space.
+    if (addr != 0) {
+      return addr;
+    }
+  }
+  return AllocateOld(size, HeapPage::kData);
 }
 
 uword Heap::AllocateOld(intptr_t size, HeapPage::PageType type) {
   ASSERT(Thread::Current()->no_safepoint_scope_depth() == 0);
   uword addr = old_space_.TryAllocate(size, type);
   if (addr != 0) {
     return addr;
   }
   // If we are in the process of running a sweep, wait for the sweeper to free
   // memory.
@@ skipping 461 matching lines @@
 
 bool Heap::Verify(MarkExpectation mark_expectation) const {
   HeapIterationScope heap_iteration_scope;
   return VerifyGC(mark_expectation);
 }
 
 bool Heap::VerifyGC(MarkExpectation mark_expectation) const {
   StackZone stack_zone(Thread::Current());
 
   // Change the new space's top_ with the more up-to-date thread's view of top_
-  new_space_.FlushTLS();
+  uword saved_top = new_space_.FlushTLS();
 
   ObjectSet* allocated_set =
       CreateAllocatedObjectSet(stack_zone.GetZone(), mark_expectation);
   VerifyPointersVisitor visitor(isolate(), allocated_set);
   VisitObjectPointers(&visitor);
 
+  new_space_.UnflushTLS(saved_top);
   // Only returning a value so that Heap::Validate can be called from an ASSERT.
   return true;
 }
 
 void Heap::PrintSizes() const {
   OS::PrintErr(
       "New space (%" Pd64 "k of %" Pd64
       "k) "
       "Old space (%" Pd64 "k of %" Pd64 "k)\n",
       (UsedInWords(kNew) / KBInWords), (CapacityInWords(kNew) / KBInWords),
@@ skipping 254 matching lines @@
     : StackResource(thread) {
   Dart::vm_isolate()->heap()->WriteProtect(false);
 }
 
 WritableVMIsolateScope::~WritableVMIsolateScope() {
   ASSERT(Dart::vm_isolate()->heap()->UsedInWords(Heap::kNew) == 0);
   Dart::vm_isolate()->heap()->WriteProtect(true);
 }
 
 }  // namespace dart