Makes new space iterable by filling up the mutator's TLAB

runtime/vm/heap.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/heap.h"
 
 #include "platform/assert.h"
 #include "platform/utils.h"
 #include "vm/flags.h"
 #include "vm/isolate.h"
@@ skipping 39 matching lines @@
 Heap::~Heap() {
   delete barrier_;
   delete barrier_done_;
 
   for (int sel = 0; sel < kNumWeakSelectors; sel++) {
     delete new_weak_tables_[sel];
     delete old_weak_tables_[sel];
   }
 }
 
+void Heap::FillRemainingTLAB(Thread* thread) {
+  uword start = thread->top();
+  uword end = thread->end();
+  ASSERT(end >= start);
+  intptr_t size = end - start;
+  if (end == new_space_.end()) {

[rmacnak, 2017/08/03 20:31:40]

Do we need this check? Won't this only happen if the remaining size is already
0?

[danunez, 2017/08/03 20:39:21]

As discussed, the TLAB may not be full. This check only checks if this TLAB
happens end right when the new space ends.

Also as discussed, This case will be removed in favor of treating each TLAB
equally. Therefore, any TLAB will be filled up through this function.

+    size = 0;
+  }
+  ASSERT(Utils::IsAligned(size, kObjectAlignment));
+  if (size >= kObjectAlignment) {
+    FreeListElement::AsElement(start, size);
+    ASSERT(RawObject::FromAddr(start)->Size() == size);
+    ASSERT((start + size) == new_space_.top());
+  }
+}
+
+void Heap::AbandonRemainingTLAB(Thread* thread) {
+  FillRemainingTLAB(thread);
+  thread->set_top(0);
+  thread->set_end(0);
+}
+
+intptr_t Heap::CalculateTLABSize() {
+  intptr_t size = new_space_.end() - new_space_.top();
+  return Utils::RoundDown(size, kObjectAlignment);
+}
+
 uword Heap::AllocateNew(intptr_t size) {
   ASSERT(Thread::Current()->no_safepoint_scope_depth() == 0);
   // Currently, only the Dart thread may allocate in new space.
   isolate()->AssertCurrentThreadIsMutator();
   Thread* thread = Thread::Current();
   uword addr = new_space_.TryAllocateInTLAB(thread, size);
-  if (addr == 0) {
-    // This call to CollectGarbage might end up "reusing" a collection spawned
-    // from a different thread and will be racing to allocate the requested
-    // memory with other threads being released after the collection.
-    CollectGarbage(kNew);
-    addr = new_space_.TryAllocateInTLAB(thread, size);
-    if (addr == 0) {
-      return AllocateOld(size, HeapPage::kData);
+  if (addr != 0) {
+    return addr;
+  }
+
+  intptr_t tlab_size = CalculateTLABSize();
+  if ((tlab_size > 0) && (size > tlab_size)) {
+    return AllocateOld(size, HeapPage::kData);
+  }
+
+  AbandonRemainingTLAB(thread);
+  if (tlab_size > 0) {
+    uword tlab_top = new_space_.TryAllocateNewTLAB(thread, tlab_size);
+    if (tlab_top != 0) {
+      addr = new_space_.TryAllocateInTLAB(thread, size);
+      ASSERT(addr != 0);
+      return addr;
     }
   }
-  return addr;
+
+  ASSERT(!thread->HasActiveTLAB());
+
+  // This call to CollectGarbage might end up "reusing" a collection spawned
+  // from a different thread and will be racing to allocate the requested
+  // memory with other threads being released after the collection.
+  CollectGarbage(kNew);
+  tlab_size = CalculateTLABSize();
+  uword tlab_top = new_space_.TryAllocateNewTLAB(thread, tlab_size);
+  if (tlab_top != 0) {
+    addr = new_space_.TryAllocateInTLAB(thread, size);
+    // It is possible a GC doesn't clear enough space.
+    // In that case, we must fall through and allocate into old space.
+    if (addr != 0) {
+      return addr;
+    }
+  }
+  return AllocateOld(size, HeapPage::kData);
 }
 
 uword Heap::AllocateOld(intptr_t size, HeapPage::PageType type) {
   ASSERT(Thread::Current()->no_safepoint_scope_depth() == 0);
   uword addr = old_space_.TryAllocate(size, type);
   if (addr != 0) {
     return addr;
   }
   // If we are in the process of running a sweep, wait for the sweeper to free
   // memory.
@@ skipping 461 matching lines @@
 
 bool Heap::Verify(MarkExpectation mark_expectation) const {
   HeapIterationScope heap_iteration_scope;
   return VerifyGC(mark_expectation);
 }
 
 bool Heap::VerifyGC(MarkExpectation mark_expectation) const {
   StackZone stack_zone(Thread::Current());
 
   // Change the new space's top_ with the more up-to-date thread's view of top_
-  new_space_.FlushTLS();
+  uword saved_top = new_space_.FlushTLS();
 
   ObjectSet* allocated_set =
       CreateAllocatedObjectSet(stack_zone.GetZone(), mark_expectation);
   VerifyPointersVisitor visitor(isolate(), allocated_set);
   VisitObjectPointers(&visitor);
 
+  new_space_.UnflushTLS(saved_top);
   // Only returning a value so that Heap::Validate can be called from an ASSERT.
   return true;
 }
 
 void Heap::PrintSizes() const {
   OS::PrintErr(
       "New space (%" Pd64 "k of %" Pd64
       "k) "
       "Old space (%" Pd64 "k of %" Pd64 "k)\n",
       (UsedInWords(kNew) / KBInWords), (CapacityInWords(kNew) / KBInWords),
@@ skipping 254 matching lines @@
     : StackResource(thread) {
   Dart::vm_isolate()->heap()->WriteProtect(false);
 }
 
 WritableVMIsolateScope::~WritableVMIsolateScope() {
   ASSERT(Dart::vm_isolate()->heap()->UsedInWords(Heap::kNew) == 0);
   Dart::vm_isolate()->heap()->WriteProtect(true);
 }
 
 }  // namespace dart