1. Add code to grow the heap instead of collecting garbage in Heap::AllocateOld when allocation is …

runtime/vm/heap.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/heap.h"
 
 #include "platform/assert.h"
 #include "platform/utils.h"
 #include "vm/flags.h"
 #include "vm/isolate.h"
@@ skipping 33 matching lines @@
 
 
 Heap::Heap(Isolate* isolate,
            intptr_t max_new_gen_semi_words,
            intptr_t max_old_gen_words,
            intptr_t max_external_words)
     : isolate_(isolate),
       new_space_(this, max_new_gen_semi_words, kNewObjectAlignmentOffset),
       old_space_(this, max_old_gen_words, max_external_words),
       read_only_(false),
-      gc_in_progress_(false),
+      gc_new_space_in_progress_(false),
+      gc_old_space_in_progress_(false),
       pretenure_policy_(0) {
   for (int sel = 0;
        sel < kNumWeakSelectors;
        sel++) {
     new_weak_tables_[sel] = new WeakTable();
     old_weak_tables_[sel] = new WeakTable();
   }
   stats_.num_ = 0;
 }
 
 
 Heap::~Heap() {
   for (int sel = 0;
        sel < kNumWeakSelectors;
        sel++) {
     delete new_weak_tables_[sel];
     delete old_weak_tables_[sel];
   }
 }
 
 
 uword Heap::AllocateNew(intptr_t size) {
   ASSERT(Thread::Current()->no_safepoint_scope_depth() == 0);
   // Currently, only the Dart thread may allocate in new space.
   isolate()->AssertCurrentThreadIsMutator();
   uword addr = new_space_.TryAllocate(size);
   if (addr == 0) {
+    // This call to CollectGarbage might end up "reusing" a collection spawned
+    // from a different thread and will be racing to allocate the requested
+    // memory with other threads being released after the collection.
     CollectGarbage(kNew);
     addr = new_space_.TryAllocate(size);
     if (addr == 0) {
       return AllocateOld(size, HeapPage::kData);
     }
   }
   return addr;
 }
 
 
 uword Heap::AllocateOld(intptr_t size, HeapPage::PageType type) {
   ASSERT(Thread::Current()->no_safepoint_scope_depth() == 0);
   uword addr = old_space_.TryAllocate(size, type);
   if (addr != 0) {
     return addr;
   }
   // If we are in the process of running a sweep wait for the sweeper to free
   // memory.
   {
     MonitorLocker ml(old_space_.tasks_lock());
     addr = old_space_.TryAllocate(size, type);
     while ((addr == 0) && (old_space_.tasks() > 0)) {
       ml.Wait();
       addr = old_space_.TryAllocate(size, type);
     }
   }
   if (addr != 0) {
     return addr;
   }
-  // All GC tasks finished without allocating successfully. Run a full GC.
-  CollectAllGarbage();
-  addr = old_space_.TryAllocate(size, type);
-  if (addr != 0) {
-    return addr;
-  }
-  // Wait for all of the concurrent tasks to finish before giving up.
-  {
-    MonitorLocker ml(old_space_.tasks_lock());
+  Thread* thread = Thread::Current();
+  if (thread->CanCollectGarbage()) {
+    // All GC tasks finished without allocating successfully. Run a full GC.
+    CollectAllGarbage();
     addr = old_space_.TryAllocate(size, type);
-    while ((addr == 0) && (old_space_.tasks() > 0)) {
-      ml.Wait();
+    if (addr != 0) {
+      return addr;
+    }
+    // Wait for all of the concurrent tasks to finish before giving up.
+    {
+      MonitorLocker ml(old_space_.tasks_lock());
       addr = old_space_.TryAllocate(size, type);
+      while ((addr == 0) && (old_space_.tasks() > 0)) {
+        ml.Wait();
+        addr = old_space_.TryAllocate(size, type);
+      }
     }
-  }
-  if (addr != 0) {
-    return addr;
-  }
-  // Force growth before attempting a synchronous GC.
-  addr = old_space_.TryAllocate(size, type, PageSpace::kForceGrowth);
-  if (addr != 0) {
-    return addr;
-  }
-  // Before throwing an out-of-memory error try a synchronous GC.
-  CollectAllGarbage();
-  {
-    MonitorLocker ml(old_space_.tasks_lock());
-    while (old_space_.tasks() > 0) {
-      ml.Wait();
+    if (addr != 0) {
+      return addr;
+    }
+    // Force growth before attempting another synchronous GC.
+    addr = old_space_.TryAllocate(size, type, PageSpace::kForceGrowth);
+    if (addr != 0) {
+      return addr;
+    }
+    // Before throwing an out-of-memory error try a synchronous GC.
+    CollectAllGarbage();
+    {
+      MonitorLocker ml(old_space_.tasks_lock());
+      while (old_space_.tasks() > 0) {
+        ml.Wait();
+      }
     }
   }
   addr = old_space_.TryAllocate(size, type, PageSpace::kForceGrowth);
   if (addr != 0) {
     return addr;
   }
   // Give up allocating this object.
   OS::PrintErr(
       "Exhausted heap space, trying to allocate %" Pd " bytes.\n", size);
   return 0;
@@ skipping 150 matching lines @@
   }
   raw_obj = FindOldObject(visitor);
   if (raw_obj != Object::null()) {
     return raw_obj;
   }
   raw_obj = FindObjectInCodeSpace(visitor);
   return raw_obj;
 }
 
 
-bool Heap::gc_in_progress() {
-  MutexLocker ml(&gc_in_progress_mutex_);
-  return gc_in_progress_;
+bool Heap::BeginNewSpaceGC() {
+  MonitorLocker ml(&gc_in_progress_monitor_);
+  bool start_gc_on_thread = true;
+  while (gc_new_space_in_progress_ ||
+         gc_old_space_in_progress_) {
+    start_gc_on_thread = !gc_new_space_in_progress_;
+    ml.Wait();
+  }
+  if (start_gc_on_thread) {
+    gc_new_space_in_progress_ = true;
+    return true;
+  }
+  return false;
 }
 
 
-void Heap::BeginGC() {
-  MutexLocker ml(&gc_in_progress_mutex_);
-  ASSERT(!gc_in_progress_);
-  gc_in_progress_ = true;
+void Heap::EndNewSpaceGC() {
+  MonitorLocker ml(&gc_in_progress_monitor_);
+  ASSERT(gc_new_space_in_progress_);
+  gc_new_space_in_progress_ = false;
+  ml.NotifyAll();
 }
 
 
-void Heap::EndGC() {
-  MutexLocker ml(&gc_in_progress_mutex_);
-  ASSERT(gc_in_progress_);
-  gc_in_progress_ = false;
+bool Heap::BeginOldSpaceGC() {
+  MonitorLocker ml(&gc_in_progress_monitor_);
+  bool start_gc_on_thread = true;
+  while (gc_new_space_in_progress_ ||
+         gc_old_space_in_progress_) {
+    start_gc_on_thread = !gc_old_space_in_progress_;
+    ml.Wait();
+  }
+  if (start_gc_on_thread) {
+    gc_old_space_in_progress_ = true;
+    return true;
+  }
+  return false;
 }
 
 
-void Heap::CollectGarbage(Space space,
-                          ApiCallbacks api_callbacks,
-                          GCReason reason) {
-  Thread* thread = Thread::Current();
-  bool invoke_api_callbacks = (api_callbacks == kInvokeApiCallbacks);
-  switch (space) {
-    case kNew: {
-      RecordBeforeGC(kNew, reason);
-      VMTagScope tagScope(thread, VMTag::kGCNewSpaceTagId);
-      TimelineDurationScope tds(thread,
-                                isolate()->GetGCStream(),
-                                "CollectNewGeneration");
-      UpdateClassHeapStatsBeforeGC(kNew);
-      new_space_.Scavenge(invoke_api_callbacks);
-      isolate()->class_table()->UpdatePromoted();
-      UpdatePretenurePolicy();
-      RecordAfterGC();
-      PrintStats();
-      if (old_space_.NeedsGarbageCollection()) {
-        // Old collections should call the API callbacks.
-        CollectGarbage(kOld, kInvokeApiCallbacks, kPromotion);
-      }
-      break;
-    }
-    case kOld:
-    case kCode: {
-      RecordBeforeGC(kOld, reason);
-      VMTagScope tagScope(thread, VMTag::kGCOldSpaceTagId);
-      TimelineDurationScope tds(thread,
-                                isolate()->GetGCStream(),
-                                "CollectOldGeneration");
-      UpdateClassHeapStatsBeforeGC(kOld);
-      old_space_.MarkSweep(invoke_api_callbacks);
-      RecordAfterGC();
-      PrintStats();
-      break;
-    }
-    default:
-      UNREACHABLE();
-  }
+void Heap::EndOldSpaceGC() {
+  MonitorLocker ml(&gc_in_progress_monitor_);
+  ASSERT(gc_old_space_in_progress_);
+  gc_old_space_in_progress_ = false;
+  ml.NotifyAll();
 }
 
 
 void Heap::UpdateClassHeapStatsBeforeGC(Heap::Space space) {
   ClassTable* class_table = isolate()->class_table();
   if (space == kNew) {
     class_table->ResetCountersNew();
   } else {
     class_table->ResetCountersOld();
   }
 }
 
 
+void Heap::CollectNewSpaceGarbage(Thread* thread,
+                                  ApiCallbacks api_callbacks,
+                                  GCReason reason) {
+  if (BeginNewSpaceGC()) {
+    bool invoke_api_callbacks = (api_callbacks == kInvokeApiCallbacks);
+    RecordBeforeGC(kNew, reason);
+    VMTagScope tagScope(thread, VMTag::kGCNewSpaceTagId);
+    TimelineDurationScope tds(thread,
+                              isolate()->GetGCStream(),
+                              "CollectNewGeneration");
+    UpdateClassHeapStatsBeforeGC(kNew);
+    new_space_.Scavenge(invoke_api_callbacks);
+    isolate()->class_table()->UpdatePromoted();
+    UpdatePretenurePolicy();
+    RecordAfterGC(kNew);
+    PrintStats();
+    EndNewSpaceGC();
+    if (old_space_.NeedsGarbageCollection()) {
+      // Old collections should call the API callbacks.
+      CollectOldSpaceGarbage(thread, kInvokeApiCallbacks, kPromotion);
+    }
+  }
+}
+
+
+void Heap::CollectOldSpaceGarbage(Thread* thread,
+                                  ApiCallbacks api_callbacks,
+                                  GCReason reason) {
+  if (BeginOldSpaceGC()) {
+    bool invoke_api_callbacks = (api_callbacks == kInvokeApiCallbacks);
+    RecordBeforeGC(kOld, reason);
+    VMTagScope tagScope(thread, VMTag::kGCOldSpaceTagId);
+    TimelineDurationScope tds(thread,
+                              isolate()->GetGCStream(),
+                              "CollectOldGeneration");
+    UpdateClassHeapStatsBeforeGC(kOld);
+    old_space_.MarkSweep(invoke_api_callbacks);
+    RecordAfterGC(kOld);
+    PrintStats();
+    EndOldSpaceGC();
+  }
+}
+
+
+void Heap::CollectGarbage(Space space,
+                          ApiCallbacks api_callbacks,
+                          GCReason reason) {
+  Thread* thread = Thread::Current();
+  switch (space) {
+    case kNew: {
+      CollectNewSpaceGarbage(thread, api_callbacks, reason);
+      break;
+    }
+    case kOld:
+    case kCode: {
+      CollectOldSpaceGarbage(thread, api_callbacks, reason);
+      break;
+    }
+    default:
+      UNREACHABLE();
+  }
+}
+
+
 void Heap::CollectGarbage(Space space) {
+  Thread* thread = Thread::Current();
   if (space == kOld) {
-    CollectGarbage(space, kInvokeApiCallbacks, kOldSpace);
+    CollectOldSpaceGarbage(thread, kInvokeApiCallbacks, kOldSpace);
   } else {
     ASSERT(space == kNew);
-    CollectGarbage(space, kInvokeApiCallbacks, kNewSpace);
+    CollectNewSpaceGarbage(thread, kInvokeApiCallbacks, kNewSpace);
   }
 }
 
 
 void Heap::CollectAllGarbage() {
   Thread* thread = Thread::Current();
-  {
-    RecordBeforeGC(kNew, kFull);
-    VMTagScope tagScope(thread, VMTag::kGCNewSpaceTagId);
-    TimelineDurationScope tds(thread,
-                              isolate()->GetGCStream(),
-                              "CollectNewGeneration");
-    UpdateClassHeapStatsBeforeGC(kNew);
-    new_space_.Scavenge(kInvokeApiCallbacks);
-    isolate()->class_table()->UpdatePromoted();
-    UpdatePretenurePolicy();
-    RecordAfterGC();
-    PrintStats();
-  }
-  {
-    RecordBeforeGC(kOld, kFull);
-    VMTagScope tagScope(thread, VMTag::kGCOldSpaceTagId);
-    TimelineDurationScope tds(thread,
-                              isolate()->GetGCStream(),
-                              "CollectOldGeneration");
-    UpdateClassHeapStatsBeforeGC(kOld);
-    old_space_.MarkSweep(kInvokeApiCallbacks);
-    RecordAfterGC();
-    PrintStats();
-  }
+  CollectNewSpaceGarbage(thread, kInvokeApiCallbacks, kFull);
+  CollectOldSpaceGarbage(thread, kInvokeApiCallbacks, kFull);
 }
 
 
 bool Heap::ShouldPretenure(intptr_t class_id) const {
   if (class_id == kOneByteStringCid) {
     return pretenure_policy_ > 0;
   } else {
     return false;
   }
 }
@@ skipping 258 matching lines @@
 void Heap::PrintToJSONObject(Space space, JSONObject* object) const {
   if (space == kNew) {
     new_space_.PrintToJSONObject(object);
   } else {
     old_space_.PrintToJSONObject(object);
   }
 }
 
 
 void Heap::RecordBeforeGC(Space space, GCReason reason) {
-  BeginGC();
+  ASSERT((space == kNew && gc_new_space_in_progress_) ||
+         (space == kOld && gc_old_space_in_progress_));
   stats_.num_++;
   stats_.space_ = space;
   stats_.reason_ = reason;
   stats_.before_.micros_ = OS::GetCurrentTimeMicros();
   stats_.before_.new_ = new_space_.GetCurrentUsage();
   stats_.before_.old_ = old_space_.GetCurrentUsage();
   stats_.times_[0] = 0;
   stats_.times_[1] = 0;
   stats_.times_[2] = 0;
   stats_.times_[3] = 0;
   stats_.data_[0] = 0;
   stats_.data_[1] = 0;
   stats_.data_[2] = 0;
   stats_.data_[3] = 0;
 }
 
 
-void Heap::RecordAfterGC() {
+void Heap::RecordAfterGC(Space space) {
   stats_.after_.micros_ = OS::GetCurrentTimeMicros();
   int64_t delta = stats_.after_.micros_ - stats_.before_.micros_;
   if (stats_.space_ == kNew) {
     new_space_.AddGCTime(delta);
     new_space_.IncrementCollections();
   } else {
     old_space_.AddGCTime(delta);
     old_space_.IncrementCollections();
   }
   stats_.after_.new_ = new_space_.GetCurrentUsage();
   stats_.after_.old_ = old_space_.GetCurrentUsage();
-  EndGC();
+  ASSERT((space == kNew && gc_new_space_in_progress_) ||
+         (space == kOld && gc_old_space_in_progress_));
   if (Service::gc_stream.enabled()) {
     ServiceEvent event(Isolate::Current(), ServiceEvent::kGC);
     event.set_gc_stats(&stats_);
     Service::HandleEvent(&event);
   }
 }
 
 
 void Heap::PrintStats() {
   if (!FLAG_verbose_gc) return;
@@ skipping 69 matching lines @@
   Dart::vm_isolate()->heap()->WriteProtect(false, include_code_pages_);
 }
 
 
 WritableVMIsolateScope::~WritableVMIsolateScope() {
   ASSERT(Dart::vm_isolate()->heap()->UsedInWords(Heap::kNew) == 0);
   Dart::vm_isolate()->heap()->WriteProtect(true, include_code_pages_);
 }
 
 }  // namespace dart