VM: Inline Scavenger and PageSpace objects into Heap.

runtime/vm/heap.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/heap.h"
 
 #include "platform/assert.h"
 #include "platform/utils.h"
 #include "vm/flags.h"
 #include "vm/isolate.h"
@@ skipping 30 matching lines @@
 DEFINE_FLAG(bool, verify_before_gc, false,
             "Enables heap verification before GC.");
 DEFINE_FLAG(bool, pretenure_all, false, "Global pretenuring (for testing).");
 
 
 Heap::Heap(Isolate* isolate,
            intptr_t max_new_gen_semi_words,
            intptr_t max_old_gen_words,
            intptr_t max_external_words)
     : isolate_(isolate),
+      new_space_(this, max_new_gen_semi_words, kNewObjectAlignmentOffset),
+      old_space_(this, max_old_gen_words, max_external_words),
       read_only_(false),
       gc_in_progress_(false),
       pretenure_policy_(0) {
   for (int sel = 0;
        sel < kNumWeakSelectors;
        sel++) {
     new_weak_tables_[sel] = new WeakTable();
     old_weak_tables_[sel] = new WeakTable();
   }
-  new_space_ = new Scavenger(this,
-                             max_new_gen_semi_words,
-                             kNewObjectAlignmentOffset);
-  old_space_ = new PageSpace(this, max_old_gen_words, max_external_words);
   stats_.num_ = 0;
 }
 
 
 Heap::~Heap() {
-  delete new_space_;
-  delete old_space_;
   for (int sel = 0;
        sel < kNumWeakSelectors;
        sel++) {
     delete new_weak_tables_[sel];
     delete old_weak_tables_[sel];
   }
 }
 
 
 uword Heap::AllocateNew(intptr_t size) {
   ASSERT(isolate()->no_safepoint_scope_depth() == 0);
-  uword addr = new_space_->TryAllocate(size);
+  uword addr = new_space_.TryAllocate(size);
   if (addr == 0) {
     CollectGarbage(kNew);
-    addr = new_space_->TryAllocate(size);
+    addr = new_space_.TryAllocate(size);
     if (addr == 0) {
       return AllocateOld(size, HeapPage::kData);
     }
   }
   return addr;
 }
 
 
 uword Heap::AllocateOld(intptr_t size, HeapPage::PageType type) {
   ASSERT(isolate()->no_safepoint_scope_depth() == 0);
-  uword addr = old_space_->TryAllocate(size, type);
+  uword addr = old_space_.TryAllocate(size, type);
   if (addr != 0) {
     return addr;
   }
   // If we are in the process of running a sweep wait for the sweeper to free
   // memory.
   {
-    MonitorLocker ml(old_space_->tasks_lock());
-    addr = old_space_->TryAllocate(size, type);
-    while ((addr == 0) && (old_space_->tasks() > 0)) {
+    MonitorLocker ml(old_space_.tasks_lock());
+    addr = old_space_.TryAllocate(size, type);
+    while ((addr == 0) && (old_space_.tasks() > 0)) {
       ml.Wait();
-      addr = old_space_->TryAllocate(size, type);
+      addr = old_space_.TryAllocate(size, type);
     }
   }
   if (addr != 0) {
     return addr;
   }
   // All GC tasks finished without allocating successfully. Run a full GC.
   CollectAllGarbage();
-  addr = old_space_->TryAllocate(size, type);
+  addr = old_space_.TryAllocate(size, type);
   if (addr != 0) {
     return addr;
   }
   // Wait for all of the concurrent tasks to finish before giving up.
   {
-    MonitorLocker ml(old_space_->tasks_lock());
-    addr = old_space_->TryAllocate(size, type);
-    while ((addr == 0) && (old_space_->tasks() > 0)) {
+    MonitorLocker ml(old_space_.tasks_lock());
+    addr = old_space_.TryAllocate(size, type);
+    while ((addr == 0) && (old_space_.tasks() > 0)) {
       ml.Wait();
-      addr = old_space_->TryAllocate(size, type);
+      addr = old_space_.TryAllocate(size, type);
     }
   }
   if (addr != 0) {
     return addr;
   }
   // Force growth before attempting a synchronous GC.
-  addr = old_space_->TryAllocate(size, type, PageSpace::kForceGrowth);
+  addr = old_space_.TryAllocate(size, type, PageSpace::kForceGrowth);
   if (addr != 0) {
     return addr;
   }
   // Before throwing an out-of-memory error try a synchronous GC.
   CollectAllGarbage();
   {
-    MonitorLocker ml(old_space_->tasks_lock());
-    while (old_space_->tasks() > 0) {
+    MonitorLocker ml(old_space_.tasks_lock());
+    while (old_space_.tasks() > 0) {
       ml.Wait();
     }
   }
-  addr = old_space_->TryAllocate(size, type, PageSpace::kForceGrowth);
+  addr = old_space_.TryAllocate(size, type, PageSpace::kForceGrowth);
   if (addr != 0) {
     return addr;
   }
   // Give up allocating this object.
   OS::PrintErr(
       "Exhausted heap space, trying to allocate %" Pd " bytes.\n", size);
   return 0;
 }
 
 
 uword Heap::AllocatePretenured(intptr_t size) {
   ASSERT(isolate()->no_safepoint_scope_depth() == 0);
-  uword addr = old_space_->TryAllocateDataBump(size, PageSpace::kControlGrowth);
+  uword addr = old_space_.TryAllocateDataBump(size, PageSpace::kControlGrowth);
   if (addr != 0) return addr;
   return AllocateOld(size, HeapPage::kData);
 }
 
 
 void Heap::AllocateExternal(intptr_t size, Space space) {
   ASSERT(isolate()->no_safepoint_scope_depth() == 0);
   if (space == kNew) {
-    new_space_->AllocateExternal(size);
-    if (new_space_->ExternalInWords() > (FLAG_new_gen_ext_limit * MBInWords)) {
+    new_space_.AllocateExternal(size);
+    if (new_space_.ExternalInWords() > (FLAG_new_gen_ext_limit * MBInWords)) {
       // Attempt to free some external allocation by a scavenge. (If the total
       // remains above the limit, next external alloc will trigger another.)
       CollectGarbage(kNew);
     }
   } else {
     ASSERT(space == kOld);
-    old_space_->AllocateExternal(size);
-    if (old_space_->NeedsGarbageCollection()) {
+    old_space_.AllocateExternal(size);
+    if (old_space_.NeedsGarbageCollection()) {
       CollectAllGarbage();
     }
   }
 }
 
 void Heap::FreeExternal(intptr_t size, Space space) {
   if (space == kNew) {
-    new_space_->FreeExternal(size);
+    new_space_.FreeExternal(size);
   } else {
     ASSERT(space == kOld);
-    old_space_->FreeExternal(size);
+    old_space_.FreeExternal(size);
   }
 }
 
 void Heap::PromoteExternal(intptr_t size) {
-  new_space_->FreeExternal(size);
-  old_space_->AllocateExternal(size);
+  new_space_.FreeExternal(size);
+  old_space_.AllocateExternal(size);
 }
 
 bool Heap::Contains(uword addr) const {
-  return new_space_->Contains(addr) ||
-      old_space_->Contains(addr);
+  return new_space_.Contains(addr) ||
+      old_space_.Contains(addr);
 }
 
 
 bool Heap::NewContains(uword addr) const {
-  return new_space_->Contains(addr);
+  return new_space_.Contains(addr);
 }
 
 
 bool Heap::OldContains(uword addr) const {
-  return old_space_->Contains(addr);
+  return old_space_.Contains(addr);
 }
 
 
 bool Heap::CodeContains(uword addr) const {
-  return old_space_->Contains(addr, HeapPage::kExecutable);
+  return old_space_.Contains(addr, HeapPage::kExecutable);
 }
 
 
 void Heap::VisitObjects(ObjectVisitor* visitor) const {
-  new_space_->VisitObjects(visitor);
-  old_space_->VisitObjects(visitor);
+  new_space_.VisitObjects(visitor);
+  old_space_.VisitObjects(visitor);
 }
 
 
 HeapIterationScope::HeapIterationScope()
     : StackResource(Thread::Current()->isolate()),
       old_space_(isolate()->heap()->old_space()) {
   // It's not yet safe to iterate over a paged space while it's concurrently
   // sweeping, so wait for any such task to complete first.
   MonitorLocker ml(old_space_->tasks_lock());
 #if defined(DEBUG)
@@ skipping 16 matching lines @@
 #endif
   ASSERT(old_space_->tasks() == 1);
   old_space_->set_tasks(0);
   ml.Notify();
 }
 
 
 void Heap::IterateObjects(ObjectVisitor* visitor) const {
   // The visitor must not allocate from the heap.
   NoSafepointScope no_safepoint_scope_;
-  new_space_->VisitObjects(visitor);
+  new_space_.VisitObjects(visitor);
   IterateOldObjects(visitor);
 }
 
 
 void Heap::IterateOldObjects(ObjectVisitor* visitor) const {
   HeapIterationScope heap_iteration_scope;
-  old_space_->VisitObjects(visitor);
+  old_space_.VisitObjects(visitor);
 }
 
 
 void Heap::VisitObjectPointers(ObjectPointerVisitor* visitor) const {
-  new_space_->VisitObjectPointers(visitor);
-  old_space_->VisitObjectPointers(visitor);
+  new_space_.VisitObjectPointers(visitor);
+  old_space_.VisitObjectPointers(visitor);
 }
 
 
 RawInstructions* Heap::FindObjectInCodeSpace(FindObjectVisitor* visitor) const {
   // Only executable pages can have RawInstructions objects.
-  RawObject* raw_obj = old_space_->FindObject(visitor, HeapPage::kExecutable);
+  RawObject* raw_obj = old_space_.FindObject(visitor, HeapPage::kExecutable);
   ASSERT((raw_obj == Object::null()) ||
          (raw_obj->GetClassId() == kInstructionsCid));
   return reinterpret_cast<RawInstructions*>(raw_obj);
 }
 
 
 RawObject* Heap::FindOldObject(FindObjectVisitor* visitor) const {
   HeapIterationScope heap_iteration_scope;
-  return old_space_->FindObject(visitor, HeapPage::kData);
+  return old_space_.FindObject(visitor, HeapPage::kData);
 }
 
 
 RawObject* Heap::FindNewObject(FindObjectVisitor* visitor) const {
-  return new_space_->FindObject(visitor);
+  return new_space_.FindObject(visitor);
 }
 
 
 RawObject* Heap::FindObject(FindObjectVisitor* visitor) const {
   // The visitor must not allocate from the heap.
   NoSafepointScope no_safepoint_scope;
   RawObject* raw_obj = FindNewObject(visitor);
   if (raw_obj != Object::null()) {
     return raw_obj;
   }
@@ skipping 12 matching lines @@
   TIMERSCOPE(isolate(), time_gc);
   bool invoke_api_callbacks = (api_callbacks == kInvokeApiCallbacks);
   switch (space) {
     case kNew: {
       VMTagScope tagScope(isolate(), VMTag::kGCNewSpaceTagId);
       TimelineDurationScope tds(isolate(),
                                 isolate()->GetGCStream(),
                                 "CollectNewGeneration");
       RecordBeforeGC(kNew, reason);
       UpdateClassHeapStatsBeforeGC(kNew);
-      new_space_->Scavenge(invoke_api_callbacks);
+      new_space_.Scavenge(invoke_api_callbacks);
       isolate()->class_table()->UpdatePromoted();
       UpdatePretenurePolicy();
       RecordAfterGC();
       PrintStats();
-      if (old_space_->NeedsGarbageCollection()) {
+      if (old_space_.NeedsGarbageCollection()) {
         // Old collections should call the API callbacks.
         CollectGarbage(kOld, kInvokeApiCallbacks, kPromotion);
       }
       break;
     }
     case kOld:
     case kCode: {
       VMTagScope tagScope(isolate(), VMTag::kGCOldSpaceTagId);
       TimelineDurationScope tds(isolate(),
                                 isolate()->GetGCStream(),
                                 "CollectOldGeneration");
       RecordBeforeGC(kOld, reason);
       UpdateClassHeapStatsBeforeGC(kOld);
-      old_space_->MarkSweep(invoke_api_callbacks);
+      old_space_.MarkSweep(invoke_api_callbacks);
       RecordAfterGC();
       PrintStats();
       break;
     }
     default:
       UNREACHABLE();
   }
 }
 
 
@@ skipping 19 matching lines @@
 
 void Heap::CollectAllGarbage() {
   TIMERSCOPE(isolate(), time_gc);
   {
     VMTagScope tagScope(isolate(), VMTag::kGCNewSpaceTagId);
     TimelineDurationScope tds(isolate(),
                               isolate()->GetGCStream(),
                               "CollectNewGeneration");
     RecordBeforeGC(kNew, kFull);
     UpdateClassHeapStatsBeforeGC(kNew);
-    new_space_->Scavenge(kInvokeApiCallbacks);
+    new_space_.Scavenge(kInvokeApiCallbacks);
     isolate()->class_table()->UpdatePromoted();
     UpdatePretenurePolicy();
     RecordAfterGC();
     PrintStats();
   }
   {
     VMTagScope tagScope(isolate(), VMTag::kGCOldSpaceTagId);
     TimelineDurationScope tds(isolate(),
                               isolate()->GetGCStream(),
                               "CollectOldGeneration");
     RecordBeforeGC(kOld, kFull);
     UpdateClassHeapStatsBeforeGC(kOld);
-    old_space_->MarkSweep(kInvokeApiCallbacks);
+    old_space_.MarkSweep(kInvokeApiCallbacks);
     RecordAfterGC();
     PrintStats();
   }
 }
 
 
 bool Heap::ShouldPretenure(intptr_t class_id) const {
   if (class_id == kOneByteStringCid) {
     return pretenure_policy_ > 0;
   } else {
@@ skipping 21 matching lines @@
       (100 * stats->promoted_count) / allocated;
   if (promo_percent >= FLAG_pretenure_threshold) {
     pretenure_policy_ += FLAG_pretenure_interval;
   } else {
     pretenure_policy_ = Utils::Maximum(0, pretenure_policy_ - 1);
   }
 }
 
 
 void Heap::SetGrowthControlState(bool state) {
-  old_space_->SetGrowthControlState(state);
+  old_space_.SetGrowthControlState(state);
 }
 
 
 bool Heap::GrowthControlState() {
-  return old_space_->GrowthControlState();
+  return old_space_.GrowthControlState();
 }
 
 
 void Heap::WriteProtect(bool read_only) {
   read_only_ = read_only;
-  new_space_->WriteProtect(read_only);
-  old_space_->WriteProtect(read_only);
+  new_space_.WriteProtect(read_only);
+  old_space_.WriteProtect(read_only);
 }
 
 
 uword Heap::TopAddress(Heap::Space space) {
   if (space == kNew) {
-    return reinterpret_cast<uword>(new_space_->TopAddress());
+    return reinterpret_cast<uword>(new_space_.TopAddress());
   } else {
     ASSERT(space == kPretenured);
-    return reinterpret_cast<uword>(old_space_->TopAddress());
+    return reinterpret_cast<uword>(old_space_.TopAddress());
   }
 }
 
 
 uword Heap::EndAddress(Heap::Space space) {
   if (space == kNew) {
-    return reinterpret_cast<uword>(new_space_->EndAddress());
+    return reinterpret_cast<uword>(new_space_.EndAddress());
   } else {
     ASSERT(space == kPretenured);
-    return reinterpret_cast<uword>(old_space_->EndAddress());
+    return reinterpret_cast<uword>(old_space_.EndAddress());
   }
 }
 
 
 Heap::Space Heap::SpaceForAllocation(intptr_t cid) const {
   return FLAG_pretenure_all ? kPretenured : kNew;
 }
 
 
 void Heap::Init(Isolate* isolate,
                 intptr_t max_new_gen_words,
                 intptr_t max_old_gen_words,
                 intptr_t max_external_words) {
   ASSERT(isolate->heap() == NULL);
   Heap* heap = new Heap(isolate,
                         max_new_gen_words,
                         max_old_gen_words,
                         max_external_words);
   isolate->set_heap(heap);
 }
 
 
 void Heap::GetMergedAddressRange(uword* start, uword* end) const {
-  if (new_space_->CapacityInWords() != 0) {
+  if (new_space_.CapacityInWords() != 0) {
     uword new_start;
     uword new_end;
-    new_space_->StartEndAddress(&new_start, &new_end);
+    new_space_.StartEndAddress(&new_start, &new_end);
     *start = Utils::Minimum(new_start, *start);
     *end = Utils::Maximum(new_end, *end);
   }
-  if (old_space_->CapacityInWords() != 0) {
+  if (old_space_.CapacityInWords() != 0) {
     uword old_start;
     uword old_end;
-    old_space_->StartEndAddress(&old_start, &old_end);
+    old_space_.StartEndAddress(&old_start, &old_end);
     *start = Utils::Minimum(old_start, *start);
     *end = Utils::Maximum(old_end, *end);
   }
   ASSERT(*start <= *end);
 }
 
 
 ObjectSet* Heap::CreateAllocatedObjectSet(
     MarkExpectation mark_expectation) const {
   uword start = static_cast<uword>(-1);
@@ skipping 38 matching lines @@
   OS::PrintErr("New space (%" Pd "k of %" Pd "k) "
                "Old space (%" Pd "k of %" Pd "k)\n",
                (UsedInWords(kNew) / KBInWords),
                (CapacityInWords(kNew) / KBInWords),
                (UsedInWords(kOld) / KBInWords),
                (CapacityInWords(kOld) / KBInWords));
 }
 
 
 intptr_t Heap::UsedInWords(Space space) const {
-  return space == kNew ? new_space_->UsedInWords() : old_space_->UsedInWords();
+  return space == kNew ? new_space_.UsedInWords() : old_space_.UsedInWords();
 }
 
 
 intptr_t Heap::CapacityInWords(Space space) const {
-  return space == kNew ? new_space_->CapacityInWords() :
-                         old_space_->CapacityInWords();
+  return space == kNew ? new_space_.CapacityInWords() :
+                         old_space_.CapacityInWords();
 }
 
 intptr_t Heap::ExternalInWords(Space space) const {
-  return space == kNew ? new_space_->ExternalInWords() :
-                         old_space_->ExternalInWords();
+  return space == kNew ? new_space_.ExternalInWords() :
+                         old_space_.ExternalInWords();
 }
 
 int64_t Heap::GCTimeInMicros(Space space) const {
   if (space == kNew) {
-    return new_space_->gc_time_micros();
+    return new_space_.gc_time_micros();
   }
-  return old_space_->gc_time_micros();
+  return old_space_.gc_time_micros();
 }
 
 
 intptr_t Heap::Collections(Space space) const {
   if (space == kNew) {
-    return new_space_->collections();
+    return new_space_.collections();
   }
-  return old_space_->collections();
+  return old_space_.collections();
 }
 
 
 const char* Heap::GCReasonToString(GCReason gc_reason) {
   switch (gc_reason) {
     case kNewSpace:
       return "new space";
     case kPromotion:
       return "promotion";
     case kOldSpace:
@@ skipping 36 matching lines @@
     new_weak_tables_[sel]->SetValue(raw_obj, val);
   } else {
     ASSERT(raw_obj->IsOldObject());
     old_weak_tables_[sel]->SetValue(raw_obj, val);
   }
 }
 
 
 void Heap::PrintToJSONObject(Space space, JSONObject* object) const {
   if (space == kNew) {
-    new_space_->PrintToJSONObject(object);
+    new_space_.PrintToJSONObject(object);
   } else {
-    old_space_->PrintToJSONObject(object);
+    old_space_.PrintToJSONObject(object);
   }
 }
 
 
 void Heap::RecordBeforeGC(Space space, GCReason reason) {
   ASSERT(!gc_in_progress_);
   gc_in_progress_ = true;
   stats_.num_++;
   stats_.space_ = space;
   stats_.reason_ = reason;
   stats_.before_.micros_ = OS::GetCurrentTimeMicros();
-  stats_.before_.new_ = new_space_->GetCurrentUsage();
-  stats_.before_.old_ = old_space_->GetCurrentUsage();
+  stats_.before_.new_ = new_space_.GetCurrentUsage();
+  stats_.before_.old_ = old_space_.GetCurrentUsage();
   stats_.times_[0] = 0;
   stats_.times_[1] = 0;
   stats_.times_[2] = 0;
   stats_.times_[3] = 0;
   stats_.data_[0] = 0;
   stats_.data_[1] = 0;
   stats_.data_[2] = 0;
   stats_.data_[3] = 0;
 }
 
 
 void Heap::RecordAfterGC() {
   stats_.after_.micros_ = OS::GetCurrentTimeMicros();
   int64_t delta = stats_.after_.micros_ - stats_.before_.micros_;
   if (stats_.space_ == kNew) {
-    new_space_->AddGCTime(delta);
-    new_space_->IncrementCollections();
+    new_space_.AddGCTime(delta);
+    new_space_.IncrementCollections();
   } else {
-    old_space_->AddGCTime(delta);
-    old_space_->IncrementCollections();
+    old_space_.AddGCTime(delta);
+    old_space_.IncrementCollections();
   }
-  stats_.after_.new_ = new_space_->GetCurrentUsage();
-  stats_.after_.old_ = old_space_->GetCurrentUsage();
+  stats_.after_.new_ = new_space_.GetCurrentUsage();
+  stats_.after_.old_ = old_space_.GetCurrentUsage();
   ASSERT(gc_in_progress_);
   gc_in_progress_ = false;
   if (Service::gc_stream.enabled()) {
     ServiceEvent event(Isolate::Current(), ServiceEvent::kGC);
     event.set_gc_stats(&stats_);
     Service::HandleEvent(&event);
   }
 }
 
 
@@ skipping 70 matching lines @@
     heap->DisableGrowthControl();
 }
 
 
 NoHeapGrowthControlScope::~NoHeapGrowthControlScope() {
     Heap* heap = reinterpret_cast<Isolate*>(isolate())->heap();
     heap->SetGrowthControlState(current_growth_controller_state_);
 }
 
 }  // namespace dart