Moves the top_ and end_ words of the Scavenger into mutator thread.

runtime/vm/scavenger.cc

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/scavenger.h"
 
 #include "vm/dart.h"
 #include "vm/dart_api_state.h"
 #include "vm/isolate.h"
 #include "vm/lockers.h"
@@ skipping 120 matching lines @@
       // Get the new location of the object.
       new_addr = ForwardedAddr(header);
     } else {
       intptr_t size = raw_obj->Size();
       NOT_IN_PRODUCT(intptr_t cid = raw_obj->GetClassId());
       NOT_IN_PRODUCT(ClassTable* class_table = isolate()->class_table());
       // Check whether object should be promoted.
       if (scavenger_->survivor_end_ <= raw_addr) {
         // Not a survivor of a previous scavenge. Just copy the object into the
         // to space.
         new_addr = scavenger_->TryAllocate(size);

[rmacnak, 2017/07/05 17:39:52]

Can we use a separate allocation function here? During a scavenge, we should be
able to use Scavenger's top/end instead of Thread's. We should also be able to
assert copying allocations during a scavenge always succeed.

[danunez, 2017/07/05 18:12:55]

I will add a function to do allocation using the thread's top and end. This
function will become the TLAB allocation function. TryAllocate(intptr_t) will be
reverted to the version in master.

         NOT_IN_PRODUCT(class_table->UpdateLiveNew(cid, size));
       } else {
         // TODO(iposva): Experiment with less aggressive promotion. For example
         // a coin toss determines if an object is promoted or whether it should
         // survive in this generation.
         //
         // This object is a survivor of a previous scavenge. Attempt to promote
         // the object.
         new_addr =
             page_space_->TryAllocatePromoLocked(size, PageSpace::kForceGrowth);
         if (new_addr != 0) {
           // If promotion succeeded then we need to remember it so that it can
           // be traversed later.
           scavenger_->PushToPromotedStack(new_addr);
           bytes_promoted_ += size;
           NOT_IN_PRODUCT(class_table->UpdateAllocatedOld(cid, size));
         } else {
           // Promotion did not succeed. Copy into the to space instead.
           new_addr = scavenger_->TryAllocate(size);

[rmacnak, 2017/07/05 17:39:52]

And here.

[danunez, 2017/07/05 18:12:55]

Ditto.

           NOT_IN_PRODUCT(class_table->UpdateLiveNew(cid, size));
         }
       }
       // During a scavenge we always succeed to at least copy all of the
       // current objects to the to space.
       ASSERT(new_addr != 0);
       // Copy the object to the new location.
       memmove(reinterpret_cast<void*>(new_addr),
               reinterpret_cast<void*>(raw_addr), size);
       // Remember forwarding address.
@@ skipping 160 matching lines @@
 Scavenger::Scavenger(Heap* heap,
                      intptr_t max_semi_capacity_in_words,
                      uword object_alignment)
     : heap_(heap),
       max_semi_capacity_in_words_(max_semi_capacity_in_words),
       object_alignment_(object_alignment),
       scavenging_(false),
       delayed_weak_properties_(NULL),
       gc_time_micros_(0),
       collections_(0),
-      external_size_(0) {
+      external_size_(0),
+      space_lock_(new Mutex()) {
   // Verify assumptions about the first word in objects which the scavenger is
   // going to use for forwarding pointers.
   ASSERT(Object::tags_offset() == 0);
 
   // Set initial size resulting in a total of three different levels.
   const intptr_t initial_semi_capacity_in_words =
       max_semi_capacity_in_words /
       (FLAG_new_gen_growth_factor * FLAG_new_gen_growth_factor);
 
   const intptr_t kVmNameSize = 128;
@@ skipping 11 matching lines @@
   survivor_end_ = FirstObjectStart();
 
   UpdateMaxHeapCapacity();
   UpdateMaxHeapUsage();
 }
 
 
 Scavenger::~Scavenger() {
   ASSERT(!scavenging_);
   to_->Delete();
+  delete space_lock_;
 }
 
 
 intptr_t Scavenger::NewSizeInWords(intptr_t old_size_in_words) const {
   if (stats_history_.Size() == 0) {
     return old_size_in_words;
   }
   double garbage = stats_history_.Get(0).GarbageFraction();
   if (garbage < (FLAG_new_gen_garbage_threshold / 100.0)) {
     return Utils::Minimum(max_semi_capacity_in_words_,
                           old_size_in_words * FLAG_new_gen_growth_factor);
   } else {
     return old_size_in_words;
   }
 }
 
 
 SemiSpace* Scavenger::Prologue(Isolate* isolate, bool invoke_api_callbacks) {
   if (invoke_api_callbacks && (isolate->gc_prologue_callback() != NULL)) {
     (isolate->gc_prologue_callback())();
   }
   isolate->PrepareForGC();
+
+  Thread* thread = Thread::Current();

[rmacnak, 2017/07/05 17:39:53]

How about

Thread* mutator = isolate->mutator_thread()

?

[danunez, 2017/07/05 18:12:55]

Done.

+  if (!thread->IsMutatorThread()) {
+    thread = isolate->mutator_thread();
+  }
+
   // Flip the two semi-spaces so that to_ is always the space for allocating
   // objects.
   SemiSpace* from = to_;
 
   const intptr_t kVmNameSize = 128;
   char vm_name[kVmNameSize];
   Heap::RegionName(heap_, Heap::kNew, vm_name, kVmNameSize);
   to_ = SemiSpace::New(NewSizeInWords(from->size_in_words()), vm_name);
   if (to_ == NULL) {
     // TODO(koda): We could try to recover (collect old space, wait for another
     // isolate to finish scavenge, etc.).
     OUT_OF_MEMORY();
   }
   UpdateMaxHeapCapacity();
   top_ = FirstObjectStart();
   resolved_top_ = top_;
   end_ = to_->end();
+
+  if (thread->heap() != NULL) {
+    thread->set_top_offset(top_);
+    thread->set_end_offset(end_);
+  }
+
   return from;
 }
 
 
 void Scavenger::Epilogue(Isolate* isolate,
                          SemiSpace* from,
                          bool invoke_api_callbacks) {
   // All objects in the to space have been copied from the from space at this
   // moment.
+
+  Thread* thread = Thread::Current();
+
+  if (!thread->IsMutatorThread()) {
+    thread = isolate->mutator_thread();
+  }
+  uword top = 0;
+  uword end = 0;
+
+
+  if (thread->heap() != NULL) {
+    top = thread->top();
+    ASSERT(thread->end() == end_);
+    end = thread->end();
+  } else {
+    top = top_;
+    end = end_;
+  }
+
   double avg_frac = stats_history_.Get(0).PromoCandidatesSuccessFraction();
   if (stats_history_.Size() >= 2) {
     // Previous scavenge is only given half as much weight.
     avg_frac += 0.5 * stats_history_.Get(1).PromoCandidatesSuccessFraction();
     avg_frac /= 1.0 + 0.5;  // Normalize.
   }
   if (avg_frac < (FLAG_early_tenuring_threshold / 100.0)) {
     // Remember the limit to which objects have been copied.
-    survivor_end_ = top_;
+    survivor_end_ = top;
   } else {
     // Move survivor end to the end of the to_ space, making all surviving
     // objects candidates for promotion next time.
-    survivor_end_ = end_;
+    survivor_end_ = end;
   }
 #if defined(DEBUG)
   // We can only safely verify the store buffers from old space if there is no
   // concurrent old space task. At the same time we prevent new tasks from
   // being spawned.
   {
     PageSpace* page_space = heap_->old_space();
     MonitorLocker ml(page_space->tasks_lock());
     if (page_space->tasks() == 0) {
       VerifyStoreBufferPointerVisitor verify_store_buffer_visitor(isolate, to_);
@@ skipping 103 matching lines @@
 }
 
 
 void Scavenger::IterateWeakRoots(Isolate* isolate, HandleVisitor* visitor) {
   isolate->VisitWeakPersistentHandles(visitor);
 }
 
 
 void Scavenger::ProcessToSpace(ScavengerVisitor* visitor) {
   // Iterate until all work has been drained.
-  while ((resolved_top_ < top_) || PromotedStackHasMore()) {
-    while (resolved_top_ < top_) {
+
+  uword top = 0;
+  Thread* thread = Thread::Current();
+
+  if (!thread->IsMutatorThread()) {
+    thread = Isolate::Current()->mutator_thread();
+  }
+
+  if (thread->heap() != NULL) {
+    top = thread->top();
+  } else {
+    top = top_;
+  }
+
+  while ((resolved_top_ < top) || PromotedStackHasMore()) {
+    while (resolved_top_ < top) {
       RawObject* raw_obj = RawObject::FromAddr(resolved_top_);
       intptr_t class_id = raw_obj->GetClassId();
       if (class_id != kWeakPropertyCid) {
         resolved_top_ += raw_obj->VisitPointersNonvirtual(visitor);
       } else {
         RawWeakProperty* raw_weak = reinterpret_cast<RawWeakProperty*>(raw_obj);
         resolved_top_ += ProcessWeakProperty(raw_weak, visitor);
       }
+
+      if (thread->heap() != NULL) {
+        top = thread->top();
+      } else {
+        top = top_;
+      }
     }
     {
       // Visit all the promoted objects and update/scavenge their internal
       // pointers. Potentially this adds more objects to the to space.
       while (PromotedStackHasMore()) {
         RawObject* raw_object = RawObject::FromAddr(PopFromPromotedStack());
         // Resolve or copy all objects referred to by the current object. This
         // can potentially push more objects on this stack as well as add more
         // objects to be resolved in the to space.
         ASSERT(!raw_object->IsRemembered());
         visitor->VisitingOldObject(raw_object);
         raw_object->VisitPointersNonvirtual(visitor);
+
+        if (thread->heap() != NULL) {
+          top = thread->top();
+        } else {
+          top = top_;
+        }
       }
       visitor->VisitingOldObject(NULL);
     }
     {
       // Finished this round of scavenging. Process the pending weak properties
       // for which the keys have become reachable. Potentially this adds more
       // objects to the to space.
       RawWeakProperty* cur_weak = delayed_weak_properties_;
       delayed_weak_properties_ = NULL;
       while (cur_weak != NULL) {
@@ skipping 10 matching lines @@
         uword raw_addr = RawObject::ToAddr(raw_key);
         ASSERT(visitor->from_->Contains(raw_addr));
         uword header = *reinterpret_cast<uword*>(raw_addr);
         // Reset the next pointer in the weak property.
         cur_weak->ptr()->next_ = 0;
         if (IsForwarding(header)) {
           cur_weak->VisitPointersNonvirtual(visitor);
         } else {
           EnqueueWeakProperty(cur_weak);
         }
+
+        if (thread->heap() != NULL) {
+          top = thread->top();
+        } else {
+          top = top_;
+        }
+
         // Advance to next weak property in the queue.
         cur_weak = reinterpret_cast<RawWeakProperty*>(next_weak);
       }
     }
   }
 }
 
 
 void Scavenger::UpdateMaxHeapCapacity() {
   if (heap_ == NULL) {
@@ skipping 106 matching lines @@
 
       // Advance to next weak property in the queue.
       cur_weak = reinterpret_cast<RawWeakProperty*>(next_weak);
     }
   }
 }
 
 
 void Scavenger::VisitObjectPointers(ObjectPointerVisitor* visitor) const {
   uword cur = FirstObjectStart();
-  while (cur < top_) {
+  Thread* thread = Thread::Current();
+
+  if (!thread->IsMutatorThread()) {

[rmacnak, 2017/07/05 17:39:53]

Ditto

[danunez, 2017/07/05 18:12:55]

Done.

+    thread = Isolate::Current()->mutator_thread();
+  }
+
+  uword top = CapacityInWords() == 0 ? 0 : thread->top();
+
+  if (thread->heap() == NULL) {
+    top = top_;
+  }
+
+  while (cur < top) {
     RawObject* raw_obj = RawObject::FromAddr(cur);
     cur += raw_obj->VisitPointers(visitor);
   }
 }
 
 
 void Scavenger::VisitObjects(ObjectVisitor* visitor) const {
   uword cur = FirstObjectStart();
-  while (cur < top_) {
+  Thread* thread = Thread::Current();
+
+  if (!thread->IsMutatorThread()) {

[rmacnak, 2017/07/05 17:39:53]

Ditto

[danunez, 2017/07/05 18:12:55]

Done.

+    thread = Isolate::Current()->mutator_thread();
+  }
+
+  uword top = CapacityInWords() == 0 ? 0 : thread->top();
+
+  if (thread->heap() == NULL) {
+    top = top_;
+  }
+
+  while (cur < top) {
     RawObject* raw_obj = RawObject::FromAddr(cur);
     visitor->VisitObject(raw_obj);
     cur += raw_obj->Size();
   }
 }
 
 
 void Scavenger::AddRegionsToObjectSet(ObjectSet* set) const {
   set->AddRegion(to_->start(), to_->end());
 }
 
 
 RawObject* Scavenger::FindObject(FindObjectVisitor* visitor) const {
   ASSERT(!scavenging_);
+  Thread* thread = Thread::Current();
+
+  if (!thread->IsMutatorThread()) {

[rmacnak, 2017/07/05 17:39:52]

Ditto

[danunez, 2017/07/05 18:12:55]

Done.

+    thread = Isolate::Current()->mutator_thread();
+  }
+
+  uword top = CapacityInWords() == 0 ? 0 : thread->top();
   uword cur = FirstObjectStart();
-  if (visitor->VisitRange(cur, top_)) {
-    while (cur < top_) {
+
+  if (thread->heap() == NULL) {
+    top = top_;
+  }
+
+  if (visitor->VisitRange(cur, top)) {
+    while (cur < top) {
       RawObject* raw_obj = RawObject::FromAddr(cur);
       uword next = cur + raw_obj->Size();
       if (visitor->VisitRange(cur, next) && raw_obj->FindObject(visitor)) {
         return raw_obj;  // Found object, return it.
       }
       cur = next;
     }
-    ASSERT(cur == top_);
   }
   return Object::null();
 }
 
 
 void Scavenger::Scavenge() {
   // TODO(cshapiro): Add a decision procedure for determining when the
   // the API callbacks should be invoked.
   Scavenge(false);
 }
@@ skipping 119 matching lines @@
 }
 
 
 void Scavenger::FreeExternal(intptr_t size) {
   ASSERT(size >= 0);
   external_size_ -= size;
   ASSERT(external_size_ >= 0);
 }
 
 }  // namespace dart