- Add and enable concurrent sweeper.

runtime/vm/pages.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/pages.h"
 
 #include "platform/assert.h"
 #include "vm/compiler_stats.h"
 #include "vm/gc_marker.h"
 #include "vm/gc_sweeper.h"
@@ skipping 15 matching lines @@
 DEFINE_FLAG(bool, print_free_list_after_gc, false,
             "Print free list statistics after a GC");
 DEFINE_FLAG(bool, collect_code, true,
             "Attempt to GC infrequently used code.");
 DEFINE_FLAG(int, code_collection_interval_in_us, 30000000,
             "Time between attempts to collect unused code.");
 DEFINE_FLAG(bool, log_code_drop, false,
             "Emit a log message when pointers to unused code are dropped.");
 DEFINE_FLAG(bool, always_drop_code, false,
             "Always try to drop code if the function's usage counter is >= 0");
+DEFINE_FLAG(bool, concurrent_sweep, true,
+            "Concurrent sweep for old generation.");
 
 HeapPage* HeapPage::Initialize(VirtualMemory* memory, PageType type) {
   ASSERT(memory->size() > VirtualMemory::PageSize());
   bool is_executable = (type == kExecutable);
   memory->Commit(is_executable);
 
   HeapPage* result = reinterpret_cast<HeapPage*>(memory->address());
   result->memory_ = memory;
   result->next_ = NULL;
   result->executable_ = is_executable;
@@ skipping 68 matching lines @@
     prot = VirtualMemory::kReadWrite;
   }
   bool status = memory_->Protect(prot);
   ASSERT(status);
 }
 
 
 PageSpace::PageSpace(Heap* heap, intptr_t max_capacity_in_words)
     : freelist_(),
       heap_(heap),
+      pages_lock_(new Mutex),

[koda, 2014/08/26 23:34:48]

Mutex -> Mutex()

[Ivan Posva, 2014/08/27 01:00:22]

Done.

       pages_(NULL),
       pages_tail_(NULL),
+      exec_pages_(NULL),
+      exec_pages_tail_(NULL),
       large_pages_(NULL),
       max_capacity_in_words_(max_capacity_in_words),
       tasks_lock_(new Monitor()),
       tasks_(0),
       page_space_controller_(heap,
                              FLAG_heap_growth_space_ratio,
                              FLAG_heap_growth_rate,
                              FLAG_heap_growth_time_ratio),
       gc_time_micros_(0),
       collections_(0) {
 }
 
 
 PageSpace::~PageSpace() {
   {
     MonitorLocker ml(tasks_lock());
     while (tasks() > 0) {
       ml.Wait();
     }
   }
   FreePages(pages_);
+  FreePages(exec_pages_);
   FreePages(large_pages_);
+  delete pages_lock_;
   delete tasks_lock_;
 }
 
 
 intptr_t PageSpace::LargePageSizeInWordsFor(intptr_t size) {
   intptr_t page_size = Utils::RoundUp(size + HeapPage::ObjectStartOffset(),
                                       VirtualMemory::PageSize());
   return page_size >> kWordSizeLog2;
 }
 
 
 HeapPage* PageSpace::AllocatePage(HeapPage::PageType type) {
   HeapPage* page = HeapPage::Allocate(kPageSizeInWords, type);
-  if (pages_ == NULL) {
-    pages_ = page;
+
+  bool is_exec = (type == HeapPage::kExecutable);
+
+  MutexLocker ml(pages_lock_);
+  if (!is_exec) {
+    if (pages_ == NULL) {
+      pages_ = page;
+    } else {
+      pages_tail_->set_next(page);
+    }
+    pages_tail_ = page;
   } else {
-    const bool is_protected = (pages_tail_->type() == HeapPage::kExecutable)
-        && FLAG_write_protect_code;
-    if (is_protected) {
-      pages_tail_->WriteProtect(false);
+    if (exec_pages_ == NULL) {
+      exec_pages_ = page;
+    } else {
+      const bool is_protected = is_exec && FLAG_write_protect_code;

[koda, 2014/08/26 23:34:49]

"is_protected" relates to exec_pages_tail_, which is always executable. So it
doesn't make sense to use "is_exec", which relates to the page you are
allocating, in this expression. Just remove "is_exec" here.

That said, I really think it's about time to create a PageList utility class to
avoid lots of duplication and risk for bugs like this one.

[Ivan Posva, 2014/08/27 01:00:22]

Done.

+      if (is_protected) {
+        exec_pages_tail_->WriteProtect(false);
+      }
+      exec_pages_tail_->set_next(page);
+      if (is_protected) {
+        exec_pages_tail_->WriteProtect(true);
+      }
     }
-    pages_tail_->set_next(page);
-    if (is_protected) {
-      pages_tail_->WriteProtect(true);
-    }
+    exec_pages_tail_ = page;
   }
-  pages_tail_ = page;
   usage_.capacity_in_words += kPageSizeInWords;
   page->set_object_end(page->memory_->end());
   return page;
 }
 
 
 HeapPage* PageSpace::AllocateLargePage(intptr_t size, HeapPage::PageType type) {
   intptr_t page_size_in_words = LargePageSizeInWordsFor(size);
   HeapPage* page = HeapPage::Allocate(page_size_in_words, type);
   page->set_next(large_pages_);
@@ skipping 18 matching lines @@
   if (new_page_size_in_words < old_page_size_in_words) {
     memory->Truncate(memory->start(), new_page_size_in_words << kWordSizeLog2);
     usage_.capacity_in_words -= old_page_size_in_words;
     usage_.capacity_in_words += new_page_size_in_words;
     page->set_object_end(page->object_start() + new_object_size_in_bytes);
   }
 }
 
 
 void PageSpace::FreePage(HeapPage* page, HeapPage* previous_page) {
-  usage_.capacity_in_words -= (page->memory_->size() >> kWordSizeLog2);
-  // Remove the page from the list.
-  if (previous_page != NULL) {
-    previous_page->set_next(page->next());
-  } else {
-    pages_ = page->next();
-  }
-  if (page == pages_tail_) {
-    pages_tail_ = previous_page;
+  bool is_exec = (page->type() == HeapPage::kExecutable);
+  {
+    MutexLocker ml(pages_lock_);
+    usage_.capacity_in_words -= (page->memory_->size() >> kWordSizeLog2);
+    if (!is_exec) {
+      // Remove the page from the list of data pages.
+      if (previous_page != NULL) {
+        previous_page->set_next(page->next());
+      } else {
+        pages_ = page->next();
+      }
+      if (page == pages_tail_) {
+        pages_tail_ = previous_page;
+      }
+    } else {
+      // Remove the page from the list of executable pages.
+      if (previous_page != NULL) {
+        previous_page->set_next(page->next());
+      } else {
+        exec_pages_ = page->next();
+      }
+      if (page == exec_pages_tail_) {
+        exec_pages_tail_ = previous_page;
+      }
+    }
   }
   // TODO(iposva): Consider adding to a pool of empty pages.
   page->Deallocate();
 }
 
 
 void PageSpace::FreeLargePage(HeapPage* page, HeapPage* previous_page) {
   usage_.capacity_in_words -= (page->memory_->size() >> kWordSizeLog2);
   // Remove the page from the list.
   if (previous_page != NULL) {
@@ skipping 98 matching lines @@
 }
 
 
 void PageSpace::FreeExternal(intptr_t size) {
   intptr_t size_in_words = size >> kWordSizeLog2;
   usage_.external_in_words -= size_in_words;
 }
 
 
 bool PageSpace::Contains(uword addr) const {
+  MutexLocker ml(pages_lock_);
+  NoGCScope no_gc;
   HeapPage* page = pages_;
   while (page != NULL) {
     if (page->Contains(addr)) {
       return true;
     }
     page = NextPageAnySize(page);
   }
   return false;
 }
 
 
 bool PageSpace::Contains(uword addr, HeapPage::PageType type) const {
+  MutexLocker ml(pages_lock_);
+  NoGCScope no_gc;
   HeapPage* page = pages_;
   while (page != NULL) {
     if ((page->type() == type) && page->Contains(addr)) {
       return true;
     }
     page = NextPageAnySize(page);
   }
   return false;
 }
 
 
 void PageSpace::StartEndAddress(uword* start, uword* end) const {
-  ASSERT(pages_ != NULL || large_pages_ != NULL);
+  MutexLocker ml(pages_lock_);
+  NoGCScope no_gc;
+  ASSERT((pages_ != NULL) || (exec_pages_ != NULL) || (large_pages_ != NULL));
   *start = static_cast<uword>(~0);
   *end = 0;
   for (HeapPage* page = pages_; page != NULL; page = NextPageAnySize(page)) {
     *start = Utils::Minimum(*start, page->object_start());
     *end = Utils::Maximum(*end, page->object_end());
   }
   ASSERT(*start != static_cast<uword>(~0));
   ASSERT(*end != 0);
 }
 
 
 void PageSpace::VisitObjects(ObjectVisitor* visitor) const {
+  MutexLocker ml(pages_lock_);
+  NoGCScope no_gc;
   HeapPage* page = pages_;
   while (page != NULL) {
     page->VisitObjects(visitor);
     page = NextPageAnySize(page);
   }
 }
 
 
 void PageSpace::VisitObjectPointers(ObjectPointerVisitor* visitor) const {
+  MutexLocker ml(pages_lock_);
+  NoGCScope no_gc;
   HeapPage* page = pages_;
   while (page != NULL) {
     page->VisitObjectPointers(visitor);
     page = NextPageAnySize(page);
   }
 }
 
 
 RawObject* PageSpace::FindObject(FindObjectVisitor* visitor,
                                  HeapPage::PageType type) const {
   ASSERT(Isolate::Current()->no_gc_scope_depth() != 0);
+  MutexLocker ml(pages_lock_);
+  NoGCScope no_gc;
   HeapPage* page = pages_;
   while (page != NULL) {
     if (page->type() == type) {
       RawObject* obj = page->FindObject(visitor);
       if (obj != Object::null()) {
         return obj;
       }
     }
     page = NextPageAnySize(page);
   }
   return Object::null();
 }
 
 
 void PageSpace::WriteProtect(bool read_only) {
+  MutexLocker ml(pages_lock_);
+  NoGCScope no_gc;
   HeapPage* page = pages_;
   while (page != NULL) {
     page->WriteProtect(read_only);
     page = NextPageAnySize(page);
   }
 }
 
 
 void PageSpace::PrintToJSONObject(JSONObject* object) {
   Isolate* isolate = Isolate::Current();
@@ skipping 44 matching lines @@
   heap_map.AddProperty("unit_size_bytes",
                        static_cast<intptr_t>(kObjectAlignment));
   heap_map.AddProperty("page_size_bytes", kPageSizeInWords * kWordSize);
   {
     JSONObject class_list(&heap_map, "class_list");
     isolate->class_table()->PrintToJSONObject(&class_list);
   }
   {
     // "pages" is an array [page0, page1, ..., pageN], each page of the form
     // {"object_start": "0x...", "objects": [size, class id, size, ...]}
+    MutexLocker ml(pages_lock_);
+    NoGCScope no_gc;
     JSONArray all_pages(&heap_map, "pages");
     for (HeapPage* page = pages_; page != NULL; page = page->next()) {
       JSONObject page_container(&all_pages);
       page_container.AddPropertyF("object_start",
                                   "0x%" Px "", page->object_start());
       JSONArray page_map(&page_container, "objects");
       HeapMapAsJSONVisitor printer(&page_map);
       page->VisitObjects(&printer);
     }
+    for (HeapPage* page = exec_pages_; page != NULL; page = page->next()) {
+      JSONObject page_container(&all_pages);
+      page_container.AddPropertyF("object_start",
+                                  "0x%" Px "", page->object_start());
+      JSONArray page_map(&page_container, "objects");
+      HeapMapAsJSONVisitor printer(&page_map);
+      page->VisitObjects(&printer);
+    }
   }
 }
 
 
 bool PageSpace::ShouldCollectCode() {
   // Try to collect code if enough time has passed since the last attempt.
   const int64_t start = OS::GetCurrentTimeMicros();
   const int64_t last_code_collection_in_us =
       page_space_controller_.last_code_collection_in_us();
 
   if ((start - last_code_collection_in_us) >
       FLAG_code_collection_interval_in_us) {
     if (FLAG_log_code_drop) {
       OS::Print("Trying to detach code.\n");
     }
     page_space_controller_.set_last_code_collection_in_us(start);
     return true;
   }
   return false;
 }
 
 
 void PageSpace::WriteProtectCode(bool read_only) {
   if (FLAG_write_protect_code) {
-    HeapPage* current_page = pages_;
-    while (current_page != NULL) {
-      if (current_page->type() == HeapPage::kExecutable) {
-        current_page->WriteProtect(read_only);
+    MutexLocker ml(pages_lock_);
+    NoGCScope no_gc;
+    // No need to go through all of the data pages first.
+    HeapPage* page = exec_pages_;
+    while (page != NULL) {
+      ASSERT(page->type() == HeapPage::kExecutable);
+      page->WriteProtect(read_only);
+      page = page->next();
+    }
+    page = large_pages_;
+    while (page != NULL) {
+      if (page->type() == HeapPage::kExecutable) {
+        page->WriteProtect(read_only);
       }
-      current_page = NextPageAnySize(current_page);
+      page = page->next();
     }
   }
 }
 
 
 void PageSpace::MarkSweep(bool invoke_api_callbacks) {
   Isolate* isolate = heap_->isolate();
   ASSERT(isolate == Isolate::Current());
 
   // Wait for pending tasks to complete and then account for the driver task.
@@ skipping 37 matching lines @@
   int64_t mid1 = OS::GetCurrentTimeMicros();
 
   // Reset the bump allocation page to unused.
   // Reset the freelists and setup sweeping.
   freelist_[HeapPage::kData].Reset();
   freelist_[HeapPage::kExecutable].Reset();
 
   int64_t mid2 = OS::GetCurrentTimeMicros();
   int64_t mid3 = 0;
 
-  GCSweeper sweeper(heap_);
+  {
+    GCSweeper sweeper(heap_);
 
-  // During stop-the-world phases we should use bulk lock when adding elements
-  // to the free list.
-  {
+    // During stop-the-world phases we should use bulk lock when adding elements
+    // to the free list.
     MutexLocker mld(freelist_[HeapPage::kData].mutex());
     MutexLocker mle(freelist_[HeapPage::kExecutable].mutex());
 
+    // Large and executable pages are always swept immediately.
     HeapPage* prev_page = NULL;
-    HeapPage* page = pages_;
-    while (page != NULL) {
-      HeapPage* next_page = page->next();
-      bool page_in_use = sweeper.SweepPage(page, &freelist_[page->type()]);
-      if (page_in_use) {
-        prev_page = page;
-      } else {
-        FreePage(page, prev_page);
-      }
-      // Advance to the next page.
-      page = next_page;
-    }
-
-    mid3 = OS::GetCurrentTimeMicros();
-
-    prev_page = NULL;
-    page = large_pages_;
+    HeapPage* page = large_pages_;
     while (page != NULL) {
       HeapPage* next_page = page->next();
       const intptr_t words_to_end = sweeper.SweepLargePage(page);
       if (words_to_end == 0) {
         FreeLargePage(page, prev_page);
       } else {
         TruncateLargePage(page, words_to_end << kWordSizeLog2);
         prev_page = page;
       }
       // Advance to the next page.
       page = next_page;
     }
+
+    prev_page = NULL;
+    page = exec_pages_;
+    FreeList* freelist = &freelist_[HeapPage::kExecutable];
+    while (page != NULL) {
+      HeapPage* next_page = page->next();
+      bool page_in_use = sweeper.SweepPage(page, freelist);
+      if (page_in_use) {
+        prev_page = page;
+      } else {
+        FreePage(page, prev_page);
+      }
+      // Advance to the next page.
+      page = next_page;
+    }
+
+    mid3 = OS::GetCurrentTimeMicros();
+
+    if (!FLAG_concurrent_sweep) {
+      // Sweep all regular sized pages now.
+      prev_page = NULL;
+      page = pages_;
+      while (page != NULL) {
+        HeapPage* next_page = page->next();
+        bool page_in_use = sweeper.SweepPage(page, &freelist_[page->type()]);
+        if (page_in_use) {
+          prev_page = page;
+        } else {
+          FreePage(page, prev_page);
+        }
+        // Advance to the next page.
+        page = next_page;
+      }
+    } else {
+      // Start the concurrent sweeper task now.
+      GCSweeper::SweepConcurrent(
+          isolate, this, pages_, pages_tail_, &freelist_[HeapPage::kData]);
+    }
   }
 
   // Make code pages read-only.
   WriteProtectCode(true);
 
   int64_t end = OS::GetCurrentTimeMicros();
 
   // Record signals for growth control. Include size of external allocations.
   page_space_controller_.EvaluateGarbageCollection(usage_before, usage_,
                                                    start, end);
@@ skipping 12 matching lines @@
 
   if (FLAG_verify_after_gc) {
     OS::PrintErr("Verifying after MarkSweep...");
     heap_->Verify();
     OS::PrintErr(" done.\n");
   }
 
   // Done, reset the task count.
   {
     MonitorLocker ml(tasks_lock());
-    ASSERT(tasks() == 1);
     set_tasks(tasks() - 1);
     ml.Notify();
   }
 }
 
 
 PageSpaceController::PageSpaceController(Heap* heap,
                                          int heap_growth_ratio,
                                          int heap_growth_max,
                                          int garbage_collection_time_ratio)
@@ skipping 12 matching lines @@
 
 
 bool PageSpaceController::NeedsGarbageCollection(SpaceUsage after) const {
   if (!is_enabled_) {
     return false;
   }
   if (heap_growth_ratio_ == 100) {
     return false;
   }
   intptr_t capacity_increase_in_words =
-    after.capacity_in_words - last_usage_.capacity_in_words;
-  ASSERT(capacity_increase_in_words >= 0);
+      after.capacity_in_words - last_usage_.capacity_in_words;
+  // The concurrent sweeper might have freed more capacity than was allocated.
+  capacity_increase_in_words =
+      Utils::Maximum<intptr_t>(0, capacity_increase_in_words);
   capacity_increase_in_words =
       Utils::RoundUp(capacity_increase_in_words, PageSpace::kPageSizeInWords);
   intptr_t capacity_increase_in_pages =
       capacity_increase_in_words / PageSpace::kPageSizeInWords;
   double multiplier = 1.0;
   // To avoid waste, the first GC should be triggered before too long. After
   // kInitialTimeoutSeconds, gradually lower the capacity limit.
   static const double kInitialTimeoutSeconds = 1.00;
   if (history_.IsEmpty()) {
     double seconds_since_init = MicrosecondsToSeconds(
@@ skipping 73 matching lines @@
     return 0;
   } else {
     ASSERT(total_time >= gc_time);
     int result= static_cast<int>((static_cast<double>(gc_time) /
                              static_cast<double>(total_time)) * 100);
     return result;
   }
 }
 
 }  // namespace dart