Remove sequential sweeping mode and perform lazy sweeping when no sweeper threads are active.

src/mark-compact.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/v8.h"
 
 #include "src/base/atomicops.h"
 #include "src/code-stubs.h"
 #include "src/compilation-cache.h"
 #include "src/cpu-profiler.h"
@@ skipping 27 matching lines @@
 MarkCompactCollector::MarkCompactCollector(Heap* heap) :  // NOLINT
 #ifdef DEBUG
       state_(IDLE),
 #endif
       sweep_precisely_(false),
       reduce_memory_footprint_(false),
       abort_incremental_marking_(false),
       marking_parity_(ODD_MARKING_PARITY),
       compacting_(false),
       was_marked_incrementally_(false),
-      sweeping_pending_(false),
+      sweeping_in_progress_(false),
       pending_sweeper_jobs_semaphore_(0),
       sequential_sweeping_(false),
       tracer_(NULL),
       migration_slots_buffer_(NULL),
       heap_(heap),
       code_flusher_(NULL),
       have_code_to_deoptimize_(false) { }
 
 #ifdef VERIFY_HEAP
 class VerifyMarkingVisitor: public ObjectVisitor {
@@ skipping 507 matching lines @@
   Heap* heap_;
   PagedSpace* space_;
 
   DISALLOW_COPY_AND_ASSIGN(SweeperTask);
 };
 
 
 void MarkCompactCollector::StartSweeperThreads() {
   ASSERT(free_list_old_pointer_space_.get()->IsEmpty());
   ASSERT(free_list_old_data_space_.get()->IsEmpty());
-  sweeping_pending_ = true;
+  sweeping_in_progress_ = true;
   for (int i = 0; i < isolate()->num_sweeper_threads(); i++) {
     isolate()->sweeper_threads()[i]->StartSweeping();
   }
   if (FLAG_job_based_sweeping) {
     V8::GetCurrentPlatform()->CallOnBackgroundThread(
         new SweeperTask(heap(), heap()->old_data_space()),
         v8::Platform::kShortRunningTask);
     V8::GetCurrentPlatform()->CallOnBackgroundThread(
         new SweeperTask(heap(), heap()->old_pointer_space()),
         v8::Platform::kShortRunningTask);
   }
 }
 
 
-void MarkCompactCollector::WaitUntilSweepingCompleted() {
-  ASSERT(sweeping_pending_ == true);
+void MarkCompactCollector::EnsureSweepingCompleted() {
+  ASSERT(sweeping_in_progress_ == true);
+
+  // If sweeping is not completed, we try to complete it here. If we do not
+  // have sweeper threads we have to complete since we do not have a good
+  // indicator for a swept space in that case.
+  if (!AreSweeperThreadsActivated() || !IsSweepingCompleted()) {
+    SweepInParallel(heap()->paged_space(OLD_DATA_SPACE), 0);
+    SweepInParallel(heap()->paged_space(OLD_POINTER_SPACE), 0);
+  }
+
   for (int i = 0; i < isolate()->num_sweeper_threads(); i++) {
     isolate()->sweeper_threads()[i]->WaitForSweeperThread();
   }
   if (FLAG_job_based_sweeping) {
     // Wait twice for both jobs.
     pending_sweeper_jobs_semaphore_.Wait();
     pending_sweeper_jobs_semaphore_.Wait();
   }
   ParallelSweepSpacesComplete();
-  sweeping_pending_ = false;
+  sweeping_in_progress_ = false;
   RefillFreeList(heap()->paged_space(OLD_DATA_SPACE));
   RefillFreeList(heap()->paged_space(OLD_POINTER_SPACE));
   heap()->paged_space(OLD_DATA_SPACE)->ResetUnsweptFreeBytes();
   heap()->paged_space(OLD_POINTER_SPACE)->ResetUnsweptFreeBytes();
 }
 
 
 bool MarkCompactCollector::IsSweepingCompleted() {
   for (int i = 0; i < isolate()->num_sweeper_threads(); i++) {
     if (!isolate()->sweeper_threads()[i]->SweepingCompleted()) {
       return false;
     }
   }
+
   if (FLAG_job_based_sweeping) {
     if (!pending_sweeper_jobs_semaphore_.WaitFor(
             base::TimeDelta::FromSeconds(0))) {
       return false;
     }
     pending_sweeper_jobs_semaphore_.Signal();
   }
+
   return true;
 }
 
 
 void MarkCompactCollector::RefillFreeList(PagedSpace* space) {
   FreeList* free_list;
 
   if (space == heap()->old_pointer_space()) {
     free_list = free_list_old_pointer_space_.get();
   } else if (space == heap()->old_data_space()) {
     free_list = free_list_old_data_space_.get();
   } else {
     // Any PagedSpace might invoke RefillFreeLists, so we need to make sure
     // to only refill them for old data and pointer spaces.
     return;
   }
 
   intptr_t freed_bytes = space->free_list()->Concatenate(free_list);
   space->AddToAccountingStats(freed_bytes);
   space->DecrementUnsweptFreeBytes(freed_bytes);
 }
 
 
 bool MarkCompactCollector::AreSweeperThreadsActivated() {
   return isolate()->sweeper_threads() != NULL || FLAG_job_based_sweeping;
 }
 
 
-bool MarkCompactCollector::IsConcurrentSweepingInProgress(PagedSpace* space) {
-  return (space == NULL || space->is_swept_concurrently()) &&
-      sweeping_pending_;
-}
-
-
 void Marking::TransferMark(Address old_start, Address new_start) {
   // This is only used when resizing an object.
   ASSERT(MemoryChunk::FromAddress(old_start) ==
          MemoryChunk::FromAddress(new_start));
 
   if (!heap_->incremental_marking()->IsMarking()) return;
 
   // If the mark doesn't move, we don't check the color of the object.
   // It doesn't matter whether the object is black, since it hasn't changed
   // size, so the adjustment to the live data count will be zero anyway.
@@ skipping 285 matching lines @@
   // variable.
   tracer_ = tracer;
 
 #ifdef DEBUG
   ASSERT(state_ == IDLE);
   state_ = PREPARE_GC;
 #endif
 
   ASSERT(!FLAG_never_compact || !FLAG_always_compact);
 
-  if (IsConcurrentSweepingInProgress()) {
+  if (sweeping_in_progress()) {
     // Instead of waiting we could also abort the sweeper threads here.
-    WaitUntilSweepingCompleted();
+    EnsureSweepingCompleted();
   }
 
   // Clear marking bits if incremental marking is aborted.
   if (was_marked_incrementally_ && abort_incremental_marking_) {
     heap()->incremental_marking()->Abort();
     ClearMarkbits();
     AbortCompaction();
     was_marked_incrementally_ = false;
   }
 
@@ skipping 3024 matching lines @@
     cell = it.CurrentCell();
     if (*cell != 0) break;
   }
 
   if (it.Done()) {
     size = p->area_end() - p->area_start();
     freed_bytes = Free<mode>(space, free_list, p->area_start(),
                              static_cast<int>(size));
     max_freed_bytes = Max(freed_bytes, max_freed_bytes);
     ASSERT_EQ(0, p->LiveBytes());
-    return freed_bytes;
+    return free_list->GuaranteedAllocatable(max_freed_bytes);
   }
 
   // Grow the size of the start-of-page free space a little to get up to the
   // first live object.
   Address free_end = StartOfLiveObject(cell_base, *cell);
   // Free the first free space.
   size = free_end - p->area_start();
   freed_bytes = Free<mode>(space, free_list, p->area_start(),
                            static_cast<int>(size));
   max_freed_bytes = Max(freed_bytes, max_freed_bytes);
@@ skipping 36 matching lines @@
 
   // Handle the free space at the end of the page.
   if (cell_base - free_start > 32 * kPointerSize) {
     free_start = DigestFreeStart(free_start, free_start_cell);
     freed_bytes = Free<mode>(space, free_list, free_start,
                              static_cast<int>(p->area_end() - free_start));
     max_freed_bytes = Max(freed_bytes, max_freed_bytes);
   }
 
   p->ResetLiveBytes();
-  return max_freed_bytes;
+  return free_list->GuaranteedAllocatable(max_freed_bytes);
 }
 
 
 int MarkCompactCollector::SweepInParallel(PagedSpace* space,
                                           int required_freed_bytes) {
   PageIterator it(space);
   FreeList* free_list = space == heap()->old_pointer_space()
                             ? free_list_old_pointer_space_.get()
                             : free_list_old_data_space_.get();
   FreeList private_free_list(space);
@@ skipping 13 matching lines @@
       max_freed_overall = Max(max_freed, max_freed_overall);
     }
     if (p == space->end_of_unswept_pages()) break;
   }
   return max_freed_overall;
 }
 
 
 void MarkCompactCollector::SweepSpace(PagedSpace* space, SweeperType sweeper) {
   space->set_is_iterable(sweeper == PRECISE);
-  space->set_is_swept_concurrently(sweeper == CONCURRENT_CONSERVATIVE);
   space->ClearStats();
 
   // We defensively initialize end_of_unswept_pages_ here with the first page
   // of the pages list.
   space->set_end_of_unswept_pages(space->FirstPage());
 
   PageIterator it(space);
 
   int pages_swept = 0;
   bool unused_page_present = false;
@@ skipping 24 matching lines @@
         // Adjust unswept free bytes because releasing a page expects said
         // counter to be accurate for unswept pages.
         space->IncreaseUnsweptFreeBytes(p);
         space->ReleasePage(p);
         continue;
       }
       unused_page_present = true;
     }
 
     switch (sweeper) {
-      case CONSERVATIVE: {
-        if (FLAG_gc_verbose) {
-          PrintF("Sweeping 0x%" V8PRIxPTR " conservatively.\n",
-                 reinterpret_cast<intptr_t>(p));
-        }
-        SweepConservatively<SWEEP_ON_MAIN_THREAD>(space, NULL, p);
-        pages_swept++;
-        break;
-      }
       case CONCURRENT_CONSERVATIVE:
       case PARALLEL_CONSERVATIVE: {
         if (!parallel_sweeping_active) {
           if (FLAG_gc_verbose) {
             PrintF("Sweeping 0x%" V8PRIxPTR " conservatively.\n",
                    reinterpret_cast<intptr_t>(p));
           }
           SweepConservatively<SWEEP_ON_MAIN_THREAD>(space, NULL, p);
           pages_swept++;
           parallel_sweeping_active = true;
@@ skipping 41 matching lines @@
   // Give pages that are queued to be freed back to the OS.
   heap()->FreeQueuedChunks();
 }
 
 
 void MarkCompactCollector::SweepSpaces() {
   GCTracer::Scope gc_scope(tracer_, GCTracer::Scope::MC_SWEEP);
 #ifdef DEBUG
   state_ = SWEEP_SPACES;
 #endif
-  SweeperType how_to_sweep = CONSERVATIVE;
-  if (AreSweeperThreadsActivated()) {
-    if (FLAG_parallel_sweeping) how_to_sweep = PARALLEL_CONSERVATIVE;
-    if (FLAG_concurrent_sweeping) how_to_sweep = CONCURRENT_CONSERVATIVE;
-  }
+  SweeperType how_to_sweep = CONCURRENT_CONSERVATIVE;
+  if (FLAG_parallel_sweeping) how_to_sweep = PARALLEL_CONSERVATIVE;
+  if (FLAG_concurrent_sweeping) how_to_sweep = CONCURRENT_CONSERVATIVE;
+
   if (sweep_precisely_) how_to_sweep = PRECISE;
 
   MoveEvacuationCandidatesToEndOfPagesList();
 
   // Noncompacting collections simply sweep the spaces to clear the mark
   // bits and free the nonlive blocks (for old and map spaces).  We sweep
   // the map space last because freeing non-live maps overwrites them and
   // the other spaces rely on possibly non-live maps to get the sizes for
   // non-live objects.
   { GCTracer::Scope sweep_scope(tracer_, GCTracer::Scope::MC_SWEEP_OLDSPACE);
     { SequentialSweepingScope scope(this);
       SweepSpace(heap()->old_pointer_space(), how_to_sweep);
       SweepSpace(heap()->old_data_space(), how_to_sweep);
     }
 
     if (how_to_sweep == PARALLEL_CONSERVATIVE ||
         how_to_sweep == CONCURRENT_CONSERVATIVE) {
       StartSweeperThreads();
     }
 
     if (how_to_sweep == PARALLEL_CONSERVATIVE) {
-      WaitUntilSweepingCompleted();
+      EnsureSweepingCompleted();
     }
   }
   RemoveDeadInvalidatedCode();
 
   { GCTracer::Scope sweep_scope(tracer_, GCTracer::Scope::MC_SWEEP_CODE);
     SweepSpace(heap()->code_space(), PRECISE);
   }
 
   { GCTracer::Scope sweep_scope(tracer_, GCTracer::Scope::MC_SWEEP_CELL);
     SweepSpace(heap()->cell_space(), PRECISE);
@@ skipping 249 matching lines @@
   while (buffer != NULL) {
     SlotsBuffer* next_buffer = buffer->next();
     DeallocateBuffer(buffer);
     buffer = next_buffer;
   }
   *buffer_address = NULL;
 }
 
 
 } }  // namespace v8::internal