Allow main thread to contribute to the sweeping phase.

src/mark-compact.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/v8.h"
 
 #include "src/base/atomicops.h"
 #include "src/code-stubs.h"
 #include "src/compilation-cache.h"
 #include "src/cpu-profiler.h"
@@ skipping 541 matching lines @@
 class MarkCompactCollector::SweeperTask : public v8::Task {
  public:
   SweeperTask(Heap* heap, PagedSpace* space)
     : heap_(heap), space_(space) {}
 
   virtual ~SweeperTask() {}
 
  private:
   // v8::Task overrides.
   virtual void Run() V8_OVERRIDE {
-    heap_->mark_compact_collector()->SweepInParallel(space_);
+    heap_->mark_compact_collector()->SweepInParallel<SWEEP_IN_PARALLEL>(
+        space_, 0);
     heap_->mark_compact_collector()->pending_sweeper_jobs_semaphore_.Signal();
   }
 
   Heap* heap_;
   PagedSpace* space_;
 
   DISALLOW_COPY_AND_ASSIGN(SweeperTask);
 };
 
 
@@ skipping 2964 matching lines @@
       } else {
         if (FLAG_gc_verbose) {
           PrintF("Sweeping 0x%" V8PRIxPTR " during evacuation.\n",
                  reinterpret_cast<intptr_t>(p));
         }
         PagedSpace* space = static_cast<PagedSpace*>(p->owner());
         p->ClearFlag(MemoryChunk::RESCAN_ON_EVACUATION);
 
         switch (space->identity()) {
           case OLD_DATA_SPACE:
-            SweepConservatively<SWEEP_SEQUENTIALLY>(space, NULL, p);
+            SweepConservatively<SWEEP_ON_MAIN_THREAD>(space, NULL, p);
             break;
           case OLD_POINTER_SPACE:
             SweepPrecisely<SWEEP_AND_VISIT_LIVE_OBJECTS,
                            IGNORE_SKIP_LIST,
                            IGNORE_FREE_SPACE>(
                 space, p, &updating_visitor);
             break;
           case CODE_SPACE:
             if (FLAG_zap_code_space) {
               SweepPrecisely<SWEEP_AND_VISIT_LIVE_OBJECTS,
@@ skipping 374 matching lines @@
   USE(live_objects);
   return block_address + offsets[0] * kPointerSize;
 }
 
 
 template<MarkCompactCollector::SweepingParallelism mode>
 static intptr_t Free(PagedSpace* space,
                      FreeList* free_list,
                      Address start,
                      int size) {
-  if (mode == MarkCompactCollector::SWEEP_SEQUENTIALLY) {
+  if (mode == MarkCompactCollector::SWEEP_ON_MAIN_THREAD) {
     return space->Free(start, size);
   } else {
     return size - free_list->Free(start, size);
   }
 }
 
 
 // Force instantiation of templatized SweepConservatively method for
-// SWEEP_SEQUENTIALLY mode.
+// SWEEP_ON_MAIN_THREAD mode.
 template intptr_t MarkCompactCollector::
-    SweepConservatively<MarkCompactCollector::SWEEP_SEQUENTIALLY>(
+    SweepConservatively<MarkCompactCollector::SWEEP_ON_MAIN_THREAD>(
         PagedSpace*, FreeList*, Page*);
 
 
 // Force instantiation of templatized SweepConservatively method for
 // SWEEP_IN_PARALLEL mode.
 template intptr_t MarkCompactCollector::
     SweepConservatively<MarkCompactCollector::SWEEP_IN_PARALLEL>(
         PagedSpace*, FreeList*, Page*);
 
 
 // Sweeps a space conservatively.  After this has been done the larger free
 // spaces have been put on the free list and the smaller ones have been
 // ignored and left untouched.  A free space is always either ignored or put
 // on the free list, never split up into two parts.  This is important
 // because it means that any FreeSpace maps left actually describe a region of
 // memory that can be ignored when scanning.  Dead objects other than free
 // spaces will not contain the free space map.
 template<MarkCompactCollector::SweepingParallelism mode>
 intptr_t MarkCompactCollector::SweepConservatively(PagedSpace* space,
                                                    FreeList* free_list,
                                                    Page* p) {
   ASSERT(!p->IsEvacuationCandidate() && !p->WasSwept());
   ASSERT((mode == MarkCompactCollector::SWEEP_IN_PARALLEL &&
          free_list != NULL) ||
-         (mode == MarkCompactCollector::SWEEP_SEQUENTIALLY &&
+         (mode == MarkCompactCollector::SWEEP_ON_MAIN_THREAD &&
          free_list == NULL));
 
   // When parallel sweeping is active, the page will be marked after
   // sweeping by the main thread.
   if (mode != MarkCompactCollector::SWEEP_IN_PARALLEL) {
     p->MarkSweptConservatively();
   }
 
   intptr_t freed_bytes = 0;
+  intptr_t max_freed_bytes = 0;
   size_t size = 0;
 
   // Skip over all the dead objects at the start of the page and mark them free.
   Address cell_base = 0;
   MarkBit::CellType* cell = NULL;
   MarkBitCellIterator it(p);
   for (; !it.Done(); it.Advance()) {
     cell_base = it.CurrentCellBase();
     cell = it.CurrentCell();
     if (*cell != 0) break;
   }
 
   if (it.Done()) {
     size = p->area_end() - p->area_start();
-    freed_bytes += Free<mode>(space, free_list, p->area_start(),
-                              static_cast<int>(size));
+    freed_bytes = Free<mode>(space, free_list, p->area_start(),
+                             static_cast<int>(size));
+    max_freed_bytes = Max(freed_bytes, max_freed_bytes);
     ASSERT_EQ(0, p->LiveBytes());
     return freed_bytes;
   }
 
   // Grow the size of the start-of-page free space a little to get up to the
   // first live object.
   Address free_end = StartOfLiveObject(cell_base, *cell);
   // Free the first free space.
   size = free_end - p->area_start();
-  freed_bytes += Free<mode>(space, free_list, p->area_start(),
-                            static_cast<int>(size));
+  freed_bytes = Free<mode>(space, free_list, p->area_start(),
+                           static_cast<int>(size));
+  max_freed_bytes = Max(freed_bytes, max_freed_bytes);
 
   // The start of the current free area is represented in undigested form by
   // the address of the last 32-word section that contained a live object and
   // the marking bitmap for that cell, which describes where the live object
   // started.  Unless we find a large free space in the bitmap we will not
   // digest this pair into a real address.  We start the iteration here at the
   // first word in the marking bit map that indicates a live object.
   Address free_start = cell_base;
   MarkBit::CellType free_start_cell = *cell;
 
   for (; !it.Done(); it.Advance()) {
     cell_base = it.CurrentCellBase();
     cell = it.CurrentCell();
     if (*cell != 0) {
       // We have a live object.  Check approximately whether it is more than 32
       // words since the last live object.
       if (cell_base - free_start > 32 * kPointerSize) {
         free_start = DigestFreeStart(free_start, free_start_cell);
         if (cell_base - free_start > 32 * kPointerSize) {
           // Now that we know the exact start of the free space it still looks
           // like we have a large enough free space to be worth bothering with.
           // so now we need to find the start of the first live object at the
           // end of the free space.
           free_end = StartOfLiveObject(cell_base, *cell);
-          freed_bytes += Free<mode>(space, free_list, free_start,
-                                    static_cast<int>(free_end - free_start));
+          freed_bytes = Free<mode>(space, free_list, free_start,
+                                   static_cast<int>(free_end - free_start));
+          max_freed_bytes = Max(freed_bytes, max_freed_bytes);
         }
       }
       // Update our undigested record of where the current free area started.
       free_start = cell_base;
       free_start_cell = *cell;
       // Clear marking bits for current cell.
       *cell = 0;
     }
   }
 
   // Handle the free space at the end of the page.
   if (cell_base - free_start > 32 * kPointerSize) {
     free_start = DigestFreeStart(free_start, free_start_cell);
-    freed_bytes += Free<mode>(space, free_list, free_start,
-                              static_cast<int>(p->area_end() - free_start));
+    freed_bytes = Free<mode>(space, free_list, free_start,
+                             static_cast<int>(p->area_end() - free_start));
+    max_freed_bytes = Max(freed_bytes, max_freed_bytes);
   }
 
   p->ResetLiveBytes();
-  return freed_bytes;
+  return max_freed_bytes;
 }
 
 
-void MarkCompactCollector::SweepInParallel(PagedSpace* space) {
+// Force instantiation of templatized SweepInParallel method for

[Jarin, 2014/07/09 11:02:02]

Is the templatization of SweepInParallel really necessary? Would not it just
work if you always called with SWEEP_IN_PARALLEL mode?

[Hannes Payer (out of office), 2014/07/10 12:03:45]

Done.

It would make sense to remove the template for SweepConservatively. However,
this is a major refactoring of the sweeping code. I will try to do this with a
foll-up cl. I basically have to move all the sweeping modes over to use the lazy
mechanism for bookkeeping instead of eagerly bookkeeping in the non-concurrent
case.

+// SWEEP_ON_MAIN_THREAD mode.
+template int MarkCompactCollector::
+    SweepInParallel<MarkCompactCollector::SWEEP_ON_MAIN_THREAD>(
+        PagedSpace*, int);
+
+
+// Force instantiation of templatized SweepInParallel method for
+// SWEEP_IN_PARALLEL mode.
+template int MarkCompactCollector::
+    SweepInParallel<MarkCompactCollector::SWEEP_IN_PARALLEL>(
+        PagedSpace*, int);
+
+
+template<MarkCompactCollector::SweepingParallelism type>
+int MarkCompactCollector::SweepInParallel(PagedSpace* space,
+                                          int required_freed_bytes) {
   PageIterator it(space);
   FreeList* free_list = space == heap()->old_pointer_space()
                             ? free_list_old_pointer_space_.get()
                             : free_list_old_data_space_.get();
   FreeList private_free_list(space);
+  int max_freed = 0;
   while (it.has_next()) {
     Page* p = it.next();
 
     if (p->TryParallelSweeping()) {
-      SweepConservatively<SWEEP_IN_PARALLEL>(space, &private_free_list, p);
-      free_list->Concatenate(&private_free_list);
+      if (type == SWEEP_IN_PARALLEL) {
+        max_freed = SweepConservatively<SWEEP_IN_PARALLEL>(
+            space, &private_free_list, p);
+        free_list->Concatenate(&private_free_list);
+      } else {
+        max_freed = SweepConservatively<SWEEP_ON_MAIN_THREAD>(space, NULL, p);
+      }
       p->set_parallel_sweeping(MemoryChunk::PARALLEL_SWEEPING_FINALIZE);
+      if (required_freed_bytes > 0 && max_freed >= required_freed_bytes) {
+        return max_freed;
+      }
     }
     if (p == space->end_of_unswept_pages()) break;
   }
+  return max_freed;

[Jarin, 2014/07/09 11:02:02]

For consistency, should not this return the maximum? (Rather than the last value
of returned by the SweepConservatively.)

[Hannes Payer (out of office), 2014/07/10 12:03:45]

Done.

 }
 
 
 void MarkCompactCollector::SweepSpace(PagedSpace* space, SweeperType sweeper) {
   space->set_is_iterable(sweeper == PRECISE);
   space->set_is_swept_concurrently(sweeper == CONCURRENT_CONSERVATIVE);
   space->ClearStats();
 
   // We defensively initialize end_of_unswept_pages_ here with the first page
   // of the pages list.
@@ skipping 35 matching lines @@
       }
       unused_page_present = true;
     }
 
     switch (sweeper) {
       case CONSERVATIVE: {
         if (FLAG_gc_verbose) {
           PrintF("Sweeping 0x%" V8PRIxPTR " conservatively.\n",
                  reinterpret_cast<intptr_t>(p));
         }
-        SweepConservatively<SWEEP_SEQUENTIALLY>(space, NULL, p);
+        SweepConservatively<SWEEP_ON_MAIN_THREAD>(space, NULL, p);
         pages_swept++;
         break;
       }
       case CONCURRENT_CONSERVATIVE:
       case PARALLEL_CONSERVATIVE: {
         if (!parallel_sweeping_active) {
           if (FLAG_gc_verbose) {
             PrintF("Sweeping 0x%" V8PRIxPTR " conservatively.\n",
                    reinterpret_cast<intptr_t>(p));
           }
-          SweepConservatively<SWEEP_SEQUENTIALLY>(space, NULL, p);
+          SweepConservatively<SWEEP_ON_MAIN_THREAD>(space, NULL, p);
           pages_swept++;
           parallel_sweeping_active = true;
         } else {
           if (p->scan_on_scavenge()) {
             SweepPrecisely<SWEEP_ONLY, IGNORE_SKIP_LIST, IGNORE_FREE_SPACE>(
                 space, p, NULL);
             pages_swept++;
             if (FLAG_gc_verbose) {
               PrintF("Sweeping 0x%" V8PRIxPTR
                   " scan on scavenge page precisely.\n",
@@ skipping 345 matching lines @@
   while (buffer != NULL) {
     SlotsBuffer* next_buffer = buffer->next();
     DeallocateBuffer(buffer);
     buffer = next_buffer;
   }
   *buffer_address = NULL;
 }
 
 
 } }  // namespace v8::internal