Fix lazy sweeping for new marking bit clearing.

src/mark-compact.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 2476 matching lines @@
     }
   }
 
   heap_->IncrementYoungSurvivorsCounter(survivors_size);
   new_space->set_age_mark(new_space->top());
 }
 
 
 void MarkCompactCollector::EvacuateLiveObjectsFromPage(Page* p) {
   AlwaysAllocateScope always_allocate;
-
-  ASSERT(p->IsEvacuationCandidate() && !p->WasSwept());
-
   PagedSpace* space = static_cast<PagedSpace*>(p->owner());
-
+  ASSERT(p->IsEvacuationCandidate() &&
+         !p->WasSweptPrecisely() &&

[Erik Corry, 2011/08/25 09:19:48]

I think it makes sense to have a
  bool WasSwept() { return WasSweptPrecisely() || WasSweptConservatively(); }
I think there are 4 places this could be used.

[Michael Starzinger, 2011/08/25 09:33:03]

Done.

+         !p->WasSweptConservatively());
   MarkBit::CellType* cells = p->markbits()->cells();
+  p->MarkSweptPrecisely();
 
   int last_cell_index =
       Bitmap::IndexToCell(
           Bitmap::CellAlignIndex(
               p->AddressToMarkbitIndex(p->ObjectAreaEnd())));
 
   int cell_index = Page::kFirstUsedCell;
   Address cell_base = p->ObjectAreaStart();
   int offsets[16];
 
@@ skipping 16 matching lines @@
 
       // This should never fail as we are in always allocate scope.
       Object* target = space->AllocateRaw(size)->ToObjectUnchecked();
 
       MigrateObject(HeapObject::cast(target)->address(),
                     object_addr,
                     size,
                     space->identity());
       ASSERT(object->map_word().IsForwardingAddress());
     }
+
+    // Clear marking bits for current cell.
+    cells[cell_index] = 0;
   }
 }
 
 
 void MarkCompactCollector::EvacuatePages() {
   int npages = evacuation_candidates_.length();
   for (int i = 0; i < npages; i++) {
     Page* p = evacuation_candidates_[i];
     ASSERT(p->IsEvacuationCandidate() ||
            p->IsFlagSet(Page::RESCAN_ON_EVACUATION));
@@ skipping 57 matching lines @@
 };
 
 
 // Sweep a space precisely.  After this has been done the space can
 // be iterated precisely, hitting only the live objects.  Code space
 // is always swept precisely because we want to be able to iterate
 // over it.  Map space is swept precisely, because it is not compacted.
 // Slots in live objects pointing into evacuation candidates are updated
 // if requested.
 static void SweepPrecisely(PagedSpace* space, Page* p, SweepingMode mode) {
-  ASSERT(!p->IsEvacuationCandidate() && !p->WasSwept());
-  ASSERT(!p->IsFlagSet(MemoryChunk::WAS_SWEPT_CONSERVATIVELY));
+  ASSERT(!p->IsEvacuationCandidate() &&
+         !p->WasSweptPrecisely() &&
+         !p->WasSweptConservatively());
   MarkBit::CellType* cells = p->markbits()->cells();
-  p->MarkSwept();
+  p->MarkSweptPrecisely();
 
   int last_cell_index =
       Bitmap::IndexToCell(
           Bitmap::CellAlignIndex(
               p->AddressToMarkbitIndex(p->ObjectAreaEnd())));
 
   int cell_index = Page::kFirstUsedCell;
   Address free_start = p->ObjectAreaStart();
   ASSERT(reinterpret_cast<intptr_t>(free_start) % (32 * kPointerSize) == 0);
   Address object_address = p->ObjectAreaStart();
@@ skipping 74 matching lines @@
            p->IsFlagSet(Page::RESCAN_ON_EVACUATION));
 
     if (p->IsEvacuationCandidate()) {
       SlotsBuffer::UpdateSlotsRecordedIn(p->slots_buffer());
       if (FLAG_trace_fragmentation) {
         PrintF("  page %p slots buffer: %d\n",
                reinterpret_cast<void*>(p),
                SlotsBuffer::SizeOfChain(p->slots_buffer()));
       }
     } else {
+      if (FLAG_gc_verbose) {
+        PrintF("Sweeping 0x%" V8PRIxPTR " during evacuation.\n",
+               reinterpret_cast<intptr_t>(p));
+      }
       PagedSpace* space = static_cast<PagedSpace*>(p->owner());
       p->ClearFlag(MemoryChunk::RESCAN_ON_EVACUATION);
       SweepPrecisely(space, p, SWEEP_AND_UPDATE_SLOTS);
     }
   }
 
   // Update pointers from cells.
   HeapObjectIterator cell_iterator(heap_->cell_space());
   for (HeapObject* cell = cell_iterator.Next();
        cell != NULL;
@@ skipping 30 matching lines @@
   slots_buffer_allocator_.DeallocateChain(&migration_slots_buffer_);
   ASSERT(migration_slots_buffer_ == NULL);
   for (int i = 0; i < npages; i++) {
     Page* p = evacuation_candidates_[i];
     if (!p->IsEvacuationCandidate()) continue;
     PagedSpace* space = static_cast<PagedSpace*>(p->owner());
     space->Free(p->ObjectAreaStart(), Page::kObjectAreaSize);
     p->set_scan_on_scavenge(false);
     slots_buffer_allocator_.DeallocateChain(p->slots_buffer_address());
     p->ClearEvacuationCandidate();
-    p->MarkSwept();
-    p->ClearFlag(MemoryChunk::WAS_SWEPT_CONSERVATIVELY);
   }
   evacuation_candidates_.Rewind(0);
   compacting_ = false;
 }
 
 
 INLINE(static uint32_t SweepFree(PagedSpace* space,
                                  Page* p,
                                  uint32_t free_start,
                                  uint32_t region_end,
@@ skipping 310 matching lines @@
 
 
 // Sweeps a space conservatively.  After this has been done the larger free
 // spaces have been put on the free list and the smaller ones have been
 // ignored and left untouched.  A free space is always either ignored or put
 // on the free list, never split up into two parts.  This is important
 // because it means that any FreeSpace maps left actually describe a region of
 // memory that can be ignored when scanning.  Dead objects other than free
 // spaces will not contain the free space map.
 intptr_t MarkCompactCollector::SweepConservatively(PagedSpace* space, Page* p) {
-  ASSERT(!p->IsEvacuationCandidate() && !p->WasSwept());
+  ASSERT(!p->IsEvacuationCandidate() &&
+         !p->WasSweptPrecisely() &&
+         !p->WasSweptConservatively());
   MarkBit::CellType* cells = p->markbits()->cells();
-  p->SetFlag(MemoryChunk::WAS_SWEPT_CONSERVATIVELY);
+  p->MarkSweptConservatively();
 
   int last_cell_index =
       Bitmap::IndexToCell(
           Bitmap::CellAlignIndex(
               p->AddressToMarkbitIndex(p->ObjectAreaEnd())));
 
   int cell_index = Page::kFirstUsedCell;
   intptr_t freed_bytes = 0;
 
   // This is the start of the 32 word block that we are currently looking at.
@@ skipping 73 matching lines @@
                                       SweeperType sweeper) {
   space->set_was_swept_conservatively(sweeper == CONSERVATIVE ||
                                       sweeper == LAZY_CONSERVATIVE);
 
   space->ClearStats();
 
   PageIterator it(space);
 
   intptr_t freed_bytes = 0;
   intptr_t newspace_size = space->heap()->new_space()->Size();
+  bool lazy_sweeping_active = false;
 
   while (it.has_next()) {
     Page* p = it.next();
 
+    // Clear sweeping flags indicating that marking bits are still intact.
+    p->ClearSweptPrecisely();
+    p->ClearSweptConservatively();
+
     if (p->IsEvacuationCandidate()) {
       ASSERT(evacuation_candidates_.length() > 0);
       continue;
     }
 
     if (p->IsFlagSet(Page::RESCAN_ON_EVACUATION)) {
       // Will be processed in EvacuateNewSpaceAndCandidates.
       continue;
     }
 
+    if (lazy_sweeping_active) {
+      if (FLAG_gc_verbose) {
+        PrintF("Sweeping 0x%" V8PRIxPTR " lazily postponed.\n",
+               reinterpret_cast<intptr_t>(p));
+      }
+      continue;
+    }
+
+    if (FLAG_gc_verbose) {
+      PrintF("Sweeping 0x%" V8PRIxPTR " with sweeper %d.\n",
+             reinterpret_cast<intptr_t>(p),
+             sweeper);
+    }
+
     switch (sweeper) {
       case CONSERVATIVE: {
         SweepConservatively(space, p);
         break;
       }
       case LAZY_CONSERVATIVE: {
-        Page* next_page = p->next_page();
         freed_bytes += SweepConservatively(space, p);
         if (freed_bytes >= newspace_size && p != space->LastPage()) {
-          space->SetPagesToSweep(next_page, space->LastPage());
-          return;
+          space->SetPagesToSweep(p->next_page(), space->LastPage());
+          lazy_sweeping_active = true;
         }
         break;
       }
       case PRECISE: {
         SweepPrecisely(space, p, SWEEP_ONLY);
         break;
       }
       default: {
         UNREACHABLE();
       }
@@ skipping 89 matching lines @@
   while (buffer != NULL) {
     SlotsBuffer* next_buffer = buffer->next();
     DeallocateBuffer(buffer);
     buffer = next_buffer;
   }
   *buffer_address = NULL;
 }
 
 
 } }  // namespace v8::internal