Implemented lazy sweeping of new space.

src/store-buffer.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 346 matching lines @@
 void StoreBuffer::VerifyPointers(PagedSpace* space,
                                  RegionCallback region_callback) {
   PageIterator it(space);
 
   while (it.has_next()) {
     Page* page = it.next();
     FindPointersToNewSpaceOnPage(
         reinterpret_cast<PagedSpace*>(page->owner()),
         page,
         region_callback,
-        &DummyScavengePointer);
+        &DummyScavengePointer,
+        false);
   }
 }
 
 
 void StoreBuffer::VerifyPointers(LargeObjectSpace* space) {
   LargeObjectIterator it(space);
   for (HeapObject* object = it.Next(); object != NULL; object = it.Next()) {
     if (object->IsFixedArray()) {
       Address slot_address = object->address();
       Address end = object->address() + object->Size();
@@ skipping 27 matching lines @@
   during_gc_ = false;
 #ifdef VERIFY_HEAP
   if (FLAG_verify_heap) {
     Verify();
   }
 #endif
 }
 
 
 void StoreBuffer::FindPointersToNewSpaceInRegion(
-    Address start, Address end, ObjectSlotCallback slot_callback) {
+    Address start,
+    Address end,
+    ObjectSlotCallback slot_callback,
+    bool clear_maps) {
   for (Address slot_address = start;
        slot_address < end;
        slot_address += kPointerSize) {
     Object** slot = reinterpret_cast<Object**>(slot_address);
     if (heap_->InNewSpace(*slot)) {
       HeapObject* object = reinterpret_cast<HeapObject*>(*slot);
+      Address& map_field = Memory::Address_at(object->address());

[titzer, 2013/07/10 17:06:35]

I'm a little dedup, short and stout. Rip me over and factor me out!

[Hannes Payer (out of office), 2013/07/11 07:30:40]

Done.

+      // The new space object was not promoted if it still contains a map
+      // pointer. Clear the map field now lazily.
+      if (clear_maps && heap_->map_space()->Contains(map_field)) {
+          map_field = NULL;
+      }
       ASSERT(object->IsHeapObject());
       slot_callback(reinterpret_cast<HeapObject**>(slot), object);
       if (heap_->InNewSpace(*slot)) {
         EnterDirectlyIntoStoreBuffer(slot_address);
       }
     }
   }
 }
 
 
 // Compute start address of the first map following given addr.
 static inline Address MapStartAlign(Address addr) {
   Address page = Page::FromAddress(addr)->area_start();
   return page + (((addr - page) + (Map::kSize - 1)) / Map::kSize * Map::kSize);
 }
 
 
 // Compute end address of the first map preceding given addr.
 static inline Address MapEndAlign(Address addr) {
   Address page = Page::FromAllocationTop(addr)->area_start();
   return page + ((addr - page) / Map::kSize * Map::kSize);
 }
 
 
 void StoreBuffer::FindPointersToNewSpaceInMaps(
     Address start,
     Address end,
-    ObjectSlotCallback slot_callback) {
+    ObjectSlotCallback slot_callback,
+    bool clear_maps) {
   ASSERT(MapStartAlign(start) == start);
   ASSERT(MapEndAlign(end) == end);
 
   Address map_address = start;
   while (map_address < end) {
     ASSERT(!heap_->InNewSpace(Memory::Object_at(map_address)));
     ASSERT(Memory::Object_at(map_address)->IsMap());
 
     Address pointer_fields_start = map_address + Map::kPointerFieldsBeginOffset;
     Address pointer_fields_end = map_address + Map::kPointerFieldsEndOffset;
 
     FindPointersToNewSpaceInRegion(pointer_fields_start,
                                    pointer_fields_end,
-                                   slot_callback);
+                                   slot_callback,
+                                   clear_maps);
     map_address += Map::kSize;
   }
 }
 
 
 void StoreBuffer::FindPointersToNewSpaceInMapsRegion(
     Address start,
     Address end,
-    ObjectSlotCallback slot_callback) {
+    ObjectSlotCallback slot_callback,
+    bool clear_maps) {
   Address map_aligned_start = MapStartAlign(start);
   Address map_aligned_end   = MapEndAlign(end);
 
   ASSERT(map_aligned_start == start);
   ASSERT(map_aligned_end == end);
 
   FindPointersToNewSpaceInMaps(map_aligned_start,
                                map_aligned_end,
-                               slot_callback);
+                               slot_callback,
+                               clear_maps);
 }
 
 
 // This function iterates over all the pointers in a paged space in the heap,
 // looking for pointers into new space.  Within the pages there may be dead
 // objects that have not been overwritten by free spaces or fillers because of
 // lazy sweeping.  These dead objects may not contain pointers to new space.
 // The garbage areas that have been swept properly (these will normally be the
 // large ones) will be marked with free space and filler map words.  In
 // addition any area that has never been used at all for object allocation must
 // be marked with a free space or filler.  Because the free space and filler
 // maps do not move we can always recognize these even after a compaction.
 // Normal objects like FixedArrays and JSObjects should not contain references
 // to these maps.  The special garbage section (see comment in spaces.h) is
 // skipped since it can contain absolutely anything.  Any objects that are
 // allocated during iteration may or may not be visited by the iteration, but
 // they will not be partially visited.
 void StoreBuffer::FindPointersToNewSpaceOnPage(
     PagedSpace* space,
     Page* page,
     RegionCallback region_callback,
-    ObjectSlotCallback slot_callback) {
+    ObjectSlotCallback slot_callback,
+    bool clear_maps) {
   Address visitable_start = page->area_start();
   Address end_of_page = page->area_end();
 
   Address visitable_end = visitable_start;
 
   Object* free_space_map = heap_->free_space_map();
   Object* two_pointer_filler_map = heap_->two_pointer_filler_map();
 
   while (visitable_end < end_of_page) {
     Object* o = *reinterpret_cast<Object**>(visitable_end);
     // Skip fillers but not things that look like fillers in the special
     // garbage section which can contain anything.
     if (o == free_space_map ||
         o == two_pointer_filler_map ||
         (visitable_end == space->top() && visitable_end != space->limit())) {
       if (visitable_start != visitable_end) {
         // After calling this the special garbage section may have moved.
         (this->*region_callback)(visitable_start,
                                  visitable_end,
-                                 slot_callback);
+                                 slot_callback,
+                                 clear_maps);
         if (visitable_end >= space->top() && visitable_end < space->limit()) {
           visitable_end = space->limit();
           visitable_start = visitable_end;
           continue;
         }
       }
       if (visitable_end == space->top() && visitable_end != space->limit()) {
         visitable_start = visitable_end = space->limit();
       } else {
         // At this point we are either at the start of a filler or we are at
@@ skipping 10 matching lines @@
       ASSERT(o != free_space_map);
       ASSERT(o != two_pointer_filler_map);
       ASSERT(visitable_end < space->top() || visitable_end >= space->limit());
       visitable_end += kPointerSize;
     }
   }
   ASSERT(visitable_end == end_of_page);
   if (visitable_start != visitable_end) {
     (this->*region_callback)(visitable_start,
                              visitable_end,
-                             slot_callback);
+                             slot_callback,
+                             clear_maps);
   }
 }
 
 
 void StoreBuffer::IteratePointersInStoreBuffer(
-    ObjectSlotCallback slot_callback) {
+    ObjectSlotCallback slot_callback,
+    bool clear_maps) {
   Address* limit = old_top_;
   old_top_ = old_start_;
   {
     DontMoveStoreBufferEntriesScope scope(this);
     for (Address* current = old_start_; current < limit; current++) {
 #ifdef DEBUG
       Address* saved_top = old_top_;
 #endif
       Object** slot = reinterpret_cast<Object**>(*current);
       Object* object = *slot;
       if (heap_->InFromSpace(object)) {
         HeapObject* heap_object = reinterpret_cast<HeapObject*>(object);
+        Address& map_field = Memory::Address_at(heap_object->address());
+        // The new space object was not promoted if it still contains a map
+        // pointer. Clear the map field now lazily.
+        if (clear_maps && heap_->map_space()->Contains(map_field)) {
+          map_field = NULL;
+        }

[titzer, 2013/07/10 17:06:35]

BTW, do you still want to call the slot callback if the object is dead?

[Hannes Payer (out of office), 2013/07/11 07:30:40]

As soon as we clear the todo in the callback, we can make that call conditional.

         slot_callback(reinterpret_cast<HeapObject**>(slot), heap_object);
         if (heap_->InNewSpace(*slot)) {
           EnterDirectlyIntoStoreBuffer(reinterpret_cast<Address>(slot));
         }
       }
       ASSERT(old_top_ == saved_top + 1 || old_top_ == saved_top);
     }
   }
 }
 
 
-void StoreBuffer::IteratePointersToNewSpace(ObjectSlotCallback slot_callback) {
+void StoreBuffer::IteratePointersToNewSpace(ObjectSlotCallback slot_callback,
+                                            bool clear_maps) {
   // We do not sort or remove duplicated entries from the store buffer because
   // we expect that callback will rebuild the store buffer thus removing
   // all duplicates and pointers to old space.
   bool some_pages_to_scan = PrepareForIteration();
 
   // TODO(gc): we want to skip slots on evacuation candidates
   // but we can't simply figure that out from slot address
   // because slot can belong to a large object.
-  IteratePointersInStoreBuffer(slot_callback);
+  IteratePointersInStoreBuffer(slot_callback, clear_maps);
 
   // We are done scanning all the pointers that were in the store buffer, but
   // there may be some pages marked scan_on_scavenge that have pointers to new
   // space that are not in the store buffer.  We must scan them now.  As we
   // scan, the surviving pointers to new space will be added to the store
   // buffer.  If there are still a lot of pointers to new space then we will
   // keep the scan_on_scavenge flag on the page and discard the pointers that
   // were added to the store buffer.  If there are not many pointers to new
   // space left on the page we will keep the pointers in the store buffer and
   // remove the flag from the page.
   if (some_pages_to_scan) {
     if (callback_ != NULL) {
       (*callback_)(heap_, NULL, kStoreBufferStartScanningPagesEvent);
     }
     PointerChunkIterator it(heap_);
     MemoryChunk* chunk;
     while ((chunk = it.next()) != NULL) {
       if (chunk->scan_on_scavenge()) {
         chunk->set_scan_on_scavenge(false);
         if (callback_ != NULL) {
           (*callback_)(heap_, chunk, kStoreBufferScanningPageEvent);
         }
         if (chunk->owner() == heap_->lo_space()) {
           LargePage* large_page = reinterpret_cast<LargePage*>(chunk);
           HeapObject* array = large_page->GetObject();
           ASSERT(array->IsFixedArray());
           Address start = array->address();
           Address end = start + array->Size();
-          FindPointersToNewSpaceInRegion(start, end, slot_callback);
+          FindPointersToNewSpaceInRegion(start, end, slot_callback, clear_maps);
         } else {
           Page* page = reinterpret_cast<Page*>(chunk);
           PagedSpace* owner = reinterpret_cast<PagedSpace*>(page->owner());
           FindPointersToNewSpaceOnPage(
               owner,
               page,
               (owner == heap_->map_space() ?
                  &StoreBuffer::FindPointersToNewSpaceInMapsRegion :
                  &StoreBuffer::FindPointersToNewSpaceInRegion),
-              slot_callback);
+              slot_callback,
+              clear_maps);
         }
       }
     }
     if (callback_ != NULL) {
       (*callback_)(heap_, NULL, kStoreBufferScanningPageEvent);
     }
   }
 }
 
 
@@ skipping 44 matching lines @@
     }
     old_buffer_is_sorted_ = false;
     old_buffer_is_filtered_ = false;
     *old_top_++ = reinterpret_cast<Address>(int_addr << kPointerSizeLog2);
     ASSERT(old_top_ <= old_limit_);
   }
   heap_->isolate()->counters()->store_buffer_compactions()->Increment();
 }
 
 } }  // namespace v8::internal