On store buffer overflow we mark individidual pages for...

src/heap.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 42 matching lines @@
 #endif
 
 
 namespace v8 {
 namespace internal {
 
 
 String* Heap::hidden_symbol_;
 Object* Heap::roots_[Heap::kRootListLength];
 Object* Heap::global_contexts_list_;
+StoreBufferRebuilder Heap::store_buffer_rebuilder_;
 
 
 NewSpace Heap::new_space_;
 OldSpace* Heap::old_pointer_space_ = NULL;
 OldSpace* Heap::old_data_space_ = NULL;
 OldSpace* Heap::code_space_ = NULL;
 MapSpace* Heap::map_space_ = NULL;
 CellSpace* Heap::cell_space_ = NULL;
 LargeObjectSpace* Heap::lo_space_ = NULL;
 
@@ skipping 16 matching lines @@
 int Heap::initial_semispace_size_ = 128*KB;
 intptr_t Heap::code_range_size_ = 0;
 intptr_t Heap::max_executable_size_ = max_old_generation_size_;
 #elif defined(V8_TARGET_ARCH_X64)
 static const int default_max_semispace_size_  = 16*MB;
 intptr_t Heap::max_old_generation_size_ = 1*GB;
 int Heap::initial_semispace_size_ = 1*MB;
 intptr_t Heap::code_range_size_ = 512*MB;
 intptr_t Heap::max_executable_size_ = 256*MB;
 #else
-static const int default_max_semispace_size_  = 8*MB;
+static const int default_max_semispace_size_  = 4*MB;
 intptr_t Heap::max_old_generation_size_ = 512*MB;
 int Heap::initial_semispace_size_ = 512*KB;
 intptr_t Heap::code_range_size_ = 0;
 intptr_t Heap::max_executable_size_ = 128*MB;
 #endif
 
 // Allow build-time customization of the max semispace size. Building
 // V8 with snapshots and a non-default max semispace size is much
 // easier if you can define it as part of the build environment.
 #if defined(V8_MAX_SEMISPACE_SIZE)
@@ skipping 363 matching lines @@
   if (collector == SCAVENGER &&
       IncrementalMarking::state() != IncrementalMarking::STOPPED) {
     if (FLAG_trace_incremental_marking) {
       PrintF("[IncrementalMarking] Scavenge during marking.\n");
     }
   }
 
   if (collector == MARK_COMPACTOR &&
       !MarkCompactCollector::PreciseSweepingRequired() &&
       IncrementalMarking::state() == IncrementalMarking::MARKING &&
+      !IncrementalMarking::should_hurry() &&
       FLAG_incremental_marking_steps) {
     if (FLAG_trace_incremental_marking) {
       PrintF("[IncrementalMarking] Delaying MarkSweep.\n");
     }
     collector = SCAVENGER;
   }
 
+  IncrementalMarking::set_should_hurry(false);

[Vyacheslav Egorov (Chromium), 2011/03/28 15:13:19]

I think nicer place to clean the flag is in the IncrementalMarking::Finalize()

[Erik Corry, 2011/03/28 15:56:07]

Done and made it private.

+
   bool next_gc_likely_to_collect_more = false;
 
   { GCTracer tracer;
     GarbageCollectionPrologue();
     // The GC count was incremented in the prologue.  Tell the tracer about
     // it.
     tracer.set_gc_count(gc_count_);
 
     // Tell the tracer which collector we've selected.
     tracer.set_collector(collector);
@@ skipping 444 matching lines @@
   if (new_space_.Capacity() < new_space_.MaximumCapacity() &&
       survived_since_last_expansion_ > new_space_.Capacity()) {
     // Grow the size of new space if there is room to grow and enough
     // data has survived scavenge since the last expansion.
     new_space_.Grow();
     survived_since_last_expansion_ = 0;
   }
 }
 
 
+void Heap::ScavengeStoreBufferCallback(MemoryChunk* page,
+                                       StoreBufferEvent event) {
+  store_buffer_rebuilder_.Callback(page, event);
+}
+
+
+void StoreBufferRebuilder::Callback(MemoryChunk* page, StoreBufferEvent event) {
+  if (event == kStoreBufferStartScanningPagesEvent) {
+    start_of_current_page_ = NULL;
+    current_page_ = NULL;
+  } else if (event == kStoreBufferScanningPageEvent) {
+    if (current_page_ != NULL) {
+      // If this page already overflowed the store buffer during this iteration.
+      if (current_page_->scan_on_scavenge()) {
+        // Then we should wipe out the entries that have been added for it.
+        StoreBuffer::SetTop(start_of_current_page_);
+      } else if (StoreBuffer::Top() - start_of_current_page_ >=
+                 (StoreBuffer::Limit() - StoreBuffer::Top()) >> 2) {
+        // Did we find too many pointers in the previous page?  The heuristic is
+        // that no page can take more then 1/5 the remaining slots in the store
+        // buffer.
+        current_page_->set_scan_on_scavenge(true);
+        StoreBuffer::SetTop(start_of_current_page_);
+      } else {
+        // In this case the page we scanned took a reasonable number of slots in
+        // the store buffer.  It has now been rehabilitated and is no longer
+        // marked scan_on_scavenge.
+        ASSERT(!current_page_->scan_on_scavenge());
+      }
+    }
+    start_of_current_page_ = StoreBuffer::Top();
+    current_page_ = page;
+  } else if (event == kStoreBufferFullEvent) {
+    // The current page overflowed the store buffer again.  Wipe out its entries
+    // in the store buffer and mark it scan-on-scavenge again.  This may happen
+    // several times while scanning.
+    if (current_page_ == NULL) {
+      // Store Buffer overflowed while scanning promoted objects.  These are not
+      // in any particular page, though they are likely to be clustered by the
+      // allocation routines.
+      StoreBuffer::HandleFullness();
+    } else {
+      // Store Buffer overflowed while scanning a particular old space page for
+      // pointers to new space.
+      ASSERT(current_page_ == page);
+      ASSERT(page != NULL);
+      current_page_->set_scan_on_scavenge(true);
+      ASSERT(start_of_current_page_ != StoreBuffer::Top());
+      StoreBuffer::SetTop(start_of_current_page_);
+    }
+  } else {
+    UNREACHABLE();
+  }
+}
+
+
 void Heap::Scavenge() {
 #ifdef DEBUG
   if (FLAG_enable_slow_asserts) VerifyNonPointerSpacePointers();
 #endif
 
   gc_state_ = SCAVENGE;
 
   // Implements Cheney's copying algorithm
   LOG(ResourceEvent("scavenge", "begin"));
 
@@ skipping 37 matching lines @@
 #ifdef DEBUG
   StoreBuffer::Clean();
 #endif
 
   ScavengeVisitor scavenge_visitor;
   // Copy roots.
   IterateRoots(&scavenge_visitor, VISIT_ALL_IN_SCAVENGE);
 
   // Copy objects reachable from the old generation.
   {
-    StoreBufferRebuildScope scope;
+    StoreBufferRebuildScope scope(&ScavengeStoreBufferCallback);
     StoreBuffer::IteratePointersToNewSpace(&ScavengeObject);
   }
 
   // Copy objects reachable from cells by scavenging cell values directly.
   HeapObjectIterator cell_iterator(cell_space_);
   for (HeapObject* cell = cell_iterator.Next();
        cell != NULL; cell = cell_iterator.Next()) {
     if (cell->IsJSGlobalPropertyCell()) {
       Address value_address =
           reinterpret_cast<Address>(cell) +
@@ skipping 176 matching lines @@
     // The addresses new_space_front and new_space_.top() define a
     // queue of unprocessed copied objects.  Process them until the
     // queue is empty.
     while (new_space_front < new_space_.top()) {
       HeapObject* object = HeapObject::FromAddress(new_space_front);
       new_space_front += NewSpaceScavenger::IterateBody(object->map(), object);
     }
 
     // Promote and process all the to-be-promoted objects.
     {
-      StoreBufferRebuildScope scope;
+      StoreBufferRebuildScope scope(&ScavengeStoreBufferCallback);
       while (!promotion_queue.is_empty()) {
         HeapObject* target;
         int size;
         promotion_queue.remove(&target, &size);
 
         // Promoted object might be already partially visited
         // during old space pointer iteration. Thus we search specificly
         // for pointers to from semispace instead of looking for pointers
         // to new space.
         ASSERT(!target->IsMap());
@@ skipping 2751 matching lines @@
 }
 
 
 static void VerifyPointers(
     PagedSpace* space,
     PointerRegionCallback visit_pointer_region) {
   PageIterator it(space);
 
   while (it.has_next()) {
     Page* page = it.next();
-    Address start = page->ObjectAreaStart();
-    Address end = page->ObjectAreaEnd();
-
-    Heap::IteratePointersToNewSpace(start,
-                                    end,
-                                    &DummyScavengePointer);
+    Heap::IteratePointersOnPage(
+        reinterpret_cast<PagedSpace*>(page->owner()),
+        &Heap::IteratePointersToNewSpace,
+        &DummyScavengePointer,
+        page);
   }
 }
 
 
 static void VerifyPointers(LargeObjectSpace* space) {
   LargeObjectIterator it(space);
   for (HeapObject* object = it.next(); object != NULL; object = it.next()) {
     if (object->IsFixedArray()) {
       Address slot_address = object->address();
       Address end = object->address() + object->Size();
@@ skipping 190 matching lines @@
 }
 
 
 void Heap::IterateAndMarkPointersToFromSpace(Address start,
                                              Address end,
                                              ObjectSlotCallback callback) {
   Address slot_address = start;
   while (slot_address < end) {
     Object** slot = reinterpret_cast<Object**>(slot_address);
     Object* object = *slot;
-    // In normal store buffer operation we use this function to process the
-    // promotion queue and we never scan an object twice so we will not see
-    // pointers that have already been updated to point to to-space.  But
-    // in the case of store buffer overflow we scan the entire old space to
-    // find pointers that point to new-space and in that case we may hit
-    // newly promoted objects and fix the pointers before the promotion
-    // queue gets to them.
-    ASSERT(StoreBuffer::store_buffer_mode() !=
-               StoreBuffer::kStoreBufferFunctional ||
-           !Heap::InToSpace(object));
+    // If the store buffer becomes overfull we mark pages as being exempt from
+    // the store buffer.  These pages are scanned to find pointers that point
+    // to the new space.  In that case we may hit newly promoted objects and
+    // fix the pointers before the promotion queue gets to them.  Thus the 'if'.
     if (Heap::InFromSpace(object)) {
       callback(reinterpret_cast<HeapObject**>(slot), HeapObject::cast(object));
       if (Heap::InNewSpace(*slot)) {
         ASSERT(Heap::InToSpace(*slot));
         ASSERT((*slot)->IsHeapObject());
-        ASSERT(StoreBuffer::CellIsInStoreBuffer(
-            reinterpret_cast<Address>(slot)));
       }
     }
     slot_address += kPointerSize;
   }
 }
 
 
 #ifdef DEBUG
 typedef bool (*CheckStoreBufferFilter)(Object** addr);
 
@@ skipping 60 matching lines @@
       while (!(*obj_start)->IsMap()) obj_start--;
       UNREACHABLE();
     }
   }
 }
 
 
 // Check that the store buffer contains all intergenerational pointers by
 // scanning a page and ensuring that all pointers to young space are in the
 // store buffer.
-void Heap::OldPointerSpaceCheckStoreBuffer(
-    ExpectedPageWatermarkState watermark_state) {
+void Heap::OldPointerSpaceCheckStoreBuffer() {
   OldSpace* space = old_pointer_space();
   PageIterator pages(space);
 
   StoreBuffer::SortUniq();
 
   while (pages.has_next()) {
     Page* page = pages.next();
     Object** current = reinterpret_cast<Object**>(page->ObjectAreaStart());
 
     Address end = page->ObjectAreaEnd();
 
     Object*** store_buffer_position = StoreBuffer::Start();
     Object*** store_buffer_top = StoreBuffer::Top();
 
     Object** limit = reinterpret_cast<Object**>(end);
     CheckStoreBuffer(current,
                      limit,
                      &store_buffer_position,
                      store_buffer_top,
                      &EverythingsAPointer,
                      space->top(),
                      space->limit());
   }
 }
 
 
-void Heap::MapSpaceCheckStoreBuffer(
-    ExpectedPageWatermarkState watermark_state) {
+void Heap::MapSpaceCheckStoreBuffer() {
   MapSpace* space = map_space();
   PageIterator pages(space);
 
   StoreBuffer::SortUniq();
 
   while (pages.has_next()) {
     Page* page = pages.next();
     Object** current = reinterpret_cast<Object**>(page->ObjectAreaStart());
 
     Address end = page->ObjectAreaEnd();
@@ skipping 50 matching lines @@
 // be marked with a free space or filler.  Because the free space and filler
 // maps do not move we can always recognize these even after a compaction.
 // Normal objects like FixedArrays and JSObjects should not contain references
 // to these maps.  The special garbage section (see comment in spaces.h) is
 // skipped since it can contain absolutely anything.  Any objects that are
 // allocated during iteration may or may not be visited by the iteration, but
 // they will not be partially visited.
 void Heap::IteratePointers(
     PagedSpace* space,
     PointerRegionCallback visit_pointer_region,
-    ObjectSlotCallback copy_object_func,
-    ExpectedPageWatermarkState expected_page_watermark_state) {
+    ObjectSlotCallback copy_object_func) {
 
   PageIterator pages(space);
 
   while (pages.has_next()) {
     Page* page = pages.next();
-    Address start = page->ObjectAreaStart();
-    Address limit = page->ObjectAreaEnd();
-
-    Address end = start;
-
-    Object* free_space_map = Heap::free_space_map();
-    Object* two_pointer_filler_map = Heap::two_pointer_filler_map();
-
-    while (end < limit) {
-      Object* o = *reinterpret_cast<Object**>(end);
-      // Skip fillers but not things that look like fillers in the special
-      // garbage section which can contain anything.
-      if (o == free_space_map ||
-          o == two_pointer_filler_map ||
-          end == space->top()) {
-        if (start != end) {
-          // After calling this the special garbage section may have moved.
-          visit_pointer_region(start, end, copy_object_func);
-          if (end >= space->top() && end < space->limit()) {
-            end = space->limit();
-            start = end;
-            continue;
-          }
-        }
-        if (end == space->top()) {
-          start = end = space->limit();
-        } else {
-          // At this point we are either at the start of a filler or we are at
-          // the point where the space->top() used to be before the
-          // visit_pointer_region call above.  Either way we can skip the
-          // object at the current spot:  We don't promise to visit objects
-          // allocated during heap traversal, and if space->top() moved then it
-          // must be because an object was allocated at this point.
-          start = end + HeapObject::FromAddress(end)->Size();
-          end = start;
-        }
-      } else {
-        ASSERT(o != free_space_map);
-        ASSERT(o != two_pointer_filler_map);
-        ASSERT(end < space->top() || end >= space->limit());
-        end += kPointerSize;
-      }
-    }
-    ASSERT(end == limit);
-    if (start != end) {
-      visit_pointer_region(start, end, copy_object_func);
-    }
+    IteratePointersOnPage(space, visit_pointer_region, copy_object_func, page);
   }
 }
 
 
+void Heap::IteratePointersOnPage(
+    PagedSpace* space,
+    PointerRegionCallback visit_pointer_region,
+    ObjectSlotCallback copy_object_func,
+    Page* page) {
+  Address visitable_start = page->ObjectAreaStart();
+  Address end_of_page = page->ObjectAreaEnd();
+
+  Address visitable_end = visitable_start;
+
+  Object* free_space_map = Heap::free_space_map();
+  Object* two_pointer_filler_map = Heap::two_pointer_filler_map();
+
+  while (visitable_end < end_of_page) {
+    Object* o = *reinterpret_cast<Object**>(visitable_end);
+    // Skip fillers but not things that look like fillers in the special
+    // garbage section which can contain anything.
+    if (o == free_space_map ||
+        o == two_pointer_filler_map ||
+        visitable_end == space->top()) {
+      if (visitable_start != visitable_end) {
+        // After calling this the special garbage section may have moved.
+        visit_pointer_region(visitable_start, visitable_end, copy_object_func);
+        if (visitable_end >= space->top() && visitable_end < space->limit()) {
+          visitable_end = space->limit();
+          visitable_start = visitable_end;
+          continue;
+        }
+      }
+      if (visitable_end == space->top() && visitable_end != space->limit()) {
+        visitable_start = visitable_end = space->limit();
+      } else {
+        // At this point we are either at the start of a filler or we are at
+        // the point where the space->top() used to be before the
+        // visit_pointer_region call above.  Either way we can skip the
+        // object at the current spot:  We don't promise to visit objects
+        // allocated during heap traversal, and if space->top() moved then it
+        // must be because an object was allocated at this point.
+        visitable_start =
+            visitable_end + HeapObject::FromAddress(visitable_end)->Size();
+        visitable_end = visitable_start;
+      }
+    } else {
+      ASSERT(o != free_space_map);
+      ASSERT(o != two_pointer_filler_map);
+      ASSERT(visitable_end < space->top() || visitable_end >= space->limit());
+      visitable_end += kPointerSize;
+    }
+  }
+  ASSERT(visitable_end == end_of_page);
+  if (visitable_start != visitable_end) {
+    visit_pointer_region(visitable_start, visitable_end, copy_object_func);
+  }
+}
+
+
 void Heap::IterateRoots(ObjectVisitor* v, VisitMode mode) {
   IterateStrongRoots(v, mode);
   IterateWeakRoots(v, mode);
 }
 
 
 void Heap::IterateWeakRoots(ObjectVisitor* v, VisitMode mode) {
   v->VisitPointer(reinterpret_cast<Object**>(&roots_[kSymbolTableRootIndex]));
   v->Synchronize("symbol_table");
   if (mode != VISIT_ALL_IN_SCAVENGE) {
@@ skipping 1139 matching lines @@
 void ExternalStringTable::TearDown() {
   new_space_strings_.Free();
   old_space_strings_.Free();
 }
 
 
 List<Object*> ExternalStringTable::new_space_strings_;
 List<Object*> ExternalStringTable::old_space_strings_;
 
 } }  // namespace v8::internal