Avoid scan-on-scavenge pages during full collection.

src/store-buffer.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/store-buffer.h"
 
 #include <algorithm>
 
 #include "src/v8.h"
 
 #include "src/base/atomicops.h"
 #include "src/counters.h"
 #include "src/store-buffer-inl.h"
 
 namespace v8 {
 namespace internal {
 
 StoreBuffer::StoreBuffer(Heap* heap)
     : heap_(heap),
       start_(NULL),
       limit_(NULL),
       old_start_(NULL),
       old_limit_(NULL),
       old_top_(NULL),
+      old_regular_limit_(NULL),
       old_reserved_limit_(NULL),
+      old_virtual_memory_(NULL),
+      old_store_buffer_length_(0),
       old_buffer_is_sorted_(false),
       old_buffer_is_filtered_(false),
-      during_gc_(false),
+      allow_overflow_(false),
       store_buffer_rebuilding_enabled_(false),
       callback_(NULL),
       may_move_store_buffer_entries_(true),
       virtual_memory_(NULL),
       hash_set_1_(NULL),
       hash_set_2_(NULL),
       hash_sets_are_empty_(true) {
 }
 
 
 void StoreBuffer::SetUp() {
   virtual_memory_ = new base::VirtualMemory(kStoreBufferSize * 3);
   uintptr_t start_as_int =
       reinterpret_cast<uintptr_t>(virtual_memory_->address());
   start_ =
       reinterpret_cast<Address*>(RoundUp(start_as_int, kStoreBufferSize * 2));
   limit_ = start_ + (kStoreBufferSize / kPointerSize);
 
+  // We set the maximum store buffer size to the maximum size of a semi-space.
+  // The store buffer may reach this limit during a full garbage collection.
+  // Note that half of the semi-space should be good enough since half of the
+  // memory in the semi-space are not object pointers.
+  old_store_buffer_length_ = heap_->MaxSemiSpaceSize() / sizeof(Address);
+
   old_virtual_memory_ =
-      new base::VirtualMemory(kOldStoreBufferLength * kPointerSize);
+      new base::VirtualMemory(old_store_buffer_length_ * kPointerSize);
   old_top_ = old_start_ =
       reinterpret_cast<Address*>(old_virtual_memory_->address());
   // Don't know the alignment requirements of the OS, but it is certainly not
   // less than 0xfff.
   ASSERT((reinterpret_cast<uintptr_t>(old_start_) & 0xfff) == 0);
   int initial_length =
       static_cast<int>(base::OS::CommitPageSize() / kPointerSize);
   ASSERT(initial_length > 0);
-  ASSERT(initial_length <= kOldStoreBufferLength);
+  ASSERT(initial_length <= kOldRegularStoreBufferLength);
+  ASSERT(initial_length <= old_store_buffer_length_);
+  ASSERT(kOldRegularStoreBufferLength <= old_store_buffer_length_);
   old_limit_ = old_start_ + initial_length;
-  old_reserved_limit_ = old_start_ + kOldStoreBufferLength;
+  old_regular_limit_ = old_start_ + kOldRegularStoreBufferLength;
+  old_reserved_limit_ = old_start_ + old_store_buffer_length_;
 
   CHECK(old_virtual_memory_->Commit(
             reinterpret_cast<void*>(old_start_),
             (old_limit_ - old_start_) * kPointerSize,
             false));
 
   ASSERT(reinterpret_cast<Address>(start_) >= virtual_memory_->address());
   ASSERT(reinterpret_cast<Address>(limit_) >= virtual_memory_->address());
   Address* vm_limit = reinterpret_cast<Address*>(
       reinterpret_cast<char*>(virtual_memory_->address()) +
@@ skipping 16 matching lines @@
 
   ClearFilteringHashSets();
 }
 
 
 void StoreBuffer::TearDown() {
   delete virtual_memory_;
   delete old_virtual_memory_;
   delete[] hash_set_1_;
   delete[] hash_set_2_;
-  old_start_ = old_top_ = old_limit_ = old_reserved_limit_ = NULL;
-  start_ = limit_ = NULL;
+  old_start_ = NULL;
+  old_top_ = NULL;
+  old_limit_ = NULL;
+  old_reserved_limit_ = NULL;
+  old_regular_limit_ = NULL;
+  start_ = NULL;
+  limit_ = NULL;
   heap_->public_set_store_buffer_top(start_);
 }
 
 
 void StoreBuffer::StoreBufferOverflow(Isolate* isolate) {
   isolate->heap()->store_buffer()->Compact();
   isolate->counters()->store_buffer_overflows()->Increment();
 }
 
 
@@ skipping 13 matching lines @@
   }
   old_top_ = write;
 }
 
 
 bool StoreBuffer::SpaceAvailable(intptr_t space_needed) {
   return old_limit_ - old_top_ >= space_needed;
 }
 
 
+template<StoreBuffer::ExemptPopularPagesMode mode>
+void StoreBuffer::IterativelyExemptPopularPages(intptr_t space_needed) {
+  // Sample 1 entry in 97 and filter out the pages where we estimate that more
+  // than 1 in 8 pointers are to new space.
+  static const int kSampleFinenesses = 5;
+  static const struct Samples {
+    int prime_sample_step;
+    int threshold;
+  } samples[kSampleFinenesses] =  {
+    { 97, ((Page::kPageSize / kPointerSize) / 97) / 8 },
+    { 23, ((Page::kPageSize / kPointerSize) / 23) / 16 },
+    { 7, ((Page::kPageSize / kPointerSize) / 7) / 32 },
+    { 3, ((Page::kPageSize / kPointerSize) / 3) / 256 },
+    { 1, 0}
+  };
+  for (int i = 0; i < kSampleFinenesses; i++) {
+    ExemptPopularPages(samples[i].prime_sample_step, samples[i].threshold);
+    // As a last resort we mark all pages as being exempt from the store buffer.
+    ASSERT(i != (kSampleFinenesses - 1) || old_top_ == old_start_);
+    if (mode == ENSURE_SPACE && SpaceAvailable(space_needed)) return;
+    else if (mode == SHRINK_TO_REGULAR_SIZE && old_top_ < old_limit_) return;
+  }
+}
+
+
 void StoreBuffer::EnsureSpace(intptr_t space_needed) {
   while (old_limit_ - old_top_ < space_needed &&
-         old_limit_ < old_reserved_limit_) {
+      ((!allow_overflow_ && old_limit_ < old_regular_limit_) ||
+          (allow_overflow_ && old_limit_ < old_reserved_limit_))) {
     size_t grow = old_limit_ - old_start_;  // Double size.
     CHECK(old_virtual_memory_->Commit(reinterpret_cast<void*>(old_limit_),
                                       grow * kPointerSize,
                                       false));
     old_limit_ += grow;
   }
 
   if (SpaceAvailable(space_needed)) return;
 
   if (old_buffer_is_filtered_) return;
@@ skipping 11 matching lines @@
       break;
     }
   }
 
   if (page_has_scan_on_scavenge_flag) {
     Filter(MemoryChunk::SCAN_ON_SCAVENGE);
   }
 
   if (SpaceAvailable(space_needed)) return;
 
-  // Sample 1 entry in 97 and filter out the pages where we estimate that more
-  // than 1 in 8 pointers are to new space.
-  static const int kSampleFinenesses = 5;
-  static const struct Samples {
-    int prime_sample_step;
-    int threshold;
-  } samples[kSampleFinenesses] =  {
-    { 97, ((Page::kPageSize / kPointerSize) / 97) / 8 },
-    { 23, ((Page::kPageSize / kPointerSize) / 23) / 16 },
-    { 7, ((Page::kPageSize / kPointerSize) / 7) / 32 },
-    { 3, ((Page::kPageSize / kPointerSize) / 3) / 256 },
-    { 1, 0}
-  };
-  for (int i = 0; i < kSampleFinenesses; i++) {
-    ExemptPopularPages(samples[i].prime_sample_step, samples[i].threshold);
-    // As a last resort we mark all pages as being exempt from the store buffer.
-    ASSERT(i != (kSampleFinenesses - 1) || old_top_ == old_start_);
-    if (SpaceAvailable(space_needed)) return;
-  }
-  UNREACHABLE();
+  IterativelyExemptPopularPages<ENSURE_SPACE>(space_needed);
+  ASSERT(SpaceAvailable(space_needed));
 }
 
 
 // Sample the store buffer to see if some pages are taking up a lot of space
 // in the store buffer.
 void StoreBuffer::ExemptPopularPages(int prime_sample_step, int threshold) {
   PointerChunkIterator it(heap_);
   MemoryChunk* chunk;
   while ((chunk = it.next()) != NULL) {
     chunk->set_store_buffer_counter(0);
@@ skipping 126 matching lines @@
            0,
            sizeof(uintptr_t) * kHashSetLength);
     memset(reinterpret_cast<void*>(hash_set_2_),
            0,
            sizeof(uintptr_t) * kHashSetLength);
     hash_sets_are_empty_ = true;
   }
 }
 
 
-void StoreBuffer::GCPrologue() {
+void StoreBuffer::GCPrologue(bool allow_overflow) {
   ClearFilteringHashSets();
-  during_gc_ = true;
+  allow_overflow_ = allow_overflow;
 }
 
 
 #ifdef VERIFY_HEAP
 void StoreBuffer::VerifyPointers(LargeObjectSpace* space) {
   LargeObjectIterator it(space);
   for (HeapObject* object = it.Next(); object != NULL; object = it.Next()) {
     if (object->IsFixedArray()) {
       Address slot_address = object->address();
       Address end = object->address() + object->Size();
@@ skipping 15 matching lines @@
 
 
 void StoreBuffer::Verify() {
 #ifdef VERIFY_HEAP
   VerifyPointers(heap_->lo_space());
 #endif
 }
 
 
 void StoreBuffer::GCEpilogue() {
-  during_gc_ = false;
+  if (allow_overflow_ && old_limit_ > old_regular_limit_) {
+    IterativelyExemptPopularPages<SHRINK_TO_REGULAR_SIZE>(0);
+    ASSERT(old_limit_ < old_regular_limit_);
+    old_virtual_memory_->Uncommit(old_limit_, old_regular_limit_ - old_limit_);
+  }
+
+  allow_overflow_ = false;
 #ifdef VERIFY_HEAP
   if (FLAG_verify_heap) {
     Verify();
   }
 #endif
 }
 
 
 void StoreBuffer::FindPointersToNewSpaceInRegion(
     Address start,
@@ skipping 185 matching lines @@
     }
     old_buffer_is_sorted_ = false;
     old_buffer_is_filtered_ = false;
     *old_top_++ = reinterpret_cast<Address>(int_addr << kPointerSizeLog2);
     ASSERT(old_top_ <= old_limit_);
   }
   heap_->isolate()->counters()->store_buffer_compactions()->Increment();
 }
 
 } }  // namespace v8::internal