Remove slots that point to unboxed doubles from the StoreBuffer/SlotsBuffer.

src/heap/store-buffer.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <algorithm>
 
 #include "src/v8.h"
 
 #include "src/base/atomicops.h"
 #include "src/counters.h"
@@ skipping 97 matching lines @@
 }
 
 
 void StoreBuffer::Uniq() {
   // Remove adjacent duplicates and cells that do not point at new space.
   Address previous = NULL;
   Address* write = old_start_;
   DCHECK(may_move_store_buffer_entries_);
   for (Address* read = old_start_; read < old_top_; read++) {
     Address current = *read;
+    DCHECK_NOT_NULL(current);
     if (current != previous) {
       Object* object = reinterpret_cast<Object*>(
           base::NoBarrier_Load(reinterpret_cast<base::AtomicWord*>(current)));
       if (heap_->InNewSpace(object)) {
         *write++ = current;
       }
     }
     previous = current;
   }
   old_top_ = write;
@@ skipping 67 matching lines @@
 void StoreBuffer::ExemptPopularPages(int prime_sample_step, int threshold) {
   PointerChunkIterator it(heap_);
   MemoryChunk* chunk;
   while ((chunk = it.next()) != NULL) {
     chunk->set_store_buffer_counter(0);
   }
   bool created_new_scan_on_scavenge_pages = false;
   MemoryChunk* previous_chunk = NULL;
   for (Address* p = old_start_; p < old_top_; p += prime_sample_step) {
     Address addr = *p;
+    if (addr == NULL) continue;
     MemoryChunk* containing_chunk = NULL;
     if (previous_chunk != NULL && previous_chunk->Contains(addr)) {
       containing_chunk = previous_chunk;
     } else {
       containing_chunk = MemoryChunk::FromAnyPointerAddress(heap_, addr);
     }
     int old_counter = containing_chunk->store_buffer_counter();
     if (old_counter >= threshold) {
       containing_chunk->set_scan_on_scavenge(true);
       created_new_scan_on_scavenge_pages = true;
     }
     containing_chunk->set_store_buffer_counter(old_counter + 1);
     previous_chunk = containing_chunk;
   }
   if (created_new_scan_on_scavenge_pages) {
     Filter(MemoryChunk::SCAN_ON_SCAVENGE);
   }
   old_buffer_is_filtered_ = true;
 }
 
 
 void StoreBuffer::Filter(int flag) {
   Address* new_top = old_start_;
   MemoryChunk* previous_chunk = NULL;
   for (Address* p = old_start_; p < old_top_; p++) {
     Address addr = *p;
+    if (addr == NULL) continue;
     MemoryChunk* containing_chunk = NULL;
     if (previous_chunk != NULL && previous_chunk->Contains(addr)) {
       containing_chunk = previous_chunk;
     } else {
       containing_chunk = MemoryChunk::FromAnyPointerAddress(heap_, addr);
       previous_chunk = containing_chunk;
     }
     if (!containing_chunk->IsFlagSet(flag)) {
       *new_top++ = addr;
     }
   }
   old_top_ = new_top;
 
   // Filtering hash sets are inconsistent with the store buffer after this
   // operation.
   ClearFilteringHashSets();
 }
 
 
+void StoreBuffer::RemoveSlots(Address start_address, Address end_address) {
+  struct IsValueInRangePredicate {
+    Address start_address_;
+    Address end_address_;
+
+    IsValueInRangePredicate(Address start_address, Address end_address)
+        : start_address_(start_address), end_address_(end_address) {}
+
+    bool operator()(Address addr) {
+      return start_address_ <= addr && addr < end_address_;
+    }
+  };
+
+  IsValueInRangePredicate predicate(start_address, end_address);
+  const Address kRemovedSlot = NULL;

[Hannes Payer (out of office), 2015/03/05 21:34:43]

The NULL value is not much fun. Maybe we should point it to an address in old
space that does not move. Then we do not have to guard everywhere against NULL
pointer.

[Igor Sheludko, 2015/03/06 09:06:59]

Good point!

+
+  {
+    Address* top = reinterpret_cast<Address*>(heap_->store_buffer_top());
+    std::replace_if(start_, top, predicate, kRemovedSlot);
+  }
+
+  if (old_buffer_is_sorted_) {
+    // Remove slots from an old buffer preserving the order.
+    Address* lower = std::lower_bound(old_start_, old_top_, start_address);
+    if (lower != old_top_) {
+      Address* upper = std::lower_bound(lower, old_top_, end_address);
+      // Remove [lower, upper) from the buffer.
+      if (upper == old_top_) {
+        old_top_ = lower;
+      } else {
+        Address* new_top = lower;
+        for (Address* p = upper; p < old_top_; p++) {
+          *new_top++ = *p;
+        }
+        old_top_ = new_top;

[Hannes Payer (out of office), 2015/03/05 21:34:43]

Why do you set old_top_ here, the old top should remain unchanged because upper
!= old_top_

[Igor Sheludko, 2015/03/06 09:06:59]

Sorry, I didn't get the question. Added more comments to sources.

+      }
+    }
+  } else {
+    std::replace_if(old_start_, old_top_, predicate, kRemovedSlot);
+  }
+}
+
+
 void StoreBuffer::SortUniq() {
   Compact();
   if (old_buffer_is_sorted_) return;
   std::sort(old_start_, old_top_);
   Uniq();
 
   old_buffer_is_sorted_ = true;
 
   // Filtering hash sets are inconsistent with the store buffer after this
   // operation.
@@ skipping 30 matching lines @@
   ClearFilteringHashSets();
   Uniq();  // Also removes things that no longer point to new space.
   EnsureSpace(kStoreBufferSize / 2);
 }
 
 
 static Address* in_store_buffer_1_element_cache = NULL;
 
 
 bool StoreBuffer::CellIsInStoreBuffer(Address cell_address) {
-  if (!FLAG_enable_slow_asserts) return true;
+  DCHECK_NOT_NULL(cell_address);
+  Address* top = reinterpret_cast<Address*>(heap_->store_buffer_top());
   if (in_store_buffer_1_element_cache != NULL &&
       *in_store_buffer_1_element_cache == cell_address) {
-    return true;
+    // Check if the cache still points into the active part of the buffer.
+    if ((start_ <= in_store_buffer_1_element_cache &&
+         in_store_buffer_1_element_cache < top) ||
+        (old_start_ <= in_store_buffer_1_element_cache &&
+         in_store_buffer_1_element_cache < old_top_)) {
+      return true;
+    }
   }
-  Address* top = reinterpret_cast<Address*>(heap_->store_buffer_top());
   for (Address* current = top - 1; current >= start_; current--) {
     if (*current == cell_address) {
       in_store_buffer_1_element_cache = current;
       return true;
     }
   }
   for (Address* current = old_top_ - 1; current >= old_start_; current--) {
     if (*current == cell_address) {
       in_store_buffer_1_element_cache = current;
       return true;
@@ skipping 101 matching lines @@
 }
 
 
 void StoreBuffer::IteratePointersInStoreBuffer(ObjectSlotCallback slot_callback,
                                                bool clear_maps) {
   Address* limit = old_top_;
   old_top_ = old_start_;
   {
     DontMoveStoreBufferEntriesScope scope(this);
     for (Address* current = old_start_; current < limit; current++) {
+      Address addr = *current;
+      if (addr == NULL) continue;
 #ifdef DEBUG
       Address* saved_top = old_top_;
 #endif
-      ProcessOldToNewSlot(*current, slot_callback, clear_maps);
+      ProcessOldToNewSlot(addr, slot_callback, clear_maps);
       DCHECK(old_top_ == saved_top + 1 || old_top_ == saved_top);
     }
   }
 }
 
 
 void StoreBuffer::IteratePointersToNewSpace(ObjectSlotCallback slot_callback) {
   IteratePointersToNewSpace(slot_callback, false);
 }
 
@@ skipping 134 matching lines @@
   DCHECK(top <= limit_);
   heap_->public_set_store_buffer_top(start_);
   EnsureSpace(top - start_);
   DCHECK(may_move_store_buffer_entries_);
   // Goes through the addresses in the store buffer attempting to remove
   // duplicates.  In the interest of speed this is a lossy operation.  Some
   // duplicates will remain.  We have two hash sets with different hash
   // functions to reduce the number of unnecessary clashes.
   hash_sets_are_empty_ = false;  // Hash sets are in use.
   for (Address* current = start_; current < top; current++) {
-    DCHECK(!heap_->cell_space()->Contains(*current));
-    DCHECK(!heap_->code_space()->Contains(*current));
-    DCHECK(!heap_->old_data_space()->Contains(*current));
-    uintptr_t int_addr = reinterpret_cast<uintptr_t>(*current);
+    Address addr = *current;
+    if (addr == NULL) continue;
+    DCHECK(!heap_->cell_space()->Contains(addr));
+    DCHECK(!heap_->code_space()->Contains(addr));
+    DCHECK(!heap_->old_data_space()->Contains(addr));
+    uintptr_t int_addr = reinterpret_cast<uintptr_t>(addr);
     // Shift out the last bits including any tags.
     int_addr >>= kPointerSizeLog2;
     // The upper part of an address is basically random because of ASLR and OS
     // non-determinism, so we use only the bits within a page for hashing to
     // make v8's behavior (more) deterministic.
     uintptr_t hash_addr =
         int_addr & (Page::kPageAlignmentMask >> kPointerSizeLog2);
     int hash1 = ((hash_addr ^ (hash_addr >> kHashSetLengthLog2)) &
                  (kHashSetLength - 1));
     if (hash_set_1_[hash1] == int_addr) continue;
@@ skipping 13 matching lines @@
     }
     old_buffer_is_sorted_ = false;
     old_buffer_is_filtered_ = false;
     *old_top_++ = reinterpret_cast<Address>(int_addr << kPointerSizeLog2);
     DCHECK(old_top_ <= old_limit_);
   }
   heap_->isolate()->counters()->store_buffer_compactions()->Increment();
 }
 }
 }  // namespace v8::internal