VERIFY_PREDICTABLE build mode introduced. Shadow pages support added.

src/spaces.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "v8.h"
 
 #include "full-codegen.h"
 #include "macro-assembler.h"
 #include "mark-compact.h"
 #include "msan.h"
@@ skipping 257 matching lines @@
       capacity_(0),
       capacity_executable_(0),
       size_(0),
       size_executable_(0),
       lowest_ever_allocated_(reinterpret_cast<void*>(-1)),
       highest_ever_allocated_(reinterpret_cast<void*>(0)) {
 }
 
 
 bool MemoryAllocator::SetUp(intptr_t capacity, intptr_t capacity_executable) {
+  if (FLAG_shadow_pages) {
+    // We need up to two times more non-executable memory for shadow pages.
+    capacity *= 2;
+  }
   capacity_ = RoundUp(capacity, Page::kPageSize);
   capacity_executable_ = RoundUp(capacity_executable, Page::kPageSize);
   ASSERT_GE(capacity_, capacity_executable_);
 
   size_ = 0;
   size_executable_ = 0;
 
   return true;
 }
 
@@ skipping 171 matching lines @@
   MemoryChunk* chunk = FromAddress(base);
 
   ASSERT(base == chunk->address());
 
   chunk->heap_ = heap;
   chunk->size_ = size;
   chunk->area_start_ = area_start;
   chunk->area_end_ = area_end;
   chunk->flags_ = 0;
   chunk->set_owner(owner);
+  chunk->shadow_chunk_ = NULL;
+  chunk->shadow_data_offset_ = 0;
+  chunk->large_object_shadow_data_ = reinterpret_cast<intptr_t>(kZapValue);
   chunk->InitializeReservedMemory();
   chunk->slots_buffer_ = NULL;
   chunk->skip_list_ = NULL;
   chunk->write_barrier_counter_ = kWriteBarrierCounterGranularity;
   chunk->progress_bar_ = 0;
   chunk->high_water_mark_ = static_cast<int>(area_start - base);
   chunk->set_parallel_sweeping(PARALLEL_SWEEPING_DONE);
   chunk->available_in_small_free_list_ = 0;
   chunk->available_in_medium_free_list_ = 0;
   chunk->available_in_large_free_list_ = 0;
@@ skipping 215 matching lines @@
                                                 area_start,
                                                 area_end,
                                                 executable,
                                                 owner);
   result->set_reserved_memory(&reservation);
   MSAN_MEMORY_IS_INITIALIZED_IN_JIT(base, chunk_size);
   return result;
 }
 
 
+bool MemoryAllocator::AllocateShadowChunkFor(MemoryChunk* chunk) {
+  ASSERT(FLAG_shadow_pages);
+  ASSERT(chunk->shadow_chunk() == NULL);
+
+  int size =
+      chunk->area_end() - (chunk->address() + MemoryChunk::kObjectStartOffset);
+  MemoryChunk* shadow = AllocateChunk(size, size, NOT_EXECUTABLE, NULL);
+  if (shadow == NULL) {
+    return false;
+  }
+  ASSERT((chunk->area_end() - chunk->address()) ==
+         (shadow->area_end() - shadow->address()));
+  chunk->set_shadow_chunk(shadow);
+  chunk->shadow_data_offset_ = shadow->address() - chunk->address();
+  ASSERT(IsAligned(chunk->shadow_data_offset_, Page::kPageSize));
+
+  return true;
+}
+
+
+void MemoryAllocator::FreeShadowChunkFor(MemoryChunk* chunk) {
+  ASSERT(FLAG_shadow_pages);
+
+  if (chunk->shadow_chunk() != NULL) {
+    Free(chunk->shadow_chunk());
+    chunk->set_shadow_chunk(NULL);
+  }
+}
+
+
 void Page::ResetFreeListStatistics() {
   non_available_small_blocks_ = 0;
   available_in_small_free_list_ = 0;
   available_in_medium_free_list_ = 0;
   available_in_large_free_list_ = 0;
   available_in_huge_free_list_ = 0;
 }
 
 
 Page* MemoryAllocator::AllocatePage(intptr_t size,
                                     PagedSpace* owner,
                                     Executability executable) {
   MemoryChunk* chunk = AllocateChunk(size, size, executable, owner);
 
   if (chunk == NULL) return NULL;
 
-  return Page::Initialize(isolate_->heap(), chunk, executable, owner);
+  Page* p = Page::Initialize(isolate_->heap(), chunk, executable, owner);
+  if (FLAG_shadow_pages) {
+    if (!AllocateShadowChunkFor(p)) {
+      Free(p);
+      return NULL;
+    }
+  }
+  return p;
 }
 
 
 LargePage* MemoryAllocator::AllocateLargePage(intptr_t object_size,
                                               Space* owner,
                                               Executability executable) {
   MemoryChunk* chunk = AllocateChunk(object_size,
                                      object_size,
                                      executable,
                                      owner);
   if (chunk == NULL) return NULL;
-  return LargePage::Initialize(isolate_->heap(), chunk);
+
+  LargePage* p = LargePage::Initialize(isolate_->heap(), chunk);
+
+  if (FLAG_shadow_pages) {
+    // Instead of allocating shadow chunk we initialize shadow offset so that
+    // it points directly into p.large_object_shadow_data_.
+    p->shadow_data_offset_ =
+        reinterpret_cast<Address>(&p->large_object_shadow_data_) -
+        p->GetObject()->address();
+  }
+  return p;
 }
 
 
 void MemoryAllocator::Free(MemoryChunk* chunk) {
   LOG(isolate_, DeleteEvent("MemoryChunk", chunk));
   if (chunk->owner() != NULL) {
     ObjectSpace space =
         static_cast<ObjectSpace>(1 << chunk->owner()->identity());
     PerformAllocationCallback(space, kAllocationActionFree, chunk->size());
   }
 
   isolate_->heap()->RememberUnmappedPage(
       reinterpret_cast<Address>(chunk), chunk->IsEvacuationCandidate());
 
   delete chunk->slots_buffer();
   delete chunk->skip_list();
 
+  if (FLAG_shadow_pages) {
+    FreeShadowChunkFor(chunk);
+  }
+
   VirtualMemory* reservation = chunk->reserved_memory();
   if (reservation->IsReserved()) {
     FreeMemory(reservation, chunk->executable());
   } else {
     FreeMemory(chunk->address(),
                chunk->size(),
                chunk->executable());
   }
 }
 
@@ skipping 741 matching lines @@
   maximum_committed_ = 0;
   committed_ = false;
   start_ = start;
   address_mask_ = ~(maximum_capacity - 1);
   object_mask_ = address_mask_ | kHeapObjectTagMask;
   object_expected_ = reinterpret_cast<uintptr_t>(start) | kHeapObjectTag;
   age_mark_ = start_;
 }
 
 
+bool SemiSpace::AllocateShadowChunksForPages(int start_index, int end_index) {
+  ASSERT(FLAG_shadow_pages);
+
+  MemoryAllocator* allocator = heap()->isolate()->memory_allocator();
+
+  for (int i = start_index; i < end_index; i++) {
+    NewSpacePage* page =
+        NewSpacePage::FromAddress(start_ + i * Page::kPageSize);
+
+    if (!allocator->AllocateShadowChunkFor(page)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+
+void SemiSpace::FreeShadowChunksForPages(int start_index, int end_index) {
+  ASSERT(FLAG_shadow_pages);
+  MemoryAllocator* allocator = heap()->isolate()->memory_allocator();
+
+  for (int i = start_index; i < end_index; i++) {
+    NewSpacePage* page =
+        NewSpacePage::FromAddress(start_ + i * Page::kPageSize);
+
+    allocator->FreeShadowChunkFor(page);
+  }
+}
+
+
 void SemiSpace::TearDown() {
+  if (FLAG_shadow_pages && is_committed()) {
+    int pages = capacity_ / Page::kPageSize;
+    FreeShadowChunksForPages(0, pages);
+  }
   start_ = NULL;
   capacity_ = 0;
 }
 
 
 bool SemiSpace::Commit() {
   ASSERT(!is_committed());
   int pages = capacity_ / Page::kPageSize;
   if (!heap()->isolate()->memory_allocator()->CommitBlock(start_,
                                                           capacity_,
                                                           executable())) {
     return false;
   }
 
   NewSpacePage* current = anchor();
   for (int i = 0; i < pages; i++) {
     NewSpacePage* new_page =
       NewSpacePage::Initialize(heap(), start_ + i * Page::kPageSize, this);
     new_page->InsertAfter(current);
     current = new_page;
   }
+  if (FLAG_shadow_pages) {
+    if (!AllocateShadowChunksForPages(0, pages)) {
+      return false;
+    }
+  }
 
   SetCapacity(capacity_);
   committed_ = true;
   Reset();
   return true;
 }
 
 
 bool SemiSpace::Uncommit() {
   ASSERT(is_committed());
+
+  if (FLAG_shadow_pages) {
+    int pages_before = capacity_ / Page::kPageSize;
+    FreeShadowChunksForPages(0, pages_before);
+  }
+
   Address start = start_ + maximum_capacity_ - capacity_;
   if (!heap()->isolate()->memory_allocator()->UncommitBlock(start, capacity_)) {
     return false;
   }
   anchor()->set_next_page(anchor());
   anchor()->set_prev_page(anchor());
 
   committed_ = false;
   return true;
 }
@@ skipping 35 matching lines @@
     NewSpacePage* new_page = NewSpacePage::Initialize(heap(),
                                                       page_address,
                                                       this);
     new_page->InsertAfter(last_page);
     Bitmap::Clear(new_page);
     // Duplicate the flags that was set on the old page.
     new_page->SetFlags(last_page->GetFlags(),
                        NewSpacePage::kCopyOnFlipFlagsMask);
     last_page = new_page;
   }
+
+  if (FLAG_shadow_pages) {
+    if (!AllocateShadowChunksForPages(pages_before, pages_after)) {
+      return false;
+    }
+  }
+
   return true;
 }
 
 
 bool SemiSpace::ShrinkTo(int new_capacity) {
   ASSERT((new_capacity & Page::kPageAlignmentMask) == 0);
   ASSERT(new_capacity >= initial_capacity_);
   ASSERT(new_capacity < capacity_);
   if (is_committed()) {
     size_t delta = capacity_ - new_capacity;
     ASSERT(IsAligned(delta, OS::AllocateAlignment()));
 
     MemoryAllocator* allocator = heap()->isolate()->memory_allocator();
+    int pages_after = new_capacity / Page::kPageSize;
+
+    if (FLAG_shadow_pages) {
+      int pages_before = capacity_ / Page::kPageSize;
+      FreeShadowChunksForPages(pages_after, pages_before);
+    }
+
     if (!allocator->UncommitBlock(start_ + new_capacity, delta)) {
       return false;
     }
 
-    int pages_after = new_capacity / Page::kPageSize;
     NewSpacePage* new_last_page =
         NewSpacePage::FromAddress(start_ + (pages_after - 1) * Page::kPageSize);
     new_last_page->set_next_page(anchor());
     anchor()->set_prev_page(new_last_page);
     ASSERT((current_page_ >= first_page()) && (current_page_ <= new_last_page));
   }
 
   SetCapacity(new_capacity);
 
   return true;
@@ skipping 1485 matching lines @@
     object->ShortPrint();
     PrintF("\n");
   }
   printf(" --------------------------------------\n");
   printf(" Marked: %x, LiveCount: %x\n", mark_size, LiveBytes());
 }
 
 #endif  // DEBUG
 
 } }  // namespace v8::internal