Cardmarking writebarrier.

src/spaces.cc

 // Copyright 2006-2008 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 23 matching lines @@
 namespace v8 {
 namespace internal {
 
 // For contiguous spaces, top should be in the space (or at the end) and limit
 // should be the end of the space.
 #define ASSERT_SEMISPACE_ALLOCATION_INFO(info, space) \
   ASSERT((space).low() <= (info).top                  \
          && (info).top <= (space).high()              \
          && (info).limit == (space).high())
 
+intptr_t Page::watermark_invalidated_mark_ = Page::WATERMARK_INVALIDATED;
 
 // ----------------------------------------------------------------------------
 // HeapObjectIterator
 
 HeapObjectIterator::HeapObjectIterator(PagedSpace* space) {
   Initialize(space->bottom(), space->top(), NULL);
 }
 
 
 HeapObjectIterator::HeapObjectIterator(PagedSpace* space,
@@ skipping 78 matching lines @@
         }
       }
 #endif
       stop_page_ = space->last_page_;
       break;
   }
 }
 
 
 // -----------------------------------------------------------------------------
-// Page
-
-#ifdef DEBUG
-Page::RSetState Page::rset_state_ = Page::IN_USE;
-#endif
-
-// -----------------------------------------------------------------------------
 // CodeRange
 
 List<CodeRange::FreeBlock> CodeRange::free_list_(0);
 List<CodeRange::FreeBlock> CodeRange::allocation_list_(0);
 int CodeRange::current_allocation_block_index_ = 0;
 VirtualMemory* CodeRange::code_range_ = NULL;
 
 
 bool CodeRange::Setup(const size_t requested) {
   ASSERT(code_range_ == NULL);
@@ skipping 358 matching lines @@
   size_t chunk_size = chunks_[chunk_id].size();
   Address high = RoundDown(chunk_start + chunk_size, Page::kPageSize);
   ASSERT(pages_in_chunk <=
         ((OffsetFrom(high) - OffsetFrom(low)) / Page::kPageSize));
 #endif
 
   Address page_addr = low;
   for (int i = 0; i < pages_in_chunk; i++) {
     Page* p = Page::FromAddress(page_addr);
     p->opaque_header = OffsetFrom(page_addr + Page::kPageSize) | chunk_id;
+    p->InvalidateWatermark(true);
     p->SetIsLargeObjectPage(false);
+    p->SetAllocationWatermark(p->ObjectAreaStart());
+    p->SetCachedAllocationWatermark(p->ObjectAreaStart());
     page_addr += Page::kPageSize;
   }
 
   // Set the next page of the last page to 0.
   Page* last_page = Page::FromAddress(page_addr - Page::kPageSize);
   last_page->opaque_header = OffsetFrom(0) | chunk_id;
 
   return Page::FromAddress(low);
 }
 
@@ skipping 136 matching lines @@
 
   if (prev->is_valid()) {
     SetNextPage(prev, Page::FromAddress(page_addr));
   }
 
   for (int i = 0; i < pages_in_chunk; i++) {
     Page* p = Page::FromAddress(page_addr);
     p->opaque_header = OffsetFrom(page_addr + Page::kPageSize) | chunk_id;
     page_addr += Page::kPageSize;
 
+    p->InvalidateWatermark(true);
     if (p->WasInUseBeforeMC()) {
       *last_page_in_use = p;
     }
   }
 
   // Set the next page of the last page to 0.
   Page* last_page = Page::FromAddress(page_addr - Page::kPageSize);
   last_page->opaque_header = OffsetFrom(0) | chunk_id;
 
   if (last_page->WasInUseBeforeMC()) {
@@ skipping 43 matching lines @@
     if (!first_page_->is_valid()) return false;
   }
 
   // We are sure that the first page is valid and that we have at least one
   // page.
   ASSERT(first_page_->is_valid());
   ASSERT(num_pages > 0);
   accounting_stats_.ExpandSpace(num_pages * Page::kObjectAreaSize);
   ASSERT(Capacity() <= max_capacity_);
 
-  // Sequentially initialize remembered sets in the newly allocated
+  // Sequentially clear region marks in the newly allocated
   // pages and cache the current last page in the space.
   for (Page* p = first_page_; p->is_valid(); p = p->next_page()) {
-    p->ClearRSet();
+    p->SetRegionMarks(Page::kAllRegionsCleanMarks);
     last_page_ = p;
   }
 
   // Use first_page_ for allocation.
   SetAllocationInfo(&allocation_info_, first_page_);
 
   page_list_is_chunk_ordered_ = true;
 
   return true;
 }
@@ skipping 26 matching lines @@
   Page* page = first_page_;
   while (page->is_valid()) {
     MemoryAllocator::UnprotectChunkFromPage(page);
     page = MemoryAllocator::FindLastPageInSameChunk(page)->next_page();
   }
 }
 
 #endif
 
 
-void PagedSpace::ClearRSet() {
+void PagedSpace::MarkAllPagesClean() {
   PageIterator it(this, PageIterator::ALL_PAGES);
   while (it.has_next()) {
-    it.next()->ClearRSet();
+    it.next()->SetRegionMarks(Page::kAllRegionsCleanMarks);
   }
 }
 
 
 Object* PagedSpace::FindObject(Address addr) {
   // Note: this function can only be called before or after mark-compact GC
   // because it accesses map pointers.
   ASSERT(!MarkCompactCollector::in_use());
 
   if (!Contains(addr)) return Failure::Exception();
@@ skipping 82 matching lines @@
   ASSERT(current_page->next_page()->is_valid());
   // We do not add the top of page block for current page to the space's
   // free list---the block may contain live objects so we cannot write
   // bookkeeping information to it.  Instead, we will recover top of page
   // blocks when we move objects to their new locations.
   //
   // We do however write the allocation pointer to the page.  The encoding
   // of forwarding addresses is as an offset in terms of live bytes, so we
   // need quick access to the allocation top of each page to decode
   // forwarding addresses.
-  current_page->mc_relocation_top = mc_forwarding_info_.top;
+  current_page->SetAllocationWatermark(mc_forwarding_info_.top);
+  current_page->next_page()->InvalidateWatermark(true);
   SetAllocationInfo(&mc_forwarding_info_, current_page->next_page());
   return AllocateLinearly(&mc_forwarding_info_, size_in_bytes);
 }
 
 
 bool PagedSpace::Expand(Page* last_page) {
   ASSERT(max_capacity_ % Page::kObjectAreaSize == 0);
   ASSERT(Capacity() % Page::kObjectAreaSize == 0);
 
   if (Capacity() == max_capacity_) return false;
 
   ASSERT(Capacity() < max_capacity_);
   // Last page must be valid and its next page is invalid.
   ASSERT(last_page->is_valid() && !last_page->next_page()->is_valid());
 
   int available_pages = (max_capacity_ - Capacity()) / Page::kObjectAreaSize;
   if (available_pages <= 0) return false;
 
   int desired_pages = Min(available_pages, MemoryAllocator::kPagesPerChunk);
   Page* p = MemoryAllocator::AllocatePages(desired_pages, &desired_pages, this);
   if (!p->is_valid()) return false;
 
   accounting_stats_.ExpandSpace(desired_pages * Page::kObjectAreaSize);
   ASSERT(Capacity() <= max_capacity_);
 
   MemoryAllocator::SetNextPage(last_page, p);
 
-  // Sequentially clear remembered set of new pages and and cache the
+  // Sequentially clear region marks of new pages and and cache the
   // new last page in the space.
   while (p->is_valid()) {
-    p->ClearRSet();
+    p->SetRegionMarks(Page::kAllRegionsCleanMarks);
     last_page_ = p;
     p = p->next_page();
   }
 
   return true;
 }
 
 
 #ifdef DEBUG
 int PagedSpace::CountTotalPages() {
@@ skipping 78 matching lines @@
   Page* top_page = Page::FromAllocationTop(allocation_info_.top);
   ASSERT(MemoryAllocator::IsPageInSpace(top_page, this));
 
   // Loop over all the pages.
   bool above_allocation_top = false;
   Page* current_page = first_page_;
   while (current_page->is_valid()) {
     if (above_allocation_top) {
       // We don't care what's above the allocation top.
     } else {
-      // Unless this is the last page in the space containing allocated
-      // objects, the allocation top should be at a constant offset from the
-      // object area end.
       Address top = current_page->AllocationTop();
       if (current_page == top_page) {
         ASSERT(top == allocation_info_.top);
         // The next page will be above the allocation top.
         above_allocation_top = true;
-      } else {
-        ASSERT(top == PageAllocationLimit(current_page));
       }
 
       // It should be packed with objects from the bottom to the top.
       Address current = current_page->ObjectAreaStart();
       while (current < top) {
         HeapObject* object = HeapObject::FromAddress(current);
 
         // The first word should be a map, and we expect all map pointers to
         // be in map space.
         Map* map = object->map();
         ASSERT(map->IsMap());
         ASSERT(Heap::map_space()->Contains(map));
 
         // Perform space-specific object verification.
         VerifyObject(object);
 
         // The object itself should look OK.
         object->Verify();
 
         // All the interior pointers should be contained in the heap and
-        // have their remembered set bits set if required as determined
-        // by the visitor.
+        // have page regions covering intergenerational references should be
+        // marked dirty.
         int size = object->Size();
         object->IterateBody(map->instance_type(), size, visitor);
 
         current += size;
       }
 
       // The allocation pointer should not be in the middle of an object.
       ASSERT(current == top);
     }
 
@@ skipping 38 matching lines @@
     return false;
   }
   if (!from_space_.Setup(start + maximum_semispace_capacity,
                          initial_semispace_capacity,
                          maximum_semispace_capacity)) {
     return false;
   }
 
   start_ = start;
   address_mask_ = ~(size - 1);
-  object_mask_ = address_mask_ | kHeapObjectTag;
+  object_mask_ = address_mask_ | kHeapObjectTagMask;
   object_expected_ = reinterpret_cast<uintptr_t>(start) | kHeapObjectTag;
 
   allocation_info_.top = to_space_.low();
   allocation_info_.limit = to_space_.high();
   mc_forwarding_info_.top = NULL;
   mc_forwarding_info_.limit = NULL;
 
   ASSERT_SEMISPACE_ALLOCATION_INFO(allocation_info_, to_space_);
   return true;
 }
@@ skipping 183 matching lines @@
   // otherwise.  In the mark-compact collector, the memory region of the from
   // space is used as the marking stack. It requires contiguous memory
   // addresses.
   initial_capacity_ = initial_capacity;
   capacity_ = initial_capacity;
   maximum_capacity_ = maximum_capacity;
   committed_ = false;
 
   start_ = start;
   address_mask_ = ~(maximum_capacity - 1);
-  object_mask_ = address_mask_ | kHeapObjectTag;
+  object_mask_ = address_mask_ | kHeapObjectTagMask;
   object_expected_ = reinterpret_cast<uintptr_t>(start) | kHeapObjectTag;
   age_mark_ = start_;
 
   return Commit();
 }
 
 
 void SemiSpace::TearDown() {
   start_ = NULL;
   capacity_ = 0;
@@ skipping 289 matching lines @@
   ASSERT(size_in_bytes > 0);
   ASSERT(IsAligned(size_in_bytes, kPointerSize));
 
   // We write a map and possibly size information to the block.  If the block
   // is big enough to be a ByteArray with at least one extra word (the next
   // pointer), we set its map to be the byte array map and its size to an
   // appropriate array length for the desired size from HeapObject::Size().
   // If the block is too small (eg, one or two words), to hold both a size
   // field and a next pointer, we give it a filler map that gives it the
   // correct size.
-  if (size_in_bytes > ByteArray::kAlignedSize) {
+  if (size_in_bytes > ByteArray::kHeaderSize) {
     set_map(Heap::raw_unchecked_byte_array_map());
     // Can't use ByteArray::cast because it fails during deserialization.
     ByteArray* this_as_byte_array = reinterpret_cast<ByteArray*>(this);
     this_as_byte_array->set_length(ByteArray::LengthFor(size_in_bytes));
   } else if (size_in_bytes == kPointerSize) {
     set_map(Heap::raw_unchecked_one_pointer_filler_map());
   } else if (size_in_bytes == 2 * kPointerSize) {
     set_map(Heap::raw_unchecked_two_pointer_filler_map());
   } else {
     UNREACHABLE();
@@ skipping 176 matching lines @@
 
 
 FixedSizeFreeList::FixedSizeFreeList(AllocationSpace owner, int object_size)
     : owner_(owner), object_size_(object_size) {
   Reset();
 }
 
 
 void FixedSizeFreeList::Reset() {
   available_ = 0;
-  head_ = NULL;
+  head_ = tail_ = NULL;
 }
 
 
 void FixedSizeFreeList::Free(Address start) {
 #ifdef DEBUG
   MemoryAllocator::ZapBlock(start, object_size_);
 #endif
   // We only use the freelists with mark-sweep.
   ASSERT(!MarkCompactCollector::IsCompacting());
   FreeListNode* node = FreeListNode::FromAddress(start);
   node->set_size(object_size_);
-  node->set_next(head_);
-  head_ = node->address();
+  node->set_next(NULL);
+  if (head_ == NULL) {
+    tail_ = head_ = node->address();
+  } else {
+    FreeListNode::FromAddress(tail_)->set_next(node->address());
+    tail_ = node->address();
+  }
   available_ += object_size_;
 }
 
 
 Object* FixedSizeFreeList::Allocate() {
   if (head_ == NULL) {
     return Failure::RetryAfterGC(object_size_, owner_);
   }
 
   ASSERT(!FLAG_always_compact);  // We only use the freelists with mark-sweep.
@@ skipping 42 matching lines @@
   ASSERT(Waste() == 0);
   ASSERT(AvailableFree() == 0);
 
   // Build the free list for the space.
   int computed_size = 0;
   PageIterator it(this, PageIterator::PAGES_USED_BY_MC);
   while (it.has_next()) {
     Page* p = it.next();
     // Space below the relocation pointer is allocated.
     computed_size +=
-        static_cast<int>(p->mc_relocation_top - p->ObjectAreaStart());
+        static_cast<int>(p->AllocationWatermark() - p->ObjectAreaStart());
     if (it.has_next()) {
-      // Free the space at the top of the page.  We cannot use
-      // p->mc_relocation_top after the call to Free (because Free will clear
-      // remembered set bits).
+      // Free the space at the top of the page.
       int extra_size =
-          static_cast<int>(p->ObjectAreaEnd() - p->mc_relocation_top);
+          static_cast<int>(p->ObjectAreaEnd() - p->AllocationWatermark());
       if (extra_size > 0) {
-        int wasted_bytes = free_list_.Free(p->mc_relocation_top, extra_size);
+        int wasted_bytes = free_list_.Free(p->AllocationWatermark(),
+                                           extra_size);
         // The bytes we have just "freed" to add to the free list were
         // already accounted as available.
         accounting_stats_.WasteBytes(wasted_bytes);
       }
     }
   }
 
   // Make sure the computed size - based on the used portion of the pages in
   // use - matches the size obtained while computing forwarding addresses.
   ASSERT(computed_size == Size());
@@ skipping 27 matching lines @@
     MemoryAllocator::SetNextPage(prev, last->next_page());
   }
 
   // Attach it after the last page.
   MemoryAllocator::SetNextPage(last_page_, first);
   last_page_ = last;
   MemoryAllocator::SetNextPage(last, NULL);
 
   // Clean them up.
   do {
-    first->ClearRSet();
+    first->InvalidateWatermark(true);
+    first->SetAllocationWatermark(first->ObjectAreaStart());
+    first->SetCachedAllocationWatermark(first->ObjectAreaStart());
+    first->SetRegionMarks(Page::kAllRegionsCleanMarks);
     first = first->next_page();
   } while (first != NULL);
 
   // Order of pages in this space might no longer be consistent with
   // order of pages in chunks.
   page_list_is_chunk_ordered_ = false;
 }
 
 
 void PagedSpace::PrepareForMarkCompact(bool will_compact) {
@@ skipping 19 matching lines @@
       MemoryAllocator::RelinkPageListInChunkOrder(this,
                                                   &first_page_,
                                                   &last_page_,
                                                   &new_last_in_use);
       ASSERT(new_last_in_use->is_valid());
 
       if (new_last_in_use != last_in_use) {
         // Current allocation top points to a page which is now in the middle
         // of page list. We should move allocation top forward to the new last
         // used page so various object iterators will continue to work properly.
+        last_in_use->SetAllocationWatermark(last_in_use->AllocationTop());
 
         int size_in_bytes = static_cast<int>(PageAllocationLimit(last_in_use) -
                                              last_in_use->AllocationTop());
 
         if (size_in_bytes > 0) {
           // There is still some space left on this page. Create a fake
           // object which will occupy all free space on this page.
           // Otherwise iterators would not be able to scan this page
           // correctly.
 
@@ skipping 12 matching lines @@
       PageIterator pages_in_use_iterator(this, PageIterator::PAGES_IN_USE);
       while (pages_in_use_iterator.has_next()) {
         Page* p = pages_in_use_iterator.next();
         if (!p->WasInUseBeforeMC()) {
           // Empty page is in the middle of a sequence of used pages.
           // Create a fake object which will occupy all free space on this page.
           // Otherwise iterators would not be able to scan this page correctly.
           int size_in_bytes = static_cast<int>(PageAllocationLimit(p) -
                                                p->ObjectAreaStart());
 
+          p->SetAllocationWatermark(p->ObjectAreaStart());
           Heap::CreateFillerObjectAt(p->ObjectAreaStart(), size_in_bytes);
         }
       }
 
       page_list_is_chunk_ordered_ = true;
     }
   }
 }
 
 
@@ skipping 11 matching lines @@
   while (bytes_left_to_reserve > 0) {
     if (!reserved_page->next_page()->is_valid()) {
       if (Heap::OldGenerationAllocationLimitReached()) return false;
       Expand(reserved_page);
     }
     bytes_left_to_reserve -= Page::kPageSize;
     reserved_page = reserved_page->next_page();
     if (!reserved_page->is_valid()) return false;
   }
   ASSERT(TopPageOf(allocation_info_)->next_page()->is_valid());
+  TopPageOf(allocation_info_)->next_page()->InvalidateWatermark(true);
   SetAllocationInfo(&allocation_info_,
                     TopPageOf(allocation_info_)->next_page());
   return true;
 }
 
 
 // You have to call this last, since the implementation from PagedSpace
 // doesn't know that memory was 'promised' to large object space.
 bool LargeObjectSpace::ReserveSpace(int bytes) {
   return Heap::OldGenerationSpaceAvailable() >= bytes;
@@ skipping 14 matching lines @@
   }
 
   // There is no next page in this space.  Try free list allocation unless that
   // is currently forbidden.
   if (!Heap::linear_allocation()) {
     int wasted_bytes;
     Object* result = free_list_.Allocate(size_in_bytes, &wasted_bytes);
     accounting_stats_.WasteBytes(wasted_bytes);
     if (!result->IsFailure()) {
       accounting_stats_.AllocateBytes(size_in_bytes);
-      return HeapObject::cast(result);
+
+      HeapObject* obj = HeapObject::cast(result);
+      Page* p = Page::FromAddress(obj->address());
+
+      if (obj->address() >= p->AllocationWatermark()) {
+        // There should be no hole between the allocation watermark
+        // and allocated object address.
+        // Memory above the allocation watermark was not swept and
+        // might contain garbage pointers to new space.
+        ASSERT(obj->address() == p->AllocationWatermark());
+        p->SetAllocationWatermark(obj->address() + size_in_bytes);
+      }
+
+      return obj;
     }
   }
 
   // Free list allocation failed and there is no next page.  Fail if we have
   // hit the old generation size limit that should cause a garbage
   // collection.
   if (!Heap::always_allocate() && Heap::OldGenerationAllocationLimitReached()) {
     return NULL;
   }
 
   // Try to expand the space and allocate in the new next page.
   ASSERT(!current_page->next_page()->is_valid());
   if (Expand(current_page)) {
     return AllocateInNextPage(current_page, size_in_bytes);
   }
 
   // Finally, fail.
   return NULL;
 }
 
 
 void OldSpace::PutRestOfCurrentPageOnFreeList(Page* current_page) {
+  current_page->SetAllocationWatermark(allocation_info_.top);
   int free_size =
       static_cast<int>(current_page->ObjectAreaEnd() - allocation_info_.top);
   if (free_size > 0) {
     int wasted_bytes = free_list_.Free(allocation_info_.top, free_size);
     accounting_stats_.WasteBytes(wasted_bytes);
   }
 }
 
 
 void FixedSpace::PutRestOfCurrentPageOnFreeList(Page* current_page) {
+  current_page->SetAllocationWatermark(allocation_info_.top);
   int free_size =
       static_cast<int>(current_page->ObjectAreaEnd() - allocation_info_.top);
   // In the fixed space free list all the free list items have the right size.
   // We use up the rest of the page while preserving this invariant.
   while (free_size >= object_size_in_bytes_) {
     free_list_.Free(allocation_info_.top);
     allocation_info_.top += object_size_in_bytes_;
     free_size -= object_size_in_bytes_;
     accounting_stats_.WasteBytes(object_size_in_bytes_);
   }
 }
 
 
 // Add the block at the top of the page to the space's free list, set the
 // allocation info to the next page (assumed to be one), and allocate
 // linearly there.
 HeapObject* OldSpace::AllocateInNextPage(Page* current_page,
                                          int size_in_bytes) {
   ASSERT(current_page->next_page()->is_valid());
+  Page* next_page = current_page->next_page();
+  next_page->ClearGCFields();
   PutRestOfCurrentPageOnFreeList(current_page);
-  SetAllocationInfo(&allocation_info_, current_page->next_page());
+  SetAllocationInfo(&allocation_info_, next_page);
   return AllocateLinearly(&allocation_info_, size_in_bytes);
 }
 
 
 #ifdef DEBUG
 struct CommentStatistic {
   const char* comment;
   int size;
   int count;
   void Clear() {
@@ skipping 122 matching lines @@
     }
   }
 }
 
 
 void OldSpace::ReportStatistics() {
   int pct = Available() * 100 / Capacity();
   PrintF("  capacity: %d, waste: %d, available: %d, %%%d\n",
          Capacity(), Waste(), Available(), pct);
 
-  // Report remembered set statistics.
-  int rset_marked_pointers = 0;
-  int rset_marked_arrays = 0;
-  int rset_marked_array_elements = 0;
-  int cross_gen_pointers = 0;
-  int cross_gen_array_elements = 0;
-
-  PageIterator page_it(this, PageIterator::PAGES_IN_USE);
-  while (page_it.has_next()) {
-    Page* p = page_it.next();
-
-    for (Address rset_addr = p->RSetStart();
-         rset_addr < p->RSetEnd();
-         rset_addr += kIntSize) {
-      int rset = Memory::int_at(rset_addr);
-      if (rset != 0) {
-        // Bits were set
-        int intoff =
-            static_cast<int>(rset_addr - p->address() - Page::kRSetOffset);
-        int bitoff = 0;
-        for (; bitoff < kBitsPerInt; ++bitoff) {
-          if ((rset & (1 << bitoff)) != 0) {
-            int bitpos = intoff*kBitsPerByte + bitoff;
-            Address slot = p->OffsetToAddress(bitpos << kObjectAlignmentBits);
-            Object** obj = reinterpret_cast<Object**>(slot);
-            if (*obj == Heap::raw_unchecked_fixed_array_map()) {
-              rset_marked_arrays++;
-              FixedArray* fa = FixedArray::cast(HeapObject::FromAddress(slot));
-
-              rset_marked_array_elements += fa->length();
-              // Manually inline FixedArray::IterateBody
-              Address elm_start = slot + FixedArray::kHeaderSize;
-              Address elm_stop = elm_start + fa->length() * kPointerSize;
-              for (Address elm_addr = elm_start;
-                   elm_addr < elm_stop; elm_addr += kPointerSize) {
-                // Filter non-heap-object pointers
-                Object** elm_p = reinterpret_cast<Object**>(elm_addr);
-                if (Heap::InNewSpace(*elm_p))
-                  cross_gen_array_elements++;
-              }
-            } else {
-              rset_marked_pointers++;
-              if (Heap::InNewSpace(*obj))
-                cross_gen_pointers++;
-            }
-          }
-        }
-      }
-    }
-  }
-
-  pct = rset_marked_pointers == 0 ?
-        0 : cross_gen_pointers * 100 / rset_marked_pointers;
-  PrintF("  rset-marked pointers %d, to-new-space %d (%%%d)\n",
-            rset_marked_pointers, cross_gen_pointers, pct);
-  PrintF("  rset_marked arrays %d, ", rset_marked_arrays);
-  PrintF("  elements %d, ", rset_marked_array_elements);
-  pct = rset_marked_array_elements == 0 ? 0
-           : cross_gen_array_elements * 100 / rset_marked_array_elements;
-  PrintF("  pointers to new space %d (%%%d)\n", cross_gen_array_elements, pct);
-  PrintF("  total rset-marked bits %d\n",
-            (rset_marked_pointers + rset_marked_arrays));
-  pct = (rset_marked_pointers + rset_marked_array_elements) == 0 ? 0
-        : (cross_gen_pointers + cross_gen_array_elements) * 100 /
-          (rset_marked_pointers + rset_marked_array_elements);
-  PrintF("  total rset pointers %d, true cross generation ones %d (%%%d)\n",
-         (rset_marked_pointers + rset_marked_array_elements),
-         (cross_gen_pointers + cross_gen_array_elements),
-         pct);
-
   ClearHistograms();
   HeapObjectIterator obj_it(this);
   for (HeapObject* obj = obj_it.next(); obj != NULL; obj = obj_it.next())
     CollectHistogramInfo(obj);
   ReportHistogram(true);
 }
-
-
-// Dump the range of remembered set words between [start, end) corresponding
-// to the pointers starting at object_p.  The allocation_top is an object
-// pointer which should not be read past.  This is important for large object
-// pages, where some bits in the remembered set range do not correspond to
-// allocated addresses.
-static void PrintRSetRange(Address start, Address end, Object** object_p,
-                           Address allocation_top) {
-  Address rset_address = start;
-
-  // If the range starts on on odd numbered word (eg, for large object extra
-  // remembered set ranges), print some spaces.
-  if ((reinterpret_cast<uintptr_t>(start) / kIntSize) % 2 == 1) {
-    PrintF("                                    ");
-  }
-
-  // Loop over all the words in the range.
-  while (rset_address < end) {
-    uint32_t rset_word = Memory::uint32_at(rset_address);
-    int bit_position = 0;
-
-    // Loop over all the bits in the word.
-    while (bit_position < kBitsPerInt) {
-      if (object_p == reinterpret_cast<Object**>(allocation_top)) {
-        // Print a bar at the allocation pointer.
-        PrintF("|");
-      } else if (object_p > reinterpret_cast<Object**>(allocation_top)) {
-        // Do not dereference object_p past the allocation pointer.
-        PrintF("#");
-      } else if ((rset_word & (1 << bit_position)) == 0) {
-        // Print a dot for zero bits.
-        PrintF(".");
-      } else if (Heap::InNewSpace(*object_p)) {
-        // Print an X for one bits for pointers to new space.
-        PrintF("X");
-      } else {
-        // Print a circle for one bits for pointers to old space.
-        PrintF("o");
-      }
-
-      // Print a space after every 8th bit except the last.
-      if (bit_position % 8 == 7 && bit_position != (kBitsPerInt - 1)) {
-        PrintF(" ");
-      }
-
-      // Advance to next bit.
-      bit_position++;
-      object_p++;
-    }
-
-    // Print a newline after every odd numbered word, otherwise a space.
-    if ((reinterpret_cast<uintptr_t>(rset_address) / kIntSize) % 2 == 1) {
-      PrintF("\n");
-    } else {
-      PrintF(" ");
-    }
-
-    // Advance to next remembered set word.
-    rset_address += kIntSize;
-  }
-}
-
-
-void PagedSpace::DoPrintRSet(const char* space_name) {
-  PageIterator it(this, PageIterator::PAGES_IN_USE);
-  while (it.has_next()) {
-    Page* p = it.next();
-    PrintF("%s page 0x%x:\n", space_name, p);
-    PrintRSetRange(p->RSetStart(), p->RSetEnd(),
-                   reinterpret_cast<Object**>(p->ObjectAreaStart()),
-                   p->AllocationTop());
-    PrintF("\n");
-  }
-}
-
-
-void OldSpace::PrintRSet() { DoPrintRSet("old"); }
 #endif
 
 // -----------------------------------------------------------------------------
 // FixedSpace implementation
 
 void FixedSpace::PrepareForMarkCompact(bool will_compact) {
   // Call prepare of the super class.
   PagedSpace::PrepareForMarkCompact(will_compact);
 
   if (will_compact) {
@@ skipping 29 matching lines @@
   // Update allocation_top of each page in use and compute waste.
   int computed_size = 0;
   PageIterator it(this, PageIterator::PAGES_USED_BY_MC);
   while (it.has_next()) {
     Page* page = it.next();
     Address page_top = page->AllocationTop();
     computed_size += static_cast<int>(page_top - page->ObjectAreaStart());
     if (it.has_next()) {
       accounting_stats_.WasteBytes(
           static_cast<int>(page->ObjectAreaEnd() - page_top));
+      page->SetAllocationWatermark(page_top);
     }
   }
 
   // Make sure the computed size - based on the used portion of the
   // pages in use - matches the size we adjust during allocation.
   ASSERT(computed_size == Size());
 }
 
 
 // Slow case for normal allocation. Try in order: (1) allocate in the next
 // page in the space, (2) allocate off the space's free list, (3) expand the
 // space, (4) fail.
 HeapObject* FixedSpace::SlowAllocateRaw(int size_in_bytes) {
   ASSERT_EQ(object_size_in_bytes_, size_in_bytes);
   // Linear allocation in this space has failed.  If there is another page
   // in the space, move to that page and allocate there.  This allocation
   // should succeed.
   Page* current_page = TopPageOf(allocation_info_);
   if (current_page->next_page()->is_valid()) {
     return AllocateInNextPage(current_page, size_in_bytes);
   }
 
   // There is no next page in this space.  Try free list allocation unless
   // that is currently forbidden.  The fixed space free list implicitly assumes
   // that all free blocks are of the fixed size.
   if (!Heap::linear_allocation()) {
     Object* result = free_list_.Allocate();
     if (!result->IsFailure()) {
       accounting_stats_.AllocateBytes(size_in_bytes);
-      return HeapObject::cast(result);
+      HeapObject* obj = HeapObject::cast(result);
+      Page* p = Page::FromAddress(obj->address());
+
+      if (obj->address() >= p->AllocationWatermark()) {
+        // There should be no hole between the allocation watermark
+        // and allocated object address.
+        // Memory above the allocation watermark was not swept and
+        // might contain garbage pointers to new space.
+        ASSERT(obj->address() == p->AllocationWatermark());
+        p->SetAllocationWatermark(obj->address() + size_in_bytes);
+      }
+
+      return obj;
     }
   }
 
   // Free list allocation failed and there is no next page.  Fail if we have
   // hit the old generation size limit that should cause a garbage
   // collection.
   if (!Heap::always_allocate() && Heap::OldGenerationAllocationLimitReached()) {
     return NULL;
   }
 
   // Try to expand the space and allocate in the new next page.
   ASSERT(!current_page->next_page()->is_valid());
   if (Expand(current_page)) {
     return AllocateInNextPage(current_page, size_in_bytes);
   }
 
   // Finally, fail.
   return NULL;
 }
 
 
 // Move to the next page (there is assumed to be one) and allocate there.
 // The top of page block is always wasted, because it is too small to hold a
 // map.
 HeapObject* FixedSpace::AllocateInNextPage(Page* current_page,
                                            int size_in_bytes) {
   ASSERT(current_page->next_page()->is_valid());
   ASSERT(allocation_info_.top == PageAllocationLimit(current_page));
   ASSERT_EQ(object_size_in_bytes_, size_in_bytes);
+  Page* next_page = current_page->next_page();
+  next_page->ClearGCFields();
+  current_page->SetAllocationWatermark(allocation_info_.top);
   accounting_stats_.WasteBytes(page_extra_);
-  SetAllocationInfo(&allocation_info_, current_page->next_page());
+  SetAllocationInfo(&allocation_info_, next_page);
   return AllocateLinearly(&allocation_info_, size_in_bytes);
 }
 
 
 #ifdef DEBUG
 void FixedSpace::ReportStatistics() {
   int pct = Available() * 100 / Capacity();
   PrintF("  capacity: %d, waste: %d, available: %d, %%%d\n",
          Capacity(), Waste(), Available(), pct);
 
-  // Report remembered set statistics.
-  int rset_marked_pointers = 0;
-  int cross_gen_pointers = 0;
-
-  PageIterator page_it(this, PageIterator::PAGES_IN_USE);
-  while (page_it.has_next()) {
-    Page* p = page_it.next();
-
-    for (Address rset_addr = p->RSetStart();
-         rset_addr < p->RSetEnd();
-         rset_addr += kIntSize) {
-      int rset = Memory::int_at(rset_addr);
-      if (rset != 0) {
-        // Bits were set
-        int intoff =
-            static_cast<int>(rset_addr - p->address() - Page::kRSetOffset);
-        int bitoff = 0;
-        for (; bitoff < kBitsPerInt; ++bitoff) {
-          if ((rset & (1 << bitoff)) != 0) {
-            int bitpos = intoff*kBitsPerByte + bitoff;
-            Address slot = p->OffsetToAddress(bitpos << kObjectAlignmentBits);
-            Object** obj = reinterpret_cast<Object**>(slot);
-            rset_marked_pointers++;
-            if (Heap::InNewSpace(*obj))
-              cross_gen_pointers++;
-          }
-        }
-      }
-    }
-  }
-
-  pct = rset_marked_pointers == 0 ?
-          0 : cross_gen_pointers * 100 / rset_marked_pointers;
-  PrintF("  rset-marked pointers %d, to-new-space %d (%%%d)\n",
-            rset_marked_pointers, cross_gen_pointers, pct);
-
   ClearHistograms();
   HeapObjectIterator obj_it(this);
   for (HeapObject* obj = obj_it.next(); obj != NULL; obj = obj_it.next())
     CollectHistogramInfo(obj);
   ReportHistogram(false);
 }
-
-
-void FixedSpace::PrintRSet() { DoPrintRSet(name_); }
 #endif
 
 
 // -----------------------------------------------------------------------------
 // MapSpace implementation
 
 void MapSpace::PrepareForMarkCompact(bool will_compact) {
   // Call prepare of the super class.
   FixedSpace::PrepareForMarkCompact(will_compact);
 
@@ skipping 158 matching lines @@
   if (chunk == NULL) {
     return Failure::RetryAfterGC(requested_size, identity());
   }
 
   size_ += static_cast<int>(chunk_size);
   page_count_++;
   chunk->set_next(first_chunk_);
   chunk->set_size(chunk_size);
   first_chunk_ = chunk;
 
-  // Set the object address and size in the page header and clear its
-  // remembered set.
+  // Initialize page header.
   Page* page = Page::FromAddress(RoundUp(chunk->address(), Page::kPageSize));
   Address object_address = page->ObjectAreaStart();
   // Clear the low order bit of the second word in the page to flag it as a
   // large object page.  If the chunk_size happened to be written there, its
   // low order bit should already be clear.
   ASSERT((chunk_size & 0x1) == 0);
   page->SetIsLargeObjectPage(true);
-  page->ClearRSet();
-  int extra_bytes = requested_size - object_size;
-  if (extra_bytes > 0) {
-    // The extra memory for the remembered set should be cleared.
-    memset(object_address + object_size, 0, extra_bytes);
-  }
-
+  page->SetRegionMarks(Page::kAllRegionsCleanMarks);
   return HeapObject::FromAddress(object_address);
 }
 
 
 Object* LargeObjectSpace::AllocateRawCode(int size_in_bytes) {
   ASSERT(0 < size_in_bytes);
   return AllocateRawInternal(size_in_bytes,
                              size_in_bytes,
                              EXECUTABLE);
 }
 
 
 Object* LargeObjectSpace::AllocateRawFixedArray(int size_in_bytes) {
   ASSERT(0 < size_in_bytes);
-  int extra_rset_bytes = ExtraRSetBytesFor(size_in_bytes);
-  return AllocateRawInternal(size_in_bytes + extra_rset_bytes,
+  return AllocateRawInternal(size_in_bytes,
                              size_in_bytes,
                              NOT_EXECUTABLE);
 }
 
 
 Object* LargeObjectSpace::AllocateRaw(int size_in_bytes) {
   ASSERT(0 < size_in_bytes);
   return AllocateRawInternal(size_in_bytes,
                              size_in_bytes,
                              NOT_EXECUTABLE);
 }
 
 
 // GC support
 Object* LargeObjectSpace::FindObject(Address a) {
   for (LargeObjectChunk* chunk = first_chunk_;
        chunk != NULL;
        chunk = chunk->next()) {
     Address chunk_address = chunk->address();
     if (chunk_address <= a && a < chunk_address + chunk->size()) {
       return chunk->GetObject();
     }
   }
   return Failure::Exception();
 }
 
-
-void LargeObjectSpace::ClearRSet() {
-  ASSERT(Page::is_rset_in_use());
-
-  LargeObjectIterator it(this);
-  for (HeapObject* object = it.next(); object != NULL; object = it.next()) {
-    // We only have code, sequential strings, or fixed arrays in large
-    // object space, and only fixed arrays need remembered set support.
-    if (object->IsFixedArray()) {
-      // Clear the normal remembered set region of the page;
-      Page* page = Page::FromAddress(object->address());
-      page->ClearRSet();
-
-      // Clear the extra remembered set.
-      int size = object->Size();
-      int extra_rset_bytes = ExtraRSetBytesFor(size);
-      memset(object->address() + size, 0, extra_rset_bytes);
-    }
-  }
-}
-
-
-void LargeObjectSpace::IterateRSet(ObjectSlotCallback copy_object_func) {
-  ASSERT(Page::is_rset_in_use());
-
-  static void* lo_rset_histogram = StatsTable::CreateHistogram(
-      "V8.RSetLO",
-      0,
-      // Keeping this histogram's buckets the same as the paged space histogram.
-      Page::kObjectAreaSize / kPointerSize,
-      30);
-
+void LargeObjectSpace::IterateDirtyRegions(ObjectSlotCallback copy_object) {
   LargeObjectIterator it(this);
   for (HeapObject* object = it.next(); object != NULL; object = it.next()) {
     // We only have code, sequential strings, or fixed arrays in large
     // object space, and only fixed arrays can possibly contain pointers to
     // the young generation.
     if (object->IsFixedArray()) {
-      // Iterate the normal page remembered set range.
       Page* page = Page::FromAddress(object->address());
-      Address object_end = object->address() + object->Size();
-      int count = Heap::IterateRSetRange(page->ObjectAreaStart(),
-                                         Min(page->ObjectAreaEnd(), object_end),
-                                         page->RSetStart(),
-                                         copy_object_func);
+      uint32_t marks = page->GetRegionMarks();
+      uint32_t newmarks = Page::kAllRegionsCleanMarks;
 
-      // Iterate the extra array elements.
-      if (object_end > page->ObjectAreaEnd()) {
-        count += Heap::IterateRSetRange(page->ObjectAreaEnd(), object_end,
-                                        object_end, copy_object_func);
-      }
-      if (lo_rset_histogram != NULL) {
-        StatsTable::AddHistogramSample(lo_rset_histogram, count);
+      if (marks != Page::kAllRegionsCleanMarks) {
+        // For a large page a single dirty mark corresponds to several
+        // regions (modulo 32). So we treat a large page as a sequence of
+        // normal pages of size Page::kPageSize having same dirty marks
+        // and subsequently iterate dirty regions on each of these pages.
+        Address start = object->address();
+        Address end = page->ObjectAreaEnd();
+        Address object_end = start + object->Size();
+
+        // Iterate regions of the first normal page covering object.
+        uint32_t first_region_number = page->GetRegionNumberForAddress(start);
+        newmarks |=
+            Heap::IterateDirtyRegions(marks >> first_region_number,
+                                      start,
+                                      end,
+                                      &Heap::IteratePointersInDirtyRegion,
+                                      copy_object) << first_region_number;
+
+        start = end;
+        end = start + Page::kPageSize;
+        while (end <= object_end) {
+          // Iterate next 32 regions.
+          newmarks |=
+              Heap::IterateDirtyRegions(marks,
+                                        start,
+                                        end,
+                                        &Heap::IteratePointersInDirtyRegion,
+                                        copy_object);
+          start = end;
+          end = start + Page::kPageSize;
+        }
+
+        if (start != object_end) {
+          // Iterate the last piece of an object which is less than
+          // Page::kPageSize.
+          newmarks |=
+              Heap::IterateDirtyRegions(marks,
+                                        start,
+                                        object_end,
+                                        &Heap::IteratePointersInDirtyRegion,
+                                        copy_object);
+        }
+
+        page->SetRegionMarks(newmarks);
       }
     }
   }
 }
 
 
 void LargeObjectSpace::FreeUnmarkedObjects() {
   LargeObjectChunk* previous = NULL;
   LargeObjectChunk* current = first_chunk_;
   while (current != NULL) {
@@ skipping 71 matching lines @@
 
     // Byte arrays and strings don't have interior pointers.
     if (object->IsCode()) {
       VerifyPointersVisitor code_visitor;
       object->IterateBody(map->instance_type(),
                           object->Size(),
                           &code_visitor);
     } else if (object->IsFixedArray()) {
       // We loop over fixed arrays ourselves, rather then using the visitor,
       // because the visitor doesn't support the start/offset iteration
-      // needed for IsRSetSet.
+      // needed for IsRegionDirty.
       FixedArray* array = FixedArray::cast(object);
       for (int j = 0; j < array->length(); j++) {
         Object* element = array->get(j);
         if (element->IsHeapObject()) {
           HeapObject* element_object = HeapObject::cast(element);
           ASSERT(Heap::Contains(element_object));
           ASSERT(element_object->map()->IsMap());
           if (Heap::InNewSpace(element_object)) {
-            ASSERT(Page::IsRSetSet(object->address(),
-                                   FixedArray::kHeaderSize + j * kPointerSize));
+            Address array_addr = object->address();
+            Address element_addr = array_addr + FixedArray::kHeaderSize +
+                j * kPointerSize;
+
+            ASSERT(Page::FromAddress(array_addr)->IsRegionDirty(element_addr));
           }
         }
       }
     }
   }
 }
 
 
 void LargeObjectSpace::Print() {
   LargeObjectIterator it(this);
@@ skipping 20 matching lines @@
 
 void LargeObjectSpace::CollectCodeStatistics() {
   LargeObjectIterator obj_it(this);
   for (HeapObject* obj = obj_it.next(); obj != NULL; obj = obj_it.next()) {
     if (obj->IsCode()) {
       Code* code = Code::cast(obj);
       code_kind_statistics[code->kind()] += code->Size();
     }
   }
 }
-
-
-void LargeObjectSpace::PrintRSet() {
-  LargeObjectIterator it(this);
-  for (HeapObject* object = it.next(); object != NULL; object = it.next()) {
-    if (object->IsFixedArray()) {
-      Page* page = Page::FromAddress(object->address());
-
-      Address allocation_top = object->address() + object->Size();
-      PrintF("large page 0x%x:\n", page);
-      PrintRSetRange(page->RSetStart(), page->RSetEnd(),
-                     reinterpret_cast<Object**>(object->address()),
-                     allocation_top);
-      int extra_array_bytes = object->Size() - Page::kObjectAreaSize;
-      int extra_rset_bits = RoundUp(extra_array_bytes / kPointerSize,
-                                    kBitsPerInt);
-      PrintF("------------------------------------------------------------"
-             "-----------\n");
-      PrintRSetRange(allocation_top,
-                     allocation_top + extra_rset_bits / kBitsPerByte,
-                     reinterpret_cast<Object**>(object->address()
-                                                + Page::kObjectAreaSize),
-                     allocation_top);
-      PrintF("\n");
-    }
-  }
-}
 #endif  // DEBUG
 
 } }  // namespace v8::internal