Get rid of distinction between below- and above-watermark in page allocation....

src/heap.cc

-// Copyright 2010 the V8 project authors. All rights reserved.
+// Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
 //       with the distribution.
@@ skipping 404 matching lines @@
   Counters::number_of_symbols.Set(symbol_table()->NumberOfElements());
 #if defined(DEBUG) || defined(ENABLE_LOGGING_AND_PROFILING)
   ReportStatisticsAfterGC();
 #endif
 #ifdef ENABLE_DEBUGGER_SUPPORT
   Debug::AfterGarbageCollection();
 #endif
 }
 
 
-void Heap::CollectAllGarbage(bool force_compaction) {
+void Heap::CollectAllGarbage(int flags) {
   // Since we are ignoring the return value, the exact choice of space does
   // not matter, so long as we do not specify NEW_SPACE, which would not
   // cause a full GC.
-  MarkCompactCollector::SetForceCompaction(force_compaction);
+  MarkCompactCollector::SetFlags(flags);
   CollectGarbage(OLD_POINTER_SPACE);
-  MarkCompactCollector::SetForceCompaction(false);
+  MarkCompactCollector::SetFlags(kNoGCFlags);
 }
 
 
 void Heap::CollectAllAvailableGarbage() {
   // Since we are ignoring the return value, the exact choice of space does
   // not matter, so long as we do not specify NEW_SPACE, which would not
   // cause a full GC.
-  MarkCompactCollector::SetForceCompaction(true);
+  MarkCompactCollector::SetFlags(kMakeHeapIterableMask | kForceCompactionMask);
 
   // Major GC would invoke weak handle callbacks on weakly reachable
   // handles, but won't collect weakly reachable objects until next
   // major GC.  Therefore if we collect aggressively and weak handle callback
   // has been invoked, we rerun major GC to release objects which become
   // garbage.
   // Note: as weak callbacks can execute arbitrary code, we cannot
   // hope that eventually there will be no weak callbacks invocations.
   // Therefore stop recollecting after several attempts.
   const int kMaxNumberOfAttempts = 7;
   for (int attempt = 0; attempt < kMaxNumberOfAttempts; attempt++) {
     if (!CollectGarbage(OLD_POINTER_SPACE, MARK_COMPACTOR)) {
       break;
     }
   }
-  MarkCompactCollector::SetForceCompaction(false);
+  MarkCompactCollector::SetFlags(kNoGCFlags);
 }
 
 
 bool Heap::CollectGarbage(AllocationSpace space, GarbageCollector collector) {
   // The VM is in the GC state until exiting this function.
   VMState state(GC);
 
 #ifdef DEBUG
   // Reset the allocation timeout to the GC interval, but make sure to
   // allow at least a few allocations after a collection. The reason
@@ skipping 438 matching lines @@
     }
   }
 };
 
 
 static void VerifyNonPointerSpacePointers() {
   // Verify that there are no pointers to new space in spaces where we
   // do not expect them.
   VerifyNonPointerSpacePointersVisitor v;
   HeapObjectIterator code_it(Heap::code_space());
-  for (HeapObject* object = code_it.next();
-       object != NULL; object = code_it.next())
+  for (HeapObject* object = code_it.Next();
+       object != NULL; object = code_it.Next())
     object->Iterate(&v);
 
-  HeapObjectIterator data_it(Heap::old_data_space());
-  for (HeapObject* object = data_it.next();
-       object != NULL; object = data_it.next())
-    object->Iterate(&v);
+  // The old data space was normally swept conservatively so that the iterator
+  // doesn't work, so we normally skip the next bit.
+  if (!Heap::old_data_space()->was_swept_conservatively()) {
+    HeapObjectIterator data_it(Heap::old_data_space());
+    for (HeapObject* object = data_it.Next();
+         object != NULL; object = data_it.Next())
+      object->Iterate(&v);
+  }
 }
 #endif
 
 
 void Heap::CheckNewSpaceExpansionCriteria() {
   if (new_space_.Capacity() < new_space_.MaximumCapacity() &&
       survived_since_last_expansion_ > new_space_.Capacity()) {
     // Grow the size of new space if there is room to grow and enough
     // data has survived scavenge since the last expansion.
     new_space_.Grow();
     survived_since_last_expansion_ = 0;
   }
 }
 
 
 void Heap::Scavenge() {
 #ifdef DEBUG
   if (FLAG_enable_slow_asserts) VerifyNonPointerSpacePointers();
 #endif
 
   gc_state_ = SCAVENGE;
 
-  Page::FlipMeaningOfInvalidatedWatermarkFlag();
-
-  // We do not update an allocation watermark of the top page during linear
-  // allocation to avoid overhead. So to maintain the watermark invariant
-  // we have to manually cache the watermark and mark the top page as having an
-  // invalid watermark.  This guarantees that old space pointer iteration will
-  // use a correct watermark even if a linear allocation happens.
-  old_pointer_space_->FlushTopPageWatermark();
-  map_space_->FlushTopPageWatermark();
-
   // Implements Cheney's copying algorithm
   LOG(ResourceEvent("scavenge", "begin"));
 
   // Clear descriptor cache.
   DescriptorLookupCache::Clear();
 
   // Used for updating survived_since_last_expansion_ at function end.
   intptr_t survived_watermark = PromotedSpaceSize();
 
   CheckNewSpaceExpansionCriteria();
@@ skipping 32 matching lines @@
   IterateRoots(&scavenge_visitor, VISIT_ALL_IN_SCAVENGE);
 
   // Copy objects reachable from the old generation.
   {
     StoreBufferRebuildScope scope;
     StoreBuffer::IteratePointersToNewSpace(&ScavengeObject);
   }
 
   // Copy objects reachable from cells by scavenging cell values directly.
   HeapObjectIterator cell_iterator(cell_space_);
-  for (HeapObject* cell = cell_iterator.next();
-       cell != NULL; cell = cell_iterator.next()) {
+  for (HeapObject* cell = cell_iterator.Next();
+       cell != NULL; cell = cell_iterator.Next()) {
     if (cell->IsJSGlobalPropertyCell()) {
       Address value_address =
           reinterpret_cast<Address>(cell) +
           (JSGlobalPropertyCell::kValueOffset - kHeapObjectTag);
       scavenge_visitor.VisitPointer(reinterpret_cast<Object**>(value_address));
     }
   }
 
   // Scavenge object reachable from the global contexts list directly.
   scavenge_visitor.VisitPointer(BitCast<Object**>(&global_contexts_list_));
@@ skipping 646 matching lines @@
   }
   set_undetectable_ascii_string_map(Map::cast(obj));
   Map::cast(obj)->set_is_undetectable();
 
   { MaybeObject* maybe_obj =
         AllocateMap(BYTE_ARRAY_TYPE, kVariableSizeSentinel);
     if (!maybe_obj->ToObject(&obj)) return false;
   }
   set_byte_array_map(Map::cast(obj));
 
+  { MaybeObject* maybe_obj =
+        AllocateMap(FREE_SPACE_TYPE, kVariableSizeSentinel);
+    if (!maybe_obj->ToObject(&obj)) return false;
+  }
+  set_free_space_map(Map::cast(obj));
+
   { MaybeObject* maybe_obj = AllocateByteArray(0, TENURED);
     if (!maybe_obj->ToObject(&obj)) return false;
   }
   set_empty_byte_array(ByteArray::cast(obj));
 
   { MaybeObject* maybe_obj =
         AllocateMap(EXTERNAL_PIXEL_ARRAY_TYPE, ExternalArray::kAlignedSize);
     if (!maybe_obj->ToObject(&obj)) return false;
   }
   set_external_pixel_array_map(Map::cast(obj));
@@ skipping 937 matching lines @@
 
 
 void Heap::CreateFillerObjectAt(Address addr, int size) {
   if (size == 0) return;
   HeapObject* filler = HeapObject::FromAddress(addr);
   if (size == kPointerSize) {
     filler->set_map(one_pointer_filler_map());
   } else if (size == 2 * kPointerSize) {
     filler->set_map(two_pointer_filler_map());
   } else {
-    filler->set_map(byte_array_map());
-    ByteArray::cast(filler)->set_length(ByteArray::LengthFor(size));
+    filler->set_map(free_space_map());
+    FreeSpace::cast(filler)->set_size(size);
   }
 }
 
 
 MaybeObject* Heap::AllocateExternalArray(int length,
                                          ExternalArrayType array_type,
                                          void* external_pointer,
                                          PretenureFlag pretenure) {
   AllocationSpace space = (pretenure == TENURED) ? OLD_DATA_SPACE : NEW_SPACE;
   Object* result;
@@ skipping 1058 matching lines @@
       (size > MaxObjectSizeInPagedSpace()) ? LO_SPACE : OLD_POINTER_SPACE;
   Object* result;
   { MaybeObject* maybe_result = Heap::Allocate(map, space);
     if (!maybe_result->ToObject(&result)) return maybe_result;
   }
   Struct::cast(result)->InitializeBody(size);
   return result;
 }
 
 
+void Heap::EnsureHeapIsIterable() {
+  ASSERT(IsAllocationAllowed());
+  if (IncrementalMarking::state() != IncrementalMarking::STOPPED ||
+      old_pointer_space()->was_swept_conservatively() ||
+      old_data_space()->was_swept_conservatively()) {
+    CollectAllGarbage(kMakeHeapIterableMask);
+  }
+  ASSERT(!old_pointer_space()->was_swept_conservatively());
+  ASSERT(!old_data_space()->was_swept_conservatively());
+}
+
+
 bool Heap::IdleNotification() {
   static const int kIdlesBeforeScavenge = 4;
   static const int kIdlesBeforeMarkSweep = 7;
   static const int kIdlesBeforeMarkCompact = 8;
   static const int kMaxIdleCount = kIdlesBeforeMarkCompact + 1;
   static const unsigned int kGCsBetweenCleanup = 4;
   static int number_idle_notifications = 0;
   static unsigned int last_gc_count = gc_count_;
 
   bool uncommit = true;
   bool finished = false;
 
   // Reset the number of idle notifications received when a number of
   // GCs have taken place. This allows another round of cleanup based
   // on idle notifications if enough work has been carried out to
   // provoke a number of garbage collections.
   if (gc_count_ - last_gc_count < kGCsBetweenCleanup) {
     number_idle_notifications =
         Min(number_idle_notifications + 1, kMaxIdleCount);
   } else {
     number_idle_notifications = 0;
     last_gc_count = gc_count_;
   }
 
   if (number_idle_notifications == kIdlesBeforeScavenge) {
     if (contexts_disposed_ > 0) {
       HistogramTimerScope scope(&Counters::gc_context);
-      CollectAllGarbage(false);
+      CollectAllGarbage(kNoGCFlags);
     } else {
       CollectGarbage(NEW_SPACE);
     }
     new_space_.Shrink();
     last_gc_count = gc_count_;
   } else if (number_idle_notifications == kIdlesBeforeMarkSweep) {
     // Before doing the mark-sweep collections we clear the
     // compilation cache to avoid hanging on to source code and
     // generated code for cached functions.
     CompilationCache::Clear();
 
-    CollectAllGarbage(false);
+    CollectAllGarbage(kNoGCFlags);
     new_space_.Shrink();
     last_gc_count = gc_count_;
 
   } else if (number_idle_notifications == kIdlesBeforeMarkCompact) {
-    CollectAllGarbage(true);
+    CollectAllGarbage(kForceCompactionMask);
     new_space_.Shrink();
     last_gc_count = gc_count_;
     finished = true;
 
   } else if (contexts_disposed_ > 0) {
     if (FLAG_expose_gc) {
       contexts_disposed_ = 0;
     } else {
       HistogramTimerScope scope(&Counters::gc_context);
-      CollectAllGarbage(false);
+      CollectAllGarbage(kNoGCFlags);
       last_gc_count = gc_count_;
     }
     // If this is the first idle notification, we reset the
     // notification count to avoid letting idle notifications for
     // context disposal garbage collections start a potentially too
     // aggressive idle GC cycle.
     if (number_idle_notifications <= 1) {
       number_idle_notifications = 0;
       uncommit = false;
     }
@@ skipping 123 matching lines @@
 
 #ifdef DEBUG
 static void DummyScavengePointer(HeapObject** p, HeapObject* o) {
   // When we are not in GC the Heap::InNewSpace() predicate
   // checks that pointers which satisfy predicate point into
   // the active semispace.
   Heap::InNewSpace(*p);
 }
 
 
-static void VerifyPointersUnderWatermark(
+static void VerifyPointers(
     PagedSpace* space,
     PointerRegionCallback visit_pointer_region) {
-  PageIterator it(space, PageIterator::PAGES_IN_USE);
+  PageIterator it(space);
 
   while (it.has_next()) {
     Page* page = it.next();
     Address start = page->ObjectAreaStart();
-    Address end = page->AllocationWatermark();
+    Address end = page->ObjectAreaEnd();
 
     Heap::IteratePointersToNewSpace(start,
                                     end,
                                     &DummyScavengePointer);
   }
 }
 
 
-static void VerifyPointersUnderWatermark(LargeObjectSpace* space) {
+static void VerifyPointers(LargeObjectSpace* space) {
   LargeObjectIterator it(space);
   for (HeapObject* object = it.next(); object != NULL; object = it.next()) {
     if (object->IsFixedArray()) {
       Address slot_address = object->address();
       Address end = object->address() + object->Size();
 
       while (slot_address < end) {
         HeapObject** slot = reinterpret_cast<HeapObject**>(slot_address);
         // When we are not in GC the Heap::InNewSpace() predicate
         // checks that pointers which satisfy predicate point into
@@ skipping 12 matching lines @@
   StoreBuffer::Verify();
 
   VerifyPointersVisitor visitor;
   IterateRoots(&visitor, VISIT_ONLY_STRONG);
 
   new_space_.Verify();
 
   old_pointer_space_->Verify(&visitor);
   map_space_->Verify(&visitor);
 
-  VerifyPointersUnderWatermark(old_pointer_space_,
-                               &IteratePointersToNewSpace);
-  VerifyPointersUnderWatermark(map_space_,
-                               &IteratePointersFromMapsToNewSpace);
-  VerifyPointersUnderWatermark(lo_space_);
+  VerifyPointers(old_pointer_space_, &IteratePointersToNewSpace);
+  VerifyPointers(map_space_, &IteratePointersFromMapsToNewSpace);
+  VerifyPointers(lo_space_);
 
   VerifyPointersVisitor no_dirty_regions_visitor;
   old_data_space_->Verify(&no_dirty_regions_visitor);
   code_space_->Verify(&no_dirty_regions_visitor);
   cell_space_->Verify(&no_dirty_regions_visitor);
 
   lo_space_->Verify();
 }
 #endif  // DEBUG
 
@@ skipping 175 matching lines @@
         ASSERT(StoreBuffer::CellIsInStoreBuffer(
             reinterpret_cast<Address>(slot)));
       }
     }
     slot_address += kPointerSize;
   }
 }
 
 
 #ifdef DEBUG
+typedef bool (*CheckStoreBufferFilter)(Object** addr);
+
+
+bool IsAMapPointerAddress(Object** addr) {
+  uintptr_t a = reinterpret_cast<uintptr_t>(addr);
+  int mod = a % Map::kSize;
+  return mod >= Map::kPointerFieldsBeginOffset &&
+         mod < Map::kPointerFieldsEndOffset;
+}
+
+
+bool EverythingsAPointer(Object** addr) {
+  return true;
+}
+
+
 static void CheckStoreBuffer(Object** current,
                              Object** limit,
                              Object**** store_buffer_position,
-                             Object*** store_buffer_top) {
+                             Object*** store_buffer_top,
+                             CheckStoreBufferFilter filter,
+                             Address special_garbage_start,
+                             Address special_garbage_end) {
   for ( ; current < limit; current++) {
     Object* o = *current;
-    if (reinterpret_cast<uintptr_t>(o) == kFreeListZapValue) {
-      Object*** zap_checker = *store_buffer_position;
-      while (*zap_checker < current) {
-        zap_checker++;
-        if (zap_checker >= store_buffer_top) break;
-      }
-      if (zap_checker < store_buffer_top) {
-        // Objects in the free list shouldn't be in the store buffer.
-        ASSERT(*zap_checker != current);
-      }
+    Address current_address = reinterpret_cast<Address>(current);
+    // Skip free space.
+    if (o == Heap::free_space_map()) {
+      Address current_address = reinterpret_cast<Address>(current);
+      FreeSpace* free_space =
+          FreeSpace::cast(HeapObject::FromAddress(current_address));
+      int skip = free_space->Size();
+      ASSERT(current_address + skip <= reinterpret_cast<Address>(limit));
+      ASSERT(skip > 0);
+      current_address += skip - kPointerSize;
+      current = reinterpret_cast<Object**>(current_address);
       continue;
     }
+    // Skip the current linear allocation space between top and limit which is
+    // unmarked with the free space map, but can contain junk.
+    if (current_address == special_garbage_start &&
+        special_garbage_end != special_garbage_start) {
+      current_address = special_garbage_end - kPointerSize;
+      current = reinterpret_cast<Object**>(current_address);
+      continue;
+    }
+    if (!(*filter)(current)) continue;
+    ASSERT(current_address < special_garbage_start ||
+           current_address >= special_garbage_end);
+    ASSERT(reinterpret_cast<uintptr_t>(o) != kFreeListZapValue);
     // We have to check that the pointer does not point into new space
     // without trying to cast it to a heap object since the hash field of
     // a string can contain values like 1 and 3 which are tagged null
     // pointers.
     if (!Heap::InNewSpace(o)) continue;
     while (**store_buffer_position < current &&
            *store_buffer_position < store_buffer_top) {
       (*store_buffer_position)++;
     }
     if (**store_buffer_position != current ||
         *store_buffer_position == store_buffer_top) {
       Object** obj_start = current;
       while (!(*obj_start)->IsMap()) obj_start--;
       UNREACHABLE();
     }
   }
 }
 
 
 // Check that the store buffer contains all intergenerational pointers by
 // scanning a page and ensuring that all pointers to young space are in the
 // store buffer.
 void Heap::OldPointerSpaceCheckStoreBuffer(
     ExpectedPageWatermarkState watermark_state) {
   OldSpace* space = old_pointer_space();
-  PageIterator pages(space, PageIterator::PAGES_IN_USE);
-
-  space->free_list()->Zap();
+  PageIterator pages(space);
 
   StoreBuffer::SortUniq();
 
   while (pages.has_next()) {
     Page* page = pages.next();
     Object** current = reinterpret_cast<Object**>(page->ObjectAreaStart());
 
-    // Do not try to visit pointers beyond page allocation watermark.
-    // Page can contain garbage pointers there.
-    Address end;
-
-    if (watermark_state == WATERMARK_SHOULD_BE_VALID ||
-        page->IsWatermarkValid()) {
-      end = page->AllocationWatermark();
-    } else {
-      end = page->CachedAllocationWatermark();
-    }
+    Address end = page->ObjectAreaEnd();
 
     Object*** store_buffer_position = StoreBuffer::Start();
     Object*** store_buffer_top = StoreBuffer::Top();
 
     Object** limit = reinterpret_cast<Object**>(end);
-    CheckStoreBuffer(current, limit, &store_buffer_position, store_buffer_top);
+    CheckStoreBuffer(current,
+                     limit,
+                     &store_buffer_position,
+                     store_buffer_top,
+                     &EverythingsAPointer,
+                     space->top(),
+                     space->limit());
   }
 }
 
 
 void Heap::MapSpaceCheckStoreBuffer(
     ExpectedPageWatermarkState watermark_state) {
   MapSpace* space = map_space();
-  PageIterator pages(space, PageIterator::PAGES_IN_USE);
-
-  space->free_list()->Zap();
+  PageIterator pages(space);
 
   StoreBuffer::SortUniq();
 
   while (pages.has_next()) {
     Page* page = pages.next();
+    Object** current = reinterpret_cast<Object**>(page->ObjectAreaStart());
 
-    // Do not try to visit pointers beyond page allocation watermark.
-    // Page can contain garbage pointers there.
-    Address end;
-
-    if (watermark_state == WATERMARK_SHOULD_BE_VALID ||
-        page->IsWatermarkValid()) {
-      end = page->AllocationWatermark();
-    } else {
-      end = page->CachedAllocationWatermark();
-    }
-
-    Address map_aligned_current = page->ObjectAreaStart();
-
-    ASSERT(map_aligned_current == MapStartAlign(map_aligned_current));
-    ASSERT(end == MapEndAlign(end));
+    Address end = page->ObjectAreaEnd();
 
     Object*** store_buffer_position = StoreBuffer::Start();
     Object*** store_buffer_top = StoreBuffer::Top();
 
-    for ( ; map_aligned_current < end; map_aligned_current += Map::kSize) {
-      ASSERT(!Heap::InNewSpace(Memory::Object_at(map_aligned_current)));
-      ASSERT(Memory::Object_at(map_aligned_current)->IsMap());
-
-      Object** current = reinterpret_cast<Object**>(
-          map_aligned_current + Map::kPointerFieldsBeginOffset);
-      Object** limit = reinterpret_cast<Object**>(
-          map_aligned_current + Map::kPointerFieldsEndOffset);
-
-      CheckStoreBuffer(current,
-                       limit,
-                       &store_buffer_position,
-                       store_buffer_top);
-    }
+    Object** limit = reinterpret_cast<Object**>(end);
+    CheckStoreBuffer(current,
+                     limit,
+                     &store_buffer_position,
+                     store_buffer_top,
+                     &IsAMapPointerAddress,
+                     space->top(),
+                     space->limit());
   }
 }
 
 
 void Heap::LargeObjectSpaceCheckStoreBuffer() {
   LargeObjectIterator it(lo_space());
   for (HeapObject* object = it.next(); object != NULL; object = it.next()) {
     // We only have code, sequential strings, or fixed arrays in large
     // object space, and only fixed arrays can possibly contain pointers to
     // the young generation.
     if (object->IsFixedArray()) {
       Object*** store_buffer_position = StoreBuffer::Start();
       Object*** store_buffer_top = StoreBuffer::Top();
       Object** current = reinterpret_cast<Object**>(object->address());
       Object** limit =
           reinterpret_cast<Object**>(object->address() + object->Size());
       CheckStoreBuffer(current,
                        limit,
                        &store_buffer_position,
-                       store_buffer_top);
+                       store_buffer_top,
+                       &EverythingsAPointer,
+                       NULL,
+                       NULL);
     }
   }
 }
 
 
 #endif
 
 
+// This function iterates over all the pointers in a paged space in the heap,
+// looking for pointers into new space.  Within the pages there may be dead
+// objects that have not been overwritten by free spaces or fillers because of
+// lazy sweeping.  These dead objects may not contain pointers to new space.
+// The garbage areas that have been swept properly (these will normally be the
+// large ones) will be marked with free space and filler map words.  In
+// addition any area that has never been used at all for object allocation must
+// be marked with a free space or filler.  Because the free space and filler
+// maps do not move we can always recognize these even after a compaction.
+// Normal objects like FixedArrays and JSObjects should not contain references
+// to these maps.  The special garbage section (see comment in spaces.h) is
+// skipped since it can contain absolutely anything.  Any objects that are
+// allocated during iteration may or may not be visited by the iteration, but
+// they will not be partially visited.
 void Heap::IteratePointers(
     PagedSpace* space,
     PointerRegionCallback visit_pointer_region,
     ObjectSlotCallback copy_object_func,
     ExpectedPageWatermarkState expected_page_watermark_state) {
 
-  PageIterator pages(space, PageIterator::PAGES_IN_USE);
+  PageIterator pages(space);
 
   while (pages.has_next()) {
     Page* page = pages.next();
     Address start = page->ObjectAreaStart();
+    Address limit = page->ObjectAreaEnd();
 
-    // Do not try to visit pointers beyond page allocation watermark.
-    // Page can contain garbage pointers there.
-    Address end;
+    Address end = start;
 
-    if ((expected_page_watermark_state == WATERMARK_SHOULD_BE_VALID) ||
-        page->IsWatermarkValid()) {
-      end = page->AllocationWatermark();
-    } else {
-      end = page->CachedAllocationWatermark();
+    Object* free_space_map = Heap::free_space_map();
+    Object* two_pointer_filler_map = Heap::two_pointer_filler_map();
+
+    while (end < limit) {
+      Object* o = *reinterpret_cast<Object**>(end);
+      // Skip fillers but not things that look like fillers in the special
+      // garbage section which can contain anything.
+      if (o == free_space_map ||
+          o == two_pointer_filler_map ||
+          end == space->top()) {
+        if (start != end) {
+          // After calling this the special garbage section may have moved.
+          visit_pointer_region(start, end, copy_object_func);
+          if (end >= space->top() && end < space->limit()) {
+            end = space->limit();
+            start = end;
+            continue;
+          }
+        }
+        if (end == space->top()) {
+          start = end = space->limit();
+        } else {
+          // At this point we are either at the start of a filler or we are at
+          // the point where the space->top() used to be before the
+          // visit_pointer_region call above.  Either way we can skip the
+          // object at the current spot:  We don't promise to visit objects
+          // allocated during heap traversal, and if space->top() moved then it
+          // must be because an object was allocated at this point.
+          start = end + HeapObject::FromAddress(end)->Size();
+          end = start;
+        }
+      } else {
+        ASSERT(o != free_space_map);
+        ASSERT(o != two_pointer_filler_map);
+        ASSERT(end < space->top() || end >= space->limit());
+        end += kPointerSize;
+      }
     }
-
-    ASSERT(space == old_pointer_space_ ||
-           (space == map_space_ &&
-            ((page->ObjectAreaStart() - end) % Map::kSize == 0)));
-
-    visit_pointer_region(start, end, copy_object_func);
-
-    // Mark page watermark as invalid to maintain watermark validity invariant.
-    // See Page::FlipMeaningOfInvalidatedWatermarkFlag() for details.
-    page->InvalidateWatermark(true);
+    ASSERT(end == limit);
+    if (start != end) {
+      visit_pointer_region(start, end, copy_object_func);
+    }
   }
 }
 
 
 void Heap::IterateRoots(ObjectVisitor* v, VisitMode mode) {
   IterateStrongRoots(v, mode);
   IterateWeakRoots(v, mode);
 }
 
 
@@ skipping 146 matching lines @@
   *stats->map_space_capacity = map_space_->Capacity();
   *stats->cell_space_size = cell_space_->Size();
   *stats->cell_space_capacity = cell_space_->Capacity();
   *stats->lo_space_size = lo_space_->Size();
   GlobalHandles::RecordStats(stats);
   *stats->memory_allocator_size = MemoryAllocator::Size();
   *stats->memory_allocator_capacity =
       MemoryAllocator::Size() + MemoryAllocator::Available();
   *stats->os_error = OS::GetLastError();
   if (take_snapshot) {
-    HeapIterator iterator(HeapIterator::kFilterFreeListNodes);
-    for (HeapObject* obj = iterator.next();
+    HeapIterator iterator;
+    for (HeapObject* obj = iterator.Next();
          obj != NULL;
-         obj = iterator.next()) {
+         obj = iterator.Next()) {
       InstanceType type = obj->map()->instance_type();
       ASSERT(0 <= type && type <= LAST_TYPE);
       stats->objects_per_type[type]++;
       stats->size_per_type[type] += obj->Size();
     }
   }
 }
 
 
 intptr_t Heap::PromotedSpaceSize() {
@@ skipping 410 matching lines @@
 }
 
 
 class HeapObjectsFilter {
  public:
   virtual ~HeapObjectsFilter() {}
   virtual bool SkipObject(HeapObject* object) = 0;
 };
 
 
-class FreeListNodesFilter : public HeapObjectsFilter {
- public:
-  FreeListNodesFilter() {
-    MarkFreeListNodes();
-  }
-
-  bool SkipObject(HeapObject* object) {
-    if (IntrusiveMarking::IsMarked(object)) {
-      IntrusiveMarking::ClearMark(object);
-      return true;
-    } else {
-      return false;
-    }
-  }
-
- private:
-  void MarkFreeListNodes() {
-    Heap::old_pointer_space()->MarkFreeListNodes();
-    Heap::old_data_space()->MarkFreeListNodes();
-    MarkCodeSpaceFreeListNodes();
-    Heap::map_space()->MarkFreeListNodes();
-    Heap::cell_space()->MarkFreeListNodes();
-  }
-
-  void MarkCodeSpaceFreeListNodes() {
-    // For code space, using FreeListNode::IsFreeListNode is OK.
-    HeapObjectIterator iter(Heap::code_space());
-    for (HeapObject* obj = iter.next_object();
-         obj != NULL;
-         obj = iter.next_object()) {
-      if (FreeListNode::IsFreeListNode(obj)) {
-        IntrusiveMarking::SetMark(obj);
-      }
-    }
-  }
-
-  AssertNoAllocation no_alloc;
-};
-
-
 class UnreachableObjectsFilter : public HeapObjectsFilter {
  public:
   UnreachableObjectsFilter() {
     MarkUnreachableObjects();
   }
 
   bool SkipObject(HeapObject* object) {
     if (IntrusiveMarking::IsMarked(object)) {
       IntrusiveMarking::ClearMark(object);
       return true;
@@ skipping 24 matching lines @@
       HeapObject* obj = list_.RemoveLast();
       obj->Iterate(this);
     }
 
    private:
     List<HeapObject*> list_;
   };
 
   void MarkUnreachableObjects() {
     HeapIterator iterator;
-    for (HeapObject* obj = iterator.next();
+    for (HeapObject* obj = iterator.Next();
          obj != NULL;
-         obj = iterator.next()) {
+         obj = iterator.Next()) {
       IntrusiveMarking::SetMark(obj);
     }
     UnmarkingVisitor visitor;
     Heap::IterateRoots(&visitor, VISIT_ALL);
     while (visitor.can_process())
       visitor.ProcessNext();
   }
 
   AssertNoAllocation no_alloc;
 };
 
 
-HeapIterator::HeapIterator()
-    : filtering_(HeapIterator::kNoFiltering),
-      filter_(NULL) {
-  Init();
-}
-
-
-HeapIterator::HeapIterator(HeapIterator::HeapObjectsFiltering filtering)
-    : filtering_(filtering),
-      filter_(NULL) {
+HeapIterator::HeapIterator() {
   Init();
 }
 
 
 HeapIterator::~HeapIterator() {
   Shutdown();
 }
 
 
 void HeapIterator::Init() {
   // Start the iteration.
-  space_iterator_ = filtering_ == kNoFiltering ? new SpaceIterator :
-      new SpaceIterator(IntrusiveMarking::SizeOfMarkedObject);
-  switch (filtering_) {
-    case kFilterFreeListNodes:
-      filter_ = new FreeListNodesFilter;
-      break;
-    case kFilterUnreachable:
-      filter_ = new UnreachableObjectsFilter;
-      break;
-    default:
-      break;
-  }
+  Heap::EnsureHeapIsIterable();
+  space_iterator_ = new SpaceIterator();
   object_iterator_ = space_iterator_->next();
 }
 
 
 void HeapIterator::Shutdown() {
-#ifdef DEBUG
-  // Assert that in filtering mode we have iterated through all
-  // objects. Otherwise, heap will be left in an inconsistent state.
-  if (filtering_ != kNoFiltering) {
-    ASSERT(object_iterator_ == NULL);
-  }
-#endif
   // Make sure the last iterator is deallocated.
   delete space_iterator_;
   space_iterator_ = NULL;
   object_iterator_ = NULL;
-  delete filter_;
-  filter_ = NULL;
 }
 
 
-HeapObject* HeapIterator::next() {
-  if (filter_ == NULL) return NextObject();
-
-  HeapObject* obj = NextObject();
-  while (obj != NULL && filter_->SkipObject(obj)) obj = NextObject();
-  return obj;
-}
-
-
-HeapObject* HeapIterator::NextObject() {
+HeapObject* HeapIterator::Next() {
   // No iterator means we are done.
   if (object_iterator_ == NULL) return NULL;
 
   if (HeapObject* obj = object_iterator_->next_object()) {
     // If the current iterator has more objects we are fine.
     return obj;
   } else {
     // Go though the spaces looking for one that has objects.
     while (space_iterator_->has_next()) {
       object_iterator_ = space_iterator_->next();
       if (HeapObject* obj = object_iterator_->next_object()) {
         return obj;
       }
     }
   }
   // Done with the last space.
   object_iterator_ = NULL;
   return NULL;
 }
 
 
-void HeapIterator::reset() {
+void HeapIterator::Reset() {
   // Restart the iterator.
   Shutdown();
   Init();
 }
 
 
 #if defined(DEBUG) || defined(LIVE_OBJECT_LIST)
 
 Object* const PathTracer::kAnyGlobalObject = reinterpret_cast<Object*>(NULL);
 
@@ skipping 182 matching lines @@
 }
 #endif
 
 
 static intptr_t CountTotalHolesSize() {
   intptr_t holes_size = 0;
   OldSpaces spaces;
   for (OldSpace* space = spaces.next();
        space != NULL;
        space = spaces.next()) {
-    holes_size += space->Waste() + space->AvailableFree();
+    holes_size += space->Waste() + space->Available();
   }
   return holes_size;
 }
 
 
 GCTracer::GCTracer()
     : start_time_(0.0),
       start_size_(0),
       gc_count_(0),
       full_gc_count_(0),
@@ skipping 220 matching lines @@
 void ExternalStringTable::TearDown() {
   new_space_strings_.Free();
   old_space_strings_.Free();
 }
 
 
 List<Object*> ExternalStringTable::new_space_strings_;
 List<Object*> ExternalStringTable::old_space_strings_;
 
 } }  // namespace v8::internal