MarkBit cleanup: They have to be accessed through Marking accessors.

src/heap/mark-compact.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/v8.h"
 
 #include "src/base/atomicops.h"
 #include "src/base/bits.h"
 #include "src/code-stubs.h"
 #include "src/compilation-cache.h"
@@ skipping 456 matching lines @@
 
 void MarkCompactCollector::ClearMarkbits() {
   ClearMarkbitsInPagedSpace(heap_->code_space());
   ClearMarkbitsInPagedSpace(heap_->map_space());
   ClearMarkbitsInPagedSpace(heap_->old_space());
   ClearMarkbitsInPagedSpace(heap_->cell_space());
   ClearMarkbitsInNewSpace(heap_->new_space());
 
   LargeObjectIterator it(heap_->lo_space());
   for (HeapObject* obj = it.Next(); obj != NULL; obj = it.Next()) {
-    MarkBit mark_bit = Marking::MarkBitFrom(obj);
-    mark_bit.Clear();
-    mark_bit.Next().Clear();
+    Marking::MarkWhite(Marking::MarkBitFrom(obj));
     Page::FromAddress(obj->address())->ResetProgressBar();
     Page::FromAddress(obj->address())->ResetLiveBytes();
   }
 }
 
 
 class MarkCompactCollector::SweeperTask : public v8::Task {
  public:
   SweeperTask(Heap* heap, PagedSpace* space) : heap_(heap), space_(space) {}
 
@@ skipping 66 matching lines @@
     // to only refill them for the old space.
     return;
   }
 
   intptr_t freed_bytes = space->free_list()->Concatenate(free_list);
   space->AddToAccountingStats(freed_bytes);
   space->DecrementUnsweptFreeBytes(freed_bytes);
 }
 
 
+void Marking::SetAllMarkBitsInRange(MarkBit start, MarkBit end) {
+  MarkBit::CellType* start_cell = start.cell();
+  MarkBit::CellType* end_cell = end.cell();
+  MarkBit::CellType start_mask = ~(start.mask() - 1);
+  MarkBit::CellType end_mask = (end.mask() << 1) - 1;
+
+  if (start_cell == end_cell) {
+    *start_cell |= start_mask & end_mask;
+  } else {
+    *start_cell |= start_mask;
+    for (MarkBit::CellType* cell = start_cell + 1; cell < end_cell; cell++) {
+      *cell = ~0;
+    }
+    *end_cell |= end_mask;
+  }
+}
+
+
+void Marking::ClearAllMarkBitsOfCellsContainedInRange(MarkBit start,
+                                                      MarkBit end) {
+  MarkBit::CellType* start_cell = start.cell();
+  MarkBit::CellType* end_cell = end.cell();
+  for (MarkBit::CellType* cell = start_cell; cell <= end_cell; cell++) {
+    *cell = 0;
+  }
+}
+
+
 void Marking::TransferMark(Address old_start, Address new_start) {
   // This is only used when resizing an object.
   DCHECK(MemoryChunk::FromAddress(old_start) ==
          MemoryChunk::FromAddress(new_start));
 
   if (!heap_->incremental_marking()->IsMarking()) return;
 
   // If the mark doesn't move, we don't check the color of the object.
   // It doesn't matter whether the object is black, since it hasn't changed
   // size, so the adjustment to the live data count will be zero anyway.
   if (old_start == new_start) return;
 
   MarkBit new_mark_bit = MarkBitFrom(new_start);
   MarkBit old_mark_bit = MarkBitFrom(old_start);
 
 #ifdef DEBUG
   ObjectColor old_color = Color(old_mark_bit);
 #endif
 
   if (Marking::IsBlack(old_mark_bit)) {
-    old_mark_bit.Clear();
-    DCHECK(IsWhite(old_mark_bit));
+    Marking::BlackToWhite(old_mark_bit);
     Marking::MarkBlack(new_mark_bit);
     return;
   } else if (Marking::IsGrey(old_mark_bit)) {
-    old_mark_bit.Clear();
-    old_mark_bit.Next().Clear();
-    DCHECK(IsWhite(old_mark_bit));
+    Marking::GreyToWhite(old_mark_bit);
     heap_->incremental_marking()->WhiteToGreyAndPush(
         HeapObject::FromAddress(new_start), new_mark_bit);
     heap_->incremental_marking()->RestartIfNotMarking();
   }
 
 #ifdef DEBUG
   ObjectColor new_color = Color(new_mark_bit);
   DCHECK(new_color == old_color);
 #endif
 }
@@ skipping 359 matching lines @@
   JSFunction* candidate = jsfunction_candidates_head_;
   JSFunction* next_candidate;
   while (candidate != NULL) {
     next_candidate = GetNextCandidate(candidate);
     ClearNextCandidate(candidate, undefined);
 
     SharedFunctionInfo* shared = candidate->shared();
 
     Code* code = shared->code();
     MarkBit code_mark = Marking::MarkBitFrom(code);
-    if (!code_mark.Get()) {
+    if (Marking::IsWhite(code_mark)) {
       if (FLAG_trace_code_flushing && shared->is_compiled()) {
         PrintF("[code-flushing clears: ");
         shared->ShortPrint();
         PrintF(" - age: %d]\n", code->GetAge());
       }
       shared->set_code(lazy_compile);
       candidate->set_code(lazy_compile);
     } else {
+      DCHECK(Marking::IsBlack(code_mark));
       candidate->set_code(code);
     }
 
     // We are in the middle of a GC cycle so the write barrier in the code
     // setter did not record the slot update and we have to do that manually.
     Address slot = candidate->address() + JSFunction::kCodeEntryOffset;
     Code* target = Code::cast(Code::GetObjectFromEntryAddress(slot));
     isolate_->heap()->mark_compact_collector()->RecordCodeEntrySlot(slot,
                                                                     target);
 
@@ skipping 13 matching lines @@
   Code* lazy_compile = isolate_->builtins()->builtin(Builtins::kCompileLazy);
 
   SharedFunctionInfo* candidate = shared_function_info_candidates_head_;
   SharedFunctionInfo* next_candidate;
   while (candidate != NULL) {
     next_candidate = GetNextCandidate(candidate);
     ClearNextCandidate(candidate);
 
     Code* code = candidate->code();
     MarkBit code_mark = Marking::MarkBitFrom(code);
-    if (!code_mark.Get()) {
+    if (Marking::IsWhite(code_mark)) {
       if (FLAG_trace_code_flushing && candidate->is_compiled()) {
         PrintF("[code-flushing clears: ");
         candidate->ShortPrint();
         PrintF(" - age: %d]\n", code->GetAge());
       }
       candidate->set_code(lazy_compile);
     }
 
     Object** code_slot =
         HeapObject::RawField(candidate, SharedFunctionInfo::kCodeOffset);
@@ skipping 17 matching lines @@
     next_holder = GetNextCodeMap(holder);
     ClearNextCodeMap(holder);
 
     FixedArray* code_map = FixedArray::cast(holder->optimized_code_map());
     int new_length = SharedFunctionInfo::kEntriesStart;
     int old_length = code_map->length();
     for (int i = SharedFunctionInfo::kEntriesStart; i < old_length;
          i += SharedFunctionInfo::kEntryLength) {
       Code* code =
           Code::cast(code_map->get(i + SharedFunctionInfo::kCachedCodeOffset));
-      if (!Marking::MarkBitFrom(code).Get()) continue;
-
+      if (Marking::IsWhite(Marking::MarkBitFrom(code))) continue;
+      DCHECK(Marking::IsBlack(Marking::MarkBitFrom(code)));
       // Move every slot in the entry.
       for (int j = 0; j < SharedFunctionInfo::kEntryLength; j++) {
         int dst_index = new_length++;
         Object** slot = code_map->RawFieldOfElementAt(dst_index);
         Object* object = code_map->get(i + j);
         code_map->set(dst_index, object);
         if (j == SharedFunctionInfo::kOsrAstIdOffset) {
           DCHECK(object->IsSmi());
         } else {
           DCHECK(
@@ skipping 255 matching lines @@
   // Marks the object black and pushes it on the marking stack.
   INLINE(static void MarkObject(Heap* heap, HeapObject* object)) {
     MarkBit mark = Marking::MarkBitFrom(object);
     heap->mark_compact_collector()->MarkObject(object, mark);
   }
 
   // Marks the object black without pushing it on the marking stack.
   // Returns true if object needed marking and false otherwise.
   INLINE(static bool MarkObjectWithoutPush(Heap* heap, HeapObject* object)) {
     MarkBit mark_bit = Marking::MarkBitFrom(object);
-    if (!mark_bit.Get()) {
+    if (Marking::IsWhite(mark_bit)) {
       heap->mark_compact_collector()->SetMark(object, mark_bit);
       return true;
     }
     return false;
   }
 
   // Mark object pointed to by p.
   INLINE(static void MarkObjectByPointer(MarkCompactCollector* collector,
                                          Object** anchor_slot, Object** p)) {
     if (!(*p)->IsHeapObject()) return;
@@ skipping 30 matching lines @@
     if (check.HasOverflowed()) return false;
 
     MarkCompactCollector* collector = heap->mark_compact_collector();
     // Visit the unmarked objects.
     for (Object** p = start; p < end; p++) {
       Object* o = *p;
       if (!o->IsHeapObject()) continue;
       collector->RecordSlot(start, p, o);
       HeapObject* obj = HeapObject::cast(o);
       MarkBit mark = Marking::MarkBitFrom(obj);
-      if (mark.Get()) continue;
+      if (Marking::IsBlackOrGrey(mark)) continue;
       VisitUnmarkedObject(collector, obj);
     }
     return true;
   }
 
  private:
   template <int id>
   static inline void TrackObjectStatsAndVisit(Map* map, HeapObject* obj);
 
   // Code flushing support.
@@ skipping 324 matching lines @@
   // ProcessTopOptimizedFrame.
   void VisitNextCodeLink(Object** p) {}
 
  private:
   void MarkObjectByPointer(Object** p) {
     if (!(*p)->IsHeapObject()) return;
 
     // Replace flat cons strings in place.
     HeapObject* object = ShortCircuitConsString(p);
     MarkBit mark_bit = Marking::MarkBitFrom(object);
-    if (mark_bit.Get()) return;
+    if (Marking::IsBlackOrGrey(mark_bit)) return;
 
     Map* map = object->map();
     // Mark the object.
     collector_->SetMark(object, mark_bit);
 
     // Mark the map pointer and body, and push them on the marking stack.
     MarkBit map_mark = Marking::MarkBitFrom(map);
     collector_->MarkObject(map, map_mark);
     MarkCompactMarkingVisitor::IterateBody(map, object);
 
@@ skipping 10 matching lines @@
 template <bool finalize_external_strings>
 class StringTableCleaner : public ObjectVisitor {
  public:
   explicit StringTableCleaner(Heap* heap) : heap_(heap), pointers_removed_(0) {}
 
   virtual void VisitPointers(Object** start, Object** end) {
     // Visit all HeapObject pointers in [start, end).
     for (Object** p = start; p < end; p++) {
       Object* o = *p;
       if (o->IsHeapObject() &&
-          !Marking::MarkBitFrom(HeapObject::cast(o)).Get()) {
+          Marking::IsWhite(Marking::MarkBitFrom(HeapObject::cast(o)))) {
         if (finalize_external_strings) {
           DCHECK(o->IsExternalString());
           heap_->FinalizeExternalString(String::cast(*p));
         } else {
           pointers_removed_++;
         }
         // Set the entry to the_hole_value (as deleted).
         *p = heap_->the_hole_value();
       }
     }
@@ skipping 12 matching lines @@
 
 typedef StringTableCleaner<false> InternalizedStringTableCleaner;
 typedef StringTableCleaner<true> ExternalStringTableCleaner;
 
 
 // Implementation of WeakObjectRetainer for mark compact GCs. All marked objects
 // are retained.
 class MarkCompactWeakObjectRetainer : public WeakObjectRetainer {
  public:
   virtual Object* RetainAs(Object* object) {
-    if (Marking::MarkBitFrom(HeapObject::cast(object)).Get()) {
+    if (Marking::IsBlackOrGrey(
+            Marking::MarkBitFrom(HeapObject::cast(object)))) {
       return object;
     } else if (object->IsAllocationSite() &&
                !(AllocationSite::cast(object)->IsZombie())) {
       // "dead" AllocationSites need to live long enough for a traversal of new
       // space. These sites get a one-time reprieve.
       AllocationSite* site = AllocationSite::cast(object);
       site->MarkZombie();
       site->GetHeap()->mark_compact_collector()->MarkAllocationSite(site);
       return object;
     } else {
@@ skipping 93 matching lines @@
     MarkBit::CellType current_cell = *cell;
     if (current_cell == 0) continue;
 
     int offset = 0;
     while (current_cell != 0) {
       int trailing_zeros = base::bits::CountTrailingZeros32(current_cell);
       current_cell >>= trailing_zeros;
       offset += trailing_zeros;
       Address address = cell_base + offset * kPointerSize;
       HeapObject* object = HeapObject::FromAddress(address);
+      DCHECK(Marking::IsBlack(Marking::MarkBitFrom(object)));
 
       int size = object->Size();
       survivors_size += size;
 
       Heap::UpdateAllocationSiteFeedback(object, Heap::RECORD_SCRATCHPAD_SLOT);
 
-      offset++;
-      current_cell >>= 1;
+      offset += 2;
+      current_cell >>= 2;
 
       // TODO(hpayer): Refactor EvacuateObject and call this function instead.
       if (heap()->ShouldBePromoted(object->address(), size) &&
           TryPromoteObject(object, size)) {
         continue;
       }
 
       AllocationResult allocation = new_space->AllocateRaw(size);
       if (allocation.IsRetry()) {
         if (!new_space->AddFreshPage()) {
@@ skipping 37 matching lines @@
     if (marking_deque->IsFull()) return;
   }
 }
 
 
 bool MarkCompactCollector::IsUnmarkedHeapObject(Object** p) {
   Object* o = *p;
   if (!o->IsHeapObject()) return false;
   HeapObject* heap_object = HeapObject::cast(o);
   MarkBit mark = Marking::MarkBitFrom(heap_object);
-  return !mark.Get();
+  return Marking::IsWhite(mark);
 }
 
 
 bool MarkCompactCollector::IsUnmarkedHeapObjectWithHeap(Heap* heap,
                                                         Object** p) {
   Object* o = *p;
   DCHECK(o->IsHeapObject());
   HeapObject* heap_object = HeapObject::cast(o);
   MarkBit mark = Marking::MarkBitFrom(heap_object);
-  return !mark.Get();
+  return Marking::IsWhite(mark);
 }
 
 
 void MarkCompactCollector::MarkStringTable(RootMarkingVisitor* visitor) {
   StringTable* string_table = heap()->string_table();
   // Mark the string table itself.
   MarkBit string_table_mark = Marking::MarkBitFrom(string_table);
-  if (!string_table_mark.Get()) {
+  if (Marking::IsWhite(string_table_mark)) {
     // String table could have already been marked by visiting the handles list.
     SetMark(string_table, string_table_mark);
   }
   // Explicitly mark the prefix.
   string_table->IteratePrefix(visitor);
   ProcessMarkingDeque();
 }
 
 
 void MarkCompactCollector::MarkAllocationSite(AllocationSite* site) {
@@ skipping 169 matching lines @@
   int length = retained_maps->Length();
   int new_length = 0;
   for (int i = 0; i < length; i += 2) {
     DCHECK(retained_maps->Get(i)->IsWeakCell());
     WeakCell* cell = WeakCell::cast(retained_maps->Get(i));
     if (cell->cleared()) continue;
     int age = Smi::cast(retained_maps->Get(i + 1))->value();
     int new_age;
     Map* map = Map::cast(cell->value());
     MarkBit map_mark = Marking::MarkBitFrom(map);
-    if (!map_mark.Get()) {
+    if (Marking::IsWhite(map_mark)) {
       if (age == 0) {
         // The map has aged. Do not retain this map.
         continue;
       }
       Object* constructor = map->GetConstructor();
-      if (!constructor->IsHeapObject() ||
-          !Marking::MarkBitFrom(HeapObject::cast(constructor)).Get()) {
+      if (!constructor->IsHeapObject() || Marking::IsWhite(Marking::MarkBitFrom(
+                                              HeapObject::cast(constructor)))) {
         // The constructor is dead, no new objects with this map can
         // be created. Do not retain this map.
         continue;
       }
       Object* prototype = map->prototype();
       if (prototype->IsHeapObject() &&
-          !Marking::MarkBitFrom(HeapObject::cast(prototype)).Get()) {
+          Marking::IsWhite(Marking::MarkBitFrom(HeapObject::cast(prototype)))) {
         // The prototype is not marked, age the map.
         new_age = age - 1;
       } else {
         // The prototype and the constructor are marked, this map keeps only
         // transition tree alive, not JSObjects. Do not age the map.
         new_age = age;
       }
       MarkObject(map, map_mark);
     } else {
       new_age = FLAG_retain_maps_for_n_gc;
@@ skipping 193 matching lines @@
   for (HeapObject* obj = map_iterator.Next(); obj != NULL;
        obj = map_iterator.Next()) {
     Map* map = Map::cast(obj);
 
     if (!map->CanTransition()) continue;
 
     MarkBit map_mark = Marking::MarkBitFrom(map);
     ClearNonLivePrototypeTransitions(map);
     ClearNonLiveMapTransitions(map, map_mark);
 
-    if (!map_mark.Get()) {
+    if (Marking::IsWhite(map_mark)) {
       have_code_to_deoptimize_ |=
           map->dependent_code()->MarkCodeForDeoptimization(
               isolate(), DependentCode::kWeakCodeGroup);
       map->set_dependent_code(DependentCode::cast(heap()->empty_fixed_array()));
     }
   }
 
   WeakHashTable* table = heap_->weak_object_to_code_table();
   uint32_t capacity = table->Capacity();
   for (uint32_t i = 0; i < capacity; i++) {
@@ skipping 47 matching lines @@
 
 
 void MarkCompactCollector::ClearNonLiveMapTransitions(Map* map,
                                                       MarkBit map_mark) {
   Object* potential_parent = map->GetBackPointer();
   if (!potential_parent->IsMap()) return;
   Map* parent = Map::cast(potential_parent);
 
   // Follow back pointer, check whether we are dealing with a map transition
   // from a live map to a dead path and in case clear transitions of parent.
-  bool current_is_alive = map_mark.Get();
-  bool parent_is_alive = Marking::MarkBitFrom(parent).Get();
+  bool current_is_alive = Marking::IsBlackOrGrey(map_mark);
+  bool parent_is_alive = Marking::IsBlackOrGrey(Marking::MarkBitFrom(parent));
   if (!current_is_alive && parent_is_alive) {
     ClearMapTransitions(parent, map);
   }
 }
 
 
 // Clear a possible back pointer in case the transition leads to a dead map.
 // Return true in case a back pointer has been cleared and false otherwise.
 bool MarkCompactCollector::ClearMapBackPointer(Map* target) {
-  if (Marking::MarkBitFrom(target).Get()) return false;
+  if (Marking::IsBlackOrGrey(Marking::MarkBitFrom(target))) return false;
   target->SetBackPointer(heap_->undefined_value(), SKIP_WRITE_BARRIER);
   return true;
 }
 
 
 void MarkCompactCollector::ClearMapTransitions(Map* map, Map* dead_transition) {
   Object* transitions = map->raw_transitions();
   int num_transitions = TransitionArray::NumberOfTransitions(transitions);
 
   int number_of_own_descriptors = map->NumberOfOwnDescriptors();
@@ skipping 1048 matching lines @@
     return false;
   }
 
   Address code_start = code->address();
   Address code_end = code_start + code->Size();
 
   uint32_t start_index = MemoryChunk::FastAddressToMarkbitIndex(code_start);
   uint32_t end_index =
       MemoryChunk::FastAddressToMarkbitIndex(code_end - kPointerSize);
 
+  // TODO(hpayer): Filter out invalidated code in
+  // ClearInvalidSlotsBufferEntries.
   Bitmap* b = p->markbits();
 
   MarkBit start_mark_bit = b->MarkBitFromIndex(start_index);
   MarkBit end_mark_bit = b->MarkBitFromIndex(end_index);
 
-  MarkBit::CellType* start_cell = start_mark_bit.cell();
-  MarkBit::CellType* end_cell = end_mark_bit.cell();
-
   if (value) {
-    MarkBit::CellType start_mask = ~(start_mark_bit.mask() - 1);
-    MarkBit::CellType end_mask = (end_mark_bit.mask() << 1) - 1;
-
-    if (start_cell == end_cell) {
-      *start_cell |= start_mask & end_mask;
-    } else {
-      *start_cell |= start_mask;
-      for (MarkBit::CellType* cell = start_cell + 1; cell < end_cell; cell++) {
-        *cell = ~0;
-      }
-      *end_cell |= end_mask;
-    }
+    Marking::SetAllMarkBitsInRange(start_mark_bit, end_mark_bit);
   } else {
-    for (MarkBit::CellType* cell = start_cell; cell <= end_cell; cell++) {
-      *cell = 0;
-    }
+    Marking::ClearAllMarkBitsOfCellsContainedInRange(start_mark_bit,
+                                                     end_mark_bit);
   }
 
   return true;
 }
 
 
 static bool IsOnInvalidatedCodeObject(Address addr) {
   // We did not record any slots in large objects thus
   // we can safely go to the page from the slot address.
   Page* p = Page::FromAddress(addr);
 
   // First check owner's identity because old space is swept concurrently or
   // lazily and might still have non-zero mark-bits on some pages.
   if (p->owner()->identity() != CODE_SPACE) return false;
 
   // In code space only bits on evacuation candidates (but we don't record
   // any slots on them) and under invalidated code objects are non-zero.
   MarkBit mark_bit =
       p->markbits()->MarkBitFromIndex(Page::FastAddressToMarkbitIndex(addr));
 
-  return mark_bit.Get();
+  return Marking::IsBlackOrGrey(mark_bit);
 }
 
 
 void MarkCompactCollector::InvalidateCode(Code* code) {
   if (heap_->incremental_marking()->IsCompacting() &&
       !ShouldSkipEvacuationSlotRecording(code)) {
     DCHECK(compacting_);
 
     // If the object is white than no slots were recorded on it yet.
     MarkBit mark_bit = Marking::MarkBitFrom(code);
@@ skipping 1142 matching lines @@
   SlotsBuffer* buffer = *buffer_address;
   while (buffer != NULL) {
     SlotsBuffer* next_buffer = buffer->next();
     DeallocateBuffer(buffer);
     buffer = next_buffer;
   }
   *buffer_address = NULL;
 }
 }
 }  // namespace v8::internal