Revert of Only use FreeSpace objects in the free list.

src/heap/spaces.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/v8.h"
 
 #include "src/base/bits.h"
 #include "src/base/platform/platform.h"
 #include "src/full-codegen.h"
 #include "src/heap/mark-compact.h"
@@ skipping 1010 matching lines @@
 
   return true;
 }
 
 
 bool PagedSpace::Expand() {
   if (!CanExpand()) return false;
 
   intptr_t size = AreaSize();
 
+  if (anchor_.next_page() == &anchor_) {
+    size = SizeOfFirstPage();
+  }
+
   Page* p = heap()->isolate()->memory_allocator()->AllocatePage(size, this,
                                                                 executable());
   if (p == NULL) return false;
 
   DCHECK(Capacity() <= max_capacity_);
 
   p->InsertAfter(anchor_.prev_page());
 
   return true;
 }
 
 
+intptr_t PagedSpace::SizeOfFirstPage() {
+  // If the snapshot contains a custom script, all size guarantees are off.
+  if (Snapshot::EmbedsScript()) return AreaSize();
+  // If using an ool constant pool then transfer the constant pool allowance
+  // from the code space to the old pointer space.
+  static const int constant_pool_delta = FLAG_enable_ool_constant_pool ? 48 : 0;
+  int size = 0;
+  switch (identity()) {
+    case OLD_POINTER_SPACE:
+      size = (128 + constant_pool_delta) * kPointerSize * KB;
+      break;
+    case OLD_DATA_SPACE:
+      size = 192 * KB;
+      break;
+    case MAP_SPACE:
+      size = 16 * kPointerSize * KB;
+      break;
+    case CELL_SPACE:
+      size = 16 * kPointerSize * KB;
+      break;
+    case PROPERTY_CELL_SPACE:
+      size = 8 * kPointerSize * KB;
+      break;
+    case CODE_SPACE: {
+      CodeRange* code_range = heap()->isolate()->code_range();
+      if (code_range != NULL && code_range->valid()) {
+        // When code range exists, code pages are allocated in a special way
+        // (from the reserved code range). That part of the code is not yet
+        // upgraded to handle small pages.
+        size = AreaSize();
+      } else {
+        size = RoundUp((480 - constant_pool_delta) * KB *
+                           FullCodeGenerator::kBootCodeSizeMultiplier / 100,
+                       kPointerSize);
+      }
+      break;
+    }
+    default:
+      UNREACHABLE();
+  }
+  return Min(size, AreaSize());
+}
+
+
 int PagedSpace::CountTotalPages() {
   PageIterator it(this);
   int count = 0;
   while (it.has_next()) {
     it.next();
     count++;
   }
   return count;
 }
 
@@ skipping 974 matching lines @@
   if (from_space_.is_committed()) {
     size += from_space_.CommittedPhysicalMemory();
   }
   return size;
 }
 
 
 // -----------------------------------------------------------------------------
 // Free lists for old object spaces implementation
 
+void FreeListNode::set_size(Heap* heap, int size_in_bytes) {
+  DCHECK(size_in_bytes > 0);
+  DCHECK(IsAligned(size_in_bytes, kPointerSize));
+
+  // We write a map and possibly size information to the block.  If the block
+  // is big enough to be a FreeSpace with at least one extra word (the next
+  // pointer), we set its map to be the free space map and its size to an
+  // appropriate array length for the desired size from HeapObject::Size().
+  // If the block is too small (eg, one or two words), to hold both a size
+  // field and a next pointer, we give it a filler map that gives it the
+  // correct size.
+  if (size_in_bytes > FreeSpace::kHeaderSize) {
+    // Can't use FreeSpace::cast because it fails during deserialization.
+    // We have to set the size first with a release store before we store
+    // the map because a concurrent store buffer scan on scavenge must not
+    // observe a map with an invalid size.
+    FreeSpace* this_as_free_space = reinterpret_cast<FreeSpace*>(this);
+    this_as_free_space->nobarrier_set_size(size_in_bytes);
+    synchronized_set_map_no_write_barrier(heap->raw_unchecked_free_space_map());
+  } else if (size_in_bytes == kPointerSize) {
+    set_map_no_write_barrier(heap->raw_unchecked_one_pointer_filler_map());
+  } else if (size_in_bytes == 2 * kPointerSize) {
+    set_map_no_write_barrier(heap->raw_unchecked_two_pointer_filler_map());
+  } else {
+    UNREACHABLE();
+  }
+  // We would like to DCHECK(Size() == size_in_bytes) but this would fail during
+  // deserialization because the free space map is not done yet.
+}
+
+
+FreeListNode* FreeListNode::next() {
+  DCHECK(IsFreeListNode(this));
+  if (map() == GetHeap()->raw_unchecked_free_space_map()) {
+    DCHECK(map() == NULL || Size() >= kNextOffset + kPointerSize);
+    return reinterpret_cast<FreeListNode*>(
+        Memory::Address_at(address() + kNextOffset));
+  } else {
+    return reinterpret_cast<FreeListNode*>(
+        Memory::Address_at(address() + kPointerSize));
+  }
+}
+
+
+FreeListNode** FreeListNode::next_address() {
+  DCHECK(IsFreeListNode(this));
+  if (map() == GetHeap()->raw_unchecked_free_space_map()) {
+    DCHECK(Size() >= kNextOffset + kPointerSize);
+    return reinterpret_cast<FreeListNode**>(address() + kNextOffset);
+  } else {
+    return reinterpret_cast<FreeListNode**>(address() + kPointerSize);
+  }
+}
+
+
+void FreeListNode::set_next(FreeListNode* next) {
+  DCHECK(IsFreeListNode(this));
+  // While we are booting the VM the free space map will actually be null.  So
+  // we have to make sure that we don't try to use it for anything at that
+  // stage.
+  if (map() == GetHeap()->raw_unchecked_free_space_map()) {
+    DCHECK(map() == NULL || Size() >= kNextOffset + kPointerSize);
+    base::NoBarrier_Store(
+        reinterpret_cast<base::AtomicWord*>(address() + kNextOffset),
+        reinterpret_cast<base::AtomicWord>(next));
+  } else {
+    base::NoBarrier_Store(
+        reinterpret_cast<base::AtomicWord*>(address() + kPointerSize),
+        reinterpret_cast<base::AtomicWord>(next));
+  }
+}
+
+
 intptr_t FreeListCategory::Concatenate(FreeListCategory* category) {
   intptr_t free_bytes = 0;
   if (category->top() != NULL) {
     // This is safe (not going to deadlock) since Concatenate operations
     // are never performed on the same free lists at the same time in
     // reverse order.
     base::LockGuard<base::Mutex> target_lock_guard(mutex());
     base::LockGuard<base::Mutex> source_lock_guard(category->mutex());
     DCHECK(category->end_ != NULL);
     free_bytes = category->available();
@@ skipping 13 matching lines @@
 
 void FreeListCategory::Reset() {
   set_top(NULL);
   set_end(NULL);
   set_available(0);
 }
 
 
 intptr_t FreeListCategory::EvictFreeListItemsInList(Page* p) {
   int sum = 0;
-  FreeSpace* t = top();
-  FreeSpace** n = &t;
+  FreeListNode* t = top();
+  FreeListNode** n = &t;
   while (*n != NULL) {
     if (Page::FromAddress((*n)->address()) == p) {
-      FreeSpace* free_space = *n;
+      FreeSpace* free_space = reinterpret_cast<FreeSpace*>(*n);
       sum += free_space->Size();
       *n = (*n)->next();
     } else {
       n = (*n)->next_address();
     }
   }
   set_top(t);
   if (top() == NULL) {
     set_end(NULL);
   }
   available_ -= sum;
   return sum;
 }
 
 
 bool FreeListCategory::ContainsPageFreeListItemsInList(Page* p) {
-  FreeSpace* node = top();
+  FreeListNode* node = top();
   while (node != NULL) {
     if (Page::FromAddress(node->address()) == p) return true;
     node = node->next();
   }
   return false;
 }
 
 
-FreeSpace* FreeListCategory::PickNodeFromList(int* node_size) {
-  FreeSpace* node = top();
+FreeListNode* FreeListCategory::PickNodeFromList(int* node_size) {
+  FreeListNode* node = top();
 
   if (node == NULL) return NULL;
 
   while (node != NULL &&
          Page::FromAddress(node->address())->IsEvacuationCandidate()) {
-    available_ -= node->Size();
+    available_ -= reinterpret_cast<FreeSpace*>(node)->Size();
     node = node->next();
   }
 
   if (node != NULL) {
     set_top(node->next());
-    *node_size = node->Size();
+    *node_size = reinterpret_cast<FreeSpace*>(node)->Size();
     available_ -= *node_size;
   } else {
     set_top(NULL);
   }
 
   if (top() == NULL) {
     set_end(NULL);
   }
 
   return node;
 }
 
 
-FreeSpace* FreeListCategory::PickNodeFromList(int size_in_bytes,
-                                              int* node_size) {
-  FreeSpace* node = PickNodeFromList(node_size);
+FreeListNode* FreeListCategory::PickNodeFromList(int size_in_bytes,
+                                                 int* node_size) {
+  FreeListNode* node = PickNodeFromList(node_size);
   if (node != NULL && *node_size < size_in_bytes) {
     Free(node, *node_size);
     *node_size = 0;
     return NULL;
   }
   return node;
 }
 
 
-void FreeListCategory::Free(FreeSpace* free_space, int size_in_bytes) {
-  DCHECK_LE(FreeList::kSmallListMin, size_in_bytes);
-  free_space->set_next(top());
-  set_top(free_space);
+void FreeListCategory::Free(FreeListNode* node, int size_in_bytes) {
+  node->set_next(top());
+  set_top(node);
   if (end_ == NULL) {
-    end_ = free_space;
+    end_ = node;
   }
   available_ += size_in_bytes;
 }
 
 
 void FreeListCategory::RepairFreeList(Heap* heap) {
-  FreeSpace* n = top();
+  FreeListNode* n = top();
   while (n != NULL) {
     Map** map_location = reinterpret_cast<Map**>(n->address());
     if (*map_location == NULL) {
       *map_location = heap->free_space_map();
     } else {
       DCHECK(*map_location == heap->free_space_map());
     }
     n = n->next();
   }
 }
@@ skipping 18 matching lines @@
   small_list_.Reset();
   medium_list_.Reset();
   large_list_.Reset();
   huge_list_.Reset();
 }
 
 
 int FreeList::Free(Address start, int size_in_bytes) {
   if (size_in_bytes == 0) return 0;
 
-  heap_->CreateFillerObjectAt(start, size_in_bytes);
-
+  FreeListNode* node = FreeListNode::FromAddress(start);
+  node->set_size(heap_, size_in_bytes);
   Page* page = Page::FromAddress(start);
 
   // Early return to drop too-small blocks on the floor.
   if (size_in_bytes < kSmallListMin) {
     page->add_non_available_small_blocks(size_in_bytes);
     return size_in_bytes;
   }
 
-  FreeSpace* free_space = FreeSpace::cast(HeapObject::FromAddress(start));
   // Insert other blocks at the head of a free list of the appropriate
   // magnitude.
   if (size_in_bytes <= kSmallListMax) {
-    small_list_.Free(free_space, size_in_bytes);
+    small_list_.Free(node, size_in_bytes);
     page->add_available_in_small_free_list(size_in_bytes);
   } else if (size_in_bytes <= kMediumListMax) {
-    medium_list_.Free(free_space, size_in_bytes);
+    medium_list_.Free(node, size_in_bytes);
     page->add_available_in_medium_free_list(size_in_bytes);
   } else if (size_in_bytes <= kLargeListMax) {
-    large_list_.Free(free_space, size_in_bytes);
+    large_list_.Free(node, size_in_bytes);
     page->add_available_in_large_free_list(size_in_bytes);
   } else {
-    huge_list_.Free(free_space, size_in_bytes);
+    huge_list_.Free(node, size_in_bytes);
     page->add_available_in_huge_free_list(size_in_bytes);
   }
 
   DCHECK(IsVeryLong() || available() == SumFreeLists());
   return 0;
 }
 
 
-FreeSpace* FreeList::FindNodeFor(int size_in_bytes, int* node_size) {
-  FreeSpace* node = NULL;
+FreeListNode* FreeList::FindNodeFor(int size_in_bytes, int* node_size) {
+  FreeListNode* node = NULL;
   Page* page = NULL;
 
   if (size_in_bytes <= kSmallAllocationMax) {
     node = small_list_.PickNodeFromList(node_size);
     if (node != NULL) {
       DCHECK(size_in_bytes <= *node_size);
       page = Page::FromAddress(node->address());
       page->add_available_in_small_free_list(-(*node_size));
       DCHECK(IsVeryLong() || available() == SumFreeLists());
       return node;
@@ skipping 16 matching lines @@
     if (node != NULL) {
       DCHECK(size_in_bytes <= *node_size);
       page = Page::FromAddress(node->address());
       page->add_available_in_large_free_list(-(*node_size));
       DCHECK(IsVeryLong() || available() == SumFreeLists());
       return node;
     }
   }
 
   int huge_list_available = huge_list_.available();
-  FreeSpace* top_node = huge_list_.top();
-  for (FreeSpace** cur = &top_node; *cur != NULL;
+  FreeListNode* top_node = huge_list_.top();
+  for (FreeListNode** cur = &top_node; *cur != NULL;
        cur = (*cur)->next_address()) {
-    FreeSpace* cur_node = *cur;
+    FreeListNode* cur_node = *cur;
     while (cur_node != NULL &&
            Page::FromAddress(cur_node->address())->IsEvacuationCandidate()) {
-      int size = cur_node->Size();
+      int size = reinterpret_cast<FreeSpace*>(cur_node)->Size();
       huge_list_available -= size;
       page = Page::FromAddress(cur_node->address());
       page->add_available_in_huge_free_list(-size);
       cur_node = cur_node->next();
     }
 
     *cur = cur_node;
     if (cur_node == NULL) {
       huge_list_.set_end(NULL);
       break;
     }
 
-    int size = cur_node->Size();
+    DCHECK((*cur)->map() == heap_->raw_unchecked_free_space_map());
+    FreeSpace* cur_as_free_space = reinterpret_cast<FreeSpace*>(*cur);
+    int size = cur_as_free_space->Size();
     if (size >= size_in_bytes) {
       // Large enough node found.  Unlink it from the list.
       node = *cur;
       *cur = node->next();
       *node_size = size;
       huge_list_available -= size;
       page = Page::FromAddress(node->address());
       page->add_available_in_huge_free_list(-size);
       break;
     }
@@ skipping 52 matching lines @@
   int old_linear_size = static_cast<int>(owner_->limit() - owner_->top());
   // Mark the old linear allocation area with a free space map so it can be
   // skipped when scanning the heap.  This also puts it back in the free list
   // if it is big enough.
   owner_->Free(owner_->top(), old_linear_size);
 
   owner_->heap()->incremental_marking()->OldSpaceStep(size_in_bytes -
                                                       old_linear_size);
 
   int new_node_size = 0;
-  FreeSpace* new_node = FindNodeFor(size_in_bytes, &new_node_size);
+  FreeListNode* new_node = FindNodeFor(size_in_bytes, &new_node_size);
   if (new_node == NULL) {
     owner_->SetTopAndLimit(NULL, NULL);
     return NULL;
   }
 
   int bytes_left = new_node_size - size_in_bytes;
   DCHECK(bytes_left >= 0);
 
 #ifdef DEBUG
   for (int i = 0; i < size_in_bytes / kPointerSize; i++) {
@@ skipping 73 matching lines @@
   small_list_.RepairFreeList(heap);
   medium_list_.RepairFreeList(heap);
   large_list_.RepairFreeList(heap);
   huge_list_.RepairFreeList(heap);
 }
 
 
 #ifdef DEBUG
 intptr_t FreeListCategory::SumFreeList() {
   intptr_t sum = 0;
-  FreeSpace* cur = top();
+  FreeListNode* cur = top();
   while (cur != NULL) {
     DCHECK(cur->map() == cur->GetHeap()->raw_unchecked_free_space_map());
-    sum += cur->nobarrier_size();
+    FreeSpace* cur_as_free_space = reinterpret_cast<FreeSpace*>(cur);
+    sum += cur_as_free_space->nobarrier_size();
     cur = cur->next();
   }
   return sum;
 }
 
 
 static const int kVeryLongFreeList = 500;
 
 
 int FreeListCategory::FreeListLength() {
   int length = 0;
-  FreeSpace* cur = top();
+  FreeListNode* cur = top();
   while (cur != NULL) {
     length++;
     cur = cur->next();
     if (length == kVeryLongFreeList) return length;
   }
   return length;
 }
 
 
 bool FreeList::IsVeryLong() {
@@ skipping 40 matching lines @@
          heap()->mark_compact_collector()->sweeping_in_progress() ||
          (unswept_free_bytes_ == 0));
   return Size() - unswept_free_bytes_ - (limit() - top());
 }
 
 
 // After we have booted, we have created a map which represents free space
 // on the heap.  If there was already a free list then the elements on it
 // were created with the wrong FreeSpaceMap (normally NULL), so we need to
 // fix them.
-void PagedSpace::RepairFreeListsAfterDeserialization() {
-  free_list_.RepairLists(heap());
-}
+void PagedSpace::RepairFreeListsAfterBoot() { free_list_.RepairLists(heap()); }
 
 
 void PagedSpace::EvictEvacuationCandidatesFromFreeLists() {
   if (allocation_info_.top() >= allocation_info_.limit()) return;
 
   if (Page::FromAllocationTop(allocation_info_.top())
           ->IsEvacuationCandidate()) {
     // Create filler object to keep page iterable if it was iterable.
     int remaining =
         static_cast<int>(allocation_info_.limit() - allocation_info_.top());
@@ skipping 572 matching lines @@
     object->ShortPrint();
     PrintF("\n");
   }
   printf(" --------------------------------------\n");
   printf(" Marked: %x, LiveCount: %x\n", mark_size, LiveBytes());
 }
 
 #endif  // DEBUG
 }
 }  // namespace v8::internal