Clean up aligned allocation code in preparation for SIMD alignments.

src/heap/heap.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/v8.h"
 
 #include "src/accessors.h"
 #include "src/api.h"
 #include "src/base/bits.h"
 #include "src/base/once.h"
@@ skipping 1957 matching lines @@
 STATIC_ASSERT((ConstantPoolArray::kExtendedFirstOffset &
                kDoubleAlignmentMask) == 0);  // NOLINT
 STATIC_ASSERT((FixedTypedArrayBase::kDataOffset & kDoubleAlignmentMask) ==
               0);  // NOLINT
 #ifdef V8_HOST_ARCH_32_BIT
 STATIC_ASSERT((HeapNumber::kValueOffset & kDoubleAlignmentMask) !=
               0);  // NOLINT
 #endif
 
 
-HeapObject* Heap::EnsureAligned(HeapObject* object, int size,
-                                AllocationAlignment alignment) {
-  if (alignment == kDoubleAligned &&
-      (OffsetFrom(object->address()) & kDoubleAlignmentMask) != 0) {
-    CreateFillerObjectAt(object->address(), kPointerSize);
-    return HeapObject::FromAddress(object->address() + kPointerSize);
-  } else if (alignment == kDoubleUnaligned &&
-             (OffsetFrom(object->address()) & kDoubleAlignmentMask) == 0) {
-    CreateFillerObjectAt(object->address(), kPointerSize);
-    return HeapObject::FromAddress(object->address() + kPointerSize);
-  } else {
-    CreateFillerObjectAt(object->address() + size - kPointerSize, kPointerSize);
-    return object;
+int GetMaximumMisalignment(AllocationAlignment alignment) {
+  switch (alignment) {
+    case kWordAligned:
+      return 0;
+    case kDoubleAligned:
+    case kDoubleValueAligned:
+      return kDoubleSize / 2;
+    default:
+      UNREACHABLE();
   }
 }
 
 
-HeapObject* Heap::PrecedeWithFiller(HeapObject* object) {
-  CreateFillerObjectAt(object->address(), kPointerSize);
-  return HeapObject::FromAddress(object->address() + kPointerSize);
+int Heap::GetMisalignment(Address address, AllocationAlignment alignment) {
+  intptr_t offset = OffsetFrom(address);
+  if (alignment == kDoubleAligned && (offset & kDoubleAlignmentMask) != 0)
+    return kPointerSize;
+  if (alignment == kDoubleValueAligned && (offset & kDoubleAlignmentMask) == 0)
+    return kPointerSize;
+  return 0;
+}
+
+
+HeapObject* Heap::PrecedeWithFiller(HeapObject* object, int filler_size) {
+  CreateFillerObjectAt(object->address(), filler_size);
+  return HeapObject::FromAddress(object->address() + filler_size);
 }
 
 
 HeapObject* Heap::DoubleAlignForDeserialization(HeapObject* object, int size) {
-  return EnsureAligned(object, size, kDoubleAligned);
+  Address address = object->address();
+  int fill_size = GetMisalignment(address, kDoubleAligned);
+  // If object is not aligned, add fill to align it.
+  if (fill_size) return PrecedeWithFiller(object, kPointerSize);
+
+  // object is aligned. Add fill in the extra space at the end.
+  // TODO(bbudge) Calculate alignment fill earlier to avoid this.
+  CreateFillerObjectAt(address + size - kPointerSize, kPointerSize);
+  return object;
 }
 
 
 enum LoggingAndProfiling {
   LOGGING_AND_PROFILING_ENABLED,
   LOGGING_AND_PROFILING_DISABLED
 };
 
 
 enum MarksHandling { TRANSFER_MARKS, IGNORE_MARKS };
@@ skipping 124 matching lines @@
       heap->OnMoveEvent(target, source, size);
     }
 
     if (marks_handling == TRANSFER_MARKS) {
       if (Marking::TransferColor(source, target)) {
         MemoryChunk::IncrementLiveBytesFromGC(target->address(), size);
       }
     }
   }
 
-  template <int alignment>
+  template <AllocationAlignment alignment>
   static inline bool SemiSpaceCopyObject(Map* map, HeapObject** slot,
                                          HeapObject* object, int object_size) {
     Heap* heap = map->GetHeap();
 
     DCHECK(heap->AllowedToBeMigrated(object, NEW_SPACE));
-    AllocationAlignment align =
-        alignment == kDoubleAlignment ? kDoubleAligned : kWordAligned;
     AllocationResult allocation =
-        heap->new_space()->AllocateRaw(object_size, align);
+        heap->new_space()->AllocateRaw(object_size, alignment);
 
     HeapObject* target = NULL;  // Initialization to please compiler.
     if (allocation.To(&target)) {
       // Order is important here: Set the promotion limit before storing a
       // filler for double alignment or migrating the object. Otherwise we
       // may end up overwriting promotion queue entries when we migrate the
       // object.
       heap->promotion_queue()->SetNewLimit(heap->new_space()->top());
 
       MigrateObject(heap, object, target, object_size);
 
       // Update slot to new target.
       *slot = target;
 
       heap->IncrementSemiSpaceCopiedObjectSize(object_size);
       return true;
     }
     return false;
   }
 
 
-  template <ObjectContents object_contents, int alignment>
+  template <ObjectContents object_contents, AllocationAlignment alignment>
   static inline bool PromoteObject(Map* map, HeapObject** slot,
                                    HeapObject* object, int object_size) {
     Heap* heap = map->GetHeap();
 
-    AllocationAlignment align =
-        alignment == kDoubleAlignment ? kDoubleAligned : kWordAligned;
     AllocationResult allocation =
-        heap->old_space()->AllocateRaw(object_size, align);
+        heap->old_space()->AllocateRaw(object_size, alignment);
 
     HeapObject* target = NULL;  // Initialization to please compiler.
     if (allocation.To(&target)) {
       MigrateObject(heap, object, target, object_size);
 
       // Update slot to new target.
       *slot = target;
 
       if (object_contents == POINTER_OBJECT) {
         if (map->instance_type() == JS_FUNCTION_TYPE) {
           heap->promotion_queue()->insert(target,
                                           JSFunction::kNonWeakFieldsEndOffset);
         } else {
           heap->promotion_queue()->insert(target, object_size);
         }
       }
       heap->IncrementPromotedObjectsSize(object_size);
       return true;
     }
     return false;
   }
 
 
-  template <ObjectContents object_contents, int alignment>
+  template <ObjectContents object_contents, AllocationAlignment alignment>
   static inline void EvacuateObject(Map* map, HeapObject** slot,
                                     HeapObject* object, int object_size) {
     SLOW_DCHECK(object_size <= Page::kMaxRegularHeapObjectSize);
     SLOW_DCHECK(object->Size() == object_size);
     Heap* heap = map->GetHeap();
 
     if (!heap->ShouldBePromoted(object->address(), object_size)) {
       // A semi-space copy may fail due to fragmentation. In that case, we
       // try to promote the object.
       if (SemiSpaceCopyObject<alignment>(map, slot, object, object_size)) {
@@ skipping 29 matching lines @@
       // promotion queue processing (IterateAndMarkPointersToFromSpace) will
       // miss it as it is not HeapObject-tagged.
       Address code_entry_slot =
           target->address() + JSFunction::kCodeEntryOffset;
       Code* code = Code::cast(Code::GetObjectFromEntryAddress(code_entry_slot));
       map->GetHeap()->mark_compact_collector()->RecordCodeEntrySlot(
           code_entry_slot, code);
     }
   }
 
+#if V8_HOST_ARCH_64_BIT
+#define kObjectAligned kDoubleAligned
+#else
+#define kObjectAligned kWordAligned
+#endif
 
   static inline void EvacuateFixedArray(Map* map, HeapObject** slot,
                                         HeapObject* object) {
     int object_size = FixedArray::BodyDescriptor::SizeOf(map, object);
-    EvacuateObject<POINTER_OBJECT, kObjectAlignment>(map, slot, object,
-                                                     object_size);
+    EvacuateObject<POINTER_OBJECT, kObjectAligned>(map, slot, object,
+                                                   object_size);
   }
 
 
   static inline void EvacuateFixedDoubleArray(Map* map, HeapObject** slot,
                                               HeapObject* object) {
     int length = reinterpret_cast<FixedDoubleArray*>(object)->length();
     int object_size = FixedDoubleArray::SizeFor(length);
-    EvacuateObject<DATA_OBJECT, kDoubleAlignment>(map, slot, object,
-                                                  object_size);
+    EvacuateObject<DATA_OBJECT, kDoubleAligned>(map, slot, object, object_size);
   }
 
 
   static inline void EvacuateFixedTypedArray(Map* map, HeapObject** slot,
                                              HeapObject* object) {
     int object_size = reinterpret_cast<FixedTypedArrayBase*>(object)->size();
-    EvacuateObject<DATA_OBJECT, kObjectAlignment>(map, slot, object,
-                                                  object_size);
+    EvacuateObject<DATA_OBJECT, kObjectAligned>(map, slot, object, object_size);
   }
 
 
   static inline void EvacuateFixedFloat64Array(Map* map, HeapObject** slot,
                                                HeapObject* object) {
     int object_size = reinterpret_cast<FixedFloat64Array*>(object)->size();
-    EvacuateObject<DATA_OBJECT, kDoubleAlignment>(map, slot, object,
-                                                  object_size);
+    EvacuateObject<DATA_OBJECT, kDoubleAligned>(map, slot, object, object_size);
   }
 
 
   static inline void EvacuateByteArray(Map* map, HeapObject** slot,
                                        HeapObject* object) {
     int object_size = reinterpret_cast<ByteArray*>(object)->ByteArraySize();
-    EvacuateObject<DATA_OBJECT, kObjectAlignment>(map, slot, object,
-                                                  object_size);
+    EvacuateObject<DATA_OBJECT, kObjectAligned>(map, slot, object, object_size);
   }
 
 
   static inline void EvacuateSeqOneByteString(Map* map, HeapObject** slot,
                                               HeapObject* object) {
     int object_size = SeqOneByteString::cast(object)
                           ->SeqOneByteStringSize(map->instance_type());
-    EvacuateObject<DATA_OBJECT, kObjectAlignment>(map, slot, object,
-                                                  object_size);
+    EvacuateObject<DATA_OBJECT, kObjectAligned>(map, slot, object, object_size);
   }
 
 
   static inline void EvacuateSeqTwoByteString(Map* map, HeapObject** slot,
                                               HeapObject* object) {
     int object_size = SeqTwoByteString::cast(object)
                           ->SeqTwoByteStringSize(map->instance_type());
-    EvacuateObject<DATA_OBJECT, kObjectAlignment>(map, slot, object,
-                                                  object_size);
+    EvacuateObject<DATA_OBJECT, kObjectAligned>(map, slot, object, object_size);
   }
 
 
   static inline void EvacuateShortcutCandidate(Map* map, HeapObject** slot,
                                                HeapObject* object) {
     DCHECK(IsShortcutCandidate(map->instance_type()));
 
     Heap* heap = map->GetHeap();
 
     if (marks_handling == IGNORE_MARKS &&
@@ skipping 16 matching lines @@
         object->set_map_word(MapWord::FromForwardingAddress(target));
         return;
       }
 
       heap->DoScavengeObject(first->map(), slot, first);
       object->set_map_word(MapWord::FromForwardingAddress(*slot));
       return;
     }
 
     int object_size = ConsString::kSize;
-    EvacuateObject<POINTER_OBJECT, kObjectAlignment>(map, slot, object,
-                                                     object_size);
+    EvacuateObject<POINTER_OBJECT, kObjectAligned>(map, slot, object,
+                                                   object_size);
   }
 
   template <ObjectContents object_contents>
   class ObjectEvacuationStrategy {
    public:
     template <int object_size>
     static inline void VisitSpecialized(Map* map, HeapObject** slot,
                                         HeapObject* object) {
-      EvacuateObject<object_contents, kObjectAlignment>(map, slot, object,
-                                                        object_size);
+      EvacuateObject<object_contents, kObjectAligned>(map, slot, object,
+                                                      object_size);
     }
 
     static inline void Visit(Map* map, HeapObject** slot, HeapObject* object) {
       int object_size = map->instance_size();
-      EvacuateObject<object_contents, kObjectAlignment>(map, slot, object,
-                                                        object_size);
+      EvacuateObject<object_contents, kObjectAligned>(map, slot, object,
+                                                      object_size);
     }
   };
 
   static VisitorDispatchTable<ScavengingCallback> table_;
 };
 
 
 template <MarksHandling marks_handling,
           LoggingAndProfiling logging_and_profiling_mode>
 VisitorDispatchTable<ScavengingCallback>
@@ skipping 462 matching lines @@
   // Statically ensure that it is safe to allocate heap numbers in paged
   // spaces.
   int size = HeapNumber::kSize;
   STATIC_ASSERT(HeapNumber::kSize <= Page::kMaxRegularHeapObjectSize);
 
   AllocationSpace space = SelectSpace(size, pretenure);
 
   HeapObject* result;
   {
     AllocationResult allocation =
-        AllocateRaw(size, space, OLD_SPACE, kDoubleUnaligned);
+        AllocateRaw(size, space, OLD_SPACE, kDoubleValueAligned);
     if (!allocation.To(&result)) return allocation;
   }
 
   Map* map = mode == MUTABLE ? mutable_heap_number_map() : heap_number_map();
   HeapObject::cast(result)->set_map_no_write_barrier(map);
   HeapNumber::cast(result)->set_value(value);
   return result;
 }
 
 
@@ skipping 3723 matching lines @@
     *object_type = "CODE_TYPE";                                                \
     *object_sub_type = "CODE_AGE/" #name;                                      \
     return true;
     CODE_AGE_LIST_COMPLETE(COMPARE_AND_RETURN_NAME)
 #undef COMPARE_AND_RETURN_NAME
   }
   return false;
 }
 }
 }  // namespace v8::internal