In-object double fields unboxing (for 64-bit only).

src/objects-inl.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // Review notes:
 //
 // - The use of macros in these inline functions may seem superfluous
 // but it is absolutely needed to make sure gcc generates optimal
 // code. gcc is not happy when attempting to inline too deep.
 //
@@ skipping 38 matching lines @@
   return Smi::FromInt(value >> 1);
 }
 
 
 PropertyDetails PropertyDetails::AsDeleted() const {
   Smi* smi = Smi::FromInt(value_ | DeletedField::encode(1));
   return PropertyDetails(smi);
 }
 
 
+int PropertyDetails::field_width_in_words() const {
+  DCHECK(type() == FIELD);
+  if (!FLAG_unbox_double_fields) return 1;
+  if (kDoubleSize == kPointerSize) return 1;
+  return representation().IsDouble() ? kDoubleSize / kPointerSize : 1;
+}
+
+
 #define TYPE_CHECKER(type, instancetype)                                \
   bool Object::Is##type() const {                                       \
   return Object::IsHeapObject() &&                                      \
       HeapObject::cast(this)->map()->instance_type() == instancetype;   \
   }
 
 
 #define CAST_ACCESSOR(type)                       \
   type* type::cast(Object* object) {              \
     SLOW_DCHECK(object->Is##type());              \
@@ skipping 628 matching lines @@
 bool Object::IsJSWeakCollection() const {
   return IsJSWeakMap() || IsJSWeakSet();
 }
 
 
 bool Object::IsDescriptorArray() const {
   return IsFixedArray();
 }
 
 
+bool Object::IsLayoutDescriptor() const {
+  return IsSmi() || IsFixedTypedArrayBase();
+}
+
+
 bool Object::IsTransitionArray() const {
   return IsFixedArray();
 }
 
 
 bool Object::IsTypeFeedbackVector() const { return IsFixedArray(); }
 
 
 bool Object::IsDeoptimizationInputData() const {
   // Must be a fixed array.
@@ skipping 1338 matching lines @@
 void JSObject::SetInternalField(int index, Smi* value) {
   DCHECK(index < GetInternalFieldCount() && index >= 0);
   // Internal objects do follow immediately after the header, whereas in-object
   // properties are at the end of the object. Therefore there is no need
   // to adjust the index here.
   int offset = GetHeaderSize() + (kPointerSize * index);
   WRITE_FIELD(this, offset, value);
 }
 
 
+bool JSObject::IsUnboxedDoubleField(FieldIndex index) {
+  if (!FLAG_unbox_double_fields) return false;
+  if (index.is_hidden_field() || !index.is_inobject()) return false;
+  return !map()->layout_descriptor()->IsTagged(index.property_index());
+}
+
+
+bool Map::IsUnboxedDoubleField(FieldIndex index) {
+  if (!FLAG_unbox_double_fields) return false;
+  if (index.is_hidden_field() || !index.is_inobject()) return false;
+  return !layout_descriptor()->IsTagged(index.property_index());
+}
+
+
 // Access fast-case object properties at index. The use of these routines
 // is needed to correctly distinguish between properties stored in-object and
 // properties stored in the properties array.
 Object* JSObject::RawFastPropertyAt(FieldIndex index) {
+  DCHECK(!IsUnboxedDoubleField(index));
   if (index.is_inobject()) {
     return READ_FIELD(this, index.offset());
   } else {
     return properties()->get(index.outobject_array_index());
   }
 }
 
 
-void JSObject::FastPropertyAtPut(FieldIndex index, Object* value) {
+double JSObject::RawFastDoublePropertyAt(FieldIndex index) {
+  DCHECK(IsUnboxedDoubleField(index));
+  return READ_DOUBLE_FIELD(this, index.offset());
+}
+
+
+void JSObject::RawFastPropertyAtPut(FieldIndex index, Object* value) {
   if (index.is_inobject()) {
     int offset = index.offset();
     WRITE_FIELD(this, offset, value);
     WRITE_BARRIER(GetHeap(), this, offset, value);
   } else {
     properties()->set(index.outobject_array_index(), value);
   }
 }
 
 
+void JSObject::RawFastDoublePropertyAtPut(FieldIndex index, double value) {
+  WRITE_DOUBLE_FIELD(this, index.offset(), value);
+}
+
+
+void JSObject::FastPropertyAtPut(FieldIndex index, Object* value) {
+  if (IsUnboxedDoubleField(index)) {
+    DCHECK(value->IsMutableHeapNumber());
+    RawFastDoublePropertyAtPut(index, HeapNumber::cast(value)->value());
+  } else {
+    RawFastPropertyAtPut(index, value);
+  }
+}
+
+
 int JSObject::GetInObjectPropertyOffset(int index) {
   return map()->GetInObjectPropertyOffset(index);
 }
 
 
 Object* JSObject::InObjectPropertyAt(int index) {
   int offset = GetInObjectPropertyOffset(index);
   return READ_FIELD(this, offset);
 }
 
@@ skipping 977 matching lines @@
                           const WhitenessWitness&) {
   // Range check.
   DCHECK(descriptor_number < number_of_descriptors());
 
   NoIncrementalWriteBarrierSet(this,
                                ToKeyIndex(descriptor_number),
                                *desc->GetKey());
   NoIncrementalWriteBarrierSet(this,
                                ToValueIndex(descriptor_number),
                                *desc->GetValue());
-  NoIncrementalWriteBarrierSet(this,
-                               ToDetailsIndex(descriptor_number),
+  NoIncrementalWriteBarrierSet(this, ToDetailsIndex(descriptor_number),
                                desc->GetDetails().AsSmi());
 }
 
 
 void DescriptorArray::Set(int descriptor_number, Descriptor* desc) {
   // Range check.
   DCHECK(descriptor_number < number_of_descriptors());
 
   set(ToKeyIndex(descriptor_number), *desc->GetKey());
   set(ToValueIndex(descriptor_number), *desc->GetValue());
@@ skipping 34 matching lines @@
   DCHECK(!marking_->IsMarking() ||
          Marking::Color(array) == Marking::WHITE_OBJECT);
 }
 
 
 DescriptorArray::WhitenessWitness::~WhitenessWitness() {
   marking_->LeaveNoMarkingScope();
 }
 
 
+LayoutDescriptor* LayoutDescriptor::FromSmi(Smi* smi) {
+  return LayoutDescriptor::cast(smi);
+}
+
+
+Handle<LayoutDescriptor> LayoutDescriptor::New(Isolate* isolate, int length) {
+  if (length <= kSmiValueSize) {
+    // The whole bit vector fits into a smi.
+    return handle(LayoutDescriptor::FromSmi(Smi::FromInt(0)), isolate);
+  }
+
+  length = (length + kNumberOfBits - 1) / kNumberOfBits;
+  DCHECK(length > 0);
+
+  if (SmiValuesAre32Bits() && (length & 1)) {
+    // On 64-bit systems if the length is odd then the half-word space would be
+    // lost anyway (due to alignment and the fact that we are allocating
+    // uint32-typed array), so we increase the length of allocated array
+    // to utilize that "lost" space which could also help to avoid layout
+    // descriptor reallocations.
+    ++length;
+  }
+  return Handle<LayoutDescriptor>::cast(
+      isolate->factory()->NewFixedTypedArray(length, kExternalUint32Array));
+}
+
+
+bool LayoutDescriptor::InobjectUnboxedField(int inobject_properties,
+                                            PropertyDetails details) {
+  if (details.type() != FIELD || !details.representation().IsDouble()) {
+    return false;
+  }
+  // We care only about in-object properties.
+  return details.field_index() < inobject_properties;
+}
+
+
+LayoutDescriptor* LayoutDescriptor::FastPointerLayout() {
+  return LayoutDescriptor::FromSmi(Smi::FromInt(0));
+}
+
+
+bool LayoutDescriptor::GetIndexes(int field_index, int* layout_word_index,
+                                  uint32_t* layout_mask) {
+  if (static_cast<unsigned>(field_index) >= static_cast<unsigned>(capacity())) {
+    return false;
+  }
+
+  *layout_word_index = field_index / kNumberOfBits;
+  CHECK((!IsSmi() && (*layout_word_index < length())) ||
+        (IsSmi() && (*layout_word_index < 1)));
+
+  int layout_bit_index = field_index % kNumberOfBits;
+  *layout_mask = static_cast<uint32_t>(1) << layout_bit_index;
+  return true;
+}
+
+
+LayoutDescriptor* LayoutDescriptor::SetTagged(int field_index, bool tagged) {
+  int layout_word_index;
+  uint32_t layout_mask;
+
+  if (!GetIndexes(field_index, &layout_word_index, &layout_mask)) {
+    CHECK(false);
+    return this;
+  }
+
+  if (IsSlowLayout()) {
+    uint32_t value = get_scalar(layout_word_index);
+    if (tagged) {
+      value &= ~layout_mask;
+    } else {
+      value |= layout_mask;
+    }
+    set(layout_word_index, value);
+    return this;
+  } else {
+    uint32_t value = static_cast<uint32_t>(Smi::cast(this)->value());
+    if (tagged) {
+      value &= ~layout_mask;
+    } else {
+      value |= layout_mask;
+    }
+    return LayoutDescriptor::FromSmi(Smi::FromInt(static_cast<int>(value)));
+  }
+}
+
+
+bool LayoutDescriptor::IsTagged(int field_index) {
+  if (IsFastPointerLayout()) return true;
+
+  int layout_word_index;
+  uint32_t layout_mask;
+
+  if (!GetIndexes(field_index, &layout_word_index, &layout_mask)) {
+    // All bits after Out of bounds queries
+    return true;
+  }
+
+  if (IsSlowLayout()) {
+    uint32_t value = get_scalar(layout_word_index);
+    return (value & layout_mask) == 0;
+  } else {
+    uint32_t value = static_cast<uint32_t>(Smi::cast(this)->value());
+    return (value & layout_mask) == 0;
+  }
+}
+
+
+bool LayoutDescriptor::IsFastPointerLayout() {
+  return IsSmi() && (Smi::cast(this)->value() == 0);
+}
+
+
+bool LayoutDescriptor::IsSlowLayout() { return !IsSmi(); }
+
+
+int LayoutDescriptor::capacity() {
+  return IsSlowLayout() ? (length() * kNumberOfBits) : kSmiValueSize;
+}
+
+
+LayoutDescriptor* LayoutDescriptor::cast_gc_safe(Object* object) {
+  if (object->IsSmi()) {
+    // Fast mode layout descriptor.
+    return reinterpret_cast<LayoutDescriptor*>(object);
+  }
+
+  // This is a mixed descriptor which is a fixed typed array.
+  MapWord map_word = reinterpret_cast<HeapObject*>(object)->map_word();
+  if (map_word.IsForwardingAddress()) {
+    // Mark-compact has already moved layout descriptor.
+    object = map_word.ToForwardingAddress();
+  }
+  return LayoutDescriptor::cast(object);
+}
+
+
+// InobjectPropertiesHelper is a helper class for querying whether inobject
+// property at offset is Double or not.
+InobjectPropertiesHelper::InobjectPropertiesHelper(Map* map)
+    : all_fields_tagged_(true),
+      header_size_(0),
+      inobject_properties_count_(0),
+      layout_descriptor_(LayoutDescriptor::FastPointerLayout()) {
+  if (!FLAG_unbox_double_fields) return;
+
+  layout_descriptor_ = map->layout_descriptor_gc_safe();
+  if (layout_descriptor_->IsFastPointerLayout()) {
+    return;
+  }
+
+  int inobject_properties = map->inobject_properties();
+  DCHECK(inobject_properties > 0);
+  header_size_ = map->instance_size() - (inobject_properties * kPointerSize);
+  DCHECK(header_size_ >= 0);
+
+  all_fields_tagged_ = false;
+}
+
+
+bool InobjectPropertiesHelper::IsTagged(int offset_in_bytes) {
+  DCHECK(IsAligned(offset_in_bytes, kPointerSize));
+  if (all_fields_tagged_) return true;
+  // Object headers do not contain non-tagged fields.
+  if (offset_in_bytes < header_size_) return true;
+  int field_index = (offset_in_bytes - header_size_) / kPointerSize;
+
+  return layout_descriptor_->IsTagged(field_index);
+}
+
+
 template<typename Derived, typename Shape, typename Key>
 int HashTable<Derived, Shape, Key>::ComputeCapacity(int at_least_space_for) {
   const int kMinCapacity = 32;
   int capacity = base::bits::RoundUpToPowerOfTwo32(at_least_space_for * 2);
   if (capacity < kMinCapacity) {
     capacity = kMinCapacity;  // Guarantee min capacity.
   }
   return capacity;
 }
 
@@ skipping 100 matching lines @@
 CAST_ACCESSOR(JSObject)
 CAST_ACCESSOR(JSProxy)
 CAST_ACCESSOR(JSReceiver)
 CAST_ACCESSOR(JSRegExp)
 CAST_ACCESSOR(JSSet)
 CAST_ACCESSOR(JSSetIterator)
 CAST_ACCESSOR(JSTypedArray)
 CAST_ACCESSOR(JSValue)
 CAST_ACCESSOR(JSWeakMap)
 CAST_ACCESSOR(JSWeakSet)
+CAST_ACCESSOR(LayoutDescriptor)
 CAST_ACCESSOR(Map)
 CAST_ACCESSOR(MapCache)
 CAST_ACCESSOR(Name)
 CAST_ACCESSOR(NameDictionary)
 CAST_ACCESSOR(NormalizedMapCache)
 CAST_ACCESSOR(Object)
 CAST_ACCESSOR(ObjectHashTable)
 CAST_ACCESSOR(Oddball)
 CAST_ACCESSOR(OrderedHashMap)
 CAST_ACCESSOR(OrderedHashSet)
@@ skipping 1863 matching lines @@
     transitions->set_back_pointer_storage(map->GetBackPointer());
   } else if (!map->transitions()->IsFullTransitionArray()) {
     transitions = TransitionArray::ExtendToFullTransitionArray(map);
   } else {
     return;
   }
   map->set_transitions(*transitions);
 }
 
 
-void Map::InitializeDescriptors(DescriptorArray* descriptors) {
+LayoutDescriptor* Map::layout_descriptor_gc_safe() {
+  Object* layout_desc = READ_FIELD(this, kLayoutDecriptorOffset);
+  return LayoutDescriptor::cast_gc_safe(layout_desc);
+}
+
+
+void Map::UpdateDescriptors(DescriptorArray* descriptors,
+                            LayoutDescriptor* layout_desc) {
+  set_instance_descriptors(descriptors);
+  if (FLAG_unbox_double_fields) {
+    if (layout_descriptor()->IsSlowLayout()) {
+      set_layout_descriptor(layout_desc);
+    }
+    SLOW_DCHECK(layout_descriptor()->IsConsistentWithMap(this));
+    DCHECK(visitor_id() == StaticVisitorBase::GetVisitorId(this));
+  }
+}
+
+
+void Map::InitializeDescriptors(DescriptorArray* descriptors,
+                                LayoutDescriptor* layout_desc) {
   int len = descriptors->number_of_descriptors();
   set_instance_descriptors(descriptors);
   SetNumberOfOwnDescriptors(len);
+
+  if (FLAG_unbox_double_fields) {
+    set_layout_descriptor(layout_desc);
+    SLOW_DCHECK(layout_descriptor()->IsConsistentWithMap(this));
+    set_visitor_id(StaticVisitorBase::GetVisitorId(this));
+  }
 }
 
 
 ACCESSORS(Map, instance_descriptors, DescriptorArray, kDescriptorsOffset)
+ACCESSORS(Map, layout_descriptor, LayoutDescriptor, kLayoutDecriptorOffset)
 
 
 void Map::set_bit_field3(uint32_t bits) {
   if (kInt32Size != kPointerSize) {
     WRITE_UINT32_FIELD(this, kBitField3Offset + kInt32Size, 0);
   }
   WRITE_UINT32_FIELD(this, kBitField3Offset, bits);
 }
 
 
 uint32_t Map::bit_field3() {
   return READ_UINT32_FIELD(this, kBitField3Offset);
 }
 
 
+Handle<LayoutDescriptor> Map::GetLayoutDescriptor() {
+  LayoutDescriptor* layout_desc = FLAG_unbox_double_fields
+                                      ? layout_descriptor()
+                                      : LayoutDescriptor::FastPointerLayout();
+  return handle(layout_desc, GetIsolate());
+}
+
+
 void Map::AppendDescriptor(Descriptor* desc) {
   DescriptorArray* descriptors = instance_descriptors();
   int number_of_own_descriptors = NumberOfOwnDescriptors();
   DCHECK(descriptors->number_of_descriptors() == number_of_own_descriptors);
   descriptors->Append(desc);
   SetNumberOfOwnDescriptors(number_of_own_descriptors + 1);
+
+// This function does not support appending double field descriptors and
+// it should never try to (otherwise, layout descriptor must be updated too).
+#ifdef DEBUG
+  PropertyDetails details = desc->GetDetails();
+  CHECK(details.type() != FIELD || !details.representation().IsDouble());
+#endif
 }
 
 
 Object* Map::GetBackPointer() {
   Object* object = READ_FIELD(this, kTransitionsOrBackPointerOffset);
   if (object->IsDescriptorArray()) {
     return TransitionArray::cast(object)->back_pointer_storage();
   } else {
     DCHECK(object->IsMap() || object->IsUndefined());
     return object;
@@ skipping 2011 matching lines @@
 
 
 template<typename StaticVisitor>
 void ExternalTwoByteString::ExternalTwoByteStringIterateBody() {
   typedef v8::String::ExternalStringResource Resource;
   StaticVisitor::VisitExternalTwoByteString(
       reinterpret_cast<Resource**>(FIELD_ADDR(this, kResourceOffset)));
 }
 
 
+static void IterateBodyUsingLayoutDescriptor(HeapObject* object,
+                                             int start_offset, int end_offset,
+                                             ObjectVisitor* v) {
+  DCHECK(FLAG_unbox_double_fields);
+  DCHECK(IsAligned(start_offset, kPointerSize) &&
+         IsAligned(end_offset, kPointerSize));
+
+  InobjectPropertiesHelper helper(object->map());
+  DCHECK(!helper.all_fields_tagged());
+
+  for (int offset = start_offset; offset < end_offset; offset += kPointerSize) {
+    // Visit all tagged fields.
+    if (helper.IsTagged(offset)) {
+      v->VisitPointer(HeapObject::RawField(object, offset));
+    }
+  }
+}
+
+
 template<int start_offset, int end_offset, int size>
 void FixedBodyDescriptor<start_offset, end_offset, size>::IterateBody(
     HeapObject* obj,
     ObjectVisitor* v) {
+  if (!FLAG_unbox_double_fields ||
+      obj->map()->layout_descriptor()->IsFastPointerLayout()) {
     v->VisitPointers(HeapObject::RawField(obj, start_offset),
                      HeapObject::RawField(obj, end_offset));
+  } else {
+    IterateBodyUsingLayoutDescriptor(obj, start_offset, end_offset, v);
+  }
 }
 
 
 template<int start_offset>
 void FlexibleBodyDescriptor<start_offset>::IterateBody(HeapObject* obj,
                                                        int object_size,
                                                        ObjectVisitor* v) {
-  v->VisitPointers(HeapObject::RawField(obj, start_offset),
-                   HeapObject::RawField(obj, object_size));
+  if (!FLAG_unbox_double_fields ||
+      obj->map()->layout_descriptor()->IsFastPointerLayout()) {
+    v->VisitPointers(HeapObject::RawField(obj, start_offset),
+                     HeapObject::RawField(obj, object_size));
+  } else {
+    IterateBodyUsingLayoutDescriptor(obj, start_offset, object_size, v);
+  }
 }
 
 
 template<class Derived, class TableType>
 Object* OrderedHashTableIterator<Derived, TableType>::CurrentKey() {
   TableType* table(TableType::cast(this->table()));
   int index = Smi::cast(this->index())->value();
   Object* key = table->KeyAt(index);
   DCHECK(!key->IsTheHole());
   return key;
@@ skipping 49 matching lines @@
 #undef READ_SHORT_FIELD
 #undef WRITE_SHORT_FIELD
 #undef READ_BYTE_FIELD
 #undef WRITE_BYTE_FIELD
 #undef NOBARRIER_READ_BYTE_FIELD
 #undef NOBARRIER_WRITE_BYTE_FIELD
 
 } }  // namespace v8::internal
 
 #endif  // V8_OBJECTS_INL_H_