In-object double fields unboxing (for 64-bit only).

src/objects.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef V8_OBJECTS_H_
 #define V8_OBJECTS_H_
 
 #include "src/allocation.h"
 #include "src/assert-scope.h"
 #include "src/builtins.h"
@@ skipping 863 matching lines @@
 
 class AccessorPair;
 class AllocationSite;
 class AllocationSiteCreationContext;
 class AllocationSiteUsageContext;
 class DictionaryElementsAccessor;
 class ElementsAccessor;
 class FixedArrayBase;
 class GlobalObject;
 class ObjectVisitor;
+class LayoutDescriptor;
 class LookupIterator;
 class StringStream;
 // We cannot just say "class HeapType;" if it is created from a template... =8-?
 template<class> class TypeImpl;
 struct HeapTypeConfig;
 typedef TypeImpl<HeapTypeConfig> HeapType;
 
 
 // A template-ized version of the IsXXX functions.
 template <class C> inline bool Is(Object* obj);
 
 #ifdef VERIFY_HEAP
 #define DECLARE_VERIFIER(Name) void Name##Verify();
 #else
 #define DECLARE_VERIFIER(Name)
 #endif
 
 #ifdef OBJECT_PRINT
 #define DECLARE_PRINTER(Name) void Name##Print(OStream& os);  // NOLINT
 #else
 #define DECLARE_PRINTER(Name)
 #endif
 
 
-#define OBJECT_TYPE_LIST(V)                    \
-  V(Smi)                                       \
-  V(HeapObject)                                \
-  V(Number)                                    \
+#define OBJECT_TYPE_LIST(V) \
+  V(Smi)                    \
+  V(HeapObject)             \
+  V(Number)
 
-#define HEAP_OBJECT_TYPE_LIST(V)               \
-  V(HeapNumber)                                \
-  V(MutableHeapNumber)                         \
-  V(Name)                                      \
-  V(UniqueName)                                \
-  V(String)                                    \
-  V(SeqString)                                 \
-  V(ExternalString)                            \
-  V(ConsString)                                \
-  V(SlicedString)                              \
-  V(ExternalTwoByteString)                     \
-  V(ExternalAsciiString)                       \
-  V(SeqTwoByteString)                          \
-  V(SeqOneByteString)                          \
-  V(InternalizedString)                        \
-  V(Symbol)                                    \
-                                               \
-  V(ExternalArray)                             \
-  V(ExternalInt8Array)                         \
-  V(ExternalUint8Array)                        \
-  V(ExternalInt16Array)                        \
-  V(ExternalUint16Array)                       \
-  V(ExternalInt32Array)                        \
-  V(ExternalUint32Array)                       \
-  V(ExternalFloat32Array)                      \
-  V(ExternalFloat64Array)                      \
-  V(ExternalUint8ClampedArray)                 \
-  V(FixedTypedArrayBase)                       \
-  V(FixedUint8Array)                           \
-  V(FixedInt8Array)                            \
-  V(FixedUint16Array)                          \
-  V(FixedInt16Array)                           \
-  V(FixedUint32Array)                          \
-  V(FixedInt32Array)                           \
-  V(FixedFloat32Array)                         \
-  V(FixedFloat64Array)                         \
-  V(FixedUint8ClampedArray)                    \
-  V(ByteArray)                                 \
-  V(FreeSpace)                                 \
-  V(JSReceiver)                                \
-  V(JSObject)                                  \
-  V(JSContextExtensionObject)                  \
-  V(JSGeneratorObject)                         \
-  V(JSModule)                                  \
-  V(Map)                                       \
-  V(DescriptorArray)                           \
-  V(TransitionArray)                           \
-  V(DeoptimizationInputData)                   \
-  V(DeoptimizationOutputData)                  \
-  V(DependentCode)                             \
-  V(FixedArray)                                \
-  V(FixedDoubleArray)                          \
-  V(ConstantPoolArray)                         \
-  V(Context)                                   \
-  V(NativeContext)                             \
-  V(ScopeInfo)                                 \
-  V(JSFunction)                                \
-  V(Code)                                      \
-  V(Oddball)                                   \
-  V(SharedFunctionInfo)                        \
-  V(JSValue)                                   \
-  V(JSDate)                                    \
-  V(JSMessageObject)                           \
-  V(StringWrapper)                             \
-  V(Foreign)                                   \
-  V(Boolean)                                   \
-  V(JSArray)                                   \
-  V(JSArrayBuffer)                             \
-  V(JSArrayBufferView)                         \
-  V(JSTypedArray)                              \
-  V(JSDataView)                                \
-  V(JSProxy)                                   \
-  V(JSFunctionProxy)                           \
-  V(JSSet)                                     \
-  V(JSMap)                                     \
-  V(JSSetIterator)                             \
-  V(JSMapIterator)                             \
-  V(JSWeakCollection)                          \
-  V(JSWeakMap)                                 \
-  V(JSWeakSet)                                 \
-  V(JSRegExp)                                  \
-  V(HashTable)                                 \
-  V(Dictionary)                                \
-  V(StringTable)                               \
-  V(JSFunctionResultCache)                     \
-  V(NormalizedMapCache)                        \
-  V(CompilationCacheTable)                     \
-  V(CodeCacheHashTable)                        \
-  V(PolymorphicCodeCacheHashTable)             \
-  V(MapCache)                                  \
-  V(Primitive)                                 \
-  V(GlobalObject)                              \
-  V(JSGlobalObject)                            \
-  V(JSBuiltinsObject)                          \
-  V(JSGlobalProxy)                             \
-  V(UndetectableObject)                        \
-  V(AccessCheckNeeded)                         \
-  V(Cell)                                      \
-  V(PropertyCell)                              \
-  V(ObjectHashTable)                           \
-  V(WeakHashTable)                             \
+#define HEAP_OBJECT_TYPE_LIST(V)   \
+  V(HeapNumber)                    \
+  V(MutableHeapNumber)             \
+  V(Name)                          \
+  V(UniqueName)                    \
+  V(String)                        \
+  V(SeqString)                     \
+  V(ExternalString)                \
+  V(ConsString)                    \
+  V(SlicedString)                  \
+  V(ExternalTwoByteString)         \
+  V(ExternalAsciiString)           \
+  V(SeqTwoByteString)              \
+  V(SeqOneByteString)              \
+  V(InternalizedString)            \
+  V(Symbol)                        \
+                                   \
+  V(ExternalArray)                 \
+  V(ExternalInt8Array)             \
+  V(ExternalUint8Array)            \
+  V(ExternalInt16Array)            \
+  V(ExternalUint16Array)           \
+  V(ExternalInt32Array)            \
+  V(ExternalUint32Array)           \
+  V(ExternalFloat32Array)          \
+  V(ExternalFloat64Array)          \
+  V(ExternalUint8ClampedArray)     \
+  V(FixedTypedArrayBase)           \
+  V(FixedUint8Array)               \
+  V(FixedInt8Array)                \
+  V(FixedUint16Array)              \
+  V(FixedInt16Array)               \
+  V(FixedUint32Array)              \
+  V(FixedInt32Array)               \
+  V(FixedFloat32Array)             \
+  V(FixedFloat64Array)             \
+  V(FixedUint8ClampedArray)        \
+  V(ByteArray)                     \
+  V(FreeSpace)                     \
+  V(JSReceiver)                    \
+  V(JSObject)                      \
+  V(JSContextExtensionObject)      \
+  V(JSGeneratorObject)             \
+  V(JSModule)                      \
+  V(Map)                           \
+  V(DescriptorArray)               \
+  V(LayoutDescriptor)              \
+  V(TransitionArray)               \
+  V(DeoptimizationInputData)       \
+  V(DeoptimizationOutputData)      \
+  V(DependentCode)                 \
+  V(FixedArray)                    \
+  V(FixedDoubleArray)              \
+  V(ConstantPoolArray)             \
+  V(Context)                       \
+  V(NativeContext)                 \
+  V(ScopeInfo)                     \
+  V(JSFunction)                    \
+  V(Code)                          \
+  V(Oddball)                       \
+  V(SharedFunctionInfo)            \
+  V(JSValue)                       \
+  V(JSDate)                        \
+  V(JSMessageObject)               \
+  V(StringWrapper)                 \
+  V(Foreign)                       \
+  V(Boolean)                       \
+  V(JSArray)                       \
+  V(JSArrayBuffer)                 \
+  V(JSArrayBufferView)             \
+  V(JSTypedArray)                  \
+  V(JSDataView)                    \
+  V(JSProxy)                       \
+  V(JSFunctionProxy)               \
+  V(JSSet)                         \
+  V(JSMap)                         \
+  V(JSSetIterator)                 \
+  V(JSMapIterator)                 \
+  V(JSWeakCollection)              \
+  V(JSWeakMap)                     \
+  V(JSWeakSet)                     \
+  V(JSRegExp)                      \
+  V(HashTable)                     \
+  V(Dictionary)                    \
+  V(StringTable)                   \
+  V(JSFunctionResultCache)         \
+  V(NormalizedMapCache)            \
+  V(CompilationCacheTable)         \
+  V(CodeCacheHashTable)            \
+  V(PolymorphicCodeCacheHashTable) \
+  V(MapCache)                      \
+  V(Primitive)                     \
+  V(GlobalObject)                  \
+  V(JSGlobalObject)                \
+  V(JSBuiltinsObject)              \
+  V(JSGlobalProxy)                 \
+  V(UndetectableObject)            \
+  V(AccessCheckNeeded)             \
+  V(Cell)                          \
+  V(PropertyCell)                  \
+  V(ObjectHashTable)               \
+  V(WeakHashTable)                 \
   V(OrderedHashTable)
 
 
 #define ERROR_MESSAGES_LIST(V) \
   V(kNoReason, "no reason")                                                   \
                                                                               \
   V(k32BitValueInRegisterIsNotZeroExtended,                                   \
     "32 bit value in register is not zero-extended")                          \
   V(kAlignmentMarkerExpected, "Alignment marker expected")                    \
   V(kAllocationIsNotDoubleAligned, "Allocation is not double aligned")        \
@@ skipping 1439 matching lines @@
 
   // Transform slow named properties to fast variants.
   static void MigrateSlowToFast(Handle<JSObject> object,
                                 int unused_property_fields);
 
   // Access fast-case object properties at index.
   static Handle<Object> FastPropertyAt(Handle<JSObject> object,
                                        Representation representation,
                                        FieldIndex index);
   inline Object* RawFastPropertyAt(FieldIndex index);
+  static inline Handle<Object> RawFastBoxedPropertyAt(Handle<JSObject> object,
+                                                      FieldIndex index);
+  inline double RawFastDoublePropertyAt(FieldIndex index);
+  inline void FastPropertyAtPut(Map* map, FieldIndex index, Object* value);
   inline void FastPropertyAtPut(FieldIndex index, Object* value);
   void WriteToField(int descriptor, Object* value);
+  inline void FastDoublePropertyAtPut(FieldIndex index, double value);
 
   // Access to in object properties.
   inline int GetInObjectPropertyOffset(int index);
   inline Object* InObjectPropertyAt(int index);
   inline Object* InObjectPropertyAtPut(int index,
                                        Object* value,
                                        WriteBarrierMode mode
                                        = UPDATE_WRITE_BARRIER);
 
   // Set the object's prototype (only JSReceiver and null are allowed values).
@@ skipping 921 matching lines @@
     return len == 0 ? 0 : (len - kFirstIndex) / kDescriptorSize;
   }
 
   int NumberOfSlackDescriptors() {
     return number_of_descriptors_storage() - number_of_descriptors();
   }
 
   inline void SetNumberOfDescriptors(int number_of_descriptors);
   inline int number_of_entries() { return number_of_descriptors(); }
 
+  inline Object* cached_layout_descriptor();
+  inline void set_cached_layout_descriptor(LayoutDescriptor* cached);
+  inline void drop_cached_layout_descriptor();
+
   bool HasEnumCache() {
     return !IsEmpty() && !get(kEnumCacheIndex)->IsSmi();
   }
 
   void CopyEnumCacheFrom(DescriptorArray* array) {
     set(kEnumCacheIndex, array->get(kEnumCacheIndex));
   }
 
   FixedArray* GetEnumCache() {
     ASSERT(HasEnumCache());
@@ skipping 86 matching lines @@
   static Handle<DescriptorArray> Allocate(Isolate* isolate,
                                           int number_of_descriptors,
                                           int slack = 0);
 
   DECLARE_CAST(DescriptorArray)
 
   // Constant for denoting key was not found.
   static const int kNotFound = -1;
 
   static const int kDescriptorLengthIndex = 0;
+#if V8_DOUBLE_FIELDS_UNBOXING
+  static const int kLayoutDescriptorCacheIndex = 1;
+  static const int kEnumCacheIndex = 2;
+  static const int kFirstIndex = 3;
+#else
+  static const int kLayoutDescriptorCacheIndex = -1;
   static const int kEnumCacheIndex = 1;
   static const int kFirstIndex = 2;
+#endif
 
   // The length of the "bridge" to the enum cache.
   static const int kEnumCacheBridgeLength = 2;
   static const int kEnumCacheBridgeCacheIndex = 0;
   static const int kEnumCacheBridgeIndicesCacheIndex = 1;
 
   // Layout description.
   static const int kDescriptorLengthOffset = FixedArray::kHeaderSize;
+#if V8_DOUBLE_FIELDS_UNBOXING
+  static const int kLayoutDescriptorOffset =
+      kDescriptorLengthOffset + kPointerSize;
+  static const int kEnumCacheOffset = kLayoutDescriptorOffset + kPointerSize;
+#else
+  static const int kLayoutDescriptorOffset = 1;  // Must not be used.
   static const int kEnumCacheOffset = kDescriptorLengthOffset + kPointerSize;
+#endif
   static const int kFirstOffset = kEnumCacheOffset + kPointerSize;
 
   // Layout description for the bridge array.
   static const int kEnumCacheBridgeCacheOffset = FixedArray::kHeaderSize;
 
   // Layout of descriptor.
   static const int kDescriptorKey = 0;
   static const int kDescriptorDetails = 1;
   static const int kDescriptorValue = 2;
   static const int kDescriptorSize = 3;
@@ skipping 66 matching lines @@
   }
 
   static int ToValueIndex(int descriptor_number) {
     return kFirstIndex +
            (descriptor_number * kDescriptorSize) +
            kDescriptorValue;
   }
 
   // Transfer a complete descriptor from the src descriptor array to this
   // descriptor array.
-  void CopyFrom(int index,
-                DescriptorArray* src,
-                const WhitenessWitness&);
+  void CopyFrom(int index, DescriptorArray* src, const WhitenessWitness&);
 
   inline void Set(int descriptor_number,
                   Descriptor* desc,
                   const WhitenessWitness&);
 
   inline void Append(Descriptor* desc, const WhitenessWitness&);
 
   // Swap first and second descriptor.
   inline void SwapSortedKeys(int first, int second);
 
@@ skipping 1638 matching lines @@
                                           elementType scalar);                \
     static inline elementType defaultValue();                                 \
   };                                                                          \
                                                                               \
   typedef FixedTypedArray<Type##ArrayTraits> Fixed##Type##Array;
 
 TYPED_ARRAYS(FIXED_TYPED_ARRAY_TRAITS)
 
 #undef FIXED_TYPED_ARRAY_TRAITS
 
+
+// LayoutDescriptor is a bit vector defining which fields contain non-tagged
+// values. It could either be a fixed typed array (slow form) or a Smi
+// if the length fits (fast form).
+// Each bit in the layout represents a FIELD. The bits are referenced by
+// field_index which is a field number. If the bit is set then corresponding
+// field contains non tagged value and therefore must be skipped by GC.
+// Otherwise the field is considered tagged.
+class LayoutDescriptor : public FixedTypedArray<Uint32ArrayTraits> {
+ public:
+  INLINE(MUST_USE_RESULT LayoutDescriptor* SetTagged(int field_index,
+                                                     bool tagged));
+  INLINE(bool IsTagged(int field_index));
+
+  // Returns true if this is a layout of the object having only tagged fields.
+  INLINE(bool IsFastPointerLayout());
+
+  DECLARE_CAST(LayoutDescriptor)
+  INLINE(static LayoutDescriptor* cast_gc_safe(Object* object));
+
+  // Builds layout descriptor by given descriptors array.
+  static Handle<LayoutDescriptor> New(Handle<DescriptorArray> descriptors);
+
+  // Layout descriptor that corresponds to an object all fields of which are
+  // tagged (FastPointerLayout).
+  INLINE(static LayoutDescriptor* FastPointerLayout());
+
+  // If the descriptor has fixed array form then this method tries to convert
+  // make an optimized descriptor which has a smi form if particular |map|
+  // refers only to a small part of the layout.
+  // Otherwise |this| is returned.
+  INLINE(LayoutDescriptor* OptimizeFor(Map* map));
+
+#ifdef DEBUG
+  // Check that this layout descriptor corresponds to given map.
+  bool IsConsistentWithMap(Map* map);
+#endif
+
+#ifdef OBJECT_PRINT
+  void Print(OStream& os);  // NOLINT
+#endif
+
+ private:
+  static const int kNumberOfBits = 32;
+
+  INLINE(bool IsSlowLayout());
+
+  INLINE(static Handle<LayoutDescriptor> New(Isolate* isolate, int length));
+  INLINE(static LayoutDescriptor* FromSmi(Smi* smi));
+
+  INLINE(void GetIndexes(int field_index, int* layout_word_index,
+                         uint32_t* layout_mask));
+};
+
 // DeoptimizationInputData is a fixed array used to hold the deoptimization
 // data for code generated by the Hydrogen/Lithium compiler.  It also
 // contains information about functions that were inlined.  If N different
 // functions were inlined then first N elements of the literal array will
 // contain these functions.
 //
 // It can be empty.
 class DeoptimizationInputData: public FixedArray {
  public:
   // Layout description.  Indices in the array.
@@ skipping 1071 matching lines @@
   inline bool has_code_cache();
 
   // [prototype]: implicit prototype object.
   DECL_ACCESSORS(prototype, Object)
 
   // [constructor]: points back to the function responsible for this map.
   DECL_ACCESSORS(constructor, Object)
 
   // [instance descriptors]: describes the object.
   DECL_ACCESSORS(instance_descriptors, DescriptorArray)
-  inline void InitializeDescriptors(DescriptorArray* descriptors);
+
+  // [layout descriptor]: describes the object layout.
+  DECL_ACCESSORS(layout_descriptor, LayoutDescriptor)
+  // |layout descriptor| accessor which can be used from GC.
+  inline LayoutDescriptor* layout_descriptor_gc_safe();
+
+  inline static void RebuildLayoutDescriptor(Handle<Map> map);

[Toon Verwaest, 2014/07/29 15:02:09]

Avoid having a Rebuild method. Just make sure that the map is always in a
consistent state. So update on append if necessary. However, for bootstrapper
related objects we can't do field type tracking anyway, so no unboxing is
necessary.

+  inline void InitializeDescriptors(DescriptorArray* descriptors,

[Toon Verwaest, 2014/07/29 15:02:09]

UpdateDescriptors / SetDescriptors

[Igor Sheludko, 2014/10/30 14:23:44]

Done.

+                                    LayoutDescriptor* layout_descriptor);
+  inline void InitializeOwnDescriptors(DescriptorArray* descriptors,

[Toon Verwaest, 2014/07/29 15:02:09]

InitializeDescriptors

[Igor Sheludko, 2014/10/30 14:23:44]

Done.

+                                       LayoutDescriptor* layout_descriptor);
 
   // [stub cache]: contains stubs compiled for this map.
   DECL_ACCESSORS(code_cache, Object)
 
   // [dependent code]: list of optimized codes that weakly embed this map.
   DECL_ACCESSORS(dependent_code, DependentCode)
 
   // [back pointer]: points back to the parent map from which a transition
   // leads to this map. The field overlaps with prototype transitions and the
   // back pointer will be moved into the prototype transitions array if
@@ skipping 289 matching lines @@
   static const int kPrototypeOffset = kBitField3Offset + kPointerSize;
   static const int kConstructorOffset = kPrototypeOffset + kPointerSize;
   // Storage for the transition array is overloaded to directly contain a back
   // pointer if unused. When the map has transitions, the back pointer is
   // transferred to the transition array and accessed through an extra
   // indirection.
   static const int kTransitionsOrBackPointerOffset =
       kConstructorOffset + kPointerSize;
   static const int kDescriptorsOffset =
       kTransitionsOrBackPointerOffset + kPointerSize;
+#if V8_DOUBLE_FIELDS_UNBOXING
+  static const int kLayoutDecriptorOffset = kDescriptorsOffset + kPointerSize;
+  static const int kCodeCacheOffset = kLayoutDecriptorOffset + kPointerSize;
+#else
+  static const int kLayoutDecriptorOffset = 1;  // Must not be ever accessed.
   static const int kCodeCacheOffset = kDescriptorsOffset + kPointerSize;
+#endif
   static const int kDependentCodeOffset = kCodeCacheOffset + kPointerSize;
   static const int kSize = kDependentCodeOffset + kPointerSize;
 
   // Layout of pointer fields. Heap iteration code relies on them
   // being continuously allocated.
   static const int kPointerFieldsBeginOffset = Map::kPrototypeOffset;
   static const int kPointerFieldsEndOffset = kSize;
 
   // Byte offsets within kInstanceSizesOffset.
   static const int kInstanceSizeOffset = kInstanceSizesOffset + 0;
@@ skipping 57 matching lines @@
   typedef FixedBodyDescriptor<kPointerFieldsBeginOffset,
                               kPointerFieldsEndOffset,
                               kSize> BodyDescriptor;
 
   // Compares this map to another to see if they describe equivalent objects.
   // If |mode| is set to CLEAR_INOBJECT_PROPERTIES, |other| is treated as if
   // it had exactly zero inobject properties.
   // The "shared" flags of both this map and |other| are ignored.
   bool EquivalentToForNormalization(Map* other, PropertyNormalizationMode mode);
 
+  // Returns true if given field is unboxed double.
+  inline bool IsUnboxedDoubleField(FieldIndex index);
+
  private:
   static Handle<TransitionArray> SetElementsTransitionMap(
       Handle<Map> map, Handle<Map> transitioned_map);
 
   bool EquivalentToForTransition(Map* other);
   static Handle<Map> RawCopy(Handle<Map> map, int instance_size);
   static Handle<Map> ShareDescriptor(Handle<Map> map,
                                      Handle<DescriptorArray> descriptors,
                                      Descriptor* descriptor);
   static Handle<Map> CopyInstallDescriptors(
@@ skipping 30 matching lines @@
   // Zaps the contents of backing data structures. Note that the
   // heap verifier (i.e. VerifyMarkingVisitor) relies on zapping of objects
   // holding weak references when incremental marking is used, because it also
   // iterates over objects that are otherwise unreachable.
   // In general we only want to call these functions in release mode when
   // heap verification is turned on.
   void ZapPrototypeTransitions();
   void ZapTransitions();
 
   void DeprecateTransitionTree();
-  void DeprecateTarget(Name* key, DescriptorArray* new_descriptors);
+  void DeprecateTarget(Handle<Name> key,
+                       Handle<DescriptorArray> new_descriptors);
 
   Map* FindLastMatchMap(int verbatim, int length, DescriptorArray* descriptors);
 
   void UpdateDescriptor(int descriptor_number, Descriptor* desc);
 
   void PrintGeneralization(FILE* file,
                            const char* reason,
                            int modify_index,
                            int split,
                            int descriptors,
@@ skipping 4405 matching lines @@
   static inline int set(int value, int bit_position, bool v) {
     if (v) {
       value |= (1 << bit_position);
     } else {
       value &= ~(1 << bit_position);
     }
     return value;
   }
 };
 
+
+// InobjectPropertiesHelper is a helper class for querying layout descriptor
+// about whether the field at given offset is tagged or not.
+class InobjectPropertiesHelper {
+ public:
+  inline explicit InobjectPropertiesHelper(Map* map);
+
+  bool all_fields_tagged() { return all_fields_tagged_; }
+  inline bool IsTagged(int offset_in_bytes);
+
+ private:
+  bool all_fields_tagged_;
+  int header_size_;
+  int inobject_properties_count_;
+  LayoutDescriptor* layout_descriptor_;
+};
 } }  // namespace v8::internal
 
 #endif  // V8_OBJECTS_H_