VM: Reland Inline instance object hash code into object header on 64bit.

runtime/vm/raw_object.h

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #ifndef RUNTIME_VM_RAW_OBJECT_H_
 #define RUNTIME_VM_RAW_OBJECT_H_
 
 #include "platform/assert.h"
 #include "vm/atomic.h"
 #include "vm/exceptions.h"
@@ skipping 253 matching lines @@
     kMarkBit = 0,
     kCanonicalBit = 1,
     kVMHeapObjectBit = 2,
     kRememberedBit = 3,
     kReservedTagPos = 4,  // kReservedBit{100K,1M,10M}
     kReservedTagSize = 4,
     kSizeTagPos = kReservedTagPos + kReservedTagSize,  // = 8
     kSizeTagSize = 8,
     kClassIdTagPos = kSizeTagPos + kSizeTagSize,  // = 16
     kClassIdTagSize = 16,
+#if defined(HASH_IN_OBJECT_HEADER)
+    kHashTagPos = kClassIdTagPos + kClassIdTagSize,  // = 32
+    kHashTagSize = 16,
+#endif
   };
 
   COMPILE_ASSERT(kClassIdTagSize == (sizeof(classid_t) * kBitsPerByte));
 
   // Encodes the object size in the tag in units of object alignment.
   class SizeTag {
    public:
     static const intptr_t kMaxSizeTag = ((1 << RawObject::kSizeTagSize) - 1)
                                         << kObjectAlignmentLog2;
 
     static uword encode(intptr_t size) {
       return SizeBits::encode(SizeToTagValue(size));
     }
 
     static intptr_t decode(uword tag) {
       return TagValueToSize(SizeBits::decode(tag));
     }
 
     static uword update(intptr_t size, uword tag) {
       return SizeBits::update(SizeToTagValue(size), tag);
     }
 
    private:
     // The actual unscaled bit field used within the tag field.
     class SizeBits
-        : public BitField<uword, intptr_t, kSizeTagPos, kSizeTagSize> {};
+        : public BitField<uint32_t, intptr_t, kSizeTagPos, kSizeTagSize> {};
 
     static intptr_t SizeToTagValue(intptr_t size) {
       ASSERT(Utils::IsAligned(size, kObjectAlignment));
       return (size > kMaxSizeTag) ? 0 : (size >> kObjectAlignmentLog2);
     }
     static intptr_t TagValueToSize(intptr_t value) {
       return value << kObjectAlignmentLog2;
     }
   };
 
   class ClassIdTag
-      : public BitField<uword, intptr_t, kClassIdTagPos, kClassIdTagSize> {};
+      : public BitField<uint32_t, intptr_t, kClassIdTagPos, kClassIdTagSize> {};
 
   bool IsWellFormed() const {
     uword value = reinterpret_cast<uword>(this);
     return (value & kSmiTagMask) == 0 ||
            Utils::IsAligned(value - kHeapObjectTag, kWordSize);
   }
   bool IsHeapObject() const {
     ASSERT(IsWellFormed());
     uword value = reinterpret_cast<uword>(this);
     return (value & kSmiTagMask) == kHeapObjectTag;
@@ skipping 23 matching lines @@
   }
 
   // Support for GC marking bit.
   bool IsMarked() const { return MarkBit::decode(ptr()->tags_); }
   void SetMarkBit() {
     ASSERT(!IsMarked());
     UpdateTagBit<MarkBit>(true);
   }
   void SetMarkBitUnsynchronized() {
     ASSERT(!IsMarked());
-    uword tags = ptr()->tags_;
+    uint32_t tags = ptr()->tags_;
     ptr()->tags_ = MarkBit::update(true, tags);
   }
   void ClearMarkBit() {
     ASSERT(IsMarked());
     UpdateTagBit<MarkBit>(false);
   }
   // Returns false if the bit was already set.
   // TODO(koda): Add "must use result" annotation here, after we add support.
   bool TryAcquireMarkBit() { return TryAcquireTagBit<MarkBit>(); }
 
   // Support for object tags.
   bool IsCanonical() const { return CanonicalObjectTag::decode(ptr()->tags_); }
   void SetCanonical() { UpdateTagBit<CanonicalObjectTag>(true); }
   void ClearCanonical() { UpdateTagBit<CanonicalObjectTag>(false); }
   bool IsVMHeapObject() const { return VMHeapObjectTag::decode(ptr()->tags_); }
   void SetVMHeapObject() { UpdateTagBit<VMHeapObjectTag>(true); }
 
   // Support for GC remembered bit.
   bool IsRemembered() const { return RememberedBit::decode(ptr()->tags_); }
   void SetRememberedBit() {
     ASSERT(!IsRemembered());
     UpdateTagBit<RememberedBit>(true);
   }
   void SetRememberedBitUnsynchronized() {
     ASSERT(!IsRemembered());
-    uword tags = ptr()->tags_;
+    uint32_t tags = ptr()->tags_;
     ptr()->tags_ = RememberedBit::update(true, tags);
   }
   void ClearRememberedBit() { UpdateTagBit<RememberedBit>(false); }
   void ClearRememberedBitUnsynchronized() {
-    uword tags = ptr()->tags_;
+    uint32_t tags = ptr()->tags_;
     ptr()->tags_ = RememberedBit::update(false, tags);
   }
   // Returns false if the bit was already set.
   // TODO(koda): Add "must use result" annotation here, after we add support.
   bool TryAcquireRememberedBit() { return TryAcquireTagBit<RememberedBit>(); }
 
 #define DEFINE_IS_CID(clazz)                                                   \
   bool Is##clazz() const { return ((GetClassId() == k##clazz##Cid)); }
   CLASS_LIST(DEFINE_IS_CID)
 #undef DEFINE_IS_CID
 
 #define DEFINE_IS_CID(clazz)                                                   \
   bool IsTypedData##clazz() const {                                            \
     return ((GetClassId() == kTypedData##clazz##Cid));                         \
   }                                                                            \
   bool IsTypedDataView##clazz() const {                                        \
     return ((GetClassId() == kTypedData##clazz##ViewCid));                     \
   }                                                                            \
   bool IsExternalTypedData##clazz() const {                                    \
     return ((GetClassId() == kExternalTypedData##clazz##Cid));                 \
   }
   CLASS_LIST_TYPED_DATA(DEFINE_IS_CID)
 #undef DEFINE_IS_CID
 
   bool IsStringInstance() const { return IsStringClassId(GetClassId()); }
+  bool IsRawNull() const { return GetClassId() == kNullCid; }
   bool IsDartInstance() const {
     return (!IsHeapObject() || (GetClassId() >= kInstanceCid));
   }
   bool IsFreeListElement() const {
     return ((GetClassId() == kFreeListElement));
   }
   bool IsForwardingCorpse() const {
     return ((GetClassId() == kForwardingCorpse));
   }
   bool IsPseudoObject() const {
     return IsFreeListElement() || IsForwardingCorpse();
   }
 
   intptr_t Size() const {
-    uword tags = ptr()->tags_;
+    uint32_t tags = ptr()->tags_;
     intptr_t result = SizeTag::decode(tags);
     if (result != 0) {
 #if defined(DEBUG)
       // TODO(22501) Array::MakeFixedLength has a race with this code: we might
       // have loaded tags field and then MakeFixedLength could have updated it
       // leading to inconsistency between SizeFromClass() and
       // SizeTag::decode(tags). We are working around it by reloading tags_ and
       // recomputing size from tags.
       const intptr_t size_from_class = SizeFromClass();
       if ((result > size_from_class) && (GetClassId() == kArrayCid) &&
@@ skipping 89 matching lines @@
   static bool IsTypedDataClassId(intptr_t index);
   static bool IsTypedDataViewClassId(intptr_t index);
   static bool IsExternalTypedDataClassId(intptr_t index);
   static bool IsInternalVMdefinedClassId(intptr_t index);
   static bool IsVariableSizeClassId(intptr_t index);
   static bool IsImplicitFieldClassId(intptr_t index);
 
   static intptr_t NumberOfTypedDataClasses();
 
  private:
-  uword tags_;  // Various object tags (bits).
+  uint32_t tags_;  // Various object tags (bits).
+#if defined(HASH_IN_OBJECT_HEADER)
+  // On 64 bit there is a hash field in the header for the identity hash.
+  uint32_t hash_;
+#endif
 
-  class MarkBit : public BitField<uword, bool, kMarkBit, 1> {};
+  class MarkBit : public BitField<uint32_t, bool, kMarkBit, 1> {};
 
-  class RememberedBit : public BitField<uword, bool, kRememberedBit, 1> {};
+  class RememberedBit : public BitField<uint32_t, bool, kRememberedBit, 1> {};
 
-  class CanonicalObjectTag : public BitField<uword, bool, kCanonicalBit, 1> {};
+  class CanonicalObjectTag : public BitField<uint32_t, bool, kCanonicalBit, 1> {
+  };
 
-  class VMHeapObjectTag : public BitField<uword, bool, kVMHeapObjectBit, 1> {};
+  class VMHeapObjectTag : public BitField<uint32_t, bool, kVMHeapObjectBit, 1> {
+  };
 
   class ReservedBits
-      : public BitField<uword, intptr_t, kReservedTagPos, kReservedTagSize> {};
+      : public BitField<uint32_t, intptr_t, kReservedTagPos, kReservedTagSize> {
+  };
 
   // TODO(koda): After handling tags_, return const*, like Object::raw_ptr().
   RawObject* ptr() const {
     ASSERT(IsHeapObject());
     return reinterpret_cast<RawObject*>(reinterpret_cast<uword>(this) -
                                         kHeapObjectTag);
   }
 
   intptr_t VisitPointersPredefined(ObjectPointerVisitor* visitor,
                                    intptr_t class_id);
 
   intptr_t SizeFromClass() const;
 
   intptr_t GetClassId() const {
-    uword tags = ptr()->tags_;
+    uint32_t tags = ptr()->tags_;
     return ClassIdTag::decode(tags);
   }
 
   void SetClassId(intptr_t new_cid) {
-    uword tags = ptr()->tags_;
+    uint32_t tags = ptr()->tags_;
     ptr()->tags_ = ClassIdTag::update(new_cid, tags);
   }
 
   template <class TagBitField>
   void UpdateTagBit(bool value) {
-    uword tags = ptr()->tags_;
-    uword old_tags;
+    uint32_t tags = ptr()->tags_;
+    uint32_t old_tags;
     do {
       old_tags = tags;
-      uword new_tags = TagBitField::update(value, old_tags);
-      tags = AtomicOperations::CompareAndSwapWord(&ptr()->tags_, old_tags,
-                                                  new_tags);
+      uint32_t new_tags = TagBitField::update(value, old_tags);
+      tags = AtomicOperations::CompareAndSwapUint32(&ptr()->tags_, old_tags,
+                                                    new_tags);
     } while (tags != old_tags);
   }
 
   template <class TagBitField>
   bool TryAcquireTagBit() {
-    uword tags = ptr()->tags_;
-    uword old_tags;
+    uint32_t tags = ptr()->tags_;
+    uint32_t old_tags;
     do {
       old_tags = tags;
       if (TagBitField::decode(tags)) return false;
-      uword new_tags = TagBitField::update(true, old_tags);
-      tags = AtomicOperations::CompareAndSwapWord(&ptr()->tags_, old_tags,
-                                                  new_tags);
+      uint32_t new_tags = TagBitField::update(true, old_tags);
+      tags = AtomicOperations::CompareAndSwapUint32(&ptr()->tags_, old_tags,
+                                                    new_tags);
     } while (tags != old_tags);
     return true;
   }
 
   // All writes to heap objects should ultimately pass through one of the
   // methods below or their counterparts in Object, to ensure that the
   // write barrier is correctly applied.
 
   template <typename type>
   void StorePointer(type const* addr, type value) {
@@ skipping 708 matching lines @@
     // Is kKindShiftSize enough bits?
     COMPILE_ASSERT(kLastKind <= 1 << ((1 << kKindShiftSize) - 1));
 
     static const intptr_t kTryIndexPos = kKindShiftSize;
     static const intptr_t kTryIndexSize = kBitsPerWord - kKindShiftSize;
   };
 
  private:
   RAW_HEAP_OBJECT_IMPLEMENTATION(PcDescriptors);
 
-  int32_t length_;  // Number of descriptors.
+  // Number of descriptors.  This only needs to be an int32_t, but we make it a
+  // uword so that the variable length data is 64 bit aligned on 64 bit
+  // platforms.
+  uword length_;
 
   // Variable length data follows here.
   uint8_t* data() { OPEN_ARRAY_START(uint8_t, intptr_t); }
   const uint8_t* data() const { OPEN_ARRAY_START(uint8_t, intptr_t); }
 
   friend class Object;
 };
 
 
 // CodeSourceMap encodes a mapping from code PC ranges to source token
 // positions and the stack of inlined functions.
 class RawCodeSourceMap : public RawObject {
  private:
   RAW_HEAP_OBJECT_IMPLEMENTATION(CodeSourceMap);
 
-  int32_t length_;  // Length in bytes.
+  // Length in bytes.  This only needs to be an int32_t, but we make it a uword
+  // so that the variable length data is 64 bit aligned on 64 bit platforms.
+  uword length_;
 
   // Variable length data follows here.
   uint8_t* data() { OPEN_ARRAY_START(uint8_t, intptr_t); }
   const uint8_t* data() const { OPEN_ARRAY_START(uint8_t, intptr_t); }
 
   friend class Object;
 };
 
 
 // StackMap is an immutable representation of the layout of the stack at a
 // PC. The stack map representation consists of a bit map which marks each
 // live object index starting from the base of the frame.
 //
 // The bit map representation is optimized for dense and small bit maps, without
 // any upper bound.
 class RawStackMap : public RawObject {
   RAW_HEAP_OBJECT_IMPLEMENTATION(StackMap);
 
   // Regarding changing this to a bitfield: ARM64 requires register_bit_count_
   // to be as large as 96, meaning 7 bits, leaving 25 bits for the length, or
   // as large as ~33 million entries. If that is sufficient, then these two
   // fields can be merged into a BitField.
   int32_t length_;               // Length of payload, in bits.
   int32_t slow_path_bit_count_;  // Slow path live values, included in length_.
 
   // Offset from code entry point corresponding to this stack map
-  // representation.
-  uint32_t pc_offset_;
+  // representation. This only needs to be an int32_t, but we make it a uword
+  // so that the variable length data is 64 bit aligned on 64 bit platforms.
+  uword pc_offset_;
 
   // Variable length data follows here (bitmap of the stack layout).
   uint8_t* data() { OPEN_ARRAY_START(uint8_t, uint8_t); }
   const uint8_t* data() const { OPEN_ARRAY_START(uint8_t, uint8_t); }
 };
 
 
 class RawLocalVarDescriptors : public RawObject {
  public:
   enum VarInfoKind {
@@ skipping 33 matching lines @@
       index_kind = KindBits::update(kind, index_kind);
     }
     int32_t index() const { return IndexBits::decode(index_kind) - kIndexBias; }
     void set_index(int32_t index) {
       index_kind = IndexBits::update(index + kIndexBias, index_kind);
     }
   };
 
  private:
   RAW_HEAP_OBJECT_IMPLEMENTATION(LocalVarDescriptors);
-  int32_t num_entries_;  // Number of descriptors.
+  // Number of descriptors. This only needs to be an int32_t, but we make it a
+  // uword so that the variable length data is 64 bit aligned on 64 bit
+  // platforms.
+  uword num_entries_;
 
   RawObject** from() {
     return reinterpret_cast<RawObject**>(&ptr()->names()[0]);
   }
   RawString** names() {
     // Array of [num_entries_] variable names.
     OPEN_ARRAY_START(RawString*, RawString*);
   }
   RawString** nameAddrAt(intptr_t i) { return &(ptr()->names()[i]); }
 
@@ skipping 487 matching lines @@
 
 
 class RawOneByteString : public RawString {
   RAW_HEAP_OBJECT_IMPLEMENTATION(OneByteString);
 
   // Variable length data follows here.
   uint8_t* data() { OPEN_ARRAY_START(uint8_t, uint8_t); }
   const uint8_t* data() const { OPEN_ARRAY_START(uint8_t, uint8_t); }
 
   friend class ApiMessageReader;
+  friend class RODataSerializationCluster;
   friend class SnapshotReader;
-  friend class RODataSerializationCluster;
+  friend class String;
 };
 
 
 class RawTwoByteString : public RawString {
   RAW_HEAP_OBJECT_IMPLEMENTATION(TwoByteString);
 
   // Variable length data follows here.
   uint16_t* data() { OPEN_ARRAY_START(uint16_t, uint16_t); }
   const uint16_t* data() const { OPEN_ARRAY_START(uint16_t, uint16_t); }
 
+  friend class RODataSerializationCluster;
   friend class SnapshotReader;
-  friend class RODataSerializationCluster;
+  friend class String;
 };
 
 
 template <typename T>
 class ExternalStringData {
  public:
   ExternalStringData(const T* data, void* peer, Dart_PeerFinalizer callback)
       : data_(data), peer_(peer), callback_(callback) {}
   ~ExternalStringData() {
     if (callback_ != NULL) (*callback_)(peer_);
@@ skipping 15 matching lines @@
 
 class RawExternalOneByteString : public RawString {
   RAW_HEAP_OBJECT_IMPLEMENTATION(ExternalOneByteString);
 
  public:
   typedef ExternalStringData<uint8_t> ExternalData;
 
  private:
   ExternalData* external_data_;
   friend class Api;
+  friend class String;
 };
 
 
 class RawExternalTwoByteString : public RawString {
   RAW_HEAP_OBJECT_IMPLEMENTATION(ExternalTwoByteString);
 
  public:
   typedef ExternalStringData<uint16_t> ExternalData;
 
  private:
   ExternalData* external_data_;
   friend class Api;
+  friend class String;
 };
 
 
 class RawBool : public RawInstance {
   RAW_HEAP_OBJECT_IMPLEMENTATION(Bool);
 
   bool value_;
 };
 
 
@@ skipping 498 matching lines @@
                  kTypedDataInt8ArrayViewCid + 15);
   COMPILE_ASSERT(kByteBufferCid == kExternalTypedDataInt8ArrayCid + 14);
   COMPILE_ASSERT(kNullCid == kByteBufferCid + 1);
   return (kNullCid - kTypedDataInt8ArrayCid);
 }
 
 
 }  // namespace dart
 
 #endif  // RUNTIME_VM_RAW_OBJECT_H_