VM: Reland Inline instance object hash code into object header on 64bit.

runtime/vm/object.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/object.h"
 
 #include "include/dart_api.h"
 #include "platform/assert.h"
 #include "vm/assembler.h"
 #include "vm/become.h"
@@ skipping 974 matching lines @@
   ASSERT(!speculative_inlining_error_->IsSmi());
   ASSERT(speculative_inlining_error_->IsLanguageError());
   ASSERT(!background_compilation_error_->IsSmi());
   ASSERT(background_compilation_error_->IsLanguageError());
   ASSERT(!vm_isolate_snapshot_object_table_->IsSmi());
   ASSERT(vm_isolate_snapshot_object_table_->IsArray());
 }
 
 
 // An object visitor which will mark all visited objects. This is used to
-// premark all objects in the vm_isolate_ heap.
-class PremarkingVisitor : public ObjectVisitor {
+// premark all objects in the vm_isolate_ heap.  Also precalculates hash
+// codes so that we can get the identity hash code of objects in the read-
+// only VM isolate.
+class FinalizeVMIsolateVisitor : public ObjectVisitor {
  public:
-  PremarkingVisitor() {}
+  FinalizeVMIsolateVisitor()
+#if defined(HASH_IN_OBJECT_HEADER)
+      : counter_(1337)
+#endif
+  {
+  }
 
   void VisitObject(RawObject* obj) {
     // Free list elements should never be marked.
     ASSERT(!obj->IsMarked());
     // No forwarding corpses in the VM isolate.
     ASSERT(!obj->IsForwardingCorpse());
     if (!obj->IsFreeListElement()) {
       ASSERT(obj->IsVMHeapObject());
       obj->SetMarkBitUnsynchronized();
+      if (obj->IsStringInstance()) {
+        RawString* str = reinterpret_cast<RawString*>(obj);
+        intptr_t hash = String::Hash(str);
+        String::SetCachedHash(str, hash);
+      }
+#if defined(HASH_IN_OBJECT_HEADER)
+      // These objects end up in the read-only VM isolate which is shared
+      // between isolates, so we have to prepopulate them with identity hash
+      // codes, since we can't add hash codes later.
+      if (Object::GetCachedHash(obj) == 0) {
+        // Some classes have identity hash codes that depend on their contents,
+        // not per object.
+        ASSERT(!obj->IsStringInstance());
+        if (!obj->IsMint() && !obj->IsDouble() && !obj->IsBigint() &&
+            !obj->IsRawNull() && !obj->IsBool()) {
+          counter_ += 2011;  // The year Dart was announced and a prime.
+          counter_ &= 0x3fffffff;
+          if (counter_ == 0) counter_++;
+          Object::SetCachedHash(obj, counter_);
+        }
+      }
+#endif
     }
   }
+
+ private:
+#if defined(HASH_IN_OBJECT_HEADER)
+  int32_t counter_;
+#endif
 };
 
 
 #define SET_CLASS_NAME(class_name, name)                                       \
   cls = class_name##_class();                                                  \
   cls.set_name(Symbols::name());
 
 void Object::FinalizeVMIsolate(Isolate* isolate) {
   // Should only be run by the vm isolate.
   ASSERT(isolate == Dart::vm_isolate());
@@ skipping 59 matching lines @@
   // Set up names for the pseudo-classes for free list elements and forwarding
   // corpses. Mainly this makes VM debugging easier.
   cls = isolate->class_table()->At(kFreeListElement);
   cls.set_name(Symbols::FreeListElement());
   cls = isolate->class_table()->At(kForwardingCorpse);
   cls.set_name(Symbols::ForwardingCorpse());
 
   {
     ASSERT(isolate == Dart::vm_isolate());
     WritableVMIsolateScope scope(Thread::Current());
-    PremarkingVisitor premarker;
+    FinalizeVMIsolateVisitor premarker;
     ASSERT(isolate->heap()->UsedInWords(Heap::kNew) == 0);
     isolate->heap()->IterateOldObjectsNoImagePages(&premarker);
     // Make the VM isolate read-only again after setting all objects as marked.
     // Note objects in image pages are already pre-marked.
   }
 }
 
 
 void Object::set_vm_isolate_snapshot_object_table(const Array& table) {
   ASSERT(Isolate::Current() == Dart::vm_isolate());
@@ skipping 14 matching lines @@
   if (original_size > used_size) {
     intptr_t leftover_size = original_size - used_size;
 
     uword addr = RawObject::ToAddr(obj.raw()) + used_size;
     if (leftover_size >= TypedData::InstanceSize(0)) {
       // Update the leftover space as a TypedDataInt8Array object.
       RawTypedData* raw =
           reinterpret_cast<RawTypedData*>(RawObject::FromAddr(addr));
       uword new_tags = RawObject::ClassIdTag::update(kTypedDataInt8ArrayCid, 0);
       new_tags = RawObject::SizeTag::update(leftover_size, new_tags);
-      uword tags = raw->ptr()->tags_;
-      uword old_tags;
+      uint32_t tags = raw->ptr()->tags_;
+      uint32_t old_tags;
       // TODO(iposva): Investigate whether CompareAndSwapWord is necessary.
       do {
         old_tags = tags;
-        tags = AtomicOperations::CompareAndSwapWord(&raw->ptr()->tags_,
-                                                    old_tags, new_tags);
+        // We can't use obj.CompareAndSwapTags here because we don't have a
+        // handle for the new object.
+        tags = AtomicOperations::CompareAndSwapUint32(&raw->ptr()->tags_,
+                                                      old_tags, new_tags);
       } while (tags != old_tags);
 
       intptr_t leftover_len = (leftover_size - TypedData::InstanceSize(0));
       ASSERT(TypedData::InstanceSize(leftover_len) == leftover_size);
       raw->StoreSmi(&(raw->ptr()->length_), Smi::New(leftover_len));
     } else {
       // Update the leftover space as a basic object.
       ASSERT(leftover_size == Object::InstanceSize());
       RawObject* raw = reinterpret_cast<RawObject*>(RawObject::FromAddr(addr));
       uword new_tags = RawObject::ClassIdTag::update(kInstanceCid, 0);
       new_tags = RawObject::SizeTag::update(leftover_size, new_tags);
-      uword tags = raw->ptr()->tags_;
-      uword old_tags;
+      uint32_t tags = raw->ptr()->tags_;
+      uint32_t old_tags;
       // TODO(iposva): Investigate whether CompareAndSwapWord is necessary.
       do {
         old_tags = tags;
-        tags = AtomicOperations::CompareAndSwapWord(&raw->ptr()->tags_,
-                                                    old_tags, new_tags);
+        tags = obj.CompareAndSwapTags(old_tags, new_tags);
       } while (tags != old_tags);
     }
   }
 }
 
 
 void Object::VerifyBuiltinVtables() {
 #if defined(DEBUG)
   Thread* thread = Thread::Current();
   Isolate* isolate = thread->isolate();
@@ skipping 701 matching lines @@
                               bool is_vm_object) {
   uword initial_value = (class_id == kInstructionsCid)
                             ? Assembler::GetBreakInstructionFiller()
                             : reinterpret_cast<uword>(null_);
   uword cur = address;
   uword end = address + size;
   while (cur < end) {
     *reinterpret_cast<uword*>(cur) = initial_value;
     cur += kWordSize;
   }
-  uword tags = 0;
+  uint32_t tags = 0;
   ASSERT(class_id != kIllegalCid);
   tags = RawObject::ClassIdTag::update(class_id, tags);
   tags = RawObject::SizeTag::update(size, tags);
   tags = RawObject::VMHeapObjectTag::update(is_vm_object, tags);
   reinterpret_cast<RawObject*>(address)->tags_ = tags;
+#if defined(HASH_IN_OBJECT_HEADER)
+  reinterpret_cast<RawObject*>(address)->hash_ = 0;
+#endif
   ASSERT(is_vm_object == RawObject::IsVMHeapObject(tags));
 }
 
 
 void Object::CheckHandle() const {
 #if defined(DEBUG)
   if (raw_ != Object::null()) {
     if ((reinterpret_cast<uword>(raw_) & kSmiTagMask) == kSmiTag) {
       ASSERT(vtable() == Smi::handle_vtable_);
       return;
@@ skipping 18446 matching lines @@
 static intptr_t HashImpl(const T* characters, intptr_t len) {
   ASSERT(len >= 0);
   StringHasher hasher;
   for (intptr_t i = 0; i < len; i++) {
     hasher.Add(characters[i]);
   }
   return hasher.Finalize(String::kHashBits);
 }
 
 
+intptr_t String::Hash(RawString* raw) {
+  StringHasher hasher;
+  uword length = Smi::Value(raw->ptr()->length_);
+  if (raw->IsOneByteString() || raw->IsExternalOneByteString()) {
+    const uint8_t* data;
+    if (raw->IsOneByteString()) {
+      data = reinterpret_cast<RawOneByteString*>(raw)->ptr()->data();
+    } else {
+      ASSERT(raw->IsExternalOneByteString());
+      RawExternalOneByteString* str =
+          reinterpret_cast<RawExternalOneByteString*>(raw);
+      data = str->ptr()->external_data_->data();
+    }
+    return String::Hash(data, length);
+  } else {
+    const uint16_t* data;
+    if (raw->IsTwoByteString()) {
+      data = reinterpret_cast<RawTwoByteString*>(raw)->ptr()->data();
+    } else {
+      ASSERT(raw->IsExternalTwoByteString());
+      RawExternalTwoByteString* str =
+          reinterpret_cast<RawExternalTwoByteString*>(raw);
+      data = str->ptr()->external_data_->data();
+    }
+    return String::Hash(data, length);
+  }
+}
+
+
 intptr_t String::Hash(const char* characters, intptr_t len) {
   return HashImpl(characters, len);
 }
 
 
 intptr_t String::Hash(const uint8_t* characters, intptr_t len) {
   return HashImpl(characters, len);
 }
 
 
@@ skipping 800 matching lines @@
         memmove(array, OneByteString::CharAddr(*this, 0), str_length);
       }
 
       // If there is any left over space fill it with either an Array object or
       // just a plain object (depending on the amount of left over space) so
       // that it can be traversed over successfully during garbage collection.
       Object::MakeUnusedSpaceTraversable(*this, original_size, used_size);
 
       // Update the class information of the object.
       const intptr_t class_id = kExternalOneByteStringCid;
-      uword tags = raw_ptr()->tags_;
-      uword old_tags;
+      uint32_t tags = raw_ptr()->tags_;
+      uint32_t old_tags;
       do {
         old_tags = tags;
-        uword new_tags = RawObject::SizeTag::update(used_size, old_tags);
+        uint32_t new_tags = RawObject::SizeTag::update(used_size, old_tags);
         new_tags = RawObject::ClassIdTag::update(class_id, new_tags);
         tags = CompareAndSwapTags(old_tags, new_tags);
       } while (tags != old_tags);
       result = this->raw();
       const uint8_t* ext_array = reinterpret_cast<const uint8_t*>(array);
       ExternalStringData<uint8_t>* ext_data =
           new ExternalStringData<uint8_t>(ext_array, peer, cback);
       ASSERT(result.Length() == str_length);
       ASSERT(!result.HasHash() ||
              (result.Hash() == String::Hash(ext_array, str_length)));
@@ skipping 12 matching lines @@
                 (str_length * kTwoByteChar));
       }
 
       // If there is any left over space fill it with either an Array object or
       // just a plain object (depending on the amount of left over space) so
       // that it can be traversed over successfully during garbage collection.
       Object::MakeUnusedSpaceTraversable(*this, original_size, used_size);
 
       // Update the class information of the object.
       const intptr_t class_id = kExternalTwoByteStringCid;
-      uword tags = raw_ptr()->tags_;
-      uword old_tags;
+      uint32_t tags = raw_ptr()->tags_;
+      uint32_t old_tags;
       do {
         old_tags = tags;
-        uword new_tags = RawObject::SizeTag::update(used_size, old_tags);
+        uint32_t new_tags = RawObject::SizeTag::update(used_size, old_tags);
         new_tags = RawObject::ClassIdTag::update(class_id, new_tags);
         tags = CompareAndSwapTags(old_tags, new_tags);
       } while (tags != old_tags);
       result = this->raw();
       const uint16_t* ext_array = reinterpret_cast<const uint16_t*>(array);
       ExternalStringData<uint16_t>* ext_data =
           new ExternalStringData<uint16_t>(ext_array, peer, cback);
       ASSERT(result.Length() == str_length);
       ASSERT(!result.HasHash() ||
              (result.Hash() == String::Hash(ext_array, str_length)));
@@ skipping 867 matching lines @@
   }
 
   return dest.raw();
 }
 
 
 void Array::MakeImmutable() const {
   if (IsImmutable()) return;
   ASSERT(!IsCanonical());
   NoSafepointScope no_safepoint;
-  uword tags = raw_ptr()->tags_;
-  uword old_tags;
+  uint32_t tags = raw_ptr()->tags_;
+  uint32_t old_tags;
   do {
     old_tags = tags;
-    uword new_tags =
+    uint32_t new_tags =
         RawObject::ClassIdTag::update(kImmutableArrayCid, old_tags);
     tags = CompareAndSwapTags(old_tags, new_tags);
   } while (tags != old_tags);
 }
 
 
 const char* Array::ToCString() const {
   if (IsNull()) {
     return IsImmutable() ? "_ImmutableList NULL" : "_List NULL";
   }
@@ skipping 44 matching lines @@
 
     // The backing array may be a shared instance, or may not have correct
     // type parameters. Create a new empty array.
     Heap::Space space = thread->IsMutatorThread() ? Heap::kNew : Heap::kOld;
     Array& array = Array::Handle(zone, Array::New(0, space));
     array.SetTypeArguments(type_arguments);
     return array.raw();
   }
   intptr_t capacity_len = growable_array.Capacity();
   const Array& array = Array::Handle(zone, growable_array.data());
+  ASSERT(array.IsArray());
   array.SetTypeArguments(type_arguments);
   intptr_t capacity_size = Array::InstanceSize(capacity_len);
   intptr_t used_size = Array::InstanceSize(used_len);
   NoSafepointScope no_safepoint;
 
   // If there is any left over space fill it with either an Array object or
   // just a plain object (depending on the amount of left over space) so
   // that it can be traversed over successfully during garbage collection.
   Object::MakeUnusedSpaceTraversable(array, capacity_size, used_size);
 
   // Update the size in the header field and length of the array object.
   uword tags = array.raw_ptr()->tags_;
   ASSERT(kArrayCid == RawObject::ClassIdTag::decode(tags));
-  uword old_tags;
+  uint32_t old_tags;
   do {
     old_tags = tags;
-    uword new_tags = RawObject::SizeTag::update(used_size, old_tags);
+    uint32_t new_tags = RawObject::SizeTag::update(used_size, old_tags);
     tags = array.CompareAndSwapTags(old_tags, new_tags);
   } while (tags != old_tags);
   // TODO(22501): For the heap to remain walkable by the sweeper, it must
   // observe the creation of the filler object no later than the new length
   // of the array. This assumption holds on ia32/x64 or if the CAS above is a
   // full memory barrier.
   //
   // Also, between the CAS of the header above and the SetLength below,
   // the array is temporarily in an inconsistent state. The header is considered
   // the overriding source of object size by RawObject::Size, but the ASSERTs
@@ skipping 1315 matching lines @@
   return UserTag::null();
 }
 
 
 const char* UserTag::ToCString() const {
   const String& tag_label = String::Handle(label());
   return tag_label.ToCString();
 }
 
 }  // namespace dart