[stubs,interpreter] Optimise SMI loading for 64-bit targets.

src/code-stub-assembler.cc

 // Copyright 2016 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/code-stub-assembler.h"
 #include "src/code-factory.h"
 #include "src/frames-inl.h"
 #include "src/frames.h"
 #include "src/ic/stub-cache.h"
 
@@ skipping 56 matching lines @@
 
 Node* CodeStubAssembler::UndefinedConstant() {
   return LoadRoot(Heap::kUndefinedValueRootIndex);
 }
 
 Node* CodeStubAssembler::TheHoleConstant() {
   return LoadRoot(Heap::kTheHoleValueRootIndex);
 }
 
 Node* CodeStubAssembler::HashSeed() {
-  return SmiToWord32(LoadRoot(Heap::kHashSeedRootIndex));
+  return LoadAndUntagToWord32Root(Heap::kHashSeedRootIndex);
 }
 
 Node* CodeStubAssembler::StaleRegisterConstant() {
   return LoadRoot(Heap::kStaleRegisterRootIndex);
 }
 
 Node* CodeStubAssembler::Float64Round(Node* x) {
   Node* one = Float64Constant(1.0);
   Node* one_half = Float64Constant(0.5);
 
@@ skipping 819 matching lines @@
 Node* CodeStubAssembler::LoadObjectField(Node* object, int offset,
                                          MachineType rep) {
   return Load(rep, object, IntPtrConstant(offset - kHeapObjectTag));
 }
 
 Node* CodeStubAssembler::LoadObjectField(Node* object, Node* offset,
                                          MachineType rep) {
   return Load(rep, object, IntPtrSub(offset, IntPtrConstant(kHeapObjectTag)));
 }
 
+Node* CodeStubAssembler::LoadAndUntagObjectField(Node* object, int offset) {
+  if (Is64()) {
+#if V8_TARGET_LITTLE_ENDIAN
+    offset += kPointerSize / 2;
+#endif
+    return ChangeInt32ToInt64(
+        LoadObjectField(object, offset, MachineType::Int32()));
+  } else {
+    return SmiToWord(LoadObjectField(object, offset, MachineType::AnyTagged()));
+  }
+}
+
+Node* CodeStubAssembler::LoadAndUntagToWord32ObjectField(Node* object,
+                                                         int offset) {
+  if (Is64()) {
+#if V8_TARGET_LITTLE_ENDIAN
+    offset += kPointerSize / 2;
+#endif
+    return LoadObjectField(object, offset, MachineType::Int32());
+  } else {
+    return SmiToWord32(
+        LoadObjectField(object, offset, MachineType::AnyTagged()));
+  }
+}
+
+Node* CodeStubAssembler::LoadAndUntagSmi(Node* base, int index) {
+  if (Is64()) {
+#if V8_TARGET_LITTLE_ENDIAN
+    index += kPointerSize / 2;
+#endif
+    return ChangeInt32ToInt64(
+        Load(MachineType::Int32(), base, IntPtrConstant(index)));
+  } else {
+    return SmiToWord(
+        Load(MachineType::AnyTagged(), base, IntPtrConstant(index)));
+  }
+}
+
+Node* CodeStubAssembler::LoadAndUntagToWord32Root(
+    Heap::RootListIndex root_index) {
+  Node* roots_array_start =
+      ExternalConstant(ExternalReference::roots_array_start(isolate()));
+  int index = root_index * kPointerSize;
+  if (Is64()) {
+#if V8_TARGET_LITTLE_ENDIAN
+    index += kPointerSize / 2;
+#endif
+    return Load(MachineType::Int32(), roots_array_start, IntPtrConstant(index));
+  } else {
+    return SmiToWord32(Load(MachineType::AnyTagged(), roots_array_start,
+                            IntPtrConstant(index)));
+  }
+}
+
 Node* CodeStubAssembler::LoadHeapNumberValue(Node* object) {
   return LoadObjectField(object, HeapNumber::kValueOffset,
                          MachineType::Float64());
 }
 
 Node* CodeStubAssembler::LoadMap(Node* object) {
   return LoadObjectField(object, HeapObject::kMapOffset);
 }
 
 Node* CodeStubAssembler::LoadInstanceType(Node* object) {
   return LoadMapInstanceType(LoadMap(object));
 }
 
 void CodeStubAssembler::AssertInstanceType(Node* object,
                                            InstanceType instance_type) {
   Assert(Word32Equal(LoadInstanceType(object), Int32Constant(instance_type)));
 }
 
 Node* CodeStubAssembler::LoadProperties(Node* object) {
   return LoadObjectField(object, JSObject::kPropertiesOffset);
 }
 
 Node* CodeStubAssembler::LoadElements(Node* object) {
   return LoadObjectField(object, JSObject::kElementsOffset);
 }
 
-Node* CodeStubAssembler::LoadFixedArrayBaseLength(Node* array) {
-  return LoadObjectField(array, FixedArrayBase::kLengthOffset);
+Node* CodeStubAssembler::LoadAndUntagFixedArrayBaseLength(Node* array) {
+  return LoadAndUntagObjectField(array, FixedArrayBase::kLengthOffset);
 }
 
 Node* CodeStubAssembler::LoadMapBitField(Node* map) {
   return LoadObjectField(map, Map::kBitFieldOffset, MachineType::Uint8());
 }
 
 Node* CodeStubAssembler::LoadMapBitField2(Node* map) {
   return LoadObjectField(map, Map::kBitField2Offset, MachineType::Uint8());
 }
 
@@ skipping 94 matching lines @@
 Node* CodeStubAssembler::LoadFixedArrayElement(Node* object, Node* index_node,
                                                int additional_offset,
                                                ParameterMode parameter_mode) {
   int32_t header_size =
       FixedArray::kHeaderSize + additional_offset - kHeapObjectTag;
   Node* offset = ElementOffsetFromIndex(index_node, FAST_HOLEY_ELEMENTS,
                                         parameter_mode, header_size);
   return Load(MachineType::AnyTagged(), object, offset);
 }
 
+Node* CodeStubAssembler::LoadAndUntagToWord32FixedArrayElement(
+    Node* object, Node* index_node, int additional_offset,
+    ParameterMode parameter_mode) {
+  int32_t header_size =
+      FixedArray::kHeaderSize + additional_offset - kHeapObjectTag;
+#if V8_TARGET_LITTLE_ENDIAN
+  if (Is64()) {
+    header_size += kPointerSize / 2;
+  }
+#endif
+  Node* offset = ElementOffsetFromIndex(index_node, FAST_HOLEY_ELEMENTS,
+                                        parameter_mode, header_size);
+  if (Is64()) {
+    return Load(MachineType::Int32(), object, offset);
+  } else {
+    return SmiToWord32(Load(MachineType::AnyTagged(), object, offset));
+  }
+}
+
 Node* CodeStubAssembler::LoadFixedDoubleArrayElement(
     Node* object, Node* index_node, MachineType machine_type,
     int additional_offset, ParameterMode parameter_mode) {
   int32_t header_size =
       FixedDoubleArray::kHeaderSize + additional_offset - kHeapObjectTag;
   Node* offset = ElementOffsetFromIndex(index_node, FAST_HOLEY_DOUBLE_ELEMENTS,
                                         parameter_mode, header_size);
   return Load(machine_type, object, offset);
 }
 
@@ skipping 839 matching lines @@
                 CallRuntime(Runtime::kExternalStringGetChar,
                             NoContextConstant(), string, SmiTag(index))));
             Goto(&done_loop);
           }
         }
 
         Bind(&if_stringisnotexternal);
         {
           // The {string} is a SlicedString, continue with its parent.
           Node* string_offset =
-              SmiToWord(LoadObjectField(string, SlicedString::kOffsetOffset));
+              LoadAndUntagObjectField(string, SlicedString::kOffsetOffset);
           Node* string_parent =
               LoadObjectField(string, SlicedString::kParentOffset);
           var_index.Bind(IntPtrAdd(index, string_offset));
           var_string.Bind(string_parent);
           Goto(&loop);
         }
       }
     }
   }
 
@@ skipping 150 matching lines @@
 
 template <typename Dictionary>
 void CodeStubAssembler::NameDictionaryLookup(Node* dictionary,
                                              Node* unique_name, Label* if_found,
                                              Variable* var_name_index,
                                              Label* if_not_found,
                                              int inlined_probes) {
   DCHECK_EQ(MachineRepresentation::kWord32, var_name_index->rep());
   Comment("NameDictionaryLookup");
 
-  Node* capacity = SmiToWord32(LoadFixedArrayElement(
-      dictionary, Int32Constant(Dictionary::kCapacityIndex)));
+  Node* capacity = LoadAndUntagToWord32FixedArrayElement(
+      dictionary, Int32Constant(Dictionary::kCapacityIndex));
   Node* mask = Int32Sub(capacity, Int32Constant(1));
   Node* hash = LoadNameHash(unique_name);
 
   // See Dictionary::FirstProbe().
   Node* count = Int32Constant(0);
   Node* entry = Word32And(hash, mask);
 
   for (int i = 0; i < inlined_probes; i++) {
     Node* index = EntryToIndex<Dictionary>(entry);
     var_name_index->Bind(index);
@@ skipping 58 matching lines @@
 }
 
 template <typename Dictionary>
 void CodeStubAssembler::NumberDictionaryLookup(Node* dictionary, Node* key,
                                                Label* if_found,
                                                Variable* var_entry,
                                                Label* if_not_found) {
   DCHECK_EQ(MachineRepresentation::kWord32, var_entry->rep());
   Comment("NumberDictionaryLookup");
 
-  Node* capacity = SmiToWord32(LoadFixedArrayElement(
-      dictionary, Int32Constant(Dictionary::kCapacityIndex)));
+  Node* capacity = LoadAndUntagToWord32FixedArrayElement(
+      dictionary, Int32Constant(Dictionary::kCapacityIndex));
   Node* mask = Int32Sub(capacity, Int32Constant(1));
 
   Node* seed;
   if (Dictionary::ShapeT::UsesSeed) {
     seed = HashSeed();
   } else {
     seed = Int32Constant(kZeroHashSeed);
   }
   Node* hash = ComputeIntegerHash(key, seed);
   Node* key_as_float64 = ChangeUint32ToFloat64(key);
@@ skipping 170 matching lines @@
   DCHECK_EQ(MachineRepresentation::kTagged, var_value->rep());
   Comment("[ LoadPropertyFromFastObject");
 
   const int name_to_details_offset =
       (DescriptorArray::kDescriptorDetails - DescriptorArray::kDescriptorKey) *
       kPointerSize;
   const int name_to_value_offset =
       (DescriptorArray::kDescriptorValue - DescriptorArray::kDescriptorKey) *
       kPointerSize;
 
-  Node* details = SmiToWord32(
-      LoadFixedArrayElement(descriptors, name_index, name_to_details_offset));
+  Node* details = LoadAndUntagToWord32FixedArrayElement(descriptors, name_index,
+                                                        name_to_details_offset);
   var_details->Bind(details);
 
   Node* location = BitFieldDecode<PropertyDetails::LocationField>(details);
 
   Label if_in_field(this), if_in_descriptor(this), done(this);
   Branch(Word32Equal(location, Int32Constant(kField)), &if_in_field,
          &if_in_descriptor);
   Bind(&if_in_field);
   {
     Node* field_index =
@@ skipping 85 matching lines @@
                                                        Variable* var_value) {
   Comment("LoadPropertyFromNameDictionary");
 
   const int name_to_details_offset =
       (NameDictionary::kEntryDetailsIndex - NameDictionary::kEntryKeyIndex) *
       kPointerSize;
   const int name_to_value_offset =
       (NameDictionary::kEntryValueIndex - NameDictionary::kEntryKeyIndex) *
       kPointerSize;
 
-  Node* details = SmiToWord32(
-      LoadFixedArrayElement(dictionary, name_index, name_to_details_offset));
+  Node* details = LoadAndUntagToWord32FixedArrayElement(dictionary, name_index,
+                                                        name_to_details_offset);
 
   var_details->Bind(details);
   var_value->Bind(
       LoadFixedArrayElement(dictionary, name_index, name_to_value_offset));
 
   Comment("] LoadPropertyFromNameDictionary");
 }
 
 void CodeStubAssembler::LoadPropertyFromGlobalDictionary(Node* dictionary,
                                                          Node* name_index,
                                                          Variable* var_details,
                                                          Variable* var_value,
                                                          Label* if_deleted) {
   Comment("[ LoadPropertyFromGlobalDictionary");
 
   const int name_to_value_offset =
       (GlobalDictionary::kEntryValueIndex - GlobalDictionary::kEntryKeyIndex) *
       kPointerSize;
 
   Node* property_cell =
       LoadFixedArrayElement(dictionary, name_index, name_to_value_offset);
 
   Node* value = LoadObjectField(property_cell, PropertyCell::kValueOffset);
   GotoIf(WordEqual(value, TheHoleConstant()), if_deleted);
 
   var_value->Bind(value);
 
-  Node* details =
-      SmiToWord32(LoadObjectField(property_cell, PropertyCell::kDetailsOffset));
+  Node* details = LoadAndUntagToWord32ObjectField(property_cell,
+                                                  PropertyCell::kDetailsOffset);
   var_details->Bind(details);
 
   Comment("] LoadPropertyFromGlobalDictionary");
 }
 
 void CodeStubAssembler::TryGetOwnProperty(
     Node* context, Node* receiver, Node* object, Node* map, Node* instance_type,
     Node* unique_name, Label* if_found_value, Variable* var_value,
     Label* if_not_found, Label* if_bailout) {
   DCHECK_EQ(MachineRepresentation::kTagged, var_value->rep());
@@ skipping 117 matching lines @@
       &if_isslowstringwrapper,
       if_not_found,
   };
   // clang-format on
   STATIC_ASSERT(arraysize(values) == arraysize(labels));
   Switch(elements_kind, if_bailout, values, labels, arraysize(values));
 
   Bind(&if_isobjectorsmi);
   {
     Node* elements = LoadElements(object);
-    Node* length = LoadFixedArrayBaseLength(elements);
+    Node* length = LoadAndUntagFixedArrayBaseLength(elements);
 
-    GotoUnless(Uint32LessThan(index, SmiToWord32(length)), if_not_found);
+    GotoUnless(Uint32LessThan(index, length), if_not_found);
 
     Node* element = LoadFixedArrayElement(elements, index);
     Node* the_hole = TheHoleConstant();
     Branch(WordEqual(element, the_hole), if_not_found, if_found);
   }
   Bind(&if_isdouble);
   {
     Node* elements = LoadElements(object);
-    Node* length = LoadFixedArrayBaseLength(elements);
+    Node* length = LoadAndUntagFixedArrayBaseLength(elements);
 
-    GotoUnless(Uint32LessThan(index, SmiToWord32(length)), if_not_found);
+    GotoUnless(Uint32LessThan(index, length), if_not_found);
 
     if (kPointerSize == kDoubleSize) {
       Node* element =
           LoadFixedDoubleArrayElement(elements, index, MachineType::Uint64());
       Node* the_hole = Int64Constant(kHoleNanInt64);
       Branch(Word64Equal(element, the_hole), if_not_found, if_found);
     } else {
       Node* element_upper =
           LoadFixedDoubleArrayElement(elements, index, MachineType::Uint32(),
                                       kIeeeDoubleExponentWordOffset);
@@ skipping 386 matching lines @@
     GotoIf(WordNotEqual(receiver_map, cached_map), &next_entry);
 
     // Found, now call handler.
     Node* handler =
         LoadFixedArrayElement(feedback, Int32Constant(i * kEntrySize + 1));
     var_handler->Bind(handler);
     Goto(if_handler);
 
     Bind(&next_entry);
   }
-  Node* length = SmiToWord32(LoadFixedArrayBaseLength(feedback));
+  Node* length = LoadAndUntagFixedArrayBaseLength(feedback);
 
   // Loop from {unroll_count}*kEntrySize to {length}.
   Variable var_index(this, MachineRepresentation::kWord32);
   Label loop(this, &var_index);
   var_index.Bind(Int32Constant(unroll_count * kEntrySize));
   Goto(&loop);
   Bind(&loop);
   {
     Node* index = var_index.value();
     GotoIf(Int32GreaterThanOrEqual(index, length), if_miss);
@@ skipping 419 matching lines @@
                        Heap::kTheHoleValueRootIndex);
 
   // Store the WeakCell in the feedback vector.
   StoreFixedArrayElement(feedback_vector, slot, cell, UPDATE_WRITE_BARRIER,
                          CodeStubAssembler::SMI_PARAMETERS);
   return cell;
 }
 
 }  // namespace internal
 }  // namespace v8