[interpreter] Add support for scalable operands.

src/interpreter/interpreter-assembler.cc

 // Copyright 2015 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/interpreter/interpreter-assembler.h"
 
 #include <ostream>
 
 #include "src/code-factory.h"
 #include "src/frames.h"
 #include "src/interface-descriptors.h"
 #include "src/interpreter/bytecodes.h"
 #include "src/interpreter/interpreter.h"
 #include "src/machine-type.h"
 #include "src/macro-assembler.h"
 #include "src/zone.h"
 
 namespace v8 {
 namespace internal {
 namespace interpreter {
 
 using compiler::Node;
 
 InterpreterAssembler::InterpreterAssembler(Isolate* isolate, Zone* zone,
-                                           Bytecode bytecode)
+                                           Bytecode bytecode, int operand_scale)
     : compiler::CodeStubAssembler(isolate, zone,
                                   InterpreterDispatchDescriptor(isolate),
                                   Code::ComputeFlags(Code::BYTECODE_HANDLER),
                                   Bytecodes::ToString(bytecode), 0),
       bytecode_(bytecode),
+      operand_scale_(operand_scale),
       accumulator_(this, MachineRepresentation::kTagged),
       context_(this, MachineRepresentation::kTagged),
       bytecode_array_(this, MachineRepresentation::kTagged),
+      disable_stack_check_across_call_(false),
+      stack_pointer_before_call_(nullptr) {
+  accumulator_.Bind(
+      Parameter(InterpreterDispatchDescriptor::kAccumulatorParameter));
+  context_.Bind(Parameter(InterpreterDispatchDescriptor::kContextParameter));
+  bytecode_array_.Bind(
+      Parameter(InterpreterDispatchDescriptor::kBytecodeArrayParameter));
+  if (FLAG_trace_ignition) {
+    TraceBytecode(Runtime::kInterpreterTraceBytecodeEntry);
+  }
+}
+
+InterpreterAssembler::InterpreterAssembler(Isolate* isolate, Zone* zone)
+    : compiler::CodeStubAssembler(
+          isolate, zone, InterpreterDispatchDescriptor(isolate),
+          Code::ComputeFlags(Code::STUB), "InvalidBytecode", 0),
+      bytecode_(static_cast<Bytecode>(-1)),
+      operand_scale_(-1),
+      accumulator_(this, MachineRepresentation::kTagged),
+      context_(this, MachineRepresentation::kTagged),
+      bytecode_array_(this, MachineRepresentation::kTagged),
       disable_stack_check_across_call_(false),
       stack_pointer_before_call_(nullptr) {
   accumulator_.Bind(
       Parameter(InterpreterDispatchDescriptor::kAccumulatorParameter));
   context_.Bind(Parameter(InterpreterDispatchDescriptor::kContextParameter));
   bytecode_array_.Bind(
       Parameter(InterpreterDispatchDescriptor::kBytecodeArrayParameter));
   if (FLAG_trace_ignition) {
     TraceBytecode(Runtime::kInterpreterTraceBytecodeEntry);
   }
@@ skipping 68 matching lines @@
                              RegisterFrameOffset(reg_index), value);
 }
 
 Node* InterpreterAssembler::NextRegister(Node* reg_index) {
   // Register indexes are negative, so the next index is minus one.
   return IntPtrAdd(reg_index, IntPtrConstant(-1));
 }
 
 Node* InterpreterAssembler::BytecodeOperand(int operand_index) {
   DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_));
-  DCHECK_EQ(OperandSize::kByte,
-            Bytecodes::GetOperandSize(bytecode_, operand_index));
-  return Load(
-      MachineType::Uint8(), BytecodeArrayTaggedPointer(),
-      IntPtrAdd(BytecodeOffset(), IntPtrConstant(Bytecodes::GetOperandOffset(
-                                      bytecode_, operand_index))));
+  DCHECK_EQ(OperandSize::kByte, Bytecodes::GetOperandSize(
+                                    bytecode_, operand_index, operand_scale()));
+  return Load(MachineType::Uint8(), BytecodeArrayTaggedPointer(),
+              IntPtrAdd(BytecodeOffset(),
+                        IntPtrConstant(Bytecodes::GetOperandOffset(
+                            bytecode_, operand_index, operand_scale()))));
 }
 
 Node* InterpreterAssembler::BytecodeOperandSignExtended(int operand_index) {
   DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_));
-  DCHECK_EQ(OperandSize::kByte,
-            Bytecodes::GetOperandSize(bytecode_, operand_index));
-  Node* load = Load(
-      MachineType::Int8(), BytecodeArrayTaggedPointer(),
-      IntPtrAdd(BytecodeOffset(), IntPtrConstant(Bytecodes::GetOperandOffset(
-                                      bytecode_, operand_index))));
+  DCHECK_EQ(OperandSize::kByte, Bytecodes::GetOperandSize(
+                                    bytecode_, operand_index, operand_scale()));
+  Node* load = Load(MachineType::Int8(), BytecodeArrayTaggedPointer(),
+                    IntPtrAdd(BytecodeOffset(),
+                              IntPtrConstant(Bytecodes::GetOperandOffset(
+                                  bytecode_, operand_index, operand_scale()))));
   // Ensure that we sign extend to full pointer size
   if (kPointerSize == 8) {
     load = ChangeInt32ToInt64(load);
   }
   return load;
 }
 
 Node* InterpreterAssembler::BytecodeOperandShort(int operand_index) {
   DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_));
-  DCHECK_EQ(OperandSize::kShort,
-            Bytecodes::GetOperandSize(bytecode_, operand_index));
+  DCHECK_EQ(
+      OperandSize::kShort,
+      Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale()));
   if (TargetSupportsUnalignedAccess()) {
-    return Load(
-        MachineType::Uint16(), BytecodeArrayTaggedPointer(),
-        IntPtrAdd(BytecodeOffset(), IntPtrConstant(Bytecodes::GetOperandOffset(
-                                        bytecode_, operand_index))));
+    return Load(MachineType::Uint16(), BytecodeArrayTaggedPointer(),
+                IntPtrAdd(BytecodeOffset(),
+                          IntPtrConstant(Bytecodes::GetOperandOffset(
+                              bytecode_, operand_index, operand_scale()))));
   } else {
-    int offset = Bytecodes::GetOperandOffset(bytecode_, operand_index);
+    int offset =
+        Bytecodes::GetOperandOffset(bytecode_, operand_index, operand_scale());
     Node* first_byte =
         Load(MachineType::Uint8(), BytecodeArrayTaggedPointer(),
              IntPtrAdd(BytecodeOffset(), IntPtrConstant(offset)));
     Node* second_byte =
         Load(MachineType::Uint8(), BytecodeArrayTaggedPointer(),
              IntPtrAdd(BytecodeOffset(), IntPtrConstant(offset + 1)));
 #if V8_TARGET_LITTLE_ENDIAN
-    return WordOr(WordShl(second_byte, kBitsPerByte), first_byte);
+    return WordOr(WordShl(second_byte, IntPtrConstant(kBitsPerByte)),
+                  first_byte);
 #elif V8_TARGET_BIG_ENDIAN
-    return WordOr(WordShl(first_byte, kBitsPerByte), second_byte);
+    return WordOr(WordShl(first_byte, IntPtrConstant(kBitsPerByte)),
+                  second_byte);
 #else
 #error "Unknown Architecture"
 #endif
   }
 }
 
 Node* InterpreterAssembler::BytecodeOperandShortSignExtended(
     int operand_index) {
   DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_));
-  DCHECK_EQ(OperandSize::kShort,
-            Bytecodes::GetOperandSize(bytecode_, operand_index));
-  int operand_offset = Bytecodes::GetOperandOffset(bytecode_, operand_index);
+  DCHECK_EQ(
+      OperandSize::kShort,
+      Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale()));
+  int operand_offset =
+      Bytecodes::GetOperandOffset(bytecode_, operand_index, operand_scale());
   Node* load;
   if (TargetSupportsUnalignedAccess()) {
     load = Load(MachineType::Int16(), BytecodeArrayTaggedPointer(),
                 IntPtrAdd(BytecodeOffset(), IntPtrConstant(operand_offset)));
   } else {
 #if V8_TARGET_LITTLE_ENDIAN
     Node* hi_byte_offset = IntPtrConstant(operand_offset + 1);
     Node* lo_byte_offset = IntPtrConstant(operand_offset);
 #elif V8_TARGET_BIG_ENDIAN
     Node* hi_byte_offset = IntPtrConstant(operand_offset);
     Node* lo_byte_offset = IntPtrConstant(operand_offset + 1);
 #else
 #error "Unknown Architecture"
 #endif
     Node* hi_byte = Load(MachineType::Int8(), BytecodeArrayTaggedPointer(),
                          IntPtrAdd(BytecodeOffset(), hi_byte_offset));
     Node* lo_byte = Load(MachineType::Uint8(), BytecodeArrayTaggedPointer(),
                          IntPtrAdd(BytecodeOffset(), lo_byte_offset));
     hi_byte = Word32Shl(hi_byte, Int32Constant(kBitsPerByte));
     load = Word32Or(hi_byte, lo_byte);
   }
 
   // Ensure that we sign extend to full pointer size
   if (kPointerSize == 8) {
     load = ChangeInt32ToInt64(load);
   }
   return load;
 }
 
+Node* InterpreterAssembler::BytecodeOperandQuad(int operand_index) {
+  DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_));
+  DCHECK_EQ(OperandSize::kQuad, Bytecodes::GetOperandSize(
+                                    bytecode_, operand_index, operand_scale()));
+  int operand_offset =
+      Bytecodes::GetOperandOffset(bytecode_, operand_index, operand_scale());
+  if (TargetSupportsUnalignedAccess()) {
+    return Load(MachineType::Uint32(), BytecodeArrayTaggedPointer(),
+                IntPtrAdd(BytecodeOffset(), IntPtrConstant(operand_offset)));
+  } else {
+    Node* bytes[4];
+    for (int i = 0; i < static_cast<int>(arraysize(bytes)); ++i) {
+      bytes[i] =
+          Load(MachineType::Uint8(), BytecodeArrayTaggedPointer(),
+               IntPtrAdd(BytecodeOffset(), IntPtrConstant(operand_offset + i)));
+    }
+#if V8_TARGET_LITTLE_ENDIAN
+    Node* upper =
+        WordOr(WordShl(bytes[3], IntPtrConstant(kBitsPerByte)), bytes[2]);
+    Node* lower =
+        WordOr(WordShl(bytes[1], IntPtrConstant(kBitsPerByte)), bytes[0]);
+    return WordOr(WordShl(upper, IntPtrConstant(2 * kBitsPerByte)), lower);
+#elif V8_TARGET_BIG_ENDIAN
+    Node* upper =
+        WordOr(WordShl(bytes[0], IntPtrConstant(kBitsPerByte)), bytes[1]);
+    Node* lower =
+        WordOr(WordShl(bytes[2], IntPtrConstant(kBitsPerByte)), bytes[3]);
+    return WordOr(WordShl(upper, IntPtrConstant(2 * kBitsPerByte)), lower);
+#else
+#error "Unknown Architecture"
+#endif
+  }
+}
+
+Node* InterpreterAssembler::BytecodeOperandQuadSignExtended(int operand_index) {
+  DCHECK_LT(operand_index, Bytecodes::NumberOfOperands(bytecode_));
+  DCHECK_EQ(OperandSize::kQuad, Bytecodes::GetOperandSize(
+                                    bytecode_, operand_index, operand_scale()));
+  int operand_offset =
+      Bytecodes::GetOperandOffset(bytecode_, operand_index, operand_scale());
+  Node* load;
+  if (TargetSupportsUnalignedAccess()) {
+    load = Load(MachineType::Int32(), BytecodeArrayTaggedPointer(),
+                IntPtrAdd(BytecodeOffset(), IntPtrConstant(operand_offset)));
+  } else {
+    Node* bytes[4];
+    for (int i = 0; i < static_cast<int>(arraysize(bytes)); ++i) {
+      bytes[i] =
+          Load(MachineType::Uint8(), BytecodeArrayTaggedPointer(),
+               IntPtrAdd(BytecodeOffset(), IntPtrConstant(operand_offset + i)));
+    }
+#if V8_TARGET_LITTLE_ENDIAN
+    Node* upper =
+        WordOr(WordShl(bytes[3], IntPtrConstant(kBitsPerByte)), bytes[2]);
+    Node* lower =
+        WordOr(WordShl(bytes[1], IntPtrConstant(kBitsPerByte)), bytes[0]);
+    load = WordOr(WordShl(upper, IntPtrConstant(2 * kBitsPerByte)), lower);
+#elif V8_TARGET_BIG_ENDIAN
+    Node* upper =
+        WordOr(WordShl(bytes[0], IntPtrConstant(kBitsPerByte)), bytes[1]);
+    Node* lower =
+        WordOr(WordShl(bytes[2], IntPtrConstant(kBitsPerByte)), bytes[3]);
+    load = WordOr(WordShl(upper, IntPtrConstant(2 * kBitsPerByte)), lower);
+#else
+#error "Unknown Architecture"
+#endif
+  }
+
+  // Ensure that we sign extend to full pointer size
+  if (kPointerSize == 8) {
+    load = ChangeInt32ToInt64(load);
+  }
+  return load;
+}
+
 Node* InterpreterAssembler::BytecodeOperandCount(int operand_index) {
-  switch (Bytecodes::GetOperandSize(bytecode_, operand_index)) {
+  DCHECK_EQ(OperandType::kRegCount8,
+            Bytecodes::GetOperandType(bytecode_, operand_index));
+  OperandSize operand_size =
+      Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale());
+  switch (operand_size) {
     case OperandSize::kByte:
-      DCHECK_EQ(OperandType::kRegCount8,
-                Bytecodes::GetOperandType(bytecode_, operand_index));
       return BytecodeOperand(operand_index);
     case OperandSize::kShort:
-      DCHECK_EQ(OperandType::kRegCount16,
-                Bytecodes::GetOperandType(bytecode_, operand_index));
       return BytecodeOperandShort(operand_index);
+    case OperandSize::kQuad:
+      return BytecodeOperandQuad(operand_index);
     case OperandSize::kNone:
       UNREACHABLE();
   }
+  return nullptr;
+}
+
+Node* InterpreterAssembler::BytecodeOperandFlag(int operand_index) {
+  DCHECK_EQ(OperandType::kFlag8,
+            Bytecodes::GetOperandType(bytecode_, operand_index));
+  OperandSize operand_size =
+      Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale());
+  switch (operand_size) {
+    case OperandSize::kByte:
+      return BytecodeOperand(operand_index);
+    case OperandSize::kShort:
+    case OperandSize::kQuad:
+    case OperandSize::kNone:
+      UNREACHABLE();
+  }
   return nullptr;
 }
 
 Node* InterpreterAssembler::BytecodeOperandImm(int operand_index) {
   DCHECK_EQ(OperandType::kImm8,
             Bytecodes::GetOperandType(bytecode_, operand_index));
-  return BytecodeOperandSignExtended(operand_index);
+  OperandSize operand_size =
+      Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale());
+  switch (operand_size) {
+    case OperandSize::kByte:
+      return BytecodeOperandSignExtended(operand_index);
+    case OperandSize::kShort:
+      return BytecodeOperandShortSignExtended(operand_index);
+    case OperandSize::kQuad:
+      return BytecodeOperandQuadSignExtended(operand_index);
+    case OperandSize::kNone:
+      UNREACHABLE();
+  }
+  return nullptr;
 }
 
 Node* InterpreterAssembler::BytecodeOperandIdx(int operand_index) {
-  switch (Bytecodes::GetOperandSize(bytecode_, operand_index)) {
+  DCHECK(OperandType::kIdx8 ==
+             Bytecodes::GetOperandType(bytecode_, operand_index) ||
+         OperandType::kRuntimeId ==
+             Bytecodes::GetOperandType(bytecode_, operand_index));
+  OperandSize operand_size =
+      Bytecodes::GetOperandSize(bytecode_, operand_index, operand_scale());
+  switch (operand_size) {
     case OperandSize::kByte:
-      DCHECK_EQ(OperandType::kIdx8,
-                Bytecodes::GetOperandType(bytecode_, operand_index));
       return BytecodeOperand(operand_index);
     case OperandSize::kShort:
-      DCHECK_EQ(OperandType::kIdx16,
-                Bytecodes::GetOperandType(bytecode_, operand_index));
       return BytecodeOperandShort(operand_index);
+    case OperandSize::kQuad:
+      return BytecodeOperandQuad(operand_index);
     case OperandSize::kNone:
       UNREACHABLE();
   }
   return nullptr;
 }
 
 Node* InterpreterAssembler::BytecodeOperandReg(int operand_index) {
   OperandType operand_type =
       Bytecodes::GetOperandType(bytecode_, operand_index);
-  if (Bytecodes::IsRegisterOperandType(operand_type)) {
-    OperandSize operand_size = Bytecodes::SizeOfOperand(operand_type);
-    if (operand_size == OperandSize::kByte) {
+  DCHECK(Bytecodes::IsRegisterOperandType(operand_type));
+  switch (Bytecodes::SizeOfOperand(operand_type, operand_scale())) {
+    case OperandSize::kByte:
       return BytecodeOperandSignExtended(operand_index);
-    } else if (operand_size == OperandSize::kShort) {
+    case OperandSize::kShort:
       return BytecodeOperandShortSignExtended(operand_index);
-    }
+    case OperandSize::kQuad:
+      return BytecodeOperandQuadSignExtended(operand_index);
+    case OperandSize::kNone:
+      UNREACHABLE();
   }
-  UNREACHABLE();
   return nullptr;
 }
 
 Node* InterpreterAssembler::LoadConstantPoolEntry(Node* index) {
   Node* constant_pool = LoadObjectField(BytecodeArrayTaggedPointer(),
                                         BytecodeArray::kConstantPoolOffset);
   Node* entry_offset =
       IntPtrAdd(IntPtrConstant(FixedArray::kHeaderSize - kHeapObjectTag),
                 WordShl(index, kPointerSizeLog2));
   return Load(MachineType::AnyTagged(), constant_pool, entry_offset);
@@ skipping 158 matching lines @@
 void InterpreterAssembler::JumpIfWordEqual(Node* lhs, Node* rhs, Node* delta) {
   JumpConditional(WordEqual(lhs, rhs), delta);
 }
 
 void InterpreterAssembler::JumpIfWordNotEqual(Node* lhs, Node* rhs,
                                               Node* delta) {
   JumpConditional(WordNotEqual(lhs, rhs), delta);
 }
 
 void InterpreterAssembler::Dispatch() {
-  DispatchTo(Advance(Bytecodes::Size(bytecode_)));
+  DCHECK(base::bits::IsPowerOfTwo32(operand_scale_));
+  DispatchTo(Advance(Bytecodes::Size(bytecode_, operand_scale_)));
 }
 
 void InterpreterAssembler::DispatchTo(Node* new_bytecode_offset) {
   Node* target_bytecode = Load(
       MachineType::Uint8(), BytecodeArrayTaggedPointer(), new_bytecode_offset);
   if (kPointerSize == 8) {
     target_bytecode = ChangeUint32ToUint64(target_bytecode);
   }
 
   // TODO(rmcilroy): Create a code target dispatch table to avoid conversion
@@ skipping 11 matching lines @@
     TraceBytecode(Runtime::kInterpreterTraceBytecodeExit);
   }
 
   InterpreterDispatchDescriptor descriptor(isolate());
   Node* args[] = {GetAccumulator(),          RegisterFileRawPointer(),
                   bytecode_offset,           BytecodeArrayTaggedPointer(),
                   DispatchTableRawPointer(), GetContext()};
   TailCall(descriptor, handler, args, 0);
 }
 
+void InterpreterAssembler::RedispatchWide() {
+  STATIC_ASSERT(static_cast<int>(Bytecode::kExtraWide) ==
+                static_cast<int>(Bytecode::kWide) + 1);
+  Node* upper = Load(MachineType::Uint8(), BytecodeArrayTaggedPointer(),
+                     BytecodeOffset());
+  upper =
+      IntPtrAdd(upper, Int32Constant(-static_cast<int>(Bytecode::kWide) + 1));
+  Node* lower = Load(MachineType::Uint8(), BytecodeArrayTaggedPointer(),
+                     IntPtrAdd(BytecodeOffset(), Int32Constant(1)));
+  Node* target_bytecode =
+      Word32Or(Word32Shl(upper, Int32Constant(kBitsPerByte)), lower);
+  Node* target_code_object =
+      Load(MachineType::Pointer(), DispatchTableRawPointer(),
+           Word32Shl(target_bytecode, Int32Constant(kPointerSizeLog2)));
+  DispatchToBytecodeHandler(target_code_object, Advance(1));
+}
+
 void InterpreterAssembler::InterpreterReturn() {
   // TODO(rmcilroy): Investigate whether it is worth supporting self
   // optimization of primitive functions like FullCodegen.
 
   // Update profiling count by -BytecodeOffset to simulate backedge to start of
   // function.
   Node* profiling_weight =
       Int32Sub(Int32Constant(kHeapObjectTag + BytecodeArray::kHeaderSize),
                BytecodeOffset());
   UpdateInterruptBudget(profiling_weight);
@@ skipping 58 matching lines @@
     V8_TARGET_ARCH_S390
   return true;
 #else
 #error "Unknown Architecture"
 #endif
 }
 
 }  // namespace interpreter
 }  // namespace internal
 }  // namespace v8