[wasm] implement simd lowering for replaceLane, load, store and test for phi

src/compiler/simd-scalar-lowering.cc

 // Copyright 2016 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "src/compiler/simd-scalar-lowering.h"
 #include "src/compiler/diamond.h"
 #include "src/compiler/linkage.h"
 #include "src/compiler/node-matchers.h"
 #include "src/compiler/node-properties.h"
 
@@ skipping 40 matching lines @@
     } else {
       // Push the next input onto the stack.
       Node* input = top.node->InputAt(top.input_index++);
       if (state_.Get(input) == State::kUnvisited) {
         SetLoweredType(input, top.node);
         if (input->opcode() == IrOpcode::kPhi) {
           // To break cycles with phi nodes we push phis on a separate stack so
           // that they are processed after all other nodes.
           PreparePhiReplacement(input);
           stack_.push_front({input, 0});
+        } else if (input->opcode() == IrOpcode::kEffectPhi ||
+                   input->opcode() == IrOpcode::kLoop) {
+          stack_.push_front({input, 0});
         } else {
           stack_.push_back({input, 0});
         }
         state_.Set(input, State::kOnStack);
       }
     }
   }
 }
 
 #define FOREACH_INT32X4_OPCODE(V) \
   V(Int32x4Add)                   \
   V(Int32x4ExtractLane)           \
-  V(CreateInt32x4)
+  V(CreateInt32x4)                \
+  V(Int32x4ReplaceLane)
 
 #define FOREACH_FLOAT32X4_OPCODE(V) \
   V(Float32x4Add)                   \
   V(Float32x4ExtractLane)           \
-  V(CreateFloat32x4)
+  V(CreateFloat32x4)                \
+  V(Float32x4ReplaceLane)
 
 void SimdScalarLowering::SetLoweredType(Node* node, Node* output) {
   switch (node->opcode()) {
 #define CASE_STMT(name) case IrOpcode::k##name:
     FOREACH_INT32X4_OPCODE(CASE_STMT)
     case IrOpcode::kReturn:
     case IrOpcode::kParameter:
     case IrOpcode::kCall: {
       replacements_[node->id()].type = SimdType::kInt32;
       break;
     }
       FOREACH_FLOAT32X4_OPCODE(CASE_STMT) {
         replacements_[node->id()].type = SimdType::kFloat32;
         break;
       }
 #undef CASE_STMT
     default:
       replacements_[node->id()].type = replacements_[output->id()].type;
   }
 }
 
 static int GetParameterIndexAfterLowering(
     Signature<MachineRepresentation>* signature, int old_index) {
   // In function calls, the simd128 types are passed as 4 Int32 types. The
   // parameters are typecast to the types as needed for various operations.
   int result = old_index;
-  for (int i = 0; i < old_index; i++) {
+  for (int i = 0; i < old_index; ++i) {
     if (signature->GetParam(i) == MachineRepresentation::kSimd128) {
       result += 3;
     }
   }
   return result;
 }
 
 int SimdScalarLowering::GetParameterCountAfterLowering() {
   if (parameter_count_after_lowering_ == -1) {
     // GetParameterIndexAfterLowering(parameter_count) returns the parameter
     // count after lowering.
     parameter_count_after_lowering_ = GetParameterIndexAfterLowering(
         signature(), static_cast<int>(signature()->parameter_count()));
   }
   return parameter_count_after_lowering_;
 }
 
 static int GetReturnCountAfterLowering(
     Signature<MachineRepresentation>* signature) {
   int result = static_cast<int>(signature->return_count());
-  for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
+  for (int i = 0; i < static_cast<int>(signature->return_count()); ++i) {
     if (signature->GetReturn(i) == MachineRepresentation::kSimd128) {
       result += 3;
     }
   }
   return result;
 }
 
+void SimdScalarLowering::GetIndexNodes(Node* index, Node** new_indices) {
+  new_indices[0] = index;
+  for (size_t i = 1; i < kMaxLanes; ++i) {
+    new_indices[i] = graph()->NewNode(machine()->Int32Add(), index,
+                                      graph()->NewNode(common()->Int32Constant(
+                                          static_cast<int>(i) * kLaneWidth)));
+  }
+}
+
+void SimdScalarLowering::LowerLoadOp(MachineRepresentation rep, Node* node,
+                                     const Operator* load_op) {
+  if (rep == MachineRepresentation::kSimd128) {
+    Node* base = node->InputAt(0);
+    Node* index = node->InputAt(1);
+    Node* indices[kMaxLanes];
+    GetIndexNodes(index, indices);
+    Node* rep_nodes[kMaxLanes];
+    rep_nodes[0] = node;
+    NodeProperties::ChangeOp(rep_nodes[0], load_op);
+    if (node->InputCount() > 2) {
+      DCHECK(node->InputCount() > 3);
+      Node* effect_input = node->InputAt(2);
+      Node* control_input = node->InputAt(3);
+      rep_nodes[3] = graph()->NewNode(load_op, base, indices[3], effect_input,
+                                      control_input);
+      rep_nodes[2] = graph()->NewNode(load_op, base, indices[2], rep_nodes[3],
+                                      control_input);
+      rep_nodes[1] = graph()->NewNode(load_op, base, indices[1], rep_nodes[2],
+                                      control_input);
+      rep_nodes[0]->ReplaceInput(2, rep_nodes[1]);
+    } else {
+      for (size_t i = 1; i < kMaxLanes; ++i) {
+        rep_nodes[i] = graph()->NewNode(load_op, base, indices[i]);
+      }
+    }
+    ReplaceNode(node, rep_nodes);
+  } else {
+    DefaultLowering(node);
+  }
+}
+
+void SimdScalarLowering::LowerStoreOp(MachineRepresentation rep, Node* node,
+                                      const Operator* store_op,
+                                      SimdType rep_type) {
+  if (rep == MachineRepresentation::kSimd128) {
+    Node* base = node->InputAt(0);
+    Node* index = node->InputAt(1);
+    Node* indices[kMaxLanes];
+    GetIndexNodes(index, indices);
+    DCHECK(node->InputCount() > 2);
+    Node* value = node->InputAt(2);
+    DCHECK(HasReplacement(1, value));
+    Node* rep_nodes[kMaxLanes];
+    rep_nodes[0] = node;
+    Node** rep_inputs = GetReplacementsWithType(value, rep_type);
+    rep_nodes[0]->ReplaceInput(2, rep_inputs[0]);
+    NodeProperties::ChangeOp(node, store_op);
+    if (node->InputCount() > 3) {
+      DCHECK(node->InputCount() > 4);
+      Node* effect_input = node->InputAt(3);
+      Node* control_input = node->InputAt(4);
+      rep_nodes[3] = graph()->NewNode(store_op, base, indices[3], rep_inputs[3],
+                                      effect_input, control_input);
+      rep_nodes[2] = graph()->NewNode(store_op, base, indices[2], rep_inputs[2],
+                                      rep_nodes[3], control_input);
+      rep_nodes[1] = graph()->NewNode(store_op, base, indices[1], rep_inputs[1],
+                                      rep_nodes[2], control_input);
+      rep_nodes[0]->ReplaceInput(3, rep_nodes[1]);
+
+    } else {
+      for (size_t i = 1; i < kMaxLanes; ++i) {
+        rep_nodes[i] =
+            graph()->NewNode(store_op, base, indices[i], rep_inputs[i]);
+      }
+    }
+
+    ReplaceNode(node, rep_nodes);
+  } else {
+    DefaultLowering(node);
+  }
+}
+
+void SimdScalarLowering::LowerBinaryOp(Node* node, SimdType rep_type,
+                                       const Operator* op) {
+  DCHECK(node->InputCount() == 2);
+  Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type);
+  Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type);
+  Node* rep_node[kMaxLanes];
+  for (int i = 0; i < kMaxLanes; ++i) {
+    rep_node[i] = graph()->NewNode(op, rep_left[i], rep_right[i]);
+  }
+  ReplaceNode(node, rep_node);
+}
+
 void SimdScalarLowering::LowerNode(Node* node) {
   SimdType rep_type = ReplacementType(node);
   switch (node->opcode()) {
     case IrOpcode::kStart: {
       int parameter_count = GetParameterCountAfterLowering();
       // Only exchange the node if the parameter count actually changed.
       if (parameter_count != static_cast<int>(signature()->parameter_count())) {
         int delta =
             parameter_count - static_cast<int>(signature()->parameter_count());
         int new_output_count = node->op()->ValueOutputCount() + delta;
         NodeProperties::ChangeOp(node, common()->Start(new_output_count));
       }
       break;
     }
     case IrOpcode::kParameter: {
       DCHECK(node->InputCount() == 1);
       // Only exchange the node if the parameter count actually changed. We do
       // not even have to do the default lowering because the the start node,
       // the only input of a parameter node, only changes if the parameter count
       // changes.
       if (GetParameterCountAfterLowering() !=
           static_cast<int>(signature()->parameter_count())) {
         int old_index = ParameterIndexOf(node->op());
         int new_index = GetParameterIndexAfterLowering(signature(), old_index);
         if (old_index == new_index) {
           NodeProperties::ChangeOp(node, common()->Parameter(new_index));
 
           Node* new_node[kMaxLanes];
-          for (int i = 0; i < kMaxLanes; i++) {
+          for (int i = 0; i < kMaxLanes; ++i) {
             new_node[i] = nullptr;
           }
           new_node[0] = node;
           if (signature()->GetParam(old_index) ==
               MachineRepresentation::kSimd128) {
-            for (int i = 1; i < kMaxLanes; i++) {
+            for (int i = 1; i < kMaxLanes; ++i) {
               new_node[i] = graph()->NewNode(common()->Parameter(new_index + i),
                                              graph()->start());
             }
           }
           ReplaceNode(node, new_node);
         }
       }
       break;
     }
+    case IrOpcode::kLoad: {
+      MachineRepresentation rep =
+          LoadRepresentationOf(node->op()).representation();
+      const Operator* load_op;
+      if (rep_type == SimdType::kInt32) {
+        load_op = machine()->Load(MachineType::Int32());
+      } else if (rep_type == SimdType::kFloat32) {
+        load_op = machine()->Load(MachineType::Float32());
+      }
+      LowerLoadOp(rep, node, load_op);
+      break;
+    }
+    case IrOpcode::kUnalignedLoad: {
+      MachineRepresentation rep =
+          UnalignedLoadRepresentationOf(node->op()).representation();
+      const Operator* load_op;
+      if (rep_type == SimdType::kInt32) {
+        load_op = machine()->UnalignedLoad(MachineType::Int32());
+      } else if (rep_type == SimdType::kFloat32) {
+        load_op = machine()->UnalignedLoad(MachineType::Float32());
+      }
+      LowerLoadOp(rep, node, load_op);
+      break;
+    }
+    case IrOpcode::kStore: {
+      MachineRepresentation rep =
+          StoreRepresentationOf(node->op()).representation();
+      WriteBarrierKind write_barrier_kind =
+          StoreRepresentationOf(node->op()).write_barrier_kind();
+      const Operator* store_op;
+      if (rep_type == SimdType::kInt32) {
+        store_op = machine()->Store(StoreRepresentation(
+            MachineRepresentation::kWord32, write_barrier_kind));
+      } else {
+        store_op = machine()->Store(StoreRepresentation(
+            MachineRepresentation::kFloat32, write_barrier_kind));
+      }
+      LowerStoreOp(rep, node, store_op, rep_type);
+      break;
+    }
+    case IrOpcode::kUnalignedStore: {
+      MachineRepresentation rep = UnalignedStoreRepresentationOf(node->op());
+      const Operator* store_op;
+      if (rep_type == SimdType::kInt32) {
+        store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
+      } else {
+        store_op = machine()->UnalignedStore(MachineRepresentation::kFloat32);
+      }
+      LowerStoreOp(rep, node, store_op, rep_type);
+      break;
+    }
     case IrOpcode::kReturn: {
       DefaultLowering(node);
       int new_return_count = GetReturnCountAfterLowering(signature());
       if (static_cast<int>(signature()->return_count()) != new_return_count) {
         NodeProperties::ChangeOp(node, common()->Return(new_return_count));
       }
       break;
     }
     case IrOpcode::kCall: {
       // TODO(turbofan): Make WASM code const-correct wrt. CallDescriptor.
       CallDescriptor* descriptor =
           const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
       if (DefaultLowering(node) ||
           (descriptor->ReturnCount() == 1 &&
            descriptor->GetReturnType(0) == MachineType::Simd128())) {
         // We have to adjust the call descriptor.
         const Operator* op =
             common()->Call(wasm::ModuleEnv::GetI32WasmCallDescriptorForSimd(
                 zone(), descriptor));
         NodeProperties::ChangeOp(node, op);
       }
       if (descriptor->ReturnCount() == 1 &&
           descriptor->GetReturnType(0) == MachineType::Simd128()) {
         // We access the additional return values through projections.
         Node* rep_node[kMaxLanes];
-        for (int i = 0; i < kMaxLanes; i++) {
+        for (int i = 0; i < kMaxLanes; ++i) {
           rep_node[i] =
               graph()->NewNode(common()->Projection(i), node, graph()->start());
         }
         ReplaceNode(node, rep_node);
       }
       break;
     }
     case IrOpcode::kPhi: {
       MachineRepresentation rep = PhiRepresentationOf(node->op());
       if (rep == MachineRepresentation::kSimd128) {
         // The replacement nodes have already been created, we only have to
         // replace placeholder nodes.
         Node** rep_node = GetReplacements(node);
-        for (int i = 0; i < node->op()->ValueInputCount(); i++) {
+        for (int i = 0; i < node->op()->ValueInputCount(); ++i) {
           Node** rep_input =
               GetReplacementsWithType(node->InputAt(i), rep_type);
           for (int j = 0; j < kMaxLanes; j++) {
             rep_node[j]->ReplaceInput(i, rep_input[j]);
           }
         }
       } else {
         DefaultLowering(node);
       }
       break;
     }
-
     case IrOpcode::kInt32x4Add: {
-      DCHECK(node->InputCount() == 2);
-      Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type);
-      Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type);
+      LowerBinaryOp(node, rep_type, machine()->Int32Add());
+      break;
+    }
+    case IrOpcode::kFloat32x4Add: {
+      LowerBinaryOp(node, rep_type, machine()->Float32Add());
+      break;
+    }
+    case IrOpcode::kCreateInt32x4:
+    case IrOpcode::kCreateFloat32x4: {
       Node* rep_node[kMaxLanes];
-      for (int i = 0; i < kMaxLanes; i++) {
-        rep_node[i] =
-            graph()->NewNode(machine()->Int32Add(), rep_left[i], rep_right[i]);
+      for (int i = 0; i < kMaxLanes; ++i) {
+        if (HasReplacement(0, node->InputAt(i))) {
+          rep_node[i] = GetReplacements(node->InputAt(i))[0];
+        } else {
+          rep_node[i] = node->InputAt(i);
+        }
       }
       ReplaceNode(node, rep_node);
       break;
     }
-
-    case IrOpcode::kCreateInt32x4: {
-      Node* rep_node[kMaxLanes];
-      for (int i = 0; i < kMaxLanes; i++) {
-        DCHECK(!HasReplacement(1, node->InputAt(i)));
-        rep_node[i] = node->InputAt(i);
-      }
-      ReplaceNode(node, rep_node);
-      break;
-    }
-
-    case IrOpcode::kInt32x4ExtractLane: {
+    case IrOpcode::kInt32x4ExtractLane:
+    case IrOpcode::kFloat32x4ExtractLane: {
       Node* laneNode = node->InputAt(1);
       DCHECK_EQ(laneNode->opcode(), IrOpcode::kInt32Constant);
       int32_t lane = OpParameter<int32_t>(laneNode);
       Node* rep_node[kMaxLanes] = {
           GetReplacementsWithType(node->InputAt(0), rep_type)[lane], nullptr,
           nullptr, nullptr};
       ReplaceNode(node, rep_node);
       break;
     }
-
-    case IrOpcode::kFloat32x4Add: {
-      DCHECK(node->InputCount() == 2);
-      Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type);
-      Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type);
-      Node* rep_node[kMaxLanes];
-      for (int i = 0; i < kMaxLanes; i++) {
-        rep_node[i] = graph()->NewNode(machine()->Float32Add(), rep_left[i],
-                                       rep_right[i]);
+    case IrOpcode::kInt32x4ReplaceLane:
+    case IrOpcode::kFloat32x4ReplaceLane: {
+      DCHECK_EQ(3, node->InputCount());
+      Node* laneNode = node->InputAt(1);
+      Node* repNode = node->InputAt(2);
+      DCHECK_EQ(laneNode->opcode(), IrOpcode::kInt32Constant);
+      int32_t lane = OpParameter<int32_t>(laneNode);
+      DCHECK(lane >= 0 && lane <= 3);
+      Node** rep_node = GetReplacementsWithType(node->InputAt(0), rep_type);
+      if (HasReplacement(0, repNode)) {
+        rep_node[lane] = GetReplacements(repNode)[0];
+      } else {
+        rep_node[lane] = repNode;
       }
       ReplaceNode(node, rep_node);
       break;
     }
-
-    case IrOpcode::kCreateFloat32x4: {
-      Node* rep_node[kMaxLanes];
-      for (int i = 0; i < kMaxLanes; i++) {
-        DCHECK(!HasReplacement(1, node->InputAt(i)));
-        rep_node[i] = node->InputAt(i);
-      }
-      ReplaceNode(node, rep_node);
-      break;
-    }
-
-    case IrOpcode::kFloat32x4ExtractLane: {
-      Node* laneNode = node->InputAt(1);
-      DCHECK_EQ(laneNode->opcode(), IrOpcode::kInt32Constant);
-      int32_t lane = OpParameter<int32_t>(laneNode);
-      Node* rep_node[kMaxLanes] = {
-          GetReplacementsWithType(node->InputAt(0), rep_type)[lane], nullptr,
-          nullptr, nullptr};
-      ReplaceNode(node, rep_node);
-      break;
-    }
-
     default: { DefaultLowering(node); }
   }
 }
 
 bool SimdScalarLowering::DefaultLowering(Node* node) {
   bool something_changed = false;
   for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
     Node* input = node->InputAt(i);
     if (HasReplacement(0, input)) {
       something_changed = true;
       node->ReplaceInput(i, GetReplacements(input)[0]);
     }
     if (HasReplacement(1, input)) {
       something_changed = true;
       for (int j = 1; j < kMaxLanes; j++) {
         node->InsertInput(zone(), i + j, GetReplacements(input)[j]);
       }
     }
   }
   return something_changed;
 }
 
 void SimdScalarLowering::ReplaceNode(Node* old, Node** new_node) {
   // if new_low == nullptr, then also new_high == nullptr.
   DCHECK(new_node[0] != nullptr ||
          (new_node[1] == nullptr && new_node[2] == nullptr &&
           new_node[3] == nullptr));
-  for (int i = 0; i < kMaxLanes; i++) {
+  for (int i = 0; i < kMaxLanes; ++i) {
     replacements_[old->id()].node[i] = new_node[i];
   }
 }
 
 bool SimdScalarLowering::HasReplacement(size_t index, Node* node) {
   return replacements_[node->id()].node[index] != nullptr;
 }
 
 SimdScalarLowering::SimdType SimdScalarLowering::ReplacementType(Node* node) {
   return replacements_[node->id()].type;
 }
 
 Node** SimdScalarLowering::GetReplacements(Node* node) {
   Node** result = replacements_[node->id()].node;
   DCHECK(result);
   return result;
 }
 
 Node** SimdScalarLowering::GetReplacementsWithType(Node* node, SimdType type) {
   Node** replacements = GetReplacements(node);
   if (ReplacementType(node) == type) {
     return GetReplacements(node);
   }
   Node** result = zone()->NewArray<Node*>(kMaxLanes);
   if (ReplacementType(node) == SimdType::kInt32 && type == SimdType::kFloat32) {
-    for (int i = 0; i < kMaxLanes; i++) {
+    for (int i = 0; i < kMaxLanes; ++i) {
       if (replacements[i] != nullptr) {
         result[i] = graph()->NewNode(machine()->BitcastInt32ToFloat32(),
                                      replacements[i]);
       } else {
         result[i] = nullptr;
       }
     }
   } else {
-    for (int i = 0; i < kMaxLanes; i++) {
+    for (int i = 0; i < kMaxLanes; ++i) {
       if (replacements[i] != nullptr) {
         result[i] = graph()->NewNode(machine()->BitcastFloat32ToInt32(),
                                      replacements[i]);
       } else {
         result[i] = nullptr;
       }
     }
   }
   return result;
 }
 
 void SimdScalarLowering::PreparePhiReplacement(Node* phi) {
   MachineRepresentation rep = PhiRepresentationOf(phi->op());
   if (rep == MachineRepresentation::kSimd128) {
     // We have to create the replacements for a phi node before we actually
     // lower the phi to break potential cycles in the graph. The replacements of
     // input nodes do not exist yet, so we use a placeholder node to pass the
     // graph verifier.
     int value_count = phi->op()->ValueInputCount();
     SimdType type = ReplacementType(phi);
     Node** inputs_rep[kMaxLanes];
-    for (int i = 0; i < kMaxLanes; i++) {
+    for (int i = 0; i < kMaxLanes; ++i) {
       inputs_rep[i] = zone()->NewArray<Node*>(value_count + 1);
       inputs_rep[i][value_count] = NodeProperties::GetControlInput(phi, 0);
     }
-    for (int i = 0; i < value_count; i++) {
+    for (int i = 0; i < value_count; ++i) {
       for (int j = 0; j < kMaxLanes; j++) {
         inputs_rep[j][i] = placeholder_;
       }
     }
     Node* rep_nodes[kMaxLanes];
-    for (int i = 0; i < kMaxLanes; i++) {
+    for (int i = 0; i < kMaxLanes; ++i) {
       if (type == SimdType::kInt32) {
         rep_nodes[i] = graph()->NewNode(
             common()->Phi(MachineRepresentation::kWord32, value_count),
             value_count + 1, inputs_rep[i], false);
       } else if (type == SimdType::kFloat32) {
         rep_nodes[i] = graph()->NewNode(
             common()->Phi(MachineRepresentation::kFloat32, value_count),
             value_count + 1, inputs_rep[i], false);
       } else {
         UNREACHABLE();
       }
     }
     ReplaceNode(phi, rep_nodes);
   }
 }
 }  // namespace compiler
 }  // namespace internal
 }  // namespace v8