Subzero. ARM32. De-scalarizes icmp and fcmp for vectors.

src/IceTargetLoweringARM32.cpp

 //===- subzero/src/IceTargetLoweringARM32.cpp - ARM32 lowering ------------===//
 //
 //                        The Subzero Code Generator
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 ///
 /// \file
@@ skipping 106 matching lines @@
 
 namespace {
 
 // The following table summarizes the logic for lowering the icmp instruction
 // for i32 and narrower types. Each icmp condition has a clear mapping to an
 // ARM32 conditional move instruction.
 
 const struct TableIcmp32_ {
   CondARM32::Cond Mapping;
 } TableIcmp32[] = {
-#define X(val, is_signed, swapped64, C_32, C1_64, C2_64)                       \
+#define X(val, is_signed, swapped64, C_32, C1_64, C2_64, C_V, INV_V, NEG_V)    \
   { CondARM32::C_32 }                                                          \
   ,
     ICMPARM32_TABLE
 #undef X
 };
 
 // The following table summarizes the logic for lowering the icmp instruction
 // for the i64 type. Two conditional moves are needed for setting to 1 or 0.
 // The operands may need to be swapped, and there is a slight difference for
 // signed vs unsigned (comparing hi vs lo first, and using cmp vs sbc).
 const struct TableIcmp64_ {
   bool IsSigned;
   bool Swapped;
   CondARM32::Cond C1, C2;
 } TableIcmp64[] = {
-#define X(val, is_signed, swapped64, C_32, C1_64, C2_64)                       \
+#define X(val, is_signed, swapped64, C_32, C1_64, C2_64, C_V, INV_V, NEG_V)    \
   { is_signed, swapped64, CondARM32::C1_64, CondARM32::C2_64 }                 \
   ,
     ICMPARM32_TABLE
 #undef X
 };
 
 CondARM32::Cond getIcmp32Mapping(InstIcmp::ICond Cond) {
   assert(Cond < llvm::array_lengthof(TableIcmp32));
   return TableIcmp32[Cond].Mapping;
 }
 
 // In some cases, there are x-macros tables for both high-level and low-level
 // instructions/operands that use the same enum key value. The tables are kept
 // separate to maintain a proper separation between abstraction layers. There
 // is a risk that the tables could get out of sync if enum values are reordered
 // or if entries are added or deleted. The following anonymous namespaces use
 // static_asserts to ensure everything is kept in sync.
 
 // Validate the enum values in ICMPARM32_TABLE.
 namespace {
 // Define a temporary set of enum values based on low-level table entries.
 enum _icmp_ll_enum {
-#define X(val, signed, swapped64, C_32, C1_64, C2_64) _icmp_ll_##val,
+#define X(val, is_signed, swapped64, C_32, C1_64, C2_64, C_V, INV_V, NEG_V)    \
+  _icmp_ll_##val,
   ICMPARM32_TABLE
 #undef X
       _num
 };
 // Define a set of constants based on high-level table entries.
 #define X(tag, str) static constexpr int _icmp_hl_##tag = InstIcmp::tag;
 ICEINSTICMP_TABLE
 #undef X
 // Define a set of constants based on low-level table entries, and ensure the
 // table entry keys are consistent.
-#define X(val, signed, swapped64, C_32, C1_64, C2_64)                          \
+#define X(val, is_signed, swapped64, C_32, C1_64, C2_64, C_V, INV_V, NEG_V)    \
   static_assert(                                                               \
       _icmp_ll_##val == _icmp_hl_##val,                                        \
       "Inconsistency between ICMPARM32_TABLE and ICEINSTICMP_TABLE: " #val);
 ICMPARM32_TABLE
 #undef X
 // Repeat the static asserts with respect to the high-level table entries in
 // case the high-level table has extra entries.
 #define X(tag, str)                                                            \
   static_assert(                                                               \
       _icmp_hl_##tag == _icmp_ll_##tag,                                        \
@@ skipping 634 matching lines @@
           Ctx->getRuntimeHelperFunc(RuntimeHelper::H_call_setjmp);
       auto *Call = Context.insert<InstCall>(MaxArgs, Dest, TargetHelper,
                                             NoTailCall, IsTargetHelperCall);
       Call->addArg(IntrinsicCall->getArg(0));
       Instr->setDeleted();
       return;
     }
     }
     llvm::report_fatal_error("Control flow should never have reached here.");
   }
-  case Inst::Icmp: {
-    Variable *Dest = Instr->getDest();
-    const Type DestTy = Dest->getType();
-    if (isVectorType(DestTy)) {
-      auto *CmpInstr = llvm::cast<InstIcmp>(Instr);
-      const auto Condition = CmpInstr->getCondition();
-      scalarizeInstruction(
-          Dest,
-          [this, Condition](Variable *Dest, Variable *Src0, Variable *Src1) {
-            return Context.insert<InstIcmp>(Condition, Dest, Src0, Src1);
-          },
-          CmpInstr->getSrc(0), CmpInstr->getSrc(1));
-      CmpInstr->setDeleted();
-    }
-    return;
-  }
-  case Inst::Fcmp: {
-    Variable *Dest = Instr->getDest();
-    const Type DestTy = Dest->getType();
-    if (isVectorType(DestTy)) {
-      auto *CmpInstr = llvm::cast<InstFcmp>(Instr);
-      const auto Condition = CmpInstr->getCondition();
-      scalarizeInstruction(
-          Dest,
-          [this, Condition](Variable *Dest, Variable *Src0, Variable *Src1) {
-            return Context.insert<InstFcmp>(Condition, Dest, Src0, Src1);
-          },
-          CmpInstr->getSrc(0), CmpInstr->getSrc(1));
-      CmpInstr->setDeleted();
-    }
-    return;
-  }
   }
 }
 
 void TargetARM32::findMaxStackOutArgsSize() {
   // MinNeededOutArgsBytes should be updated if the Target ever creates a
   // high-level InstCall that requires more stack bytes.
   constexpr size_t MinNeededOutArgsBytes = 0;
   MaxOutArgsSizeBytes = MinNeededOutArgsBytes;
   for (CfgNode *Node : Func->getNodes()) {
     Context.init(Node);
@@ skipping 3370 matching lines @@
     _mov(Dest, T);
     return;
   }
   assert(false && "extractelement requires a constant index");
 }
 
 namespace {
 // Validates FCMPARM32_TABLE's declaration w.r.t. InstFcmp::FCondition ordering
 // (and naming).
 enum {
-#define X(val, CC0, CC1) _fcmp_ll_##val,
+#define X(val, CC0, CC1, CC0_V, CC1_V, INV_V, NEG_V) _fcmp_ll_##val,
   FCMPARM32_TABLE
 #undef X
       _fcmp_ll_NUM
 };
 
 enum {
 #define X(tag, str) _fcmp_hl_##tag = InstFcmp::tag,
   ICEINSTFCMP_TABLE
 #undef X
       _fcmp_hl_NUM
 };
 
 static_assert((uint32_t)_fcmp_hl_NUM == (uint32_t)_fcmp_ll_NUM,
               "Inconsistency between high-level and low-level fcmp tags.");
 #define X(tag, str)                                                            \
   static_assert(                                                               \
       (uint32_t)_fcmp_hl_##tag == (uint32_t)_fcmp_ll_##tag,                    \
       "Inconsistency between high-level and low-level fcmp tag " #tag);
 ICEINSTFCMP_TABLE
 #undef X
 
 struct {
   CondARM32::Cond CC0;
   CondARM32::Cond CC1;
 } TableFcmp[] = {
-#define X(val, CC0, CC1)                                                       \
+#define X(val, CC0, CC1, CC0_V, CC1_V, INV_V, NEG_V)                           \
   { CondARM32::CC0, CondARM32::CC1 }                                           \
   ,
     FCMPARM32_TABLE
 #undef X
 };
 
 bool isFloatingPointZero(const Operand *Src) {
   if (const auto *F32 = llvm::dyn_cast<const ConstantFloat>(Src)) {
     return Utils::isPositiveZero(F32->getValue());
   }
@@ skipping 24 matching lines @@
     }
     _vmrs();
     assert(Condition < llvm::array_lengthof(TableFcmp));
     return CondWhenTrue(TableFcmp[Condition].CC0, TableFcmp[Condition].CC1);
   }
   }
 }
 
 void TargetARM32::lowerFcmp(const InstFcmp *Instr) {
   Variable *Dest = Instr->getDest();
-  if (isVectorType(Dest->getType())) {
-    UnimplementedLoweringError(this, Instr);
+  const Type DestTy = Dest->getType();
+
+  if (isVectorType(DestTy)) {
+    if (Instr->getCondition() == InstFcmp::False) {
+      constexpr Type SafeTypeForMovingConstant = IceType_v4i32;
+      auto *T = makeReg(SafeTypeForMovingConstant);
+      _mov(T, llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(0)));
+      _mov(Dest, T);
+      return;
+    }
+
+    if (Instr->getCondition() == InstFcmp::True) {
+      constexpr Type SafeTypeForMovingConstant = IceType_v4i32;
+      auto *T = makeReg(SafeTypeForMovingConstant);
+      _mov(T, llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(1)));
+      _mov(Dest, T);
+      return;
+    }
+
+    Variable *T0;
+    Variable *T1;
+    bool Negate = false;
+    auto *Src0 = legalizeToReg(Instr->getSrc(0));
+    auto *Src1 = legalizeToReg(Instr->getSrc(1));
+
+    switch (Instr->getCondition()) {
+    default:
+      llvm::report_fatal_error("Unhandled fp comparison.");
+#define _Vcnone(Tptr, S0, S1)                                                  \
+  do {                                                                         \
+    *(Tptr) = nullptr;                                                         \
+  } while (0)
+#define _Vceq(Tptr, S0, S1)                                                    \
+  do {                                                                         \
+    *(Tptr) = makeReg(DestTy);                                                 \
+    _vceq(*(Tptr), S0, S1);                                                    \
+  } while (0)
+#define _Vcge(Tptr, S0, S1)                                                    \
+  do {                                                                         \
+    *(Tptr) = makeReg(DestTy);                                                 \
+    _vcge(*(Tptr), S0, S1)->setSignType(InstARM32::FS_Signed);                 \
+  } while (0)
+#define _Vcgt(Tptr, S0, S1)                                                    \
+  do {                                                                         \
+    *(Tptr) = makeReg(DestTy);                                                 \
+    _vcgt(*(Tptr), S0, S1)->setSignType(InstARM32::FS_Signed);                 \
+  } while (0)
+#define X(val, CC0, CC1, CC0_V, CC1_V, INV_V, NEG_V)                           \
+  case InstFcmp::val: {                                                        \
+    _Vc##CC0_V(&T0, (INV_V) ? Src1 : Src0, (INV_V) ? Src0 : Src1);             \
+    _Vc##CC1_V(&T1, (INV_V) ? Src0 : Src1, (INV_V) ? Src1 : Src0);             \
+    Negate = NEG_V;                                                            \
+  } break;
+      FCMPARM32_TABLE
+#undef X
+#undef _Vcgt
+#undef _Vcge
+#undef _Vceq
+#undef _Vcnone
+    }
+    assert(T0 != nullptr);
+    Variable *T = T0;
+    if (T1 != nullptr) {
+      T = makeReg(DestTy);
+      _vorr(T, T0, T1);
+    }
+
+    if (Negate) {
+      auto *TNeg = makeReg(DestTy);
+      _vmvn(TNeg, T);
+      T = TNeg;
+    }
+
+    _mov(Dest, T);
     return;
   }
 
   Variable *T = makeReg(IceType_i1);
   Operand *_1 = legalize(Ctx->getConstantInt32(1), Legal_Reg | Legal_Flex);
   Operand *_0 =
       legalize(Ctx->getConstantZero(IceType_i32), Legal_Reg | Legal_Flex);
 
   CondWhenTrue Cond = lowerFcmpCond(Instr);
 
@@ skipping 277 matching lines @@
     return lowerInt8AndInt16IcmpCond(Condition, Src0, Src1);
   case IceType_i32:
     return lowerInt32IcmpCond(Condition, Src0, Src1);
   case IceType_i64:
     return lowerInt64IcmpCond(Condition, Src0, Src1);
   }
 }
 
 void TargetARM32::lowerIcmp(const InstIcmp *Instr) {
   Variable *Dest = Instr->getDest();
+  const Type DestTy = Dest->getType();
 
-  if (isVectorType(Dest->getType())) {
-    UnimplementedLoweringError(this, Instr);
+  if (isVectorType(DestTy)) {
+    auto *T = makeReg(DestTy);
+    auto *Src0 = legalizeToReg(Instr->getSrc(0));
+    auto *Src1 = legalizeToReg(Instr->getSrc(1));
+    const Type SrcTy = Src0->getType();
+
+    bool NeedsShl = false;
+    Type NewTypeAfterShl;
+    SizeT ShAmt;
+    switch (SrcTy) {
+    default:
+      break;
+    case IceType_v16i1:
+      NeedsShl = true;
+      NewTypeAfterShl = IceType_v16i8;
+      ShAmt = 7;
+      break;
+    case IceType_v8i1:
+      NeedsShl = true;
+      NewTypeAfterShl = IceType_v8i16;
+      ShAmt = 15;
+      break;
+    case IceType_v4i1:
+      NeedsShl = true;
+      NewTypeAfterShl = IceType_v4i32;
+      ShAmt = 31;
+      break;
+    }
+
+    if (NeedsShl) {
+      auto *Imm = llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(ShAmt));
+      auto *Src0T = makeReg(NewTypeAfterShl);
+      auto *Src0Shl = makeReg(NewTypeAfterShl);
+      _mov(Src0T, Src0);
+      _vshl(Src0Shl, Src0T, Imm);
+      Src0 = Src0Shl;
+
+      auto *Src1T = makeReg(NewTypeAfterShl);
+      auto *Src1Shl = makeReg(NewTypeAfterShl);
+      _mov(Src1T, Src1);
+      _vshl(Src1Shl, Src1T, Imm);
+      Src1 = Src1Shl;
+    }
+
+    switch (Instr->getCondition()) {
+    default:
+      llvm::report_fatal_error("Unhandled integer comparison.");
+#define _Vceq(T, S0, S1, Signed) _vceq(T, S0, S1)
+#define _Vcge(T, S0, S1, Signed)                                               \
+  _vcge(T, S0, S1)                                                             \
+      ->setSignType(Signed ? InstARM32::FS_Signed : InstARM32::FS_Unsigned)
+#define _Vcgt(T, S0, S1, Signed)                                               \
+  _vcgt(T, S0, S1)                                                             \
+      ->setSignType(Signed ? InstARM32::FS_Signed : InstARM32::FS_Unsigned)
+#define X(val, is_signed, swapped64, C_32, C1_64, C2_64, C_V, INV_V, NEG_V)    \
+  case InstIcmp::val: {                                                        \
+    _Vc##C_V(T, (INV_V) ? Src1 : Src0, (INV_V) ? Src0 : Src1, is_signed);      \
+    if (NEG_V) {                                                               \
+      auto *TInv = makeReg(DestTy);                                            \
+      _vmvn(TInv, T);                                                          \
+      T = TInv;                                                                \
+    }                                                                          \
+  } break;
+      ICMPARM32_TABLE
+#undef X
+#undef _Vcgt
+#undef _Vcge
+#undef _Vceq
+    }
+    _mov(Dest, T);
     return;
   }
 
   Operand *_0 =
       legalize(Ctx->getConstantZero(IceType_i32), Legal_Reg | Legal_Flex);
   Operand *_1 = legalize(Ctx->getConstantInt32(1), Legal_Reg | Legal_Flex);
   Variable *T = makeReg(IceType_i1);
 
   _mov(T, _0);
   CondWhenTrue Cond = lowerIcmpCond(Instr);
@@ skipping 2357 matching lines @@
   // However, for compatibility with current NaCl LLVM, don't claim that.
   Str << ".eabi_attribute 14, 3   @ Tag_ABI_PCS_R9_use: Not used\n";
 }
 
 SmallBitVector TargetARM32::TypeToRegisterSet[RegARM32::RCARM32_NUM];
 SmallBitVector TargetARM32::TypeToRegisterSetUnfiltered[RegARM32::RCARM32_NUM];
 SmallBitVector TargetARM32::RegisterAliases[RegARM32::Reg_NUM];
 
 } // end of namespace ARM32
 } // end of namespace Ice