Lower the fcmp instruction for <4 x float> operands.

src/IceTargetLoweringX8632.cpp

 //===- subzero/src/IceTargetLoweringX8632.cpp - x86-32 lowering -----------===//
 //
 //                        The Subzero Code Generator
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the TargetLoweringX8632 class, which
 // consists almost entirely of the lowering sequence for each
 // high-level instruction.  It also implements
 // TargetX8632Fast::postLower() which does the simplest possible
 // register allocation for the "fast" target.
 //
 //===----------------------------------------------------------------------===//
 
 #include "IceDefs.h"
 #include "IceCfg.h"
 #include "IceCfgNode.h"
 #include "IceInstX8632.h"
 #include "IceOperand.h"
 #include "IceTargetLoweringX8632.def"
 #include "IceTargetLoweringX8632.h"
 
 namespace Ice {
 
 namespace {
 
-// The following table summarizes the logic for lowering the fcmp instruction.
+// The following table summarizes the logic for lowering the fcmp
+// instruction when the operands are floating point scalar values.
 // There is one table entry for each of the 16 conditions.  A comment in
-// lowerFcmp() describes the lowering template.  In the most general case, there
-// is a compare followed by two conditional branches, because some fcmp
-// conditions don't map to a single x86 conditional branch.  However, in many
-// cases it is possible to swap the operands in the comparison and have a single
-// conditional branch.  Since it's quite tedious to validate the table by hand,
-// good execution tests are helpful.
+// lowerFcmp() describes the lowering template.  In the most general
+// case, there is a compare followed by two conditional branches,
+// because some fcmp conditions don't map to a single x86 conditional
+// branch.  However, in many cases it is possible to swap the operands
+// in the comparison and have a single conditional branch.  Since it's
+// quite tedious to validate the table by hand, good execution tests are
+// helpful.
 
-const struct TableFcmp_ {
+const struct TableScalarFcmp_ {
   uint32_t Default;
   bool SwapOperands;
   InstX8632::BrCond C1, C2;
-} TableFcmp[] = {
+} TableScalarFcmp[] = {
 #define X(val, dflt, swap, C1, C2)                                             \
   { dflt, swap, InstX8632Br::C1, InstX8632Br::C2 }                             \
   ,
-    FCMPX8632_TABLE
+    SCALAR_FCMPX8632_TABLE
 #undef X
   };
-const size_t TableFcmpSize = llvm::array_lengthof(TableFcmp);
+const size_t TableScalarFcmpSize = llvm::array_lengthof(TableScalarFcmp);
+
+// The following table summarizes the logic for lowering the fcmp
+// instruction when the operands are vectors of floating point values.
+// For most fcmp conditions, there is a clear mapping to a single x86
+// cmpps instruction variant.  Some fcmp conditions require special code
+// to handle and these are marked in the table with a Cmpps_Invalid
+// predicate.
+
+const struct TableVectorFcmp_ {
+  bool SwapOperands;
+  InstX8632Cmpps::CmppsCond Predicate;
+} TableVectorFcmp[] = {
+#define X(val, swap, pred)                                                     \
+  { swap, InstX8632Cmpps::pred }                                               \
+  ,
+    VECTOR_FCMPX8632_TABLE
+#undef X
+  };
+const size_t TableVectorFcmpSize = llvm::array_lengthof(TableVectorFcmp);
 
 // The following table summarizes the logic for lowering the icmp instruction
 // for i32 and narrower types.  Each icmp condition has a clear mapping to an
 // x86 conditional branch instruction.
 
 const struct TableIcmp32_ {
   InstX8632::BrCond Mapping;
 } TableIcmp32[] = {
 #define X(val, C_32, C1_64, C2_64, C3_64)                                      \
   { InstX8632Br::C_32 }                                                        \
@@ skipping 65 matching lines @@
 }
 
 // In some cases, there are x-macros tables for both high-level and
 // low-level instructions/operands that use the same enum key value.
 // The tables are kept separate to maintain a proper separation
 // between abstraction layers.  There is a risk that the tables
 // could get out of sync if enum values are reordered or if entries
 // are added or deleted.  This dummy function uses static_assert to
 // ensure everything is kept in sync.
 void xMacroIntegrityCheck() {
-  // Validate the enum values in FCMPX8632_TABLE.
+  // Validate the enum values in SCALAR_FCMPX8632_TABLE.
   {
     // Define a temporary set of enum values based on low-level
     // table entries.
     enum _tmp_enum {
 #define X(val, dflt, swap, C1, C2) _tmp_##val,
-      FCMPX8632_TABLE
+      SCALAR_FCMPX8632_TABLE
 #undef X
           _num
     };
 // Define a set of constants based on high-level table entries.
 #define X(tag, str) static const int _table1_##tag = InstFcmp::tag;
     ICEINSTFCMP_TABLE;
 #undef X
 // Define a set of constants based on low-level table entries,
 // and ensure the table entry keys are consistent.
 #define X(val, dflt, swap, C1, C2)                                             \
   static const int _table2_##val = _tmp_##val;                                 \
   STATIC_ASSERT(_table1_##val == _table2_##val);
-    FCMPX8632_TABLE;
+    SCALAR_FCMPX8632_TABLE;
 #undef X
 // Repeat the static asserts with respect to the high-level
 // table entries in case the high-level table has extra entries.
+#define X(tag, str) STATIC_ASSERT(_table1_##tag == _table2_##tag);
+    ICEINSTFCMP_TABLE;
+#undef X
+  }
+
+  // Validate the enum values in VECTOR_FCMPX8632_TABLE.
+  {
+    // Define a temporary set of enum values based on low-level
+    // table entries.
+    enum _tmp_enum {
+#define X(val, swap, pred) _tmp_##val,
+      VECTOR_FCMPX8632_TABLE
+#undef X
+          _num
+    };
+// Define a set of constants based on high-level table entries.
+#define X(tag, str) static const int _table1_##tag = InstFcmp::tag;
+    ICEINSTFCMP_TABLE;
+#undef X
+// Define a set of constants based on low-level table entries,
+// and ensure the table entry keys are consistent.
+#define X(val, swap, pred)                                                     \
+  static const int _table2_##val = _tmp_##val;                                 \
+  STATIC_ASSERT(_table1_##val == _table2_##val);
+    VECTOR_FCMPX8632_TABLE;
+#undef X
+// Repeat the static asserts with respect to the high-level
+// table entries in case the high-level table has extra entries.
 #define X(tag, str) STATIC_ASSERT(_table1_##tag == _table2_##tag);
     ICEINSTFCMP_TABLE;
 #undef X
   }
 
   // Validate the enum values in ICMPX8632_TABLE.
   {
     // Define a temporary set of enum values based on low-level
     // table entries.
     enum _tmp_enum {
@@ skipping 2037 matching lines @@
 
   // Copy the element to the destination.
   Variable *Dest = Inst->getDest();
   _mov(Dest, ExtractedElement);
 }
 
 void TargetX8632::lowerFcmp(const InstFcmp *Inst) {
   Operand *Src0 = Inst->getSrc(0);
   Operand *Src1 = Inst->getSrc(1);
   Variable *Dest = Inst->getDest();
+
+  if (isVectorType(Dest->getType())) {
+    InstFcmp::FCond Condition = Inst->getCondition();
+    size_t Index = static_cast<size_t>(Condition);
+    assert(Index < TableVectorFcmpSize);
+
+    if (TableVectorFcmp[Index].SwapOperands) {
+      Operand *T = Src0;
+      Src0 = Src1;
+      Src1 = T;
+    }
+
+    Variable *T = NULL;
+
+    // ALIGNHACK: Without support for stack alignment, both operands to
+    // cmpps need to be forced into registers.  Once support for stack
+    // alignment is implemented, remove LEGAL_HACK.
+#define LEGAL_HACK(Vect) legalizeToVar((Vect))
+    switch (Condition) {
+    default: {
+      InstX8632Cmpps::CmppsCond Predicate = TableVectorFcmp[Index].Predicate;
+      assert(Predicate != InstX8632Cmpps::Cmpps_Invalid);
+      T = makeReg(Src0->getType());
+      _movp(T, Src0);
+      _cmpps(T, LEGAL_HACK(Src1), Predicate);
+    } break;
+    case InstFcmp::False:
+      T = makeVectorOfZeros(Src0->getType());
+      break;
+    case InstFcmp::One: {
+      // Check both unequal and ordered.
+      T = makeReg(Src0->getType());
+      Variable *T2 = makeReg(Src0->getType());
+      Src1 = LEGAL_HACK(Src1);
+      _movp(T, Src0);
+      _cmpps(T, Src1, InstX8632Cmpps::Cmpps_neq);
+      _movp(T2, Src0);
+      _cmpps(T2, Src1, InstX8632Cmpps::Cmpps_ord);
+      _pand(T, T2);
+    } break;
+    case InstFcmp::Ueq: {
+      // Check both equal or unordered.
+      T = makeReg(Src0->getType());
+      Variable *T2 = makeReg(Src0->getType());
+      Src1 = LEGAL_HACK(Src1);
+      _movp(T, Src0);
+      _cmpps(T, Src1, InstX8632Cmpps::Cmpps_eq);
+      _movp(T2, Src0);
+      _cmpps(T2, Src1, InstX8632Cmpps::Cmpps_unord);
+      _por(T, T2);
+    } break;
+    case InstFcmp::True:
+      T = makeVectorOfMinusOnes(IceType_v4i32);
+      break;
+    }
+#undef LEGAL_HACK
+
+    _movp(Dest, T);
+    eliminateNextVectorSextInstruction(Dest);
+    return;
+  }
+
   // Lowering a = fcmp cond, b, c
   //   ucomiss b, c       /* only if C1 != Br_None */
   //                      /* but swap b,c order if SwapOperands==true */
   //   mov a, <default>
   //   j<C1> label        /* only if C1 != Br_None */
   //   j<C2> label        /* only if C2 != Br_None */
   //   FakeUse(a)         /* only if C1 != Br_None */
   //   mov a, !<default>  /* only if C1 != Br_None */
   //   label:             /* only if C1 != Br_None */
   InstFcmp::FCond Condition = Inst->getCondition();
   size_t Index = static_cast<size_t>(Condition);
-  assert(Index < TableFcmpSize);
-  if (TableFcmp[Index].SwapOperands) {
+  assert(Index < TableScalarFcmpSize);
+  if (TableScalarFcmp[Index].SwapOperands) {
     Operand *Tmp = Src0;
     Src0 = Src1;
     Src1 = Tmp;
   }
-  bool HasC1 = (TableFcmp[Index].C1 != InstX8632Br::Br_None);
-  bool HasC2 = (TableFcmp[Index].C2 != InstX8632Br::Br_None);
+  bool HasC1 = (TableScalarFcmp[Index].C1 != InstX8632Br::Br_None);
+  bool HasC2 = (TableScalarFcmp[Index].C2 != InstX8632Br::Br_None);
   if (HasC1) {
     Src0 = legalize(Src0);
     Operand *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem);
     Variable *T = NULL;
     _mov(T, Src0);
     _ucomiss(T, Src1RM);
   }
   Constant *Default =
-      Ctx->getConstantInt(IceType_i32, TableFcmp[Index].Default);
+      Ctx->getConstantInt(IceType_i32, TableScalarFcmp[Index].Default);
   _mov(Dest, Default);
   if (HasC1) {
     InstX8632Label *Label = InstX8632Label::create(Func, this);
-    _br(TableFcmp[Index].C1, Label);
+    _br(TableScalarFcmp[Index].C1, Label);
     if (HasC2) {
-      _br(TableFcmp[Index].C2, Label);
+      _br(TableScalarFcmp[Index].C2, Label);
     }
     Context.insert(InstFakeUse::create(Func, Dest));
     Constant *NonDefault =
-        Ctx->getConstantInt(IceType_i32, !TableFcmp[Index].Default);
+        Ctx->getConstantInt(IceType_i32, !TableScalarFcmp[Index].Default);
     _mov(Dest, NonDefault);
     Context.insert(Label);
   }
 }
 
 void TargetX8632::lowerIcmp(const InstIcmp *Inst) {
   Operand *Src0 = legalize(Inst->getSrc(0));
   Operand *Src1 = legalize(Inst->getSrc(1));
   Variable *Dest = Inst->getDest();
 
@@ skipping 85 matching lines @@
       // !(Src0 > Src1)
       _movp(T, Src0);
       _pcmpgt(T, LEGAL_HACK(Src1));
       Variable *MinusOne = makeVectorOfMinusOnes(Ty);
       _pxor(T, MinusOne);
     } break;
     }
 #undef LEGAL_HACK
 
     _movp(Dest, T);
-
-    // The following pattern occurs often in lowered C and C++ code:
-    //
-    //   %cmp     = icmp pred <n x ty> %src0, %src1
-    //   %cmp.ext = sext <n x i1> %cmp to <n x ty>
-    //
-    // We can avoid the sext operation by copying the result from pcmpgt
-    // and pcmpeq, which is already sign extended, to the result of the
-    // sext operation
-    if (InstCast *NextCast =
-            llvm::dyn_cast_or_null<InstCast>(Context.getNextInst())) {
-      if (NextCast->getCastKind() == InstCast::Sext &&
-          NextCast->getSrc(0) == Dest) {
-        _movp(NextCast->getDest(), T);
-        // Skip over the instruction.
-        NextCast->setDeleted();
-        Context.advanceNext();
-      }
-    }
-
+    eliminateNextVectorSextInstruction(Dest);
     return;
   }
 
   // If Src1 is an immediate, or known to be a physical register, we can
   // allow Src0 to be a memory operand.  Otherwise, Src0 must be copied into
   // a physical register.  (Actually, either Src0 or Src1 can be chosen for
   // the physical register, but unfortunately we have to commit to one or
   // the other before register allocation.)
   bool IsSrc1ImmOrReg = false;
   if (llvm::isa<Constant>(Src1)) {
@@ skipping 1113 matching lines @@
     Src0 = legalize(Src0, Legal_All, true);
   for (SizeT I = 0; I < NumCases; ++I) {
     Operand *Value = Ctx->getConstantInt(IceType_i32, Inst->getValue(I));
     _cmp(Src0, Value);
     _br(InstX8632Br::Br_e, Inst->getLabel(I));
   }
 
   _br(Inst->getLabelDefault());
 }
 
+// The following pattern occurs often in lowered C and C++ code:
+//
+//   %cmp     = fcmp/icmp pred <n x ty> %src0, %src1
+//   %cmp.ext = sext <n x i1> %cmp to <n x ty>
+//
+// We can eliminate the sext operation by copying the result of pcmpeqd,
+// pcmpgtd, or cmpps (which produce sign extended results) the result of
+// the sext operation.
+void
+TargetX8632::eliminateNextVectorSextInstruction(Variable *SignExtendedResult) {
+  if (InstCast *NextCast =
+          llvm::dyn_cast_or_null<InstCast>(Context.getNextInst())) {
+    if (NextCast->getCastKind() == InstCast::Sext &&
+        NextCast->getSrc(0) == SignExtendedResult) {
+      _movp(NextCast->getDest(), legalizeToVar(SignExtendedResult));
+      // Skip over the instruction.
+      NextCast->setDeleted();
+      Context.advanceNext();
+    }
+  }
+}
+
 void TargetX8632::lowerUnreachable(const InstUnreachable * /*Inst*/) {
   const SizeT MaxSrcs = 0;
   Variable *Dest = NULL;
   InstCall *Call = makeHelperCall("ice_unreachable", Dest, MaxSrcs);
   lowerCall(Call);
 }
 
 // There is no support for loading or emitting vector constants, so the
 // vector values returned from makeVectorOfZeros, makeVectorOfOnes,
 // etc. are initialized with register operations.
@@ skipping 376 matching lines @@
     for (SizeT i = 0; i < Size; ++i) {
       Str << "\t.byte\t" << (((unsigned)Data[i]) & 0xff) << "\n";
     }
     Str << "\t.size\t" << MangledName << ", " << Size << "\n";
   }
   Str << "\t" << (IsInternal ? ".local" : ".global") << "\t" << MangledName
       << "\n";
 }
 
 } // end of namespace Ice