Subzero: Support non sequentially consistent memory orderings for atomic ops.

src/IceTargetLoweringX8632.cpp

 //===- subzero/src/IceTargetLoweringX8632.cpp - x86-32 lowering -----------===//
 //
 //                        The Subzero Code Generator
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the TargetLoweringX8632 class, which
@@ skipping 2838 matching lines @@
     OperandX8632Mem *Loc =
         getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset);
     _store(legalizeToVar(ElementToInsertNotLegalized), Loc);
 
     Variable *T = makeReg(Ty);
     _movp(T, Slot);
     _movp(Inst->getDest(), T);
   }
 }
 
+namespace {
+
+// Converts a ConstantInteger32 operand into its constant value, or
+// MemoryOrderInvalid if the operand is not a ConstantInteger32.
+uint64_t getConstantMemoryOrder(Operand *Opnd) {
+  if (auto Integer = llvm::dyn_cast<ConstantInteger32>(Opnd))
+    return Integer->getValue();
+  return Intrinsics::MemoryOrderInvalid;
+}
+
+} // end of anonymous namespace
+
 void TargetX8632::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
-  switch (Instr->getIntrinsicInfo().ID) {
+  switch (Intrinsics::IntrinsicID ID = Instr->getIntrinsicInfo().ID) {
   case Intrinsics::AtomicCmpxchg: {
-    if (!Intrinsics::VerifyMemoryOrder(
-            llvm::cast<ConstantInteger32>(Instr->getArg(3))->getValue())) {
-      Func->setError("Unexpected memory ordering (success) for AtomicCmpxchg");
-      return;
-    }
-    if (!Intrinsics::VerifyMemoryOrder(
-            llvm::cast<ConstantInteger32>(Instr->getArg(4))->getValue())) {
-      Func->setError("Unexpected memory ordering (failure) for AtomicCmpxchg");
+    if (!Intrinsics::isMemoryOrderValid(
+            ID, getConstantMemoryOrder(Instr->getArg(3)),
+            getConstantMemoryOrder(Instr->getArg(4)))) {
+      Func->setError("Unexpected memory ordering for AtomicCmpxchg");
       return;
     }
     Variable *DestPrev = Instr->getDest();
     Operand *PtrToMem = Instr->getArg(0);
     Operand *Expected = Instr->getArg(1);
     Operand *Desired = Instr->getArg(2);
     if (tryOptimizedCmpxchgCmpBr(DestPrev, PtrToMem, Expected, Desired))
       return;
     lowerAtomicCmpxchg(DestPrev, PtrToMem, Expected, Desired);
     return;
   }
   case Intrinsics::AtomicFence:
-    if (!Intrinsics::VerifyMemoryOrder(
-            llvm::cast<ConstantInteger32>(Instr->getArg(0))->getValue())) {
+    if (!Intrinsics::isMemoryOrderValid(
+            ID, getConstantMemoryOrder(Instr->getArg(0)))) {
       Func->setError("Unexpected memory ordering for AtomicFence");
       return;
     }
     _mfence();
     return;
   case Intrinsics::AtomicFenceAll:
     // NOTE: FenceAll should prevent and load/store from being moved
     // across the fence (both atomic and non-atomic). The InstX8632Mfence
     // instruction is currently marked coarsely as "HasSideEffects".
     _mfence();
@@ skipping 24 matching lines @@
       _mov(Dest, Result);
       return;
     }
     // The PNaCl ABI requires the byte size to be a compile-time constant.
     Func->setError("AtomicIsLockFree byte size should be compile-time const");
     return;
   }
   case Intrinsics::AtomicLoad: {
     // We require the memory address to be naturally aligned.
     // Given that is the case, then normal loads are atomic.
-    if (!Intrinsics::VerifyMemoryOrder(
-            llvm::cast<ConstantInteger32>(Instr->getArg(1))->getValue())) {
+    if (!Intrinsics::isMemoryOrderValid(
+            ID, getConstantMemoryOrder(Instr->getArg(1)))) {
       Func->setError("Unexpected memory ordering for AtomicLoad");
       return;
     }
     Variable *Dest = Instr->getDest();
     if (Dest->getType() == IceType_i64) {
       // Follow what GCC does and use a movq instead of what lowerLoad()
       // normally does (split the load into two).
       // Thus, this skips load/arithmetic op folding. Load/arithmetic folding
       // can't happen anyway, since this is x86-32 and integer arithmetic only
       // happens on 32-bit quantities.
@@ skipping 11 matching lines @@
     InstLoad *Load = InstLoad::create(Func, Dest, Instr->getArg(0));
     lowerLoad(Load);
     // Make sure the atomic load isn't elided when unused, by adding a FakeUse.
     // Since lowerLoad may fuse the load w/ an arithmetic instruction,
     // insert the FakeUse on the last-inserted instruction's dest.
     Context.insert(
         InstFakeUse::create(Func, Context.getLastInserted()->getDest()));
     return;
   }
   case Intrinsics::AtomicRMW:
-    if (!Intrinsics::VerifyMemoryOrder(
-            llvm::cast<ConstantInteger32>(Instr->getArg(3))->getValue())) {
+    if (!Intrinsics::isMemoryOrderValid(
+            ID, getConstantMemoryOrder(Instr->getArg(3)))) {
       Func->setError("Unexpected memory ordering for AtomicRMW");
       return;
     }
     lowerAtomicRMW(Instr->getDest(),
                    static_cast<uint32_t>(llvm::cast<ConstantInteger32>(
                                              Instr->getArg(0))->getValue()),
                    Instr->getArg(1), Instr->getArg(2));
     return;
   case Intrinsics::AtomicStore: {
-    if (!Intrinsics::VerifyMemoryOrder(
-            llvm::cast<ConstantInteger32>(Instr->getArg(2))->getValue())) {
+    if (!Intrinsics::isMemoryOrderValid(
+            ID, getConstantMemoryOrder(Instr->getArg(2)))) {
       Func->setError("Unexpected memory ordering for AtomicStore");
       return;
     }
     // We require the memory address to be naturally aligned.
     // Given that is the case, then normal stores are atomic.
     // Add a fence after the store to make it visible.
     Operand *Value = Instr->getArg(0);
     Operand *Ptr = Instr->getArg(1);
     if (Value->getType() == IceType_i64) {
       // Use a movq instead of what lowerStore() normally does
@@ skipping 1494 matching lines @@
 OperandX8632Mem *TargetX8632::FormMemoryOperand(Operand *Operand, Type Ty) {
   OperandX8632Mem *Mem = llvm::dyn_cast<OperandX8632Mem>(Operand);
   // It may be the case that address mode optimization already creates
   // an OperandX8632Mem, so in that case it wouldn't need another level
   // of transformation.
   if (!Mem) {
     Variable *Base = llvm::dyn_cast<Variable>(Operand);
     Constant *Offset = llvm::dyn_cast<Constant>(Operand);
     assert(Base || Offset);
     if (Offset) {
+      // Make sure Offset is not undef.
+      Offset = llvm::cast<Constant>(legalize(Offset));
       assert(llvm::isa<ConstantInteger32>(Offset) ||
              llvm::isa<ConstantRelocatable>(Offset));
     }
     Mem = OperandX8632Mem::create(Func, Ty, Base, Offset);
   }
   return llvm::cast<OperandX8632Mem>(legalize(Mem));
 }
 
 Variable *TargetX8632::makeReg(Type Type, int32_t RegNum) {
   // There aren't any 64-bit integer registers for x86-32.
@@ skipping 290 matching lines @@
   case FT_Asm:
   case FT_Iasm: {
     OstreamLocker L(Ctx);
     emitConstantPool<PoolTypeConverter<float>>(Ctx);
     emitConstantPool<PoolTypeConverter<double>>(Ctx);
   } break;
   }
 }
 
 } // end of namespace Ice