PNaCl: Impl the other atomicrmw operations: nand, max, min, umax, and umin.

lib/Transforms/NaCl/RewriteAtomics.cpp

 //===- RewriteAtomics.cpp - Stabilize instructions used for concurrency ---===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This pass encodes atomics, volatiles and fences using NaCl intrinsics
 // instead of LLVM's regular IR instructions.
 //
 // All of the above are transformed into one of the
 // @llvm.nacl.atomic.* intrinsics.
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/NaCl/PNaClSimplificationAnalyses.h"
+#include "llvm/CodeGen/AtomicExpandUtils.h"
+#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/InstVisitor.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/NaClAtomicIntrinsics.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/TargetRegistry.h"
 #include "llvm/Transforms/NaCl.h"
 #include <climits>
 #include <string>
 
 using namespace llvm;
 
 static cl::opt<bool> PNaClMemoryOrderSeqCstOnly(
     "pnacl-memory-order-seq-cst-only",
     cl::desc("PNaCl should upgrade all atomic memory orders to seq_cst"),
     cl::init(false));
 
 namespace {
 
-class RewriteAtomics : public ModulePass {
+class RewriteAtomicsPass {
 public:
-  static char ID; // Pass identification, replacement for typeid
-  RewriteAtomics() : ModulePass(ID) {
-    // This is a module pass because it may have to introduce
-    // intrinsic declarations into the module and modify a global function.
-    initializeRewriteAtomicsPass(*PassRegistry::getPassRegistry());
-  }
+  static StringRef name() { return "RewriteAtomicsPass"; }
 
-  virtual bool runOnModule(Module &M);
+  RewriteAtomicsPass() { }
+  RewriteAtomicsPass(RewriteAtomicsPass &&Rhs) { }
+  RewriteAtomicsPass &operator=(RewriteAtomicsPass &&Rhs) { return *this; }
+
+  PreservedAnalyses run(Function &F, AnalysisManager<Function> *AM);
 };
 
 template <class T> std::string ToStr(const T &V) {
   std::string S;
   raw_string_ostream OS(S);
   OS << const_cast<T &>(V);
   return OS.str();
 }
 
 class AtomicVisitor : public InstVisitor<AtomicVisitor> {
 public:
-  AtomicVisitor(Module &M, Pass &P)
-      : M(M), C(M.getContext()),
-        TD(M.getDataLayout()), AI(C),
-        ModifiedModule(false) {}
+  AtomicVisitor(Module &M)
+      : M(M), C(M.getContext()), TD(M.getDataLayout()), AI(C) {}
   ~AtomicVisitor() {}
-  bool modifiedModule() const { return ModifiedModule; }
+  bool modifiedFunction() const { return Modified; }
 
   void visitLoadInst(LoadInst &I);
   void visitStoreInst(StoreInst &I);
   void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
   void visitAtomicRMWInst(AtomicRMWInst &I);
   void visitFenceInst(FenceInst &I);
 
 private:
   Module &M;
   LLVMContext &C;
   const DataLayout TD;
   NaCl::AtomicIntrinsics AI;
-  bool ModifiedModule;
+  bool Modified = false;
 
   AtomicVisitor() = delete;
   AtomicVisitor(const AtomicVisitor &) = delete;
   AtomicVisitor &operator=(const AtomicVisitor &) = delete;
 
   /// Create an integer constant holding a NaCl::MemoryOrder that can be
   /// passed as an argument to one of the @llvm.nacl.atomic.*
   /// intrinsics. This function may strengthen the ordering initially
   /// specified by the instruction \p I for stability purpose.
   template <class Instruction>
   ConstantInt *freezeMemoryOrder(const Instruction &I, AtomicOrdering O) const;
   std::pair<ConstantInt *, ConstantInt *>
-  freezeMemoryOrder(const AtomicCmpXchgInst &I, AtomicOrdering S,
+  freezeMemoryOrder(const Instruction &I, AtomicOrdering S,
                     AtomicOrdering F) const;
 
   /// Sanity-check that instruction \p I which has pointer and value
   /// parameters have matching sizes \p BitSize for the type-pointed-to
   /// and the value's type \p T.
-  void checkSizeMatchesType(const Instruction &I, unsigned BitSize,
+  void checkSizeMatchesType(const Value &I, unsigned BitSize,
                             const Type *T) const;
 
   /// Verify that loads and stores are at least naturally aligned. Use
   /// byte alignment because converting to bits could truncate the
   /// value.
   void checkAlignment(const Instruction &I, unsigned ByteAlignment,
                       unsigned ByteSize) const;
 
   /// Create a cast before Instruction \p I from \p Src to \p Dst with \p Name.
   CastInst *createCast(Instruction &I, Value *Src, Type *Dst, Twine Name) const;
@@ skipping 12 matching lines @@
       Instruction &I, const NaCl::AtomicIntrinsics::AtomicIntrinsic *Intrinsic,
       Type *DstType, Type *OverloadedType, ArrayRef<Value *> Args);
 
   /// Most atomics instructions deal with at least one pointer, this
   /// struct automates some of this and has generic sanity checks.
   template <class Instruction> struct PointerHelper {
     Value *P;
     Type *OriginalPET;
     Type *PET;
     unsigned BitSize;
-    PointerHelper(const AtomicVisitor &AV, Instruction &I)
+    PointerHelper(const AtomicVisitor &AV, Instruction &I,
+                  IRBuilder<> *Builder = nullptr)
         : P(I.getPointerOperand()) {
       if (I.getPointerAddressSpace() != 0)
         report_fatal_error("unhandled pointer address space " +
                            Twine(I.getPointerAddressSpace()) + " for atomic: " +
                            ToStr(I));
       assert(P->getType()->isPointerTy() && "expected a pointer");
       PET = OriginalPET = P->getType()->getPointerElementType();
       BitSize = AV.TD.getTypeSizeInBits(OriginalPET);
       if (!OriginalPET->isIntegerTy()) {
         // The pointer wasn't to an integer type. We define atomics in
         // terms of integers, so bitcast the pointer to an integer of
         // the proper width.
         Type *IntNPtr = Type::getIntNPtrTy(AV.C, BitSize);
-        P = AV.createCast(I, P, IntNPtr, P->getName() + ".cast");
+        if(!Builder) {
+          P = AV.createCast(I, P, IntNPtr, P->getName() + ".cast");
+        } else {
+          P = Builder->CreateBitOrPointerCast(P, IntNPtr, P->getName() + ".cast");
+        }
         PET = P->getType()->getPointerElementType();
       }
       AV.checkSizeMatchesType(I, BitSize, PET);
     }
   };
 };
 }
 
-char RewriteAtomics::ID = 0;
-INITIALIZE_PASS(RewriteAtomics, "nacl-rewrite-atomics",
-                "rewrite atomics, volatiles and fences into stable "
-                "@llvm.nacl.atomics.* intrinsics",
-                false, false)
+static bool
+ExpandAtomicInstructions(Function &F, AtomicInfo &Info) {
+  bool Changed = false;
+  AtomicVisitor AV(*F.getParent());
 
-bool RewriteAtomics::runOnModule(Module &M) {
-  AtomicVisitor AV(M, *this);
-  AV.visit(M);
-  return AV.modifiedModule();
+  auto &CmpXchgs = Info.getCmpXchgs();
+  for (auto *CmpXchg : CmpXchgs) {
+    AV.visitAtomicCmpXchgInst(*CmpXchg);
+    Changed = true;
+  }
+
+  auto &Loads = Info.getLoads();
+  for (auto *Load : Loads) {
+    AV.visitLoadInst(*Load);
+    Changed = true;
+  }
+
+  auto &Stores = Info.getStores();
+  for (auto *Store : Stores) {
+    AV.visitStoreInst(*Store);
+    Changed = true;
+  }
+
+  auto &RMWs = Info.getRMWs();
+  for (auto *RMW : RMWs) {
+    AV.visitAtomicRMWInst(*RMW);
+    Changed = true;
+  }
+
+  auto &Fences = Info.getFences();
+  for (auto *Fence : Fences) {
+    AV.visitFenceInst(*Fence);
+    Changed = true;
+  }
+
+  return Changed;
+}
+
+PreservedAnalyses RewriteAtomicsPass::run(Function &F,
+                                          AnalysisManager<Function> *AM) {
+  auto &Info = AM->getResult<AtomicAnalysis>(F);
+
+  if (ExpandAtomicInstructions(F, Info)) {
+    return PreservedAnalyses::none();
+  } else {
+    return PreservedAnalyses::all();
+  }
 }
 
 template <class Instruction>
 ConstantInt *AtomicVisitor::freezeMemoryOrder(const Instruction &I,
                                               AtomicOrdering O) const {
   NaCl::MemoryOrder AO = NaCl::MemoryOrderInvalid;
 
   // TODO Volatile load/store are promoted to sequentially consistent
   //      for now. We could do something weaker.
   if (const LoadInst *L = dyn_cast<LoadInst>(&I)) {
     if (L->isVolatile())
       AO = NaCl::MemoryOrderSequentiallyConsistent;
   } else if (const StoreInst *S = dyn_cast<StoreInst>(&I)) {
     if (S->isVolatile())
       AO = NaCl::MemoryOrderSequentiallyConsistent;
   }
 
   if (AO == NaCl::MemoryOrderInvalid) {
     switch (O) {
-    case NotAtomic: llvm_unreachable("unexpected memory order");
+    case NotAtomic:
+      llvm_unreachable("unexpected memory order");
     // Monotonic is a strict superset of Unordered. Both can therefore
     // map to Relaxed ordering, which is in the C11/C++11 standard.
-    case Unordered: AO = NaCl::MemoryOrderRelaxed; break;
-    case Monotonic: AO = NaCl::MemoryOrderRelaxed; break;
+    case Unordered:
+      AO = NaCl::MemoryOrderRelaxed;
+      break;
+    case Monotonic:
+      AO = NaCl::MemoryOrderRelaxed;
+      break;
     // TODO Consume is currently unspecified by LLVM's internal IR.
-    case Acquire: AO = NaCl::MemoryOrderAcquire; break;
-    case Release: AO = NaCl::MemoryOrderRelease; break;
-    case AcquireRelease: AO = NaCl::MemoryOrderAcquireRelease; break;
+    case Acquire:
+      AO = NaCl::MemoryOrderAcquire;
+      break;
+    case Release:
+      AO = NaCl::MemoryOrderRelease;
+      break;
+    case AcquireRelease:
+      AO = NaCl::MemoryOrderAcquireRelease;
+      break;
     case SequentiallyConsistent:
-      AO = NaCl::MemoryOrderSequentiallyConsistent; break;
+      AO = NaCl::MemoryOrderSequentiallyConsistent;
+      break;
     }
   }
 
   // TODO For now only acquire/release/acq_rel/seq_cst are allowed.
   if (PNaClMemoryOrderSeqCstOnly || AO == NaCl::MemoryOrderRelaxed)
     AO = NaCl::MemoryOrderSequentiallyConsistent;
 
   return ConstantInt::get(Type::getInt32Ty(C), AO);
 }
 
 std::pair<ConstantInt *, ConstantInt *>
-AtomicVisitor::freezeMemoryOrder(const AtomicCmpXchgInst &I, AtomicOrdering S,
+AtomicVisitor::freezeMemoryOrder(const Instruction &I, AtomicOrdering S,
                                  AtomicOrdering F) const {
   if (S == Release || (S == AcquireRelease && F != Acquire))
     // According to C++11's [atomics.types.operations.req], cmpxchg with release
     // success memory ordering must have relaxed failure memory ordering, which
     // PNaCl currently disallows. The next-strongest ordering is acq_rel which
     // is also an invalid failure ordering, we therefore have to change the
     // success ordering to seq_cst, which can then fail as seq_cst.
     S = F = SequentiallyConsistent;
   if (F == Unordered || F == Monotonic) // Both are treated as relaxed.
     F = AtomicCmpXchgInst::getStrongestFailureOrdering(S);
   return std::make_pair(freezeMemoryOrder(I, S), freezeMemoryOrder(I, F));
 }
 
-void AtomicVisitor::checkSizeMatchesType(const Instruction &I, unsigned BitSize,
+void AtomicVisitor::checkSizeMatchesType(const Value &I, unsigned BitSize,
                                          const Type *T) const {
   Type *IntType = Type::getIntNTy(C, BitSize);
   if (IntType && T == IntType)
     return;
   report_fatal_error("unsupported atomic type " + ToStr(*T) + " of size " +
                      Twine(BitSize) + " bits in: " + ToStr(I));
 }
 
 void AtomicVisitor::checkAlignment(const Instruction &I, unsigned ByteAlignment,
                                    unsigned ByteSize) const {
@@ skipping 56 matching lines @@
         Success, 1, Name + ".insert.success", &I);
   } else if (!Call->getType()->isVoidTy() && DstType != OverloadedType) {
     // The call returns a value which needs to be cast to a non-integer.
     Res = createCast(I, Call, DstType, Name + ".cast");
     Res->setDebugLoc(I.getDebugLoc());
   }
 
   I.replaceAllUsesWith(Res);
   I.eraseFromParent();
   Call->setName(Name);
-  ModifiedModule = true;
+  Modified = true;
 }
 
 ///   %res = load {atomic|volatile} T* %ptr memory_order, align sizeof(T)
 /// becomes:
 ///   %res = call T @llvm.nacl.atomic.load.i<size>(%ptr, memory_order)
 void AtomicVisitor::visitLoadInst(LoadInst &I) {
   if (I.isSimple())
     return;
   PointerHelper<LoadInst> PH(*this, I);
   const NaCl::AtomicIntrinsics::AtomicIntrinsic *Intrinsic =
@@ skipping 29 matching lines @@
   replaceInstructionWithIntrinsicCall(I, Intrinsic, PH.OriginalPET, PH.PET,
                                       Args);
 }
 
 ///   %res = atomicrmw OP T* %ptr, T %val memory_order
 /// becomes:
 ///   %res = call T @llvm.nacl.atomic.rmw.i<size>(OP, %ptr, %val, memory_order)
 void AtomicVisitor::visitAtomicRMWInst(AtomicRMWInst &I) {
   NaCl::AtomicRMWOperation Op;
   switch (I.getOperation()) {
-  default: report_fatal_error("unsupported atomicrmw operation: " + ToStr(I));
-  case AtomicRMWInst::Add: Op = NaCl::AtomicAdd; break;
-  case AtomicRMWInst::Sub: Op = NaCl::AtomicSub; break;
-  case AtomicRMWInst::And: Op = NaCl::AtomicAnd; break;
-  case AtomicRMWInst::Or:  Op = NaCl::AtomicOr;  break;
-  case AtomicRMWInst::Xor: Op = NaCl::AtomicXor; break;
-  case AtomicRMWInst::Xchg: Op = NaCl::AtomicExchange; break;
+  default: {
+    auto Factory = [this, &I] (IRBuilder<> &Builder, Value *Addr,
+                               Value *Loaded, Value *NewVal,
+                               AtomicOrdering MemOpOrder,
+                               Value *&Success, Value *&NewLoaded) {
+      PointerHelper<AtomicRMWInst> PH(*this, I, &Builder);
+
+      const NaCl::AtomicIntrinsics::AtomicIntrinsic *Intrinsic =
+      findAtomicIntrinsic(I, Intrinsic::nacl_atomic_cmpxchg, PH.PET);
+
+      auto Order = freezeMemoryOrder(I, MemOpOrder, MemOpOrder);
+      Value *Args[] = {PH.P, Loaded, NewVal,
+                       Order.first, Order.second};
+
+      Function *F = Intrinsic->getDeclaration(&this->M);
+
+      Value* UnCastedValue = Builder.CreateCall(F, Args, "");
+      if(PH.P->getType() != Addr->getType()) {
+        NewLoaded = Builder.CreateBitOrPointerCast(UnCastedValue, Addr->getType(),
+                                                   "cast." + Addr->getName());
+      } else {
+        NewLoaded = UnCastedValue;
+      }
+      Success = Builder.CreateICmp(CmpInst::ICMP_EQ, NewLoaded, Loaded, "success");
+    };
+    Modified = expandAtomicRMWToCmpXchg(&I, Factory) || Modified;
+    return;
+  }
+  case AtomicRMWInst::Add:
+    Op = NaCl::AtomicAdd;
+    break;
+  case AtomicRMWInst::Sub:
+    Op = NaCl::AtomicSub;
+    break;
+  case AtomicRMWInst::And:
+    Op = NaCl::AtomicAnd;
+    break;
+  case AtomicRMWInst::Or:
+    Op = NaCl::AtomicOr;
+    break;
+  case AtomicRMWInst::Xor:
+    Op = NaCl::AtomicXor;
+    break;
+  case AtomicRMWInst::Xchg:
+    Op = NaCl::AtomicExchange;
+    break;
   }
   PointerHelper<AtomicRMWInst> PH(*this, I);
   const NaCl::AtomicIntrinsics::AtomicIntrinsic *Intrinsic =
       findAtomicIntrinsic(I, Intrinsic::nacl_atomic_rmw, PH.PET);
   checkSizeMatchesType(I, PH.BitSize, I.getValOperand()->getType());
   Value *Args[] = {ConstantInt::get(Type::getInt32Ty(C), Op), PH.P,
                    I.getValOperand(), freezeMemoryOrder(I, I.getOrdering())};
   replaceInstructionWithIntrinsicCall(I, Intrinsic, PH.OriginalPET, PH.PET,
                                       Args);
 }
@@ skipping 52 matching lines @@
     replaceInstructionWithIntrinsicCall(I, Intrinsic, T, T,
                                         ArrayRef<Value *>());
   } else {
     const NaCl::AtomicIntrinsics::AtomicIntrinsic *Intrinsic =
         findAtomicIntrinsic(I, Intrinsic::nacl_atomic_fence, T);
     Value *Args[] = {freezeMemoryOrder(I, I.getOrdering())};
     replaceInstructionWithIntrinsicCall(I, Intrinsic, T, T, Args);
   }
 }
 
-ModulePass *llvm::createRewriteAtomicsPass() { return new RewriteAtomics(); }
+/// Wrapper for the legacy pass manager.
+class RewriteAtomics : public FunctionPass {
+public:
+  static char ID; // Pass identification, replacement for typeid
+  RewriteAtomics() : FunctionPass(ID) {
+    initializeRewriteAtomicsPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnFunction(Function &F) override {
+    auto &Info = getAnalysis<AtomicAnalysisWrapperPass>().getInfo();
+    return ExpandAtomicInstructions(F, Info);
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<AtomicAnalysisWrapperPass>();
+  }
+};
+char RewriteAtomics::ID = 0;
+INITIALIZE_PASS_BEGIN(RewriteAtomics, "nacl-rewrite-atomics",
+                      "rewrite atomics, volatiles and fences into stable "
+                      "@llvm.nacl.atomics.* intrinsics",
+                      false, false)
+INITIALIZE_PASS_DEPENDENCY(AtomicAnalysisWrapperPass);
+INITIALIZE_PASS_END(RewriteAtomics, "nacl-rewrite-atomics",
+                    "rewrite atomics, volatiles and fences into stable "
+                    "@llvm.nacl.atomics.* intrinsics",
+                    false, false)
+
+FunctionPass *llvm::createRewriteAtomicsPass() { return new RewriteAtomics(); }