Add the new @llvm.nacl.atomic.fence.all intrinsic

lib/Transforms/NaCl/RewriteAtomics.cpp

 //===- RewriteAtomics.cpp - Stabilize instructions used for concurrency ---===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This pass encodes atomics, volatiles and fences using NaCl intrinsics
 // instead of LLVM's regular IR instructions.
 //
 // All of the above are transformed into one of the
 // @llvm.nacl.atomic.* intrinsics.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/Twine.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/NaClAtomicIntrinsics.h"
 #include "llvm/InstVisitor.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/NaCl.h"
 #include <climits>
@@ skipping 203 matching lines @@
     Res = createCast(I, Call, DstType, Name + ".cast");
     Res->setDebugLoc(I.getDebugLoc());
   }
   Call->setDebugLoc(I.getDebugLoc());
   I.replaceAllUsesWith(Res);
   I.eraseFromParent();
   Call->setName(Name);
   ModifiedModule = true;
 }
 
-/// %res = load {atomic|volatile} T* %ptr memory_order, align sizeof(T)
+///   %res = load {atomic|volatile} T* %ptr memory_order, align sizeof(T)
 /// becomes:
-/// %res = call T @llvm.nacl.atomic.load.i<size>(%ptr, memory_order)
+///   %res = call T @llvm.nacl.atomic.load.i<size>(%ptr, memory_order)
 void AtomicVisitor::visitLoadInst(LoadInst &I) {
   if (I.isSimple())
     return;
   PointerHelper<LoadInst> PH(*this, I);
   checkAlignment(I, I.getAlignment(), PH.BitSize / CHAR_BIT);
   Value *Args[] = { PH.P, freezeMemoryOrder(I) };
   replaceInstructionWithIntrinsicCall(I, Intrinsic::nacl_atomic_load,
                                       PH.OriginalPET, PH.PET, Args);
 }
 
-/// store {atomic|volatile} T %val, T* %ptr memory_order, align sizeof(T)
+///   store {atomic|volatile} T %val, T* %ptr memory_order, align sizeof(T)
 /// becomes:
-/// call void @llvm.nacl.atomic.store.i<size>(%val, %ptr, memory_order)
+///   call void @llvm.nacl.atomic.store.i<size>(%val, %ptr, memory_order)
 void AtomicVisitor::visitStoreInst(StoreInst &I) {
   if (I.isSimple())
     return;
   PointerHelper<StoreInst> PH(*this, I);
   checkAlignment(I, I.getAlignment(), PH.BitSize / CHAR_BIT);
   Value *V = I.getValueOperand();
   if (!V->getType()->isIntegerTy()) {
     // The store isn't of an integer type. We define atomics in terms of
     // integers, so bitcast the value to store to an integer of the
     // proper width.
     CastInst *Cast = createCast(I, V, Type::getIntNTy(C, PH.BitSize),
                                 V->getName() + ".cast");
     Cast->setDebugLoc(I.getDebugLoc());
     V = Cast;
   }
   checkSizeMatchesType(I, PH.BitSize, V->getType());
   Value *Args[] = { V, PH.P, freezeMemoryOrder(I) };
   replaceInstructionWithIntrinsicCall(I, Intrinsic::nacl_atomic_store,
                                       PH.OriginalPET, PH.PET, Args);
 }
 
-/// %res = atomicrmw OP T* %ptr, T %val memory_order
+///   %res = atomicrmw OP T* %ptr, T %val memory_order
 /// becomes:
-/// %res = call T @llvm.nacl.atomic.rmw.i<size>(OP, %ptr, %val, memory_order)
+///   %res = call T @llvm.nacl.atomic.rmw.i<size>(OP, %ptr, %val, memory_order)
 void AtomicVisitor::visitAtomicRMWInst(AtomicRMWInst &I) {
   NaCl::AtomicRMWOperation Op;
   switch (I.getOperation()) {
   default: report_fatal_error("unsupported atomicrmw operation: " + ToStr(I));
   case AtomicRMWInst::Add: Op = NaCl::AtomicAdd; break;
   case AtomicRMWInst::Sub: Op = NaCl::AtomicSub; break;
   case AtomicRMWInst::And: Op = NaCl::AtomicAnd; break;
   case AtomicRMWInst::Or:  Op = NaCl::AtomicOr;  break;
   case AtomicRMWInst::Xor: Op = NaCl::AtomicXor; break;
   case AtomicRMWInst::Xchg: Op = NaCl::AtomicExchange; break;
   }
   PointerHelper<AtomicRMWInst> PH(*this, I);
   checkSizeMatchesType(I, PH.BitSize, I.getValOperand()->getType());
   Value *Args[] = { ConstantInt::get(Type::getInt32Ty(C), Op), PH.P,
                     I.getValOperand(), freezeMemoryOrder(I) };
   replaceInstructionWithIntrinsicCall(I, Intrinsic::nacl_atomic_rmw,
                                       PH.OriginalPET, PH.PET, Args);
 }
 
-/// %res = cmpxchg T* %ptr, T %old, T %new memory_order
+///   %res = cmpxchg T* %ptr, T %old, T %new memory_order
 /// becomes:
-/// %res = call T @llvm.nacl.atomic.cmpxchg.i<size>(
-///     %object, %expected, %desired, memory_order_success,
-///     memory_order_failure)
+///   %res = call T @llvm.nacl.atomic.cmpxchg.i<size>(
+///       %object, %expected, %desired, memory_order_success,
+///       memory_order_failure)
 void AtomicVisitor::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
   PointerHelper<AtomicCmpXchgInst> PH(*this, I);
   checkSizeMatchesType(I, PH.BitSize, I.getCompareOperand()->getType());
   checkSizeMatchesType(I, PH.BitSize, I.getNewValOperand()->getType());
   // TODO LLVM currently doesn't support specifying separate memory
   //      orders for compare exchange's success and failure cases: LLVM
   //      IR implicitly drops the Release part of the specified memory
   //      order on failure.
   Value *Args[] = { PH.P, I.getCompareOperand(), I.getNewValOperand(),
                     freezeMemoryOrder(I), freezeMemoryOrder(I) };
   replaceInstructionWithIntrinsicCall(I, Intrinsic::nacl_atomic_cmpxchg,
                                       PH.OriginalPET, PH.PET, Args);
 }
 
-/// fence memory_order
+///   fence memory_order
 /// becomes:
-/// call void @llvm.nacl.atomic.fence(memory_order)
+///   call void @llvm.nacl.atomic.fence(memory_order)
+/// and
+///   call void asm sideeffect "", "~{memory}"()
+///   fence seq_cst
+///   call void asm sideeffect "", "~{memory}"()
+/// becomes:
+///   call void asm sideeffect "", "~{memory}"()
+///   call void @llvm.nacl.atomic.fence.all()
+///   call void asm sideeffect "", "~{memory}"()
+/// Note that the assembly gets eliminated by the -remove-asm-memory pass.
 void AtomicVisitor::visitFenceInst(FenceInst &I) {
   Type *T = Type::getInt32Ty(C); // Fences aren't overloaded on type.
-  Value *Args[] = { freezeMemoryOrder(I) };
-  replaceInstructionWithIntrinsicCall(I, Intrinsic::nacl_atomic_fence, T, T,
-                                      Args);
+  BasicBlock::InstListType &IL(I.getParent()->getInstList());
+  bool isFirst = IL.empty() || &*I.getParent()->getInstList().begin() == &I;
+  bool isLast = IL.empty() || &*I.getParent()->getInstList().rbegin() == &I;
+  CallInst *PrevC = isFirst ? 0 : dyn_cast<CallInst>(I.getPrevNode());
+  CallInst *NextC = isLast ? 0 : dyn_cast<CallInst>(I.getNextNode());
+
+  if ((PrevC && PrevC->isInlineAsm() &&
+       cast<InlineAsm>(PrevC->getCalledValue())->isAsmMemory()) &&
+      (NextC && NextC->isInlineAsm() &&
+       cast<InlineAsm>(NextC->getCalledValue())->isAsmMemory()) &&
+      I.getOrdering() == SequentiallyConsistent) {
+    replaceInstructionWithIntrinsicCall(I, Intrinsic::nacl_atomic_fence_all, T,
+                                        T, ArrayRef<Value *>());
+  } else {
+    Value *Args[] = { freezeMemoryOrder(I) };
+    replaceInstructionWithIntrinsicCall(I, Intrinsic::nacl_atomic_fence, T, T,
+                                        Args);
+  }
 }
 
 ModulePass *llvm::createRewriteAtomicsPass() { return new RewriteAtomics(); }