PNaCl: Add support for GCC/LLVM vector extensions

lib/Transforms/NaCl/FixVectorLoadStoreAlignment.cpp

+//===- FixVectorLoadStoreAlignment.cpp - Vector load/store alignment ------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Replace all vector load/store instructions by loads/stores of each
+// individual element since different architectures have different
+// faults on unaligned memory access. This pass pessimizes all vector
+// memory accesses. It's expected that backends with more liberal
+// alignment restrictions recognize this pattern and reconstruct the
+// original vector load/store.
+//
+// Volatile load/store are broken up as allowed by C/C++, and atomic
+// accesses cause errors at compile-time.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Transforms/NaCl.h"
+
+using namespace llvm;
+
+namespace {
+class FixVectorLoadStoreAlignment : public BasicBlockPass {
+public:
+  static char ID; // Pass identification, replacement for typeid
+  FixVectorLoadStoreAlignment() : BasicBlockPass(ID), M(0), TD(0) {
+    initializeFixVectorLoadStoreAlignmentPass(
+        *PassRegistry::getPassRegistry());
+  }
+  virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+    AU.addRequired<DataLayout>();
+    BasicBlockPass::getAnalysisUsage(AU);
+  }
+  virtual bool doInitialization(Module &Mod) {
+    M = &Mod;
+    return false; // Unchanged.
+  }
+  virtual bool runOnBasicBlock(BasicBlock &BB);
+
+private:
+  typedef SmallVector<Instruction *, 8> Instructions;
+  const Module *M;
+  const DataLayout *TD;

[jvoung (off chromium), 2014/04/04 23:20:21]

DL instead of TD

+
+  /// Some sub-classes of Instruction have a non-virtual function
+  /// indicating which operand is the pointer operand. This template
+  /// function returns the pointer operand's type, and required that

[Jim Stichnoth, 2014/04/04 23:08:17]

required --> requires

[JF, 2014/04/15 01:52:27]

Done.

+  /// InstTy have a getPointerOperand function.
+  template <typename InstTy>
+  static PointerType *pointerOperandType(const InstTy *I) {
+    return cast<PointerType>(I->getPointerOperand()->getType());
+  }
+
+  /// Similar to pointerOperandType, this template function checks
+  /// whether the pointer operand is a pointer to a vector type.
+  template <typename InstTy>
+  static bool pointerOperandIsVectorPointer(const Instruction *I) {
+    return pointerOperandType(cast<InstTy>(I))->getElementType()->isVectorTy();
+  }
+
+  /// Returns true if one of the Instruction's operands is a pointer to
+  /// a vector type. This is more general than the above and assumes we
+  /// don't know which Instruction type is provided.
+  static bool hasVectorPointerOperand(const Instruction *I) {
+    for (User::const_op_iterator IB = I->op_begin(), IE = I->op_end(); IB != IE;
+         ++IB)
+      if (PointerType *PtrTy = dyn_cast<PointerType>((*IB)->getType()))
+        if (isa<VectorType>(PtrTy->getElementType()))
+          return true;
+    return false;
+  }
+
+  void findVectorLoadStore(const BasicBlock &BB, Instructions &Loads,
+                           Instructions &Stores) const;
+  void fixVectorLoadStoreAlignment(BasicBlock &BB, const Instructions &Loads,
+                                   const Instructions &Stores) const;
+};
+}

[Jim Stichnoth, 2014/04/04 23:08:17]

I prefer end-of-namespace comments (here and elsewhere), but it's your call

[JF, 2014/04/15 01:52:27]

Done here and in other files.

+
+char FixVectorLoadStoreAlignment::ID = 0;
+INITIALIZE_PASS(FixVectorLoadStoreAlignment, "fix-vector-load-store-alignment",
+                "Replace vector load/store by loads/stores of each element",
+                false, false)
+
+void FixVectorLoadStoreAlignment::findVectorLoadStore(
+    const BasicBlock &BB, Instructions &Loads, Instructions &Stores) const {
+  for (BasicBlock::const_iterator BBI = BB.begin(), BBE = BB.end(); BBI != BBE;
+       ++BBI) {
+    Instruction *I = const_cast<Instruction *>(&*BBI);
+    switch (I->getOpcode()) {
+    case Instruction::Load:
+      if (pointerOperandIsVectorPointer<LoadInst>(I)) {
+        if (cast<LoadInst>(I)->isAtomic())
+          report_fatal_error("unhandled: atomic vector store");
+        Loads.push_back(I);
+      }
+      break;
+    case Instruction::Store:
+      if (pointerOperandIsVectorPointer<StoreInst>(I)) {
+        if (cast<StoreInst>(I)->isAtomic())
+          report_fatal_error("unhandled: atomic vector store");
+        Stores.push_back(I);
+      }
+      break;
+    case Instruction::Alloca:
+    case Instruction::GetElementPtr:

[jvoung (off chromium), 2014/04/04 23:20:21]

GetElementPtr is just arithmetic so doesn't touch memory?

[JF, 2014/04/15 01:52:27]

Hmm, I'm not sure why I had this there, I'll remove it.

+    case Instruction::Fence:
+    case Instruction::VAArg:

[jvoung (off chromium), 2014/04/04 23:20:21]

Any more detail on why these are left as-is?

For "fence" you may not be able to tell if they are related to vector types at
all (e.g., fence just has the memory order and no other operands).

I guess alloca is just reserving stack space, so it doesn't matter.

For va_arg, the element might be loaded from the stack... but there's the NaCl
pass that gets rid of / transforms va_arg instructions into something that
explicitly uses the stack so that would handle this?

Actually, expand va args is run before this in the pass list, so we shouldn't
see VAArg?

[JF, 2014/04/15 01:52:27]

I added a comment, it's based on mayReadOrWriteMemory.

Fence shouldn't be in mayReadOrWriteMemory's list, but it is, so I have to
include it too. It's indeed unrelated to vectors, so I leave it as-is.

Yes, if it's generally helpful to handle va_args I can add it, but I don't think
it is in this case.

+      // Leave these memory operations as-is, even when they deal with
+      // vectors.
+      break;
+    case Instruction::Call:
+    case Instruction::Invoke:
+      // Call/invoke don't touch memory per-se, leave them as-is.
+      break;
+    case Instruction::AtomicCmpXchg:
+      if (pointerOperandIsVectorPointer<AtomicCmpXchgInst>(I))
+        report_fatal_error(
+            "unhandled: atomic compare and exchange operation on vector");
+      break;
+    case Instruction::AtomicRMW:
+      if (pointerOperandIsVectorPointer<AtomicRMWInst>(I))
+        report_fatal_error("unhandled: atomic RMW operation on vector");
+      break;
+    default:
+      if (I->mayReadOrWriteMemory() && hasVectorPointerOperand(I))
+        report_fatal_error(

[jvoung (off chromium), 2014/04/04 23:20:21]

Print the instruction for (a bit more) developer friendliness?

[JF, 2014/04/15 01:52:27]

Done.

+            "unexpected: vector operations which may read/write memory");
+      break;
+    }
+  }
+}
+
+void FixVectorLoadStoreAlignment::fixVectorLoadStoreAlignment(
+    BasicBlock &BB, const Instructions &Loads,
+    const Instructions &Stores) const {
+  for (Instructions::const_iterator IB = Loads.begin(), IE = Loads.end();
+       IB != IE; ++IB) {
+    LoadInst *VecLoad = cast<LoadInst>(*IB);

[jvoung (off chromium), 2014/04/04 23:20:21]

copydebug?

[JF, 2014/04/15 01:52:27]

Same as before on IRBuilder.

+    VectorType *LoadedVecTy =
+        cast<VectorType>(pointerOperandType(VecLoad)->getElementType());
+    Type *ElemTy = LoadedVecTy->getElementType();
+
+    // The base of the vector is as aligned as the vector load (where
+    // zero means ABI alignment for the vector), whereas subsequent
+    // elements are as aligned as the base+offset can be.
+    unsigned BaseAlign = VecLoad->getAlignment()
+                             ? VecLoad->getAlignment()
+                             : TD->getABITypeAlignment(LoadedVecTy);
+    unsigned ElemAllocSize = TD->getTypeAllocSize(ElemTy);
+
+    // Fill in the vector element by element.
+    IRBuilder<> IRB(VecLoad);
+    Value *Loaded = UndefValue::get(LoadedVecTy);
+    Value *Base =
+        IRB.CreateBitCast(VecLoad->getPointerOperand(), ElemTy->getPointerTo());
+
+    for (unsigned Elem = 0, NumElems = LoadedVecTy->getNumElements();
+         Elem != NumElems; ++Elem) {
+      unsigned Align = MinAlign(BaseAlign, ElemAllocSize * Elem);
+      Value *GEP = IRB.CreateConstInBoundsGEP1_32(Base, Elem);
+      LoadInst *LoadedElem =
+          IRB.CreateAlignedLoad(GEP, Align, VecLoad->isVolatile());
+      LoadedElem->setSynchScope(VecLoad->getSynchScope());
+      Loaded = IRB.CreateInsertElement(
+          Loaded, LoadedElem,
+          ConstantInt::get(Type::getInt32Ty(M->getContext()), Elem));
+    }
+
+    VecLoad->replaceAllUsesWith(Loaded);
+    VecLoad->eraseFromParent();
+  }
+
+  for (Instructions::const_iterator IB = Stores.begin(), IE = Stores.end();
+       IB != IE; ++IB) {
+    StoreInst *VecStore = cast<StoreInst>(*IB);
+    Value *StoredVec = VecStore->getValueOperand();
+    VectorType *StoredVecTy = cast<VectorType>(StoredVec->getType());
+    Type *ElemTy = StoredVecTy->getElementType();
+
+    unsigned BaseAlign = VecStore->getAlignment()
+                             ? VecStore->getAlignment()
+                             : TD->getABITypeAlignment(StoredVecTy);
+    unsigned ElemAllocSize = TD->getTypeAllocSize(ElemTy);
+
+    // Fill in the vector element by element.
+    IRBuilder<> IRB(VecStore);
+    Value *Base =
+        IRB.CreateBitCast(VecStore->getPointerOperand(), ElemTy->getPointerTo());

[Jim Stichnoth, 2014/04/04 23:08:17]

line longer than 80 chars

[JF, 2014/04/15 01:52:27]

Done.

+
+    for (unsigned Elem = 0, NumElems = StoredVecTy->getNumElements();
+         Elem != NumElems; ++Elem) {
+      unsigned Align = MinAlign(BaseAlign, ElemAllocSize * Elem);
+      Value *GEP = IRB.CreateConstInBoundsGEP1_32(Base, Elem);
+      Value *ElemToStore = IRB.CreateExtractElement(
+          StoredVec, ConstantInt::get(Type::getInt32Ty(M->getContext()), Elem));
+      StoreInst *StoredElem = IRB.CreateAlignedStore(ElemToStore, GEP, Align,
+                                                     VecStore->isVolatile());
+      StoredElem->setSynchScope(VecStore->getSynchScope());
+    }
+
+    VecStore->eraseFromParent();
+  }
+}
+
+bool FixVectorLoadStoreAlignment::runOnBasicBlock(BasicBlock &BB) {
+  bool Changed = false;
+  if (!TD)
+    TD = &getAnalysis<DataLayout>();
+  Instructions Loads;
+  Instructions Stores;
+  findVectorLoadStore(BB, Loads, Stores);
+  if (!(Loads.empty() && Stores.empty())) {
+    Changed = true;
+    fixVectorLoadStoreAlignment(BB, Loads, Stores);
+  }
+  return Changed;
+}
+
+BasicBlockPass *llvm::createFixVectorLoadStoreAlignmentPass() {
+  return new FixVectorLoadStoreAlignment();
+}