PNaCl SIMD: allow element-aligned vector load/store

lib/Transforms/NaCl/FixVectorLoadStoreAlignment.cpp

 //===- FixVectorLoadStoreAlignment.cpp - Vector load/store alignment ------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
-// Replace all vector load/store instructions by loads/stores of each
-// individual element since different architectures have different
-// faults on unaligned memory access. This pass pessimizes all vector
-// memory accesses. It's expected that backends with more liberal
-// alignment restrictions recognize this pattern and reconstruct the
-// original vector load/store.
+// Fix vector load/store alignment by:
+// - Leaving as-is if the alignment is equal to the vector's element width.
+// - Reducing the alignment to vector's element width if it's greater and the
+//   current alignment is a factor of the element alignment.
+// - Scalarizing if the alignment is smaller than the element-wise alignment.
 //
-// Volatile load/store are broken up as allowed by C/C++, and atomic
-// accesses cause errors at compile-time.
+// Volatile vector load/store are handled the same, and can therefore be broken
+// up as allowed by C/C++.
+//
+// TODO(jfb) Atomic accesses cause errors at compile-time. This could be
+//           implemented as a call to the C++ runtime, since 128-bit atomics
+//           aren't usually lock-free.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Debug.h"
@@ skipping 45 matching lines @@
   /// don't know which Instruction type is provided.
   static bool hasVectorPointerOperand(const Instruction *I) {
     for (User::const_op_iterator IB = I->op_begin(), IE = I->op_end(); IB != IE;
          ++IB)
       if (PointerType *PtrTy = dyn_cast<PointerType>((*IB)->getType()))
         if (isa<VectorType>(PtrTy->getElementType()))
           return true;
     return false;
   }
 
-  void findVectorLoadStore(const BasicBlock &BB, Instructions &Loads,
-                           Instructions &Stores) const;
-  void fixVectorLoadStoreAlignment(BasicBlock &BB, const Instructions &Loads,
-                                   const Instructions &Stores) const;
+  /// Vectors are expected to be element-aligned. If they are, leave as-is; if
+  /// the alignment is too much then narrow the alignment (when possible);
+  /// otherwise return false.
+  template <typename InstTy>
+  static bool tryFixVectorAlignment(const DataLayout *DL, Instruction *I) {
+    InstTy *LoadStore = cast<InstTy>(I);
+    VectorType *VecTy =
+        cast<VectorType>(pointerOperandType(LoadStore)->getElementType());
+    Type *ElemTy = VecTy->getElementType();
+    uint64_t ElemBitSize = DL->getTypeSizeInBits(ElemTy);
+    uint64_t ElemByteSize = ElemBitSize / CHAR_BIT;
+    uint64_t CurrentByteAlign = LoadStore->getAlignment();
+    bool isABIAligned = CurrentByteAlign == 0;
+    uint64_t VecABIByteAlign = DL->getABITypeAlignment(VecTy);
+    CurrentByteAlign = isABIAligned ? VecABIByteAlign : CurrentByteAlign;
+
+    if (CHAR_BIT * ElemByteSize != ElemBitSize)
+      return false; // Minimum byte-size elements.
+    if (MinAlign(ElemByteSize, CurrentByteAlign) == ElemByteSize) {
+      // Element-aligned, or compatible over-aligned. Keep element-aligned.
+      LoadStore->setAlignment(ElemByteSize);
+      return true;
+    }
+    return false; // Under-aligned.
+  }
+
+  void visitVectorLoadStore(BasicBlock &BB, Instructions &Loads,
+                            Instructions &Stores) const;
+  void scalarizeVectorLoadStore(BasicBlock &BB, const Instructions &Loads,
+                                const Instructions &Stores) const;
 };
 } // anonymous namespace
 
 char FixVectorLoadStoreAlignment::ID = 0;
 INITIALIZE_PASS(FixVectorLoadStoreAlignment, "fix-vector-load-store-alignment",
-                "Replace vector load/store by loads/stores of each element",
+                "Ensure vector load/store have element-size alignment",
                 false, false)
 
-void FixVectorLoadStoreAlignment::findVectorLoadStore(
-    const BasicBlock &BB, Instructions &Loads, Instructions &Stores) const {
-  for (BasicBlock::const_iterator BBI = BB.begin(), BBE = BB.end(); BBI != BBE;
+void FixVectorLoadStoreAlignment::visitVectorLoadStore(
+    BasicBlock &BB, Instructions &Loads, Instructions &Stores) const {
+  for (BasicBlock::iterator BBI = BB.begin(), BBE = BB.end(); BBI != BBE;
        ++BBI) {
-    Instruction *I = const_cast<Instruction *>(&*BBI);
+    Instruction *I = &*BBI;
     // The following list of instructions is based on mayReadOrWriteMemory.
     switch (I->getOpcode()) {
     case Instruction::Load:
       if (pointerOperandIsVectorPointer<LoadInst>(I)) {
         if (cast<LoadInst>(I)->isAtomic())
           report_fatal_error("unhandled: atomic vector store");
-        Loads.push_back(I);
+        if (!tryFixVectorAlignment<LoadInst>(DL, I))
+          Loads.push_back(I);
       }
       break;
     case Instruction::Store:
       if (pointerOperandIsVectorPointer<StoreInst>(I)) {
         if (cast<StoreInst>(I)->isAtomic())
           report_fatal_error("unhandled: atomic vector store");
-        Stores.push_back(I);
+        if (!tryFixVectorAlignment<StoreInst>(DL, I))
+          Stores.push_back(I);
       }
       break;
     case Instruction::Alloca:
     case Instruction::Fence:
     case Instruction::VAArg:
       // Leave these memory operations as-is, even when they deal with
       // vectors.
       break;
     case Instruction::Call:
     case Instruction::Invoke:
@@ skipping 12 matching lines @@
       if (I->mayReadOrWriteMemory() && hasVectorPointerOperand(I)) {
         errs() << "Not handled: " << *I << '\n';
         report_fatal_error(
             "unexpected: vector operations which may read/write memory");
       }
       break;
     }
   }
 }
 
-void FixVectorLoadStoreAlignment::fixVectorLoadStoreAlignment(
+void FixVectorLoadStoreAlignment::scalarizeVectorLoadStore(
     BasicBlock &BB, const Instructions &Loads,
     const Instructions &Stores) const {
   for (Instructions::const_iterator IB = Loads.begin(), IE = Loads.end();
        IB != IE; ++IB) {
     LoadInst *VecLoad = cast<LoadInst>(*IB);
     VectorType *LoadedVecTy =
         cast<VectorType>(pointerOperandType(VecLoad)->getElementType());
     Type *ElemTy = LoadedVecTy->getElementType();
 
     // The base of the vector is as aligned as the vector load (where
@@ skipping 57 matching lines @@
     VecStore->eraseFromParent();
   }
 }
 
 bool FixVectorLoadStoreAlignment::runOnBasicBlock(BasicBlock &BB) {
   bool Changed = false;
   if (!DL)
     DL = &getAnalysis<DataLayout>();
   Instructions Loads;
   Instructions Stores;
-  findVectorLoadStore(BB, Loads, Stores);
+  visitVectorLoadStore(BB, Loads, Stores);
   if (!(Loads.empty() && Stores.empty())) {
     Changed = true;
-    fixVectorLoadStoreAlignment(BB, Loads, Stores);
+    scalarizeVectorLoadStore(BB, Loads, Stores);
   }
   return Changed;
 }
 
 BasicBlockPass *llvm::createFixVectorLoadStoreAlignmentPass() {
   return new FixVectorLoadStoreAlignment();
 }