Subzero. Enables (most) crosstests for ARM32.

src/IceTargetLoweringARM32.cpp

 //===- subzero/src/IceTargetLoweringARM32.cpp - ARM32 lowering ------------===//
 //
 //                        The Subzero Code Generator
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 ///
 /// \file
 /// This file implements the TargetLoweringARM32 class, which consists almost
 /// entirely of the lowering sequence for each high-level instruction.
 ///
 //===----------------------------------------------------------------------===//
 #include "IceTargetLoweringARM32.h"
 
 #include "IceCfg.h"
 #include "IceCfgNode.h"
 #include "IceClFlags.h"
 #include "IceDefs.h"
 #include "IceELFObjectWriter.h"
 #include "IceGlobalInits.h"
+#include "IceInstARM32.def"
 #include "IceInstARM32.h"
 #include "IceLiveness.h"
 #include "IceOperand.h"
 #include "IcePhiLoweringImpl.h"
 #include "IceRegistersARM32.h"
 #include "IceTargetLoweringARM32.def"
 #include "IceUtils.h"
 #include "llvm/Support/MathExtras.h"
 
 #include <algorithm>
+#include <utility>
 
 namespace Ice {
 
 namespace {
 
 // UnimplementedError is defined as a macro so that we can get actual line
 // numbers.
 #define UnimplementedError(Flags)                                              \
   do {                                                                         \
     if (!static_cast<const ClFlags &>(Flags).getSkipUnimplemented()) {         \
@@ skipping 330 matching lines @@
           isFP32, isFP64, isVec128, alias_init)                                \
   name,
       REGARM32_TABLE
 #undef X
   };
 
   return RegNames[RegNum];
 }
 
 Variable *TargetARM32::getPhysicalRegister(SizeT RegNum, Type Ty) {
-  if (Ty == IceType_void)
-    Ty = IceType_i32;
+  static const Type DefaultType[] = {
+#define X(val, encode, name, scratch, preserved, stackptr, frameptr, isInt,    \
+          isFP32, isFP64, isVec128, alias_init)                                \
+  (isFP32)                                                                     \
+      ? IceType_f32                                                            \
+      : ((isFP64) ? IceType_f64 : ((isVec128 ? IceType_v4i32 : IceType_i32))),
+      REGARM32_TABLE
+#undef X
+  };
+
+  assert(RegNum < RegARM32::Reg_NUM);
+  if (Ty == IceType_void) {
+    assert(RegNum < llvm::array_lengthof(DefaultType));
+    Ty = DefaultType[RegNum];
+  }
   if (PhysicalRegisters[Ty].empty())
     PhysicalRegisters[Ty].resize(RegARM32::Reg_NUM);
   assert(RegNum < PhysicalRegisters[Ty].size());
   Variable *Reg = PhysicalRegisters[Ty][RegNum];
   if (Reg == nullptr) {
     Reg = Func->makeVariable(Ty);
     Reg->setRegNum(RegNum);
     PhysicalRegisters[Ty][RegNum] = Reg;
     // Specially mark a named physical register as an "argument" so that it is
     // considered live upon function entry.  Otherwise it's possible to get
@@ skipping 23 matching lines @@
     llvm::report_fatal_error(
         "Infinite-weight Variable has no register assigned");
   }
   int32_t Offset = Var->getStackOffset();
   int32_t BaseRegNum = Var->getBaseRegNum();
   if (BaseRegNum == Variable::NoRegister) {
     BaseRegNum = getFrameOrStackReg();
     if (!hasFramePointer())
       Offset += getStackAdjustment();
   }
-  if (!isLegalVariableStackOffset(Offset)) {
+  const Type VarTy = Var->getType();
+  // In general, no Variable64On32 should be emited in textual asm output. It
+  // turns out that some lowering sequences Fake-Def/Fake-Use such a variables.
+  // If they end up being assigned an illegal offset we get a runtime error. We
+  // liberally allow Variable64On32 to have illegal offsets because offsets
+  // don't matter in FakeDefs/FakeUses.
+  if (!llvm::isa<Variable64On32>(Var) &&
+      !isLegalVariableStackOffset(VarTy, Offset)) {
     llvm::report_fatal_error("Illegal stack offset");
   }
-  const Type FrameSPTy = stackSlotType();
-  Str << "[" << getRegName(BaseRegNum, FrameSPTy);
+  Str << "[" << getRegName(BaseRegNum, VarTy);
   if (Offset != 0) {
     Str << ", " << getConstantPrefix() << Offset;
   }
   Str << "]";
 }
 
 bool TargetARM32::CallingConv::I64InRegs(std::pair<int32_t, int32_t> *Regs) {
   if (NumGPRRegsUsed >= ARM32_MAX_GPR_ARG)
     return false;
   int32_t RegLo, RegHi;
@@ skipping 142 matching lines @@
     return;
   }
   Type Ty = Arg->getType();
   InArgsSizeBytes = applyStackAlignmentTy(InArgsSizeBytes, Ty);
   Arg->setStackOffset(BasicFrameOffset + InArgsSizeBytes);
   InArgsSizeBytes += typeWidthInBytesOnStack(Ty);
   // If the argument variable has been assigned a register, we need to load the
   // value from the stack slot.
   if (Arg->hasReg()) {
     assert(Ty != IceType_i64);
-    OperandARM32Mem *Mem = OperandARM32Mem::create(
+    // This should be simple, just load the parameter off the stack using a nice
+    // sp + imm addressing mode. Because ARM, we can't do that (e.g., VLDR, for
+    // fp types, cannot have an index register), so we legalize the memory
+    // operand instead.
+    auto *Mem = OperandARM32Mem::create(
         Func, Ty, FramePtr, llvm::cast<ConstantInteger32>(
                                 Ctx->getConstantInt32(Arg->getStackOffset())));
-    if (isVectorType(Arg->getType())) {
-      // Use vld1.$elem or something?
-      UnimplementedError(Func->getContext()->getFlags());
-    } else if (isFloatingType(Arg->getType())) {
-      _vldr(Arg, Mem);
-    } else {
-      _ldr(Arg, Mem);
-    }
+    legalizeToReg(Mem, Arg->getRegNum());
     // This argument-copying instruction uses an explicit OperandARM32Mem
     // operand instead of a Variable, so its fill-from-stack operation has to
     // be tracked separately for statistics.
     Ctx->statsUpdateFills();
   }
 }
 
 Type TargetARM32::stackSlotType() { return IceType_i32; }
 
 void TargetARM32::addProlog(CfgNode *Node) {
@@ skipping 271 matching lines @@
   Variable *RetValue = nullptr;
   if (RI->getSrcSize())
     RetValue = llvm::cast<Variable>(RI->getSrc(0));
   _bundle_lock();
   _bic(LR, LR, RetMask);
   _ret(LR, RetValue);
   _bundle_unlock();
   RI->setDeleted();
 }
 
-bool TargetARM32::isLegalVariableStackOffset(int32_t Offset) const {
+bool TargetARM32::isLegalVariableStackOffset(Type Ty, int32_t Offset) const {
   constexpr bool SignExt = false;
-  // TODO(jvoung): vldr of FP stack slots has a different limit from the plain
-  // stackSlotType().
-  return OperandARM32Mem::canHoldOffset(stackSlotType(), SignExt, Offset);
+  return OperandARM32Mem::canHoldOffset(Ty, SignExt, Offset);
 }
 
 StackVariable *TargetARM32::legalizeVariableSlot(Variable *Var,
+                                                 int32_t StackAdjust,
                                                  Variable *OrigBaseReg) {
-  int32_t Offset = Var->getStackOffset();
+  int32_t Offset = Var->getStackOffset() + StackAdjust;
   // Legalize will likely need a movw/movt combination, but if the top bits are
   // all 0 from negating the offset and subtracting, we could use that instead.
   bool ShouldSub = (-Offset & 0xFFFF0000) == 0;
   if (ShouldSub)
     Offset = -Offset;
   Operand *OffsetVal = legalize(Ctx->getConstantInt32(Offset),
                                 Legal_Reg | Legal_Flex, getReservedTmpReg());
   Variable *ScratchReg = makeReg(IceType_i32, getReservedTmpReg());
   if (ShouldSub)
     _sub(ScratchReg, OrigBaseReg, OffsetVal);
@@ skipping 13 matching lines @@
   // to:
   //   movw/movt TMP, OFF_PART
   //   add TMP, TMP, SP
   //   ldr X, OFF_MORE[TMP]
   //
   // This is safe because we have reserved TMP, and add for ARM does not
   // clobber the flags register.
   Func->dump("Before legalizeStackSlots");
   assert(hasComputedFrame());
   // Early exit, if SpillAreaSizeBytes is really small.
-  if (isLegalVariableStackOffset(SpillAreaSizeBytes))
+  // TODO(jpp): this is not safe -- loads and stores of q registers can't have
+  // offsets.
+  if (isLegalVariableStackOffset(IceType_v4i32, SpillAreaSizeBytes))
     return;
   Variable *OrigBaseReg = getPhysicalRegister(getFrameOrStackReg());
   int32_t StackAdjust = 0;
   // Do a fairly naive greedy clustering for now. Pick the first stack slot
   // that's out of bounds and make a new base reg using the architecture's temp
   // register. If that works for the next slot, then great. Otherwise, create a
   // new base register, clobbering the previous base register. Never share a
   // base reg across different basic blocks. This isn't ideal if local and
   // multi-block variables are far apart and their references are interspersed.
   // It may help to be more coordinated about assign stack slot numbers and may
@@ skipping 20 matching lines @@
           StackAdjust += AdjInst->getAmount();
           NewBaseOffset += AdjInst->getAmount();
           continue;
         }
         if (llvm::isa<InstARM32Call>(CurInstr)) {
           NewBaseOffset -= StackAdjust;
           StackAdjust = 0;
           continue;
         }
       }
+
       // For now, only Mov instructions can have stack variables. We need to
       // know the type of instruction because we currently create a fresh one
       // to replace Dest/Source, rather than mutate in place.
-      auto *MovInst = llvm::dyn_cast<InstARM32Mov>(CurInstr);
-      if (!MovInst) {
+      bool MayNeedOffsetRewrite = false;
+      if (auto *MovInstr = llvm::dyn_cast<InstARM32Mov>(CurInstr)) {
+        MayNeedOffsetRewrite =
+            !MovInstr->isMultiDest() && !MovInstr->isMultiSource();
+      }
+
+      if (!MayNeedOffsetRewrite) {
         continue;
       }
+
+      assert(Dest != nullptr);
+      Type DestTy = Dest->getType();
+      assert(DestTy != IceType_i64);
       if (!Dest->hasReg()) {
         int32_t Offset = Dest->getStackOffset();
         Offset += StackAdjust;
-        if (!isLegalVariableStackOffset(Offset)) {
+        if (!isLegalVariableStackOffset(DestTy, Offset)) {
           if (NewBaseReg) {
             int32_t OffsetDiff = Offset - NewBaseOffset;
-            if (isLegalVariableStackOffset(OffsetDiff)) {
+            if (isLegalVariableStackOffset(DestTy, OffsetDiff)) {
               StackVariable *NewDest =
                   Func->makeVariable<StackVariable>(stackSlotType());
               NewDest->setMustNotHaveReg();
               NewDest->setBaseRegNum(NewBaseReg->getBaseRegNum());
               NewDest->setStackOffset(OffsetDiff);
               Variable *NewDestVar = NewDest;
-              _mov(NewDestVar, MovInst->getSrc(0));
-              MovInst->setDeleted();
+              _mov(NewDestVar, CurInstr->getSrc(0));
+              CurInstr->setDeleted();
               continue;
             }
           }
-          StackVariable *LegalDest = legalizeVariableSlot(Dest, OrigBaseReg);
+          StackVariable *LegalDest =
+              legalizeVariableSlot(Dest, StackAdjust, OrigBaseReg);
           assert(LegalDest != Dest);
           Variable *LegalDestVar = LegalDest;
-          _mov(LegalDestVar, MovInst->getSrc(0));
-          MovInst->setDeleted();
+          _mov(LegalDestVar, CurInstr->getSrc(0));
+          CurInstr->setDeleted();
           NewBaseReg = LegalDest;
           NewBaseOffset = Offset;
           continue;
         }
       }
-      assert(MovInst->getSrcSize() == 1);
-      Variable *Var = llvm::dyn_cast<Variable>(MovInst->getSrc(0));
+      assert(CurInstr->getSrcSize() == 1);
+      Variable *Var = llvm::dyn_cast<Variable>(CurInstr->getSrc(0));
       if (Var && !Var->hasReg()) {
+        Type VarTy = Var->getType();
         int32_t Offset = Var->getStackOffset();
         Offset += StackAdjust;
-        if (!isLegalVariableStackOffset(Offset)) {
+        if (!isLegalVariableStackOffset(VarTy, Offset)) {
           if (NewBaseReg) {
             int32_t OffsetDiff = Offset - NewBaseOffset;
-            if (isLegalVariableStackOffset(OffsetDiff)) {
+            if (isLegalVariableStackOffset(VarTy, OffsetDiff)) {
               StackVariable *NewVar =
                   Func->makeVariable<StackVariable>(stackSlotType());
               NewVar->setMustNotHaveReg();
               NewVar->setBaseRegNum(NewBaseReg->getBaseRegNum());
               NewVar->setStackOffset(OffsetDiff);
               _mov(Dest, NewVar);
-              MovInst->setDeleted();
+              CurInstr->setDeleted();
               continue;
             }
           }
-          StackVariable *LegalVar = legalizeVariableSlot(Var, OrigBaseReg);
+          StackVariable *LegalVar =
+              legalizeVariableSlot(Var, StackAdjust, OrigBaseReg);
           assert(LegalVar != Var);
           _mov(Dest, LegalVar);
-          MovInst->setDeleted();
+          CurInstr->setDeleted();
           NewBaseReg = LegalVar;
           NewBaseOffset = Offset;
           continue;
         }
       }
     }
   }
 }
 
 Operand *TargetARM32::loOperand(Operand *Operand) {
@@ skipping 371 matching lines @@
       _mul(T_Acc, Src0RLo, Src1RHi);
       _mla(T_Acc1, Src1RLo, Src0RHi, T_Acc);
       _umull(T_Lo, T_Hi1, Src0RLo, Src1RLo);
       _add(T_Hi, T_Hi1, T_Acc1);
       _mov(DestLo, T_Lo);
       _mov(DestHi, T_Hi);
       return;
     }
     case InstArithmetic::Shl: {
       // a=b<<c ==>
+      // pnacl-llc does:
+      // mov     t_b.lo, b.lo
+      // mov     t_b.hi, b.hi
+      // mov     t_c.lo, c.lo
+      // rsb     T0, t_c.lo, #32
+      // lsr     T1, t_b.lo, T0
+      // orr     t_a.hi, T1, t_b.hi, lsl t_c.lo
+      // sub     T2, t_c.lo, #32
+      // cmp     T2, #0
+      // lslge   t_a.hi, t_b.lo, T2
+      // lsl     t_a.lo, t_b.lo, t_c.lo
+      // mov     a.lo, t_a.lo
+      // mov     a.hi, t_a.hi
+      //
       // GCC 4.8 does:
       // sub t_c1, c.lo, #32
       // lsl t_hi, b.hi, c.lo
       // orr t_hi, t_hi, b.lo, lsl t_c1
       // rsb t_c2, c.lo, #32
       // orr t_hi, t_hi, b.lo, lsr t_c2
       // lsl t_lo, b.lo, c.lo
       // a.lo = t_lo
       // a.hi = t_hi
+      //
+      // These are incompatible, therefore we mimic pnacl-llc.
       // Can be strength-reduced for constant-shifts, but we don't do that for
       // now.
       // Given the sub/rsb T_C, C.lo, #32, one of the T_C will be negative. On
       // ARM, shifts only take the lower 8 bits of the shift register, and
       // saturate to the range 0-32, so the negative value will saturate to 32.
-      Variable *T_Hi = makeReg(IceType_i32);
+      Constant *_32 = Ctx->getConstantInt32(32);
+      Constant *_0 = Ctx->getConstantZero(IceType_i32);
       Variable *Src1RLo = legalizeToReg(Src1Lo);
-      Constant *ThirtyTwo = Ctx->getConstantInt32(32);
-      Variable *T_C1 = makeReg(IceType_i32);
-      Variable *T_C2 = makeReg(IceType_i32);
-      _sub(T_C1, Src1RLo, ThirtyTwo);
-      _lsl(T_Hi, Src0RHi, Src1RLo);
-      _orr(T_Hi, T_Hi, OperandARM32FlexReg::create(Func, IceType_i32, Src0RLo,
-                                                   OperandARM32::LSL, T_C1));
-      _rsb(T_C2, Src1RLo, ThirtyTwo);
-      _orr(T_Hi, T_Hi, OperandARM32FlexReg::create(Func, IceType_i32, Src0RLo,
-                                                   OperandARM32::LSR, T_C2));
-      _mov(DestHi, T_Hi);
-      Variable *T_Lo = makeReg(IceType_i32);
-      // _mov seems to sometimes have better register preferencing than lsl.
-      // Otherwise mov w/ lsl shifted register is a pseudo-instruction that
-      // maps to lsl.
-      _mov(T_Lo, OperandARM32FlexReg::create(Func, IceType_i32, Src0RLo,
-                                             OperandARM32::LSL, Src1RLo));
-      _mov(DestLo, T_Lo);
+      Variable *T0 = makeReg(IceType_i32);
+      Variable *T1 = makeReg(IceType_i32);
+      Variable *T2 = makeReg(IceType_i32);
+      Variable *TA_Hi = makeReg(IceType_i32);
+      Variable *TA_Lo = makeReg(IceType_i32);
+      _rsb(T0, Src1RLo, _32);
+      _lsr(T1, Src0RLo, T0);
+      _orr(TA_Hi, T1, OperandARM32FlexReg::create(Func, IceType_i32, Src0RHi,
+                                                  OperandARM32::LSL, Src1RLo));
+      _sub(T2, Src1RLo, _32);
+      _cmp(T2, _0);
+      _lsl(TA_Hi, Src0RLo, T2, CondARM32::GE);
+      _set_dest_nonkillable();
+      _lsl(TA_Lo, Src0RLo, Src1RLo);
+      _mov(DestLo, TA_Lo);
+      _mov(DestHi, TA_Hi);
       return;
     }
     case InstArithmetic::Lshr:
-    // a=b>>c (unsigned) ==>
-    // GCC 4.8 does:
-    // rsb t_c1, c.lo, #32
-    // lsr t_lo, b.lo, c.lo
-    // orr t_lo, t_lo, b.hi, lsl t_c1
-    // sub t_c2, c.lo, #32
-    // orr t_lo, t_lo, b.hi, lsr t_c2
-    // lsr t_hi, b.hi, c.lo
-    // a.lo = t_lo
-    // a.hi = t_hi
     case InstArithmetic::Ashr: {
-      // a=b>>c (signed) ==> ...
-      // Ashr is similar, but the sub t_c2, c.lo, #32 should set flags, and the
-      // next orr should be conditioned on PLUS. The last two right shifts
-      // should also be arithmetic.
-      bool IsAshr = Inst->getOp() == InstArithmetic::Ashr;
-      Variable *T_Lo = makeReg(IceType_i32);
+      // a=b>>c
+      // pnacl-llc does:
+      // mov        t_b.lo, b.lo
+      // mov        t_b.hi, b.hi
+      // mov        t_c.lo, c.lo
+      // lsr        T0, t_b.lo, t_c.lo
+      // rsb        T1, t_c.lo, #32
+      // orr        t_a.lo, T0, t_b.hi, lsl T1
+      // sub        T2, t_c.lo, #32
+      // cmp        T2, #0
+      // [al]srge   t_a.lo, t_b.hi, T2
+      // [al]sr     t_a.hi, t_b.hi, t_c.lo
+      // mov        a.lo, t_a.lo
+      // mov        a.hi, t_a.hi
+      //
+      // GCC 4.8 does (lsr):
+      // rsb        t_c1, c.lo, #32
+      // lsr        t_lo, b.lo, c.lo
+      // orr        t_lo, t_lo, b.hi, lsl t_c1
+      // sub        t_c2, c.lo, #32
+      // orr        t_lo, t_lo, b.hi, lsr t_c2
+      // lsr        t_hi, b.hi, c.lo
+      // mov        a.lo, t_lo
+      // mov        a.hi, t_hi
+      //
+      // These are incompatible, therefore we mimic pnacl-llc.
+      const bool IsAshr = Inst->getOp() == InstArithmetic::Ashr;
+      Constant *_32 = Ctx->getConstantInt32(32);
+      Constant *_0 = Ctx->getConstantZero(IceType_i32);
       Variable *Src1RLo = legalizeToReg(Src1Lo);
-      Constant *ThirtyTwo = Ctx->getConstantInt32(32);
-      Variable *T_C1 = makeReg(IceType_i32);
-      Variable *T_C2 = makeReg(IceType_i32);
-      _rsb(T_C1, Src1RLo, ThirtyTwo);
-      _lsr(T_Lo, Src0RLo, Src1RLo);
-      _orr(T_Lo, T_Lo, OperandARM32FlexReg::create(Func, IceType_i32, Src0RHi,
-                                                   OperandARM32::LSL, T_C1));
-      OperandARM32::ShiftKind RShiftKind;
-      CondARM32::Cond Pred;
+      Variable *T0 = makeReg(IceType_i32);
+      Variable *T1 = makeReg(IceType_i32);
+      Variable *T2 = makeReg(IceType_i32);
+      Variable *TA_Lo = makeReg(IceType_i32);
+      Variable *TA_Hi = makeReg(IceType_i32);
+      _lsr(T0, Src0RLo, Src1RLo);
+      _rsb(T1, Src1RLo, _32);
+      _orr(TA_Lo, T0, OperandARM32FlexReg::create(Func, IceType_i32, Src0RHi,
+                                                  OperandARM32::LSL, T1));
+      _sub(T2, Src1RLo, _32);
+      _cmp(T2, _0);
       if (IsAshr) {
-        _subs(T_C2, Src1RLo, ThirtyTwo);
-        RShiftKind = OperandARM32::ASR;
-        Pred = CondARM32::PL;
+        _asr(TA_Lo, Src0RHi, T2, CondARM32::GE);
+        _set_dest_nonkillable();
+        _asr(TA_Hi, Src0RHi, Src1RLo);
       } else {
-        _sub(T_C2, Src1RLo, ThirtyTwo);
-        RShiftKind = OperandARM32::LSR;
-        Pred = CondARM32::AL;
+        _lsr(TA_Lo, Src0RHi, T2, CondARM32::GE);
+        _set_dest_nonkillable();
+        _lsr(TA_Hi, Src0RHi, Src1RLo);
       }
-      _orr(T_Lo, T_Lo, OperandARM32FlexReg::create(Func, IceType_i32, Src0RHi,
-                                                   RShiftKind, T_C2),
-           Pred);
-      _mov(DestLo, T_Lo);
-      Variable *T_Hi = makeReg(IceType_i32);
-      _mov(T_Hi, OperandARM32FlexReg::create(Func, IceType_i32, Src0RHi,
-                                             RShiftKind, Src1RLo));
-      _mov(DestHi, T_Hi);
+      _mov(DestLo, TA_Lo);
+      _mov(DestHi, TA_Hi);
       return;
     }
     case InstArithmetic::Fadd:
     case InstArithmetic::Fsub:
     case InstArithmetic::Fmul:
     case InstArithmetic::Fdiv:
     case InstArithmetic::Frem:
       llvm_unreachable("FP instruction with i64 type");
       return;
     case InstArithmetic::Udiv:
     case InstArithmetic::Sdiv:
     case InstArithmetic::Urem:
     case InstArithmetic::Srem:
       llvm_unreachable("Call-helper-involved instruction for i64 type "
                        "should have already been handled before");
       return;
     }
     return;
   } else if (isVectorType(Dest->getType())) {
+    // Add a fake def to keep liveness consistent in the meantime.
+    Variable *T = makeReg(Dest->getType());
+    Context.insert(InstFakeDef::create(Func, T));
+    _mov(Dest, T);
     UnimplementedError(Func->getContext()->getFlags());
-    // Add a fake def to keep liveness consistent in the meantime.
-    Context.insert(InstFakeDef::create(Func, Dest));
     return;
   }
   // Dest->getType() is a non-i64 scalar.
   Variable *Src0R = legalizeToReg(Src0);
   Variable *T = makeReg(Dest->getType());
   // Handle div/rem separately. They require a non-legalized Src1 to inspect
   // whether or not Src1 is a non-zero constant. Once legalized it is more
   // difficult to determine (constant may be moved to a register).
   switch (Inst->getOp()) {
   default:
@@ skipping 35 matching lines @@
   }
 
   // Handle floating point arithmetic separately: they require Src1 to be
   // legalized to a register.
   switch (Inst->getOp()) {
   default:
     break;
   case InstArithmetic::Fadd: {
     Variable *Src1R = legalizeToReg(Src1);
     _vadd(T, Src0R, Src1R);
-    _vmov(Dest, T);
+    _mov(Dest, T);
     return;
   }
   case InstArithmetic::Fsub: {
     Variable *Src1R = legalizeToReg(Src1);
     _vsub(T, Src0R, Src1R);
-    _vmov(Dest, T);
+    _mov(Dest, T);
     return;
   }
   case InstArithmetic::Fmul: {
     Variable *Src1R = legalizeToReg(Src1);
     _vmul(T, Src0R, Src1R);
-    _vmov(Dest, T);
+    _mov(Dest, T);
     return;
   }
   case InstArithmetic::Fdiv: {
     Variable *Src1R = legalizeToReg(Src1);
     _vdiv(T, Src0R, Src1R);
-    _vmov(Dest, T);
+    _mov(Dest, T);
     return;
   }
   }
 
   Operand *Src1RF = legalize(Src1, Legal_Reg | Legal_Flex);
   switch (Inst->getOp()) {
   case InstArithmetic::_num:
     llvm_unreachable("Unknown arithmetic operator");
     return;
   case InstArithmetic::Add:
@@ skipping 53 matching lines @@
 void TargetARM32::lowerAssign(const InstAssign *Inst) {
   Variable *Dest = Inst->getDest();
   Operand *Src0 = Inst->getSrc(0);
   assert(Dest->getType() == Src0->getType());
   if (Dest->getType() == IceType_i64) {
     Src0 = legalizeUndef(Src0);
     Operand *Src0Lo = legalize(loOperand(Src0), Legal_Reg | Legal_Flex);
     Operand *Src0Hi = legalize(hiOperand(Src0), Legal_Reg | Legal_Flex);
     Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
     Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
-    Variable *T_Lo = nullptr, *T_Hi = nullptr;
+    Variable *T_Lo = makeReg(IceType_i32);
+    Variable *T_Hi = makeReg(IceType_i32);
     _mov(T_Lo, Src0Lo);
     _mov(DestLo, T_Lo);
     _mov(T_Hi, Src0Hi);
     _mov(DestHi, T_Hi);
   } else {
     Operand *NewSrc;
     if (Dest->hasReg()) {
       // If Dest already has a physical register, then legalize the Src operand
       // into a Variable with the same register assignment. This especially
       // helps allow the use of Flex operands.
       NewSrc = legalize(Src0, Legal_Reg | Legal_Flex, Dest->getRegNum());
     } else {
       // Dest could be a stack operand. Since we could potentially need to do a
       // Store (and store can only have Register operands), legalize this to a
       // register.
       NewSrc = legalize(Src0, Legal_Reg);
     }
     if (isVectorType(Dest->getType())) {
-      UnimplementedError(Func->getContext()->getFlags());
+      Variable *SrcR = legalizeToReg(NewSrc);
+      _mov(Dest, SrcR);
     } else if (isFloatingType(Dest->getType())) {
       Variable *SrcR = legalizeToReg(NewSrc);
-      _vmov(Dest, SrcR);
+      _mov(Dest, SrcR);
     } else {
       _mov(Dest, NewSrc);
     }
   }
 }
 
 void TargetARM32::lowerBr(const InstBr *Inst) {
   if (Inst->isUnconditional()) {
     _br(Inst->getTargetUnconditional());
     return;
@@ skipping 49 matching lines @@
       if (CC.I32InReg(&Reg)) {
         InRegs = true;
         GPRArgs.push_back(std::make_pair(Arg, Reg));
       }
     }
 
     if (!InRegs) {
       ParameterAreaSizeBytes =
           applyStackAlignmentTy(ParameterAreaSizeBytes, Ty);
       StackArgs.push_back(std::make_pair(Arg, ParameterAreaSizeBytes));
-      ParameterAreaSizeBytes += typeWidthInBytesOnStack(Arg->getType());
+      ParameterAreaSizeBytes += typeWidthInBytesOnStack(Ty);
     }
   }
 
   // Adjust the parameter area so that the stack is aligned. It is assumed that
   // the stack is already aligned at the start of the calling sequence.
   ParameterAreaSizeBytes = applyStackAlignment(ParameterAreaSizeBytes);
 
   // Subtract the appropriate amount for the argument area. This also takes
   // care of setting the stack adjustment during emission.
   //
@@ skipping 19 matching lines @@
       Addr = OperandARM32Mem::create(Func, Ty, SP, Loc);
     } else {
       Variable *NewBase = Func->makeVariable(SP->getType());
       lowerArithmetic(
           InstArithmetic::create(Func, InstArithmetic::Add, NewBase, SP, Loc));
       Addr = formMemoryOperand(NewBase, Ty);
     }
     lowerStore(InstStore::create(Func, StackArg.first, Addr));
   }
 
-  // Copy arguments to be passed in registers to the appropriate registers.
-  for (auto &GPRArg : GPRArgs) {
-    Variable *Reg = legalizeToReg(GPRArg.first, GPRArg.second);
-    // Generate a FakeUse of register arguments so that they do not get dead
-    // code eliminated as a result of the FakeKill of scratch registers after
-    // the call.
-    Context.insert(InstFakeUse::create(Func, Reg));
-  }
-  for (auto &FPArg : FPArgs) {
-    Variable *Reg = legalizeToReg(FPArg.first, FPArg.second);
-    Context.insert(InstFakeUse::create(Func, Reg));
-  }
-
   // Generate the call instruction. Assign its result to a temporary with high
   // register allocation weight.
   Variable *Dest = Instr->getDest();
   // ReturnReg doubles as ReturnRegLo as necessary.
   Variable *ReturnReg = nullptr;
   Variable *ReturnRegHi = nullptr;
   if (Dest) {
     switch (Dest->getType()) {
     case IceType_NUM:
       llvm_unreachable("Invalid Call dest type");
@@ skipping 30 matching lines @@
   Operand *CallTarget = Instr->getCallTarget();
   // TODO(jvoung): Handle sandboxing. const bool NeedSandboxing =
   // Ctx->getFlags().getUseSandboxing();
 
   // Allow ConstantRelocatable to be left alone as a direct call, but force
   // other constants like ConstantInteger32 to be in a register and make it an
   // indirect call.
   if (!llvm::isa<ConstantRelocatable>(CallTarget)) {
     CallTarget = legalize(CallTarget, Legal_Reg);
   }
+
+  // Copy arguments to be passed in registers to the appropriate registers.
+  for (auto &FPArg : FPArgs) {
+    Variable *Reg = legalizeToReg(FPArg.first, FPArg.second);
+    Context.insert(InstFakeUse::create(Func, Reg));
+  }
+  for (auto &GPRArg : GPRArgs) {
+    Variable *Reg = legalizeToReg(GPRArg.first, GPRArg.second);
+    // Generate a FakeUse of register arguments so that they do not get dead
+    // code eliminated as a result of the FakeKill of scratch registers after
+    // the call.
+    Context.insert(InstFakeUse::create(Func, Reg));
+  }
   Inst *NewCall = InstARM32Call::create(Func, ReturnReg, CallTarget);
   Context.insert(NewCall);
   if (ReturnRegHi)
     Context.insert(InstFakeDef::create(Func, ReturnRegHi));
 
   // Add the appropriate offset to SP. The call instruction takes care of
   // resetting the stack offset during emission.
   if (ParameterAreaSizeBytes) {
     Operand *AddAmount = legalize(Ctx->getConstantInt32(ParameterAreaSizeBytes),
                                   Legal_Reg | Legal_Flex);
@@ skipping 16 matching lines @@
   // Assign the result of the call to Dest.
   if (ReturnReg) {
     if (ReturnRegHi) {
       auto *Dest64On32 = llvm::cast<Variable64On32>(Dest);
       Variable *DestLo = Dest64On32->getLo();
       Variable *DestHi = Dest64On32->getHi();
       _mov(DestLo, ReturnReg);
       _mov(DestHi, ReturnRegHi);
     } else {
       if (isFloatingType(Dest->getType()) || isVectorType(Dest->getType())) {
-        _vmov(Dest, ReturnReg);
+        _mov(Dest, ReturnReg);
       } else {
         assert(isIntegerType(Dest->getType()) &&
                typeWidthInBytes(Dest->getType()) <= 4);
         _mov(Dest, ReturnReg);
       }
     }
   }
 }
 
+namespace {
+void forceHiLoInReg(Variable64On32 *Var) {
+  Var->getHi()->setMustHaveReg();
+  Var->getLo()->setMustHaveReg();
+}
+} // end of anonymous namespace
+
 void TargetARM32::lowerCast(const InstCast *Inst) {
   InstCast::OpKind CastKind = Inst->getCastKind();
   Variable *Dest = Inst->getDest();
   Operand *Src0 = legalizeUndef(Inst->getSrc(0));
   switch (CastKind) {
   default:
     Func->setError("Cast type not supported");
     return;
   case InstCast::Sext: {
     if (isVectorType(Dest->getType())) {
+      Variable *T = makeReg(Dest->getType());
+      Context.insert(InstFakeDef::create(Func, T, legalizeToReg(Src0)));
+      _mov(Dest, T);
       UnimplementedError(Func->getContext()->getFlags());
     } else if (Dest->getType() == IceType_i64) {
       // t1=sxtb src; t2= mov t1 asr #31; dst.lo=t1; dst.hi=t2
       Constant *ShiftAmt = Ctx->getConstantInt32(31);
       Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
       Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
       Variable *T_Lo = makeReg(DestLo->getType());
       if (Src0->getType() == IceType_i32) {
         Operand *Src0RF = legalize(Src0, Legal_Reg | Legal_Flex);
         _mov(T_Lo, Src0RF);
@@ skipping 30 matching lines @@
       // t1 = sxt src; dst = t1
       Variable *Src0R = legalizeToReg(Src0);
       Variable *T = makeReg(Dest->getType());
       _sxt(T, Src0R);
       _mov(Dest, T);
     }
     break;
   }
   case InstCast::Zext: {
     if (isVectorType(Dest->getType())) {
+      Variable *T = makeReg(Dest->getType());
+      Context.insert(InstFakeDef::create(Func, T, legalizeToReg(Src0)));
+      _mov(Dest, T);
       UnimplementedError(Func->getContext()->getFlags());
     } else if (Dest->getType() == IceType_i64) {
       // t1=uxtb src; dst.lo=t1; dst.hi=0
       Constant *Zero = Ctx->getConstantZero(IceType_i32);
       Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
       Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
       Variable *T_Lo = makeReg(DestLo->getType());
       // i32 and i1 can just take up the whole register. i32 doesn't need uxt,
       // while i1 will have an and mask later anyway.
       if (Src0->getType() == IceType_i32 || Src0->getType() == IceType_i1) {
@@ skipping 26 matching lines @@
       // t1 = uxt src; dst = t1
       Variable *Src0R = legalizeToReg(Src0);
       Variable *T = makeReg(Dest->getType());
       _uxt(T, Src0R);
       _mov(Dest, T);
     }
     break;
   }
   case InstCast::Trunc: {
     if (isVectorType(Dest->getType())) {
+      Variable *T = makeReg(Dest->getType());
+      Context.insert(InstFakeDef::create(Func, T, legalizeToReg(Src0)));
+      _mov(Dest, T);
       UnimplementedError(Func->getContext()->getFlags());
     } else {
       if (Src0->getType() == IceType_i64)
         Src0 = loOperand(Src0);
       Operand *Src0RF = legalize(Src0, Legal_Reg | Legal_Flex);
       // t1 = trunc Src0RF; Dest = t1
       Variable *T = makeReg(Dest->getType());
       _mov(T, Src0RF);
       if (Dest->getType() == IceType_i1)
         _and(T, T, Ctx->getConstantInt1(1));
       _mov(Dest, T);
     }
     break;
   }
   case InstCast::Fptrunc:
   case InstCast::Fpext: {
     // fptrunc: dest.f32 = fptrunc src0.fp64
     // fpext: dest.f64 = fptrunc src0.fp32
     const bool IsTrunc = CastKind == InstCast::Fptrunc;
     if (isVectorType(Dest->getType())) {
+      Variable *T = makeReg(Dest->getType());
+      Context.insert(InstFakeDef::create(Func, T, legalizeToReg(Src0)));
+      _mov(Dest, T);
       UnimplementedError(Func->getContext()->getFlags());
       break;
     }
     assert(Dest->getType() == (IsTrunc ? IceType_f32 : IceType_f64));
     assert(Src0->getType() == (IsTrunc ? IceType_f64 : IceType_f32));
     Variable *Src0R = legalizeToReg(Src0);
     Variable *T = makeReg(Dest->getType());
     _vcvt(T, Src0R, IsTrunc ? InstARM32Vcvt::D2s : InstARM32Vcvt::S2d);
     _mov(Dest, T);
     break;
   }
   case InstCast::Fptosi:
   case InstCast::Fptoui: {
+    if (isVectorType(Dest->getType())) {
+      Variable *T = makeReg(Dest->getType());
+      Context.insert(InstFakeDef::create(Func, T, legalizeToReg(Src0)));
+      _mov(Dest, T);
+      UnimplementedError(Func->getContext()->getFlags());
+      break;
+    }
+
+    const bool DestIsSigned = CastKind == InstCast::Fptosi;
+    const bool Src0IsF32 = isFloat32Asserting32Or64(Src0->getType());
+    if (llvm::isa<Variable64On32>(Dest)) {
+      const char *HelperName =
+          Src0IsF32 ? (DestIsSigned ? H_fptosi_f32_i64 : H_fptoui_f32_i64)
+                    : (DestIsSigned ? H_fptosi_f64_i64 : H_fptoui_f64_i64);
+      static constexpr SizeT MaxSrcs = 1;
+      InstCall *Call = makeHelperCall(HelperName, Dest, MaxSrcs);
+      Call->addArg(Src0);
+      lowerCall(Call);
+      break;
+    }
     // fptosi:
     //     t1.fp = vcvt src0.fp
     //     t2.i32 = vmov t1.fp
     //     dest.int = conv t2.i32     @ Truncates the result if needed.
     // fptoui:
     //     t1.fp = vcvt src0.fp
     //     t2.u32 = vmov t1.fp
     //     dest.uint = conv t2.u32    @ Truncates the result if needed.
-    if (isVectorType(Dest->getType())) {
-      UnimplementedError(Func->getContext()->getFlags());
-      break;
-    }
-    if (auto *Dest64On32 = llvm::dyn_cast<Variable64On32>(Dest)) {
-      Context.insert(InstFakeDef::create(Func, Dest64On32->getLo()));
-      Context.insert(InstFakeDef::create(Func, Dest64On32->getHi()));
-      UnimplementedError(Func->getContext()->getFlags());
-      break;
-    }
-    const bool DestIsSigned = CastKind == InstCast::Fptosi;
     Variable *Src0R = legalizeToReg(Src0);
     Variable *T_fp = makeReg(IceType_f32);
-    if (isFloat32Asserting32Or64(Src0->getType())) {
-      _vcvt(T_fp, Src0R,
-            DestIsSigned ? InstARM32Vcvt::S2si : InstARM32Vcvt::S2ui);
-    } else {
-      _vcvt(T_fp, Src0R,
-            DestIsSigned ? InstARM32Vcvt::D2si : InstARM32Vcvt::D2ui);
-    }
+    const InstARM32Vcvt::VcvtVariant Conversion =
+        Src0IsF32 ? (DestIsSigned ? InstARM32Vcvt::S2si : InstARM32Vcvt::S2ui)
+                  : (DestIsSigned ? InstARM32Vcvt::D2si : InstARM32Vcvt::D2ui);
+    _vcvt(T_fp, Src0R, Conversion);
     Variable *T = makeReg(IceType_i32);
-    _vmov(T, T_fp);
+    _mov(T, T_fp);
     if (Dest->getType() != IceType_i32) {
       Variable *T_1 = makeReg(Dest->getType());
       lowerCast(InstCast::create(Func, InstCast::Trunc, T_1, T));
       T = T_1;
     }
     _mov(Dest, T);
     break;
   }
   case InstCast::Sitofp:
   case InstCast::Uitofp: {
+    if (isVectorType(Dest->getType())) {
+      Variable *T = makeReg(Dest->getType());
+      Context.insert(InstFakeDef::create(Func, T, legalizeToReg(Src0)));
+      _mov(Dest, T);
+      UnimplementedError(Func->getContext()->getFlags());
+      break;
+    }
+    const bool SourceIsSigned = CastKind == InstCast::Sitofp;
+    const bool DestIsF32 = isFloat32Asserting32Or64(Dest->getType());
+    if (Src0->getType() == IceType_i64) {
+      const char *HelperName =
+          DestIsF32 ? (SourceIsSigned ? H_sitofp_i64_f32 : H_uitofp_i64_f32)
+                    : (SourceIsSigned ? H_sitofp_i64_f64 : H_uitofp_i64_f64);
+      static constexpr SizeT MaxSrcs = 1;
+      InstCall *Call = makeHelperCall(HelperName, Dest, MaxSrcs);
+      Call->addArg(Src0);
+      lowerCall(Call);
+      break;
+    }
     // sitofp:
     //     t1.i32 = sext src.int    @ sign-extends src0 if needed.
     //     t2.fp32 = vmov t1.i32
     //     t3.fp = vcvt.{fp}.s32    @ fp is either f32 or f64
     // uitofp:
     //     t1.i32 = zext src.int    @ zero-extends src0 if needed.
     //     t2.fp32 = vmov t1.i32
     //     t3.fp = vcvt.{fp}.s32    @ fp is either f32 or f64
-    if (isVectorType(Dest->getType())) {
-      UnimplementedError(Func->getContext()->getFlags());
-      break;
-    }
-    if (Src0->getType() == IceType_i64) {
-      // avoid cryptic liveness errors
-      Context.insert(InstFakeDef::create(Func, Dest));
-      UnimplementedError(Func->getContext()->getFlags());
-      break;
-    }
-    const bool SourceIsSigned = CastKind == InstCast::Sitofp;
     if (Src0->getType() != IceType_i32) {
       Variable *Src0R_32 = makeReg(IceType_i32);
       lowerCast(InstCast::create(Func, SourceIsSigned ? InstCast::Sext
                                                       : InstCast::Zext,
                                  Src0R_32, Src0));
       Src0 = Src0R_32;
     }
     Variable *Src0R = legalizeToReg(Src0);
     Variable *Src0R_f32 = makeReg(IceType_f32);
-    _vmov(Src0R_f32, Src0R);
+    _mov(Src0R_f32, Src0R);
     Src0R = Src0R_f32;
     Variable *T = makeReg(Dest->getType());
-    if (isFloat32Asserting32Or64(Dest->getType())) {
-      _vcvt(T, Src0R,
-            SourceIsSigned ? InstARM32Vcvt::Si2s : InstARM32Vcvt::Ui2s);
-    } else {
-      _vcvt(T, Src0R,
-            SourceIsSigned ? InstARM32Vcvt::Si2d : InstARM32Vcvt::Ui2d);
-    }
+    const InstARM32Vcvt::VcvtVariant Conversion =
+        DestIsF32
+            ? (SourceIsSigned ? InstARM32Vcvt::Si2s : InstARM32Vcvt::Ui2s)
+            : (SourceIsSigned ? InstARM32Vcvt::Si2d : InstARM32Vcvt::Ui2d);
+    _vcvt(T, Src0R, Conversion);
     _mov(Dest, T);
     break;
   }
   case InstCast::Bitcast: {
     Operand *Src0 = Inst->getSrc(0);
     if (Dest->getType() == Src0->getType()) {
       InstAssign *Assign = InstAssign::create(Func, Dest, Src0);
       lowerAssign(Assign);
       return;
     }
     Type DestType = Dest->getType();
     switch (DestType) {
     case IceType_NUM:
     case IceType_void:
       llvm::report_fatal_error("Unexpected bitcast.");
     case IceType_i1:
       UnimplementedError(Func->getContext()->getFlags());
       break;
-    case IceType_v4i1:
-      UnimplementedError(Func->getContext()->getFlags());
-      break;
     case IceType_i8:
       UnimplementedError(Func->getContext()->getFlags());
       break;
     case IceType_i16:
       UnimplementedError(Func->getContext()->getFlags());
       break;
     case IceType_i32:
     case IceType_f32: {
       Variable *Src0R = legalizeToReg(Src0);
       Variable *T = makeReg(DestType);
-      _vmov(T, Src0R);
+      _mov(T, Src0R);
       lowerAssign(InstAssign::create(Func, Dest, T));
       break;
     }
     case IceType_i64: {
       // t0, t1 <- src0
       // dest[31..0]  = t0
       // dest[63..32] = t1
       assert(Src0->getType() == IceType_f64);
-      Variable *T0 = makeReg(IceType_i32);
-      Variable *T1 = makeReg(IceType_i32);
+      auto *T = llvm::cast<Variable64On32>(Func->makeVariable(IceType_i64));
+      T->initHiLo(Func);
+      forceHiLoInReg(T);
       Variable *Src0R = legalizeToReg(Src0);
-      _vmov(InstARM32Vmov::RegisterPair(T0, T1), Src0R);
+      _mov(T, Src0R);
+      Context.insert(InstFakeDef::create(Func, T->getLo()));
+      Context.insert(InstFakeDef::create(Func, T->getHi()));
       auto *Dest64On32 = llvm::cast<Variable64On32>(Dest);
-      lowerAssign(InstAssign::create(Func, Dest64On32->getLo(), T0));
-      lowerAssign(InstAssign::create(Func, Dest64On32->getHi(), T1));
+      lowerAssign(InstAssign::create(Func, Dest64On32->getLo(), T->getLo()));
+      lowerAssign(InstAssign::create(Func, Dest64On32->getHi(), T->getHi()));
+      Context.insert(InstFakeUse::create(Func, T));
       break;
     }
     case IceType_f64: {
       // T0 <- lo(src)
       // T1 <- hi(src)
       // vmov T2, T0, T1
       // Dest <- T2
       assert(Src0->getType() == IceType_i64);
-      Variable *SrcLo = legalizeToReg(loOperand(Src0));
-      Variable *SrcHi = legalizeToReg(hiOperand(Src0));
-      Variable *T = makeReg(IceType_f64);
-      _vmov(T, InstARM32Vmov::RegisterPair(SrcLo, SrcHi));
+      auto *Src64 = llvm::cast<Variable64On32>(Func->makeVariable(IceType_i64));
+      Src64->initHiLo(Func);
+      forceHiLoInReg(Src64);
+      Variable *T = Src64->getLo();
+      _mov(T, legalizeToReg(loOperand(Src0)));
+      T = Src64->getHi();
+      _mov(T, legalizeToReg(hiOperand(Src0)));
+      T = makeReg(IceType_f64);
+      Context.insert(InstFakeDef::create(Func, Src64));
+      _mov(T, Src64);
+      Context.insert(InstFakeUse::create(Func, Src64->getLo()));
+      Context.insert(InstFakeUse::create(Func, Src64->getHi()));
       lowerAssign(InstAssign::create(Func, Dest, T));
       break;
     }
+    case IceType_v4i1:
     case IceType_v8i1:
+    case IceType_v16i1:
+    case IceType_v8i16:
+    case IceType_v16i8:
+    case IceType_v4f32:
+    case IceType_v4i32: {
+      // avoid cryptic liveness errors
+      Variable *T = makeReg(DestType);
+      Context.insert(InstFakeDef::create(Func, T, legalizeToReg(Src0)));
+      _mov(Dest, T);
       UnimplementedError(Func->getContext()->getFlags());
       break;
-    case IceType_v16i1:
-      UnimplementedError(Func->getContext()->getFlags());
-      break;
-    case IceType_v8i16:
-      UnimplementedError(Func->getContext()->getFlags());
-      break;
-    case IceType_v16i8:
-      UnimplementedError(Func->getContext()->getFlags());
-      break;
-    case IceType_v4i32:
-      // avoid cryptic liveness errors
-      Context.insert(InstFakeDef::create(Func, Dest));
-      UnimplementedError(Func->getContext()->getFlags());
-      break;
-    case IceType_v4f32:
-      UnimplementedError(Func->getContext()->getFlags());
-      break;
+    }
     }
     break;
   }
   }
 }
 
 void TargetARM32::lowerExtractElement(const InstExtractElement *Inst) {
-  (void)Inst;
+  Variable *Dest = Inst->getDest();
+  Type DestType = Dest->getType();
+  Variable *T = makeReg(DestType);
+  Context.insert(InstFakeDef::create(Func, T));
+  _mov(Dest, T);
   UnimplementedError(Func->getContext()->getFlags());
 }
 
 namespace {
 // Validates FCMPARM32_TABLE's declaration w.r.t. InstFcmp::FCondition ordering
 // (and naming).
 enum {
 #define X(val, CC0, CC1) _fcmp_ll_##val,
   FCMPARM32_TABLE
 #undef X
@@ skipping 24 matching lines @@
   { CondARM32::CC0, CondARM32::CC1 }                                           \
   ,
     FCMPARM32_TABLE
 #undef X
 };
 } // end of anonymous namespace
 
 void TargetARM32::lowerFcmp(const InstFcmp *Inst) {
   Variable *Dest = Inst->getDest();
   if (isVectorType(Dest->getType())) {
+    Variable *T = makeReg(Dest->getType());
+    Context.insert(InstFakeDef::create(Func, T));
+    _mov(Dest, T);
     UnimplementedError(Func->getContext()->getFlags());
     return;
   }
 
   Variable *Src0R = legalizeToReg(Inst->getSrc(0));
   Variable *Src1R = legalizeToReg(Inst->getSrc(1));
   Variable *T = makeReg(IceType_i32);
   _vcmp(Src0R, Src1R);
   _mov(T, Ctx->getConstantZero(IceType_i32));
   _vmrs();
@@ skipping 17 matching lines @@
   }
   _mov(Dest, T);
 }
 
 void TargetARM32::lowerIcmp(const InstIcmp *Inst) {
   Variable *Dest = Inst->getDest();
   Operand *Src0 = legalizeUndef(Inst->getSrc(0));
   Operand *Src1 = legalizeUndef(Inst->getSrc(1));
 
   if (isVectorType(Dest->getType())) {
+    Variable *T = makeReg(Dest->getType());
+    Context.insert(InstFakeDef::create(Func, T));
+    _mov(Dest, T);
     UnimplementedError(Func->getContext()->getFlags());
     return;
   }
 
   // a=icmp cond, b, c ==>
   // GCC does:
   //   cmp      b.hi, c.hi     or  cmp      b.lo, c.lo
   //   cmp.eq   b.lo, c.lo         sbcs t1, b.hi, c.hi
   //   mov.<C1> t, #1              mov.<C1> t, #1
   //   mov.<C2> t, #0              mov.<C2> t, #0
@@ skipping 188 matching lines @@
     Call->addArg(Val);
     lowerCall(Call);
     // The popcount helpers always return 32-bit values, while the intrinsic's
     // signature matches some 64-bit platform's native instructions and expect
     // to fill a 64-bit reg. Thus, clear the upper bits of the dest just in
     // case the user doesn't do that in the IR or doesn't toss the bits via
     // truncate.
     if (Val->getType() == IceType_i64) {
       Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
       Constant *Zero = Ctx->getConstantZero(IceType_i32);
-      Variable *T = nullptr;
+      Variable *T = makeReg(Zero->getType());
       _mov(T, Zero);
       _mov(DestHi, T);
     }
     return;
   }
   case Intrinsics::Ctlz: {
     // The "is zero undef" parameter is ignored and we always return a
     // well-defined value.
     Operand *Val = Instr->getArg(0);
     Variable *ValLoR;
@@ skipping 26 matching lines @@
     } else {
       ValLoR = legalizeToReg(Val);
       Variable *T = makeReg(IceType_i32);
       _rbit(T, ValLoR);
       ValLoR = T;
     }
     lowerCLZ(Instr->getDest(), ValLoR, ValHiR);
     return;
   }
   case Intrinsics::Fabs: {
-    // Add a fake def to keep liveness consistent in the meantime.
-    Context.insert(InstFakeDef::create(Func, Instr->getDest()));
-    UnimplementedError(Func->getContext()->getFlags());
+    Variable *Dest = Instr->getDest();
+    Type DestTy = Dest->getType();
+    Variable *T = makeReg(DestTy);
+    if (isVectorType(DestTy)) {
+      // Add a fake def to keep liveness consistent in the meantime.
+      Context.insert(InstFakeDef::create(Func, T));
+      _mov(Instr->getDest(), T);
+      UnimplementedError(Func->getContext()->getFlags());
+      return;
+    }
+    _vabs(T, legalizeToReg(Instr->getArg(0)));
+    _mov(Dest, T);
     return;
   }
   case Intrinsics::Longjmp: {
     InstCall *Call = makeHelperCall(H_call_longjmp, nullptr, 2);
     Call->addArg(Instr->getArg(0));
     Call->addArg(Instr->getArg(1));
     lowerCall(Call);
     return;
   }
   case Intrinsics::Memcpy: {
@@ skipping 44 matching lines @@
     InstCall *Call = makeHelperCall(H_call_setjmp, Instr->getDest(), 1);
     Call->addArg(Instr->getArg(0));
     lowerCall(Call);
     return;
   }
   case Intrinsics::Sqrt: {
     Variable *Src = legalizeToReg(Instr->getArg(0));
     Variable *Dest = Instr->getDest();
     Variable *T = makeReg(Dest->getType());
     _vsqrt(T, Src);
-    _vmov(Dest, T);
+    _mov(Dest, T);
     return;
   }
   case Intrinsics::Stacksave: {
     Variable *SP = getPhysicalRegister(RegARM32::Reg_sp);
     Variable *Dest = Instr->getDest();
     _mov(Dest, SP);
     return;
   }
   case Intrinsics::Stackrestore: {
     Variable *SP = getPhysicalRegister(RegARM32::Reg_sp);
@@ skipping 25 matching lines @@
         legalize(Ctx->getConstantInt32(32), Legal_Reg | Legal_Flex);
     _cmp(ValHiR, Zero);
     Variable *T2 = makeReg(IceType_i32);
     _add(T2, T, ThirtyTwo);
     _clz(T2, ValHiR, CondARM32::NE);
     // T2 is actually a source as well when the predicate is not AL (since it
     // may leave T2 alone). We use set_dest_nonkillable to prolong the liveness
     // of T2 as if it was used as a source.
     _set_dest_nonkillable();
     _mov(DestLo, T2);
-    Variable *T3 = nullptr;
+    Variable *T3 = makeReg(Zero->getType());
     _mov(T3, Zero);
     _mov(DestHi, T3);
     return;
   }
   _mov(Dest, T);
   return;
 }
 
 void TargetARM32::lowerLoad(const InstLoad *Load) {
   // A Load instruction can be treated the same as an Assign instruction, after
@@ skipping 39 matching lines @@
       Variable *S0 = legalizeToReg(Src0, RegARM32::Reg_s0);
       Reg = S0;
     } else if (Ty == IceType_f64) {
       Variable *D0 = legalizeToReg(Src0, RegARM32::Reg_d0);
       Reg = D0;
     } else if (isVectorType(Src0->getType())) {
       Variable *Q0 = legalizeToReg(Src0, RegARM32::Reg_q0);
       Reg = Q0;
     } else {
       Operand *Src0F = legalize(Src0, Legal_Reg | Legal_Flex);
-      _mov(Reg, Src0F, CondARM32::AL, RegARM32::Reg_r0);
+      Reg = makeReg(Src0F->getType(), RegARM32::Reg_r0);
+      _mov(Reg, Src0F, CondARM32::AL);
     }
   }
   // Add a ret instruction even if sandboxing is enabled, because addEpilog
   // explicitly looks for a ret instruction as a marker for where to insert the
   // frame removal instructions. addEpilog is responsible for restoring the
   // "lr" register as needed prior to this ret instruction.
   _ret(getPhysicalRegister(RegARM32::Reg_lr), Reg);
   // Add a fake use of sp to make sure sp stays alive for the entire function.
   // Otherwise post-call sp adjustments get dead-code eliminated.
   // TODO: Are there more places where the fake use should be inserted? E.g.
   // "void f(int n){while(1) g(n);}" may not have a ret instruction.
   Variable *SP = getPhysicalRegister(RegARM32::Reg_sp);
   Context.insert(InstFakeUse::create(Func, SP));
 }
 
 void TargetARM32::lowerSelect(const InstSelect *Inst) {
   Variable *Dest = Inst->getDest();
   Type DestTy = Dest->getType();
   Operand *SrcT = Inst->getTrueOperand();
   Operand *SrcF = Inst->getFalseOperand();
   Operand *Condition = Inst->getCondition();
 
   if (isVectorType(DestTy)) {
+    Variable *T = makeReg(DestTy);
+    Context.insert(InstFakeDef::create(Func, T));
+    _mov(Dest, T);
     UnimplementedError(Func->getContext()->getFlags());
     return;
   }
   // TODO(jvoung): handle folding opportunities.
   // cmp cond, #0; mov t, SrcF; mov_cond t, SrcT; mov dest, t
   Variable *CmpOpnd0 = legalizeToReg(Condition);
   Operand *CmpOpnd1 = Ctx->getConstantZero(IceType_i32);
   _cmp(CmpOpnd0, CmpOpnd1);
   static constexpr CondARM32::Cond Cond = CondARM32::NE;
   if (DestTy == IceType_i64) {
     SrcT = legalizeUndef(SrcT);
     SrcF = legalizeUndef(SrcF);
     // Set the low portion.
     Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
-    Variable *TLo = nullptr;
     Operand *SrcFLo = legalize(loOperand(SrcF), Legal_Reg | Legal_Flex);
+    Variable *TLo = makeReg(SrcFLo->getType());
     _mov(TLo, SrcFLo);
     Operand *SrcTLo = legalize(loOperand(SrcT), Legal_Reg | Legal_Flex);
     _mov_nonkillable(TLo, SrcTLo, Cond);
     _mov(DestLo, TLo);
     // Set the high portion.
     Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
-    Variable *THi = nullptr;
     Operand *SrcFHi = legalize(hiOperand(SrcF), Legal_Reg | Legal_Flex);
+    Variable *THi = makeReg(SrcFHi->getType());
     _mov(THi, SrcFHi);
     Operand *SrcTHi = legalize(hiOperand(SrcT), Legal_Reg | Legal_Flex);
     _mov_nonkillable(THi, SrcTHi, Cond);
     _mov(DestHi, THi);
     return;
   }
 
   if (isFloatingType(DestTy)) {
     Variable *T = makeReg(DestTy);
     SrcF = legalizeToReg(SrcF);
     assert(DestTy == SrcF->getType());
-    _vmov(T, SrcF);
+    _mov(T, SrcF);
     SrcT = legalizeToReg(SrcT);
     assert(DestTy == SrcT->getType());
-    _vmov(T, SrcT, Cond);
+    _mov(T, SrcT, Cond);
     _set_dest_nonkillable();
-    _vmov(Dest, T);
+    _mov(Dest, T);
     return;
   }
 
-  Variable *T = nullptr;
   SrcF = legalize(SrcF, Legal_Reg | Legal_Flex);
+  Variable *T = makeReg(SrcF->getType());
   _mov(T, SrcF);
   SrcT = legalize(SrcT, Legal_Reg | Legal_Flex);
   _mov_nonkillable(T, SrcT, Cond);
   _mov(Dest, T);
 }
 
 void TargetARM32::lowerStore(const InstStore *Inst) {
   Operand *Value = Inst->getData();
   Operand *Addr = Inst->getAddr();
   OperandARM32Mem *NewAddr = formMemoryOperand(Addr, Value->getType());
   Type Ty = NewAddr->getType();
 
   if (Ty == IceType_i64) {
     Value = legalizeUndef(Value);
     Variable *ValueHi = legalizeToReg(hiOperand(Value));
     Variable *ValueLo = legalizeToReg(loOperand(Value));
     _str(ValueHi, llvm::cast<OperandARM32Mem>(hiOperand(NewAddr)));
     _str(ValueLo, llvm::cast<OperandARM32Mem>(loOperand(NewAddr)));
   } else {
-    if (isVectorType(Ty)) {
-      UnimplementedError(Func->getContext()->getFlags());
-    }
     Variable *ValueR = legalizeToReg(Value);
     _str(ValueR, NewAddr);
   }
 }
 
 void TargetARM32::doAddressOptStore() {
   UnimplementedError(Func->getContext()->getFlags());
 }
 
 void TargetARM32::lowerSwitch(const InstSwitch *Inst) {
@@ skipping 32 matching lines @@
 void TargetARM32::lowerUnreachable(const InstUnreachable * /*Inst*/) {
   _trap();
 }
 
 void TargetARM32::prelowerPhis() {
   PhiLowering::prelowerPhis32Bit<TargetARM32>(this, Context.getNode(), Func);
 }
 
 Variable *TargetARM32::makeVectorOfZeros(Type Ty, int32_t RegNum) {
   Variable *Reg = makeReg(Ty, RegNum);
+  Context.insert(InstFakeDef::create(Func, Reg));
   UnimplementedError(Func->getContext()->getFlags());
   return Reg;
 }
 
 // Helper for legalize() to emit the right code to lower an operand to a
 // register of the appropriate type.
 Variable *TargetARM32::copyToReg(Operand *Src, int32_t RegNum) {
   Type Ty = Src->getType();
   Variable *Reg = makeReg(Ty, RegNum);
-  if (isVectorType(Ty)) {
-    // TODO(jpp): Src must be a register, or an address with base register.
-    _vmov(Reg, Src);
-  } else if (isFloatingType(Ty)) {
-    _vmov(Reg, Src);
-  } else {
-    // Mov's Src operand can really only be the flexible second operand type or
-    // a register. Users should guarantee that.
-    _mov(Reg, Src);
-  }
+  _mov(Reg, Src);
   return Reg;
 }
 
 Operand *TargetARM32::legalize(Operand *From, LegalMask Allowed,
                                int32_t RegNum) {
   Type Ty = From->getType();
   // Assert that a physical register is allowed. To date, all calls to
   // legalize() allow a physical register. Legal_Flex converts registers to the
   // right type OperandARM32FlexReg as needed.
   assert(Allowed & Legal_Reg);
   // Go through the various types of operands: OperandARM32Mem,
   // OperandARM32Flex, Constant, and Variable. Given the above assertion, if
   // type of operand is not legal (e.g., OperandARM32Mem and !Legal_Mem), we
   // can always copy to a register.
   if (auto Mem = llvm::dyn_cast<OperandARM32Mem>(From)) {
+    static const struct {
+      bool CanHaveOffset;
+      bool CanHaveIndex;
+    } MemTraits[] = {
+#define X(tag, elementty, int_width, vec_width, sbits, ubits, rraddr)          \
+  { (ubits) > 0, rraddr }                                                      \
+  ,
+        ICETYPEARM32_TABLE
+#undef X
+    };
     // Before doing anything with a Mem operand, we need to ensure that the
     // Base and Index components are in physical registers.
     Variable *Base = Mem->getBase();
     Variable *Index = Mem->getIndex();
+    ConstantInteger32 *Offset = Mem->getOffset();
+    assert(Index == nullptr || Offset == nullptr);
     Variable *RegBase = nullptr;
     Variable *RegIndex = nullptr;
     if (Base) {
       RegBase = legalizeToReg(Base);
     }
     if (Index) {
       RegIndex = legalizeToReg(Index);
+      if (!MemTraits[Ty].CanHaveIndex) {
+        Variable *T = makeReg(IceType_i32, getReservedTmpReg());
+        _add(T, RegBase, RegIndex);
+        RegBase = T;
+        RegIndex = nullptr;
+      }
     }
+    if (Offset && Offset->getValue() != 0) {
+      static constexpr bool SignExt = false;
+      if (!MemTraits[Ty].CanHaveOffset ||
+          !OperandARM32Mem::canHoldOffset(Ty, SignExt, Offset->getValue())) {
+        Variable *T = legalizeToReg(Offset, getReservedTmpReg());
+        _add(T, T, RegBase);
+        RegBase = T;
+        Offset = llvm::cast<ConstantInteger32>(Ctx->getConstantInt32(0));
+      }
+    }
+
     // Create a new operand if there was a change.
     if (Base != RegBase || Index != RegIndex) {
       // There is only a reg +/- reg or reg + imm form.
       // Figure out which to re-create.
-      if (Mem->isRegReg()) {
+      if (RegBase && RegIndex) {
         Mem = OperandARM32Mem::create(Func, Ty, RegBase, RegIndex,
                                       Mem->getShiftOp(), Mem->getShiftAmt(),
                                       Mem->getAddrMode());
       } else {
-        Mem = OperandARM32Mem::create(Func, Ty, RegBase, Mem->getOffset(),
+        Mem = OperandARM32Mem::create(Func, Ty, RegBase, Offset,
                                       Mem->getAddrMode());
       }
     }
-    if (!(Allowed & Legal_Mem)) {
+    if (Allowed & Legal_Mem) {
+      From = Mem;
+    } else {
       Variable *Reg = makeReg(Ty, RegNum);
-      if (isVectorType(Ty)) {
-        UnimplementedError(Func->getContext()->getFlags());
-      } else if (isFloatingType(Ty)) {
-        _vldr(Reg, Mem);
-      } else {
-        _ldr(Reg, Mem);
-      }
+      _ldr(Reg, Mem);
       From = Reg;
-    } else {
-      From = Mem;
     }
     return From;
   }
 
   if (auto Flex = llvm::dyn_cast<OperandARM32Flex>(From)) {
     if (!(Allowed & Legal_Flex)) {
       if (auto FlexReg = llvm::dyn_cast<OperandARM32FlexReg>(Flex)) {
         if (FlexReg->getShiftOp() == OperandARM32::kNoShift) {
           From = FlexReg->getReg();
           // Fall through and let From be checked as a Variable below, where it
@@ skipping 392 matching lines @@
       << ".eabi_attribute 68, 1   @ Tag_Virtualization_use\n";
   if (CPUFeatures.hasFeature(TargetARM32Features::HWDivArm)) {
     Str << ".eabi_attribute 44, 2   @ Tag_DIV_use\n";
   }
   // Technically R9 is used for TLS with Sandboxing, and we reserve it.
   // However, for compatibility with current NaCl LLVM, don't claim that.
   Str << ".eabi_attribute 14, 3   @ Tag_ABI_PCS_R9_use: Not used\n";
 }
 
 } // end of namespace Ice