Subzero. ARM32. Pre-lowers calls to ARM32 Helpers.

src/IceTargetLoweringARM32.cpp

 //===- subzero/src/IceTargetLoweringARM32.cpp - ARM32 lowering ------------===//
 //
 //                        The Subzero Code Generator
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 ///
 /// \file
@@ skipping 247 matching lines @@
       }
     }
 
     OutArgsSizeBytes = applyStackAlignmentTy(OutArgsSizeBytes, Ty);
     OutArgsSizeBytes += typeWidthInBytesOnStack(Ty);
   }
 
   return applyStackAlignment(OutArgsSizeBytes);
 }
 
+void TargetARM32::genTargetHelperCallFor(Inst *Instr) {
+  constexpr bool NoTailCall = false;
+  constexpr bool IsTargetHelperCall = true;
+
+  switch (Instr->getKind()) {
+  default:
+    return;
+  case Inst::Arithmetic: {
+    Variable *Dest = Instr->getDest();
+    const Type DestTy = Dest->getType();
+    const InstArithmetic::OpKind Op =
+        llvm::cast<InstArithmetic>(Instr)->getOp();
+    switch (DestTy) {
+    default:
+      return;
+    case IceType_i64: {
+      // Technically, ARM has its own aeabi routines, but we can use the
+      // non-aeabi routine as well. LLVM uses __aeabi_ldivmod for div, but uses
+      // the more standard __moddi3 for rem.
+      Operand *TargetHelper = nullptr;
+      switch (Op) {
+      default:
+        return;
+      case InstArithmetic::Udiv:
+        TargetHelper = Ctx->getConstantExternSym(H_udiv_i64);
+        break;
+      case InstArithmetic::Sdiv:
+        TargetHelper = Ctx->getConstantExternSym(H_sdiv_i64);
+        break;
+      case InstArithmetic::Urem:
+        TargetHelper = Ctx->getConstantExternSym(H_urem_i64);
+        break;
+      case InstArithmetic::Srem:
+        TargetHelper = Ctx->getConstantExternSym(H_srem_i64);
+        break;
+      }
+      assert(TargetHelper != nullptr);
+      ARM32HelpersPreamble[TargetHelper] = &TargetARM32::DivRemPreamble;
+      constexpr SizeT MaxArgs = 2;
+      auto *Call = InstCall::create(Func, MaxArgs, Dest, TargetHelper,
+                                    NoTailCall, IsTargetHelperCall);
+      Call->addArg(Instr->getSrc(0));
+      Call->addArg(Instr->getSrc(1));
+      Context.insert(Call);
+      Instr->setDeleted();
+      return;
+    }
+    case IceType_i32:
+    case IceType_i16:
+    case IceType_i8: {
+      if (hasCPUFeature(TargetARM32Features::HWDivArm)) {
+        return;
+      }
+      InstCast::OpKind CastKind;
+      Operand *TargetHelper;
+      switch (Op) {
+      default:
+        return;
+      case InstArithmetic::Udiv:
+        TargetHelper = Ctx->getConstantExternSym(H_udiv_i32);
+        CastKind = InstCast::Zext;
+        break;
+      case InstArithmetic::Sdiv:
+        TargetHelper = Ctx->getConstantExternSym(H_sdiv_i32);
+        CastKind = InstCast::Sext;
+        break;
+      case InstArithmetic::Urem:
+        TargetHelper = Ctx->getConstantExternSym(H_urem_i32);
+        CastKind = InstCast::Zext;
+        break;
+      case InstArithmetic::Srem:
+        TargetHelper = Ctx->getConstantExternSym(H_srem_i32);
+        CastKind = InstCast::Sext;
+        break;
+      }
+      Operand *Src0 = Instr->getSrc(0);
+      Operand *Src1 = Instr->getSrc(1);
+      if (DestTy != IceType_i32) {
+        // Src0 and Src1 have to be zero-, or signed-extended to i32. For Src0,
+        // we just insert a InstCast right before the call to the helper.
+        Variable *Src0_32 = Func->makeVariable(IceType_i32);
+        Context.insert(InstCast::create(Func, CastKind, Src0_32, Src0));
+        Src0 = Src0_32;
+
+        // For extending Src1, we will just insert an InstCast if Src1 is not a
+        // Constant. If it is, then we extend it here, and not during program
+        // runtime. This allows DivRemPreamble to optimize-out the div-by-0
+        // check.
+        if (auto *C = llvm::dyn_cast<ConstantInteger32>(Src1)) {
+          const int32_t ShAmt = (DestTy == IceType_i16) ? 16 : 24;
+          int32_t NewC = C->getValue();
+          if (CastKind == InstCast::Zext) {
+            NewC &= !(0x80000000 >> ShAmt);

[Jim Stichnoth, 2015/11/30 21:47:20]

Did you mean "~" instead of "!" ?

Also, what do you think about using int32_t(0x80000000)>>ShAmt to make it more
clear that it's an arithmetic shift of a negative number?

[John, 2015/11/30 22:16:25]

are you OK with 0x80000000l instead? I don't really like using PODs as if they
were ctors.

[Jim Stichnoth, 2015/12/01 17:01:56]

OK.  I think at least the presence of a suffix hints at the special intent, so
that's good.

+          } else {
+            NewC = (NewC << ShAmt) >> ShAmt;
+          }
+          Src1 = Ctx->getConstantInt32(NewC);
+        } else {
+          Variable *Src1_32 = Func->makeVariable(IceType_i32);
+          Context.insert(InstCast::create(Func, CastKind, Src1_32, Src1));
+          Src1 = Src1_32;
+        }
+      }
+      assert(TargetHelper != nullptr);
+      ARM32HelpersPreamble[TargetHelper] = &TargetARM32::DivRemPreamble;
+      constexpr SizeT MaxArgs = 2;
+      auto *Call = InstCall::create(Func, MaxArgs, Dest, TargetHelper,
+                                    NoTailCall, IsTargetHelperCall);
+      assert(Src0->getType() == IceType_i32);
+      Call->addArg(Src0);
+      assert(Src1->getType() == IceType_i32);
+      Call->addArg(Src1);
+      Context.insert(Call);
+      Instr->setDeleted();
+      return;
+    }
+    case IceType_f64:
+    case IceType_f32: {
+      if (Op != InstArithmetic::Frem) {
+        return;
+      }
+      constexpr SizeT MaxArgs = 2;
+      Operand *TargetHelper = Ctx->getConstantExternSym(
+          DestTy == IceType_f32 ? H_frem_f32 : H_frem_f64);
+      auto *Call = InstCall::create(Func, MaxArgs, Dest, TargetHelper,
+                                    NoTailCall, IsTargetHelperCall);
+      Call->addArg(Instr->getSrc(0));
+      Call->addArg(Instr->getSrc(1));
+      Context.insert(Call);
+      Instr->setDeleted();
+      return;
+    }
+    }
+    llvm_unreachable("Control flow should never have reached here.");

[Jim Stichnoth, 2015/11/30 21:47:19]

Except for vector instructions...  How about making this report_fatal_error()
just to be safe?

[John, 2015/11/30 22:16:25]

Done.

[Jim Stichnoth, 2015/12/01 17:01:56]

Thanks.  I marked a few more cases that I think ought to be report_fatal_error. 
Optionally, make them all into report_fatal_error.

+  }
+  case Inst::Cast: {
+    Variable *Dest = Instr->getDest();
+    Operand *Src0 = Instr->getSrc(0);
+    const Type DestTy = Dest->getType();
+    const InstCast::OpKind CastKind =
+        llvm::cast<InstCast>(Instr)->getCastKind();
+    switch (CastKind) {
+    default:
+      return;
+    case InstCast::Fptosi:
+    case InstCast::Fptoui: {
+      if (DestTy != IceType_i64) {
+        return;
+      }
+      const bool DestIsSigned = CastKind == InstCast::Fptosi;
+      const bool Src0IsF32 = isFloat32Asserting32Or64(Src0->getType());
+      Operand *TargetHelper = Ctx->getConstantExternSym(
+          Src0IsF32 ? (DestIsSigned ? H_fptosi_f32_i64 : H_fptoui_f32_i64)
+                    : (DestIsSigned ? H_fptosi_f64_i64 : H_fptoui_f64_i64));
+      static constexpr SizeT MaxArgs = 1;
+      auto *Call = InstCall::create(Func, MaxArgs, Dest, TargetHelper,
+                                    NoTailCall, IsTargetHelperCall);
+      Call->addArg(Src0);
+      Context.insert(Call);
+      Instr->setDeleted();
+      return;
+    }
+    case InstCast::Sitofp:
+    case InstCast::Uitofp: {
+      if (Src0->getType() != IceType_i64) {
+        return;
+      }
+      const bool SourceIsSigned = CastKind == InstCast::Sitofp;
+      const bool DestIsF32 = isFloat32Asserting32Or64(Dest->getType());
+      Operand *TargetHelper = Ctx->getConstantExternSym(
+          DestIsF32 ? (SourceIsSigned ? H_sitofp_i64_f32 : H_uitofp_i64_f32)
+                    : (SourceIsSigned ? H_sitofp_i64_f64 : H_uitofp_i64_f64));
+      static constexpr SizeT MaxArgs = 1;
+      auto *Call = InstCall::create(Func, MaxArgs, Dest, TargetHelper,
+                                    NoTailCall, IsTargetHelperCall);
+      Call->addArg(Src0);
+      Context.insert(Call);
+      Instr->setDeleted();
+      return;
+    }
+    }
+    llvm_unreachable("Control flow should never have reached here.");
+  }
+  case Inst::IntrinsicCall: {
+    Variable *Dest = Instr->getDest();
+    auto *IntrinsicCall = llvm::cast<InstIntrinsicCall>(Instr);
+    Intrinsics::IntrinsicID ID = IntrinsicCall->getIntrinsicInfo().ID;
+    switch (ID) {
+    default:
+      return;
+    case Intrinsics::Ctpop: {
+      Operand *Src0 = IntrinsicCall->getArg(0);
+      Operand *TargetHelper = Ctx->getConstantExternSym(
+          isInt32Asserting32Or64(Src0->getType()) ? H_call_ctpop_i32
+                                                  : H_call_ctpop_i64);
+      static constexpr SizeT MaxArgs = 1;
+      auto *Call = InstCall::create(Func, MaxArgs, Dest, TargetHelper,
+                                    NoTailCall, IsTargetHelperCall);
+      Call->addArg(Src0);
+      Context.insert(Call);
+      Instr->setDeleted();
+      if (Src0->getType() == IceType_i64) {
+        ARM32HelpersPostamble[TargetHelper] = &TargetARM32::Ctpop64Postamble;
+      }
+      return;
+    }
+    case Intrinsics::Longjmp: {
+      static constexpr SizeT MaxArgs = 2;
+      static constexpr Variable *NoDest = nullptr;
+      Operand *TargetHelper = Ctx->getConstantExternSym(H_call_longjmp);
+      auto *Call = InstCall::create(Func, MaxArgs, NoDest, TargetHelper,
+                                    NoTailCall, IsTargetHelperCall);
+      Call->addArg(IntrinsicCall->getArg(0));
+      Call->addArg(IntrinsicCall->getArg(1));
+      Context.insert(Call);
+      Instr->setDeleted();
+      return;
+    }
+    case Intrinsics::Memcpy: {
+      // In the future, we could potentially emit an inline memcpy/memset, etc.
+      // for intrinsic calls w/ a known length.
+      static constexpr SizeT MaxArgs = 3;
+      static constexpr Variable *NoDest = nullptr;
+      Operand *TargetHelper = Ctx->getConstantExternSym(H_call_memcpy);
+      auto *Call = InstCall::create(Func, MaxArgs, NoDest, TargetHelper,
+                                    NoTailCall, IsTargetHelperCall);
+      Call->addArg(IntrinsicCall->getArg(0));
+      Call->addArg(IntrinsicCall->getArg(1));
+      Call->addArg(IntrinsicCall->getArg(2));
+      Context.insert(Call);
+      Instr->setDeleted();
+      return;
+    }
+    case Intrinsics::Memmove: {
+      static constexpr SizeT MaxArgs = 3;
+      static constexpr Variable *NoDest = nullptr;
+      Operand *TargetHelper = Ctx->getConstantExternSym(H_call_memmove);
+      auto *Call = InstCall::create(Func, MaxArgs, NoDest, TargetHelper,
+                                    NoTailCall, IsTargetHelperCall);
+      Call->addArg(IntrinsicCall->getArg(0));
+      Call->addArg(IntrinsicCall->getArg(1));
+      Call->addArg(IntrinsicCall->getArg(2));
+      Context.insert(Call);
+      Instr->setDeleted();
+      return;
+    }
+    case Intrinsics::Memset: {
+      // The value operand needs to be extended to a stack slot size because the
+      // PNaCl ABI requires arguments to be at least 32 bits wide.
+      Operand *ValOp = IntrinsicCall->getArg(1);
+      assert(ValOp->getType() == IceType_i8);
+      Variable *ValExt = Func->makeVariable(stackSlotType());
+      Context.insert(InstCast::create(Func, InstCast::Zext, ValExt, ValOp));
+
+      // Technically, ARM has its own __aeabi_memset, but we can use plain
+      // memset too. The value and size argument need to be flipped if we ever
+      // decide to use __aeabi_memset.
+      static constexpr SizeT MaxArgs = 3;
+      static constexpr Variable *NoDest = nullptr;
+      Operand *TargetHelper = Ctx->getConstantExternSym(H_call_memset);
+      auto *Call = InstCall::create(Func, MaxArgs, NoDest, TargetHelper,
+                                    NoTailCall, IsTargetHelperCall);
+      Call->addArg(IntrinsicCall->getArg(0));
+      Call->addArg(ValExt);
+      Call->addArg(IntrinsicCall->getArg(2));
+      Context.insert(Call);
+      Instr->setDeleted();
+      return;
+    }
+    case Intrinsics::NaClReadTP: {
+      if (Ctx->getFlags().getUseSandboxing()) {
+        UnimplementedError(Func->getContext()->getFlags());
+        return;
+      }
+      static constexpr SizeT MaxArgs = 0;
+      Operand *TargetHelper = Ctx->getConstantExternSym(H_call_read_tp);
+      auto *Call = InstCall::create(Func, MaxArgs, Dest, TargetHelper,
+                                    NoTailCall, IsTargetHelperCall);
+      Context.insert(Call);
+      Instr->setDeleted();
+      return;
+    }
+    case Intrinsics::Setjmp: {
+      static constexpr SizeT MaxArgs = 1;
+      Operand *TargetHelper = Ctx->getConstantExternSym(H_call_setjmp);
+      auto *Call = InstCall::create(Func, MaxArgs, Dest, TargetHelper,
+                                    NoTailCall, IsTargetHelperCall);
+      Call->addArg(IntrinsicCall->getArg(0));
+      Context.insert(Call);
+      Instr->setDeleted();
+      return;
+    }
+    }
+    llvm_unreachable("Control flow should never have reached here.");
+  }
+  }
+}
+
 void TargetARM32::findMaxStackOutArgsSize() {
   // MinNeededOutArgsBytes should be updated if the Target ever creates a
   // high-level InstCall that requires more stack bytes.
   constexpr size_t MinNeededOutArgsBytes = 0;
   MaxOutArgsSizeBytes = MinNeededOutArgsBytes;
   for (CfgNode *Node : Func->getNodes()) {
     Context.init(Node);
     while (!Context.atEnd()) {
       PostIncrLoweringContext PostIncrement(Context);
       Inst *CurInstr = Context.getCur();
@@ skipping 1128 matching lines @@
   }
   }
   InstARM32Label *Label = InstARM32Label::create(Func, this);
   _br(Label, CondARM32::NE);
   _trap();
   Context.insert(Label);
 }
 
 void TargetARM32::lowerIDivRem(Variable *Dest, Variable *T, Variable *Src0R,
                                Operand *Src1, ExtInstr ExtFunc,
-                               DivInstr DivFunc, const char *DivHelperName,
-                               bool IsRemainder) {
+                               DivInstr DivFunc, bool IsRemainder) {
   div0Check(Dest->getType(), Src1, nullptr);
   Variable *Src1R = legalizeToReg(Src1);
   Variable *T0R = Src0R;
   Variable *T1R = Src1R;
   if (Dest->getType() != IceType_i32) {
     T0R = makeReg(IceType_i32);
     (this->*ExtFunc)(T0R, Src0R, CondARM32::AL);
     T1R = makeReg(IceType_i32);
     (this->*ExtFunc)(T1R, Src1R, CondARM32::AL);
   }
   if (hasCPUFeature(TargetARM32Features::HWDivArm)) {
     (this->*DivFunc)(T, T0R, T1R, CondARM32::AL);
     if (IsRemainder) {
       Variable *T2 = makeReg(IceType_i32);
       _mls(T2, T, T1R, T0R);
       T = T2;
     }
     _mov(Dest, T);
   } else {
-    constexpr SizeT MaxSrcs = 2;
-    InstCall *Call = makeHelperCall(DivHelperName, Dest, MaxSrcs);
-    Call->addArg(T0R);
-    Call->addArg(T1R);
-    lowerCall(Call);
+    llvm::report_fatal_error("div should have already been turned into a call");
   }
-  return;
 }
 
 TargetARM32::SafeBoolChain
 TargetARM32::lowerInt1Arithmetic(const InstArithmetic *Inst) {
   Variable *Dest = Inst->getDest();
   assert(Dest->getType() == IceType_i1);
 
   // So folding didn't work for Inst. Not a problem: We just need to
   // materialize the Sources, and perform the operation. We create regular
   // Variables (and not infinite-weight ones) because this call might recurse a
@@ skipping 181 matching lines @@
   }
 
   Operand *invertedSrc1F(TargetARM32 *Target) const {
     return legalizeToRegOrFlex(Target,
                                Target->getCtx()->getConstantInt32(
                                    ~static_cast<uint32_t>(getConstantValue())));
   }
 };
 } // end of anonymous namespace
 
+void TargetARM32::DivRemPreamble(const InstCall *Instr) {
+  Operand *Src1 = Instr->getArg(1);
+
+  switch (Src1->getType()) {
+  default:
+    llvm::report_fatal_error("Invalid type for idiv.");
+  case IceType_i64: {
+    if (auto *C = llvm::dyn_cast<ConstantInteger64>(Src1)) {
+      if (C->getValue() == 0) {
+        _trap();
+        return;
+      }
+    }
+    div0Check(IceType_i64, loOperand(Src1), hiOperand(Src1));
+    return;
+  }
+  case IceType_i32: {
+    // Src0 and Src1 have already been appropriately extended to an i32, so we
+    // don't check for i8 and i16.
+    if (auto *C = llvm::dyn_cast<ConstantInteger32>(Src1)) {
+      if (C->getValue() == 0) {
+        _trap();
+        return;
+      }
+    }
+    div0Check(IceType_i32, Src1, nullptr);
+    return;
+  }
+  }
+}
+
 void TargetARM32::lowerInt64Arithmetic(InstArithmetic::OpKind Op,
                                        Variable *Dest, Operand *Src0,
                                        Operand *Src1) {
   Int32Operands SrcsLo(loOperand(Src0), loOperand(Src1));
   Int32Operands SrcsHi(hiOperand(Src0), hiOperand(Src1));
   assert(SrcsLo.swappedOperands() == SrcsHi.swappedOperands());
   assert(SrcsLo.hasConstOperand() == SrcsHi.hasConstOperand());
 
-  // These helper-call-involved instructions are lowered in this separate
-  // switch. This is because we would otherwise assume that we need to
-  // legalize Src0 to Src0RLo and Src0Hi. However, those go unused with
-  // helper calls, and such unused/redundant instructions will fail liveness
-  // analysis under -Om1 setting.
-  switch (Op) {
-  default:
-    break;
-  case InstArithmetic::Udiv:
-  case InstArithmetic::Sdiv:
-  case InstArithmetic::Urem:
-  case InstArithmetic::Srem: {
-    // Check for divide by 0 (ARM normally doesn't trap, but we want it to
-    // trap for NaCl). Src1Lo and Src1Hi may have already been legalized to a
-    // register, which will hide a constant source operand. Instead, check
-    // the not-yet-legalized Src1 to optimize-out a divide by 0 check.
-    if (!SrcsLo.swappedOperands() && SrcsLo.hasConstOperand()) {
-      if (SrcsLo.getConstantValue() == 0 && SrcsHi.getConstantValue() == 0) {
-        _trap();
-        return;
-      }
-    } else {
-      Operand *Src1Lo = SrcsLo.unswappedSrc1R(this);
-      Operand *Src1Hi = SrcsHi.unswappedSrc1R(this);
-      div0Check(IceType_i64, Src1Lo, Src1Hi);
-    }
-    // Technically, ARM has its own aeabi routines, but we can use the
-    // non-aeabi routine as well. LLVM uses __aeabi_ldivmod for div, but uses
-    // the more standard __moddi3 for rem.
-    const char *HelperName = "";
-    switch (Op) {
-    default:
-      llvm::report_fatal_error("Should have only matched div ops.");
-      break;
-    case InstArithmetic::Udiv:
-      HelperName = H_udiv_i64;
-      break;
-    case InstArithmetic::Sdiv:
-      HelperName = H_sdiv_i64;
-      break;
-    case InstArithmetic::Urem:
-      HelperName = H_urem_i64;
-      break;
-    case InstArithmetic::Srem:
-      HelperName = H_srem_i64;
-      break;
-    }
-    constexpr SizeT MaxSrcs = 2;
-    InstCall *Call = makeHelperCall(HelperName, Dest, MaxSrcs);
-    Call->addArg(Src0);
-    Call->addArg(Src1);
-    lowerCall(Call);
-    return;
-  }
-  }
-
   Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
   Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
   Variable *T_Lo = makeReg(DestLo->getType());
   Variable *T_Hi = makeReg(DestHi->getType());
 
   switch (Op) {
   case InstArithmetic::_num:
     llvm::report_fatal_error("Unknown arithmetic operator");
     return;
   case InstArithmetic::Add: {
@@ skipping 504 matching lines @@
   // difficult to determine (constant may be moved to a register).
   // * Handle floating point arithmetic separately: they require Src1 to be
   // legalized to a register.
   switch (Inst->getOp()) {
   default:
     break;
   case InstArithmetic::Udiv: {
     constexpr bool NotRemainder = false;
     Variable *Src0R = legalizeToReg(Src0);
     lowerIDivRem(Dest, T, Src0R, Src1, &TargetARM32::_uxt, &TargetARM32::_udiv,
-                 H_udiv_i32, NotRemainder);
+                 NotRemainder);
     return;
   }
   case InstArithmetic::Sdiv: {
     constexpr bool NotRemainder = false;
     Variable *Src0R = legalizeToReg(Src0);
     lowerIDivRem(Dest, T, Src0R, Src1, &TargetARM32::_sxt, &TargetARM32::_sdiv,
-                 H_sdiv_i32, NotRemainder);
+                 NotRemainder);
     return;
   }
   case InstArithmetic::Urem: {
     constexpr bool IsRemainder = true;
     Variable *Src0R = legalizeToReg(Src0);
     lowerIDivRem(Dest, T, Src0R, Src1, &TargetARM32::_uxt, &TargetARM32::_udiv,
-                 H_urem_i32, IsRemainder);
+                 IsRemainder);
     return;
   }
   case InstArithmetic::Srem: {
     constexpr bool IsRemainder = true;
     Variable *Src0R = legalizeToReg(Src0);
     lowerIDivRem(Dest, T, Src0R, Src1, &TargetARM32::_sxt, &TargetARM32::_sdiv,
-                 H_srem_i32, IsRemainder);
+                 IsRemainder);
     return;
   }
   case InstArithmetic::Frem: {
-    constexpr SizeT MaxSrcs = 2;
-    Variable *Src0R = legalizeToReg(Src0);
-    Type Ty = DestTy;
-    InstCall *Call = makeHelperCall(
-        isFloat32Asserting32Or64(Ty) ? H_frem_f32 : H_frem_f64, Dest, MaxSrcs);
-    Call->addArg(Src0R);
-    Call->addArg(Src1);
-    lowerCall(Call);
-    return;
+    if (!isScalarFloatingType(DestTy)) {
+      llvm::report_fatal_error("Unexpected type when lowering frem.");
+    }
+    llvm::report_fatal_error("Frem should have already been lowered.");
   }
   case InstArithmetic::Fadd: {
     Variable *Src0R = legalizeToReg(Src0);
     Variable *Src1R = legalizeToReg(Src1);
     _vadd(T, Src0R, Src1R);
     _mov(Dest, T);
     return;
   }
   case InstArithmetic::Fsub: {
     Variable *Src0R = legalizeToReg(Src0);
@@ skipping 398 matching lines @@
   case CondARM32::kNone:
     _br(TargetFalse);
     break;
   case CondARM32::AL:
     _br(TargetTrue);
     break;
   }
 }
 
 void TargetARM32::lowerCall(const InstCall *Instr) {
+  Operand *CallTarget = Instr->getCallTarget();
+  if (Instr->isTargetHelperCall()) {
+    auto TargetHelperPreamble = ARM32HelpersPreamble.find(CallTarget);
+    if (TargetHelperPreamble != ARM32HelpersPreamble.end()) {
+      (this->*TargetHelperPreamble->second)(Instr);
+    }
+  }
   MaybeLeafFunc = false;
   NeedsStackAlignment = true;
 
   // Assign arguments to registers and stack. Also reserve stack.
   TargetARM32::CallingConv CC;
   // Pair of Arg Operand -> GPR number assignments.
   llvm::SmallVector<std::pair<Operand *, int32_t>,
                     TargetARM32::CallingConv::ARM32_MAX_GPR_ARG> GPRArgs;
   llvm::SmallVector<std::pair<Operand *, int32_t>,
                     TargetARM32::CallingConv::ARM32_MAX_FP_REG_UNITS> FPArgs;
@@ skipping 102 matching lines @@
     case IceType_v8i1:
     case IceType_v16i1:
     case IceType_v16i8:
     case IceType_v8i16:
     case IceType_v4i32:
     case IceType_v4f32:
       ReturnReg = makeReg(Dest->getType(), RegARM32::Reg_q0);
       break;
     }
   }
-  Operand *CallTarget = Instr->getCallTarget();
   // TODO(jvoung): Handle sandboxing. const bool NeedSandboxing =
   // Ctx->getFlags().getUseSandboxing();
 
   // Allow ConstantRelocatable to be left alone as a direct call, but force
   // other constants like ConstantInteger32 to be in a register and make it an
   // indirect call.
   if (!llvm::isa<ConstantRelocatable>(CallTarget)) {
     CallTarget = legalize(CallTarget, Legal_Reg);
   }
 
@@ skipping 16 matching lines @@
 
   // Insert a register-kill pseudo instruction.
   Context.insert(InstFakeKill::create(Func, NewCall));
 
   // Generate a FakeUse to keep the call live if necessary.
   if (Instr->hasSideEffects() && ReturnReg) {
     Inst *FakeUse = InstFakeUse::create(Func, ReturnReg);
     Context.insert(FakeUse);
   }
 
-  if (!Dest)
-    return;
+  if (Dest != nullptr) {
+    // Assign the result of the call to Dest.
+    if (ReturnReg != nullptr) {
+      if (ReturnRegHi) {
+        auto *Dest64On32 = llvm::cast<Variable64On32>(Dest);
+        Variable *DestLo = Dest64On32->getLo();
+        Variable *DestHi = Dest64On32->getHi();
+        _mov(DestLo, ReturnReg);
+        _mov(DestHi, ReturnRegHi);
+      } else {
+        if (isFloatingType(Dest->getType()) || isVectorType(Dest->getType())) {
+          _mov(Dest, ReturnReg);
+        } else {
+          assert(isIntegerType(Dest->getType()) &&
+                 typeWidthInBytes(Dest->getType()) <= 4);
+          _mov(Dest, ReturnReg);
+        }
+      }
+    }
+  }
 
-  // Assign the result of the call to Dest.
-  if (ReturnReg) {
-    if (ReturnRegHi) {
-      auto *Dest64On32 = llvm::cast<Variable64On32>(Dest);
-      Variable *DestLo = Dest64On32->getLo();
-      Variable *DestHi = Dest64On32->getHi();
-      _mov(DestLo, ReturnReg);
-      _mov(DestHi, ReturnRegHi);
-    } else {
-      if (isFloatingType(Dest->getType()) || isVectorType(Dest->getType())) {
-        _mov(Dest, ReturnReg);
-      } else {
-        assert(isIntegerType(Dest->getType()) &&
-               typeWidthInBytes(Dest->getType()) <= 4);
-        _mov(Dest, ReturnReg);
-      }
+  if (Instr->isTargetHelperCall()) {
+    auto TargetHelpersPostamble = ARM32HelpersPostamble.find(CallTarget);
+    if (TargetHelpersPostamble != ARM32HelpersPostamble.end()) {
+      (this->*TargetHelpersPostamble->second)(Instr);
     }
   }
 }
 
 namespace {
 void configureBitcastTemporary(Variable64On32 *Var) {
   Var->setMustNotHaveReg();
   Var->getHi()->setMustHaveReg();
   Var->getLo()->setMustHaveReg();
 }
@@ skipping 156 matching lines @@
       Variable *T = makeReg(Dest->getType());
       Context.insert(InstFakeDef::create(Func, T, legalizeToReg(Src0)));
       _mov(Dest, T);
       UnimplementedError(Func->getContext()->getFlags());
       break;
     }
 
     const bool DestIsSigned = CastKind == InstCast::Fptosi;
     const bool Src0IsF32 = isFloat32Asserting32Or64(Src0->getType());
     if (llvm::isa<Variable64On32>(Dest)) {
-      const char *HelperName =
-          Src0IsF32 ? (DestIsSigned ? H_fptosi_f32_i64 : H_fptoui_f32_i64)
-                    : (DestIsSigned ? H_fptosi_f64_i64 : H_fptoui_f64_i64);
-      static constexpr SizeT MaxSrcs = 1;
-      InstCall *Call = makeHelperCall(HelperName, Dest, MaxSrcs);
-      Call->addArg(Src0);
-      lowerCall(Call);
-      break;
+      llvm::report_fatal_error("fp-to-i64 should have been pre-lowered.");
     }
     // fptosi:
     //     t1.fp = vcvt src0.fp
     //     t2.i32 = vmov t1.fp
     //     dest.int = conv t2.i32     @ Truncates the result if needed.
     // fptoui:
     //     t1.fp = vcvt src0.fp
     //     t2.u32 = vmov t1.fp
     //     dest.uint = conv t2.u32    @ Truncates the result if needed.
     Variable *Src0R = legalizeToReg(Src0);
@@ skipping 17 matching lines @@
     if (isVectorType(Dest->getType())) {
       Variable *T = makeReg(Dest->getType());
       Context.insert(InstFakeDef::create(Func, T, legalizeToReg(Src0)));
       _mov(Dest, T);
       UnimplementedError(Func->getContext()->getFlags());
       break;
     }
     const bool SourceIsSigned = CastKind == InstCast::Sitofp;
     const bool DestIsF32 = isFloat32Asserting32Or64(Dest->getType());
     if (Src0->getType() == IceType_i64) {
-      const char *HelperName =
-          DestIsF32 ? (SourceIsSigned ? H_sitofp_i64_f32 : H_uitofp_i64_f32)
-                    : (SourceIsSigned ? H_sitofp_i64_f64 : H_uitofp_i64_f64);
-      static constexpr SizeT MaxSrcs = 1;
-      InstCall *Call = makeHelperCall(HelperName, Dest, MaxSrcs);
-      Call->addArg(Src0);
-      lowerCall(Call);
-      break;
+      llvm::report_fatal_error("i64-to-fp should have been pre-lowered.");
     }
     // sitofp:
     //     t1.i32 = sext src.int    @ sign-extends src0 if needed.
     //     t2.fp32 = vmov t1.i32
     //     t3.fp = vcvt.{fp}.s32    @ fp is either f32 or f64
     // uitofp:
     //     t1.i32 = zext src.int    @ zero-extends src0 if needed.
     //     t2.fp32 = vmov t1.i32
     //     t3.fp = vcvt.{fp}.s32    @ fp is either f32 or f64
     if (Src0->getType() != IceType_i32) {
@@ skipping 665 matching lines @@
   }
   lowerAssign(InstAssign::create(Func, Dest, PtrContentsReg));
   if (auto *Dest64 = llvm::dyn_cast<Variable64On32>(Dest)) {
     Context.insert(InstFakeUse::create(Func, Dest64->getLo()));
     Context.insert(InstFakeUse::create(Func, Dest64->getHi()));
   } else {
     Context.insert(InstFakeUse::create(Func, Dest));
   }
 }
 
+void TargetARM32::Ctpop64Postamble(const InstCall *Instr) {
+  Operand *Arg0 = Instr->getArg(0);
+  if (isInt32Asserting32Or64(Arg0->getType())) {
+    return;
+  }
+  // The popcount helpers always return 32-bit values, while the intrinsic's
+  // signature matches some 64-bit platform's native instructions and expect to
+  // fill a 64-bit reg. Thus, clear the upper bits of the dest just in case the
+  // user doesn't do that in the IR or doesn't toss the bits via truncate.
+  Variable *DestHi = llvm::cast<Variable>(hiOperand(Instr->getDest()));
+  Variable *T = makeReg(IceType_i32);
+  Operand *_0 =
+      legalize(Ctx->getConstantZero(IceType_i32), Legal_Reg | Legal_Flex);
+  _mov(T, _0);
+  _mov(DestHi, T);
+}
+
 void TargetARM32::lowerIntrinsicCall(const InstIntrinsicCall *Instr) {
   Variable *Dest = Instr->getDest();
   Type DestTy = (Dest != nullptr) ? Dest->getType() : IceType_void;
   Intrinsics::IntrinsicID ID = Instr->getIntrinsicInfo().ID;
   switch (ID) {
   case Intrinsics::AtomicFence:
   case Intrinsics::AtomicFenceAll:
     assert(Dest == nullptr);
     _dmb();
     return;
@@ skipping 296 matching lines @@
       _rev(T, ValR);
       if (Val->getType() == IceType_i16) {
         Operand *_16 = shAmtImm(16);
         _lsr(T, T, _16);
       }
       _mov(Dest, T);
     }
     return;
   }
   case Intrinsics::Ctpop: {
-    Operand *Val = Instr->getArg(0);
-    InstCall *Call = makeHelperCall(isInt32Asserting32Or64(Val->getType())
-                                        ? H_call_ctpop_i32
-                                        : H_call_ctpop_i64,
-                                    Dest, 1);
-    Call->addArg(Val);
-    lowerCall(Call);
-    // The popcount helpers always return 32-bit values, while the intrinsic's
-    // signature matches some 64-bit platform's native instructions and expect
-    // to fill a 64-bit reg. Thus, clear the upper bits of the dest just in
-    // case the user doesn't do that in the IR or doesn't toss the bits via
-    // truncate.
-    if (Val->getType() == IceType_i64) {
-      Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
-      Constant *Zero = Ctx->getConstantZero(IceType_i32);
-      Variable *T = makeReg(Zero->getType());
-      _mov(T, Zero);
-      _mov(DestHi, T);
-    }
-    return;
+    llvm::report_fatal_error("Ctpop should have been prelowered.");
   }
   case Intrinsics::Ctlz: {
     // The "is zero undef" parameter is ignored and we always return a
     // well-defined value.
     Operand *Val = Instr->getArg(0);
     Variable *ValLoR;
     Variable *ValHiR = nullptr;
     if (Val->getType() == IceType_i64) {
       Val = legalizeUndef(Val);
       ValLoR = legalizeToReg(loOperand(Val));
@@ skipping 36 matching lines @@
       Context.insert(InstFakeDef::create(Func, T));
       _mov(Dest, T);
       UnimplementedError(Func->getContext()->getFlags());
       return;
     }
     _vabs(T, legalizeToReg(Instr->getArg(0)));
     _mov(Dest, T);
     return;
   }
   case Intrinsics::Longjmp: {
-    InstCall *Call = makeHelperCall(H_call_longjmp, nullptr, 2);
-    Call->addArg(Instr->getArg(0));
-    Call->addArg(Instr->getArg(1));
-    lowerCall(Call);
-    return;
+    llvm::report_fatal_error("longjmp should have been prelowered.");
   }
   case Intrinsics::Memcpy: {
-    // In the future, we could potentially emit an inline memcpy/memset, etc.
-    // for intrinsic calls w/ a known length.
-    InstCall *Call = makeHelperCall(H_call_memcpy, nullptr, 3);
-    Call->addArg(Instr->getArg(0));
-    Call->addArg(Instr->getArg(1));
-    Call->addArg(Instr->getArg(2));
-    lowerCall(Call);
-    return;
+    llvm::report_fatal_error("memcpy should have been prelowered.");
   }
   case Intrinsics::Memmove: {
-    InstCall *Call = makeHelperCall(H_call_memmove, nullptr, 3);
-    Call->addArg(Instr->getArg(0));
-    Call->addArg(Instr->getArg(1));
-    Call->addArg(Instr->getArg(2));
-    lowerCall(Call);
-    return;
+    llvm::report_fatal_error("memmove should have been prelowered.");
   }
   case Intrinsics::Memset: {
-    // The value operand needs to be extended to a stack slot size because the
-    // PNaCl ABI requires arguments to be at least 32 bits wide.
-    Operand *ValOp = Instr->getArg(1);
-    assert(ValOp->getType() == IceType_i8);
-    Variable *ValExt = Func->makeVariable(stackSlotType());
-    lowerCast(InstCast::create(Func, InstCast::Zext, ValExt, ValOp));
-    // Technically, ARM has their own __aeabi_memset, but we can use plain
-    // memset too. The value and size argument need to be flipped if we ever
-    // decide to use __aeabi_memset.
-    InstCall *Call = makeHelperCall(H_call_memset, nullptr, 3);
-    Call->addArg(Instr->getArg(0));
-    Call->addArg(ValExt);
-    Call->addArg(Instr->getArg(2));
-    lowerCall(Call);
-    return;
+    llvm::report_fatal_error("memmove should have been prelowered.");
   }
   case Intrinsics::NaClReadTP: {
-    if (Ctx->getFlags().getUseSandboxing()) {
-      UnimplementedError(Func->getContext()->getFlags());
-    } else {
-      InstCall *Call = makeHelperCall(H_call_read_tp, Dest, 0);
-      lowerCall(Call);
-    }
-    return;
+    llvm::report_fatal_error("nacl-read-tp should have been prelowered.");
   }
   case Intrinsics::Setjmp: {
-    InstCall *Call = makeHelperCall(H_call_setjmp, Dest, 1);
-    Call->addArg(Instr->getArg(0));
-    lowerCall(Call);
-    return;
+    llvm::report_fatal_error("setjmp should have been prelowered.");
   }
   case Intrinsics::Sqrt: {
     Variable *Src = legalizeToReg(Instr->getArg(0));
     Variable *T = makeReg(Dest->getType());
     _vsqrt(T, Src);
     _mov(Dest, T);
     return;
   }
   case Intrinsics::Stacksave: {
     Variable *SP = getPhysicalRegister(RegARM32::Reg_sp);
@@ skipping 1496 matching lines @@
   // Technically R9 is used for TLS with Sandboxing, and we reserve it.
   // However, for compatibility with current NaCl LLVM, don't claim that.
   Str << ".eabi_attribute 14, 3   @ Tag_ABI_PCS_R9_use: Not used\n";
 }
 
 llvm::SmallBitVector TargetARM32::TypeToRegisterSet[IceType_NUM];
 llvm::SmallBitVector TargetARM32::RegisterAliases[RegARM32::Reg_NUM];
 llvm::SmallBitVector TargetARM32::ScratchRegs;
 
 } // end of namespace Ice