Subzero. Native 64-bit int arithmetic on x86-64.

src/IceTargetLoweringX86BaseImpl.h

 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==//
 //
 //                        The Subzero Code Generator
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 ///
 /// \file
@@ skipping 59 matching lines @@
   // NumUses counts the number of times Var is used as a source operand in the
   // basic block.  If IsComplex is true and there is more than one use of Var,
   // then the folding optimization is disabled for Var.
   uint32_t NumUses = 0;
 };
 
 template <class MachineTraits> class BoolFolding {
 public:
   enum BoolFoldingProducerKind {
     PK_None,
+    // TODO(jpp): PK_Icmp32 is no longer meaningful. Rename to PK_IcmpNative.
     PK_Icmp32,
     PK_Icmp64,
     PK_Fcmp,
     PK_Trunc
   };
 
   /// Currently the actual enum values are not used (other than CK_None), but we
   /// go ahead and produce them anyway for symmetry with the
   /// BoolFoldingProducerKind.
   enum BoolFoldingConsumerKind { CK_None, CK_Br, CK_Select, CK_Sext, CK_Zext };
@@ skipping 23 matching lines @@
 };
 
 template <class MachineTraits>
 BoolFoldingEntry<MachineTraits>::BoolFoldingEntry(Inst *I)
     : Instr(I), IsComplex(BoolFolding<MachineTraits>::hasComplexLowering(I)) {}
 
 template <class MachineTraits>
 typename BoolFolding<MachineTraits>::BoolFoldingProducerKind
 BoolFolding<MachineTraits>::getProducerKind(const Inst *Instr) {
   if (llvm::isa<InstIcmp>(Instr)) {
-    if (Instr->getSrc(0)->getType() != IceType_i64)
+    if (MachineTraits::Is64Bit || Instr->getSrc(0)->getType() != IceType_i64)
       return PK_Icmp32;
     return PK_None; // TODO(stichnot): actually PK_Icmp64;
   }
   return PK_None; // TODO(stichnot): remove this
 
   if (llvm::isa<InstFcmp>(Instr))
     return PK_Fcmp;
   if (auto *Cast = llvm::dyn_cast<InstCast>(Instr)) {
     switch (Cast->getCastKind()) {
     default:
@@ skipping 502 matching lines @@
       // instruction or equivalent.
       if (auto *Load = llvm::dyn_cast<InstLoad>(CurInst)) {
         // An InstLoad always qualifies.
         LoadDest = Load->getDest();
         const bool DoLegalize = false;
         LoadSrc = formMemoryOperand(Load->getSourceAddress(),
                                     LoadDest->getType(), DoLegalize);
       } else if (auto *Intrin = llvm::dyn_cast<InstIntrinsicCall>(CurInst)) {
         // An AtomicLoad intrinsic qualifies as long as it has a valid
         // memory ordering, and can be implemented in a single
-        // instruction (i.e., not i64).
+        // instruction (i.e., not i64 on x86-32).
         Intrinsics::IntrinsicID ID = Intrin->getIntrinsicInfo().ID;
         if (ID == Intrinsics::AtomicLoad &&
-            Intrin->getDest()->getType() != IceType_i64 &&
+            (Traits::Is64Bit || Intrin->getDest()->getType() != IceType_i64) &&
             Intrinsics::isMemoryOrderValid(
                 ID, getConstantMemoryOrder(Intrin->getArg(1)))) {
           LoadDest = Intrin->getDest();
           const bool DoLegalize = false;
           LoadSrc = formMemoryOperand(Intrin->getArg(0), LoadDest->getType(),
                                       DoLegalize);
         }
       }
       // A Load instruction can be folded into the following
       // instruction only if the following instruction ends the Load's
@@ skipping 57 matching lines @@
 template <class Machine>
 bool TargetX86Base<Machine>::doBranchOpt(Inst *I, const CfgNode *NextNode) {
   if (auto *Br = llvm::dyn_cast<typename Traits::Insts::Br>(I)) {
     return Br->optimizeBranch(NextNode);
   }
   return false;
 }
 
 template <class Machine>
 Variable *TargetX86Base<Machine>::getPhysicalRegister(SizeT RegNum, Type Ty) {
+  // Special case: never allow partial reads/writes to/from %rBP and %rSP.
+  if (RegNum == Traits::RegisterSet::Reg_esp ||
+      RegNum == Traits::RegisterSet::Reg_ebp)
+    Ty = Traits::WordType;
   if (Ty == IceType_void)
     Ty = IceType_i32;
   if (PhysicalRegisters[Ty].empty())
     PhysicalRegisters[Ty].resize(Traits::RegisterSet::Reg_NUM);
   assert(RegNum < PhysicalRegisters[Ty].size());
   Variable *Reg = PhysicalRegisters[Ty][RegNum];
   if (Reg == nullptr) {
     Reg = Func->makeVariable(Ty);
     Reg->setRegNum(RegNum);
     PhysicalRegisters[Ty][RegNum] = Reg;
@@ skipping 26 matching lines @@
   }
   int32_t Offset = Var->getStackOffset();
   int32_t BaseRegNum = Var->getBaseRegNum();
   if (BaseRegNum == Variable::NoRegister) {
     BaseRegNum = getFrameOrStackReg();
     if (!hasFramePointer())
       Offset += getStackAdjustment();
   }
   if (Offset)
     Str << Offset;
-  const Type FrameSPTy = IceType_i32;
+  const Type FrameSPTy = Traits::WordType;
   Str << "(%" << getRegName(BaseRegNum, FrameSPTy) << ")";
 }
 
 template <class Machine>
 typename TargetX86Base<Machine>::Traits::Address
 TargetX86Base<Machine>::stackVarToAsmOperand(const Variable *Var) const {
   if (Var->hasReg())
     llvm_unreachable("Stack Variable has a register assigned");
   if (Var->getWeight().isInf()) {
     llvm_unreachable("Infinite-weight Variable has no register assigned");
@@ skipping 19 matching lines @@
 /// function generates an instruction to copy Arg into its assigned
 /// register if applicable.
 template <class Machine>
 void TargetX86Base<Machine>::finishArgumentLowering(Variable *Arg,
                                                     Variable *FramePtr,
                                                     size_t BasicFrameOffset,
                                                     size_t &InArgsSizeBytes) {
   Variable *Lo = Arg->getLo();
   Variable *Hi = Arg->getHi();
   Type Ty = Arg->getType();
-  if (Lo && Hi && Ty == IceType_i64) {
-    // TODO(jpp): This special case is not needed for x86-64.
+  if (!Traits::Is64Bit && Lo && Hi && Ty == IceType_i64) {
     assert(Lo->getType() != IceType_i64); // don't want infinite recursion
     assert(Hi->getType() != IceType_i64); // don't want infinite recursion
     finishArgumentLowering(Lo, FramePtr, BasicFrameOffset, InArgsSizeBytes);
     finishArgumentLowering(Hi, FramePtr, BasicFrameOffset, InArgsSizeBytes);
     return;
   }
   if (isVectorType(Ty)) {
     InArgsSizeBytes = Traits::applyStackAlignment(InArgsSizeBytes);
   }
   Arg->setStackOffset(BasicFrameOffset + InArgsSizeBytes);
   InArgsSizeBytes += typeWidthInBytesOnStack(Ty);
   if (Arg->hasReg()) {
-    assert(Ty != IceType_i64);
+    assert(Ty != IceType_i64 || Traits::Is64Bit);
     typename Traits::X86OperandMem *Mem = Traits::X86OperandMem::create(
         Func, Ty, FramePtr, Ctx->getConstantInt32(Arg->getStackOffset()));
     if (isVectorType(Arg->getType())) {
       _movp(Arg, Mem);
     } else {
       _mov(Arg, Mem);
     }
     // This argument-copying instruction uses an explicit Traits::X86OperandMem
     // operand instead of a Variable, so its fill-from-stack operation has to be
     // tracked separately for statistics.
     Ctx->statsUpdateFills();
   }
 }
 
 template <class Machine> Type TargetX86Base<Machine>::stackSlotType() {
-  // TODO(jpp): this is wrong for x86-64.
-  return IceType_i32;
+  return Traits::WordType;
 }
 
-template <class Machine> void TargetX86Base<Machine>::split64(Variable *Var) {
+template <class Machine>
+template <typename T>
+typename std::enable_if<!T::Is64Bit, void>::type
+TargetX86Base<Machine>::split64(Variable *Var) {
   switch (Var->getType()) {
   default:
     return;
   case IceType_i64:
   // TODO: Only consider F64 if we need to push each half when
   // passing as an argument to a function call.  Note that each half
   // is still typed as I32.
   case IceType_f64:
     break;
   }
@@ skipping 11 matching lines @@
     Hi->setName(Func, Var->getName(Func) + "__hi");
   }
   Var->setLoHi(Lo, Hi);
   if (Var->getIsArg()) {
     Lo->setIsArg();
     Hi->setIsArg();
   }
 }
 
 template <class Machine>
-Operand *TargetX86Base<Machine>::loOperand(Operand *Operand) {
+template <typename T>
+typename std::enable_if<!T::Is64Bit, Operand>::type *
+TargetX86Base<Machine>::loOperand(Operand *Operand) {
   assert(Operand->getType() == IceType_i64 ||
          Operand->getType() == IceType_f64);
   if (Operand->getType() != IceType_i64 && Operand->getType() != IceType_f64)
     return Operand;
   if (auto *Var = llvm::dyn_cast<Variable>(Operand)) {
     split64(Var);
     return Var->getLo();
   }
   if (auto *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) {
     auto *ConstInt = llvm::dyn_cast<ConstantInteger32>(
         Ctx->getConstantInt32(static_cast<int32_t>(Const->getValue())));
     // Check if we need to blind/pool the constant.
     return legalize(ConstInt);
   }
   if (auto *Mem = llvm::dyn_cast<typename Traits::X86OperandMem>(Operand)) {
     auto *MemOperand = Traits::X86OperandMem::create(
         Func, IceType_i32, Mem->getBase(), Mem->getOffset(), Mem->getIndex(),
         Mem->getShift(), Mem->getSegmentRegister());
     // Test if we should randomize or pool the offset, if so randomize it or
     // pool it then create mem operand with the blinded/pooled constant.
     // Otherwise, return the mem operand as ordinary mem operand.
     return legalize(MemOperand);
   }
   llvm_unreachable("Unsupported operand type");
   return nullptr;
 }
 
 template <class Machine>
-Operand *TargetX86Base<Machine>::hiOperand(Operand *Operand) {
+template <typename T>
+typename std::enable_if<!T::Is64Bit, Operand>::type *
+TargetX86Base<Machine>::hiOperand(Operand *Operand) {
   assert(Operand->getType() == IceType_i64 ||
          Operand->getType() == IceType_f64);
   if (Operand->getType() != IceType_i64 && Operand->getType() != IceType_f64)
     return Operand;
   if (auto *Var = llvm::dyn_cast<Variable>(Operand)) {
     split64(Var);
     return Var->getHi();
   }
   if (auto *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) {
     auto *ConstInt = llvm::dyn_cast<ConstantInteger32>(
@@ skipping 181 matching lines @@
 void TargetX86Base<Machine>::lowerArithmetic(const InstArithmetic *Inst) {
   Variable *Dest = Inst->getDest();
   Operand *Src0 = legalize(Inst->getSrc(0));
   Operand *Src1 = legalize(Inst->getSrc(1));
   if (Inst->isCommutative()) {
     if (!llvm::isa<Variable>(Src0) && llvm::isa<Variable>(Src1))
       std::swap(Src0, Src1);
     if (llvm::isa<Constant>(Src0) && !llvm::isa<Constant>(Src1))
       std::swap(Src0, Src1);
   }
-  if (Dest->getType() == IceType_i64) {
-    // These helper-call-involved instructions are lowered in this
+  if (!Traits::Is64Bit && Dest->getType() == IceType_i64) {
+    // These x86-32 helper-call-involved instructions are lowered in this
     // separate switch. This is because loOperand() and hiOperand()
     // may insert redundant instructions for constant blinding and
     // pooling. Such redundant instructions will fail liveness analysis
     // under -Om1 setting. And, actually these arguments do not need
     // to be processed with loOperand() and hiOperand() to be used.
     switch (Inst->getOp()) {
     case InstArithmetic::Udiv: {
       const SizeT MaxSrcs = 2;
       InstCall *Call = makeHelperCall(H_udiv_i64, Dest, MaxSrcs);
       Call->addArg(Inst->getSrc(0));
@@ skipping 527 matching lines @@
       // mov %ah, %al because it would make x86-64 codegen more complicated. If
       // this ever becomes a problem we can introduce a pseudo rem instruction
       // that returns the remainder in %al directly (and uses a mov for copying
       // %ah to %al.)
       static constexpr uint8_t AlSizeInBits = 8;
       _shr(T_eax, Ctx->getConstantInt8(AlSizeInBits));
       _mov(Dest, T);
       Context.insert(InstFakeUse::create(Func, T_eax));
     } else {
       Constant *Zero = Ctx->getConstantZero(IceType_i32);
-      _mov(T_edx, Zero, Traits::RegisterSet::Reg_edx);
+      T_edx = makeReg(Dest->getType(), Traits::RegisterSet::Reg_edx);
+      _mov(T_edx, Zero);
       _mov(T, Src0, Traits::RegisterSet::Reg_eax);
       _div(T_edx, Src1, T);
       _mov(Dest, T_edx);
     }
     break;
   case InstArithmetic::Srem:
     // TODO(stichnot): Enable this after doing better performance
     // and cross testing.
     if (false && Ctx->getFlags().getOptLevel() >= Opt_1) {
       // Optimize mod by constant power of 2, but not for Om1 or O0,
@@ skipping 44 matching lines @@
       // shr $8, %eax shifts ah (i.e., the 8 bit remainder) into al. We don't
       // mov %ah, %al because it would make x86-64 codegen more complicated. If
       // this ever becomes a problem we can introduce a pseudo rem instruction
       // that returns the remainder in %al directly (and uses a mov for copying
       // %ah to %al.)
       static constexpr uint8_t AlSizeInBits = 8;
       _shr(T_eax, Ctx->getConstantInt8(AlSizeInBits));
       _mov(Dest, T);
       Context.insert(InstFakeUse::create(Func, T_eax));
     } else {
-      T_edx = makeReg(IceType_i32, Traits::RegisterSet::Reg_edx);
+      T_edx = makeReg(Dest->getType(), Traits::RegisterSet::Reg_edx);
       _mov(T, Src0, Traits::RegisterSet::Reg_eax);
       _cbwdq(T_edx, T);
       _idiv(T_edx, Src1, T);
       _mov(Dest, T_edx);
     }
     break;
   case InstArithmetic::Fadd:
     _mov(T, Src0);
     _addss(T, Src1);
     _mov(Dest, T);
@@ skipping 23 matching lines @@
     return lowerCall(Call);
   }
   }
 }
 
 template <class Machine>
 void TargetX86Base<Machine>::lowerAssign(const InstAssign *Inst) {
   Variable *Dest = Inst->getDest();
   Operand *Src0 = Inst->getSrc(0);
   assert(Dest->getType() == Src0->getType());
-  if (Dest->getType() == IceType_i64) {
+  if (!Traits::Is64Bit && Dest->getType() == IceType_i64) {
     Src0 = legalize(Src0);
     Operand *Src0Lo = loOperand(Src0);
     Operand *Src0Hi = hiOperand(Src0);
     Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
     Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
     Variable *T_Lo = nullptr, *T_Hi = nullptr;
     _mov(T_Lo, Src0Lo);
     _mov(DestLo, T_Lo);
     _mov(T_Hi, Src0Hi);
     _mov(DestHi, T_Hi);
@@ skipping 84 matching lines @@
         SizeT ShiftAmount =
             Traits::X86_CHAR_BIT * typeWidthInBytes(typeElementType(DestTy)) -
             1;
         Constant *ShiftConstant = Ctx->getConstantInt8(ShiftAmount);
         Variable *T = makeReg(DestTy);
         _movp(T, Src0RM);
         _psll(T, ShiftConstant);
         _psra(T, ShiftConstant);
         _movp(Dest, T);
       }
-    } else if (Dest->getType() == IceType_i64) {
+    } else if (!Traits::Is64Bit && Dest->getType() == IceType_i64) {
       // t1=movsx src; t2=t1; t2=sar t2, 31; dst.lo=t1; dst.hi=t2
       Constant *Shift = Ctx->getConstantInt32(31);
       Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
       Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
       Variable *T_Lo = makeReg(DestLo->getType());
       if (Src0RM->getType() == IceType_i32) {
         _mov(T_Lo, Src0RM);
       } else if (Src0RM->getType() == IceType_i1) {
         _movzx(T_Lo, Src0RM);
         _shl(T_Lo, Shift);
@@ skipping 39 matching lines @@
   case InstCast::Zext: {
     Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
     if (isVectorType(Dest->getType())) {
       // onemask = materialize(1,1,...); dest = onemask & src
       Type DestTy = Dest->getType();
       Variable *OneMask = makeVectorOfOnes(DestTy);
       Variable *T = makeReg(DestTy);
       _movp(T, Src0RM);
       _pand(T, OneMask);
       _movp(Dest, T);
-    } else if (Dest->getType() == IceType_i64) {
+    } else if (!Traits::Is64Bit && Dest->getType() == IceType_i64) {
       // t1=movzx src; dst.lo=t1; dst.hi=0
       Constant *Zero = Ctx->getConstantZero(IceType_i32);
       Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
       Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
       Variable *Tmp = makeReg(DestLo->getType());
       if (Src0RM->getType() == IceType_i32) {
         _mov(Tmp, Src0RM);
       } else {
         _movzx(Tmp, Src0RM);
       }
       if (Src0RM->getType() == IceType_i1) {
         Constant *One = Ctx->getConstantInt32(1);
         _and(Tmp, One);
       }
       _mov(DestLo, Tmp);
       _mov(DestHi, Zero);
     } else if (Src0RM->getType() == IceType_i1) {
       // t = Src0RM; t &= 1; Dest = t
       Constant *One = Ctx->getConstantInt32(1);
       Type DestTy = Dest->getType();
-      Variable *T;
+      Variable *T = nullptr;
       if (DestTy == IceType_i8) {
-        T = makeReg(DestTy);
         _mov(T, Src0RM);
       } else {
+        assert(DestTy != IceType_i1);
+        assert(Traits::Is64Bit || DestTy != IceType_i64);
         // Use 32-bit for both 16-bit and 32-bit, since 32-bit ops are shorter.
-        T = makeReg(IceType_i32);
+        // In x86-64 we need to widen T to 64-bits to ensure that T -- if
+        // written to the stack (i.e., in -Om1) will be fully zero-extended.
+        T = makeReg(DestTy == IceType_i64 ? IceType_i64 : IceType_i32);
         _movzx(T, Src0RM);
       }
       _and(T, One);
       _mov(Dest, T);
     } else {
       // t1 = movzx src; dst = t1
       Variable *T = makeReg(Dest->getType());
       _movzx(T, Src0RM);
       _mov(Dest, T);
     }
     break;
   }
   case InstCast::Trunc: {
     if (isVectorType(Dest->getType())) {
       // onemask = materialize(1,1,...); dst = src & onemask
       Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
       Type Src0Ty = Src0RM->getType();
       Variable *OneMask = makeVectorOfOnes(Src0Ty);
       Variable *T = makeReg(Dest->getType());
       _movp(T, Src0RM);
       _pand(T, OneMask);
       _movp(Dest, T);
     } else {
       Operand *Src0 = legalizeUndef(Inst->getSrc(0));
-      if (Src0->getType() == IceType_i64)
+      if (!Traits::Is64Bit && Src0->getType() == IceType_i64)
         Src0 = loOperand(Src0);
       Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem);
       // t1 = trunc Src0RM; Dest = t1
       Variable *T = nullptr;
       _mov(T, Src0RM);
       if (Dest->getType() == IceType_i1)
         _and(T, Ctx->getConstantInt1(1));
       _mov(Dest, T);
     }
     break;
@@ skipping 10 matching lines @@
   case InstCast::Fptosi:
     if (isVectorType(Dest->getType())) {
       assert(Dest->getType() == IceType_v4i32 &&
              Inst->getSrc(0)->getType() == IceType_v4f32);
       Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
       if (llvm::isa<typename Traits::X86OperandMem>(Src0RM))
         Src0RM = legalizeToReg(Src0RM);
       Variable *T = makeReg(Dest->getType());
       _cvt(T, Src0RM, Traits::Insts::Cvt::Tps2dq);
       _movp(Dest, T);
-    } else if (Dest->getType() == IceType_i64) {
+    } else if (!Traits::Is64Bit && Dest->getType() == IceType_i64) {
       // Use a helper for converting floating-point values to 64-bit
       // integers.  SSE2 appears to have no way to convert from xmm
       // registers to something like the edx:eax register pair, and
       // gcc and clang both want to use x87 instructions complete with
       // temporary manipulation of the status word.  This helper is
       // not needed for x86-64.
       split64(Dest);
       const SizeT MaxSrcs = 1;
       Type SrcType = Inst->getSrc(0)->getType();
       InstCall *Call =
           makeHelperCall(isFloat32Asserting32Or64(SrcType) ? H_fptosi_f32_i64
                                                            : H_fptosi_f64_i64,
                          Dest, MaxSrcs);
       Call->addArg(Inst->getSrc(0));
       lowerCall(Call);
     } else {
       Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
       // t1.i32 = cvt Src0RM; t2.dest_type = t1; Dest = t2.dest_type
-      Variable *T_1 = makeReg(IceType_i32);
+      Variable *T_1 = nullptr;
+      if (Traits::Is64Bit && Dest->getType() == IceType_i64) {
+        T_1 = makeReg(IceType_i64);
+      } else {
+        assert(Dest->getType() != IceType_i64);
+        T_1 = makeReg(IceType_i32);
+      }
+      // cvt() requires its integer argument to be a GPR.
+      T_1->setWeightInfinite();
       Variable *T_2 = makeReg(Dest->getType());
       _cvt(T_1, Src0RM, Traits::Insts::Cvt::Tss2si);
       _mov(T_2, T_1); // T_1 and T_2 may have different integer types
       if (Dest->getType() == IceType_i1)
         _and(T_2, Ctx->getConstantInt1(1));
       _mov(Dest, T_2);
     }
     break;
   case InstCast::Fptoui:
     if (isVectorType(Dest->getType())) {
       assert(Dest->getType() == IceType_v4i32 &&
              Inst->getSrc(0)->getType() == IceType_v4f32);
       const SizeT MaxSrcs = 1;
       InstCall *Call = makeHelperCall(H_fptoui_4xi32_f32, Dest, MaxSrcs);
       Call->addArg(Inst->getSrc(0));
       lowerCall(Call);
     } else if (Dest->getType() == IceType_i64 ||
-               Dest->getType() == IceType_i32) {
+               (!Traits::Is64Bit && Dest->getType() == IceType_i32)) {
       // Use a helper for both x86-32 and x86-64.
-      split64(Dest);
+      if (!Traits::Is64Bit)
+        split64(Dest);
       const SizeT MaxSrcs = 1;
       Type DestType = Dest->getType();
       Type SrcType = Inst->getSrc(0)->getType();
       IceString TargetString;
-      if (isInt32Asserting32Or64(DestType)) {
+      if (Traits::Is64Bit) {
+        TargetString = isFloat32Asserting32Or64(SrcType) ? H_fptoui_f32_i64
+                                                         : H_fptoui_f64_i64;
+      } else if (isInt32Asserting32Or64(DestType)) {
         TargetString = isFloat32Asserting32Or64(SrcType) ? H_fptoui_f32_i32
                                                          : H_fptoui_f64_i32;
       } else {
         TargetString = isFloat32Asserting32Or64(SrcType) ? H_fptoui_f32_i64
                                                          : H_fptoui_f64_i64;
       }
       InstCall *Call = makeHelperCall(TargetString, Dest, MaxSrcs);
       Call->addArg(Inst->getSrc(0));
       lowerCall(Call);
       return;
     } else {
       Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
       // t1.i32 = cvt Src0RM; t2.dest_type = t1; Dest = t2.dest_type
-      Variable *T_1 = makeReg(IceType_i32);
+      assert(Dest->getType() != IceType_i64);
+      Variable *T_1 = nullptr;
+      if (Traits::Is64Bit && Dest->getType() == IceType_i32) {
+        T_1 = makeReg(IceType_i64);
+      } else {
+        assert(Dest->getType() != IceType_i32);
+        T_1 = makeReg(IceType_i32);
+      }
+      T_1->setWeightInfinite();
       Variable *T_2 = makeReg(Dest->getType());
       _cvt(T_1, Src0RM, Traits::Insts::Cvt::Tss2si);
       _mov(T_2, T_1); // T_1 and T_2 may have different integer types
       if (Dest->getType() == IceType_i1)
         _and(T_2, Ctx->getConstantInt1(1));
       _mov(Dest, T_2);
     }
     break;
   case InstCast::Sitofp:
     if (isVectorType(Dest->getType())) {
       assert(Dest->getType() == IceType_v4f32 &&
              Inst->getSrc(0)->getType() == IceType_v4i32);
       Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
       if (llvm::isa<typename Traits::X86OperandMem>(Src0RM))
         Src0RM = legalizeToReg(Src0RM);
       Variable *T = makeReg(Dest->getType());
       _cvt(T, Src0RM, Traits::Insts::Cvt::Dq2ps);
       _movp(Dest, T);
-    } else if (Inst->getSrc(0)->getType() == IceType_i64) {
+    } else if (!Traits::Is64Bit && Inst->getSrc(0)->getType() == IceType_i64) {
       // Use a helper for x86-32.
       const SizeT MaxSrcs = 1;
       Type DestType = Dest->getType();
       InstCall *Call =
           makeHelperCall(isFloat32Asserting32Or64(DestType) ? H_sitofp_i64_f32
                                                             : H_sitofp_i64_f64,
                          Dest, MaxSrcs);
       // TODO: Call the correct compiler-rt helper function.
       Call->addArg(Inst->getSrc(0));
       lowerCall(Call);
       return;
     } else {
       Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
       // Sign-extend the operand.
       // t1.i32 = movsx Src0RM; t2 = Cvt t1.i32; Dest = t2
-      Variable *T_1 = makeReg(IceType_i32);
+      Variable *T_1 = nullptr;
+      if (Traits::Is64Bit && Src0RM->getType() == IceType_i64) {
+        T_1 = makeReg(IceType_i64);
+      } else {
+        assert(Src0RM->getType() != IceType_i64);
+        T_1 = makeReg(IceType_i32);
+      }
+      T_1->setWeightInfinite();
       Variable *T_2 = makeReg(Dest->getType());
-      if (Src0RM->getType() == IceType_i32)
+      if (Src0RM->getType() == T_1->getType())
         _mov(T_1, Src0RM);
       else
         _movsx(T_1, Src0RM);
       _cvt(T_2, T_1, Traits::Insts::Cvt::Si2ss);
       _mov(Dest, T_2);
     }
     break;
   case InstCast::Uitofp: {
     Operand *Src0 = Inst->getSrc(0);
     if (isVectorType(Src0->getType())) {
       assert(Dest->getType() == IceType_v4f32 &&
              Src0->getType() == IceType_v4i32);
       const SizeT MaxSrcs = 1;
       InstCall *Call = makeHelperCall(H_uitofp_4xi32_4xf32, Dest, MaxSrcs);
       Call->addArg(Src0);
       lowerCall(Call);
     } else if (Src0->getType() == IceType_i64 ||
-               Src0->getType() == IceType_i32) {
+               (!Traits::Is64Bit && Src0->getType() == IceType_i32)) {
       // Use a helper for x86-32 and x86-64.  Also use a helper for
       // i32 on x86-32.
       const SizeT MaxSrcs = 1;
       Type DestType = Dest->getType();
       IceString TargetString;
       if (isInt32Asserting32Or64(Src0->getType())) {
         TargetString = isFloat32Asserting32Or64(DestType) ? H_uitofp_i32_f32
                                                           : H_uitofp_i32_f64;
       } else {
         TargetString = isFloat32Asserting32Or64(DestType) ? H_uitofp_i64_f32
                                                           : H_uitofp_i64_f64;
       }
       InstCall *Call = makeHelperCall(TargetString, Dest, MaxSrcs);
       Call->addArg(Src0);
       lowerCall(Call);
       return;
     } else {
       Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem);
       // Zero-extend the operand.
       // t1.i32 = movzx Src0RM; t2 = Cvt t1.i32; Dest = t2
-      Variable *T_1 = makeReg(IceType_i32);
+      Variable *T_1 = nullptr;
+      if (Traits::Is64Bit && Src0RM->getType() == IceType_i32) {
+        T_1 = makeReg(IceType_i64);
+      } else {
+        assert(Src0RM->getType() != IceType_i64);
+        assert(Traits::Is64Bit || Src0RM->getType() != IceType_i32);
+        T_1 = makeReg(IceType_i32);
+      }
+      T_1->setWeightInfinite();
       Variable *T_2 = makeReg(Dest->getType());
-      if (Src0RM->getType() == IceType_i32)
+      if (Src0RM->getType() == T_1->getType())
         _mov(T_1, Src0RM);
       else
         _movzx(T_1, Src0RM);
       _cvt(T_2, T_1, Traits::Insts::Cvt::Si2ss);
       _mov(Dest, T_2);
     }
     break;
   }
   case InstCast::Bitcast: {
     Operand *Src0 = Inst->getSrc(0);
@@ skipping 35 matching lines @@
       typename Traits::SpillVariable *SpillVar =
           Func->makeVariable<typename Traits::SpillVariable>(SrcType);
       SpillVar->setLinkedTo(Dest);
       Variable *Spill = SpillVar;
       Spill->setWeight(RegWeight::Zero);
       _mov(T, Src0RM);
       _mov(Spill, T);
       _mov(Dest, Spill);
     } break;
     case IceType_i64: {
-      Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem);
-      assert(Src0RM->getType() == IceType_f64);
-      // a.i64 = bitcast b.f64 ==>
-      //   s.f64 = spill b.f64
-      //   t_lo.i32 = lo(s.f64)
-      //   a_lo.i32 = t_lo.i32
-      //   t_hi.i32 = hi(s.f64)
-      //   a_hi.i32 = t_hi.i32
-      Operand *SpillLo, *SpillHi;
-      if (auto *Src0Var = llvm::dyn_cast<Variable>(Src0RM)) {
+      assert(Src0->getType() == IceType_f64);
+      if (Traits::Is64Bit) {
+        // Movd requires its fp argument (in this case, the bitcast source) to
+        // be an xmm register.
+        Variable *Src0R = legalizeToReg(Src0);
+        Variable *T = makeReg(IceType_i64);
+        _movd(T, Src0R);
+        _mov(Dest, T);
+      } else {
+        Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem);
+        // a.i64 = bitcast b.f64 ==>
+        //   s.f64 = spill b.f64
+        //   t_lo.i32 = lo(s.f64)
+        //   a_lo.i32 = t_lo.i32
+        //   t_hi.i32 = hi(s.f64)
+        //   a_hi.i32 = t_hi.i32
+        Operand *SpillLo, *SpillHi;
+        if (auto *Src0Var = llvm::dyn_cast<Variable>(Src0RM)) {
+          typename Traits::SpillVariable *SpillVar =
+              Func->makeVariable<typename Traits::SpillVariable>(IceType_f64);
+          SpillVar->setLinkedTo(Src0Var);
+          Variable *Spill = SpillVar;
+          Spill->setWeight(RegWeight::Zero);
+          _movq(Spill, Src0RM);
+          SpillLo = Traits::VariableSplit::create(Func, Spill,
+                                                  Traits::VariableSplit::Low);
+          SpillHi = Traits::VariableSplit::create(Func, Spill,
+                                                  Traits::VariableSplit::High);
+        } else {
+          SpillLo = loOperand(Src0RM);
+          SpillHi = hiOperand(Src0RM);
+        }
+
+        Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
+        Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
+        Variable *T_Lo = makeReg(IceType_i32);
+        Variable *T_Hi = makeReg(IceType_i32);
+
+        _mov(T_Lo, SpillLo);
+        _mov(DestLo, T_Lo);
+        _mov(T_Hi, SpillHi);
+        _mov(DestHi, T_Hi);
+      }
+    } break;
+    case IceType_f64: {
+      assert(Src0->getType() == IceType_i64);
+      if (Traits::Is64Bit) {
+        Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem);
+        Variable *T = makeReg(IceType_f64);
+        // Movd requires its fp argument (in this case, the bitcast destination)
+        // to be an xmm register.
+        T->setWeightInfinite();
+        _movd(T, Src0RM);
+        _mov(Dest, T);
+      } else {
+        Src0 = legalize(Src0);
+        if (llvm::isa<typename Traits::X86OperandMem>(Src0)) {
+          Variable *T = Func->makeVariable(Dest->getType());
+          _movq(T, Src0);
+          _movq(Dest, T);
+          break;
+        }
+        // a.f64 = bitcast b.i64 ==>
+        //   t_lo.i32 = b_lo.i32
+        //   FakeDef(s.f64)
+        //   lo(s.f64) = t_lo.i32
+        //   t_hi.i32 = b_hi.i32
+        //   hi(s.f64) = t_hi.i32
+        //   a.f64 = s.f64
         typename Traits::SpillVariable *SpillVar =
             Func->makeVariable<typename Traits::SpillVariable>(IceType_f64);
-        SpillVar->setLinkedTo(Src0Var);
+        SpillVar->setLinkedTo(Dest);
         Variable *Spill = SpillVar;
         Spill->setWeight(RegWeight::Zero);
-        _movq(Spill, Src0RM);
-        SpillLo = Traits::VariableSplit::create(Func, Spill,
-                                                Traits::VariableSplit::Low);
-        SpillHi = Traits::VariableSplit::create(Func, Spill,
-                                                Traits::VariableSplit::High);
-      } else {
-        SpillLo = loOperand(Src0RM);
-        SpillHi = hiOperand(Src0RM);
+
+        Variable *T_Lo = nullptr, *T_Hi = nullptr;
+        typename Traits::VariableSplit *SpillLo = Traits::VariableSplit::create(
+            Func, Spill, Traits::VariableSplit::Low);
+        typename Traits::VariableSplit *SpillHi = Traits::VariableSplit::create(
+            Func, Spill, Traits::VariableSplit::High);
+        _mov(T_Lo, loOperand(Src0));
+        // Technically, the Spill is defined after the _store happens, but
+        // SpillLo is considered a "use" of Spill so define Spill before it
+        // is used.
+        Context.insert(InstFakeDef::create(Func, Spill));
+        _store(T_Lo, SpillLo);
+        _mov(T_Hi, hiOperand(Src0));
+        _store(T_Hi, SpillHi);
+        _movq(Dest, Spill);
       }
-
-      Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
-      Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
-      Variable *T_Lo = makeReg(IceType_i32);
-      Variable *T_Hi = makeReg(IceType_i32);
-
-      _mov(T_Lo, SpillLo);
-      _mov(DestLo, T_Lo);
-      _mov(T_Hi, SpillHi);
-      _mov(DestHi, T_Hi);
-    } break;
-    case IceType_f64: {
-      Src0 = legalize(Src0);
-      assert(Src0->getType() == IceType_i64);
-      if (llvm::isa<typename Traits::X86OperandMem>(Src0)) {
-        Variable *T = Func->makeVariable(Dest->getType());
-        _movq(T, Src0);
-        _movq(Dest, T);
-        break;
-      }
-      // a.f64 = bitcast b.i64 ==>
-      //   t_lo.i32 = b_lo.i32
-      //   FakeDef(s.f64)
-      //   lo(s.f64) = t_lo.i32
-      //   t_hi.i32 = b_hi.i32
-      //   hi(s.f64) = t_hi.i32
-      //   a.f64 = s.f64
-      typename Traits::SpillVariable *SpillVar =
-          Func->makeVariable<typename Traits::SpillVariable>(IceType_f64);
-      SpillVar->setLinkedTo(Dest);
-      Variable *Spill = SpillVar;
-      Spill->setWeight(RegWeight::Zero);
-
-      Variable *T_Lo = nullptr, *T_Hi = nullptr;
-      typename Traits::VariableSplit *SpillLo = Traits::VariableSplit::create(
-          Func, Spill, Traits::VariableSplit::Low);
-      typename Traits::VariableSplit *SpillHi = Traits::VariableSplit::create(
-          Func, Spill, Traits::VariableSplit::High);
-      _mov(T_Lo, loOperand(Src0));
-      // Technically, the Spill is defined after the _store happens, but
-      // SpillLo is considered a "use" of Spill so define Spill before it
-      // is used.
-      Context.insert(InstFakeDef::create(Func, Spill));
-      _store(T_Lo, SpillLo);
-      _mov(T_Hi, hiOperand(Src0));
-      _store(T_Hi, SpillHi);
-      _movq(Dest, Spill);
     } break;
     case IceType_v8i1: {
       assert(Src0->getType() == IceType_i8);
       InstCall *Call = makeHelperCall(H_bitcast_i8_8xi1, Dest, 1);
       Variable *Src0AsI32 = Func->makeVariable(stackSlotType());
       // Arguments to functions are required to be at least 32 bits wide.
       lowerCast(InstCast::create(Func, InstCast::Zext, Src0AsI32, Src0));
       Call->addArg(Src0AsI32);
       lowerCall(Call);
     } break;
@@ skipping 319 matching lines @@
       Variable *MinusOne = makeVectorOfMinusOnes(Ty);
       _pxor(T, MinusOne);
     } break;
     }
 
     _movp(Dest, T);
     eliminateNextVectorSextInstruction(Dest);
     return;
   }
 
-  // a=icmp cond, b, c ==> cmp b,c; a=1; br cond,L1; FakeUse(a); a=0; L1:
-  if (Src0->getType() == IceType_i64) {
-    InstIcmp::ICond Condition = Inst->getCondition();
-    size_t Index = static_cast<size_t>(Condition);
-    assert(Index < Traits::TableIcmp64Size);
-    Operand *Src0LoRM = legalize(loOperand(Src0), Legal_Reg | Legal_Mem);
-    Operand *Src0HiRM = legalize(hiOperand(Src0), Legal_Reg | Legal_Mem);
-    Operand *Src1LoRI = legalize(loOperand(Src1), Legal_Reg | Legal_Imm);
-    Operand *Src1HiRI = legalize(hiOperand(Src1), Legal_Reg | Legal_Imm);
-    Constant *Zero = Ctx->getConstantZero(IceType_i32);
-    Constant *One = Ctx->getConstantInt32(1);
-    typename Traits::Insts::Label *LabelFalse =
-        Traits::Insts::Label::create(Func, this);
-    typename Traits::Insts::Label *LabelTrue =
-        Traits::Insts::Label::create(Func, this);
-    _mov(Dest, One);
-    _cmp(Src0HiRM, Src1HiRI);
-    if (Traits::TableIcmp64[Index].C1 != Traits::Cond::Br_None)
-      _br(Traits::TableIcmp64[Index].C1, LabelTrue);
-    if (Traits::TableIcmp64[Index].C2 != Traits::Cond::Br_None)
-      _br(Traits::TableIcmp64[Index].C2, LabelFalse);
-    _cmp(Src0LoRM, Src1LoRI);
-    _br(Traits::TableIcmp64[Index].C3, LabelTrue);
-    Context.insert(LabelFalse);
-    _mov_nonkillable(Dest, Zero);
-    Context.insert(LabelTrue);
+  if (!Traits::Is64Bit && Src0->getType() == IceType_i64) {
+    lowerIcmp64(Inst);
     return;
   }
 
   // cmp b, c
   Operand *Src0RM = legalizeSrc0ForCmp(Src0, Src1);
   _cmp(Src0RM, Src1);
   _setcc(Dest, Traits::getIcmp32Mapping(Inst->getCondition()));
 }
 
+template <typename Machine>
+template <typename T>
+typename std::enable_if<!T::Is64Bit, void>::type
+TargetX86Base<Machine>::lowerIcmp64(const InstIcmp *Inst) {
+  // a=icmp cond, b, c ==> cmp b,c; a=1; br cond,L1; FakeUse(a); a=0; L1:
+  Operand *Src0 = legalize(Inst->getSrc(0));
+  Operand *Src1 = legalize(Inst->getSrc(1));
+  Variable *Dest = Inst->getDest();
+  InstIcmp::ICond Condition = Inst->getCondition();
+  size_t Index = static_cast<size_t>(Condition);
+  assert(Index < Traits::TableIcmp64Size);
+  Operand *Src0LoRM = legalize(loOperand(Src0), Legal_Reg | Legal_Mem);
+  Operand *Src0HiRM = legalize(hiOperand(Src0), Legal_Reg | Legal_Mem);
+  Operand *Src1LoRI = legalize(loOperand(Src1), Legal_Reg | Legal_Imm);
+  Operand *Src1HiRI = legalize(hiOperand(Src1), Legal_Reg | Legal_Imm);
+  Constant *Zero = Ctx->getConstantZero(IceType_i32);
+  Constant *One = Ctx->getConstantInt32(1);
+  typename Traits::Insts::Label *LabelFalse =
+      Traits::Insts::Label::create(Func, this);
+  typename Traits::Insts::Label *LabelTrue =
+      Traits::Insts::Label::create(Func, this);
+  _mov(Dest, One);
+  _cmp(Src0HiRM, Src1HiRI);
+  if (Traits::TableIcmp64[Index].C1 != Traits::Cond::Br_None)
+    _br(Traits::TableIcmp64[Index].C1, LabelTrue);
+  if (Traits::TableIcmp64[Index].C2 != Traits::Cond::Br_None)
+    _br(Traits::TableIcmp64[Index].C2, LabelFalse);
+  _cmp(Src0LoRM, Src1LoRI);
+  _br(Traits::TableIcmp64[Index].C3, LabelTrue);
+  Context.insert(LabelFalse);
+  _mov_nonkillable(Dest, Zero);
+  Context.insert(LabelTrue);
+}
+
 template <class Machine>
 void TargetX86Base<Machine>::lowerInsertElement(const InstInsertElement *Inst) {
   Operand *SourceVectNotLegalized = Inst->getSrc(0);
   Operand *ElementToInsertNotLegalized = Inst->getSrc(1);
   ConstantInteger32 *ElementIndex =
       llvm::dyn_cast<ConstantInteger32>(Inst->getSrc(2));
   // Only constant indices are allowed in PNaCl IR.
   assert(ElementIndex);
   unsigned Index = ElementIndex->getValue();
   assert(Index < typeNumElements(SourceVectNotLegalized->getType()));
@@ skipping 178 matching lines @@
   }
   case Intrinsics::AtomicLoad: {
     // We require the memory address to be naturally aligned.
     // Given that is the case, then normal loads are atomic.
     if (!Intrinsics::isMemoryOrderValid(
             ID, getConstantMemoryOrder(Instr->getArg(1)))) {
       Func->setError("Unexpected memory ordering for AtomicLoad");
       return;
     }
     Variable *Dest = Instr->getDest();
-    if (Dest->getType() == IceType_i64) {
+    if (!Traits::Is64Bit && Dest->getType() == IceType_i64) {
       // Follow what GCC does and use a movq instead of what lowerLoad()
       // normally does (split the load into two).
       // Thus, this skips load/arithmetic op folding. Load/arithmetic folding
       // can't happen anyway, since this is x86-32 and integer arithmetic only
       // happens on 32-bit quantities.
       Variable *T = makeReg(IceType_f64);
       typename Traits::X86OperandMem *Addr =
           formMemoryOperand(Instr->getArg(0), IceType_f64);
       _movq(T, Addr);
       // Then cast the bits back out of the XMM register to the i64 Dest.
@@ skipping 29 matching lines @@
     if (!Intrinsics::isMemoryOrderValid(
             ID, getConstantMemoryOrder(Instr->getArg(2)))) {
       Func->setError("Unexpected memory ordering for AtomicStore");
       return;
     }
     // We require the memory address to be naturally aligned.
     // Given that is the case, then normal stores are atomic.
     // Add a fence after the store to make it visible.
     Operand *Value = Instr->getArg(0);
     Operand *Ptr = Instr->getArg(1);
-    if (Value->getType() == IceType_i64) {
+    if (!Traits::Is64Bit && Value->getType() == IceType_i64) {
       // Use a movq instead of what lowerStore() normally does
       // (split the store into two), following what GCC does.
       // Cast the bits from int -> to an xmm register first.
       Variable *T = makeReg(IceType_f64);
       InstCast *Cast = InstCast::create(Func, InstCast::Bitcast, T, Value);
       lowerCast(Cast);
       // Then store XMM w/ a movq.
       typename Traits::X86OperandMem *Addr =
           formMemoryOperand(Ptr, IceType_f64);
       _storeq(T, Addr);
       _mfence();
       return;
     }
     InstStore *Store = InstStore::create(Func, Value, Ptr);
     lowerStore(Store);
     _mfence();
     return;
   }
   case Intrinsics::Bswap: {
     Variable *Dest = Instr->getDest();
     Operand *Val = Instr->getArg(0);
     // In 32-bit mode, bswap only works on 32-bit arguments, and the
     // argument must be a register. Use rotate left for 16-bit bswap.
-    if (Val->getType() == IceType_i64) {
+    if (!Traits::Is64Bit && Val->getType() == IceType_i64) {
       Val = legalizeUndef(Val);
       Variable *T_Lo = legalizeToReg(loOperand(Val));
       Variable *T_Hi = legalizeToReg(hiOperand(Val));
       Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
       Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
       _bswap(T_Lo);
       _bswap(T_Hi);
       _mov(DestLo, T_Hi);
       _mov(DestHi, T_Lo);
-    } else if (Val->getType() == IceType_i32) {
+    } else if ((Traits::Is64Bit && Val->getType() == IceType_i64) ||
+               Val->getType() == IceType_i32) {
       Variable *T = legalizeToReg(Val);
       _bswap(T);
       _mov(Dest, T);
     } else {
       assert(Val->getType() == IceType_i16);
       Constant *Eight = Ctx->getConstantInt16(8);
       Variable *T = nullptr;
       Val = legalize(Val);
       _mov(T, Val);
       _rol(T, Eight);
       _mov(Dest, T);
     }
     return;
   }
   case Intrinsics::Ctpop: {
     Variable *Dest = Instr->getDest();
+    Variable *T = nullptr;
     Operand *Val = Instr->getArg(0);
-    InstCall *Call = makeHelperCall(isInt32Asserting32Or64(Val->getType())
-                                        ? H_call_ctpop_i32
-                                        : H_call_ctpop_i64,
-                                    Dest, 1);
+    Type ValTy = Val->getType();
+    assert(ValTy == IceType_i32 || ValTy == IceType_i64);
+
+    if (!Traits::Is64Bit) {
+      T = Dest;
+    } else {
+      T = makeReg(IceType_i64);
+      if (ValTy == IceType_i32) {
+        // in x86-64, __popcountsi2 is not defined, so we cheat a bit by
+        // converting it to a 64-bit value, and using ctpop_i64. _movzx should
+        // ensure we will not have any bits set on Val's upper 32 bits.
+        Variable *V = makeReg(IceType_i64);
+        _movzx(V, Val);
+        Val = V;
+      }
+      ValTy = IceType_i64;
+    }
+
+    InstCall *Call = makeHelperCall(
+        ValTy == IceType_i32 ? H_call_ctpop_i32 : H_call_ctpop_i64, T, 1);
     Call->addArg(Val);
     lowerCall(Call);
     // The popcount helpers always return 32-bit values, while the intrinsic's
     // signature matches the native POPCNT instruction and fills a 64-bit reg
     // (in 64-bit mode). Thus, clear the upper bits of the dest just in case
     // the user doesn't do that in the IR. If the user does that in the IR,
     // then this zero'ing instruction is dead and gets optimized out.
-    if (Val->getType() == IceType_i64) {
-      Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
-      Constant *Zero = Ctx->getConstantZero(IceType_i32);
-      _mov(DestHi, Zero);
+    if (!Traits::Is64Bit) {
+      assert(T == Dest);
+      if (Val->getType() == IceType_i64) {
+        Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
+        Constant *Zero = Ctx->getConstantZero(IceType_i32);
+        _mov(DestHi, Zero);
+      }
+    } else {
+      assert(Val->getType() == IceType_i64);
+      // T is 64 bit. It needs to be copied to dest. We need to:
+      //
+      // T_1.32 = trunc T.64 to i32
+      // T_2.64 = zext T_1.32 to i64
+      // Dest.<<right_size>> = T_2.<<right_size>>
+      //
+      // which ensures the upper 32 bits will always be cleared. Just doing a
+      //
+      // mov Dest.32 = trunc T.32 to i32
+      //
+      // is dangerous because there's a chance the compiler will optimize this
+      // copy out. To use _movzx we need two new registers (one 32-, and
+      // another 64-bit wide.)
+      Variable *T_1 = makeReg(IceType_i32);
+      _mov(T_1, T);
+      Variable *T_2 = makeReg(IceType_i64);
+      _movzx(T_2, T_1);
+      _mov(Dest, T_2);
     }
     return;
   }
   case Intrinsics::Ctlz: {
     // The "is zero undef" parameter is ignored and we always return
     // a well-defined value.
     Operand *Val = legalize(Instr->getArg(0));
     Operand *FirstVal;
     Operand *SecondVal = nullptr;
-    if (Val->getType() == IceType_i64) {
+    if (!Traits::Is64Bit && Val->getType() == IceType_i64) {
       FirstVal = loOperand(Val);
       SecondVal = hiOperand(Val);
     } else {
       FirstVal = Val;
     }
     const bool IsCttz = false;
     lowerCountZeros(IsCttz, Val->getType(), Instr->getDest(), FirstVal,
                     SecondVal);
     return;
   }
   case Intrinsics::Cttz: {
     // The "is zero undef" parameter is ignored and we always return
     // a well-defined value.
     Operand *Val = legalize(Instr->getArg(0));
     Operand *FirstVal;
     Operand *SecondVal = nullptr;
-    if (Val->getType() == IceType_i64) {
+    if (!Traits::Is64Bit && Val->getType() == IceType_i64) {
       FirstVal = hiOperand(Val);
       SecondVal = loOperand(Val);
     } else {
       FirstVal = Val;
     }
     const bool IsCttz = true;
     lowerCountZeros(IsCttz, Val->getType(), Instr->getDest(), FirstVal,
                     SecondVal);
     return;
   }
@@ skipping 93 matching lines @@
     Func->setError("Should not be lowering UnknownIntrinsic");
     return;
   }
   return;
 }
 
 template <class Machine>
 void TargetX86Base<Machine>::lowerAtomicCmpxchg(Variable *DestPrev,
                                                 Operand *Ptr, Operand *Expected,
                                                 Operand *Desired) {
-  if (Expected->getType() == IceType_i64) {
+  if (!Traits::Is64Bit && Expected->getType() == IceType_i64) {
     // Reserve the pre-colored registers first, before adding any more
     // infinite-weight variables from formMemoryOperand's legalization.
     Variable *T_edx = makeReg(IceType_i32, Traits::RegisterSet::Reg_edx);
     Variable *T_eax = makeReg(IceType_i32, Traits::RegisterSet::Reg_eax);
     Variable *T_ecx = makeReg(IceType_i32, Traits::RegisterSet::Reg_ecx);
     Variable *T_ebx = makeReg(IceType_i32, Traits::RegisterSet::Reg_ebx);
     _mov(T_eax, loOperand(Expected));
     _mov(T_edx, hiOperand(Expected));
     _mov(T_ebx, loOperand(Desired));
     _mov(T_ecx, hiOperand(Desired));
@@ skipping 97 matching lines @@
 void TargetX86Base<Machine>::lowerAtomicRMW(Variable *Dest, uint32_t Operation,
                                             Operand *Ptr, Operand *Val) {
   bool NeedsCmpxchg = false;
   LowerBinOp Op_Lo = nullptr;
   LowerBinOp Op_Hi = nullptr;
   switch (Operation) {
   default:
     Func->setError("Unknown AtomicRMW operation");
     return;
   case Intrinsics::AtomicAdd: {
-    if (Dest->getType() == IceType_i64) {
+    if (!Traits::Is64Bit && Dest->getType() == IceType_i64) {
       // All the fall-through paths must set this to true, but use this
       // for asserting.
       NeedsCmpxchg = true;
       Op_Lo = &TargetX86Base<Machine>::_add;
       Op_Hi = &TargetX86Base<Machine>::_adc;
       break;
     }
     typename Traits::X86OperandMem *Addr =
         formMemoryOperand(Ptr, Dest->getType());
     const bool Locked = true;
     Variable *T = nullptr;
     _mov(T, Val);
     _xadd(Addr, T, Locked);
     _mov(Dest, T);
     return;
   }
   case Intrinsics::AtomicSub: {
-    if (Dest->getType() == IceType_i64) {
+    if (!Traits::Is64Bit && Dest->getType() == IceType_i64) {
       NeedsCmpxchg = true;
       Op_Lo = &TargetX86Base<Machine>::_sub;
       Op_Hi = &TargetX86Base<Machine>::_sbb;
       break;
     }
     typename Traits::X86OperandMem *Addr =
         formMemoryOperand(Ptr, Dest->getType());
     const bool Locked = true;
     Variable *T = nullptr;
     _mov(T, Val);
@@ skipping 16 matching lines @@
     NeedsCmpxchg = true;
     Op_Lo = &TargetX86Base<Machine>::_and;
     Op_Hi = &TargetX86Base<Machine>::_and;
     break;
   case Intrinsics::AtomicXor:
     NeedsCmpxchg = true;
     Op_Lo = &TargetX86Base<Machine>::_xor;
     Op_Hi = &TargetX86Base<Machine>::_xor;
     break;
   case Intrinsics::AtomicExchange:
-    if (Dest->getType() == IceType_i64) {
+    if (!Traits::Is64Bit && Dest->getType() == IceType_i64) {
       NeedsCmpxchg = true;
       // NeedsCmpxchg, but no real Op_Lo/Op_Hi need to be done. The values
       // just need to be moved to the ecx and ebx registers.
       Op_Lo = nullptr;
       Op_Hi = nullptr;
       break;
     }
     typename Traits::X86OperandMem *Addr =
         formMemoryOperand(Ptr, Dest->getType());
     Variable *T = nullptr;
@@ skipping 33 matching lines @@
   // .LABEL:
   //   mov     <reg>, eax
   //   op      <reg>, [desired_adj]
   //   lock cmpxchg [ptr], <reg>
   //   jne     .LABEL
   //   mov     <dest>, eax
   //
   // If Op_{Lo,Hi} are nullptr, then just copy the value.
   Val = legalize(Val);
   Type Ty = Val->getType();
-  if (Ty == IceType_i64) {
+  if (!Traits::Is64Bit && Ty == IceType_i64) {
     Variable *T_edx = makeReg(IceType_i32, Traits::RegisterSet::Reg_edx);
     Variable *T_eax = makeReg(IceType_i32, Traits::RegisterSet::Reg_eax);
     typename Traits::X86OperandMem *Addr = formMemoryOperand(Ptr, Ty);
     _mov(T_eax, loOperand(Addr));
     _mov(T_edx, hiOperand(Addr));
     Variable *T_ecx = makeReg(IceType_i32, Traits::RegisterSet::Reg_ecx);
     Variable *T_ebx = makeReg(IceType_i32, Traits::RegisterSet::Reg_ebx);
     typename Traits::Insts::Label *Label =
         Traits::Insts::Label::create(Func, this);
     const bool IsXchg8b = Op_Lo == nullptr && Op_Hi == nullptr;
@@ skipping 111 matching lines @@
   if (Cttz) {
     _mov(T_Dest, ThirtyTwo);
   } else {
     Constant *SixtyThree = Ctx->getConstantInt32(63);
     _mov(T_Dest, SixtyThree);
   }
   _cmov(T_Dest, T, Traits::Cond::Br_ne);
   if (!Cttz) {
     _xor(T_Dest, ThirtyOne);
   }
-  if (Ty == IceType_i32) {
+  if (Traits::Is64Bit || Ty == IceType_i32) {
     _mov(Dest, T_Dest);
     return;
   }
   _add(T_Dest, ThirtyTwo);
   Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
   Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
   // Will be using "test" on this, so we need a registerized variable.
   Variable *SecondVar = legalizeToReg(SecondVal);
   Variable *T_Dest2 = makeReg(IceType_i32);
   if (Cttz) {
@@ skipping 568 matching lines @@
     return;
   }
   // mov t, SrcF; cmov_cond t, SrcT; mov dest, t
   // But if SrcT is immediate, we might be able to do better, as
   // the cmov instruction doesn't allow an immediate operand:
   // mov t, SrcT; cmov_!cond t, SrcF; mov dest, t
   if (llvm::isa<Constant>(SrcT) && !llvm::isa<Constant>(SrcF)) {
     std::swap(SrcT, SrcF);
     Cond = InstX86Base<Machine>::getOppositeCondition(Cond);
   }
-  if (DestTy == IceType_i64) {
+  if (!Traits::Is64Bit && DestTy == IceType_i64) {
     SrcT = legalizeUndef(SrcT);
     SrcF = legalizeUndef(SrcF);
     // Set the low portion.
     Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
     Variable *TLo = nullptr;
     Operand *SrcFLo = legalize(loOperand(SrcF));
     _mov(TLo, SrcFLo);
     Operand *SrcTLo = legalize(loOperand(SrcT), Legal_Reg | Legal_Mem);
     _cmov(TLo, SrcTLo, Cond);
     _mov(DestLo, TLo);
     // Set the high portion.
     Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
     Variable *THi = nullptr;
     Operand *SrcFHi = legalize(hiOperand(SrcF));
     _mov(THi, SrcFHi);
     Operand *SrcTHi = legalize(hiOperand(SrcT), Legal_Reg | Legal_Mem);
     _cmov(THi, SrcTHi, Cond);
     _mov(DestHi, THi);
     return;
   }
 
-  assert(DestTy == IceType_i16 || DestTy == IceType_i32);
+  assert(DestTy == IceType_i16 || DestTy == IceType_i32 ||
+         (Traits::Is64Bit && DestTy == IceType_i64));
   Variable *T = nullptr;
   SrcF = legalize(SrcF);
   _mov(T, SrcF);
   SrcT = legalize(SrcT, Legal_Reg | Legal_Mem);
   _cmov(T, SrcT, Cond);
   _mov(Dest, T);
 }
 
 template <class Machine>
 void TargetX86Base<Machine>::lowerStore(const InstStore *Inst) {
   Operand *Value = Inst->getData();
   Operand *Addr = Inst->getAddr();
   typename Traits::X86OperandMem *NewAddr =
       formMemoryOperand(Addr, Value->getType());
   Type Ty = NewAddr->getType();
 
-  if (Ty == IceType_i64) {
+  if (!Traits::Is64Bit && Ty == IceType_i64) {
     Value = legalizeUndef(Value);
     Operand *ValueHi = legalize(hiOperand(Value), Legal_Reg | Legal_Imm);
     Operand *ValueLo = legalize(loOperand(Value), Legal_Reg | Legal_Imm);
     _store(ValueHi,
            llvm::cast<typename Traits::X86OperandMem>(hiOperand(NewAddr)));
     _store(ValueLo,
            llvm::cast<typename Traits::X86OperandMem>(loOperand(NewAddr)));
   } else if (isVectorType(Ty)) {
     _storep(legalizeToReg(Value), NewAddr);
   } else {
@@ skipping 27 matching lines @@
       NewStore->setRmwBeacon(Inst->getRmwBeacon());
     Context.insert(NewStore);
   }
 }
 
 template <class Machine>
 Operand *TargetX86Base<Machine>::lowerCmpRange(Operand *Comparison,
                                                uint64_t Min, uint64_t Max) {
   // TODO(ascull): 64-bit should not reach here but only because it is not
   // implemented yet. This should be able to handle the 64-bit case.
-  assert(Comparison->getType() != IceType_i64);
+  assert(Traits::Is64Bit || Comparison->getType() != IceType_i64);
   // Subtracting 0 is a nop so don't do it
   if (Min != 0) {
     // Avoid clobbering the comparison by copying it
     Variable *T = nullptr;
     _mov(T, Comparison);
     _sub(T, Ctx->getConstantInt32(Min));
     Comparison = T;
   }
 
   _cmp(Comparison, Ctx->getConstantInt32(Max - Min));
@@ skipping 78 matching lines @@
 
 template <class Machine>
 void TargetX86Base<Machine>::lowerSwitch(const InstSwitch *Inst) {
   // Group cases together and navigate through them with a binary search
   CaseClusterArray CaseClusters = CaseCluster::clusterizeSwitch(Func, Inst);
   Operand *Src0 = Inst->getComparison();
   CfgNode *DefaultTarget = Inst->getLabelDefault();
 
   assert(CaseClusters.size() != 0); // Should always be at least one
 
-  if (Src0->getType() == IceType_i64) {
+  if (!Traits::Is64Bit && Src0->getType() == IceType_i64) {
     Src0 = legalize(Src0); // get Base/Index into physical registers
     Operand *Src0Lo = loOperand(Src0);
     Operand *Src0Hi = hiOperand(Src0);
     if (CaseClusters.back().getHigh() > UINT32_MAX) {
       // TODO(ascull): handle 64-bit case properly (currently naive version)
       // This might be handled by a higher level lowering of switches.
       SizeT NumCases = Inst->getNumCases();
       if (NumCases >= 2) {
         Src0Lo = legalizeToReg(Src0Lo);
         Src0Hi = legalizeToReg(Src0Hi);
@@ skipping 184 matching lines @@
   // that follows.  This means that the original Store instruction is
   // still there, either because the value being stored is used beyond
   // the Store instruction, or because dead code elimination did not
   // happen.  In either case, we cancel RMW lowering (and the caller
   // deletes the RMW instruction).
   if (!RMW->isLastUse(RMW->getBeacon()))
     return;
   Operand *Src = RMW->getData();
   Type Ty = Src->getType();
   typename Traits::X86OperandMem *Addr = formMemoryOperand(RMW->getAddr(), Ty);
-  if (Ty == IceType_i64) {
+  if (!Traits::Is64Bit && Ty == IceType_i64) {
     Src = legalizeUndef(Src);
     Operand *SrcLo = legalize(loOperand(Src), Legal_Reg | Legal_Imm);
     Operand *SrcHi = legalize(hiOperand(Src), Legal_Reg | Legal_Imm);
     typename Traits::X86OperandMem *AddrLo =
         llvm::cast<typename Traits::X86OperandMem>(loOperand(Addr));
     typename Traits::X86OperandMem *AddrHi =
         llvm::cast<typename Traits::X86OperandMem>(hiOperand(Addr));
     switch (RMW->getOp()) {
     default:
       // TODO(stichnot): Implement other arithmetic operators.
@@ skipping 13 matching lines @@
     case InstArithmetic::Or:
       _or_rmw(AddrLo, SrcLo);
       _or_rmw(AddrHi, SrcHi);
       return;
     case InstArithmetic::Xor:
       _xor_rmw(AddrLo, SrcLo);
       _xor_rmw(AddrHi, SrcHi);
       return;
     }
   } else {
-    // i8, i16, i32
+    // x86-32: i8, i16, i32
+    // x86-64: i8, i16, i32, i64
     switch (RMW->getOp()) {
     default:
       // TODO(stichnot): Implement other arithmetic operators.
       break;
     case InstArithmetic::Add:
       Src = legalize(Src, Legal_Reg | Legal_Imm);
       _add_rmw(Addr, Src);
       return;
     case InstArithmetic::Sub:
       Src = legalize(Src, Legal_Reg | Legal_Imm);
@@ skipping 24 matching lines @@
   } else {
     TargetLowering::lowerOther(Instr);
   }
 }
 
 /// Turn an i64 Phi instruction into a pair of i32 Phi instructions, to
 /// preserve integrity of liveness analysis.  Undef values are also
 /// turned into zeroes, since loOperand() and hiOperand() don't expect
 /// Undef input.
 template <class Machine> void TargetX86Base<Machine>::prelowerPhis() {
-  // Pause constant blinding or pooling, blinding or pooling will be done later
-  // during phi lowering assignments
+  if (Traits::Is64Bit) {
+    // On x86-64 we don't need to prelower phis -- the architecture can handle
+    // 64-bit integer natively.
+    return;
+  }
+
+  // Pause constant blinding or pooling, blinding or pooling will be done
+  // later during phi lowering assignments
   BoolFlagSaver B(RandomizationPoolingPaused, true);
   PhiLowering::prelowerPhis32Bit<TargetX86Base<Machine>>(
       this, Context.getNode(), Func);
 }
 
 // There is no support for loading or emitting vector constants, so the
 // vector values returned from makeVectorOfZeros, makeVectorOfOnes,
 // etc. are initialized with register operations.
 //
 // TODO(wala): Add limited support for vector constants so that
@@ skipping 140 matching lines @@
   if (auto *Const = llvm::dyn_cast<Constant>(From)) {
     if (llvm::isa<ConstantUndef>(Const)) {
       From = legalizeUndef(Const, RegNum);
       if (isVectorType(Ty))
         return From;
       Const = llvm::cast<Constant>(From);
     }
     // There should be no constants of vector type (other than undef).
     assert(!isVectorType(Ty));
 
+    // If the operand is a 64 bit constant integer we need to legalize it to a
+    // register in x86-64.
+    if (Traits::Is64Bit) {
+      if (llvm::isa<ConstantInteger64>(Const)) {
+        Variable *V = copyToReg(Const, RegNum);
+        V->setWeightInfinite();
+        return V;
+      }
+    }
+
     // If the operand is an 32 bit constant integer, we should check
     // whether we need to randomize it or pool it.
     if (ConstantInteger32 *C = llvm::dyn_cast<ConstantInteger32>(Const)) {
       Operand *NewConst = randomizeOrPoolImmediate(C, RegNum);
       if (NewConst != Const) {
         return NewConst;
       }
     }
 
     // Convert a scalar floating point constant into an explicit
@@ skipping 117 matching lines @@
   }
   // Do legalization, which contains randomization/pooling
   // or do randomization/pooling.
   return llvm::cast<typename Traits::X86OperandMem>(
       DoLegalize ? legalize(Mem) : randomizeOrPoolImmediate(Mem));
 }
 
 template <class Machine>
 Variable *TargetX86Base<Machine>::makeReg(Type Type, int32_t RegNum) {
   // There aren't any 64-bit integer registers for x86-32.
-  assert(Type != IceType_i64);
+  assert(Traits::Is64Bit || Type != IceType_i64);
   Variable *Reg = Func->makeVariable(Type);
   if (RegNum == Variable::NoRegister)
     Reg->setWeightInfinite();
   else
     Reg->setRegNum(RegNum);
   return Reg;
 }
 
 template <class Machine> void TargetX86Base<Machine>::postLower() {
   if (Ctx->getFlags().getOptLevel() == Opt_m1)
@@ skipping 11 matching lines @@
 
 template <class Machine>
 void TargetX86Base<Machine>::emit(const ConstantInteger32 *C) const {
   if (!BuildDefs::dump())
     return;
   Ostream &Str = Ctx->getStrEmit();
   Str << getConstantPrefix() << C->getValue();
 }
 
 template <class Machine>
-void TargetX86Base<Machine>::emit(const ConstantInteger64 *) const {
-  llvm::report_fatal_error("Not expecting to emit 64-bit integers");
+void TargetX86Base<Machine>::emit(const ConstantInteger64 *C) const {
+  if (!Traits::Is64Bit) {
+    llvm::report_fatal_error("Not expecting to emit 64-bit integers");
+  } else {
+    if (!BuildDefs::dump())
+      return;
+    Ostream &Str = Ctx->getStrEmit();
+    Str << getConstantPrefix() << C->getValue();
+  }
 }
 
 template <class Machine>
 void TargetX86Base<Machine>::emit(const ConstantFloat *C) const {
   if (!BuildDefs::dump())
     return;
   Ostream &Str = Ctx->getStrEmit();
   C->emitPoolLabel(Str);
 }
 
@@ skipping 124 matching lines @@
         Constant *Mask1 = Ctx->getConstantInt(
             MemOperand->getOffset()->getType(), Cookie + Value);
         Constant *Mask2 =
             Ctx->getConstantInt(MemOperand->getOffset()->getType(), 0 - Cookie);
 
         typename Traits::X86OperandMem *TempMemOperand =
             Traits::X86OperandMem::create(Func, MemOperand->getType(),
                                           MemOperand->getBase(), Mask1);
         // If we have already assigned a physical register, we must come from
         // advancedPhiLowering()=>lowerAssign(). In this case we should reuse
-        // the assigned register as this assignment is that start of its use-def
-        // chain. So we add RegNum argument here.
+        // the assigned register as this assignment is that start of its
+        // use-def chain. So we add RegNum argument here.
         Variable *RegTemp = makeReg(MemOperand->getOffset()->getType(), RegNum);
         _lea(RegTemp, TempMemOperand);
         // As source operand doesn't use the dstreg, we don't need to add
         // _set_dest_nonkillable().
         // But if we use the same Dest Reg, that is, with RegNum
         // assigned, we should add this _set_dest_nonkillable()
         if (RegNum != Variable::NoRegister)
           _set_dest_nonkillable();
 
         typename Traits::X86OperandMem *NewMemOperand =
@@ skipping 62 matching lines @@
   }
   // the offset is not eligible for blinding or pooling, return the original
   // mem operand
   return MemOperand;
 }
 
 } // end of namespace X86Internal
 } // end of namespace Ice
 
 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H