Subzero: Refactor x86 register representation to actively use aliases.

src/IceTargetLoweringX86BaseImpl.h

 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==//
 //
 //                        The Subzero Code Generator
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 ///
 /// \file
@@ skipping 1198 matching lines @@
     }
   } else {
     // NON-CONSTANT CASES.
     Constant *BitTest = Ctx->getConstantInt32(0x20);
     typename Traits::Insts::Label *Label =
         Traits::Insts::Label::create(Func, this);
     // COMMON PREFIX OF: a=b SHIFT_OP c ==>
     //   t1:ecx = c.lo & 0xff
     //   t2 = b.lo
     //   t3 = b.hi
-    T_1 = makeReg(IceType_i8, Traits::RegisterSet::Reg_cl);
-    _mov(T_1, Src1Lo);
+    T_1 = copyToReg8(Src1Lo, Traits::RegisterSet::Reg_cl);
     _mov(T_2, Src0Lo);
     _mov(T_3, Src0Hi);
     switch (Op) {
     default:
       assert(0 && "non-shift op");
       break;
     case InstArithmetic::Shl: {
       // a=b<<c ==>
       //   t3 = shld t3, t2, t1
       //   t2 = shl t2, t1
@@ skipping 57 matching lines @@
     //   a.hi = t3
     Context.insert(Label);
     _mov(DestLo, T_2);
     _mov(DestHi, T_3);
   }
 }
 
 template <class Machine>
 void TargetX86Base<Machine>::lowerArithmetic(const InstArithmetic *Inst) {
   Variable *Dest = Inst->getDest();
+  Type Ty = Dest->getType();
   Operand *Src0 = legalize(Inst->getSrc(0));
   Operand *Src1 = legalize(Inst->getSrc(1));
   if (Inst->isCommutative()) {
     uint32_t SwapCount = 0;
     if (!llvm::isa<Variable>(Src0) && llvm::isa<Variable>(Src1)) {
       std::swap(Src0, Src1);
       ++SwapCount;
     }
     if (llvm::isa<Constant>(Src0) && !llvm::isa<Constant>(Src1)) {
       std::swap(Src0, Src1);
       ++SwapCount;
     }
     // Improve two-address code patterns by avoiding a copy to the dest
     // register when one of the source operands ends its lifetime here.
     if (!Inst->isLastUse(Src0) && Inst->isLastUse(Src1)) {
       std::swap(Src0, Src1);
       ++SwapCount;
     }
     assert(SwapCount <= 1);
     (void)SwapCount;
   }
-  if (!Traits::Is64Bit && Dest->getType() == IceType_i64) {
+  if (!Traits::Is64Bit && Ty == IceType_i64) {
     // These x86-32 helper-call-involved instructions are lowered in this
     // separate switch. This is because loOperand() and hiOperand() may insert
     // redundant instructions for constant blinding and pooling. Such redundant
     // instructions will fail liveness analysis under -Om1 setting. And,
     // actually these arguments do not need to be processed with loOperand()
     // and hiOperand() to be used.
     switch (Inst->getOp()) {
     case InstArithmetic::Udiv: {
       constexpr SizeT MaxSrcs = 2;
       InstCall *Call = makeHelperCall(H_udiv_i64, Dest, MaxSrcs);
@@ skipping 126 matching lines @@
     case InstArithmetic::Udiv:
     case InstArithmetic::Sdiv:
     case InstArithmetic::Urem:
     case InstArithmetic::Srem:
       llvm_unreachable("Call-helper-involved instruction for i64 type \
                        should have already been handled before");
       break;
     }
     return;
   }
-  if (isVectorType(Dest->getType())) {
+  if (isVectorType(Ty)) {
     // TODO: Trap on integer divide and integer modulo by zero. See:
     // https://code.google.com/p/nativeclient/issues/detail?id=3899
     if (llvm::isa<typename Traits::X86OperandMem>(Src1))
       Src1 = legalizeToReg(Src1);
     switch (Inst->getOp()) {
     case InstArithmetic::_num:
       llvm_unreachable("Unknown arithmetic operator");
       break;
     case InstArithmetic::Add: {
-      Variable *T = makeReg(Dest->getType());
+      Variable *T = makeReg(Ty);
       _movp(T, Src0);
       _padd(T, Src1);
       _movp(Dest, T);
     } break;
     case InstArithmetic::And: {
-      Variable *T = makeReg(Dest->getType());
+      Variable *T = makeReg(Ty);
       _movp(T, Src0);
       _pand(T, Src1);
       _movp(Dest, T);
     } break;
     case InstArithmetic::Or: {
-      Variable *T = makeReg(Dest->getType());
+      Variable *T = makeReg(Ty);
       _movp(T, Src0);
       _por(T, Src1);
       _movp(Dest, T);
     } break;
     case InstArithmetic::Xor: {
-      Variable *T = makeReg(Dest->getType());
+      Variable *T = makeReg(Ty);
       _movp(T, Src0);
       _pxor(T, Src1);
       _movp(Dest, T);
     } break;
     case InstArithmetic::Sub: {
-      Variable *T = makeReg(Dest->getType());
+      Variable *T = makeReg(Ty);
       _movp(T, Src0);
       _psub(T, Src1);
       _movp(Dest, T);
     } break;
     case InstArithmetic::Mul: {
-      bool TypesAreValidForPmull =
-          Dest->getType() == IceType_v4i32 || Dest->getType() == IceType_v8i16;
+      bool TypesAreValidForPmull = Ty == IceType_v4i32 || Ty == IceType_v8i16;
       bool InstructionSetIsValidForPmull =
-          Dest->getType() == IceType_v8i16 || InstructionSet >= Traits::SSE4_1;
+          Ty == IceType_v8i16 || InstructionSet >= Traits::SSE4_1;
       if (TypesAreValidForPmull && InstructionSetIsValidForPmull) {
-        Variable *T = makeReg(Dest->getType());
+        Variable *T = makeReg(Ty);
         _movp(T, Src0);
         _pmull(T, Src0 == Src1 ? T : Src1);
         _movp(Dest, T);
-      } else if (Dest->getType() == IceType_v4i32) {
+      } else if (Ty == IceType_v4i32) {
         // Lowering sequence:
         // Note: The mask arguments have index 0 on the left.
         //
         // movups  T1, Src0
         // pshufd  T2, Src0, {1,0,3,0}
         // pshufd  T3, Src1, {1,0,3,0}
         // # T1 = {Src0[0] * Src1[0], Src0[2] * Src1[2]}
         // pmuludq T1, Src1
         // # T2 = {Src0[1] * Src1[1], Src0[3] * Src1[3]}
         // pmuludq T2, T3
@@ skipping 17 matching lines @@
         Variable *T3 = makeReg(IceType_v4i32);
         Variable *T4 = makeReg(IceType_v4i32);
         _movp(T1, Src0);
         _pshufd(T2, Src0, Mask1030);
         _pshufd(T3, Src1, Mask1030);
         _pmuludq(T1, Src1);
         _pmuludq(T2, T3);
         _shufps(T1, T2, Ctx->getConstantInt32(Mask0202));
         _pshufd(T4, T1, Ctx->getConstantInt32(Mask0213));
         _movp(Dest, T4);
-      } else if (Dest->getType() == IceType_v16i8) {
+      } else if (Ty == IceType_v16i8) {
         scalarizeArithmetic(Inst->getOp(), Dest, Src0, Src1);
       } else {
         llvm::report_fatal_error("Invalid vector multiply type");
       }
     } break;
     case InstArithmetic::Shl:
     case InstArithmetic::Lshr:
     case InstArithmetic::Ashr:
     case InstArithmetic::Udiv:
     case InstArithmetic::Urem:
     case InstArithmetic::Sdiv:
     case InstArithmetic::Srem:
       scalarizeArithmetic(Inst->getOp(), Dest, Src0, Src1);
       break;
     case InstArithmetic::Fadd: {
-      Variable *T = makeReg(Dest->getType());
+      Variable *T = makeReg(Ty);
       _movp(T, Src0);
       _addps(T, Src1);
       _movp(Dest, T);
     } break;
     case InstArithmetic::Fsub: {
-      Variable *T = makeReg(Dest->getType());
+      Variable *T = makeReg(Ty);
       _movp(T, Src0);
       _subps(T, Src1);
       _movp(Dest, T);
     } break;
     case InstArithmetic::Fmul: {
-      Variable *T = makeReg(Dest->getType());
+      Variable *T = makeReg(Ty);
       _movp(T, Src0);
       _mulps(T, Src0 == Src1 ? T : Src1);
       _movp(Dest, T);
     } break;
     case InstArithmetic::Fdiv: {
-      Variable *T = makeReg(Dest->getType());
+      Variable *T = makeReg(Ty);
       _movp(T, Src0);
       _divps(T, Src1);
       _movp(Dest, T);
     } break;
     case InstArithmetic::Frem:
       scalarizeArithmetic(Inst->getOp(), Dest, Src0, Src1);
       break;
     }
     return;
   }
@@ skipping 28 matching lines @@
     _sub(T, Src1);
     _mov(Dest, T);
     break;
   case InstArithmetic::Mul:
     if (auto *C = llvm::dyn_cast<ConstantInteger32>(Src1)) {
       if (optimizeScalarMul(Dest, Src0, C->getValue()))
         return;
     }
     // The 8-bit version of imul only allows the form "imul r/m8" where T must
     // be in al.
-    if (isByteSizedArithType(Dest->getType())) {
+    if (isByteSizedArithType(Ty)) {
       _mov(T, Src0, Traits::RegisterSet::Reg_al);
       Src1 = legalize(Src1, Legal_Reg | Legal_Mem);
       _imul(T, Src0 == Src1 ? T : Src1);
       _mov(Dest, T);
     } else if (auto *ImmConst = llvm::dyn_cast<ConstantInteger32>(Src1)) {
-      T = makeReg(Dest->getType());
+      T = makeReg(Ty);
       _imul_imm(T, Src0, ImmConst);
       _mov(Dest, T);
     } else {
       _mov(T, Src0);
       _imul(T, Src0 == Src1 ? T : Src1);
       _mov(Dest, T);
     }
     break;
   case InstArithmetic::Shl:
     _mov(T, Src0);
-    if (!llvm::isa<ConstantInteger32>(Src1)) {
-      Variable *Cl = makeReg(IceType_i8, Traits::RegisterSet::Reg_cl);
-      _mov(Cl, Src1);
-      Src1 = Cl;
-    }
+    if (!llvm::isa<ConstantInteger32>(Src1))
+      Src1 = copyToReg8(Src1, Traits::RegisterSet::Reg_cl);
     _shl(T, Src1);
     _mov(Dest, T);
     break;
   case InstArithmetic::Lshr:
     _mov(T, Src0);
-    if (!llvm::isa<ConstantInteger32>(Src1)) {
-      Variable *Cl = makeReg(IceType_i8, Traits::RegisterSet::Reg_cl);
-      _mov(Cl, Src1);
-      Src1 = Cl;
-    }
+    if (!llvm::isa<ConstantInteger32>(Src1))
+      Src1 = copyToReg8(Src1, Traits::RegisterSet::Reg_cl);
     _shr(T, Src1);
     _mov(Dest, T);
     break;
   case InstArithmetic::Ashr:
     _mov(T, Src0);
-    if (!llvm::isa<ConstantInteger32>(Src1)) {
-      Variable *Cl = makeReg(IceType_i8, Traits::RegisterSet::Reg_cl);
-      _mov(Cl, Src1);
-      Src1 = Cl;
-    }
+    if (!llvm::isa<ConstantInteger32>(Src1))
+      Src1 = copyToReg8(Src1, Traits::RegisterSet::Reg_cl);
     _sar(T, Src1);
     _mov(Dest, T);
     break;
-  case InstArithmetic::Udiv:
+  case InstArithmetic::Udiv: {
     // div and idiv are the few arithmetic operators that do not allow
     // immediates as the operand.
     Src1 = legalize(Src1, Legal_Reg | Legal_Mem);
-    if (isByteSizedArithType(Dest->getType())) {
-      // For 8-bit unsigned division we need to zero-extend al into ah. A mov
-      // $0, %ah (or xor %ah, %ah) would work just fine, except that the x86-64
-      // assembler refuses to encode %ah (encoding %spl with a REX prefix
-      // instead.) Accessing %ah in 64-bit is "tricky" as you can't encode %ah
-      // with any other 8-bit register except for %a[lh], %b[lh], %c[lh], and
-      // d[%lh], which means the X86 target lowering (and the register
-      // allocator) would have to be aware of this restriction. For now, we
-      // simply zero %eax completely, and move the dividend into %al.
-      Variable *T_eax = makeReg(IceType_i32, Traits::RegisterSet::Reg_eax);
-      Context.insert(InstFakeDef::create(Func, T_eax));
-      _xor(T_eax, T_eax);
-      _mov(T, Src0, Traits::RegisterSet::Reg_al);
-      _div(T, Src1, T);
-      _mov(Dest, T);
-      Context.insert(InstFakeUse::create(Func, T_eax));
-    } else {
-      Type Ty = Dest->getType();
-      uint32_t Eax = Traits::RegisterSet::Reg_eax;
-      uint32_t Edx = Traits::RegisterSet::Reg_edx;
-      switch (Ty) {
-      default:
-        llvm_unreachable("Bad type for udiv");
-      // fallthrough
-      case IceType_i32:
-        break;
-      case IceType_i16:
-        Eax = Traits::RegisterSet::Reg_ax;
-        Edx = Traits::RegisterSet::Reg_dx;
-        break;
-      }
-      Constant *Zero = Ctx->getConstantZero(Ty);
-      _mov(T, Src0, Eax);
-      _mov(T_edx, Zero, Edx);
-      _div(T, Src1, T_edx);
-      _mov(Dest, T);
+    uint32_t Eax = Traits::RegisterSet::Reg_eax;
+    uint32_t Edx = Traits::RegisterSet::Reg_edx;
+    switch (Ty) {
+    default:
+      llvm_unreachable("Bad type for udiv");
+    // fallthrough
+    case IceType_i32:
+      break;
+    case IceType_i16:
+      Eax = Traits::RegisterSet::Reg_ax;
+      Edx = Traits::RegisterSet::Reg_dx;
+      break;
+    case IceType_i8:
+      Eax = Traits::RegisterSet::Reg_al;
+      Edx = Traits::RegisterSet::Reg_ah;
+      break;
     }
-    break;
+    _mov(T, Src0, Eax);
+    _mov(T_edx, Ctx->getConstantZero(Ty), Edx);
+    _div(T, Src1, T_edx);
+    _mov(Dest, T);
+  } break;
   case InstArithmetic::Sdiv:
     // TODO(stichnot): Enable this after doing better performance and cross
     // testing.
     if (false && Ctx->getFlags().getOptLevel() >= Opt_1) {
       // Optimize division by constant power of 2, but not for Om1 or O0, just
       // to keep things simple there.
       if (auto *C = llvm::dyn_cast<ConstantInteger32>(Src1)) {
         int32_t Divisor = C->getValue();
         uint32_t UDivisor = static_cast<uint32_t>(Divisor);
         if (Divisor > 0 && llvm::isPowerOf2_32(UDivisor)) {
           uint32_t LogDiv = llvm::Log2_32(UDivisor);
-          Type Ty = Dest->getType();
           // LLVM does the following for dest=src/(1<<log):
           //   t=src
           //   sar t,typewidth-1 // -1 if src is negative, 0 if not
           //   shr t,typewidth-log
           //   add t,src
           //   sar t,log
           //   dest=t
           uint32_t TypeWidth = Traits::X86_CHAR_BIT * typeWidthInBytes(Ty);
           _mov(T, Src0);
           // If for some reason we are dividing by 1, just treat it like an
           // assignment.
           if (LogDiv > 0) {
             // The initial sar is unnecessary when dividing by 2.
             if (LogDiv > 1)
               _sar(T, Ctx->getConstantInt(Ty, TypeWidth - 1));
             _shr(T, Ctx->getConstantInt(Ty, TypeWidth - LogDiv));
             _add(T, Src0);
             _sar(T, Ctx->getConstantInt(Ty, LogDiv));
           }
           _mov(Dest, T);
           return;
         }
       }
     }
     Src1 = legalize(Src1, Legal_Reg | Legal_Mem);
-    switch (Type Ty = Dest->getType()) {
+    switch (Ty) {
     default:
       llvm_unreachable("Bad type for sdiv");
     // fallthrough
     case IceType_i32:
       T_edx = makeReg(Ty, Traits::RegisterSet::Reg_edx);
       _mov(T, Src0, Traits::RegisterSet::Reg_eax);
       break;
     case IceType_i16:
       T_edx = makeReg(Ty, Traits::RegisterSet::Reg_dx);
       _mov(T, Src0, Traits::RegisterSet::Reg_ax);
       break;
     case IceType_i8:
       T_edx = makeReg(IceType_i16, Traits::RegisterSet::Reg_ax);
       _mov(T, Src0, Traits::RegisterSet::Reg_al);
       break;
     }
     _cbwdq(T_edx, T);
     _idiv(T, Src1, T_edx);
     _mov(Dest, T);
     break;
-  case InstArithmetic::Urem:
+  case InstArithmetic::Urem: {
     Src1 = legalize(Src1, Legal_Reg | Legal_Mem);
-    if (isByteSizedArithType(Dest->getType())) {
-      Variable *T_eax = makeReg(IceType_i32, Traits::RegisterSet::Reg_eax);
-      Context.insert(InstFakeDef::create(Func, T_eax));
-      _xor(T_eax, T_eax);
-      _mov(T, Src0, Traits::RegisterSet::Reg_al);
-      _div(T, Src1, T);
-      // shr $8, %eax shifts ah (i.e., the 8 bit remainder) into al. We don't
-      // mov %ah, %al because it would make x86-64 codegen more complicated. If
-      // this ever becomes a problem we can introduce a pseudo rem instruction
-      // that returns the remainder in %al directly (and uses a mov for copying
-      // %ah to %al.)
-      static constexpr uint8_t AlSizeInBits = 8;
-      _shr(T_eax, Ctx->getConstantInt8(AlSizeInBits));
-      _mov(Dest, T);
-      Context.insert(InstFakeUse::create(Func, T_eax));
-    } else {
-      Type Ty = Dest->getType();
-      uint32_t Eax = Traits::RegisterSet::Reg_eax;
-      uint32_t Edx = Traits::RegisterSet::Reg_edx;
-      switch (Ty) {
-      default:
-        llvm_unreachable("Bad type for urem");
-      // fallthrough
-      case IceType_i32:
-        break;
-      case IceType_i16:
-        Eax = Traits::RegisterSet::Reg_ax;
-        Edx = Traits::RegisterSet::Reg_dx;
-        break;
-      }
-      Constant *Zero = Ctx->getConstantZero(Ty);
-      T_edx = makeReg(Dest->getType(), Edx);
-      _mov(T_edx, Zero);
-      _mov(T, Src0, Eax);
-      _div(T_edx, Src1, T);
-      _mov(Dest, T_edx);
+    uint32_t Eax = Traits::RegisterSet::Reg_eax;
+    uint32_t Edx = Traits::RegisterSet::Reg_edx;
+    switch (Ty) {
+    default:
+      llvm_unreachable("Bad type for urem");
+    // fallthrough
+    case IceType_i32:
+      break;
+    case IceType_i16:
+      Eax = Traits::RegisterSet::Reg_ax;
+      Edx = Traits::RegisterSet::Reg_dx;
+      break;
+    case IceType_i8:
+      Eax = Traits::RegisterSet::Reg_al;
+      Edx = Traits::RegisterSet::Reg_ah;
+      break;
     }
-    break;
-  case InstArithmetic::Srem:
+    T_edx = makeReg(Ty, Edx);
+    _mov(T_edx, Ctx->getConstantZero(Ty));
+    _mov(T, Src0, Eax);
+    _div(T_edx, Src1, T);
+    _mov(Dest, T_edx);
+  } break;
+  case InstArithmetic::Srem: {
     // TODO(stichnot): Enable this after doing better performance and cross
     // testing.
     if (false && Ctx->getFlags().getOptLevel() >= Opt_1) {
       // Optimize mod by constant power of 2, but not for Om1 or O0, just to
       // keep things simple there.
       if (auto *C = llvm::dyn_cast<ConstantInteger32>(Src1)) {
         int32_t Divisor = C->getValue();
         uint32_t UDivisor = static_cast<uint32_t>(Divisor);
         if (Divisor > 0 && llvm::isPowerOf2_32(UDivisor)) {
           uint32_t LogDiv = llvm::Log2_32(UDivisor);
-          Type Ty = Dest->getType();
           // LLVM does the following for dest=src%(1<<log):
           //   t=src
           //   sar t,typewidth-1 // -1 if src is negative, 0 if not
           //   shr t,typewidth-log
           //   add t,src
           //   and t, -(1<<log)
           //   sub t,src
           //   neg t
           //   dest=t
           uint32_t TypeWidth = Traits::X86_CHAR_BIT * typeWidthInBytes(Ty);
@@ skipping 10 matching lines @@
           _add(T, Src0);
           _and(T, Ctx->getConstantInt(Ty, -(1 << LogDiv)));
           _sub(T, Src0);
           _neg(T);
           _mov(Dest, T);
           return;
         }
       }
     }
     Src1 = legalize(Src1, Legal_Reg | Legal_Mem);
-    switch (Type Ty = Dest->getType()) {
+    uint32_t Eax = Traits::RegisterSet::Reg_eax;
+    uint32_t Edx = Traits::RegisterSet::Reg_edx;
+    switch (Ty) {
     default:
       llvm_unreachable("Bad type for srem");
     // fallthrough
     case IceType_i32:
-      T_edx = makeReg(Ty, Traits::RegisterSet::Reg_edx);
-      _mov(T, Src0, Traits::RegisterSet::Reg_eax);
-      _cbwdq(T_edx, T);
-      _idiv(T_edx, Src1, T);
-      _mov(Dest, T_edx);
       break;
     case IceType_i16:
-      T_edx = makeReg(Ty, Traits::RegisterSet::Reg_dx);
-      _mov(T, Src0, Traits::RegisterSet::Reg_ax);
-      _cbwdq(T_edx, T);
-      _idiv(T_edx, Src1, T);
-      _mov(Dest, T_edx);
+      Eax = Traits::RegisterSet::Reg_ax;
+      Edx = Traits::RegisterSet::Reg_dx;
       break;
     case IceType_i8:
-      T_edx = makeReg(IceType_i16, Traits::RegisterSet::Reg_ax);
-      // TODO(stichnot): Use register ah for T_edx, and remove the _shr().
-      // T_edx = makeReg(Ty, Traits::RegisterSet::Reg_ah);
-      _mov(T, Src0, Traits::RegisterSet::Reg_al);
-      _cbwdq(T_edx, T);
-      _idiv(T_edx, Src1, T);
-      static constexpr uint8_t AlSizeInBits = 8;
-      _shr(T_edx, Ctx->getConstantInt8(AlSizeInBits));
-      _mov(Dest, T_edx);
+      Eax = Traits::RegisterSet::Reg_al;
+      Edx = Traits::RegisterSet::Reg_ah;
       break;
     }
-    break;
+    T_edx = makeReg(Ty, Edx);
+    _mov(T, Src0, Eax);
+    _cbwdq(T_edx, T);
+    _idiv(T_edx, Src1, T);
+    _mov(Dest, T_edx);
+  } break;
   case InstArithmetic::Fadd:
     _mov(T, Src0);
     _addss(T, Src1);
     _mov(Dest, T);
     break;
   case InstArithmetic::Fsub:
     _mov(T, Src0);
     _subss(T, Src1);
     _mov(Dest, T);
     break;
   case InstArithmetic::Fmul:
     _mov(T, Src0);
     _mulss(T, Src0 == Src1 ? T : Src1);
     _mov(Dest, T);
     break;
   case InstArithmetic::Fdiv:
     _mov(T, Src0);
     _divss(T, Src1);
     _mov(Dest, T);
     break;
   case InstArithmetic::Frem: {
     constexpr SizeT MaxSrcs = 2;
-    Type Ty = Dest->getType();
     InstCall *Call = makeHelperCall(
         isFloat32Asserting32Or64(Ty) ? H_frem_f32 : H_frem_f64, Dest, MaxSrcs);
     Call->addArg(Src0);
     Call->addArg(Src1);
     return lowerCall(Call);
   }
   }
 }
 
 template <class Machine>
@@ skipping 57 matching lines @@
   Constant *Zero = Ctx->getConstantZero(IceType_i32);
   _cmp(Src0, Zero);
   _br(Traits::Cond::Br_ne, Inst->getTargetTrue(), Inst->getTargetFalse());
 }
 
 template <class Machine>
 void TargetX86Base<Machine>::lowerCast(const InstCast *Inst) {
   // a = cast(b) ==> t=cast(b); a=t; (link t->b, link a->t, no overlap)
   InstCast::OpKind CastKind = Inst->getCastKind();
   Variable *Dest = Inst->getDest();
+  Type DestTy = Dest->getType();
   switch (CastKind) {
   default:
     Func->setError("Cast type not supported");
     return;
   case InstCast::Sext: {
     // Src0RM is the source operand legalized to physical register or memory,
     // but not immediate, since the relevant x86 native instructions don't
     // allow an immediate operand. If the operand is an immediate, we could
     // consider computing the strength-reduced result at translation time, but
     // we're unlikely to see something like that in the bitcode that the
     // optimizer wouldn't have already taken care of.
     Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
-    if (isVectorType(Dest->getType())) {
-      Type DestTy = Dest->getType();
+    if (isVectorType(DestTy)) {
       if (DestTy == IceType_v16i8) {
         // onemask = materialize(1,1,...); dst = (src & onemask) > 0
-        Variable *OneMask = makeVectorOfOnes(Dest->getType());
+        Variable *OneMask = makeVectorOfOnes(DestTy);
         Variable *T = makeReg(DestTy);
         _movp(T, Src0RM);
         _pand(T, OneMask);
-        Variable *Zeros = makeVectorOfZeros(Dest->getType());
+        Variable *Zeros = makeVectorOfZeros(DestTy);
         _pcmpgt(T, Zeros);
         _movp(Dest, T);
       } else {
         /// width = width(elty) - 1; dest = (src << width) >> width
         SizeT ShiftAmount =
             Traits::X86_CHAR_BIT * typeWidthInBytes(typeElementType(DestTy)) -
             1;
         Constant *ShiftConstant = Ctx->getConstantInt8(ShiftAmount);
         Variable *T = makeReg(DestTy);
         _movp(T, Src0RM);
         _psll(T, ShiftConstant);
         _psra(T, ShiftConstant);
         _movp(Dest, T);
       }
-    } else if (!Traits::Is64Bit && Dest->getType() == IceType_i64) {
+    } else if (!Traits::Is64Bit && DestTy == IceType_i64) {
       // t1=movsx src; t2=t1; t2=sar t2, 31; dst.lo=t1; dst.hi=t2
       Constant *Shift = Ctx->getConstantInt32(31);
       Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
       Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
       Variable *T_Lo = makeReg(DestLo->getType());
       if (Src0RM->getType() == IceType_i32) {
         _mov(T_Lo, Src0RM);
       } else if (Src0RM->getType() == IceType_i1) {
         _movzx(T_Lo, Src0RM);
         _shl(T_Lo, Shift);
         _sar(T_Lo, Shift);
       } else {
         _movsx(T_Lo, Src0RM);
       }
       _mov(DestLo, T_Lo);
       Variable *T_Hi = nullptr;
       _mov(T_Hi, T_Lo);
       if (Src0RM->getType() != IceType_i1)
         // For i1, the sar instruction is already done above.
         _sar(T_Hi, Shift);
       _mov(DestHi, T_Hi);
     } else if (Src0RM->getType() == IceType_i1) {
       // t1 = src
       // shl t1, dst_bitwidth - 1
       // sar t1, dst_bitwidth - 1
       // dst = t1
-      size_t DestBits =
-          Traits::X86_CHAR_BIT * typeWidthInBytes(Dest->getType());
+      size_t DestBits = Traits::X86_CHAR_BIT * typeWidthInBytes(DestTy);
       Constant *ShiftAmount = Ctx->getConstantInt32(DestBits - 1);
-      Variable *T = makeReg(Dest->getType());
-      if (typeWidthInBytes(Dest->getType()) <=
-          typeWidthInBytes(Src0RM->getType())) {
+      Variable *T = makeReg(DestTy);
+      if (typeWidthInBytes(DestTy) <= typeWidthInBytes(Src0RM->getType())) {
         _mov(T, Src0RM);
       } else {
         // Widen the source using movsx or movzx. (It doesn't matter which one,
         // since the following shl/sar overwrite the bits.)
         _movzx(T, Src0RM);
       }
       _shl(T, ShiftAmount);
       _sar(T, ShiftAmount);
       _mov(Dest, T);
     } else {
       // t1 = movsx src; dst = t1
-      Variable *T = makeReg(Dest->getType());
+      Variable *T = makeReg(DestTy);
       _movsx(T, Src0RM);
       _mov(Dest, T);
     }
     break;
   }
   case InstCast::Zext: {
     Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
-    if (isVectorType(Dest->getType())) {
+    if (isVectorType(DestTy)) {
       // onemask = materialize(1,1,...); dest = onemask & src
-      Type DestTy = Dest->getType();
       Variable *OneMask = makeVectorOfOnes(DestTy);
       Variable *T = makeReg(DestTy);
       _movp(T, Src0RM);
       _pand(T, OneMask);
       _movp(Dest, T);
-    } else if (!Traits::Is64Bit && Dest->getType() == IceType_i64) {
+    } else if (!Traits::Is64Bit && DestTy == IceType_i64) {
       // t1=movzx src; dst.lo=t1; dst.hi=0
       Constant *Zero = Ctx->getConstantZero(IceType_i32);
       Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
       Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
       Variable *Tmp = makeReg(DestLo->getType());
       if (Src0RM->getType() == IceType_i32) {
         _mov(Tmp, Src0RM);
       } else {
         _movzx(Tmp, Src0RM);
       }
       _mov(DestLo, Tmp);
       _mov(DestHi, Zero);
     } else if (Src0RM->getType() == IceType_i1) {
       // t = Src0RM; Dest = t
-      Type DestTy = Dest->getType();
       Variable *T = nullptr;
       if (DestTy == IceType_i8) {
         _mov(T, Src0RM);
       } else {
         assert(DestTy != IceType_i1);
         assert(Traits::Is64Bit || DestTy != IceType_i64);
         // Use 32-bit for both 16-bit and 32-bit, since 32-bit ops are shorter.
         // In x86-64 we need to widen T to 64-bits to ensure that T -- if
         // written to the stack (i.e., in -Om1) will be fully zero-extended.
         T = makeReg(DestTy == IceType_i64 ? IceType_i64 : IceType_i32);
         _movzx(T, Src0RM);
       }
       _mov(Dest, T);
     } else {
       // t1 = movzx src; dst = t1
-      Variable *T = makeReg(Dest->getType());
+      Variable *T = makeReg(DestTy);
       _movzx(T, Src0RM);
       _mov(Dest, T);
     }
     break;
   }
   case InstCast::Trunc: {
-    if (isVectorType(Dest->getType())) {
+    if (isVectorType(DestTy)) {
       // onemask = materialize(1,1,...); dst = src & onemask
       Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
       Type Src0Ty = Src0RM->getType();
       Variable *OneMask = makeVectorOfOnes(Src0Ty);
-      Variable *T = makeReg(Dest->getType());
+      Variable *T = makeReg(DestTy);
       _movp(T, Src0RM);
       _pand(T, OneMask);
       _movp(Dest, T);
+    } else if (DestTy == IceType_i1 || DestTy == IceType_i8) {
+      // Make sure we truncate from and into valid registers.
+      Operand *Src0 = legalizeUndef(Inst->getSrc(0));
+      if (!Traits::Is64Bit && Src0->getType() == IceType_i64)
+        Src0 = loOperand(Src0);
+      Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem);
+      Variable *T = copyToReg8(Src0RM);
+      if (DestTy == IceType_i1)
+        _and(T, Ctx->getConstantInt1(1));
+      _mov(Dest, T);
     } else {
       Operand *Src0 = legalizeUndef(Inst->getSrc(0));
       if (!Traits::Is64Bit && Src0->getType() == IceType_i64)
         Src0 = loOperand(Src0);
       Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem);
       // t1 = trunc Src0RM; Dest = t1
-      Variable *T = nullptr;
+      Variable *T = makeReg(DestTy);
       _mov(T, Src0RM);
-      if (Dest->getType() == IceType_i1)
-        _and(T, Ctx->getConstantInt1(1));
       _mov(Dest, T);
     }
     break;
   }
   case InstCast::Fptrunc:
   case InstCast::Fpext: {
     Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
     // t1 = cvt Src0RM; Dest = t1
-    Variable *T = makeReg(Dest->getType());
+    Variable *T = makeReg(DestTy);
     _cvt(T, Src0RM, Traits::Insts::Cvt::Float2float);
     _mov(Dest, T);
     break;
   }
   case InstCast::Fptosi:
-    if (isVectorType(Dest->getType())) {
-      assert(Dest->getType() == IceType_v4i32 &&
+    if (isVectorType(DestTy)) {
+      assert(DestTy == IceType_v4i32 &&
              Inst->getSrc(0)->getType() == IceType_v4f32);
       Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
       if (llvm::isa<typename Traits::X86OperandMem>(Src0RM))
         Src0RM = legalizeToReg(Src0RM);
-      Variable *T = makeReg(Dest->getType());
+      Variable *T = makeReg(DestTy);
       _cvt(T, Src0RM, Traits::Insts::Cvt::Tps2dq);
       _movp(Dest, T);
-    } else if (!Traits::Is64Bit && Dest->getType() == IceType_i64) {
+    } else if (!Traits::Is64Bit && DestTy == IceType_i64) {
       constexpr SizeT MaxSrcs = 1;
       Type SrcType = Inst->getSrc(0)->getType();
       InstCall *Call =
           makeHelperCall(isFloat32Asserting32Or64(SrcType) ? H_fptosi_f32_i64
                                                            : H_fptosi_f64_i64,
                          Dest, MaxSrcs);
       Call->addArg(Inst->getSrc(0));
       lowerCall(Call);
     } else {
       Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
       // t1.i32 = cvt Src0RM; t2.dest_type = t1; Dest = t2.dest_type
       Variable *T_1 = nullptr;
-      if (Traits::Is64Bit && Dest->getType() == IceType_i64) {
+      if (Traits::Is64Bit && DestTy == IceType_i64) {
         T_1 = makeReg(IceType_i64);
       } else {
-        assert(Dest->getType() != IceType_i64);
+        assert(DestTy != IceType_i64);
         T_1 = makeReg(IceType_i32);
       }
       // cvt() requires its integer argument to be a GPR.
-      Variable *T_2 = makeReg(Dest->getType());
+      Variable *T_2 = makeReg(DestTy);
+      if (isByteSizedType(DestTy)) {
+        assert(T_1->getType() == IceType_i32);
+        T_1->setRegClass(RCX86_Is32To8);
+        T_2->setRegClass(RCX86_IsTrunc8Rcvr);
+      }
       _cvt(T_1, Src0RM, Traits::Insts::Cvt::Tss2si);
       _mov(T_2, T_1); // T_1 and T_2 may have different integer types
-      if (Dest->getType() == IceType_i1)
+      if (DestTy == IceType_i1)
         _and(T_2, Ctx->getConstantInt1(1));
       _mov(Dest, T_2);
     }
     break;
   case InstCast::Fptoui:
-    if (isVectorType(Dest->getType())) {
-      assert(Dest->getType() == IceType_v4i32 &&
+    if (isVectorType(DestTy)) {
+      assert(DestTy == IceType_v4i32 &&
              Inst->getSrc(0)->getType() == IceType_v4f32);
       constexpr SizeT MaxSrcs = 1;
       InstCall *Call = makeHelperCall(H_fptoui_4xi32_f32, Dest, MaxSrcs);
       Call->addArg(Inst->getSrc(0));
       lowerCall(Call);
-    } else if (Dest->getType() == IceType_i64 ||
-               (!Traits::Is64Bit && Dest->getType() == IceType_i32)) {
+    } else if (DestTy == IceType_i64 ||
+               (!Traits::Is64Bit && DestTy == IceType_i32)) {
       // Use a helper for both x86-32 and x86-64.
       constexpr SizeT MaxSrcs = 1;
-      Type DestType = Dest->getType();
       Type SrcType = Inst->getSrc(0)->getType();
       IceString TargetString;
       if (Traits::Is64Bit) {
         TargetString = isFloat32Asserting32Or64(SrcType) ? H_fptoui_f32_i64
                                                          : H_fptoui_f64_i64;
-      } else if (isInt32Asserting32Or64(DestType)) {
+      } else if (isInt32Asserting32Or64(DestTy)) {
         TargetString = isFloat32Asserting32Or64(SrcType) ? H_fptoui_f32_i32
                                                          : H_fptoui_f64_i32;
       } else {
         TargetString = isFloat32Asserting32Or64(SrcType) ? H_fptoui_f32_i64
                                                          : H_fptoui_f64_i64;
       }
       InstCall *Call = makeHelperCall(TargetString, Dest, MaxSrcs);
       Call->addArg(Inst->getSrc(0));
       lowerCall(Call);
       return;
     } else {
       Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
       // t1.i32 = cvt Src0RM; t2.dest_type = t1; Dest = t2.dest_type
-      assert(Dest->getType() != IceType_i64);
+      assert(DestTy != IceType_i64);
       Variable *T_1 = nullptr;
-      if (Traits::Is64Bit && Dest->getType() == IceType_i32) {
+      if (Traits::Is64Bit && DestTy == IceType_i32) {
         T_1 = makeReg(IceType_i64);
       } else {
-        assert(Dest->getType() != IceType_i32);
+        assert(DestTy != IceType_i32);
         T_1 = makeReg(IceType_i32);
       }
-      Variable *T_2 = makeReg(Dest->getType());
+      Variable *T_2 = makeReg(DestTy);
+      if (isByteSizedType(DestTy)) {
+        assert(T_1->getType() == IceType_i32);
+        T_1->setRegClass(RCX86_Is32To8);
+        T_2->setRegClass(RCX86_IsTrunc8Rcvr);
+      }
       _cvt(T_1, Src0RM, Traits::Insts::Cvt::Tss2si);
       _mov(T_2, T_1); // T_1 and T_2 may have different integer types
-      if (Dest->getType() == IceType_i1)
+      if (DestTy == IceType_i1)
         _and(T_2, Ctx->getConstantInt1(1));
       _mov(Dest, T_2);
     }
     break;
   case InstCast::Sitofp:
-    if (isVectorType(Dest->getType())) {
-      assert(Dest->getType() == IceType_v4f32 &&
+    if (isVectorType(DestTy)) {
+      assert(DestTy == IceType_v4f32 &&
              Inst->getSrc(0)->getType() == IceType_v4i32);
       Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
       if (llvm::isa<typename Traits::X86OperandMem>(Src0RM))
         Src0RM = legalizeToReg(Src0RM);
-      Variable *T = makeReg(Dest->getType());
+      Variable *T = makeReg(DestTy);
       _cvt(T, Src0RM, Traits::Insts::Cvt::Dq2ps);
       _movp(Dest, T);
     } else if (!Traits::Is64Bit && Inst->getSrc(0)->getType() == IceType_i64) {
       // Use a helper for x86-32.
       constexpr SizeT MaxSrcs = 1;
-      Type DestType = Dest->getType();
       InstCall *Call =
-          makeHelperCall(isFloat32Asserting32Or64(DestType) ? H_sitofp_i64_f32
-                                                            : H_sitofp_i64_f64,
+          makeHelperCall(isFloat32Asserting32Or64(DestTy) ? H_sitofp_i64_f32
+                                                          : H_sitofp_i64_f64,
                          Dest, MaxSrcs);
       // TODO: Call the correct compiler-rt helper function.
       Call->addArg(Inst->getSrc(0));
       lowerCall(Call);
       return;
     } else {
       Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem);
       // Sign-extend the operand.
       // t1.i32 = movsx Src0RM; t2 = Cvt t1.i32; Dest = t2
       Variable *T_1 = nullptr;
       if (Traits::Is64Bit && Src0RM->getType() == IceType_i64) {
         T_1 = makeReg(IceType_i64);
       } else {
         assert(Src0RM->getType() != IceType_i64);
         T_1 = makeReg(IceType_i32);
       }
-      Variable *T_2 = makeReg(Dest->getType());
+      Variable *T_2 = makeReg(DestTy);
       if (Src0RM->getType() == T_1->getType())
         _mov(T_1, Src0RM);
       else
         _movsx(T_1, Src0RM);
       _cvt(T_2, T_1, Traits::Insts::Cvt::Si2ss);
       _mov(Dest, T_2);
     }
     break;
   case InstCast::Uitofp: {
     Operand *Src0 = Inst->getSrc(0);
     if (isVectorType(Src0->getType())) {
-      assert(Dest->getType() == IceType_v4f32 &&
-             Src0->getType() == IceType_v4i32);
+      assert(DestTy == IceType_v4f32 && Src0->getType() == IceType_v4i32);
       constexpr SizeT MaxSrcs = 1;
       InstCall *Call = makeHelperCall(H_uitofp_4xi32_4xf32, Dest, MaxSrcs);
       Call->addArg(Src0);
       lowerCall(Call);
     } else if (Src0->getType() == IceType_i64 ||
                (!Traits::Is64Bit && Src0->getType() == IceType_i32)) {
       // Use a helper for x86-32 and x86-64. Also use a helper for i32 on
       // x86-32.
       constexpr SizeT MaxSrcs = 1;
-      Type DestType = Dest->getType();
       IceString TargetString;
       if (isInt32Asserting32Or64(Src0->getType())) {
-        TargetString = isFloat32Asserting32Or64(DestType) ? H_uitofp_i32_f32
-                                                          : H_uitofp_i32_f64;
+        TargetString = isFloat32Asserting32Or64(DestTy) ? H_uitofp_i32_f32
+                                                        : H_uitofp_i32_f64;
       } else {
-        TargetString = isFloat32Asserting32Or64(DestType) ? H_uitofp_i64_f32
-                                                          : H_uitofp_i64_f64;
+        TargetString = isFloat32Asserting32Or64(DestTy) ? H_uitofp_i64_f32
+                                                        : H_uitofp_i64_f64;
       }
       InstCall *Call = makeHelperCall(TargetString, Dest, MaxSrcs);
       Call->addArg(Src0);
       lowerCall(Call);
       return;
     } else {
       Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem);
       // Zero-extend the operand.
       // t1.i32 = movzx Src0RM; t2 = Cvt t1.i32; Dest = t2
       Variable *T_1 = nullptr;
       if (Traits::Is64Bit && Src0RM->getType() == IceType_i32) {
         T_1 = makeReg(IceType_i64);
       } else {
         assert(Src0RM->getType() != IceType_i64);
         assert(Traits::Is64Bit || Src0RM->getType() != IceType_i32);
         T_1 = makeReg(IceType_i32);
       }
-      Variable *T_2 = makeReg(Dest->getType());
+      Variable *T_2 = makeReg(DestTy);
       if (Src0RM->getType() == T_1->getType())
         _mov(T_1, Src0RM);
       else
         _movzx(T_1, Src0RM);
       _cvt(T_2, T_1, Traits::Insts::Cvt::Si2ss);
       _mov(Dest, T_2);
     }
     break;
   }
   case InstCast::Bitcast: {
     Operand *Src0 = Inst->getSrc(0);
-    if (Dest->getType() == Src0->getType()) {
+    if (DestTy == Src0->getType()) {
       InstAssign *Assign = InstAssign::create(Func, Dest, Src0);
       lowerAssign(Assign);
       return;
     }
-    switch (Dest->getType()) {
+    switch (DestTy) {
     default:
       llvm_unreachable("Unexpected Bitcast dest type");
     case IceType_i8: {
       assert(Src0->getType() == IceType_v8i1);
       InstCall *Call = makeHelperCall(H_bitcast_8xi1_i8, Dest, 1);
       Call->addArg(Src0);
       lowerCall(Call);
     } break;
     case IceType_i16: {
       assert(Src0->getType() == IceType_v16i1);
       InstCall *Call = makeHelperCall(H_bitcast_16xi1_i16, Dest, 1);
       Call->addArg(Src0);
       lowerCall(Call);
     } break;
     case IceType_i32:
     case IceType_f32: {
       Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem);
-      Type DestType = Dest->getType();
       Type SrcType = Src0RM->getType();
-      (void)DestType;
-      assert((DestType == IceType_i32 && SrcType == IceType_f32) ||
-             (DestType == IceType_f32 && SrcType == IceType_i32));
+      assert((DestTy == IceType_i32 && SrcType == IceType_f32) ||
+             (DestTy == IceType_f32 && SrcType == IceType_i32));
       // a.i32 = bitcast b.f32 ==>
       //   t.f32 = b.f32
       //   s.f32 = spill t.f32
       //   a.i32 = s.f32
       Variable *T = nullptr;
       // TODO: Should be able to force a spill setup by calling legalize() with
       // Legal_Mem and not Legal_Reg or Legal_Imm.
       typename Traits::SpillVariable *SpillVar =
           Func->makeVariable<typename Traits::SpillVariable>(SrcType);
       SpillVar->setLinkedTo(Dest);
@@ skipping 53 matching lines @@
       if (Traits::Is64Bit) {
         Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem);
         Variable *T = makeReg(IceType_f64);
         // Movd requires its fp argument (in this case, the bitcast
         // destination) to be an xmm register.
         _movd(T, Src0RM);
         _mov(Dest, T);
       } else {
         Src0 = legalize(Src0);
         if (llvm::isa<typename Traits::X86OperandMem>(Src0)) {
-          Variable *T = Func->makeVariable(Dest->getType());
+          Variable *T = Func->makeVariable(DestTy);
           _movq(T, Src0);
           _movq(Dest, T);
           break;
         }
         // a.f64 = bitcast b.i64 ==>
         //   t_lo.i32 = b_lo.i32
         //   FakeDef(s.f64)
         //   lo(s.f64) = t_lo.i32
         //   t_hi.i32 = b_hi.i32
         //   hi(s.f64) = t_hi.i32
@@ skipping 580 matching lines @@
 
   if (Ty == IceType_v8i16 || Ty == IceType_v8i1 ||
       InstructionSet >= Traits::SSE4_1) {
     // Use insertps, pinsrb, pinsrw, or pinsrd.
     Operand *ElementRM =
         legalize(ElementToInsertNotLegalized, Legal_Reg | Legal_Mem);
     Operand *SourceVectRM =
         legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem);
     Variable *T = makeReg(Ty);
     _movp(T, SourceVectRM);
-    if (Ty == IceType_v4f32)
+    if (Ty == IceType_v4f32) {
       _insertps(T, ElementRM, Ctx->getConstantInt32(Index << 4));
-    else
-      // TODO(stichnot): For the pinsrb and pinsrw instructions, when the source
-      // operand is a register, it must be a full r32 register like eax, and not
-      // ax/al/ah.  For filetype=asm, InstX86Pinsr<Machine>::emit() compensates
-      // for the use of r16 and r8 by converting them through getBaseReg(),
-      // while emitIAS() validates that the original and base register encodings
-      // are the same.  But for an "interior" register like ah, it should
-      // probably be copied into an r32 via movzx so that the types work out.
+    } else {
+      // For the pinsrb and pinsrw instructions, when the source operand is a
+      // register, it must be a full r32 register like eax, and not ax/al/ah.
+      // For filetype=asm, InstX86Pinsr<Machine>::emit() compensates for the use
+      // of r16 and r8 by converting them through getBaseReg(), while emitIAS()
+      // validates that the original and base register encodings are the same.
+      if (ElementRM->getType() == IceType_i8 &&
+          llvm::isa<Variable>(ElementRM)) {
+        // Don't use ah/bh/ch/dh for pinsrb.
+        ElementRM = copyToReg8(ElementRM);
+      }
       _pinsr(T, ElementRM, Ctx->getConstantInt32(Index));
+    }
     _movp(Inst->getDest(), T);
   } else if (Ty == IceType_v4i32 || Ty == IceType_v4f32 || Ty == IceType_v4i1) {
     // Use shufps or movss.
     Variable *ElementR = nullptr;
     Operand *SourceVectRM =
         legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem);
 
     if (InVectorElementTy == IceType_f32) {
       // ElementR will be in an XMM register since it is floating point.
       ElementR = legalizeToReg(ElementToInsertNotLegalized);
@@ skipping 2286 matching lines @@
   // TODO(wala,stichnot): lea should not
   // be required. The address of the stack slot is known at compile time
   // (although not until after addProlog()).
   constexpr Type PointerType = IceType_i32;
   Variable *Loc = makeReg(PointerType);
   _lea(Loc, Slot);
   Constant *ConstantOffset = Ctx->getConstantInt32(Offset);
   return Traits::X86OperandMem::create(Func, Ty, Loc, ConstantOffset);
 }
 
+/// Lowering helper to copy a scalar integer source operand into some 8-bit GPR.
+/// Src is assumed to already be legalized.  If the source operand is known to
+/// be a memory or immediate operand, a simple mov will suffice.  But if the
+/// source operand can be a physical register, then it must first be copied into
+/// a physical register that is truncable to 8-bit, then truncated into a
+/// physical register that can receive a truncation, and finally copied into the
+/// result 8-bit register (which in general can be any 8-bit register).  For
+/// example, moving %ebp into %ah may be accomplished as:
+///   movl %ebp, %edx
+///   mov_trunc %edx, %dl  // this redundant assignment is ultimately elided
+///   movb %dl, %ah
+/// On the other hand, moving a memory or immediate operand into ah:
+///   movb 4(%ebp), %ah
+///   movb $my_imm, %ah
+///
+/// Note #1.  On a 64-bit target, the "movb 4(%ebp), %ah" is likely not
+/// encodable, so RegNum=Reg_ah should NOT be given as an argument.  Instead,
+/// use RegNum=NoRegister and then let the caller do a separate copy into
+/// Reg_ah.
+///
+/// Note #2.  ConstantRelocatable operands are also put through this process
+/// (not truncated directly) because our ELF emitter does R_386_32 relocations
+/// but not R_386_8 relocations.
+///
+/// Note #3.  If Src is a Variable, the result will be an infinite-weight i8
+/// Variable with the RCX86_IsTrunc8Rcvr register class.  As such, this helper
+/// is a convenient way to prevent ah/bh/ch/dh from being an (invalid) argument
+/// to the pinsrb instruction.
+template <class Machine>
+Variable *TargetX86Base<Machine>::copyToReg8(Operand *Src, int32_t RegNum) {
+  Type Ty = Src->getType();
+  assert(isScalarIntegerType(Ty));
+  assert(Ty != IceType_i1);
+  Variable *Reg = makeReg(IceType_i8, RegNum);
+  Reg->setRegClass(RCX86_IsTrunc8Rcvr);
+  if (llvm::isa<Variable>(Src) || llvm::isa<ConstantRelocatable>(Src)) {
+    Variable *SrcTruncable = makeReg(Ty);
+    switch (Ty) {
+    case IceType_i64:
+      SrcTruncable->setRegClass(RCX86_Is64To8);
+      break;
+    case IceType_i32:
+      SrcTruncable->setRegClass(RCX86_Is32To8);
+      break;
+    case IceType_i16:
+      SrcTruncable->setRegClass(RCX86_Is16To8);
+      break;
+    default:
+      // i8 - just use default register class
+      break;
+    }
+    Variable *SrcRcvr = makeReg(IceType_i8);
+    SrcRcvr->setRegClass(RCX86_IsTrunc8Rcvr);
+    _mov(SrcTruncable, Src);
+    _mov(SrcRcvr, SrcTruncable);
+    Src = SrcRcvr;
+  }
+  _mov(Reg, Src);
+  return Reg;
+}
+
 /// Helper for legalize() to emit the right code to lower an operand to a
 /// register of the appropriate type.
 template <class Machine>
 Variable *TargetX86Base<Machine>::copyToReg(Operand *Src, int32_t RegNum) {
   Type Ty = Src->getType();
   Variable *Reg = makeReg(Ty, RegNum);
   if (isVectorType(Ty)) {
     _movp(Reg, Src);
   } else {
     _mov(Reg, Src);
@@ skipping 493 matching lines @@
   }
   // the offset is not eligible for blinding or pooling, return the original
   // mem operand
   return MemOperand;
 }
 
 } // end of namespace X86Internal
 } // end of namespace Ice
 
 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H