Subzero: Fold the load instruction into the next cast instruction.

src/IceTargetLoweringX8632.cpp

 //===- subzero/src/IceTargetLoweringX8632.cpp - x86-32 lowering -----------===//
 //
 //                        The Subzero Code Generator
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the TargetLoweringX8632 class, which
@@ skipping 865 matching lines @@
     // register) or already has one (if passed on the stack).
     if (Var->getIsArg())
       continue;
     // An unreferenced variable doesn't need a stack slot.
     if (!IsVarReferenced[Var->getIndex()])
       continue;
     // A spill slot linked to a variable with a stack slot should reuse
     // that stack slot.
     if (SpillVariable *SpillVar = llvm::dyn_cast<SpillVariable>(Var)) {
       assert(Var->getWeight().isZero());
-      if (!SpillVar->getLinkedTo()->hasReg()) {
+      if (SpillVar->getLinkedTo() && !SpillVar->getLinkedTo()->hasReg()) {
         VariablesLinkedToSpillSlots.push_back(Var);
         continue;
       }
     }
     SpilledVariables.push_back(Var);
   }
 
   SortedSpilledVariables.reserve(SpilledVariables.size());
   sortByAlignment(SortedSpilledVariables, SpilledVariables);
   for (Variable *Var : SortedSpilledVariables) {
@@ skipping 256 matching lines @@
     Hi->setName(Func, Var->getName(Func) + "__hi");
   }
   Var->setLoHi(Lo, Hi);
   if (Var->getIsArg()) {
     Lo->setIsArg();
     Hi->setIsArg();
   }
 }
 
 Operand *TargetX8632::loOperand(Operand *Operand) {
-  assert(Operand->getType() == IceType_i64);
-  if (Operand->getType() != IceType_i64)
+  assert(Operand->getType() == IceType_i64 ||
+         Operand->getType() == IceType_f64);
+  if (Operand->getType() != IceType_i64 && Operand->getType() != IceType_f64)
     return Operand;
   if (Variable *Var = llvm::dyn_cast<Variable>(Operand)) {
     split64(Var);
     return Var->getLo();
   }
   if (ConstantInteger64 *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) {
     return Ctx->getConstantInt32(static_cast<uint32_t>(Const->getValue()));
   }
   if (OperandX8632Mem *Mem = llvm::dyn_cast<OperandX8632Mem>(Operand)) {
     return OperandX8632Mem::create(Func, IceType_i32, Mem->getBase(),
                                    Mem->getOffset(), Mem->getIndex(),
                                    Mem->getShift(), Mem->getSegmentRegister());
   }
   llvm_unreachable("Unsupported operand type");
   return nullptr;
 }
 
 Operand *TargetX8632::hiOperand(Operand *Operand) {
-  assert(Operand->getType() == IceType_i64);
-  if (Operand->getType() != IceType_i64)
+  assert(Operand->getType() == IceType_i64 ||
+         Operand->getType() == IceType_f64);
+  if (Operand->getType() != IceType_i64 && Operand->getType() != IceType_f64)
     return Operand;
   if (Variable *Var = llvm::dyn_cast<Variable>(Operand)) {
     split64(Var);
     return Var->getHi();
   }
   if (ConstantInteger64 *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) {
     return Ctx->getConstantInt32(
         static_cast<uint32_t>(Const->getValue() >> 32));
   }
   if (OperandX8632Mem *Mem = llvm::dyn_cast<OperandX8632Mem>(Operand)) {
@@ skipping 1261 matching lines @@
     } break;
     case IceType_i64: {
       Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem);
       assert(Src0RM->getType() == IceType_f64);
       // a.i64 = bitcast b.f64 ==>
       //   s.f64 = spill b.f64
       //   t_lo.i32 = lo(s.f64)
       //   a_lo.i32 = t_lo.i32
       //   t_hi.i32 = hi(s.f64)
       //   a_hi.i32 = t_hi.i32
-      SpillVariable *SpillVar = Func->makeVariable<SpillVariable>(IceType_f64);
-      SpillVar->setLinkedTo(llvm::dyn_cast<Variable>(Src0RM));
-      Variable *Spill = SpillVar;
-      Spill->setWeight(RegWeight::Zero);
-      _movq(Spill, Src0RM);
+      Operand *SpillLo, *SpillHi;
+      if (auto *Src0Var = llvm::dyn_cast<Variable>(Src0RM)) {
+        SpillVariable *SpillVar =
+            Func->makeVariable<SpillVariable>(IceType_f64);
+        SpillVar->setLinkedTo(Src0Var);
+        Variable *Spill = SpillVar;
+        Spill->setWeight(RegWeight::Zero);
+        _movq(Spill, Src0RM);
+        SpillLo = VariableSplit::create(Func, Spill, VariableSplit::Low);
+        SpillHi = VariableSplit::create(Func, Spill, VariableSplit::High);
+      } else {
+        SpillLo = loOperand(Src0RM);
+        SpillHi = hiOperand(Src0RM);
+      }
 
       Variable *DestLo = llvm::cast<Variable>(loOperand(Dest));
       Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest));
       Variable *T_Lo = makeReg(IceType_i32);
       Variable *T_Hi = makeReg(IceType_i32);
-      VariableSplit *SpillLo =
-          VariableSplit::create(Func, Spill, VariableSplit::Low);
-      VariableSplit *SpillHi =
-          VariableSplit::create(Func, Spill, VariableSplit::High);
 
       _mov(T_Lo, SpillLo);
       _mov(DestLo, T_Lo);
       _mov(T_Hi, SpillHi);
       _mov(DestHi, T_Hi);
     } break;
     case IceType_f64: {
       Src0 = legalize(Src0);
       assert(Src0->getType() == IceType_i64);
+      if (llvm::isa<OperandX8632Mem>(Src0)) {
+        Variable *T = Func->makeVariable(Dest->getType());
+        _movq(T, Src0);
+        _movq(Dest, T);
+        break;
+      }
       // a.f64 = bitcast b.i64 ==>
       //   t_lo.i32 = b_lo.i32
       //   FakeDef(s.f64)
       //   lo(s.f64) = t_lo.i32
       //   t_hi.i32 = b_hi.i32
       //   hi(s.f64) = t_hi.i32
       //   a.f64 = s.f64
       SpillVariable *SpillVar = Func->makeVariable<SpillVariable>(IceType_f64);
       SpillVar->setLinkedTo(Dest);
       Variable *Spill = SpillVar;
@@ skipping 1449 matching lines @@
     // Index is Index=Var-Const ==>
     //   set Index=Var, Offset-=(Const<<Shift)
 
     // TODO: consider overflow issues with respect to Offset.
     // TODO: handle symbolic constants.
   }
 }
 
 } // anonymous namespace
 
-void TargetX8632::lowerLoad(const InstLoad *Inst) {
+void TargetX8632::lowerLoad(const InstLoad *Load) {
   // A Load instruction can be treated the same as an Assign
   // instruction, after the source operand is transformed into an
   // OperandX8632Mem operand.  Note that the address mode
   // optimization already creates an OperandX8632Mem operand, so it
   // doesn't need another level of transformation.
-  Type Ty = Inst->getDest()->getType();
-  Operand *Src0 = FormMemoryOperand(Inst->getSourceAddress(), Ty);
+  Type Ty = Load->getDest()->getType();
+  Operand *Src0 = FormMemoryOperand(Load->getSourceAddress(), Ty);
 
   // Fuse this load with a subsequent Arithmetic instruction in the
   // following situations:
   //   a=[mem]; c=b+a ==> c=b+[mem] if last use of a and a not in b
   //   a=[mem]; c=a+b ==> c=b+[mem] if commutative and above is true
   //
+  // Fuse this load with a subsequent Cast instruction:
+  //   a=[mem]; b=cast(a) ==> b=cast([mem]) if last use of a
+  //
   // TODO: Clean up and test thoroughly.
   // (E.g., if there is an mfence-all make sure the load ends up on the
   // same side of the fence).
   //
   // TODO: Why limit to Arithmetic instructions?  This could probably be
   // applied to most any instruction type.  Look at all source operands
   // in the following instruction, and if there is one instance of the
   // load instruction's dest variable, and that instruction ends that
   // variable's live range, then make the substitution.  Deal with
   // commutativity optimization in the arithmetic instruction lowering.
-  InstArithmetic *NewArith = nullptr;
-  if (InstArithmetic *Arith =
-          llvm::dyn_cast_or_null<InstArithmetic>(Context.getNextInst())) {
-    Variable *DestLoad = Inst->getDest();
-    Variable *Src0Arith = llvm::dyn_cast<Variable>(Arith->getSrc(0));
-    Variable *Src1Arith = llvm::dyn_cast<Variable>(Arith->getSrc(1));
-    if (Src1Arith == DestLoad && Arith->isLastUse(Src1Arith) &&
-        DestLoad != Src0Arith) {
-      NewArith = InstArithmetic::create(Func, Arith->getOp(), Arith->getDest(),
-                                        Arith->getSrc(0), Src0);
-    } else if (Src0Arith == DestLoad && Arith->isCommutative() &&
-               Arith->isLastUse(Src0Arith) && DestLoad != Src1Arith) {
-      NewArith = InstArithmetic::create(Func, Arith->getOp(), Arith->getDest(),
-                                        Arith->getSrc(1), Src0);
-    }
-    if (NewArith) {
-      Arith->setDeleted();
-      Context.advanceNext();
-      lowerArithmetic(NewArith);
-      return;
+  //
+  // TODO(stichnot): Do load fusing as a separate pass.  Run it before
+  // the bool folding pass.  Modify Ice::Inst to allow src operands to
+  // be replaced, including updating Inst::LiveRangesEnded, to avoid
+  // having to manually mostly clone each instruction type.
+  Inst *NextInst = Context.getNextInst();
+  Variable *DestLoad = Load->getDest();
+  if (NextInst && NextInst->isLastUse(DestLoad)) {
+    if (auto *Arith = llvm::dyn_cast<InstArithmetic>(NextInst)) {
+      InstArithmetic *NewArith = nullptr;
+      Variable *Src0Arith = llvm::dyn_cast<Variable>(Arith->getSrc(0));
+      Variable *Src1Arith = llvm::dyn_cast<Variable>(Arith->getSrc(1));
+      if (Src1Arith == DestLoad && DestLoad != Src0Arith) {
+        NewArith = InstArithmetic::create(
+            Func, Arith->getOp(), Arith->getDest(), Arith->getSrc(0), Src0);
+      } else if (Src0Arith == DestLoad && Arith->isCommutative() &&
+                 DestLoad != Src1Arith) {
+        NewArith = InstArithmetic::create(
+            Func, Arith->getOp(), Arith->getDest(), Arith->getSrc(1), Src0);
+      }
+      if (NewArith) {
+        Arith->setDeleted();
+        Context.advanceNext();
+        lowerArithmetic(NewArith);
+        return;
+      }
+    } else if (auto *Cast = llvm::dyn_cast<InstCast>(NextInst)) {
+      Variable *Src0Cast = llvm::dyn_cast<Variable>(Cast->getSrc(0));
+      if (Src0Cast == DestLoad) {
+        InstCast *NewCast =
+            InstCast::create(Func, Cast->getCastKind(), Cast->getDest(), Src0);
+        Cast->setDeleted();
+        Context.advanceNext();
+        lowerCast(NewCast);
+        return;
+      }
     }
   }
 
-  InstAssign *Assign = InstAssign::create(Func, Inst->getDest(), Src0);
+  InstAssign *Assign = InstAssign::create(Func, DestLoad, Src0);
   lowerAssign(Assign);
 }
 
 void TargetX8632::doAddressOptLoad() {
   Inst *Inst = Context.getCur();
   Variable *Dest = Inst->getDest();
   Operand *Addr = Inst->getSrc(0);
   Variable *Index = nullptr;
   uint16_t Shift = 0;
   int32_t Offset = 0; // TODO: make Constant
@@ skipping 1016 matching lines @@
   case FT_Asm:
   case FT_Iasm: {
     OstreamLocker L(Ctx);
     emitConstantPool<PoolTypeConverter<float>>(Ctx);
     emitConstantPool<PoolTypeConverter<double>>(Ctx);
   } break;
   }
 }
 
 } // end of namespace Ice