| OLD | NEW |
| 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// | 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// |
| 2 // | 2 // |
| 3 // The Subzero Code Generator | 3 // The Subzero Code Generator |
| 4 // | 4 // |
| 5 // This file is distributed under the University of Illinois Open Source | 5 // This file is distributed under the University of Illinois Open Source |
| 6 // License. See LICENSE.TXT for details. | 6 // License. See LICENSE.TXT for details. |
| 7 // | 7 // |
| 8 //===----------------------------------------------------------------------===// | 8 //===----------------------------------------------------------------------===// |
| 9 // | 9 // |
| 10 // This file implements the TargetLoweringX86Base class, which | 10 // This file implements the TargetLoweringX86Base class, which |
| (...skipping 571 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 582 continue; | 582 continue; |
| 583 if (Func->isVerbose(IceV_RMW)) { | 583 if (Func->isVerbose(IceV_RMW)) { |
| 584 Str << "Found RMW in " << Func->getFunctionName() << ":\n "; | 584 Str << "Found RMW in " << Func->getFunctionName() << ":\n "; |
| 585 Load->dump(Func); | 585 Load->dump(Func); |
| 586 Str << "\n "; | 586 Str << "\n "; |
| 587 Arith->dump(Func); | 587 Arith->dump(Func); |
| 588 Str << "\n "; | 588 Str << "\n "; |
| 589 Store->dump(Func); | 589 Store->dump(Func); |
| 590 Str << "\n"; | 590 Str << "\n"; |
| 591 } | 591 } |
| 592 Variable *Beacon = Func->makeVariable(IceType_i32); | 592 Variable *Beacon = Func->template makeVariable(IceType_i32); |
| 593 Beacon->setWeight(0); | 593 Beacon->setWeight(0); |
| 594 Store->setRmwBeacon(Beacon); | 594 Store->setRmwBeacon(Beacon); |
| 595 InstFakeDef *BeaconDef = InstFakeDef::create(Func, Beacon); | 595 InstFakeDef *BeaconDef = InstFakeDef::create(Func, Beacon); |
| 596 Node->getInsts().insert(I3, BeaconDef); | 596 Node->getInsts().insert(I3, BeaconDef); |
| 597 InstX8632FakeRMW *RMW = InstX8632FakeRMW::create( | 597 InstX8632FakeRMW *RMW = InstX8632FakeRMW::create( |
| 598 Func, ArithSrcOther, Store->getAddr(), Beacon, Arith->getOp()); | 598 Func, ArithSrcOther, Store->getAddr(), Beacon, Arith->getOp()); |
| 599 Node->getInsts().insert(I3, RMW); | 599 Node->getInsts().insert(I3, RMW); |
| 600 } | 600 } |
| 601 } | 601 } |
| 602 } | 602 } |
| (...skipping 136 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 739 | 739 |
| 740 template <class Machine> | 740 template <class Machine> |
| 741 Variable *TargetX86Base<Machine>::getPhysicalRegister(SizeT RegNum, Type Ty) { | 741 Variable *TargetX86Base<Machine>::getPhysicalRegister(SizeT RegNum, Type Ty) { |
| 742 if (Ty == IceType_void) | 742 if (Ty == IceType_void) |
| 743 Ty = IceType_i32; | 743 Ty = IceType_i32; |
| 744 if (PhysicalRegisters[Ty].empty()) | 744 if (PhysicalRegisters[Ty].empty()) |
| 745 PhysicalRegisters[Ty].resize(RegX8632::Reg_NUM); | 745 PhysicalRegisters[Ty].resize(RegX8632::Reg_NUM); |
| 746 assert(RegNum < PhysicalRegisters[Ty].size()); | 746 assert(RegNum < PhysicalRegisters[Ty].size()); |
| 747 Variable *Reg = PhysicalRegisters[Ty][RegNum]; | 747 Variable *Reg = PhysicalRegisters[Ty][RegNum]; |
| 748 if (Reg == nullptr) { | 748 if (Reg == nullptr) { |
| 749 Reg = Func->makeVariable(Ty); | 749 Reg = Func->template makeVariable(Ty); |
| 750 Reg->setRegNum(RegNum); | 750 Reg->setRegNum(RegNum); |
| 751 PhysicalRegisters[Ty][RegNum] = Reg; | 751 PhysicalRegisters[Ty][RegNum] = Reg; |
| 752 // Specially mark esp as an "argument" so that it is considered | 752 // Specially mark esp as an "argument" so that it is considered |
| 753 // live upon function entry. | 753 // live upon function entry. |
| 754 if (RegNum == RegX8632::Reg_esp) { | 754 if (RegNum == RegX8632::Reg_esp) { |
| 755 Func->addImplicitArg(Reg); | 755 Func->addImplicitArg(Reg); |
| 756 Reg->setIgnoreLiveness(); | 756 Reg->setIgnoreLiveness(); |
| 757 } | 757 } |
| 758 } | 758 } |
| 759 return Reg; | 759 return Reg; |
| (...skipping 74 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 834 I < E && NumXmmArgs < Traits::X86_MAX_XMM_ARGS; ++I) { | 834 I < E && NumXmmArgs < Traits::X86_MAX_XMM_ARGS; ++I) { |
| 835 Variable *Arg = Args[I]; | 835 Variable *Arg = Args[I]; |
| 836 Type Ty = Arg->getType(); | 836 Type Ty = Arg->getType(); |
| 837 if (!isVectorType(Ty)) | 837 if (!isVectorType(Ty)) |
| 838 continue; | 838 continue; |
| 839 // Replace Arg in the argument list with the home register. Then | 839 // Replace Arg in the argument list with the home register. Then |
| 840 // generate an instruction in the prolog to copy the home register | 840 // generate an instruction in the prolog to copy the home register |
| 841 // to the assigned location of Arg. | 841 // to the assigned location of Arg. |
| 842 int32_t RegNum = RegX8632::Reg_xmm0 + NumXmmArgs; | 842 int32_t RegNum = RegX8632::Reg_xmm0 + NumXmmArgs; |
| 843 ++NumXmmArgs; | 843 ++NumXmmArgs; |
| 844 Variable *RegisterArg = Func->makeVariable(Ty); | 844 Variable *RegisterArg = Func->template makeVariable(Ty); |
| 845 if (ALLOW_DUMP) | 845 if (ALLOW_DUMP) |
| 846 RegisterArg->setName(Func, "home_reg:" + Arg->getName(Func)); | 846 RegisterArg->setName(Func, "home_reg:" + Arg->getName(Func)); |
| 847 RegisterArg->setRegNum(RegNum); | 847 RegisterArg->setRegNum(RegNum); |
| 848 RegisterArg->setIsArg(); | 848 RegisterArg->setIsArg(); |
| 849 Arg->setIsArg(false); | 849 Arg->setIsArg(false); |
| 850 | 850 |
| 851 Args[I] = RegisterArg; | 851 Args[I] = RegisterArg; |
| 852 Context.insert(InstAssign::create(Func, Arg, RegisterArg)); | 852 Context.insert(InstAssign::create(Func, Arg, RegisterArg)); |
| 853 } | 853 } |
| 854 } | 854 } |
| (...skipping 326 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1181 case IceType_f64: | 1181 case IceType_f64: |
| 1182 break; | 1182 break; |
| 1183 } | 1183 } |
| 1184 Variable *Lo = Var->getLo(); | 1184 Variable *Lo = Var->getLo(); |
| 1185 Variable *Hi = Var->getHi(); | 1185 Variable *Hi = Var->getHi(); |
| 1186 if (Lo) { | 1186 if (Lo) { |
| 1187 assert(Hi); | 1187 assert(Hi); |
| 1188 return; | 1188 return; |
| 1189 } | 1189 } |
| 1190 assert(Hi == nullptr); | 1190 assert(Hi == nullptr); |
| 1191 Lo = Func->makeVariable(IceType_i32); | 1191 Lo = Func->template makeVariable(IceType_i32); |
| 1192 Hi = Func->makeVariable(IceType_i32); | 1192 Hi = Func->template makeVariable(IceType_i32); |
| 1193 if (ALLOW_DUMP) { | 1193 if (ALLOW_DUMP) { |
| 1194 Lo->setName(Func, Var->getName(Func) + "__lo"); | 1194 Lo->setName(Func, Var->getName(Func) + "__lo"); |
| 1195 Hi->setName(Func, Var->getName(Func) + "__hi"); | 1195 Hi->setName(Func, Var->getName(Func) + "__hi"); |
| 1196 } | 1196 } |
| 1197 Var->setLoHi(Lo, Hi); | 1197 Var->setLoHi(Lo, Hi); |
| 1198 if (Var->getIsArg()) { | 1198 if (Var->getIsArg()) { |
| 1199 Lo->setIsArg(); | 1199 Lo->setIsArg(); |
| 1200 Hi->setIsArg(); | 1200 Hi->setIsArg(); |
| 1201 } | 1201 } |
| 1202 } | 1202 } |
| (...skipping 1515 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2718 (void)DestType; | 2718 (void)DestType; |
| 2719 assert((DestType == IceType_i32 && SrcType == IceType_f32) || | 2719 assert((DestType == IceType_i32 && SrcType == IceType_f32) || |
| 2720 (DestType == IceType_f32 && SrcType == IceType_i32)); | 2720 (DestType == IceType_f32 && SrcType == IceType_i32)); |
| 2721 // a.i32 = bitcast b.f32 ==> | 2721 // a.i32 = bitcast b.f32 ==> |
| 2722 // t.f32 = b.f32 | 2722 // t.f32 = b.f32 |
| 2723 // s.f32 = spill t.f32 | 2723 // s.f32 = spill t.f32 |
| 2724 // a.i32 = s.f32 | 2724 // a.i32 = s.f32 |
| 2725 Variable *T = nullptr; | 2725 Variable *T = nullptr; |
| 2726 // TODO: Should be able to force a spill setup by calling legalize() with | 2726 // TODO: Should be able to force a spill setup by calling legalize() with |
| 2727 // Legal_Mem and not Legal_Reg or Legal_Imm. | 2727 // Legal_Mem and not Legal_Reg or Legal_Imm. |
| 2728 SpillVariable *SpillVar = Func->makeVariable<SpillVariable>(SrcType); | 2728 SpillVariable *SpillVar = |
| 2729 Func->template makeVariable<SpillVariable>(SrcType); |
| 2729 SpillVar->setLinkedTo(Dest); | 2730 SpillVar->setLinkedTo(Dest); |
| 2730 Variable *Spill = SpillVar; | 2731 Variable *Spill = SpillVar; |
| 2731 Spill->setWeight(RegWeight::Zero); | 2732 Spill->setWeight(RegWeight::Zero); |
| 2732 _mov(T, Src0RM); | 2733 _mov(T, Src0RM); |
| 2733 _mov(Spill, T); | 2734 _mov(Spill, T); |
| 2734 _mov(Dest, Spill); | 2735 _mov(Dest, Spill); |
| 2735 } break; | 2736 } break; |
| 2736 case IceType_i64: { | 2737 case IceType_i64: { |
| 2737 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); | 2738 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); |
| 2738 assert(Src0RM->getType() == IceType_f64); | 2739 assert(Src0RM->getType() == IceType_f64); |
| 2739 // a.i64 = bitcast b.f64 ==> | 2740 // a.i64 = bitcast b.f64 ==> |
| 2740 // s.f64 = spill b.f64 | 2741 // s.f64 = spill b.f64 |
| 2741 // t_lo.i32 = lo(s.f64) | 2742 // t_lo.i32 = lo(s.f64) |
| 2742 // a_lo.i32 = t_lo.i32 | 2743 // a_lo.i32 = t_lo.i32 |
| 2743 // t_hi.i32 = hi(s.f64) | 2744 // t_hi.i32 = hi(s.f64) |
| 2744 // a_hi.i32 = t_hi.i32 | 2745 // a_hi.i32 = t_hi.i32 |
| 2745 Operand *SpillLo, *SpillHi; | 2746 Operand *SpillLo, *SpillHi; |
| 2746 if (auto *Src0Var = llvm::dyn_cast<Variable>(Src0RM)) { | 2747 if (auto *Src0Var = llvm::dyn_cast<Variable>(Src0RM)) { |
| 2747 SpillVariable *SpillVar = | 2748 SpillVariable *SpillVar = |
| 2748 Func->makeVariable<SpillVariable>(IceType_f64); | 2749 Func->template makeVariable<SpillVariable>(IceType_f64); |
| 2749 SpillVar->setLinkedTo(Src0Var); | 2750 SpillVar->setLinkedTo(Src0Var); |
| 2750 Variable *Spill = SpillVar; | 2751 Variable *Spill = SpillVar; |
| 2751 Spill->setWeight(RegWeight::Zero); | 2752 Spill->setWeight(RegWeight::Zero); |
| 2752 _movq(Spill, Src0RM); | 2753 _movq(Spill, Src0RM); |
| 2753 SpillLo = VariableSplit::create(Func, Spill, VariableSplit::Low); | 2754 SpillLo = VariableSplit::create(Func, Spill, VariableSplit::Low); |
| 2754 SpillHi = VariableSplit::create(Func, Spill, VariableSplit::High); | 2755 SpillHi = VariableSplit::create(Func, Spill, VariableSplit::High); |
| 2755 } else { | 2756 } else { |
| 2756 SpillLo = loOperand(Src0RM); | 2757 SpillLo = loOperand(Src0RM); |
| 2757 SpillHi = hiOperand(Src0RM); | 2758 SpillHi = hiOperand(Src0RM); |
| 2758 } | 2759 } |
| 2759 | 2760 |
| 2760 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 2761 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
| 2761 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 2762 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
| 2762 Variable *T_Lo = makeReg(IceType_i32); | 2763 Variable *T_Lo = makeReg(IceType_i32); |
| 2763 Variable *T_Hi = makeReg(IceType_i32); | 2764 Variable *T_Hi = makeReg(IceType_i32); |
| 2764 | 2765 |
| 2765 _mov(T_Lo, SpillLo); | 2766 _mov(T_Lo, SpillLo); |
| 2766 _mov(DestLo, T_Lo); | 2767 _mov(DestLo, T_Lo); |
| 2767 _mov(T_Hi, SpillHi); | 2768 _mov(T_Hi, SpillHi); |
| 2768 _mov(DestHi, T_Hi); | 2769 _mov(DestHi, T_Hi); |
| 2769 } break; | 2770 } break; |
| 2770 case IceType_f64: { | 2771 case IceType_f64: { |
| 2771 Src0 = legalize(Src0); | 2772 Src0 = legalize(Src0); |
| 2772 assert(Src0->getType() == IceType_i64); | 2773 assert(Src0->getType() == IceType_i64); |
| 2773 if (llvm::isa<OperandX8632Mem>(Src0)) { | 2774 if (llvm::isa<OperandX8632Mem>(Src0)) { |
| 2774 Variable *T = Func->makeVariable(Dest->getType()); | 2775 Variable *T = Func->template makeVariable(Dest->getType()); |
| 2775 _movq(T, Src0); | 2776 _movq(T, Src0); |
| 2776 _movq(Dest, T); | 2777 _movq(Dest, T); |
| 2777 break; | 2778 break; |
| 2778 } | 2779 } |
| 2779 // a.f64 = bitcast b.i64 ==> | 2780 // a.f64 = bitcast b.i64 ==> |
| 2780 // t_lo.i32 = b_lo.i32 | 2781 // t_lo.i32 = b_lo.i32 |
| 2781 // FakeDef(s.f64) | 2782 // FakeDef(s.f64) |
| 2782 // lo(s.f64) = t_lo.i32 | 2783 // lo(s.f64) = t_lo.i32 |
| 2783 // t_hi.i32 = b_hi.i32 | 2784 // t_hi.i32 = b_hi.i32 |
| 2784 // hi(s.f64) = t_hi.i32 | 2785 // hi(s.f64) = t_hi.i32 |
| 2785 // a.f64 = s.f64 | 2786 // a.f64 = s.f64 |
| 2786 SpillVariable *SpillVar = Func->makeVariable<SpillVariable>(IceType_f64); | 2787 SpillVariable *SpillVar = |
| 2788 Func->template makeVariable<SpillVariable>(IceType_f64); |
| 2787 SpillVar->setLinkedTo(Dest); | 2789 SpillVar->setLinkedTo(Dest); |
| 2788 Variable *Spill = SpillVar; | 2790 Variable *Spill = SpillVar; |
| 2789 Spill->setWeight(RegWeight::Zero); | 2791 Spill->setWeight(RegWeight::Zero); |
| 2790 | 2792 |
| 2791 Variable *T_Lo = nullptr, *T_Hi = nullptr; | 2793 Variable *T_Lo = nullptr, *T_Hi = nullptr; |
| 2792 VariableSplit *SpillLo = | 2794 VariableSplit *SpillLo = |
| 2793 VariableSplit::create(Func, Spill, VariableSplit::Low); | 2795 VariableSplit::create(Func, Spill, VariableSplit::Low); |
| 2794 VariableSplit *SpillHi = | 2796 VariableSplit *SpillHi = |
| 2795 VariableSplit::create(Func, Spill, VariableSplit::High); | 2797 VariableSplit::create(Func, Spill, VariableSplit::High); |
| 2796 _mov(T_Lo, loOperand(Src0)); | 2798 _mov(T_Lo, loOperand(Src0)); |
| 2797 // Technically, the Spill is defined after the _store happens, but | 2799 // Technically, the Spill is defined after the _store happens, but |
| 2798 // SpillLo is considered a "use" of Spill so define Spill before it | 2800 // SpillLo is considered a "use" of Spill so define Spill before it |
| 2799 // is used. | 2801 // is used. |
| 2800 Context.insert(InstFakeDef::create(Func, Spill)); | 2802 Context.insert(InstFakeDef::create(Func, Spill)); |
| 2801 _store(T_Lo, SpillLo); | 2803 _store(T_Lo, SpillLo); |
| 2802 _mov(T_Hi, hiOperand(Src0)); | 2804 _mov(T_Hi, hiOperand(Src0)); |
| 2803 _store(T_Hi, SpillHi); | 2805 _store(T_Hi, SpillHi); |
| 2804 _movq(Dest, Spill); | 2806 _movq(Dest, Spill); |
| 2805 } break; | 2807 } break; |
| 2806 case IceType_v8i1: { | 2808 case IceType_v8i1: { |
| 2807 assert(Src0->getType() == IceType_i8); | 2809 assert(Src0->getType() == IceType_i8); |
| 2808 InstCall *Call = makeHelperCall(H_bitcast_i8_8xi1, Dest, 1); | 2810 InstCall *Call = makeHelperCall(H_bitcast_i8_8xi1, Dest, 1); |
| 2809 Variable *Src0AsI32 = Func->makeVariable(stackSlotType()); | 2811 Variable *Src0AsI32 = Func->template makeVariable(stackSlotType()); |
| 2810 // Arguments to functions are required to be at least 32 bits wide. | 2812 // Arguments to functions are required to be at least 32 bits wide. |
| 2811 lowerCast(InstCast::create(Func, InstCast::Zext, Src0AsI32, Src0)); | 2813 lowerCast(InstCast::create(Func, InstCast::Zext, Src0AsI32, Src0)); |
| 2812 Call->addArg(Src0AsI32); | 2814 Call->addArg(Src0AsI32); |
| 2813 lowerCall(Call); | 2815 lowerCall(Call); |
| 2814 } break; | 2816 } break; |
| 2815 case IceType_v16i1: { | 2817 case IceType_v16i1: { |
| 2816 assert(Src0->getType() == IceType_i16); | 2818 assert(Src0->getType() == IceType_i16); |
| 2817 InstCall *Call = makeHelperCall(H_bitcast_i16_16xi1, Dest, 1); | 2819 InstCall *Call = makeHelperCall(H_bitcast_i16_16xi1, Dest, 1); |
| 2818 Variable *Src0AsI32 = Func->makeVariable(stackSlotType()); | 2820 Variable *Src0AsI32 = Func->template makeVariable(stackSlotType()); |
| 2819 // Arguments to functions are required to be at least 32 bits wide. | 2821 // Arguments to functions are required to be at least 32 bits wide. |
| 2820 lowerCast(InstCast::create(Func, InstCast::Zext, Src0AsI32, Src0)); | 2822 lowerCast(InstCast::create(Func, InstCast::Zext, Src0AsI32, Src0)); |
| 2821 Call->addArg(Src0AsI32); | 2823 Call->addArg(Src0AsI32); |
| 2822 lowerCall(Call); | 2824 lowerCall(Call); |
| 2823 } break; | 2825 } break; |
| 2824 case IceType_v8i16: | 2826 case IceType_v8i16: |
| 2825 case IceType_v16i8: | 2827 case IceType_v16i8: |
| 2826 case IceType_v4i32: | 2828 case IceType_v4i32: |
| 2827 case IceType_v4f32: { | 2829 case IceType_v4f32: { |
| 2828 _movp(Dest, legalizeToVar(Src0)); | 2830 _movp(Dest, legalizeToVar(Src0)); |
| (...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2879 // keep the live range analysis consistent. | 2881 // keep the live range analysis consistent. |
| 2880 Context.insert(InstFakeDef::create(Func, ExtractedElementR)); | 2882 Context.insert(InstFakeDef::create(Func, ExtractedElementR)); |
| 2881 _movss(ExtractedElementR, T); | 2883 _movss(ExtractedElementR, T); |
| 2882 } | 2884 } |
| 2883 } else { | 2885 } else { |
| 2884 assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); | 2886 assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); |
| 2885 // Spill the value to a stack slot and do the extraction in memory. | 2887 // Spill the value to a stack slot and do the extraction in memory. |
| 2886 // | 2888 // |
| 2887 // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when | 2889 // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when |
| 2888 // support for legalizing to mem is implemented. | 2890 // support for legalizing to mem is implemented. |
| 2889 Variable *Slot = Func->makeVariable(Ty); | 2891 Variable *Slot = Func->template makeVariable(Ty); |
| 2890 Slot->setWeight(RegWeight::Zero); | 2892 Slot->setWeight(RegWeight::Zero); |
| 2891 _movp(Slot, legalizeToVar(SourceVectNotLegalized)); | 2893 _movp(Slot, legalizeToVar(SourceVectNotLegalized)); |
| 2892 | 2894 |
| 2893 // Compute the location of the element in memory. | 2895 // Compute the location of the element in memory. |
| 2894 unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); | 2896 unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); |
| 2895 OperandX8632Mem *Loc = | 2897 OperandX8632Mem *Loc = |
| 2896 getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); | 2898 getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); |
| 2897 _mov(ExtractedElementR, Loc); | 2899 _mov(ExtractedElementR, Loc); |
| 2898 } | 2900 } |
| 2899 | 2901 |
| (...skipping 142 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3042 case IceType_v4i1: | 3044 case IceType_v4i1: |
| 3043 NewTy = IceType_v4i32; | 3045 NewTy = IceType_v4i32; |
| 3044 break; | 3046 break; |
| 3045 case IceType_v8i1: | 3047 case IceType_v8i1: |
| 3046 NewTy = IceType_v8i16; | 3048 NewTy = IceType_v8i16; |
| 3047 break; | 3049 break; |
| 3048 case IceType_v16i1: | 3050 case IceType_v16i1: |
| 3049 NewTy = IceType_v16i8; | 3051 NewTy = IceType_v16i8; |
| 3050 break; | 3052 break; |
| 3051 } | 3053 } |
| 3052 Variable *NewSrc0 = Func->makeVariable(NewTy); | 3054 Variable *NewSrc0 = Func->template makeVariable(NewTy); |
| 3053 Variable *NewSrc1 = Func->makeVariable(NewTy); | 3055 Variable *NewSrc1 = Func->template makeVariable(NewTy); |
| 3054 lowerCast(InstCast::create(Func, InstCast::Sext, NewSrc0, Src0)); | 3056 lowerCast(InstCast::create(Func, InstCast::Sext, NewSrc0, Src0)); |
| 3055 lowerCast(InstCast::create(Func, InstCast::Sext, NewSrc1, Src1)); | 3057 lowerCast(InstCast::create(Func, InstCast::Sext, NewSrc1, Src1)); |
| 3056 Src0 = NewSrc0; | 3058 Src0 = NewSrc0; |
| 3057 Src1 = NewSrc1; | 3059 Src1 = NewSrc1; |
| 3058 Ty = NewTy; | 3060 Ty = NewTy; |
| 3059 } | 3061 } |
| 3060 | 3062 |
| 3061 InstIcmp::ICond Condition = Inst->getCondition(); | 3063 InstIcmp::ICond Condition = Inst->getCondition(); |
| 3062 | 3064 |
| 3063 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); | 3065 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); |
| (...skipping 119 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3183 unsigned Index = ElementIndex->getValue(); | 3185 unsigned Index = ElementIndex->getValue(); |
| 3184 assert(Index < typeNumElements(SourceVectNotLegalized->getType())); | 3186 assert(Index < typeNumElements(SourceVectNotLegalized->getType())); |
| 3185 | 3187 |
| 3186 Type Ty = SourceVectNotLegalized->getType(); | 3188 Type Ty = SourceVectNotLegalized->getType(); |
| 3187 Type ElementTy = typeElementType(Ty); | 3189 Type ElementTy = typeElementType(Ty); |
| 3188 Type InVectorElementTy = Traits::getInVectorElementType(Ty); | 3190 Type InVectorElementTy = Traits::getInVectorElementType(Ty); |
| 3189 | 3191 |
| 3190 if (ElementTy == IceType_i1) { | 3192 if (ElementTy == IceType_i1) { |
| 3191 // Expand the element to the appropriate size for it to be inserted | 3193 // Expand the element to the appropriate size for it to be inserted |
| 3192 // in the vector. | 3194 // in the vector. |
| 3193 Variable *Expanded = Func->makeVariable(InVectorElementTy); | 3195 Variable *Expanded = Func->template makeVariable(InVectorElementTy); |
| 3194 InstCast *Cast = InstCast::create(Func, InstCast::Zext, Expanded, | 3196 InstCast *Cast = InstCast::create(Func, InstCast::Zext, Expanded, |
| 3195 ElementToInsertNotLegalized); | 3197 ElementToInsertNotLegalized); |
| 3196 lowerCast(Cast); | 3198 lowerCast(Cast); |
| 3197 ElementToInsertNotLegalized = Expanded; | 3199 ElementToInsertNotLegalized = Expanded; |
| 3198 } | 3200 } |
| 3199 | 3201 |
| 3200 if (Ty == IceType_v8i16 || Ty == IceType_v8i1 || | 3202 if (Ty == IceType_v8i16 || Ty == IceType_v8i1 || |
| 3201 InstructionSet >= Machine::SSE4_1) { | 3203 InstructionSet >= Machine::SSE4_1) { |
| 3202 // Use insertps, pinsrb, pinsrw, or pinsrd. | 3204 // Use insertps, pinsrb, pinsrw, or pinsrd. |
| 3203 Operand *ElementRM = | 3205 Operand *ElementRM = |
| (...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3274 _shufps(T, ElementR, Mask2Constant); | 3276 _shufps(T, ElementR, Mask2Constant); |
| 3275 _movp(Inst->getDest(), T); | 3277 _movp(Inst->getDest(), T); |
| 3276 } | 3278 } |
| 3277 } else { | 3279 } else { |
| 3278 assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); | 3280 assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); |
| 3279 // Spill the value to a stack slot and perform the insertion in | 3281 // Spill the value to a stack slot and perform the insertion in |
| 3280 // memory. | 3282 // memory. |
| 3281 // | 3283 // |
| 3282 // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when | 3284 // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when |
| 3283 // support for legalizing to mem is implemented. | 3285 // support for legalizing to mem is implemented. |
| 3284 Variable *Slot = Func->makeVariable(Ty); | 3286 Variable *Slot = Func->template makeVariable(Ty); |
| 3285 Slot->setWeight(RegWeight::Zero); | 3287 Slot->setWeight(RegWeight::Zero); |
| 3286 _movp(Slot, legalizeToVar(SourceVectNotLegalized)); | 3288 _movp(Slot, legalizeToVar(SourceVectNotLegalized)); |
| 3287 | 3289 |
| 3288 // Compute the location of the position to insert in memory. | 3290 // Compute the location of the position to insert in memory. |
| 3289 unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); | 3291 unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); |
| 3290 OperandX8632Mem *Loc = | 3292 OperandX8632Mem *Loc = |
| 3291 getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); | 3293 getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); |
| 3292 _store(legalizeToVar(ElementToInsertNotLegalized), Loc); | 3294 _store(legalizeToVar(ElementToInsertNotLegalized), Loc); |
| 3293 | 3295 |
| 3294 Variable *T = makeReg(Ty); | 3296 Variable *T = makeReg(Ty); |
| (...skipping 269 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3564 Call->addArg(Instr->getArg(2)); | 3566 Call->addArg(Instr->getArg(2)); |
| 3565 lowerCall(Call); | 3567 lowerCall(Call); |
| 3566 return; | 3568 return; |
| 3567 } | 3569 } |
| 3568 case Intrinsics::Memset: { | 3570 case Intrinsics::Memset: { |
| 3569 // The value operand needs to be extended to a stack slot size | 3571 // The value operand needs to be extended to a stack slot size |
| 3570 // because the PNaCl ABI requires arguments to be at least 32 bits | 3572 // because the PNaCl ABI requires arguments to be at least 32 bits |
| 3571 // wide. | 3573 // wide. |
| 3572 Operand *ValOp = Instr->getArg(1); | 3574 Operand *ValOp = Instr->getArg(1); |
| 3573 assert(ValOp->getType() == IceType_i8); | 3575 assert(ValOp->getType() == IceType_i8); |
| 3574 Variable *ValExt = Func->makeVariable(stackSlotType()); | 3576 Variable *ValExt = Func->template makeVariable(stackSlotType()); |
| 3575 lowerCast(InstCast::create(Func, InstCast::Zext, ValExt, ValOp)); | 3577 lowerCast(InstCast::create(Func, InstCast::Zext, ValExt, ValOp)); |
| 3576 InstCall *Call = makeHelperCall(H_call_memset, nullptr, 3); | 3578 InstCall *Call = makeHelperCall(H_call_memset, nullptr, 3); |
| 3577 Call->addArg(Instr->getArg(0)); | 3579 Call->addArg(Instr->getArg(0)); |
| 3578 Call->addArg(ValExt); | 3580 Call->addArg(ValExt); |
| 3579 Call->addArg(Instr->getArg(2)); | 3581 Call->addArg(Instr->getArg(2)); |
| 3580 lowerCall(Call); | 3582 lowerCall(Call); |
| 3581 return; | 3583 return; |
| 3582 } | 3584 } |
| 3583 case Intrinsics::NaClReadTP: { | 3585 case Intrinsics::NaClReadTP: { |
| 3584 if (Ctx->getFlags().getUseSandboxing()) { | 3586 if (Ctx->getFlags().getUseSandboxing()) { |
| (...skipping 786 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4371 } | 4373 } |
| 4372 // Lower select without Machine::SSE4.1: | 4374 // Lower select without Machine::SSE4.1: |
| 4373 // a=d?b:c ==> | 4375 // a=d?b:c ==> |
| 4374 // if elementtype(d) != i1: | 4376 // if elementtype(d) != i1: |
| 4375 // d=sext(d); | 4377 // d=sext(d); |
| 4376 // a=(b&d)|(c&~d); | 4378 // a=(b&d)|(c&~d); |
| 4377 Variable *T2 = makeReg(SrcTy); | 4379 Variable *T2 = makeReg(SrcTy); |
| 4378 // Sign extend the condition operand if applicable. | 4380 // Sign extend the condition operand if applicable. |
| 4379 if (SrcTy == IceType_v4f32) { | 4381 if (SrcTy == IceType_v4f32) { |
| 4380 // The sext operation takes only integer arguments. | 4382 // The sext operation takes only integer arguments. |
| 4381 Variable *T3 = Func->makeVariable(IceType_v4i32); | 4383 Variable *T3 = Func->template makeVariable(IceType_v4i32); |
| 4382 lowerCast(InstCast::create(Func, InstCast::Sext, T3, Condition)); | 4384 lowerCast(InstCast::create(Func, InstCast::Sext, T3, Condition)); |
| 4383 _movp(T, T3); | 4385 _movp(T, T3); |
| 4384 } else if (typeElementType(SrcTy) != IceType_i1) { | 4386 } else if (typeElementType(SrcTy) != IceType_i1) { |
| 4385 lowerCast(InstCast::create(Func, InstCast::Sext, T, Condition)); | 4387 lowerCast(InstCast::create(Func, InstCast::Sext, T, Condition)); |
| 4386 } else { | 4388 } else { |
| 4387 Operand *ConditionRM = legalize(Condition, Legal_Reg | Legal_Mem); | 4389 Operand *ConditionRM = legalize(Condition, Legal_Reg | Legal_Mem); |
| 4388 _movp(T, ConditionRM); | 4390 _movp(T, ConditionRM); |
| 4389 } | 4391 } |
| 4390 _movp(T2, T); | 4392 _movp(T2, T); |
| 4391 _pand(T, SrcTRM); | 4393 _pand(T, SrcTRM); |
| (...skipping 179 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4571 assert(isVectorType(Dest->getType())); | 4573 assert(isVectorType(Dest->getType())); |
| 4572 Type Ty = Dest->getType(); | 4574 Type Ty = Dest->getType(); |
| 4573 Type ElementTy = typeElementType(Ty); | 4575 Type ElementTy = typeElementType(Ty); |
| 4574 SizeT NumElements = typeNumElements(Ty); | 4576 SizeT NumElements = typeNumElements(Ty); |
| 4575 | 4577 |
| 4576 Operand *T = Ctx->getConstantUndef(Ty); | 4578 Operand *T = Ctx->getConstantUndef(Ty); |
| 4577 for (SizeT I = 0; I < NumElements; ++I) { | 4579 for (SizeT I = 0; I < NumElements; ++I) { |
| 4578 Constant *Index = Ctx->getConstantInt32(I); | 4580 Constant *Index = Ctx->getConstantInt32(I); |
| 4579 | 4581 |
| 4580 // Extract the next two inputs. | 4582 // Extract the next two inputs. |
| 4581 Variable *Op0 = Func->makeVariable(ElementTy); | 4583 Variable *Op0 = Func->template makeVariable(ElementTy); |
| 4582 lowerExtractElement(InstExtractElement::create(Func, Op0, Src0, Index)); | 4584 lowerExtractElement(InstExtractElement::create(Func, Op0, Src0, Index)); |
| 4583 Variable *Op1 = Func->makeVariable(ElementTy); | 4585 Variable *Op1 = Func->template makeVariable(ElementTy); |
| 4584 lowerExtractElement(InstExtractElement::create(Func, Op1, Src1, Index)); | 4586 lowerExtractElement(InstExtractElement::create(Func, Op1, Src1, Index)); |
| 4585 | 4587 |
| 4586 // Perform the arithmetic as a scalar operation. | 4588 // Perform the arithmetic as a scalar operation. |
| 4587 Variable *Res = Func->makeVariable(ElementTy); | 4589 Variable *Res = Func->template makeVariable(ElementTy); |
| 4588 lowerArithmetic(InstArithmetic::create(Func, Kind, Res, Op0, Op1)); | 4590 lowerArithmetic(InstArithmetic::create(Func, Kind, Res, Op0, Op1)); |
| 4589 | 4591 |
| 4590 // Insert the result into position. | 4592 // Insert the result into position. |
| 4591 Variable *DestT = Func->makeVariable(Ty); | 4593 Variable *DestT = Func->template makeVariable(Ty); |
| 4592 lowerInsertElement(InstInsertElement::create(Func, DestT, T, Res, Index)); | 4594 lowerInsertElement(InstInsertElement::create(Func, DestT, T, Res, Index)); |
| 4593 T = DestT; | 4595 T = DestT; |
| 4594 } | 4596 } |
| 4595 | 4597 |
| 4596 lowerAssign(InstAssign::create(Func, Dest, T)); | 4598 lowerAssign(InstAssign::create(Func, Dest, T)); |
| 4597 } | 4599 } |
| 4598 | 4600 |
| 4599 // The following pattern occurs often in lowered C and C++ code: | 4601 // The following pattern occurs often in lowered C and C++ code: |
| 4600 // | 4602 // |
| 4601 // %cmp = fcmp/icmp pred <n x ty> %src0, %src1 | 4603 // %cmp = fcmp/icmp pred <n x ty> %src0, %src1 |
| (...skipping 248 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4850 Variable *Preg = nullptr; | 4852 Variable *Preg = nullptr; |
| 4851 // TODO(stichnot): Opportunity for register randomization. | 4853 // TODO(stichnot): Opportunity for register randomization. |
| 4852 int32_t RegNum = AvailRegsForType.find_first(); | 4854 int32_t RegNum = AvailRegsForType.find_first(); |
| 4853 bool IsVector = isVectorType(Dest->getType()); | 4855 bool IsVector = isVectorType(Dest->getType()); |
| 4854 bool NeedSpill = (RegNum == -1); | 4856 bool NeedSpill = (RegNum == -1); |
| 4855 if (NeedSpill) { | 4857 if (NeedSpill) { |
| 4856 // Pick some register to spill and update RegNum. | 4858 // Pick some register to spill and update RegNum. |
| 4857 // TODO(stichnot): Opportunity for register randomization. | 4859 // TODO(stichnot): Opportunity for register randomization. |
| 4858 RegNum = RegsForType.find_first(); | 4860 RegNum = RegsForType.find_first(); |
| 4859 Preg = getPhysicalRegister(RegNum, Dest->getType()); | 4861 Preg = getPhysicalRegister(RegNum, Dest->getType()); |
| 4860 SpillLoc = Func->makeVariable(Dest->getType()); | 4862 SpillLoc = Func->template makeVariable(Dest->getType()); |
| 4861 // Create a fake def of the physical register to avoid | 4863 // Create a fake def of the physical register to avoid |
| 4862 // liveness inconsistency problems during late-stage liveness | 4864 // liveness inconsistency problems during late-stage liveness |
| 4863 // analysis (e.g. asm-verbose mode). | 4865 // analysis (e.g. asm-verbose mode). |
| 4864 Context.insert(InstFakeDef::create(Func, Preg)); | 4866 Context.insert(InstFakeDef::create(Func, Preg)); |
| 4865 if (IsVector) | 4867 if (IsVector) |
| 4866 _movp(SpillLoc, Preg); | 4868 _movp(SpillLoc, Preg); |
| 4867 else | 4869 else |
| 4868 _mov(SpillLoc, Preg); | 4870 _mov(SpillLoc, Preg); |
| 4869 } | 4871 } |
| 4870 assert(RegNum >= 0); | 4872 assert(RegNum >= 0); |
| (...skipping 309 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 5180 // Do legalization, which contains randomization/pooling | 5182 // Do legalization, which contains randomization/pooling |
| 5181 // or do randomization/pooling. | 5183 // or do randomization/pooling. |
| 5182 return llvm::cast<OperandX8632Mem>( | 5184 return llvm::cast<OperandX8632Mem>( |
| 5183 DoLegalize ? legalize(Mem) : randomizeOrPoolImmediate(Mem)); | 5185 DoLegalize ? legalize(Mem) : randomizeOrPoolImmediate(Mem)); |
| 5184 } | 5186 } |
| 5185 | 5187 |
| 5186 template <class Machine> | 5188 template <class Machine> |
| 5187 Variable *TargetX86Base<Machine>::makeReg(Type Type, int32_t RegNum) { | 5189 Variable *TargetX86Base<Machine>::makeReg(Type Type, int32_t RegNum) { |
| 5188 // There aren't any 64-bit integer registers for x86-32. | 5190 // There aren't any 64-bit integer registers for x86-32. |
| 5189 assert(Type != IceType_i64); | 5191 assert(Type != IceType_i64); |
| 5190 Variable *Reg = Func->makeVariable(Type); | 5192 Variable *Reg = Func->template makeVariable(Type); |
| 5191 if (RegNum == Variable::NoRegister) | 5193 if (RegNum == Variable::NoRegister) |
| 5192 Reg->setWeightInfinite(); | 5194 Reg->setWeightInfinite(); |
| 5193 else | 5195 else |
| 5194 Reg->setRegNum(RegNum); | 5196 Reg->setRegNum(RegNum); |
| 5195 return Reg; | 5197 return Reg; |
| 5196 } | 5198 } |
| 5197 | 5199 |
| 5198 template <class Machine> void TargetX86Base<Machine>::postLower() { | 5200 template <class Machine> void TargetX86Base<Machine>::postLower() { |
| 5199 if (Ctx->getFlags().getOptLevel() == Opt_m1) | 5201 if (Ctx->getFlags().getOptLevel() == Opt_m1) |
| 5200 return; | 5202 return; |
| (...skipping 293 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 5494 } | 5496 } |
| 5495 // the offset is not eligible for blinding or pooling, return the original | 5497 // the offset is not eligible for blinding or pooling, return the original |
| 5496 // mem operand | 5498 // mem operand |
| 5497 return MemOperand; | 5499 return MemOperand; |
| 5498 } | 5500 } |
| 5499 | 5501 |
| 5500 } // end of namespace X86Internal | 5502 } // end of namespace X86Internal |
| 5501 } // end of namespace Ice | 5503 } // end of namespace Ice |
| 5502 | 5504 |
| 5503 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H | 5505 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H |
| OLD | NEW |