| OLD | NEW |
| 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// | 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// |
| 2 // | 2 // |
| 3 // The Subzero Code Generator | 3 // The Subzero Code Generator |
| 4 // | 4 // |
| 5 // This file is distributed under the University of Illinois Open Source | 5 // This file is distributed under the University of Illinois Open Source |
| 6 // License. See LICENSE.TXT for details. | 6 // License. See LICENSE.TXT for details. |
| 7 // | 7 // |
| 8 //===----------------------------------------------------------------------===// | 8 //===----------------------------------------------------------------------===// |
| 9 /// | 9 /// |
| 10 /// \file | 10 /// \file |
| (...skipping 1632 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1643 llvm_unreachable("Call-helper-involved instruction for i64 type \ | 1643 llvm_unreachable("Call-helper-involved instruction for i64 type \ |
| 1644 should have already been handled before"); | 1644 should have already been handled before"); |
| 1645 break; | 1645 break; |
| 1646 } | 1646 } |
| 1647 return; | 1647 return; |
| 1648 } | 1648 } |
| 1649 if (isVectorType(Dest->getType())) { | 1649 if (isVectorType(Dest->getType())) { |
| 1650 // TODO: Trap on integer divide and integer modulo by zero. | 1650 // TODO: Trap on integer divide and integer modulo by zero. |
| 1651 // See: https://code.google.com/p/nativeclient/issues/detail?id=3899 | 1651 // See: https://code.google.com/p/nativeclient/issues/detail?id=3899 |
| 1652 if (llvm::isa<typename Traits::X86OperandMem>(Src1)) | 1652 if (llvm::isa<typename Traits::X86OperandMem>(Src1)) |
| 1653 Src1 = legalizeToVar(Src1); | 1653 Src1 = legalizeToReg(Src1); |
| 1654 switch (Inst->getOp()) { | 1654 switch (Inst->getOp()) { |
| 1655 case InstArithmetic::_num: | 1655 case InstArithmetic::_num: |
| 1656 llvm_unreachable("Unknown arithmetic operator"); | 1656 llvm_unreachable("Unknown arithmetic operator"); |
| 1657 break; | 1657 break; |
| 1658 case InstArithmetic::Add: { | 1658 case InstArithmetic::Add: { |
| 1659 Variable *T = makeReg(Dest->getType()); | 1659 Variable *T = makeReg(Dest->getType()); |
| 1660 _movp(T, Src0); | 1660 _movp(T, Src0); |
| 1661 _padd(T, Src1); | 1661 _padd(T, Src1); |
| 1662 _movp(Dest, T); | 1662 _movp(Dest, T); |
| 1663 } break; | 1663 } break; |
| (...skipping 156 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1820 Src1 = legalize(Src1, Legal_Reg | Legal_Mem); | 1820 Src1 = legalize(Src1, Legal_Reg | Legal_Mem); |
| 1821 } else { | 1821 } else { |
| 1822 _mov(T, Src0); | 1822 _mov(T, Src0); |
| 1823 } | 1823 } |
| 1824 _imul(T, Src1); | 1824 _imul(T, Src1); |
| 1825 _mov(Dest, T); | 1825 _mov(Dest, T); |
| 1826 break; | 1826 break; |
| 1827 case InstArithmetic::Shl: | 1827 case InstArithmetic::Shl: |
| 1828 _mov(T, Src0); | 1828 _mov(T, Src0); |
| 1829 if (!llvm::isa<Constant>(Src1)) | 1829 if (!llvm::isa<Constant>(Src1)) |
| 1830 Src1 = legalizeToVar(Src1, Traits::RegisterSet::Reg_ecx); | 1830 Src1 = legalizeToReg(Src1, Traits::RegisterSet::Reg_ecx); |
| 1831 _shl(T, Src1); | 1831 _shl(T, Src1); |
| 1832 _mov(Dest, T); | 1832 _mov(Dest, T); |
| 1833 break; | 1833 break; |
| 1834 case InstArithmetic::Lshr: | 1834 case InstArithmetic::Lshr: |
| 1835 _mov(T, Src0); | 1835 _mov(T, Src0); |
| 1836 if (!llvm::isa<Constant>(Src1)) | 1836 if (!llvm::isa<Constant>(Src1)) |
| 1837 Src1 = legalizeToVar(Src1, Traits::RegisterSet::Reg_ecx); | 1837 Src1 = legalizeToReg(Src1, Traits::RegisterSet::Reg_ecx); |
| 1838 _shr(T, Src1); | 1838 _shr(T, Src1); |
| 1839 _mov(Dest, T); | 1839 _mov(Dest, T); |
| 1840 break; | 1840 break; |
| 1841 case InstArithmetic::Ashr: | 1841 case InstArithmetic::Ashr: |
| 1842 _mov(T, Src0); | 1842 _mov(T, Src0); |
| 1843 if (!llvm::isa<Constant>(Src1)) | 1843 if (!llvm::isa<Constant>(Src1)) |
| 1844 Src1 = legalizeToVar(Src1, Traits::RegisterSet::Reg_ecx); | 1844 Src1 = legalizeToReg(Src1, Traits::RegisterSet::Reg_ecx); |
| 1845 _sar(T, Src1); | 1845 _sar(T, Src1); |
| 1846 _mov(Dest, T); | 1846 _mov(Dest, T); |
| 1847 break; | 1847 break; |
| 1848 case InstArithmetic::Udiv: | 1848 case InstArithmetic::Udiv: |
| 1849 // div and idiv are the few arithmetic operators that do not allow | 1849 // div and idiv are the few arithmetic operators that do not allow |
| 1850 // immediates as the operand. | 1850 // immediates as the operand. |
| 1851 Src1 = legalize(Src1, Legal_Reg | Legal_Mem); | 1851 Src1 = legalize(Src1, Legal_Reg | Legal_Mem); |
| 1852 if (isByteSizedArithType(Dest->getType())) { | 1852 if (isByteSizedArithType(Dest->getType())) { |
| 1853 Variable *T_ah = nullptr; | 1853 Variable *T_ah = nullptr; |
| 1854 Constant *Zero = Ctx->getConstantZero(IceType_i8); | 1854 Constant *Zero = Ctx->getConstantZero(IceType_i8); |
| (...skipping 324 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2179 // registers. | 2179 // registers. |
| 2180 // TODO: Investigate the impact of lowering arguments passed in | 2180 // TODO: Investigate the impact of lowering arguments passed in |
| 2181 // registers after lowering stack arguments as opposed to the other | 2181 // registers after lowering stack arguments as opposed to the other |
| 2182 // way around. Lowering register arguments after stack arguments may | 2182 // way around. Lowering register arguments after stack arguments may |
| 2183 // reduce register pressure. On the other hand, lowering register | 2183 // reduce register pressure. On the other hand, lowering register |
| 2184 // arguments first (before stack arguments) may result in more compact | 2184 // arguments first (before stack arguments) may result in more compact |
| 2185 // code, as the memory operand displacements may end up being smaller | 2185 // code, as the memory operand displacements may end up being smaller |
| 2186 // before any stack adjustment is done. | 2186 // before any stack adjustment is done. |
| 2187 for (SizeT i = 0, NumXmmArgs = XmmArgs.size(); i < NumXmmArgs; ++i) { | 2187 for (SizeT i = 0, NumXmmArgs = XmmArgs.size(); i < NumXmmArgs; ++i) { |
| 2188 Variable *Reg = | 2188 Variable *Reg = |
| 2189 legalizeToVar(XmmArgs[i], Traits::RegisterSet::Reg_xmm0 + i); | 2189 legalizeToReg(XmmArgs[i], Traits::RegisterSet::Reg_xmm0 + i); |
| 2190 // Generate a FakeUse of register arguments so that they do not get | 2190 // Generate a FakeUse of register arguments so that they do not get |
| 2191 // dead code eliminated as a result of the FakeKill of scratch | 2191 // dead code eliminated as a result of the FakeKill of scratch |
| 2192 // registers after the call. | 2192 // registers after the call. |
| 2193 Context.insert(InstFakeUse::create(Func, Reg)); | 2193 Context.insert(InstFakeUse::create(Func, Reg)); |
| 2194 } | 2194 } |
| 2195 // Generate the call instruction. Assign its result to a temporary | 2195 // Generate the call instruction. Assign its result to a temporary |
| 2196 // with high register allocation weight. | 2196 // with high register allocation weight. |
| 2197 Variable *Dest = Instr->getDest(); | 2197 Variable *Dest = Instr->getDest(); |
| 2198 // ReturnReg doubles as ReturnRegLo as necessary. | 2198 // ReturnReg doubles as ReturnRegLo as necessary. |
| 2199 Variable *ReturnReg = nullptr; | 2199 Variable *ReturnReg = nullptr; |
| (...skipping 276 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2476 _cvt(T, Src0RM, Traits::Insts::Cvt::Float2float); | 2476 _cvt(T, Src0RM, Traits::Insts::Cvt::Float2float); |
| 2477 _mov(Dest, T); | 2477 _mov(Dest, T); |
| 2478 break; | 2478 break; |
| 2479 } | 2479 } |
| 2480 case InstCast::Fptosi: | 2480 case InstCast::Fptosi: |
| 2481 if (isVectorType(Dest->getType())) { | 2481 if (isVectorType(Dest->getType())) { |
| 2482 assert(Dest->getType() == IceType_v4i32 && | 2482 assert(Dest->getType() == IceType_v4i32 && |
| 2483 Inst->getSrc(0)->getType() == IceType_v4f32); | 2483 Inst->getSrc(0)->getType() == IceType_v4f32); |
| 2484 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); | 2484 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); |
| 2485 if (llvm::isa<typename Traits::X86OperandMem>(Src0RM)) | 2485 if (llvm::isa<typename Traits::X86OperandMem>(Src0RM)) |
| 2486 Src0RM = legalizeToVar(Src0RM); | 2486 Src0RM = legalizeToReg(Src0RM); |
| 2487 Variable *T = makeReg(Dest->getType()); | 2487 Variable *T = makeReg(Dest->getType()); |
| 2488 _cvt(T, Src0RM, Traits::Insts::Cvt::Tps2dq); | 2488 _cvt(T, Src0RM, Traits::Insts::Cvt::Tps2dq); |
| 2489 _movp(Dest, T); | 2489 _movp(Dest, T); |
| 2490 } else if (Dest->getType() == IceType_i64) { | 2490 } else if (Dest->getType() == IceType_i64) { |
| 2491 // Use a helper for converting floating-point values to 64-bit | 2491 // Use a helper for converting floating-point values to 64-bit |
| 2492 // integers. SSE2 appears to have no way to convert from xmm | 2492 // integers. SSE2 appears to have no way to convert from xmm |
| 2493 // registers to something like the edx:eax register pair, and | 2493 // registers to something like the edx:eax register pair, and |
| 2494 // gcc and clang both want to use x87 instructions complete with | 2494 // gcc and clang both want to use x87 instructions complete with |
| 2495 // temporary manipulation of the status word. This helper is | 2495 // temporary manipulation of the status word. This helper is |
| 2496 // not needed for x86-64. | 2496 // not needed for x86-64. |
| (...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2553 _and(T_2, Ctx->getConstantInt1(1)); | 2553 _and(T_2, Ctx->getConstantInt1(1)); |
| 2554 _mov(Dest, T_2); | 2554 _mov(Dest, T_2); |
| 2555 } | 2555 } |
| 2556 break; | 2556 break; |
| 2557 case InstCast::Sitofp: | 2557 case InstCast::Sitofp: |
| 2558 if (isVectorType(Dest->getType())) { | 2558 if (isVectorType(Dest->getType())) { |
| 2559 assert(Dest->getType() == IceType_v4f32 && | 2559 assert(Dest->getType() == IceType_v4f32 && |
| 2560 Inst->getSrc(0)->getType() == IceType_v4i32); | 2560 Inst->getSrc(0)->getType() == IceType_v4i32); |
| 2561 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); | 2561 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); |
| 2562 if (llvm::isa<typename Traits::X86OperandMem>(Src0RM)) | 2562 if (llvm::isa<typename Traits::X86OperandMem>(Src0RM)) |
| 2563 Src0RM = legalizeToVar(Src0RM); | 2563 Src0RM = legalizeToReg(Src0RM); |
| 2564 Variable *T = makeReg(Dest->getType()); | 2564 Variable *T = makeReg(Dest->getType()); |
| 2565 _cvt(T, Src0RM, Traits::Insts::Cvt::Dq2ps); | 2565 _cvt(T, Src0RM, Traits::Insts::Cvt::Dq2ps); |
| 2566 _movp(Dest, T); | 2566 _movp(Dest, T); |
| 2567 } else if (Inst->getSrc(0)->getType() == IceType_i64) { | 2567 } else if (Inst->getSrc(0)->getType() == IceType_i64) { |
| 2568 // Use a helper for x86-32. | 2568 // Use a helper for x86-32. |
| 2569 const SizeT MaxSrcs = 1; | 2569 const SizeT MaxSrcs = 1; |
| 2570 Type DestType = Dest->getType(); | 2570 Type DestType = Dest->getType(); |
| 2571 InstCall *Call = | 2571 InstCall *Call = |
| 2572 makeHelperCall(isFloat32Asserting32Or64(DestType) ? H_sitofp_i64_f32 | 2572 makeHelperCall(isFloat32Asserting32Or64(DestType) ? H_sitofp_i64_f32 |
| 2573 : H_sitofp_i64_f64, | 2573 : H_sitofp_i64_f64, |
| (...skipping 194 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2768 Variable *Src0AsI32 = Func->template makeVariable(stackSlotType()); | 2768 Variable *Src0AsI32 = Func->template makeVariable(stackSlotType()); |
| 2769 // Arguments to functions are required to be at least 32 bits wide. | 2769 // Arguments to functions are required to be at least 32 bits wide. |
| 2770 lowerCast(InstCast::create(Func, InstCast::Zext, Src0AsI32, Src0)); | 2770 lowerCast(InstCast::create(Func, InstCast::Zext, Src0AsI32, Src0)); |
| 2771 Call->addArg(Src0AsI32); | 2771 Call->addArg(Src0AsI32); |
| 2772 lowerCall(Call); | 2772 lowerCall(Call); |
| 2773 } break; | 2773 } break; |
| 2774 case IceType_v8i16: | 2774 case IceType_v8i16: |
| 2775 case IceType_v16i8: | 2775 case IceType_v16i8: |
| 2776 case IceType_v4i32: | 2776 case IceType_v4i32: |
| 2777 case IceType_v4f32: { | 2777 case IceType_v4f32: { |
| 2778 _movp(Dest, legalizeToVar(Src0)); | 2778 _movp(Dest, legalizeToReg(Src0)); |
| 2779 } break; | 2779 } break; |
| 2780 } | 2780 } |
| 2781 break; | 2781 break; |
| 2782 } | 2782 } |
| 2783 } | 2783 } |
| 2784 } | 2784 } |
| 2785 | 2785 |
| 2786 template <class Machine> | 2786 template <class Machine> |
| 2787 void TargetX86Base<Machine>::lowerExtractElement( | 2787 void TargetX86Base<Machine>::lowerExtractElement( |
| 2788 const InstExtractElement *Inst) { | 2788 const InstExtractElement *Inst) { |
| 2789 Operand *SourceVectNotLegalized = Inst->getSrc(0); | 2789 Operand *SourceVectNotLegalized = Inst->getSrc(0); |
| 2790 ConstantInteger32 *ElementIndex = | 2790 ConstantInteger32 *ElementIndex = |
| 2791 llvm::dyn_cast<ConstantInteger32>(Inst->getSrc(1)); | 2791 llvm::dyn_cast<ConstantInteger32>(Inst->getSrc(1)); |
| 2792 // Only constant indices are allowed in PNaCl IR. | 2792 // Only constant indices are allowed in PNaCl IR. |
| 2793 assert(ElementIndex); | 2793 assert(ElementIndex); |
| 2794 | 2794 |
| 2795 unsigned Index = ElementIndex->getValue(); | 2795 unsigned Index = ElementIndex->getValue(); |
| 2796 Type Ty = SourceVectNotLegalized->getType(); | 2796 Type Ty = SourceVectNotLegalized->getType(); |
| 2797 Type ElementTy = typeElementType(Ty); | 2797 Type ElementTy = typeElementType(Ty); |
| 2798 Type InVectorElementTy = Traits::getInVectorElementType(Ty); | 2798 Type InVectorElementTy = Traits::getInVectorElementType(Ty); |
| 2799 Variable *ExtractedElementR = makeReg(InVectorElementTy); | 2799 Variable *ExtractedElementR = makeReg(InVectorElementTy); |
| 2800 | 2800 |
| 2801 // TODO(wala): Determine the best lowering sequences for each type. | 2801 // TODO(wala): Determine the best lowering sequences for each type. |
| 2802 bool CanUsePextr = Ty == IceType_v8i16 || Ty == IceType_v8i1 || | 2802 bool CanUsePextr = Ty == IceType_v8i16 || Ty == IceType_v8i1 || |
| 2803 InstructionSet >= Traits::SSE4_1; | 2803 InstructionSet >= Traits::SSE4_1; |
| 2804 if (CanUsePextr && Ty != IceType_v4f32) { | 2804 if (CanUsePextr && Ty != IceType_v4f32) { |
| 2805 // Use pextrb, pextrw, or pextrd. | 2805 // Use pextrb, pextrw, or pextrd. |
| 2806 Constant *Mask = Ctx->getConstantInt32(Index); | 2806 Constant *Mask = Ctx->getConstantInt32(Index); |
| 2807 Variable *SourceVectR = legalizeToVar(SourceVectNotLegalized); | 2807 Variable *SourceVectR = legalizeToReg(SourceVectNotLegalized); |
| 2808 _pextr(ExtractedElementR, SourceVectR, Mask); | 2808 _pextr(ExtractedElementR, SourceVectR, Mask); |
| 2809 } else if (Ty == IceType_v4i32 || Ty == IceType_v4f32 || Ty == IceType_v4i1) { | 2809 } else if (Ty == IceType_v4i32 || Ty == IceType_v4f32 || Ty == IceType_v4i1) { |
| 2810 // Use pshufd and movd/movss. | 2810 // Use pshufd and movd/movss. |
| 2811 Variable *T = nullptr; | 2811 Variable *T = nullptr; |
| 2812 if (Index) { | 2812 if (Index) { |
| 2813 // The shuffle only needs to occur if the element to be extracted | 2813 // The shuffle only needs to occur if the element to be extracted |
| 2814 // is not at the lowest index. | 2814 // is not at the lowest index. |
| 2815 Constant *Mask = Ctx->getConstantInt32(Index); | 2815 Constant *Mask = Ctx->getConstantInt32(Index); |
| 2816 T = makeReg(Ty); | 2816 T = makeReg(Ty); |
| 2817 _pshufd(T, legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem), Mask); | 2817 _pshufd(T, legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem), Mask); |
| 2818 } else { | 2818 } else { |
| 2819 T = legalizeToVar(SourceVectNotLegalized); | 2819 T = legalizeToReg(SourceVectNotLegalized); |
| 2820 } | 2820 } |
| 2821 | 2821 |
| 2822 if (InVectorElementTy == IceType_i32) { | 2822 if (InVectorElementTy == IceType_i32) { |
| 2823 _movd(ExtractedElementR, T); | 2823 _movd(ExtractedElementR, T); |
| 2824 } else { // Ty == IceType_f32 | 2824 } else { // Ty == IceType_f32 |
| 2825 // TODO(wala): _movss is only used here because _mov does not | 2825 // TODO(wala): _movss is only used here because _mov does not |
| 2826 // allow a vector source and a scalar destination. _mov should be | 2826 // allow a vector source and a scalar destination. _mov should be |
| 2827 // able to be used here. | 2827 // able to be used here. |
| 2828 // _movss is a binary instruction, so the FakeDef is needed to | 2828 // _movss is a binary instruction, so the FakeDef is needed to |
| 2829 // keep the live range analysis consistent. | 2829 // keep the live range analysis consistent. |
| 2830 Context.insert(InstFakeDef::create(Func, ExtractedElementR)); | 2830 Context.insert(InstFakeDef::create(Func, ExtractedElementR)); |
| 2831 _movss(ExtractedElementR, T); | 2831 _movss(ExtractedElementR, T); |
| 2832 } | 2832 } |
| 2833 } else { | 2833 } else { |
| 2834 assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); | 2834 assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); |
| 2835 // Spill the value to a stack slot and do the extraction in memory. | 2835 // Spill the value to a stack slot and do the extraction in memory. |
| 2836 // | 2836 // |
| 2837 // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when | 2837 // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when |
| 2838 // support for legalizing to mem is implemented. | 2838 // support for legalizing to mem is implemented. |
| 2839 Variable *Slot = Func->template makeVariable(Ty); | 2839 Variable *Slot = Func->template makeVariable(Ty); |
| 2840 Slot->setWeight(RegWeight::Zero); | 2840 Slot->setWeight(RegWeight::Zero); |
| 2841 _movp(Slot, legalizeToVar(SourceVectNotLegalized)); | 2841 _movp(Slot, legalizeToReg(SourceVectNotLegalized)); |
| 2842 | 2842 |
| 2843 // Compute the location of the element in memory. | 2843 // Compute the location of the element in memory. |
| 2844 unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); | 2844 unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); |
| 2845 typename Traits::X86OperandMem *Loc = | 2845 typename Traits::X86OperandMem *Loc = |
| 2846 getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); | 2846 getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); |
| 2847 _mov(ExtractedElementR, Loc); | 2847 _mov(ExtractedElementR, Loc); |
| 2848 } | 2848 } |
| 2849 | 2849 |
| 2850 if (ElementTy == IceType_i1) { | 2850 if (ElementTy == IceType_i1) { |
| 2851 // Truncate extracted integers to i1s if necessary. | 2851 // Truncate extracted integers to i1s if necessary. |
| (...skipping 30 matching lines...) Expand all Loading... |
| 2882 | 2882 |
| 2883 if (Condition == InstFcmp::True) { | 2883 if (Condition == InstFcmp::True) { |
| 2884 // makeVectorOfOnes() requires an integer vector type. | 2884 // makeVectorOfOnes() requires an integer vector type. |
| 2885 T = makeVectorOfMinusOnes(IceType_v4i32); | 2885 T = makeVectorOfMinusOnes(IceType_v4i32); |
| 2886 } else if (Condition == InstFcmp::False) { | 2886 } else if (Condition == InstFcmp::False) { |
| 2887 T = makeVectorOfZeros(Dest->getType()); | 2887 T = makeVectorOfZeros(Dest->getType()); |
| 2888 } else { | 2888 } else { |
| 2889 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); | 2889 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); |
| 2890 Operand *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); | 2890 Operand *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); |
| 2891 if (llvm::isa<typename Traits::X86OperandMem>(Src1RM)) | 2891 if (llvm::isa<typename Traits::X86OperandMem>(Src1RM)) |
| 2892 Src1RM = legalizeToVar(Src1RM); | 2892 Src1RM = legalizeToReg(Src1RM); |
| 2893 | 2893 |
| 2894 switch (Condition) { | 2894 switch (Condition) { |
| 2895 default: { | 2895 default: { |
| 2896 typename Traits::Cond::CmppsCond Predicate = | 2896 typename Traits::Cond::CmppsCond Predicate = |
| 2897 Traits::TableFcmp[Index].Predicate; | 2897 Traits::TableFcmp[Index].Predicate; |
| 2898 assert(Predicate != Traits::Cond::Cmpps_Invalid); | 2898 assert(Predicate != Traits::Cond::Cmpps_Invalid); |
| 2899 T = makeReg(Src0RM->getType()); | 2899 T = makeReg(Src0RM->getType()); |
| 2900 _movp(T, Src0RM); | 2900 _movp(T, Src0RM); |
| 2901 _cmpps(T, Src1RM, Predicate); | 2901 _cmpps(T, Src1RM, Predicate); |
| 2902 } break; | 2902 } break; |
| (...skipping 128 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3031 Src1RM = T1; | 3031 Src1RM = T1; |
| 3032 } | 3032 } |
| 3033 | 3033 |
| 3034 Variable *T = makeReg(Ty); | 3034 Variable *T = makeReg(Ty); |
| 3035 switch (Condition) { | 3035 switch (Condition) { |
| 3036 default: | 3036 default: |
| 3037 llvm_unreachable("unexpected condition"); | 3037 llvm_unreachable("unexpected condition"); |
| 3038 break; | 3038 break; |
| 3039 case InstIcmp::Eq: { | 3039 case InstIcmp::Eq: { |
| 3040 if (llvm::isa<typename Traits::X86OperandMem>(Src1RM)) | 3040 if (llvm::isa<typename Traits::X86OperandMem>(Src1RM)) |
| 3041 Src1RM = legalizeToVar(Src1RM); | 3041 Src1RM = legalizeToReg(Src1RM); |
| 3042 _movp(T, Src0RM); | 3042 _movp(T, Src0RM); |
| 3043 _pcmpeq(T, Src1RM); | 3043 _pcmpeq(T, Src1RM); |
| 3044 } break; | 3044 } break; |
| 3045 case InstIcmp::Ne: { | 3045 case InstIcmp::Ne: { |
| 3046 if (llvm::isa<typename Traits::X86OperandMem>(Src1RM)) | 3046 if (llvm::isa<typename Traits::X86OperandMem>(Src1RM)) |
| 3047 Src1RM = legalizeToVar(Src1RM); | 3047 Src1RM = legalizeToReg(Src1RM); |
| 3048 _movp(T, Src0RM); | 3048 _movp(T, Src0RM); |
| 3049 _pcmpeq(T, Src1RM); | 3049 _pcmpeq(T, Src1RM); |
| 3050 Variable *MinusOne = makeVectorOfMinusOnes(Ty); | 3050 Variable *MinusOne = makeVectorOfMinusOnes(Ty); |
| 3051 _pxor(T, MinusOne); | 3051 _pxor(T, MinusOne); |
| 3052 } break; | 3052 } break; |
| 3053 case InstIcmp::Ugt: | 3053 case InstIcmp::Ugt: |
| 3054 case InstIcmp::Sgt: { | 3054 case InstIcmp::Sgt: { |
| 3055 if (llvm::isa<typename Traits::X86OperandMem>(Src1RM)) | 3055 if (llvm::isa<typename Traits::X86OperandMem>(Src1RM)) |
| 3056 Src1RM = legalizeToVar(Src1RM); | 3056 Src1RM = legalizeToReg(Src1RM); |
| 3057 _movp(T, Src0RM); | 3057 _movp(T, Src0RM); |
| 3058 _pcmpgt(T, Src1RM); | 3058 _pcmpgt(T, Src1RM); |
| 3059 } break; | 3059 } break; |
| 3060 case InstIcmp::Uge: | 3060 case InstIcmp::Uge: |
| 3061 case InstIcmp::Sge: { | 3061 case InstIcmp::Sge: { |
| 3062 // !(Src1RM > Src0RM) | 3062 // !(Src1RM > Src0RM) |
| 3063 if (llvm::isa<typename Traits::X86OperandMem>(Src0RM)) | 3063 if (llvm::isa<typename Traits::X86OperandMem>(Src0RM)) |
| 3064 Src0RM = legalizeToVar(Src0RM); | 3064 Src0RM = legalizeToReg(Src0RM); |
| 3065 _movp(T, Src1RM); | 3065 _movp(T, Src1RM); |
| 3066 _pcmpgt(T, Src0RM); | 3066 _pcmpgt(T, Src0RM); |
| 3067 Variable *MinusOne = makeVectorOfMinusOnes(Ty); | 3067 Variable *MinusOne = makeVectorOfMinusOnes(Ty); |
| 3068 _pxor(T, MinusOne); | 3068 _pxor(T, MinusOne); |
| 3069 } break; | 3069 } break; |
| 3070 case InstIcmp::Ult: | 3070 case InstIcmp::Ult: |
| 3071 case InstIcmp::Slt: { | 3071 case InstIcmp::Slt: { |
| 3072 if (llvm::isa<typename Traits::X86OperandMem>(Src0RM)) | 3072 if (llvm::isa<typename Traits::X86OperandMem>(Src0RM)) |
| 3073 Src0RM = legalizeToVar(Src0RM); | 3073 Src0RM = legalizeToReg(Src0RM); |
| 3074 _movp(T, Src1RM); | 3074 _movp(T, Src1RM); |
| 3075 _pcmpgt(T, Src0RM); | 3075 _pcmpgt(T, Src0RM); |
| 3076 } break; | 3076 } break; |
| 3077 case InstIcmp::Ule: | 3077 case InstIcmp::Ule: |
| 3078 case InstIcmp::Sle: { | 3078 case InstIcmp::Sle: { |
| 3079 // !(Src0RM > Src1RM) | 3079 // !(Src0RM > Src1RM) |
| 3080 if (llvm::isa<typename Traits::X86OperandMem>(Src1RM)) | 3080 if (llvm::isa<typename Traits::X86OperandMem>(Src1RM)) |
| 3081 Src1RM = legalizeToVar(Src1RM); | 3081 Src1RM = legalizeToReg(Src1RM); |
| 3082 _movp(T, Src0RM); | 3082 _movp(T, Src0RM); |
| 3083 _pcmpgt(T, Src1RM); | 3083 _pcmpgt(T, Src1RM); |
| 3084 Variable *MinusOne = makeVectorOfMinusOnes(Ty); | 3084 Variable *MinusOne = makeVectorOfMinusOnes(Ty); |
| 3085 _pxor(T, MinusOne); | 3085 _pxor(T, MinusOne); |
| 3086 } break; | 3086 } break; |
| 3087 } | 3087 } |
| 3088 | 3088 |
| 3089 _movp(Dest, T); | 3089 _movp(Dest, T); |
| 3090 eliminateNextVectorSextInstruction(Dest); | 3090 eliminateNextVectorSextInstruction(Dest); |
| 3091 return; | 3091 return; |
| (...skipping 74 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3166 _pinsr(T, ElementRM, Ctx->getConstantInt32(Index)); | 3166 _pinsr(T, ElementRM, Ctx->getConstantInt32(Index)); |
| 3167 _movp(Inst->getDest(), T); | 3167 _movp(Inst->getDest(), T); |
| 3168 } else if (Ty == IceType_v4i32 || Ty == IceType_v4f32 || Ty == IceType_v4i1) { | 3168 } else if (Ty == IceType_v4i32 || Ty == IceType_v4f32 || Ty == IceType_v4i1) { |
| 3169 // Use shufps or movss. | 3169 // Use shufps or movss. |
| 3170 Variable *ElementR = nullptr; | 3170 Variable *ElementR = nullptr; |
| 3171 Operand *SourceVectRM = | 3171 Operand *SourceVectRM = |
| 3172 legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem); | 3172 legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem); |
| 3173 | 3173 |
| 3174 if (InVectorElementTy == IceType_f32) { | 3174 if (InVectorElementTy == IceType_f32) { |
| 3175 // ElementR will be in an XMM register since it is floating point. | 3175 // ElementR will be in an XMM register since it is floating point. |
| 3176 ElementR = legalizeToVar(ElementToInsertNotLegalized); | 3176 ElementR = legalizeToReg(ElementToInsertNotLegalized); |
| 3177 } else { | 3177 } else { |
| 3178 // Copy an integer to an XMM register. | 3178 // Copy an integer to an XMM register. |
| 3179 Operand *T = legalize(ElementToInsertNotLegalized, Legal_Reg | Legal_Mem); | 3179 Operand *T = legalize(ElementToInsertNotLegalized, Legal_Reg | Legal_Mem); |
| 3180 ElementR = makeReg(Ty); | 3180 ElementR = makeReg(Ty); |
| 3181 _movd(ElementR, T); | 3181 _movd(ElementR, T); |
| 3182 } | 3182 } |
| 3183 | 3183 |
| 3184 if (Index == 0) { | 3184 if (Index == 0) { |
| 3185 Variable *T = makeReg(Ty); | 3185 Variable *T = makeReg(Ty); |
| 3186 _movp(T, SourceVectRM); | 3186 _movp(T, SourceVectRM); |
| (...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3230 } | 3230 } |
| 3231 } else { | 3231 } else { |
| 3232 assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); | 3232 assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); |
| 3233 // Spill the value to a stack slot and perform the insertion in | 3233 // Spill the value to a stack slot and perform the insertion in |
| 3234 // memory. | 3234 // memory. |
| 3235 // | 3235 // |
| 3236 // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when | 3236 // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when |
| 3237 // support for legalizing to mem is implemented. | 3237 // support for legalizing to mem is implemented. |
| 3238 Variable *Slot = Func->template makeVariable(Ty); | 3238 Variable *Slot = Func->template makeVariable(Ty); |
| 3239 Slot->setWeight(RegWeight::Zero); | 3239 Slot->setWeight(RegWeight::Zero); |
| 3240 _movp(Slot, legalizeToVar(SourceVectNotLegalized)); | 3240 _movp(Slot, legalizeToReg(SourceVectNotLegalized)); |
| 3241 | 3241 |
| 3242 // Compute the location of the position to insert in memory. | 3242 // Compute the location of the position to insert in memory. |
| 3243 unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); | 3243 unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); |
| 3244 typename Traits::X86OperandMem *Loc = | 3244 typename Traits::X86OperandMem *Loc = |
| 3245 getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); | 3245 getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); |
| 3246 _store(legalizeToVar(ElementToInsertNotLegalized), Loc); | 3246 _store(legalizeToReg(ElementToInsertNotLegalized), Loc); |
| 3247 | 3247 |
| 3248 Variable *T = makeReg(Ty); | 3248 Variable *T = makeReg(Ty); |
| 3249 _movp(T, Slot); | 3249 _movp(T, Slot); |
| 3250 _movp(Inst->getDest(), T); | 3250 _movp(Inst->getDest(), T); |
| 3251 } | 3251 } |
| 3252 } | 3252 } |
| 3253 | 3253 |
| 3254 template <class Machine> | 3254 template <class Machine> |
| 3255 void TargetX86Base<Machine>::lowerIntrinsicCall( | 3255 void TargetX86Base<Machine>::lowerIntrinsicCall( |
| 3256 const InstIntrinsicCall *Instr) { | 3256 const InstIntrinsicCall *Instr) { |
| (...skipping 136 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3393 _mfence(); | 3393 _mfence(); |
| 3394 return; | 3394 return; |
| 3395 } | 3395 } |
| 3396 case Intrinsics::Bswap: { | 3396 case Intrinsics::Bswap: { |
| 3397 Variable *Dest = Instr->getDest(); | 3397 Variable *Dest = Instr->getDest(); |
| 3398 Operand *Val = Instr->getArg(0); | 3398 Operand *Val = Instr->getArg(0); |
| 3399 // In 32-bit mode, bswap only works on 32-bit arguments, and the | 3399 // In 32-bit mode, bswap only works on 32-bit arguments, and the |
| 3400 // argument must be a register. Use rotate left for 16-bit bswap. | 3400 // argument must be a register. Use rotate left for 16-bit bswap. |
| 3401 if (Val->getType() == IceType_i64) { | 3401 if (Val->getType() == IceType_i64) { |
| 3402 Val = legalizeUndef(Val); | 3402 Val = legalizeUndef(Val); |
| 3403 Variable *T_Lo = legalizeToVar(loOperand(Val)); | 3403 Variable *T_Lo = legalizeToReg(loOperand(Val)); |
| 3404 Variable *T_Hi = legalizeToVar(hiOperand(Val)); | 3404 Variable *T_Hi = legalizeToReg(hiOperand(Val)); |
| 3405 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 3405 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
| 3406 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 3406 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
| 3407 _bswap(T_Lo); | 3407 _bswap(T_Lo); |
| 3408 _bswap(T_Hi); | 3408 _bswap(T_Hi); |
| 3409 _mov(DestLo, T_Hi); | 3409 _mov(DestLo, T_Hi); |
| 3410 _mov(DestHi, T_Lo); | 3410 _mov(DestHi, T_Lo); |
| 3411 } else if (Val->getType() == IceType_i32) { | 3411 } else if (Val->getType() == IceType_i32) { |
| 3412 Variable *T = legalizeToVar(Val); | 3412 Variable *T = legalizeToReg(Val); |
| 3413 _bswap(T); | 3413 _bswap(T); |
| 3414 _mov(Dest, T); | 3414 _mov(Dest, T); |
| 3415 } else { | 3415 } else { |
| 3416 assert(Val->getType() == IceType_i16); | 3416 assert(Val->getType() == IceType_i16); |
| 3417 Constant *Eight = Ctx->getConstantInt16(8); | 3417 Constant *Eight = Ctx->getConstantInt16(8); |
| 3418 Variable *T = nullptr; | 3418 Variable *T = nullptr; |
| 3419 Val = legalize(Val); | 3419 Val = legalize(Val); |
| 3420 _mov(T, Val); | 3420 _mov(T, Val); |
| 3421 _rol(T, Eight); | 3421 _rol(T, Eight); |
| 3422 _mov(Dest, T); | 3422 _mov(Dest, T); |
| (...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3480 } | 3480 } |
| 3481 case Intrinsics::Fabs: { | 3481 case Intrinsics::Fabs: { |
| 3482 Operand *Src = legalize(Instr->getArg(0)); | 3482 Operand *Src = legalize(Instr->getArg(0)); |
| 3483 Type Ty = Src->getType(); | 3483 Type Ty = Src->getType(); |
| 3484 Variable *Dest = Instr->getDest(); | 3484 Variable *Dest = Instr->getDest(); |
| 3485 Variable *T = makeVectorOfFabsMask(Ty); | 3485 Variable *T = makeVectorOfFabsMask(Ty); |
| 3486 // The pand instruction operates on an m128 memory operand, so if | 3486 // The pand instruction operates on an m128 memory operand, so if |
| 3487 // Src is an f32 or f64, we need to make sure it's in a register. | 3487 // Src is an f32 or f64, we need to make sure it's in a register. |
| 3488 if (isVectorType(Ty)) { | 3488 if (isVectorType(Ty)) { |
| 3489 if (llvm::isa<typename Traits::X86OperandMem>(Src)) | 3489 if (llvm::isa<typename Traits::X86OperandMem>(Src)) |
| 3490 Src = legalizeToVar(Src); | 3490 Src = legalizeToReg(Src); |
| 3491 } else { | 3491 } else { |
| 3492 Src = legalizeToVar(Src); | 3492 Src = legalizeToReg(Src); |
| 3493 } | 3493 } |
| 3494 _pand(T, Src); | 3494 _pand(T, Src); |
| 3495 if (isVectorType(Ty)) | 3495 if (isVectorType(Ty)) |
| 3496 _movp(Dest, T); | 3496 _movp(Dest, T); |
| 3497 else | 3497 else |
| 3498 _mov(Dest, T); | 3498 _mov(Dest, T); |
| 3499 return; | 3499 return; |
| 3500 } | 3500 } |
| 3501 case Intrinsics::Longjmp: { | 3501 case Intrinsics::Longjmp: { |
| 3502 InstCall *Call = makeHelperCall(H_call_longjmp, nullptr, 2); | 3502 InstCall *Call = makeHelperCall(H_call_longjmp, nullptr, 2); |
| (...skipping 109 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3612 Variable *DestLo = llvm::cast<Variable>(loOperand(DestPrev)); | 3612 Variable *DestLo = llvm::cast<Variable>(loOperand(DestPrev)); |
| 3613 Variable *DestHi = llvm::cast<Variable>(hiOperand(DestPrev)); | 3613 Variable *DestHi = llvm::cast<Variable>(hiOperand(DestPrev)); |
| 3614 _mov(DestLo, T_eax); | 3614 _mov(DestLo, T_eax); |
| 3615 _mov(DestHi, T_edx); | 3615 _mov(DestHi, T_edx); |
| 3616 return; | 3616 return; |
| 3617 } | 3617 } |
| 3618 Variable *T_eax = makeReg(Expected->getType(), Traits::RegisterSet::Reg_eax); | 3618 Variable *T_eax = makeReg(Expected->getType(), Traits::RegisterSet::Reg_eax); |
| 3619 _mov(T_eax, Expected); | 3619 _mov(T_eax, Expected); |
| 3620 typename Traits::X86OperandMem *Addr = | 3620 typename Traits::X86OperandMem *Addr = |
| 3621 formMemoryOperand(Ptr, Expected->getType()); | 3621 formMemoryOperand(Ptr, Expected->getType()); |
| 3622 Variable *DesiredReg = legalizeToVar(Desired); | 3622 Variable *DesiredReg = legalizeToReg(Desired); |
| 3623 const bool Locked = true; | 3623 const bool Locked = true; |
| 3624 _cmpxchg(Addr, T_eax, DesiredReg, Locked); | 3624 _cmpxchg(Addr, T_eax, DesiredReg, Locked); |
| 3625 _mov(DestPrev, T_eax); | 3625 _mov(DestPrev, T_eax); |
| 3626 } | 3626 } |
| 3627 | 3627 |
| 3628 template <class Machine> | 3628 template <class Machine> |
| 3629 bool TargetX86Base<Machine>::tryOptimizedCmpxchgCmpBr(Variable *Dest, | 3629 bool TargetX86Base<Machine>::tryOptimizedCmpxchgCmpBr(Variable *Dest, |
| 3630 Operand *PtrToMem, | 3630 Operand *PtrToMem, |
| 3631 Operand *Expected, | 3631 Operand *Expected, |
| 3632 Operand *Desired) { | 3632 Operand *Desired) { |
| (...skipping 321 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3954 _xor(T_Dest, ThirtyOne); | 3954 _xor(T_Dest, ThirtyOne); |
| 3955 } | 3955 } |
| 3956 if (Ty == IceType_i32) { | 3956 if (Ty == IceType_i32) { |
| 3957 _mov(Dest, T_Dest); | 3957 _mov(Dest, T_Dest); |
| 3958 return; | 3958 return; |
| 3959 } | 3959 } |
| 3960 _add(T_Dest, ThirtyTwo); | 3960 _add(T_Dest, ThirtyTwo); |
| 3961 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 3961 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
| 3962 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 3962 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
| 3963 // Will be using "test" on this, so we need a registerized variable. | 3963 // Will be using "test" on this, so we need a registerized variable. |
| 3964 Variable *SecondVar = legalizeToVar(SecondVal); | 3964 Variable *SecondVar = legalizeToReg(SecondVal); |
| 3965 Variable *T_Dest2 = makeReg(IceType_i32); | 3965 Variable *T_Dest2 = makeReg(IceType_i32); |
| 3966 if (Cttz) { | 3966 if (Cttz) { |
| 3967 _bsf(T_Dest2, SecondVar); | 3967 _bsf(T_Dest2, SecondVar); |
| 3968 } else { | 3968 } else { |
| 3969 _bsr(T_Dest2, SecondVar); | 3969 _bsr(T_Dest2, SecondVar); |
| 3970 _xor(T_Dest2, ThirtyOne); | 3970 _xor(T_Dest2, ThirtyOne); |
| 3971 } | 3971 } |
| 3972 _test(SecondVar, SecondVar); | 3972 _test(SecondVar, SecondVar); |
| 3973 _cmov(T_Dest2, T_Dest, Traits::Cond::Br_e); | 3973 _cmov(T_Dest2, T_Dest, Traits::Cond::Br_e); |
| 3974 _mov(DestLo, T_Dest2); | 3974 _mov(DestLo, T_Dest2); |
| (...skipping 291 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4266 Func->setError("Phi found in regular instruction list"); | 4266 Func->setError("Phi found in regular instruction list"); |
| 4267 } | 4267 } |
| 4268 | 4268 |
| 4269 template <class Machine> | 4269 template <class Machine> |
| 4270 void TargetX86Base<Machine>::lowerRet(const InstRet *Inst) { | 4270 void TargetX86Base<Machine>::lowerRet(const InstRet *Inst) { |
| 4271 Variable *Reg = nullptr; | 4271 Variable *Reg = nullptr; |
| 4272 if (Inst->hasRetValue()) { | 4272 if (Inst->hasRetValue()) { |
| 4273 Operand *Src0 = legalize(Inst->getRetValue()); | 4273 Operand *Src0 = legalize(Inst->getRetValue()); |
| 4274 if (Src0->getType() == IceType_i64) { | 4274 if (Src0->getType() == IceType_i64) { |
| 4275 Variable *eax = | 4275 Variable *eax = |
| 4276 legalizeToVar(loOperand(Src0), Traits::RegisterSet::Reg_eax); | 4276 legalizeToReg(loOperand(Src0), Traits::RegisterSet::Reg_eax); |
| 4277 Variable *edx = | 4277 Variable *edx = |
| 4278 legalizeToVar(hiOperand(Src0), Traits::RegisterSet::Reg_edx); | 4278 legalizeToReg(hiOperand(Src0), Traits::RegisterSet::Reg_edx); |
| 4279 Reg = eax; | 4279 Reg = eax; |
| 4280 Context.insert(InstFakeUse::create(Func, edx)); | 4280 Context.insert(InstFakeUse::create(Func, edx)); |
| 4281 } else if (isScalarFloatingType(Src0->getType())) { | 4281 } else if (isScalarFloatingType(Src0->getType())) { |
| 4282 _fld(Src0); | 4282 _fld(Src0); |
| 4283 } else if (isVectorType(Src0->getType())) { | 4283 } else if (isVectorType(Src0->getType())) { |
| 4284 Reg = legalizeToVar(Src0, Traits::RegisterSet::Reg_xmm0); | 4284 Reg = legalizeToReg(Src0, Traits::RegisterSet::Reg_xmm0); |
| 4285 } else { | 4285 } else { |
| 4286 _mov(Reg, Src0, Traits::RegisterSet::Reg_eax); | 4286 _mov(Reg, Src0, Traits::RegisterSet::Reg_eax); |
| 4287 } | 4287 } |
| 4288 } | 4288 } |
| 4289 // Add a ret instruction even if sandboxing is enabled, because | 4289 // Add a ret instruction even if sandboxing is enabled, because |
| 4290 // addEpilog explicitly looks for a ret instruction as a marker for | 4290 // addEpilog explicitly looks for a ret instruction as a marker for |
| 4291 // where to insert the frame removal instructions. | 4291 // where to insert the frame removal instructions. |
| 4292 _ret(Reg); | 4292 _ret(Reg); |
| 4293 // Add a fake use of esp to make sure esp stays alive for the entire | 4293 // Add a fake use of esp to make sure esp stays alive for the entire |
| 4294 // function. Otherwise post-call esp adjustments get dead-code | 4294 // function. Otherwise post-call esp adjustments get dead-code |
| (...skipping 159 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4454 | 4454 |
| 4455 if (Ty == IceType_i64) { | 4455 if (Ty == IceType_i64) { |
| 4456 Value = legalizeUndef(Value); | 4456 Value = legalizeUndef(Value); |
| 4457 Operand *ValueHi = legalize(hiOperand(Value), Legal_Reg | Legal_Imm); | 4457 Operand *ValueHi = legalize(hiOperand(Value), Legal_Reg | Legal_Imm); |
| 4458 Operand *ValueLo = legalize(loOperand(Value), Legal_Reg | Legal_Imm); | 4458 Operand *ValueLo = legalize(loOperand(Value), Legal_Reg | Legal_Imm); |
| 4459 _store(ValueHi, | 4459 _store(ValueHi, |
| 4460 llvm::cast<typename Traits::X86OperandMem>(hiOperand(NewAddr))); | 4460 llvm::cast<typename Traits::X86OperandMem>(hiOperand(NewAddr))); |
| 4461 _store(ValueLo, | 4461 _store(ValueLo, |
| 4462 llvm::cast<typename Traits::X86OperandMem>(loOperand(NewAddr))); | 4462 llvm::cast<typename Traits::X86OperandMem>(loOperand(NewAddr))); |
| 4463 } else if (isVectorType(Ty)) { | 4463 } else if (isVectorType(Ty)) { |
| 4464 _storep(legalizeToVar(Value), NewAddr); | 4464 _storep(legalizeToReg(Value), NewAddr); |
| 4465 } else { | 4465 } else { |
| 4466 Value = legalize(Value, Legal_Reg | Legal_Imm); | 4466 Value = legalize(Value, Legal_Reg | Legal_Imm); |
| 4467 _store(Value, NewAddr); | 4467 _store(Value, NewAddr); |
| 4468 } | 4468 } |
| 4469 } | 4469 } |
| 4470 | 4470 |
| 4471 template <class Machine> void TargetX86Base<Machine>::doAddressOptStore() { | 4471 template <class Machine> void TargetX86Base<Machine>::doAddressOptStore() { |
| 4472 InstStore *Inst = llvm::cast<InstStore>(Context.getCur()); | 4472 InstStore *Inst = llvm::cast<InstStore>(Context.getCur()); |
| 4473 Operand *Data = Inst->getData(); | 4473 Operand *Data = Inst->getData(); |
| 4474 Operand *Addr = Inst->getAddr(); | 4474 Operand *Addr = Inst->getAddr(); |
| (...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4535 | 4535 |
| 4536 InstJumpTable *JumpTable = Case.getJumpTable(); | 4536 InstJumpTable *JumpTable = Case.getJumpTable(); |
| 4537 Context.insert(JumpTable); | 4537 Context.insert(JumpTable); |
| 4538 | 4538 |
| 4539 // Make sure the index is a register of the same width as the base | 4539 // Make sure the index is a register of the same width as the base |
| 4540 Variable *Index; | 4540 Variable *Index; |
| 4541 if (RangeIndex->getType() != getPointerType()) { | 4541 if (RangeIndex->getType() != getPointerType()) { |
| 4542 Index = makeReg(getPointerType()); | 4542 Index = makeReg(getPointerType()); |
| 4543 _movzx(Index, RangeIndex); | 4543 _movzx(Index, RangeIndex); |
| 4544 } else { | 4544 } else { |
| 4545 Index = legalizeToVar(RangeIndex); | 4545 Index = legalizeToReg(RangeIndex); |
| 4546 } | 4546 } |
| 4547 | 4547 |
| 4548 constexpr RelocOffsetT RelocOffset = 0; | 4548 constexpr RelocOffsetT RelocOffset = 0; |
| 4549 constexpr bool SuppressMangling = true; | 4549 constexpr bool SuppressMangling = true; |
| 4550 Constant *Base = Ctx->getConstantSym(RelocOffset, JumpTable->getName(Func), | 4550 Constant *Base = Ctx->getConstantSym(RelocOffset, JumpTable->getName(Func), |
| 4551 SuppressMangling); | 4551 SuppressMangling); |
| 4552 Constant *Offset = nullptr; | 4552 Constant *Offset = nullptr; |
| 4553 uint16_t Shift = typeWidthInBytesLog2(getPointerType()); | 4553 uint16_t Shift = typeWidthInBytesLog2(getPointerType()); |
| 4554 // TODO(ascull): remove need for legalize by allowing null base in memop | 4554 // TODO(ascull): remove need for legalize by allowing null base in memop |
| 4555 auto *MemTarget = Traits::X86OperandMem::create( | 4555 auto *MemTarget = Traits::X86OperandMem::create( |
| 4556 Func, getPointerType(), legalizeToVar(Base), Offset, Index, Shift); | 4556 Func, getPointerType(), legalizeToReg(Base), Offset, Index, Shift); |
| 4557 Variable *Target = nullptr; | 4557 Variable *Target = nullptr; |
| 4558 _mov(Target, MemTarget); | 4558 _mov(Target, MemTarget); |
| 4559 _jmp(Target); | 4559 _jmp(Target); |
| 4560 // TODO(ascull): sandboxing for indirect jump | 4560 // TODO(ascull): sandboxing for indirect jump |
| 4561 | 4561 |
| 4562 if (DefaultLabel == nullptr) | 4562 if (DefaultLabel == nullptr) |
| 4563 Context.insert(SkipJumpTable); | 4563 Context.insert(SkipJumpTable); |
| 4564 return; | 4564 return; |
| 4565 } | 4565 } |
| 4566 case CaseCluster::Range: { | 4566 case CaseCluster::Range: { |
| (...skipping 23 matching lines...) Expand all Loading... |
| 4590 if (!Ctx->getFlags().getUseAdvancedSwitchLowering()) { | 4590 if (!Ctx->getFlags().getUseAdvancedSwitchLowering()) { |
| 4591 // This implements the most naive possible lowering. | 4591 // This implements the most naive possible lowering. |
| 4592 // cmp a,val[0]; jeq label[0]; cmp a,val[1]; jeq label[1]; ... jmp default | 4592 // cmp a,val[0]; jeq label[0]; cmp a,val[1]; jeq label[1]; ... jmp default |
| 4593 Operand *Src0 = Inst->getComparison(); | 4593 Operand *Src0 = Inst->getComparison(); |
| 4594 SizeT NumCases = Inst->getNumCases(); | 4594 SizeT NumCases = Inst->getNumCases(); |
| 4595 if (Src0->getType() == IceType_i64) { | 4595 if (Src0->getType() == IceType_i64) { |
| 4596 Src0 = legalize(Src0); // get Base/Index into physical registers | 4596 Src0 = legalize(Src0); // get Base/Index into physical registers |
| 4597 Operand *Src0Lo = loOperand(Src0); | 4597 Operand *Src0Lo = loOperand(Src0); |
| 4598 Operand *Src0Hi = hiOperand(Src0); | 4598 Operand *Src0Hi = hiOperand(Src0); |
| 4599 if (NumCases >= 2) { | 4599 if (NumCases >= 2) { |
| 4600 Src0Lo = legalizeToVar(Src0Lo); | 4600 Src0Lo = legalizeToReg(Src0Lo); |
| 4601 Src0Hi = legalizeToVar(Src0Hi); | 4601 Src0Hi = legalizeToReg(Src0Hi); |
| 4602 } else { | 4602 } else { |
| 4603 Src0Lo = legalize(Src0Lo, Legal_Reg | Legal_Mem); | 4603 Src0Lo = legalize(Src0Lo, Legal_Reg | Legal_Mem); |
| 4604 Src0Hi = legalize(Src0Hi, Legal_Reg | Legal_Mem); | 4604 Src0Hi = legalize(Src0Hi, Legal_Reg | Legal_Mem); |
| 4605 } | 4605 } |
| 4606 for (SizeT I = 0; I < NumCases; ++I) { | 4606 for (SizeT I = 0; I < NumCases; ++I) { |
| 4607 Constant *ValueLo = Ctx->getConstantInt32(Inst->getValue(I)); | 4607 Constant *ValueLo = Ctx->getConstantInt32(Inst->getValue(I)); |
| 4608 Constant *ValueHi = Ctx->getConstantInt32(Inst->getValue(I) >> 32); | 4608 Constant *ValueHi = Ctx->getConstantInt32(Inst->getValue(I) >> 32); |
| 4609 typename Traits::Insts::Label *Label = | 4609 typename Traits::Insts::Label *Label = |
| 4610 Traits::Insts::Label::create(Func, this); | 4610 Traits::Insts::Label::create(Func, this); |
| 4611 _cmp(Src0Lo, ValueLo); | 4611 _cmp(Src0Lo, ValueLo); |
| 4612 _br(Traits::Cond::Br_ne, Label); | 4612 _br(Traits::Cond::Br_ne, Label); |
| 4613 _cmp(Src0Hi, ValueHi); | 4613 _cmp(Src0Hi, ValueHi); |
| 4614 _br(Traits::Cond::Br_e, Inst->getLabel(I)); | 4614 _br(Traits::Cond::Br_e, Inst->getLabel(I)); |
| 4615 Context.insert(Label); | 4615 Context.insert(Label); |
| 4616 } | 4616 } |
| 4617 _br(Inst->getLabelDefault()); | 4617 _br(Inst->getLabelDefault()); |
| 4618 return; | 4618 return; |
| 4619 } | 4619 } |
| 4620 // OK, we'll be slightly less naive by forcing Src into a physical | 4620 // OK, we'll be slightly less naive by forcing Src into a physical |
| 4621 // register if there are 2 or more uses. | 4621 // register if there are 2 or more uses. |
| 4622 if (NumCases >= 2) | 4622 if (NumCases >= 2) |
| 4623 Src0 = legalizeToVar(Src0); | 4623 Src0 = legalizeToReg(Src0); |
| 4624 else | 4624 else |
| 4625 Src0 = legalize(Src0, Legal_Reg | Legal_Mem); | 4625 Src0 = legalize(Src0, Legal_Reg | Legal_Mem); |
| 4626 for (SizeT I = 0; I < NumCases; ++I) { | 4626 for (SizeT I = 0; I < NumCases; ++I) { |
| 4627 Constant *Value = Ctx->getConstantInt32(Inst->getValue(I)); | 4627 Constant *Value = Ctx->getConstantInt32(Inst->getValue(I)); |
| 4628 _cmp(Src0, Value); | 4628 _cmp(Src0, Value); |
| 4629 _br(Traits::Cond::Br_e, Inst->getLabel(I)); | 4629 _br(Traits::Cond::Br_e, Inst->getLabel(I)); |
| 4630 } | 4630 } |
| 4631 | 4631 |
| 4632 _br(Inst->getLabelDefault()); | 4632 _br(Inst->getLabelDefault()); |
| 4633 return; | 4633 return; |
| 4634 } | 4634 } |
| 4635 | 4635 |
| 4636 // Group cases together and navigate through them with a binary search | 4636 // Group cases together and navigate through them with a binary search |
| 4637 CaseClusterArray CaseClusters = CaseCluster::clusterizeSwitch(Func, Inst); | 4637 CaseClusterArray CaseClusters = CaseCluster::clusterizeSwitch(Func, Inst); |
| 4638 Operand *Src0 = Inst->getComparison(); | 4638 Operand *Src0 = Inst->getComparison(); |
| 4639 CfgNode *DefaultLabel = Inst->getLabelDefault(); | 4639 CfgNode *DefaultLabel = Inst->getLabelDefault(); |
| 4640 | 4640 |
| 4641 assert(CaseClusters.size() != 0); // Should always be at least one | 4641 assert(CaseClusters.size() != 0); // Should always be at least one |
| 4642 | 4642 |
| 4643 if (Src0->getType() == IceType_i64) { | 4643 if (Src0->getType() == IceType_i64) { |
| 4644 Src0 = legalize(Src0); // get Base/Index into physical registers | 4644 Src0 = legalize(Src0); // get Base/Index into physical registers |
| 4645 Operand *Src0Lo = loOperand(Src0); | 4645 Operand *Src0Lo = loOperand(Src0); |
| 4646 Operand *Src0Hi = hiOperand(Src0); | 4646 Operand *Src0Hi = hiOperand(Src0); |
| 4647 if (CaseClusters.back().getHigh() > UINT32_MAX) { | 4647 if (CaseClusters.back().getHigh() > UINT32_MAX) { |
| 4648 // TODO(ascull): handle 64-bit case properly (currently naive version) | 4648 // TODO(ascull): handle 64-bit case properly (currently naive version) |
| 4649 // This might be handled by a higher level lowering of switches. | 4649 // This might be handled by a higher level lowering of switches. |
| 4650 SizeT NumCases = Inst->getNumCases(); | 4650 SizeT NumCases = Inst->getNumCases(); |
| 4651 if (NumCases >= 2) { | 4651 if (NumCases >= 2) { |
| 4652 Src0Lo = legalizeToVar(Src0Lo); | 4652 Src0Lo = legalizeToReg(Src0Lo); |
| 4653 Src0Hi = legalizeToVar(Src0Hi); | 4653 Src0Hi = legalizeToReg(Src0Hi); |
| 4654 } else { | 4654 } else { |
| 4655 Src0Lo = legalize(Src0Lo, Legal_Reg | Legal_Mem); | 4655 Src0Lo = legalize(Src0Lo, Legal_Reg | Legal_Mem); |
| 4656 Src0Hi = legalize(Src0Hi, Legal_Reg | Legal_Mem); | 4656 Src0Hi = legalize(Src0Hi, Legal_Reg | Legal_Mem); |
| 4657 } | 4657 } |
| 4658 for (SizeT I = 0; I < NumCases; ++I) { | 4658 for (SizeT I = 0; I < NumCases; ++I) { |
| 4659 Constant *ValueLo = Ctx->getConstantInt32(Inst->getValue(I)); | 4659 Constant *ValueLo = Ctx->getConstantInt32(Inst->getValue(I)); |
| 4660 Constant *ValueHi = Ctx->getConstantInt32(Inst->getValue(I) >> 32); | 4660 Constant *ValueHi = Ctx->getConstantInt32(Inst->getValue(I) >> 32); |
| 4661 typename Traits::Insts::Label *Label = | 4661 typename Traits::Insts::Label *Label = |
| 4662 Traits::Insts::Label::create(Func, this); | 4662 Traits::Insts::Label::create(Func, this); |
| 4663 _cmp(Src0Lo, ValueLo); | 4663 _cmp(Src0Lo, ValueLo); |
| (...skipping 19 matching lines...) Expand all Loading... |
| 4683 | 4683 |
| 4684 if (CaseClusters.size() == 1) { | 4684 if (CaseClusters.size() == 1) { |
| 4685 // Jump straight to default if needed. Currently a common case as jump | 4685 // Jump straight to default if needed. Currently a common case as jump |
| 4686 // tables occur on their own. | 4686 // tables occur on their own. |
| 4687 constexpr bool DoneCmp = false; | 4687 constexpr bool DoneCmp = false; |
| 4688 lowerCaseCluster(CaseClusters.front(), Src0, DoneCmp, DefaultLabel); | 4688 lowerCaseCluster(CaseClusters.front(), Src0, DoneCmp, DefaultLabel); |
| 4689 return; | 4689 return; |
| 4690 } | 4690 } |
| 4691 | 4691 |
| 4692 // Going to be using multiple times so get it in a register early | 4692 // Going to be using multiple times so get it in a register early |
| 4693 Variable *Comparison = legalizeToVar(Src0); | 4693 Variable *Comparison = legalizeToReg(Src0); |
| 4694 | 4694 |
| 4695 // A span is over the clusters | 4695 // A span is over the clusters |
| 4696 struct SearchSpan { | 4696 struct SearchSpan { |
| 4697 SearchSpan(SizeT Begin, SizeT Size, typename Traits::Insts::Label *Label) | 4697 SearchSpan(SizeT Begin, SizeT Size, typename Traits::Insts::Label *Label) |
| 4698 : Begin(Begin), Size(Size), Label(Label) {} | 4698 : Begin(Begin), Size(Size), Label(Label) {} |
| 4699 | 4699 |
| 4700 SizeT Begin; | 4700 SizeT Begin; |
| 4701 SizeT Size; | 4701 SizeT Size; |
| 4702 typename Traits::Insts::Label *Label; | 4702 typename Traits::Insts::Label *Label; |
| 4703 }; | 4703 }; |
| (...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4793 /// pcmpgtd, or cmpps (which produce sign extended results) to the result | 4793 /// pcmpgtd, or cmpps (which produce sign extended results) to the result |
| 4794 /// of the sext operation. | 4794 /// of the sext operation. |
| 4795 template <class Machine> | 4795 template <class Machine> |
| 4796 void TargetX86Base<Machine>::eliminateNextVectorSextInstruction( | 4796 void TargetX86Base<Machine>::eliminateNextVectorSextInstruction( |
| 4797 Variable *SignExtendedResult) { | 4797 Variable *SignExtendedResult) { |
| 4798 if (InstCast *NextCast = | 4798 if (InstCast *NextCast = |
| 4799 llvm::dyn_cast_or_null<InstCast>(Context.getNextInst())) { | 4799 llvm::dyn_cast_or_null<InstCast>(Context.getNextInst())) { |
| 4800 if (NextCast->getCastKind() == InstCast::Sext && | 4800 if (NextCast->getCastKind() == InstCast::Sext && |
| 4801 NextCast->getSrc(0) == SignExtendedResult) { | 4801 NextCast->getSrc(0) == SignExtendedResult) { |
| 4802 NextCast->setDeleted(); | 4802 NextCast->setDeleted(); |
| 4803 _movp(NextCast->getDest(), legalizeToVar(SignExtendedResult)); | 4803 _movp(NextCast->getDest(), legalizeToReg(SignExtendedResult)); |
| 4804 // Skip over the instruction. | 4804 // Skip over the instruction. |
| 4805 Context.advanceNext(); | 4805 Context.advanceNext(); |
| 4806 } | 4806 } |
| 4807 } | 4807 } |
| 4808 } | 4808 } |
| 4809 | 4809 |
| 4810 template <class Machine> | 4810 template <class Machine> |
| 4811 void TargetX86Base<Machine>::lowerUnreachable( | 4811 void TargetX86Base<Machine>::lowerUnreachable( |
| 4812 const InstUnreachable * /*Inst*/) { | 4812 const InstUnreachable * /*Inst*/) { |
| 4813 _ud2(); | 4813 _ud2(); |
| (...skipping 377 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 5191 assert(RegNum == Variable::NoRegister || Allowed == Legal_Reg); | 5191 assert(RegNum == Variable::NoRegister || Allowed == Legal_Reg); |
| 5192 | 5192 |
| 5193 if (auto Mem = llvm::dyn_cast<typename Traits::X86OperandMem>(From)) { | 5193 if (auto Mem = llvm::dyn_cast<typename Traits::X86OperandMem>(From)) { |
| 5194 // Before doing anything with a Mem operand, we need to ensure | 5194 // Before doing anything with a Mem operand, we need to ensure |
| 5195 // that the Base and Index components are in physical registers. | 5195 // that the Base and Index components are in physical registers. |
| 5196 Variable *Base = Mem->getBase(); | 5196 Variable *Base = Mem->getBase(); |
| 5197 Variable *Index = Mem->getIndex(); | 5197 Variable *Index = Mem->getIndex(); |
| 5198 Variable *RegBase = nullptr; | 5198 Variable *RegBase = nullptr; |
| 5199 Variable *RegIndex = nullptr; | 5199 Variable *RegIndex = nullptr; |
| 5200 if (Base) { | 5200 if (Base) { |
| 5201 RegBase = legalizeToVar(Base); | 5201 RegBase = legalizeToReg(Base); |
| 5202 } | 5202 } |
| 5203 if (Index) { | 5203 if (Index) { |
| 5204 RegIndex = legalizeToVar(Index); | 5204 RegIndex = legalizeToReg(Index); |
| 5205 } | 5205 } |
| 5206 if (Base != RegBase || Index != RegIndex) { | 5206 if (Base != RegBase || Index != RegIndex) { |
| 5207 Mem = Traits::X86OperandMem::create(Func, Ty, RegBase, Mem->getOffset(), | 5207 Mem = Traits::X86OperandMem::create(Func, Ty, RegBase, Mem->getOffset(), |
| 5208 RegIndex, Mem->getShift(), | 5208 RegIndex, Mem->getShift(), |
| 5209 Mem->getSegmentRegister()); | 5209 Mem->getSegmentRegister()); |
| 5210 } | 5210 } |
| 5211 | 5211 |
| 5212 // For all Memory Operands, we do randomization/pooling here | 5212 // For all Memory Operands, we do randomization/pooling here |
| 5213 From = randomizeOrPoolImmediate(Mem); | 5213 From = randomizeOrPoolImmediate(Mem); |
| 5214 | 5214 |
| (...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 5273 From = copyToReg(From, RegNum); | 5273 From = copyToReg(From, RegNum); |
| 5274 } | 5274 } |
| 5275 return From; | 5275 return From; |
| 5276 } | 5276 } |
| 5277 llvm_unreachable("Unhandled operand kind in legalize()"); | 5277 llvm_unreachable("Unhandled operand kind in legalize()"); |
| 5278 return From; | 5278 return From; |
| 5279 } | 5279 } |
| 5280 | 5280 |
| 5281 /// Provide a trivial wrapper to legalize() for this common usage. | 5281 /// Provide a trivial wrapper to legalize() for this common usage. |
| 5282 template <class Machine> | 5282 template <class Machine> |
| 5283 Variable *TargetX86Base<Machine>::legalizeToVar(Operand *From, int32_t RegNum) { | 5283 Variable *TargetX86Base<Machine>::legalizeToReg(Operand *From, int32_t RegNum) { |
| 5284 return llvm::cast<Variable>(legalize(From, Legal_Reg, RegNum)); | 5284 return llvm::cast<Variable>(legalize(From, Legal_Reg, RegNum)); |
| 5285 } | 5285 } |
| 5286 | 5286 |
| 5287 /// Legalize undef values to concrete values. | 5287 /// Legalize undef values to concrete values. |
| 5288 template <class Machine> | 5288 template <class Machine> |
| 5289 Operand *TargetX86Base<Machine>::legalizeUndef(Operand *From, int32_t RegNum) { | 5289 Operand *TargetX86Base<Machine>::legalizeUndef(Operand *From, int32_t RegNum) { |
| 5290 Type Ty = From->getType(); | 5290 Type Ty = From->getType(); |
| 5291 if (llvm::isa<ConstantUndef>(From)) { | 5291 if (llvm::isa<ConstantUndef>(From)) { |
| 5292 // Lower undefs to zero. Another option is to lower undefs to an | 5292 // Lower undefs to zero. Another option is to lower undefs to an |
| 5293 // uninitialized register; however, using an uninitialized register | 5293 // uninitialized register; however, using an uninitialized register |
| (...skipping 325 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 5619 } | 5619 } |
| 5620 // the offset is not eligible for blinding or pooling, return the original | 5620 // the offset is not eligible for blinding or pooling, return the original |
| 5621 // mem operand | 5621 // mem operand |
| 5622 return MemOperand; | 5622 return MemOperand; |
| 5623 } | 5623 } |
| 5624 | 5624 |
| 5625 } // end of namespace X86Internal | 5625 } // end of namespace X86Internal |
| 5626 } // end of namespace Ice | 5626 } // end of namespace Ice |
| 5627 | 5627 |
| 5628 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H | 5628 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H |
| OLD | NEW |