| OLD | NEW |
| 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// | 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// |
| 2 // | 2 // |
| 3 // The Subzero Code Generator | 3 // The Subzero Code Generator |
| 4 // | 4 // |
| 5 // This file is distributed under the University of Illinois Open Source | 5 // This file is distributed under the University of Illinois Open Source |
| 6 // License. See LICENSE.TXT for details. | 6 // License. See LICENSE.TXT for details. |
| 7 // | 7 // |
| 8 //===----------------------------------------------------------------------===// | 8 //===----------------------------------------------------------------------===// |
| 9 /// | 9 /// |
| 10 /// \file | 10 /// \file |
| (...skipping 1361 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1372 } | 1372 } |
| 1373 | 1373 |
| 1374 template <typename TraitsType> | 1374 template <typename TraitsType> |
| 1375 llvm::SmallBitVector | 1375 llvm::SmallBitVector |
| 1376 TargetX86Base<TraitsType>::getRegisterSet(RegSetMask Include, | 1376 TargetX86Base<TraitsType>::getRegisterSet(RegSetMask Include, |
| 1377 RegSetMask Exclude) const { | 1377 RegSetMask Exclude) const { |
| 1378 return Traits::getRegisterSet(Ctx->getFlags(), Include, Exclude); | 1378 return Traits::getRegisterSet(Ctx->getFlags(), Include, Exclude); |
| 1379 } | 1379 } |
| 1380 | 1380 |
| 1381 template <typename TraitsType> | 1381 template <typename TraitsType> |
| 1382 void TargetX86Base<TraitsType>::lowerAlloca(const InstAlloca *Inst) { | 1382 void TargetX86Base<TraitsType>::lowerAlloca(const InstAlloca *Instr) { |
| 1383 // Conservatively require the stack to be aligned. Some stack adjustment | 1383 // Conservatively require the stack to be aligned. Some stack adjustment |
| 1384 // operations implemented below assume that the stack is aligned before the | 1384 // operations implemented below assume that the stack is aligned before the |
| 1385 // alloca. All the alloca code ensures that the stack alignment is preserved | 1385 // alloca. All the alloca code ensures that the stack alignment is preserved |
| 1386 // after the alloca. The stack alignment restriction can be relaxed in some | 1386 // after the alloca. The stack alignment restriction can be relaxed in some |
| 1387 // cases. | 1387 // cases. |
| 1388 NeedsStackAlignment = true; | 1388 NeedsStackAlignment = true; |
| 1389 | 1389 |
| 1390 // For default align=0, set it to the real value 1, to avoid any | 1390 // For default align=0, set it to the real value 1, to avoid any |
| 1391 // bit-manipulation problems below. | 1391 // bit-manipulation problems below. |
| 1392 const uint32_t AlignmentParam = std::max(1u, Inst->getAlignInBytes()); | 1392 const uint32_t AlignmentParam = std::max(1u, Instr->getAlignInBytes()); |
| 1393 | 1393 |
| 1394 // LLVM enforces power of 2 alignment. | 1394 // LLVM enforces power of 2 alignment. |
| 1395 assert(llvm::isPowerOf2_32(AlignmentParam)); | 1395 assert(llvm::isPowerOf2_32(AlignmentParam)); |
| 1396 assert(llvm::isPowerOf2_32(Traits::X86_STACK_ALIGNMENT_BYTES)); | 1396 assert(llvm::isPowerOf2_32(Traits::X86_STACK_ALIGNMENT_BYTES)); |
| 1397 | 1397 |
| 1398 const uint32_t Alignment = | 1398 const uint32_t Alignment = |
| 1399 std::max(AlignmentParam, Traits::X86_STACK_ALIGNMENT_BYTES); | 1399 std::max(AlignmentParam, Traits::X86_STACK_ALIGNMENT_BYTES); |
| 1400 const bool OverAligned = Alignment > Traits::X86_STACK_ALIGNMENT_BYTES; | 1400 const bool OverAligned = Alignment > Traits::X86_STACK_ALIGNMENT_BYTES; |
| 1401 const bool OptM1 = Ctx->getFlags().getOptLevel() == Opt_m1; | 1401 const bool OptM1 = Ctx->getFlags().getOptLevel() == Opt_m1; |
| 1402 const bool AllocaWithKnownOffset = Inst->getKnownFrameOffset(); | 1402 const bool AllocaWithKnownOffset = Instr->getKnownFrameOffset(); |
| 1403 const bool UseFramePointer = | 1403 const bool UseFramePointer = |
| 1404 hasFramePointer() || OverAligned || !AllocaWithKnownOffset || OptM1; | 1404 hasFramePointer() || OverAligned || !AllocaWithKnownOffset || OptM1; |
| 1405 | 1405 |
| 1406 if (UseFramePointer) | 1406 if (UseFramePointer) |
| 1407 setHasFramePointer(); | 1407 setHasFramePointer(); |
| 1408 | 1408 |
| 1409 Variable *esp = getPhysicalRegister(getStackReg(), Traits::WordType); | 1409 Variable *esp = getPhysicalRegister(getStackReg(), Traits::WordType); |
| 1410 if (OverAligned) { | 1410 if (OverAligned) { |
| 1411 _and(esp, Ctx->getConstantInt32(-Alignment)); | 1411 _and(esp, Ctx->getConstantInt32(-Alignment)); |
| 1412 } | 1412 } |
| 1413 | 1413 |
| 1414 Variable *Dest = Inst->getDest(); | 1414 Variable *Dest = Instr->getDest(); |
| 1415 Operand *TotalSize = legalize(Inst->getSizeInBytes()); | 1415 Operand *TotalSize = legalize(Instr->getSizeInBytes()); |
| 1416 | 1416 |
| 1417 if (const auto *ConstantTotalSize = | 1417 if (const auto *ConstantTotalSize = |
| 1418 llvm::dyn_cast<ConstantInteger32>(TotalSize)) { | 1418 llvm::dyn_cast<ConstantInteger32>(TotalSize)) { |
| 1419 const uint32_t Value = | 1419 const uint32_t Value = |
| 1420 Utils::applyAlignment(ConstantTotalSize->getValue(), Alignment); | 1420 Utils::applyAlignment(ConstantTotalSize->getValue(), Alignment); |
| 1421 if (UseFramePointer) { | 1421 if (UseFramePointer) { |
| 1422 _sub_sp(Ctx->getConstantInt32(Value)); | 1422 _sub_sp(Ctx->getConstantInt32(Value)); |
| 1423 } else { | 1423 } else { |
| 1424 // If we don't need a Frame Pointer, this alloca has a known offset to the | 1424 // If we don't need a Frame Pointer, this alloca has a known offset to the |
| 1425 // stack pointer. We don't need adjust the stack pointer, nor assign any | 1425 // stack pointer. We don't need adjust the stack pointer, nor assign any |
| (...skipping 399 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1825 // L1: | 1825 // L1: |
| 1826 // a.lo = t2 | 1826 // a.lo = t2 |
| 1827 // a.hi = t3 | 1827 // a.hi = t3 |
| 1828 Context.insert(Label); | 1828 Context.insert(Label); |
| 1829 _mov(DestLo, T_2); | 1829 _mov(DestLo, T_2); |
| 1830 _mov(DestHi, T_3); | 1830 _mov(DestHi, T_3); |
| 1831 } | 1831 } |
| 1832 } | 1832 } |
| 1833 | 1833 |
| 1834 template <typename TraitsType> | 1834 template <typename TraitsType> |
| 1835 void TargetX86Base<TraitsType>::lowerArithmetic(const InstArithmetic *Inst) { | 1835 void TargetX86Base<TraitsType>::lowerArithmetic(const InstArithmetic *Instr) { |
| 1836 Variable *Dest = Inst->getDest(); | 1836 Variable *Dest = Instr->getDest(); |
| 1837 if (Dest->isRematerializable()) { | 1837 if (Dest->isRematerializable()) { |
| 1838 Context.insert<InstFakeDef>(Dest); | 1838 Context.insert<InstFakeDef>(Dest); |
| 1839 return; | 1839 return; |
| 1840 } | 1840 } |
| 1841 Type Ty = Dest->getType(); | 1841 Type Ty = Dest->getType(); |
| 1842 Operand *Src0 = legalize(Inst->getSrc(0)); | 1842 Operand *Src0 = legalize(Instr->getSrc(0)); |
| 1843 Operand *Src1 = legalize(Inst->getSrc(1)); | 1843 Operand *Src1 = legalize(Instr->getSrc(1)); |
| 1844 if (Inst->isCommutative()) { | 1844 if (Instr->isCommutative()) { |
| 1845 uint32_t SwapCount = 0; | 1845 uint32_t SwapCount = 0; |
| 1846 if (!llvm::isa<Variable>(Src0) && llvm::isa<Variable>(Src1)) { | 1846 if (!llvm::isa<Variable>(Src0) && llvm::isa<Variable>(Src1)) { |
| 1847 std::swap(Src0, Src1); | 1847 std::swap(Src0, Src1); |
| 1848 ++SwapCount; | 1848 ++SwapCount; |
| 1849 } | 1849 } |
| 1850 if (llvm::isa<Constant>(Src0) && !llvm::isa<Constant>(Src1)) { | 1850 if (llvm::isa<Constant>(Src0) && !llvm::isa<Constant>(Src1)) { |
| 1851 std::swap(Src0, Src1); | 1851 std::swap(Src0, Src1); |
| 1852 ++SwapCount; | 1852 ++SwapCount; |
| 1853 } | 1853 } |
| 1854 // Improve two-address code patterns by avoiding a copy to the dest | 1854 // Improve two-address code patterns by avoiding a copy to the dest |
| 1855 // register when one of the source operands ends its lifetime here. | 1855 // register when one of the source operands ends its lifetime here. |
| 1856 if (!Inst->isLastUse(Src0) && Inst->isLastUse(Src1)) { | 1856 if (!Instr->isLastUse(Src0) && Instr->isLastUse(Src1)) { |
| 1857 std::swap(Src0, Src1); | 1857 std::swap(Src0, Src1); |
| 1858 ++SwapCount; | 1858 ++SwapCount; |
| 1859 } | 1859 } |
| 1860 assert(SwapCount <= 1); | 1860 assert(SwapCount <= 1); |
| 1861 (void)SwapCount; | 1861 (void)SwapCount; |
| 1862 } | 1862 } |
| 1863 if (!Traits::Is64Bit && Ty == IceType_i64) { | 1863 if (!Traits::Is64Bit && Ty == IceType_i64) { |
| 1864 // These x86-32 helper-call-involved instructions are lowered in this | 1864 // These x86-32 helper-call-involved instructions are lowered in this |
| 1865 // separate switch. This is because loOperand() and hiOperand() may insert | 1865 // separate switch. This is because loOperand() and hiOperand() may insert |
| 1866 // redundant instructions for constant blinding and pooling. Such redundant | 1866 // redundant instructions for constant blinding and pooling. Such redundant |
| 1867 // instructions will fail liveness analysis under -Om1 setting. And, | 1867 // instructions will fail liveness analysis under -Om1 setting. And, |
| 1868 // actually these arguments do not need to be processed with loOperand() | 1868 // actually these arguments do not need to be processed with loOperand() |
| 1869 // and hiOperand() to be used. | 1869 // and hiOperand() to be used. |
| 1870 switch (Inst->getOp()) { | 1870 switch (Instr->getOp()) { |
| 1871 case InstArithmetic::Udiv: | 1871 case InstArithmetic::Udiv: |
| 1872 case InstArithmetic::Sdiv: | 1872 case InstArithmetic::Sdiv: |
| 1873 case InstArithmetic::Urem: | 1873 case InstArithmetic::Urem: |
| 1874 case InstArithmetic::Srem: | 1874 case InstArithmetic::Srem: |
| 1875 llvm::report_fatal_error("Helper call was expected"); | 1875 llvm::report_fatal_error("Helper call was expected"); |
| 1876 return; | 1876 return; |
| 1877 default: | 1877 default: |
| 1878 break; | 1878 break; |
| 1879 } | 1879 } |
| 1880 | 1880 |
| 1881 auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 1881 auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
| 1882 auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 1882 auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
| 1883 Operand *Src0Lo = loOperand(Src0); | 1883 Operand *Src0Lo = loOperand(Src0); |
| 1884 Operand *Src0Hi = hiOperand(Src0); | 1884 Operand *Src0Hi = hiOperand(Src0); |
| 1885 Operand *Src1Lo = loOperand(Src1); | 1885 Operand *Src1Lo = loOperand(Src1); |
| 1886 Operand *Src1Hi = hiOperand(Src1); | 1886 Operand *Src1Hi = hiOperand(Src1); |
| 1887 Variable *T_Lo = nullptr, *T_Hi = nullptr; | 1887 Variable *T_Lo = nullptr, *T_Hi = nullptr; |
| 1888 switch (Inst->getOp()) { | 1888 switch (Instr->getOp()) { |
| 1889 case InstArithmetic::_num: | 1889 case InstArithmetic::_num: |
| 1890 llvm_unreachable("Unknown arithmetic operator"); | 1890 llvm_unreachable("Unknown arithmetic operator"); |
| 1891 break; | 1891 break; |
| 1892 case InstArithmetic::Add: | 1892 case InstArithmetic::Add: |
| 1893 _mov(T_Lo, Src0Lo); | 1893 _mov(T_Lo, Src0Lo); |
| 1894 _add(T_Lo, Src1Lo); | 1894 _add(T_Lo, Src1Lo); |
| 1895 _mov(DestLo, T_Lo); | 1895 _mov(DestLo, T_Lo); |
| 1896 _mov(T_Hi, Src0Hi); | 1896 _mov(T_Hi, Src0Hi); |
| 1897 _adc(T_Hi, Src1Hi); | 1897 _adc(T_Hi, Src1Hi); |
| 1898 _mov(DestHi, T_Hi); | 1898 _mov(DestHi, T_Hi); |
| (...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1955 _mov(DestLo, T_4Lo); | 1955 _mov(DestLo, T_4Lo); |
| 1956 _add(T_4Hi, T_1); | 1956 _add(T_4Hi, T_1); |
| 1957 _mov(T_2, Src1Hi); | 1957 _mov(T_2, Src1Hi); |
| 1958 _imul(T_2, Src0Lo); | 1958 _imul(T_2, Src0Lo); |
| 1959 _add(T_4Hi, T_2); | 1959 _add(T_4Hi, T_2); |
| 1960 _mov(DestHi, T_4Hi); | 1960 _mov(DestHi, T_4Hi); |
| 1961 } break; | 1961 } break; |
| 1962 case InstArithmetic::Shl: | 1962 case InstArithmetic::Shl: |
| 1963 case InstArithmetic::Lshr: | 1963 case InstArithmetic::Lshr: |
| 1964 case InstArithmetic::Ashr: | 1964 case InstArithmetic::Ashr: |
| 1965 lowerShift64(Inst->getOp(), Src0Lo, Src0Hi, Src1Lo, DestLo, DestHi); | 1965 lowerShift64(Instr->getOp(), Src0Lo, Src0Hi, Src1Lo, DestLo, DestHi); |
| 1966 break; | 1966 break; |
| 1967 case InstArithmetic::Fadd: | 1967 case InstArithmetic::Fadd: |
| 1968 case InstArithmetic::Fsub: | 1968 case InstArithmetic::Fsub: |
| 1969 case InstArithmetic::Fmul: | 1969 case InstArithmetic::Fmul: |
| 1970 case InstArithmetic::Fdiv: | 1970 case InstArithmetic::Fdiv: |
| 1971 case InstArithmetic::Frem: | 1971 case InstArithmetic::Frem: |
| 1972 llvm_unreachable("FP instruction with i64 type"); | 1972 llvm_unreachable("FP instruction with i64 type"); |
| 1973 break; | 1973 break; |
| 1974 case InstArithmetic::Udiv: | 1974 case InstArithmetic::Udiv: |
| 1975 case InstArithmetic::Sdiv: | 1975 case InstArithmetic::Sdiv: |
| 1976 case InstArithmetic::Urem: | 1976 case InstArithmetic::Urem: |
| 1977 case InstArithmetic::Srem: | 1977 case InstArithmetic::Srem: |
| 1978 llvm_unreachable("Call-helper-involved instruction for i64 type \ | 1978 llvm_unreachable("Call-helper-involved instruction for i64 type \ |
| 1979 should have already been handled before"); | 1979 should have already been handled before"); |
| 1980 break; | 1980 break; |
| 1981 } | 1981 } |
| 1982 return; | 1982 return; |
| 1983 } | 1983 } |
| 1984 if (isVectorType(Ty)) { | 1984 if (isVectorType(Ty)) { |
| 1985 // TODO: Trap on integer divide and integer modulo by zero. See: | 1985 // TODO: Trap on integer divide and integer modulo by zero. See: |
| 1986 // https://code.google.com/p/nativeclient/issues/detail?id=3899 | 1986 // https://code.google.com/p/nativeclient/issues/detail?id=3899 |
| 1987 if (llvm::isa<X86OperandMem>(Src1)) | 1987 if (llvm::isa<X86OperandMem>(Src1)) |
| 1988 Src1 = legalizeToReg(Src1); | 1988 Src1 = legalizeToReg(Src1); |
| 1989 switch (Inst->getOp()) { | 1989 switch (Instr->getOp()) { |
| 1990 case InstArithmetic::_num: | 1990 case InstArithmetic::_num: |
| 1991 llvm_unreachable("Unknown arithmetic operator"); | 1991 llvm_unreachable("Unknown arithmetic operator"); |
| 1992 break; | 1992 break; |
| 1993 case InstArithmetic::Add: { | 1993 case InstArithmetic::Add: { |
| 1994 Variable *T = makeReg(Ty); | 1994 Variable *T = makeReg(Ty); |
| 1995 _movp(T, Src0); | 1995 _movp(T, Src0); |
| 1996 _padd(T, Src1); | 1996 _padd(T, Src1); |
| 1997 _movp(Dest, T); | 1997 _movp(Dest, T); |
| 1998 } break; | 1998 } break; |
| 1999 case InstArithmetic::And: { | 1999 case InstArithmetic::And: { |
| (...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2107 _movp(Dest, T); | 2107 _movp(Dest, T); |
| 2108 } break; | 2108 } break; |
| 2109 case InstArithmetic::Frem: | 2109 case InstArithmetic::Frem: |
| 2110 llvm::report_fatal_error("Scalarized operation was expected"); | 2110 llvm::report_fatal_error("Scalarized operation was expected"); |
| 2111 break; | 2111 break; |
| 2112 } | 2112 } |
| 2113 return; | 2113 return; |
| 2114 } | 2114 } |
| 2115 Variable *T_edx = nullptr; | 2115 Variable *T_edx = nullptr; |
| 2116 Variable *T = nullptr; | 2116 Variable *T = nullptr; |
| 2117 switch (Inst->getOp()) { | 2117 switch (Instr->getOp()) { |
| 2118 case InstArithmetic::_num: | 2118 case InstArithmetic::_num: |
| 2119 llvm_unreachable("Unknown arithmetic operator"); | 2119 llvm_unreachable("Unknown arithmetic operator"); |
| 2120 break; | 2120 break; |
| 2121 case InstArithmetic::Add: | 2121 case InstArithmetic::Add: |
| 2122 _mov(T, Src0); | 2122 _mov(T, Src0); |
| 2123 _add(T, Src1); | 2123 _add(T, Src1); |
| 2124 _mov(Dest, T); | 2124 _mov(Dest, T); |
| 2125 break; | 2125 break; |
| 2126 case InstArithmetic::And: | 2126 case InstArithmetic::And: |
| 2127 _mov(T, Src0); | 2127 _mov(T, Src0); |
| (...skipping 269 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2397 _divss(T, Src1); | 2397 _divss(T, Src1); |
| 2398 _mov(Dest, T); | 2398 _mov(Dest, T); |
| 2399 break; | 2399 break; |
| 2400 case InstArithmetic::Frem: | 2400 case InstArithmetic::Frem: |
| 2401 llvm::report_fatal_error("Helper call was expected"); | 2401 llvm::report_fatal_error("Helper call was expected"); |
| 2402 break; | 2402 break; |
| 2403 } | 2403 } |
| 2404 } | 2404 } |
| 2405 | 2405 |
| 2406 template <typename TraitsType> | 2406 template <typename TraitsType> |
| 2407 void TargetX86Base<TraitsType>::lowerAssign(const InstAssign *Inst) { | 2407 void TargetX86Base<TraitsType>::lowerAssign(const InstAssign *Instr) { |
| 2408 Variable *Dest = Inst->getDest(); | 2408 Variable *Dest = Instr->getDest(); |
| 2409 if (Dest->isRematerializable()) { | 2409 if (Dest->isRematerializable()) { |
| 2410 Context.insert<InstFakeDef>(Dest); | 2410 Context.insert<InstFakeDef>(Dest); |
| 2411 return; | 2411 return; |
| 2412 } | 2412 } |
| 2413 Operand *Src = Inst->getSrc(0); | 2413 Operand *Src = Instr->getSrc(0); |
| 2414 assert(Dest->getType() == Src->getType()); | 2414 assert(Dest->getType() == Src->getType()); |
| 2415 lowerMove(Dest, Src, false); | 2415 lowerMove(Dest, Src, false); |
| 2416 } | 2416 } |
| 2417 | 2417 |
| 2418 template <typename TraitsType> | 2418 template <typename TraitsType> |
| 2419 void TargetX86Base<TraitsType>::lowerBr(const InstBr *Br) { | 2419 void TargetX86Base<TraitsType>::lowerBr(const InstBr *Br) { |
| 2420 if (Br->isUnconditional()) { | 2420 if (Br->isUnconditional()) { |
| 2421 _br(Br->getTargetUnconditional()); | 2421 _br(Br->getTargetUnconditional()); |
| 2422 return; | 2422 return; |
| 2423 } | 2423 } |
| (...skipping 217 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2641 Variable *DestHi = Dest64On32->getHi(); | 2641 Variable *DestHi = Dest64On32->getHi(); |
| 2642 _mov(DestLo, ReturnReg); | 2642 _mov(DestLo, ReturnReg); |
| 2643 _mov(DestHi, ReturnRegHi); | 2643 _mov(DestHi, ReturnRegHi); |
| 2644 } else { | 2644 } else { |
| 2645 _mov(Dest, ReturnReg); | 2645 _mov(Dest, ReturnReg); |
| 2646 } | 2646 } |
| 2647 } | 2647 } |
| 2648 } | 2648 } |
| 2649 | 2649 |
| 2650 template <typename TraitsType> | 2650 template <typename TraitsType> |
| 2651 void TargetX86Base<TraitsType>::lowerCast(const InstCast *Inst) { | 2651 void TargetX86Base<TraitsType>::lowerCast(const InstCast *Instr) { |
| 2652 // a = cast(b) ==> t=cast(b); a=t; (link t->b, link a->t, no overlap) | 2652 // a = cast(b) ==> t=cast(b); a=t; (link t->b, link a->t, no overlap) |
| 2653 InstCast::OpKind CastKind = Inst->getCastKind(); | 2653 InstCast::OpKind CastKind = Instr->getCastKind(); |
| 2654 Variable *Dest = Inst->getDest(); | 2654 Variable *Dest = Instr->getDest(); |
| 2655 Type DestTy = Dest->getType(); | 2655 Type DestTy = Dest->getType(); |
| 2656 switch (CastKind) { | 2656 switch (CastKind) { |
| 2657 default: | 2657 default: |
| 2658 Func->setError("Cast type not supported"); | 2658 Func->setError("Cast type not supported"); |
| 2659 return; | 2659 return; |
| 2660 case InstCast::Sext: { | 2660 case InstCast::Sext: { |
| 2661 // Src0RM is the source operand legalized to physical register or memory, | 2661 // Src0RM is the source operand legalized to physical register or memory, |
| 2662 // but not immediate, since the relevant x86 native instructions don't | 2662 // but not immediate, since the relevant x86 native instructions don't |
| 2663 // allow an immediate operand. If the operand is an immediate, we could | 2663 // allow an immediate operand. If the operand is an immediate, we could |
| 2664 // consider computing the strength-reduced result at translation time, but | 2664 // consider computing the strength-reduced result at translation time, but |
| 2665 // we're unlikely to see something like that in the bitcode that the | 2665 // we're unlikely to see something like that in the bitcode that the |
| 2666 // optimizer wouldn't have already taken care of. | 2666 // optimizer wouldn't have already taken care of. |
| 2667 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); | 2667 Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); |
| 2668 if (isVectorType(DestTy)) { | 2668 if (isVectorType(DestTy)) { |
| 2669 if (DestTy == IceType_v16i8) { | 2669 if (DestTy == IceType_v16i8) { |
| 2670 // onemask = materialize(1,1,...); dst = (src & onemask) > 0 | 2670 // onemask = materialize(1,1,...); dst = (src & onemask) > 0 |
| 2671 Variable *OneMask = makeVectorOfOnes(DestTy); | 2671 Variable *OneMask = makeVectorOfOnes(DestTy); |
| 2672 Variable *T = makeReg(DestTy); | 2672 Variable *T = makeReg(DestTy); |
| 2673 _movp(T, Src0RM); | 2673 _movp(T, Src0RM); |
| 2674 _pand(T, OneMask); | 2674 _pand(T, OneMask); |
| 2675 Variable *Zeros = makeVectorOfZeros(DestTy); | 2675 Variable *Zeros = makeVectorOfZeros(DestTy); |
| 2676 _pcmpgt(T, Zeros); | 2676 _pcmpgt(T, Zeros); |
| 2677 _movp(Dest, T); | 2677 _movp(Dest, T); |
| (...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2729 _mov(Dest, T); | 2729 _mov(Dest, T); |
| 2730 } else { | 2730 } else { |
| 2731 // t1 = movsx src; dst = t1 | 2731 // t1 = movsx src; dst = t1 |
| 2732 Variable *T = makeReg(DestTy); | 2732 Variable *T = makeReg(DestTy); |
| 2733 _movsx(T, Src0RM); | 2733 _movsx(T, Src0RM); |
| 2734 _mov(Dest, T); | 2734 _mov(Dest, T); |
| 2735 } | 2735 } |
| 2736 break; | 2736 break; |
| 2737 } | 2737 } |
| 2738 case InstCast::Zext: { | 2738 case InstCast::Zext: { |
| 2739 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); | 2739 Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); |
| 2740 if (isVectorType(DestTy)) { | 2740 if (isVectorType(DestTy)) { |
| 2741 // onemask = materialize(1,1,...); dest = onemask & src | 2741 // onemask = materialize(1,1,...); dest = onemask & src |
| 2742 Variable *OneMask = makeVectorOfOnes(DestTy); | 2742 Variable *OneMask = makeVectorOfOnes(DestTy); |
| 2743 Variable *T = makeReg(DestTy); | 2743 Variable *T = makeReg(DestTy); |
| 2744 _movp(T, Src0RM); | 2744 _movp(T, Src0RM); |
| 2745 _pand(T, OneMask); | 2745 _pand(T, OneMask); |
| 2746 _movp(Dest, T); | 2746 _movp(Dest, T); |
| 2747 } else if (!Traits::Is64Bit && DestTy == IceType_i64) { | 2747 } else if (!Traits::Is64Bit && DestTy == IceType_i64) { |
| 2748 // t1=movzx src; dst.lo=t1; dst.hi=0 | 2748 // t1=movzx src; dst.lo=t1; dst.hi=0 |
| 2749 Constant *Zero = Ctx->getConstantZero(IceType_i32); | 2749 Constant *Zero = Ctx->getConstantZero(IceType_i32); |
| (...skipping 26 matching lines...) Expand all Loading... |
| 2776 // t1 = movzx src; dst = t1 | 2776 // t1 = movzx src; dst = t1 |
| 2777 Variable *T = makeReg(DestTy); | 2777 Variable *T = makeReg(DestTy); |
| 2778 _movzx(T, Src0RM); | 2778 _movzx(T, Src0RM); |
| 2779 _mov(Dest, T); | 2779 _mov(Dest, T); |
| 2780 } | 2780 } |
| 2781 break; | 2781 break; |
| 2782 } | 2782 } |
| 2783 case InstCast::Trunc: { | 2783 case InstCast::Trunc: { |
| 2784 if (isVectorType(DestTy)) { | 2784 if (isVectorType(DestTy)) { |
| 2785 // onemask = materialize(1,1,...); dst = src & onemask | 2785 // onemask = materialize(1,1,...); dst = src & onemask |
| 2786 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); | 2786 Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); |
| 2787 Type Src0Ty = Src0RM->getType(); | 2787 Type Src0Ty = Src0RM->getType(); |
| 2788 Variable *OneMask = makeVectorOfOnes(Src0Ty); | 2788 Variable *OneMask = makeVectorOfOnes(Src0Ty); |
| 2789 Variable *T = makeReg(DestTy); | 2789 Variable *T = makeReg(DestTy); |
| 2790 _movp(T, Src0RM); | 2790 _movp(T, Src0RM); |
| 2791 _pand(T, OneMask); | 2791 _pand(T, OneMask); |
| 2792 _movp(Dest, T); | 2792 _movp(Dest, T); |
| 2793 } else if (DestTy == IceType_i1 || DestTy == IceType_i8) { | 2793 } else if (DestTy == IceType_i1 || DestTy == IceType_i8) { |
| 2794 // Make sure we truncate from and into valid registers. | 2794 // Make sure we truncate from and into valid registers. |
| 2795 Operand *Src0 = legalizeUndef(Inst->getSrc(0)); | 2795 Operand *Src0 = legalizeUndef(Instr->getSrc(0)); |
| 2796 if (!Traits::Is64Bit && Src0->getType() == IceType_i64) | 2796 if (!Traits::Is64Bit && Src0->getType() == IceType_i64) |
| 2797 Src0 = loOperand(Src0); | 2797 Src0 = loOperand(Src0); |
| 2798 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); | 2798 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); |
| 2799 Variable *T = copyToReg8(Src0RM); | 2799 Variable *T = copyToReg8(Src0RM); |
| 2800 if (DestTy == IceType_i1) | 2800 if (DestTy == IceType_i1) |
| 2801 _and(T, Ctx->getConstantInt1(1)); | 2801 _and(T, Ctx->getConstantInt1(1)); |
| 2802 _mov(Dest, T); | 2802 _mov(Dest, T); |
| 2803 } else { | 2803 } else { |
| 2804 Operand *Src0 = legalizeUndef(Inst->getSrc(0)); | 2804 Operand *Src0 = legalizeUndef(Instr->getSrc(0)); |
| 2805 if (!Traits::Is64Bit && Src0->getType() == IceType_i64) | 2805 if (!Traits::Is64Bit && Src0->getType() == IceType_i64) |
| 2806 Src0 = loOperand(Src0); | 2806 Src0 = loOperand(Src0); |
| 2807 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); | 2807 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); |
| 2808 // t1 = trunc Src0RM; Dest = t1 | 2808 // t1 = trunc Src0RM; Dest = t1 |
| 2809 Variable *T = makeReg(DestTy); | 2809 Variable *T = makeReg(DestTy); |
| 2810 _mov(T, Src0RM); | 2810 _mov(T, Src0RM); |
| 2811 _mov(Dest, T); | 2811 _mov(Dest, T); |
| 2812 } | 2812 } |
| 2813 break; | 2813 break; |
| 2814 } | 2814 } |
| 2815 case InstCast::Fptrunc: | 2815 case InstCast::Fptrunc: |
| 2816 case InstCast::Fpext: { | 2816 case InstCast::Fpext: { |
| 2817 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); | 2817 Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); |
| 2818 // t1 = cvt Src0RM; Dest = t1 | 2818 // t1 = cvt Src0RM; Dest = t1 |
| 2819 Variable *T = makeReg(DestTy); | 2819 Variable *T = makeReg(DestTy); |
| 2820 _cvt(T, Src0RM, Traits::Insts::Cvt::Float2float); | 2820 _cvt(T, Src0RM, Traits::Insts::Cvt::Float2float); |
| 2821 _mov(Dest, T); | 2821 _mov(Dest, T); |
| 2822 break; | 2822 break; |
| 2823 } | 2823 } |
| 2824 case InstCast::Fptosi: | 2824 case InstCast::Fptosi: |
| 2825 if (isVectorType(DestTy)) { | 2825 if (isVectorType(DestTy)) { |
| 2826 assert(DestTy == IceType_v4i32 && | 2826 assert(DestTy == IceType_v4i32 && |
| 2827 Inst->getSrc(0)->getType() == IceType_v4f32); | 2827 Instr->getSrc(0)->getType() == IceType_v4f32); |
| 2828 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); | 2828 Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); |
| 2829 if (llvm::isa<X86OperandMem>(Src0RM)) | 2829 if (llvm::isa<X86OperandMem>(Src0RM)) |
| 2830 Src0RM = legalizeToReg(Src0RM); | 2830 Src0RM = legalizeToReg(Src0RM); |
| 2831 Variable *T = makeReg(DestTy); | 2831 Variable *T = makeReg(DestTy); |
| 2832 _cvt(T, Src0RM, Traits::Insts::Cvt::Tps2dq); | 2832 _cvt(T, Src0RM, Traits::Insts::Cvt::Tps2dq); |
| 2833 _movp(Dest, T); | 2833 _movp(Dest, T); |
| 2834 } else if (!Traits::Is64Bit && DestTy == IceType_i64) { | 2834 } else if (!Traits::Is64Bit && DestTy == IceType_i64) { |
| 2835 llvm::report_fatal_error("Helper call was expected"); | 2835 llvm::report_fatal_error("Helper call was expected"); |
| 2836 } else { | 2836 } else { |
| 2837 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); | 2837 Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); |
| 2838 // t1.i32 = cvt Src0RM; t2.dest_type = t1; Dest = t2.dest_type | 2838 // t1.i32 = cvt Src0RM; t2.dest_type = t1; Dest = t2.dest_type |
| 2839 Variable *T_1 = nullptr; | 2839 Variable *T_1 = nullptr; |
| 2840 if (Traits::Is64Bit && DestTy == IceType_i64) { | 2840 if (Traits::Is64Bit && DestTy == IceType_i64) { |
| 2841 T_1 = makeReg(IceType_i64); | 2841 T_1 = makeReg(IceType_i64); |
| 2842 } else { | 2842 } else { |
| 2843 assert(DestTy != IceType_i64); | 2843 assert(DestTy != IceType_i64); |
| 2844 T_1 = makeReg(IceType_i32); | 2844 T_1 = makeReg(IceType_i32); |
| 2845 } | 2845 } |
| 2846 // cvt() requires its integer argument to be a GPR. | 2846 // cvt() requires its integer argument to be a GPR. |
| 2847 Variable *T_2 = makeReg(DestTy); | 2847 Variable *T_2 = makeReg(DestTy); |
| 2848 if (isByteSizedType(DestTy)) { | 2848 if (isByteSizedType(DestTy)) { |
| 2849 assert(T_1->getType() == IceType_i32); | 2849 assert(T_1->getType() == IceType_i32); |
| 2850 T_1->setRegClass(RCX86_Is32To8); | 2850 T_1->setRegClass(RCX86_Is32To8); |
| 2851 T_2->setRegClass(RCX86_IsTrunc8Rcvr); | 2851 T_2->setRegClass(RCX86_IsTrunc8Rcvr); |
| 2852 } | 2852 } |
| 2853 _cvt(T_1, Src0RM, Traits::Insts::Cvt::Tss2si); | 2853 _cvt(T_1, Src0RM, Traits::Insts::Cvt::Tss2si); |
| 2854 _mov(T_2, T_1); // T_1 and T_2 may have different integer types | 2854 _mov(T_2, T_1); // T_1 and T_2 may have different integer types |
| 2855 if (DestTy == IceType_i1) | 2855 if (DestTy == IceType_i1) |
| 2856 _and(T_2, Ctx->getConstantInt1(1)); | 2856 _and(T_2, Ctx->getConstantInt1(1)); |
| 2857 _mov(Dest, T_2); | 2857 _mov(Dest, T_2); |
| 2858 } | 2858 } |
| 2859 break; | 2859 break; |
| 2860 case InstCast::Fptoui: | 2860 case InstCast::Fptoui: |
| 2861 if (isVectorType(DestTy)) { | 2861 if (isVectorType(DestTy)) { |
| 2862 llvm::report_fatal_error("Helper call was expected"); | 2862 llvm::report_fatal_error("Helper call was expected"); |
| 2863 } else if (DestTy == IceType_i64 || | 2863 } else if (DestTy == IceType_i64 || |
| 2864 (!Traits::Is64Bit && DestTy == IceType_i32)) { | 2864 (!Traits::Is64Bit && DestTy == IceType_i32)) { |
| 2865 llvm::report_fatal_error("Helper call was expected"); | 2865 llvm::report_fatal_error("Helper call was expected"); |
| 2866 } else { | 2866 } else { |
| 2867 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); | 2867 Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); |
| 2868 // t1.i32 = cvt Src0RM; t2.dest_type = t1; Dest = t2.dest_type | 2868 // t1.i32 = cvt Src0RM; t2.dest_type = t1; Dest = t2.dest_type |
| 2869 assert(DestTy != IceType_i64); | 2869 assert(DestTy != IceType_i64); |
| 2870 Variable *T_1 = nullptr; | 2870 Variable *T_1 = nullptr; |
| 2871 if (Traits::Is64Bit && DestTy == IceType_i32) { | 2871 if (Traits::Is64Bit && DestTy == IceType_i32) { |
| 2872 T_1 = makeReg(IceType_i64); | 2872 T_1 = makeReg(IceType_i64); |
| 2873 } else { | 2873 } else { |
| 2874 assert(DestTy != IceType_i32); | 2874 assert(DestTy != IceType_i32); |
| 2875 T_1 = makeReg(IceType_i32); | 2875 T_1 = makeReg(IceType_i32); |
| 2876 } | 2876 } |
| 2877 Variable *T_2 = makeReg(DestTy); | 2877 Variable *T_2 = makeReg(DestTy); |
| 2878 if (isByteSizedType(DestTy)) { | 2878 if (isByteSizedType(DestTy)) { |
| 2879 assert(T_1->getType() == IceType_i32); | 2879 assert(T_1->getType() == IceType_i32); |
| 2880 T_1->setRegClass(RCX86_Is32To8); | 2880 T_1->setRegClass(RCX86_Is32To8); |
| 2881 T_2->setRegClass(RCX86_IsTrunc8Rcvr); | 2881 T_2->setRegClass(RCX86_IsTrunc8Rcvr); |
| 2882 } | 2882 } |
| 2883 _cvt(T_1, Src0RM, Traits::Insts::Cvt::Tss2si); | 2883 _cvt(T_1, Src0RM, Traits::Insts::Cvt::Tss2si); |
| 2884 _mov(T_2, T_1); // T_1 and T_2 may have different integer types | 2884 _mov(T_2, T_1); // T_1 and T_2 may have different integer types |
| 2885 if (DestTy == IceType_i1) | 2885 if (DestTy == IceType_i1) |
| 2886 _and(T_2, Ctx->getConstantInt1(1)); | 2886 _and(T_2, Ctx->getConstantInt1(1)); |
| 2887 _mov(Dest, T_2); | 2887 _mov(Dest, T_2); |
| 2888 } | 2888 } |
| 2889 break; | 2889 break; |
| 2890 case InstCast::Sitofp: | 2890 case InstCast::Sitofp: |
| 2891 if (isVectorType(DestTy)) { | 2891 if (isVectorType(DestTy)) { |
| 2892 assert(DestTy == IceType_v4f32 && | 2892 assert(DestTy == IceType_v4f32 && |
| 2893 Inst->getSrc(0)->getType() == IceType_v4i32); | 2893 Instr->getSrc(0)->getType() == IceType_v4i32); |
| 2894 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); | 2894 Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); |
| 2895 if (llvm::isa<X86OperandMem>(Src0RM)) | 2895 if (llvm::isa<X86OperandMem>(Src0RM)) |
| 2896 Src0RM = legalizeToReg(Src0RM); | 2896 Src0RM = legalizeToReg(Src0RM); |
| 2897 Variable *T = makeReg(DestTy); | 2897 Variable *T = makeReg(DestTy); |
| 2898 _cvt(T, Src0RM, Traits::Insts::Cvt::Dq2ps); | 2898 _cvt(T, Src0RM, Traits::Insts::Cvt::Dq2ps); |
| 2899 _movp(Dest, T); | 2899 _movp(Dest, T); |
| 2900 } else if (!Traits::Is64Bit && Inst->getSrc(0)->getType() == IceType_i64) { | 2900 } else if (!Traits::Is64Bit && Instr->getSrc(0)->getType() == IceType_i64) { |
| 2901 llvm::report_fatal_error("Helper call was expected"); | 2901 llvm::report_fatal_error("Helper call was expected"); |
| 2902 } else { | 2902 } else { |
| 2903 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); | 2903 Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); |
| 2904 // Sign-extend the operand. | 2904 // Sign-extend the operand. |
| 2905 // t1.i32 = movsx Src0RM; t2 = Cvt t1.i32; Dest = t2 | 2905 // t1.i32 = movsx Src0RM; t2 = Cvt t1.i32; Dest = t2 |
| 2906 Variable *T_1 = nullptr; | 2906 Variable *T_1 = nullptr; |
| 2907 if (Traits::Is64Bit && Src0RM->getType() == IceType_i64) { | 2907 if (Traits::Is64Bit && Src0RM->getType() == IceType_i64) { |
| 2908 T_1 = makeReg(IceType_i64); | 2908 T_1 = makeReg(IceType_i64); |
| 2909 } else { | 2909 } else { |
| 2910 assert(Src0RM->getType() != IceType_i64); | 2910 assert(Src0RM->getType() != IceType_i64); |
| 2911 T_1 = makeReg(IceType_i32); | 2911 T_1 = makeReg(IceType_i32); |
| 2912 } | 2912 } |
| 2913 Variable *T_2 = makeReg(DestTy); | 2913 Variable *T_2 = makeReg(DestTy); |
| 2914 if (Src0RM->getType() == T_1->getType()) | 2914 if (Src0RM->getType() == T_1->getType()) |
| 2915 _mov(T_1, Src0RM); | 2915 _mov(T_1, Src0RM); |
| 2916 else | 2916 else |
| 2917 _movsx(T_1, Src0RM); | 2917 _movsx(T_1, Src0RM); |
| 2918 _cvt(T_2, T_1, Traits::Insts::Cvt::Si2ss); | 2918 _cvt(T_2, T_1, Traits::Insts::Cvt::Si2ss); |
| 2919 _mov(Dest, T_2); | 2919 _mov(Dest, T_2); |
| 2920 } | 2920 } |
| 2921 break; | 2921 break; |
| 2922 case InstCast::Uitofp: { | 2922 case InstCast::Uitofp: { |
| 2923 Operand *Src0 = Inst->getSrc(0); | 2923 Operand *Src0 = Instr->getSrc(0); |
| 2924 if (isVectorType(Src0->getType())) { | 2924 if (isVectorType(Src0->getType())) { |
| 2925 llvm::report_fatal_error("Helper call was expected"); | 2925 llvm::report_fatal_error("Helper call was expected"); |
| 2926 } else if (Src0->getType() == IceType_i64 || | 2926 } else if (Src0->getType() == IceType_i64 || |
| 2927 (!Traits::Is64Bit && Src0->getType() == IceType_i32)) { | 2927 (!Traits::Is64Bit && Src0->getType() == IceType_i32)) { |
| 2928 llvm::report_fatal_error("Helper call was expected"); | 2928 llvm::report_fatal_error("Helper call was expected"); |
| 2929 } else { | 2929 } else { |
| 2930 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); | 2930 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); |
| 2931 // Zero-extend the operand. | 2931 // Zero-extend the operand. |
| 2932 // t1.i32 = movzx Src0RM; t2 = Cvt t1.i32; Dest = t2 | 2932 // t1.i32 = movzx Src0RM; t2 = Cvt t1.i32; Dest = t2 |
| 2933 Variable *T_1 = nullptr; | 2933 Variable *T_1 = nullptr; |
| 2934 if (Traits::Is64Bit && Src0RM->getType() == IceType_i32) { | 2934 if (Traits::Is64Bit && Src0RM->getType() == IceType_i32) { |
| 2935 T_1 = makeReg(IceType_i64); | 2935 T_1 = makeReg(IceType_i64); |
| 2936 } else { | 2936 } else { |
| 2937 assert(Src0RM->getType() != IceType_i64); | 2937 assert(Src0RM->getType() != IceType_i64); |
| 2938 assert(Traits::Is64Bit || Src0RM->getType() != IceType_i32); | 2938 assert(Traits::Is64Bit || Src0RM->getType() != IceType_i32); |
| 2939 T_1 = makeReg(IceType_i32); | 2939 T_1 = makeReg(IceType_i32); |
| 2940 } | 2940 } |
| 2941 Variable *T_2 = makeReg(DestTy); | 2941 Variable *T_2 = makeReg(DestTy); |
| 2942 if (Src0RM->getType() == T_1->getType()) | 2942 if (Src0RM->getType() == T_1->getType()) |
| 2943 _mov(T_1, Src0RM); | 2943 _mov(T_1, Src0RM); |
| 2944 else | 2944 else |
| 2945 _movzx(T_1, Src0RM); | 2945 _movzx(T_1, Src0RM); |
| 2946 _cvt(T_2, T_1, Traits::Insts::Cvt::Si2ss); | 2946 _cvt(T_2, T_1, Traits::Insts::Cvt::Si2ss); |
| 2947 _mov(Dest, T_2); | 2947 _mov(Dest, T_2); |
| 2948 } | 2948 } |
| 2949 break; | 2949 break; |
| 2950 } | 2950 } |
| 2951 case InstCast::Bitcast: { | 2951 case InstCast::Bitcast: { |
| 2952 Operand *Src0 = Inst->getSrc(0); | 2952 Operand *Src0 = Instr->getSrc(0); |
| 2953 if (DestTy == Src0->getType()) { | 2953 if (DestTy == Src0->getType()) { |
| 2954 auto *Assign = InstAssign::create(Func, Dest, Src0); | 2954 auto *Assign = InstAssign::create(Func, Dest, Src0); |
| 2955 lowerAssign(Assign); | 2955 lowerAssign(Assign); |
| 2956 return; | 2956 return; |
| 2957 } | 2957 } |
| 2958 switch (DestTy) { | 2958 switch (DestTy) { |
| 2959 default: | 2959 default: |
| 2960 llvm_unreachable("Unexpected Bitcast dest type"); | 2960 llvm_unreachable("Unexpected Bitcast dest type"); |
| 2961 case IceType_i8: { | 2961 case IceType_i8: { |
| 2962 llvm::report_fatal_error("Helper call was expected"); | 2962 llvm::report_fatal_error("Helper call was expected"); |
| (...skipping 122 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3085 _movp(Dest, legalizeToReg(Src0)); | 3085 _movp(Dest, legalizeToReg(Src0)); |
| 3086 } break; | 3086 } break; |
| 3087 } | 3087 } |
| 3088 break; | 3088 break; |
| 3089 } | 3089 } |
| 3090 } | 3090 } |
| 3091 } | 3091 } |
| 3092 | 3092 |
| 3093 template <typename TraitsType> | 3093 template <typename TraitsType> |
| 3094 void TargetX86Base<TraitsType>::lowerExtractElement( | 3094 void TargetX86Base<TraitsType>::lowerExtractElement( |
| 3095 const InstExtractElement *Inst) { | 3095 const InstExtractElement *Instr) { |
| 3096 Operand *SourceVectNotLegalized = Inst->getSrc(0); | 3096 Operand *SourceVectNotLegalized = Instr->getSrc(0); |
| 3097 ConstantInteger32 *ElementIndex = | 3097 ConstantInteger32 *ElementIndex = |
| 3098 llvm::dyn_cast<ConstantInteger32>(Inst->getSrc(1)); | 3098 llvm::dyn_cast<ConstantInteger32>(Instr->getSrc(1)); |
| 3099 // Only constant indices are allowed in PNaCl IR. | 3099 // Only constant indices are allowed in PNaCl IR. |
| 3100 assert(ElementIndex); | 3100 assert(ElementIndex); |
| 3101 | 3101 |
| 3102 unsigned Index = ElementIndex->getValue(); | 3102 unsigned Index = ElementIndex->getValue(); |
| 3103 Type Ty = SourceVectNotLegalized->getType(); | 3103 Type Ty = SourceVectNotLegalized->getType(); |
| 3104 Type ElementTy = typeElementType(Ty); | 3104 Type ElementTy = typeElementType(Ty); |
| 3105 Type InVectorElementTy = Traits::getInVectorElementType(Ty); | 3105 Type InVectorElementTy = Traits::getInVectorElementType(Ty); |
| 3106 | 3106 |
| 3107 // TODO(wala): Determine the best lowering sequences for each type. | 3107 // TODO(wala): Determine the best lowering sequences for each type. |
| 3108 bool CanUsePextr = Ty == IceType_v8i16 || Ty == IceType_v8i1 || | 3108 bool CanUsePextr = Ty == IceType_v8i16 || Ty == IceType_v8i1 || |
| (...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3161 if (ElementTy == IceType_i1) { | 3161 if (ElementTy == IceType_i1) { |
| 3162 // Truncate extracted integers to i1s if necessary. | 3162 // Truncate extracted integers to i1s if necessary. |
| 3163 Variable *T = makeReg(IceType_i1); | 3163 Variable *T = makeReg(IceType_i1); |
| 3164 InstCast *Cast = | 3164 InstCast *Cast = |
| 3165 InstCast::create(Func, InstCast::Trunc, T, ExtractedElementR); | 3165 InstCast::create(Func, InstCast::Trunc, T, ExtractedElementR); |
| 3166 lowerCast(Cast); | 3166 lowerCast(Cast); |
| 3167 ExtractedElementR = T; | 3167 ExtractedElementR = T; |
| 3168 } | 3168 } |
| 3169 | 3169 |
| 3170 // Copy the element to the destination. | 3170 // Copy the element to the destination. |
| 3171 Variable *Dest = Inst->getDest(); | 3171 Variable *Dest = Instr->getDest(); |
| 3172 _mov(Dest, ExtractedElementR); | 3172 _mov(Dest, ExtractedElementR); |
| 3173 } | 3173 } |
| 3174 | 3174 |
| 3175 template <typename TraitsType> | 3175 template <typename TraitsType> |
| 3176 void TargetX86Base<TraitsType>::lowerFcmp(const InstFcmp *Fcmp) { | 3176 void TargetX86Base<TraitsType>::lowerFcmp(const InstFcmp *Fcmp) { |
| 3177 Variable *Dest = Fcmp->getDest(); | 3177 Variable *Dest = Fcmp->getDest(); |
| 3178 | 3178 |
| 3179 if (isVectorType(Dest->getType())) { | 3179 if (isVectorType(Dest->getType())) { |
| 3180 lowerFcmpVector(Fcmp); | 3180 lowerFcmpVector(Fcmp); |
| 3181 } else { | 3181 } else { |
| (...skipping 562 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3744 Context.insert<InstFakeUse>(T); | 3744 Context.insert<InstFakeUse>(T); |
| 3745 Context.insert<InstFakeDef>(Dest); | 3745 Context.insert<InstFakeDef>(Dest); |
| 3746 _br(Traits::Cond::Br_ne, Br->getTargetTrue(), Br->getTargetFalse()); | 3746 _br(Traits::Cond::Br_ne, Br->getTargetTrue(), Br->getTargetFalse()); |
| 3747 return; | 3747 return; |
| 3748 } | 3748 } |
| 3749 llvm::report_fatal_error("Unexpected consumer type"); | 3749 llvm::report_fatal_error("Unexpected consumer type"); |
| 3750 } | 3750 } |
| 3751 | 3751 |
| 3752 template <typename TraitsType> | 3752 template <typename TraitsType> |
| 3753 void TargetX86Base<TraitsType>::lowerInsertElement( | 3753 void TargetX86Base<TraitsType>::lowerInsertElement( |
| 3754 const InstInsertElement *Inst) { | 3754 const InstInsertElement *Instr) { |
| 3755 Operand *SourceVectNotLegalized = Inst->getSrc(0); | 3755 Operand *SourceVectNotLegalized = Instr->getSrc(0); |
| 3756 Operand *ElementToInsertNotLegalized = Inst->getSrc(1); | 3756 Operand *ElementToInsertNotLegalized = Instr->getSrc(1); |
| 3757 ConstantInteger32 *ElementIndex = | 3757 ConstantInteger32 *ElementIndex = |
| 3758 llvm::dyn_cast<ConstantInteger32>(Inst->getSrc(2)); | 3758 llvm::dyn_cast<ConstantInteger32>(Instr->getSrc(2)); |
| 3759 // Only constant indices are allowed in PNaCl IR. | 3759 // Only constant indices are allowed in PNaCl IR. |
| 3760 assert(ElementIndex); | 3760 assert(ElementIndex); |
| 3761 unsigned Index = ElementIndex->getValue(); | 3761 unsigned Index = ElementIndex->getValue(); |
| 3762 assert(Index < typeNumElements(SourceVectNotLegalized->getType())); | 3762 assert(Index < typeNumElements(SourceVectNotLegalized->getType())); |
| 3763 | 3763 |
| 3764 Type Ty = SourceVectNotLegalized->getType(); | 3764 Type Ty = SourceVectNotLegalized->getType(); |
| 3765 Type ElementTy = typeElementType(Ty); | 3765 Type ElementTy = typeElementType(Ty); |
| 3766 Type InVectorElementTy = Traits::getInVectorElementType(Ty); | 3766 Type InVectorElementTy = Traits::getInVectorElementType(Ty); |
| 3767 | 3767 |
| 3768 if (ElementTy == IceType_i1) { | 3768 if (ElementTy == IceType_i1) { |
| (...skipping 24 matching lines...) Expand all Loading... |
| 3793 // the use | 3793 // the use |
| 3794 // of r16 and r8 by converting them through getBaseReg(), while emitIAS() | 3794 // of r16 and r8 by converting them through getBaseReg(), while emitIAS() |
| 3795 // validates that the original and base register encodings are the same. | 3795 // validates that the original and base register encodings are the same. |
| 3796 if (ElementRM->getType() == IceType_i8 && | 3796 if (ElementRM->getType() == IceType_i8 && |
| 3797 llvm::isa<Variable>(ElementRM)) { | 3797 llvm::isa<Variable>(ElementRM)) { |
| 3798 // Don't use ah/bh/ch/dh for pinsrb. | 3798 // Don't use ah/bh/ch/dh for pinsrb. |
| 3799 ElementRM = copyToReg8(ElementRM); | 3799 ElementRM = copyToReg8(ElementRM); |
| 3800 } | 3800 } |
| 3801 _pinsr(T, ElementRM, Ctx->getConstantInt32(Index)); | 3801 _pinsr(T, ElementRM, Ctx->getConstantInt32(Index)); |
| 3802 } | 3802 } |
| 3803 _movp(Inst->getDest(), T); | 3803 _movp(Instr->getDest(), T); |
| 3804 } else if (Ty == IceType_v4i32 || Ty == IceType_v4f32 || Ty == IceType_v4i1) { | 3804 } else if (Ty == IceType_v4i32 || Ty == IceType_v4f32 || Ty == IceType_v4i1) { |
| 3805 // Use shufps or movss. | 3805 // Use shufps or movss. |
| 3806 Variable *ElementR = nullptr; | 3806 Variable *ElementR = nullptr; |
| 3807 Operand *SourceVectRM = | 3807 Operand *SourceVectRM = |
| 3808 legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem); | 3808 legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem); |
| 3809 | 3809 |
| 3810 if (InVectorElementTy == IceType_f32) { | 3810 if (InVectorElementTy == IceType_f32) { |
| 3811 // ElementR will be in an XMM register since it is floating point. | 3811 // ElementR will be in an XMM register since it is floating point. |
| 3812 ElementR = legalizeToReg(ElementToInsertNotLegalized); | 3812 ElementR = legalizeToReg(ElementToInsertNotLegalized); |
| 3813 } else { | 3813 } else { |
| 3814 // Copy an integer to an XMM register. | 3814 // Copy an integer to an XMM register. |
| 3815 Operand *T = legalize(ElementToInsertNotLegalized, Legal_Reg | Legal_Mem); | 3815 Operand *T = legalize(ElementToInsertNotLegalized, Legal_Reg | Legal_Mem); |
| 3816 ElementR = makeReg(Ty); | 3816 ElementR = makeReg(Ty); |
| 3817 _movd(ElementR, T); | 3817 _movd(ElementR, T); |
| 3818 } | 3818 } |
| 3819 | 3819 |
| 3820 if (Index == 0) { | 3820 if (Index == 0) { |
| 3821 Variable *T = makeReg(Ty); | 3821 Variable *T = makeReg(Ty); |
| 3822 _movp(T, SourceVectRM); | 3822 _movp(T, SourceVectRM); |
| 3823 _movss(T, ElementR); | 3823 _movss(T, ElementR); |
| 3824 _movp(Inst->getDest(), T); | 3824 _movp(Instr->getDest(), T); |
| 3825 return; | 3825 return; |
| 3826 } | 3826 } |
| 3827 | 3827 |
| 3828 // shufps treats the source and destination operands as vectors of four | 3828 // shufps treats the source and destination operands as vectors of four |
| 3829 // doublewords. The destination's two high doublewords are selected from | 3829 // doublewords. The destination's two high doublewords are selected from |
| 3830 // the source operand and the two low doublewords are selected from the | 3830 // the source operand and the two low doublewords are selected from the |
| 3831 // (original value of) the destination operand. An insertelement operation | 3831 // (original value of) the destination operand. An insertelement operation |
| 3832 // can be effected with a sequence of two shufps operations with | 3832 // can be effected with a sequence of two shufps operations with |
| 3833 // appropriate masks. In all cases below, Element[0] is being inserted into | 3833 // appropriate masks. In all cases below, Element[0] is being inserted into |
| 3834 // SourceVectOperand. Indices are ordered from left to right. | 3834 // SourceVectOperand. Indices are ordered from left to right. |
| (...skipping 13 matching lines...) Expand all Loading... |
| 3848 // T := T[0, 1] ElementR[3, 0] | 3848 // T := T[0, 1] ElementR[3, 0] |
| 3849 const unsigned char Mask1[3] = {0, 192, 128}; | 3849 const unsigned char Mask1[3] = {0, 192, 128}; |
| 3850 const unsigned char Mask2[3] = {227, 196, 52}; | 3850 const unsigned char Mask2[3] = {227, 196, 52}; |
| 3851 | 3851 |
| 3852 Constant *Mask1Constant = Ctx->getConstantInt32(Mask1[Index - 1]); | 3852 Constant *Mask1Constant = Ctx->getConstantInt32(Mask1[Index - 1]); |
| 3853 Constant *Mask2Constant = Ctx->getConstantInt32(Mask2[Index - 1]); | 3853 Constant *Mask2Constant = Ctx->getConstantInt32(Mask2[Index - 1]); |
| 3854 | 3854 |
| 3855 if (Index == 1) { | 3855 if (Index == 1) { |
| 3856 _shufps(ElementR, SourceVectRM, Mask1Constant); | 3856 _shufps(ElementR, SourceVectRM, Mask1Constant); |
| 3857 _shufps(ElementR, SourceVectRM, Mask2Constant); | 3857 _shufps(ElementR, SourceVectRM, Mask2Constant); |
| 3858 _movp(Inst->getDest(), ElementR); | 3858 _movp(Instr->getDest(), ElementR); |
| 3859 } else { | 3859 } else { |
| 3860 Variable *T = makeReg(Ty); | 3860 Variable *T = makeReg(Ty); |
| 3861 _movp(T, SourceVectRM); | 3861 _movp(T, SourceVectRM); |
| 3862 _shufps(ElementR, T, Mask1Constant); | 3862 _shufps(ElementR, T, Mask1Constant); |
| 3863 _shufps(T, ElementR, Mask2Constant); | 3863 _shufps(T, ElementR, Mask2Constant); |
| 3864 _movp(Inst->getDest(), T); | 3864 _movp(Instr->getDest(), T); |
| 3865 } | 3865 } |
| 3866 } else { | 3866 } else { |
| 3867 assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); | 3867 assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); |
| 3868 // Spill the value to a stack slot and perform the insertion in memory. | 3868 // Spill the value to a stack slot and perform the insertion in memory. |
| 3869 // | 3869 // |
| 3870 // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when support | 3870 // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when support |
| 3871 // for legalizing to mem is implemented. | 3871 // for legalizing to mem is implemented. |
| 3872 Variable *Slot = Func->makeVariable(Ty); | 3872 Variable *Slot = Func->makeVariable(Ty); |
| 3873 Slot->setMustNotHaveReg(); | 3873 Slot->setMustNotHaveReg(); |
| 3874 _movp(Slot, legalizeToReg(SourceVectNotLegalized)); | 3874 _movp(Slot, legalizeToReg(SourceVectNotLegalized)); |
| 3875 | 3875 |
| 3876 // Compute the location of the position to insert in memory. | 3876 // Compute the location of the position to insert in memory. |
| 3877 unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); | 3877 unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); |
| 3878 X86OperandMem *Loc = | 3878 X86OperandMem *Loc = |
| 3879 getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); | 3879 getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); |
| 3880 _store(legalizeToReg(ElementToInsertNotLegalized), Loc); | 3880 _store(legalizeToReg(ElementToInsertNotLegalized), Loc); |
| 3881 | 3881 |
| 3882 Variable *T = makeReg(Ty); | 3882 Variable *T = makeReg(Ty); |
| 3883 _movp(T, Slot); | 3883 _movp(T, Slot); |
| 3884 _movp(Inst->getDest(), T); | 3884 _movp(Instr->getDest(), T); |
| 3885 } | 3885 } |
| 3886 } | 3886 } |
| 3887 | 3887 |
| 3888 template <typename TraitsType> | 3888 template <typename TraitsType> |
| 3889 void TargetX86Base<TraitsType>::lowerIntrinsicCall( | 3889 void TargetX86Base<TraitsType>::lowerIntrinsicCall( |
| 3890 const InstIntrinsicCall *Instr) { | 3890 const InstIntrinsicCall *Instr) { |
| 3891 switch (Intrinsics::IntrinsicID ID = Instr->getIntrinsicInfo().ID) { | 3891 switch (Intrinsics::IntrinsicID ID = Instr->getIntrinsicInfo().ID) { |
| 3892 case Intrinsics::AtomicCmpxchg: { | 3892 case Intrinsics::AtomicCmpxchg: { |
| 3893 if (!Intrinsics::isMemoryOrderValid( | 3893 if (!Intrinsics::isMemoryOrderValid( |
| 3894 ID, getConstantMemoryOrder(Instr->getArg(3)), | 3894 ID, getConstantMemoryOrder(Instr->getArg(3)), |
| (...skipping 1097 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4992 inline const Inst *matchShiftedIndex(Variable **Index, uint16_t *Shift); | 4992 inline const Inst *matchShiftedIndex(Variable **Index, uint16_t *Shift); |
| 4993 | 4993 |
| 4994 inline const Inst *matchOffsetBase(Variable **Base, | 4994 inline const Inst *matchOffsetBase(Variable **Base, |
| 4995 ConstantRelocatable **Relocatable, | 4995 ConstantRelocatable **Relocatable, |
| 4996 int32_t *Offset); | 4996 int32_t *Offset); |
| 4997 | 4997 |
| 4998 private: | 4998 private: |
| 4999 const Cfg *const Func; | 4999 const Cfg *const Func; |
| 5000 const VariablesMetadata *const VMetadata; | 5000 const VariablesMetadata *const VMetadata; |
| 5001 | 5001 |
| 5002 static bool isAdd(const Inst *Inst) { | 5002 static bool isAdd(const Inst *Instr) { |
| 5003 if (auto *Arith = llvm::dyn_cast_or_null<const InstArithmetic>(Inst)) { | 5003 if (auto *Arith = llvm::dyn_cast_or_null<const InstArithmetic>(Instr)) { |
| 5004 return (Arith->getOp() == InstArithmetic::Add); | 5004 return (Arith->getOp() == InstArithmetic::Add); |
| 5005 } | 5005 } |
| 5006 return false; | 5006 return false; |
| 5007 } | 5007 } |
| 5008 }; | 5008 }; |
| 5009 | 5009 |
| 5010 void AddressOptimizer::dumpAddressOpt( | 5010 void AddressOptimizer::dumpAddressOpt( |
| 5011 const ConstantRelocatable *const Relocatable, int32_t Offset, | 5011 const ConstantRelocatable *const Relocatable, int32_t Offset, |
| 5012 const Variable *Base, const Variable *Index, uint16_t Shift, | 5012 const Variable *Base, const Variable *Index, uint16_t Shift, |
| 5013 const Inst *Reason) const { | 5013 const Inst *Reason) const { |
| (...skipping 492 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 5506 Variable *DestLoad = Load->getDest(); | 5506 Variable *DestLoad = Load->getDest(); |
| 5507 Type Ty = DestLoad->getType(); | 5507 Type Ty = DestLoad->getType(); |
| 5508 Operand *Src0 = formMemoryOperand(Load->getSourceAddress(), Ty); | 5508 Operand *Src0 = formMemoryOperand(Load->getSourceAddress(), Ty); |
| 5509 doMockBoundsCheck(Src0); | 5509 doMockBoundsCheck(Src0); |
| 5510 auto *Assign = InstAssign::create(Func, DestLoad, Src0); | 5510 auto *Assign = InstAssign::create(Func, DestLoad, Src0); |
| 5511 lowerAssign(Assign); | 5511 lowerAssign(Assign); |
| 5512 } | 5512 } |
| 5513 | 5513 |
| 5514 template <typename TraitsType> | 5514 template <typename TraitsType> |
| 5515 void TargetX86Base<TraitsType>::doAddressOptLoad() { | 5515 void TargetX86Base<TraitsType>::doAddressOptLoad() { |
| 5516 Inst *Inst = Context.getCur(); | 5516 Inst *Instr = Context.getCur(); |
| 5517 Operand *Addr = Inst->getSrc(0); | 5517 Operand *Addr = Instr->getSrc(0); |
| 5518 Variable *Dest = Inst->getDest(); | 5518 Variable *Dest = Instr->getDest(); |
| 5519 if (auto *OptAddr = computeAddressOpt(Inst, Dest->getType(), Addr)) { | 5519 if (auto *OptAddr = computeAddressOpt(Instr, Dest->getType(), Addr)) { |
| 5520 Inst->setDeleted(); | 5520 Instr->setDeleted(); |
| 5521 Context.insert<InstLoad>(Dest, OptAddr); | 5521 Context.insert<InstLoad>(Dest, OptAddr); |
| 5522 } | 5522 } |
| 5523 } | 5523 } |
| 5524 | 5524 |
| 5525 template <typename TraitsType> | 5525 template <typename TraitsType> |
| 5526 void TargetX86Base<TraitsType>::randomlyInsertNop(float Probability, | 5526 void TargetX86Base<TraitsType>::randomlyInsertNop(float Probability, |
| 5527 RandomNumberGenerator &RNG) { | 5527 RandomNumberGenerator &RNG) { |
| 5528 RandomNumberGeneratorWrapper RNGW(RNG); | 5528 RandomNumberGeneratorWrapper RNGW(RNG); |
| 5529 if (RNGW.getTrueWithProbability(Probability)) { | 5529 if (RNGW.getTrueWithProbability(Probability)) { |
| 5530 _nop(RNGW(Traits::X86_NUM_NOP_VARIANTS)); | 5530 _nop(RNGW(Traits::X86_NUM_NOP_VARIANTS)); |
| 5531 } | 5531 } |
| 5532 } | 5532 } |
| 5533 | 5533 |
| 5534 template <typename TraitsType> | 5534 template <typename TraitsType> |
| 5535 void TargetX86Base<TraitsType>::lowerPhi(const InstPhi * /*Inst*/) { | 5535 void TargetX86Base<TraitsType>::lowerPhi(const InstPhi * /*Instr*/) { |
| 5536 Func->setError("Phi found in regular instruction list"); | 5536 Func->setError("Phi found in regular instruction list"); |
| 5537 } | 5537 } |
| 5538 | 5538 |
| 5539 template <typename TraitsType> | 5539 template <typename TraitsType> |
| 5540 void TargetX86Base<TraitsType>::lowerRet(const InstRet *Inst) { | 5540 void TargetX86Base<TraitsType>::lowerRet(const InstRet *Instr) { |
| 5541 Variable *Reg = nullptr; | 5541 Variable *Reg = nullptr; |
| 5542 if (Inst->hasRetValue()) { | 5542 if (Instr->hasRetValue()) { |
| 5543 Operand *RetValue = legalize(Inst->getRetValue()); | 5543 Operand *RetValue = legalize(Instr->getRetValue()); |
| 5544 const Type ReturnType = RetValue->getType(); | 5544 const Type ReturnType = RetValue->getType(); |
| 5545 assert(isVectorType(ReturnType) || isScalarFloatingType(ReturnType) || | 5545 assert(isVectorType(ReturnType) || isScalarFloatingType(ReturnType) || |
| 5546 (ReturnType == IceType_i32) || (ReturnType == IceType_i64)); | 5546 (ReturnType == IceType_i32) || (ReturnType == IceType_i64)); |
| 5547 Reg = moveReturnValueToRegister(RetValue, ReturnType); | 5547 Reg = moveReturnValueToRegister(RetValue, ReturnType); |
| 5548 } | 5548 } |
| 5549 // Add a ret instruction even if sandboxing is enabled, because addEpilog | 5549 // Add a ret instruction even if sandboxing is enabled, because addEpilog |
| 5550 // explicitly looks for a ret instruction as a marker for where to insert the | 5550 // explicitly looks for a ret instruction as a marker for where to insert the |
| 5551 // frame removal instructions. | 5551 // frame removal instructions. |
| 5552 _ret(Reg); | 5552 _ret(Reg); |
| 5553 // Add a fake use of esp to make sure esp stays alive for the entire | 5553 // Add a fake use of esp to make sure esp stays alive for the entire |
| (...skipping 162 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 5716 Variable *Dest = Icmp->getDest(); | 5716 Variable *Dest = Icmp->getDest(); |
| 5717 if (isVectorType(Dest->getType())) { | 5717 if (isVectorType(Dest->getType())) { |
| 5718 lowerIcmpVector(Icmp); | 5718 lowerIcmpVector(Icmp); |
| 5719 } else { | 5719 } else { |
| 5720 constexpr Inst *Consumer = nullptr; | 5720 constexpr Inst *Consumer = nullptr; |
| 5721 lowerIcmpAndConsumer(Icmp, Consumer); | 5721 lowerIcmpAndConsumer(Icmp, Consumer); |
| 5722 } | 5722 } |
| 5723 } | 5723 } |
| 5724 | 5724 |
| 5725 template <typename TraitsType> | 5725 template <typename TraitsType> |
| 5726 void TargetX86Base<TraitsType>::lowerSelectVector(const InstSelect *Inst) { | 5726 void TargetX86Base<TraitsType>::lowerSelectVector(const InstSelect *Instr) { |
| 5727 Variable *Dest = Inst->getDest(); | 5727 Variable *Dest = Instr->getDest(); |
| 5728 Type DestTy = Dest->getType(); | 5728 Type DestTy = Dest->getType(); |
| 5729 Operand *SrcT = Inst->getTrueOperand(); | 5729 Operand *SrcT = Instr->getTrueOperand(); |
| 5730 Operand *SrcF = Inst->getFalseOperand(); | 5730 Operand *SrcF = Instr->getFalseOperand(); |
| 5731 Operand *Condition = Inst->getCondition(); | 5731 Operand *Condition = Instr->getCondition(); |
| 5732 | 5732 |
| 5733 if (!isVectorType(DestTy)) | 5733 if (!isVectorType(DestTy)) |
| 5734 llvm::report_fatal_error("Expected a vector select"); | 5734 llvm::report_fatal_error("Expected a vector select"); |
| 5735 | 5735 |
| 5736 Type SrcTy = SrcT->getType(); | 5736 Type SrcTy = SrcT->getType(); |
| 5737 Variable *T = makeReg(SrcTy); | 5737 Variable *T = makeReg(SrcTy); |
| 5738 Operand *SrcTRM = legalize(SrcT, Legal_Reg | Legal_Mem); | 5738 Operand *SrcTRM = legalize(SrcT, Legal_Reg | Legal_Mem); |
| 5739 Operand *SrcFRM = legalize(SrcF, Legal_Reg | Legal_Mem); | 5739 Operand *SrcFRM = legalize(SrcF, Legal_Reg | Legal_Mem); |
| 5740 if (InstructionSet >= Traits::SSE4_1) { | 5740 if (InstructionSet >= Traits::SSE4_1) { |
| 5741 // TODO(wala): If the condition operand is a constant, use blendps or | 5741 // TODO(wala): If the condition operand is a constant, use blendps or |
| (...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 5784 _movp(T2, T); | 5784 _movp(T2, T); |
| 5785 _pand(T, SrcTRM); | 5785 _pand(T, SrcTRM); |
| 5786 _pandn(T2, SrcFRM); | 5786 _pandn(T2, SrcFRM); |
| 5787 _por(T, T2); | 5787 _por(T, T2); |
| 5788 _movp(Dest, T); | 5788 _movp(Dest, T); |
| 5789 | 5789 |
| 5790 return; | 5790 return; |
| 5791 } | 5791 } |
| 5792 | 5792 |
| 5793 template <typename TraitsType> | 5793 template <typename TraitsType> |
| 5794 void TargetX86Base<TraitsType>::lowerStore(const InstStore *Inst) { | 5794 void TargetX86Base<TraitsType>::lowerStore(const InstStore *Instr) { |
| 5795 Operand *Value = Inst->getData(); | 5795 Operand *Value = Instr->getData(); |
| 5796 Operand *Addr = Inst->getAddr(); | 5796 Operand *Addr = Instr->getAddr(); |
| 5797 X86OperandMem *NewAddr = formMemoryOperand(Addr, Value->getType()); | 5797 X86OperandMem *NewAddr = formMemoryOperand(Addr, Value->getType()); |
| 5798 doMockBoundsCheck(NewAddr); | 5798 doMockBoundsCheck(NewAddr); |
| 5799 Type Ty = NewAddr->getType(); | 5799 Type Ty = NewAddr->getType(); |
| 5800 | 5800 |
| 5801 if (!Traits::Is64Bit && Ty == IceType_i64) { | 5801 if (!Traits::Is64Bit && Ty == IceType_i64) { |
| 5802 Value = legalizeUndef(Value); | 5802 Value = legalizeUndef(Value); |
| 5803 Operand *ValueHi = legalize(hiOperand(Value), Legal_Reg | Legal_Imm); | 5803 Operand *ValueHi = legalize(hiOperand(Value), Legal_Reg | Legal_Imm); |
| 5804 _store(ValueHi, llvm::cast<X86OperandMem>(hiOperand(NewAddr))); | 5804 _store(ValueHi, llvm::cast<X86OperandMem>(hiOperand(NewAddr))); |
| 5805 Operand *ValueLo = legalize(loOperand(Value), Legal_Reg | Legal_Imm); | 5805 Operand *ValueLo = legalize(loOperand(Value), Legal_Reg | Legal_Imm); |
| 5806 _store(ValueLo, llvm::cast<X86OperandMem>(loOperand(NewAddr))); | 5806 _store(ValueLo, llvm::cast<X86OperandMem>(loOperand(NewAddr))); |
| 5807 } else if (isVectorType(Ty)) { | 5807 } else if (isVectorType(Ty)) { |
| 5808 _storep(legalizeToReg(Value), NewAddr); | 5808 _storep(legalizeToReg(Value), NewAddr); |
| 5809 } else { | 5809 } else { |
| 5810 Value = legalize(Value, Legal_Reg | Legal_Imm); | 5810 Value = legalize(Value, Legal_Reg | Legal_Imm); |
| 5811 _store(Value, NewAddr); | 5811 _store(Value, NewAddr); |
| 5812 } | 5812 } |
| 5813 } | 5813 } |
| 5814 | 5814 |
| 5815 template <typename TraitsType> | 5815 template <typename TraitsType> |
| 5816 void TargetX86Base<TraitsType>::doAddressOptStore() { | 5816 void TargetX86Base<TraitsType>::doAddressOptStore() { |
| 5817 auto *Inst = llvm::cast<InstStore>(Context.getCur()); | 5817 auto *Instr = llvm::cast<InstStore>(Context.getCur()); |
| 5818 Operand *Addr = Inst->getAddr(); | 5818 Operand *Addr = Instr->getAddr(); |
| 5819 Operand *Data = Inst->getData(); | 5819 Operand *Data = Instr->getData(); |
| 5820 if (auto *OptAddr = computeAddressOpt(Inst, Data->getType(), Addr)) { | 5820 if (auto *OptAddr = computeAddressOpt(Instr, Data->getType(), Addr)) { |
| 5821 Inst->setDeleted(); | 5821 Instr->setDeleted(); |
| 5822 auto *NewStore = Context.insert<InstStore>(Data, OptAddr); | 5822 auto *NewStore = Context.insert<InstStore>(Data, OptAddr); |
| 5823 if (Inst->getDest()) | 5823 if (Instr->getDest()) |
| 5824 NewStore->setRmwBeacon(Inst->getRmwBeacon()); | 5824 NewStore->setRmwBeacon(Instr->getRmwBeacon()); |
| 5825 } | 5825 } |
| 5826 } | 5826 } |
| 5827 | 5827 |
| 5828 template <typename TraitsType> | 5828 template <typename TraitsType> |
| 5829 Operand *TargetX86Base<TraitsType>::lowerCmpRange(Operand *Comparison, | 5829 Operand *TargetX86Base<TraitsType>::lowerCmpRange(Operand *Comparison, |
| 5830 uint64_t Min, uint64_t Max) { | 5830 uint64_t Min, uint64_t Max) { |
| 5831 // TODO(ascull): 64-bit should not reach here but only because it is not | 5831 // TODO(ascull): 64-bit should not reach here but only because it is not |
| 5832 // implemented yet. This should be able to handle the 64-bit case. | 5832 // implemented yet. This should be able to handle the 64-bit case. |
| 5833 assert(Traits::Is64Bit || Comparison->getType() != IceType_i64); | 5833 assert(Traits::Is64Bit || Comparison->getType() != IceType_i64); |
| 5834 // Subtracting 0 is a nop so don't do it | 5834 // Subtracting 0 is a nop so don't do it |
| (...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 5921 _br(Traits::Cond::Br_be, Case.getTarget()); | 5921 _br(Traits::Cond::Br_be, Case.getTarget()); |
| 5922 } | 5922 } |
| 5923 if (DefaultTarget != nullptr) | 5923 if (DefaultTarget != nullptr) |
| 5924 _br(DefaultTarget); | 5924 _br(DefaultTarget); |
| 5925 return; | 5925 return; |
| 5926 } | 5926 } |
| 5927 } | 5927 } |
| 5928 } | 5928 } |
| 5929 | 5929 |
| 5930 template <typename TraitsType> | 5930 template <typename TraitsType> |
| 5931 void TargetX86Base<TraitsType>::lowerSwitch(const InstSwitch *Inst) { | 5931 void TargetX86Base<TraitsType>::lowerSwitch(const InstSwitch *Instr) { |
| 5932 // Group cases together and navigate through them with a binary search | 5932 // Group cases together and navigate through them with a binary search |
| 5933 CaseClusterArray CaseClusters = CaseCluster::clusterizeSwitch(Func, Inst); | 5933 CaseClusterArray CaseClusters = CaseCluster::clusterizeSwitch(Func, Instr); |
| 5934 Operand *Src0 = Inst->getComparison(); | 5934 Operand *Src0 = Instr->getComparison(); |
| 5935 CfgNode *DefaultTarget = Inst->getLabelDefault(); | 5935 CfgNode *DefaultTarget = Instr->getLabelDefault(); |
| 5936 | 5936 |
| 5937 assert(CaseClusters.size() != 0); // Should always be at least one | 5937 assert(CaseClusters.size() != 0); // Should always be at least one |
| 5938 | 5938 |
| 5939 if (!Traits::Is64Bit && Src0->getType() == IceType_i64) { | 5939 if (!Traits::Is64Bit && Src0->getType() == IceType_i64) { |
| 5940 Src0 = legalize(Src0); // get Base/Index into physical registers | 5940 Src0 = legalize(Src0); // get Base/Index into physical registers |
| 5941 Operand *Src0Lo = loOperand(Src0); | 5941 Operand *Src0Lo = loOperand(Src0); |
| 5942 Operand *Src0Hi = hiOperand(Src0); | 5942 Operand *Src0Hi = hiOperand(Src0); |
| 5943 if (CaseClusters.back().getHigh() > UINT32_MAX) { | 5943 if (CaseClusters.back().getHigh() > UINT32_MAX) { |
| 5944 // TODO(ascull): handle 64-bit case properly (currently naive version) | 5944 // TODO(ascull): handle 64-bit case properly (currently naive version) |
| 5945 // This might be handled by a higher level lowering of switches. | 5945 // This might be handled by a higher level lowering of switches. |
| 5946 SizeT NumCases = Inst->getNumCases(); | 5946 SizeT NumCases = Instr->getNumCases(); |
| 5947 if (NumCases >= 2) { | 5947 if (NumCases >= 2) { |
| 5948 Src0Lo = legalizeToReg(Src0Lo); | 5948 Src0Lo = legalizeToReg(Src0Lo); |
| 5949 Src0Hi = legalizeToReg(Src0Hi); | 5949 Src0Hi = legalizeToReg(Src0Hi); |
| 5950 } else { | 5950 } else { |
| 5951 Src0Lo = legalize(Src0Lo, Legal_Reg | Legal_Mem); | 5951 Src0Lo = legalize(Src0Lo, Legal_Reg | Legal_Mem); |
| 5952 Src0Hi = legalize(Src0Hi, Legal_Reg | Legal_Mem); | 5952 Src0Hi = legalize(Src0Hi, Legal_Reg | Legal_Mem); |
| 5953 } | 5953 } |
| 5954 for (SizeT I = 0; I < NumCases; ++I) { | 5954 for (SizeT I = 0; I < NumCases; ++I) { |
| 5955 Constant *ValueLo = Ctx->getConstantInt32(Inst->getValue(I)); | 5955 Constant *ValueLo = Ctx->getConstantInt32(Instr->getValue(I)); |
| 5956 Constant *ValueHi = Ctx->getConstantInt32(Inst->getValue(I) >> 32); | 5956 Constant *ValueHi = Ctx->getConstantInt32(Instr->getValue(I) >> 32); |
| 5957 InstX86Label *Label = InstX86Label::create(Func, this); | 5957 InstX86Label *Label = InstX86Label::create(Func, this); |
| 5958 _cmp(Src0Lo, ValueLo); | 5958 _cmp(Src0Lo, ValueLo); |
| 5959 _br(Traits::Cond::Br_ne, Label); | 5959 _br(Traits::Cond::Br_ne, Label); |
| 5960 _cmp(Src0Hi, ValueHi); | 5960 _cmp(Src0Hi, ValueHi); |
| 5961 _br(Traits::Cond::Br_e, Inst->getLabel(I)); | 5961 _br(Traits::Cond::Br_e, Instr->getLabel(I)); |
| 5962 Context.insert(Label); | 5962 Context.insert(Label); |
| 5963 } | 5963 } |
| 5964 _br(Inst->getLabelDefault()); | 5964 _br(Instr->getLabelDefault()); |
| 5965 return; | 5965 return; |
| 5966 } else { | 5966 } else { |
| 5967 // All the values are 32-bit so just check the operand is too and then | 5967 // All the values are 32-bit so just check the operand is too and then |
| 5968 // fall through to the 32-bit implementation. This is a common case. | 5968 // fall through to the 32-bit implementation. This is a common case. |
| 5969 Src0Hi = legalize(Src0Hi, Legal_Reg | Legal_Mem); | 5969 Src0Hi = legalize(Src0Hi, Legal_Reg | Legal_Mem); |
| 5970 Constant *Zero = Ctx->getConstantInt32(0); | 5970 Constant *Zero = Ctx->getConstantInt32(0); |
| 5971 _cmp(Src0Hi, Zero); | 5971 _cmp(Src0Hi, Zero); |
| 5972 _br(Traits::Cond::Br_ne, DefaultTarget); | 5972 _br(Traits::Cond::Br_ne, DefaultTarget); |
| 5973 Src0 = Src0Lo; | 5973 Src0 = Src0Lo; |
| 5974 } | 5974 } |
| (...skipping 141 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 6116 NextCast->setDeleted(); | 6116 NextCast->setDeleted(); |
| 6117 _movp(NextCast->getDest(), legalizeToReg(SignExtendedResult)); | 6117 _movp(NextCast->getDest(), legalizeToReg(SignExtendedResult)); |
| 6118 // Skip over the instruction. | 6118 // Skip over the instruction. |
| 6119 Context.advanceNext(); | 6119 Context.advanceNext(); |
| 6120 } | 6120 } |
| 6121 } | 6121 } |
| 6122 } | 6122 } |
| 6123 | 6123 |
| 6124 template <typename TraitsType> | 6124 template <typename TraitsType> |
| 6125 void TargetX86Base<TraitsType>::lowerUnreachable( | 6125 void TargetX86Base<TraitsType>::lowerUnreachable( |
| 6126 const InstUnreachable * /*Inst*/) { | 6126 const InstUnreachable * /*Instr*/) { |
| 6127 _ud2(); | 6127 _ud2(); |
| 6128 // Add a fake use of esp to make sure esp adjustments after the unreachable | 6128 // Add a fake use of esp to make sure esp adjustments after the unreachable |
| 6129 // do not get dead-code eliminated. | 6129 // do not get dead-code eliminated. |
| 6130 keepEspLiveAtExit(); | 6130 keepEspLiveAtExit(); |
| 6131 } | 6131 } |
| 6132 | 6132 |
| 6133 template <typename TraitsType> | 6133 template <typename TraitsType> |
| 6134 void TargetX86Base<TraitsType>::lowerRMW(const InstX86FakeRMW *RMW) { | 6134 void TargetX86Base<TraitsType>::lowerRMW(const InstX86FakeRMW *RMW) { |
| 6135 // If the beacon variable's live range does not end in this instruction, then | 6135 // If the beacon variable's live range does not end in this instruction, then |
| 6136 // it must end in the modified Store instruction that follows. This means | 6136 // it must end in the modified Store instruction that follows. This means |
| (...skipping 1318 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 7455 emitGlobal(*Var, SectionSuffix); | 7455 emitGlobal(*Var, SectionSuffix); |
| 7456 } | 7456 } |
| 7457 } | 7457 } |
| 7458 } break; | 7458 } break; |
| 7459 } | 7459 } |
| 7460 } | 7460 } |
| 7461 } // end of namespace X86NAMESPACE | 7461 } // end of namespace X86NAMESPACE |
| 7462 } // end of namespace Ice | 7462 } // end of namespace Ice |
| 7463 | 7463 |
| 7464 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H | 7464 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H |
| OLD | NEW |