OLD | NEW |
1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// | 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// |
2 // | 2 // |
3 // The Subzero Code Generator | 3 // The Subzero Code Generator |
4 // | 4 // |
5 // This file is distributed under the University of Illinois Open Source | 5 // This file is distributed under the University of Illinois Open Source |
6 // License. See LICENSE.TXT for details. | 6 // License. See LICENSE.TXT for details. |
7 // | 7 // |
8 //===----------------------------------------------------------------------===// | 8 //===----------------------------------------------------------------------===// |
9 /// | 9 /// |
10 /// \file | 10 /// \file |
(...skipping 583 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
594 Load->dump(Func); | 594 Load->dump(Func); |
595 Str << "\n "; | 595 Str << "\n "; |
596 Arith->dump(Func); | 596 Arith->dump(Func); |
597 Str << "\n "; | 597 Str << "\n "; |
598 Store->dump(Func); | 598 Store->dump(Func); |
599 Str << "\n"; | 599 Str << "\n"; |
600 } | 600 } |
601 Variable *Beacon = Func->makeVariable(IceType_i32); | 601 Variable *Beacon = Func->makeVariable(IceType_i32); |
602 Beacon->setMustNotHaveReg(); | 602 Beacon->setMustNotHaveReg(); |
603 Store->setRmwBeacon(Beacon); | 603 Store->setRmwBeacon(Beacon); |
604 InstFakeDef *BeaconDef = InstFakeDef::create(Func, Beacon); | 604 auto *BeaconDef = InstFakeDef::create(Func, Beacon); |
605 Node->getInsts().insert(I3, BeaconDef); | 605 Node->getInsts().insert(I3, BeaconDef); |
606 auto *RMW = Traits::Insts::FakeRMW::create( | 606 auto *RMW = Traits::Insts::FakeRMW::create( |
607 Func, ArithSrcOther, Store->getAddr(), Beacon, Arith->getOp()); | 607 Func, ArithSrcOther, Store->getAddr(), Beacon, Arith->getOp()); |
608 Node->getInsts().insert(I3, RMW); | 608 Node->getInsts().insert(I3, RMW); |
609 } | 609 } |
610 } | 610 } |
611 if (Func->isVerbose(IceV_RMW)) | 611 if (Func->isVerbose(IceV_RMW)) |
612 Func->getContext()->unlockStr(); | 612 Func->getContext()->unlockStr(); |
613 } | 613 } |
614 | 614 |
(...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
697 } | 697 } |
698 } else if (auto *Select = llvm::dyn_cast<InstSelect>(Next)) { | 698 } else if (auto *Select = llvm::dyn_cast<InstSelect>(Next)) { |
699 Operand *Src0 = Select->getTrueOperand(); | 699 Operand *Src0 = Select->getTrueOperand(); |
700 Operand *Src1 = Select->getFalseOperand(); | 700 Operand *Src1 = Select->getFalseOperand(); |
701 if (canFoldLoadIntoBinaryInst(LoadSrc, LoadDest, Src0, Src1)) { | 701 if (canFoldLoadIntoBinaryInst(LoadSrc, LoadDest, Src0, Src1)) { |
702 NewInst = InstSelect::create(Func, Select->getDest(), | 702 NewInst = InstSelect::create(Func, Select->getDest(), |
703 Select->getCondition(), Src0, Src1); | 703 Select->getCondition(), Src0, Src1); |
704 } | 704 } |
705 } else if (auto *Cast = llvm::dyn_cast<InstCast>(Next)) { | 705 } else if (auto *Cast = llvm::dyn_cast<InstCast>(Next)) { |
706 // The load dest can always be folded into a Cast instruction. | 706 // The load dest can always be folded into a Cast instruction. |
707 Variable *Src0 = llvm::dyn_cast<Variable>(Cast->getSrc(0)); | 707 auto *Src0 = llvm::dyn_cast<Variable>(Cast->getSrc(0)); |
708 if (Src0 == LoadDest) { | 708 if (Src0 == LoadDest) { |
709 NewInst = InstCast::create(Func, Cast->getCastKind(), | 709 NewInst = InstCast::create(Func, Cast->getCastKind(), |
710 Cast->getDest(), LoadSrc); | 710 Cast->getDest(), LoadSrc); |
711 } | 711 } |
712 } | 712 } |
713 if (NewInst) { | 713 if (NewInst) { |
714 CurInst->setDeleted(); | 714 CurInst->setDeleted(); |
715 Next->setDeleted(); | 715 Next->setDeleted(); |
716 Context.insert(NewInst); | 716 Context.insert(NewInst); |
717 // Update NewInst->LiveRangesEnded so that target lowering may | 717 // Update NewInst->LiveRangesEnded so that target lowering may |
(...skipping 121 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
839 } | 839 } |
840 } | 840 } |
841 Type Ty = Arg->getType(); | 841 Type Ty = Arg->getType(); |
842 if (isVectorType(Ty)) { | 842 if (isVectorType(Ty)) { |
843 InArgsSizeBytes = Traits::applyStackAlignment(InArgsSizeBytes); | 843 InArgsSizeBytes = Traits::applyStackAlignment(InArgsSizeBytes); |
844 } | 844 } |
845 Arg->setStackOffset(BasicFrameOffset + InArgsSizeBytes); | 845 Arg->setStackOffset(BasicFrameOffset + InArgsSizeBytes); |
846 InArgsSizeBytes += typeWidthInBytesOnStack(Ty); | 846 InArgsSizeBytes += typeWidthInBytesOnStack(Ty); |
847 if (Arg->hasReg()) { | 847 if (Arg->hasReg()) { |
848 assert(Ty != IceType_i64 || Traits::Is64Bit); | 848 assert(Ty != IceType_i64 || Traits::Is64Bit); |
849 typename Traits::X86OperandMem *Mem = Traits::X86OperandMem::create( | 849 auto *Mem = Traits::X86OperandMem::create( |
850 Func, Ty, FramePtr, | 850 Func, Ty, FramePtr, |
851 Ctx->getConstantInt32(Arg->getStackOffset() + StackAdjBytes)); | 851 Ctx->getConstantInt32(Arg->getStackOffset() + StackAdjBytes)); |
852 if (isVectorType(Arg->getType())) { | 852 if (isVectorType(Arg->getType())) { |
853 _movp(Arg, Mem); | 853 _movp(Arg, Mem); |
854 } else { | 854 } else { |
855 _mov(Arg, Mem); | 855 _mov(Arg, Mem); |
856 } | 856 } |
857 // This argument-copying instruction uses an explicit Traits::X86OperandMem | 857 // This argument-copying instruction uses an explicit Traits::X86OperandMem |
858 // operand instead of a Variable, so its fill-from-stack operation has to | 858 // operand instead of a Variable, so its fill-from-stack operation has to |
859 // be tracked separately for statistics. | 859 // be tracked separately for statistics. |
(...skipping 505 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1365 case InstArithmetic::Udiv: | 1365 case InstArithmetic::Udiv: |
1366 case InstArithmetic::Sdiv: | 1366 case InstArithmetic::Sdiv: |
1367 case InstArithmetic::Urem: | 1367 case InstArithmetic::Urem: |
1368 case InstArithmetic::Srem: | 1368 case InstArithmetic::Srem: |
1369 llvm::report_fatal_error("Helper call was expected"); | 1369 llvm::report_fatal_error("Helper call was expected"); |
1370 return; | 1370 return; |
1371 default: | 1371 default: |
1372 break; | 1372 break; |
1373 } | 1373 } |
1374 | 1374 |
1375 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 1375 auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
1376 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 1376 auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
1377 Operand *Src0Lo = loOperand(Src0); | 1377 Operand *Src0Lo = loOperand(Src0); |
1378 Operand *Src0Hi = hiOperand(Src0); | 1378 Operand *Src0Hi = hiOperand(Src0); |
1379 Operand *Src1Lo = loOperand(Src1); | 1379 Operand *Src1Lo = loOperand(Src1); |
1380 Operand *Src1Hi = hiOperand(Src1); | 1380 Operand *Src1Hi = hiOperand(Src1); |
1381 Variable *T_Lo = nullptr, *T_Hi = nullptr; | 1381 Variable *T_Lo = nullptr, *T_Hi = nullptr; |
1382 switch (Inst->getOp()) { | 1382 switch (Inst->getOp()) { |
1383 case InstArithmetic::_num: | 1383 case InstArithmetic::_num: |
1384 llvm_unreachable("Unknown arithmetic operator"); | 1384 llvm_unreachable("Unknown arithmetic operator"); |
1385 break; | 1385 break; |
1386 case InstArithmetic::Add: | 1386 case InstArithmetic::Add: |
(...skipping 497 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1884 if (Dest->isRematerializable()) { | 1884 if (Dest->isRematerializable()) { |
1885 Context.insert(InstFakeDef::create(Func, Dest)); | 1885 Context.insert(InstFakeDef::create(Func, Dest)); |
1886 return; | 1886 return; |
1887 } | 1887 } |
1888 Operand *Src0 = Inst->getSrc(0); | 1888 Operand *Src0 = Inst->getSrc(0); |
1889 assert(Dest->getType() == Src0->getType()); | 1889 assert(Dest->getType() == Src0->getType()); |
1890 if (!Traits::Is64Bit && Dest->getType() == IceType_i64) { | 1890 if (!Traits::Is64Bit && Dest->getType() == IceType_i64) { |
1891 Src0 = legalize(Src0); | 1891 Src0 = legalize(Src0); |
1892 Operand *Src0Lo = loOperand(Src0); | 1892 Operand *Src0Lo = loOperand(Src0); |
1893 Operand *Src0Hi = hiOperand(Src0); | 1893 Operand *Src0Hi = hiOperand(Src0); |
1894 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 1894 auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
1895 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 1895 auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
1896 Variable *T_Lo = nullptr, *T_Hi = nullptr; | 1896 Variable *T_Lo = nullptr, *T_Hi = nullptr; |
1897 _mov(T_Lo, Src0Lo); | 1897 _mov(T_Lo, Src0Lo); |
1898 _mov(DestLo, T_Lo); | 1898 _mov(DestLo, T_Lo); |
1899 _mov(T_Hi, Src0Hi); | 1899 _mov(T_Hi, Src0Hi); |
1900 _mov(DestHi, T_Hi); | 1900 _mov(DestHi, T_Hi); |
1901 } else { | 1901 } else { |
1902 Operand *Src0Legal; | 1902 Operand *Src0Legal; |
1903 if (Dest->hasReg()) { | 1903 if (Dest->hasReg()) { |
1904 // If Dest already has a physical register, then only basic legalization | 1904 // If Dest already has a physical register, then only basic legalization |
1905 // is needed, as the source operand can be a register, immediate, or | 1905 // is needed, as the source operand can be a register, immediate, or |
(...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1988 Constant *ShiftConstant = Ctx->getConstantInt8(ShiftAmount); | 1988 Constant *ShiftConstant = Ctx->getConstantInt8(ShiftAmount); |
1989 Variable *T = makeReg(DestTy); | 1989 Variable *T = makeReg(DestTy); |
1990 _movp(T, Src0RM); | 1990 _movp(T, Src0RM); |
1991 _psll(T, ShiftConstant); | 1991 _psll(T, ShiftConstant); |
1992 _psra(T, ShiftConstant); | 1992 _psra(T, ShiftConstant); |
1993 _movp(Dest, T); | 1993 _movp(Dest, T); |
1994 } | 1994 } |
1995 } else if (!Traits::Is64Bit && DestTy == IceType_i64) { | 1995 } else if (!Traits::Is64Bit && DestTy == IceType_i64) { |
1996 // t1=movsx src; t2=t1; t2=sar t2, 31; dst.lo=t1; dst.hi=t2 | 1996 // t1=movsx src; t2=t1; t2=sar t2, 31; dst.lo=t1; dst.hi=t2 |
1997 Constant *Shift = Ctx->getConstantInt32(31); | 1997 Constant *Shift = Ctx->getConstantInt32(31); |
1998 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 1998 auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
1999 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 1999 auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
2000 Variable *T_Lo = makeReg(DestLo->getType()); | 2000 Variable *T_Lo = makeReg(DestLo->getType()); |
2001 if (Src0RM->getType() == IceType_i32) { | 2001 if (Src0RM->getType() == IceType_i32) { |
2002 _mov(T_Lo, Src0RM); | 2002 _mov(T_Lo, Src0RM); |
2003 } else if (Src0RM->getType() == IceType_i1) { | 2003 } else if (Src0RM->getType() == IceType_i1) { |
2004 _movzx(T_Lo, Src0RM); | 2004 _movzx(T_Lo, Src0RM); |
2005 _shl(T_Lo, Shift); | 2005 _shl(T_Lo, Shift); |
2006 _sar(T_Lo, Shift); | 2006 _sar(T_Lo, Shift); |
2007 } else { | 2007 } else { |
2008 _movsx(T_Lo, Src0RM); | 2008 _movsx(T_Lo, Src0RM); |
2009 } | 2009 } |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2045 if (isVectorType(DestTy)) { | 2045 if (isVectorType(DestTy)) { |
2046 // onemask = materialize(1,1,...); dest = onemask & src | 2046 // onemask = materialize(1,1,...); dest = onemask & src |
2047 Variable *OneMask = makeVectorOfOnes(DestTy); | 2047 Variable *OneMask = makeVectorOfOnes(DestTy); |
2048 Variable *T = makeReg(DestTy); | 2048 Variable *T = makeReg(DestTy); |
2049 _movp(T, Src0RM); | 2049 _movp(T, Src0RM); |
2050 _pand(T, OneMask); | 2050 _pand(T, OneMask); |
2051 _movp(Dest, T); | 2051 _movp(Dest, T); |
2052 } else if (!Traits::Is64Bit && DestTy == IceType_i64) { | 2052 } else if (!Traits::Is64Bit && DestTy == IceType_i64) { |
2053 // t1=movzx src; dst.lo=t1; dst.hi=0 | 2053 // t1=movzx src; dst.lo=t1; dst.hi=0 |
2054 Constant *Zero = Ctx->getConstantZero(IceType_i32); | 2054 Constant *Zero = Ctx->getConstantZero(IceType_i32); |
2055 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 2055 auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
2056 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 2056 auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
2057 Variable *Tmp = makeReg(DestLo->getType()); | 2057 Variable *Tmp = makeReg(DestLo->getType()); |
2058 if (Src0RM->getType() == IceType_i32) { | 2058 if (Src0RM->getType() == IceType_i32) { |
2059 _mov(Tmp, Src0RM); | 2059 _mov(Tmp, Src0RM); |
2060 } else { | 2060 } else { |
2061 _movzx(Tmp, Src0RM); | 2061 _movzx(Tmp, Src0RM); |
2062 } | 2062 } |
2063 _mov(DestLo, Tmp); | 2063 _mov(DestLo, Tmp); |
2064 _mov(DestHi, Zero); | 2064 _mov(DestHi, Zero); |
2065 } else if (Src0RM->getType() == IceType_i1) { | 2065 } else if (Src0RM->getType() == IceType_i1) { |
2066 // t = Src0RM; Dest = t | 2066 // t = Src0RM; Dest = t |
(...skipping 182 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2249 else | 2249 else |
2250 _movzx(T_1, Src0RM); | 2250 _movzx(T_1, Src0RM); |
2251 _cvt(T_2, T_1, Traits::Insts::Cvt::Si2ss); | 2251 _cvt(T_2, T_1, Traits::Insts::Cvt::Si2ss); |
2252 _mov(Dest, T_2); | 2252 _mov(Dest, T_2); |
2253 } | 2253 } |
2254 break; | 2254 break; |
2255 } | 2255 } |
2256 case InstCast::Bitcast: { | 2256 case InstCast::Bitcast: { |
2257 Operand *Src0 = Inst->getSrc(0); | 2257 Operand *Src0 = Inst->getSrc(0); |
2258 if (DestTy == Src0->getType()) { | 2258 if (DestTy == Src0->getType()) { |
2259 InstAssign *Assign = InstAssign::create(Func, Dest, Src0); | 2259 auto *Assign = InstAssign::create(Func, Dest, Src0); |
2260 lowerAssign(Assign); | 2260 lowerAssign(Assign); |
2261 return; | 2261 return; |
2262 } | 2262 } |
2263 switch (DestTy) { | 2263 switch (DestTy) { |
2264 default: | 2264 default: |
2265 llvm_unreachable("Unexpected Bitcast dest type"); | 2265 llvm_unreachable("Unexpected Bitcast dest type"); |
2266 case IceType_i8: { | 2266 case IceType_i8: { |
2267 llvm::report_fatal_error("Helper call was expected"); | 2267 llvm::report_fatal_error("Helper call was expected"); |
2268 } break; | 2268 } break; |
2269 case IceType_i16: { | 2269 case IceType_i16: { |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2318 _movq(Spill, Src0RM); | 2318 _movq(Spill, Src0RM); |
2319 SpillLo = Traits::VariableSplit::create(Func, Spill, | 2319 SpillLo = Traits::VariableSplit::create(Func, Spill, |
2320 Traits::VariableSplit::Low); | 2320 Traits::VariableSplit::Low); |
2321 SpillHi = Traits::VariableSplit::create(Func, Spill, | 2321 SpillHi = Traits::VariableSplit::create(Func, Spill, |
2322 Traits::VariableSplit::High); | 2322 Traits::VariableSplit::High); |
2323 } else { | 2323 } else { |
2324 SpillLo = loOperand(Src0RM); | 2324 SpillLo = loOperand(Src0RM); |
2325 SpillHi = hiOperand(Src0RM); | 2325 SpillHi = hiOperand(Src0RM); |
2326 } | 2326 } |
2327 | 2327 |
2328 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 2328 auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
2329 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 2329 auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
2330 Variable *T_Lo = makeReg(IceType_i32); | 2330 Variable *T_Lo = makeReg(IceType_i32); |
2331 Variable *T_Hi = makeReg(IceType_i32); | 2331 Variable *T_Hi = makeReg(IceType_i32); |
2332 | 2332 |
2333 _mov(T_Lo, SpillLo); | 2333 _mov(T_Lo, SpillLo); |
2334 _mov(DestLo, T_Lo); | 2334 _mov(DestLo, T_Lo); |
2335 _mov(T_Hi, SpillHi); | 2335 _mov(T_Hi, SpillHi); |
2336 _mov(DestHi, T_Hi); | 2336 _mov(DestHi, T_Hi); |
2337 } | 2337 } |
2338 } break; | 2338 } break; |
2339 case IceType_f64: { | 2339 case IceType_f64: { |
(...skipping 20 matching lines...) Expand all Loading... |
2360 // t_hi.i32 = b_hi.i32 | 2360 // t_hi.i32 = b_hi.i32 |
2361 // hi(s.f64) = t_hi.i32 | 2361 // hi(s.f64) = t_hi.i32 |
2362 // a.f64 = s.f64 | 2362 // a.f64 = s.f64 |
2363 typename Traits::SpillVariable *SpillVar = | 2363 typename Traits::SpillVariable *SpillVar = |
2364 Func->makeVariable<typename Traits::SpillVariable>(IceType_f64); | 2364 Func->makeVariable<typename Traits::SpillVariable>(IceType_f64); |
2365 SpillVar->setLinkedTo(Dest); | 2365 SpillVar->setLinkedTo(Dest); |
2366 Variable *Spill = SpillVar; | 2366 Variable *Spill = SpillVar; |
2367 Spill->setMustNotHaveReg(); | 2367 Spill->setMustNotHaveReg(); |
2368 | 2368 |
2369 Variable *T_Lo = nullptr, *T_Hi = nullptr; | 2369 Variable *T_Lo = nullptr, *T_Hi = nullptr; |
2370 typename Traits::VariableSplit *SpillLo = Traits::VariableSplit::create( | 2370 auto *SpillLo = Traits::VariableSplit::create( |
2371 Func, Spill, Traits::VariableSplit::Low); | 2371 Func, Spill, Traits::VariableSplit::Low); |
2372 typename Traits::VariableSplit *SpillHi = Traits::VariableSplit::create( | 2372 auto *SpillHi = Traits::VariableSplit::create( |
2373 Func, Spill, Traits::VariableSplit::High); | 2373 Func, Spill, Traits::VariableSplit::High); |
2374 _mov(T_Lo, loOperand(Src0)); | 2374 _mov(T_Lo, loOperand(Src0)); |
2375 // Technically, the Spill is defined after the _store happens, but | 2375 // Technically, the Spill is defined after the _store happens, but |
2376 // SpillLo is considered a "use" of Spill so define Spill before it is | 2376 // SpillLo is considered a "use" of Spill so define Spill before it is |
2377 // used. | 2377 // used. |
2378 Context.insert(InstFakeDef::create(Func, Spill)); | 2378 Context.insert(InstFakeDef::create(Func, Spill)); |
2379 _store(T_Lo, SpillLo); | 2379 _store(T_Lo, SpillLo); |
2380 _mov(T_Hi, hiOperand(Src0)); | 2380 _mov(T_Hi, hiOperand(Src0)); |
2381 _store(T_Hi, SpillHi); | 2381 _store(T_Hi, SpillHi); |
2382 _movq(Dest, Spill); | 2382 _movq(Dest, Spill); |
(...skipping 584 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2967 assert(Index < typeNumElements(SourceVectNotLegalized->getType())); | 2967 assert(Index < typeNumElements(SourceVectNotLegalized->getType())); |
2968 | 2968 |
2969 Type Ty = SourceVectNotLegalized->getType(); | 2969 Type Ty = SourceVectNotLegalized->getType(); |
2970 Type ElementTy = typeElementType(Ty); | 2970 Type ElementTy = typeElementType(Ty); |
2971 Type InVectorElementTy = Traits::getInVectorElementType(Ty); | 2971 Type InVectorElementTy = Traits::getInVectorElementType(Ty); |
2972 | 2972 |
2973 if (ElementTy == IceType_i1) { | 2973 if (ElementTy == IceType_i1) { |
2974 // Expand the element to the appropriate size for it to be inserted in the | 2974 // Expand the element to the appropriate size for it to be inserted in the |
2975 // vector. | 2975 // vector. |
2976 Variable *Expanded = Func->makeVariable(InVectorElementTy); | 2976 Variable *Expanded = Func->makeVariable(InVectorElementTy); |
2977 InstCast *Cast = InstCast::create(Func, InstCast::Zext, Expanded, | 2977 auto *Cast = InstCast::create(Func, InstCast::Zext, Expanded, |
2978 ElementToInsertNotLegalized); | 2978 ElementToInsertNotLegalized); |
2979 lowerCast(Cast); | 2979 lowerCast(Cast); |
2980 ElementToInsertNotLegalized = Expanded; | 2980 ElementToInsertNotLegalized = Expanded; |
2981 } | 2981 } |
2982 | 2982 |
2983 if (Ty == IceType_v8i16 || Ty == IceType_v8i1 || | 2983 if (Ty == IceType_v8i16 || Ty == IceType_v8i1 || |
2984 InstructionSet >= Traits::SSE4_1) { | 2984 InstructionSet >= Traits::SSE4_1) { |
2985 // Use insertps, pinsrb, pinsrw, or pinsrd. | 2985 // Use insertps, pinsrb, pinsrw, or pinsrd. |
2986 Operand *ElementRM = | 2986 Operand *ElementRM = |
2987 legalize(ElementToInsertNotLegalized, Legal_Reg | Legal_Mem); | 2987 legalize(ElementToInsertNotLegalized, Legal_Reg | Legal_Mem); |
2988 Operand *SourceVectRM = | 2988 Operand *SourceVectRM = |
(...skipping 133 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3122 // fence (both atomic and non-atomic). The InstX8632Mfence instruction is | 3122 // fence (both atomic and non-atomic). The InstX8632Mfence instruction is |
3123 // currently marked coarsely as "HasSideEffects". | 3123 // currently marked coarsely as "HasSideEffects". |
3124 _mfence(); | 3124 _mfence(); |
3125 return; | 3125 return; |
3126 case Intrinsics::AtomicIsLockFree: { | 3126 case Intrinsics::AtomicIsLockFree: { |
3127 // X86 is always lock free for 8/16/32/64 bit accesses. | 3127 // X86 is always lock free for 8/16/32/64 bit accesses. |
3128 // TODO(jvoung): Since the result is constant when given a constant byte | 3128 // TODO(jvoung): Since the result is constant when given a constant byte |
3129 // size, this opens up DCE opportunities. | 3129 // size, this opens up DCE opportunities. |
3130 Operand *ByteSize = Instr->getArg(0); | 3130 Operand *ByteSize = Instr->getArg(0); |
3131 Variable *Dest = Instr->getDest(); | 3131 Variable *Dest = Instr->getDest(); |
3132 if (ConstantInteger32 *CI = llvm::dyn_cast<ConstantInteger32>(ByteSize)) { | 3132 if (auto *CI = llvm::dyn_cast<ConstantInteger32>(ByteSize)) { |
3133 Constant *Result; | 3133 Constant *Result; |
3134 switch (CI->getValue()) { | 3134 switch (CI->getValue()) { |
3135 default: | 3135 default: |
3136 // Some x86-64 processors support the cmpxchg16b instruction, which can | 3136 // Some x86-64 processors support the cmpxchg16b instruction, which can |
3137 // make 16-byte operations lock free (when used with the LOCK prefix). | 3137 // make 16-byte operations lock free (when used with the LOCK prefix). |
3138 // However, that's not supported in 32-bit mode, so just return 0 even | 3138 // However, that's not supported in 32-bit mode, so just return 0 even |
3139 // for large sizes. | 3139 // for large sizes. |
3140 Result = Ctx->getConstantZero(IceType_i32); | 3140 Result = Ctx->getConstantZero(IceType_i32); |
3141 break; | 3141 break; |
3142 case 1: | 3142 case 1: |
(...skipping 24 matching lines...) Expand all Loading... |
3167 // Follow what GCC does and use a movq instead of what lowerLoad() | 3167 // Follow what GCC does and use a movq instead of what lowerLoad() |
3168 // normally does (split the load into two). Thus, this skips | 3168 // normally does (split the load into two). Thus, this skips |
3169 // load/arithmetic op folding. Load/arithmetic folding can't happen | 3169 // load/arithmetic op folding. Load/arithmetic folding can't happen |
3170 // anyway, since this is x86-32 and integer arithmetic only happens on | 3170 // anyway, since this is x86-32 and integer arithmetic only happens on |
3171 // 32-bit quantities. | 3171 // 32-bit quantities. |
3172 Variable *T = makeReg(IceType_f64); | 3172 Variable *T = makeReg(IceType_f64); |
3173 typename Traits::X86OperandMem *Addr = | 3173 typename Traits::X86OperandMem *Addr = |
3174 formMemoryOperand(Instr->getArg(0), IceType_f64); | 3174 formMemoryOperand(Instr->getArg(0), IceType_f64); |
3175 _movq(T, Addr); | 3175 _movq(T, Addr); |
3176 // Then cast the bits back out of the XMM register to the i64 Dest. | 3176 // Then cast the bits back out of the XMM register to the i64 Dest. |
3177 InstCast *Cast = InstCast::create(Func, InstCast::Bitcast, Dest, T); | 3177 auto *Cast = InstCast::create(Func, InstCast::Bitcast, Dest, T); |
3178 lowerCast(Cast); | 3178 lowerCast(Cast); |
3179 // Make sure that the atomic load isn't elided when unused. | 3179 // Make sure that the atomic load isn't elided when unused. |
3180 Context.insert(InstFakeUse::create(Func, Dest64On32->getLo())); | 3180 Context.insert(InstFakeUse::create(Func, Dest64On32->getLo())); |
3181 Context.insert(InstFakeUse::create(Func, Dest64On32->getHi())); | 3181 Context.insert(InstFakeUse::create(Func, Dest64On32->getHi())); |
3182 return; | 3182 return; |
3183 } | 3183 } |
3184 } | 3184 } |
3185 InstLoad *Load = InstLoad::create(Func, Dest, Instr->getArg(0)); | 3185 auto *Load = InstLoad::create(Func, Dest, Instr->getArg(0)); |
3186 lowerLoad(Load); | 3186 lowerLoad(Load); |
3187 // Make sure the atomic load isn't elided when unused, by adding a FakeUse. | 3187 // Make sure the atomic load isn't elided when unused, by adding a FakeUse. |
3188 // Since lowerLoad may fuse the load w/ an arithmetic instruction, insert | 3188 // Since lowerLoad may fuse the load w/ an arithmetic instruction, insert |
3189 // the FakeUse on the last-inserted instruction's dest. | 3189 // the FakeUse on the last-inserted instruction's dest. |
3190 Context.insert( | 3190 Context.insert( |
3191 InstFakeUse::create(Func, Context.getLastInserted()->getDest())); | 3191 InstFakeUse::create(Func, Context.getLastInserted()->getDest())); |
3192 return; | 3192 return; |
3193 } | 3193 } |
3194 case Intrinsics::AtomicRMW: | 3194 case Intrinsics::AtomicRMW: |
3195 if (!Intrinsics::isMemoryOrderValid( | 3195 if (!Intrinsics::isMemoryOrderValid( |
(...skipping 16 matching lines...) Expand all Loading... |
3212 // We require the memory address to be naturally aligned. Given that is the | 3212 // We require the memory address to be naturally aligned. Given that is the |
3213 // case, then normal stores are atomic. Add a fence after the store to make | 3213 // case, then normal stores are atomic. Add a fence after the store to make |
3214 // it visible. | 3214 // it visible. |
3215 Operand *Value = Instr->getArg(0); | 3215 Operand *Value = Instr->getArg(0); |
3216 Operand *Ptr = Instr->getArg(1); | 3216 Operand *Ptr = Instr->getArg(1); |
3217 if (!Traits::Is64Bit && Value->getType() == IceType_i64) { | 3217 if (!Traits::Is64Bit && Value->getType() == IceType_i64) { |
3218 // Use a movq instead of what lowerStore() normally does (split the store | 3218 // Use a movq instead of what lowerStore() normally does (split the store |
3219 // into two), following what GCC does. Cast the bits from int -> to an | 3219 // into two), following what GCC does. Cast the bits from int -> to an |
3220 // xmm register first. | 3220 // xmm register first. |
3221 Variable *T = makeReg(IceType_f64); | 3221 Variable *T = makeReg(IceType_f64); |
3222 InstCast *Cast = InstCast::create(Func, InstCast::Bitcast, T, Value); | 3222 auto *Cast = InstCast::create(Func, InstCast::Bitcast, T, Value); |
3223 lowerCast(Cast); | 3223 lowerCast(Cast); |
3224 // Then store XMM w/ a movq. | 3224 // Then store XMM w/ a movq. |
3225 typename Traits::X86OperandMem *Addr = | 3225 typename Traits::X86OperandMem *Addr = |
3226 formMemoryOperand(Ptr, IceType_f64); | 3226 formMemoryOperand(Ptr, IceType_f64); |
3227 _storeq(T, Addr); | 3227 _storeq(T, Addr); |
3228 _mfence(); | 3228 _mfence(); |
3229 return; | 3229 return; |
3230 } | 3230 } |
3231 InstStore *Store = InstStore::create(Func, Value, Ptr); | 3231 auto *Store = InstStore::create(Func, Value, Ptr); |
3232 lowerStore(Store); | 3232 lowerStore(Store); |
3233 _mfence(); | 3233 _mfence(); |
3234 return; | 3234 return; |
3235 } | 3235 } |
3236 case Intrinsics::Bswap: { | 3236 case Intrinsics::Bswap: { |
3237 Variable *Dest = Instr->getDest(); | 3237 Variable *Dest = Instr->getDest(); |
3238 Operand *Val = Instr->getArg(0); | 3238 Operand *Val = Instr->getArg(0); |
3239 // In 32-bit mode, bswap only works on 32-bit arguments, and the argument | 3239 // In 32-bit mode, bswap only works on 32-bit arguments, and the argument |
3240 // must be a register. Use rotate left for 16-bit bswap. | 3240 // must be a register. Use rotate left for 16-bit bswap. |
3241 if (!Traits::Is64Bit && Val->getType() == IceType_i64) { | 3241 if (!Traits::Is64Bit && Val->getType() == IceType_i64) { |
3242 Val = legalizeUndef(Val); | 3242 Val = legalizeUndef(Val); |
3243 Variable *T_Lo = legalizeToReg(loOperand(Val)); | 3243 Variable *T_Lo = legalizeToReg(loOperand(Val)); |
3244 Variable *T_Hi = legalizeToReg(hiOperand(Val)); | 3244 Variable *T_Hi = legalizeToReg(hiOperand(Val)); |
3245 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 3245 auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
3246 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 3246 auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
3247 _bswap(T_Lo); | 3247 _bswap(T_Lo); |
3248 _bswap(T_Hi); | 3248 _bswap(T_Hi); |
3249 _mov(DestLo, T_Hi); | 3249 _mov(DestLo, T_Hi); |
3250 _mov(DestHi, T_Lo); | 3250 _mov(DestHi, T_Lo); |
3251 } else if ((Traits::Is64Bit && Val->getType() == IceType_i64) || | 3251 } else if ((Traits::Is64Bit && Val->getType() == IceType_i64) || |
3252 Val->getType() == IceType_i32) { | 3252 Val->getType() == IceType_i32) { |
3253 Variable *T = legalizeToReg(Val); | 3253 Variable *T = legalizeToReg(Val); |
3254 _bswap(T); | 3254 _bswap(T); |
3255 _mov(Dest, T); | 3255 _mov(Dest, T); |
3256 } else { | 3256 } else { |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3291 Call->addArg(Val); | 3291 Call->addArg(Val); |
3292 lowerCall(Call); | 3292 lowerCall(Call); |
3293 // The popcount helpers always return 32-bit values, while the intrinsic's | 3293 // The popcount helpers always return 32-bit values, while the intrinsic's |
3294 // signature matches the native POPCNT instruction and fills a 64-bit reg | 3294 // signature matches the native POPCNT instruction and fills a 64-bit reg |
3295 // (in 64-bit mode). Thus, clear the upper bits of the dest just in case | 3295 // (in 64-bit mode). Thus, clear the upper bits of the dest just in case |
3296 // the user doesn't do that in the IR. If the user does that in the IR, | 3296 // the user doesn't do that in the IR. If the user does that in the IR, |
3297 // then this zero'ing instruction is dead and gets optimized out. | 3297 // then this zero'ing instruction is dead and gets optimized out. |
3298 if (!Traits::Is64Bit) { | 3298 if (!Traits::Is64Bit) { |
3299 assert(T == Dest); | 3299 assert(T == Dest); |
3300 if (Val->getType() == IceType_i64) { | 3300 if (Val->getType() == IceType_i64) { |
3301 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 3301 auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
3302 Constant *Zero = Ctx->getConstantZero(IceType_i32); | 3302 Constant *Zero = Ctx->getConstantZero(IceType_i32); |
3303 _mov(DestHi, Zero); | 3303 _mov(DestHi, Zero); |
3304 } | 3304 } |
3305 } else { | 3305 } else { |
3306 assert(Val->getType() == IceType_i64); | 3306 assert(Val->getType() == IceType_i64); |
3307 // T is 64 bit. It needs to be copied to dest. We need to: | 3307 // T is 64 bit. It needs to be copied to dest. We need to: |
3308 // | 3308 // |
3309 // T_1.32 = trunc T.64 to i32 | 3309 // T_1.32 = trunc T.64 to i32 |
3310 // T_2.64 = zext T_1.32 to i64 | 3310 // T_2.64 = zext T_1.32 to i64 |
3311 // Dest.<<right_size>> = T_2.<<right_size>> | 3311 // Dest.<<right_size>> = T_2.<<right_size>> |
(...skipping 148 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3460 Variable *T_eax = makeReg(IceType_i32, Traits::RegisterSet::Reg_eax); | 3460 Variable *T_eax = makeReg(IceType_i32, Traits::RegisterSet::Reg_eax); |
3461 Variable *T_ecx = makeReg(IceType_i32, Traits::RegisterSet::Reg_ecx); | 3461 Variable *T_ecx = makeReg(IceType_i32, Traits::RegisterSet::Reg_ecx); |
3462 Variable *T_ebx = makeReg(IceType_i32, Traits::RegisterSet::Reg_ebx); | 3462 Variable *T_ebx = makeReg(IceType_i32, Traits::RegisterSet::Reg_ebx); |
3463 _mov(T_eax, loOperand(Expected)); | 3463 _mov(T_eax, loOperand(Expected)); |
3464 _mov(T_edx, hiOperand(Expected)); | 3464 _mov(T_edx, hiOperand(Expected)); |
3465 _mov(T_ebx, loOperand(Desired)); | 3465 _mov(T_ebx, loOperand(Desired)); |
3466 _mov(T_ecx, hiOperand(Desired)); | 3466 _mov(T_ecx, hiOperand(Desired)); |
3467 typename Traits::X86OperandMem *Addr = formMemoryOperand(Ptr, Ty); | 3467 typename Traits::X86OperandMem *Addr = formMemoryOperand(Ptr, Ty); |
3468 constexpr bool Locked = true; | 3468 constexpr bool Locked = true; |
3469 _cmpxchg8b(Addr, T_edx, T_eax, T_ecx, T_ebx, Locked); | 3469 _cmpxchg8b(Addr, T_edx, T_eax, T_ecx, T_ebx, Locked); |
3470 Variable *DestLo = llvm::cast<Variable>(loOperand(DestPrev)); | 3470 auto *DestLo = llvm::cast<Variable>(loOperand(DestPrev)); |
3471 Variable *DestHi = llvm::cast<Variable>(hiOperand(DestPrev)); | 3471 auto *DestHi = llvm::cast<Variable>(hiOperand(DestPrev)); |
3472 _mov(DestLo, T_eax); | 3472 _mov(DestLo, T_eax); |
3473 _mov(DestHi, T_edx); | 3473 _mov(DestHi, T_edx); |
3474 return; | 3474 return; |
3475 } | 3475 } |
3476 int32_t Eax; | 3476 int32_t Eax; |
3477 switch (Ty) { | 3477 switch (Ty) { |
3478 default: | 3478 default: |
3479 llvm_unreachable("Bad type for cmpxchg"); | 3479 llvm_unreachable("Bad type for cmpxchg"); |
3480 // fallthrough | 3480 // fallthrough |
3481 case IceType_i32: | 3481 case IceType_i32: |
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3522 // This assumes that the atomic cmpxchg has not been lowered yet, | 3522 // This assumes that the atomic cmpxchg has not been lowered yet, |
3523 // so that the instructions seen in the scan from "Cur" is simple. | 3523 // so that the instructions seen in the scan from "Cur" is simple. |
3524 assert(llvm::isa<InstIntrinsicCall>(*I)); | 3524 assert(llvm::isa<InstIntrinsicCall>(*I)); |
3525 Inst *NextInst = Context.getNextInst(I); | 3525 Inst *NextInst = Context.getNextInst(I); |
3526 if (!NextInst) | 3526 if (!NextInst) |
3527 return false; | 3527 return false; |
3528 // There might be phi assignments right before the compare+branch, since this | 3528 // There might be phi assignments right before the compare+branch, since this |
3529 // could be a backward branch for a loop. This placement of assignments is | 3529 // could be a backward branch for a loop. This placement of assignments is |
3530 // determined by placePhiStores(). | 3530 // determined by placePhiStores(). |
3531 std::vector<InstAssign *> PhiAssigns; | 3531 std::vector<InstAssign *> PhiAssigns; |
3532 while (InstAssign *PhiAssign = llvm::dyn_cast<InstAssign>(NextInst)) { | 3532 while (auto *PhiAssign = llvm::dyn_cast<InstAssign>(NextInst)) { |
3533 if (PhiAssign->getDest() == Dest) | 3533 if (PhiAssign->getDest() == Dest) |
3534 return false; | 3534 return false; |
3535 PhiAssigns.push_back(PhiAssign); | 3535 PhiAssigns.push_back(PhiAssign); |
3536 NextInst = Context.getNextInst(I); | 3536 NextInst = Context.getNextInst(I); |
3537 if (!NextInst) | 3537 if (!NextInst) |
3538 return false; | 3538 return false; |
3539 } | 3539 } |
3540 if (InstIcmp *NextCmp = llvm::dyn_cast<InstIcmp>(NextInst)) { | 3540 if (auto *NextCmp = llvm::dyn_cast<InstIcmp>(NextInst)) { |
3541 if (!(NextCmp->getCondition() == InstIcmp::Eq && | 3541 if (!(NextCmp->getCondition() == InstIcmp::Eq && |
3542 ((NextCmp->getSrc(0) == Dest && NextCmp->getSrc(1) == Expected) || | 3542 ((NextCmp->getSrc(0) == Dest && NextCmp->getSrc(1) == Expected) || |
3543 (NextCmp->getSrc(1) == Dest && NextCmp->getSrc(0) == Expected)))) { | 3543 (NextCmp->getSrc(1) == Dest && NextCmp->getSrc(0) == Expected)))) { |
3544 return false; | 3544 return false; |
3545 } | 3545 } |
3546 NextInst = Context.getNextInst(I); | 3546 NextInst = Context.getNextInst(I); |
3547 if (!NextInst) | 3547 if (!NextInst) |
3548 return false; | 3548 return false; |
3549 if (InstBr *NextBr = llvm::dyn_cast<InstBr>(NextInst)) { | 3549 if (auto *NextBr = llvm::dyn_cast<InstBr>(NextInst)) { |
3550 if (!NextBr->isUnconditional() && | 3550 if (!NextBr->isUnconditional() && |
3551 NextCmp->getDest() == NextBr->getCondition() && | 3551 NextCmp->getDest() == NextBr->getCondition() && |
3552 NextBr->isLastUse(NextCmp->getDest())) { | 3552 NextBr->isLastUse(NextCmp->getDest())) { |
3553 lowerAtomicCmpxchg(Dest, PtrToMem, Expected, Desired); | 3553 lowerAtomicCmpxchg(Dest, PtrToMem, Expected, Desired); |
3554 for (size_t i = 0; i < PhiAssigns.size(); ++i) { | 3554 for (size_t i = 0; i < PhiAssigns.size(); ++i) { |
3555 // Lower the phi assignments now, before the branch (same placement | 3555 // Lower the phi assignments now, before the branch (same placement |
3556 // as before). | 3556 // as before). |
3557 InstAssign *PhiAssign = PhiAssigns[i]; | 3557 InstAssign *PhiAssign = PhiAssigns[i]; |
3558 PhiAssign->setDeleted(); | 3558 PhiAssign->setDeleted(); |
3559 lowerAssign(PhiAssign); | 3559 lowerAssign(PhiAssign); |
(...skipping 157 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3717 _mov(T_ebx, loOperand(Val)); | 3717 _mov(T_ebx, loOperand(Val)); |
3718 _mov(T_ecx, hiOperand(Val)); | 3718 _mov(T_ecx, hiOperand(Val)); |
3719 Context.insert(Label); | 3719 Context.insert(Label); |
3720 } | 3720 } |
3721 constexpr bool Locked = true; | 3721 constexpr bool Locked = true; |
3722 _cmpxchg8b(Addr, T_edx, T_eax, T_ecx, T_ebx, Locked); | 3722 _cmpxchg8b(Addr, T_edx, T_eax, T_ecx, T_ebx, Locked); |
3723 _br(Traits::Cond::Br_ne, Label); | 3723 _br(Traits::Cond::Br_ne, Label); |
3724 if (!IsXchg8b) { | 3724 if (!IsXchg8b) { |
3725 // If Val is a variable, model the extended live range of Val through | 3725 // If Val is a variable, model the extended live range of Val through |
3726 // the end of the loop, since it will be re-used by the loop. | 3726 // the end of the loop, since it will be re-used by the loop. |
3727 if (Variable *ValVar = llvm::dyn_cast<Variable>(Val)) { | 3727 if (auto *ValVar = llvm::dyn_cast<Variable>(Val)) { |
3728 Variable *ValLo = llvm::cast<Variable>(loOperand(ValVar)); | 3728 auto *ValLo = llvm::cast<Variable>(loOperand(ValVar)); |
3729 Variable *ValHi = llvm::cast<Variable>(hiOperand(ValVar)); | 3729 auto *ValHi = llvm::cast<Variable>(hiOperand(ValVar)); |
3730 Context.insert(InstFakeUse::create(Func, ValLo)); | 3730 Context.insert(InstFakeUse::create(Func, ValLo)); |
3731 Context.insert(InstFakeUse::create(Func, ValHi)); | 3731 Context.insert(InstFakeUse::create(Func, ValHi)); |
3732 } | 3732 } |
3733 } else { | 3733 } else { |
3734 // For xchg, the loop is slightly smaller and ebx/ecx are used. | 3734 // For xchg, the loop is slightly smaller and ebx/ecx are used. |
3735 Context.insert(InstFakeUse::create(Func, T_ebx)); | 3735 Context.insert(InstFakeUse::create(Func, T_ebx)); |
3736 Context.insert(InstFakeUse::create(Func, T_ecx)); | 3736 Context.insert(InstFakeUse::create(Func, T_ecx)); |
3737 } | 3737 } |
3738 // The address base (if any) is also reused in the loop. | 3738 // The address base (if any) is also reused in the loop. |
3739 if (Variable *Base = Addr->getBase()) | 3739 if (Variable *Base = Addr->getBase()) |
3740 Context.insert(InstFakeUse::create(Func, Base)); | 3740 Context.insert(InstFakeUse::create(Func, Base)); |
3741 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 3741 auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
3742 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 3742 auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
3743 _mov(DestLo, T_eax); | 3743 _mov(DestLo, T_eax); |
3744 _mov(DestHi, T_edx); | 3744 _mov(DestHi, T_edx); |
3745 return; | 3745 return; |
3746 } | 3746 } |
3747 typename Traits::X86OperandMem *Addr = formMemoryOperand(Ptr, Ty); | 3747 typename Traits::X86OperandMem *Addr = formMemoryOperand(Ptr, Ty); |
3748 int32_t Eax; | 3748 int32_t Eax; |
3749 switch (Ty) { | 3749 switch (Ty) { |
3750 default: | 3750 default: |
3751 llvm_unreachable("Bad type for atomicRMW"); | 3751 llvm_unreachable("Bad type for atomicRMW"); |
3752 // fallthrough | 3752 // fallthrough |
(...skipping 15 matching lines...) Expand all Loading... |
3768 // We want to pick a different register for T than Eax, so don't use | 3768 // We want to pick a different register for T than Eax, so don't use |
3769 // _mov(T == nullptr, T_eax). | 3769 // _mov(T == nullptr, T_eax). |
3770 Variable *T = makeReg(Ty); | 3770 Variable *T = makeReg(Ty); |
3771 _mov(T, T_eax); | 3771 _mov(T, T_eax); |
3772 (this->*Op_Lo)(T, Val); | 3772 (this->*Op_Lo)(T, Val); |
3773 constexpr bool Locked = true; | 3773 constexpr bool Locked = true; |
3774 _cmpxchg(Addr, T_eax, T, Locked); | 3774 _cmpxchg(Addr, T_eax, T, Locked); |
3775 _br(Traits::Cond::Br_ne, Label); | 3775 _br(Traits::Cond::Br_ne, Label); |
3776 // If Val is a variable, model the extended live range of Val through | 3776 // If Val is a variable, model the extended live range of Val through |
3777 // the end of the loop, since it will be re-used by the loop. | 3777 // the end of the loop, since it will be re-used by the loop. |
3778 if (Variable *ValVar = llvm::dyn_cast<Variable>(Val)) { | 3778 if (auto *ValVar = llvm::dyn_cast<Variable>(Val)) { |
3779 Context.insert(InstFakeUse::create(Func, ValVar)); | 3779 Context.insert(InstFakeUse::create(Func, ValVar)); |
3780 } | 3780 } |
3781 // The address base (if any) is also reused in the loop. | 3781 // The address base (if any) is also reused in the loop. |
3782 if (Variable *Base = Addr->getBase()) | 3782 if (Variable *Base = Addr->getBase()) |
3783 Context.insert(InstFakeUse::create(Func, Base)); | 3783 Context.insert(InstFakeUse::create(Func, Base)); |
3784 _mov(Dest, T_eax); | 3784 _mov(Dest, T_eax); |
3785 } | 3785 } |
3786 | 3786 |
3787 /// Lowers count {trailing, leading} zeros intrinsic. | 3787 /// Lowers count {trailing, leading} zeros intrinsic. |
3788 /// | 3788 /// |
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3838 } | 3838 } |
3839 _cmov(T_Dest, T, Traits::Cond::Br_ne); | 3839 _cmov(T_Dest, T, Traits::Cond::Br_ne); |
3840 if (!Cttz) { | 3840 if (!Cttz) { |
3841 _xor(T_Dest, ThirtyOne); | 3841 _xor(T_Dest, ThirtyOne); |
3842 } | 3842 } |
3843 if (Traits::Is64Bit || Ty == IceType_i32) { | 3843 if (Traits::Is64Bit || Ty == IceType_i32) { |
3844 _mov(Dest, T_Dest); | 3844 _mov(Dest, T_Dest); |
3845 return; | 3845 return; |
3846 } | 3846 } |
3847 _add(T_Dest, ThirtyTwo); | 3847 _add(T_Dest, ThirtyTwo); |
3848 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 3848 auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
3849 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 3849 auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
3850 // Will be using "test" on this, so we need a registerized variable. | 3850 // Will be using "test" on this, so we need a registerized variable. |
3851 Variable *SecondVar = legalizeToReg(SecondVal); | 3851 Variable *SecondVar = legalizeToReg(SecondVal); |
3852 Variable *T_Dest2 = makeReg(IceType_i32); | 3852 Variable *T_Dest2 = makeReg(IceType_i32); |
3853 if (Cttz) { | 3853 if (Cttz) { |
3854 _bsf(T_Dest2, SecondVar); | 3854 _bsf(T_Dest2, SecondVar); |
3855 } else { | 3855 } else { |
3856 _bsr(T_Dest2, SecondVar); | 3856 _bsr(T_Dest2, SecondVar); |
3857 _xor(T_Dest2, ThirtyOne); | 3857 _xor(T_Dest2, ThirtyOne); |
3858 } | 3858 } |
3859 _test(SecondVar, SecondVar); | 3859 _test(SecondVar, SecondVar); |
(...skipping 700 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
4560 void TargetX86Base<Machine>::lowerLoad(const InstLoad *Load) { | 4560 void TargetX86Base<Machine>::lowerLoad(const InstLoad *Load) { |
4561 // A Load instruction can be treated the same as an Assign instruction, after | 4561 // A Load instruction can be treated the same as an Assign instruction, after |
4562 // the source operand is transformed into an Traits::X86OperandMem operand. | 4562 // the source operand is transformed into an Traits::X86OperandMem operand. |
4563 // Note that the address mode optimization already creates an | 4563 // Note that the address mode optimization already creates an |
4564 // Traits::X86OperandMem operand, so it doesn't need another level of | 4564 // Traits::X86OperandMem operand, so it doesn't need another level of |
4565 // transformation. | 4565 // transformation. |
4566 Variable *DestLoad = Load->getDest(); | 4566 Variable *DestLoad = Load->getDest(); |
4567 Type Ty = DestLoad->getType(); | 4567 Type Ty = DestLoad->getType(); |
4568 Operand *Src0 = formMemoryOperand(Load->getSourceAddress(), Ty); | 4568 Operand *Src0 = formMemoryOperand(Load->getSourceAddress(), Ty); |
4569 doMockBoundsCheck(Src0); | 4569 doMockBoundsCheck(Src0); |
4570 InstAssign *Assign = InstAssign::create(Func, DestLoad, Src0); | 4570 auto *Assign = InstAssign::create(Func, DestLoad, Src0); |
4571 lowerAssign(Assign); | 4571 lowerAssign(Assign); |
4572 } | 4572 } |
4573 | 4573 |
4574 template <class Machine> void TargetX86Base<Machine>::doAddressOptLoad() { | 4574 template <class Machine> void TargetX86Base<Machine>::doAddressOptLoad() { |
4575 Inst *Inst = Context.getCur(); | 4575 Inst *Inst = Context.getCur(); |
4576 Variable *Dest = Inst->getDest(); | 4576 Variable *Dest = Inst->getDest(); |
4577 Operand *Addr = Inst->getSrc(0); | 4577 Operand *Addr = Inst->getSrc(0); |
4578 Variable *Index = nullptr; | 4578 Variable *Index = nullptr; |
4579 ConstantRelocatable *Relocatable = nullptr; | 4579 ConstantRelocatable *Relocatable = nullptr; |
4580 uint16_t Shift = 0; | 4580 uint16_t Shift = 0; |
(...skipping 145 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
4726 // instruction doesn't allow an immediate operand: | 4726 // instruction doesn't allow an immediate operand: |
4727 // mov t, SrcT; cmov_!cond t, SrcF; mov dest, t | 4727 // mov t, SrcT; cmov_!cond t, SrcF; mov dest, t |
4728 if (llvm::isa<Constant>(SrcT) && !llvm::isa<Constant>(SrcF)) { | 4728 if (llvm::isa<Constant>(SrcT) && !llvm::isa<Constant>(SrcF)) { |
4729 std::swap(SrcT, SrcF); | 4729 std::swap(SrcT, SrcF); |
4730 Cond = InstX86Base<Machine>::getOppositeCondition(Cond); | 4730 Cond = InstX86Base<Machine>::getOppositeCondition(Cond); |
4731 } | 4731 } |
4732 if (!Traits::Is64Bit && DestTy == IceType_i64) { | 4732 if (!Traits::Is64Bit && DestTy == IceType_i64) { |
4733 SrcT = legalizeUndef(SrcT); | 4733 SrcT = legalizeUndef(SrcT); |
4734 SrcF = legalizeUndef(SrcF); | 4734 SrcF = legalizeUndef(SrcF); |
4735 // Set the low portion. | 4735 // Set the low portion. |
4736 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 4736 auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
4737 Variable *TLo = nullptr; | 4737 Variable *TLo = nullptr; |
4738 Operand *SrcFLo = legalize(loOperand(SrcF)); | 4738 Operand *SrcFLo = legalize(loOperand(SrcF)); |
4739 _mov(TLo, SrcFLo); | 4739 _mov(TLo, SrcFLo); |
4740 Operand *SrcTLo = legalize(loOperand(SrcT), Legal_Reg | Legal_Mem); | 4740 Operand *SrcTLo = legalize(loOperand(SrcT), Legal_Reg | Legal_Mem); |
4741 _cmov(TLo, SrcTLo, Cond); | 4741 _cmov(TLo, SrcTLo, Cond); |
4742 _mov(DestLo, TLo); | 4742 _mov(DestLo, TLo); |
4743 // Set the high portion. | 4743 // Set the high portion. |
4744 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 4744 auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
4745 Variable *THi = nullptr; | 4745 Variable *THi = nullptr; |
4746 Operand *SrcFHi = legalize(hiOperand(SrcF)); | 4746 Operand *SrcFHi = legalize(hiOperand(SrcF)); |
4747 _mov(THi, SrcFHi); | 4747 _mov(THi, SrcFHi); |
4748 Operand *SrcTHi = legalize(hiOperand(SrcT), Legal_Reg | Legal_Mem); | 4748 Operand *SrcTHi = legalize(hiOperand(SrcT), Legal_Reg | Legal_Mem); |
4749 _cmov(THi, SrcTHi, Cond); | 4749 _cmov(THi, SrcTHi, Cond); |
4750 _mov(DestHi, THi); | 4750 _mov(DestHi, THi); |
4751 return; | 4751 return; |
4752 } | 4752 } |
4753 | 4753 |
4754 assert(DestTy == IceType_i16 || DestTy == IceType_i32 || | 4754 assert(DestTy == IceType_i16 || DestTy == IceType_i32 || |
(...skipping 25 matching lines...) Expand all Loading... |
4780 llvm::cast<typename Traits::X86OperandMem>(loOperand(NewAddr))); | 4780 llvm::cast<typename Traits::X86OperandMem>(loOperand(NewAddr))); |
4781 } else if (isVectorType(Ty)) { | 4781 } else if (isVectorType(Ty)) { |
4782 _storep(legalizeToReg(Value), NewAddr); | 4782 _storep(legalizeToReg(Value), NewAddr); |
4783 } else { | 4783 } else { |
4784 Value = legalize(Value, Legal_Reg | Legal_Imm); | 4784 Value = legalize(Value, Legal_Reg | Legal_Imm); |
4785 _store(Value, NewAddr); | 4785 _store(Value, NewAddr); |
4786 } | 4786 } |
4787 } | 4787 } |
4788 | 4788 |
4789 template <class Machine> void TargetX86Base<Machine>::doAddressOptStore() { | 4789 template <class Machine> void TargetX86Base<Machine>::doAddressOptStore() { |
4790 InstStore *Inst = llvm::cast<InstStore>(Context.getCur()); | 4790 auto *Inst = llvm::cast<InstStore>(Context.getCur()); |
4791 Operand *Data = Inst->getData(); | 4791 Operand *Data = Inst->getData(); |
4792 Operand *Addr = Inst->getAddr(); | 4792 Operand *Addr = Inst->getAddr(); |
4793 Variable *Index = nullptr; | 4793 Variable *Index = nullptr; |
4794 ConstantRelocatable *Relocatable = nullptr; | 4794 ConstantRelocatable *Relocatable = nullptr; |
4795 uint16_t Shift = 0; | 4795 uint16_t Shift = 0; |
4796 int32_t Offset = 0; | 4796 int32_t Offset = 0; |
4797 auto *Base = llvm::dyn_cast<Variable>(Addr); | 4797 auto *Base = llvm::dyn_cast<Variable>(Addr); |
4798 // Vanilla ICE store instructions should not use the segment registers, and | 4798 // Vanilla ICE store instructions should not use the segment registers, and |
4799 // computeAddressOpt only works at the level of Variables and Constants, not | 4799 // computeAddressOpt only works at the level of Variables and Constants, not |
4800 // other Traits::X86OperandMem, so there should be no mention of segment | 4800 // other Traits::X86OperandMem, so there should be no mention of segment |
4801 // registers there either. | 4801 // registers there either. |
4802 const typename Traits::X86OperandMem::SegmentRegisters SegmentReg = | 4802 const typename Traits::X86OperandMem::SegmentRegisters SegmentReg = |
4803 Traits::X86OperandMem::DefaultSegment; | 4803 Traits::X86OperandMem::DefaultSegment; |
4804 if (computeAddressOpt(Func, Inst, Relocatable, Offset, Base, Index, Shift)) { | 4804 if (computeAddressOpt(Func, Inst, Relocatable, Offset, Base, Index, Shift)) { |
4805 Inst->setDeleted(); | 4805 Inst->setDeleted(); |
4806 Constant *OffsetOp = nullptr; | 4806 Constant *OffsetOp = nullptr; |
4807 if (Relocatable == nullptr) { | 4807 if (Relocatable == nullptr) { |
4808 OffsetOp = Ctx->getConstantInt32(Offset); | 4808 OffsetOp = Ctx->getConstantInt32(Offset); |
4809 } else { | 4809 } else { |
4810 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, | 4810 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, |
4811 Relocatable->getName(), | 4811 Relocatable->getName(), |
4812 Relocatable->getSuppressMangling()); | 4812 Relocatable->getSuppressMangling()); |
4813 } | 4813 } |
4814 Addr = Traits::X86OperandMem::create(Func, Data->getType(), Base, OffsetOp, | 4814 Addr = Traits::X86OperandMem::create(Func, Data->getType(), Base, OffsetOp, |
4815 Index, Shift, SegmentReg); | 4815 Index, Shift, SegmentReg); |
4816 InstStore *NewStore = InstStore::create(Func, Data, Addr); | 4816 auto *NewStore = InstStore::create(Func, Data, Addr); |
4817 if (Inst->getDest()) | 4817 if (Inst->getDest()) |
4818 NewStore->setRmwBeacon(Inst->getRmwBeacon()); | 4818 NewStore->setRmwBeacon(Inst->getRmwBeacon()); |
4819 Context.insert(NewStore); | 4819 Context.insert(NewStore); |
4820 } | 4820 } |
4821 } | 4821 } |
4822 | 4822 |
4823 template <class Machine> | 4823 template <class Machine> |
4824 Operand *TargetX86Base<Machine>::lowerCmpRange(Operand *Comparison, | 4824 Operand *TargetX86Base<Machine>::lowerCmpRange(Operand *Comparison, |
4825 uint64_t Min, uint64_t Max) { | 4825 uint64_t Min, uint64_t Max) { |
4826 // TODO(ascull): 64-bit should not reach here but only because it is not | 4826 // TODO(ascull): 64-bit should not reach here but only because it is not |
(...skipping 265 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5092 /// | 5092 /// |
5093 /// %cmp = fcmp/icmp pred <n x ty> %src0, %src1 | 5093 /// %cmp = fcmp/icmp pred <n x ty> %src0, %src1 |
5094 /// %cmp.ext = sext <n x i1> %cmp to <n x ty> | 5094 /// %cmp.ext = sext <n x i1> %cmp to <n x ty> |
5095 /// | 5095 /// |
5096 /// We can eliminate the sext operation by copying the result of pcmpeqd, | 5096 /// We can eliminate the sext operation by copying the result of pcmpeqd, |
5097 /// pcmpgtd, or cmpps (which produce sign extended results) to the result of the | 5097 /// pcmpgtd, or cmpps (which produce sign extended results) to the result of the |
5098 /// sext operation. | 5098 /// sext operation. |
5099 template <class Machine> | 5099 template <class Machine> |
5100 void TargetX86Base<Machine>::eliminateNextVectorSextInstruction( | 5100 void TargetX86Base<Machine>::eliminateNextVectorSextInstruction( |
5101 Variable *SignExtendedResult) { | 5101 Variable *SignExtendedResult) { |
5102 if (InstCast *NextCast = | 5102 if (auto *NextCast = |
5103 llvm::dyn_cast_or_null<InstCast>(Context.getNextInst())) { | 5103 llvm::dyn_cast_or_null<InstCast>(Context.getNextInst())) { |
5104 if (NextCast->getCastKind() == InstCast::Sext && | 5104 if (NextCast->getCastKind() == InstCast::Sext && |
5105 NextCast->getSrc(0) == SignExtendedResult) { | 5105 NextCast->getSrc(0) == SignExtendedResult) { |
5106 NextCast->setDeleted(); | 5106 NextCast->setDeleted(); |
5107 _movp(NextCast->getDest(), legalizeToReg(SignExtendedResult)); | 5107 _movp(NextCast->getDest(), legalizeToReg(SignExtendedResult)); |
5108 // Skip over the instruction. | 5108 // Skip over the instruction. |
5109 Context.advanceNext(); | 5109 Context.advanceNext(); |
5110 } | 5110 } |
5111 } | 5111 } |
5112 } | 5112 } |
(...skipping 654 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5767 // register in x86-64. | 5767 // register in x86-64. |
5768 if (Traits::Is64Bit) { | 5768 if (Traits::Is64Bit) { |
5769 if (llvm::isa<ConstantInteger64>(Const)) { | 5769 if (llvm::isa<ConstantInteger64>(Const)) { |
5770 Variable *V = copyToReg(Const, RegNum); | 5770 Variable *V = copyToReg(Const, RegNum); |
5771 return V; | 5771 return V; |
5772 } | 5772 } |
5773 } | 5773 } |
5774 | 5774 |
5775 // If the operand is an 32 bit constant integer, we should check whether we | 5775 // If the operand is an 32 bit constant integer, we should check whether we |
5776 // need to randomize it or pool it. | 5776 // need to randomize it or pool it. |
5777 if (ConstantInteger32 *C = llvm::dyn_cast<ConstantInteger32>(Const)) { | 5777 if (auto *C = llvm::dyn_cast<ConstantInteger32>(Const)) { |
5778 Operand *NewConst = randomizeOrPoolImmediate(C, RegNum); | 5778 Operand *NewConst = randomizeOrPoolImmediate(C, RegNum); |
5779 if (NewConst != Const) { | 5779 if (NewConst != Const) { |
5780 return NewConst; | 5780 return NewConst; |
5781 } | 5781 } |
5782 } | 5782 } |
5783 | 5783 |
5784 // Convert a scalar floating point constant into an explicit memory | 5784 // Convert a scalar floating point constant into an explicit memory |
5785 // operand. | 5785 // operand. |
5786 if (isScalarFloatingType(Ty)) { | 5786 if (isScalarFloatingType(Ty)) { |
5787 if (auto *ConstFloat = llvm::dyn_cast<ConstantFloat>(Const)) { | 5787 if (auto *ConstFloat = llvm::dyn_cast<ConstantFloat>(Const)) { |
(...skipping 104 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5892 | 5892 |
5893 template <class Machine> | 5893 template <class Machine> |
5894 typename TargetX86Base<Machine>::Traits::X86OperandMem * | 5894 typename TargetX86Base<Machine>::Traits::X86OperandMem * |
5895 TargetX86Base<Machine>::formMemoryOperand(Operand *Opnd, Type Ty, | 5895 TargetX86Base<Machine>::formMemoryOperand(Operand *Opnd, Type Ty, |
5896 bool DoLegalize) { | 5896 bool DoLegalize) { |
5897 auto *Mem = llvm::dyn_cast<typename Traits::X86OperandMem>(Opnd); | 5897 auto *Mem = llvm::dyn_cast<typename Traits::X86OperandMem>(Opnd); |
5898 // It may be the case that address mode optimization already creates an | 5898 // It may be the case that address mode optimization already creates an |
5899 // Traits::X86OperandMem, so in that case it wouldn't need another level of | 5899 // Traits::X86OperandMem, so in that case it wouldn't need another level of |
5900 // transformation. | 5900 // transformation. |
5901 if (!Mem) { | 5901 if (!Mem) { |
5902 Variable *Base = llvm::dyn_cast<Variable>(Opnd); | 5902 auto *Base = llvm::dyn_cast<Variable>(Opnd); |
5903 Constant *Offset = llvm::dyn_cast<Constant>(Opnd); | 5903 auto *Offset = llvm::dyn_cast<Constant>(Opnd); |
5904 assert(Base || Offset); | 5904 assert(Base || Offset); |
5905 if (Offset) { | 5905 if (Offset) { |
5906 // During memory operand building, we do not blind or pool the constant | 5906 // During memory operand building, we do not blind or pool the constant |
5907 // offset, we will work on the whole memory operand later as one entity | 5907 // offset, we will work on the whole memory operand later as one entity |
5908 // later, this save one instruction. By turning blinding and pooling off, | 5908 // later, this save one instruction. By turning blinding and pooling off, |
5909 // we guarantee legalize(Offset) will return a Constant*. | 5909 // we guarantee legalize(Offset) will return a Constant*. |
5910 { | 5910 { |
5911 BoolFlagSaver B(RandomizationPoolingPaused, true); | 5911 BoolFlagSaver B(RandomizationPoolingPaused, true); |
5912 | 5912 |
5913 Offset = llvm::cast<Constant>(legalize(Offset)); | 5913 Offset = llvm::cast<Constant>(legalize(Offset)); |
(...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
6041 // TO: | 6041 // TO: |
6042 // insert: mov imm+cookie, Reg | 6042 // insert: mov imm+cookie, Reg |
6043 // insert: lea -cookie[Reg], Reg | 6043 // insert: lea -cookie[Reg], Reg |
6044 // => Reg | 6044 // => Reg |
6045 // If we have already assigned a phy register, we must come from | 6045 // If we have already assigned a phy register, we must come from |
6046 // advancedPhiLowering()=>lowerAssign(). In this case we should reuse the | 6046 // advancedPhiLowering()=>lowerAssign(). In this case we should reuse the |
6047 // assigned register as this assignment is that start of its use-def | 6047 // assigned register as this assignment is that start of its use-def |
6048 // chain. So we add RegNum argument here. Note we use 'lea' instruction | 6048 // chain. So we add RegNum argument here. Note we use 'lea' instruction |
6049 // instead of 'xor' to avoid affecting the flags. | 6049 // instead of 'xor' to avoid affecting the flags. |
6050 Variable *Reg = makeReg(IceType_i32, RegNum); | 6050 Variable *Reg = makeReg(IceType_i32, RegNum); |
6051 ConstantInteger32 *Integer = llvm::cast<ConstantInteger32>(Immediate); | 6051 auto *Integer = llvm::cast<ConstantInteger32>(Immediate); |
6052 uint32_t Value = Integer->getValue(); | 6052 uint32_t Value = Integer->getValue(); |
6053 uint32_t Cookie = Func->getConstantBlindingCookie(); | 6053 uint32_t Cookie = Func->getConstantBlindingCookie(); |
6054 _mov(Reg, Ctx->getConstantInt(IceType_i32, Cookie + Value)); | 6054 _mov(Reg, Ctx->getConstantInt(IceType_i32, Cookie + Value)); |
6055 Constant *Offset = Ctx->getConstantInt(IceType_i32, 0 - Cookie); | 6055 Constant *Offset = Ctx->getConstantInt(IceType_i32, 0 - Cookie); |
6056 _lea(Reg, Traits::X86OperandMem::create(Func, IceType_i32, Reg, Offset, | 6056 _lea(Reg, Traits::X86OperandMem::create(Func, IceType_i32, Reg, Offset, |
6057 nullptr, 0)); | 6057 nullptr, 0)); |
6058 if (Immediate->getType() != IceType_i32) { | 6058 if (Immediate->getType() != IceType_i32) { |
6059 Variable *TruncReg = makeReg(Immediate->getType(), RegNum); | 6059 Variable *TruncReg = makeReg(Immediate->getType(), RegNum); |
6060 _mov(TruncReg, Reg); | 6060 _mov(TruncReg, Reg); |
6061 return TruncReg; | 6061 return TruncReg; |
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
6104 RandomizationPoolingPaused == true) { | 6104 RandomizationPoolingPaused == true) { |
6105 // immediates randomization/pooling is turned off | 6105 // immediates randomization/pooling is turned off |
6106 return MemOperand; | 6106 return MemOperand; |
6107 } | 6107 } |
6108 | 6108 |
6109 // If this memory operand is already a randomized one, we do not randomize it | 6109 // If this memory operand is already a randomized one, we do not randomize it |
6110 // again. | 6110 // again. |
6111 if (MemOperand->getRandomized()) | 6111 if (MemOperand->getRandomized()) |
6112 return MemOperand; | 6112 return MemOperand; |
6113 | 6113 |
6114 if (Constant *C = llvm::dyn_cast_or_null<Constant>(MemOperand->getOffset())) { | 6114 if (auto *C = llvm::dyn_cast_or_null<Constant>(MemOperand->getOffset())) { |
6115 if (C->shouldBeRandomizedOrPooled(Ctx)) { | 6115 if (C->shouldBeRandomizedOrPooled(Ctx)) { |
6116 // The offset of this mem operand should be blinded or pooled | 6116 // The offset of this mem operand should be blinded or pooled |
6117 Ctx->statsUpdateRPImms(); | 6117 Ctx->statsUpdateRPImms(); |
6118 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == | 6118 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == |
6119 RPI_Randomize) { | 6119 RPI_Randomize) { |
6120 // blind the constant offset | 6120 // blind the constant offset |
6121 // FROM: | 6121 // FROM: |
6122 // offset[base, index, shift] | 6122 // offset[base, index, shift] |
6123 // TO: | 6123 // TO: |
6124 // insert: lea offset+cookie[base], RegTemp | 6124 // insert: lea offset+cookie[base], RegTemp |
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
6206 } | 6206 } |
6207 // the offset is not eligible for blinding or pooling, return the original | 6207 // the offset is not eligible for blinding or pooling, return the original |
6208 // mem operand | 6208 // mem operand |
6209 return MemOperand; | 6209 return MemOperand; |
6210 } | 6210 } |
6211 | 6211 |
6212 } // end of namespace X86Internal | 6212 } // end of namespace X86Internal |
6213 } // end of namespace Ice | 6213 } // end of namespace Ice |
6214 | 6214 |
6215 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H | 6215 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H |
OLD | NEW |