| OLD | NEW |
| 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// | 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// |
| 2 // | 2 // |
| 3 // The Subzero Code Generator | 3 // The Subzero Code Generator |
| 4 // | 4 // |
| 5 // This file is distributed under the University of Illinois Open Source | 5 // This file is distributed under the University of Illinois Open Source |
| 6 // License. See LICENSE.TXT for details. | 6 // License. See LICENSE.TXT for details. |
| 7 // | 7 // |
| 8 //===----------------------------------------------------------------------===// | 8 //===----------------------------------------------------------------------===// |
| 9 /// | 9 /// |
| 10 /// \file | 10 /// \file |
| (...skipping 290 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 301 } | 301 } |
| 302 | 302 |
| 303 template <typename TraitsType> | 303 template <typename TraitsType> |
| 304 void TargetX86Base<TraitsType>::initNodeForLowering(CfgNode *Node) { | 304 void TargetX86Base<TraitsType>::initNodeForLowering(CfgNode *Node) { |
| 305 FoldingInfo.init(Node); | 305 FoldingInfo.init(Node); |
| 306 FoldingInfo.dump(Func); | 306 FoldingInfo.dump(Func); |
| 307 } | 307 } |
| 308 | 308 |
| 309 template <typename TraitsType> | 309 template <typename TraitsType> |
| 310 TargetX86Base<TraitsType>::TargetX86Base(Cfg *Func) | 310 TargetX86Base<TraitsType>::TargetX86Base(Cfg *Func) |
| 311 : TargetLowering(Func) { | 311 : TargetLowering(Func), NeedSandboxing(Ctx->getFlags().getUseSandboxing()) { |
| 312 static_assert( | 312 static_assert( |
| 313 (Traits::InstructionSet::End - Traits::InstructionSet::Begin) == | 313 (Traits::InstructionSet::End - Traits::InstructionSet::Begin) == |
| 314 (TargetInstructionSet::X86InstructionSet_End - | 314 (TargetInstructionSet::X86InstructionSet_End - |
| 315 TargetInstructionSet::X86InstructionSet_Begin), | 315 TargetInstructionSet::X86InstructionSet_Begin), |
| 316 "Traits::InstructionSet range different from TargetInstructionSet"); | 316 "Traits::InstructionSet range different from TargetInstructionSet"); |
| 317 if (Func->getContext()->getFlags().getTargetInstructionSet() != | 317 if (Func->getContext()->getFlags().getTargetInstructionSet() != |
| 318 TargetInstructionSet::BaseInstructionSet) { | 318 TargetInstructionSet::BaseInstructionSet) { |
| 319 InstructionSet = static_cast<InstructionSetEnum>( | 319 InstructionSet = static_cast<InstructionSetEnum>( |
| 320 (Func->getContext()->getFlags().getTargetInstructionSet() - | 320 (Func->getContext()->getFlags().getTargetInstructionSet() - |
| 321 TargetInstructionSet::X86InstructionSet_Begin) + | 321 TargetInstructionSet::X86InstructionSet_Begin) + |
| 322 Traits::InstructionSet::Begin); | 322 Traits::InstructionSet::Begin); |
| 323 } | 323 } |
| 324 } | 324 } |
| 325 | 325 |
| 326 template <typename TraitsType> | 326 template <typename TraitsType> |
| 327 void TargetX86Base<TraitsType>::staticInit(const ClFlags &Flags) { | 327 void TargetX86Base<TraitsType>::staticInit(const ClFlags &Flags) { |
| 328 Traits::initRegisterSet(&TypeToRegisterSet, &RegisterAliases, &ScratchRegs); | 328 Traits::initRegisterSet(Flags, &TypeToRegisterSet, &RegisterAliases, |
| 329 &ScratchRegs); |
| 329 PcRelFixup = Traits::FK_PcRel; | 330 PcRelFixup = Traits::FK_PcRel; |
| 330 AbsFixup = Flags.getUseNonsfi() ? Traits::FK_Gotoff : Traits::FK_Abs; | 331 AbsFixup = Flags.getUseNonsfi() ? Traits::FK_Gotoff : Traits::FK_Abs; |
| 331 } | 332 } |
| 332 | 333 |
| 333 template <typename TraitsType> void TargetX86Base<TraitsType>::translateO2() { | 334 template <typename TraitsType> void TargetX86Base<TraitsType>::translateO2() { |
| 334 TimerMarker T(TimerStack::TT_O2, Func); | 335 TimerMarker T(TimerStack::TT_O2, Func); |
| 335 | 336 |
| 336 if (!Traits::Is64Bit && Func->getContext()->getFlags().getUseNonsfi()) { | 337 if (!Traits::Is64Bit && Func->getContext()->getFlags().getUseNonsfi()) { |
| 337 GotVar = Func->makeVariable(IceType_i32); | 338 GotVar = Func->makeVariable(IceType_i32); |
| 338 } | 339 } |
| 340 |
| 341 if (NeedSandboxing) { |
| 342 initSandbox(); |
| 343 } |
| 344 |
| 339 genTargetHelperCalls(); | 345 genTargetHelperCalls(); |
| 340 Func->dump("After target helper call insertion"); | 346 Func->dump("After target helper call insertion"); |
| 341 | 347 |
| 342 // Merge Alloca instructions, and lay out the stack. | 348 // Merge Alloca instructions, and lay out the stack. |
| 343 static constexpr bool SortAndCombineAllocas = true; | 349 static constexpr bool SortAndCombineAllocas = true; |
| 344 Func->processAllocas(SortAndCombineAllocas); | 350 Func->processAllocas(SortAndCombineAllocas); |
| 345 Func->dump("After Alloca processing"); | 351 Func->dump("After Alloca processing"); |
| 346 | 352 |
| 347 if (!Ctx->getFlags().getPhiEdgeSplit()) { | 353 if (!Ctx->getFlags().getPhiEdgeSplit()) { |
| 348 // Lower Phi instructions. | 354 // Lower Phi instructions. |
| (...skipping 100 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 449 // particular, no transformations that insert or reorder CfgNodes should be | 455 // particular, no transformations that insert or reorder CfgNodes should be |
| 450 // done after branch optimization. We go ahead and do it before nop insertion | 456 // done after branch optimization. We go ahead and do it before nop insertion |
| 451 // to reduce the amount of work needed for searching for opportunities. | 457 // to reduce the amount of work needed for searching for opportunities. |
| 452 Func->doBranchOpt(); | 458 Func->doBranchOpt(); |
| 453 Func->dump("After branch optimization"); | 459 Func->dump("After branch optimization"); |
| 454 | 460 |
| 455 // Nop insertion if -nop-insertion is enabled. | 461 // Nop insertion if -nop-insertion is enabled. |
| 456 Func->doNopInsertion(); | 462 Func->doNopInsertion(); |
| 457 | 463 |
| 458 // Mark nodes that require sandbox alignment | 464 // Mark nodes that require sandbox alignment |
| 459 if (Ctx->getFlags().getUseSandboxing()) | 465 if (NeedSandboxing) { |
| 460 Func->markNodesForSandboxing(); | 466 Func->markNodesForSandboxing(); |
| 467 } |
| 461 } | 468 } |
| 462 | 469 |
| 463 template <typename TraitsType> void TargetX86Base<TraitsType>::translateOm1() { | 470 template <typename TraitsType> void TargetX86Base<TraitsType>::translateOm1() { |
| 464 TimerMarker T(TimerStack::TT_Om1, Func); | 471 TimerMarker T(TimerStack::TT_Om1, Func); |
| 465 | 472 |
| 466 if (!Traits::Is64Bit && Func->getContext()->getFlags().getUseNonsfi()) { | 473 if (!Traits::Is64Bit && Func->getContext()->getFlags().getUseNonsfi()) { |
| 467 GotVar = Func->makeVariable(IceType_i32); | 474 GotVar = Func->makeVariable(IceType_i32); |
| 468 } | 475 } |
| 476 |
| 477 if (NeedSandboxing) { |
| 478 initSandbox(); |
| 479 } |
| 480 |
| 469 genTargetHelperCalls(); | 481 genTargetHelperCalls(); |
| 470 | 482 |
| 471 // Do not merge Alloca instructions, and lay out the stack. | 483 // Do not merge Alloca instructions, and lay out the stack. |
| 472 static constexpr bool SortAndCombineAllocas = false; | 484 static constexpr bool SortAndCombineAllocas = false; |
| 473 Func->processAllocas(SortAndCombineAllocas); | 485 Func->processAllocas(SortAndCombineAllocas); |
| 474 Func->dump("After Alloca processing"); | 486 Func->dump("After Alloca processing"); |
| 475 | 487 |
| 476 Func->placePhiLoads(); | 488 Func->placePhiLoads(); |
| 477 if (Func->hasError()) | 489 if (Func->hasError()) |
| 478 return; | 490 return; |
| (...skipping 22 matching lines...) Expand all Loading... |
| 501 return; | 513 return; |
| 502 Func->dump("After stack frame mapping"); | 514 Func->dump("After stack frame mapping"); |
| 503 | 515 |
| 504 // Shuffle basic block order if -reorder-basic-blocks is enabled. | 516 // Shuffle basic block order if -reorder-basic-blocks is enabled. |
| 505 Func->shuffleNodes(); | 517 Func->shuffleNodes(); |
| 506 | 518 |
| 507 // Nop insertion if -nop-insertion is enabled. | 519 // Nop insertion if -nop-insertion is enabled. |
| 508 Func->doNopInsertion(); | 520 Func->doNopInsertion(); |
| 509 | 521 |
| 510 // Mark nodes that require sandbox alignment | 522 // Mark nodes that require sandbox alignment |
| 511 if (Ctx->getFlags().getUseSandboxing()) | 523 if (NeedSandboxing) |
| 512 Func->markNodesForSandboxing(); | 524 Func->markNodesForSandboxing(); |
| 513 } | 525 } |
| 514 | 526 |
| 515 inline bool canRMW(const InstArithmetic *Arith) { | 527 inline bool canRMW(const InstArithmetic *Arith) { |
| 516 Type Ty = Arith->getDest()->getType(); | 528 Type Ty = Arith->getDest()->getType(); |
| 517 // X86 vector instructions write to a register and have no RMW option. | 529 // X86 vector instructions write to a register and have no RMW option. |
| 518 if (isVectorType(Ty)) | 530 if (isVectorType(Ty)) |
| 519 return false; | 531 return false; |
| 520 bool isI64 = Ty == IceType_i64; | 532 bool isI64 = Ty == IceType_i64; |
| 521 | 533 |
| (...skipping 281 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 803 IceString TargetX86Base<TraitsType>::getRegName(SizeT RegNum, Type Ty) const { | 815 IceString TargetX86Base<TraitsType>::getRegName(SizeT RegNum, Type Ty) const { |
| 804 return Traits::getRegName(Traits::getGprForType(Ty, RegNum)); | 816 return Traits::getRegName(Traits::getGprForType(Ty, RegNum)); |
| 805 } | 817 } |
| 806 | 818 |
| 807 template <typename TraitsType> | 819 template <typename TraitsType> |
| 808 void TargetX86Base<TraitsType>::emitVariable(const Variable *Var) const { | 820 void TargetX86Base<TraitsType>::emitVariable(const Variable *Var) const { |
| 809 if (!BuildDefs::dump()) | 821 if (!BuildDefs::dump()) |
| 810 return; | 822 return; |
| 811 Ostream &Str = Ctx->getStrEmit(); | 823 Ostream &Str = Ctx->getStrEmit(); |
| 812 if (Var->hasReg()) { | 824 if (Var->hasReg()) { |
| 813 Str << "%" << getRegName(Var->getRegNum(), Var->getType()); | 825 const bool Is64BitSandboxing = Traits::Is64Bit && NeedSandboxing; |
| 826 const Type VarType = (Var->isRematerializable() && Is64BitSandboxing) |
| 827 ? IceType_i64 |
| 828 : Var->getType(); |
| 829 Str << "%" << getRegName(Var->getRegNum(), VarType); |
| 814 return; | 830 return; |
| 815 } | 831 } |
| 816 if (Var->mustHaveReg()) { | 832 if (Var->mustHaveReg()) { |
| 817 llvm::report_fatal_error( | 833 llvm::report_fatal_error( |
| 818 "Infinite-weight Variable has no register assigned"); | 834 "Infinite-weight Variable has no register assigned"); |
| 819 } | 835 } |
| 820 const int32_t Offset = Var->getStackOffset(); | 836 const int32_t Offset = Var->getStackOffset(); |
| 821 int32_t BaseRegNum = Var->getBaseRegNum(); | 837 int32_t BaseRegNum = Var->getBaseRegNum(); |
| 822 if (BaseRegNum == Variable::NoRegister) | 838 if (BaseRegNum == Variable::NoRegister) |
| 823 BaseRegNum = getFrameOrStackReg(); | 839 BaseRegNum = getFrameOrStackReg(); |
| (...skipping 92 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 916 return Var64On32->getLo(); | 932 return Var64On32->getLo(); |
| 917 if (auto *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { | 933 if (auto *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { |
| 918 auto *ConstInt = llvm::dyn_cast<ConstantInteger32>( | 934 auto *ConstInt = llvm::dyn_cast<ConstantInteger32>( |
| 919 Ctx->getConstantInt32(static_cast<int32_t>(Const->getValue()))); | 935 Ctx->getConstantInt32(static_cast<int32_t>(Const->getValue()))); |
| 920 // Check if we need to blind/pool the constant. | 936 // Check if we need to blind/pool the constant. |
| 921 return legalize(ConstInt); | 937 return legalize(ConstInt); |
| 922 } | 938 } |
| 923 if (auto *Mem = llvm::dyn_cast<X86OperandMem>(Operand)) { | 939 if (auto *Mem = llvm::dyn_cast<X86OperandMem>(Operand)) { |
| 924 auto *MemOperand = X86OperandMem::create( | 940 auto *MemOperand = X86OperandMem::create( |
| 925 Func, IceType_i32, Mem->getBase(), Mem->getOffset(), Mem->getIndex(), | 941 Func, IceType_i32, Mem->getBase(), Mem->getOffset(), Mem->getIndex(), |
| 926 Mem->getShift(), Mem->getSegmentRegister(), Mem->getIsPIC()); | 942 Mem->getShift(), Mem->getSegmentRegister(), Mem->getIsRebased()); |
| 927 // Test if we should randomize or pool the offset, if so randomize it or | 943 // Test if we should randomize or pool the offset, if so randomize it or |
| 928 // pool it then create mem operand with the blinded/pooled constant. | 944 // pool it then create mem operand with the blinded/pooled constant. |
| 929 // Otherwise, return the mem operand as ordinary mem operand. | 945 // Otherwise, return the mem operand as ordinary mem operand. |
| 930 return legalize(MemOperand); | 946 return legalize(MemOperand); |
| 931 } | 947 } |
| 932 llvm_unreachable("Unsupported operand type"); | 948 llvm_unreachable("Unsupported operand type"); |
| 933 return nullptr; | 949 return nullptr; |
| 934 } | 950 } |
| 935 | 951 |
| 936 template <typename TraitsType> | 952 template <typename TraitsType> |
| (...skipping 19 matching lines...) Expand all Loading... |
| 956 } else if (auto *IntOffset = llvm::dyn_cast<ConstantInteger32>(Offset)) { | 972 } else if (auto *IntOffset = llvm::dyn_cast<ConstantInteger32>(Offset)) { |
| 957 Offset = Ctx->getConstantInt32(4 + IntOffset->getValue()); | 973 Offset = Ctx->getConstantInt32(4 + IntOffset->getValue()); |
| 958 } else if (auto *SymOffset = llvm::dyn_cast<ConstantRelocatable>(Offset)) { | 974 } else if (auto *SymOffset = llvm::dyn_cast<ConstantRelocatable>(Offset)) { |
| 959 assert(!Utils::WouldOverflowAdd(SymOffset->getOffset(), 4)); | 975 assert(!Utils::WouldOverflowAdd(SymOffset->getOffset(), 4)); |
| 960 Offset = | 976 Offset = |
| 961 Ctx->getConstantSym(4 + SymOffset->getOffset(), SymOffset->getName(), | 977 Ctx->getConstantSym(4 + SymOffset->getOffset(), SymOffset->getName(), |
| 962 SymOffset->getSuppressMangling()); | 978 SymOffset->getSuppressMangling()); |
| 963 } | 979 } |
| 964 auto *MemOperand = X86OperandMem::create( | 980 auto *MemOperand = X86OperandMem::create( |
| 965 Func, IceType_i32, Mem->getBase(), Offset, Mem->getIndex(), | 981 Func, IceType_i32, Mem->getBase(), Offset, Mem->getIndex(), |
| 966 Mem->getShift(), Mem->getSegmentRegister(), Mem->getIsPIC()); | 982 Mem->getShift(), Mem->getSegmentRegister(), Mem->getIsRebased()); |
| 967 // Test if the Offset is an eligible i32 constants for randomization and | 983 // Test if the Offset is an eligible i32 constants for randomization and |
| 968 // pooling. Blind/pool it if it is. Otherwise return as oridinary mem | 984 // pooling. Blind/pool it if it is. Otherwise return as oridinary mem |
| 969 // operand. | 985 // operand. |
| 970 return legalize(MemOperand); | 986 return legalize(MemOperand); |
| 971 } | 987 } |
| 972 llvm_unreachable("Unsupported operand type"); | 988 llvm_unreachable("Unsupported operand type"); |
| 973 return nullptr; | 989 return nullptr; |
| 974 } | 990 } |
| 975 | 991 |
| 976 template <typename TraitsType> | 992 template <typename TraitsType> |
| 977 llvm::SmallBitVector | 993 llvm::SmallBitVector |
| 978 TargetX86Base<TraitsType>::getRegisterSet(RegSetMask Include, | 994 TargetX86Base<TraitsType>::getRegisterSet(RegSetMask Include, |
| 979 RegSetMask Exclude) const { | 995 RegSetMask Exclude) const { |
| 980 return Traits::getRegisterSet(Include, Exclude); | 996 return Traits::getRegisterSet(Ctx->getFlags(), Include, Exclude); |
| 981 } | 997 } |
| 982 | 998 |
| 983 template <typename TraitsType> | 999 template <typename TraitsType> |
| 984 void TargetX86Base<TraitsType>::initGotVarIfNeeded() { | 1000 void TargetX86Base<TraitsType>::initGotVarIfNeeded() { |
| 985 if (!Func->getContext()->getFlags().getUseNonsfi()) | 1001 if (!Func->getContext()->getFlags().getUseNonsfi()) |
| 986 return; | 1002 return; |
| 987 if (Traits::Is64Bit) { | 1003 if (Traits::Is64Bit) { |
| 988 // Probably no implementation is needed, but error to be safe for now. | 1004 // Probably no implementation is needed, but error to be safe for now. |
| 989 llvm::report_fatal_error( | 1005 llvm::report_fatal_error( |
| 990 "Need to implement initGotVarIfNeeded() for 64-bit."); | 1006 "Need to implement initGotVarIfNeeded() for 64-bit."); |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1030 _and(esp, Ctx->getConstantInt32(-Alignment)); | 1046 _and(esp, Ctx->getConstantInt32(-Alignment)); |
| 1031 } | 1047 } |
| 1032 | 1048 |
| 1033 Variable *Dest = Inst->getDest(); | 1049 Variable *Dest = Inst->getDest(); |
| 1034 Operand *TotalSize = legalize(Inst->getSizeInBytes()); | 1050 Operand *TotalSize = legalize(Inst->getSizeInBytes()); |
| 1035 | 1051 |
| 1036 if (const auto *ConstantTotalSize = | 1052 if (const auto *ConstantTotalSize = |
| 1037 llvm::dyn_cast<ConstantInteger32>(TotalSize)) { | 1053 llvm::dyn_cast<ConstantInteger32>(TotalSize)) { |
| 1038 const uint32_t Value = | 1054 const uint32_t Value = |
| 1039 Utils::applyAlignment(ConstantTotalSize->getValue(), Alignment); | 1055 Utils::applyAlignment(ConstantTotalSize->getValue(), Alignment); |
| 1040 if (!UseFramePointer) { | 1056 if (UseFramePointer) { |
| 1057 _sub_sp(Ctx->getConstantInt32(Value)); |
| 1058 } else { |
| 1041 // If we don't need a Frame Pointer, this alloca has a known offset to the | 1059 // If we don't need a Frame Pointer, this alloca has a known offset to the |
| 1042 // stack pointer. We don't need adjust the stack pointer, nor assign any | 1060 // stack pointer. We don't need adjust the stack pointer, nor assign any |
| 1043 // value to Dest, as Dest is rematerializable. | 1061 // value to Dest, as Dest is rematerializable. |
| 1044 assert(Dest->isRematerializable()); | 1062 assert(Dest->isRematerializable()); |
| 1045 FixedAllocaSizeBytes += Value; | 1063 FixedAllocaSizeBytes += Value; |
| 1046 Context.insert<InstFakeDef>(Dest); | 1064 Context.insert<InstFakeDef>(Dest); |
| 1047 } else { | |
| 1048 _sub(esp, Ctx->getConstantInt32(Value)); | |
| 1049 } | 1065 } |
| 1050 } else { | 1066 } else { |
| 1051 // Non-constant sizes need to be adjusted to the next highest multiple of | 1067 // Non-constant sizes need to be adjusted to the next highest multiple of |
| 1052 // the required alignment at runtime. | 1068 // the required alignment at runtime. |
| 1053 Variable *T = makeReg(Traits::WordType); | 1069 Variable *T = nullptr; |
| 1054 if (Traits::Is64Bit && TotalSize->getType() != IceType_i64) { | 1070 if (Traits::Is64Bit && TotalSize->getType() != IceType_i64 && |
| 1071 !NeedSandboxing) { |
| 1072 T = makeReg(IceType_i64); |
| 1055 _movzx(T, TotalSize); | 1073 _movzx(T, TotalSize); |
| 1056 } else { | 1074 } else { |
| 1075 T = makeReg(IceType_i32); |
| 1057 _mov(T, TotalSize); | 1076 _mov(T, TotalSize); |
| 1058 } | 1077 } |
| 1059 _add(T, Ctx->getConstantInt32(Alignment - 1)); | 1078 _add(T, Ctx->getConstantInt32(Alignment - 1)); |
| 1060 _and(T, Ctx->getConstantInt32(-Alignment)); | 1079 _and(T, Ctx->getConstantInt32(-Alignment)); |
| 1061 _sub(esp, T); | 1080 _sub_sp(T); |
| 1062 } | 1081 } |
| 1063 // Add enough to the returned address to account for the out args area. | 1082 // Add enough to the returned address to account for the out args area. |
| 1064 uint32_t OutArgsSize = maxOutArgsSizeBytes(); | 1083 uint32_t OutArgsSize = maxOutArgsSizeBytes(); |
| 1065 if (OutArgsSize > 0) { | 1084 if (OutArgsSize > 0) { |
| 1066 Variable *T = makeReg(IceType_i32); | 1085 Variable *T = makeReg(IceType_i32); |
| 1067 auto *CalculateOperand = X86OperandMem::create( | 1086 auto *CalculateOperand = X86OperandMem::create( |
| 1068 Func, IceType_i32, esp, Ctx->getConstantInt(IceType_i32, OutArgsSize)); | 1087 Func, IceType_void, esp, Ctx->getConstantInt(IceType_i32, OutArgsSize)); |
| 1069 _lea(T, CalculateOperand); | 1088 _lea(T, CalculateOperand); |
| 1070 _mov(Dest, T); | 1089 _mov(Dest, T); |
| 1071 } else { | 1090 } else { |
| 1072 _mov(Dest, esp); | 1091 _mov(Dest, esp); |
| 1073 } | 1092 } |
| 1074 } | 1093 } |
| 1075 | 1094 |
| 1076 /// Strength-reduce scalar integer multiplication by a constant (for i32 or | 1095 /// Strength-reduce scalar integer multiplication by a constant (for i32 or |
| 1077 /// narrower) for certain constants. The lea instruction can be used to multiply | 1096 /// narrower) for certain constants. The lea instruction can be used to multiply |
| 1078 /// by 3, 5, or 9, and the lsh instruction can be used to multiply by powers of | 1097 /// by 3, 5, or 9, and the lsh instruction can be used to multiply by powers of |
| 1079 /// 2. These can be combined such that e.g. multiplying by 100 can be done as 2 | 1098 /// 2. These can be combined such that e.g. multiplying by 100 can be done as 2 |
| 1080 /// lea-based multiplies by 5, combined with left-shifting by 2. | 1099 /// lea-based multiplies by 5, combined with left-shifting by 2. |
| 1081 template <typename TraitsType> | 1100 template <typename TraitsType> |
| 1082 bool TargetX86Base<TraitsType>::optimizeScalarMul(Variable *Dest, Operand *Src0, | 1101 bool TargetX86Base<TraitsType>::optimizeScalarMul(Variable *Dest, Operand *Src0, |
| 1083 int32_t Src1) { | 1102 int32_t Src1) { |
| 1084 // Disable this optimization for Om1 and O0, just to keep things simple | 1103 // Disable this optimization for Om1 and O0, just to keep things simple |
| 1085 // there. | 1104 // there. |
| 1086 if (Ctx->getFlags().getOptLevel() < Opt_1) | 1105 if (Ctx->getFlags().getOptLevel() < Opt_1) |
| 1087 return false; | 1106 return false; |
| 1088 Type Ty = Dest->getType(); | 1107 Type Ty = Dest->getType(); |
| 1089 Variable *T = nullptr; | |
| 1090 if (Src1 == -1) { | 1108 if (Src1 == -1) { |
| 1109 Variable *T = nullptr; |
| 1091 _mov(T, Src0); | 1110 _mov(T, Src0); |
| 1092 _neg(T); | 1111 _neg(T); |
| 1093 _mov(Dest, T); | 1112 _mov(Dest, T); |
| 1094 return true; | 1113 return true; |
| 1095 } | 1114 } |
| 1096 if (Src1 == 0) { | 1115 if (Src1 == 0) { |
| 1097 _mov(Dest, Ctx->getConstantZero(Ty)); | 1116 _mov(Dest, Ctx->getConstantZero(Ty)); |
| 1098 return true; | 1117 return true; |
| 1099 } | 1118 } |
| 1100 if (Src1 == 1) { | 1119 if (Src1 == 1) { |
| 1120 Variable *T = nullptr; |
| 1101 _mov(T, Src0); | 1121 _mov(T, Src0); |
| 1102 _mov(Dest, T); | 1122 _mov(Dest, T); |
| 1103 return true; | 1123 return true; |
| 1104 } | 1124 } |
| 1105 // Don't bother with the edge case where Src1 == MININT. | 1125 // Don't bother with the edge case where Src1 == MININT. |
| 1106 if (Src1 == -Src1) | 1126 if (Src1 == -Src1) |
| 1107 return false; | 1127 return false; |
| 1108 const bool Src1IsNegative = Src1 < 0; | 1128 const bool Src1IsNegative = Src1 < 0; |
| 1109 if (Src1IsNegative) | 1129 if (Src1IsNegative) |
| 1110 Src1 = -Src1; | 1130 Src1 = -Src1; |
| (...skipping 17 matching lines...) Expand all Loading... |
| 1128 Src1 /= 3; | 1148 Src1 /= 3; |
| 1129 } else if (Src1 % 2 == 0) { | 1149 } else if (Src1 % 2 == 0) { |
| 1130 if (Count2 == 0) | 1150 if (Count2 == 0) |
| 1131 ++CountOps; | 1151 ++CountOps; |
| 1132 ++Count2; | 1152 ++Count2; |
| 1133 Src1 /= 2; | 1153 Src1 /= 2; |
| 1134 } else { | 1154 } else { |
| 1135 return false; | 1155 return false; |
| 1136 } | 1156 } |
| 1137 } | 1157 } |
| 1138 // Lea optimization only works for i32 type, not i1/i8/i16/i64. | 1158 // Lea optimization only works for i16 and i32 types, not i8. |
| 1139 if (Ty != IceType_i32 && (Count3 || Count5 || Count9)) | 1159 if (Ty != IceType_i32 && !(Traits::Is64Bit && Ty == IceType_i64) && |
| 1160 (Count3 || Count5 || Count9)) |
| 1140 return false; | 1161 return false; |
| 1141 // Limit the number of lea/shl operations for a single multiply, to a | 1162 // Limit the number of lea/shl operations for a single multiply, to a |
| 1142 // somewhat arbitrary choice of 3. | 1163 // somewhat arbitrary choice of 3. |
| 1143 constexpr uint32_t MaxOpsForOptimizedMul = 3; | 1164 constexpr uint32_t MaxOpsForOptimizedMul = 3; |
| 1144 if (CountOps > MaxOpsForOptimizedMul) | 1165 if (CountOps > MaxOpsForOptimizedMul) |
| 1145 return false; | 1166 return false; |
| 1146 _mov(T, Src0); | 1167 Variable *T = makeReg(Traits::WordType); |
| 1168 if (typeWidthInBytes(Src0->getType()) < typeWidthInBytes(T->getType())) { |
| 1169 _movzx(T, Src0); |
| 1170 } else { |
| 1171 _mov(T, Src0); |
| 1172 } |
| 1147 Constant *Zero = Ctx->getConstantZero(IceType_i32); | 1173 Constant *Zero = Ctx->getConstantZero(IceType_i32); |
| 1148 for (uint32_t i = 0; i < Count9; ++i) { | 1174 for (uint32_t i = 0; i < Count9; ++i) { |
| 1149 constexpr uint16_t Shift = 3; // log2(9-1) | 1175 constexpr uint16_t Shift = 3; // log2(9-1) |
| 1150 _lea(T, X86OperandMem::create(Func, IceType_void, T, Zero, T, Shift)); | 1176 _lea(T, X86OperandMem::create(Func, IceType_void, T, Zero, T, Shift)); |
| 1151 } | 1177 } |
| 1152 for (uint32_t i = 0; i < Count5; ++i) { | 1178 for (uint32_t i = 0; i < Count5; ++i) { |
| 1153 constexpr uint16_t Shift = 2; // log2(5-1) | 1179 constexpr uint16_t Shift = 2; // log2(5-1) |
| 1154 _lea(T, X86OperandMem::create(Func, IceType_void, T, Zero, T, Shift)); | 1180 _lea(T, X86OperandMem::create(Func, IceType_void, T, Zero, T, Shift)); |
| 1155 } | 1181 } |
| 1156 for (uint32_t i = 0; i < Count3; ++i) { | 1182 for (uint32_t i = 0; i < Count3; ++i) { |
| (...skipping 2379 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3536 } | 3562 } |
| 3537 case Intrinsics::Memmove: { | 3563 case Intrinsics::Memmove: { |
| 3538 lowerMemmove(Instr->getArg(0), Instr->getArg(1), Instr->getArg(2)); | 3564 lowerMemmove(Instr->getArg(0), Instr->getArg(1), Instr->getArg(2)); |
| 3539 return; | 3565 return; |
| 3540 } | 3566 } |
| 3541 case Intrinsics::Memset: { | 3567 case Intrinsics::Memset: { |
| 3542 lowerMemset(Instr->getArg(0), Instr->getArg(1), Instr->getArg(2)); | 3568 lowerMemset(Instr->getArg(0), Instr->getArg(1), Instr->getArg(2)); |
| 3543 return; | 3569 return; |
| 3544 } | 3570 } |
| 3545 case Intrinsics::NaClReadTP: { | 3571 case Intrinsics::NaClReadTP: { |
| 3546 if (Ctx->getFlags().getUseSandboxing()) { | 3572 if (NeedSandboxing) { |
| 3547 Operand *Src = | 3573 Operand *Src = |
| 3548 dispatchToConcrete(&ConcreteTarget::createNaClReadTPSrcOperand); | 3574 dispatchToConcrete(&ConcreteTarget::createNaClReadTPSrcOperand); |
| 3549 Variable *Dest = Instr->getDest(); | 3575 Variable *Dest = Instr->getDest(); |
| 3550 Variable *T = nullptr; | 3576 Variable *T = nullptr; |
| 3551 _mov(T, Src); | 3577 _mov(T, Src); |
| 3552 _mov(Dest, T); | 3578 _mov(Dest, T); |
| 3553 } else { | 3579 } else { |
| 3554 InstCall *Call = makeHelperCall(H_call_read_tp, Instr->getDest(), 0); | 3580 InstCall *Call = makeHelperCall(H_call_read_tp, Instr->getDest(), 0); |
| 3555 lowerCall(Call); | 3581 lowerCall(Call); |
| 3556 } | 3582 } |
| 3557 return; | 3583 return; |
| 3558 } | 3584 } |
| 3559 case Intrinsics::Setjmp: { | 3585 case Intrinsics::Setjmp: { |
| 3560 InstCall *Call = makeHelperCall(H_call_setjmp, Instr->getDest(), 1); | 3586 InstCall *Call = makeHelperCall(H_call_setjmp, Instr->getDest(), 1); |
| 3561 Call->addArg(Instr->getArg(0)); | 3587 Call->addArg(Instr->getArg(0)); |
| 3562 lowerCall(Call); | 3588 lowerCall(Call); |
| 3563 return; | 3589 return; |
| 3564 } | 3590 } |
| 3565 case Intrinsics::Sqrt: { | 3591 case Intrinsics::Sqrt: { |
| 3566 Operand *Src = legalize(Instr->getArg(0)); | 3592 Operand *Src = legalize(Instr->getArg(0)); |
| 3567 Variable *Dest = Instr->getDest(); | 3593 Variable *Dest = Instr->getDest(); |
| 3568 Variable *T = makeReg(Dest->getType()); | 3594 Variable *T = makeReg(Dest->getType()); |
| 3569 _sqrtss(T, Src); | 3595 _sqrtss(T, Src); |
| 3570 _mov(Dest, T); | 3596 _mov(Dest, T); |
| 3571 return; | 3597 return; |
| 3572 } | 3598 } |
| 3573 case Intrinsics::Stacksave: { | 3599 case Intrinsics::Stacksave: { |
| 3574 Variable *esp = | 3600 if (!Traits::Is64Bit || !NeedSandboxing) { |
| 3575 Func->getTarget()->getPhysicalRegister(getStackReg(), Traits::WordType); | 3601 Variable *esp = Func->getTarget()->getPhysicalRegister(getStackReg(), |
| 3602 Traits::WordType); |
| 3603 Variable *Dest = Instr->getDest(); |
| 3604 _mov(Dest, esp); |
| 3605 return; |
| 3606 } |
| 3607 Variable *esp = Func->getTarget()->getPhysicalRegister( |
| 3608 Traits::RegisterSet::Reg_esp, IceType_i32); |
| 3576 Variable *Dest = Instr->getDest(); | 3609 Variable *Dest = Instr->getDest(); |
| 3577 _mov(Dest, esp); | 3610 _mov(Dest, esp); |
| 3611 |
| 3578 return; | 3612 return; |
| 3579 } | 3613 } |
| 3580 case Intrinsics::Stackrestore: { | 3614 case Intrinsics::Stackrestore: { |
| 3581 Operand *Src = Instr->getArg(0); | 3615 Operand *Src = Instr->getArg(0); |
| 3582 const Type SrcTy = Src->getType(); | 3616 _mov_sp(Src); |
| 3583 Variable *esp = Func->getTarget()->getPhysicalRegister( | |
| 3584 Traits::getGprForType(SrcTy, getStackReg()), SrcTy); | |
| 3585 _redefined(_mov(esp, Src)); | |
| 3586 return; | 3617 return; |
| 3587 } | 3618 } |
| 3619 |
| 3588 case Intrinsics::Trap: | 3620 case Intrinsics::Trap: |
| 3589 _ud2(); | 3621 _ud2(); |
| 3590 return; | 3622 return; |
| 3591 case Intrinsics::UnknownIntrinsic: | 3623 case Intrinsics::UnknownIntrinsic: |
| 3592 Func->setError("Should not be lowering UnknownIntrinsic"); | 3624 Func->setError("Should not be lowering UnknownIntrinsic"); |
| 3593 return; | 3625 return; |
| 3594 } | 3626 } |
| 3595 return; | 3627 return; |
| 3596 } | 3628 } |
| 3597 | 3629 |
| (...skipping 688 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4286 lowerCast(InstCast::create(Func, InstCast::Zext, ValExtVar, Val)); | 4318 lowerCast(InstCast::create(Func, InstCast::Zext, ValExtVar, Val)); |
| 4287 ValExt = ValExtVar; | 4319 ValExt = ValExtVar; |
| 4288 } | 4320 } |
| 4289 InstCall *Call = makeHelperCall(H_call_memset, nullptr, 3); | 4321 InstCall *Call = makeHelperCall(H_call_memset, nullptr, 3); |
| 4290 Call->addArg(Dest); | 4322 Call->addArg(Dest); |
| 4291 Call->addArg(ValExt); | 4323 Call->addArg(ValExt); |
| 4292 Call->addArg(Count); | 4324 Call->addArg(Count); |
| 4293 lowerCall(Call); | 4325 lowerCall(Call); |
| 4294 } | 4326 } |
| 4295 | 4327 |
| 4296 template <typename TraitsType> | |
| 4297 void TargetX86Base<TraitsType>::lowerIndirectJump(Variable *JumpTarget) { | |
| 4298 const bool NeedSandboxing = Ctx->getFlags().getUseSandboxing(); | |
| 4299 if (Traits::Is64Bit) { | |
| 4300 Variable *T = makeReg(IceType_i64); | |
| 4301 _movzx(T, JumpTarget); | |
| 4302 JumpTarget = T; | |
| 4303 } | |
| 4304 if (NeedSandboxing) { | |
| 4305 _bundle_lock(); | |
| 4306 const SizeT BundleSize = | |
| 4307 1 << Func->getAssembler<>()->getBundleAlignLog2Bytes(); | |
| 4308 _and(JumpTarget, Ctx->getConstantInt32(~(BundleSize - 1))); | |
| 4309 } | |
| 4310 _jmp(JumpTarget); | |
| 4311 if (NeedSandboxing) | |
| 4312 _bundle_unlock(); | |
| 4313 } | |
| 4314 | |
| 4315 inline bool isAdd(const Inst *Inst) { | 4328 inline bool isAdd(const Inst *Inst) { |
| 4316 if (auto *Arith = llvm::dyn_cast_or_null<const InstArithmetic>(Inst)) { | 4329 if (auto *Arith = llvm::dyn_cast_or_null<const InstArithmetic>(Inst)) { |
| 4317 return (Arith->getOp() == InstArithmetic::Add); | 4330 return (Arith->getOp() == InstArithmetic::Add); |
| 4318 } | 4331 } |
| 4319 return false; | 4332 return false; |
| 4320 } | 4333 } |
| 4321 | 4334 |
| 4322 inline void dumpAddressOpt(const Cfg *Func, | 4335 inline void dumpAddressOpt(const Cfg *Func, |
| 4323 const ConstantRelocatable *Relocatable, | 4336 const ConstantRelocatable *Relocatable, |
| 4324 int32_t Offset, const Variable *Base, | 4337 int32_t Offset, const Variable *Base, |
| (...skipping 219 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4544 return false; | 4557 return false; |
| 4545 // Incorporate ConstantRelocatables. | 4558 // Incorporate ConstantRelocatables. |
| 4546 if (Reloc0) | 4559 if (Reloc0) |
| 4547 NewRelocatable = Reloc0; | 4560 NewRelocatable = Reloc0; |
| 4548 else if (Reloc1) | 4561 else if (Reloc1) |
| 4549 NewRelocatable = Reloc1; | 4562 NewRelocatable = Reloc1; |
| 4550 if ((Reloc0 || Reloc1) && BaseOther && GotVar) | 4563 if ((Reloc0 || Reloc1) && BaseOther && GotVar) |
| 4551 return false; | 4564 return false; |
| 4552 // Compute the updated constant offset. | 4565 // Compute the updated constant offset. |
| 4553 if (Const0) { | 4566 if (Const0) { |
| 4554 int32_t MoreOffset = IsAdd ? Const0->getValue() : -Const0->getValue(); | 4567 const int32_t MoreOffset = |
| 4568 IsAdd ? Const0->getValue() : -Const0->getValue(); |
| 4555 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) | 4569 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) |
| 4556 return false; | 4570 return false; |
| 4557 NewOffset += MoreOffset; | 4571 NewOffset += MoreOffset; |
| 4558 } | 4572 } |
| 4559 if (Const1) { | 4573 if (Const1) { |
| 4560 int32_t MoreOffset = IsAdd ? Const1->getValue() : -Const1->getValue(); | 4574 const int32_t MoreOffset = |
| 4575 IsAdd ? Const1->getValue() : -Const1->getValue(); |
| 4561 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) | 4576 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) |
| 4562 return false; | 4577 return false; |
| 4563 NewOffset += MoreOffset; | 4578 NewOffset += MoreOffset; |
| 4564 } | 4579 } |
| 4565 // Update the computed address parameters once we are sure optimization | 4580 // Update the computed address parameters once we are sure optimization |
| 4566 // is valid. | 4581 // is valid. |
| 4567 if ((Reloc0 || Reloc1) && GotVar) { | 4582 if ((Reloc0 || Reloc1) && GotVar) { |
| 4568 assert(BaseOther == nullptr); | 4583 assert(BaseOther == nullptr); |
| 4569 BaseOther = GotVar; | 4584 BaseOther = GotVar; |
| 4570 } | 4585 } |
| 4571 Base = NewBase; | 4586 Base = NewBase; |
| 4572 Offset = NewOffset; | 4587 Offset = NewOffset; |
| 4573 Relocatable = NewRelocatable; | 4588 Relocatable = NewRelocatable; |
| 4574 Reason = BaseInst; | 4589 Reason = BaseInst; |
| 4575 return true; | 4590 return true; |
| 4576 } | 4591 } |
| 4577 return false; | 4592 return false; |
| 4578 } | 4593 } |
| 4579 | 4594 |
| 4580 // Builds information for a canonical address expresion: | 4595 // Builds information for a canonical address expresion: |
| 4581 // <Relocatable + Offset>(Base, Index, Shift) | 4596 // <Relocatable + Offset>(Base, Index, Shift) |
| 4582 // On entry: | 4597 // On entry: |
| 4583 // Relocatable == null, | 4598 // Relocatable == null, |
| 4584 // Offset == 0, | 4599 // Offset == 0, |
| 4585 // Base is a Variable, | 4600 // Base is a Variable, |
| 4586 // Index == nullptr, | 4601 // Index == nullptr, |
| 4587 // Shift == 0 | 4602 // Shift == 0 |
| 4588 inline bool computeAddressOpt(Cfg *Func, const Inst *Instr, Variable *GotVar, | 4603 inline bool computeAddressOpt(Cfg *Func, const Inst *Instr, Variable *GotVar, |
| 4604 bool ReserveSlot, |
| 4589 ConstantRelocatable *&Relocatable, | 4605 ConstantRelocatable *&Relocatable, |
| 4590 int32_t &Offset, Variable *&Base, | 4606 int32_t &Offset, Variable *&Base, |
| 4591 Variable *&Index, uint16_t &Shift) { | 4607 Variable *&Index, uint16_t &Shift) { |
| 4592 bool AddressWasOptimized = false; | 4608 bool AddressWasOptimized = false; |
| 4593 Func->resetCurrentNode(); | 4609 Func->resetCurrentNode(); |
| 4594 if (Func->isVerbose(IceV_AddrOpt)) { | 4610 if (Func->isVerbose(IceV_AddrOpt)) { |
| 4595 OstreamLocker L(Func->getContext()); | 4611 OstreamLocker L(Func->getContext()); |
| 4596 Ostream &Str = Func->getContext()->getStrDump(); | 4612 Ostream &Str = Func->getContext()->getStrDump(); |
| 4597 Str << "\nStarting computeAddressOpt for instruction:\n "; | 4613 Str << "\nStarting computeAddressOpt for instruction:\n "; |
| 4598 Instr->dumpDecorated(Func); | 4614 Instr->dumpDecorated(Func); |
| 4599 } | 4615 } |
| 4600 if (Base == nullptr) | 4616 if (Base == nullptr) |
| 4601 return AddressWasOptimized; | 4617 return AddressWasOptimized; |
| 4602 // If the Base has more than one use or is live across multiple blocks, then | 4618 // If the Base has more than one use or is live across multiple blocks, then |
| 4603 // don't go further. Alternatively (?), never consider a transformation that | 4619 // don't go further. Alternatively (?), never consider a transformation that |
| 4604 // would change a variable that is currently *not* live across basic block | 4620 // would change a variable that is currently *not* live across basic block |
| 4605 // boundaries into one that *is*. | 4621 // boundaries into one that *is*. |
| 4606 if (Func->getVMetadata()->isMultiBlock(Base) /* || Base->getUseCount() > 1*/) | 4622 if (Func->getVMetadata()->isMultiBlock(Base) /* || Base->getUseCount() > 1*/) |
| 4607 return AddressWasOptimized; | 4623 return AddressWasOptimized; |
| 4608 | 4624 |
| 4609 const bool MockBounds = Func->getContext()->getFlags().getMockBoundsCheck(); | 4625 const bool MockBounds = Func->getContext()->getFlags().getMockBoundsCheck(); |
| 4610 const VariablesMetadata *VMetadata = Func->getVMetadata(); | 4626 const VariablesMetadata *VMetadata = Func->getVMetadata(); |
| 4611 const Inst *Reason = nullptr; | 4627 const Inst *Reason = nullptr; |
| 4612 do { | 4628 do { |
| 4629 assert(!ReserveSlot || Base == nullptr || Index == nullptr); |
| 4613 if (Reason) { | 4630 if (Reason) { |
| 4614 dumpAddressOpt(Func, Relocatable, Offset, Base, Index, Shift, Reason); | 4631 dumpAddressOpt(Func, Relocatable, Offset, Base, Index, Shift, Reason); |
| 4615 AddressWasOptimized = true; | 4632 AddressWasOptimized = true; |
| 4616 Reason = nullptr; | 4633 Reason = nullptr; |
| 4617 } | 4634 } |
| 4618 // Update Base and Index to follow through assignments to definitions. | 4635 // Update Base and Index to follow through assignments to definitions. |
| 4619 if (matchAssign(VMetadata, GotVar, Base, Relocatable, Offset, Reason)) { | 4636 if (matchAssign(VMetadata, GotVar, Base, Relocatable, Offset, Reason)) { |
| 4620 // Assignments of Base from a Relocatable or ConstantInt32 can result | 4637 // Assignments of Base from a Relocatable or ConstantInt32 can result |
| 4621 // in Base becoming nullptr. To avoid code duplication in this loop we | 4638 // in Base becoming nullptr. To avoid code duplication in this loop we |
| 4622 // prefer that Base be non-nullptr if possible. | 4639 // prefer that Base be non-nullptr if possible. |
| 4623 if ((Base == nullptr) && (Index != nullptr) && Shift == 0) | 4640 if ((Base == nullptr) && (Index != nullptr) && Shift == 0) |
| 4624 std::swap(Base, Index); | 4641 std::swap(Base, Index); |
| 4625 continue; | 4642 continue; |
| 4626 } | 4643 } |
| 4627 if (matchAssign(VMetadata, GotVar, Index, Relocatable, Offset, Reason)) | 4644 if (matchAssign(VMetadata, GotVar, Index, Relocatable, Offset, Reason)) |
| 4628 continue; | 4645 continue; |
| 4629 | 4646 |
| 4630 if (!MockBounds) { | 4647 if (!MockBounds) { |
| 4631 // Transition from: | 4648 // Transition from: |
| 4632 // <Relocatable + Offset>(Base) to | 4649 // <Relocatable + Offset>(Base) to |
| 4633 // <Relocatable + Offset>(Base, Index) | 4650 // <Relocatable + Offset>(Base, Index) |
| 4634 if (matchCombinedBaseIndex(VMetadata, Base, Index, Shift, Reason)) | 4651 if (!ReserveSlot && |
| 4652 matchCombinedBaseIndex(VMetadata, Base, Index, Shift, Reason)) |
| 4635 continue; | 4653 continue; |
| 4636 // Recognize multiply/shift and update Shift amount. | 4654 // Recognize multiply/shift and update Shift amount. |
| 4637 // Index becomes Index=Var<<Const && Const+Shift<=3 ==> | 4655 // Index becomes Index=Var<<Const && Const+Shift<=3 ==> |
| 4638 // Index=Var, Shift+=Const | 4656 // Index=Var, Shift+=Const |
| 4639 // Index becomes Index=Const*Var && log2(Const)+Shift<=3 ==> | 4657 // Index becomes Index=Const*Var && log2(Const)+Shift<=3 ==> |
| 4640 // Index=Var, Shift+=log2(Const) | 4658 // Index=Var, Shift+=log2(Const) |
| 4641 if (matchShiftedIndex(VMetadata, Index, Shift, Reason)) | 4659 if (matchShiftedIndex(VMetadata, Index, Shift, Reason)) |
| 4642 continue; | 4660 continue; |
| 4643 // If Shift is zero, the choice of Base and Index was purely arbitrary. | 4661 // If Shift is zero, the choice of Base and Index was purely arbitrary. |
| 4644 // Recognize multiply/shift and set Shift amount. | 4662 // Recognize multiply/shift and set Shift amount. |
| (...skipping 112 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4757 Variable *Index = nullptr; | 4775 Variable *Index = nullptr; |
| 4758 ConstantRelocatable *Relocatable = nullptr; | 4776 ConstantRelocatable *Relocatable = nullptr; |
| 4759 uint16_t Shift = 0; | 4777 uint16_t Shift = 0; |
| 4760 int32_t Offset = 0; | 4778 int32_t Offset = 0; |
| 4761 // Vanilla ICE load instructions should not use the segment registers, and | 4779 // Vanilla ICE load instructions should not use the segment registers, and |
| 4762 // computeAddressOpt only works at the level of Variables and Constants, not | 4780 // computeAddressOpt only works at the level of Variables and Constants, not |
| 4763 // other X86OperandMem, so there should be no mention of segment | 4781 // other X86OperandMem, so there should be no mention of segment |
| 4764 // registers there either. | 4782 // registers there either. |
| 4765 constexpr auto SegmentReg = X86OperandMem::SegmentRegisters::DefaultSegment; | 4783 constexpr auto SegmentReg = X86OperandMem::SegmentRegisters::DefaultSegment; |
| 4766 auto *Base = llvm::dyn_cast<Variable>(Addr); | 4784 auto *Base = llvm::dyn_cast<Variable>(Addr); |
| 4767 if (computeAddressOpt(Func, Inst, GotVar, Relocatable, Offset, Base, Index, | 4785 const bool ReserveSlot = Traits::Is64Bit && NeedSandboxing; |
| 4768 Shift)) { | 4786 if (computeAddressOpt(Func, Inst, GotVar, ReserveSlot, Relocatable, Offset, |
| 4787 Base, Index, Shift)) { |
| 4769 Inst->setDeleted(); | 4788 Inst->setDeleted(); |
| 4770 Constant *OffsetOp = nullptr; | 4789 Constant *OffsetOp = nullptr; |
| 4771 if (Relocatable == nullptr) { | 4790 if (Relocatable == nullptr) { |
| 4772 OffsetOp = Ctx->getConstantInt32(Offset); | 4791 OffsetOp = Ctx->getConstantInt32(Offset); |
| 4773 } else { | 4792 } else { |
| 4774 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, | 4793 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, |
| 4775 Relocatable->getName(), | 4794 Relocatable->getName(), |
| 4776 Relocatable->getSuppressMangling()); | 4795 Relocatable->getSuppressMangling()); |
| 4777 } | 4796 } |
| 4778 // The new mem operand is created without IsPIC being set, because | 4797 // The new mem operand is created without IsRebased being set, because |
| 4779 // computeAddressOpt() doesn't include GotVar in its final result. | 4798 // computeAddressOpt() doesn't include GotVar in its final result. |
| 4780 Addr = X86OperandMem::create(Func, Dest->getType(), Base, OffsetOp, Index, | 4799 Addr = X86OperandMem::create(Func, Dest->getType(), Base, OffsetOp, Index, |
| 4781 Shift, SegmentReg); | 4800 Shift, SegmentReg); |
| 4782 Context.insert<InstLoad>(Dest, Addr); | 4801 Context.insert<InstLoad>(Dest, Addr); |
| 4783 } | 4802 } |
| 4784 } | 4803 } |
| 4785 | 4804 |
| 4786 template <typename TraitsType> | 4805 template <typename TraitsType> |
| 4787 void TargetX86Base<TraitsType>::randomlyInsertNop(float Probability, | 4806 void TargetX86Base<TraitsType>::randomlyInsertNop(float Probability, |
| 4788 RandomNumberGenerator &RNG) { | 4807 RandomNumberGenerator &RNG) { |
| (...skipping 273 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 5062 Variable *Index = nullptr; | 5081 Variable *Index = nullptr; |
| 5063 ConstantRelocatable *Relocatable = nullptr; | 5082 ConstantRelocatable *Relocatable = nullptr; |
| 5064 uint16_t Shift = 0; | 5083 uint16_t Shift = 0; |
| 5065 int32_t Offset = 0; | 5084 int32_t Offset = 0; |
| 5066 auto *Base = llvm::dyn_cast<Variable>(Addr); | 5085 auto *Base = llvm::dyn_cast<Variable>(Addr); |
| 5067 // Vanilla ICE store instructions should not use the segment registers, and | 5086 // Vanilla ICE store instructions should not use the segment registers, and |
| 5068 // computeAddressOpt only works at the level of Variables and Constants, not | 5087 // computeAddressOpt only works at the level of Variables and Constants, not |
| 5069 // other X86OperandMem, so there should be no mention of segment | 5088 // other X86OperandMem, so there should be no mention of segment |
| 5070 // registers there either. | 5089 // registers there either. |
| 5071 constexpr auto SegmentReg = X86OperandMem::SegmentRegisters::DefaultSegment; | 5090 constexpr auto SegmentReg = X86OperandMem::SegmentRegisters::DefaultSegment; |
| 5072 if (computeAddressOpt(Func, Inst, GotVar, Relocatable, Offset, Base, Index, | 5091 const bool ReserveSlot = Traits::Is64Bit && NeedSandboxing; |
| 5073 Shift)) { | 5092 if (computeAddressOpt(Func, Inst, GotVar, ReserveSlot, Relocatable, Offset, |
| 5093 Base, Index, Shift)) { |
| 5074 Inst->setDeleted(); | 5094 Inst->setDeleted(); |
| 5075 Constant *OffsetOp = nullptr; | 5095 Constant *OffsetOp = nullptr; |
| 5076 if (Relocatable == nullptr) { | 5096 if (Relocatable == nullptr) { |
| 5077 OffsetOp = Ctx->getConstantInt32(Offset); | 5097 OffsetOp = Ctx->getConstantInt32(Offset); |
| 5078 } else { | 5098 } else { |
| 5079 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, | 5099 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, |
| 5080 Relocatable->getName(), | 5100 Relocatable->getName(), |
| 5081 Relocatable->getSuppressMangling()); | 5101 Relocatable->getSuppressMangling()); |
| 5082 } | 5102 } |
| 5083 // The new mem operand is created without IsPIC being set, because | 5103 // The new mem operand is created without IsRebased being set, because |
| 5084 // computeAddressOpt() doesn't include GotVar in its final result. | 5104 // computeAddressOpt() doesn't include GotVar in its final result. |
| 5085 Addr = X86OperandMem::create(Func, Data->getType(), Base, OffsetOp, Index, | 5105 Addr = X86OperandMem::create(Func, Data->getType(), Base, OffsetOp, Index, |
| 5086 Shift, SegmentReg); | 5106 Shift, SegmentReg); |
| 5087 auto *NewStore = Context.insert<InstStore>(Data, Addr); | 5107 auto *NewStore = Context.insert<InstStore>(Data, Addr); |
| 5088 if (Inst->getDest()) | 5108 if (Inst->getDest()) |
| 5089 NewStore->setRmwBeacon(Inst->getRmwBeacon()); | 5109 NewStore->setRmwBeacon(Inst->getRmwBeacon()); |
| 5090 } | 5110 } |
| 5091 } | 5111 } |
| 5092 | 5112 |
| 5093 template <typename TraitsType> | 5113 template <typename TraitsType> |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 5127 _br(Traits::Cond::Br_a, SkipJumpTable); | 5147 _br(Traits::Cond::Br_a, SkipJumpTable); |
| 5128 } else { | 5148 } else { |
| 5129 _br(Traits::Cond::Br_a, DefaultTarget); | 5149 _br(Traits::Cond::Br_a, DefaultTarget); |
| 5130 } | 5150 } |
| 5131 | 5151 |
| 5132 InstJumpTable *JumpTable = Case.getJumpTable(); | 5152 InstJumpTable *JumpTable = Case.getJumpTable(); |
| 5133 Context.insert(JumpTable); | 5153 Context.insert(JumpTable); |
| 5134 | 5154 |
| 5135 // Make sure the index is a register of the same width as the base | 5155 // Make sure the index is a register of the same width as the base |
| 5136 Variable *Index; | 5156 Variable *Index; |
| 5137 if (RangeIndex->getType() != getPointerType()) { | 5157 const Type PointerType = getPointerType(); |
| 5138 Index = makeReg(getPointerType()); | 5158 if (RangeIndex->getType() != PointerType) { |
| 5159 Index = makeReg(PointerType); |
| 5139 _movzx(Index, RangeIndex); | 5160 _movzx(Index, RangeIndex); |
| 5140 } else { | 5161 } else { |
| 5141 Index = legalizeToReg(RangeIndex); | 5162 Index = legalizeToReg(RangeIndex); |
| 5142 } | 5163 } |
| 5143 | 5164 |
| 5144 constexpr RelocOffsetT RelocOffset = 0; | 5165 constexpr RelocOffsetT RelocOffset = 0; |
| 5145 constexpr bool SuppressMangling = true; | 5166 constexpr bool SuppressMangling = true; |
| 5146 const bool IsPIC = Ctx->getFlags().getUseNonsfi(); | 5167 const bool IsRebased = Ctx->getFlags().getUseNonsfi(); |
| 5147 IceString MangledName = Ctx->mangleName(Func->getFunctionName()); | 5168 IceString MangledName = Ctx->mangleName(Func->getFunctionName()); |
| 5148 Variable *Base = IsPIC ? legalizeToReg(GotVar) : nullptr; | 5169 Variable *Base = IsRebased ? legalizeToReg(GotVar) : nullptr; |
| 5149 Constant *Offset = Ctx->getConstantSym( | 5170 Constant *Offset = Ctx->getConstantSym( |
| 5150 RelocOffset, InstJumpTable::makeName(MangledName, JumpTable->getId()), | 5171 RelocOffset, InstJumpTable::makeName(MangledName, JumpTable->getId()), |
| 5151 SuppressMangling); | 5172 SuppressMangling); |
| 5152 uint16_t Shift = typeWidthInBytesLog2(getPointerType()); | 5173 uint16_t Shift = typeWidthInBytesLog2(PointerType); |
| 5153 constexpr auto Segment = X86OperandMem::SegmentRegisters::DefaultSegment; | 5174 constexpr auto Segment = X86OperandMem::SegmentRegisters::DefaultSegment; |
| 5175 |
| 5176 Variable *Target = nullptr; |
| 5177 if (Traits::Is64Bit && NeedSandboxing) { |
| 5178 assert(Base == nullptr); |
| 5179 assert(Index != nullptr && Index->getType() == IceType_i32); |
| 5180 } |
| 5154 auto *TargetInMemory = X86OperandMem::create( | 5181 auto *TargetInMemory = X86OperandMem::create( |
| 5155 Func, getPointerType(), Base, Offset, Index, Shift, Segment, IsPIC); | 5182 Func, PointerType, Base, Offset, Index, Shift, Segment, IsRebased); |
| 5156 Variable *Target = nullptr; | |
| 5157 _mov(Target, TargetInMemory); | 5183 _mov(Target, TargetInMemory); |
| 5184 |
| 5158 lowerIndirectJump(Target); | 5185 lowerIndirectJump(Target); |
| 5159 | 5186 |
| 5160 if (DefaultTarget == nullptr) | 5187 if (DefaultTarget == nullptr) |
| 5161 Context.insert(SkipJumpTable); | 5188 Context.insert(SkipJumpTable); |
| 5162 return; | 5189 return; |
| 5163 } | 5190 } |
| 5164 case CaseCluster::Range: { | 5191 case CaseCluster::Range: { |
| 5165 if (Case.isUnitRange()) { | 5192 if (Case.isUnitRange()) { |
| 5166 // Single item | 5193 // Single item |
| 5167 if (!DoneCmp) { | 5194 if (!DoneCmp) { |
| (...skipping 862 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 6030 } | 6057 } |
| 6031 | 6058 |
| 6032 if (auto *Mem = llvm::dyn_cast<X86OperandMem>(From)) { | 6059 if (auto *Mem = llvm::dyn_cast<X86OperandMem>(From)) { |
| 6033 // Before doing anything with a Mem operand, we need to ensure that the | 6060 // Before doing anything with a Mem operand, we need to ensure that the |
| 6034 // Base and Index components are in physical registers. | 6061 // Base and Index components are in physical registers. |
| 6035 Variable *Base = Mem->getBase(); | 6062 Variable *Base = Mem->getBase(); |
| 6036 Variable *Index = Mem->getIndex(); | 6063 Variable *Index = Mem->getIndex(); |
| 6037 Constant *Offset = Mem->getOffset(); | 6064 Constant *Offset = Mem->getOffset(); |
| 6038 Variable *RegBase = nullptr; | 6065 Variable *RegBase = nullptr; |
| 6039 Variable *RegIndex = nullptr; | 6066 Variable *RegIndex = nullptr; |
| 6067 uint16_t Shift = Mem->getShift(); |
| 6040 if (Base) { | 6068 if (Base) { |
| 6041 RegBase = llvm::cast<Variable>( | 6069 RegBase = llvm::cast<Variable>( |
| 6042 legalize(Base, Legal_Reg | Legal_Rematerializable)); | 6070 legalize(Base, Legal_Reg | Legal_Rematerializable)); |
| 6043 } | 6071 } |
| 6044 if (Index) { | 6072 if (Index) { |
| 6073 // TODO(jpp): perhaps we should only allow Legal_Reg if |
| 6074 // Base->isRematerializable. |
| 6045 RegIndex = llvm::cast<Variable>( | 6075 RegIndex = llvm::cast<Variable>( |
| 6046 legalize(Index, Legal_Reg | Legal_Rematerializable)); | 6076 legalize(Index, Legal_Reg | Legal_Rematerializable)); |
| 6047 } | 6077 } |
| 6048 // For Non-SFI mode, if the Offset field is a ConstantRelocatable, we | 6078 // For Non-SFI mode, if the Offset field is a ConstantRelocatable, we |
| 6049 // replace either Base or Index with a legalized GotVar. At emission time, | 6079 // replace either Base or Index with a legalized GotVar. At emission time, |
| 6050 // the ConstantRelocatable will be emitted with the @GOTOFF relocation. | 6080 // the ConstantRelocatable will be emitted with the @GOTOFF relocation. |
| 6051 bool NeedPIC = false; | 6081 bool IsRebased = false; |
| 6052 if (UseNonsfi && !Mem->getIsPIC() && Offset && | 6082 if (UseNonsfi && !Mem->getIsRebased() && Offset && |
| 6053 llvm::isa<ConstantRelocatable>(Offset)) { | 6083 llvm::isa<ConstantRelocatable>(Offset)) { |
| 6054 assert(!(Allowed & Legal_AddrAbs)); | 6084 assert(!(Allowed & Legal_AddrAbs)); |
| 6055 NeedPIC = true; | 6085 IsRebased = true; |
| 6056 if (RegBase == nullptr) { | 6086 if (RegBase == nullptr) { |
| 6057 RegBase = legalizeToReg(GotVar); | 6087 RegBase = legalizeToReg(GotVar); |
| 6058 } else if (RegIndex == nullptr) { | 6088 } else if (RegIndex == nullptr) { |
| 6059 RegIndex = legalizeToReg(GotVar); | 6089 RegIndex = legalizeToReg(GotVar); |
| 6060 } else { | 6090 } else { |
| 6061 llvm::report_fatal_error( | 6091 llvm::report_fatal_error( |
| 6062 "Either Base or Index must be unused in Non-SFI mode"); | 6092 "Either Base or Index must be unused in Non-SFI mode"); |
| 6063 } | 6093 } |
| 6064 } | 6094 } |
| 6065 if (Base != RegBase || Index != RegIndex) { | 6095 if (Base != RegBase || Index != RegIndex) { |
| 6066 Mem = X86OperandMem::create(Func, Ty, RegBase, Offset, RegIndex, | 6096 Mem = X86OperandMem::create(Func, Ty, RegBase, Offset, RegIndex, Shift, |
| 6067 Mem->getShift(), Mem->getSegmentRegister(), | 6097 Mem->getSegmentRegister(), IsRebased); |
| 6068 NeedPIC); | |
| 6069 } | 6098 } |
| 6070 | 6099 |
| 6071 // For all Memory Operands, we do randomization/pooling here | 6100 // For all Memory Operands, we do randomization/pooling here |
| 6072 From = randomizeOrPoolImmediate(Mem); | 6101 From = randomizeOrPoolImmediate(Mem); |
| 6073 | 6102 |
| 6074 if (!(Allowed & Legal_Mem)) { | 6103 if (!(Allowed & Legal_Mem)) { |
| 6075 From = copyToReg(From, RegNum); | 6104 From = copyToReg(From, RegNum); |
| 6076 } | 6105 } |
| 6077 return From; | 6106 return From; |
| 6078 } | 6107 } |
| (...skipping 28 matching lines...) Expand all Loading... |
| 6107 } | 6136 } |
| 6108 } | 6137 } |
| 6109 | 6138 |
| 6110 // If the operand is a ConstantRelocatable, and Legal_AddrAbs is not | 6139 // If the operand is a ConstantRelocatable, and Legal_AddrAbs is not |
| 6111 // specified, and UseNonsfi is indicated, we need to add GotVar. | 6140 // specified, and UseNonsfi is indicated, we need to add GotVar. |
| 6112 if (auto *CR = llvm::dyn_cast<ConstantRelocatable>(Const)) { | 6141 if (auto *CR = llvm::dyn_cast<ConstantRelocatable>(Const)) { |
| 6113 if (UseNonsfi && !(Allowed & Legal_AddrAbs)) { | 6142 if (UseNonsfi && !(Allowed & Legal_AddrAbs)) { |
| 6114 assert(Ty == IceType_i32); | 6143 assert(Ty == IceType_i32); |
| 6115 Variable *RegBase = legalizeToReg(GotVar); | 6144 Variable *RegBase = legalizeToReg(GotVar); |
| 6116 Variable *NewVar = makeReg(Ty, RegNum); | 6145 Variable *NewVar = makeReg(Ty, RegNum); |
| 6117 auto *Mem = Traits::X86OperandMem::create(Func, Ty, RegBase, CR); | 6146 static constexpr bool IsRebased = true; |
| 6118 Mem->setIsPIC(); | 6147 auto *Mem = |
| 6148 Traits::X86OperandMem::create(Func, Ty, RegBase, CR, IsRebased); |
| 6119 _lea(NewVar, Mem); | 6149 _lea(NewVar, Mem); |
| 6120 From = NewVar; | 6150 From = NewVar; |
| 6121 } | 6151 } |
| 6122 } | 6152 } |
| 6123 | 6153 |
| 6124 // Convert a scalar floating point constant into an explicit memory | 6154 // Convert a scalar floating point constant into an explicit memory |
| 6125 // operand. | 6155 // operand. |
| 6126 if (isScalarFloatingType(Ty)) { | 6156 if (isScalarFloatingType(Ty)) { |
| 6127 if (auto *ConstFloat = llvm::dyn_cast<ConstantFloat>(Const)) { | 6157 if (auto *ConstFloat = llvm::dyn_cast<ConstantFloat>(Const)) { |
| 6128 if (Utils::isPositiveZero(ConstFloat->getValue())) | 6158 if (Utils::isPositiveZero(ConstFloat->getValue())) |
| 6129 return makeZeroedRegister(Ty, RegNum); | 6159 return makeZeroedRegister(Ty, RegNum); |
| 6130 } else if (auto *ConstDouble = llvm::dyn_cast<ConstantDouble>(Const)) { | 6160 } else if (auto *ConstDouble = llvm::dyn_cast<ConstantDouble>(Const)) { |
| 6131 if (Utils::isPositiveZero(ConstDouble->getValue())) | 6161 if (Utils::isPositiveZero(ConstDouble->getValue())) |
| 6132 return makeZeroedRegister(Ty, RegNum); | 6162 return makeZeroedRegister(Ty, RegNum); |
| 6133 } | 6163 } |
| 6134 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; | 6164 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; |
| 6135 std::string Buffer; | 6165 std::string Buffer; |
| 6136 llvm::raw_string_ostream StrBuf(Buffer); | 6166 llvm::raw_string_ostream StrBuf(Buffer); |
| 6137 llvm::cast<Constant>(From)->emitPoolLabel(StrBuf, Ctx); | 6167 llvm::cast<Constant>(From)->emitPoolLabel(StrBuf, Ctx); |
| 6138 llvm::cast<Constant>(From)->setShouldBePooled(true); | 6168 llvm::cast<Constant>(From)->setShouldBePooled(true); |
| 6139 Constant *Offset = Ctx->getConstantSym(0, StrBuf.str(), true); | 6169 Constant *Offset = Ctx->getConstantSym(0, StrBuf.str(), true); |
| 6140 auto *Mem = X86OperandMem::create(Func, Ty, Base, Offset); | 6170 const bool IsRebased = Base != nullptr; |
| 6141 if (UseNonsfi) | 6171 auto *Mem = X86OperandMem::create(Func, Ty, Base, Offset, IsRebased); |
| 6142 Mem->setIsPIC(); | |
| 6143 From = Mem; | 6172 From = Mem; |
| 6144 } | 6173 } |
| 6145 bool NeedsReg = false; | 6174 bool NeedsReg = false; |
| 6146 if (!(Allowed & Legal_Imm) && !isScalarFloatingType(Ty)) | 6175 if (!(Allowed & Legal_Imm) && !isScalarFloatingType(Ty)) |
| 6147 // Immediate specifically not allowed | 6176 // Immediate specifically not allowed |
| 6148 NeedsReg = true; | 6177 NeedsReg = true; |
| 6149 if (!(Allowed & Legal_Mem) && isScalarFloatingType(Ty)) | 6178 if (!(Allowed & Legal_Mem) && isScalarFloatingType(Ty)) |
| 6150 // On x86, FP constants are lowered to mem operands. | 6179 // On x86, FP constants are lowered to mem operands. |
| 6151 NeedsReg = true; | 6180 NeedsReg = true; |
| 6152 if (NeedsReg) { | 6181 if (NeedsReg) { |
| (...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 6256 // we guarantee legalize(Offset) will return a Constant*. | 6285 // we guarantee legalize(Offset) will return a Constant*. |
| 6257 if (!llvm::isa<ConstantRelocatable>(Offset)) { | 6286 if (!llvm::isa<ConstantRelocatable>(Offset)) { |
| 6258 BoolFlagSaver B(RandomizationPoolingPaused, true); | 6287 BoolFlagSaver B(RandomizationPoolingPaused, true); |
| 6259 | 6288 |
| 6260 Offset = llvm::cast<Constant>(legalize(Offset)); | 6289 Offset = llvm::cast<Constant>(legalize(Offset)); |
| 6261 } | 6290 } |
| 6262 | 6291 |
| 6263 assert(llvm::isa<ConstantInteger32>(Offset) || | 6292 assert(llvm::isa<ConstantInteger32>(Offset) || |
| 6264 llvm::isa<ConstantRelocatable>(Offset)); | 6293 llvm::isa<ConstantRelocatable>(Offset)); |
| 6265 } | 6294 } |
| 6266 // Not completely sure whether it's OK to leave IsPIC unset when creating | 6295 // Not completely sure whether it's OK to leave IsRebased unset when |
| 6267 // the mem operand. If DoLegalize is true, it will definitely be applied | 6296 // creating the mem operand. If DoLegalize is true, it will definitely be |
| 6268 // during the legalize() call, but perhaps not during the | 6297 // applied during the legalize() call, but perhaps not during the |
| 6269 // randomizeOrPoolImmediate() call. In any case, the emit routines will | 6298 // randomizeOrPoolImmediate() call. In any case, the emit routines will |
| 6270 // assert that PIC legalization has been applied. | 6299 // assert that PIC legalization has been applied. |
| 6271 Mem = X86OperandMem::create(Func, Ty, Base, Offset); | 6300 Mem = X86OperandMem::create(Func, Ty, Base, Offset); |
| 6272 } | 6301 } |
| 6273 // Do legalization, which contains randomization/pooling or do | 6302 // Do legalization, which contains randomization/pooling or do |
| 6274 // randomization/pooling. | 6303 // randomization/pooling. |
| 6275 return llvm::cast<X86OperandMem>(DoLegalize ? legalize(Mem) | 6304 return llvm::cast<X86OperandMem>(DoLegalize ? legalize(Mem) |
| 6276 : randomizeOrPoolImmediate(Mem)); | 6305 : randomizeOrPoolImmediate(Mem)); |
| 6277 } | 6306 } |
| 6278 | 6307 |
| (...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 6386 Operand * | 6415 Operand * |
| 6387 TargetX86Base<TraitsType>::randomizeOrPoolImmediate(Constant *Immediate, | 6416 TargetX86Base<TraitsType>::randomizeOrPoolImmediate(Constant *Immediate, |
| 6388 int32_t RegNum) { | 6417 int32_t RegNum) { |
| 6389 assert(llvm::isa<ConstantInteger32>(Immediate) || | 6418 assert(llvm::isa<ConstantInteger32>(Immediate) || |
| 6390 llvm::isa<ConstantRelocatable>(Immediate)); | 6419 llvm::isa<ConstantRelocatable>(Immediate)); |
| 6391 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_None || | 6420 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_None || |
| 6392 RandomizationPoolingPaused == true) { | 6421 RandomizationPoolingPaused == true) { |
| 6393 // Immediates randomization/pooling off or paused | 6422 // Immediates randomization/pooling off or paused |
| 6394 return Immediate; | 6423 return Immediate; |
| 6395 } | 6424 } |
| 6396 if (Immediate->shouldBeRandomizedOrPooled(Ctx)) { | 6425 |
| 6397 Ctx->statsUpdateRPImms(); | 6426 if (Traits::Is64Bit && NeedSandboxing) { |
| 6398 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == | 6427 // Immediate randomization/pooling is currently disabled for x86-64 |
| 6399 RPI_Randomize) { | 6428 // sandboxing for it could generate invalid memory operands. |
| 6400 // blind the constant | 6429 assert(false && |
| 6401 // FROM: | 6430 "Constant pooling/randomization is disabled for x8664 sandbox."); |
| 6402 // imm | 6431 return Immediate; |
| 6403 // TO: | 6432 } |
| 6404 // insert: mov imm+cookie, Reg | 6433 |
| 6405 // insert: lea -cookie[Reg], Reg | 6434 if (!Immediate->shouldBeRandomizedOrPooled(Ctx)) { |
| 6406 // => Reg | 6435 // the constant Immediate is not eligible for blinding/pooling |
| 6407 // If we have already assigned a phy register, we must come from | 6436 return Immediate; |
| 6408 // advancedPhiLowering()=>lowerAssign(). In this case we should reuse the | 6437 } |
| 6409 // assigned register as this assignment is that start of its use-def | 6438 Ctx->statsUpdateRPImms(); |
| 6410 // chain. So we add RegNum argument here. Note we use 'lea' instruction | 6439 switch (Ctx->getFlags().getRandomizeAndPoolImmediatesOption()) { |
| 6411 // instead of 'xor' to avoid affecting the flags. | 6440 default: |
| 6412 Variable *Reg = makeReg(IceType_i32, RegNum); | 6441 llvm::report_fatal_error("Unsupported -randomize-pool-immediates option"); |
| 6413 auto *Integer = llvm::cast<ConstantInteger32>(Immediate); | 6442 case RPI_Randomize: { |
| 6414 uint32_t Value = Integer->getValue(); | 6443 // blind the constant |
| 6415 uint32_t Cookie = Func->getConstantBlindingCookie(); | 6444 // FROM: |
| 6416 _mov(Reg, Ctx->getConstantInt(IceType_i32, Cookie + Value)); | 6445 // imm |
| 6417 Constant *Offset = Ctx->getConstantInt(IceType_i32, 0 - Cookie); | 6446 // TO: |
| 6418 _lea(Reg, | 6447 // insert: mov imm+cookie, Reg |
| 6419 X86OperandMem::create(Func, IceType_i32, Reg, Offset, nullptr, 0)); | 6448 // insert: lea -cookie[Reg], Reg |
| 6420 if (Immediate->getType() != IceType_i32) { | 6449 // => Reg |
| 6421 Variable *TruncReg = makeReg(Immediate->getType(), RegNum); | 6450 // If we have already assigned a phy register, we must come from |
| 6422 _mov(TruncReg, Reg); | 6451 // advancedPhiLowering()=>lowerAssign(). In this case we should reuse the |
| 6423 return TruncReg; | 6452 // assigned register as this assignment is that start of its use-def |
| 6424 } | 6453 // chain. So we add RegNum argument here. Note we use 'lea' instruction |
| 6454 // instead of 'xor' to avoid affecting the flags. |
| 6455 Variable *Reg = makeReg(IceType_i32, RegNum); |
| 6456 auto *Integer = llvm::cast<ConstantInteger32>(Immediate); |
| 6457 uint32_t Value = Integer->getValue(); |
| 6458 uint32_t Cookie = Func->getConstantBlindingCookie(); |
| 6459 _mov(Reg, Ctx->getConstantInt(IceType_i32, Cookie + Value)); |
| 6460 Constant *Offset = Ctx->getConstantInt(IceType_i32, 0 - Cookie); |
| 6461 _lea(Reg, X86OperandMem::create(Func, IceType_i32, Reg, Offset)); |
| 6462 if (Immediate->getType() == IceType_i32) { |
| 6425 return Reg; | 6463 return Reg; |
| 6426 } | 6464 } |
| 6427 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_Pool) { | 6465 Variable *TruncReg = makeReg(Immediate->getType(), RegNum); |
| 6428 // pool the constant | 6466 _mov(TruncReg, Reg); |
| 6429 // FROM: | 6467 return TruncReg; |
| 6430 // imm | |
| 6431 // TO: | |
| 6432 // insert: mov $label, Reg | |
| 6433 // => Reg | |
| 6434 assert(Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_Pool); | |
| 6435 Immediate->setShouldBePooled(true); | |
| 6436 // if we have already assigned a phy register, we must come from | |
| 6437 // advancedPhiLowering()=>lowerAssign(). In this case we should reuse the | |
| 6438 // assigned register as this assignment is that start of its use-def | |
| 6439 // chain. So we add RegNum argument here. | |
| 6440 Variable *Reg = makeReg(Immediate->getType(), RegNum); | |
| 6441 IceString Label; | |
| 6442 llvm::raw_string_ostream Label_stream(Label); | |
| 6443 Immediate->emitPoolLabel(Label_stream, Ctx); | |
| 6444 constexpr RelocOffsetT Offset = 0; | |
| 6445 constexpr bool SuppressMangling = true; | |
| 6446 Constant *Symbol = | |
| 6447 Ctx->getConstantSym(Offset, Label_stream.str(), SuppressMangling); | |
| 6448 const bool UseNonsfi = Ctx->getFlags().getUseNonsfi(); | |
| 6449 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; | |
| 6450 X86OperandMem *MemOperand = | |
| 6451 X86OperandMem::create(Func, Immediate->getType(), Base, Symbol); | |
| 6452 if (UseNonsfi) | |
| 6453 MemOperand->setIsPIC(); | |
| 6454 _mov(Reg, MemOperand); | |
| 6455 return Reg; | |
| 6456 } | |
| 6457 assert("Unsupported -randomize-pool-immediates option" && false); | |
| 6458 } | 6468 } |
| 6459 // the constant Immediate is not eligible for blinding/pooling | 6469 case RPI_Pool: { |
| 6460 return Immediate; | 6470 // pool the constant |
| 6471 // FROM: |
| 6472 // imm |
| 6473 // TO: |
| 6474 // insert: mov $label, Reg |
| 6475 // => Reg |
| 6476 assert(Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_Pool); |
| 6477 Immediate->setShouldBePooled(true); |
| 6478 // if we have already assigned a phy register, we must come from |
| 6479 // advancedPhiLowering()=>lowerAssign(). In this case we should reuse the |
| 6480 // assigned register as this assignment is that start of its use-def |
| 6481 // chain. So we add RegNum argument here. |
| 6482 Variable *Reg = makeReg(Immediate->getType(), RegNum); |
| 6483 IceString Label; |
| 6484 llvm::raw_string_ostream Label_stream(Label); |
| 6485 Immediate->emitPoolLabel(Label_stream, Ctx); |
| 6486 constexpr RelocOffsetT Offset = 0; |
| 6487 constexpr bool SuppressMangling = true; |
| 6488 Constant *Symbol = |
| 6489 Ctx->getConstantSym(Offset, Label_stream.str(), SuppressMangling); |
| 6490 const bool UseNonsfi = Ctx->getFlags().getUseNonsfi(); |
| 6491 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; |
| 6492 const bool IsRebased = Base != nullptr; |
| 6493 X86OperandMem *MemOperand = X86OperandMem::create( |
| 6494 Func, Immediate->getType(), Base, Symbol, IsRebased); |
| 6495 _mov(Reg, MemOperand); |
| 6496 return Reg; |
| 6497 } |
| 6498 } |
| 6461 } | 6499 } |
| 6462 | 6500 |
| 6463 template <typename TraitsType> | 6501 template <typename TraitsType> |
| 6464 typename TargetX86Base<TraitsType>::X86OperandMem * | 6502 typename TargetX86Base<TraitsType>::X86OperandMem * |
| 6465 TargetX86Base<TraitsType>::randomizeOrPoolImmediate(X86OperandMem *MemOperand, | 6503 TargetX86Base<TraitsType>::randomizeOrPoolImmediate(X86OperandMem *MemOperand, |
| 6466 int32_t RegNum) { | 6504 int32_t RegNum) { |
| 6467 assert(MemOperand); | 6505 assert(MemOperand); |
| 6468 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_None || | 6506 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_None || |
| 6469 RandomizationPoolingPaused == true) { | 6507 RandomizationPoolingPaused == true) { |
| 6470 // immediates randomization/pooling is turned off | 6508 // immediates randomization/pooling is turned off |
| 6471 return MemOperand; | 6509 return MemOperand; |
| 6472 } | 6510 } |
| 6473 | 6511 |
| 6512 if (Traits::Is64Bit && NeedSandboxing) { |
| 6513 // Immediate randomization/pooling is currently disabled for x86-64 |
| 6514 // sandboxing for it could generate invalid memory operands. |
| 6515 assert(false && |
| 6516 "Constant pooling/randomization is disabled for x8664 sandbox."); |
| 6517 return MemOperand; |
| 6518 } |
| 6519 |
| 6474 // If this memory operand is already a randomized one, we do not randomize it | 6520 // If this memory operand is already a randomized one, we do not randomize it |
| 6475 // again. | 6521 // again. |
| 6476 if (MemOperand->getRandomized()) | 6522 if (MemOperand->getRandomized()) |
| 6477 return MemOperand; | 6523 return MemOperand; |
| 6478 | 6524 |
| 6479 if (auto *C = llvm::dyn_cast_or_null<Constant>(MemOperand->getOffset())) { | 6525 auto *C = llvm::dyn_cast_or_null<Constant>(MemOperand->getOffset()); |
| 6480 if (C->shouldBeRandomizedOrPooled(Ctx)) { | |
| 6481 // The offset of this mem operand should be blinded or pooled | |
| 6482 Ctx->statsUpdateRPImms(); | |
| 6483 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == | |
| 6484 RPI_Randomize) { | |
| 6485 // blind the constant offset | |
| 6486 // FROM: | |
| 6487 // offset[base, index, shift] | |
| 6488 // TO: | |
| 6489 // insert: lea offset+cookie[base], RegTemp | |
| 6490 // => -cookie[RegTemp, index, shift] | |
| 6491 uint32_t Value = | |
| 6492 llvm::dyn_cast<ConstantInteger32>(MemOperand->getOffset()) | |
| 6493 ->getValue(); | |
| 6494 uint32_t Cookie = Func->getConstantBlindingCookie(); | |
| 6495 Constant *Mask1 = Ctx->getConstantInt( | |
| 6496 MemOperand->getOffset()->getType(), Cookie + Value); | |
| 6497 Constant *Mask2 = | |
| 6498 Ctx->getConstantInt(MemOperand->getOffset()->getType(), 0 - Cookie); | |
| 6499 | 6526 |
| 6500 X86OperandMem *TempMemOperand = X86OperandMem::create( | 6527 if (C == nullptr) { |
| 6501 Func, MemOperand->getType(), MemOperand->getBase(), Mask1); | 6528 return MemOperand; |
| 6502 // If we have already assigned a physical register, we must come from | 6529 } |
| 6503 // advancedPhiLowering()=>lowerAssign(). In this case we should reuse | |
| 6504 // the assigned register as this assignment is that start of its | |
| 6505 // use-def chain. So we add RegNum argument here. | |
| 6506 Variable *RegTemp = makeReg(MemOperand->getOffset()->getType(), RegNum); | |
| 6507 _lea(RegTemp, TempMemOperand); | |
| 6508 | 6530 |
| 6509 X86OperandMem *NewMemOperand = X86OperandMem::create( | 6531 if (!C->shouldBeRandomizedOrPooled(Ctx)) { |
| 6510 Func, MemOperand->getType(), RegTemp, Mask2, MemOperand->getIndex(), | 6532 return MemOperand; |
| 6511 MemOperand->getShift(), MemOperand->getSegmentRegister()); | 6533 } |
| 6512 | 6534 |
| 6513 // Label this memory operand as randomized, so we won't randomize it | 6535 // The offset of this mem operand should be blinded or pooled |
| 6514 // again in case we call legalize() multiple times on this memory | 6536 Ctx->statsUpdateRPImms(); |
| 6515 // operand. | 6537 switch (Ctx->getFlags().getRandomizeAndPoolImmediatesOption()) { |
| 6516 NewMemOperand->setRandomized(true); | 6538 default: |
| 6517 return NewMemOperand; | 6539 llvm::report_fatal_error("Unsupported -randomize-pool-immediates option"); |
| 6518 } | 6540 case RPI_Randomize: { |
| 6519 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_Pool) { | 6541 // blind the constant offset |
| 6520 // pool the constant offset | 6542 // FROM: |
| 6521 // FROM: | 6543 // offset[base, index, shift] |
| 6522 // offset[base, index, shift] | 6544 // TO: |
| 6523 // TO: | 6545 // insert: lea offset+cookie[base], RegTemp |
| 6524 // insert: mov $label, RegTemp | 6546 // => -cookie[RegTemp, index, shift] |
| 6525 // insert: lea [base, RegTemp], RegTemp | 6547 uint32_t Value = |
| 6526 // =>[RegTemp, index, shift] | 6548 llvm::dyn_cast<ConstantInteger32>(MemOperand->getOffset())->getValue(); |
| 6527 assert(Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == | 6549 uint32_t Cookie = Func->getConstantBlindingCookie(); |
| 6528 RPI_Pool); | 6550 Constant *Mask1 = |
| 6529 // Memory operand should never exist as source operands in phi lowering | 6551 Ctx->getConstantInt(MemOperand->getOffset()->getType(), Cookie + Value); |
| 6530 // assignments, so there is no need to reuse any registers here. For | 6552 Constant *Mask2 = |
| 6531 // phi lowering, we should not ask for new physical registers in | 6553 Ctx->getConstantInt(MemOperand->getOffset()->getType(), 0 - Cookie); |
| 6532 // general. However, if we do meet Memory Operand during phi lowering, | 6554 |
| 6533 // we should not blind or pool the immediates for now. | 6555 X86OperandMem *TempMemOperand = X86OperandMem::create( |
| 6534 if (RegNum != Variable::NoRegister) | 6556 Func, MemOperand->getType(), MemOperand->getBase(), Mask1); |
| 6535 return MemOperand; | 6557 // If we have already assigned a physical register, we must come from |
| 6536 Variable *RegTemp = makeReg(IceType_i32); | 6558 // advancedPhiLowering()=>lowerAssign(). In this case we should reuse |
| 6537 IceString Label; | 6559 // the assigned register as this assignment is that start of its |
| 6538 llvm::raw_string_ostream Label_stream(Label); | 6560 // use-def chain. So we add RegNum argument here. |
| 6539 MemOperand->getOffset()->emitPoolLabel(Label_stream, Ctx); | 6561 Variable *RegTemp = makeReg(MemOperand->getOffset()->getType(), RegNum); |
| 6540 MemOperand->getOffset()->setShouldBePooled(true); | 6562 _lea(RegTemp, TempMemOperand); |
| 6541 constexpr RelocOffsetT SymOffset = 0; | 6563 |
| 6542 constexpr bool SuppressMangling = true; | 6564 X86OperandMem *NewMemOperand = X86OperandMem::create( |
| 6543 Constant *Symbol = Ctx->getConstantSym(SymOffset, Label_stream.str(), | 6565 Func, MemOperand->getType(), RegTemp, Mask2, MemOperand->getIndex(), |
| 6544 SuppressMangling); | 6566 MemOperand->getShift(), MemOperand->getSegmentRegister(), |
| 6545 const bool UseNonsfi = Ctx->getFlags().getUseNonsfi(); | 6567 MemOperand->getIsRebased()); |
| 6546 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; | 6568 |
| 6547 X86OperandMem *SymbolOperand = X86OperandMem::create( | 6569 // Label this memory operand as randomized, so we won't randomize it |
| 6548 Func, MemOperand->getOffset()->getType(), Base, Symbol); | 6570 // again in case we call legalize() multiple times on this memory |
| 6549 if (UseNonsfi) | 6571 // operand. |
| 6550 SymbolOperand->setIsPIC(); | 6572 NewMemOperand->setRandomized(true); |
| 6551 _mov(RegTemp, SymbolOperand); | 6573 return NewMemOperand; |
| 6552 // If we have a base variable here, we should add the lea instruction | 6574 } |
| 6553 // to add the value of the base variable to RegTemp. If there is no | 6575 case RPI_Pool: { |
| 6554 // base variable, we won't need this lea instruction. | 6576 // pool the constant offset |
| 6555 if (MemOperand->getBase()) { | 6577 // FROM: |
| 6556 X86OperandMem *CalculateOperand = X86OperandMem::create( | 6578 // offset[base, index, shift] |
| 6557 Func, MemOperand->getType(), MemOperand->getBase(), nullptr, | 6579 // TO: |
| 6558 RegTemp, 0, MemOperand->getSegmentRegister()); | 6580 // insert: mov $label, RegTemp |
| 6559 _lea(RegTemp, CalculateOperand); | 6581 // insert: lea [base, RegTemp], RegTemp |
| 6560 } | 6582 // =>[RegTemp, index, shift] |
| 6561 X86OperandMem *NewMemOperand = X86OperandMem::create( | 6583 |
| 6562 Func, MemOperand->getType(), RegTemp, nullptr, | 6584 // Memory operand should never exist as source operands in phi lowering |
| 6563 MemOperand->getIndex(), MemOperand->getShift(), | 6585 // assignments, so there is no need to reuse any registers here. For |
| 6564 MemOperand->getSegmentRegister()); | 6586 // phi lowering, we should not ask for new physical registers in |
| 6565 return NewMemOperand; | 6587 // general. However, if we do meet Memory Operand during phi lowering, |
| 6566 } | 6588 // we should not blind or pool the immediates for now. |
| 6567 assert("Unsupported -randomize-pool-immediates option" && false); | 6589 if (RegNum != Variable::NoRegister) |
| 6590 return MemOperand; |
| 6591 Variable *RegTemp = makeReg(IceType_i32); |
| 6592 IceString Label; |
| 6593 llvm::raw_string_ostream Label_stream(Label); |
| 6594 MemOperand->getOffset()->emitPoolLabel(Label_stream, Ctx); |
| 6595 MemOperand->getOffset()->setShouldBePooled(true); |
| 6596 constexpr RelocOffsetT SymOffset = 0; |
| 6597 constexpr bool SuppressMangling = true; |
| 6598 Constant *Symbol = |
| 6599 Ctx->getConstantSym(SymOffset, Label_stream.str(), SuppressMangling); |
| 6600 const bool UseNonsfi = Ctx->getFlags().getUseNonsfi(); |
| 6601 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; |
| 6602 const bool IsRebased = Base != nullptr; |
| 6603 X86OperandMem *SymbolOperand = X86OperandMem::create( |
| 6604 Func, MemOperand->getOffset()->getType(), Base, Symbol, IsRebased); |
| 6605 _mov(RegTemp, SymbolOperand); |
| 6606 // If we have a base variable here, we should add the lea instruction |
| 6607 // to add the value of the base variable to RegTemp. If there is no |
| 6608 // base variable, we won't need this lea instruction. |
| 6609 if (MemOperand->getBase()) { |
| 6610 X86OperandMem *CalculateOperand = X86OperandMem::create( |
| 6611 Func, MemOperand->getType(), MemOperand->getBase(), nullptr, RegTemp, |
| 6612 0, MemOperand->getSegmentRegister()); |
| 6613 _lea(RegTemp, CalculateOperand); |
| 6568 } | 6614 } |
| 6615 X86OperandMem *NewMemOperand = X86OperandMem::create( |
| 6616 Func, MemOperand->getType(), RegTemp, nullptr, MemOperand->getIndex(), |
| 6617 MemOperand->getShift(), MemOperand->getSegmentRegister()); |
| 6618 return NewMemOperand; |
| 6569 } | 6619 } |
| 6570 // the offset is not eligible for blinding or pooling, return the original | 6620 } |
| 6571 // mem operand | |
| 6572 return MemOperand; | |
| 6573 } | 6621 } |
| 6574 | |
| 6575 } // end of namespace X86NAMESPACE | 6622 } // end of namespace X86NAMESPACE |
| 6576 } // end of namespace Ice | 6623 } // end of namespace Ice |
| 6577 | 6624 |
| 6578 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H | 6625 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H |
| OLD | NEW |