OLD | NEW |
1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// | 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// |
2 // | 2 // |
3 // The Subzero Code Generator | 3 // The Subzero Code Generator |
4 // | 4 // |
5 // This file is distributed under the University of Illinois Open Source | 5 // This file is distributed under the University of Illinois Open Source |
6 // License. See LICENSE.TXT for details. | 6 // License. See LICENSE.TXT for details. |
7 // | 7 // |
8 //===----------------------------------------------------------------------===// | 8 //===----------------------------------------------------------------------===// |
9 /// | 9 /// |
10 /// \file | 10 /// \file |
(...skipping 290 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
301 } | 301 } |
302 | 302 |
303 template <typename TraitsType> | 303 template <typename TraitsType> |
304 void TargetX86Base<TraitsType>::initNodeForLowering(CfgNode *Node) { | 304 void TargetX86Base<TraitsType>::initNodeForLowering(CfgNode *Node) { |
305 FoldingInfo.init(Node); | 305 FoldingInfo.init(Node); |
306 FoldingInfo.dump(Func); | 306 FoldingInfo.dump(Func); |
307 } | 307 } |
308 | 308 |
309 template <typename TraitsType> | 309 template <typename TraitsType> |
310 TargetX86Base<TraitsType>::TargetX86Base(Cfg *Func) | 310 TargetX86Base<TraitsType>::TargetX86Base(Cfg *Func) |
311 : TargetLowering(Func) { | 311 : TargetLowering(Func), NeedSandboxing(Ctx->getFlags().getUseSandboxing()) { |
312 static_assert( | 312 static_assert( |
313 (Traits::InstructionSet::End - Traits::InstructionSet::Begin) == | 313 (Traits::InstructionSet::End - Traits::InstructionSet::Begin) == |
314 (TargetInstructionSet::X86InstructionSet_End - | 314 (TargetInstructionSet::X86InstructionSet_End - |
315 TargetInstructionSet::X86InstructionSet_Begin), | 315 TargetInstructionSet::X86InstructionSet_Begin), |
316 "Traits::InstructionSet range different from TargetInstructionSet"); | 316 "Traits::InstructionSet range different from TargetInstructionSet"); |
317 if (Func->getContext()->getFlags().getTargetInstructionSet() != | 317 if (Func->getContext()->getFlags().getTargetInstructionSet() != |
318 TargetInstructionSet::BaseInstructionSet) { | 318 TargetInstructionSet::BaseInstructionSet) { |
319 InstructionSet = static_cast<InstructionSetEnum>( | 319 InstructionSet = static_cast<InstructionSetEnum>( |
320 (Func->getContext()->getFlags().getTargetInstructionSet() - | 320 (Func->getContext()->getFlags().getTargetInstructionSet() - |
321 TargetInstructionSet::X86InstructionSet_Begin) + | 321 TargetInstructionSet::X86InstructionSet_Begin) + |
322 Traits::InstructionSet::Begin); | 322 Traits::InstructionSet::Begin); |
323 } | 323 } |
324 } | 324 } |
325 | 325 |
326 template <typename TraitsType> | 326 template <typename TraitsType> |
327 void TargetX86Base<TraitsType>::staticInit(const ClFlags &Flags) { | 327 void TargetX86Base<TraitsType>::staticInit(const ClFlags &Flags) { |
328 Traits::initRegisterSet(&TypeToRegisterSet, &RegisterAliases, &ScratchRegs); | 328 Traits::initRegisterSet(Flags, &TypeToRegisterSet, &RegisterAliases, |
| 329 &ScratchRegs); |
329 PcRelFixup = Traits::FK_PcRel; | 330 PcRelFixup = Traits::FK_PcRel; |
330 AbsFixup = Flags.getUseNonsfi() ? Traits::FK_Gotoff : Traits::FK_Abs; | 331 AbsFixup = Flags.getUseNonsfi() ? Traits::FK_Gotoff : Traits::FK_Abs; |
331 } | 332 } |
332 | 333 |
333 template <typename TraitsType> void TargetX86Base<TraitsType>::translateO2() { | 334 template <typename TraitsType> void TargetX86Base<TraitsType>::translateO2() { |
334 TimerMarker T(TimerStack::TT_O2, Func); | 335 TimerMarker T(TimerStack::TT_O2, Func); |
335 | 336 |
336 if (!Traits::Is64Bit && Func->getContext()->getFlags().getUseNonsfi()) { | 337 if (!Traits::Is64Bit && Func->getContext()->getFlags().getUseNonsfi()) { |
337 GotVar = Func->makeVariable(IceType_i32); | 338 GotVar = Func->makeVariable(IceType_i32); |
338 } | 339 } |
| 340 |
| 341 if (NeedSandboxing) { |
| 342 initSandbox(); |
| 343 } |
| 344 |
339 genTargetHelperCalls(); | 345 genTargetHelperCalls(); |
340 Func->dump("After target helper call insertion"); | 346 Func->dump("After target helper call insertion"); |
341 | 347 |
342 // Merge Alloca instructions, and lay out the stack. | 348 // Merge Alloca instructions, and lay out the stack. |
343 static constexpr bool SortAndCombineAllocas = true; | 349 static constexpr bool SortAndCombineAllocas = true; |
344 Func->processAllocas(SortAndCombineAllocas); | 350 Func->processAllocas(SortAndCombineAllocas); |
345 Func->dump("After Alloca processing"); | 351 Func->dump("After Alloca processing"); |
346 | 352 |
347 if (!Ctx->getFlags().getPhiEdgeSplit()) { | 353 if (!Ctx->getFlags().getPhiEdgeSplit()) { |
348 // Lower Phi instructions. | 354 // Lower Phi instructions. |
(...skipping 100 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
449 // particular, no transformations that insert or reorder CfgNodes should be | 455 // particular, no transformations that insert or reorder CfgNodes should be |
450 // done after branch optimization. We go ahead and do it before nop insertion | 456 // done after branch optimization. We go ahead and do it before nop insertion |
451 // to reduce the amount of work needed for searching for opportunities. | 457 // to reduce the amount of work needed for searching for opportunities. |
452 Func->doBranchOpt(); | 458 Func->doBranchOpt(); |
453 Func->dump("After branch optimization"); | 459 Func->dump("After branch optimization"); |
454 | 460 |
455 // Nop insertion if -nop-insertion is enabled. | 461 // Nop insertion if -nop-insertion is enabled. |
456 Func->doNopInsertion(); | 462 Func->doNopInsertion(); |
457 | 463 |
458 // Mark nodes that require sandbox alignment | 464 // Mark nodes that require sandbox alignment |
459 if (Ctx->getFlags().getUseSandboxing()) | 465 if (NeedSandboxing) { |
460 Func->markNodesForSandboxing(); | 466 Func->markNodesForSandboxing(); |
| 467 } |
461 } | 468 } |
462 | 469 |
463 template <typename TraitsType> void TargetX86Base<TraitsType>::translateOm1() { | 470 template <typename TraitsType> void TargetX86Base<TraitsType>::translateOm1() { |
464 TimerMarker T(TimerStack::TT_Om1, Func); | 471 TimerMarker T(TimerStack::TT_Om1, Func); |
465 | 472 |
466 if (!Traits::Is64Bit && Func->getContext()->getFlags().getUseNonsfi()) { | 473 if (!Traits::Is64Bit && Func->getContext()->getFlags().getUseNonsfi()) { |
467 GotVar = Func->makeVariable(IceType_i32); | 474 GotVar = Func->makeVariable(IceType_i32); |
468 } | 475 } |
| 476 |
| 477 if (NeedSandboxing) { |
| 478 initSandbox(); |
| 479 } |
| 480 |
469 genTargetHelperCalls(); | 481 genTargetHelperCalls(); |
470 | 482 |
471 // Do not merge Alloca instructions, and lay out the stack. | 483 // Do not merge Alloca instructions, and lay out the stack. |
472 static constexpr bool SortAndCombineAllocas = false; | 484 static constexpr bool SortAndCombineAllocas = false; |
473 Func->processAllocas(SortAndCombineAllocas); | 485 Func->processAllocas(SortAndCombineAllocas); |
474 Func->dump("After Alloca processing"); | 486 Func->dump("After Alloca processing"); |
475 | 487 |
476 Func->placePhiLoads(); | 488 Func->placePhiLoads(); |
477 if (Func->hasError()) | 489 if (Func->hasError()) |
478 return; | 490 return; |
(...skipping 22 matching lines...) Expand all Loading... |
501 return; | 513 return; |
502 Func->dump("After stack frame mapping"); | 514 Func->dump("After stack frame mapping"); |
503 | 515 |
504 // Shuffle basic block order if -reorder-basic-blocks is enabled. | 516 // Shuffle basic block order if -reorder-basic-blocks is enabled. |
505 Func->shuffleNodes(); | 517 Func->shuffleNodes(); |
506 | 518 |
507 // Nop insertion if -nop-insertion is enabled. | 519 // Nop insertion if -nop-insertion is enabled. |
508 Func->doNopInsertion(); | 520 Func->doNopInsertion(); |
509 | 521 |
510 // Mark nodes that require sandbox alignment | 522 // Mark nodes that require sandbox alignment |
511 if (Ctx->getFlags().getUseSandboxing()) | 523 if (NeedSandboxing) |
512 Func->markNodesForSandboxing(); | 524 Func->markNodesForSandboxing(); |
513 } | 525 } |
514 | 526 |
515 inline bool canRMW(const InstArithmetic *Arith) { | 527 inline bool canRMW(const InstArithmetic *Arith) { |
516 Type Ty = Arith->getDest()->getType(); | 528 Type Ty = Arith->getDest()->getType(); |
517 // X86 vector instructions write to a register and have no RMW option. | 529 // X86 vector instructions write to a register and have no RMW option. |
518 if (isVectorType(Ty)) | 530 if (isVectorType(Ty)) |
519 return false; | 531 return false; |
520 bool isI64 = Ty == IceType_i64; | 532 bool isI64 = Ty == IceType_i64; |
521 | 533 |
(...skipping 281 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
803 IceString TargetX86Base<TraitsType>::getRegName(SizeT RegNum, Type Ty) const { | 815 IceString TargetX86Base<TraitsType>::getRegName(SizeT RegNum, Type Ty) const { |
804 return Traits::getRegName(Traits::getGprForType(Ty, RegNum)); | 816 return Traits::getRegName(Traits::getGprForType(Ty, RegNum)); |
805 } | 817 } |
806 | 818 |
807 template <typename TraitsType> | 819 template <typename TraitsType> |
808 void TargetX86Base<TraitsType>::emitVariable(const Variable *Var) const { | 820 void TargetX86Base<TraitsType>::emitVariable(const Variable *Var) const { |
809 if (!BuildDefs::dump()) | 821 if (!BuildDefs::dump()) |
810 return; | 822 return; |
811 Ostream &Str = Ctx->getStrEmit(); | 823 Ostream &Str = Ctx->getStrEmit(); |
812 if (Var->hasReg()) { | 824 if (Var->hasReg()) { |
813 Str << "%" << getRegName(Var->getRegNum(), Var->getType()); | 825 const bool Is64BitSandboxing = Traits::Is64Bit && NeedSandboxing; |
| 826 const Type VarType = (Var->isRematerializable() && Is64BitSandboxing) |
| 827 ? IceType_i64 |
| 828 : Var->getType(); |
| 829 Str << "%" << getRegName(Var->getRegNum(), VarType); |
814 return; | 830 return; |
815 } | 831 } |
816 if (Var->mustHaveReg()) { | 832 if (Var->mustHaveReg()) { |
817 llvm::report_fatal_error( | 833 llvm::report_fatal_error( |
818 "Infinite-weight Variable has no register assigned"); | 834 "Infinite-weight Variable has no register assigned"); |
819 } | 835 } |
820 const int32_t Offset = Var->getStackOffset(); | 836 const int32_t Offset = Var->getStackOffset(); |
821 int32_t BaseRegNum = Var->getBaseRegNum(); | 837 int32_t BaseRegNum = Var->getBaseRegNum(); |
822 if (BaseRegNum == Variable::NoRegister) | 838 if (BaseRegNum == Variable::NoRegister) |
823 BaseRegNum = getFrameOrStackReg(); | 839 BaseRegNum = getFrameOrStackReg(); |
(...skipping 92 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
916 return Var64On32->getLo(); | 932 return Var64On32->getLo(); |
917 if (auto *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { | 933 if (auto *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { |
918 auto *ConstInt = llvm::dyn_cast<ConstantInteger32>( | 934 auto *ConstInt = llvm::dyn_cast<ConstantInteger32>( |
919 Ctx->getConstantInt32(static_cast<int32_t>(Const->getValue()))); | 935 Ctx->getConstantInt32(static_cast<int32_t>(Const->getValue()))); |
920 // Check if we need to blind/pool the constant. | 936 // Check if we need to blind/pool the constant. |
921 return legalize(ConstInt); | 937 return legalize(ConstInt); |
922 } | 938 } |
923 if (auto *Mem = llvm::dyn_cast<X86OperandMem>(Operand)) { | 939 if (auto *Mem = llvm::dyn_cast<X86OperandMem>(Operand)) { |
924 auto *MemOperand = X86OperandMem::create( | 940 auto *MemOperand = X86OperandMem::create( |
925 Func, IceType_i32, Mem->getBase(), Mem->getOffset(), Mem->getIndex(), | 941 Func, IceType_i32, Mem->getBase(), Mem->getOffset(), Mem->getIndex(), |
926 Mem->getShift(), Mem->getSegmentRegister(), Mem->getIsPIC()); | 942 Mem->getShift(), Mem->getSegmentRegister(), Mem->getIsRebased()); |
927 // Test if we should randomize or pool the offset, if so randomize it or | 943 // Test if we should randomize or pool the offset, if so randomize it or |
928 // pool it then create mem operand with the blinded/pooled constant. | 944 // pool it then create mem operand with the blinded/pooled constant. |
929 // Otherwise, return the mem operand as ordinary mem operand. | 945 // Otherwise, return the mem operand as ordinary mem operand. |
930 return legalize(MemOperand); | 946 return legalize(MemOperand); |
931 } | 947 } |
932 llvm_unreachable("Unsupported operand type"); | 948 llvm_unreachable("Unsupported operand type"); |
933 return nullptr; | 949 return nullptr; |
934 } | 950 } |
935 | 951 |
936 template <typename TraitsType> | 952 template <typename TraitsType> |
(...skipping 19 matching lines...) Expand all Loading... |
956 } else if (auto *IntOffset = llvm::dyn_cast<ConstantInteger32>(Offset)) { | 972 } else if (auto *IntOffset = llvm::dyn_cast<ConstantInteger32>(Offset)) { |
957 Offset = Ctx->getConstantInt32(4 + IntOffset->getValue()); | 973 Offset = Ctx->getConstantInt32(4 + IntOffset->getValue()); |
958 } else if (auto *SymOffset = llvm::dyn_cast<ConstantRelocatable>(Offset)) { | 974 } else if (auto *SymOffset = llvm::dyn_cast<ConstantRelocatable>(Offset)) { |
959 assert(!Utils::WouldOverflowAdd(SymOffset->getOffset(), 4)); | 975 assert(!Utils::WouldOverflowAdd(SymOffset->getOffset(), 4)); |
960 Offset = | 976 Offset = |
961 Ctx->getConstantSym(4 + SymOffset->getOffset(), SymOffset->getName(), | 977 Ctx->getConstantSym(4 + SymOffset->getOffset(), SymOffset->getName(), |
962 SymOffset->getSuppressMangling()); | 978 SymOffset->getSuppressMangling()); |
963 } | 979 } |
964 auto *MemOperand = X86OperandMem::create( | 980 auto *MemOperand = X86OperandMem::create( |
965 Func, IceType_i32, Mem->getBase(), Offset, Mem->getIndex(), | 981 Func, IceType_i32, Mem->getBase(), Offset, Mem->getIndex(), |
966 Mem->getShift(), Mem->getSegmentRegister(), Mem->getIsPIC()); | 982 Mem->getShift(), Mem->getSegmentRegister(), Mem->getIsRebased()); |
967 // Test if the Offset is an eligible i32 constants for randomization and | 983 // Test if the Offset is an eligible i32 constants for randomization and |
968 // pooling. Blind/pool it if it is. Otherwise return as oridinary mem | 984 // pooling. Blind/pool it if it is. Otherwise return as oridinary mem |
969 // operand. | 985 // operand. |
970 return legalize(MemOperand); | 986 return legalize(MemOperand); |
971 } | 987 } |
972 llvm_unreachable("Unsupported operand type"); | 988 llvm_unreachable("Unsupported operand type"); |
973 return nullptr; | 989 return nullptr; |
974 } | 990 } |
975 | 991 |
976 template <typename TraitsType> | 992 template <typename TraitsType> |
977 llvm::SmallBitVector | 993 llvm::SmallBitVector |
978 TargetX86Base<TraitsType>::getRegisterSet(RegSetMask Include, | 994 TargetX86Base<TraitsType>::getRegisterSet(RegSetMask Include, |
979 RegSetMask Exclude) const { | 995 RegSetMask Exclude) const { |
980 return Traits::getRegisterSet(Include, Exclude); | 996 return Traits::getRegisterSet(Ctx->getFlags(), Include, Exclude); |
981 } | 997 } |
982 | 998 |
983 template <typename TraitsType> | 999 template <typename TraitsType> |
984 void TargetX86Base<TraitsType>::initGotVarIfNeeded() { | 1000 void TargetX86Base<TraitsType>::initGotVarIfNeeded() { |
985 if (!Func->getContext()->getFlags().getUseNonsfi()) | 1001 if (!Func->getContext()->getFlags().getUseNonsfi()) |
986 return; | 1002 return; |
987 if (Traits::Is64Bit) { | 1003 if (Traits::Is64Bit) { |
988 // Probably no implementation is needed, but error to be safe for now. | 1004 // Probably no implementation is needed, but error to be safe for now. |
989 llvm::report_fatal_error( | 1005 llvm::report_fatal_error( |
990 "Need to implement initGotVarIfNeeded() for 64-bit."); | 1006 "Need to implement initGotVarIfNeeded() for 64-bit."); |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1030 _and(esp, Ctx->getConstantInt32(-Alignment)); | 1046 _and(esp, Ctx->getConstantInt32(-Alignment)); |
1031 } | 1047 } |
1032 | 1048 |
1033 Variable *Dest = Inst->getDest(); | 1049 Variable *Dest = Inst->getDest(); |
1034 Operand *TotalSize = legalize(Inst->getSizeInBytes()); | 1050 Operand *TotalSize = legalize(Inst->getSizeInBytes()); |
1035 | 1051 |
1036 if (const auto *ConstantTotalSize = | 1052 if (const auto *ConstantTotalSize = |
1037 llvm::dyn_cast<ConstantInteger32>(TotalSize)) { | 1053 llvm::dyn_cast<ConstantInteger32>(TotalSize)) { |
1038 const uint32_t Value = | 1054 const uint32_t Value = |
1039 Utils::applyAlignment(ConstantTotalSize->getValue(), Alignment); | 1055 Utils::applyAlignment(ConstantTotalSize->getValue(), Alignment); |
1040 if (!UseFramePointer) { | 1056 if (UseFramePointer) { |
| 1057 _sub_sp(Ctx->getConstantInt32(Value)); |
| 1058 } else { |
1041 // If we don't need a Frame Pointer, this alloca has a known offset to the | 1059 // If we don't need a Frame Pointer, this alloca has a known offset to the |
1042 // stack pointer. We don't need adjust the stack pointer, nor assign any | 1060 // stack pointer. We don't need adjust the stack pointer, nor assign any |
1043 // value to Dest, as Dest is rematerializable. | 1061 // value to Dest, as Dest is rematerializable. |
1044 assert(Dest->isRematerializable()); | 1062 assert(Dest->isRematerializable()); |
1045 FixedAllocaSizeBytes += Value; | 1063 FixedAllocaSizeBytes += Value; |
1046 Context.insert<InstFakeDef>(Dest); | 1064 Context.insert<InstFakeDef>(Dest); |
1047 } else { | |
1048 _sub(esp, Ctx->getConstantInt32(Value)); | |
1049 } | 1065 } |
1050 } else { | 1066 } else { |
1051 // Non-constant sizes need to be adjusted to the next highest multiple of | 1067 // Non-constant sizes need to be adjusted to the next highest multiple of |
1052 // the required alignment at runtime. | 1068 // the required alignment at runtime. |
1053 Variable *T = makeReg(Traits::WordType); | 1069 Variable *T = nullptr; |
1054 if (Traits::Is64Bit && TotalSize->getType() != IceType_i64) { | 1070 if (Traits::Is64Bit && TotalSize->getType() != IceType_i64 && |
| 1071 !NeedSandboxing) { |
| 1072 T = makeReg(IceType_i64); |
1055 _movzx(T, TotalSize); | 1073 _movzx(T, TotalSize); |
1056 } else { | 1074 } else { |
| 1075 T = makeReg(IceType_i32); |
1057 _mov(T, TotalSize); | 1076 _mov(T, TotalSize); |
1058 } | 1077 } |
1059 _add(T, Ctx->getConstantInt32(Alignment - 1)); | 1078 _add(T, Ctx->getConstantInt32(Alignment - 1)); |
1060 _and(T, Ctx->getConstantInt32(-Alignment)); | 1079 _and(T, Ctx->getConstantInt32(-Alignment)); |
1061 _sub(esp, T); | 1080 _sub_sp(T); |
1062 } | 1081 } |
1063 // Add enough to the returned address to account for the out args area. | 1082 // Add enough to the returned address to account for the out args area. |
1064 uint32_t OutArgsSize = maxOutArgsSizeBytes(); | 1083 uint32_t OutArgsSize = maxOutArgsSizeBytes(); |
1065 if (OutArgsSize > 0) { | 1084 if (OutArgsSize > 0) { |
1066 Variable *T = makeReg(IceType_i32); | 1085 Variable *T = makeReg(IceType_i32); |
1067 auto *CalculateOperand = X86OperandMem::create( | 1086 auto *CalculateOperand = X86OperandMem::create( |
1068 Func, IceType_i32, esp, Ctx->getConstantInt(IceType_i32, OutArgsSize)); | 1087 Func, IceType_void, esp, Ctx->getConstantInt(IceType_i32, OutArgsSize)); |
1069 _lea(T, CalculateOperand); | 1088 _lea(T, CalculateOperand); |
1070 _mov(Dest, T); | 1089 _mov(Dest, T); |
1071 } else { | 1090 } else { |
1072 _mov(Dest, esp); | 1091 _mov(Dest, esp); |
1073 } | 1092 } |
1074 } | 1093 } |
1075 | 1094 |
1076 /// Strength-reduce scalar integer multiplication by a constant (for i32 or | 1095 /// Strength-reduce scalar integer multiplication by a constant (for i32 or |
1077 /// narrower) for certain constants. The lea instruction can be used to multiply | 1096 /// narrower) for certain constants. The lea instruction can be used to multiply |
1078 /// by 3, 5, or 9, and the lsh instruction can be used to multiply by powers of | 1097 /// by 3, 5, or 9, and the lsh instruction can be used to multiply by powers of |
1079 /// 2. These can be combined such that e.g. multiplying by 100 can be done as 2 | 1098 /// 2. These can be combined such that e.g. multiplying by 100 can be done as 2 |
1080 /// lea-based multiplies by 5, combined with left-shifting by 2. | 1099 /// lea-based multiplies by 5, combined with left-shifting by 2. |
1081 template <typename TraitsType> | 1100 template <typename TraitsType> |
1082 bool TargetX86Base<TraitsType>::optimizeScalarMul(Variable *Dest, Operand *Src0, | 1101 bool TargetX86Base<TraitsType>::optimizeScalarMul(Variable *Dest, Operand *Src0, |
1083 int32_t Src1) { | 1102 int32_t Src1) { |
1084 // Disable this optimization for Om1 and O0, just to keep things simple | 1103 // Disable this optimization for Om1 and O0, just to keep things simple |
1085 // there. | 1104 // there. |
1086 if (Ctx->getFlags().getOptLevel() < Opt_1) | 1105 if (Ctx->getFlags().getOptLevel() < Opt_1) |
1087 return false; | 1106 return false; |
1088 Type Ty = Dest->getType(); | 1107 Type Ty = Dest->getType(); |
1089 Variable *T = nullptr; | |
1090 if (Src1 == -1) { | 1108 if (Src1 == -1) { |
| 1109 Variable *T = nullptr; |
1091 _mov(T, Src0); | 1110 _mov(T, Src0); |
1092 _neg(T); | 1111 _neg(T); |
1093 _mov(Dest, T); | 1112 _mov(Dest, T); |
1094 return true; | 1113 return true; |
1095 } | 1114 } |
1096 if (Src1 == 0) { | 1115 if (Src1 == 0) { |
1097 _mov(Dest, Ctx->getConstantZero(Ty)); | 1116 _mov(Dest, Ctx->getConstantZero(Ty)); |
1098 return true; | 1117 return true; |
1099 } | 1118 } |
1100 if (Src1 == 1) { | 1119 if (Src1 == 1) { |
| 1120 Variable *T = nullptr; |
1101 _mov(T, Src0); | 1121 _mov(T, Src0); |
1102 _mov(Dest, T); | 1122 _mov(Dest, T); |
1103 return true; | 1123 return true; |
1104 } | 1124 } |
1105 // Don't bother with the edge case where Src1 == MININT. | 1125 // Don't bother with the edge case where Src1 == MININT. |
1106 if (Src1 == -Src1) | 1126 if (Src1 == -Src1) |
1107 return false; | 1127 return false; |
1108 const bool Src1IsNegative = Src1 < 0; | 1128 const bool Src1IsNegative = Src1 < 0; |
1109 if (Src1IsNegative) | 1129 if (Src1IsNegative) |
1110 Src1 = -Src1; | 1130 Src1 = -Src1; |
(...skipping 17 matching lines...) Expand all Loading... |
1128 Src1 /= 3; | 1148 Src1 /= 3; |
1129 } else if (Src1 % 2 == 0) { | 1149 } else if (Src1 % 2 == 0) { |
1130 if (Count2 == 0) | 1150 if (Count2 == 0) |
1131 ++CountOps; | 1151 ++CountOps; |
1132 ++Count2; | 1152 ++Count2; |
1133 Src1 /= 2; | 1153 Src1 /= 2; |
1134 } else { | 1154 } else { |
1135 return false; | 1155 return false; |
1136 } | 1156 } |
1137 } | 1157 } |
1138 // Lea optimization only works for i32 type, not i1/i8/i16/i64. | 1158 // Lea optimization only works for i16 and i32 types, not i8. |
1139 if (Ty != IceType_i32 && (Count3 || Count5 || Count9)) | 1159 if (Ty != IceType_i32 && !(Traits::Is64Bit && Ty == IceType_i64) && |
| 1160 (Count3 || Count5 || Count9)) |
1140 return false; | 1161 return false; |
1141 // Limit the number of lea/shl operations for a single multiply, to a | 1162 // Limit the number of lea/shl operations for a single multiply, to a |
1142 // somewhat arbitrary choice of 3. | 1163 // somewhat arbitrary choice of 3. |
1143 constexpr uint32_t MaxOpsForOptimizedMul = 3; | 1164 constexpr uint32_t MaxOpsForOptimizedMul = 3; |
1144 if (CountOps > MaxOpsForOptimizedMul) | 1165 if (CountOps > MaxOpsForOptimizedMul) |
1145 return false; | 1166 return false; |
1146 _mov(T, Src0); | 1167 Variable *T = makeReg(Traits::WordType); |
| 1168 if (typeWidthInBytes(Src0->getType()) < typeWidthInBytes(T->getType())) { |
| 1169 _movzx(T, Src0); |
| 1170 } else { |
| 1171 _mov(T, Src0); |
| 1172 } |
1147 Constant *Zero = Ctx->getConstantZero(IceType_i32); | 1173 Constant *Zero = Ctx->getConstantZero(IceType_i32); |
1148 for (uint32_t i = 0; i < Count9; ++i) { | 1174 for (uint32_t i = 0; i < Count9; ++i) { |
1149 constexpr uint16_t Shift = 3; // log2(9-1) | 1175 constexpr uint16_t Shift = 3; // log2(9-1) |
1150 _lea(T, X86OperandMem::create(Func, IceType_void, T, Zero, T, Shift)); | 1176 _lea(T, X86OperandMem::create(Func, IceType_void, T, Zero, T, Shift)); |
1151 } | 1177 } |
1152 for (uint32_t i = 0; i < Count5; ++i) { | 1178 for (uint32_t i = 0; i < Count5; ++i) { |
1153 constexpr uint16_t Shift = 2; // log2(5-1) | 1179 constexpr uint16_t Shift = 2; // log2(5-1) |
1154 _lea(T, X86OperandMem::create(Func, IceType_void, T, Zero, T, Shift)); | 1180 _lea(T, X86OperandMem::create(Func, IceType_void, T, Zero, T, Shift)); |
1155 } | 1181 } |
1156 for (uint32_t i = 0; i < Count3; ++i) { | 1182 for (uint32_t i = 0; i < Count3; ++i) { |
(...skipping 2379 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3536 } | 3562 } |
3537 case Intrinsics::Memmove: { | 3563 case Intrinsics::Memmove: { |
3538 lowerMemmove(Instr->getArg(0), Instr->getArg(1), Instr->getArg(2)); | 3564 lowerMemmove(Instr->getArg(0), Instr->getArg(1), Instr->getArg(2)); |
3539 return; | 3565 return; |
3540 } | 3566 } |
3541 case Intrinsics::Memset: { | 3567 case Intrinsics::Memset: { |
3542 lowerMemset(Instr->getArg(0), Instr->getArg(1), Instr->getArg(2)); | 3568 lowerMemset(Instr->getArg(0), Instr->getArg(1), Instr->getArg(2)); |
3543 return; | 3569 return; |
3544 } | 3570 } |
3545 case Intrinsics::NaClReadTP: { | 3571 case Intrinsics::NaClReadTP: { |
3546 if (Ctx->getFlags().getUseSandboxing()) { | 3572 if (NeedSandboxing) { |
3547 Operand *Src = | 3573 Operand *Src = |
3548 dispatchToConcrete(&ConcreteTarget::createNaClReadTPSrcOperand); | 3574 dispatchToConcrete(&ConcreteTarget::createNaClReadTPSrcOperand); |
3549 Variable *Dest = Instr->getDest(); | 3575 Variable *Dest = Instr->getDest(); |
3550 Variable *T = nullptr; | 3576 Variable *T = nullptr; |
3551 _mov(T, Src); | 3577 _mov(T, Src); |
3552 _mov(Dest, T); | 3578 _mov(Dest, T); |
3553 } else { | 3579 } else { |
3554 InstCall *Call = makeHelperCall(H_call_read_tp, Instr->getDest(), 0); | 3580 InstCall *Call = makeHelperCall(H_call_read_tp, Instr->getDest(), 0); |
3555 lowerCall(Call); | 3581 lowerCall(Call); |
3556 } | 3582 } |
3557 return; | 3583 return; |
3558 } | 3584 } |
3559 case Intrinsics::Setjmp: { | 3585 case Intrinsics::Setjmp: { |
3560 InstCall *Call = makeHelperCall(H_call_setjmp, Instr->getDest(), 1); | 3586 InstCall *Call = makeHelperCall(H_call_setjmp, Instr->getDest(), 1); |
3561 Call->addArg(Instr->getArg(0)); | 3587 Call->addArg(Instr->getArg(0)); |
3562 lowerCall(Call); | 3588 lowerCall(Call); |
3563 return; | 3589 return; |
3564 } | 3590 } |
3565 case Intrinsics::Sqrt: { | 3591 case Intrinsics::Sqrt: { |
3566 Operand *Src = legalize(Instr->getArg(0)); | 3592 Operand *Src = legalize(Instr->getArg(0)); |
3567 Variable *Dest = Instr->getDest(); | 3593 Variable *Dest = Instr->getDest(); |
3568 Variable *T = makeReg(Dest->getType()); | 3594 Variable *T = makeReg(Dest->getType()); |
3569 _sqrtss(T, Src); | 3595 _sqrtss(T, Src); |
3570 _mov(Dest, T); | 3596 _mov(Dest, T); |
3571 return; | 3597 return; |
3572 } | 3598 } |
3573 case Intrinsics::Stacksave: { | 3599 case Intrinsics::Stacksave: { |
3574 Variable *esp = | 3600 if (!Traits::Is64Bit || !NeedSandboxing) { |
3575 Func->getTarget()->getPhysicalRegister(getStackReg(), Traits::WordType); | 3601 Variable *esp = Func->getTarget()->getPhysicalRegister(getStackReg(), |
| 3602 Traits::WordType); |
| 3603 Variable *Dest = Instr->getDest(); |
| 3604 _mov(Dest, esp); |
| 3605 return; |
| 3606 } |
| 3607 Variable *esp = Func->getTarget()->getPhysicalRegister( |
| 3608 Traits::RegisterSet::Reg_esp, IceType_i32); |
3576 Variable *Dest = Instr->getDest(); | 3609 Variable *Dest = Instr->getDest(); |
3577 _mov(Dest, esp); | 3610 _mov(Dest, esp); |
| 3611 |
3578 return; | 3612 return; |
3579 } | 3613 } |
3580 case Intrinsics::Stackrestore: { | 3614 case Intrinsics::Stackrestore: { |
3581 Operand *Src = Instr->getArg(0); | 3615 Operand *Src = Instr->getArg(0); |
3582 const Type SrcTy = Src->getType(); | 3616 _mov_sp(Src); |
3583 Variable *esp = Func->getTarget()->getPhysicalRegister( | |
3584 Traits::getGprForType(SrcTy, getStackReg()), SrcTy); | |
3585 _redefined(_mov(esp, Src)); | |
3586 return; | 3617 return; |
3587 } | 3618 } |
| 3619 |
3588 case Intrinsics::Trap: | 3620 case Intrinsics::Trap: |
3589 _ud2(); | 3621 _ud2(); |
3590 return; | 3622 return; |
3591 case Intrinsics::UnknownIntrinsic: | 3623 case Intrinsics::UnknownIntrinsic: |
3592 Func->setError("Should not be lowering UnknownIntrinsic"); | 3624 Func->setError("Should not be lowering UnknownIntrinsic"); |
3593 return; | 3625 return; |
3594 } | 3626 } |
3595 return; | 3627 return; |
3596 } | 3628 } |
3597 | 3629 |
(...skipping 688 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
4286 lowerCast(InstCast::create(Func, InstCast::Zext, ValExtVar, Val)); | 4318 lowerCast(InstCast::create(Func, InstCast::Zext, ValExtVar, Val)); |
4287 ValExt = ValExtVar; | 4319 ValExt = ValExtVar; |
4288 } | 4320 } |
4289 InstCall *Call = makeHelperCall(H_call_memset, nullptr, 3); | 4321 InstCall *Call = makeHelperCall(H_call_memset, nullptr, 3); |
4290 Call->addArg(Dest); | 4322 Call->addArg(Dest); |
4291 Call->addArg(ValExt); | 4323 Call->addArg(ValExt); |
4292 Call->addArg(Count); | 4324 Call->addArg(Count); |
4293 lowerCall(Call); | 4325 lowerCall(Call); |
4294 } | 4326 } |
4295 | 4327 |
4296 template <typename TraitsType> | |
4297 void TargetX86Base<TraitsType>::lowerIndirectJump(Variable *JumpTarget) { | |
4298 const bool NeedSandboxing = Ctx->getFlags().getUseSandboxing(); | |
4299 if (Traits::Is64Bit) { | |
4300 Variable *T = makeReg(IceType_i64); | |
4301 _movzx(T, JumpTarget); | |
4302 JumpTarget = T; | |
4303 } | |
4304 if (NeedSandboxing) { | |
4305 _bundle_lock(); | |
4306 const SizeT BundleSize = | |
4307 1 << Func->getAssembler<>()->getBundleAlignLog2Bytes(); | |
4308 _and(JumpTarget, Ctx->getConstantInt32(~(BundleSize - 1))); | |
4309 } | |
4310 _jmp(JumpTarget); | |
4311 if (NeedSandboxing) | |
4312 _bundle_unlock(); | |
4313 } | |
4314 | |
4315 inline bool isAdd(const Inst *Inst) { | 4328 inline bool isAdd(const Inst *Inst) { |
4316 if (auto *Arith = llvm::dyn_cast_or_null<const InstArithmetic>(Inst)) { | 4329 if (auto *Arith = llvm::dyn_cast_or_null<const InstArithmetic>(Inst)) { |
4317 return (Arith->getOp() == InstArithmetic::Add); | 4330 return (Arith->getOp() == InstArithmetic::Add); |
4318 } | 4331 } |
4319 return false; | 4332 return false; |
4320 } | 4333 } |
4321 | 4334 |
4322 inline void dumpAddressOpt(const Cfg *Func, | 4335 inline void dumpAddressOpt(const Cfg *Func, |
4323 const ConstantRelocatable *Relocatable, | 4336 const ConstantRelocatable *Relocatable, |
4324 int32_t Offset, const Variable *Base, | 4337 int32_t Offset, const Variable *Base, |
(...skipping 219 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
4544 return false; | 4557 return false; |
4545 // Incorporate ConstantRelocatables. | 4558 // Incorporate ConstantRelocatables. |
4546 if (Reloc0) | 4559 if (Reloc0) |
4547 NewRelocatable = Reloc0; | 4560 NewRelocatable = Reloc0; |
4548 else if (Reloc1) | 4561 else if (Reloc1) |
4549 NewRelocatable = Reloc1; | 4562 NewRelocatable = Reloc1; |
4550 if ((Reloc0 || Reloc1) && BaseOther && GotVar) | 4563 if ((Reloc0 || Reloc1) && BaseOther && GotVar) |
4551 return false; | 4564 return false; |
4552 // Compute the updated constant offset. | 4565 // Compute the updated constant offset. |
4553 if (Const0) { | 4566 if (Const0) { |
4554 int32_t MoreOffset = IsAdd ? Const0->getValue() : -Const0->getValue(); | 4567 const int32_t MoreOffset = |
| 4568 IsAdd ? Const0->getValue() : -Const0->getValue(); |
4555 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) | 4569 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) |
4556 return false; | 4570 return false; |
4557 NewOffset += MoreOffset; | 4571 NewOffset += MoreOffset; |
4558 } | 4572 } |
4559 if (Const1) { | 4573 if (Const1) { |
4560 int32_t MoreOffset = IsAdd ? Const1->getValue() : -Const1->getValue(); | 4574 const int32_t MoreOffset = |
| 4575 IsAdd ? Const1->getValue() : -Const1->getValue(); |
4561 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) | 4576 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) |
4562 return false; | 4577 return false; |
4563 NewOffset += MoreOffset; | 4578 NewOffset += MoreOffset; |
4564 } | 4579 } |
4565 // Update the computed address parameters once we are sure optimization | 4580 // Update the computed address parameters once we are sure optimization |
4566 // is valid. | 4581 // is valid. |
4567 if ((Reloc0 || Reloc1) && GotVar) { | 4582 if ((Reloc0 || Reloc1) && GotVar) { |
4568 assert(BaseOther == nullptr); | 4583 assert(BaseOther == nullptr); |
4569 BaseOther = GotVar; | 4584 BaseOther = GotVar; |
4570 } | 4585 } |
4571 Base = NewBase; | 4586 Base = NewBase; |
4572 Offset = NewOffset; | 4587 Offset = NewOffset; |
4573 Relocatable = NewRelocatable; | 4588 Relocatable = NewRelocatable; |
4574 Reason = BaseInst; | 4589 Reason = BaseInst; |
4575 return true; | 4590 return true; |
4576 } | 4591 } |
4577 return false; | 4592 return false; |
4578 } | 4593 } |
4579 | 4594 |
4580 // Builds information for a canonical address expresion: | 4595 // Builds information for a canonical address expresion: |
4581 // <Relocatable + Offset>(Base, Index, Shift) | 4596 // <Relocatable + Offset>(Base, Index, Shift) |
4582 // On entry: | 4597 // On entry: |
4583 // Relocatable == null, | 4598 // Relocatable == null, |
4584 // Offset == 0, | 4599 // Offset == 0, |
4585 // Base is a Variable, | 4600 // Base is a Variable, |
4586 // Index == nullptr, | 4601 // Index == nullptr, |
4587 // Shift == 0 | 4602 // Shift == 0 |
4588 inline bool computeAddressOpt(Cfg *Func, const Inst *Instr, Variable *GotVar, | 4603 inline bool computeAddressOpt(Cfg *Func, const Inst *Instr, Variable *GotVar, |
| 4604 bool ReserveSlot, |
4589 ConstantRelocatable *&Relocatable, | 4605 ConstantRelocatable *&Relocatable, |
4590 int32_t &Offset, Variable *&Base, | 4606 int32_t &Offset, Variable *&Base, |
4591 Variable *&Index, uint16_t &Shift) { | 4607 Variable *&Index, uint16_t &Shift) { |
4592 bool AddressWasOptimized = false; | 4608 bool AddressWasOptimized = false; |
4593 Func->resetCurrentNode(); | 4609 Func->resetCurrentNode(); |
4594 if (Func->isVerbose(IceV_AddrOpt)) { | 4610 if (Func->isVerbose(IceV_AddrOpt)) { |
4595 OstreamLocker L(Func->getContext()); | 4611 OstreamLocker L(Func->getContext()); |
4596 Ostream &Str = Func->getContext()->getStrDump(); | 4612 Ostream &Str = Func->getContext()->getStrDump(); |
4597 Str << "\nStarting computeAddressOpt for instruction:\n "; | 4613 Str << "\nStarting computeAddressOpt for instruction:\n "; |
4598 Instr->dumpDecorated(Func); | 4614 Instr->dumpDecorated(Func); |
4599 } | 4615 } |
4600 if (Base == nullptr) | 4616 if (Base == nullptr) |
4601 return AddressWasOptimized; | 4617 return AddressWasOptimized; |
4602 // If the Base has more than one use or is live across multiple blocks, then | 4618 // If the Base has more than one use or is live across multiple blocks, then |
4603 // don't go further. Alternatively (?), never consider a transformation that | 4619 // don't go further. Alternatively (?), never consider a transformation that |
4604 // would change a variable that is currently *not* live across basic block | 4620 // would change a variable that is currently *not* live across basic block |
4605 // boundaries into one that *is*. | 4621 // boundaries into one that *is*. |
4606 if (Func->getVMetadata()->isMultiBlock(Base) /* || Base->getUseCount() > 1*/) | 4622 if (Func->getVMetadata()->isMultiBlock(Base) /* || Base->getUseCount() > 1*/) |
4607 return AddressWasOptimized; | 4623 return AddressWasOptimized; |
4608 | 4624 |
4609 const bool MockBounds = Func->getContext()->getFlags().getMockBoundsCheck(); | 4625 const bool MockBounds = Func->getContext()->getFlags().getMockBoundsCheck(); |
4610 const VariablesMetadata *VMetadata = Func->getVMetadata(); | 4626 const VariablesMetadata *VMetadata = Func->getVMetadata(); |
4611 const Inst *Reason = nullptr; | 4627 const Inst *Reason = nullptr; |
4612 do { | 4628 do { |
| 4629 assert(!ReserveSlot || Base == nullptr || Index == nullptr); |
4613 if (Reason) { | 4630 if (Reason) { |
4614 dumpAddressOpt(Func, Relocatable, Offset, Base, Index, Shift, Reason); | 4631 dumpAddressOpt(Func, Relocatable, Offset, Base, Index, Shift, Reason); |
4615 AddressWasOptimized = true; | 4632 AddressWasOptimized = true; |
4616 Reason = nullptr; | 4633 Reason = nullptr; |
4617 } | 4634 } |
4618 // Update Base and Index to follow through assignments to definitions. | 4635 // Update Base and Index to follow through assignments to definitions. |
4619 if (matchAssign(VMetadata, GotVar, Base, Relocatable, Offset, Reason)) { | 4636 if (matchAssign(VMetadata, GotVar, Base, Relocatable, Offset, Reason)) { |
4620 // Assignments of Base from a Relocatable or ConstantInt32 can result | 4637 // Assignments of Base from a Relocatable or ConstantInt32 can result |
4621 // in Base becoming nullptr. To avoid code duplication in this loop we | 4638 // in Base becoming nullptr. To avoid code duplication in this loop we |
4622 // prefer that Base be non-nullptr if possible. | 4639 // prefer that Base be non-nullptr if possible. |
4623 if ((Base == nullptr) && (Index != nullptr) && Shift == 0) | 4640 if ((Base == nullptr) && (Index != nullptr) && Shift == 0) |
4624 std::swap(Base, Index); | 4641 std::swap(Base, Index); |
4625 continue; | 4642 continue; |
4626 } | 4643 } |
4627 if (matchAssign(VMetadata, GotVar, Index, Relocatable, Offset, Reason)) | 4644 if (matchAssign(VMetadata, GotVar, Index, Relocatable, Offset, Reason)) |
4628 continue; | 4645 continue; |
4629 | 4646 |
4630 if (!MockBounds) { | 4647 if (!MockBounds) { |
4631 // Transition from: | 4648 // Transition from: |
4632 // <Relocatable + Offset>(Base) to | 4649 // <Relocatable + Offset>(Base) to |
4633 // <Relocatable + Offset>(Base, Index) | 4650 // <Relocatable + Offset>(Base, Index) |
4634 if (matchCombinedBaseIndex(VMetadata, Base, Index, Shift, Reason)) | 4651 if (!ReserveSlot && |
| 4652 matchCombinedBaseIndex(VMetadata, Base, Index, Shift, Reason)) |
4635 continue; | 4653 continue; |
4636 // Recognize multiply/shift and update Shift amount. | 4654 // Recognize multiply/shift and update Shift amount. |
4637 // Index becomes Index=Var<<Const && Const+Shift<=3 ==> | 4655 // Index becomes Index=Var<<Const && Const+Shift<=3 ==> |
4638 // Index=Var, Shift+=Const | 4656 // Index=Var, Shift+=Const |
4639 // Index becomes Index=Const*Var && log2(Const)+Shift<=3 ==> | 4657 // Index becomes Index=Const*Var && log2(Const)+Shift<=3 ==> |
4640 // Index=Var, Shift+=log2(Const) | 4658 // Index=Var, Shift+=log2(Const) |
4641 if (matchShiftedIndex(VMetadata, Index, Shift, Reason)) | 4659 if (matchShiftedIndex(VMetadata, Index, Shift, Reason)) |
4642 continue; | 4660 continue; |
4643 // If Shift is zero, the choice of Base and Index was purely arbitrary. | 4661 // If Shift is zero, the choice of Base and Index was purely arbitrary. |
4644 // Recognize multiply/shift and set Shift amount. | 4662 // Recognize multiply/shift and set Shift amount. |
(...skipping 112 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
4757 Variable *Index = nullptr; | 4775 Variable *Index = nullptr; |
4758 ConstantRelocatable *Relocatable = nullptr; | 4776 ConstantRelocatable *Relocatable = nullptr; |
4759 uint16_t Shift = 0; | 4777 uint16_t Shift = 0; |
4760 int32_t Offset = 0; | 4778 int32_t Offset = 0; |
4761 // Vanilla ICE load instructions should not use the segment registers, and | 4779 // Vanilla ICE load instructions should not use the segment registers, and |
4762 // computeAddressOpt only works at the level of Variables and Constants, not | 4780 // computeAddressOpt only works at the level of Variables and Constants, not |
4763 // other X86OperandMem, so there should be no mention of segment | 4781 // other X86OperandMem, so there should be no mention of segment |
4764 // registers there either. | 4782 // registers there either. |
4765 constexpr auto SegmentReg = X86OperandMem::SegmentRegisters::DefaultSegment; | 4783 constexpr auto SegmentReg = X86OperandMem::SegmentRegisters::DefaultSegment; |
4766 auto *Base = llvm::dyn_cast<Variable>(Addr); | 4784 auto *Base = llvm::dyn_cast<Variable>(Addr); |
4767 if (computeAddressOpt(Func, Inst, GotVar, Relocatable, Offset, Base, Index, | 4785 const bool ReserveSlot = Traits::Is64Bit && NeedSandboxing; |
4768 Shift)) { | 4786 if (computeAddressOpt(Func, Inst, GotVar, ReserveSlot, Relocatable, Offset, |
| 4787 Base, Index, Shift)) { |
4769 Inst->setDeleted(); | 4788 Inst->setDeleted(); |
4770 Constant *OffsetOp = nullptr; | 4789 Constant *OffsetOp = nullptr; |
4771 if (Relocatable == nullptr) { | 4790 if (Relocatable == nullptr) { |
4772 OffsetOp = Ctx->getConstantInt32(Offset); | 4791 OffsetOp = Ctx->getConstantInt32(Offset); |
4773 } else { | 4792 } else { |
4774 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, | 4793 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, |
4775 Relocatable->getName(), | 4794 Relocatable->getName(), |
4776 Relocatable->getSuppressMangling()); | 4795 Relocatable->getSuppressMangling()); |
4777 } | 4796 } |
4778 // The new mem operand is created without IsPIC being set, because | 4797 // The new mem operand is created without IsRebased being set, because |
4779 // computeAddressOpt() doesn't include GotVar in its final result. | 4798 // computeAddressOpt() doesn't include GotVar in its final result. |
4780 Addr = X86OperandMem::create(Func, Dest->getType(), Base, OffsetOp, Index, | 4799 Addr = X86OperandMem::create(Func, Dest->getType(), Base, OffsetOp, Index, |
4781 Shift, SegmentReg); | 4800 Shift, SegmentReg); |
4782 Context.insert<InstLoad>(Dest, Addr); | 4801 Context.insert<InstLoad>(Dest, Addr); |
4783 } | 4802 } |
4784 } | 4803 } |
4785 | 4804 |
4786 template <typename TraitsType> | 4805 template <typename TraitsType> |
4787 void TargetX86Base<TraitsType>::randomlyInsertNop(float Probability, | 4806 void TargetX86Base<TraitsType>::randomlyInsertNop(float Probability, |
4788 RandomNumberGenerator &RNG) { | 4807 RandomNumberGenerator &RNG) { |
(...skipping 273 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5062 Variable *Index = nullptr; | 5081 Variable *Index = nullptr; |
5063 ConstantRelocatable *Relocatable = nullptr; | 5082 ConstantRelocatable *Relocatable = nullptr; |
5064 uint16_t Shift = 0; | 5083 uint16_t Shift = 0; |
5065 int32_t Offset = 0; | 5084 int32_t Offset = 0; |
5066 auto *Base = llvm::dyn_cast<Variable>(Addr); | 5085 auto *Base = llvm::dyn_cast<Variable>(Addr); |
5067 // Vanilla ICE store instructions should not use the segment registers, and | 5086 // Vanilla ICE store instructions should not use the segment registers, and |
5068 // computeAddressOpt only works at the level of Variables and Constants, not | 5087 // computeAddressOpt only works at the level of Variables and Constants, not |
5069 // other X86OperandMem, so there should be no mention of segment | 5088 // other X86OperandMem, so there should be no mention of segment |
5070 // registers there either. | 5089 // registers there either. |
5071 constexpr auto SegmentReg = X86OperandMem::SegmentRegisters::DefaultSegment; | 5090 constexpr auto SegmentReg = X86OperandMem::SegmentRegisters::DefaultSegment; |
5072 if (computeAddressOpt(Func, Inst, GotVar, Relocatable, Offset, Base, Index, | 5091 const bool ReserveSlot = Traits::Is64Bit && NeedSandboxing; |
5073 Shift)) { | 5092 if (computeAddressOpt(Func, Inst, GotVar, ReserveSlot, Relocatable, Offset, |
| 5093 Base, Index, Shift)) { |
5074 Inst->setDeleted(); | 5094 Inst->setDeleted(); |
5075 Constant *OffsetOp = nullptr; | 5095 Constant *OffsetOp = nullptr; |
5076 if (Relocatable == nullptr) { | 5096 if (Relocatable == nullptr) { |
5077 OffsetOp = Ctx->getConstantInt32(Offset); | 5097 OffsetOp = Ctx->getConstantInt32(Offset); |
5078 } else { | 5098 } else { |
5079 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, | 5099 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, |
5080 Relocatable->getName(), | 5100 Relocatable->getName(), |
5081 Relocatable->getSuppressMangling()); | 5101 Relocatable->getSuppressMangling()); |
5082 } | 5102 } |
5083 // The new mem operand is created without IsPIC being set, because | 5103 // The new mem operand is created without IsRebased being set, because |
5084 // computeAddressOpt() doesn't include GotVar in its final result. | 5104 // computeAddressOpt() doesn't include GotVar in its final result. |
5085 Addr = X86OperandMem::create(Func, Data->getType(), Base, OffsetOp, Index, | 5105 Addr = X86OperandMem::create(Func, Data->getType(), Base, OffsetOp, Index, |
5086 Shift, SegmentReg); | 5106 Shift, SegmentReg); |
5087 auto *NewStore = Context.insert<InstStore>(Data, Addr); | 5107 auto *NewStore = Context.insert<InstStore>(Data, Addr); |
5088 if (Inst->getDest()) | 5108 if (Inst->getDest()) |
5089 NewStore->setRmwBeacon(Inst->getRmwBeacon()); | 5109 NewStore->setRmwBeacon(Inst->getRmwBeacon()); |
5090 } | 5110 } |
5091 } | 5111 } |
5092 | 5112 |
5093 template <typename TraitsType> | 5113 template <typename TraitsType> |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5127 _br(Traits::Cond::Br_a, SkipJumpTable); | 5147 _br(Traits::Cond::Br_a, SkipJumpTable); |
5128 } else { | 5148 } else { |
5129 _br(Traits::Cond::Br_a, DefaultTarget); | 5149 _br(Traits::Cond::Br_a, DefaultTarget); |
5130 } | 5150 } |
5131 | 5151 |
5132 InstJumpTable *JumpTable = Case.getJumpTable(); | 5152 InstJumpTable *JumpTable = Case.getJumpTable(); |
5133 Context.insert(JumpTable); | 5153 Context.insert(JumpTable); |
5134 | 5154 |
5135 // Make sure the index is a register of the same width as the base | 5155 // Make sure the index is a register of the same width as the base |
5136 Variable *Index; | 5156 Variable *Index; |
5137 if (RangeIndex->getType() != getPointerType()) { | 5157 const Type PointerType = getPointerType(); |
5138 Index = makeReg(getPointerType()); | 5158 if (RangeIndex->getType() != PointerType) { |
| 5159 Index = makeReg(PointerType); |
5139 _movzx(Index, RangeIndex); | 5160 _movzx(Index, RangeIndex); |
5140 } else { | 5161 } else { |
5141 Index = legalizeToReg(RangeIndex); | 5162 Index = legalizeToReg(RangeIndex); |
5142 } | 5163 } |
5143 | 5164 |
5144 constexpr RelocOffsetT RelocOffset = 0; | 5165 constexpr RelocOffsetT RelocOffset = 0; |
5145 constexpr bool SuppressMangling = true; | 5166 constexpr bool SuppressMangling = true; |
5146 const bool IsPIC = Ctx->getFlags().getUseNonsfi(); | 5167 const bool IsRebased = Ctx->getFlags().getUseNonsfi(); |
5147 IceString MangledName = Ctx->mangleName(Func->getFunctionName()); | 5168 IceString MangledName = Ctx->mangleName(Func->getFunctionName()); |
5148 Variable *Base = IsPIC ? legalizeToReg(GotVar) : nullptr; | 5169 Variable *Base = IsRebased ? legalizeToReg(GotVar) : nullptr; |
5149 Constant *Offset = Ctx->getConstantSym( | 5170 Constant *Offset = Ctx->getConstantSym( |
5150 RelocOffset, InstJumpTable::makeName(MangledName, JumpTable->getId()), | 5171 RelocOffset, InstJumpTable::makeName(MangledName, JumpTable->getId()), |
5151 SuppressMangling); | 5172 SuppressMangling); |
5152 uint16_t Shift = typeWidthInBytesLog2(getPointerType()); | 5173 uint16_t Shift = typeWidthInBytesLog2(PointerType); |
5153 constexpr auto Segment = X86OperandMem::SegmentRegisters::DefaultSegment; | 5174 constexpr auto Segment = X86OperandMem::SegmentRegisters::DefaultSegment; |
| 5175 |
| 5176 Variable *Target = nullptr; |
| 5177 if (Traits::Is64Bit && NeedSandboxing) { |
| 5178 assert(Base == nullptr); |
| 5179 assert(Index != nullptr && Index->getType() == IceType_i32); |
| 5180 } |
5154 auto *TargetInMemory = X86OperandMem::create( | 5181 auto *TargetInMemory = X86OperandMem::create( |
5155 Func, getPointerType(), Base, Offset, Index, Shift, Segment, IsPIC); | 5182 Func, PointerType, Base, Offset, Index, Shift, Segment, IsRebased); |
5156 Variable *Target = nullptr; | |
5157 _mov(Target, TargetInMemory); | 5183 _mov(Target, TargetInMemory); |
| 5184 |
5158 lowerIndirectJump(Target); | 5185 lowerIndirectJump(Target); |
5159 | 5186 |
5160 if (DefaultTarget == nullptr) | 5187 if (DefaultTarget == nullptr) |
5161 Context.insert(SkipJumpTable); | 5188 Context.insert(SkipJumpTable); |
5162 return; | 5189 return; |
5163 } | 5190 } |
5164 case CaseCluster::Range: { | 5191 case CaseCluster::Range: { |
5165 if (Case.isUnitRange()) { | 5192 if (Case.isUnitRange()) { |
5166 // Single item | 5193 // Single item |
5167 if (!DoneCmp) { | 5194 if (!DoneCmp) { |
(...skipping 862 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
6030 } | 6057 } |
6031 | 6058 |
6032 if (auto *Mem = llvm::dyn_cast<X86OperandMem>(From)) { | 6059 if (auto *Mem = llvm::dyn_cast<X86OperandMem>(From)) { |
6033 // Before doing anything with a Mem operand, we need to ensure that the | 6060 // Before doing anything with a Mem operand, we need to ensure that the |
6034 // Base and Index components are in physical registers. | 6061 // Base and Index components are in physical registers. |
6035 Variable *Base = Mem->getBase(); | 6062 Variable *Base = Mem->getBase(); |
6036 Variable *Index = Mem->getIndex(); | 6063 Variable *Index = Mem->getIndex(); |
6037 Constant *Offset = Mem->getOffset(); | 6064 Constant *Offset = Mem->getOffset(); |
6038 Variable *RegBase = nullptr; | 6065 Variable *RegBase = nullptr; |
6039 Variable *RegIndex = nullptr; | 6066 Variable *RegIndex = nullptr; |
| 6067 uint16_t Shift = Mem->getShift(); |
6040 if (Base) { | 6068 if (Base) { |
6041 RegBase = llvm::cast<Variable>( | 6069 RegBase = llvm::cast<Variable>( |
6042 legalize(Base, Legal_Reg | Legal_Rematerializable)); | 6070 legalize(Base, Legal_Reg | Legal_Rematerializable)); |
6043 } | 6071 } |
6044 if (Index) { | 6072 if (Index) { |
| 6073 // TODO(jpp): perhaps we should only allow Legal_Reg if |
| 6074 // Base->isRematerializable. |
6045 RegIndex = llvm::cast<Variable>( | 6075 RegIndex = llvm::cast<Variable>( |
6046 legalize(Index, Legal_Reg | Legal_Rematerializable)); | 6076 legalize(Index, Legal_Reg | Legal_Rematerializable)); |
6047 } | 6077 } |
6048 // For Non-SFI mode, if the Offset field is a ConstantRelocatable, we | 6078 // For Non-SFI mode, if the Offset field is a ConstantRelocatable, we |
6049 // replace either Base or Index with a legalized GotVar. At emission time, | 6079 // replace either Base or Index with a legalized GotVar. At emission time, |
6050 // the ConstantRelocatable will be emitted with the @GOTOFF relocation. | 6080 // the ConstantRelocatable will be emitted with the @GOTOFF relocation. |
6051 bool NeedPIC = false; | 6081 bool IsRebased = false; |
6052 if (UseNonsfi && !Mem->getIsPIC() && Offset && | 6082 if (UseNonsfi && !Mem->getIsRebased() && Offset && |
6053 llvm::isa<ConstantRelocatable>(Offset)) { | 6083 llvm::isa<ConstantRelocatable>(Offset)) { |
6054 assert(!(Allowed & Legal_AddrAbs)); | 6084 assert(!(Allowed & Legal_AddrAbs)); |
6055 NeedPIC = true; | 6085 IsRebased = true; |
6056 if (RegBase == nullptr) { | 6086 if (RegBase == nullptr) { |
6057 RegBase = legalizeToReg(GotVar); | 6087 RegBase = legalizeToReg(GotVar); |
6058 } else if (RegIndex == nullptr) { | 6088 } else if (RegIndex == nullptr) { |
6059 RegIndex = legalizeToReg(GotVar); | 6089 RegIndex = legalizeToReg(GotVar); |
6060 } else { | 6090 } else { |
6061 llvm::report_fatal_error( | 6091 llvm::report_fatal_error( |
6062 "Either Base or Index must be unused in Non-SFI mode"); | 6092 "Either Base or Index must be unused in Non-SFI mode"); |
6063 } | 6093 } |
6064 } | 6094 } |
6065 if (Base != RegBase || Index != RegIndex) { | 6095 if (Base != RegBase || Index != RegIndex) { |
6066 Mem = X86OperandMem::create(Func, Ty, RegBase, Offset, RegIndex, | 6096 Mem = X86OperandMem::create(Func, Ty, RegBase, Offset, RegIndex, Shift, |
6067 Mem->getShift(), Mem->getSegmentRegister(), | 6097 Mem->getSegmentRegister(), IsRebased); |
6068 NeedPIC); | |
6069 } | 6098 } |
6070 | 6099 |
6071 // For all Memory Operands, we do randomization/pooling here | 6100 // For all Memory Operands, we do randomization/pooling here |
6072 From = randomizeOrPoolImmediate(Mem); | 6101 From = randomizeOrPoolImmediate(Mem); |
6073 | 6102 |
6074 if (!(Allowed & Legal_Mem)) { | 6103 if (!(Allowed & Legal_Mem)) { |
6075 From = copyToReg(From, RegNum); | 6104 From = copyToReg(From, RegNum); |
6076 } | 6105 } |
6077 return From; | 6106 return From; |
6078 } | 6107 } |
(...skipping 28 matching lines...) Expand all Loading... |
6107 } | 6136 } |
6108 } | 6137 } |
6109 | 6138 |
6110 // If the operand is a ConstantRelocatable, and Legal_AddrAbs is not | 6139 // If the operand is a ConstantRelocatable, and Legal_AddrAbs is not |
6111 // specified, and UseNonsfi is indicated, we need to add GotVar. | 6140 // specified, and UseNonsfi is indicated, we need to add GotVar. |
6112 if (auto *CR = llvm::dyn_cast<ConstantRelocatable>(Const)) { | 6141 if (auto *CR = llvm::dyn_cast<ConstantRelocatable>(Const)) { |
6113 if (UseNonsfi && !(Allowed & Legal_AddrAbs)) { | 6142 if (UseNonsfi && !(Allowed & Legal_AddrAbs)) { |
6114 assert(Ty == IceType_i32); | 6143 assert(Ty == IceType_i32); |
6115 Variable *RegBase = legalizeToReg(GotVar); | 6144 Variable *RegBase = legalizeToReg(GotVar); |
6116 Variable *NewVar = makeReg(Ty, RegNum); | 6145 Variable *NewVar = makeReg(Ty, RegNum); |
6117 auto *Mem = Traits::X86OperandMem::create(Func, Ty, RegBase, CR); | 6146 static constexpr bool IsRebased = true; |
6118 Mem->setIsPIC(); | 6147 auto *Mem = |
| 6148 Traits::X86OperandMem::create(Func, Ty, RegBase, CR, IsRebased); |
6119 _lea(NewVar, Mem); | 6149 _lea(NewVar, Mem); |
6120 From = NewVar; | 6150 From = NewVar; |
6121 } | 6151 } |
6122 } | 6152 } |
6123 | 6153 |
6124 // Convert a scalar floating point constant into an explicit memory | 6154 // Convert a scalar floating point constant into an explicit memory |
6125 // operand. | 6155 // operand. |
6126 if (isScalarFloatingType(Ty)) { | 6156 if (isScalarFloatingType(Ty)) { |
6127 if (auto *ConstFloat = llvm::dyn_cast<ConstantFloat>(Const)) { | 6157 if (auto *ConstFloat = llvm::dyn_cast<ConstantFloat>(Const)) { |
6128 if (Utils::isPositiveZero(ConstFloat->getValue())) | 6158 if (Utils::isPositiveZero(ConstFloat->getValue())) |
6129 return makeZeroedRegister(Ty, RegNum); | 6159 return makeZeroedRegister(Ty, RegNum); |
6130 } else if (auto *ConstDouble = llvm::dyn_cast<ConstantDouble>(Const)) { | 6160 } else if (auto *ConstDouble = llvm::dyn_cast<ConstantDouble>(Const)) { |
6131 if (Utils::isPositiveZero(ConstDouble->getValue())) | 6161 if (Utils::isPositiveZero(ConstDouble->getValue())) |
6132 return makeZeroedRegister(Ty, RegNum); | 6162 return makeZeroedRegister(Ty, RegNum); |
6133 } | 6163 } |
6134 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; | 6164 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; |
6135 std::string Buffer; | 6165 std::string Buffer; |
6136 llvm::raw_string_ostream StrBuf(Buffer); | 6166 llvm::raw_string_ostream StrBuf(Buffer); |
6137 llvm::cast<Constant>(From)->emitPoolLabel(StrBuf, Ctx); | 6167 llvm::cast<Constant>(From)->emitPoolLabel(StrBuf, Ctx); |
6138 llvm::cast<Constant>(From)->setShouldBePooled(true); | 6168 llvm::cast<Constant>(From)->setShouldBePooled(true); |
6139 Constant *Offset = Ctx->getConstantSym(0, StrBuf.str(), true); | 6169 Constant *Offset = Ctx->getConstantSym(0, StrBuf.str(), true); |
6140 auto *Mem = X86OperandMem::create(Func, Ty, Base, Offset); | 6170 const bool IsRebased = Base != nullptr; |
6141 if (UseNonsfi) | 6171 auto *Mem = X86OperandMem::create(Func, Ty, Base, Offset, IsRebased); |
6142 Mem->setIsPIC(); | |
6143 From = Mem; | 6172 From = Mem; |
6144 } | 6173 } |
6145 bool NeedsReg = false; | 6174 bool NeedsReg = false; |
6146 if (!(Allowed & Legal_Imm) && !isScalarFloatingType(Ty)) | 6175 if (!(Allowed & Legal_Imm) && !isScalarFloatingType(Ty)) |
6147 // Immediate specifically not allowed | 6176 // Immediate specifically not allowed |
6148 NeedsReg = true; | 6177 NeedsReg = true; |
6149 if (!(Allowed & Legal_Mem) && isScalarFloatingType(Ty)) | 6178 if (!(Allowed & Legal_Mem) && isScalarFloatingType(Ty)) |
6150 // On x86, FP constants are lowered to mem operands. | 6179 // On x86, FP constants are lowered to mem operands. |
6151 NeedsReg = true; | 6180 NeedsReg = true; |
6152 if (NeedsReg) { | 6181 if (NeedsReg) { |
(...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
6256 // we guarantee legalize(Offset) will return a Constant*. | 6285 // we guarantee legalize(Offset) will return a Constant*. |
6257 if (!llvm::isa<ConstantRelocatable>(Offset)) { | 6286 if (!llvm::isa<ConstantRelocatable>(Offset)) { |
6258 BoolFlagSaver B(RandomizationPoolingPaused, true); | 6287 BoolFlagSaver B(RandomizationPoolingPaused, true); |
6259 | 6288 |
6260 Offset = llvm::cast<Constant>(legalize(Offset)); | 6289 Offset = llvm::cast<Constant>(legalize(Offset)); |
6261 } | 6290 } |
6262 | 6291 |
6263 assert(llvm::isa<ConstantInteger32>(Offset) || | 6292 assert(llvm::isa<ConstantInteger32>(Offset) || |
6264 llvm::isa<ConstantRelocatable>(Offset)); | 6293 llvm::isa<ConstantRelocatable>(Offset)); |
6265 } | 6294 } |
6266 // Not completely sure whether it's OK to leave IsPIC unset when creating | 6295 // Not completely sure whether it's OK to leave IsRebased unset when |
6267 // the mem operand. If DoLegalize is true, it will definitely be applied | 6296 // creating the mem operand. If DoLegalize is true, it will definitely be |
6268 // during the legalize() call, but perhaps not during the | 6297 // applied during the legalize() call, but perhaps not during the |
6269 // randomizeOrPoolImmediate() call. In any case, the emit routines will | 6298 // randomizeOrPoolImmediate() call. In any case, the emit routines will |
6270 // assert that PIC legalization has been applied. | 6299 // assert that PIC legalization has been applied. |
6271 Mem = X86OperandMem::create(Func, Ty, Base, Offset); | 6300 Mem = X86OperandMem::create(Func, Ty, Base, Offset); |
6272 } | 6301 } |
6273 // Do legalization, which contains randomization/pooling or do | 6302 // Do legalization, which contains randomization/pooling or do |
6274 // randomization/pooling. | 6303 // randomization/pooling. |
6275 return llvm::cast<X86OperandMem>(DoLegalize ? legalize(Mem) | 6304 return llvm::cast<X86OperandMem>(DoLegalize ? legalize(Mem) |
6276 : randomizeOrPoolImmediate(Mem)); | 6305 : randomizeOrPoolImmediate(Mem)); |
6277 } | 6306 } |
6278 | 6307 |
(...skipping 107 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
6386 Operand * | 6415 Operand * |
6387 TargetX86Base<TraitsType>::randomizeOrPoolImmediate(Constant *Immediate, | 6416 TargetX86Base<TraitsType>::randomizeOrPoolImmediate(Constant *Immediate, |
6388 int32_t RegNum) { | 6417 int32_t RegNum) { |
6389 assert(llvm::isa<ConstantInteger32>(Immediate) || | 6418 assert(llvm::isa<ConstantInteger32>(Immediate) || |
6390 llvm::isa<ConstantRelocatable>(Immediate)); | 6419 llvm::isa<ConstantRelocatable>(Immediate)); |
6391 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_None || | 6420 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_None || |
6392 RandomizationPoolingPaused == true) { | 6421 RandomizationPoolingPaused == true) { |
6393 // Immediates randomization/pooling off or paused | 6422 // Immediates randomization/pooling off or paused |
6394 return Immediate; | 6423 return Immediate; |
6395 } | 6424 } |
6396 if (Immediate->shouldBeRandomizedOrPooled(Ctx)) { | 6425 |
6397 Ctx->statsUpdateRPImms(); | 6426 if (Traits::Is64Bit && NeedSandboxing) { |
6398 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == | 6427 // Immediate randomization/pooling is currently disabled for x86-64 |
6399 RPI_Randomize) { | 6428 // sandboxing for it could generate invalid memory operands. |
6400 // blind the constant | 6429 assert(false && |
6401 // FROM: | 6430 "Constant pooling/randomization is disabled for x8664 sandbox."); |
6402 // imm | 6431 return Immediate; |
6403 // TO: | 6432 } |
6404 // insert: mov imm+cookie, Reg | 6433 |
6405 // insert: lea -cookie[Reg], Reg | 6434 if (!Immediate->shouldBeRandomizedOrPooled(Ctx)) { |
6406 // => Reg | 6435 // the constant Immediate is not eligible for blinding/pooling |
6407 // If we have already assigned a phy register, we must come from | 6436 return Immediate; |
6408 // advancedPhiLowering()=>lowerAssign(). In this case we should reuse the | 6437 } |
6409 // assigned register as this assignment is that start of its use-def | 6438 Ctx->statsUpdateRPImms(); |
6410 // chain. So we add RegNum argument here. Note we use 'lea' instruction | 6439 switch (Ctx->getFlags().getRandomizeAndPoolImmediatesOption()) { |
6411 // instead of 'xor' to avoid affecting the flags. | 6440 default: |
6412 Variable *Reg = makeReg(IceType_i32, RegNum); | 6441 llvm::report_fatal_error("Unsupported -randomize-pool-immediates option"); |
6413 auto *Integer = llvm::cast<ConstantInteger32>(Immediate); | 6442 case RPI_Randomize: { |
6414 uint32_t Value = Integer->getValue(); | 6443 // blind the constant |
6415 uint32_t Cookie = Func->getConstantBlindingCookie(); | 6444 // FROM: |
6416 _mov(Reg, Ctx->getConstantInt(IceType_i32, Cookie + Value)); | 6445 // imm |
6417 Constant *Offset = Ctx->getConstantInt(IceType_i32, 0 - Cookie); | 6446 // TO: |
6418 _lea(Reg, | 6447 // insert: mov imm+cookie, Reg |
6419 X86OperandMem::create(Func, IceType_i32, Reg, Offset, nullptr, 0)); | 6448 // insert: lea -cookie[Reg], Reg |
6420 if (Immediate->getType() != IceType_i32) { | 6449 // => Reg |
6421 Variable *TruncReg = makeReg(Immediate->getType(), RegNum); | 6450 // If we have already assigned a phy register, we must come from |
6422 _mov(TruncReg, Reg); | 6451 // advancedPhiLowering()=>lowerAssign(). In this case we should reuse the |
6423 return TruncReg; | 6452 // assigned register as this assignment is that start of its use-def |
6424 } | 6453 // chain. So we add RegNum argument here. Note we use 'lea' instruction |
| 6454 // instead of 'xor' to avoid affecting the flags. |
| 6455 Variable *Reg = makeReg(IceType_i32, RegNum); |
| 6456 auto *Integer = llvm::cast<ConstantInteger32>(Immediate); |
| 6457 uint32_t Value = Integer->getValue(); |
| 6458 uint32_t Cookie = Func->getConstantBlindingCookie(); |
| 6459 _mov(Reg, Ctx->getConstantInt(IceType_i32, Cookie + Value)); |
| 6460 Constant *Offset = Ctx->getConstantInt(IceType_i32, 0 - Cookie); |
| 6461 _lea(Reg, X86OperandMem::create(Func, IceType_i32, Reg, Offset)); |
| 6462 if (Immediate->getType() == IceType_i32) { |
6425 return Reg; | 6463 return Reg; |
6426 } | 6464 } |
6427 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_Pool) { | 6465 Variable *TruncReg = makeReg(Immediate->getType(), RegNum); |
6428 // pool the constant | 6466 _mov(TruncReg, Reg); |
6429 // FROM: | 6467 return TruncReg; |
6430 // imm | |
6431 // TO: | |
6432 // insert: mov $label, Reg | |
6433 // => Reg | |
6434 assert(Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_Pool); | |
6435 Immediate->setShouldBePooled(true); | |
6436 // if we have already assigned a phy register, we must come from | |
6437 // advancedPhiLowering()=>lowerAssign(). In this case we should reuse the | |
6438 // assigned register as this assignment is that start of its use-def | |
6439 // chain. So we add RegNum argument here. | |
6440 Variable *Reg = makeReg(Immediate->getType(), RegNum); | |
6441 IceString Label; | |
6442 llvm::raw_string_ostream Label_stream(Label); | |
6443 Immediate->emitPoolLabel(Label_stream, Ctx); | |
6444 constexpr RelocOffsetT Offset = 0; | |
6445 constexpr bool SuppressMangling = true; | |
6446 Constant *Symbol = | |
6447 Ctx->getConstantSym(Offset, Label_stream.str(), SuppressMangling); | |
6448 const bool UseNonsfi = Ctx->getFlags().getUseNonsfi(); | |
6449 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; | |
6450 X86OperandMem *MemOperand = | |
6451 X86OperandMem::create(Func, Immediate->getType(), Base, Symbol); | |
6452 if (UseNonsfi) | |
6453 MemOperand->setIsPIC(); | |
6454 _mov(Reg, MemOperand); | |
6455 return Reg; | |
6456 } | |
6457 assert("Unsupported -randomize-pool-immediates option" && false); | |
6458 } | 6468 } |
6459 // the constant Immediate is not eligible for blinding/pooling | 6469 case RPI_Pool: { |
6460 return Immediate; | 6470 // pool the constant |
| 6471 // FROM: |
| 6472 // imm |
| 6473 // TO: |
| 6474 // insert: mov $label, Reg |
| 6475 // => Reg |
| 6476 assert(Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_Pool); |
| 6477 Immediate->setShouldBePooled(true); |
| 6478 // if we have already assigned a phy register, we must come from |
| 6479 // advancedPhiLowering()=>lowerAssign(). In this case we should reuse the |
| 6480 // assigned register as this assignment is that start of its use-def |
| 6481 // chain. So we add RegNum argument here. |
| 6482 Variable *Reg = makeReg(Immediate->getType(), RegNum); |
| 6483 IceString Label; |
| 6484 llvm::raw_string_ostream Label_stream(Label); |
| 6485 Immediate->emitPoolLabel(Label_stream, Ctx); |
| 6486 constexpr RelocOffsetT Offset = 0; |
| 6487 constexpr bool SuppressMangling = true; |
| 6488 Constant *Symbol = |
| 6489 Ctx->getConstantSym(Offset, Label_stream.str(), SuppressMangling); |
| 6490 const bool UseNonsfi = Ctx->getFlags().getUseNonsfi(); |
| 6491 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; |
| 6492 const bool IsRebased = Base != nullptr; |
| 6493 X86OperandMem *MemOperand = X86OperandMem::create( |
| 6494 Func, Immediate->getType(), Base, Symbol, IsRebased); |
| 6495 _mov(Reg, MemOperand); |
| 6496 return Reg; |
| 6497 } |
| 6498 } |
6461 } | 6499 } |
6462 | 6500 |
6463 template <typename TraitsType> | 6501 template <typename TraitsType> |
6464 typename TargetX86Base<TraitsType>::X86OperandMem * | 6502 typename TargetX86Base<TraitsType>::X86OperandMem * |
6465 TargetX86Base<TraitsType>::randomizeOrPoolImmediate(X86OperandMem *MemOperand, | 6503 TargetX86Base<TraitsType>::randomizeOrPoolImmediate(X86OperandMem *MemOperand, |
6466 int32_t RegNum) { | 6504 int32_t RegNum) { |
6467 assert(MemOperand); | 6505 assert(MemOperand); |
6468 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_None || | 6506 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_None || |
6469 RandomizationPoolingPaused == true) { | 6507 RandomizationPoolingPaused == true) { |
6470 // immediates randomization/pooling is turned off | 6508 // immediates randomization/pooling is turned off |
6471 return MemOperand; | 6509 return MemOperand; |
6472 } | 6510 } |
6473 | 6511 |
| 6512 if (Traits::Is64Bit && NeedSandboxing) { |
| 6513 // Immediate randomization/pooling is currently disabled for x86-64 |
| 6514 // sandboxing for it could generate invalid memory operands. |
| 6515 assert(false && |
| 6516 "Constant pooling/randomization is disabled for x8664 sandbox."); |
| 6517 return MemOperand; |
| 6518 } |
| 6519 |
6474 // If this memory operand is already a randomized one, we do not randomize it | 6520 // If this memory operand is already a randomized one, we do not randomize it |
6475 // again. | 6521 // again. |
6476 if (MemOperand->getRandomized()) | 6522 if (MemOperand->getRandomized()) |
6477 return MemOperand; | 6523 return MemOperand; |
6478 | 6524 |
6479 if (auto *C = llvm::dyn_cast_or_null<Constant>(MemOperand->getOffset())) { | 6525 auto *C = llvm::dyn_cast_or_null<Constant>(MemOperand->getOffset()); |
6480 if (C->shouldBeRandomizedOrPooled(Ctx)) { | |
6481 // The offset of this mem operand should be blinded or pooled | |
6482 Ctx->statsUpdateRPImms(); | |
6483 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == | |
6484 RPI_Randomize) { | |
6485 // blind the constant offset | |
6486 // FROM: | |
6487 // offset[base, index, shift] | |
6488 // TO: | |
6489 // insert: lea offset+cookie[base], RegTemp | |
6490 // => -cookie[RegTemp, index, shift] | |
6491 uint32_t Value = | |
6492 llvm::dyn_cast<ConstantInteger32>(MemOperand->getOffset()) | |
6493 ->getValue(); | |
6494 uint32_t Cookie = Func->getConstantBlindingCookie(); | |
6495 Constant *Mask1 = Ctx->getConstantInt( | |
6496 MemOperand->getOffset()->getType(), Cookie + Value); | |
6497 Constant *Mask2 = | |
6498 Ctx->getConstantInt(MemOperand->getOffset()->getType(), 0 - Cookie); | |
6499 | 6526 |
6500 X86OperandMem *TempMemOperand = X86OperandMem::create( | 6527 if (C == nullptr) { |
6501 Func, MemOperand->getType(), MemOperand->getBase(), Mask1); | 6528 return MemOperand; |
6502 // If we have already assigned a physical register, we must come from | 6529 } |
6503 // advancedPhiLowering()=>lowerAssign(). In this case we should reuse | |
6504 // the assigned register as this assignment is that start of its | |
6505 // use-def chain. So we add RegNum argument here. | |
6506 Variable *RegTemp = makeReg(MemOperand->getOffset()->getType(), RegNum); | |
6507 _lea(RegTemp, TempMemOperand); | |
6508 | 6530 |
6509 X86OperandMem *NewMemOperand = X86OperandMem::create( | 6531 if (!C->shouldBeRandomizedOrPooled(Ctx)) { |
6510 Func, MemOperand->getType(), RegTemp, Mask2, MemOperand->getIndex(), | 6532 return MemOperand; |
6511 MemOperand->getShift(), MemOperand->getSegmentRegister()); | 6533 } |
6512 | 6534 |
6513 // Label this memory operand as randomized, so we won't randomize it | 6535 // The offset of this mem operand should be blinded or pooled |
6514 // again in case we call legalize() multiple times on this memory | 6536 Ctx->statsUpdateRPImms(); |
6515 // operand. | 6537 switch (Ctx->getFlags().getRandomizeAndPoolImmediatesOption()) { |
6516 NewMemOperand->setRandomized(true); | 6538 default: |
6517 return NewMemOperand; | 6539 llvm::report_fatal_error("Unsupported -randomize-pool-immediates option"); |
6518 } | 6540 case RPI_Randomize: { |
6519 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_Pool) { | 6541 // blind the constant offset |
6520 // pool the constant offset | 6542 // FROM: |
6521 // FROM: | 6543 // offset[base, index, shift] |
6522 // offset[base, index, shift] | 6544 // TO: |
6523 // TO: | 6545 // insert: lea offset+cookie[base], RegTemp |
6524 // insert: mov $label, RegTemp | 6546 // => -cookie[RegTemp, index, shift] |
6525 // insert: lea [base, RegTemp], RegTemp | 6547 uint32_t Value = |
6526 // =>[RegTemp, index, shift] | 6548 llvm::dyn_cast<ConstantInteger32>(MemOperand->getOffset())->getValue(); |
6527 assert(Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == | 6549 uint32_t Cookie = Func->getConstantBlindingCookie(); |
6528 RPI_Pool); | 6550 Constant *Mask1 = |
6529 // Memory operand should never exist as source operands in phi lowering | 6551 Ctx->getConstantInt(MemOperand->getOffset()->getType(), Cookie + Value); |
6530 // assignments, so there is no need to reuse any registers here. For | 6552 Constant *Mask2 = |
6531 // phi lowering, we should not ask for new physical registers in | 6553 Ctx->getConstantInt(MemOperand->getOffset()->getType(), 0 - Cookie); |
6532 // general. However, if we do meet Memory Operand during phi lowering, | 6554 |
6533 // we should not blind or pool the immediates for now. | 6555 X86OperandMem *TempMemOperand = X86OperandMem::create( |
6534 if (RegNum != Variable::NoRegister) | 6556 Func, MemOperand->getType(), MemOperand->getBase(), Mask1); |
6535 return MemOperand; | 6557 // If we have already assigned a physical register, we must come from |
6536 Variable *RegTemp = makeReg(IceType_i32); | 6558 // advancedPhiLowering()=>lowerAssign(). In this case we should reuse |
6537 IceString Label; | 6559 // the assigned register as this assignment is that start of its |
6538 llvm::raw_string_ostream Label_stream(Label); | 6560 // use-def chain. So we add RegNum argument here. |
6539 MemOperand->getOffset()->emitPoolLabel(Label_stream, Ctx); | 6561 Variable *RegTemp = makeReg(MemOperand->getOffset()->getType(), RegNum); |
6540 MemOperand->getOffset()->setShouldBePooled(true); | 6562 _lea(RegTemp, TempMemOperand); |
6541 constexpr RelocOffsetT SymOffset = 0; | 6563 |
6542 constexpr bool SuppressMangling = true; | 6564 X86OperandMem *NewMemOperand = X86OperandMem::create( |
6543 Constant *Symbol = Ctx->getConstantSym(SymOffset, Label_stream.str(), | 6565 Func, MemOperand->getType(), RegTemp, Mask2, MemOperand->getIndex(), |
6544 SuppressMangling); | 6566 MemOperand->getShift(), MemOperand->getSegmentRegister(), |
6545 const bool UseNonsfi = Ctx->getFlags().getUseNonsfi(); | 6567 MemOperand->getIsRebased()); |
6546 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; | 6568 |
6547 X86OperandMem *SymbolOperand = X86OperandMem::create( | 6569 // Label this memory operand as randomized, so we won't randomize it |
6548 Func, MemOperand->getOffset()->getType(), Base, Symbol); | 6570 // again in case we call legalize() multiple times on this memory |
6549 if (UseNonsfi) | 6571 // operand. |
6550 SymbolOperand->setIsPIC(); | 6572 NewMemOperand->setRandomized(true); |
6551 _mov(RegTemp, SymbolOperand); | 6573 return NewMemOperand; |
6552 // If we have a base variable here, we should add the lea instruction | 6574 } |
6553 // to add the value of the base variable to RegTemp. If there is no | 6575 case RPI_Pool: { |
6554 // base variable, we won't need this lea instruction. | 6576 // pool the constant offset |
6555 if (MemOperand->getBase()) { | 6577 // FROM: |
6556 X86OperandMem *CalculateOperand = X86OperandMem::create( | 6578 // offset[base, index, shift] |
6557 Func, MemOperand->getType(), MemOperand->getBase(), nullptr, | 6579 // TO: |
6558 RegTemp, 0, MemOperand->getSegmentRegister()); | 6580 // insert: mov $label, RegTemp |
6559 _lea(RegTemp, CalculateOperand); | 6581 // insert: lea [base, RegTemp], RegTemp |
6560 } | 6582 // =>[RegTemp, index, shift] |
6561 X86OperandMem *NewMemOperand = X86OperandMem::create( | 6583 |
6562 Func, MemOperand->getType(), RegTemp, nullptr, | 6584 // Memory operand should never exist as source operands in phi lowering |
6563 MemOperand->getIndex(), MemOperand->getShift(), | 6585 // assignments, so there is no need to reuse any registers here. For |
6564 MemOperand->getSegmentRegister()); | 6586 // phi lowering, we should not ask for new physical registers in |
6565 return NewMemOperand; | 6587 // general. However, if we do meet Memory Operand during phi lowering, |
6566 } | 6588 // we should not blind or pool the immediates for now. |
6567 assert("Unsupported -randomize-pool-immediates option" && false); | 6589 if (RegNum != Variable::NoRegister) |
| 6590 return MemOperand; |
| 6591 Variable *RegTemp = makeReg(IceType_i32); |
| 6592 IceString Label; |
| 6593 llvm::raw_string_ostream Label_stream(Label); |
| 6594 MemOperand->getOffset()->emitPoolLabel(Label_stream, Ctx); |
| 6595 MemOperand->getOffset()->setShouldBePooled(true); |
| 6596 constexpr RelocOffsetT SymOffset = 0; |
| 6597 constexpr bool SuppressMangling = true; |
| 6598 Constant *Symbol = |
| 6599 Ctx->getConstantSym(SymOffset, Label_stream.str(), SuppressMangling); |
| 6600 const bool UseNonsfi = Ctx->getFlags().getUseNonsfi(); |
| 6601 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; |
| 6602 const bool IsRebased = Base != nullptr; |
| 6603 X86OperandMem *SymbolOperand = X86OperandMem::create( |
| 6604 Func, MemOperand->getOffset()->getType(), Base, Symbol, IsRebased); |
| 6605 _mov(RegTemp, SymbolOperand); |
| 6606 // If we have a base variable here, we should add the lea instruction |
| 6607 // to add the value of the base variable to RegTemp. If there is no |
| 6608 // base variable, we won't need this lea instruction. |
| 6609 if (MemOperand->getBase()) { |
| 6610 X86OperandMem *CalculateOperand = X86OperandMem::create( |
| 6611 Func, MemOperand->getType(), MemOperand->getBase(), nullptr, RegTemp, |
| 6612 0, MemOperand->getSegmentRegister()); |
| 6613 _lea(RegTemp, CalculateOperand); |
6568 } | 6614 } |
| 6615 X86OperandMem *NewMemOperand = X86OperandMem::create( |
| 6616 Func, MemOperand->getType(), RegTemp, nullptr, MemOperand->getIndex(), |
| 6617 MemOperand->getShift(), MemOperand->getSegmentRegister()); |
| 6618 return NewMemOperand; |
6569 } | 6619 } |
6570 // the offset is not eligible for blinding or pooling, return the original | 6620 } |
6571 // mem operand | |
6572 return MemOperand; | |
6573 } | 6621 } |
6574 | |
6575 } // end of namespace X86NAMESPACE | 6622 } // end of namespace X86NAMESPACE |
6576 } // end of namespace Ice | 6623 } // end of namespace Ice |
6577 | 6624 |
6578 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H | 6625 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H |
OLD | NEW |