OLD | NEW |
1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// | 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// |
2 // | 2 // |
3 // The Subzero Code Generator | 3 // The Subzero Code Generator |
4 // | 4 // |
5 // This file is distributed under the University of Illinois Open Source | 5 // This file is distributed under the University of Illinois Open Source |
6 // License. See LICENSE.TXT for details. | 6 // License. See LICENSE.TXT for details. |
7 // | 7 // |
8 //===----------------------------------------------------------------------===// | 8 //===----------------------------------------------------------------------===// |
9 /// | 9 /// |
10 /// \file | 10 /// \file |
(...skipping 305 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
316 "Traits::InstructionSet range different from TargetInstructionSet"); | 316 "Traits::InstructionSet range different from TargetInstructionSet"); |
317 if (Func->getContext()->getFlags().getTargetInstructionSet() != | 317 if (Func->getContext()->getFlags().getTargetInstructionSet() != |
318 TargetInstructionSet::BaseInstructionSet) { | 318 TargetInstructionSet::BaseInstructionSet) { |
319 InstructionSet = static_cast<InstructionSetEnum>( | 319 InstructionSet = static_cast<InstructionSetEnum>( |
320 (Func->getContext()->getFlags().getTargetInstructionSet() - | 320 (Func->getContext()->getFlags().getTargetInstructionSet() - |
321 TargetInstructionSet::X86InstructionSet_Begin) + | 321 TargetInstructionSet::X86InstructionSet_Begin) + |
322 Traits::InstructionSet::Begin); | 322 Traits::InstructionSet::Begin); |
323 } | 323 } |
324 } | 324 } |
325 | 325 |
326 template <typename TraitsType> void TargetX86Base<TraitsType>::staticInit() { | 326 template <typename TraitsType> |
| 327 void TargetX86Base<TraitsType>::staticInit(const ClFlags &Flags) { |
327 Traits::initRegisterSet(&TypeToRegisterSet, &RegisterAliases, &ScratchRegs); | 328 Traits::initRegisterSet(&TypeToRegisterSet, &RegisterAliases, &ScratchRegs); |
| 329 PcRelFixup = Traits::FK_PcRel; |
| 330 AbsFixup = Flags.getUseNonsfi() ? Traits::FK_Gotoff : Traits::FK_Abs; |
328 } | 331 } |
329 | 332 |
330 template <typename TraitsType> void TargetX86Base<TraitsType>::translateO2() { | 333 template <typename TraitsType> void TargetX86Base<TraitsType>::translateO2() { |
331 TimerMarker T(TimerStack::TT_O2, Func); | 334 TimerMarker T(TimerStack::TT_O2, Func); |
332 | 335 |
| 336 if (!Traits::Is64Bit && Func->getContext()->getFlags().getUseNonsfi()) { |
| 337 GotVar = Func->makeVariable(IceType_i32); |
| 338 } |
333 genTargetHelperCalls(); | 339 genTargetHelperCalls(); |
334 Func->dump("After target helper call insertion"); | 340 Func->dump("After target helper call insertion"); |
335 | 341 |
336 // Merge Alloca instructions, and lay out the stack. | 342 // Merge Alloca instructions, and lay out the stack. |
337 static constexpr bool SortAndCombineAllocas = true; | 343 static constexpr bool SortAndCombineAllocas = true; |
338 Func->processAllocas(SortAndCombineAllocas); | 344 Func->processAllocas(SortAndCombineAllocas); |
339 Func->dump("After Alloca processing"); | 345 Func->dump("After Alloca processing"); |
340 | 346 |
341 if (!Ctx->getFlags().getPhiEdgeSplit()) { | 347 if (!Ctx->getFlags().getPhiEdgeSplit()) { |
342 // Lower Phi instructions. | 348 // Lower Phi instructions. |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
391 Func->dump("After x86 address mode opt"); | 397 Func->dump("After x86 address mode opt"); |
392 | 398 |
393 // Disable constant blinding or pooling for load optimization. | 399 // Disable constant blinding or pooling for load optimization. |
394 { | 400 { |
395 BoolFlagSaver B(RandomizationPoolingPaused, true); | 401 BoolFlagSaver B(RandomizationPoolingPaused, true); |
396 doLoadOpt(); | 402 doLoadOpt(); |
397 } | 403 } |
398 Func->genCode(); | 404 Func->genCode(); |
399 if (Func->hasError()) | 405 if (Func->hasError()) |
400 return; | 406 return; |
| 407 initGotVarIfNeeded(); |
401 Func->dump("After x86 codegen"); | 408 Func->dump("After x86 codegen"); |
402 | 409 |
403 // Register allocation. This requires instruction renumbering and full | 410 // Register allocation. This requires instruction renumbering and full |
404 // liveness analysis. Loops must be identified before liveness so variable | 411 // liveness analysis. Loops must be identified before liveness so variable |
405 // use weights are correct. | 412 // use weights are correct. |
406 Func->renumberInstructions(); | 413 Func->renumberInstructions(); |
407 if (Func->hasError()) | 414 if (Func->hasError()) |
408 return; | 415 return; |
409 Func->liveness(Liveness_Intervals); | 416 Func->liveness(Liveness_Intervals); |
410 if (Func->hasError()) | 417 if (Func->hasError()) |
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
449 Func->doNopInsertion(); | 456 Func->doNopInsertion(); |
450 | 457 |
451 // Mark nodes that require sandbox alignment | 458 // Mark nodes that require sandbox alignment |
452 if (Ctx->getFlags().getUseSandboxing()) | 459 if (Ctx->getFlags().getUseSandboxing()) |
453 Func->markNodesForSandboxing(); | 460 Func->markNodesForSandboxing(); |
454 } | 461 } |
455 | 462 |
456 template <typename TraitsType> void TargetX86Base<TraitsType>::translateOm1() { | 463 template <typename TraitsType> void TargetX86Base<TraitsType>::translateOm1() { |
457 TimerMarker T(TimerStack::TT_Om1, Func); | 464 TimerMarker T(TimerStack::TT_Om1, Func); |
458 | 465 |
| 466 if (!Traits::Is64Bit && Func->getContext()->getFlags().getUseNonsfi()) { |
| 467 GotVar = Func->makeVariable(IceType_i32); |
| 468 } |
459 genTargetHelperCalls(); | 469 genTargetHelperCalls(); |
460 | 470 |
461 // Do not merge Alloca instructions, and lay out the stack. | 471 // Do not merge Alloca instructions, and lay out the stack. |
462 static constexpr bool SortAndCombineAllocas = false; | 472 static constexpr bool SortAndCombineAllocas = false; |
463 Func->processAllocas(SortAndCombineAllocas); | 473 Func->processAllocas(SortAndCombineAllocas); |
464 Func->dump("After Alloca processing"); | 474 Func->dump("After Alloca processing"); |
465 | 475 |
466 Func->placePhiLoads(); | 476 Func->placePhiLoads(); |
467 if (Func->hasError()) | 477 if (Func->hasError()) |
468 return; | 478 return; |
469 Func->placePhiStores(); | 479 Func->placePhiStores(); |
470 if (Func->hasError()) | 480 if (Func->hasError()) |
471 return; | 481 return; |
472 Func->deletePhis(); | 482 Func->deletePhis(); |
473 if (Func->hasError()) | 483 if (Func->hasError()) |
474 return; | 484 return; |
475 Func->dump("After Phi lowering"); | 485 Func->dump("After Phi lowering"); |
476 | 486 |
477 Func->doArgLowering(); | 487 Func->doArgLowering(); |
478 Func->genCode(); | 488 Func->genCode(); |
479 if (Func->hasError()) | 489 if (Func->hasError()) |
480 return; | 490 return; |
| 491 initGotVarIfNeeded(); |
481 Func->dump("After initial x8632 codegen"); | 492 Func->dump("After initial x8632 codegen"); |
482 | 493 |
483 regAlloc(RAK_InfOnly); | 494 regAlloc(RAK_InfOnly); |
484 if (Func->hasError()) | 495 if (Func->hasError()) |
485 return; | 496 return; |
486 Func->dump("After regalloc of infinite-weight variables"); | 497 Func->dump("After regalloc of infinite-weight variables"); |
487 | 498 |
488 Func->genFrame(); | 499 Func->genFrame(); |
489 if (Func->hasError()) | 500 if (Func->hasError()) |
490 return; | 501 return; |
(...skipping 305 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
796 template <typename TraitsType> | 807 template <typename TraitsType> |
797 void TargetX86Base<TraitsType>::emitVariable(const Variable *Var) const { | 808 void TargetX86Base<TraitsType>::emitVariable(const Variable *Var) const { |
798 if (!BuildDefs::dump()) | 809 if (!BuildDefs::dump()) |
799 return; | 810 return; |
800 Ostream &Str = Ctx->getStrEmit(); | 811 Ostream &Str = Ctx->getStrEmit(); |
801 if (Var->hasReg()) { | 812 if (Var->hasReg()) { |
802 Str << "%" << getRegName(Var->getRegNum(), Var->getType()); | 813 Str << "%" << getRegName(Var->getRegNum(), Var->getType()); |
803 return; | 814 return; |
804 } | 815 } |
805 if (Var->mustHaveReg()) { | 816 if (Var->mustHaveReg()) { |
806 llvm_unreachable("Infinite-weight Variable has no register assigned"); | 817 llvm::report_fatal_error( |
| 818 "Infinite-weight Variable has no register assigned"); |
807 } | 819 } |
808 const int32_t Offset = Var->getStackOffset(); | 820 const int32_t Offset = Var->getStackOffset(); |
809 int32_t BaseRegNum = Var->getBaseRegNum(); | 821 int32_t BaseRegNum = Var->getBaseRegNum(); |
810 if (BaseRegNum == Variable::NoRegister) | 822 if (BaseRegNum == Variable::NoRegister) |
811 BaseRegNum = getFrameOrStackReg(); | 823 BaseRegNum = getFrameOrStackReg(); |
812 // Print in the form "Offset(%reg)", taking care that: | 824 // Print in the form "Offset(%reg)", taking care that: |
813 // - Offset is never printed when it is 0 | 825 // - Offset is never printed when it is 0 |
814 | 826 |
815 const bool DecorateAsm = Func->getContext()->getFlags().getDecorateAsm(); | 827 const bool DecorateAsm = Func->getContext()->getFlags().getDecorateAsm(); |
816 // Only print Offset when it is nonzero, regardless of DecorateAsm. | 828 // Only print Offset when it is nonzero, regardless of DecorateAsm. |
817 if (Offset) { | 829 if (Offset) { |
818 if (DecorateAsm) { | 830 if (DecorateAsm) { |
819 Str << Var->getSymbolicStackOffset(Func); | 831 Str << Var->getSymbolicStackOffset(Func); |
820 } else { | 832 } else { |
821 Str << Offset; | 833 Str << Offset; |
822 } | 834 } |
823 } | 835 } |
824 const Type FrameSPTy = Traits::WordType; | 836 const Type FrameSPTy = Traits::WordType; |
825 Str << "(%" << getRegName(BaseRegNum, FrameSPTy) << ")"; | 837 Str << "(%" << getRegName(BaseRegNum, FrameSPTy) << ")"; |
826 } | 838 } |
827 | 839 |
828 template <typename TraitsType> | 840 template <typename TraitsType> |
829 typename TargetX86Base<TraitsType>::X86Address | 841 typename TargetX86Base<TraitsType>::X86Address |
830 TargetX86Base<TraitsType>::stackVarToAsmOperand(const Variable *Var) const { | 842 TargetX86Base<TraitsType>::stackVarToAsmOperand(const Variable *Var) const { |
831 if (Var->hasReg()) | 843 if (Var->hasReg()) |
832 llvm_unreachable("Stack Variable has a register assigned"); | 844 llvm::report_fatal_error("Stack Variable has a register assigned"); |
833 if (Var->mustHaveReg()) { | 845 if (Var->mustHaveReg()) { |
834 llvm_unreachable("Infinite-weight Variable has no register assigned"); | 846 llvm::report_fatal_error( |
| 847 "Infinite-weight Variable has no register assigned"); |
835 } | 848 } |
836 int32_t Offset = Var->getStackOffset(); | 849 int32_t Offset = Var->getStackOffset(); |
837 int32_t BaseRegNum = Var->getBaseRegNum(); | 850 int32_t BaseRegNum = Var->getBaseRegNum(); |
838 if (Var->getBaseRegNum() == Variable::NoRegister) | 851 if (Var->getBaseRegNum() == Variable::NoRegister) |
839 BaseRegNum = getFrameOrStackReg(); | 852 BaseRegNum = getFrameOrStackReg(); |
840 return X86Address(Traits::getEncodedGPR(BaseRegNum), Offset, | 853 return X86Address(Traits::getEncodedGPR(BaseRegNum), Offset, |
841 AssemblerFixup::NoFixup); | 854 AssemblerFixup::NoFixup); |
842 } | 855 } |
843 | 856 |
844 /// Helper function for addProlog(). | 857 /// Helper function for addProlog(). |
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
903 return Var64On32->getLo(); | 916 return Var64On32->getLo(); |
904 if (auto *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { | 917 if (auto *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { |
905 auto *ConstInt = llvm::dyn_cast<ConstantInteger32>( | 918 auto *ConstInt = llvm::dyn_cast<ConstantInteger32>( |
906 Ctx->getConstantInt32(static_cast<int32_t>(Const->getValue()))); | 919 Ctx->getConstantInt32(static_cast<int32_t>(Const->getValue()))); |
907 // Check if we need to blind/pool the constant. | 920 // Check if we need to blind/pool the constant. |
908 return legalize(ConstInt); | 921 return legalize(ConstInt); |
909 } | 922 } |
910 if (auto *Mem = llvm::dyn_cast<X86OperandMem>(Operand)) { | 923 if (auto *Mem = llvm::dyn_cast<X86OperandMem>(Operand)) { |
911 auto *MemOperand = X86OperandMem::create( | 924 auto *MemOperand = X86OperandMem::create( |
912 Func, IceType_i32, Mem->getBase(), Mem->getOffset(), Mem->getIndex(), | 925 Func, IceType_i32, Mem->getBase(), Mem->getOffset(), Mem->getIndex(), |
913 Mem->getShift(), Mem->getSegmentRegister()); | 926 Mem->getShift(), Mem->getSegmentRegister(), Mem->getIsPIC()); |
914 // Test if we should randomize or pool the offset, if so randomize it or | 927 // Test if we should randomize or pool the offset, if so randomize it or |
915 // pool it then create mem operand with the blinded/pooled constant. | 928 // pool it then create mem operand with the blinded/pooled constant. |
916 // Otherwise, return the mem operand as ordinary mem operand. | 929 // Otherwise, return the mem operand as ordinary mem operand. |
917 return legalize(MemOperand); | 930 return legalize(MemOperand); |
918 } | 931 } |
919 llvm_unreachable("Unsupported operand type"); | 932 llvm_unreachable("Unsupported operand type"); |
920 return nullptr; | 933 return nullptr; |
921 } | 934 } |
922 | 935 |
923 template <typename TraitsType> | 936 template <typename TraitsType> |
(...skipping 19 matching lines...) Expand all Loading... |
943 } else if (auto *IntOffset = llvm::dyn_cast<ConstantInteger32>(Offset)) { | 956 } else if (auto *IntOffset = llvm::dyn_cast<ConstantInteger32>(Offset)) { |
944 Offset = Ctx->getConstantInt32(4 + IntOffset->getValue()); | 957 Offset = Ctx->getConstantInt32(4 + IntOffset->getValue()); |
945 } else if (auto *SymOffset = llvm::dyn_cast<ConstantRelocatable>(Offset)) { | 958 } else if (auto *SymOffset = llvm::dyn_cast<ConstantRelocatable>(Offset)) { |
946 assert(!Utils::WouldOverflowAdd(SymOffset->getOffset(), 4)); | 959 assert(!Utils::WouldOverflowAdd(SymOffset->getOffset(), 4)); |
947 Offset = | 960 Offset = |
948 Ctx->getConstantSym(4 + SymOffset->getOffset(), SymOffset->getName(), | 961 Ctx->getConstantSym(4 + SymOffset->getOffset(), SymOffset->getName(), |
949 SymOffset->getSuppressMangling()); | 962 SymOffset->getSuppressMangling()); |
950 } | 963 } |
951 auto *MemOperand = X86OperandMem::create( | 964 auto *MemOperand = X86OperandMem::create( |
952 Func, IceType_i32, Mem->getBase(), Offset, Mem->getIndex(), | 965 Func, IceType_i32, Mem->getBase(), Offset, Mem->getIndex(), |
953 Mem->getShift(), Mem->getSegmentRegister()); | 966 Mem->getShift(), Mem->getSegmentRegister(), Mem->getIsPIC()); |
954 // Test if the Offset is an eligible i32 constants for randomization and | 967 // Test if the Offset is an eligible i32 constants for randomization and |
955 // pooling. Blind/pool it if it is. Otherwise return as oridinary mem | 968 // pooling. Blind/pool it if it is. Otherwise return as oridinary mem |
956 // operand. | 969 // operand. |
957 return legalize(MemOperand); | 970 return legalize(MemOperand); |
958 } | 971 } |
959 llvm_unreachable("Unsupported operand type"); | 972 llvm_unreachable("Unsupported operand type"); |
960 return nullptr; | 973 return nullptr; |
961 } | 974 } |
962 | 975 |
963 template <typename TraitsType> | 976 template <typename TraitsType> |
964 llvm::SmallBitVector | 977 llvm::SmallBitVector |
965 TargetX86Base<TraitsType>::getRegisterSet(RegSetMask Include, | 978 TargetX86Base<TraitsType>::getRegisterSet(RegSetMask Include, |
966 RegSetMask Exclude) const { | 979 RegSetMask Exclude) const { |
967 return Traits::getRegisterSet(Include, Exclude); | 980 return Traits::getRegisterSet(Include, Exclude); |
968 } | 981 } |
969 | 982 |
970 template <typename TraitsType> | 983 template <typename TraitsType> |
| 984 void TargetX86Base<TraitsType>::initGotVarIfNeeded() { |
| 985 if (!Func->getContext()->getFlags().getUseNonsfi()) |
| 986 return; |
| 987 if (Traits::Is64Bit) { |
| 988 // Probably no implementation is needed, but error to be safe for now. |
| 989 llvm::report_fatal_error( |
| 990 "Need to implement initGotVarIfNeeded() for 64-bit."); |
| 991 } |
| 992 // Insert the GotVar assignment as the very first lowered instruction. Later, |
| 993 // it will be moved into the right place - after the stack frame is set up but |
| 994 // before in-args are copied into registers. |
| 995 Context.init(Func->getEntryNode()); |
| 996 Context.setInsertPoint(Context.getCur()); |
| 997 Context.insert<typename Traits::Insts::GetIP>(GotVar); |
| 998 } |
| 999 |
| 1000 template <typename TraitsType> |
971 void TargetX86Base<TraitsType>::lowerAlloca(const InstAlloca *Inst) { | 1001 void TargetX86Base<TraitsType>::lowerAlloca(const InstAlloca *Inst) { |
972 // Conservatively require the stack to be aligned. Some stack adjustment | 1002 // Conservatively require the stack to be aligned. Some stack adjustment |
973 // operations implemented below assume that the stack is aligned before the | 1003 // operations implemented below assume that the stack is aligned before the |
974 // alloca. All the alloca code ensures that the stack alignment is preserved | 1004 // alloca. All the alloca code ensures that the stack alignment is preserved |
975 // after the alloca. The stack alignment restriction can be relaxed in some | 1005 // after the alloca. The stack alignment restriction can be relaxed in some |
976 // cases. | 1006 // cases. |
977 NeedsStackAlignment = true; | 1007 NeedsStackAlignment = true; |
978 | 1008 |
979 // For default align=0, set it to the real value 1, to avoid any | 1009 // For default align=0, set it to the real value 1, to avoid any |
980 // bit-manipulation problems below. | 1010 // bit-manipulation problems below. |
(...skipping 2996 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3977 } | 4007 } |
3978 _test(SecondVar, SecondVar); | 4008 _test(SecondVar, SecondVar); |
3979 _cmov(T_Dest2, T_Dest, Traits::Cond::Br_e); | 4009 _cmov(T_Dest2, T_Dest, Traits::Cond::Br_e); |
3980 _mov(DestLo, T_Dest2); | 4010 _mov(DestLo, T_Dest2); |
3981 _mov(DestHi, Ctx->getConstantZero(IceType_i32)); | 4011 _mov(DestHi, Ctx->getConstantZero(IceType_i32)); |
3982 } | 4012 } |
3983 | 4013 |
3984 template <typename TraitsType> | 4014 template <typename TraitsType> |
3985 void TargetX86Base<TraitsType>::typedLoad(Type Ty, Variable *Dest, | 4015 void TargetX86Base<TraitsType>::typedLoad(Type Ty, Variable *Dest, |
3986 Variable *Base, Constant *Offset) { | 4016 Variable *Base, Constant *Offset) { |
| 4017 // If Offset is a ConstantRelocatable in Non-SFI mode, we will need to |
| 4018 // legalize Mem properly. |
| 4019 if (Offset) |
| 4020 assert(!llvm::isa<ConstantRelocatable>(Offset)); |
| 4021 |
3987 auto *Mem = X86OperandMem::create(Func, Ty, Base, Offset); | 4022 auto *Mem = X86OperandMem::create(Func, Ty, Base, Offset); |
3988 | 4023 |
3989 if (isVectorType(Ty)) | 4024 if (isVectorType(Ty)) |
3990 _movp(Dest, Mem); | 4025 _movp(Dest, Mem); |
3991 else if (Ty == IceType_f64) | 4026 else if (Ty == IceType_f64) |
3992 _movq(Dest, Mem); | 4027 _movq(Dest, Mem); |
3993 else | 4028 else |
3994 _mov(Dest, Mem); | 4029 _mov(Dest, Mem); |
3995 } | 4030 } |
3996 | 4031 |
3997 template <typename TraitsType> | 4032 template <typename TraitsType> |
3998 void TargetX86Base<TraitsType>::typedStore(Type Ty, Variable *Value, | 4033 void TargetX86Base<TraitsType>::typedStore(Type Ty, Variable *Value, |
3999 Variable *Base, Constant *Offset) { | 4034 Variable *Base, Constant *Offset) { |
| 4035 // If Offset is a ConstantRelocatable in Non-SFI mode, we will need to |
| 4036 // legalize Mem properly. |
| 4037 if (Offset) |
| 4038 assert(!llvm::isa<ConstantRelocatable>(Offset)); |
| 4039 |
4000 auto *Mem = X86OperandMem::create(Func, Ty, Base, Offset); | 4040 auto *Mem = X86OperandMem::create(Func, Ty, Base, Offset); |
4001 | 4041 |
4002 if (isVectorType(Ty)) | 4042 if (isVectorType(Ty)) |
4003 _storep(Value, Mem); | 4043 _storep(Value, Mem); |
4004 else if (Ty == IceType_f64) | 4044 else if (Ty == IceType_f64) |
4005 _storeq(Value, Mem); | 4045 _storeq(Value, Mem); |
4006 else | 4046 else |
4007 _store(Value, Mem); | 4047 _store(Value, Mem); |
4008 } | 4048 } |
4009 | 4049 |
(...skipping 289 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
4299 Str << "<null>"; | 4339 Str << "<null>"; |
4300 Str << ", Index="; | 4340 Str << ", Index="; |
4301 if (Index) | 4341 if (Index) |
4302 Index->dump(Func); | 4342 Index->dump(Func); |
4303 else | 4343 else |
4304 Str << "<null>"; | 4344 Str << "<null>"; |
4305 Str << ", Shift=" << Shift << ", Offset=" << Offset | 4345 Str << ", Shift=" << Shift << ", Offset=" << Offset |
4306 << ", Relocatable=" << Relocatable << "\n"; | 4346 << ", Relocatable=" << Relocatable << "\n"; |
4307 } | 4347 } |
4308 | 4348 |
4309 inline bool matchAssign(const VariablesMetadata *VMetadata, Variable *&Var, | 4349 inline bool matchAssign(const VariablesMetadata *VMetadata, Variable *GotVar, |
4310 ConstantRelocatable *&Relocatable, int32_t &Offset, | 4350 Variable *&Var, ConstantRelocatable *&Relocatable, |
4311 const Inst *&Reason) { | 4351 int32_t &Offset, const Inst *&Reason) { |
4312 // Var originates from Var=SrcVar ==> set Var:=SrcVar | 4352 // Var originates from Var=SrcVar ==> set Var:=SrcVar |
4313 if (Var == nullptr) | 4353 if (Var == nullptr) |
4314 return false; | 4354 return false; |
4315 if (const Inst *VarAssign = VMetadata->getSingleDefinition(Var)) { | 4355 if (const Inst *VarAssign = VMetadata->getSingleDefinition(Var)) { |
4316 assert(!VMetadata->isMultiDef(Var)); | 4356 assert(!VMetadata->isMultiDef(Var)); |
4317 if (llvm::isa<InstAssign>(VarAssign)) { | 4357 if (llvm::isa<InstAssign>(VarAssign)) { |
4318 Operand *SrcOp = VarAssign->getSrc(0); | 4358 Operand *SrcOp = VarAssign->getSrc(0); |
4319 assert(SrcOp); | 4359 assert(SrcOp); |
4320 if (auto *SrcVar = llvm::dyn_cast<Variable>(SrcOp)) { | 4360 if (auto *SrcVar = llvm::dyn_cast<Variable>(SrcOp)) { |
4321 if (!VMetadata->isMultiDef(SrcVar) && | 4361 if (!VMetadata->isMultiDef(SrcVar) && |
4322 // TODO: ensure SrcVar stays single-BB | 4362 // TODO: ensure SrcVar stays single-BB |
4323 true) { | 4363 true) { |
4324 Var = SrcVar; | 4364 Var = SrcVar; |
4325 Reason = VarAssign; | 4365 Reason = VarAssign; |
4326 return true; | 4366 return true; |
4327 } | 4367 } |
4328 } else if (auto *Const = llvm::dyn_cast<ConstantInteger32>(SrcOp)) { | 4368 } else if (auto *Const = llvm::dyn_cast<ConstantInteger32>(SrcOp)) { |
4329 int32_t MoreOffset = Const->getValue(); | 4369 int32_t MoreOffset = Const->getValue(); |
4330 if (Utils::WouldOverflowAdd(Offset, MoreOffset)) | 4370 if (Utils::WouldOverflowAdd(Offset, MoreOffset)) |
4331 return false; | 4371 return false; |
4332 Var = nullptr; | 4372 Var = nullptr; |
4333 Offset += MoreOffset; | 4373 Offset += MoreOffset; |
4334 Reason = VarAssign; | 4374 Reason = VarAssign; |
4335 return true; | 4375 return true; |
4336 } else if (auto *AddReloc = llvm::dyn_cast<ConstantRelocatable>(SrcOp)) { | 4376 } else if (auto *AddReloc = llvm::dyn_cast<ConstantRelocatable>(SrcOp)) { |
4337 if (Relocatable == nullptr) { | 4377 if (Relocatable == nullptr) { |
4338 Var = nullptr; | 4378 Var = GotVar; |
4339 Relocatable = AddReloc; | 4379 Relocatable = AddReloc; |
4340 Reason = VarAssign; | 4380 Reason = VarAssign; |
4341 return true; | 4381 return true; |
4342 } | 4382 } |
4343 } | 4383 } |
4344 } | 4384 } |
4345 } | 4385 } |
4346 return false; | 4386 return false; |
4347 } | 4387 } |
4348 | 4388 |
(...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
4447 return true; | 4487 return true; |
4448 } | 4488 } |
4449 } | 4489 } |
4450 } | 4490 } |
4451 } | 4491 } |
4452 } | 4492 } |
4453 } | 4493 } |
4454 return false; | 4494 return false; |
4455 } | 4495 } |
4456 | 4496 |
4457 inline bool matchOffsetBase(const VariablesMetadata *VMetadata, Variable *&Base, | 4497 inline bool matchOffsetBase(const VariablesMetadata *VMetadata, |
| 4498 Variable *GotVar, Variable *&Base, |
| 4499 Variable *&BaseOther, |
4458 ConstantRelocatable *&Relocatable, int32_t &Offset, | 4500 ConstantRelocatable *&Relocatable, int32_t &Offset, |
4459 const Inst *&Reason) { | 4501 const Inst *&Reason) { |
4460 // Base is Base=Var+Const || Base is Base=Const+Var ==> | 4502 // Base is Base=Var+Const || Base is Base=Const+Var ==> |
4461 // set Base=Var, Offset+=Const | 4503 // set Base=Var, Offset+=Const |
4462 // Base is Base=Var-Const ==> | 4504 // Base is Base=Var-Const ==> |
4463 // set Base=Var, Offset-=Const | 4505 // set Base=Var, Offset-=Const |
4464 if (Base == nullptr) { | 4506 if (Base == nullptr) { |
4465 return false; | 4507 return false; |
4466 } | 4508 } |
4467 const Inst *BaseInst = VMetadata->getSingleDefinition(Base); | 4509 const Inst *BaseInst = VMetadata->getSingleDefinition(Base); |
(...skipping 30 matching lines...) Expand all Loading... |
4498 if ((Relocatable && (Reloc0 || Reloc1)) || (Reloc0 && Reloc1)) | 4540 if ((Relocatable && (Reloc0 || Reloc1)) || (Reloc0 && Reloc1)) |
4499 return false; | 4541 return false; |
4500 // Don't know how to subtract a relocatable. | 4542 // Don't know how to subtract a relocatable. |
4501 if (!IsAdd && Reloc1) | 4543 if (!IsAdd && Reloc1) |
4502 return false; | 4544 return false; |
4503 // Incorporate ConstantRelocatables. | 4545 // Incorporate ConstantRelocatables. |
4504 if (Reloc0) | 4546 if (Reloc0) |
4505 NewRelocatable = Reloc0; | 4547 NewRelocatable = Reloc0; |
4506 else if (Reloc1) | 4548 else if (Reloc1) |
4507 NewRelocatable = Reloc1; | 4549 NewRelocatable = Reloc1; |
| 4550 if ((Reloc0 || Reloc1) && BaseOther && GotVar) |
| 4551 return false; |
4508 // Compute the updated constant offset. | 4552 // Compute the updated constant offset. |
4509 if (Const0) { | 4553 if (Const0) { |
4510 int32_t MoreOffset = IsAdd ? Const0->getValue() : -Const0->getValue(); | 4554 int32_t MoreOffset = IsAdd ? Const0->getValue() : -Const0->getValue(); |
4511 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) | 4555 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) |
4512 return false; | 4556 return false; |
4513 NewOffset += MoreOffset; | 4557 NewOffset += MoreOffset; |
4514 } | 4558 } |
4515 if (Const1) { | 4559 if (Const1) { |
4516 int32_t MoreOffset = IsAdd ? Const1->getValue() : -Const1->getValue(); | 4560 int32_t MoreOffset = IsAdd ? Const1->getValue() : -Const1->getValue(); |
4517 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) | 4561 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) |
4518 return false; | 4562 return false; |
4519 NewOffset += MoreOffset; | 4563 NewOffset += MoreOffset; |
4520 } | 4564 } |
4521 // Update the computed address parameters once we are sure optimization | 4565 // Update the computed address parameters once we are sure optimization |
4522 // is valid. | 4566 // is valid. |
| 4567 if ((Reloc0 || Reloc1) && GotVar) { |
| 4568 assert(BaseOther == nullptr); |
| 4569 BaseOther = GotVar; |
| 4570 } |
4523 Base = NewBase; | 4571 Base = NewBase; |
4524 Offset = NewOffset; | 4572 Offset = NewOffset; |
4525 Relocatable = NewRelocatable; | 4573 Relocatable = NewRelocatable; |
4526 Reason = BaseInst; | 4574 Reason = BaseInst; |
4527 return true; | 4575 return true; |
4528 } | 4576 } |
4529 return false; | 4577 return false; |
4530 } | 4578 } |
4531 | 4579 |
4532 // Builds information for a canonical address expresion: | 4580 // Builds information for a canonical address expresion: |
4533 // <Relocatable + Offset>(Base, Index, Shift) | 4581 // <Relocatable + Offset>(Base, Index, Shift) |
4534 // On entry: | 4582 // On entry: |
4535 // Relocatable == null, | 4583 // Relocatable == null, |
4536 // Offset == 0, | 4584 // Offset == 0, |
4537 // Base is a Variable, | 4585 // Base is a Variable, |
4538 // Index == nullptr, | 4586 // Index == nullptr, |
4539 // Shift == 0 | 4587 // Shift == 0 |
4540 inline bool computeAddressOpt(Cfg *Func, const Inst *Instr, | 4588 inline bool computeAddressOpt(Cfg *Func, const Inst *Instr, Variable *GotVar, |
4541 ConstantRelocatable *&Relocatable, | 4589 ConstantRelocatable *&Relocatable, |
4542 int32_t &Offset, Variable *&Base, | 4590 int32_t &Offset, Variable *&Base, |
4543 Variable *&Index, uint16_t &Shift) { | 4591 Variable *&Index, uint16_t &Shift) { |
4544 bool AddressWasOptimized = false; | 4592 bool AddressWasOptimized = false; |
4545 Func->resetCurrentNode(); | 4593 Func->resetCurrentNode(); |
4546 if (Func->isVerbose(IceV_AddrOpt)) { | 4594 if (Func->isVerbose(IceV_AddrOpt)) { |
4547 OstreamLocker L(Func->getContext()); | 4595 OstreamLocker L(Func->getContext()); |
4548 Ostream &Str = Func->getContext()->getStrDump(); | 4596 Ostream &Str = Func->getContext()->getStrDump(); |
4549 Str << "\nStarting computeAddressOpt for instruction:\n "; | 4597 Str << "\nStarting computeAddressOpt for instruction:\n "; |
4550 Instr->dumpDecorated(Func); | 4598 Instr->dumpDecorated(Func); |
(...skipping 10 matching lines...) Expand all Loading... |
4561 const bool MockBounds = Func->getContext()->getFlags().getMockBoundsCheck(); | 4609 const bool MockBounds = Func->getContext()->getFlags().getMockBoundsCheck(); |
4562 const VariablesMetadata *VMetadata = Func->getVMetadata(); | 4610 const VariablesMetadata *VMetadata = Func->getVMetadata(); |
4563 const Inst *Reason = nullptr; | 4611 const Inst *Reason = nullptr; |
4564 do { | 4612 do { |
4565 if (Reason) { | 4613 if (Reason) { |
4566 dumpAddressOpt(Func, Relocatable, Offset, Base, Index, Shift, Reason); | 4614 dumpAddressOpt(Func, Relocatable, Offset, Base, Index, Shift, Reason); |
4567 AddressWasOptimized = true; | 4615 AddressWasOptimized = true; |
4568 Reason = nullptr; | 4616 Reason = nullptr; |
4569 } | 4617 } |
4570 // Update Base and Index to follow through assignments to definitions. | 4618 // Update Base and Index to follow through assignments to definitions. |
4571 if (matchAssign(VMetadata, Base, Relocatable, Offset, Reason)) { | 4619 if (matchAssign(VMetadata, GotVar, Base, Relocatable, Offset, Reason)) { |
4572 // Assignments of Base from a Relocatable or ConstantInt32 can result | 4620 // Assignments of Base from a Relocatable or ConstantInt32 can result |
4573 // in Base becoming nullptr. To avoid code duplication in this loop we | 4621 // in Base becoming nullptr. To avoid code duplication in this loop we |
4574 // prefer that Base be non-nullptr if possible. | 4622 // prefer that Base be non-nullptr if possible. |
4575 if ((Base == nullptr) && (Index != nullptr) && Shift == 0) | 4623 if ((Base == nullptr) && (Index != nullptr) && Shift == 0) |
4576 std::swap(Base, Index); | 4624 std::swap(Base, Index); |
4577 continue; | 4625 continue; |
4578 } | 4626 } |
4579 if (matchAssign(VMetadata, Index, Relocatable, Offset, Reason)) | 4627 if (matchAssign(VMetadata, GotVar, Index, Relocatable, Offset, Reason)) |
4580 continue; | 4628 continue; |
4581 | 4629 |
4582 if (!MockBounds) { | 4630 if (!MockBounds) { |
4583 // Transition from: | 4631 // Transition from: |
4584 // <Relocatable + Offset>(Base) to | 4632 // <Relocatable + Offset>(Base) to |
4585 // <Relocatable + Offset>(Base, Index) | 4633 // <Relocatable + Offset>(Base, Index) |
4586 if (matchCombinedBaseIndex(VMetadata, Base, Index, Shift, Reason)) | 4634 if (matchCombinedBaseIndex(VMetadata, Base, Index, Shift, Reason)) |
4587 continue; | 4635 continue; |
4588 // Recognize multiply/shift and update Shift amount. | 4636 // Recognize multiply/shift and update Shift amount. |
4589 // Index becomes Index=Var<<Const && Const+Shift<=3 ==> | 4637 // Index becomes Index=Var<<Const && Const+Shift<=3 ==> |
4590 // Index=Var, Shift+=Const | 4638 // Index=Var, Shift+=Const |
4591 // Index becomes Index=Const*Var && log2(Const)+Shift<=3 ==> | 4639 // Index becomes Index=Const*Var && log2(Const)+Shift<=3 ==> |
4592 // Index=Var, Shift+=log2(Const) | 4640 // Index=Var, Shift+=log2(Const) |
4593 if (matchShiftedIndex(VMetadata, Index, Shift, Reason)) | 4641 if (matchShiftedIndex(VMetadata, Index, Shift, Reason)) |
4594 continue; | 4642 continue; |
4595 // If Shift is zero, the choice of Base and Index was purely arbitrary. | 4643 // If Shift is zero, the choice of Base and Index was purely arbitrary. |
4596 // Recognize multiply/shift and set Shift amount. | 4644 // Recognize multiply/shift and set Shift amount. |
4597 // Shift==0 && Base is Base=Var*Const && log2(Const)+Shift<=3 ==> | 4645 // Shift==0 && Base is Base=Var*Const && log2(Const)+Shift<=3 ==> |
4598 // swap(Index,Base) | 4646 // swap(Index,Base) |
4599 // Similar for Base=Const*Var and Base=Var<<Const | 4647 // Similar for Base=Const*Var and Base=Var<<Const |
4600 if (Shift == 0 && matchShiftedIndex(VMetadata, Base, Shift, Reason)) { | 4648 if (Shift == 0 && matchShiftedIndex(VMetadata, Base, Shift, Reason)) { |
4601 std::swap(Base, Index); | 4649 std::swap(Base, Index); |
4602 continue; | 4650 continue; |
4603 } | 4651 } |
4604 } | 4652 } |
4605 // Update Offset to reflect additions/subtractions with constants and | 4653 // Update Offset to reflect additions/subtractions with constants and |
4606 // relocatables. | 4654 // relocatables. |
4607 // TODO: consider overflow issues with respect to Offset. | 4655 // TODO: consider overflow issues with respect to Offset. |
4608 if (matchOffsetBase(VMetadata, Base, Relocatable, Offset, Reason)) | 4656 if (matchOffsetBase(VMetadata, GotVar, Base, Index, Relocatable, Offset, |
| 4657 Reason)) |
4609 continue; | 4658 continue; |
4610 if (Shift == 0 && | 4659 if (Shift == 0 && matchOffsetBase(VMetadata, GotVar, Index, Base, |
4611 matchOffsetBase(VMetadata, Index, Relocatable, Offset, Reason)) | 4660 Relocatable, Offset, Reason)) |
4612 continue; | 4661 continue; |
4613 // TODO(sehr, stichnot): Handle updates of Index with Shift != 0. | 4662 // TODO(sehr, stichnot): Handle updates of Index with Shift != 0. |
4614 // Index is Index=Var+Const ==> | 4663 // Index is Index=Var+Const ==> |
4615 // set Index=Var, Offset+=(Const<<Shift) | 4664 // set Index=Var, Offset+=(Const<<Shift) |
4616 // Index is Index=Const+Var ==> | 4665 // Index is Index=Const+Var ==> |
4617 // set Index=Var, Offset+=(Const<<Shift) | 4666 // set Index=Var, Offset+=(Const<<Shift) |
4618 // Index is Index=Var-Const ==> | 4667 // Index is Index=Var-Const ==> |
4619 // set Index=Var, Offset-=(Const<<Shift) | 4668 // set Index=Var, Offset-=(Const<<Shift) |
4620 break; | 4669 break; |
4621 } while (Reason); | 4670 } while (Reason); |
| 4671 // Undo any addition of GotVar. It will be added back when the mem operand is |
| 4672 // legalized. |
| 4673 if (Base == GotVar) |
| 4674 Base = nullptr; |
| 4675 if (Index == GotVar) |
| 4676 Index = nullptr; |
4622 return AddressWasOptimized; | 4677 return AddressWasOptimized; |
4623 } | 4678 } |
4624 | 4679 |
4625 /// Add a mock bounds check on the memory address before using it as a load or | 4680 /// Add a mock bounds check on the memory address before using it as a load or |
4626 /// store operand. The basic idea is that given a memory operand [reg], we | 4681 /// store operand. The basic idea is that given a memory operand [reg], we |
4627 /// would first add bounds-check code something like: | 4682 /// would first add bounds-check code something like: |
4628 /// | 4683 /// |
4629 /// cmp reg, <lb> | 4684 /// cmp reg, <lb> |
4630 /// jl out_of_line_error | 4685 /// jl out_of_line_error |
4631 /// cmp reg, <ub> | 4686 /// cmp reg, <ub> |
(...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
4676 _cmp(Opnd, Ctx->getConstantZero(IceType_i32)); | 4731 _cmp(Opnd, Ctx->getConstantZero(IceType_i32)); |
4677 _br(Traits::Cond::Br_e, Label); | 4732 _br(Traits::Cond::Br_e, Label); |
4678 _cmp(Opnd, Ctx->getConstantInt32(1)); | 4733 _cmp(Opnd, Ctx->getConstantInt32(1)); |
4679 _br(Traits::Cond::Br_e, Label); | 4734 _br(Traits::Cond::Br_e, Label); |
4680 Context.insert(Label); | 4735 Context.insert(Label); |
4681 } | 4736 } |
4682 | 4737 |
4683 template <typename TraitsType> | 4738 template <typename TraitsType> |
4684 void TargetX86Base<TraitsType>::lowerLoad(const InstLoad *Load) { | 4739 void TargetX86Base<TraitsType>::lowerLoad(const InstLoad *Load) { |
4685 // A Load instruction can be treated the same as an Assign instruction, after | 4740 // A Load instruction can be treated the same as an Assign instruction, after |
4686 // the source operand is transformed into an X86OperandMem operand. | 4741 // the source operand is transformed into an X86OperandMem operand. Note that |
4687 // Note that the address mode optimization already creates an | 4742 // the address mode optimization already creates an X86OperandMem operand, so |
4688 // X86OperandMem operand, so it doesn't need another level of | 4743 // it doesn't need another level of transformation. |
4689 // transformation. | |
4690 Variable *DestLoad = Load->getDest(); | 4744 Variable *DestLoad = Load->getDest(); |
4691 Type Ty = DestLoad->getType(); | 4745 Type Ty = DestLoad->getType(); |
4692 Operand *Src0 = formMemoryOperand(Load->getSourceAddress(), Ty); | 4746 Operand *Src0 = formMemoryOperand(Load->getSourceAddress(), Ty); |
4693 doMockBoundsCheck(Src0); | 4747 doMockBoundsCheck(Src0); |
4694 auto *Assign = InstAssign::create(Func, DestLoad, Src0); | 4748 auto *Assign = InstAssign::create(Func, DestLoad, Src0); |
4695 lowerAssign(Assign); | 4749 lowerAssign(Assign); |
4696 } | 4750 } |
4697 | 4751 |
4698 template <typename TraitsType> | 4752 template <typename TraitsType> |
4699 void TargetX86Base<TraitsType>::doAddressOptLoad() { | 4753 void TargetX86Base<TraitsType>::doAddressOptLoad() { |
4700 Inst *Inst = Context.getCur(); | 4754 Inst *Inst = Context.getCur(); |
4701 Variable *Dest = Inst->getDest(); | 4755 Variable *Dest = Inst->getDest(); |
4702 Operand *Addr = Inst->getSrc(0); | 4756 Operand *Addr = Inst->getSrc(0); |
4703 Variable *Index = nullptr; | 4757 Variable *Index = nullptr; |
4704 ConstantRelocatable *Relocatable = nullptr; | 4758 ConstantRelocatable *Relocatable = nullptr; |
4705 uint16_t Shift = 0; | 4759 uint16_t Shift = 0; |
4706 int32_t Offset = 0; | 4760 int32_t Offset = 0; |
4707 // Vanilla ICE load instructions should not use the segment registers, and | 4761 // Vanilla ICE load instructions should not use the segment registers, and |
4708 // computeAddressOpt only works at the level of Variables and Constants, not | 4762 // computeAddressOpt only works at the level of Variables and Constants, not |
4709 // other X86OperandMem, so there should be no mention of segment | 4763 // other X86OperandMem, so there should be no mention of segment |
4710 // registers there either. | 4764 // registers there either. |
4711 const SegmentRegisters SegmentReg = X86OperandMem::DefaultSegment; | 4765 constexpr auto SegmentReg = X86OperandMem::SegmentRegisters::DefaultSegment; |
4712 auto *Base = llvm::dyn_cast<Variable>(Addr); | 4766 auto *Base = llvm::dyn_cast<Variable>(Addr); |
4713 if (computeAddressOpt(Func, Inst, Relocatable, Offset, Base, Index, Shift)) { | 4767 if (computeAddressOpt(Func, Inst, GotVar, Relocatable, Offset, Base, Index, |
| 4768 Shift)) { |
4714 Inst->setDeleted(); | 4769 Inst->setDeleted(); |
4715 Constant *OffsetOp = nullptr; | 4770 Constant *OffsetOp = nullptr; |
4716 if (Relocatable == nullptr) { | 4771 if (Relocatable == nullptr) { |
4717 OffsetOp = Ctx->getConstantInt32(Offset); | 4772 OffsetOp = Ctx->getConstantInt32(Offset); |
4718 } else { | 4773 } else { |
4719 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, | 4774 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, |
4720 Relocatable->getName(), | 4775 Relocatable->getName(), |
4721 Relocatable->getSuppressMangling()); | 4776 Relocatable->getSuppressMangling()); |
4722 } | 4777 } |
| 4778 // The new mem operand is created without IsPIC being set, because |
| 4779 // computeAddressOpt() doesn't include GotVar in its final result. |
4723 Addr = X86OperandMem::create(Func, Dest->getType(), Base, OffsetOp, Index, | 4780 Addr = X86OperandMem::create(Func, Dest->getType(), Base, OffsetOp, Index, |
4724 Shift, SegmentReg); | 4781 Shift, SegmentReg); |
4725 Context.insert<InstLoad>(Dest, Addr); | 4782 Context.insert<InstLoad>(Dest, Addr); |
4726 } | 4783 } |
4727 } | 4784 } |
4728 | 4785 |
4729 template <typename TraitsType> | 4786 template <typename TraitsType> |
4730 void TargetX86Base<TraitsType>::randomlyInsertNop(float Probability, | 4787 void TargetX86Base<TraitsType>::randomlyInsertNop(float Probability, |
4731 RandomNumberGenerator &RNG) { | 4788 RandomNumberGenerator &RNG) { |
4732 RandomNumberGeneratorWrapper RNGW(RNG); | 4789 RandomNumberGeneratorWrapper RNGW(RNG); |
(...skipping 271 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5004 Operand *Addr = Inst->getAddr(); | 5061 Operand *Addr = Inst->getAddr(); |
5005 Variable *Index = nullptr; | 5062 Variable *Index = nullptr; |
5006 ConstantRelocatable *Relocatable = nullptr; | 5063 ConstantRelocatable *Relocatable = nullptr; |
5007 uint16_t Shift = 0; | 5064 uint16_t Shift = 0; |
5008 int32_t Offset = 0; | 5065 int32_t Offset = 0; |
5009 auto *Base = llvm::dyn_cast<Variable>(Addr); | 5066 auto *Base = llvm::dyn_cast<Variable>(Addr); |
5010 // Vanilla ICE store instructions should not use the segment registers, and | 5067 // Vanilla ICE store instructions should not use the segment registers, and |
5011 // computeAddressOpt only works at the level of Variables and Constants, not | 5068 // computeAddressOpt only works at the level of Variables and Constants, not |
5012 // other X86OperandMem, so there should be no mention of segment | 5069 // other X86OperandMem, so there should be no mention of segment |
5013 // registers there either. | 5070 // registers there either. |
5014 const SegmentRegisters SegmentReg = X86OperandMem::DefaultSegment; | 5071 constexpr auto SegmentReg = X86OperandMem::SegmentRegisters::DefaultSegment; |
5015 if (computeAddressOpt(Func, Inst, Relocatable, Offset, Base, Index, Shift)) { | 5072 if (computeAddressOpt(Func, Inst, GotVar, Relocatable, Offset, Base, Index, |
| 5073 Shift)) { |
5016 Inst->setDeleted(); | 5074 Inst->setDeleted(); |
5017 Constant *OffsetOp = nullptr; | 5075 Constant *OffsetOp = nullptr; |
5018 if (Relocatable == nullptr) { | 5076 if (Relocatable == nullptr) { |
5019 OffsetOp = Ctx->getConstantInt32(Offset); | 5077 OffsetOp = Ctx->getConstantInt32(Offset); |
5020 } else { | 5078 } else { |
5021 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, | 5079 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, |
5022 Relocatable->getName(), | 5080 Relocatable->getName(), |
5023 Relocatable->getSuppressMangling()); | 5081 Relocatable->getSuppressMangling()); |
5024 } | 5082 } |
| 5083 // The new mem operand is created without IsPIC being set, because |
| 5084 // computeAddressOpt() doesn't include GotVar in its final result. |
5025 Addr = X86OperandMem::create(Func, Data->getType(), Base, OffsetOp, Index, | 5085 Addr = X86OperandMem::create(Func, Data->getType(), Base, OffsetOp, Index, |
5026 Shift, SegmentReg); | 5086 Shift, SegmentReg); |
5027 auto *NewStore = Context.insert<InstStore>(Data, Addr); | 5087 auto *NewStore = Context.insert<InstStore>(Data, Addr); |
5028 if (Inst->getDest()) | 5088 if (Inst->getDest()) |
5029 NewStore->setRmwBeacon(Inst->getRmwBeacon()); | 5089 NewStore->setRmwBeacon(Inst->getRmwBeacon()); |
5030 } | 5090 } |
5031 } | 5091 } |
5032 | 5092 |
5033 template <typename TraitsType> | 5093 template <typename TraitsType> |
5034 Operand *TargetX86Base<TraitsType>::lowerCmpRange(Operand *Comparison, | 5094 Operand *TargetX86Base<TraitsType>::lowerCmpRange(Operand *Comparison, |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5076 Variable *Index; | 5136 Variable *Index; |
5077 if (RangeIndex->getType() != getPointerType()) { | 5137 if (RangeIndex->getType() != getPointerType()) { |
5078 Index = makeReg(getPointerType()); | 5138 Index = makeReg(getPointerType()); |
5079 _movzx(Index, RangeIndex); | 5139 _movzx(Index, RangeIndex); |
5080 } else { | 5140 } else { |
5081 Index = legalizeToReg(RangeIndex); | 5141 Index = legalizeToReg(RangeIndex); |
5082 } | 5142 } |
5083 | 5143 |
5084 constexpr RelocOffsetT RelocOffset = 0; | 5144 constexpr RelocOffsetT RelocOffset = 0; |
5085 constexpr bool SuppressMangling = true; | 5145 constexpr bool SuppressMangling = true; |
| 5146 const bool IsPIC = Ctx->getFlags().getUseNonsfi(); |
5086 IceString MangledName = Ctx->mangleName(Func->getFunctionName()); | 5147 IceString MangledName = Ctx->mangleName(Func->getFunctionName()); |
5087 Constant *Base = Ctx->getConstantSym( | 5148 Variable *Base = IsPIC ? legalizeToReg(GotVar) : nullptr; |
| 5149 Constant *Offset = Ctx->getConstantSym( |
5088 RelocOffset, InstJumpTable::makeName(MangledName, JumpTable->getId()), | 5150 RelocOffset, InstJumpTable::makeName(MangledName, JumpTable->getId()), |
5089 SuppressMangling); | 5151 SuppressMangling); |
5090 Constant *Offset = nullptr; | |
5091 uint16_t Shift = typeWidthInBytesLog2(getPointerType()); | 5152 uint16_t Shift = typeWidthInBytesLog2(getPointerType()); |
5092 // TODO(ascull): remove need for legalize by allowing null base in memop | 5153 constexpr auto Segment = X86OperandMem::SegmentRegisters::DefaultSegment; |
5093 auto *TargetInMemory = X86OperandMem::create( | 5154 auto *TargetInMemory = X86OperandMem::create( |
5094 Func, getPointerType(), legalizeToReg(Base), Offset, Index, Shift); | 5155 Func, getPointerType(), Base, Offset, Index, Shift, Segment, IsPIC); |
5095 Variable *Target = nullptr; | 5156 Variable *Target = nullptr; |
5096 _mov(Target, TargetInMemory); | 5157 _mov(Target, TargetInMemory); |
5097 lowerIndirectJump(Target); | 5158 lowerIndirectJump(Target); |
5098 | 5159 |
5099 if (DefaultTarget == nullptr) | 5160 if (DefaultTarget == nullptr) |
5100 Context.insert(SkipJumpTable); | 5161 Context.insert(SkipJumpTable); |
5101 return; | 5162 return; |
5102 } | 5163 } |
5103 case CaseCluster::Range: { | 5164 case CaseCluster::Range: { |
5104 if (Case.isUnitRange()) { | 5165 if (Case.isUnitRange()) { |
(...skipping 305 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5410 void TargetX86Base<TraitsType>::lowerOther(const Inst *Instr) { | 5471 void TargetX86Base<TraitsType>::lowerOther(const Inst *Instr) { |
5411 if (const auto *RMW = llvm::dyn_cast<InstX86FakeRMW>(Instr)) { | 5472 if (const auto *RMW = llvm::dyn_cast<InstX86FakeRMW>(Instr)) { |
5412 lowerRMW(RMW); | 5473 lowerRMW(RMW); |
5413 } else { | 5474 } else { |
5414 TargetLowering::lowerOther(Instr); | 5475 TargetLowering::lowerOther(Instr); |
5415 } | 5476 } |
5416 } | 5477 } |
5417 | 5478 |
5418 /// Turn an i64 Phi instruction into a pair of i32 Phi instructions, to preserve | 5479 /// Turn an i64 Phi instruction into a pair of i32 Phi instructions, to preserve |
5419 /// integrity of liveness analysis. Undef values are also turned into zeroes, | 5480 /// integrity of liveness analysis. Undef values are also turned into zeroes, |
5420 /// since loOperand() and hiOperand() don't expect Undef input. | 5481 /// since loOperand() and hiOperand() don't expect Undef input. Also, in |
| 5482 /// Non-SFI mode, add a FakeUse(GotVar) for every pooled constant operand. |
5421 template <typename TraitsType> void TargetX86Base<TraitsType>::prelowerPhis() { | 5483 template <typename TraitsType> void TargetX86Base<TraitsType>::prelowerPhis() { |
| 5484 if (Ctx->getFlags().getUseNonsfi()) { |
| 5485 assert(GotVar); |
| 5486 CfgNode *Node = Context.getNode(); |
| 5487 uint32_t GotVarUseCount = 0; |
| 5488 for (Inst &I : Node->getPhis()) { |
| 5489 auto *Phi = llvm::dyn_cast<InstPhi>(&I); |
| 5490 if (Phi->isDeleted()) |
| 5491 continue; |
| 5492 for (SizeT I = 0; I < Phi->getSrcSize(); ++I) { |
| 5493 Operand *Src = Phi->getSrc(I); |
| 5494 // TODO(stichnot): This over-counts for +0.0, and under-counts for other |
| 5495 // kinds of pooling. |
| 5496 if (llvm::isa<ConstantRelocatable>(Src) || |
| 5497 llvm::isa<ConstantFloat>(Src) || llvm::isa<ConstantDouble>(Src)) { |
| 5498 ++GotVarUseCount; |
| 5499 } |
| 5500 } |
| 5501 } |
| 5502 if (GotVarUseCount) { |
| 5503 Node->getInsts().push_front(InstFakeUse::create(Func, GotVar)); |
| 5504 } |
| 5505 } |
5422 if (Traits::Is64Bit) { | 5506 if (Traits::Is64Bit) { |
5423 // On x86-64 we don't need to prelower phis -- the architecture can handle | 5507 // On x86-64 we don't need to prelower phis -- the architecture can handle |
5424 // 64-bit integer natively. | 5508 // 64-bit integer natively. |
5425 return; | 5509 return; |
5426 } | 5510 } |
5427 | 5511 |
5428 // Pause constant blinding or pooling, blinding or pooling will be done later | 5512 // Pause constant blinding or pooling, blinding or pooling will be done later |
5429 // during phi lowering assignments | 5513 // during phi lowering assignments |
5430 BoolFlagSaver B(RandomizationPoolingPaused, true); | 5514 BoolFlagSaver B(RandomizationPoolingPaused, true); |
5431 PhiLowering::prelowerPhis32Bit<TargetX86Base<TraitsType>>( | 5515 PhiLowering::prelowerPhis32Bit<TargetX86Base<TraitsType>>( |
(...skipping 462 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5894 _movp(Reg, Src); | 5978 _movp(Reg, Src); |
5895 } else { | 5979 } else { |
5896 _mov(Reg, Src); | 5980 _mov(Reg, Src); |
5897 } | 5981 } |
5898 return Reg; | 5982 return Reg; |
5899 } | 5983 } |
5900 | 5984 |
5901 template <typename TraitsType> | 5985 template <typename TraitsType> |
5902 Operand *TargetX86Base<TraitsType>::legalize(Operand *From, LegalMask Allowed, | 5986 Operand *TargetX86Base<TraitsType>::legalize(Operand *From, LegalMask Allowed, |
5903 int32_t RegNum) { | 5987 int32_t RegNum) { |
5904 Type Ty = From->getType(); | 5988 const bool UseNonsfi = Func->getContext()->getFlags().getUseNonsfi(); |
| 5989 const Type Ty = From->getType(); |
5905 // Assert that a physical register is allowed. To date, all calls to | 5990 // Assert that a physical register is allowed. To date, all calls to |
5906 // legalize() allow a physical register. If a physical register needs to be | 5991 // legalize() allow a physical register. If a physical register needs to be |
5907 // explicitly disallowed, then new code will need to be written to force a | 5992 // explicitly disallowed, then new code will need to be written to force a |
5908 // spill. | 5993 // spill. |
5909 assert(Allowed & Legal_Reg); | 5994 assert(Allowed & Legal_Reg); |
5910 // If we're asking for a specific physical register, make sure we're not | 5995 // If we're asking for a specific physical register, make sure we're not |
5911 // allowing any other operand kinds. (This could be future work, e.g. allow | 5996 // allowing any other operand kinds. (This could be future work, e.g. allow |
5912 // the shl shift amount to be either an immediate or in ecx.) | 5997 // the shl shift amount to be either an immediate or in ecx.) |
5913 assert(RegNum == Variable::NoRegister || Allowed == Legal_Reg); | 5998 assert(RegNum == Variable::NoRegister || Allowed == Legal_Reg); |
5914 | 5999 |
(...skipping 13 matching lines...) Expand all Loading... |
5928 } | 6013 } |
5929 } | 6014 } |
5930 } | 6015 } |
5931 } | 6016 } |
5932 | 6017 |
5933 if (auto *Mem = llvm::dyn_cast<X86OperandMem>(From)) { | 6018 if (auto *Mem = llvm::dyn_cast<X86OperandMem>(From)) { |
5934 // Before doing anything with a Mem operand, we need to ensure that the | 6019 // Before doing anything with a Mem operand, we need to ensure that the |
5935 // Base and Index components are in physical registers. | 6020 // Base and Index components are in physical registers. |
5936 Variable *Base = Mem->getBase(); | 6021 Variable *Base = Mem->getBase(); |
5937 Variable *Index = Mem->getIndex(); | 6022 Variable *Index = Mem->getIndex(); |
| 6023 Constant *Offset = Mem->getOffset(); |
5938 Variable *RegBase = nullptr; | 6024 Variable *RegBase = nullptr; |
5939 Variable *RegIndex = nullptr; | 6025 Variable *RegIndex = nullptr; |
5940 if (Base) { | 6026 if (Base) { |
5941 RegBase = llvm::cast<Variable>( | 6027 RegBase = llvm::cast<Variable>( |
5942 legalize(Base, Legal_Reg | Legal_Rematerializable)); | 6028 legalize(Base, Legal_Reg | Legal_Rematerializable)); |
5943 } | 6029 } |
5944 if (Index) { | 6030 if (Index) { |
5945 RegIndex = llvm::cast<Variable>( | 6031 RegIndex = llvm::cast<Variable>( |
5946 legalize(Index, Legal_Reg | Legal_Rematerializable)); | 6032 legalize(Index, Legal_Reg | Legal_Rematerializable)); |
5947 } | 6033 } |
| 6034 // For Non-SFI mode, if the Offset field is a ConstantRelocatable, we |
| 6035 // replace either Base or Index with a legalized GotVar. At emission time, |
| 6036 // the ConstantRelocatable will be emitted with the @GOTOFF relocation. |
| 6037 bool NeedPIC = false; |
| 6038 if (UseNonsfi && !Mem->getIsPIC() && Offset && |
| 6039 llvm::isa<ConstantRelocatable>(Offset)) { |
| 6040 assert(!(Allowed & Legal_AddrAbs)); |
| 6041 NeedPIC = true; |
| 6042 if (RegBase == nullptr) { |
| 6043 RegBase = legalizeToReg(GotVar); |
| 6044 } else if (RegIndex == nullptr) { |
| 6045 RegIndex = legalizeToReg(GotVar); |
| 6046 } else { |
| 6047 llvm::report_fatal_error( |
| 6048 "Either Base or Index must be unused in Non-SFI mode"); |
| 6049 } |
| 6050 } |
5948 if (Base != RegBase || Index != RegIndex) { | 6051 if (Base != RegBase || Index != RegIndex) { |
5949 Mem = X86OperandMem::create(Func, Ty, RegBase, Mem->getOffset(), RegIndex, | 6052 Mem = X86OperandMem::create(Func, Ty, RegBase, Offset, RegIndex, |
5950 Mem->getShift(), Mem->getSegmentRegister()); | 6053 Mem->getShift(), Mem->getSegmentRegister(), |
| 6054 NeedPIC); |
5951 } | 6055 } |
5952 | 6056 |
5953 // For all Memory Operands, we do randomization/pooling here | 6057 // For all Memory Operands, we do randomization/pooling here |
5954 From = randomizeOrPoolImmediate(Mem); | 6058 From = randomizeOrPoolImmediate(Mem); |
5955 | 6059 |
5956 if (!(Allowed & Legal_Mem)) { | 6060 if (!(Allowed & Legal_Mem)) { |
5957 From = copyToReg(From, RegNum); | 6061 From = copyToReg(From, RegNum); |
5958 } | 6062 } |
5959 return From; | 6063 return From; |
5960 } | 6064 } |
| 6065 |
5961 if (auto *Const = llvm::dyn_cast<Constant>(From)) { | 6066 if (auto *Const = llvm::dyn_cast<Constant>(From)) { |
5962 if (llvm::isa<ConstantUndef>(Const)) { | 6067 if (llvm::isa<ConstantUndef>(Const)) { |
5963 From = legalizeUndef(Const, RegNum); | 6068 From = legalizeUndef(Const, RegNum); |
5964 if (isVectorType(Ty)) | 6069 if (isVectorType(Ty)) |
5965 return From; | 6070 return From; |
5966 Const = llvm::cast<Constant>(From); | 6071 Const = llvm::cast<Constant>(From); |
5967 } | 6072 } |
5968 // There should be no constants of vector type (other than undef). | 6073 // There should be no constants of vector type (other than undef). |
5969 assert(!isVectorType(Ty)); | 6074 assert(!isVectorType(Ty)); |
5970 | 6075 |
(...skipping 10 matching lines...) Expand all Loading... |
5981 | 6086 |
5982 // If the operand is an 32 bit constant integer, we should check whether we | 6087 // If the operand is an 32 bit constant integer, we should check whether we |
5983 // need to randomize it or pool it. | 6088 // need to randomize it or pool it. |
5984 if (auto *C = llvm::dyn_cast<ConstantInteger32>(Const)) { | 6089 if (auto *C = llvm::dyn_cast<ConstantInteger32>(Const)) { |
5985 Operand *NewConst = randomizeOrPoolImmediate(C, RegNum); | 6090 Operand *NewConst = randomizeOrPoolImmediate(C, RegNum); |
5986 if (NewConst != Const) { | 6091 if (NewConst != Const) { |
5987 return NewConst; | 6092 return NewConst; |
5988 } | 6093 } |
5989 } | 6094 } |
5990 | 6095 |
| 6096 // If the operand is a ConstantRelocatable, and Legal_AddrAbs is not |
| 6097 // specified, and UseNonsfi is indicated, we need to add GotVar. |
| 6098 if (auto *CR = llvm::dyn_cast<ConstantRelocatable>(Const)) { |
| 6099 if (UseNonsfi && !(Allowed & Legal_AddrAbs)) { |
| 6100 assert(Ty == IceType_i32); |
| 6101 Variable *RegBase = legalizeToReg(GotVar); |
| 6102 Variable *NewVar = makeReg(Ty, RegNum); |
| 6103 auto *Mem = Traits::X86OperandMem::create(Func, Ty, RegBase, CR); |
| 6104 Mem->setIsPIC(); |
| 6105 _lea(NewVar, Mem); |
| 6106 From = NewVar; |
| 6107 } |
| 6108 } |
| 6109 |
5991 // Convert a scalar floating point constant into an explicit memory | 6110 // Convert a scalar floating point constant into an explicit memory |
5992 // operand. | 6111 // operand. |
5993 if (isScalarFloatingType(Ty)) { | 6112 if (isScalarFloatingType(Ty)) { |
5994 if (auto *ConstFloat = llvm::dyn_cast<ConstantFloat>(Const)) { | 6113 if (auto *ConstFloat = llvm::dyn_cast<ConstantFloat>(Const)) { |
5995 if (Utils::isPositiveZero(ConstFloat->getValue())) | 6114 if (Utils::isPositiveZero(ConstFloat->getValue())) |
5996 return makeZeroedRegister(Ty, RegNum); | 6115 return makeZeroedRegister(Ty, RegNum); |
5997 } else if (auto *ConstDouble = llvm::dyn_cast<ConstantDouble>(Const)) { | 6116 } else if (auto *ConstDouble = llvm::dyn_cast<ConstantDouble>(Const)) { |
5998 if (Utils::isPositiveZero(ConstDouble->getValue())) | 6117 if (Utils::isPositiveZero(ConstDouble->getValue())) |
5999 return makeZeroedRegister(Ty, RegNum); | 6118 return makeZeroedRegister(Ty, RegNum); |
6000 } | 6119 } |
6001 Variable *Base = nullptr; | 6120 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; |
6002 std::string Buffer; | 6121 std::string Buffer; |
6003 llvm::raw_string_ostream StrBuf(Buffer); | 6122 llvm::raw_string_ostream StrBuf(Buffer); |
6004 llvm::cast<Constant>(From)->emitPoolLabel(StrBuf, Ctx); | 6123 llvm::cast<Constant>(From)->emitPoolLabel(StrBuf, Ctx); |
6005 llvm::cast<Constant>(From)->setShouldBePooled(true); | 6124 llvm::cast<Constant>(From)->setShouldBePooled(true); |
6006 Constant *Offset = Ctx->getConstantSym(0, StrBuf.str(), true); | 6125 Constant *Offset = Ctx->getConstantSym(0, StrBuf.str(), true); |
6007 From = X86OperandMem::create(Func, Ty, Base, Offset); | 6126 auto *Mem = X86OperandMem::create(Func, Ty, Base, Offset); |
| 6127 if (UseNonsfi) |
| 6128 Mem->setIsPIC(); |
| 6129 From = Mem; |
6008 } | 6130 } |
6009 bool NeedsReg = false; | 6131 bool NeedsReg = false; |
6010 if (!(Allowed & Legal_Imm) && !isScalarFloatingType(Ty)) | 6132 if (!(Allowed & Legal_Imm) && !isScalarFloatingType(Ty)) |
6011 // Immediate specifically not allowed | 6133 // Immediate specifically not allowed |
6012 NeedsReg = true; | 6134 NeedsReg = true; |
6013 if (!(Allowed & Legal_Mem) && isScalarFloatingType(Ty)) | 6135 if (!(Allowed & Legal_Mem) && isScalarFloatingType(Ty)) |
6014 // On x86, FP constants are lowered to mem operands. | 6136 // On x86, FP constants are lowered to mem operands. |
6015 NeedsReg = true; | 6137 NeedsReg = true; |
6016 if (NeedsReg) { | 6138 if (NeedsReg) { |
6017 From = copyToReg(From, RegNum); | 6139 From = copyToReg(From, RegNum); |
6018 } | 6140 } |
6019 return From; | 6141 return From; |
6020 } | 6142 } |
| 6143 |
6021 if (auto *Var = llvm::dyn_cast<Variable>(From)) { | 6144 if (auto *Var = llvm::dyn_cast<Variable>(From)) { |
6022 // Check if the variable is guaranteed a physical register. This can happen | 6145 // Check if the variable is guaranteed a physical register. This can happen |
6023 // either when the variable is pre-colored or when it is assigned infinite | 6146 // either when the variable is pre-colored or when it is assigned infinite |
6024 // weight. | 6147 // weight. |
6025 bool MustHaveRegister = (Var->hasReg() || Var->mustHaveReg()); | 6148 bool MustHaveRegister = (Var->hasReg() || Var->mustHaveReg()); |
6026 bool MustRematerialize = | 6149 bool MustRematerialize = |
6027 (Var->isRematerializable() && !(Allowed & Legal_Rematerializable)); | 6150 (Var->isRematerializable() && !(Allowed & Legal_Rematerializable)); |
6028 // We need a new physical register for the operand if: | 6151 // We need a new physical register for the operand if: |
6029 // - Mem is not allowed and Var isn't guaranteed a physical register, or | 6152 // - Mem is not allowed and Var isn't guaranteed a physical register, or |
6030 // - RegNum is required and Var->getRegNum() doesn't match, or | 6153 // - RegNum is required and Var->getRegNum() doesn't match, or |
6031 // - Var is a rematerializable variable and rematerializable pass-through is | 6154 // - Var is a rematerializable variable and rematerializable pass-through is |
6032 // not allowed (in which case we need an lea instruction). | 6155 // not allowed (in which case we need an lea instruction). |
6033 if (MustRematerialize) { | 6156 if (MustRematerialize) { |
6034 assert(Ty == IceType_i32); | 6157 assert(Ty == IceType_i32); |
6035 Variable *NewVar = makeReg(Ty, RegNum); | 6158 Variable *NewVar = makeReg(Ty, RegNum); |
6036 // Since Var is rematerializable, the offset will be added when the lea is | 6159 // Since Var is rematerializable, the offset will be added when the lea is |
6037 // emitted. | 6160 // emitted. |
6038 constexpr Constant *NoOffset = nullptr; | 6161 constexpr Constant *NoOffset = nullptr; |
6039 auto *Mem = X86OperandMem::create(Func, Ty, Var, NoOffset); | 6162 auto *Mem = X86OperandMem::create(Func, Ty, Var, NoOffset); |
6040 _lea(NewVar, Mem); | 6163 _lea(NewVar, Mem); |
6041 From = NewVar; | 6164 From = NewVar; |
6042 } else if ((!(Allowed & Legal_Mem) && !MustHaveRegister) || | 6165 } else if ((!(Allowed & Legal_Mem) && !MustHaveRegister) || |
6043 (RegNum != Variable::NoRegister && RegNum != Var->getRegNum()) || | 6166 (RegNum != Variable::NoRegister && RegNum != Var->getRegNum()) || |
6044 MustRematerialize) { | 6167 MustRematerialize) { |
6045 From = copyToReg(From, RegNum); | 6168 From = copyToReg(From, RegNum); |
6046 } | 6169 } |
6047 return From; | 6170 return From; |
6048 } | 6171 } |
6049 llvm_unreachable("Unhandled operand kind in legalize()"); | 6172 |
| 6173 llvm::report_fatal_error("Unhandled operand kind in legalize()"); |
6050 return From; | 6174 return From; |
6051 } | 6175 } |
6052 | 6176 |
6053 /// Provide a trivial wrapper to legalize() for this common usage. | 6177 /// Provide a trivial wrapper to legalize() for this common usage. |
6054 template <typename TraitsType> | 6178 template <typename TraitsType> |
6055 Variable *TargetX86Base<TraitsType>::legalizeToReg(Operand *From, | 6179 Variable *TargetX86Base<TraitsType>::legalizeToReg(Operand *From, |
6056 int32_t RegNum) { | 6180 int32_t RegNum) { |
6057 return llvm::cast<Variable>(legalize(From, Legal_Reg, RegNum)); | 6181 return llvm::cast<Variable>(legalize(From, Legal_Reg, RegNum)); |
6058 } | 6182 } |
6059 | 6183 |
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
6109 // transformation. | 6233 // transformation. |
6110 if (!Mem) { | 6234 if (!Mem) { |
6111 auto *Base = llvm::dyn_cast<Variable>(Opnd); | 6235 auto *Base = llvm::dyn_cast<Variable>(Opnd); |
6112 auto *Offset = llvm::dyn_cast<Constant>(Opnd); | 6236 auto *Offset = llvm::dyn_cast<Constant>(Opnd); |
6113 assert(Base || Offset); | 6237 assert(Base || Offset); |
6114 if (Offset) { | 6238 if (Offset) { |
6115 // During memory operand building, we do not blind or pool the constant | 6239 // During memory operand building, we do not blind or pool the constant |
6116 // offset, we will work on the whole memory operand later as one entity | 6240 // offset, we will work on the whole memory operand later as one entity |
6117 // later, this save one instruction. By turning blinding and pooling off, | 6241 // later, this save one instruction. By turning blinding and pooling off, |
6118 // we guarantee legalize(Offset) will return a Constant*. | 6242 // we guarantee legalize(Offset) will return a Constant*. |
6119 { | 6243 if (!llvm::isa<ConstantRelocatable>(Offset)) { |
6120 BoolFlagSaver B(RandomizationPoolingPaused, true); | 6244 BoolFlagSaver B(RandomizationPoolingPaused, true); |
6121 | 6245 |
6122 Offset = llvm::cast<Constant>(legalize(Offset)); | 6246 Offset = llvm::cast<Constant>(legalize(Offset)); |
6123 } | 6247 } |
6124 | 6248 |
6125 assert(llvm::isa<ConstantInteger32>(Offset) || | 6249 assert(llvm::isa<ConstantInteger32>(Offset) || |
6126 llvm::isa<ConstantRelocatable>(Offset)); | 6250 llvm::isa<ConstantRelocatable>(Offset)); |
6127 } | 6251 } |
| 6252 // Not completely sure whether it's OK to leave IsPIC unset when creating |
| 6253 // the mem operand. If DoLegalize is true, it will definitely be applied |
| 6254 // during the legalize() call, but perhaps not during the |
| 6255 // randomizeOrPoolImmediate() call. In any case, the emit routines will |
| 6256 // assert that PIC legalization has been applied. |
6128 Mem = X86OperandMem::create(Func, Ty, Base, Offset); | 6257 Mem = X86OperandMem::create(Func, Ty, Base, Offset); |
6129 } | 6258 } |
6130 // Do legalization, which contains randomization/pooling or do | 6259 // Do legalization, which contains randomization/pooling or do |
6131 // randomization/pooling. | 6260 // randomization/pooling. |
6132 return llvm::cast<X86OperandMem>(DoLegalize ? legalize(Mem) | 6261 return llvm::cast<X86OperandMem>(DoLegalize ? legalize(Mem) |
6133 : randomizeOrPoolImmediate(Mem)); | 6262 : randomizeOrPoolImmediate(Mem)); |
6134 } | 6263 } |
6135 | 6264 |
6136 template <typename TraitsType> | 6265 template <typename TraitsType> |
6137 Variable *TargetX86Base<TraitsType>::makeReg(Type Type, int32_t RegNum) { | 6266 Variable *TargetX86Base<TraitsType>::makeReg(Type Type, int32_t RegNum) { |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
6185 const llvm::SmallBitVector &ExcludeRegisters, uint64_t Salt) const { | 6314 const llvm::SmallBitVector &ExcludeRegisters, uint64_t Salt) const { |
6186 Traits::makeRandomRegisterPermutation(Ctx, Func, Permutation, | 6315 Traits::makeRandomRegisterPermutation(Ctx, Func, Permutation, |
6187 ExcludeRegisters, Salt); | 6316 ExcludeRegisters, Salt); |
6188 } | 6317 } |
6189 | 6318 |
6190 template <typename TraitsType> | 6319 template <typename TraitsType> |
6191 void TargetX86Base<TraitsType>::emit(const ConstantInteger32 *C) const { | 6320 void TargetX86Base<TraitsType>::emit(const ConstantInteger32 *C) const { |
6192 if (!BuildDefs::dump()) | 6321 if (!BuildDefs::dump()) |
6193 return; | 6322 return; |
6194 Ostream &Str = Ctx->getStrEmit(); | 6323 Ostream &Str = Ctx->getStrEmit(); |
6195 Str << getConstantPrefix() << C->getValue(); | 6324 Str << "$" << C->getValue(); |
6196 } | 6325 } |
6197 | 6326 |
6198 template <typename TraitsType> | 6327 template <typename TraitsType> |
6199 void TargetX86Base<TraitsType>::emit(const ConstantInteger64 *C) const { | 6328 void TargetX86Base<TraitsType>::emit(const ConstantInteger64 *C) const { |
6200 if (!Traits::Is64Bit) { | 6329 if (!Traits::Is64Bit) { |
6201 llvm::report_fatal_error("Not expecting to emit 64-bit integers"); | 6330 llvm::report_fatal_error("Not expecting to emit 64-bit integers"); |
6202 } else { | 6331 } else { |
6203 if (!BuildDefs::dump()) | 6332 if (!BuildDefs::dump()) |
6204 return; | 6333 return; |
6205 Ostream &Str = Ctx->getStrEmit(); | 6334 Ostream &Str = Ctx->getStrEmit(); |
6206 Str << getConstantPrefix() << C->getValue(); | 6335 Str << "$" << C->getValue(); |
6207 } | 6336 } |
6208 } | 6337 } |
6209 | 6338 |
6210 template <typename TraitsType> | 6339 template <typename TraitsType> |
6211 void TargetX86Base<TraitsType>::emit(const ConstantFloat *C) const { | 6340 void TargetX86Base<TraitsType>::emit(const ConstantFloat *C) const { |
6212 if (!BuildDefs::dump()) | 6341 if (!BuildDefs::dump()) |
6213 return; | 6342 return; |
6214 Ostream &Str = Ctx->getStrEmit(); | 6343 Ostream &Str = Ctx->getStrEmit(); |
6215 C->emitPoolLabel(Str, Ctx); | 6344 C->emitPoolLabel(Str, Ctx); |
6216 } | 6345 } |
6217 | 6346 |
6218 template <typename TraitsType> | 6347 template <typename TraitsType> |
6219 void TargetX86Base<TraitsType>::emit(const ConstantDouble *C) const { | 6348 void TargetX86Base<TraitsType>::emit(const ConstantDouble *C) const { |
6220 if (!BuildDefs::dump()) | 6349 if (!BuildDefs::dump()) |
6221 return; | 6350 return; |
6222 Ostream &Str = Ctx->getStrEmit(); | 6351 Ostream &Str = Ctx->getStrEmit(); |
6223 C->emitPoolLabel(Str, Ctx); | 6352 C->emitPoolLabel(Str, Ctx); |
6224 } | 6353 } |
6225 | 6354 |
6226 template <typename TraitsType> | 6355 template <typename TraitsType> |
6227 void TargetX86Base<TraitsType>::emit(const ConstantUndef *) const { | 6356 void TargetX86Base<TraitsType>::emit(const ConstantUndef *) const { |
6228 llvm::report_fatal_error("undef value encountered by emitter."); | 6357 llvm::report_fatal_error("undef value encountered by emitter."); |
6229 } | 6358 } |
6230 | 6359 |
| 6360 template <class Machine> |
| 6361 void TargetX86Base<Machine>::emit(const ConstantRelocatable *C) const { |
| 6362 if (!BuildDefs::dump()) |
| 6363 return; |
| 6364 assert(!Ctx->getFlags().getUseNonsfi()); |
| 6365 Ostream &Str = Ctx->getStrEmit(); |
| 6366 Str << "$"; |
| 6367 emitWithoutPrefix(C); |
| 6368 } |
| 6369 |
6231 /// Randomize or pool an Immediate. | 6370 /// Randomize or pool an Immediate. |
6232 template <typename TraitsType> | 6371 template <typename TraitsType> |
6233 Operand * | 6372 Operand * |
6234 TargetX86Base<TraitsType>::randomizeOrPoolImmediate(Constant *Immediate, | 6373 TargetX86Base<TraitsType>::randomizeOrPoolImmediate(Constant *Immediate, |
6235 int32_t RegNum) { | 6374 int32_t RegNum) { |
6236 assert(llvm::isa<ConstantInteger32>(Immediate) || | 6375 assert(llvm::isa<ConstantInteger32>(Immediate) || |
6237 llvm::isa<ConstantRelocatable>(Immediate)); | 6376 llvm::isa<ConstantRelocatable>(Immediate)); |
6238 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_None || | 6377 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_None || |
6239 RandomizationPoolingPaused == true) { | 6378 RandomizationPoolingPaused == true) { |
6240 // Immediates randomization/pooling off or paused | 6379 // Immediates randomization/pooling off or paused |
(...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
6285 // assigned register as this assignment is that start of its use-def | 6424 // assigned register as this assignment is that start of its use-def |
6286 // chain. So we add RegNum argument here. | 6425 // chain. So we add RegNum argument here. |
6287 Variable *Reg = makeReg(Immediate->getType(), RegNum); | 6426 Variable *Reg = makeReg(Immediate->getType(), RegNum); |
6288 IceString Label; | 6427 IceString Label; |
6289 llvm::raw_string_ostream Label_stream(Label); | 6428 llvm::raw_string_ostream Label_stream(Label); |
6290 Immediate->emitPoolLabel(Label_stream, Ctx); | 6429 Immediate->emitPoolLabel(Label_stream, Ctx); |
6291 constexpr RelocOffsetT Offset = 0; | 6430 constexpr RelocOffsetT Offset = 0; |
6292 constexpr bool SuppressMangling = true; | 6431 constexpr bool SuppressMangling = true; |
6293 Constant *Symbol = | 6432 Constant *Symbol = |
6294 Ctx->getConstantSym(Offset, Label_stream.str(), SuppressMangling); | 6433 Ctx->getConstantSym(Offset, Label_stream.str(), SuppressMangling); |
| 6434 const bool UseNonsfi = Ctx->getFlags().getUseNonsfi(); |
| 6435 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; |
6295 X86OperandMem *MemOperand = | 6436 X86OperandMem *MemOperand = |
6296 X86OperandMem::create(Func, Immediate->getType(), nullptr, Symbol); | 6437 X86OperandMem::create(Func, Immediate->getType(), Base, Symbol); |
| 6438 if (UseNonsfi) |
| 6439 MemOperand->setIsPIC(); |
6297 _mov(Reg, MemOperand); | 6440 _mov(Reg, MemOperand); |
6298 return Reg; | 6441 return Reg; |
6299 } | 6442 } |
6300 assert("Unsupported -randomize-pool-immediates option" && false); | 6443 assert("Unsupported -randomize-pool-immediates option" && false); |
6301 } | 6444 } |
6302 // the constant Immediate is not eligible for blinding/pooling | 6445 // the constant Immediate is not eligible for blinding/pooling |
6303 return Immediate; | 6446 return Immediate; |
6304 } | 6447 } |
6305 | 6448 |
6306 template <typename TraitsType> | 6449 template <typename TraitsType> |
(...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
6378 return MemOperand; | 6521 return MemOperand; |
6379 Variable *RegTemp = makeReg(IceType_i32); | 6522 Variable *RegTemp = makeReg(IceType_i32); |
6380 IceString Label; | 6523 IceString Label; |
6381 llvm::raw_string_ostream Label_stream(Label); | 6524 llvm::raw_string_ostream Label_stream(Label); |
6382 MemOperand->getOffset()->emitPoolLabel(Label_stream, Ctx); | 6525 MemOperand->getOffset()->emitPoolLabel(Label_stream, Ctx); |
6383 MemOperand->getOffset()->setShouldBePooled(true); | 6526 MemOperand->getOffset()->setShouldBePooled(true); |
6384 constexpr RelocOffsetT SymOffset = 0; | 6527 constexpr RelocOffsetT SymOffset = 0; |
6385 constexpr bool SuppressMangling = true; | 6528 constexpr bool SuppressMangling = true; |
6386 Constant *Symbol = Ctx->getConstantSym(SymOffset, Label_stream.str(), | 6529 Constant *Symbol = Ctx->getConstantSym(SymOffset, Label_stream.str(), |
6387 SuppressMangling); | 6530 SuppressMangling); |
| 6531 const bool UseNonsfi = Ctx->getFlags().getUseNonsfi(); |
| 6532 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; |
6388 X86OperandMem *SymbolOperand = X86OperandMem::create( | 6533 X86OperandMem *SymbolOperand = X86OperandMem::create( |
6389 Func, MemOperand->getOffset()->getType(), nullptr, Symbol); | 6534 Func, MemOperand->getOffset()->getType(), Base, Symbol); |
| 6535 if (UseNonsfi) |
| 6536 SymbolOperand->setIsPIC(); |
6390 _mov(RegTemp, SymbolOperand); | 6537 _mov(RegTemp, SymbolOperand); |
6391 // If we have a base variable here, we should add the lea instruction | 6538 // If we have a base variable here, we should add the lea instruction |
6392 // to add the value of the base variable to RegTemp. If there is no | 6539 // to add the value of the base variable to RegTemp. If there is no |
6393 // base variable, we won't need this lea instruction. | 6540 // base variable, we won't need this lea instruction. |
6394 if (MemOperand->getBase()) { | 6541 if (MemOperand->getBase()) { |
6395 X86OperandMem *CalculateOperand = X86OperandMem::create( | 6542 X86OperandMem *CalculateOperand = X86OperandMem::create( |
6396 Func, MemOperand->getType(), MemOperand->getBase(), nullptr, | 6543 Func, MemOperand->getType(), MemOperand->getBase(), nullptr, |
6397 RegTemp, 0, MemOperand->getSegmentRegister()); | 6544 RegTemp, 0, MemOperand->getSegmentRegister()); |
6398 _lea(RegTemp, CalculateOperand); | 6545 _lea(RegTemp, CalculateOperand); |
6399 } | 6546 } |
6400 X86OperandMem *NewMemOperand = X86OperandMem::create( | 6547 X86OperandMem *NewMemOperand = X86OperandMem::create( |
6401 Func, MemOperand->getType(), RegTemp, nullptr, | 6548 Func, MemOperand->getType(), RegTemp, nullptr, |
6402 MemOperand->getIndex(), MemOperand->getShift(), | 6549 MemOperand->getIndex(), MemOperand->getShift(), |
6403 MemOperand->getSegmentRegister()); | 6550 MemOperand->getSegmentRegister()); |
6404 return NewMemOperand; | 6551 return NewMemOperand; |
6405 } | 6552 } |
6406 assert("Unsupported -randomize-pool-immediates option" && false); | 6553 assert("Unsupported -randomize-pool-immediates option" && false); |
6407 } | 6554 } |
6408 } | 6555 } |
6409 // the offset is not eligible for blinding or pooling, return the original | 6556 // the offset is not eligible for blinding or pooling, return the original |
6410 // mem operand | 6557 // mem operand |
6411 return MemOperand; | 6558 return MemOperand; |
6412 } | 6559 } |
6413 | 6560 |
6414 } // end of namespace X86NAMESPACE | 6561 } // end of namespace X86NAMESPACE |
6415 } // end of namespace Ice | 6562 } // end of namespace Ice |
6416 | 6563 |
6417 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H | 6564 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H |
OLD | NEW |