OLD | NEW |
1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// | 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// |
2 // | 2 // |
3 // The Subzero Code Generator | 3 // The Subzero Code Generator |
4 // | 4 // |
5 // This file is distributed under the University of Illinois Open Source | 5 // This file is distributed under the University of Illinois Open Source |
6 // License. See LICENSE.TXT for details. | 6 // License. See LICENSE.TXT for details. |
7 // | 7 // |
8 //===----------------------------------------------------------------------===// | 8 //===----------------------------------------------------------------------===// |
9 /// | 9 /// |
10 /// \file | 10 /// \file |
(...skipping 290 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
301 } | 301 } |
302 | 302 |
303 template <typename TraitsType> | 303 template <typename TraitsType> |
304 void TargetX86Base<TraitsType>::initNodeForLowering(CfgNode *Node) { | 304 void TargetX86Base<TraitsType>::initNodeForLowering(CfgNode *Node) { |
305 FoldingInfo.init(Node); | 305 FoldingInfo.init(Node); |
306 FoldingInfo.dump(Func); | 306 FoldingInfo.dump(Func); |
307 } | 307 } |
308 | 308 |
309 template <typename TraitsType> | 309 template <typename TraitsType> |
310 TargetX86Base<TraitsType>::TargetX86Base(Cfg *Func) | 310 TargetX86Base<TraitsType>::TargetX86Base(Cfg *Func) |
311 : TargetLowering(Func), NeedSandboxing(Ctx->getFlags().getUseSandboxing()) { | 311 : TargetLowering(Func), NeedSandboxing(SandboxingType == ST_NaCl) { |
312 static_assert( | 312 static_assert( |
313 (Traits::InstructionSet::End - Traits::InstructionSet::Begin) == | 313 (Traits::InstructionSet::End - Traits::InstructionSet::Begin) == |
314 (TargetInstructionSet::X86InstructionSet_End - | 314 (TargetInstructionSet::X86InstructionSet_End - |
315 TargetInstructionSet::X86InstructionSet_Begin), | 315 TargetInstructionSet::X86InstructionSet_Begin), |
316 "Traits::InstructionSet range different from TargetInstructionSet"); | 316 "Traits::InstructionSet range different from TargetInstructionSet"); |
317 if (Func->getContext()->getFlags().getTargetInstructionSet() != | 317 if (Func->getContext()->getFlags().getTargetInstructionSet() != |
318 TargetInstructionSet::BaseInstructionSet) { | 318 TargetInstructionSet::BaseInstructionSet) { |
319 InstructionSet = static_cast<InstructionSetEnum>( | 319 InstructionSet = static_cast<InstructionSetEnum>( |
320 (Func->getContext()->getFlags().getTargetInstructionSet() - | 320 (Func->getContext()->getFlags().getTargetInstructionSet() - |
321 TargetInstructionSet::X86InstructionSet_Begin) + | 321 TargetInstructionSet::X86InstructionSet_Begin) + |
322 Traits::InstructionSet::Begin); | 322 Traits::InstructionSet::Begin); |
323 } | 323 } |
324 } | 324 } |
325 | 325 |
326 template <typename TraitsType> | 326 template <typename TraitsType> |
327 void TargetX86Base<TraitsType>::staticInit(GlobalContext *Ctx) { | 327 void TargetX86Base<TraitsType>::staticInit(GlobalContext *Ctx) { |
328 Traits::initRegisterSet(Ctx->getFlags(), &TypeToRegisterSet, | 328 Traits::initRegisterSet(Ctx->getFlags(), &TypeToRegisterSet, |
329 &RegisterAliases); | 329 &RegisterAliases); |
330 filterTypeToRegisterSet(Ctx, Traits::RegisterSet::Reg_NUM, | 330 filterTypeToRegisterSet(Ctx, Traits::RegisterSet::Reg_NUM, |
331 TypeToRegisterSet.data(), TypeToRegisterSet.size(), | 331 TypeToRegisterSet.data(), TypeToRegisterSet.size(), |
332 Traits::getRegName); | 332 Traits::getRegName); |
333 PcRelFixup = Traits::FK_PcRel; | 333 PcRelFixup = Traits::FK_PcRel; |
334 AbsFixup = | 334 AbsFixup = |
335 Ctx->getFlags().getUseNonsfi() ? Traits::FK_Gotoff : Traits::FK_Abs; | 335 Ctx->getFlags().getUseNonsfi() ? Traits::FK_Gotoff : Traits::FK_Abs; |
336 } | 336 } |
337 | 337 |
338 template <typename TraitsType> void TargetX86Base<TraitsType>::translateO2() { | 338 template <typename TraitsType> void TargetX86Base<TraitsType>::translateO2() { |
339 TimerMarker T(TimerStack::TT_O2, Func); | 339 TimerMarker T(TimerStack::TT_O2, Func); |
340 | 340 |
341 if (!Traits::Is64Bit && Func->getContext()->getFlags().getUseNonsfi()) { | 341 if (SandboxingType != ST_None) { |
342 GotVar = Func->makeVariable(IceType_i32); | 342 initRebasePtr(); |
343 } | |
344 | |
345 if (NeedSandboxing) { | |
346 initSandbox(); | |
347 } | 343 } |
348 | 344 |
349 genTargetHelperCalls(); | 345 genTargetHelperCalls(); |
350 Func->dump("After target helper call insertion"); | 346 Func->dump("After target helper call insertion"); |
351 | 347 |
352 // Merge Alloca instructions, and lay out the stack. | 348 // Merge Alloca instructions, and lay out the stack. |
353 static constexpr bool SortAndCombineAllocas = true; | 349 static constexpr bool SortAndCombineAllocas = true; |
354 Func->processAllocas(SortAndCombineAllocas); | 350 Func->processAllocas(SortAndCombineAllocas); |
355 Func->dump("After Alloca processing"); | 351 Func->dump("After Alloca processing"); |
356 | 352 |
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
407 Func->dump("After x86 address mode opt"); | 403 Func->dump("After x86 address mode opt"); |
408 | 404 |
409 // Disable constant blinding or pooling for load optimization. | 405 // Disable constant blinding or pooling for load optimization. |
410 { | 406 { |
411 BoolFlagSaver B(RandomizationPoolingPaused, true); | 407 BoolFlagSaver B(RandomizationPoolingPaused, true); |
412 doLoadOpt(); | 408 doLoadOpt(); |
413 } | 409 } |
414 Func->genCode(); | 410 Func->genCode(); |
415 if (Func->hasError()) | 411 if (Func->hasError()) |
416 return; | 412 return; |
417 initGotVarIfNeeded(); | 413 if (SandboxingType != ST_None) { |
| 414 initSandbox(); |
| 415 } |
418 Func->dump("After x86 codegen"); | 416 Func->dump("After x86 codegen"); |
419 | 417 |
420 // Register allocation. This requires instruction renumbering and full | 418 // Register allocation. This requires instruction renumbering and full |
421 // liveness analysis. Loops must be identified before liveness so variable | 419 // liveness analysis. Loops must be identified before liveness so variable |
422 // use weights are correct. | 420 // use weights are correct. |
423 Func->renumberInstructions(); | 421 Func->renumberInstructions(); |
424 if (Func->hasError()) | 422 if (Func->hasError()) |
425 return; | 423 return; |
426 Func->liveness(Liveness_Intervals); | 424 Func->liveness(Liveness_Intervals); |
427 if (Func->hasError()) | 425 if (Func->hasError()) |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
467 | 465 |
468 // Mark nodes that require sandbox alignment | 466 // Mark nodes that require sandbox alignment |
469 if (NeedSandboxing) { | 467 if (NeedSandboxing) { |
470 Func->markNodesForSandboxing(); | 468 Func->markNodesForSandboxing(); |
471 } | 469 } |
472 } | 470 } |
473 | 471 |
474 template <typename TraitsType> void TargetX86Base<TraitsType>::translateOm1() { | 472 template <typename TraitsType> void TargetX86Base<TraitsType>::translateOm1() { |
475 TimerMarker T(TimerStack::TT_Om1, Func); | 473 TimerMarker T(TimerStack::TT_Om1, Func); |
476 | 474 |
477 if (!Traits::Is64Bit && Func->getContext()->getFlags().getUseNonsfi()) { | 475 if (SandboxingType != ST_None) { |
478 GotVar = Func->makeVariable(IceType_i32); | 476 initRebasePtr(); |
479 } | |
480 | |
481 if (NeedSandboxing) { | |
482 initSandbox(); | |
483 } | 477 } |
484 | 478 |
485 genTargetHelperCalls(); | 479 genTargetHelperCalls(); |
486 | 480 |
487 // Do not merge Alloca instructions, and lay out the stack. | 481 // Do not merge Alloca instructions, and lay out the stack. |
488 static constexpr bool SortAndCombineAllocas = false; | 482 static constexpr bool SortAndCombineAllocas = false; |
489 Func->processAllocas(SortAndCombineAllocas); | 483 Func->processAllocas(SortAndCombineAllocas); |
490 Func->dump("After Alloca processing"); | 484 Func->dump("After Alloca processing"); |
491 | 485 |
492 Func->placePhiLoads(); | 486 Func->placePhiLoads(); |
493 if (Func->hasError()) | 487 if (Func->hasError()) |
494 return; | 488 return; |
495 Func->placePhiStores(); | 489 Func->placePhiStores(); |
496 if (Func->hasError()) | 490 if (Func->hasError()) |
497 return; | 491 return; |
498 Func->deletePhis(); | 492 Func->deletePhis(); |
499 if (Func->hasError()) | 493 if (Func->hasError()) |
500 return; | 494 return; |
501 Func->dump("After Phi lowering"); | 495 Func->dump("After Phi lowering"); |
502 | 496 |
503 Func->doArgLowering(); | 497 Func->doArgLowering(); |
504 Func->genCode(); | 498 Func->genCode(); |
505 if (Func->hasError()) | 499 if (Func->hasError()) |
506 return; | 500 return; |
507 initGotVarIfNeeded(); | 501 if (SandboxingType != ST_None) { |
| 502 initSandbox(); |
| 503 } |
508 Func->dump("After initial x8632 codegen"); | 504 Func->dump("After initial x8632 codegen"); |
509 | 505 |
510 regAlloc(RAK_InfOnly); | 506 regAlloc(RAK_InfOnly); |
511 if (Func->hasError()) | 507 if (Func->hasError()) |
512 return; | 508 return; |
513 Func->dump("After regalloc of infinite-weight variables"); | 509 Func->dump("After regalloc of infinite-weight variables"); |
514 | 510 |
515 Func->genFrame(); | 511 Func->genFrame(); |
516 if (Func->hasError()) | 512 if (Func->hasError()) |
517 return; | 513 return; |
(...skipping 804 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1322 } | 1318 } |
1323 | 1319 |
1324 template <typename TraitsType> | 1320 template <typename TraitsType> |
1325 llvm::SmallBitVector | 1321 llvm::SmallBitVector |
1326 TargetX86Base<TraitsType>::getRegisterSet(RegSetMask Include, | 1322 TargetX86Base<TraitsType>::getRegisterSet(RegSetMask Include, |
1327 RegSetMask Exclude) const { | 1323 RegSetMask Exclude) const { |
1328 return Traits::getRegisterSet(Ctx->getFlags(), Include, Exclude); | 1324 return Traits::getRegisterSet(Ctx->getFlags(), Include, Exclude); |
1329 } | 1325 } |
1330 | 1326 |
1331 template <typename TraitsType> | 1327 template <typename TraitsType> |
1332 void TargetX86Base<TraitsType>::initGotVarIfNeeded() { | |
1333 if (!Func->getContext()->getFlags().getUseNonsfi()) | |
1334 return; | |
1335 if (Traits::Is64Bit) { | |
1336 // Probably no implementation is needed, but error to be safe for now. | |
1337 llvm::report_fatal_error( | |
1338 "Need to implement initGotVarIfNeeded() for 64-bit."); | |
1339 } | |
1340 // Insert the GotVar assignment as the very first lowered instruction. Later, | |
1341 // it will be moved into the right place - after the stack frame is set up but | |
1342 // before in-args are copied into registers. | |
1343 Context.init(Func->getEntryNode()); | |
1344 Context.setInsertPoint(Context.getCur()); | |
1345 Context.insert<typename Traits::Insts::GetIP>(GotVar); | |
1346 } | |
1347 | |
1348 template <typename TraitsType> | |
1349 void TargetX86Base<TraitsType>::lowerAlloca(const InstAlloca *Inst) { | 1328 void TargetX86Base<TraitsType>::lowerAlloca(const InstAlloca *Inst) { |
1350 // Conservatively require the stack to be aligned. Some stack adjustment | 1329 // Conservatively require the stack to be aligned. Some stack adjustment |
1351 // operations implemented below assume that the stack is aligned before the | 1330 // operations implemented below assume that the stack is aligned before the |
1352 // alloca. All the alloca code ensures that the stack alignment is preserved | 1331 // alloca. All the alloca code ensures that the stack alignment is preserved |
1353 // after the alloca. The stack alignment restriction can be relaxed in some | 1332 // after the alloca. The stack alignment restriction can be relaxed in some |
1354 // cases. | 1333 // cases. |
1355 NeedsStackAlignment = true; | 1334 NeedsStackAlignment = true; |
1356 | 1335 |
1357 // For default align=0, set it to the real value 1, to avoid any | 1336 // For default align=0, set it to the real value 1, to avoid any |
1358 // bit-manipulation problems below. | 1337 // bit-manipulation problems below. |
(...skipping 3550 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
4909 lowerCast(InstCast::create(Func, InstCast::Zext, ValExtVar, Val)); | 4888 lowerCast(InstCast::create(Func, InstCast::Zext, ValExtVar, Val)); |
4910 ValExt = ValExtVar; | 4889 ValExt = ValExtVar; |
4911 } | 4890 } |
4912 InstCall *Call = makeHelperCall(H_call_memset, nullptr, 3); | 4891 InstCall *Call = makeHelperCall(H_call_memset, nullptr, 3); |
4913 Call->addArg(Dest); | 4892 Call->addArg(Dest); |
4914 Call->addArg(ValExt); | 4893 Call->addArg(ValExt); |
4915 Call->addArg(Count); | 4894 Call->addArg(Count); |
4916 lowerCall(Call); | 4895 lowerCall(Call); |
4917 } | 4896 } |
4918 | 4897 |
4919 inline bool isAdd(const Inst *Inst) { | 4898 class AddressOptimizer { |
4920 if (auto *Arith = llvm::dyn_cast_or_null<const InstArithmetic>(Inst)) { | 4899 AddressOptimizer() = delete; |
4921 return (Arith->getOp() == InstArithmetic::Add); | 4900 AddressOptimizer(const AddressOptimizer &) = delete; |
| 4901 AddressOptimizer &operator=(const AddressOptimizer &) = delete; |
| 4902 |
| 4903 public: |
| 4904 explicit AddressOptimizer(const Cfg *Func) |
| 4905 : Func(Func), VMetadata(Func->getVMetadata()) {} |
| 4906 |
| 4907 inline void dumpAddressOpt(const ConstantRelocatable *const Relocatable, |
| 4908 int32_t Offset, const Variable *Base, |
| 4909 const Variable *Index, uint16_t Shift, |
| 4910 const Inst *Reason) const; |
| 4911 |
| 4912 inline const Inst *matchAssign(Variable **Var, |
| 4913 ConstantRelocatable **Relocatable, |
| 4914 int32_t *Offset); |
| 4915 |
| 4916 inline const Inst *matchCombinedBaseIndex(Variable **Base, Variable **Index, |
| 4917 uint16_t *Shift); |
| 4918 |
| 4919 inline const Inst *matchShiftedIndex(Variable **Index, uint16_t *Shift); |
| 4920 |
| 4921 inline const Inst *matchOffsetBase(Variable **Base, |
| 4922 ConstantRelocatable **Relocatable, |
| 4923 int32_t *Offset); |
| 4924 |
| 4925 private: |
| 4926 const Cfg *const Func; |
| 4927 const VariablesMetadata *const VMetadata; |
| 4928 |
| 4929 static bool isAdd(const Inst *Inst) { |
| 4930 if (auto *Arith = llvm::dyn_cast_or_null<const InstArithmetic>(Inst)) { |
| 4931 return (Arith->getOp() == InstArithmetic::Add); |
| 4932 } |
| 4933 return false; |
4922 } | 4934 } |
4923 return false; | 4935 }; |
4924 } | |
4925 | 4936 |
4926 inline void dumpAddressOpt(const Cfg *Func, | 4937 void AddressOptimizer::dumpAddressOpt( |
4927 const ConstantRelocatable *Relocatable, | 4938 const ConstantRelocatable *const Relocatable, int32_t Offset, |
4928 int32_t Offset, const Variable *Base, | 4939 const Variable *Base, const Variable *Index, uint16_t Shift, |
4929 const Variable *Index, uint16_t Shift, | 4940 const Inst *Reason) const { |
4930 const Inst *Reason) { | |
4931 if (!BuildDefs::dump()) | 4941 if (!BuildDefs::dump()) |
4932 return; | 4942 return; |
4933 if (!Func->isVerbose(IceV_AddrOpt)) | 4943 if (!Func->isVerbose(IceV_AddrOpt)) |
4934 return; | 4944 return; |
4935 OstreamLocker L(Func->getContext()); | 4945 OstreamLocker L(Func->getContext()); |
4936 Ostream &Str = Func->getContext()->getStrDump(); | 4946 Ostream &Str = Func->getContext()->getStrDump(); |
4937 Str << "Instruction: "; | 4947 Str << "Instruction: "; |
4938 Reason->dumpDecorated(Func); | 4948 Reason->dumpDecorated(Func); |
4939 Str << " results in Base="; | 4949 Str << " results in Base="; |
4940 if (Base) | 4950 if (Base) |
4941 Base->dump(Func); | 4951 Base->dump(Func); |
4942 else | 4952 else |
4943 Str << "<null>"; | 4953 Str << "<null>"; |
4944 Str << ", Index="; | 4954 Str << ", Index="; |
4945 if (Index) | 4955 if (Index) |
4946 Index->dump(Func); | 4956 Index->dump(Func); |
4947 else | 4957 else |
4948 Str << "<null>"; | 4958 Str << "<null>"; |
4949 Str << ", Shift=" << Shift << ", Offset=" << Offset | 4959 Str << ", Shift=" << Shift << ", Offset=" << Offset |
4950 << ", Relocatable=" << Relocatable << "\n"; | 4960 << ", Relocatable=" << Relocatable << "\n"; |
4951 } | 4961 } |
4952 | 4962 |
4953 inline bool matchAssign(const VariablesMetadata *VMetadata, Variable *GotVar, | 4963 const Inst *AddressOptimizer::matchAssign(Variable **Var, |
4954 Variable *&Var, ConstantRelocatable *&Relocatable, | 4964 ConstantRelocatable **Relocatable, |
4955 int32_t &Offset, const Inst *&Reason) { | 4965 int32_t *Offset) { |
4956 // Var originates from Var=SrcVar ==> set Var:=SrcVar | 4966 // Var originates from Var=SrcVar ==> set Var:=SrcVar |
4957 if (Var == nullptr) | 4967 if (*Var == nullptr) |
4958 return false; | 4968 return nullptr; |
4959 if (const Inst *VarAssign = VMetadata->getSingleDefinition(Var)) { | 4969 if (const Inst *VarAssign = VMetadata->getSingleDefinition(*Var)) { |
4960 assert(!VMetadata->isMultiDef(Var)); | 4970 assert(!VMetadata->isMultiDef(*Var)); |
4961 if (llvm::isa<InstAssign>(VarAssign)) { | 4971 if (llvm::isa<InstAssign>(VarAssign)) { |
4962 Operand *SrcOp = VarAssign->getSrc(0); | 4972 Operand *SrcOp = VarAssign->getSrc(0); |
4963 assert(SrcOp); | 4973 assert(SrcOp); |
4964 if (auto *SrcVar = llvm::dyn_cast<Variable>(SrcOp)) { | 4974 if (auto *SrcVar = llvm::dyn_cast<Variable>(SrcOp)) { |
4965 if (!VMetadata->isMultiDef(SrcVar) && | 4975 if (!VMetadata->isMultiDef(SrcVar) && |
4966 // TODO: ensure SrcVar stays single-BB | 4976 // TODO: ensure SrcVar stays single-BB |
4967 true) { | 4977 true) { |
4968 Var = SrcVar; | 4978 *Var = SrcVar; |
4969 Reason = VarAssign; | 4979 return VarAssign; |
4970 return true; | |
4971 } | 4980 } |
4972 } else if (auto *Const = llvm::dyn_cast<ConstantInteger32>(SrcOp)) { | 4981 } else if (auto *Const = llvm::dyn_cast<ConstantInteger32>(SrcOp)) { |
4973 int32_t MoreOffset = Const->getValue(); | 4982 int32_t MoreOffset = Const->getValue(); |
4974 if (Utils::WouldOverflowAdd(Offset, MoreOffset)) | 4983 if (Utils::WouldOverflowAdd(*Offset, MoreOffset)) |
4975 return false; | 4984 return nullptr; |
4976 Var = nullptr; | 4985 *Var = nullptr; |
4977 Offset += MoreOffset; | 4986 Offset += MoreOffset; |
4978 Reason = VarAssign; | 4987 return VarAssign; |
4979 return true; | |
4980 } else if (auto *AddReloc = llvm::dyn_cast<ConstantRelocatable>(SrcOp)) { | 4988 } else if (auto *AddReloc = llvm::dyn_cast<ConstantRelocatable>(SrcOp)) { |
4981 if (Relocatable == nullptr) { | 4989 if (*Relocatable == nullptr) { |
4982 Var = GotVar; | 4990 // It is always safe to fold a relocatable through assignment -- the |
4983 Relocatable = AddReloc; | 4991 // assignment frees a slot in the address operand that can be used to |
4984 Reason = VarAssign; | 4992 // hold the Sandbox Pointer -- if any. |
4985 return true; | 4993 *Var = nullptr; |
| 4994 *Relocatable = AddReloc; |
| 4995 return VarAssign; |
4986 } | 4996 } |
4987 } | 4997 } |
4988 } | 4998 } |
4989 } | 4999 } |
4990 return false; | 5000 return nullptr; |
4991 } | 5001 } |
4992 | 5002 |
4993 inline bool matchCombinedBaseIndex(const VariablesMetadata *VMetadata, | 5003 const Inst *AddressOptimizer::matchCombinedBaseIndex(Variable **Base, |
4994 Variable *&Base, Variable *&Index, | 5004 Variable **Index, |
4995 uint16_t &Shift, const Inst *&Reason) { | 5005 uint16_t *Shift) { |
4996 // Index==nullptr && Base is Base=Var1+Var2 ==> | 5006 // Index==nullptr && Base is Base=Var1+Var2 ==> |
4997 // set Base=Var1, Index=Var2, Shift=0 | 5007 // set Base=Var1, Index=Var2, Shift=0 |
4998 if (Base == nullptr) | 5008 if (*Base == nullptr) |
4999 return false; | 5009 return nullptr; |
5000 if (Index != nullptr) | 5010 if (*Index != nullptr) |
5001 return false; | 5011 return nullptr; |
5002 auto *BaseInst = VMetadata->getSingleDefinition(Base); | 5012 auto *BaseInst = VMetadata->getSingleDefinition(*Base); |
5003 if (BaseInst == nullptr) | 5013 if (BaseInst == nullptr) |
5004 return false; | 5014 return nullptr; |
5005 assert(!VMetadata->isMultiDef(Base)); | 5015 assert(!VMetadata->isMultiDef(*Base)); |
5006 if (BaseInst->getSrcSize() < 2) | 5016 if (BaseInst->getSrcSize() < 2) |
5007 return false; | 5017 return nullptr; |
5008 if (auto *Var1 = llvm::dyn_cast<Variable>(BaseInst->getSrc(0))) { | 5018 if (auto *Var1 = llvm::dyn_cast<Variable>(BaseInst->getSrc(0))) { |
5009 if (VMetadata->isMultiDef(Var1)) | 5019 if (VMetadata->isMultiDef(Var1)) |
5010 return false; | 5020 return nullptr; |
5011 if (auto *Var2 = llvm::dyn_cast<Variable>(BaseInst->getSrc(1))) { | 5021 if (auto *Var2 = llvm::dyn_cast<Variable>(BaseInst->getSrc(1))) { |
5012 if (VMetadata->isMultiDef(Var2)) | 5022 if (VMetadata->isMultiDef(Var2)) |
5013 return false; | 5023 return nullptr; |
5014 if (isAdd(BaseInst) && | 5024 if (isAdd(BaseInst) && |
5015 // TODO: ensure Var1 and Var2 stay single-BB | 5025 // TODO: ensure Var1 and Var2 stay single-BB |
5016 true) { | 5026 true) { |
5017 Base = Var1; | 5027 *Base = Var1; |
5018 Index = Var2; | 5028 *Index = Var2; |
5019 Shift = 0; // should already have been 0 | 5029 *Shift = 0; // should already have been 0 |
5020 Reason = BaseInst; | 5030 return BaseInst; |
5021 return true; | |
5022 } | 5031 } |
5023 } | 5032 } |
5024 } | 5033 } |
5025 return false; | 5034 return nullptr; |
5026 } | 5035 } |
5027 | 5036 |
5028 inline bool matchShiftedIndex(const VariablesMetadata *VMetadata, | 5037 const Inst *AddressOptimizer::matchShiftedIndex(Variable **Index, |
5029 Variable *&Index, uint16_t &Shift, | 5038 uint16_t *Shift) { |
5030 const Inst *&Reason) { | |
5031 // Index is Index=Var*Const && log2(Const)+Shift<=3 ==> | 5039 // Index is Index=Var*Const && log2(Const)+Shift<=3 ==> |
5032 // Index=Var, Shift+=log2(Const) | 5040 // Index=Var, Shift+=log2(Const) |
5033 if (Index == nullptr) | 5041 if (*Index == nullptr) |
5034 return false; | 5042 return nullptr; |
5035 auto *IndexInst = VMetadata->getSingleDefinition(Index); | 5043 auto *IndexInst = VMetadata->getSingleDefinition(*Index); |
5036 if (IndexInst == nullptr) | 5044 if (IndexInst == nullptr) |
5037 return false; | 5045 return nullptr; |
5038 assert(!VMetadata->isMultiDef(Index)); | 5046 assert(!VMetadata->isMultiDef(*Index)); |
5039 if (IndexInst->getSrcSize() < 2) | 5047 if (IndexInst->getSrcSize() < 2) |
5040 return false; | 5048 return nullptr; |
5041 if (auto *ArithInst = llvm::dyn_cast<InstArithmetic>(IndexInst)) { | 5049 if (auto *ArithInst = llvm::dyn_cast<InstArithmetic>(IndexInst)) { |
5042 if (auto *Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(0))) { | 5050 if (auto *Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(0))) { |
5043 if (auto *Const = | 5051 if (auto *Const = |
5044 llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1))) { | 5052 llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1))) { |
5045 if (VMetadata->isMultiDef(Var) || Const->getType() != IceType_i32) | 5053 if (VMetadata->isMultiDef(Var) || Const->getType() != IceType_i32) |
5046 return false; | 5054 return nullptr; |
5047 switch (ArithInst->getOp()) { | 5055 switch (ArithInst->getOp()) { |
5048 default: | 5056 default: |
5049 return false; | 5057 return nullptr; |
5050 case InstArithmetic::Mul: { | 5058 case InstArithmetic::Mul: { |
5051 uint32_t Mult = Const->getValue(); | 5059 uint32_t Mult = Const->getValue(); |
5052 uint32_t LogMult; | 5060 uint32_t LogMult; |
5053 switch (Mult) { | 5061 switch (Mult) { |
5054 case 1: | 5062 case 1: |
5055 LogMult = 0; | 5063 LogMult = 0; |
5056 break; | 5064 break; |
5057 case 2: | 5065 case 2: |
5058 LogMult = 1; | 5066 LogMult = 1; |
5059 break; | 5067 break; |
5060 case 4: | 5068 case 4: |
5061 LogMult = 2; | 5069 LogMult = 2; |
5062 break; | 5070 break; |
5063 case 8: | 5071 case 8: |
5064 LogMult = 3; | 5072 LogMult = 3; |
5065 break; | 5073 break; |
5066 default: | 5074 default: |
5067 return false; | 5075 return nullptr; |
5068 } | 5076 } |
5069 if (Shift + LogMult <= 3) { | 5077 if (*Shift + LogMult <= 3) { |
5070 Index = Var; | 5078 *Index = Var; |
5071 Shift += LogMult; | 5079 *Shift += LogMult; |
5072 Reason = IndexInst; | 5080 return IndexInst; |
5073 return true; | |
5074 } | 5081 } |
5075 } | 5082 } |
5076 case InstArithmetic::Shl: { | 5083 case InstArithmetic::Shl: { |
5077 uint32_t ShiftAmount = Const->getValue(); | 5084 uint32_t ShiftAmount = Const->getValue(); |
5078 switch (ShiftAmount) { | 5085 switch (ShiftAmount) { |
5079 case 0: | 5086 case 0: |
5080 case 1: | 5087 case 1: |
5081 case 2: | 5088 case 2: |
5082 case 3: | 5089 case 3: |
5083 break; | 5090 break; |
5084 default: | 5091 default: |
5085 return false; | 5092 return nullptr; |
5086 } | 5093 } |
5087 if (Shift + ShiftAmount <= 3) { | 5094 if (*Shift + ShiftAmount <= 3) { |
5088 Index = Var; | 5095 *Index = Var; |
5089 Shift += ShiftAmount; | 5096 *Shift += ShiftAmount; |
5090 Reason = IndexInst; | 5097 return IndexInst; |
5091 return true; | |
5092 } | 5098 } |
5093 } | 5099 } |
5094 } | 5100 } |
5095 } | 5101 } |
5096 } | 5102 } |
5097 } | 5103 } |
5098 return false; | 5104 return nullptr; |
5099 } | 5105 } |
5100 | 5106 |
5101 inline bool matchOffsetBase(const VariablesMetadata *VMetadata, | 5107 const Inst *AddressOptimizer::matchOffsetBase(Variable **Base, |
5102 Variable *GotVar, Variable *&Base, | 5108 ConstantRelocatable **Relocatable, |
5103 Variable *&BaseOther, | 5109 int32_t *Offset) { |
5104 ConstantRelocatable *&Relocatable, int32_t &Offset, | |
5105 const Inst *&Reason) { | |
5106 // Base is Base=Var+Const || Base is Base=Const+Var ==> | 5110 // Base is Base=Var+Const || Base is Base=Const+Var ==> |
5107 // set Base=Var, Offset+=Const | 5111 // set Base=Var, Offset+=Const |
5108 // Base is Base=Var-Const ==> | 5112 // Base is Base=Var-Const ==> |
5109 // set Base=Var, Offset-=Const | 5113 // set Base=Var, Offset-=Const |
5110 if (Base == nullptr) { | 5114 if (*Base == nullptr) { |
5111 return false; | 5115 return nullptr; |
5112 } | 5116 } |
5113 const Inst *BaseInst = VMetadata->getSingleDefinition(Base); | 5117 const Inst *BaseInst = VMetadata->getSingleDefinition(*Base); |
5114 if (BaseInst == nullptr) { | 5118 if (BaseInst == nullptr) { |
5115 return false; | 5119 return nullptr; |
5116 } | 5120 } |
5117 assert(!VMetadata->isMultiDef(Base)); | 5121 assert(!VMetadata->isMultiDef(*Base)); |
5118 if (auto *ArithInst = llvm::dyn_cast<const InstArithmetic>(BaseInst)) { | 5122 if (auto *ArithInst = llvm::dyn_cast<const InstArithmetic>(BaseInst)) { |
5119 if (ArithInst->getOp() != InstArithmetic::Add && | 5123 if (ArithInst->getOp() != InstArithmetic::Add && |
5120 ArithInst->getOp() != InstArithmetic::Sub) | 5124 ArithInst->getOp() != InstArithmetic::Sub) |
5121 return false; | 5125 return nullptr; |
5122 bool IsAdd = ArithInst->getOp() == InstArithmetic::Add; | 5126 bool IsAdd = ArithInst->getOp() == InstArithmetic::Add; |
5123 Operand *Src0 = ArithInst->getSrc(0); | 5127 Operand *Src0 = ArithInst->getSrc(0); |
5124 Operand *Src1 = ArithInst->getSrc(1); | 5128 Operand *Src1 = ArithInst->getSrc(1); |
5125 auto *Var0 = llvm::dyn_cast<Variable>(Src0); | 5129 auto *Var0 = llvm::dyn_cast<Variable>(Src0); |
5126 auto *Var1 = llvm::dyn_cast<Variable>(Src1); | 5130 auto *Var1 = llvm::dyn_cast<Variable>(Src1); |
5127 auto *Const0 = llvm::dyn_cast<ConstantInteger32>(Src0); | 5131 auto *Const0 = llvm::dyn_cast<ConstantInteger32>(Src0); |
5128 auto *Const1 = llvm::dyn_cast<ConstantInteger32>(Src1); | 5132 auto *Const1 = llvm::dyn_cast<ConstantInteger32>(Src1); |
5129 auto *Reloc0 = llvm::dyn_cast<ConstantRelocatable>(Src0); | 5133 auto *Reloc0 = llvm::dyn_cast<ConstantRelocatable>(Src0); |
5130 auto *Reloc1 = llvm::dyn_cast<ConstantRelocatable>(Src1); | 5134 auto *Reloc1 = llvm::dyn_cast<ConstantRelocatable>(Src1); |
5131 Variable *NewBase = nullptr; | 5135 Variable *NewBase = nullptr; |
5132 int32_t NewOffset = Offset; | 5136 int32_t NewOffset = *Offset; |
5133 ConstantRelocatable *NewRelocatable = Relocatable; | 5137 ConstantRelocatable *NewRelocatable = *Relocatable; |
5134 if (Var0 && Var1) | 5138 if (Var0 && Var1) |
5135 // TODO(sehr): merge base/index splitting into here. | 5139 // TODO(sehr): merge base/index splitting into here. |
5136 return false; | 5140 return nullptr; |
5137 if (!IsAdd && Var1) | 5141 if (!IsAdd && Var1) |
5138 return false; | 5142 return nullptr; |
5139 if (Var0) | 5143 if (Var0) |
5140 NewBase = Var0; | 5144 NewBase = Var0; |
5141 else if (Var1) | 5145 else if (Var1) |
5142 NewBase = Var1; | 5146 NewBase = Var1; |
5143 // Don't know how to add/subtract two relocatables. | 5147 // Don't know how to add/subtract two relocatables. |
5144 if ((Relocatable && (Reloc0 || Reloc1)) || (Reloc0 && Reloc1)) | 5148 if ((*Relocatable && (Reloc0 || Reloc1)) || (Reloc0 && Reloc1)) |
5145 return false; | 5149 return nullptr; |
5146 // Don't know how to subtract a relocatable. | 5150 // Don't know how to subtract a relocatable. |
5147 if (!IsAdd && Reloc1) | 5151 if (!IsAdd && Reloc1) |
5148 return false; | 5152 return nullptr; |
5149 // Incorporate ConstantRelocatables. | 5153 // Incorporate ConstantRelocatables. |
5150 if (Reloc0) | 5154 if (Reloc0) |
5151 NewRelocatable = Reloc0; | 5155 NewRelocatable = Reloc0; |
5152 else if (Reloc1) | 5156 else if (Reloc1) |
5153 NewRelocatable = Reloc1; | 5157 NewRelocatable = Reloc1; |
5154 if ((Reloc0 || Reloc1) && BaseOther && GotVar) | |
5155 return false; | |
5156 // Compute the updated constant offset. | 5158 // Compute the updated constant offset. |
5157 if (Const0) { | 5159 if (Const0) { |
5158 const int32_t MoreOffset = | 5160 const int32_t MoreOffset = |
5159 IsAdd ? Const0->getValue() : -Const0->getValue(); | 5161 IsAdd ? Const0->getValue() : -Const0->getValue(); |
5160 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) | 5162 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) |
5161 return false; | 5163 return nullptr; |
5162 NewOffset += MoreOffset; | 5164 NewOffset += MoreOffset; |
5163 } | 5165 } |
5164 if (Const1) { | 5166 if (Const1) { |
5165 const int32_t MoreOffset = | 5167 const int32_t MoreOffset = |
5166 IsAdd ? Const1->getValue() : -Const1->getValue(); | 5168 IsAdd ? Const1->getValue() : -Const1->getValue(); |
5167 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) | 5169 if (Utils::WouldOverflowAdd(NewOffset, MoreOffset)) |
5168 return false; | 5170 return nullptr; |
5169 NewOffset += MoreOffset; | 5171 NewOffset += MoreOffset; |
5170 } | 5172 } |
5171 // Update the computed address parameters once we are sure optimization | 5173 *Base = NewBase; |
5172 // is valid. | 5174 *Offset = NewOffset; |
5173 if ((Reloc0 || Reloc1) && GotVar) { | 5175 *Relocatable = NewRelocatable; |
5174 assert(BaseOther == nullptr); | 5176 return BaseInst; |
5175 BaseOther = GotVar; | |
5176 } | |
5177 Base = NewBase; | |
5178 Offset = NewOffset; | |
5179 Relocatable = NewRelocatable; | |
5180 Reason = BaseInst; | |
5181 return true; | |
5182 } | 5177 } |
5183 return false; | 5178 return nullptr; |
5184 } | 5179 } |
5185 | 5180 |
5186 // Builds information for a canonical address expresion: | 5181 template <typename TypeTraits> |
5187 // <Relocatable + Offset>(Base, Index, Shift) | 5182 typename TargetX86Base<TypeTraits>::X86OperandMem * |
5188 // On entry: | 5183 TargetX86Base<TypeTraits>::computeAddressOpt(const Inst *Instr, Type MemType, |
5189 // Relocatable == null, | 5184 Operand *Addr) { |
5190 // Offset == 0, | |
5191 // Base is a Variable, | |
5192 // Index == nullptr, | |
5193 // Shift == 0 | |
5194 inline bool computeAddressOpt(Cfg *Func, const Inst *Instr, Variable *GotVar, | |
5195 bool ReserveSlot, | |
5196 ConstantRelocatable *&Relocatable, | |
5197 int32_t &Offset, Variable *&Base, | |
5198 Variable *&Index, uint16_t &Shift) { | |
5199 bool AddressWasOptimized = false; | |
5200 Func->resetCurrentNode(); | 5185 Func->resetCurrentNode(); |
5201 if (Func->isVerbose(IceV_AddrOpt)) { | 5186 if (Func->isVerbose(IceV_AddrOpt)) { |
5202 OstreamLocker L(Func->getContext()); | 5187 OstreamLocker L(Func->getContext()); |
5203 Ostream &Str = Func->getContext()->getStrDump(); | 5188 Ostream &Str = Func->getContext()->getStrDump(); |
5204 Str << "\nStarting computeAddressOpt for instruction:\n "; | 5189 Str << "\nStarting computeAddressOpt for instruction:\n "; |
5205 Instr->dumpDecorated(Func); | 5190 Instr->dumpDecorated(Func); |
5206 } | 5191 } |
5207 if (Base == nullptr) | 5192 |
5208 return AddressWasOptimized; | 5193 OptAddr NewAddr; |
| 5194 NewAddr.Base = llvm::dyn_cast<Variable>(Addr); |
| 5195 if (NewAddr.Base == nullptr) |
| 5196 return nullptr; |
| 5197 |
5209 // If the Base has more than one use or is live across multiple blocks, then | 5198 // If the Base has more than one use or is live across multiple blocks, then |
5210 // don't go further. Alternatively (?), never consider a transformation that | 5199 // don't go further. Alternatively (?), never consider a transformation that |
5211 // would change a variable that is currently *not* live across basic block | 5200 // would change a variable that is currently *not* live across basic block |
5212 // boundaries into one that *is*. | 5201 // boundaries into one that *is*. |
5213 if (Func->getVMetadata()->isMultiBlock(Base) /* || Base->getUseCount() > 1*/) | 5202 if (Func->getVMetadata()->isMultiBlock( |
5214 return AddressWasOptimized; | 5203 NewAddr.Base) /* || Base->getUseCount() > 1*/) |
| 5204 return nullptr; |
5215 | 5205 |
| 5206 AddressOptimizer AddrOpt(Func); |
5216 const bool MockBounds = Func->getContext()->getFlags().getMockBoundsCheck(); | 5207 const bool MockBounds = Func->getContext()->getFlags().getMockBoundsCheck(); |
5217 const VariablesMetadata *VMetadata = Func->getVMetadata(); | |
5218 const Inst *Reason = nullptr; | 5208 const Inst *Reason = nullptr; |
| 5209 bool AddressWasOptimized = false; |
| 5210 // The following unnamed struct identifies the address mode formation steps |
| 5211 // that could potentially create an invalid memory operand (i.e., no free |
| 5212 // slots for RebasePtr.) We add all those variables to this struct so that we |
| 5213 // can use memset() to reset all members to false. |
| 5214 struct { |
| 5215 bool AssignBase = false; |
| 5216 bool AssignIndex = false; |
| 5217 bool OffsetFromBase = false; |
| 5218 bool OffsetFromIndex = false; |
| 5219 bool CombinedBaseIndex = false; |
| 5220 } Skip; |
| 5221 // This points to the boolean in Skip that represents the last folding |
| 5222 // performed. This is used to disable a pattern match that generated an |
| 5223 // invalid address. Without this, the algorithm would never finish. |
| 5224 bool *SkipLastFolding = nullptr; |
| 5225 // NewAddrCheckpoint is used to rollback the address being formed in case an |
| 5226 // invalid address is formed. |
| 5227 OptAddr NewAddrCheckpoint; |
| 5228 Reason = Instr; |
5219 do { | 5229 do { |
5220 assert(!ReserveSlot || Base == nullptr || Index == nullptr); | 5230 if (SandboxingType != ST_None) { |
| 5231 // When sandboxing, we defer the sandboxing of NewAddr to the Concrete |
| 5232 // Target. If our optimization was overly aggressive, then we simply undo |
| 5233 // what the previous iteration did, and set the previous pattern's skip |
| 5234 // bit to true. |
| 5235 if (!legalizeOptAddrForSandbox(&NewAddr)) { |
| 5236 *SkipLastFolding = true; |
| 5237 SkipLastFolding = nullptr; |
| 5238 NewAddr = NewAddrCheckpoint; |
| 5239 Reason = nullptr; |
| 5240 } |
| 5241 } |
| 5242 |
5221 if (Reason) { | 5243 if (Reason) { |
5222 dumpAddressOpt(Func, Relocatable, Offset, Base, Index, Shift, Reason); | 5244 AddrOpt.dumpAddressOpt(NewAddr.Relocatable, NewAddr.Offset, NewAddr.Base, |
| 5245 NewAddr.Index, NewAddr.Shift, Reason); |
5223 AddressWasOptimized = true; | 5246 AddressWasOptimized = true; |
5224 Reason = nullptr; | 5247 Reason = nullptr; |
| 5248 SkipLastFolding = nullptr; |
| 5249 memset(&Skip, 0, sizeof(Skip)); |
5225 } | 5250 } |
| 5251 |
| 5252 NewAddrCheckpoint = NewAddr; |
| 5253 |
5226 // Update Base and Index to follow through assignments to definitions. | 5254 // Update Base and Index to follow through assignments to definitions. |
5227 if (matchAssign(VMetadata, GotVar, Base, Relocatable, Offset, Reason)) { | 5255 if (!Skip.AssignBase && |
| 5256 (Reason = AddrOpt.matchAssign(&NewAddr.Base, &NewAddr.Relocatable, |
| 5257 &NewAddr.Offset))) { |
| 5258 SkipLastFolding = &Skip.AssignBase; |
5228 // Assignments of Base from a Relocatable or ConstantInt32 can result | 5259 // Assignments of Base from a Relocatable or ConstantInt32 can result |
5229 // in Base becoming nullptr. To avoid code duplication in this loop we | 5260 // in Base becoming nullptr. To avoid code duplication in this loop we |
5230 // prefer that Base be non-nullptr if possible. | 5261 // prefer that Base be non-nullptr if possible. |
5231 if ((Base == nullptr) && (Index != nullptr) && Shift == 0) | 5262 if ((NewAddr.Base == nullptr) && (NewAddr.Index != nullptr) && |
5232 std::swap(Base, Index); | 5263 NewAddr.Shift == 0) { |
| 5264 std::swap(NewAddr.Base, NewAddr.Index); |
| 5265 } |
5233 continue; | 5266 continue; |
5234 } | 5267 } |
5235 if (matchAssign(VMetadata, GotVar, Index, Relocatable, Offset, Reason)) | 5268 if (!Skip.AssignBase && |
| 5269 (Reason = AddrOpt.matchAssign(&NewAddr.Index, &NewAddr.Relocatable, |
| 5270 &NewAddr.Offset))) { |
| 5271 SkipLastFolding = &Skip.AssignIndex; |
5236 continue; | 5272 continue; |
| 5273 } |
5237 | 5274 |
5238 if (!MockBounds) { | 5275 if (!MockBounds) { |
5239 // Transition from: | 5276 // Transition from: |
5240 // <Relocatable + Offset>(Base) to | 5277 // <Relocatable + Offset>(Base) to |
5241 // <Relocatable + Offset>(Base, Index) | 5278 // <Relocatable + Offset>(Base, Index) |
5242 if (!ReserveSlot && | 5279 if (!Skip.CombinedBaseIndex && |
5243 matchCombinedBaseIndex(VMetadata, Base, Index, Shift, Reason)) | 5280 (Reason = AddrOpt.matchCombinedBaseIndex( |
| 5281 &NewAddr.Base, &NewAddr.Index, &NewAddr.Shift))) { |
| 5282 SkipLastFolding = &Skip.CombinedBaseIndex; |
5244 continue; | 5283 continue; |
| 5284 } |
| 5285 |
5245 // Recognize multiply/shift and update Shift amount. | 5286 // Recognize multiply/shift and update Shift amount. |
5246 // Index becomes Index=Var<<Const && Const+Shift<=3 ==> | 5287 // Index becomes Index=Var<<Const && Const+Shift<=3 ==> |
5247 // Index=Var, Shift+=Const | 5288 // Index=Var, Shift+=Const |
5248 // Index becomes Index=Const*Var && log2(Const)+Shift<=3 ==> | 5289 // Index becomes Index=Const*Var && log2(Const)+Shift<=3 ==> |
5249 // Index=Var, Shift+=log2(Const) | 5290 // Index=Var, Shift+=log2(Const) |
5250 if (matchShiftedIndex(VMetadata, Index, Shift, Reason)) | 5291 if ((Reason = |
| 5292 AddrOpt.matchShiftedIndex(&NewAddr.Index, &NewAddr.Shift))) { |
5251 continue; | 5293 continue; |
| 5294 } |
| 5295 |
5252 // If Shift is zero, the choice of Base and Index was purely arbitrary. | 5296 // If Shift is zero, the choice of Base and Index was purely arbitrary. |
5253 // Recognize multiply/shift and set Shift amount. | 5297 // Recognize multiply/shift and set Shift amount. |
5254 // Shift==0 && Base is Base=Var*Const && log2(Const)+Shift<=3 ==> | 5298 // Shift==0 && Base is Base=Var*Const && log2(Const)+Shift<=3 ==> |
5255 // swap(Index,Base) | 5299 // swap(Index,Base) |
5256 // Similar for Base=Const*Var and Base=Var<<Const | 5300 // Similar for Base=Const*Var and Base=Var<<Const |
5257 if (Shift == 0 && matchShiftedIndex(VMetadata, Base, Shift, Reason)) { | 5301 if (NewAddr.Shift == 0 && |
5258 std::swap(Base, Index); | 5302 (Reason = AddrOpt.matchShiftedIndex(&NewAddr.Base, &NewAddr.Shift))) { |
| 5303 std::swap(NewAddr.Base, NewAddr.Index); |
5259 continue; | 5304 continue; |
5260 } | 5305 } |
5261 } | 5306 } |
| 5307 |
5262 // Update Offset to reflect additions/subtractions with constants and | 5308 // Update Offset to reflect additions/subtractions with constants and |
5263 // relocatables. | 5309 // relocatables. |
5264 // TODO: consider overflow issues with respect to Offset. | 5310 // TODO: consider overflow issues with respect to Offset. |
5265 if (matchOffsetBase(VMetadata, GotVar, Base, Index, Relocatable, Offset, | 5311 if (!Skip.OffsetFromBase && |
5266 Reason)) | 5312 (Reason = AddrOpt.matchOffsetBase(&NewAddr.Base, &NewAddr.Relocatable, |
| 5313 &NewAddr.Offset))) { |
| 5314 SkipLastFolding = &Skip.OffsetFromBase; |
5267 continue; | 5315 continue; |
5268 if (Shift == 0 && matchOffsetBase(VMetadata, GotVar, Index, Base, | 5316 } |
5269 Relocatable, Offset, Reason)) | 5317 if (NewAddr.Shift == 0 && !Skip.OffsetFromIndex && |
| 5318 (Reason = AddrOpt.matchOffsetBase(&NewAddr.Index, &NewAddr.Relocatable, |
| 5319 &NewAddr.Offset))) { |
| 5320 SkipLastFolding = &Skip.OffsetFromIndex; |
5270 continue; | 5321 continue; |
| 5322 } |
| 5323 |
5271 // TODO(sehr, stichnot): Handle updates of Index with Shift != 0. | 5324 // TODO(sehr, stichnot): Handle updates of Index with Shift != 0. |
5272 // Index is Index=Var+Const ==> | 5325 // Index is Index=Var+Const ==> |
5273 // set Index=Var, Offset+=(Const<<Shift) | 5326 // set Index=Var, Offset+=(Const<<Shift) |
5274 // Index is Index=Const+Var ==> | 5327 // Index is Index=Const+Var ==> |
5275 // set Index=Var, Offset+=(Const<<Shift) | 5328 // set Index=Var, Offset+=(Const<<Shift) |
5276 // Index is Index=Var-Const ==> | 5329 // Index is Index=Var-Const ==> |
5277 // set Index=Var, Offset-=(Const<<Shift) | 5330 // set Index=Var, Offset-=(Const<<Shift) |
5278 break; | 5331 break; |
5279 } while (Reason); | 5332 } while (Reason); |
5280 // Undo any addition of GotVar. It will be added back when the mem operand is | 5333 |
5281 // legalized. | 5334 if (!AddressWasOptimized) { |
5282 if (Base == GotVar) | 5335 return nullptr; |
5283 Base = nullptr; | 5336 } |
5284 if (Index == GotVar) | 5337 |
5285 Index = nullptr; | 5338 // Undo any addition of RebasePtr. It will be added back when the mem |
5286 return AddressWasOptimized; | 5339 // operand is sandboxed. |
| 5340 if (NewAddr.Base == RebasePtr) { |
| 5341 NewAddr.Base = nullptr; |
| 5342 } |
| 5343 |
| 5344 if (NewAddr.Index == RebasePtr) { |
| 5345 NewAddr.Index = nullptr; |
| 5346 NewAddr.Shift = 0; |
| 5347 } |
| 5348 |
| 5349 Constant *OffsetOp = nullptr; |
| 5350 if (NewAddr.Relocatable == nullptr) { |
| 5351 OffsetOp = Ctx->getConstantInt32(NewAddr.Offset); |
| 5352 } else { |
| 5353 OffsetOp = |
| 5354 Ctx->getConstantSym(NewAddr.Relocatable->getOffset() + NewAddr.Offset, |
| 5355 NewAddr.Relocatable->getName(), |
| 5356 NewAddr.Relocatable->getSuppressMangling()); |
| 5357 } |
| 5358 // Vanilla ICE load instructions should not use the segment registers, and |
| 5359 // computeAddressOpt only works at the level of Variables and Constants, not |
| 5360 // other X86OperandMem, so there should be no mention of segment |
| 5361 // registers there either. |
| 5362 static constexpr auto SegmentReg = |
| 5363 X86OperandMem::SegmentRegisters::DefaultSegment; |
| 5364 |
| 5365 return X86OperandMem::create(Func, MemType, NewAddr.Base, OffsetOp, |
| 5366 NewAddr.Index, NewAddr.Shift, SegmentReg); |
5287 } | 5367 } |
5288 | 5368 |
5289 /// Add a mock bounds check on the memory address before using it as a load or | 5369 /// Add a mock bounds check on the memory address before using it as a load or |
5290 /// store operand. The basic idea is that given a memory operand [reg], we | 5370 /// store operand. The basic idea is that given a memory operand [reg], we |
5291 /// would first add bounds-check code something like: | 5371 /// would first add bounds-check code something like: |
5292 /// | 5372 /// |
5293 /// cmp reg, <lb> | 5373 /// cmp reg, <lb> |
5294 /// jl out_of_line_error | 5374 /// jl out_of_line_error |
5295 /// cmp reg, <ub> | 5375 /// cmp reg, <ub> |
5296 /// jg out_of_line_error | 5376 /// jg out_of_line_error |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5354 Type Ty = DestLoad->getType(); | 5434 Type Ty = DestLoad->getType(); |
5355 Operand *Src0 = formMemoryOperand(Load->getSourceAddress(), Ty); | 5435 Operand *Src0 = formMemoryOperand(Load->getSourceAddress(), Ty); |
5356 doMockBoundsCheck(Src0); | 5436 doMockBoundsCheck(Src0); |
5357 auto *Assign = InstAssign::create(Func, DestLoad, Src0); | 5437 auto *Assign = InstAssign::create(Func, DestLoad, Src0); |
5358 lowerAssign(Assign); | 5438 lowerAssign(Assign); |
5359 } | 5439 } |
5360 | 5440 |
5361 template <typename TraitsType> | 5441 template <typename TraitsType> |
5362 void TargetX86Base<TraitsType>::doAddressOptLoad() { | 5442 void TargetX86Base<TraitsType>::doAddressOptLoad() { |
5363 Inst *Inst = Context.getCur(); | 5443 Inst *Inst = Context.getCur(); |
| 5444 Operand *Addr = Inst->getSrc(0); |
5364 Variable *Dest = Inst->getDest(); | 5445 Variable *Dest = Inst->getDest(); |
5365 Operand *Addr = Inst->getSrc(0); | 5446 if (auto *OptAddr = computeAddressOpt(Inst, Dest->getType(), Addr)) { |
5366 Variable *Index = nullptr; | |
5367 ConstantRelocatable *Relocatable = nullptr; | |
5368 uint16_t Shift = 0; | |
5369 int32_t Offset = 0; | |
5370 // Vanilla ICE load instructions should not use the segment registers, and | |
5371 // computeAddressOpt only works at the level of Variables and Constants, not | |
5372 // other X86OperandMem, so there should be no mention of segment | |
5373 // registers there either. | |
5374 constexpr auto SegmentReg = X86OperandMem::SegmentRegisters::DefaultSegment; | |
5375 auto *Base = llvm::dyn_cast<Variable>(Addr); | |
5376 const bool ReserveSlot = Traits::Is64Bit && NeedSandboxing; | |
5377 if (computeAddressOpt(Func, Inst, GotVar, ReserveSlot, Relocatable, Offset, | |
5378 Base, Index, Shift)) { | |
5379 Inst->setDeleted(); | 5447 Inst->setDeleted(); |
5380 Constant *OffsetOp = nullptr; | 5448 Context.insert<InstLoad>(Dest, OptAddr); |
5381 if (Relocatable == nullptr) { | |
5382 OffsetOp = Ctx->getConstantInt32(Offset); | |
5383 } else { | |
5384 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, | |
5385 Relocatable->getName(), | |
5386 Relocatable->getSuppressMangling()); | |
5387 } | |
5388 // The new mem operand is created without IsRebased being set, because | |
5389 // computeAddressOpt() doesn't include GotVar in its final result. | |
5390 Addr = X86OperandMem::create(Func, Dest->getType(), Base, OffsetOp, Index, | |
5391 Shift, SegmentReg); | |
5392 Context.insert<InstLoad>(Dest, Addr); | |
5393 } | 5449 } |
5394 } | 5450 } |
5395 | 5451 |
5396 template <typename TraitsType> | 5452 template <typename TraitsType> |
5397 void TargetX86Base<TraitsType>::randomlyInsertNop(float Probability, | 5453 void TargetX86Base<TraitsType>::randomlyInsertNop(float Probability, |
5398 RandomNumberGenerator &RNG) { | 5454 RandomNumberGenerator &RNG) { |
5399 RandomNumberGeneratorWrapper RNGW(RNG); | 5455 RandomNumberGeneratorWrapper RNGW(RNG); |
5400 if (RNGW.getTrueWithProbability(Probability)) { | 5456 if (RNGW.getTrueWithProbability(Probability)) { |
5401 _nop(RNGW(Traits::X86_NUM_NOP_VARIANTS)); | 5457 _nop(RNGW(Traits::X86_NUM_NOP_VARIANTS)); |
5402 } | 5458 } |
(...skipping 276 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5679 _storep(legalizeToReg(Value), NewAddr); | 5735 _storep(legalizeToReg(Value), NewAddr); |
5680 } else { | 5736 } else { |
5681 Value = legalize(Value, Legal_Reg | Legal_Imm); | 5737 Value = legalize(Value, Legal_Reg | Legal_Imm); |
5682 _store(Value, NewAddr); | 5738 _store(Value, NewAddr); |
5683 } | 5739 } |
5684 } | 5740 } |
5685 | 5741 |
5686 template <typename TraitsType> | 5742 template <typename TraitsType> |
5687 void TargetX86Base<TraitsType>::doAddressOptStore() { | 5743 void TargetX86Base<TraitsType>::doAddressOptStore() { |
5688 auto *Inst = llvm::cast<InstStore>(Context.getCur()); | 5744 auto *Inst = llvm::cast<InstStore>(Context.getCur()); |
| 5745 Operand *Addr = Inst->getAddr(); |
5689 Operand *Data = Inst->getData(); | 5746 Operand *Data = Inst->getData(); |
5690 Operand *Addr = Inst->getAddr(); | 5747 if (auto *OptAddr = computeAddressOpt(Inst, Data->getType(), Addr)) { |
5691 Variable *Index = nullptr; | |
5692 ConstantRelocatable *Relocatable = nullptr; | |
5693 uint16_t Shift = 0; | |
5694 int32_t Offset = 0; | |
5695 auto *Base = llvm::dyn_cast<Variable>(Addr); | |
5696 // Vanilla ICE store instructions should not use the segment registers, and | |
5697 // computeAddressOpt only works at the level of Variables and Constants, not | |
5698 // other X86OperandMem, so there should be no mention of segment | |
5699 // registers there either. | |
5700 constexpr auto SegmentReg = X86OperandMem::SegmentRegisters::DefaultSegment; | |
5701 const bool ReserveSlot = Traits::Is64Bit && NeedSandboxing; | |
5702 if (computeAddressOpt(Func, Inst, GotVar, ReserveSlot, Relocatable, Offset, | |
5703 Base, Index, Shift)) { | |
5704 Inst->setDeleted(); | 5748 Inst->setDeleted(); |
5705 Constant *OffsetOp = nullptr; | 5749 auto *NewStore = Context.insert<InstStore>(Data, OptAddr); |
5706 if (Relocatable == nullptr) { | |
5707 OffsetOp = Ctx->getConstantInt32(Offset); | |
5708 } else { | |
5709 OffsetOp = Ctx->getConstantSym(Relocatable->getOffset() + Offset, | |
5710 Relocatable->getName(), | |
5711 Relocatable->getSuppressMangling()); | |
5712 } | |
5713 // The new mem operand is created without IsRebased being set, because | |
5714 // computeAddressOpt() doesn't include GotVar in its final result. | |
5715 Addr = X86OperandMem::create(Func, Data->getType(), Base, OffsetOp, Index, | |
5716 Shift, SegmentReg); | |
5717 auto *NewStore = Context.insert<InstStore>(Data, Addr); | |
5718 if (Inst->getDest()) | 5750 if (Inst->getDest()) |
5719 NewStore->setRmwBeacon(Inst->getRmwBeacon()); | 5751 NewStore->setRmwBeacon(Inst->getRmwBeacon()); |
5720 } | 5752 } |
5721 } | 5753 } |
5722 | 5754 |
5723 template <typename TraitsType> | 5755 template <typename TraitsType> |
5724 Operand *TargetX86Base<TraitsType>::lowerCmpRange(Operand *Comparison, | 5756 Operand *TargetX86Base<TraitsType>::lowerCmpRange(Operand *Comparison, |
5725 uint64_t Min, uint64_t Max) { | 5757 uint64_t Min, uint64_t Max) { |
5726 // TODO(ascull): 64-bit should not reach here but only because it is not | 5758 // TODO(ascull): 64-bit should not reach here but only because it is not |
5727 // implemented yet. This should be able to handle the 64-bit case. | 5759 // implemented yet. This should be able to handle the 64-bit case. |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5767 const Type PointerType = getPointerType(); | 5799 const Type PointerType = getPointerType(); |
5768 if (RangeIndex->getType() != PointerType) { | 5800 if (RangeIndex->getType() != PointerType) { |
5769 Index = makeReg(PointerType); | 5801 Index = makeReg(PointerType); |
5770 _movzx(Index, RangeIndex); | 5802 _movzx(Index, RangeIndex); |
5771 } else { | 5803 } else { |
5772 Index = legalizeToReg(RangeIndex); | 5804 Index = legalizeToReg(RangeIndex); |
5773 } | 5805 } |
5774 | 5806 |
5775 constexpr RelocOffsetT RelocOffset = 0; | 5807 constexpr RelocOffsetT RelocOffset = 0; |
5776 constexpr bool SuppressMangling = true; | 5808 constexpr bool SuppressMangling = true; |
5777 const bool IsRebased = Ctx->getFlags().getUseNonsfi(); | |
5778 IceString MangledName = Ctx->mangleName(Func->getFunctionName()); | 5809 IceString MangledName = Ctx->mangleName(Func->getFunctionName()); |
5779 Variable *Base = IsRebased ? legalizeToReg(GotVar) : nullptr; | 5810 constexpr Variable *NoBase = nullptr; |
5780 Constant *Offset = Ctx->getConstantSym( | 5811 Constant *Offset = Ctx->getConstantSym( |
5781 RelocOffset, InstJumpTable::makeName(MangledName, JumpTable->getId()), | 5812 RelocOffset, InstJumpTable::makeName(MangledName, JumpTable->getId()), |
5782 SuppressMangling); | 5813 SuppressMangling); |
5783 uint16_t Shift = typeWidthInBytesLog2(PointerType); | 5814 uint16_t Shift = typeWidthInBytesLog2(PointerType); |
5784 constexpr auto Segment = X86OperandMem::SegmentRegisters::DefaultSegment; | 5815 constexpr auto Segment = X86OperandMem::SegmentRegisters::DefaultSegment; |
5785 | 5816 |
5786 Variable *Target = nullptr; | 5817 Variable *Target = nullptr; |
5787 if (Traits::Is64Bit && NeedSandboxing) { | 5818 if (Traits::Is64Bit && NeedSandboxing) { |
5788 assert(Base == nullptr); | |
5789 assert(Index != nullptr && Index->getType() == IceType_i32); | 5819 assert(Index != nullptr && Index->getType() == IceType_i32); |
5790 } | 5820 } |
5791 auto *TargetInMemory = X86OperandMem::create( | 5821 auto *TargetInMemory = X86OperandMem::create(Func, PointerType, NoBase, |
5792 Func, PointerType, Base, Offset, Index, Shift, Segment, IsRebased); | 5822 Offset, Index, Shift, Segment); |
5793 _mov(Target, TargetInMemory); | 5823 _mov(Target, TargetInMemory); |
5794 | 5824 |
5795 lowerIndirectJump(Target); | 5825 lowerIndirectJump(Target); |
5796 | 5826 |
5797 if (DefaultTarget == nullptr) | 5827 if (DefaultTarget == nullptr) |
5798 Context.insert(SkipJumpTable); | 5828 Context.insert(SkipJumpTable); |
5799 return; | 5829 return; |
5800 } | 5830 } |
5801 case CaseCluster::Range: { | 5831 case CaseCluster::Range: { |
5802 if (Case.isUnitRange()) { | 5832 if (Case.isUnitRange()) { |
(...skipping 306 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
6109 if (const auto *RMW = llvm::dyn_cast<InstX86FakeRMW>(Instr)) { | 6139 if (const auto *RMW = llvm::dyn_cast<InstX86FakeRMW>(Instr)) { |
6110 lowerRMW(RMW); | 6140 lowerRMW(RMW); |
6111 } else { | 6141 } else { |
6112 TargetLowering::lowerOther(Instr); | 6142 TargetLowering::lowerOther(Instr); |
6113 } | 6143 } |
6114 } | 6144 } |
6115 | 6145 |
6116 /// Turn an i64 Phi instruction into a pair of i32 Phi instructions, to preserve | 6146 /// Turn an i64 Phi instruction into a pair of i32 Phi instructions, to preserve |
6117 /// integrity of liveness analysis. Undef values are also turned into zeroes, | 6147 /// integrity of liveness analysis. Undef values are also turned into zeroes, |
6118 /// since loOperand() and hiOperand() don't expect Undef input. Also, in | 6148 /// since loOperand() and hiOperand() don't expect Undef input. Also, in |
6119 /// Non-SFI mode, add a FakeUse(GotVar) for every pooled constant operand. | 6149 /// Non-SFI mode, add a FakeUse(RebasePtr) for every pooled constant operand. |
6120 template <typename TraitsType> void TargetX86Base<TraitsType>::prelowerPhis() { | 6150 template <typename TraitsType> void TargetX86Base<TraitsType>::prelowerPhis() { |
6121 if (Ctx->getFlags().getUseNonsfi()) { | 6151 if (Ctx->getFlags().getUseNonsfi()) { |
6122 assert(GotVar); | 6152 assert(RebasePtr); |
6123 CfgNode *Node = Context.getNode(); | 6153 CfgNode *Node = Context.getNode(); |
6124 uint32_t GotVarUseCount = 0; | 6154 uint32_t RebasePtrUseCount = 0; |
6125 for (Inst &I : Node->getPhis()) { | 6155 for (Inst &I : Node->getPhis()) { |
6126 auto *Phi = llvm::dyn_cast<InstPhi>(&I); | 6156 auto *Phi = llvm::dyn_cast<InstPhi>(&I); |
6127 if (Phi->isDeleted()) | 6157 if (Phi->isDeleted()) |
6128 continue; | 6158 continue; |
6129 for (SizeT I = 0; I < Phi->getSrcSize(); ++I) { | 6159 for (SizeT I = 0; I < Phi->getSrcSize(); ++I) { |
6130 Operand *Src = Phi->getSrc(I); | 6160 Operand *Src = Phi->getSrc(I); |
6131 // TODO(stichnot): This over-counts for +0.0, and under-counts for other | 6161 // TODO(stichnot): This over-counts for +0.0, and under-counts for other |
6132 // kinds of pooling. | 6162 // kinds of pooling. |
6133 if (llvm::isa<ConstantRelocatable>(Src) || | 6163 if (llvm::isa<ConstantRelocatable>(Src) || |
6134 llvm::isa<ConstantFloat>(Src) || llvm::isa<ConstantDouble>(Src)) { | 6164 llvm::isa<ConstantFloat>(Src) || llvm::isa<ConstantDouble>(Src)) { |
6135 ++GotVarUseCount; | 6165 ++RebasePtrUseCount; |
6136 } | 6166 } |
6137 } | 6167 } |
6138 } | 6168 } |
6139 if (GotVarUseCount) { | 6169 if (RebasePtrUseCount) { |
6140 Node->getInsts().push_front(InstFakeUse::create(Func, GotVar)); | 6170 Node->getInsts().push_front(InstFakeUse::create(Func, RebasePtr)); |
6141 } | 6171 } |
6142 } | 6172 } |
6143 if (Traits::Is64Bit) { | 6173 if (Traits::Is64Bit) { |
6144 // On x86-64 we don't need to prelower phis -- the architecture can handle | 6174 // On x86-64 we don't need to prelower phis -- the architecture can handle |
6145 // 64-bit integer natively. | 6175 // 64-bit integer natively. |
6146 return; | 6176 return; |
6147 } | 6177 } |
6148 | 6178 |
6149 // Pause constant blinding or pooling, blinding or pooling will be done later | 6179 // Pause constant blinding or pooling, blinding or pooling will be done later |
6150 // during phi lowering assignments | 6180 // during phi lowering assignments |
(...skipping 527 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
6678 if (Base) { | 6708 if (Base) { |
6679 RegBase = llvm::cast<Variable>( | 6709 RegBase = llvm::cast<Variable>( |
6680 legalize(Base, Legal_Reg | Legal_Rematerializable)); | 6710 legalize(Base, Legal_Reg | Legal_Rematerializable)); |
6681 } | 6711 } |
6682 if (Index) { | 6712 if (Index) { |
6683 // TODO(jpp): perhaps we should only allow Legal_Reg if | 6713 // TODO(jpp): perhaps we should only allow Legal_Reg if |
6684 // Base->isRematerializable. | 6714 // Base->isRematerializable. |
6685 RegIndex = llvm::cast<Variable>( | 6715 RegIndex = llvm::cast<Variable>( |
6686 legalize(Index, Legal_Reg | Legal_Rematerializable)); | 6716 legalize(Index, Legal_Reg | Legal_Rematerializable)); |
6687 } | 6717 } |
6688 // For Non-SFI mode, if the Offset field is a ConstantRelocatable, we | 6718 |
6689 // replace either Base or Index with a legalized GotVar. At emission time, | |
6690 // the ConstantRelocatable will be emitted with the @GOTOFF relocation. | |
6691 bool IsRebased = false; | |
6692 if (UseNonsfi && !Mem->getIsRebased() && Offset && | |
6693 llvm::isa<ConstantRelocatable>(Offset)) { | |
6694 assert(!(Allowed & Legal_AddrAbs)); | |
6695 IsRebased = true; | |
6696 if (RegBase == nullptr) { | |
6697 RegBase = legalizeToReg(GotVar); | |
6698 } else if (RegIndex == nullptr) { | |
6699 RegIndex = legalizeToReg(GotVar); | |
6700 } else { | |
6701 llvm::report_fatal_error( | |
6702 "Either Base or Index must be unused in Non-SFI mode"); | |
6703 } | |
6704 } | |
6705 if (Base != RegBase || Index != RegIndex) { | 6719 if (Base != RegBase || Index != RegIndex) { |
6706 Mem = X86OperandMem::create(Func, Ty, RegBase, Offset, RegIndex, Shift, | 6720 Mem = X86OperandMem::create(Func, Ty, RegBase, Offset, RegIndex, Shift, |
6707 Mem->getSegmentRegister(), IsRebased); | 6721 Mem->getSegmentRegister()); |
6708 } | 6722 } |
6709 | 6723 |
6710 // For all Memory Operands, we do randomization/pooling here | 6724 // For all Memory Operands, we do randomization/pooling here. |
6711 From = randomizeOrPoolImmediate(Mem); | 6725 From = randomizeOrPoolImmediate(Mem); |
6712 | 6726 |
6713 if (!(Allowed & Legal_Mem)) { | 6727 if (!(Allowed & Legal_Mem)) { |
6714 From = copyToReg(From, RegNum); | 6728 From = copyToReg(From, RegNum); |
6715 } | 6729 } |
6716 return From; | 6730 return From; |
6717 } | 6731 } |
6718 | 6732 |
6719 if (auto *Const = llvm::dyn_cast<Constant>(From)) { | 6733 if (auto *Const = llvm::dyn_cast<Constant>(From)) { |
6720 if (llvm::isa<ConstantUndef>(Const)) { | 6734 if (llvm::isa<ConstantUndef>(Const)) { |
(...skipping 18 matching lines...) Expand all Loading... |
6739 | 6753 |
6740 // If the operand is an 32 bit constant integer, we should check whether we | 6754 // If the operand is an 32 bit constant integer, we should check whether we |
6741 // need to randomize it or pool it. | 6755 // need to randomize it or pool it. |
6742 if (auto *C = llvm::dyn_cast<ConstantInteger32>(Const)) { | 6756 if (auto *C = llvm::dyn_cast<ConstantInteger32>(Const)) { |
6743 Operand *NewConst = randomizeOrPoolImmediate(C, RegNum); | 6757 Operand *NewConst = randomizeOrPoolImmediate(C, RegNum); |
6744 if (NewConst != Const) { | 6758 if (NewConst != Const) { |
6745 return NewConst; | 6759 return NewConst; |
6746 } | 6760 } |
6747 } | 6761 } |
6748 | 6762 |
6749 // If the operand is a ConstantRelocatable, and Legal_AddrAbs is not | |
6750 // specified, and UseNonsfi is indicated, we need to add GotVar. | |
6751 if (auto *CR = llvm::dyn_cast<ConstantRelocatable>(Const)) { | 6763 if (auto *CR = llvm::dyn_cast<ConstantRelocatable>(Const)) { |
| 6764 // If the operand is a ConstantRelocatable, and Legal_AddrAbs is not |
| 6765 // specified, and UseNonsfi is indicated, we need to add RebasePtr. |
6752 if (UseNonsfi && !(Allowed & Legal_AddrAbs)) { | 6766 if (UseNonsfi && !(Allowed & Legal_AddrAbs)) { |
6753 assert(Ty == IceType_i32); | 6767 assert(Ty == IceType_i32); |
6754 Variable *RegBase = legalizeToReg(GotVar); | |
6755 Variable *NewVar = makeReg(Ty, RegNum); | 6768 Variable *NewVar = makeReg(Ty, RegNum); |
6756 static constexpr bool IsRebased = true; | 6769 auto *Mem = Traits::X86OperandMem::create(Func, Ty, nullptr, CR); |
6757 auto *Mem = | 6770 // LEAs are not automatically sandboxed, thus we explicitly invoke |
6758 Traits::X86OperandMem::create(Func, Ty, RegBase, CR, IsRebased); | 6771 // _sandbox_mem_reference. |
6759 _lea(NewVar, Mem); | 6772 _lea(NewVar, _sandbox_mem_reference(Mem)); |
6760 From = NewVar; | 6773 From = NewVar; |
6761 } | 6774 } |
6762 } | 6775 } else if (isScalarFloatingType(Ty)) { |
6763 | 6776 // Convert a scalar floating point constant into an explicit memory |
6764 // Convert a scalar floating point constant into an explicit memory | 6777 // operand. |
6765 // operand. | |
6766 if (isScalarFloatingType(Ty)) { | |
6767 if (auto *ConstFloat = llvm::dyn_cast<ConstantFloat>(Const)) { | 6778 if (auto *ConstFloat = llvm::dyn_cast<ConstantFloat>(Const)) { |
6768 if (Utils::isPositiveZero(ConstFloat->getValue())) | 6779 if (Utils::isPositiveZero(ConstFloat->getValue())) |
6769 return makeZeroedRegister(Ty, RegNum); | 6780 return makeZeroedRegister(Ty, RegNum); |
6770 } else if (auto *ConstDouble = llvm::dyn_cast<ConstantDouble>(Const)) { | 6781 } else if (auto *ConstDouble = llvm::dyn_cast<ConstantDouble>(Const)) { |
6771 if (Utils::isPositiveZero(ConstDouble->getValue())) | 6782 if (Utils::isPositiveZero(ConstDouble->getValue())) |
6772 return makeZeroedRegister(Ty, RegNum); | 6783 return makeZeroedRegister(Ty, RegNum); |
6773 } | 6784 } |
6774 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; | 6785 |
6775 std::string Buffer; | 6786 std::string Buffer; |
6776 llvm::raw_string_ostream StrBuf(Buffer); | 6787 llvm::raw_string_ostream StrBuf(Buffer); |
6777 llvm::cast<Constant>(From)->emitPoolLabel(StrBuf, Ctx); | 6788 llvm::cast<Constant>(From)->emitPoolLabel(StrBuf, Ctx); |
6778 llvm::cast<Constant>(From)->setShouldBePooled(true); | 6789 llvm::cast<Constant>(From)->setShouldBePooled(true); |
6779 Constant *Offset = Ctx->getConstantSym(0, StrBuf.str(), true); | 6790 Constant *Offset = Ctx->getConstantSym(0, StrBuf.str(), true); |
6780 const bool IsRebased = Base != nullptr; | 6791 auto *Mem = X86OperandMem::create(Func, Ty, nullptr, Offset); |
6781 auto *Mem = X86OperandMem::create(Func, Ty, Base, Offset, IsRebased); | |
6782 From = Mem; | 6792 From = Mem; |
6783 } | 6793 } |
| 6794 |
6784 bool NeedsReg = false; | 6795 bool NeedsReg = false; |
6785 if (!(Allowed & Legal_Imm) && !isScalarFloatingType(Ty)) | 6796 if (!(Allowed & Legal_Imm) && !isScalarFloatingType(Ty)) |
6786 // Immediate specifically not allowed | 6797 // Immediate specifically not allowed. |
6787 NeedsReg = true; | 6798 NeedsReg = true; |
6788 if (!(Allowed & Legal_Mem) && isScalarFloatingType(Ty)) | 6799 if (!(Allowed & Legal_Mem) && isScalarFloatingType(Ty)) |
6789 // On x86, FP constants are lowered to mem operands. | 6800 // On x86, FP constants are lowered to mem operands. |
6790 NeedsReg = true; | 6801 NeedsReg = true; |
6791 if (NeedsReg) { | 6802 if (NeedsReg) { |
6792 From = copyToReg(From, RegNum); | 6803 From = copyToReg(From, RegNum); |
6793 } | 6804 } |
6794 return From; | 6805 return From; |
6795 } | 6806 } |
6796 | 6807 |
(...skipping 12 matching lines...) Expand all Loading... |
6809 if (MustRematerialize) { | 6820 if (MustRematerialize) { |
6810 assert(Ty == IceType_i32); | 6821 assert(Ty == IceType_i32); |
6811 Variable *NewVar = makeReg(Ty, RegNum); | 6822 Variable *NewVar = makeReg(Ty, RegNum); |
6812 // Since Var is rematerializable, the offset will be added when the lea is | 6823 // Since Var is rematerializable, the offset will be added when the lea is |
6813 // emitted. | 6824 // emitted. |
6814 constexpr Constant *NoOffset = nullptr; | 6825 constexpr Constant *NoOffset = nullptr; |
6815 auto *Mem = X86OperandMem::create(Func, Ty, Var, NoOffset); | 6826 auto *Mem = X86OperandMem::create(Func, Ty, Var, NoOffset); |
6816 _lea(NewVar, Mem); | 6827 _lea(NewVar, Mem); |
6817 From = NewVar; | 6828 From = NewVar; |
6818 } else if ((!(Allowed & Legal_Mem) && !MustHaveRegister) || | 6829 } else if ((!(Allowed & Legal_Mem) && !MustHaveRegister) || |
6819 (RegNum != Variable::NoRegister && RegNum != Var->getRegNum()) || | 6830 (RegNum != Variable::NoRegister && RegNum != Var->getRegNum())) { |
6820 MustRematerialize) { | |
6821 From = copyToReg(From, RegNum); | 6831 From = copyToReg(From, RegNum); |
6822 } | 6832 } |
6823 return From; | 6833 return From; |
6824 } | 6834 } |
6825 | 6835 |
6826 llvm::report_fatal_error("Unhandled operand kind in legalize()"); | 6836 llvm::report_fatal_error("Unhandled operand kind in legalize()"); |
6827 return From; | 6837 return From; |
6828 } | 6838 } |
6829 | 6839 |
6830 /// Provide a trivial wrapper to legalize() for this common usage. | 6840 /// Provide a trivial wrapper to legalize() for this common usage. |
(...skipping 259 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
7090 // assigned register as this assignment is that start of its use-def | 7100 // assigned register as this assignment is that start of its use-def |
7091 // chain. So we add RegNum argument here. | 7101 // chain. So we add RegNum argument here. |
7092 Variable *Reg = makeReg(Immediate->getType(), RegNum); | 7102 Variable *Reg = makeReg(Immediate->getType(), RegNum); |
7093 IceString Label; | 7103 IceString Label; |
7094 llvm::raw_string_ostream Label_stream(Label); | 7104 llvm::raw_string_ostream Label_stream(Label); |
7095 Immediate->emitPoolLabel(Label_stream, Ctx); | 7105 Immediate->emitPoolLabel(Label_stream, Ctx); |
7096 constexpr RelocOffsetT Offset = 0; | 7106 constexpr RelocOffsetT Offset = 0; |
7097 constexpr bool SuppressMangling = true; | 7107 constexpr bool SuppressMangling = true; |
7098 Constant *Symbol = | 7108 Constant *Symbol = |
7099 Ctx->getConstantSym(Offset, Label_stream.str(), SuppressMangling); | 7109 Ctx->getConstantSym(Offset, Label_stream.str(), SuppressMangling); |
7100 const bool UseNonsfi = Ctx->getFlags().getUseNonsfi(); | 7110 constexpr Variable *NoBase = nullptr; |
7101 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; | 7111 X86OperandMem *MemOperand = |
7102 const bool IsRebased = Base != nullptr; | 7112 X86OperandMem::create(Func, Immediate->getType(), NoBase, Symbol); |
7103 X86OperandMem *MemOperand = X86OperandMem::create( | |
7104 Func, Immediate->getType(), Base, Symbol, IsRebased); | |
7105 _mov(Reg, MemOperand); | 7113 _mov(Reg, MemOperand); |
7106 return Reg; | 7114 return Reg; |
7107 } | 7115 } |
7108 } | 7116 } |
7109 } | 7117 } |
7110 | 7118 |
7111 template <typename TraitsType> | 7119 template <typename TraitsType> |
7112 typename TargetX86Base<TraitsType>::X86OperandMem * | 7120 typename TargetX86Base<TraitsType>::X86OperandMem * |
7113 TargetX86Base<TraitsType>::randomizeOrPoolImmediate(X86OperandMem *MemOperand, | 7121 TargetX86Base<TraitsType>::randomizeOrPoolImmediate(X86OperandMem *MemOperand, |
7114 int32_t RegNum) { | 7122 int32_t RegNum) { |
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
7200 return MemOperand; | 7208 return MemOperand; |
7201 Variable *RegTemp = makeReg(IceType_i32); | 7209 Variable *RegTemp = makeReg(IceType_i32); |
7202 IceString Label; | 7210 IceString Label; |
7203 llvm::raw_string_ostream Label_stream(Label); | 7211 llvm::raw_string_ostream Label_stream(Label); |
7204 MemOperand->getOffset()->emitPoolLabel(Label_stream, Ctx); | 7212 MemOperand->getOffset()->emitPoolLabel(Label_stream, Ctx); |
7205 MemOperand->getOffset()->setShouldBePooled(true); | 7213 MemOperand->getOffset()->setShouldBePooled(true); |
7206 constexpr RelocOffsetT SymOffset = 0; | 7214 constexpr RelocOffsetT SymOffset = 0; |
7207 constexpr bool SuppressMangling = true; | 7215 constexpr bool SuppressMangling = true; |
7208 Constant *Symbol = | 7216 Constant *Symbol = |
7209 Ctx->getConstantSym(SymOffset, Label_stream.str(), SuppressMangling); | 7217 Ctx->getConstantSym(SymOffset, Label_stream.str(), SuppressMangling); |
7210 const bool UseNonsfi = Ctx->getFlags().getUseNonsfi(); | 7218 constexpr Variable *NoBase = nullptr; |
7211 Variable *Base = UseNonsfi ? legalizeToReg(GotVar) : nullptr; | |
7212 const bool IsRebased = Base != nullptr; | |
7213 X86OperandMem *SymbolOperand = X86OperandMem::create( | 7219 X86OperandMem *SymbolOperand = X86OperandMem::create( |
7214 Func, MemOperand->getOffset()->getType(), Base, Symbol, IsRebased); | 7220 Func, MemOperand->getOffset()->getType(), NoBase, Symbol); |
7215 _mov(RegTemp, SymbolOperand); | 7221 _mov(RegTemp, SymbolOperand); |
7216 // If we have a base variable here, we should add the lea instruction | 7222 // If we have a base variable here, we should add the lea instruction |
7217 // to add the value of the base variable to RegTemp. If there is no | 7223 // to add the value of the base variable to RegTemp. If there is no |
7218 // base variable, we won't need this lea instruction. | 7224 // base variable, we won't need this lea instruction. |
7219 if (MemOperand->getBase()) { | 7225 if (MemOperand->getBase()) { |
7220 X86OperandMem *CalculateOperand = X86OperandMem::create( | 7226 X86OperandMem *CalculateOperand = X86OperandMem::create( |
7221 Func, MemOperand->getType(), MemOperand->getBase(), nullptr, RegTemp, | 7227 Func, MemOperand->getType(), MemOperand->getBase(), nullptr, RegTemp, |
7222 0, MemOperand->getSegmentRegister()); | 7228 0, MemOperand->getSegmentRegister()); |
7223 _lea(RegTemp, CalculateOperand); | 7229 _lea(RegTemp, CalculateOperand); |
7224 } | 7230 } |
7225 X86OperandMem *NewMemOperand = X86OperandMem::create( | 7231 X86OperandMem *NewMemOperand = X86OperandMem::create( |
7226 Func, MemOperand->getType(), RegTemp, nullptr, MemOperand->getIndex(), | 7232 Func, MemOperand->getType(), RegTemp, nullptr, MemOperand->getIndex(), |
7227 MemOperand->getShift(), MemOperand->getSegmentRegister()); | 7233 MemOperand->getShift(), MemOperand->getSegmentRegister()); |
7228 return NewMemOperand; | 7234 return NewMemOperand; |
7229 } | 7235 } |
7230 } | 7236 } |
7231 } | 7237 } |
7232 } // end of namespace X86NAMESPACE | 7238 } // end of namespace X86NAMESPACE |
7233 } // end of namespace Ice | 7239 } // end of namespace Ice |
7234 | 7240 |
7235 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H | 7241 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H |
OLD | NEW |