OLD | NEW |
1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// | 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// |
2 // | 2 // |
3 // The Subzero Code Generator | 3 // The Subzero Code Generator |
4 // | 4 // |
5 // This file is distributed under the University of Illinois Open Source | 5 // This file is distributed under the University of Illinois Open Source |
6 // License. See LICENSE.TXT for details. | 6 // License. See LICENSE.TXT for details. |
7 // | 7 // |
8 //===----------------------------------------------------------------------===// | 8 //===----------------------------------------------------------------------===// |
9 /// | 9 /// |
10 /// \file | 10 /// \file |
(...skipping 1146 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1157 Hi->setIsArg(); | 1157 Hi->setIsArg(); |
1158 } | 1158 } |
1159 } | 1159 } |
1160 | 1160 |
1161 template <class Machine> | 1161 template <class Machine> |
1162 Operand *TargetX86Base<Machine>::loOperand(Operand *Operand) { | 1162 Operand *TargetX86Base<Machine>::loOperand(Operand *Operand) { |
1163 assert(Operand->getType() == IceType_i64 || | 1163 assert(Operand->getType() == IceType_i64 || |
1164 Operand->getType() == IceType_f64); | 1164 Operand->getType() == IceType_f64); |
1165 if (Operand->getType() != IceType_i64 && Operand->getType() != IceType_f64) | 1165 if (Operand->getType() != IceType_i64 && Operand->getType() != IceType_f64) |
1166 return Operand; | 1166 return Operand; |
1167 if (Variable *Var = llvm::dyn_cast<Variable>(Operand)) { | 1167 if (auto *Var = llvm::dyn_cast<Variable>(Operand)) { |
1168 split64(Var); | 1168 split64(Var); |
1169 return Var->getLo(); | 1169 return Var->getLo(); |
1170 } | 1170 } |
1171 if (ConstantInteger64 *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { | 1171 if (auto *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { |
1172 ConstantInteger32 *ConstInt = llvm::dyn_cast<ConstantInteger32>( | 1172 auto *ConstInt = llvm::dyn_cast<ConstantInteger32>( |
1173 Ctx->getConstantInt32(static_cast<int32_t>(Const->getValue()))); | 1173 Ctx->getConstantInt32(static_cast<int32_t>(Const->getValue()))); |
| 1174 // Check if we need to blind/pool the constant. |
1174 return legalize(ConstInt); | 1175 return legalize(ConstInt); |
1175 } | 1176 } |
1176 if (auto *Mem = llvm::dyn_cast<typename Traits::X86OperandMem>(Operand)) { | 1177 if (auto *Mem = llvm::dyn_cast<typename Traits::X86OperandMem>(Operand)) { |
1177 auto *MemOperand = Traits::X86OperandMem::create( | 1178 auto *MemOperand = Traits::X86OperandMem::create( |
1178 Func, IceType_i32, Mem->getBase(), Mem->getOffset(), Mem->getIndex(), | 1179 Func, IceType_i32, Mem->getBase(), Mem->getOffset(), Mem->getIndex(), |
1179 Mem->getShift(), Mem->getSegmentRegister()); | 1180 Mem->getShift(), Mem->getSegmentRegister()); |
1180 // Test if we should randomize or pool the offset, if so randomize it or | 1181 // Test if we should randomize or pool the offset, if so randomize it or |
1181 // pool it then create mem operand with the blinded/pooled constant. | 1182 // pool it then create mem operand with the blinded/pooled constant. |
1182 // Otherwise, return the mem operand as ordinary mem operand. | 1183 // Otherwise, return the mem operand as ordinary mem operand. |
1183 return legalize(MemOperand); | 1184 return legalize(MemOperand); |
1184 } | 1185 } |
1185 llvm_unreachable("Unsupported operand type"); | 1186 llvm_unreachable("Unsupported operand type"); |
1186 return nullptr; | 1187 return nullptr; |
1187 } | 1188 } |
1188 | 1189 |
1189 template <class Machine> | 1190 template <class Machine> |
1190 Operand *TargetX86Base<Machine>::hiOperand(Operand *Operand) { | 1191 Operand *TargetX86Base<Machine>::hiOperand(Operand *Operand) { |
1191 assert(Operand->getType() == IceType_i64 || | 1192 assert(Operand->getType() == IceType_i64 || |
1192 Operand->getType() == IceType_f64); | 1193 Operand->getType() == IceType_f64); |
1193 if (Operand->getType() != IceType_i64 && Operand->getType() != IceType_f64) | 1194 if (Operand->getType() != IceType_i64 && Operand->getType() != IceType_f64) |
1194 return Operand; | 1195 return Operand; |
1195 if (Variable *Var = llvm::dyn_cast<Variable>(Operand)) { | 1196 if (auto *Var = llvm::dyn_cast<Variable>(Operand)) { |
1196 split64(Var); | 1197 split64(Var); |
1197 return Var->getHi(); | 1198 return Var->getHi(); |
1198 } | 1199 } |
1199 if (ConstantInteger64 *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { | 1200 if (auto *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { |
1200 ConstantInteger32 *ConstInt = llvm::dyn_cast<ConstantInteger32>( | 1201 auto *ConstInt = llvm::dyn_cast<ConstantInteger32>( |
1201 Ctx->getConstantInt32(static_cast<int32_t>(Const->getValue() >> 32))); | 1202 Ctx->getConstantInt32(static_cast<int32_t>(Const->getValue() >> 32))); |
1202 // check if we need to blind/pool the constant | 1203 // Check if we need to blind/pool the constant. |
1203 return legalize(ConstInt); | 1204 return legalize(ConstInt); |
1204 } | 1205 } |
1205 if (auto *Mem = llvm::dyn_cast<typename Traits::X86OperandMem>(Operand)) { | 1206 if (auto *Mem = llvm::dyn_cast<typename Traits::X86OperandMem>(Operand)) { |
1206 Constant *Offset = Mem->getOffset(); | 1207 Constant *Offset = Mem->getOffset(); |
1207 if (Offset == nullptr) { | 1208 if (Offset == nullptr) { |
1208 Offset = Ctx->getConstantInt32(4); | 1209 Offset = Ctx->getConstantInt32(4); |
1209 } else if (ConstantInteger32 *IntOffset = | 1210 } else if (auto *IntOffset = llvm::dyn_cast<ConstantInteger32>(Offset)) { |
1210 llvm::dyn_cast<ConstantInteger32>(Offset)) { | |
1211 Offset = Ctx->getConstantInt32(4 + IntOffset->getValue()); | 1211 Offset = Ctx->getConstantInt32(4 + IntOffset->getValue()); |
1212 } else if (ConstantRelocatable *SymOffset = | 1212 } else if (auto *SymOffset = llvm::dyn_cast<ConstantRelocatable>(Offset)) { |
1213 llvm::dyn_cast<ConstantRelocatable>(Offset)) { | |
1214 assert(!Utils::WouldOverflowAdd(SymOffset->getOffset(), 4)); | 1213 assert(!Utils::WouldOverflowAdd(SymOffset->getOffset(), 4)); |
1215 Offset = | 1214 Offset = |
1216 Ctx->getConstantSym(4 + SymOffset->getOffset(), SymOffset->getName(), | 1215 Ctx->getConstantSym(4 + SymOffset->getOffset(), SymOffset->getName(), |
1217 SymOffset->getSuppressMangling()); | 1216 SymOffset->getSuppressMangling()); |
1218 } | 1217 } |
1219 auto *MemOperand = Traits::X86OperandMem::create( | 1218 auto *MemOperand = Traits::X86OperandMem::create( |
1220 Func, IceType_i32, Mem->getBase(), Offset, Mem->getIndex(), | 1219 Func, IceType_i32, Mem->getBase(), Offset, Mem->getIndex(), |
1221 Mem->getShift(), Mem->getSegmentRegister()); | 1220 Mem->getShift(), Mem->getSegmentRegister()); |
1222 // Test if the Offset is an eligible i32 constants for randomization and | 1221 // Test if the Offset is an eligible i32 constants for randomization and |
1223 // pooling. Blind/pool it if it is. Otherwise return as oridinary mem | 1222 // pooling. Blind/pool it if it is. Otherwise return as oridinary mem |
(...skipping 1222 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2446 if (isVectorType(Dest->getType())) { | 2445 if (isVectorType(Dest->getType())) { |
2447 // onemask = materialize(1,1,...); dst = src & onemask | 2446 // onemask = materialize(1,1,...); dst = src & onemask |
2448 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); | 2447 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); |
2449 Type Src0Ty = Src0RM->getType(); | 2448 Type Src0Ty = Src0RM->getType(); |
2450 Variable *OneMask = makeVectorOfOnes(Src0Ty); | 2449 Variable *OneMask = makeVectorOfOnes(Src0Ty); |
2451 Variable *T = makeReg(Dest->getType()); | 2450 Variable *T = makeReg(Dest->getType()); |
2452 _movp(T, Src0RM); | 2451 _movp(T, Src0RM); |
2453 _pand(T, OneMask); | 2452 _pand(T, OneMask); |
2454 _movp(Dest, T); | 2453 _movp(Dest, T); |
2455 } else { | 2454 } else { |
2456 Operand *Src0 = Inst->getSrc(0); | 2455 Operand *Src0 = legalizeUndef(Inst->getSrc(0)); |
2457 if (Src0->getType() == IceType_i64) | 2456 if (Src0->getType() == IceType_i64) |
2458 Src0 = loOperand(Src0); | 2457 Src0 = loOperand(Src0); |
2459 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); | 2458 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); |
2460 // t1 = trunc Src0RM; Dest = t1 | 2459 // t1 = trunc Src0RM; Dest = t1 |
2461 Variable *T = nullptr; | 2460 Variable *T = nullptr; |
2462 _mov(T, Src0RM); | 2461 _mov(T, Src0RM); |
2463 if (Dest->getType() == IceType_i1) | 2462 if (Dest->getType() == IceType_i1) |
2464 _and(T, Ctx->getConstantInt1(1)); | 2463 _and(T, Ctx->getConstantInt1(1)); |
2465 _mov(Dest, T); | 2464 _mov(Dest, T); |
2466 } | 2465 } |
(...skipping 787 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3254 const InstIntrinsicCall *Instr) { | 3253 const InstIntrinsicCall *Instr) { |
3255 switch (Intrinsics::IntrinsicID ID = Instr->getIntrinsicInfo().ID) { | 3254 switch (Intrinsics::IntrinsicID ID = Instr->getIntrinsicInfo().ID) { |
3256 case Intrinsics::AtomicCmpxchg: { | 3255 case Intrinsics::AtomicCmpxchg: { |
3257 if (!Intrinsics::isMemoryOrderValid( | 3256 if (!Intrinsics::isMemoryOrderValid( |
3258 ID, getConstantMemoryOrder(Instr->getArg(3)), | 3257 ID, getConstantMemoryOrder(Instr->getArg(3)), |
3259 getConstantMemoryOrder(Instr->getArg(4)))) { | 3258 getConstantMemoryOrder(Instr->getArg(4)))) { |
3260 Func->setError("Unexpected memory ordering for AtomicCmpxchg"); | 3259 Func->setError("Unexpected memory ordering for AtomicCmpxchg"); |
3261 return; | 3260 return; |
3262 } | 3261 } |
3263 Variable *DestPrev = Instr->getDest(); | 3262 Variable *DestPrev = Instr->getDest(); |
3264 Operand *PtrToMem = Instr->getArg(0); | 3263 Operand *PtrToMem = legalize(Instr->getArg(0)); |
3265 Operand *Expected = Instr->getArg(1); | 3264 Operand *Expected = legalize(Instr->getArg(1)); |
3266 Operand *Desired = Instr->getArg(2); | 3265 Operand *Desired = legalize(Instr->getArg(2)); |
3267 if (tryOptimizedCmpxchgCmpBr(DestPrev, PtrToMem, Expected, Desired)) | 3266 if (tryOptimizedCmpxchgCmpBr(DestPrev, PtrToMem, Expected, Desired)) |
3268 return; | 3267 return; |
3269 lowerAtomicCmpxchg(DestPrev, PtrToMem, Expected, Desired); | 3268 lowerAtomicCmpxchg(DestPrev, PtrToMem, Expected, Desired); |
3270 return; | 3269 return; |
3271 } | 3270 } |
3272 case Intrinsics::AtomicFence: | 3271 case Intrinsics::AtomicFence: |
3273 if (!Intrinsics::isMemoryOrderValid( | 3272 if (!Intrinsics::isMemoryOrderValid( |
3274 ID, getConstantMemoryOrder(Instr->getArg(0)))) { | 3273 ID, getConstantMemoryOrder(Instr->getArg(0)))) { |
3275 Func->setError("Unexpected memory ordering for AtomicFence"); | 3274 Func->setError("Unexpected memory ordering for AtomicFence"); |
3276 return; | 3275 return; |
(...skipping 113 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3390 lowerStore(Store); | 3389 lowerStore(Store); |
3391 _mfence(); | 3390 _mfence(); |
3392 return; | 3391 return; |
3393 } | 3392 } |
3394 case Intrinsics::Bswap: { | 3393 case Intrinsics::Bswap: { |
3395 Variable *Dest = Instr->getDest(); | 3394 Variable *Dest = Instr->getDest(); |
3396 Operand *Val = Instr->getArg(0); | 3395 Operand *Val = Instr->getArg(0); |
3397 // In 32-bit mode, bswap only works on 32-bit arguments, and the | 3396 // In 32-bit mode, bswap only works on 32-bit arguments, and the |
3398 // argument must be a register. Use rotate left for 16-bit bswap. | 3397 // argument must be a register. Use rotate left for 16-bit bswap. |
3399 if (Val->getType() == IceType_i64) { | 3398 if (Val->getType() == IceType_i64) { |
| 3399 Val = legalizeUndef(Val); |
3400 Variable *T_Lo = legalizeToVar(loOperand(Val)); | 3400 Variable *T_Lo = legalizeToVar(loOperand(Val)); |
3401 Variable *T_Hi = legalizeToVar(hiOperand(Val)); | 3401 Variable *T_Hi = legalizeToVar(hiOperand(Val)); |
3402 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 3402 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
3403 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 3403 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
3404 _bswap(T_Lo); | 3404 _bswap(T_Lo); |
3405 _bswap(T_Hi); | 3405 _bswap(T_Hi); |
3406 _mov(DestLo, T_Hi); | 3406 _mov(DestLo, T_Hi); |
3407 _mov(DestHi, T_Lo); | 3407 _mov(DestHi, T_Lo); |
3408 } else if (Val->getType() == IceType_i32) { | 3408 } else if (Val->getType() == IceType_i32) { |
3409 Variable *T = legalizeToVar(Val); | 3409 Variable *T = legalizeToVar(Val); |
3410 _bswap(T); | 3410 _bswap(T); |
3411 _mov(Dest, T); | 3411 _mov(Dest, T); |
3412 } else { | 3412 } else { |
3413 assert(Val->getType() == IceType_i16); | 3413 assert(Val->getType() == IceType_i16); |
3414 Val = legalize(Val); | |
3415 Constant *Eight = Ctx->getConstantInt16(8); | 3414 Constant *Eight = Ctx->getConstantInt16(8); |
3416 Variable *T = nullptr; | 3415 Variable *T = nullptr; |
| 3416 Val = legalize(Val); |
3417 _mov(T, Val); | 3417 _mov(T, Val); |
3418 _rol(T, Eight); | 3418 _rol(T, Eight); |
3419 _mov(Dest, T); | 3419 _mov(Dest, T); |
3420 } | 3420 } |
3421 return; | 3421 return; |
3422 } | 3422 } |
3423 case Intrinsics::Ctpop: { | 3423 case Intrinsics::Ctpop: { |
3424 Variable *Dest = Instr->getDest(); | 3424 Variable *Dest = Instr->getDest(); |
3425 Operand *Val = Instr->getArg(0); | 3425 Operand *Val = Instr->getArg(0); |
3426 InstCall *Call = makeHelperCall(isInt32Asserting32Or64(Val->getType()) | 3426 InstCall *Call = makeHelperCall(isInt32Asserting32Or64(Val->getType()) |
(...skipping 977 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
4404 } | 4404 } |
4405 // mov t, SrcF; cmov_cond t, SrcT; mov dest, t | 4405 // mov t, SrcF; cmov_cond t, SrcT; mov dest, t |
4406 // But if SrcT is immediate, we might be able to do better, as | 4406 // But if SrcT is immediate, we might be able to do better, as |
4407 // the cmov instruction doesn't allow an immediate operand: | 4407 // the cmov instruction doesn't allow an immediate operand: |
4408 // mov t, SrcT; cmov_!cond t, SrcF; mov dest, t | 4408 // mov t, SrcT; cmov_!cond t, SrcF; mov dest, t |
4409 if (llvm::isa<Constant>(SrcT) && !llvm::isa<Constant>(SrcF)) { | 4409 if (llvm::isa<Constant>(SrcT) && !llvm::isa<Constant>(SrcF)) { |
4410 std::swap(SrcT, SrcF); | 4410 std::swap(SrcT, SrcF); |
4411 Cond = InstX86Base<Machine>::getOppositeCondition(Cond); | 4411 Cond = InstX86Base<Machine>::getOppositeCondition(Cond); |
4412 } | 4412 } |
4413 if (DestTy == IceType_i64) { | 4413 if (DestTy == IceType_i64) { |
| 4414 SrcT = legalizeUndef(SrcT); |
| 4415 SrcF = legalizeUndef(SrcF); |
4414 // Set the low portion. | 4416 // Set the low portion. |
4415 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 4417 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
4416 Variable *TLo = nullptr; | 4418 Variable *TLo = nullptr; |
4417 Operand *SrcFLo = legalize(loOperand(SrcF)); | 4419 Operand *SrcFLo = legalize(loOperand(SrcF)); |
4418 _mov(TLo, SrcFLo); | 4420 _mov(TLo, SrcFLo); |
4419 Operand *SrcTLo = legalize(loOperand(SrcT), Legal_Reg | Legal_Mem); | 4421 Operand *SrcTLo = legalize(loOperand(SrcT), Legal_Reg | Legal_Mem); |
4420 _cmov(TLo, SrcTLo, Cond); | 4422 _cmov(TLo, SrcTLo, Cond); |
4421 _mov(DestLo, TLo); | 4423 _mov(DestLo, TLo); |
4422 // Set the high portion. | 4424 // Set the high portion. |
4423 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 4425 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
(...skipping 17 matching lines...) Expand all Loading... |
4441 | 4443 |
4442 template <class Machine> | 4444 template <class Machine> |
4443 void TargetX86Base<Machine>::lowerStore(const InstStore *Inst) { | 4445 void TargetX86Base<Machine>::lowerStore(const InstStore *Inst) { |
4444 Operand *Value = Inst->getData(); | 4446 Operand *Value = Inst->getData(); |
4445 Operand *Addr = Inst->getAddr(); | 4447 Operand *Addr = Inst->getAddr(); |
4446 typename Traits::X86OperandMem *NewAddr = | 4448 typename Traits::X86OperandMem *NewAddr = |
4447 formMemoryOperand(Addr, Value->getType()); | 4449 formMemoryOperand(Addr, Value->getType()); |
4448 Type Ty = NewAddr->getType(); | 4450 Type Ty = NewAddr->getType(); |
4449 | 4451 |
4450 if (Ty == IceType_i64) { | 4452 if (Ty == IceType_i64) { |
4451 Value = legalize(Value); | 4453 Value = legalizeUndef(Value); |
4452 Operand *ValueHi = legalize(hiOperand(Value), Legal_Reg | Legal_Imm); | 4454 Operand *ValueHi = legalize(hiOperand(Value), Legal_Reg | Legal_Imm); |
4453 Operand *ValueLo = legalize(loOperand(Value), Legal_Reg | Legal_Imm); | 4455 Operand *ValueLo = legalize(loOperand(Value), Legal_Reg | Legal_Imm); |
4454 _store(ValueHi, | 4456 _store(ValueHi, |
4455 llvm::cast<typename Traits::X86OperandMem>(hiOperand(NewAddr))); | 4457 llvm::cast<typename Traits::X86OperandMem>(hiOperand(NewAddr))); |
4456 _store(ValueLo, | 4458 _store(ValueLo, |
4457 llvm::cast<typename Traits::X86OperandMem>(loOperand(NewAddr))); | 4459 llvm::cast<typename Traits::X86OperandMem>(loOperand(NewAddr))); |
4458 } else if (isVectorType(Ty)) { | 4460 } else if (isVectorType(Ty)) { |
4459 _storep(legalizeToVar(Value), NewAddr); | 4461 _storep(legalizeToVar(Value), NewAddr); |
4460 } else { | 4462 } else { |
4461 Value = legalize(Value, Legal_Reg | Legal_Imm); | 4463 Value = legalize(Value, Legal_Reg | Legal_Imm); |
(...skipping 28 matching lines...) Expand all Loading... |
4490 } | 4492 } |
4491 } | 4493 } |
4492 | 4494 |
4493 template <class Machine> | 4495 template <class Machine> |
4494 void TargetX86Base<Machine>::lowerSwitch(const InstSwitch *Inst) { | 4496 void TargetX86Base<Machine>::lowerSwitch(const InstSwitch *Inst) { |
4495 // This implements the most naive possible lowering. | 4497 // This implements the most naive possible lowering. |
4496 // cmp a,val[0]; jeq label[0]; cmp a,val[1]; jeq label[1]; ... jmp default | 4498 // cmp a,val[0]; jeq label[0]; cmp a,val[1]; jeq label[1]; ... jmp default |
4497 Operand *Src0 = Inst->getComparison(); | 4499 Operand *Src0 = Inst->getComparison(); |
4498 SizeT NumCases = Inst->getNumCases(); | 4500 SizeT NumCases = Inst->getNumCases(); |
4499 if (Src0->getType() == IceType_i64) { | 4501 if (Src0->getType() == IceType_i64) { |
4500 Src0 = legalize(Src0); // get Base/Index into physical registers | 4502 Src0 = legalizeUndef(Src0); |
4501 Operand *Src0Lo = loOperand(Src0); | 4503 Operand *Src0Lo = loOperand(Src0); |
4502 Operand *Src0Hi = hiOperand(Src0); | 4504 Operand *Src0Hi = hiOperand(Src0); |
4503 if (NumCases >= 2) { | 4505 if (NumCases >= 2) { |
4504 Src0Lo = legalizeToVar(Src0Lo); | 4506 Src0Lo = legalizeToVar(Src0Lo); |
4505 Src0Hi = legalizeToVar(Src0Hi); | 4507 Src0Hi = legalizeToVar(Src0Hi); |
4506 } else { | 4508 } else { |
4507 Src0Lo = legalize(Src0Lo, Legal_Reg | Legal_Mem); | 4509 Src0Lo = legalize(Src0Lo, Legal_Reg | Legal_Mem); |
4508 Src0Hi = legalize(Src0Hi, Legal_Reg | Legal_Mem); | 4510 Src0Hi = legalize(Src0Hi, Legal_Reg | Legal_Mem); |
4509 } | 4511 } |
4510 for (SizeT I = 0; I < NumCases; ++I) { | 4512 for (SizeT I = 0; I < NumCases; ++I) { |
(...skipping 95 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
4606 // still there, either because the value being stored is used beyond | 4608 // still there, either because the value being stored is used beyond |
4607 // the Store instruction, or because dead code elimination did not | 4609 // the Store instruction, or because dead code elimination did not |
4608 // happen. In either case, we cancel RMW lowering (and the caller | 4610 // happen. In either case, we cancel RMW lowering (and the caller |
4609 // deletes the RMW instruction). | 4611 // deletes the RMW instruction). |
4610 if (!RMW->isLastUse(RMW->getBeacon())) | 4612 if (!RMW->isLastUse(RMW->getBeacon())) |
4611 return; | 4613 return; |
4612 Operand *Src = RMW->getData(); | 4614 Operand *Src = RMW->getData(); |
4613 Type Ty = Src->getType(); | 4615 Type Ty = Src->getType(); |
4614 typename Traits::X86OperandMem *Addr = formMemoryOperand(RMW->getAddr(), Ty); | 4616 typename Traits::X86OperandMem *Addr = formMemoryOperand(RMW->getAddr(), Ty); |
4615 if (Ty == IceType_i64) { | 4617 if (Ty == IceType_i64) { |
| 4618 Src = legalizeUndef(Src); |
4616 Operand *SrcLo = legalize(loOperand(Src), Legal_Reg | Legal_Imm); | 4619 Operand *SrcLo = legalize(loOperand(Src), Legal_Reg | Legal_Imm); |
4617 Operand *SrcHi = legalize(hiOperand(Src), Legal_Reg | Legal_Imm); | 4620 Operand *SrcHi = legalize(hiOperand(Src), Legal_Reg | Legal_Imm); |
4618 typename Traits::X86OperandMem *AddrLo = | 4621 typename Traits::X86OperandMem *AddrLo = |
4619 llvm::cast<typename Traits::X86OperandMem>(loOperand(Addr)); | 4622 llvm::cast<typename Traits::X86OperandMem>(loOperand(Addr)); |
4620 typename Traits::X86OperandMem *AddrHi = | 4623 typename Traits::X86OperandMem *AddrHi = |
4621 llvm::cast<typename Traits::X86OperandMem>(hiOperand(Addr)); | 4624 llvm::cast<typename Traits::X86OperandMem>(hiOperand(Addr)); |
4622 switch (RMW->getOp()) { | 4625 switch (RMW->getOp()) { |
4623 default: | 4626 default: |
4624 // TODO(stichnot): Implement other arithmetic operators. | 4627 // TODO(stichnot): Implement other arithmetic operators. |
4625 break; | 4628 break; |
(...skipping 75 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
4701 continue; | 4704 continue; |
4702 Variable *Dest = Phi->getDest(); | 4705 Variable *Dest = Phi->getDest(); |
4703 if (Dest->getType() == IceType_i64) { | 4706 if (Dest->getType() == IceType_i64) { |
4704 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 4707 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
4705 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 4708 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
4706 InstPhi *PhiLo = InstPhi::create(Func, Phi->getSrcSize(), DestLo); | 4709 InstPhi *PhiLo = InstPhi::create(Func, Phi->getSrcSize(), DestLo); |
4707 InstPhi *PhiHi = InstPhi::create(Func, Phi->getSrcSize(), DestHi); | 4710 InstPhi *PhiHi = InstPhi::create(Func, Phi->getSrcSize(), DestHi); |
4708 for (SizeT I = 0; I < Phi->getSrcSize(); ++I) { | 4711 for (SizeT I = 0; I < Phi->getSrcSize(); ++I) { |
4709 Operand *Src = Phi->getSrc(I); | 4712 Operand *Src = Phi->getSrc(I); |
4710 CfgNode *Label = Phi->getLabel(I); | 4713 CfgNode *Label = Phi->getLabel(I); |
4711 if (llvm::isa<ConstantUndef>(Src)) | 4714 Src = legalizeUndef(Src); |
4712 Src = Ctx->getConstantZero(Dest->getType()); | |
4713 PhiLo->addArgument(loOperand(Src), Label); | 4715 PhiLo->addArgument(loOperand(Src), Label); |
4714 PhiHi->addArgument(hiOperand(Src), Label); | 4716 PhiHi->addArgument(hiOperand(Src), Label); |
4715 } | 4717 } |
4716 Node->getPhis().push_back(PhiLo); | 4718 Node->getPhis().push_back(PhiLo); |
4717 Node->getPhis().push_back(PhiHi); | 4719 Node->getPhis().push_back(PhiHi); |
4718 Phi->setDeleted(); | 4720 Phi->setDeleted(); |
4719 } | 4721 } |
4720 } | 4722 } |
4721 } | 4723 } |
4722 | 4724 |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
4784 // remove Src from the set of available registers. Iteration is | 4786 // remove Src from the set of available registers. Iteration is |
4785 // done backwards to enable incremental updates of the available | 4787 // done backwards to enable incremental updates of the available |
4786 // register set, and the lowered instruction numbers may be out of | 4788 // register set, and the lowered instruction numbers may be out of |
4787 // order, but that can be worked around by renumbering the block | 4789 // order, but that can be worked around by renumbering the block |
4788 // afterwards if necessary. | 4790 // afterwards if necessary. |
4789 for (const Inst &I : reverse_range(Assignments)) { | 4791 for (const Inst &I : reverse_range(Assignments)) { |
4790 Context.rewind(); | 4792 Context.rewind(); |
4791 auto Assign = llvm::dyn_cast<InstAssign>(&I); | 4793 auto Assign = llvm::dyn_cast<InstAssign>(&I); |
4792 Variable *Dest = Assign->getDest(); | 4794 Variable *Dest = Assign->getDest(); |
4793 | 4795 |
4794 // If the source operand is ConstantUndef, do not legalize it. | 4796 // If the source operand is ConstantUndef, do not legalize it. In function |
4795 // In function test_split_undef_int_vec, the advanced phi | 4797 // test_split_undef_int_vec, the advanced phi lowering process will find an |
4796 // lowering process will find an assignment of undefined | 4798 // assignment of undefined vector. This vector, as the Src here, will crash |
4797 // vector. This vector, as the Src here, will crash if it | 4799 // if it go through legalize(). legalize() will create a new variable with |
4798 // go through legalize(). legalize() will create new variable | 4800 // makeVectorOfZeros(), but this new variable will be assigned a stack |
4799 // with makeVectorOfZeros(), but this new variable will be | 4801 // slot. This will fail with pxor(Var, Var) because it is an illegal |
4800 // assigned a stack slot. This will fail the assertion in | 4802 // instruction form. Note this failure is irrelevant to randomization or |
4801 // IceInstX8632.cpp:789, as XmmEmitterRegOp() complain: | 4803 // pooling of constants. So, we do not call legalize() to add pool label |
4802 // Var->hasReg() fails. Note this failure is irrelevant to | 4804 // for the src operands of phi assignment instructions. Instead, we |
4803 // randomization or pooling of constants. | 4805 // manually add pool label for constant float and constant double values |
4804 // So, we do not call legalize() to add pool label for the | 4806 // here. Note going through legalize() does not affect the testing results |
4805 // src operands of phi assignment instructions. | 4807 // of SPEC2K and xtests. |
4806 // Instead, we manually add pool label for constant float and | |
4807 // constant double values here. | |
4808 // Note going through legalize() does not affect the testing | |
4809 // results of SPEC2K and xtests. | |
4810 Operand *Src = Assign->getSrc(0); | 4808 Operand *Src = Assign->getSrc(0); |
4811 if (!llvm::isa<ConstantUndef>(Assign->getSrc(0))) { | 4809 if (!llvm::isa<ConstantUndef>(Assign->getSrc(0))) { |
4812 Src = legalize(Src); | 4810 Src = legalize(Src); |
4813 } | 4811 } |
4814 | 4812 |
4815 Variable *SrcVar = llvm::dyn_cast<Variable>(Src); | 4813 Variable *SrcVar = llvm::dyn_cast<Variable>(Src); |
4816 // Use normal assignment lowering, except lower mem=mem specially | 4814 // Use normal assignment lowering, except lower mem=mem specially |
4817 // so we can register-allocate at the same time. | 4815 // so we can register-allocate at the same time. |
4818 if (!isMemoryOperand(Dest) || !isMemoryOperand(Src)) { | 4816 if (!isMemoryOperand(Dest) || !isMemoryOperand(Src)) { |
4819 lowerAssign(Assign); | 4817 lowerAssign(Assign); |
(...skipping 202 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5022 // For all Memory Operands, we do randomization/pooling here | 5020 // For all Memory Operands, we do randomization/pooling here |
5023 From = randomizeOrPoolImmediate(Mem); | 5021 From = randomizeOrPoolImmediate(Mem); |
5024 | 5022 |
5025 if (!(Allowed & Legal_Mem)) { | 5023 if (!(Allowed & Legal_Mem)) { |
5026 From = copyToReg(From, RegNum); | 5024 From = copyToReg(From, RegNum); |
5027 } | 5025 } |
5028 return From; | 5026 return From; |
5029 } | 5027 } |
5030 if (auto *Const = llvm::dyn_cast<Constant>(From)) { | 5028 if (auto *Const = llvm::dyn_cast<Constant>(From)) { |
5031 if (llvm::isa<ConstantUndef>(Const)) { | 5029 if (llvm::isa<ConstantUndef>(Const)) { |
5032 // Lower undefs to zero. Another option is to lower undefs to an | 5030 From = legalizeUndef(Const, RegNum); |
5033 // uninitialized register; however, using an uninitialized register | |
5034 // results in less predictable code. | |
5035 // | |
5036 // If in the future the implementation is changed to lower undef | |
5037 // values to uninitialized registers, a FakeDef will be needed: | |
5038 // Context.insert(InstFakeDef::create(Func, Reg)); | |
5039 // This is in order to ensure that the live range of Reg is not | |
5040 // overestimated. If the constant being lowered is a 64 bit value, | |
5041 // then the result should be split and the lo and hi components will | |
5042 // need to go in uninitialized registers. | |
5043 if (isVectorType(Ty)) | 5031 if (isVectorType(Ty)) |
5044 return makeVectorOfZeros(Ty, RegNum); | 5032 return From; |
5045 Const = Ctx->getConstantZero(Ty); | 5033 Const = llvm::cast<Constant>(From); |
5046 From = Const; | |
5047 } | 5034 } |
5048 // There should be no constants of vector type (other than undef). | 5035 // There should be no constants of vector type (other than undef). |
5049 assert(!isVectorType(Ty)); | 5036 assert(!isVectorType(Ty)); |
5050 | 5037 |
5051 // If the operand is an 32 bit constant integer, we should check | 5038 // If the operand is an 32 bit constant integer, we should check |
5052 // whether we need to randomize it or pool it. | 5039 // whether we need to randomize it or pool it. |
5053 if (ConstantInteger32 *C = llvm::dyn_cast<ConstantInteger32>(Const)) { | 5040 if (ConstantInteger32 *C = llvm::dyn_cast<ConstantInteger32>(Const)) { |
5054 Operand *NewConst = randomizeOrPoolImmediate(C, RegNum); | 5041 Operand *NewConst = randomizeOrPoolImmediate(C, RegNum); |
5055 if (NewConst != Const) { | 5042 if (NewConst != Const) { |
5056 return NewConst; | 5043 return NewConst; |
(...skipping 41 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5098 llvm_unreachable("Unhandled operand kind in legalize()"); | 5085 llvm_unreachable("Unhandled operand kind in legalize()"); |
5099 return From; | 5086 return From; |
5100 } | 5087 } |
5101 | 5088 |
5102 /// Provide a trivial wrapper to legalize() for this common usage. | 5089 /// Provide a trivial wrapper to legalize() for this common usage. |
5103 template <class Machine> | 5090 template <class Machine> |
5104 Variable *TargetX86Base<Machine>::legalizeToVar(Operand *From, int32_t RegNum) { | 5091 Variable *TargetX86Base<Machine>::legalizeToVar(Operand *From, int32_t RegNum) { |
5105 return llvm::cast<Variable>(legalize(From, Legal_Reg, RegNum)); | 5092 return llvm::cast<Variable>(legalize(From, Legal_Reg, RegNum)); |
5106 } | 5093 } |
5107 | 5094 |
| 5095 /// Legalize undef values to concrete values. |
| 5096 template <class Machine> |
| 5097 Operand *TargetX86Base<Machine>::legalizeUndef(Operand *From, int32_t RegNum) { |
| 5098 Type Ty = From->getType(); |
| 5099 if (llvm::isa<ConstantUndef>(From)) { |
| 5100 // Lower undefs to zero. Another option is to lower undefs to an |
| 5101 // uninitialized register; however, using an uninitialized register |
| 5102 // results in less predictable code. |
| 5103 // |
| 5104 // If in the future the implementation is changed to lower undef |
| 5105 // values to uninitialized registers, a FakeDef will be needed: |
| 5106 // Context.insert(InstFakeDef::create(Func, Reg)); |
| 5107 // This is in order to ensure that the live range of Reg is not |
| 5108 // overestimated. If the constant being lowered is a 64 bit value, |
| 5109 // then the result should be split and the lo and hi components will |
| 5110 // need to go in uninitialized registers. |
| 5111 if (isVectorType(Ty)) |
| 5112 return makeVectorOfZeros(Ty, RegNum); |
| 5113 return Ctx->getConstantZero(Ty); |
| 5114 } |
| 5115 return From; |
| 5116 } |
| 5117 |
5108 /// For the cmp instruction, if Src1 is an immediate, or known to be a | 5118 /// For the cmp instruction, if Src1 is an immediate, or known to be a |
5109 /// physical register, we can allow Src0 to be a memory operand. | 5119 /// physical register, we can allow Src0 to be a memory operand. |
5110 /// Otherwise, Src0 must be copied into a physical register. | 5120 /// Otherwise, Src0 must be copied into a physical register. |
5111 /// (Actually, either Src0 or Src1 can be chosen for the physical | 5121 /// (Actually, either Src0 or Src1 can be chosen for the physical |
5112 /// register, but unfortunately we have to commit to one or the other | 5122 /// register, but unfortunately we have to commit to one or the other |
5113 /// before register allocation.) | 5123 /// before register allocation.) |
5114 template <class Machine> | 5124 template <class Machine> |
5115 Operand *TargetX86Base<Machine>::legalizeSrc0ForCmp(Operand *Src0, | 5125 Operand *TargetX86Base<Machine>::legalizeSrc0ForCmp(Operand *Src0, |
5116 Operand *Src1) { | 5126 Operand *Src1) { |
5117 bool IsSrc1ImmOrReg = false; | 5127 bool IsSrc1ImmOrReg = false; |
5118 if (llvm::isa<Constant>(Src1)) { | 5128 if (llvm::isa<Constant>(Src1)) { |
5119 IsSrc1ImmOrReg = true; | 5129 IsSrc1ImmOrReg = true; |
5120 } else if (Variable *Var = llvm::dyn_cast<Variable>(Src1)) { | 5130 } else if (auto *Var = llvm::dyn_cast<Variable>(Src1)) { |
5121 if (Var->hasReg()) | 5131 if (Var->hasReg()) |
5122 IsSrc1ImmOrReg = true; | 5132 IsSrc1ImmOrReg = true; |
5123 } | 5133 } |
5124 return legalize(Src0, IsSrc1ImmOrReg ? (Legal_Reg | Legal_Mem) : Legal_Reg); | 5134 return legalize(Src0, IsSrc1ImmOrReg ? (Legal_Reg | Legal_Mem) : Legal_Reg); |
5125 } | 5135 } |
5126 | 5136 |
5127 template <class Machine> | 5137 template <class Machine> |
5128 typename TargetX86Base<Machine>::Traits::X86OperandMem * | 5138 typename TargetX86Base<Machine>::Traits::X86OperandMem * |
5129 TargetX86Base<Machine>::formMemoryOperand(Operand *Opnd, Type Ty, | 5139 TargetX86Base<Machine>::formMemoryOperand(Operand *Opnd, Type Ty, |
5130 bool DoLegalize) { | 5140 bool DoLegalize) { |
5131 auto *Mem = llvm::dyn_cast<typename Traits::X86OperandMem>(Opnd); | 5141 auto *Mem = llvm::dyn_cast<typename Traits::X86OperandMem>(Opnd); |
5132 // It may be the case that address mode optimization already creates an | 5142 // It may be the case that address mode optimization already creates an |
5133 // Traits::X86OperandMem, so in that case it wouldn't need another level of | 5143 // Traits::X86OperandMem, so in that case it wouldn't need another level of |
5134 // transformation. | 5144 // transformation. |
5135 if (!Mem) { | 5145 if (!Mem) { |
5136 Variable *Base = llvm::dyn_cast<Variable>(Opnd); | 5146 Variable *Base = llvm::dyn_cast<Variable>(Opnd); |
5137 Constant *Offset = llvm::dyn_cast<Constant>(Opnd); | 5147 Constant *Offset = llvm::dyn_cast<Constant>(Opnd); |
5138 assert(Base || Offset); | 5148 assert(Base || Offset); |
5139 if (Offset) { | 5149 if (Offset) { |
5140 // During memory operand building, we do not blind or pool | 5150 // During memory operand building, we do not blind or pool |
5141 // the constant offset, we will work on the whole memory | 5151 // the constant offset, we will work on the whole memory |
5142 // operand later as one entity later, this save one instruction. | 5152 // operand later as one entity later, this save one instruction. |
5143 // By turning blinding and pooling off, we guarantee | 5153 // By turning blinding and pooling off, we guarantee |
5144 // legalize(Offset) will return a constant*. | 5154 // legalize(Offset) will return a Constant*. |
5145 { | 5155 { |
5146 BoolFlagSaver B(RandomizationPoolingPaused, true); | 5156 BoolFlagSaver B(RandomizationPoolingPaused, true); |
5147 | 5157 |
5148 Offset = llvm::cast<Constant>(legalize(Offset)); | 5158 Offset = llvm::cast<Constant>(legalize(Offset)); |
5149 } | 5159 } |
5150 | 5160 |
5151 assert(llvm::isa<ConstantInteger32>(Offset) || | 5161 assert(llvm::isa<ConstantInteger32>(Offset) || |
5152 llvm::isa<ConstantRelocatable>(Offset)); | 5162 llvm::isa<ConstantRelocatable>(Offset)); |
5153 } | 5163 } |
5154 Mem = Traits::X86OperandMem::create(Func, Ty, Base, Offset); | 5164 Mem = Traits::X86OperandMem::create(Func, Ty, Base, Offset); |
(...skipping 195 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5350 // assigned, we should add this _set_dest_nonkillable() | 5360 // assigned, we should add this _set_dest_nonkillable() |
5351 if (RegNum != Variable::NoRegister) | 5361 if (RegNum != Variable::NoRegister) |
5352 _set_dest_nonkillable(); | 5362 _set_dest_nonkillable(); |
5353 | 5363 |
5354 typename Traits::X86OperandMem *NewMemOperand = | 5364 typename Traits::X86OperandMem *NewMemOperand = |
5355 Traits::X86OperandMem::create(Func, MemOperand->getType(), RegTemp, | 5365 Traits::X86OperandMem::create(Func, MemOperand->getType(), RegTemp, |
5356 Mask2, MemOperand->getIndex(), | 5366 Mask2, MemOperand->getIndex(), |
5357 MemOperand->getShift(), | 5367 MemOperand->getShift(), |
5358 MemOperand->getSegmentRegister()); | 5368 MemOperand->getSegmentRegister()); |
5359 | 5369 |
5360 // Label this memory operand as randomize, so we won't randomize it | 5370 // Label this memory operand as randomized, so we won't randomize it |
5361 // again in case we call legalize() mutiple times on this memory | 5371 // again in case we call legalize() multiple times on this memory |
5362 // operand. | 5372 // operand. |
5363 NewMemOperand->setRandomized(true); | 5373 NewMemOperand->setRandomized(true); |
5364 return NewMemOperand; | 5374 return NewMemOperand; |
5365 } | 5375 } |
5366 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_Pool) { | 5376 if (Ctx->getFlags().getRandomizeAndPoolImmediatesOption() == RPI_Pool) { |
5367 // pool the constant offset | 5377 // pool the constant offset |
5368 // FROM: | 5378 // FROM: |
5369 // offset[base, index, shift] | 5379 // offset[base, index, shift] |
5370 // TO: | 5380 // TO: |
5371 // insert: mov $label, RegTemp | 5381 // insert: mov $label, RegTemp |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
5417 } | 5427 } |
5418 // the offset is not eligible for blinding or pooling, return the original | 5428 // the offset is not eligible for blinding or pooling, return the original |
5419 // mem operand | 5429 // mem operand |
5420 return MemOperand; | 5430 return MemOperand; |
5421 } | 5431 } |
5422 | 5432 |
5423 } // end of namespace X86Internal | 5433 } // end of namespace X86Internal |
5424 } // end of namespace Ice | 5434 } // end of namespace Ice |
5425 | 5435 |
5426 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H | 5436 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H |
OLD | NEW |