| OLD | NEW |
| 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// | 1 //===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// |
| 2 // | 2 // |
| 3 // The Subzero Code Generator | 3 // The Subzero Code Generator |
| 4 // | 4 // |
| 5 // This file is distributed under the University of Illinois Open Source | 5 // This file is distributed under the University of Illinois Open Source |
| 6 // License. See LICENSE.TXT for details. | 6 // License. See LICENSE.TXT for details. |
| 7 // | 7 // |
| 8 //===----------------------------------------------------------------------===// | 8 //===----------------------------------------------------------------------===// |
| 9 /// | 9 /// |
| 10 /// \file | 10 /// \file |
| (...skipping 2210 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2221 _div(T, Src1, T_edx); | 2221 _div(T, Src1, T_edx); |
| 2222 _mov(Dest, T); | 2222 _mov(Dest, T); |
| 2223 } break; | 2223 } break; |
| 2224 case InstArithmetic::Sdiv: | 2224 case InstArithmetic::Sdiv: |
| 2225 // TODO(stichnot): Enable this after doing better performance and cross | 2225 // TODO(stichnot): Enable this after doing better performance and cross |
| 2226 // testing. | 2226 // testing. |
| 2227 if (false && Ctx->getFlags().getOptLevel() >= Opt_1) { | 2227 if (false && Ctx->getFlags().getOptLevel() >= Opt_1) { |
| 2228 // Optimize division by constant power of 2, but not for Om1 or O0, just | 2228 // Optimize division by constant power of 2, but not for Om1 or O0, just |
| 2229 // to keep things simple there. | 2229 // to keep things simple there. |
| 2230 if (auto *C = llvm::dyn_cast<ConstantInteger32>(Src1)) { | 2230 if (auto *C = llvm::dyn_cast<ConstantInteger32>(Src1)) { |
| 2231 int32_t Divisor = C->getValue(); | 2231 const int32_t Divisor = C->getValue(); |
| 2232 uint32_t UDivisor = static_cast<uint32_t>(Divisor); | 2232 const uint32_t UDivisor = Divisor; |
| 2233 if (Divisor > 0 && llvm::isPowerOf2_32(UDivisor)) { | 2233 if (Divisor > 0 && llvm::isPowerOf2_32(UDivisor)) { |
| 2234 uint32_t LogDiv = llvm::Log2_32(UDivisor); | 2234 uint32_t LogDiv = llvm::Log2_32(UDivisor); |
| 2235 // LLVM does the following for dest=src/(1<<log): | 2235 // LLVM does the following for dest=src/(1<<log): |
| 2236 // t=src | 2236 // t=src |
| 2237 // sar t,typewidth-1 // -1 if src is negative, 0 if not | 2237 // sar t,typewidth-1 // -1 if src is negative, 0 if not |
| 2238 // shr t,typewidth-log | 2238 // shr t,typewidth-log |
| 2239 // add t,src | 2239 // add t,src |
| 2240 // sar t,log | 2240 // sar t,log |
| 2241 // dest=t | 2241 // dest=t |
| 2242 uint32_t TypeWidth = Traits::X86_CHAR_BIT * typeWidthInBytes(Ty); | 2242 uint32_t TypeWidth = Traits::X86_CHAR_BIT * typeWidthInBytes(Ty); |
| (...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2311 _div(T_edx, Src1, T); | 2311 _div(T_edx, Src1, T); |
| 2312 _mov(Dest, T_edx); | 2312 _mov(Dest, T_edx); |
| 2313 } break; | 2313 } break; |
| 2314 case InstArithmetic::Srem: { | 2314 case InstArithmetic::Srem: { |
| 2315 // TODO(stichnot): Enable this after doing better performance and cross | 2315 // TODO(stichnot): Enable this after doing better performance and cross |
| 2316 // testing. | 2316 // testing. |
| 2317 if (false && Ctx->getFlags().getOptLevel() >= Opt_1) { | 2317 if (false && Ctx->getFlags().getOptLevel() >= Opt_1) { |
| 2318 // Optimize mod by constant power of 2, but not for Om1 or O0, just to | 2318 // Optimize mod by constant power of 2, but not for Om1 or O0, just to |
| 2319 // keep things simple there. | 2319 // keep things simple there. |
| 2320 if (auto *C = llvm::dyn_cast<ConstantInteger32>(Src1)) { | 2320 if (auto *C = llvm::dyn_cast<ConstantInteger32>(Src1)) { |
| 2321 int32_t Divisor = C->getValue(); | 2321 const int32_t Divisor = C->getValue(); |
| 2322 uint32_t UDivisor = static_cast<uint32_t>(Divisor); | 2322 const uint32_t UDivisor = Divisor; |
| 2323 if (Divisor > 0 && llvm::isPowerOf2_32(UDivisor)) { | 2323 if (Divisor > 0 && llvm::isPowerOf2_32(UDivisor)) { |
| 2324 uint32_t LogDiv = llvm::Log2_32(UDivisor); | 2324 uint32_t LogDiv = llvm::Log2_32(UDivisor); |
| 2325 // LLVM does the following for dest=src%(1<<log): | 2325 // LLVM does the following for dest=src%(1<<log): |
| 2326 // t=src | 2326 // t=src |
| 2327 // sar t,typewidth-1 // -1 if src is negative, 0 if not | 2327 // sar t,typewidth-1 // -1 if src is negative, 0 if not |
| 2328 // shr t,typewidth-log | 2328 // shr t,typewidth-log |
| 2329 // add t,src | 2329 // add t,src |
| 2330 // and t, -(1<<log) | 2330 // and t, -(1<<log) |
| 2331 // sub t,src | 2331 // sub t,src |
| 2332 // neg t | 2332 // neg t |
| (...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2427 Operand *Cond = Br->getCondition(); | 2427 Operand *Cond = Br->getCondition(); |
| 2428 | 2428 |
| 2429 // Handle folding opportunities. | 2429 // Handle folding opportunities. |
| 2430 if (const Inst *Producer = FoldingInfo.getProducerFor(Cond)) { | 2430 if (const Inst *Producer = FoldingInfo.getProducerFor(Cond)) { |
| 2431 assert(Producer->isDeleted()); | 2431 assert(Producer->isDeleted()); |
| 2432 switch (BoolFolding<Traits>::getProducerKind(Producer)) { | 2432 switch (BoolFolding<Traits>::getProducerKind(Producer)) { |
| 2433 default: | 2433 default: |
| 2434 break; | 2434 break; |
| 2435 case BoolFolding<Traits>::PK_Icmp32: | 2435 case BoolFolding<Traits>::PK_Icmp32: |
| 2436 case BoolFolding<Traits>::PK_Icmp64: { | 2436 case BoolFolding<Traits>::PK_Icmp64: { |
| 2437 lowerIcmpAndConsumer(llvm::dyn_cast<InstIcmp>(Producer), Br); | 2437 lowerIcmpAndConsumer(llvm::cast<InstIcmp>(Producer), Br); |
| 2438 return; | 2438 return; |
| 2439 } | 2439 } |
| 2440 case BoolFolding<Traits>::PK_Fcmp: { | 2440 case BoolFolding<Traits>::PK_Fcmp: { |
| 2441 lowerFcmpAndConsumer(llvm::dyn_cast<InstFcmp>(Producer), Br); | 2441 lowerFcmpAndConsumer(llvm::cast<InstFcmp>(Producer), Br); |
| 2442 return; | 2442 return; |
| 2443 } | 2443 } |
| 2444 case BoolFolding<Traits>::PK_Arith: { | 2444 case BoolFolding<Traits>::PK_Arith: { |
| 2445 lowerArithAndConsumer(llvm::dyn_cast<InstArithmetic>(Producer), Br); | 2445 lowerArithAndConsumer(llvm::cast<InstArithmetic>(Producer), Br); |
| 2446 return; | 2446 return; |
| 2447 } | 2447 } |
| 2448 } | 2448 } |
| 2449 } | 2449 } |
| 2450 Operand *Src0 = legalize(Cond, Legal_Reg | Legal_Mem); | 2450 Operand *Src0 = legalize(Cond, Legal_Reg | Legal_Mem); |
| 2451 Constant *Zero = Ctx->getConstantZero(IceType_i32); | 2451 Constant *Zero = Ctx->getConstantZero(IceType_i32); |
| 2452 _cmp(Src0, Zero); | 2452 _cmp(Src0, Zero); |
| 2453 _br(Traits::Cond::Br_ne, Br->getTargetTrue(), Br->getTargetFalse()); | 2453 _br(Traits::Cond::Br_ne, Br->getTargetTrue(), Br->getTargetFalse()); |
| 2454 } | 2454 } |
| 2455 | 2455 |
| (...skipping 634 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3090 } | 3090 } |
| 3091 break; | 3091 break; |
| 3092 } | 3092 } |
| 3093 } | 3093 } |
| 3094 } | 3094 } |
| 3095 | 3095 |
| 3096 template <typename TraitsType> | 3096 template <typename TraitsType> |
| 3097 void TargetX86Base<TraitsType>::lowerExtractElement( | 3097 void TargetX86Base<TraitsType>::lowerExtractElement( |
| 3098 const InstExtractElement *Instr) { | 3098 const InstExtractElement *Instr) { |
| 3099 Operand *SourceVectNotLegalized = Instr->getSrc(0); | 3099 Operand *SourceVectNotLegalized = Instr->getSrc(0); |
| 3100 ConstantInteger32 *ElementIndex = | 3100 auto *ElementIndex = llvm::dyn_cast<ConstantInteger32>(Instr->getSrc(1)); |
| 3101 llvm::dyn_cast<ConstantInteger32>(Instr->getSrc(1)); | |
| 3102 // Only constant indices are allowed in PNaCl IR. | 3101 // Only constant indices are allowed in PNaCl IR. |
| 3103 assert(ElementIndex); | 3102 assert(ElementIndex); |
| 3104 | 3103 |
| 3105 unsigned Index = ElementIndex->getValue(); | 3104 unsigned Index = ElementIndex->getValue(); |
| 3106 Type Ty = SourceVectNotLegalized->getType(); | 3105 Type Ty = SourceVectNotLegalized->getType(); |
| 3107 Type ElementTy = typeElementType(Ty); | 3106 Type ElementTy = typeElementType(Ty); |
| 3108 Type InVectorElementTy = Traits::getInVectorElementType(Ty); | 3107 Type InVectorElementTy = Traits::getInVectorElementType(Ty); |
| 3109 | 3108 |
| 3110 // TODO(wala): Determine the best lowering sequences for each type. | 3109 // TODO(wala): Determine the best lowering sequences for each type. |
| 3111 bool CanUsePextr = Ty == IceType_v8i16 || Ty == IceType_v8i1 || | 3110 bool CanUsePextr = Ty == IceType_v8i16 || Ty == IceType_v8i1 || |
| (...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3211 // j<C1> label /* only if C1 != Br_None */ | 3210 // j<C1> label /* only if C1 != Br_None */ |
| 3212 // j<C2> label /* only if C2 != Br_None */ | 3211 // j<C2> label /* only if C2 != Br_None */ |
| 3213 // FakeUse(a) /* only if C1 != Br_None */ | 3212 // FakeUse(a) /* only if C1 != Br_None */ |
| 3214 // mov a, !<default> /* only if C1 != Br_None */ | 3213 // mov a, !<default> /* only if C1 != Br_None */ |
| 3215 // label: /* only if C1 != Br_None */ | 3214 // label: /* only if C1 != Br_None */ |
| 3216 // | 3215 // |
| 3217 // setcc lowering when C1 != Br_None && C2 == Br_None: | 3216 // setcc lowering when C1 != Br_None && C2 == Br_None: |
| 3218 // ucomiss b, c /* but swap b,c order if SwapOperands==true */ | 3217 // ucomiss b, c /* but swap b,c order if SwapOperands==true */ |
| 3219 // setcc a, C1 | 3218 // setcc a, C1 |
| 3220 InstFcmp::FCond Condition = Fcmp->getCondition(); | 3219 InstFcmp::FCond Condition = Fcmp->getCondition(); |
| 3221 size_t Index = static_cast<size_t>(Condition); | 3220 assert(Condition < Traits::TableFcmpSize); |
| 3222 assert(Index < Traits::TableFcmpSize); | 3221 if (Traits::TableFcmp[Condition].SwapScalarOperands) |
| 3223 if (Traits::TableFcmp[Index].SwapScalarOperands) | |
| 3224 std::swap(Src0, Src1); | 3222 std::swap(Src0, Src1); |
| 3225 bool HasC1 = (Traits::TableFcmp[Index].C1 != Traits::Cond::Br_None); | 3223 const bool HasC1 = (Traits::TableFcmp[Condition].C1 != Traits::Cond::Br_None); |
| 3226 bool HasC2 = (Traits::TableFcmp[Index].C2 != Traits::Cond::Br_None); | 3224 const bool HasC2 = (Traits::TableFcmp[Condition].C2 != Traits::Cond::Br_None); |
| 3227 if (HasC1) { | 3225 if (HasC1) { |
| 3228 Src0 = legalize(Src0); | 3226 Src0 = legalize(Src0); |
| 3229 Operand *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); | 3227 Operand *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); |
| 3230 Variable *T = nullptr; | 3228 Variable *T = nullptr; |
| 3231 _mov(T, Src0); | 3229 _mov(T, Src0); |
| 3232 _ucomiss(T, Src1RM); | 3230 _ucomiss(T, Src1RM); |
| 3233 if (!HasC2) { | 3231 if (!HasC2) { |
| 3234 assert(Traits::TableFcmp[Index].Default); | 3232 assert(Traits::TableFcmp[Condition].Default); |
| 3235 setccOrConsumer(Traits::TableFcmp[Index].C1, Dest, Consumer); | 3233 setccOrConsumer(Traits::TableFcmp[Condition].C1, Dest, Consumer); |
| 3236 return; | 3234 return; |
| 3237 } | 3235 } |
| 3238 } | 3236 } |
| 3239 int32_t IntDefault = Traits::TableFcmp[Index].Default; | 3237 int32_t IntDefault = Traits::TableFcmp[Condition].Default; |
| 3240 if (Consumer == nullptr) { | 3238 if (Consumer == nullptr) { |
| 3241 Constant *Default = Ctx->getConstantInt(Dest->getType(), IntDefault); | 3239 Constant *Default = Ctx->getConstantInt(Dest->getType(), IntDefault); |
| 3242 _mov(Dest, Default); | 3240 _mov(Dest, Default); |
| 3243 if (HasC1) { | 3241 if (HasC1) { |
| 3244 InstX86Label *Label = InstX86Label::create(Func, this); | 3242 InstX86Label *Label = InstX86Label::create(Func, this); |
| 3245 _br(Traits::TableFcmp[Index].C1, Label); | 3243 _br(Traits::TableFcmp[Condition].C1, Label); |
| 3246 if (HasC2) { | 3244 if (HasC2) { |
| 3247 _br(Traits::TableFcmp[Index].C2, Label); | 3245 _br(Traits::TableFcmp[Condition].C2, Label); |
| 3248 } | 3246 } |
| 3249 Constant *NonDefault = Ctx->getConstantInt(Dest->getType(), !IntDefault); | 3247 Constant *NonDefault = Ctx->getConstantInt(Dest->getType(), !IntDefault); |
| 3250 _redefined(_mov(Dest, NonDefault)); | 3248 _redefined(_mov(Dest, NonDefault)); |
| 3251 Context.insert(Label); | 3249 Context.insert(Label); |
| 3252 } | 3250 } |
| 3253 return; | 3251 return; |
| 3254 } | 3252 } |
| 3255 if (const auto *Br = llvm::dyn_cast<InstBr>(Consumer)) { | 3253 if (const auto *Br = llvm::dyn_cast<InstBr>(Consumer)) { |
| 3256 CfgNode *TrueSucc = Br->getTargetTrue(); | 3254 CfgNode *TrueSucc = Br->getTargetTrue(); |
| 3257 CfgNode *FalseSucc = Br->getTargetFalse(); | 3255 CfgNode *FalseSucc = Br->getTargetFalse(); |
| 3258 if (IntDefault != 0) | 3256 if (IntDefault != 0) |
| 3259 std::swap(TrueSucc, FalseSucc); | 3257 std::swap(TrueSucc, FalseSucc); |
| 3260 if (HasC1) { | 3258 if (HasC1) { |
| 3261 _br(Traits::TableFcmp[Index].C1, FalseSucc); | 3259 _br(Traits::TableFcmp[Condition].C1, FalseSucc); |
| 3262 if (HasC2) { | 3260 if (HasC2) { |
| 3263 _br(Traits::TableFcmp[Index].C2, FalseSucc); | 3261 _br(Traits::TableFcmp[Condition].C2, FalseSucc); |
| 3264 } | 3262 } |
| 3265 _br(TrueSucc); | 3263 _br(TrueSucc); |
| 3266 return; | 3264 return; |
| 3267 } | 3265 } |
| 3268 _br(FalseSucc); | 3266 _br(FalseSucc); |
| 3269 return; | 3267 return; |
| 3270 } | 3268 } |
| 3271 if (auto *Select = llvm::dyn_cast<InstSelect>(Consumer)) { | 3269 if (auto *Select = llvm::dyn_cast<InstSelect>(Consumer)) { |
| 3272 Operand *SrcT = Select->getTrueOperand(); | 3270 Operand *SrcT = Select->getTrueOperand(); |
| 3273 Operand *SrcF = Select->getFalseOperand(); | 3271 Operand *SrcF = Select->getFalseOperand(); |
| 3274 Variable *SelectDest = Select->getDest(); | 3272 Variable *SelectDest = Select->getDest(); |
| 3275 if (IntDefault != 0) | 3273 if (IntDefault != 0) |
| 3276 std::swap(SrcT, SrcF); | 3274 std::swap(SrcT, SrcF); |
| 3277 lowerMove(SelectDest, SrcF, false); | 3275 lowerMove(SelectDest, SrcF, false); |
| 3278 if (HasC1) { | 3276 if (HasC1) { |
| 3279 InstX86Label *Label = InstX86Label::create(Func, this); | 3277 InstX86Label *Label = InstX86Label::create(Func, this); |
| 3280 _br(Traits::TableFcmp[Index].C1, Label); | 3278 _br(Traits::TableFcmp[Condition].C1, Label); |
| 3281 if (HasC2) { | 3279 if (HasC2) { |
| 3282 _br(Traits::TableFcmp[Index].C2, Label); | 3280 _br(Traits::TableFcmp[Condition].C2, Label); |
| 3283 } | 3281 } |
| 3284 static constexpr bool IsRedefinition = true; | 3282 static constexpr bool IsRedefinition = true; |
| 3285 lowerMove(SelectDest, SrcT, IsRedefinition); | 3283 lowerMove(SelectDest, SrcT, IsRedefinition); |
| 3286 Context.insert(Label); | 3284 Context.insert(Label); |
| 3287 } | 3285 } |
| 3288 return; | 3286 return; |
| 3289 } | 3287 } |
| 3290 llvm::report_fatal_error("Unexpected consumer type"); | 3288 llvm::report_fatal_error("Unexpected consumer type"); |
| 3291 } | 3289 } |
| 3292 | 3290 |
| 3293 template <typename TraitsType> | 3291 template <typename TraitsType> |
| 3294 void TargetX86Base<TraitsType>::lowerFcmpVector(const InstFcmp *Fcmp) { | 3292 void TargetX86Base<TraitsType>::lowerFcmpVector(const InstFcmp *Fcmp) { |
| 3295 Operand *Src0 = Fcmp->getSrc(0); | 3293 Operand *Src0 = Fcmp->getSrc(0); |
| 3296 Operand *Src1 = Fcmp->getSrc(1); | 3294 Operand *Src1 = Fcmp->getSrc(1); |
| 3297 Variable *Dest = Fcmp->getDest(); | 3295 Variable *Dest = Fcmp->getDest(); |
| 3298 | 3296 |
| 3299 if (!isVectorType(Dest->getType())) | 3297 if (!isVectorType(Dest->getType())) |
| 3300 llvm::report_fatal_error("Expected vector compare"); | 3298 llvm::report_fatal_error("Expected vector compare"); |
| 3301 | 3299 |
| 3302 InstFcmp::FCond Condition = Fcmp->getCondition(); | 3300 InstFcmp::FCond Condition = Fcmp->getCondition(); |
| 3303 size_t Index = static_cast<size_t>(Condition); | 3301 assert(Condition < Traits::TableFcmpSize); |
| 3304 assert(Index < Traits::TableFcmpSize); | |
| 3305 | 3302 |
| 3306 if (Traits::TableFcmp[Index].SwapVectorOperands) | 3303 if (Traits::TableFcmp[Condition].SwapVectorOperands) |
| 3307 std::swap(Src0, Src1); | 3304 std::swap(Src0, Src1); |
| 3308 | 3305 |
| 3309 Variable *T = nullptr; | 3306 Variable *T = nullptr; |
| 3310 | 3307 |
| 3311 if (Condition == InstFcmp::True) { | 3308 if (Condition == InstFcmp::True) { |
| 3312 // makeVectorOfOnes() requires an integer vector type. | 3309 // makeVectorOfOnes() requires an integer vector type. |
| 3313 T = makeVectorOfMinusOnes(IceType_v4i32); | 3310 T = makeVectorOfMinusOnes(IceType_v4i32); |
| 3314 } else if (Condition == InstFcmp::False) { | 3311 } else if (Condition == InstFcmp::False) { |
| 3315 T = makeVectorOfZeros(Dest->getType()); | 3312 T = makeVectorOfZeros(Dest->getType()); |
| 3316 } else { | 3313 } else { |
| 3317 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); | 3314 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); |
| 3318 Operand *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); | 3315 Operand *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); |
| 3319 if (llvm::isa<X86OperandMem>(Src1RM)) | 3316 if (llvm::isa<X86OperandMem>(Src1RM)) |
| 3320 Src1RM = legalizeToReg(Src1RM); | 3317 Src1RM = legalizeToReg(Src1RM); |
| 3321 | 3318 |
| 3322 switch (Condition) { | 3319 switch (Condition) { |
| 3323 default: { | 3320 default: { |
| 3324 CmppsCond Predicate = Traits::TableFcmp[Index].Predicate; | 3321 const CmppsCond Predicate = Traits::TableFcmp[Condition].Predicate; |
| 3325 assert(Predicate != Traits::Cond::Cmpps_Invalid); | 3322 assert(Predicate != Traits::Cond::Cmpps_Invalid); |
| 3326 T = makeReg(Src0RM->getType()); | 3323 T = makeReg(Src0RM->getType()); |
| 3327 _movp(T, Src0RM); | 3324 _movp(T, Src0RM); |
| 3328 _cmpps(T, Src1RM, Predicate); | 3325 _cmpps(T, Src1RM, Predicate); |
| 3329 } break; | 3326 } break; |
| 3330 case InstFcmp::One: { | 3327 case InstFcmp::One: { |
| 3331 // Check both unequal and ordered. | 3328 // Check both unequal and ordered. |
| 3332 T = makeReg(Src0RM->getType()); | 3329 T = makeReg(Src0RM->getType()); |
| 3333 Variable *T2 = makeReg(Src0RM->getType()); | 3330 Variable *T2 = makeReg(Src0RM->getType()); |
| 3334 _movp(T, Src0RM); | 3331 _movp(T, Src0RM); |
| (...skipping 181 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3516 template <typename TraitsType> | 3513 template <typename TraitsType> |
| 3517 template <typename T> | 3514 template <typename T> |
| 3518 typename std::enable_if<!T::Is64Bit, void>::type | 3515 typename std::enable_if<!T::Is64Bit, void>::type |
| 3519 TargetX86Base<TraitsType>::lowerIcmp64(const InstIcmp *Icmp, | 3516 TargetX86Base<TraitsType>::lowerIcmp64(const InstIcmp *Icmp, |
| 3520 const Inst *Consumer) { | 3517 const Inst *Consumer) { |
| 3521 // a=icmp cond, b, c ==> cmp b,c; a=1; br cond,L1; FakeUse(a); a=0; L1: | 3518 // a=icmp cond, b, c ==> cmp b,c; a=1; br cond,L1; FakeUse(a); a=0; L1: |
| 3522 Operand *Src0 = legalize(Icmp->getSrc(0)); | 3519 Operand *Src0 = legalize(Icmp->getSrc(0)); |
| 3523 Operand *Src1 = legalize(Icmp->getSrc(1)); | 3520 Operand *Src1 = legalize(Icmp->getSrc(1)); |
| 3524 Variable *Dest = Icmp->getDest(); | 3521 Variable *Dest = Icmp->getDest(); |
| 3525 InstIcmp::ICond Condition = Icmp->getCondition(); | 3522 InstIcmp::ICond Condition = Icmp->getCondition(); |
| 3526 size_t Index = static_cast<size_t>(Condition); | 3523 assert(Condition < Traits::TableIcmp64Size); |
| 3527 assert(Index < Traits::TableIcmp64Size); | |
| 3528 Operand *Src0LoRM = nullptr; | 3524 Operand *Src0LoRM = nullptr; |
| 3529 Operand *Src0HiRM = nullptr; | 3525 Operand *Src0HiRM = nullptr; |
| 3530 // Legalize the portions of Src0 that are going to be needed. | 3526 // Legalize the portions of Src0 that are going to be needed. |
| 3531 if (isZero(Src1)) { | 3527 if (isZero(Src1)) { |
| 3532 switch (Condition) { | 3528 switch (Condition) { |
| 3533 default: | 3529 default: |
| 3534 llvm_unreachable("unexpected condition"); | 3530 llvm_unreachable("unexpected condition"); |
| 3535 break; | 3531 break; |
| 3536 // These two are not optimized, so we fall through to the general case, | 3532 // These two are not optimized, so we fall through to the general case, |
| 3537 // which needs the upper and lower halves legalized. | 3533 // which needs the upper and lower halves legalized. |
| (...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3609 // Handle general compares. | 3605 // Handle general compares. |
| 3610 Operand *Src1LoRI = legalize(loOperand(Src1), Legal_Reg | Legal_Imm); | 3606 Operand *Src1LoRI = legalize(loOperand(Src1), Legal_Reg | Legal_Imm); |
| 3611 Operand *Src1HiRI = legalize(hiOperand(Src1), Legal_Reg | Legal_Imm); | 3607 Operand *Src1HiRI = legalize(hiOperand(Src1), Legal_Reg | Legal_Imm); |
| 3612 if (Consumer == nullptr) { | 3608 if (Consumer == nullptr) { |
| 3613 Constant *Zero = Ctx->getConstantInt(Dest->getType(), 0); | 3609 Constant *Zero = Ctx->getConstantInt(Dest->getType(), 0); |
| 3614 Constant *One = Ctx->getConstantInt(Dest->getType(), 1); | 3610 Constant *One = Ctx->getConstantInt(Dest->getType(), 1); |
| 3615 InstX86Label *LabelFalse = InstX86Label::create(Func, this); | 3611 InstX86Label *LabelFalse = InstX86Label::create(Func, this); |
| 3616 InstX86Label *LabelTrue = InstX86Label::create(Func, this); | 3612 InstX86Label *LabelTrue = InstX86Label::create(Func, this); |
| 3617 _mov(Dest, One); | 3613 _mov(Dest, One); |
| 3618 _cmp(Src0HiRM, Src1HiRI); | 3614 _cmp(Src0HiRM, Src1HiRI); |
| 3619 if (Traits::TableIcmp64[Index].C1 != Traits::Cond::Br_None) | 3615 if (Traits::TableIcmp64[Condition].C1 != Traits::Cond::Br_None) |
| 3620 _br(Traits::TableIcmp64[Index].C1, LabelTrue); | 3616 _br(Traits::TableIcmp64[Condition].C1, LabelTrue); |
| 3621 if (Traits::TableIcmp64[Index].C2 != Traits::Cond::Br_None) | 3617 if (Traits::TableIcmp64[Condition].C2 != Traits::Cond::Br_None) |
| 3622 _br(Traits::TableIcmp64[Index].C2, LabelFalse); | 3618 _br(Traits::TableIcmp64[Condition].C2, LabelFalse); |
| 3623 _cmp(Src0LoRM, Src1LoRI); | 3619 _cmp(Src0LoRM, Src1LoRI); |
| 3624 _br(Traits::TableIcmp64[Index].C3, LabelTrue); | 3620 _br(Traits::TableIcmp64[Condition].C3, LabelTrue); |
| 3625 Context.insert(LabelFalse); | 3621 Context.insert(LabelFalse); |
| 3626 _redefined(_mov(Dest, Zero)); | 3622 _redefined(_mov(Dest, Zero)); |
| 3627 Context.insert(LabelTrue); | 3623 Context.insert(LabelTrue); |
| 3628 return; | 3624 return; |
| 3629 } | 3625 } |
| 3630 if (const auto *Br = llvm::dyn_cast<InstBr>(Consumer)) { | 3626 if (const auto *Br = llvm::dyn_cast<InstBr>(Consumer)) { |
| 3631 _cmp(Src0HiRM, Src1HiRI); | 3627 _cmp(Src0HiRM, Src1HiRI); |
| 3632 if (Traits::TableIcmp64[Index].C1 != Traits::Cond::Br_None) | 3628 if (Traits::TableIcmp64[Condition].C1 != Traits::Cond::Br_None) |
| 3633 _br(Traits::TableIcmp64[Index].C1, Br->getTargetTrue()); | 3629 _br(Traits::TableIcmp64[Condition].C1, Br->getTargetTrue()); |
| 3634 if (Traits::TableIcmp64[Index].C2 != Traits::Cond::Br_None) | 3630 if (Traits::TableIcmp64[Condition].C2 != Traits::Cond::Br_None) |
| 3635 _br(Traits::TableIcmp64[Index].C2, Br->getTargetFalse()); | 3631 _br(Traits::TableIcmp64[Condition].C2, Br->getTargetFalse()); |
| 3636 _cmp(Src0LoRM, Src1LoRI); | 3632 _cmp(Src0LoRM, Src1LoRI); |
| 3637 _br(Traits::TableIcmp64[Index].C3, Br->getTargetTrue(), | 3633 _br(Traits::TableIcmp64[Condition].C3, Br->getTargetTrue(), |
| 3638 Br->getTargetFalse()); | 3634 Br->getTargetFalse()); |
| 3639 return; | 3635 return; |
| 3640 } | 3636 } |
| 3641 if (auto *Select = llvm::dyn_cast<InstSelect>(Consumer)) { | 3637 if (auto *Select = llvm::dyn_cast<InstSelect>(Consumer)) { |
| 3642 Operand *SrcT = Select->getTrueOperand(); | 3638 Operand *SrcT = Select->getTrueOperand(); |
| 3643 Operand *SrcF = Select->getFalseOperand(); | 3639 Operand *SrcF = Select->getFalseOperand(); |
| 3644 Variable *SelectDest = Select->getDest(); | 3640 Variable *SelectDest = Select->getDest(); |
| 3645 InstX86Label *LabelFalse = InstX86Label::create(Func, this); | 3641 InstX86Label *LabelFalse = InstX86Label::create(Func, this); |
| 3646 InstX86Label *LabelTrue = InstX86Label::create(Func, this); | 3642 InstX86Label *LabelTrue = InstX86Label::create(Func, this); |
| 3647 lowerMove(SelectDest, SrcT, false); | 3643 lowerMove(SelectDest, SrcT, false); |
| 3648 _cmp(Src0HiRM, Src1HiRI); | 3644 _cmp(Src0HiRM, Src1HiRI); |
| 3649 if (Traits::TableIcmp64[Index].C1 != Traits::Cond::Br_None) | 3645 if (Traits::TableIcmp64[Condition].C1 != Traits::Cond::Br_None) |
| 3650 _br(Traits::TableIcmp64[Index].C1, LabelTrue); | 3646 _br(Traits::TableIcmp64[Condition].C1, LabelTrue); |
| 3651 if (Traits::TableIcmp64[Index].C2 != Traits::Cond::Br_None) | 3647 if (Traits::TableIcmp64[Condition].C2 != Traits::Cond::Br_None) |
| 3652 _br(Traits::TableIcmp64[Index].C2, LabelFalse); | 3648 _br(Traits::TableIcmp64[Condition].C2, LabelFalse); |
| 3653 _cmp(Src0LoRM, Src1LoRI); | 3649 _cmp(Src0LoRM, Src1LoRI); |
| 3654 _br(Traits::TableIcmp64[Index].C3, LabelTrue); | 3650 _br(Traits::TableIcmp64[Condition].C3, LabelTrue); |
| 3655 Context.insert(LabelFalse); | 3651 Context.insert(LabelFalse); |
| 3656 static constexpr bool IsRedefinition = true; | 3652 static constexpr bool IsRedefinition = true; |
| 3657 lowerMove(SelectDest, SrcF, IsRedefinition); | 3653 lowerMove(SelectDest, SrcF, IsRedefinition); |
| 3658 Context.insert(LabelTrue); | 3654 Context.insert(LabelTrue); |
| 3659 return; | 3655 return; |
| 3660 } | 3656 } |
| 3661 llvm::report_fatal_error("Unexpected consumer type"); | 3657 llvm::report_fatal_error("Unexpected consumer type"); |
| 3662 } | 3658 } |
| 3663 | 3659 |
| 3664 template <typename TraitsType> | 3660 template <typename TraitsType> |
| (...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3750 return; | 3746 return; |
| 3751 } | 3747 } |
| 3752 llvm::report_fatal_error("Unexpected consumer type"); | 3748 llvm::report_fatal_error("Unexpected consumer type"); |
| 3753 } | 3749 } |
| 3754 | 3750 |
| 3755 template <typename TraitsType> | 3751 template <typename TraitsType> |
| 3756 void TargetX86Base<TraitsType>::lowerInsertElement( | 3752 void TargetX86Base<TraitsType>::lowerInsertElement( |
| 3757 const InstInsertElement *Instr) { | 3753 const InstInsertElement *Instr) { |
| 3758 Operand *SourceVectNotLegalized = Instr->getSrc(0); | 3754 Operand *SourceVectNotLegalized = Instr->getSrc(0); |
| 3759 Operand *ElementToInsertNotLegalized = Instr->getSrc(1); | 3755 Operand *ElementToInsertNotLegalized = Instr->getSrc(1); |
| 3760 ConstantInteger32 *ElementIndex = | 3756 auto *ElementIndex = llvm::dyn_cast<ConstantInteger32>(Instr->getSrc(2)); |
| 3761 llvm::dyn_cast<ConstantInteger32>(Instr->getSrc(2)); | |
| 3762 // Only constant indices are allowed in PNaCl IR. | 3757 // Only constant indices are allowed in PNaCl IR. |
| 3763 assert(ElementIndex); | 3758 assert(ElementIndex); |
| 3764 unsigned Index = ElementIndex->getValue(); | 3759 unsigned Index = ElementIndex->getValue(); |
| 3765 assert(Index < typeNumElements(SourceVectNotLegalized->getType())); | 3760 assert(Index < typeNumElements(SourceVectNotLegalized->getType())); |
| 3766 | 3761 |
| 3767 Type Ty = SourceVectNotLegalized->getType(); | 3762 Type Ty = SourceVectNotLegalized->getType(); |
| 3768 Type ElementTy = typeElementType(Ty); | 3763 Type ElementTy = typeElementType(Ty); |
| 3769 Type InVectorElementTy = Traits::getInVectorElementType(Ty); | 3764 Type InVectorElementTy = Traits::getInVectorElementType(Ty); |
| 3770 | 3765 |
| 3771 if (ElementTy == IceType_i1) { | 3766 if (ElementTy == IceType_i1) { |
| (...skipping 1797 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 5569 | 5564 |
| 5570 Operand *Condition = Select->getCondition(); | 5565 Operand *Condition = Select->getCondition(); |
| 5571 // Handle folding opportunities. | 5566 // Handle folding opportunities. |
| 5572 if (const Inst *Producer = FoldingInfo.getProducerFor(Condition)) { | 5567 if (const Inst *Producer = FoldingInfo.getProducerFor(Condition)) { |
| 5573 assert(Producer->isDeleted()); | 5568 assert(Producer->isDeleted()); |
| 5574 switch (BoolFolding<Traits>::getProducerKind(Producer)) { | 5569 switch (BoolFolding<Traits>::getProducerKind(Producer)) { |
| 5575 default: | 5570 default: |
| 5576 break; | 5571 break; |
| 5577 case BoolFolding<Traits>::PK_Icmp32: | 5572 case BoolFolding<Traits>::PK_Icmp32: |
| 5578 case BoolFolding<Traits>::PK_Icmp64: { | 5573 case BoolFolding<Traits>::PK_Icmp64: { |
| 5579 lowerIcmpAndConsumer(llvm::dyn_cast<InstIcmp>(Producer), Select); | 5574 lowerIcmpAndConsumer(llvm::cast<InstIcmp>(Producer), Select); |
| 5580 return; | 5575 return; |
| 5581 } | 5576 } |
| 5582 case BoolFolding<Traits>::PK_Fcmp: { | 5577 case BoolFolding<Traits>::PK_Fcmp: { |
| 5583 lowerFcmpAndConsumer(llvm::dyn_cast<InstFcmp>(Producer), Select); | 5578 lowerFcmpAndConsumer(llvm::cast<InstFcmp>(Producer), Select); |
| 5584 return; | 5579 return; |
| 5585 } | 5580 } |
| 5586 } | 5581 } |
| 5587 } | 5582 } |
| 5588 | 5583 |
| 5589 Operand *CmpResult = legalize(Condition, Legal_Reg | Legal_Mem); | 5584 Operand *CmpResult = legalize(Condition, Legal_Reg | Legal_Mem); |
| 5590 Operand *Zero = Ctx->getConstantZero(IceType_i32); | 5585 Operand *Zero = Ctx->getConstantZero(IceType_i32); |
| 5591 _cmp(CmpResult, Zero); | 5586 _cmp(CmpResult, Zero); |
| 5592 Operand *SrcT = Select->getTrueOperand(); | 5587 Operand *SrcT = Select->getTrueOperand(); |
| 5593 Operand *SrcF = Select->getFalseOperand(); | 5588 Operand *SrcF = Select->getFalseOperand(); |
| (...skipping 23 matching lines...) Expand all Loading... |
| 5617 // instruction doesn't allow an immediate operand: | 5612 // instruction doesn't allow an immediate operand: |
| 5618 // mov t, SrcT; cmov_!cond t, SrcF; mov dest, t | 5613 // mov t, SrcT; cmov_!cond t, SrcF; mov dest, t |
| 5619 if (llvm::isa<Constant>(SrcT) && !llvm::isa<Constant>(SrcF)) { | 5614 if (llvm::isa<Constant>(SrcT) && !llvm::isa<Constant>(SrcF)) { |
| 5620 std::swap(SrcT, SrcF); | 5615 std::swap(SrcT, SrcF); |
| 5621 Cond = InstImpl<TraitsType>::InstX86Base::getOppositeCondition(Cond); | 5616 Cond = InstImpl<TraitsType>::InstX86Base::getOppositeCondition(Cond); |
| 5622 } | 5617 } |
| 5623 if (!Traits::Is64Bit && DestTy == IceType_i64) { | 5618 if (!Traits::Is64Bit && DestTy == IceType_i64) { |
| 5624 SrcT = legalizeUndef(SrcT); | 5619 SrcT = legalizeUndef(SrcT); |
| 5625 SrcF = legalizeUndef(SrcF); | 5620 SrcF = legalizeUndef(SrcF); |
| 5626 // Set the low portion. | 5621 // Set the low portion. |
| 5627 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 5622 auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
| 5628 lowerSelectIntMove(DestLo, Cond, loOperand(SrcT), loOperand(SrcF)); | 5623 lowerSelectIntMove(DestLo, Cond, loOperand(SrcT), loOperand(SrcF)); |
| 5629 // Set the high portion. | 5624 // Set the high portion. |
| 5630 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 5625 auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
| 5631 lowerSelectIntMove(DestHi, Cond, hiOperand(SrcT), hiOperand(SrcF)); | 5626 lowerSelectIntMove(DestHi, Cond, hiOperand(SrcT), hiOperand(SrcF)); |
| 5632 return; | 5627 return; |
| 5633 } | 5628 } |
| 5634 | 5629 |
| 5635 assert(DestTy == IceType_i16 || DestTy == IceType_i32 || | 5630 assert(DestTy == IceType_i16 || DestTy == IceType_i32 || |
| 5636 (Traits::Is64Bit && DestTy == IceType_i64)); | 5631 (Traits::Is64Bit && DestTy == IceType_i64)); |
| 5637 lowerSelectIntMove(Dest, Cond, SrcT, SrcF); | 5632 lowerSelectIntMove(Dest, Cond, SrcT, SrcF); |
| 5638 } | 5633 } |
| 5639 | 5634 |
| 5640 template <typename TraitsType> | 5635 template <typename TraitsType> |
| (...skipping 10 matching lines...) Expand all Loading... |
| 5651 | 5646 |
| 5652 template <typename TraitsType> | 5647 template <typename TraitsType> |
| 5653 void TargetX86Base<TraitsType>::lowerMove(Variable *Dest, Operand *Src, | 5648 void TargetX86Base<TraitsType>::lowerMove(Variable *Dest, Operand *Src, |
| 5654 bool IsRedefinition) { | 5649 bool IsRedefinition) { |
| 5655 assert(Dest->getType() == Src->getType()); | 5650 assert(Dest->getType() == Src->getType()); |
| 5656 assert(!Dest->isRematerializable()); | 5651 assert(!Dest->isRematerializable()); |
| 5657 if (!Traits::Is64Bit && Dest->getType() == IceType_i64) { | 5652 if (!Traits::Is64Bit && Dest->getType() == IceType_i64) { |
| 5658 Src = legalize(Src); | 5653 Src = legalize(Src); |
| 5659 Operand *SrcLo = loOperand(Src); | 5654 Operand *SrcLo = loOperand(Src); |
| 5660 Operand *SrcHi = hiOperand(Src); | 5655 Operand *SrcHi = hiOperand(Src); |
| 5661 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 5656 auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
| 5662 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 5657 auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
| 5663 Variable *T_Lo = nullptr, *T_Hi = nullptr; | 5658 Variable *T_Lo = nullptr, *T_Hi = nullptr; |
| 5664 _mov(T_Lo, SrcLo); | 5659 _mov(T_Lo, SrcLo); |
| 5665 _redefined(_mov(DestLo, T_Lo), IsRedefinition); | 5660 _redefined(_mov(DestLo, T_Lo), IsRedefinition); |
| 5666 _mov(T_Hi, SrcHi); | 5661 _mov(T_Hi, SrcHi); |
| 5667 _redefined(_mov(DestHi, T_Hi), IsRedefinition); | 5662 _redefined(_mov(DestHi, T_Hi), IsRedefinition); |
| 5668 } else { | 5663 } else { |
| 5669 Operand *SrcLegal; | 5664 Operand *SrcLegal; |
| 5670 if (Dest->hasReg()) { | 5665 if (Dest->hasReg()) { |
| 5671 // If Dest already has a physical register, then only basic legalization | 5666 // If Dest already has a physical register, then only basic legalization |
| 5672 // is needed, as the source operand can be a register, immediate, or | 5667 // is needed, as the source operand can be a register, immediate, or |
| (...skipping 471 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 6144 if (!RMW->isLastUse(RMW->getBeacon())) | 6139 if (!RMW->isLastUse(RMW->getBeacon())) |
| 6145 return; | 6140 return; |
| 6146 Operand *Src = RMW->getData(); | 6141 Operand *Src = RMW->getData(); |
| 6147 Type Ty = Src->getType(); | 6142 Type Ty = Src->getType(); |
| 6148 X86OperandMem *Addr = formMemoryOperand(RMW->getAddr(), Ty); | 6143 X86OperandMem *Addr = formMemoryOperand(RMW->getAddr(), Ty); |
| 6149 doMockBoundsCheck(Addr); | 6144 doMockBoundsCheck(Addr); |
| 6150 if (!Traits::Is64Bit && Ty == IceType_i64) { | 6145 if (!Traits::Is64Bit && Ty == IceType_i64) { |
| 6151 Src = legalizeUndef(Src); | 6146 Src = legalizeUndef(Src); |
| 6152 Operand *SrcLo = legalize(loOperand(Src), Legal_Reg | Legal_Imm); | 6147 Operand *SrcLo = legalize(loOperand(Src), Legal_Reg | Legal_Imm); |
| 6153 Operand *SrcHi = legalize(hiOperand(Src), Legal_Reg | Legal_Imm); | 6148 Operand *SrcHi = legalize(hiOperand(Src), Legal_Reg | Legal_Imm); |
| 6154 X86OperandMem *AddrLo = llvm::cast<X86OperandMem>(loOperand(Addr)); | 6149 auto *AddrLo = llvm::cast<X86OperandMem>(loOperand(Addr)); |
| 6155 X86OperandMem *AddrHi = llvm::cast<X86OperandMem>(hiOperand(Addr)); | 6150 auto *AddrHi = llvm::cast<X86OperandMem>(hiOperand(Addr)); |
| 6156 switch (RMW->getOp()) { | 6151 switch (RMW->getOp()) { |
| 6157 default: | 6152 default: |
| 6158 // TODO(stichnot): Implement other arithmetic operators. | 6153 // TODO(stichnot): Implement other arithmetic operators. |
| 6159 break; | 6154 break; |
| 6160 case InstArithmetic::Add: | 6155 case InstArithmetic::Add: |
| 6161 _add_rmw(AddrLo, SrcLo); | 6156 _add_rmw(AddrLo, SrcLo); |
| 6162 _adc_rmw(AddrHi, SrcHi); | 6157 _adc_rmw(AddrHi, SrcHi); |
| 6163 return; | 6158 return; |
| 6164 case InstArithmetic::Sub: | 6159 case InstArithmetic::Sub: |
| 6165 _sub_rmw(AddrLo, SrcLo); | 6160 _sub_rmw(AddrLo, SrcLo); |
| (...skipping 1292 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 7458 emitGlobal(*Var, SectionSuffix); | 7453 emitGlobal(*Var, SectionSuffix); |
| 7459 } | 7454 } |
| 7460 } | 7455 } |
| 7461 } break; | 7456 } break; |
| 7462 } | 7457 } |
| 7463 } | 7458 } |
| 7464 } // end of namespace X86NAMESPACE | 7459 } // end of namespace X86NAMESPACE |
| 7465 } // end of namespace Ice | 7460 } // end of namespace Ice |
| 7466 | 7461 |
| 7467 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H | 7462 #endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H |
| OLD | NEW |