| OLD | NEW |
| 1 //===- subzero/src/IceTargetLoweringX8632.cpp - x86-32 lowering -----------===// | 1 //===- subzero/src/IceTargetLoweringX8632.cpp - x86-32 lowering -----------===// |
| 2 // | 2 // |
| 3 // The Subzero Code Generator | 3 // The Subzero Code Generator |
| 4 // | 4 // |
| 5 // This file is distributed under the University of Illinois Open Source | 5 // This file is distributed under the University of Illinois Open Source |
| 6 // License. See LICENSE.TXT for details. | 6 // License. See LICENSE.TXT for details. |
| 7 // | 7 // |
| 8 //===----------------------------------------------------------------------===// | 8 //===----------------------------------------------------------------------===// |
| 9 // | 9 // |
| 10 // This file implements the TargetLoweringX8632 class, which | 10 // This file implements the TargetLoweringX8632 class, which |
| (...skipping 447 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 458 #undef X | 458 #undef X |
| 459 }; | 459 }; |
| 460 | 460 |
| 461 Variable *TargetX8632::getPhysicalRegister(SizeT RegNum, Type Ty) { | 461 Variable *TargetX8632::getPhysicalRegister(SizeT RegNum, Type Ty) { |
| 462 if (Ty == IceType_void) | 462 if (Ty == IceType_void) |
| 463 Ty = IceType_i32; | 463 Ty = IceType_i32; |
| 464 if (PhysicalRegisters[Ty].empty()) | 464 if (PhysicalRegisters[Ty].empty()) |
| 465 PhysicalRegisters[Ty].resize(RegX8632::Reg_NUM); | 465 PhysicalRegisters[Ty].resize(RegX8632::Reg_NUM); |
| 466 assert(RegNum < PhysicalRegisters[Ty].size()); | 466 assert(RegNum < PhysicalRegisters[Ty].size()); |
| 467 Variable *Reg = PhysicalRegisters[Ty][RegNum]; | 467 Variable *Reg = PhysicalRegisters[Ty][RegNum]; |
| 468 if (Reg == NULL) { | 468 if (Reg == nullptr) { |
| 469 Reg = Func->makeVariable(Ty); | 469 Reg = Func->makeVariable(Ty); |
| 470 Reg->setRegNum(RegNum); | 470 Reg->setRegNum(RegNum); |
| 471 PhysicalRegisters[Ty][RegNum] = Reg; | 471 PhysicalRegisters[Ty][RegNum] = Reg; |
| 472 // Specially mark esp as an "argument" so that it is considered | 472 // Specially mark esp as an "argument" so that it is considered |
| 473 // live upon function entry. | 473 // live upon function entry. |
| 474 if (RegNum == RegX8632::Reg_esp) { | 474 if (RegNum == RegX8632::Reg_esp) { |
| 475 Func->addImplicitArg(Reg); | 475 Func->addImplicitArg(Reg); |
| 476 Reg->setIgnoreLiveness(); | 476 Reg->setIgnoreLiveness(); |
| 477 } | 477 } |
| 478 } | 478 } |
| (...skipping 564 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1043 // is still typed as I32. | 1043 // is still typed as I32. |
| 1044 case IceType_f64: | 1044 case IceType_f64: |
| 1045 break; | 1045 break; |
| 1046 } | 1046 } |
| 1047 Variable *Lo = Var->getLo(); | 1047 Variable *Lo = Var->getLo(); |
| 1048 Variable *Hi = Var->getHi(); | 1048 Variable *Hi = Var->getHi(); |
| 1049 if (Lo) { | 1049 if (Lo) { |
| 1050 assert(Hi); | 1050 assert(Hi); |
| 1051 return; | 1051 return; |
| 1052 } | 1052 } |
| 1053 assert(Hi == NULL); | 1053 assert(Hi == nullptr); |
| 1054 Lo = Func->makeVariable(IceType_i32); | 1054 Lo = Func->makeVariable(IceType_i32); |
| 1055 Hi = Func->makeVariable(IceType_i32); | 1055 Hi = Func->makeVariable(IceType_i32); |
| 1056 if (ALLOW_DUMP) { | 1056 if (ALLOW_DUMP) { |
| 1057 Lo->setName(Func, Var->getName(Func) + "__lo"); | 1057 Lo->setName(Func, Var->getName(Func) + "__lo"); |
| 1058 Hi->setName(Func, Var->getName(Func) + "__hi"); | 1058 Hi->setName(Func, Var->getName(Func) + "__hi"); |
| 1059 } | 1059 } |
| 1060 Var->setLoHi(Lo, Hi); | 1060 Var->setLoHi(Lo, Hi); |
| 1061 if (Var->getIsArg()) { | 1061 if (Var->getIsArg()) { |
| 1062 Lo->setIsArg(); | 1062 Lo->setIsArg(); |
| 1063 Hi->setIsArg(); | 1063 Hi->setIsArg(); |
| (...skipping 10 matching lines...) Expand all Loading... |
| 1074 } | 1074 } |
| 1075 if (ConstantInteger64 *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { | 1075 if (ConstantInteger64 *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { |
| 1076 return Ctx->getConstantInt32(static_cast<uint32_t>(Const->getValue())); | 1076 return Ctx->getConstantInt32(static_cast<uint32_t>(Const->getValue())); |
| 1077 } | 1077 } |
| 1078 if (OperandX8632Mem *Mem = llvm::dyn_cast<OperandX8632Mem>(Operand)) { | 1078 if (OperandX8632Mem *Mem = llvm::dyn_cast<OperandX8632Mem>(Operand)) { |
| 1079 return OperandX8632Mem::create(Func, IceType_i32, Mem->getBase(), | 1079 return OperandX8632Mem::create(Func, IceType_i32, Mem->getBase(), |
| 1080 Mem->getOffset(), Mem->getIndex(), | 1080 Mem->getOffset(), Mem->getIndex(), |
| 1081 Mem->getShift(), Mem->getSegmentRegister()); | 1081 Mem->getShift(), Mem->getSegmentRegister()); |
| 1082 } | 1082 } |
| 1083 llvm_unreachable("Unsupported operand type"); | 1083 llvm_unreachable("Unsupported operand type"); |
| 1084 return NULL; | 1084 return nullptr; |
| 1085 } | 1085 } |
| 1086 | 1086 |
| 1087 Operand *TargetX8632::hiOperand(Operand *Operand) { | 1087 Operand *TargetX8632::hiOperand(Operand *Operand) { |
| 1088 assert(Operand->getType() == IceType_i64); | 1088 assert(Operand->getType() == IceType_i64); |
| 1089 if (Operand->getType() != IceType_i64) | 1089 if (Operand->getType() != IceType_i64) |
| 1090 return Operand; | 1090 return Operand; |
| 1091 if (Variable *Var = llvm::dyn_cast<Variable>(Operand)) { | 1091 if (Variable *Var = llvm::dyn_cast<Variable>(Operand)) { |
| 1092 split64(Var); | 1092 split64(Var); |
| 1093 return Var->getHi(); | 1093 return Var->getHi(); |
| 1094 } | 1094 } |
| 1095 if (ConstantInteger64 *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { | 1095 if (ConstantInteger64 *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { |
| 1096 return Ctx->getConstantInt32( | 1096 return Ctx->getConstantInt32( |
| 1097 static_cast<uint32_t>(Const->getValue() >> 32)); | 1097 static_cast<uint32_t>(Const->getValue() >> 32)); |
| 1098 } | 1098 } |
| 1099 if (OperandX8632Mem *Mem = llvm::dyn_cast<OperandX8632Mem>(Operand)) { | 1099 if (OperandX8632Mem *Mem = llvm::dyn_cast<OperandX8632Mem>(Operand)) { |
| 1100 Constant *Offset = Mem->getOffset(); | 1100 Constant *Offset = Mem->getOffset(); |
| 1101 if (Offset == NULL) { | 1101 if (Offset == nullptr) { |
| 1102 Offset = Ctx->getConstantInt32(4); | 1102 Offset = Ctx->getConstantInt32(4); |
| 1103 } else if (ConstantInteger32 *IntOffset = | 1103 } else if (ConstantInteger32 *IntOffset = |
| 1104 llvm::dyn_cast<ConstantInteger32>(Offset)) { | 1104 llvm::dyn_cast<ConstantInteger32>(Offset)) { |
| 1105 Offset = Ctx->getConstantInt32(4 + IntOffset->getValue()); | 1105 Offset = Ctx->getConstantInt32(4 + IntOffset->getValue()); |
| 1106 } else if (ConstantRelocatable *SymOffset = | 1106 } else if (ConstantRelocatable *SymOffset = |
| 1107 llvm::dyn_cast<ConstantRelocatable>(Offset)) { | 1107 llvm::dyn_cast<ConstantRelocatable>(Offset)) { |
| 1108 assert(!Utils::WouldOverflowAdd(SymOffset->getOffset(), 4)); | 1108 assert(!Utils::WouldOverflowAdd(SymOffset->getOffset(), 4)); |
| 1109 Offset = | 1109 Offset = |
| 1110 Ctx->getConstantSym(4 + SymOffset->getOffset(), SymOffset->getName(), | 1110 Ctx->getConstantSym(4 + SymOffset->getOffset(), SymOffset->getName(), |
| 1111 SymOffset->getSuppressMangling()); | 1111 SymOffset->getSuppressMangling()); |
| 1112 } | 1112 } |
| 1113 return OperandX8632Mem::create(Func, IceType_i32, Mem->getBase(), Offset, | 1113 return OperandX8632Mem::create(Func, IceType_i32, Mem->getBase(), Offset, |
| 1114 Mem->getIndex(), Mem->getShift(), | 1114 Mem->getIndex(), Mem->getShift(), |
| 1115 Mem->getSegmentRegister()); | 1115 Mem->getSegmentRegister()); |
| 1116 } | 1116 } |
| 1117 llvm_unreachable("Unsupported operand type"); | 1117 llvm_unreachable("Unsupported operand type"); |
| 1118 return NULL; | 1118 return nullptr; |
| 1119 } | 1119 } |
| 1120 | 1120 |
| 1121 llvm::SmallBitVector TargetX8632::getRegisterSet(RegSetMask Include, | 1121 llvm::SmallBitVector TargetX8632::getRegisterSet(RegSetMask Include, |
| 1122 RegSetMask Exclude) const { | 1122 RegSetMask Exclude) const { |
| 1123 llvm::SmallBitVector Registers(RegX8632::Reg_NUM); | 1123 llvm::SmallBitVector Registers(RegX8632::Reg_NUM); |
| 1124 | 1124 |
| 1125 #define X(val, encode, name, name16, name8, scratch, preserved, stackptr, \ | 1125 #define X(val, encode, name, name16, name8, scratch, preserved, stackptr, \ |
| 1126 frameptr, isI8, isInt, isFP) \ | 1126 frameptr, isI8, isInt, isFP) \ |
| 1127 if (scratch && (Include & RegSet_CallerSave)) \ | 1127 if (scratch && (Include & RegSet_CallerSave)) \ |
| 1128 Registers[RegX8632::val] = true; \ | 1128 Registers[RegX8632::val] = true; \ |
| (...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1195 Variable *Dest = Inst->getDest(); | 1195 Variable *Dest = Inst->getDest(); |
| 1196 Operand *Src0 = legalize(Inst->getSrc(0)); | 1196 Operand *Src0 = legalize(Inst->getSrc(0)); |
| 1197 Operand *Src1 = legalize(Inst->getSrc(1)); | 1197 Operand *Src1 = legalize(Inst->getSrc(1)); |
| 1198 if (Dest->getType() == IceType_i64) { | 1198 if (Dest->getType() == IceType_i64) { |
| 1199 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 1199 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
| 1200 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 1200 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
| 1201 Operand *Src0Lo = loOperand(Src0); | 1201 Operand *Src0Lo = loOperand(Src0); |
| 1202 Operand *Src0Hi = hiOperand(Src0); | 1202 Operand *Src0Hi = hiOperand(Src0); |
| 1203 Operand *Src1Lo = loOperand(Src1); | 1203 Operand *Src1Lo = loOperand(Src1); |
| 1204 Operand *Src1Hi = hiOperand(Src1); | 1204 Operand *Src1Hi = hiOperand(Src1); |
| 1205 Variable *T_Lo = NULL, *T_Hi = NULL; | 1205 Variable *T_Lo = nullptr, *T_Hi = nullptr; |
| 1206 switch (Inst->getOp()) { | 1206 switch (Inst->getOp()) { |
| 1207 case InstArithmetic::_num: | 1207 case InstArithmetic::_num: |
| 1208 llvm_unreachable("Unknown arithmetic operator"); | 1208 llvm_unreachable("Unknown arithmetic operator"); |
| 1209 break; | 1209 break; |
| 1210 case InstArithmetic::Add: | 1210 case InstArithmetic::Add: |
| 1211 _mov(T_Lo, Src0Lo); | 1211 _mov(T_Lo, Src0Lo); |
| 1212 _add(T_Lo, Src1Lo); | 1212 _add(T_Lo, Src1Lo); |
| 1213 _mov(DestLo, T_Lo); | 1213 _mov(DestLo, T_Lo); |
| 1214 _mov(T_Hi, Src0Hi); | 1214 _mov(T_Hi, Src0Hi); |
| 1215 _adc(T_Hi, Src1Hi); | 1215 _adc(T_Hi, Src1Hi); |
| (...skipping 25 matching lines...) Expand all Loading... |
| 1241 break; | 1241 break; |
| 1242 case InstArithmetic::Sub: | 1242 case InstArithmetic::Sub: |
| 1243 _mov(T_Lo, Src0Lo); | 1243 _mov(T_Lo, Src0Lo); |
| 1244 _sub(T_Lo, Src1Lo); | 1244 _sub(T_Lo, Src1Lo); |
| 1245 _mov(DestLo, T_Lo); | 1245 _mov(DestLo, T_Lo); |
| 1246 _mov(T_Hi, Src0Hi); | 1246 _mov(T_Hi, Src0Hi); |
| 1247 _sbb(T_Hi, Src1Hi); | 1247 _sbb(T_Hi, Src1Hi); |
| 1248 _mov(DestHi, T_Hi); | 1248 _mov(DestHi, T_Hi); |
| 1249 break; | 1249 break; |
| 1250 case InstArithmetic::Mul: { | 1250 case InstArithmetic::Mul: { |
| 1251 Variable *T_1 = NULL, *T_2 = NULL, *T_3 = NULL; | 1251 Variable *T_1 = nullptr, *T_2 = nullptr, *T_3 = nullptr; |
| 1252 Variable *T_4Lo = makeReg(IceType_i32, RegX8632::Reg_eax); | 1252 Variable *T_4Lo = makeReg(IceType_i32, RegX8632::Reg_eax); |
| 1253 Variable *T_4Hi = makeReg(IceType_i32, RegX8632::Reg_edx); | 1253 Variable *T_4Hi = makeReg(IceType_i32, RegX8632::Reg_edx); |
| 1254 // gcc does the following: | 1254 // gcc does the following: |
| 1255 // a=b*c ==> | 1255 // a=b*c ==> |
| 1256 // t1 = b.hi; t1 *=(imul) c.lo | 1256 // t1 = b.hi; t1 *=(imul) c.lo |
| 1257 // t2 = c.hi; t2 *=(imul) b.lo | 1257 // t2 = c.hi; t2 *=(imul) b.lo |
| 1258 // t3:eax = b.lo | 1258 // t3:eax = b.lo |
| 1259 // t4.hi:edx,t4.lo:eax = t3:eax *(mul) c.lo | 1259 // t4.hi:edx,t4.lo:eax = t3:eax *(mul) c.lo |
| 1260 // a.lo = t4.lo | 1260 // a.lo = t4.lo |
| 1261 // t4.hi += t1 | 1261 // t4.hi += t1 |
| (...skipping 25 matching lines...) Expand all Loading... |
| 1287 // t3 = shld t3, t2, t1 | 1287 // t3 = shld t3, t2, t1 |
| 1288 // t2 = shl t2, t1 | 1288 // t2 = shl t2, t1 |
| 1289 // test t1, 0x20 | 1289 // test t1, 0x20 |
| 1290 // je L1 | 1290 // je L1 |
| 1291 // use(t3) | 1291 // use(t3) |
| 1292 // t3 = t2 | 1292 // t3 = t2 |
| 1293 // t2 = 0 | 1293 // t2 = 0 |
| 1294 // L1: | 1294 // L1: |
| 1295 // a.lo = t2 | 1295 // a.lo = t2 |
| 1296 // a.hi = t3 | 1296 // a.hi = t3 |
| 1297 Variable *T_1 = NULL, *T_2 = NULL, *T_3 = NULL; | 1297 Variable *T_1 = nullptr, *T_2 = nullptr, *T_3 = nullptr; |
| 1298 Constant *BitTest = Ctx->getConstantInt32(0x20); | 1298 Constant *BitTest = Ctx->getConstantInt32(0x20); |
| 1299 Constant *Zero = Ctx->getConstantZero(IceType_i32); | 1299 Constant *Zero = Ctx->getConstantZero(IceType_i32); |
| 1300 InstX8632Label *Label = InstX8632Label::create(Func, this); | 1300 InstX8632Label *Label = InstX8632Label::create(Func, this); |
| 1301 _mov(T_1, Src1Lo, RegX8632::Reg_ecx); | 1301 _mov(T_1, Src1Lo, RegX8632::Reg_ecx); |
| 1302 _mov(T_2, Src0Lo); | 1302 _mov(T_2, Src0Lo); |
| 1303 _mov(T_3, Src0Hi); | 1303 _mov(T_3, Src0Hi); |
| 1304 _shld(T_3, T_2, T_1); | 1304 _shld(T_3, T_2, T_1); |
| 1305 _shl(T_2, T_1); | 1305 _shl(T_2, T_1); |
| 1306 _test(T_1, BitTest); | 1306 _test(T_1, BitTest); |
| 1307 _br(CondX86::Br_e, Label); | 1307 _br(CondX86::Br_e, Label); |
| (...skipping 14 matching lines...) Expand all Loading... |
| 1322 // t2 = shrd t2, t3, t1 | 1322 // t2 = shrd t2, t3, t1 |
| 1323 // t3 = shr t3, t1 | 1323 // t3 = shr t3, t1 |
| 1324 // test t1, 0x20 | 1324 // test t1, 0x20 |
| 1325 // je L1 | 1325 // je L1 |
| 1326 // use(t2) | 1326 // use(t2) |
| 1327 // t2 = t3 | 1327 // t2 = t3 |
| 1328 // t3 = 0 | 1328 // t3 = 0 |
| 1329 // L1: | 1329 // L1: |
| 1330 // a.lo = t2 | 1330 // a.lo = t2 |
| 1331 // a.hi = t3 | 1331 // a.hi = t3 |
| 1332 Variable *T_1 = NULL, *T_2 = NULL, *T_3 = NULL; | 1332 Variable *T_1 = nullptr, *T_2 = nullptr, *T_3 = nullptr; |
| 1333 Constant *BitTest = Ctx->getConstantInt32(0x20); | 1333 Constant *BitTest = Ctx->getConstantInt32(0x20); |
| 1334 Constant *Zero = Ctx->getConstantZero(IceType_i32); | 1334 Constant *Zero = Ctx->getConstantZero(IceType_i32); |
| 1335 InstX8632Label *Label = InstX8632Label::create(Func, this); | 1335 InstX8632Label *Label = InstX8632Label::create(Func, this); |
| 1336 _mov(T_1, Src1Lo, RegX8632::Reg_ecx); | 1336 _mov(T_1, Src1Lo, RegX8632::Reg_ecx); |
| 1337 _mov(T_2, Src0Lo); | 1337 _mov(T_2, Src0Lo); |
| 1338 _mov(T_3, Src0Hi); | 1338 _mov(T_3, Src0Hi); |
| 1339 _shrd(T_2, T_3, T_1); | 1339 _shrd(T_2, T_3, T_1); |
| 1340 _shr(T_3, T_1); | 1340 _shr(T_3, T_1); |
| 1341 _test(T_1, BitTest); | 1341 _test(T_1, BitTest); |
| 1342 _br(CondX86::Br_e, Label); | 1342 _br(CondX86::Br_e, Label); |
| (...skipping 14 matching lines...) Expand all Loading... |
| 1357 // t2 = shrd t2, t3, t1 | 1357 // t2 = shrd t2, t3, t1 |
| 1358 // t3 = sar t3, t1 | 1358 // t3 = sar t3, t1 |
| 1359 // test t1, 0x20 | 1359 // test t1, 0x20 |
| 1360 // je L1 | 1360 // je L1 |
| 1361 // use(t2) | 1361 // use(t2) |
| 1362 // t2 = t3 | 1362 // t2 = t3 |
| 1363 // t3 = sar t3, 0x1f | 1363 // t3 = sar t3, 0x1f |
| 1364 // L1: | 1364 // L1: |
| 1365 // a.lo = t2 | 1365 // a.lo = t2 |
| 1366 // a.hi = t3 | 1366 // a.hi = t3 |
| 1367 Variable *T_1 = NULL, *T_2 = NULL, *T_3 = NULL; | 1367 Variable *T_1 = nullptr, *T_2 = nullptr, *T_3 = nullptr; |
| 1368 Constant *BitTest = Ctx->getConstantInt32(0x20); | 1368 Constant *BitTest = Ctx->getConstantInt32(0x20); |
| 1369 Constant *SignExtend = Ctx->getConstantInt32(0x1f); | 1369 Constant *SignExtend = Ctx->getConstantInt32(0x1f); |
| 1370 InstX8632Label *Label = InstX8632Label::create(Func, this); | 1370 InstX8632Label *Label = InstX8632Label::create(Func, this); |
| 1371 _mov(T_1, Src1Lo, RegX8632::Reg_ecx); | 1371 _mov(T_1, Src1Lo, RegX8632::Reg_ecx); |
| 1372 _mov(T_2, Src0Lo); | 1372 _mov(T_2, Src0Lo); |
| 1373 _mov(T_3, Src0Hi); | 1373 _mov(T_3, Src0Hi); |
| 1374 _shrd(T_2, T_3, T_1); | 1374 _shrd(T_2, T_3, T_1); |
| 1375 _sar(T_3, T_1); | 1375 _sar(T_3, T_1); |
| 1376 _test(T_1, BitTest); | 1376 _test(T_1, BitTest); |
| 1377 _br(CondX86::Br_e, Label); | 1377 _br(CondX86::Br_e, Label); |
| (...skipping 164 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1542 Variable *T = makeReg(Dest->getType()); | 1542 Variable *T = makeReg(Dest->getType()); |
| 1543 _movp(T, Src0); | 1543 _movp(T, Src0); |
| 1544 _divps(T, Src1); | 1544 _divps(T, Src1); |
| 1545 _movp(Dest, T); | 1545 _movp(Dest, T); |
| 1546 } break; | 1546 } break; |
| 1547 case InstArithmetic::Frem: | 1547 case InstArithmetic::Frem: |
| 1548 scalarizeArithmetic(Inst->getOp(), Dest, Src0, Src1); | 1548 scalarizeArithmetic(Inst->getOp(), Dest, Src0, Src1); |
| 1549 break; | 1549 break; |
| 1550 } | 1550 } |
| 1551 } else { // Dest->getType() is non-i64 scalar | 1551 } else { // Dest->getType() is non-i64 scalar |
| 1552 Variable *T_edx = NULL; | 1552 Variable *T_edx = nullptr; |
| 1553 Variable *T = NULL; | 1553 Variable *T = nullptr; |
| 1554 switch (Inst->getOp()) { | 1554 switch (Inst->getOp()) { |
| 1555 case InstArithmetic::_num: | 1555 case InstArithmetic::_num: |
| 1556 llvm_unreachable("Unknown arithmetic operator"); | 1556 llvm_unreachable("Unknown arithmetic operator"); |
| 1557 break; | 1557 break; |
| 1558 case InstArithmetic::Add: | 1558 case InstArithmetic::Add: |
| 1559 _mov(T, Src0); | 1559 _mov(T, Src0); |
| 1560 _add(T, Src1); | 1560 _add(T, Src1); |
| 1561 _mov(Dest, T); | 1561 _mov(Dest, T); |
| 1562 break; | 1562 break; |
| 1563 case InstArithmetic::And: | 1563 case InstArithmetic::And: |
| (...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1616 if (!llvm::isa<Constant>(Src1)) | 1616 if (!llvm::isa<Constant>(Src1)) |
| 1617 Src1 = legalizeToVar(Src1, RegX8632::Reg_ecx); | 1617 Src1 = legalizeToVar(Src1, RegX8632::Reg_ecx); |
| 1618 _sar(T, Src1); | 1618 _sar(T, Src1); |
| 1619 _mov(Dest, T); | 1619 _mov(Dest, T); |
| 1620 break; | 1620 break; |
| 1621 case InstArithmetic::Udiv: | 1621 case InstArithmetic::Udiv: |
| 1622 // div and idiv are the few arithmetic operators that do not allow | 1622 // div and idiv are the few arithmetic operators that do not allow |
| 1623 // immediates as the operand. | 1623 // immediates as the operand. |
| 1624 Src1 = legalize(Src1, Legal_Reg | Legal_Mem); | 1624 Src1 = legalize(Src1, Legal_Reg | Legal_Mem); |
| 1625 if (isByteSizedArithType(Dest->getType())) { | 1625 if (isByteSizedArithType(Dest->getType())) { |
| 1626 Variable *T_ah = NULL; | 1626 Variable *T_ah = nullptr; |
| 1627 Constant *Zero = Ctx->getConstantZero(IceType_i8); | 1627 Constant *Zero = Ctx->getConstantZero(IceType_i8); |
| 1628 _mov(T, Src0, RegX8632::Reg_eax); | 1628 _mov(T, Src0, RegX8632::Reg_eax); |
| 1629 _mov(T_ah, Zero, RegX8632::Reg_ah); | 1629 _mov(T_ah, Zero, RegX8632::Reg_ah); |
| 1630 _div(T, Src1, T_ah); | 1630 _div(T, Src1, T_ah); |
| 1631 _mov(Dest, T); | 1631 _mov(Dest, T); |
| 1632 } else { | 1632 } else { |
| 1633 Constant *Zero = Ctx->getConstantZero(IceType_i32); | 1633 Constant *Zero = Ctx->getConstantZero(IceType_i32); |
| 1634 _mov(T, Src0, RegX8632::Reg_eax); | 1634 _mov(T, Src0, RegX8632::Reg_eax); |
| 1635 _mov(T_edx, Zero, RegX8632::Reg_edx); | 1635 _mov(T_edx, Zero, RegX8632::Reg_edx); |
| 1636 _div(T, Src1, T_edx); | 1636 _div(T, Src1, T_edx); |
| (...skipping 11 matching lines...) Expand all Loading... |
| 1648 T_edx = makeReg(IceType_i32, RegX8632::Reg_edx); | 1648 T_edx = makeReg(IceType_i32, RegX8632::Reg_edx); |
| 1649 _mov(T, Src0, RegX8632::Reg_eax); | 1649 _mov(T, Src0, RegX8632::Reg_eax); |
| 1650 _cbwdq(T_edx, T); | 1650 _cbwdq(T_edx, T); |
| 1651 _idiv(T, Src1, T_edx); | 1651 _idiv(T, Src1, T_edx); |
| 1652 _mov(Dest, T); | 1652 _mov(Dest, T); |
| 1653 } | 1653 } |
| 1654 break; | 1654 break; |
| 1655 case InstArithmetic::Urem: | 1655 case InstArithmetic::Urem: |
| 1656 Src1 = legalize(Src1, Legal_Reg | Legal_Mem); | 1656 Src1 = legalize(Src1, Legal_Reg | Legal_Mem); |
| 1657 if (isByteSizedArithType(Dest->getType())) { | 1657 if (isByteSizedArithType(Dest->getType())) { |
| 1658 Variable *T_ah = NULL; | 1658 Variable *T_ah = nullptr; |
| 1659 Constant *Zero = Ctx->getConstantZero(IceType_i8); | 1659 Constant *Zero = Ctx->getConstantZero(IceType_i8); |
| 1660 _mov(T, Src0, RegX8632::Reg_eax); | 1660 _mov(T, Src0, RegX8632::Reg_eax); |
| 1661 _mov(T_ah, Zero, RegX8632::Reg_ah); | 1661 _mov(T_ah, Zero, RegX8632::Reg_ah); |
| 1662 _div(T_ah, Src1, T); | 1662 _div(T_ah, Src1, T); |
| 1663 _mov(Dest, T_ah); | 1663 _mov(Dest, T_ah); |
| 1664 } else { | 1664 } else { |
| 1665 Constant *Zero = Ctx->getConstantZero(IceType_i32); | 1665 Constant *Zero = Ctx->getConstantZero(IceType_i32); |
| 1666 _mov(T_edx, Zero, RegX8632::Reg_edx); | 1666 _mov(T_edx, Zero, RegX8632::Reg_edx); |
| 1667 _mov(T, Src0, RegX8632::Reg_eax); | 1667 _mov(T, Src0, RegX8632::Reg_eax); |
| 1668 _div(T_edx, Src1, T); | 1668 _div(T_edx, Src1, T); |
| (...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1722 void TargetX8632::lowerAssign(const InstAssign *Inst) { | 1722 void TargetX8632::lowerAssign(const InstAssign *Inst) { |
| 1723 Variable *Dest = Inst->getDest(); | 1723 Variable *Dest = Inst->getDest(); |
| 1724 Operand *Src0 = Inst->getSrc(0); | 1724 Operand *Src0 = Inst->getSrc(0); |
| 1725 assert(Dest->getType() == Src0->getType()); | 1725 assert(Dest->getType() == Src0->getType()); |
| 1726 if (Dest->getType() == IceType_i64) { | 1726 if (Dest->getType() == IceType_i64) { |
| 1727 Src0 = legalize(Src0); | 1727 Src0 = legalize(Src0); |
| 1728 Operand *Src0Lo = loOperand(Src0); | 1728 Operand *Src0Lo = loOperand(Src0); |
| 1729 Operand *Src0Hi = hiOperand(Src0); | 1729 Operand *Src0Hi = hiOperand(Src0); |
| 1730 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 1730 Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); |
| 1731 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 1731 Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |
| 1732 Variable *T_Lo = NULL, *T_Hi = NULL; | 1732 Variable *T_Lo = nullptr, *T_Hi = nullptr; |
| 1733 _mov(T_Lo, Src0Lo); | 1733 _mov(T_Lo, Src0Lo); |
| 1734 _mov(DestLo, T_Lo); | 1734 _mov(DestLo, T_Lo); |
| 1735 _mov(T_Hi, Src0Hi); | 1735 _mov(T_Hi, Src0Hi); |
| 1736 _mov(DestHi, T_Hi); | 1736 _mov(DestHi, T_Hi); |
| 1737 } else { | 1737 } else { |
| 1738 // If Dest is in memory, then RI is either a physical register or | 1738 // If Dest is in memory, then RI is either a physical register or |
| 1739 // an immediate, otherwise RI can be anything. | 1739 // an immediate, otherwise RI can be anything. |
| 1740 Operand *RI = | 1740 Operand *RI = |
| 1741 legalize(Src0, Dest->hasReg() ? Legal_All : Legal_Reg | Legal_Imm); | 1741 legalize(Src0, Dest->hasReg() ? Legal_All : Legal_Reg | Legal_Imm); |
| 1742 if (isVectorType(Dest->getType())) | 1742 if (isVectorType(Dest->getType())) |
| (...skipping 97 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1840 Variable *Reg = legalizeToVar(XmmArgs[i], RegX8632::Reg_xmm0 + i); | 1840 Variable *Reg = legalizeToVar(XmmArgs[i], RegX8632::Reg_xmm0 + i); |
| 1841 // Generate a FakeUse of register arguments so that they do not get | 1841 // Generate a FakeUse of register arguments so that they do not get |
| 1842 // dead code eliminated as a result of the FakeKill of scratch | 1842 // dead code eliminated as a result of the FakeKill of scratch |
| 1843 // registers after the call. | 1843 // registers after the call. |
| 1844 Context.insert(InstFakeUse::create(Func, Reg)); | 1844 Context.insert(InstFakeUse::create(Func, Reg)); |
| 1845 } | 1845 } |
| 1846 // Generate the call instruction. Assign its result to a temporary | 1846 // Generate the call instruction. Assign its result to a temporary |
| 1847 // with high register allocation weight. | 1847 // with high register allocation weight. |
| 1848 Variable *Dest = Instr->getDest(); | 1848 Variable *Dest = Instr->getDest(); |
| 1849 // ReturnReg doubles as ReturnRegLo as necessary. | 1849 // ReturnReg doubles as ReturnRegLo as necessary. |
| 1850 Variable *ReturnReg = NULL; | 1850 Variable *ReturnReg = nullptr; |
| 1851 Variable *ReturnRegHi = NULL; | 1851 Variable *ReturnRegHi = nullptr; |
| 1852 if (Dest) { | 1852 if (Dest) { |
| 1853 switch (Dest->getType()) { | 1853 switch (Dest->getType()) { |
| 1854 case IceType_NUM: | 1854 case IceType_NUM: |
| 1855 llvm_unreachable("Invalid Call dest type"); | 1855 llvm_unreachable("Invalid Call dest type"); |
| 1856 break; | 1856 break; |
| 1857 case IceType_void: | 1857 case IceType_void: |
| 1858 break; | 1858 break; |
| 1859 case IceType_i1: | 1859 case IceType_i1: |
| 1860 case IceType_i8: | 1860 case IceType_i8: |
| 1861 case IceType_i16: | 1861 case IceType_i16: |
| 1862 case IceType_i32: | 1862 case IceType_i32: |
| 1863 ReturnReg = makeReg(Dest->getType(), RegX8632::Reg_eax); | 1863 ReturnReg = makeReg(Dest->getType(), RegX8632::Reg_eax); |
| 1864 break; | 1864 break; |
| 1865 case IceType_i64: | 1865 case IceType_i64: |
| 1866 ReturnReg = makeReg(IceType_i32, RegX8632::Reg_eax); | 1866 ReturnReg = makeReg(IceType_i32, RegX8632::Reg_eax); |
| 1867 ReturnRegHi = makeReg(IceType_i32, RegX8632::Reg_edx); | 1867 ReturnRegHi = makeReg(IceType_i32, RegX8632::Reg_edx); |
| 1868 break; | 1868 break; |
| 1869 case IceType_f32: | 1869 case IceType_f32: |
| 1870 case IceType_f64: | 1870 case IceType_f64: |
| 1871 // Leave ReturnReg==ReturnRegHi==NULL, and capture the result with | 1871 // Leave ReturnReg==ReturnRegHi==nullptr, and capture the result with |
| 1872 // the fstp instruction. | 1872 // the fstp instruction. |
| 1873 break; | 1873 break; |
| 1874 case IceType_v4i1: | 1874 case IceType_v4i1: |
| 1875 case IceType_v8i1: | 1875 case IceType_v8i1: |
| 1876 case IceType_v16i1: | 1876 case IceType_v16i1: |
| 1877 case IceType_v16i8: | 1877 case IceType_v16i8: |
| 1878 case IceType_v8i16: | 1878 case IceType_v8i16: |
| 1879 case IceType_v4i32: | 1879 case IceType_v4i32: |
| 1880 case IceType_v4f32: | 1880 case IceType_v4f32: |
| 1881 ReturnReg = makeReg(Dest->getType(), RegX8632::Reg_xmm0); | 1881 ReturnReg = makeReg(Dest->getType(), RegX8632::Reg_xmm0); |
| (...skipping 103 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1985 if (Src0RM->getType() == IceType_i32) { | 1985 if (Src0RM->getType() == IceType_i32) { |
| 1986 _mov(T_Lo, Src0RM); | 1986 _mov(T_Lo, Src0RM); |
| 1987 } else if (Src0RM->getType() == IceType_i1) { | 1987 } else if (Src0RM->getType() == IceType_i1) { |
| 1988 _movzx(T_Lo, Src0RM); | 1988 _movzx(T_Lo, Src0RM); |
| 1989 _shl(T_Lo, Shift); | 1989 _shl(T_Lo, Shift); |
| 1990 _sar(T_Lo, Shift); | 1990 _sar(T_Lo, Shift); |
| 1991 } else { | 1991 } else { |
| 1992 _movsx(T_Lo, Src0RM); | 1992 _movsx(T_Lo, Src0RM); |
| 1993 } | 1993 } |
| 1994 _mov(DestLo, T_Lo); | 1994 _mov(DestLo, T_Lo); |
| 1995 Variable *T_Hi = NULL; | 1995 Variable *T_Hi = nullptr; |
| 1996 _mov(T_Hi, T_Lo); | 1996 _mov(T_Hi, T_Lo); |
| 1997 if (Src0RM->getType() != IceType_i1) | 1997 if (Src0RM->getType() != IceType_i1) |
| 1998 // For i1, the sar instruction is already done above. | 1998 // For i1, the sar instruction is already done above. |
| 1999 _sar(T_Hi, Shift); | 1999 _sar(T_Hi, Shift); |
| 2000 _mov(DestHi, T_Hi); | 2000 _mov(DestHi, T_Hi); |
| 2001 } else if (Src0RM->getType() == IceType_i1) { | 2001 } else if (Src0RM->getType() == IceType_i1) { |
| 2002 // t1 = src | 2002 // t1 = src |
| 2003 // shl t1, dst_bitwidth - 1 | 2003 // shl t1, dst_bitwidth - 1 |
| 2004 // sar t1, dst_bitwidth - 1 | 2004 // sar t1, dst_bitwidth - 1 |
| 2005 // dst = t1 | 2005 // dst = t1 |
| (...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2084 Variable *T = makeReg(Dest->getType()); | 2084 Variable *T = makeReg(Dest->getType()); |
| 2085 _movp(T, Src0RM); | 2085 _movp(T, Src0RM); |
| 2086 _pand(T, OneMask); | 2086 _pand(T, OneMask); |
| 2087 _movp(Dest, T); | 2087 _movp(Dest, T); |
| 2088 } else { | 2088 } else { |
| 2089 Operand *Src0 = Inst->getSrc(0); | 2089 Operand *Src0 = Inst->getSrc(0); |
| 2090 if (Src0->getType() == IceType_i64) | 2090 if (Src0->getType() == IceType_i64) |
| 2091 Src0 = loOperand(Src0); | 2091 Src0 = loOperand(Src0); |
| 2092 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); | 2092 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); |
| 2093 // t1 = trunc Src0RM; Dest = t1 | 2093 // t1 = trunc Src0RM; Dest = t1 |
| 2094 Variable *T = NULL; | 2094 Variable *T = nullptr; |
| 2095 _mov(T, Src0RM); | 2095 _mov(T, Src0RM); |
| 2096 if (Dest->getType() == IceType_i1) | 2096 if (Dest->getType() == IceType_i1) |
| 2097 _and(T, Ctx->getConstantInt1(1)); | 2097 _and(T, Ctx->getConstantInt1(1)); |
| 2098 _mov(Dest, T); | 2098 _mov(Dest, T); |
| 2099 } | 2099 } |
| 2100 break; | 2100 break; |
| 2101 } | 2101 } |
| 2102 case InstCast::Fptrunc: | 2102 case InstCast::Fptrunc: |
| 2103 case InstCast::Fpext: { | 2103 case InstCast::Fpext: { |
| 2104 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); | 2104 Operand *Src0RM = legalize(Inst->getSrc(0), Legal_Reg | Legal_Mem); |
| (...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2280 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); | 2280 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); |
| 2281 Type DestType = Dest->getType(); | 2281 Type DestType = Dest->getType(); |
| 2282 Type SrcType = Src0RM->getType(); | 2282 Type SrcType = Src0RM->getType(); |
| 2283 (void)DestType; | 2283 (void)DestType; |
| 2284 assert((DestType == IceType_i32 && SrcType == IceType_f32) || | 2284 assert((DestType == IceType_i32 && SrcType == IceType_f32) || |
| 2285 (DestType == IceType_f32 && SrcType == IceType_i32)); | 2285 (DestType == IceType_f32 && SrcType == IceType_i32)); |
| 2286 // a.i32 = bitcast b.f32 ==> | 2286 // a.i32 = bitcast b.f32 ==> |
| 2287 // t.f32 = b.f32 | 2287 // t.f32 = b.f32 |
| 2288 // s.f32 = spill t.f32 | 2288 // s.f32 = spill t.f32 |
| 2289 // a.i32 = s.f32 | 2289 // a.i32 = s.f32 |
| 2290 Variable *T = NULL; | 2290 Variable *T = nullptr; |
| 2291 // TODO: Should be able to force a spill setup by calling legalize() with | 2291 // TODO: Should be able to force a spill setup by calling legalize() with |
| 2292 // Legal_Mem and not Legal_Reg or Legal_Imm. | 2292 // Legal_Mem and not Legal_Reg or Legal_Imm. |
| 2293 SpillVariable *SpillVar = Func->makeVariable<SpillVariable>(SrcType); | 2293 SpillVariable *SpillVar = Func->makeVariable<SpillVariable>(SrcType); |
| 2294 SpillVar->setLinkedTo(Dest); | 2294 SpillVar->setLinkedTo(Dest); |
| 2295 Variable *Spill = SpillVar; | 2295 Variable *Spill = SpillVar; |
| 2296 Spill->setWeight(RegWeight::Zero); | 2296 Spill->setWeight(RegWeight::Zero); |
| 2297 _mov(T, Src0RM); | 2297 _mov(T, Src0RM); |
| 2298 _mov(Spill, T); | 2298 _mov(Spill, T); |
| 2299 _mov(Dest, Spill); | 2299 _mov(Dest, Spill); |
| 2300 } break; | 2300 } break; |
| (...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2335 // FakeDef(s.f64) | 2335 // FakeDef(s.f64) |
| 2336 // lo(s.f64) = t_lo.i32 | 2336 // lo(s.f64) = t_lo.i32 |
| 2337 // t_hi.i32 = b_hi.i32 | 2337 // t_hi.i32 = b_hi.i32 |
| 2338 // hi(s.f64) = t_hi.i32 | 2338 // hi(s.f64) = t_hi.i32 |
| 2339 // a.f64 = s.f64 | 2339 // a.f64 = s.f64 |
| 2340 SpillVariable *SpillVar = Func->makeVariable<SpillVariable>(IceType_f64); | 2340 SpillVariable *SpillVar = Func->makeVariable<SpillVariable>(IceType_f64); |
| 2341 SpillVar->setLinkedTo(Dest); | 2341 SpillVar->setLinkedTo(Dest); |
| 2342 Variable *Spill = SpillVar; | 2342 Variable *Spill = SpillVar; |
| 2343 Spill->setWeight(RegWeight::Zero); | 2343 Spill->setWeight(RegWeight::Zero); |
| 2344 | 2344 |
| 2345 Variable *T_Lo = NULL, *T_Hi = NULL; | 2345 Variable *T_Lo = nullptr, *T_Hi = nullptr; |
| 2346 VariableSplit *SpillLo = | 2346 VariableSplit *SpillLo = |
| 2347 VariableSplit::create(Func, Spill, VariableSplit::Low); | 2347 VariableSplit::create(Func, Spill, VariableSplit::Low); |
| 2348 VariableSplit *SpillHi = | 2348 VariableSplit *SpillHi = |
| 2349 VariableSplit::create(Func, Spill, VariableSplit::High); | 2349 VariableSplit::create(Func, Spill, VariableSplit::High); |
| 2350 _mov(T_Lo, loOperand(Src0)); | 2350 _mov(T_Lo, loOperand(Src0)); |
| 2351 // Technically, the Spill is defined after the _store happens, but | 2351 // Technically, the Spill is defined after the _store happens, but |
| 2352 // SpillLo is considered a "use" of Spill so define Spill before it | 2352 // SpillLo is considered a "use" of Spill so define Spill before it |
| 2353 // is used. | 2353 // is used. |
| 2354 Context.insert(InstFakeDef::create(Func, Spill)); | 2354 Context.insert(InstFakeDef::create(Func, Spill)); |
| 2355 _store(T_Lo, SpillLo); | 2355 _store(T_Lo, SpillLo); |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2403 // TODO(wala): Determine the best lowering sequences for each type. | 2403 // TODO(wala): Determine the best lowering sequences for each type. |
| 2404 bool CanUsePextr = | 2404 bool CanUsePextr = |
| 2405 Ty == IceType_v8i16 || Ty == IceType_v8i1 || InstructionSet >= SSE4_1; | 2405 Ty == IceType_v8i16 || Ty == IceType_v8i1 || InstructionSet >= SSE4_1; |
| 2406 if (CanUsePextr && Ty != IceType_v4f32) { | 2406 if (CanUsePextr && Ty != IceType_v4f32) { |
| 2407 // Use pextrb, pextrw, or pextrd. | 2407 // Use pextrb, pextrw, or pextrd. |
| 2408 Constant *Mask = Ctx->getConstantInt32(Index); | 2408 Constant *Mask = Ctx->getConstantInt32(Index); |
| 2409 Variable *SourceVectR = legalizeToVar(SourceVectNotLegalized); | 2409 Variable *SourceVectR = legalizeToVar(SourceVectNotLegalized); |
| 2410 _pextr(ExtractedElementR, SourceVectR, Mask); | 2410 _pextr(ExtractedElementR, SourceVectR, Mask); |
| 2411 } else if (Ty == IceType_v4i32 || Ty == IceType_v4f32 || Ty == IceType_v4i1) { | 2411 } else if (Ty == IceType_v4i32 || Ty == IceType_v4f32 || Ty == IceType_v4i1) { |
| 2412 // Use pshufd and movd/movss. | 2412 // Use pshufd and movd/movss. |
| 2413 Variable *T = NULL; | 2413 Variable *T = nullptr; |
| 2414 if (Index) { | 2414 if (Index) { |
| 2415 // The shuffle only needs to occur if the element to be extracted | 2415 // The shuffle only needs to occur if the element to be extracted |
| 2416 // is not at the lowest index. | 2416 // is not at the lowest index. |
| 2417 Constant *Mask = Ctx->getConstantInt32(Index); | 2417 Constant *Mask = Ctx->getConstantInt32(Index); |
| 2418 T = makeReg(Ty); | 2418 T = makeReg(Ty); |
| 2419 _pshufd(T, legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem), Mask); | 2419 _pshufd(T, legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem), Mask); |
| 2420 } else { | 2420 } else { |
| 2421 T = legalizeToVar(SourceVectNotLegalized); | 2421 T = legalizeToVar(SourceVectNotLegalized); |
| 2422 } | 2422 } |
| 2423 | 2423 |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2472 InstFcmp::FCond Condition = Inst->getCondition(); | 2472 InstFcmp::FCond Condition = Inst->getCondition(); |
| 2473 size_t Index = static_cast<size_t>(Condition); | 2473 size_t Index = static_cast<size_t>(Condition); |
| 2474 assert(Index < TableFcmpSize); | 2474 assert(Index < TableFcmpSize); |
| 2475 | 2475 |
| 2476 if (TableFcmp[Index].SwapVectorOperands) { | 2476 if (TableFcmp[Index].SwapVectorOperands) { |
| 2477 Operand *T = Src0; | 2477 Operand *T = Src0; |
| 2478 Src0 = Src1; | 2478 Src0 = Src1; |
| 2479 Src1 = T; | 2479 Src1 = T; |
| 2480 } | 2480 } |
| 2481 | 2481 |
| 2482 Variable *T = NULL; | 2482 Variable *T = nullptr; |
| 2483 | 2483 |
| 2484 if (Condition == InstFcmp::True) { | 2484 if (Condition == InstFcmp::True) { |
| 2485 // makeVectorOfOnes() requires an integer vector type. | 2485 // makeVectorOfOnes() requires an integer vector type. |
| 2486 T = makeVectorOfMinusOnes(IceType_v4i32); | 2486 T = makeVectorOfMinusOnes(IceType_v4i32); |
| 2487 } else if (Condition == InstFcmp::False) { | 2487 } else if (Condition == InstFcmp::False) { |
| 2488 T = makeVectorOfZeros(Dest->getType()); | 2488 T = makeVectorOfZeros(Dest->getType()); |
| 2489 } else { | 2489 } else { |
| 2490 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); | 2490 Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); |
| 2491 Operand *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); | 2491 Operand *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); |
| 2492 | 2492 |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2541 if (TableFcmp[Index].SwapScalarOperands) { | 2541 if (TableFcmp[Index].SwapScalarOperands) { |
| 2542 Operand *Tmp = Src0; | 2542 Operand *Tmp = Src0; |
| 2543 Src0 = Src1; | 2543 Src0 = Src1; |
| 2544 Src1 = Tmp; | 2544 Src1 = Tmp; |
| 2545 } | 2545 } |
| 2546 bool HasC1 = (TableFcmp[Index].C1 != CondX86::Br_None); | 2546 bool HasC1 = (TableFcmp[Index].C1 != CondX86::Br_None); |
| 2547 bool HasC2 = (TableFcmp[Index].C2 != CondX86::Br_None); | 2547 bool HasC2 = (TableFcmp[Index].C2 != CondX86::Br_None); |
| 2548 if (HasC1) { | 2548 if (HasC1) { |
| 2549 Src0 = legalize(Src0); | 2549 Src0 = legalize(Src0); |
| 2550 Operand *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); | 2550 Operand *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); |
| 2551 Variable *T = NULL; | 2551 Variable *T = nullptr; |
| 2552 _mov(T, Src0); | 2552 _mov(T, Src0); |
| 2553 _ucomiss(T, Src1RM); | 2553 _ucomiss(T, Src1RM); |
| 2554 } | 2554 } |
| 2555 Constant *Default = Ctx->getConstantInt32(TableFcmp[Index].Default); | 2555 Constant *Default = Ctx->getConstantInt32(TableFcmp[Index].Default); |
| 2556 _mov(Dest, Default); | 2556 _mov(Dest, Default); |
| 2557 if (HasC1) { | 2557 if (HasC1) { |
| 2558 InstX8632Label *Label = InstX8632Label::create(Func, this); | 2558 InstX8632Label *Label = InstX8632Label::create(Func, this); |
| 2559 _br(TableFcmp[Index].C1, Label); | 2559 _br(TableFcmp[Index].C1, Label); |
| 2560 if (HasC2) { | 2560 if (HasC2) { |
| 2561 _br(TableFcmp[Index].C2, Label); | 2561 _br(TableFcmp[Index].C2, Label); |
| (...skipping 216 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2778 legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem); | 2778 legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem); |
| 2779 Variable *T = makeReg(Ty); | 2779 Variable *T = makeReg(Ty); |
| 2780 _movp(T, SourceVectRM); | 2780 _movp(T, SourceVectRM); |
| 2781 if (Ty == IceType_v4f32) | 2781 if (Ty == IceType_v4f32) |
| 2782 _insertps(T, ElementRM, Ctx->getConstantInt32(Index << 4)); | 2782 _insertps(T, ElementRM, Ctx->getConstantInt32(Index << 4)); |
| 2783 else | 2783 else |
| 2784 _pinsr(T, ElementRM, Ctx->getConstantInt32(Index)); | 2784 _pinsr(T, ElementRM, Ctx->getConstantInt32(Index)); |
| 2785 _movp(Inst->getDest(), T); | 2785 _movp(Inst->getDest(), T); |
| 2786 } else if (Ty == IceType_v4i32 || Ty == IceType_v4f32 || Ty == IceType_v4i1) { | 2786 } else if (Ty == IceType_v4i32 || Ty == IceType_v4f32 || Ty == IceType_v4i1) { |
| 2787 // Use shufps or movss. | 2787 // Use shufps or movss. |
| 2788 Variable *ElementR = NULL; | 2788 Variable *ElementR = nullptr; |
| 2789 Operand *SourceVectRM = | 2789 Operand *SourceVectRM = |
| 2790 legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem); | 2790 legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem); |
| 2791 | 2791 |
| 2792 if (InVectorElementTy == IceType_f32) { | 2792 if (InVectorElementTy == IceType_f32) { |
| 2793 // ElementR will be in an XMM register since it is floating point. | 2793 // ElementR will be in an XMM register since it is floating point. |
| 2794 ElementR = legalizeToVar(ElementToInsertNotLegalized); | 2794 ElementR = legalizeToVar(ElementToInsertNotLegalized); |
| 2795 } else { | 2795 } else { |
| 2796 // Copy an integer to an XMM register. | 2796 // Copy an integer to an XMM register. |
| 2797 Operand *T = legalize(ElementToInsertNotLegalized, Legal_Reg | Legal_Mem); | 2797 Operand *T = legalize(ElementToInsertNotLegalized, Legal_Reg | Legal_Mem); |
| 2798 ElementR = makeReg(Ty); | 2798 ElementR = makeReg(Ty); |
| (...skipping 226 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3025 _mov(DestLo, T_Hi); | 3025 _mov(DestLo, T_Hi); |
| 3026 _mov(DestHi, T_Lo); | 3026 _mov(DestHi, T_Lo); |
| 3027 } else if (Val->getType() == IceType_i32) { | 3027 } else if (Val->getType() == IceType_i32) { |
| 3028 Variable *T = legalizeToVar(Val); | 3028 Variable *T = legalizeToVar(Val); |
| 3029 _bswap(T); | 3029 _bswap(T); |
| 3030 _mov(Dest, T); | 3030 _mov(Dest, T); |
| 3031 } else { | 3031 } else { |
| 3032 assert(Val->getType() == IceType_i16); | 3032 assert(Val->getType() == IceType_i16); |
| 3033 Val = legalize(Val); | 3033 Val = legalize(Val); |
| 3034 Constant *Eight = Ctx->getConstantInt16(8); | 3034 Constant *Eight = Ctx->getConstantInt16(8); |
| 3035 Variable *T = NULL; | 3035 Variable *T = nullptr; |
| 3036 _mov(T, Val); | 3036 _mov(T, Val); |
| 3037 _rol(T, Eight); | 3037 _rol(T, Eight); |
| 3038 _mov(Dest, T); | 3038 _mov(Dest, T); |
| 3039 } | 3039 } |
| 3040 return; | 3040 return; |
| 3041 } | 3041 } |
| 3042 case Intrinsics::Ctpop: { | 3042 case Intrinsics::Ctpop: { |
| 3043 Variable *Dest = Instr->getDest(); | 3043 Variable *Dest = Instr->getDest(); |
| 3044 Operand *Val = Instr->getArg(0); | 3044 Operand *Val = Instr->getArg(0); |
| 3045 InstCall *Call = | 3045 InstCall *Call = |
| (...skipping 12 matching lines...) Expand all Loading... |
| 3058 Constant *Zero = Ctx->getConstantZero(IceType_i32); | 3058 Constant *Zero = Ctx->getConstantZero(IceType_i32); |
| 3059 _mov(DestHi, Zero); | 3059 _mov(DestHi, Zero); |
| 3060 } | 3060 } |
| 3061 return; | 3061 return; |
| 3062 } | 3062 } |
| 3063 case Intrinsics::Ctlz: { | 3063 case Intrinsics::Ctlz: { |
| 3064 // The "is zero undef" parameter is ignored and we always return | 3064 // The "is zero undef" parameter is ignored and we always return |
| 3065 // a well-defined value. | 3065 // a well-defined value. |
| 3066 Operand *Val = legalize(Instr->getArg(0)); | 3066 Operand *Val = legalize(Instr->getArg(0)); |
| 3067 Operand *FirstVal; | 3067 Operand *FirstVal; |
| 3068 Operand *SecondVal = NULL; | 3068 Operand *SecondVal = nullptr; |
| 3069 if (Val->getType() == IceType_i64) { | 3069 if (Val->getType() == IceType_i64) { |
| 3070 FirstVal = loOperand(Val); | 3070 FirstVal = loOperand(Val); |
| 3071 SecondVal = hiOperand(Val); | 3071 SecondVal = hiOperand(Val); |
| 3072 } else { | 3072 } else { |
| 3073 FirstVal = Val; | 3073 FirstVal = Val; |
| 3074 } | 3074 } |
| 3075 const bool IsCttz = false; | 3075 const bool IsCttz = false; |
| 3076 lowerCountZeros(IsCttz, Val->getType(), Instr->getDest(), FirstVal, | 3076 lowerCountZeros(IsCttz, Val->getType(), Instr->getDest(), FirstVal, |
| 3077 SecondVal); | 3077 SecondVal); |
| 3078 return; | 3078 return; |
| 3079 } | 3079 } |
| 3080 case Intrinsics::Cttz: { | 3080 case Intrinsics::Cttz: { |
| 3081 // The "is zero undef" parameter is ignored and we always return | 3081 // The "is zero undef" parameter is ignored and we always return |
| 3082 // a well-defined value. | 3082 // a well-defined value. |
| 3083 Operand *Val = legalize(Instr->getArg(0)); | 3083 Operand *Val = legalize(Instr->getArg(0)); |
| 3084 Operand *FirstVal; | 3084 Operand *FirstVal; |
| 3085 Operand *SecondVal = NULL; | 3085 Operand *SecondVal = nullptr; |
| 3086 if (Val->getType() == IceType_i64) { | 3086 if (Val->getType() == IceType_i64) { |
| 3087 FirstVal = hiOperand(Val); | 3087 FirstVal = hiOperand(Val); |
| 3088 SecondVal = loOperand(Val); | 3088 SecondVal = loOperand(Val); |
| 3089 } else { | 3089 } else { |
| 3090 FirstVal = Val; | 3090 FirstVal = Val; |
| 3091 } | 3091 } |
| 3092 const bool IsCttz = true; | 3092 const bool IsCttz = true; |
| 3093 lowerCountZeros(IsCttz, Val->getType(), Instr->getDest(), FirstVal, | 3093 lowerCountZeros(IsCttz, Val->getType(), Instr->getDest(), FirstVal, |
| 3094 SecondVal); | 3094 SecondVal); |
| 3095 return; | 3095 return; |
| 3096 } | 3096 } |
| 3097 case Intrinsics::Longjmp: { | 3097 case Intrinsics::Longjmp: { |
| 3098 InstCall *Call = makeHelperCall("longjmp", NULL, 2); | 3098 InstCall *Call = makeHelperCall("longjmp", nullptr, 2); |
| 3099 Call->addArg(Instr->getArg(0)); | 3099 Call->addArg(Instr->getArg(0)); |
| 3100 Call->addArg(Instr->getArg(1)); | 3100 Call->addArg(Instr->getArg(1)); |
| 3101 lowerCall(Call); | 3101 lowerCall(Call); |
| 3102 return; | 3102 return; |
| 3103 } | 3103 } |
| 3104 case Intrinsics::Memcpy: { | 3104 case Intrinsics::Memcpy: { |
| 3105 // In the future, we could potentially emit an inline memcpy/memset, etc. | 3105 // In the future, we could potentially emit an inline memcpy/memset, etc. |
| 3106 // for intrinsic calls w/ a known length. | 3106 // for intrinsic calls w/ a known length. |
| 3107 InstCall *Call = makeHelperCall("memcpy", NULL, 3); | 3107 InstCall *Call = makeHelperCall("memcpy", nullptr, 3); |
| 3108 Call->addArg(Instr->getArg(0)); | 3108 Call->addArg(Instr->getArg(0)); |
| 3109 Call->addArg(Instr->getArg(1)); | 3109 Call->addArg(Instr->getArg(1)); |
| 3110 Call->addArg(Instr->getArg(2)); | 3110 Call->addArg(Instr->getArg(2)); |
| 3111 lowerCall(Call); | 3111 lowerCall(Call); |
| 3112 return; | 3112 return; |
| 3113 } | 3113 } |
| 3114 case Intrinsics::Memmove: { | 3114 case Intrinsics::Memmove: { |
| 3115 InstCall *Call = makeHelperCall("memmove", NULL, 3); | 3115 InstCall *Call = makeHelperCall("memmove", nullptr, 3); |
| 3116 Call->addArg(Instr->getArg(0)); | 3116 Call->addArg(Instr->getArg(0)); |
| 3117 Call->addArg(Instr->getArg(1)); | 3117 Call->addArg(Instr->getArg(1)); |
| 3118 Call->addArg(Instr->getArg(2)); | 3118 Call->addArg(Instr->getArg(2)); |
| 3119 lowerCall(Call); | 3119 lowerCall(Call); |
| 3120 return; | 3120 return; |
| 3121 } | 3121 } |
| 3122 case Intrinsics::Memset: { | 3122 case Intrinsics::Memset: { |
| 3123 // The value operand needs to be extended to a stack slot size | 3123 // The value operand needs to be extended to a stack slot size |
| 3124 // because the PNaCl ABI requires arguments to be at least 32 bits | 3124 // because the PNaCl ABI requires arguments to be at least 32 bits |
| 3125 // wide. | 3125 // wide. |
| 3126 Operand *ValOp = Instr->getArg(1); | 3126 Operand *ValOp = Instr->getArg(1); |
| 3127 assert(ValOp->getType() == IceType_i8); | 3127 assert(ValOp->getType() == IceType_i8); |
| 3128 Variable *ValExt = Func->makeVariable(stackSlotType()); | 3128 Variable *ValExt = Func->makeVariable(stackSlotType()); |
| 3129 lowerCast(InstCast::create(Func, InstCast::Zext, ValExt, ValOp)); | 3129 lowerCast(InstCast::create(Func, InstCast::Zext, ValExt, ValOp)); |
| 3130 InstCall *Call = makeHelperCall("memset", NULL, 3); | 3130 InstCall *Call = makeHelperCall("memset", nullptr, 3); |
| 3131 Call->addArg(Instr->getArg(0)); | 3131 Call->addArg(Instr->getArg(0)); |
| 3132 Call->addArg(ValExt); | 3132 Call->addArg(ValExt); |
| 3133 Call->addArg(Instr->getArg(2)); | 3133 Call->addArg(Instr->getArg(2)); |
| 3134 lowerCall(Call); | 3134 lowerCall(Call); |
| 3135 return; | 3135 return; |
| 3136 } | 3136 } |
| 3137 case Intrinsics::NaClReadTP: { | 3137 case Intrinsics::NaClReadTP: { |
| 3138 if (Ctx->getFlags().UseSandboxing) { | 3138 if (Ctx->getFlags().UseSandboxing) { |
| 3139 Constant *Zero = Ctx->getConstantZero(IceType_i32); | 3139 Constant *Zero = Ctx->getConstantZero(IceType_i32); |
| 3140 Operand *Src = OperandX8632Mem::create( | 3140 Operand *Src = |
| 3141 Func, IceType_i32, NULL, Zero, NULL, 0, OperandX8632Mem::SegReg_GS); | 3141 OperandX8632Mem::create(Func, IceType_i32, nullptr, Zero, nullptr, 0, |
| 3142 OperandX8632Mem::SegReg_GS); |
| 3142 Variable *Dest = Instr->getDest(); | 3143 Variable *Dest = Instr->getDest(); |
| 3143 Variable *T = NULL; | 3144 Variable *T = nullptr; |
| 3144 _mov(T, Src); | 3145 _mov(T, Src); |
| 3145 _mov(Dest, T); | 3146 _mov(Dest, T); |
| 3146 } else { | 3147 } else { |
| 3147 InstCall *Call = makeHelperCall("__nacl_read_tp", Instr->getDest(), 0); | 3148 InstCall *Call = makeHelperCall("__nacl_read_tp", Instr->getDest(), 0); |
| 3148 lowerCall(Call); | 3149 lowerCall(Call); |
| 3149 } | 3150 } |
| 3150 return; | 3151 return; |
| 3151 } | 3152 } |
| 3152 case Intrinsics::Setjmp: { | 3153 case Intrinsics::Setjmp: { |
| 3153 InstCall *Call = makeHelperCall("setjmp", Instr->getDest(), 1); | 3154 InstCall *Call = makeHelperCall("setjmp", Instr->getDest(), 1); |
| (...skipping 130 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3284 return true; | 3285 return true; |
| 3285 } | 3286 } |
| 3286 } | 3287 } |
| 3287 } | 3288 } |
| 3288 return false; | 3289 return false; |
| 3289 } | 3290 } |
| 3290 | 3291 |
| 3291 void TargetX8632::lowerAtomicRMW(Variable *Dest, uint32_t Operation, | 3292 void TargetX8632::lowerAtomicRMW(Variable *Dest, uint32_t Operation, |
| 3292 Operand *Ptr, Operand *Val) { | 3293 Operand *Ptr, Operand *Val) { |
| 3293 bool NeedsCmpxchg = false; | 3294 bool NeedsCmpxchg = false; |
| 3294 LowerBinOp Op_Lo = NULL; | 3295 LowerBinOp Op_Lo = nullptr; |
| 3295 LowerBinOp Op_Hi = NULL; | 3296 LowerBinOp Op_Hi = nullptr; |
| 3296 switch (Operation) { | 3297 switch (Operation) { |
| 3297 default: | 3298 default: |
| 3298 Func->setError("Unknown AtomicRMW operation"); | 3299 Func->setError("Unknown AtomicRMW operation"); |
| 3299 return; | 3300 return; |
| 3300 case Intrinsics::AtomicAdd: { | 3301 case Intrinsics::AtomicAdd: { |
| 3301 if (Dest->getType() == IceType_i64) { | 3302 if (Dest->getType() == IceType_i64) { |
| 3302 // All the fall-through paths must set this to true, but use this | 3303 // All the fall-through paths must set this to true, but use this |
| 3303 // for asserting. | 3304 // for asserting. |
| 3304 NeedsCmpxchg = true; | 3305 NeedsCmpxchg = true; |
| 3305 Op_Lo = &TargetX8632::_add; | 3306 Op_Lo = &TargetX8632::_add; |
| 3306 Op_Hi = &TargetX8632::_adc; | 3307 Op_Hi = &TargetX8632::_adc; |
| 3307 break; | 3308 break; |
| 3308 } | 3309 } |
| 3309 OperandX8632Mem *Addr = FormMemoryOperand(Ptr, Dest->getType()); | 3310 OperandX8632Mem *Addr = FormMemoryOperand(Ptr, Dest->getType()); |
| 3310 const bool Locked = true; | 3311 const bool Locked = true; |
| 3311 Variable *T = NULL; | 3312 Variable *T = nullptr; |
| 3312 _mov(T, Val); | 3313 _mov(T, Val); |
| 3313 _xadd(Addr, T, Locked); | 3314 _xadd(Addr, T, Locked); |
| 3314 _mov(Dest, T); | 3315 _mov(Dest, T); |
| 3315 return; | 3316 return; |
| 3316 } | 3317 } |
| 3317 case Intrinsics::AtomicSub: { | 3318 case Intrinsics::AtomicSub: { |
| 3318 if (Dest->getType() == IceType_i64) { | 3319 if (Dest->getType() == IceType_i64) { |
| 3319 NeedsCmpxchg = true; | 3320 NeedsCmpxchg = true; |
| 3320 Op_Lo = &TargetX8632::_sub; | 3321 Op_Lo = &TargetX8632::_sub; |
| 3321 Op_Hi = &TargetX8632::_sbb; | 3322 Op_Hi = &TargetX8632::_sbb; |
| 3322 break; | 3323 break; |
| 3323 } | 3324 } |
| 3324 OperandX8632Mem *Addr = FormMemoryOperand(Ptr, Dest->getType()); | 3325 OperandX8632Mem *Addr = FormMemoryOperand(Ptr, Dest->getType()); |
| 3325 const bool Locked = true; | 3326 const bool Locked = true; |
| 3326 Variable *T = NULL; | 3327 Variable *T = nullptr; |
| 3327 _mov(T, Val); | 3328 _mov(T, Val); |
| 3328 _neg(T); | 3329 _neg(T); |
| 3329 _xadd(Addr, T, Locked); | 3330 _xadd(Addr, T, Locked); |
| 3330 _mov(Dest, T); | 3331 _mov(Dest, T); |
| 3331 return; | 3332 return; |
| 3332 } | 3333 } |
| 3333 case Intrinsics::AtomicOr: | 3334 case Intrinsics::AtomicOr: |
| 3334 // TODO(jvoung): If Dest is null or dead, then some of these | 3335 // TODO(jvoung): If Dest is null or dead, then some of these |
| 3335 // operations do not need an "exchange", but just a locked op. | 3336 // operations do not need an "exchange", but just a locked op. |
| 3336 // That appears to be "worth" it for sub, or, and, and xor. | 3337 // That appears to be "worth" it for sub, or, and, and xor. |
| (...skipping 11 matching lines...) Expand all Loading... |
| 3348 case Intrinsics::AtomicXor: | 3349 case Intrinsics::AtomicXor: |
| 3349 NeedsCmpxchg = true; | 3350 NeedsCmpxchg = true; |
| 3350 Op_Lo = &TargetX8632::_xor; | 3351 Op_Lo = &TargetX8632::_xor; |
| 3351 Op_Hi = &TargetX8632::_xor; | 3352 Op_Hi = &TargetX8632::_xor; |
| 3352 break; | 3353 break; |
| 3353 case Intrinsics::AtomicExchange: | 3354 case Intrinsics::AtomicExchange: |
| 3354 if (Dest->getType() == IceType_i64) { | 3355 if (Dest->getType() == IceType_i64) { |
| 3355 NeedsCmpxchg = true; | 3356 NeedsCmpxchg = true; |
| 3356 // NeedsCmpxchg, but no real Op_Lo/Op_Hi need to be done. The values | 3357 // NeedsCmpxchg, but no real Op_Lo/Op_Hi need to be done. The values |
| 3357 // just need to be moved to the ecx and ebx registers. | 3358 // just need to be moved to the ecx and ebx registers. |
| 3358 Op_Lo = NULL; | 3359 Op_Lo = nullptr; |
| 3359 Op_Hi = NULL; | 3360 Op_Hi = nullptr; |
| 3360 break; | 3361 break; |
| 3361 } | 3362 } |
| 3362 OperandX8632Mem *Addr = FormMemoryOperand(Ptr, Dest->getType()); | 3363 OperandX8632Mem *Addr = FormMemoryOperand(Ptr, Dest->getType()); |
| 3363 Variable *T = NULL; | 3364 Variable *T = nullptr; |
| 3364 _mov(T, Val); | 3365 _mov(T, Val); |
| 3365 _xchg(Addr, T); | 3366 _xchg(Addr, T); |
| 3366 _mov(Dest, T); | 3367 _mov(Dest, T); |
| 3367 return; | 3368 return; |
| 3368 } | 3369 } |
| 3369 // Otherwise, we need a cmpxchg loop. | 3370 // Otherwise, we need a cmpxchg loop. |
| 3370 (void)NeedsCmpxchg; | 3371 (void)NeedsCmpxchg; |
| 3371 assert(NeedsCmpxchg); | 3372 assert(NeedsCmpxchg); |
| 3372 expandAtomicRMWAsCmpxchg(Op_Lo, Op_Hi, Dest, Ptr, Val); | 3373 expandAtomicRMWAsCmpxchg(Op_Lo, Op_Hi, Dest, Ptr, Val); |
| 3373 } | 3374 } |
| (...skipping 17 matching lines...) Expand all Loading... |
| 3391 // | 3392 // |
| 3392 // For 32-bit: | 3393 // For 32-bit: |
| 3393 // mov eax, [ptr] | 3394 // mov eax, [ptr] |
| 3394 // .LABEL: | 3395 // .LABEL: |
| 3395 // mov <reg>, eax | 3396 // mov <reg>, eax |
| 3396 // op <reg>, [desired_adj] | 3397 // op <reg>, [desired_adj] |
| 3397 // lock cmpxchg [ptr], <reg> | 3398 // lock cmpxchg [ptr], <reg> |
| 3398 // jne .LABEL | 3399 // jne .LABEL |
| 3399 // mov <dest>, eax | 3400 // mov <dest>, eax |
| 3400 // | 3401 // |
| 3401 // If Op_{Lo,Hi} are NULL, then just copy the value. | 3402 // If Op_{Lo,Hi} are nullptr, then just copy the value. |
| 3402 Val = legalize(Val); | 3403 Val = legalize(Val); |
| 3403 Type Ty = Val->getType(); | 3404 Type Ty = Val->getType(); |
| 3404 if (Ty == IceType_i64) { | 3405 if (Ty == IceType_i64) { |
| 3405 Variable *T_edx = makeReg(IceType_i32, RegX8632::Reg_edx); | 3406 Variable *T_edx = makeReg(IceType_i32, RegX8632::Reg_edx); |
| 3406 Variable *T_eax = makeReg(IceType_i32, RegX8632::Reg_eax); | 3407 Variable *T_eax = makeReg(IceType_i32, RegX8632::Reg_eax); |
| 3407 OperandX8632Mem *Addr = FormMemoryOperand(Ptr, Ty); | 3408 OperandX8632Mem *Addr = FormMemoryOperand(Ptr, Ty); |
| 3408 _mov(T_eax, loOperand(Addr)); | 3409 _mov(T_eax, loOperand(Addr)); |
| 3409 _mov(T_edx, hiOperand(Addr)); | 3410 _mov(T_edx, hiOperand(Addr)); |
| 3410 Variable *T_ecx = makeReg(IceType_i32, RegX8632::Reg_ecx); | 3411 Variable *T_ecx = makeReg(IceType_i32, RegX8632::Reg_ecx); |
| 3411 Variable *T_ebx = makeReg(IceType_i32, RegX8632::Reg_ebx); | 3412 Variable *T_ebx = makeReg(IceType_i32, RegX8632::Reg_ebx); |
| 3412 InstX8632Label *Label = InstX8632Label::create(Func, this); | 3413 InstX8632Label *Label = InstX8632Label::create(Func, this); |
| 3413 const bool IsXchg8b = Op_Lo == NULL && Op_Hi == NULL; | 3414 const bool IsXchg8b = Op_Lo == nullptr && Op_Hi == nullptr; |
| 3414 if (!IsXchg8b) { | 3415 if (!IsXchg8b) { |
| 3415 Context.insert(Label); | 3416 Context.insert(Label); |
| 3416 _mov(T_ebx, T_eax); | 3417 _mov(T_ebx, T_eax); |
| 3417 (this->*Op_Lo)(T_ebx, loOperand(Val)); | 3418 (this->*Op_Lo)(T_ebx, loOperand(Val)); |
| 3418 _mov(T_ecx, T_edx); | 3419 _mov(T_ecx, T_edx); |
| 3419 (this->*Op_Hi)(T_ecx, hiOperand(Val)); | 3420 (this->*Op_Hi)(T_ecx, hiOperand(Val)); |
| 3420 } else { | 3421 } else { |
| 3421 // This is for xchg, which doesn't need an actual Op_Lo/Op_Hi. | 3422 // This is for xchg, which doesn't need an actual Op_Lo/Op_Hi. |
| 3422 // It just needs the Val loaded into ebx and ecx. | 3423 // It just needs the Val loaded into ebx and ecx. |
| 3423 // That can also be done before the loop. | 3424 // That can also be done before the loop. |
| (...skipping 25 matching lines...) Expand all Loading... |
| 3449 _mov(DestLo, T_eax); | 3450 _mov(DestLo, T_eax); |
| 3450 _mov(DestHi, T_edx); | 3451 _mov(DestHi, T_edx); |
| 3451 return; | 3452 return; |
| 3452 } | 3453 } |
| 3453 OperandX8632Mem *Addr = FormMemoryOperand(Ptr, Ty); | 3454 OperandX8632Mem *Addr = FormMemoryOperand(Ptr, Ty); |
| 3454 Variable *T_eax = makeReg(Ty, RegX8632::Reg_eax); | 3455 Variable *T_eax = makeReg(Ty, RegX8632::Reg_eax); |
| 3455 _mov(T_eax, Addr); | 3456 _mov(T_eax, Addr); |
| 3456 InstX8632Label *Label = InstX8632Label::create(Func, this); | 3457 InstX8632Label *Label = InstX8632Label::create(Func, this); |
| 3457 Context.insert(Label); | 3458 Context.insert(Label); |
| 3458 // We want to pick a different register for T than Eax, so don't use | 3459 // We want to pick a different register for T than Eax, so don't use |
| 3459 // _mov(T == NULL, T_eax). | 3460 // _mov(T == nullptr, T_eax). |
| 3460 Variable *T = makeReg(Ty); | 3461 Variable *T = makeReg(Ty); |
| 3461 _mov(T, T_eax); | 3462 _mov(T, T_eax); |
| 3462 (this->*Op_Lo)(T, Val); | 3463 (this->*Op_Lo)(T, Val); |
| 3463 const bool Locked = true; | 3464 const bool Locked = true; |
| 3464 _cmpxchg(Addr, T_eax, T, Locked); | 3465 _cmpxchg(Addr, T_eax, T, Locked); |
| 3465 _br(CondX86::Br_ne, Label); | 3466 _br(CondX86::Br_ne, Label); |
| 3466 // If Val is a variable, model the extended live range of Val through | 3467 // If Val is a variable, model the extended live range of Val through |
| 3467 // the end of the loop, since it will be re-used by the loop. | 3468 // the end of the loop, since it will be re-used by the loop. |
| 3468 if (Variable *ValVar = llvm::dyn_cast<Variable>(Val)) { | 3469 if (Variable *ValVar = llvm::dyn_cast<Variable>(Val)) { |
| 3469 Context.insert(InstFakeUse::create(Func, ValVar)); | 3470 Context.insert(InstFakeUse::create(Func, ValVar)); |
| (...skipping 109 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3579 Index->dump(Func); | 3580 Index->dump(Func); |
| 3580 else | 3581 else |
| 3581 Str << "<null>"; | 3582 Str << "<null>"; |
| 3582 Str << ", Shift=" << Shift << ", Offset=" << Offset << "\n"; | 3583 Str << ", Shift=" << Shift << ", Offset=" << Offset << "\n"; |
| 3583 } | 3584 } |
| 3584 | 3585 |
| 3585 bool matchTransitiveAssign(const VariablesMetadata *VMetadata, Variable *&Var, | 3586 bool matchTransitiveAssign(const VariablesMetadata *VMetadata, Variable *&Var, |
| 3586 const Inst *&Reason) { | 3587 const Inst *&Reason) { |
| 3587 // Var originates from Var=SrcVar ==> | 3588 // Var originates from Var=SrcVar ==> |
| 3588 // set Var:=SrcVar | 3589 // set Var:=SrcVar |
| 3589 if (Var == NULL) | 3590 if (Var == nullptr) |
| 3590 return false; | 3591 return false; |
| 3591 if (const Inst *VarAssign = VMetadata->getSingleDefinition(Var)) { | 3592 if (const Inst *VarAssign = VMetadata->getSingleDefinition(Var)) { |
| 3592 assert(!VMetadata->isMultiDef(Var)); | 3593 assert(!VMetadata->isMultiDef(Var)); |
| 3593 if (llvm::isa<InstAssign>(VarAssign)) { | 3594 if (llvm::isa<InstAssign>(VarAssign)) { |
| 3594 Operand *SrcOp = VarAssign->getSrc(0); | 3595 Operand *SrcOp = VarAssign->getSrc(0); |
| 3595 assert(SrcOp); | 3596 assert(SrcOp); |
| 3596 if (Variable *SrcVar = llvm::dyn_cast<Variable>(SrcOp)) { | 3597 if (Variable *SrcVar = llvm::dyn_cast<Variable>(SrcOp)) { |
| 3597 if (!VMetadata->isMultiDef(SrcVar) && | 3598 if (!VMetadata->isMultiDef(SrcVar) && |
| 3598 // TODO: ensure SrcVar stays single-BB | 3599 // TODO: ensure SrcVar stays single-BB |
| 3599 true) { | 3600 true) { |
| 3600 Var = SrcVar; | 3601 Var = SrcVar; |
| 3601 Reason = VarAssign; | 3602 Reason = VarAssign; |
| 3602 return true; | 3603 return true; |
| 3603 } | 3604 } |
| 3604 } | 3605 } |
| 3605 } | 3606 } |
| 3606 } | 3607 } |
| 3607 return false; | 3608 return false; |
| 3608 } | 3609 } |
| 3609 | 3610 |
| 3610 bool matchCombinedBaseIndex(const VariablesMetadata *VMetadata, Variable *&Base, | 3611 bool matchCombinedBaseIndex(const VariablesMetadata *VMetadata, Variable *&Base, |
| 3611 Variable *&Index, uint16_t &Shift, | 3612 Variable *&Index, uint16_t &Shift, |
| 3612 const Inst *&Reason) { | 3613 const Inst *&Reason) { |
| 3613 // Index==NULL && Base is Base=Var1+Var2 ==> | 3614 // Index==nullptr && Base is Base=Var1+Var2 ==> |
| 3614 // set Base=Var1, Index=Var2, Shift=0 | 3615 // set Base=Var1, Index=Var2, Shift=0 |
| 3615 if (Base == NULL) | 3616 if (Base == nullptr) |
| 3616 return false; | 3617 return false; |
| 3617 if (Index != NULL) | 3618 if (Index != nullptr) |
| 3618 return false; | 3619 return false; |
| 3619 const Inst *BaseInst = VMetadata->getSingleDefinition(Base); | 3620 const Inst *BaseInst = VMetadata->getSingleDefinition(Base); |
| 3620 if (BaseInst == NULL) | 3621 if (BaseInst == nullptr) |
| 3621 return false; | 3622 return false; |
| 3622 assert(!VMetadata->isMultiDef(Base)); | 3623 assert(!VMetadata->isMultiDef(Base)); |
| 3623 if (BaseInst->getSrcSize() < 2) | 3624 if (BaseInst->getSrcSize() < 2) |
| 3624 return false; | 3625 return false; |
| 3625 if (Variable *Var1 = llvm::dyn_cast<Variable>(BaseInst->getSrc(0))) { | 3626 if (Variable *Var1 = llvm::dyn_cast<Variable>(BaseInst->getSrc(0))) { |
| 3626 if (VMetadata->isMultiDef(Var1)) | 3627 if (VMetadata->isMultiDef(Var1)) |
| 3627 return false; | 3628 return false; |
| 3628 if (Variable *Var2 = llvm::dyn_cast<Variable>(BaseInst->getSrc(1))) { | 3629 if (Variable *Var2 = llvm::dyn_cast<Variable>(BaseInst->getSrc(1))) { |
| 3629 if (VMetadata->isMultiDef(Var2)) | 3630 if (VMetadata->isMultiDef(Var2)) |
| 3630 return false; | 3631 return false; |
| 3631 if (isAdd(BaseInst) && | 3632 if (isAdd(BaseInst) && |
| 3632 // TODO: ensure Var1 and Var2 stay single-BB | 3633 // TODO: ensure Var1 and Var2 stay single-BB |
| 3633 true) { | 3634 true) { |
| 3634 Base = Var1; | 3635 Base = Var1; |
| 3635 Index = Var2; | 3636 Index = Var2; |
| 3636 Shift = 0; // should already have been 0 | 3637 Shift = 0; // should already have been 0 |
| 3637 Reason = BaseInst; | 3638 Reason = BaseInst; |
| 3638 return true; | 3639 return true; |
| 3639 } | 3640 } |
| 3640 } | 3641 } |
| 3641 } | 3642 } |
| 3642 return false; | 3643 return false; |
| 3643 } | 3644 } |
| 3644 | 3645 |
| 3645 bool matchShiftedIndex(const VariablesMetadata *VMetadata, Variable *&Index, | 3646 bool matchShiftedIndex(const VariablesMetadata *VMetadata, Variable *&Index, |
| 3646 uint16_t &Shift, const Inst *&Reason) { | 3647 uint16_t &Shift, const Inst *&Reason) { |
| 3647 // Index is Index=Var*Const && log2(Const)+Shift<=3 ==> | 3648 // Index is Index=Var*Const && log2(Const)+Shift<=3 ==> |
| 3648 // Index=Var, Shift+=log2(Const) | 3649 // Index=Var, Shift+=log2(Const) |
| 3649 if (Index == NULL) | 3650 if (Index == nullptr) |
| 3650 return false; | 3651 return false; |
| 3651 const Inst *IndexInst = VMetadata->getSingleDefinition(Index); | 3652 const Inst *IndexInst = VMetadata->getSingleDefinition(Index); |
| 3652 if (IndexInst == NULL) | 3653 if (IndexInst == nullptr) |
| 3653 return false; | 3654 return false; |
| 3654 assert(!VMetadata->isMultiDef(Index)); | 3655 assert(!VMetadata->isMultiDef(Index)); |
| 3655 if (IndexInst->getSrcSize() < 2) | 3656 if (IndexInst->getSrcSize() < 2) |
| 3656 return false; | 3657 return false; |
| 3657 if (const InstArithmetic *ArithInst = | 3658 if (const InstArithmetic *ArithInst = |
| 3658 llvm::dyn_cast<InstArithmetic>(IndexInst)) { | 3659 llvm::dyn_cast<InstArithmetic>(IndexInst)) { |
| 3659 if (Variable *Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(0))) { | 3660 if (Variable *Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(0))) { |
| 3660 if (ConstantInteger32 *Const = | 3661 if (ConstantInteger32 *Const = |
| 3661 llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1))) { | 3662 llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1))) { |
| 3662 if (ArithInst->getOp() == InstArithmetic::Mul && | 3663 if (ArithInst->getOp() == InstArithmetic::Mul && |
| (...skipping 28 matching lines...) Expand all Loading... |
| 3691 } | 3692 } |
| 3692 return false; | 3693 return false; |
| 3693 } | 3694 } |
| 3694 | 3695 |
| 3695 bool matchOffsetBase(const VariablesMetadata *VMetadata, Variable *&Base, | 3696 bool matchOffsetBase(const VariablesMetadata *VMetadata, Variable *&Base, |
| 3696 int32_t &Offset, const Inst *&Reason) { | 3697 int32_t &Offset, const Inst *&Reason) { |
| 3697 // Base is Base=Var+Const || Base is Base=Const+Var ==> | 3698 // Base is Base=Var+Const || Base is Base=Const+Var ==> |
| 3698 // set Base=Var, Offset+=Const | 3699 // set Base=Var, Offset+=Const |
| 3699 // Base is Base=Var-Const ==> | 3700 // Base is Base=Var-Const ==> |
| 3700 // set Base=Var, Offset-=Const | 3701 // set Base=Var, Offset-=Const |
| 3701 if (Base == NULL) | 3702 if (Base == nullptr) |
| 3702 return false; | 3703 return false; |
| 3703 const Inst *BaseInst = VMetadata->getSingleDefinition(Base); | 3704 const Inst *BaseInst = VMetadata->getSingleDefinition(Base); |
| 3704 if (BaseInst == NULL) | 3705 if (BaseInst == nullptr) |
| 3705 return false; | 3706 return false; |
| 3706 assert(!VMetadata->isMultiDef(Base)); | 3707 assert(!VMetadata->isMultiDef(Base)); |
| 3707 if (const InstArithmetic *ArithInst = | 3708 if (const InstArithmetic *ArithInst = |
| 3708 llvm::dyn_cast<const InstArithmetic>(BaseInst)) { | 3709 llvm::dyn_cast<const InstArithmetic>(BaseInst)) { |
| 3709 if (ArithInst->getOp() != InstArithmetic::Add && | 3710 if (ArithInst->getOp() != InstArithmetic::Add && |
| 3710 ArithInst->getOp() != InstArithmetic::Sub) | 3711 ArithInst->getOp() != InstArithmetic::Sub) |
| 3711 return false; | 3712 return false; |
| 3712 bool IsAdd = ArithInst->getOp() == InstArithmetic::Add; | 3713 bool IsAdd = ArithInst->getOp() == InstArithmetic::Add; |
| 3713 Variable *Var = NULL; | 3714 Variable *Var = nullptr; |
| 3714 ConstantInteger32 *Const = NULL; | 3715 ConstantInteger32 *Const = nullptr; |
| 3715 if (Variable *VariableOperand = | 3716 if (Variable *VariableOperand = |
| 3716 llvm::dyn_cast<Variable>(ArithInst->getSrc(0))) { | 3717 llvm::dyn_cast<Variable>(ArithInst->getSrc(0))) { |
| 3717 Var = VariableOperand; | 3718 Var = VariableOperand; |
| 3718 Const = llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1)); | 3719 Const = llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1)); |
| 3719 } else if (IsAdd) { | 3720 } else if (IsAdd) { |
| 3720 Const = llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(0)); | 3721 Const = llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(0)); |
| 3721 Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(1)); | 3722 Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(1)); |
| 3722 } | 3723 } |
| 3723 if (Var == NULL || Const == NULL || VMetadata->isMultiDef(Var)) | 3724 if (Var == nullptr || Const == nullptr || VMetadata->isMultiDef(Var)) |
| 3724 return false; | 3725 return false; |
| 3725 int32_t MoreOffset = IsAdd ? Const->getValue() : -Const->getValue(); | 3726 int32_t MoreOffset = IsAdd ? Const->getValue() : -Const->getValue(); |
| 3726 if (Utils::WouldOverflowAdd(Offset, MoreOffset)) | 3727 if (Utils::WouldOverflowAdd(Offset, MoreOffset)) |
| 3727 return false; | 3728 return false; |
| 3728 Base = Var; | 3729 Base = Var; |
| 3729 Offset += MoreOffset; | 3730 Offset += MoreOffset; |
| 3730 Reason = BaseInst; | 3731 Reason = BaseInst; |
| 3731 return true; | 3732 return true; |
| 3732 } | 3733 } |
| 3733 return false; | 3734 return false; |
| 3734 } | 3735 } |
| 3735 | 3736 |
| 3736 void computeAddressOpt(Cfg *Func, const Inst *Instr, Variable *&Base, | 3737 void computeAddressOpt(Cfg *Func, const Inst *Instr, Variable *&Base, |
| 3737 Variable *&Index, uint16_t &Shift, int32_t &Offset) { | 3738 Variable *&Index, uint16_t &Shift, int32_t &Offset) { |
| 3738 Func->resetCurrentNode(); | 3739 Func->resetCurrentNode(); |
| 3739 if (Func->getContext()->isVerbose(IceV_AddrOpt)) { | 3740 if (Func->getContext()->isVerbose(IceV_AddrOpt)) { |
| 3740 Ostream &Str = Func->getContext()->getStrDump(); | 3741 Ostream &Str = Func->getContext()->getStrDump(); |
| 3741 Str << "\nStarting computeAddressOpt for instruction:\n "; | 3742 Str << "\nStarting computeAddressOpt for instruction:\n "; |
| 3742 Instr->dumpDecorated(Func); | 3743 Instr->dumpDecorated(Func); |
| 3743 } | 3744 } |
| 3744 (void)Offset; // TODO: pattern-match for non-zero offsets. | 3745 (void)Offset; // TODO: pattern-match for non-zero offsets. |
| 3745 if (Base == NULL) | 3746 if (Base == nullptr) |
| 3746 return; | 3747 return; |
| 3747 // If the Base has more than one use or is live across multiple | 3748 // If the Base has more than one use or is live across multiple |
| 3748 // blocks, then don't go further. Alternatively (?), never consider | 3749 // blocks, then don't go further. Alternatively (?), never consider |
| 3749 // a transformation that would change a variable that is currently | 3750 // a transformation that would change a variable that is currently |
| 3750 // *not* live across basic block boundaries into one that *is*. | 3751 // *not* live across basic block boundaries into one that *is*. |
| 3751 if (Func->getVMetadata()->isMultiBlock(Base) /* || Base->getUseCount() > 1*/) | 3752 if (Func->getVMetadata()->isMultiBlock(Base) /* || Base->getUseCount() > 1*/) |
| 3752 return; | 3753 return; |
| 3753 | 3754 |
| 3754 const VariablesMetadata *VMetadata = Func->getVMetadata(); | 3755 const VariablesMetadata *VMetadata = Func->getVMetadata(); |
| 3755 bool Continue = true; | 3756 bool Continue = true; |
| 3756 while (Continue) { | 3757 while (Continue) { |
| 3757 const Inst *Reason = NULL; | 3758 const Inst *Reason = nullptr; |
| 3758 if (matchTransitiveAssign(VMetadata, Base, Reason) || | 3759 if (matchTransitiveAssign(VMetadata, Base, Reason) || |
| 3759 matchTransitiveAssign(VMetadata, Index, Reason) || | 3760 matchTransitiveAssign(VMetadata, Index, Reason) || |
| 3760 matchCombinedBaseIndex(VMetadata, Base, Index, Shift, Reason) || | 3761 matchCombinedBaseIndex(VMetadata, Base, Index, Shift, Reason) || |
| 3761 matchShiftedIndex(VMetadata, Index, Shift, Reason) || | 3762 matchShiftedIndex(VMetadata, Index, Shift, Reason) || |
| 3762 matchOffsetBase(VMetadata, Base, Offset, Reason)) { | 3763 matchOffsetBase(VMetadata, Base, Offset, Reason)) { |
| 3763 dumpAddressOpt(Func, Base, Index, Shift, Offset, Reason); | 3764 dumpAddressOpt(Func, Base, Index, Shift, Offset, Reason); |
| 3764 } else { | 3765 } else { |
| 3765 Continue = false; | 3766 Continue = false; |
| 3766 } | 3767 } |
| 3767 | 3768 |
| (...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 3808 // TODO: Clean up and test thoroughly. | 3809 // TODO: Clean up and test thoroughly. |
| 3809 // (E.g., if there is an mfence-all make sure the load ends up on the | 3810 // (E.g., if there is an mfence-all make sure the load ends up on the |
| 3810 // same side of the fence). | 3811 // same side of the fence). |
| 3811 // | 3812 // |
| 3812 // TODO: Why limit to Arithmetic instructions? This could probably be | 3813 // TODO: Why limit to Arithmetic instructions? This could probably be |
| 3813 // applied to most any instruction type. Look at all source operands | 3814 // applied to most any instruction type. Look at all source operands |
| 3814 // in the following instruction, and if there is one instance of the | 3815 // in the following instruction, and if there is one instance of the |
| 3815 // load instruction's dest variable, and that instruction ends that | 3816 // load instruction's dest variable, and that instruction ends that |
| 3816 // variable's live range, then make the substitution. Deal with | 3817 // variable's live range, then make the substitution. Deal with |
| 3817 // commutativity optimization in the arithmetic instruction lowering. | 3818 // commutativity optimization in the arithmetic instruction lowering. |
| 3818 InstArithmetic *NewArith = NULL; | 3819 InstArithmetic *NewArith = nullptr; |
| 3819 if (InstArithmetic *Arith = | 3820 if (InstArithmetic *Arith = |
| 3820 llvm::dyn_cast_or_null<InstArithmetic>(Context.getNextInst())) { | 3821 llvm::dyn_cast_or_null<InstArithmetic>(Context.getNextInst())) { |
| 3821 Variable *DestLoad = Inst->getDest(); | 3822 Variable *DestLoad = Inst->getDest(); |
| 3822 Variable *Src0Arith = llvm::dyn_cast<Variable>(Arith->getSrc(0)); | 3823 Variable *Src0Arith = llvm::dyn_cast<Variable>(Arith->getSrc(0)); |
| 3823 Variable *Src1Arith = llvm::dyn_cast<Variable>(Arith->getSrc(1)); | 3824 Variable *Src1Arith = llvm::dyn_cast<Variable>(Arith->getSrc(1)); |
| 3824 if (Src1Arith == DestLoad && Arith->isLastUse(Src1Arith) && | 3825 if (Src1Arith == DestLoad && Arith->isLastUse(Src1Arith) && |
| 3825 DestLoad != Src0Arith) { | 3826 DestLoad != Src0Arith) { |
| 3826 NewArith = InstArithmetic::create(Func, Arith->getOp(), Arith->getDest(), | 3827 NewArith = InstArithmetic::create(Func, Arith->getOp(), Arith->getDest(), |
| 3827 Arith->getSrc(0), Src0); | 3828 Arith->getSrc(0), Src0); |
| 3828 } else if (Src0Arith == DestLoad && Arith->isCommutative() && | 3829 } else if (Src0Arith == DestLoad && Arith->isCommutative() && |
| (...skipping 10 matching lines...) Expand all Loading... |
| 3839 } | 3840 } |
| 3840 | 3841 |
| 3841 InstAssign *Assign = InstAssign::create(Func, Inst->getDest(), Src0); | 3842 InstAssign *Assign = InstAssign::create(Func, Inst->getDest(), Src0); |
| 3842 lowerAssign(Assign); | 3843 lowerAssign(Assign); |
| 3843 } | 3844 } |
| 3844 | 3845 |
| 3845 void TargetX8632::doAddressOptLoad() { | 3846 void TargetX8632::doAddressOptLoad() { |
| 3846 Inst *Inst = Context.getCur(); | 3847 Inst *Inst = Context.getCur(); |
| 3847 Variable *Dest = Inst->getDest(); | 3848 Variable *Dest = Inst->getDest(); |
| 3848 Operand *Addr = Inst->getSrc(0); | 3849 Operand *Addr = Inst->getSrc(0); |
| 3849 Variable *Index = NULL; | 3850 Variable *Index = nullptr; |
| 3850 uint16_t Shift = 0; | 3851 uint16_t Shift = 0; |
| 3851 int32_t Offset = 0; // TODO: make Constant | 3852 int32_t Offset = 0; // TODO: make Constant |
| 3852 // Vanilla ICE load instructions should not use the segment registers, | 3853 // Vanilla ICE load instructions should not use the segment registers, |
| 3853 // and computeAddressOpt only works at the level of Variables and Constants, | 3854 // and computeAddressOpt only works at the level of Variables and Constants, |
| 3854 // not other OperandX8632Mem, so there should be no mention of segment | 3855 // not other OperandX8632Mem, so there should be no mention of segment |
| 3855 // registers there either. | 3856 // registers there either. |
| 3856 const OperandX8632Mem::SegmentRegisters SegmentReg = | 3857 const OperandX8632Mem::SegmentRegisters SegmentReg = |
| 3857 OperandX8632Mem::DefaultSegment; | 3858 OperandX8632Mem::DefaultSegment; |
| 3858 Variable *Base = llvm::dyn_cast<Variable>(Addr); | 3859 Variable *Base = llvm::dyn_cast<Variable>(Addr); |
| 3859 computeAddressOpt(Func, Inst, Base, Index, Shift, Offset); | 3860 computeAddressOpt(Func, Inst, Base, Index, Shift, Offset); |
| (...skipping 11 matching lines...) Expand all Loading... |
| 3871 if (RNG.getTrueWithProbability(Probability)) { | 3872 if (RNG.getTrueWithProbability(Probability)) { |
| 3872 _nop(RNG(X86_NUM_NOP_VARIANTS)); | 3873 _nop(RNG(X86_NUM_NOP_VARIANTS)); |
| 3873 } | 3874 } |
| 3874 } | 3875 } |
| 3875 | 3876 |
| 3876 void TargetX8632::lowerPhi(const InstPhi * /*Inst*/) { | 3877 void TargetX8632::lowerPhi(const InstPhi * /*Inst*/) { |
| 3877 Func->setError("Phi found in regular instruction list"); | 3878 Func->setError("Phi found in regular instruction list"); |
| 3878 } | 3879 } |
| 3879 | 3880 |
| 3880 void TargetX8632::lowerRet(const InstRet *Inst) { | 3881 void TargetX8632::lowerRet(const InstRet *Inst) { |
| 3881 Variable *Reg = NULL; | 3882 Variable *Reg = nullptr; |
| 3882 if (Inst->hasRetValue()) { | 3883 if (Inst->hasRetValue()) { |
| 3883 Operand *Src0 = legalize(Inst->getRetValue()); | 3884 Operand *Src0 = legalize(Inst->getRetValue()); |
| 3884 if (Src0->getType() == IceType_i64) { | 3885 if (Src0->getType() == IceType_i64) { |
| 3885 Variable *eax = legalizeToVar(loOperand(Src0), RegX8632::Reg_eax); | 3886 Variable *eax = legalizeToVar(loOperand(Src0), RegX8632::Reg_eax); |
| 3886 Variable *edx = legalizeToVar(hiOperand(Src0), RegX8632::Reg_edx); | 3887 Variable *edx = legalizeToVar(hiOperand(Src0), RegX8632::Reg_edx); |
| 3887 Reg = eax; | 3888 Reg = eax; |
| 3888 Context.insert(InstFakeUse::create(Func, edx)); | 3889 Context.insert(InstFakeUse::create(Func, edx)); |
| 3889 } else if (isScalarFloatingType(Src0->getType())) { | 3890 } else if (isScalarFloatingType(Src0->getType())) { |
| 3890 _fld(Src0); | 3891 _fld(Src0); |
| 3891 } else if (isVectorType(Src0->getType())) { | 3892 } else if (isVectorType(Src0->getType())) { |
| (...skipping 125 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4017 } else { | 4018 } else { |
| 4018 Value = legalize(Value, Legal_Reg | Legal_Imm); | 4019 Value = legalize(Value, Legal_Reg | Legal_Imm); |
| 4019 _store(Value, NewAddr); | 4020 _store(Value, NewAddr); |
| 4020 } | 4021 } |
| 4021 } | 4022 } |
| 4022 | 4023 |
| 4023 void TargetX8632::doAddressOptStore() { | 4024 void TargetX8632::doAddressOptStore() { |
| 4024 InstStore *Inst = llvm::cast<InstStore>(Context.getCur()); | 4025 InstStore *Inst = llvm::cast<InstStore>(Context.getCur()); |
| 4025 Operand *Data = Inst->getData(); | 4026 Operand *Data = Inst->getData(); |
| 4026 Operand *Addr = Inst->getAddr(); | 4027 Operand *Addr = Inst->getAddr(); |
| 4027 Variable *Index = NULL; | 4028 Variable *Index = nullptr; |
| 4028 uint16_t Shift = 0; | 4029 uint16_t Shift = 0; |
| 4029 int32_t Offset = 0; // TODO: make Constant | 4030 int32_t Offset = 0; // TODO: make Constant |
| 4030 Variable *Base = llvm::dyn_cast<Variable>(Addr); | 4031 Variable *Base = llvm::dyn_cast<Variable>(Addr); |
| 4031 // Vanilla ICE store instructions should not use the segment registers, | 4032 // Vanilla ICE store instructions should not use the segment registers, |
| 4032 // and computeAddressOpt only works at the level of Variables and Constants, | 4033 // and computeAddressOpt only works at the level of Variables and Constants, |
| 4033 // not other OperandX8632Mem, so there should be no mention of segment | 4034 // not other OperandX8632Mem, so there should be no mention of segment |
| 4034 // registers there either. | 4035 // registers there either. |
| 4035 const OperandX8632Mem::SegmentRegisters SegmentReg = | 4036 const OperandX8632Mem::SegmentRegisters SegmentReg = |
| 4036 OperandX8632Mem::DefaultSegment; | 4037 OperandX8632Mem::DefaultSegment; |
| 4037 computeAddressOpt(Func, Inst, Base, Index, Shift, Offset); | 4038 computeAddressOpt(Func, Inst, Base, Index, Shift, Offset); |
| (...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4136 NextCast->setDeleted(); | 4137 NextCast->setDeleted(); |
| 4137 _movp(NextCast->getDest(), legalizeToVar(SignExtendedResult)); | 4138 _movp(NextCast->getDest(), legalizeToVar(SignExtendedResult)); |
| 4138 // Skip over the instruction. | 4139 // Skip over the instruction. |
| 4139 Context.advanceNext(); | 4140 Context.advanceNext(); |
| 4140 } | 4141 } |
| 4141 } | 4142 } |
| 4142 } | 4143 } |
| 4143 | 4144 |
| 4144 void TargetX8632::lowerUnreachable(const InstUnreachable * /*Inst*/) { | 4145 void TargetX8632::lowerUnreachable(const InstUnreachable * /*Inst*/) { |
| 4145 const SizeT MaxSrcs = 0; | 4146 const SizeT MaxSrcs = 0; |
| 4146 Variable *Dest = NULL; | 4147 Variable *Dest = nullptr; |
| 4147 InstCall *Call = makeHelperCall("ice_unreachable", Dest, MaxSrcs); | 4148 InstCall *Call = makeHelperCall("ice_unreachable", Dest, MaxSrcs); |
| 4148 lowerCall(Call); | 4149 lowerCall(Call); |
| 4149 } | 4150 } |
| 4150 | 4151 |
| 4151 // Turn an i64 Phi instruction into a pair of i32 Phi instructions, to | 4152 // Turn an i64 Phi instruction into a pair of i32 Phi instructions, to |
| 4152 // preserve integrity of liveness analysis. Undef values are also | 4153 // preserve integrity of liveness analysis. Undef values are also |
| 4153 // turned into zeroes, since loOperand() and hiOperand() don't expect | 4154 // turned into zeroes, since loOperand() and hiOperand() don't expect |
| 4154 // Undef input. | 4155 // Undef input. |
| 4155 void TargetX8632::prelowerPhis() { | 4156 void TargetX8632::prelowerPhis() { |
| 4156 CfgNode *Node = Context.getNode(); | 4157 CfgNode *Node = Context.getNode(); |
| (...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4251 Variable *SrcVar = llvm::dyn_cast<Variable>(Src); | 4252 Variable *SrcVar = llvm::dyn_cast<Variable>(Src); |
| 4252 // Use normal assignment lowering, except lower mem=mem specially | 4253 // Use normal assignment lowering, except lower mem=mem specially |
| 4253 // so we can register-allocate at the same time. | 4254 // so we can register-allocate at the same time. |
| 4254 if (!isMemoryOperand(Dest) || !isMemoryOperand(Src)) { | 4255 if (!isMemoryOperand(Dest) || !isMemoryOperand(Src)) { |
| 4255 lowerAssign(Assign); | 4256 lowerAssign(Assign); |
| 4256 } else { | 4257 } else { |
| 4257 assert(Dest->getType() == Src->getType()); | 4258 assert(Dest->getType() == Src->getType()); |
| 4258 const llvm::SmallBitVector &RegsForType = | 4259 const llvm::SmallBitVector &RegsForType = |
| 4259 getRegisterSetForType(Dest->getType()); | 4260 getRegisterSetForType(Dest->getType()); |
| 4260 llvm::SmallBitVector AvailRegsForType = RegsForType & Available; | 4261 llvm::SmallBitVector AvailRegsForType = RegsForType & Available; |
| 4261 Variable *SpillLoc = NULL; | 4262 Variable *SpillLoc = nullptr; |
| 4262 Variable *Preg = NULL; | 4263 Variable *Preg = nullptr; |
| 4263 // TODO(stichnot): Opportunity for register randomization. | 4264 // TODO(stichnot): Opportunity for register randomization. |
| 4264 int32_t RegNum = AvailRegsForType.find_first(); | 4265 int32_t RegNum = AvailRegsForType.find_first(); |
| 4265 bool IsVector = isVectorType(Dest->getType()); | 4266 bool IsVector = isVectorType(Dest->getType()); |
| 4266 bool NeedSpill = (RegNum == -1); | 4267 bool NeedSpill = (RegNum == -1); |
| 4267 if (NeedSpill) { | 4268 if (NeedSpill) { |
| 4268 // Pick some register to spill and update RegNum. | 4269 // Pick some register to spill and update RegNum. |
| 4269 // TODO(stichnot): Opportunity for register randomization. | 4270 // TODO(stichnot): Opportunity for register randomization. |
| 4270 RegNum = RegsForType.find_first(); | 4271 RegNum = RegsForType.find_first(); |
| 4271 Preg = getPhysicalRegister(RegNum, Dest->getType()); | 4272 Preg = getPhysicalRegister(RegNum, Dest->getType()); |
| 4272 SpillLoc = Func->makeVariable(Dest->getType()); | 4273 SpillLoc = Func->makeVariable(Dest->getType()); |
| (...skipping 128 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4401 // If we're asking for a specific physical register, make sure we're | 4402 // If we're asking for a specific physical register, make sure we're |
| 4402 // not allowing any other operand kinds. (This could be future | 4403 // not allowing any other operand kinds. (This could be future |
| 4403 // work, e.g. allow the shl shift amount to be either an immediate | 4404 // work, e.g. allow the shl shift amount to be either an immediate |
| 4404 // or in ecx.) | 4405 // or in ecx.) |
| 4405 assert(RegNum == Variable::NoRegister || Allowed == Legal_Reg); | 4406 assert(RegNum == Variable::NoRegister || Allowed == Legal_Reg); |
| 4406 if (OperandX8632Mem *Mem = llvm::dyn_cast<OperandX8632Mem>(From)) { | 4407 if (OperandX8632Mem *Mem = llvm::dyn_cast<OperandX8632Mem>(From)) { |
| 4407 // Before doing anything with a Mem operand, we need to ensure | 4408 // Before doing anything with a Mem operand, we need to ensure |
| 4408 // that the Base and Index components are in physical registers. | 4409 // that the Base and Index components are in physical registers. |
| 4409 Variable *Base = Mem->getBase(); | 4410 Variable *Base = Mem->getBase(); |
| 4410 Variable *Index = Mem->getIndex(); | 4411 Variable *Index = Mem->getIndex(); |
| 4411 Variable *RegBase = NULL; | 4412 Variable *RegBase = nullptr; |
| 4412 Variable *RegIndex = NULL; | 4413 Variable *RegIndex = nullptr; |
| 4413 if (Base) { | 4414 if (Base) { |
| 4414 RegBase = legalizeToVar(Base); | 4415 RegBase = legalizeToVar(Base); |
| 4415 } | 4416 } |
| 4416 if (Index) { | 4417 if (Index) { |
| 4417 RegIndex = legalizeToVar(Index); | 4418 RegIndex = legalizeToVar(Index); |
| 4418 } | 4419 } |
| 4419 if (Base != RegBase || Index != RegIndex) { | 4420 if (Base != RegBase || Index != RegIndex) { |
| 4420 From = OperandX8632Mem::create( | 4421 From = OperandX8632Mem::create( |
| 4421 Func, Mem->getType(), RegBase, Mem->getOffset(), RegIndex, | 4422 Func, Mem->getType(), RegBase, Mem->getOffset(), RegIndex, |
| 4422 Mem->getShift(), Mem->getSegmentRegister()); | 4423 Mem->getShift(), Mem->getSegmentRegister()); |
| (...skipping 295 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 4718 } else if (IsConstant || IsExternal) | 4719 } else if (IsConstant || IsExternal) |
| 4719 Str << "\t.zero\t" << Size << "\n"; | 4720 Str << "\t.zero\t" << Size << "\n"; |
| 4720 // Size is part of .comm. | 4721 // Size is part of .comm. |
| 4721 | 4722 |
| 4722 if (IsConstant || HasNonzeroInitializer || IsExternal) | 4723 if (IsConstant || HasNonzeroInitializer || IsExternal) |
| 4723 Str << "\t.size\t" << MangledName << ", " << Size << "\n"; | 4724 Str << "\t.size\t" << MangledName << ", " << Size << "\n"; |
| 4724 // Size is part of .comm. | 4725 // Size is part of .comm. |
| 4725 } | 4726 } |
| 4726 | 4727 |
| 4727 } // end of namespace Ice | 4728 } // end of namespace Ice |
| OLD | NEW |