| OLD | NEW | 
|     1 //===- subzero/src/IceTargetLoweringX8632.cpp - x86-32 lowering -----------===// |     1 //===- subzero/src/IceTargetLoweringX8632.cpp - x86-32 lowering -----------===// | 
|     2 // |     2 // | 
|     3 //                        The Subzero Code Generator |     3 //                        The Subzero Code Generator | 
|     4 // |     4 // | 
|     5 // This file is distributed under the University of Illinois Open Source |     5 // This file is distributed under the University of Illinois Open Source | 
|     6 // License. See LICENSE.TXT for details. |     6 // License. See LICENSE.TXT for details. | 
|     7 // |     7 // | 
|     8 //===----------------------------------------------------------------------===// |     8 //===----------------------------------------------------------------------===// | 
|     9 // |     9 // | 
|    10 // This file implements the TargetLoweringX8632 class, which |    10 // This file implements the TargetLoweringX8632 class, which | 
| (...skipping 304 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|   315   T_placePhiStores.printElapsedUs(Context, "placePhiStores()"); |   315   T_placePhiStores.printElapsedUs(Context, "placePhiStores()"); | 
|   316   Timer T_deletePhis; |   316   Timer T_deletePhis; | 
|   317   Func->deletePhis(); |   317   Func->deletePhis(); | 
|   318   if (Func->hasError()) |   318   if (Func->hasError()) | 
|   319     return; |   319     return; | 
|   320   T_deletePhis.printElapsedUs(Context, "deletePhis()"); |   320   T_deletePhis.printElapsedUs(Context, "deletePhis()"); | 
|   321   Func->dump("After Phi lowering"); |   321   Func->dump("After Phi lowering"); | 
|   322  |   322  | 
|   323   // Address mode optimization. |   323   // Address mode optimization. | 
|   324   Timer T_doAddressOpt; |   324   Timer T_doAddressOpt; | 
 |   325   Func->getVMetadata()->init(); | 
|   325   Func->doAddressOpt(); |   326   Func->doAddressOpt(); | 
|   326   T_doAddressOpt.printElapsedUs(Context, "doAddressOpt()"); |   327   T_doAddressOpt.printElapsedUs(Context, "doAddressOpt()"); | 
|   327  |   328  | 
|   328   // Argument lowering |   329   // Argument lowering | 
|   329   Timer T_argLowering; |   330   Timer T_argLowering; | 
|   330   Func->doArgLowering(); |   331   Func->doArgLowering(); | 
|   331   T_argLowering.printElapsedUs(Context, "lowerArguments()"); |   332   T_argLowering.printElapsedUs(Context, "lowerArguments()"); | 
|   332  |   333  | 
|   333   // Target lowering.  This requires liveness analysis for some parts |   334   // Target lowering.  This requires liveness analysis for some parts | 
|   334   // of the lowering decisions, such as compare/branch fusing.  If |   335   // of the lowering decisions, such as compare/branch fusing.  If | 
| (...skipping 128 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|   463           frameptr, isI8, isInt, isFP)                                         \ |   464           frameptr, isI8, isInt, isFP)                                         \ | 
|   464   name, |   465   name, | 
|   465   REGX8632_TABLE |   466   REGX8632_TABLE | 
|   466 #undef X |   467 #undef X | 
|   467 }; |   468 }; | 
|   468  |   469  | 
|   469 Variable *TargetX8632::getPhysicalRegister(SizeT RegNum) { |   470 Variable *TargetX8632::getPhysicalRegister(SizeT RegNum) { | 
|   470   assert(RegNum < PhysicalRegisters.size()); |   471   assert(RegNum < PhysicalRegisters.size()); | 
|   471   Variable *Reg = PhysicalRegisters[RegNum]; |   472   Variable *Reg = PhysicalRegisters[RegNum]; | 
|   472   if (Reg == NULL) { |   473   if (Reg == NULL) { | 
|   473     CfgNode *Node = NULL; // NULL means multi-block lifetime |   474     Reg = Func->makeVariable(IceType_i32); | 
|   474     Reg = Func->makeVariable(IceType_i32, Node); |  | 
|   475     Reg->setRegNum(RegNum); |   475     Reg->setRegNum(RegNum); | 
|   476     PhysicalRegisters[RegNum] = Reg; |   476     PhysicalRegisters[RegNum] = Reg; | 
 |   477     // Specially mark esp as an "argument" so that it is considered | 
 |   478     // live upon function entry. | 
 |   479     if (RegNum == RegX8632::Reg_esp) | 
 |   480       Reg->setIsArg(); | 
|   477   } |   481   } | 
|   478   return Reg; |   482   return Reg; | 
|   479 } |   483 } | 
|   480  |   484  | 
|   481 IceString TargetX8632::getRegName(SizeT RegNum, Type Ty) const { |   485 IceString TargetX8632::getRegName(SizeT RegNum, Type Ty) const { | 
|   482   assert(RegNum < RegX8632::Reg_NUM); |   486   assert(RegNum < RegX8632::Reg_NUM); | 
|   483   static IceString RegNames8[] = { |   487   static IceString RegNames8[] = { | 
|   484 #define X(val, encode, name, name16, name8, scratch, preserved, stackptr,      \ |   488 #define X(val, encode, name, name16, name8, scratch, preserved, stackptr,      \ | 
|   485           frameptr, isI8, isInt, isFP)                                         \ |   489           frameptr, isI8, isInt, isFP)                                         \ | 
|   486   name8, |   490   name8, | 
| (...skipping 11 matching lines...) Expand all  Loading... | 
|   498   case IceType_i1: |   502   case IceType_i1: | 
|   499   case IceType_i8: |   503   case IceType_i8: | 
|   500     return RegNames8[RegNum]; |   504     return RegNames8[RegNum]; | 
|   501   case IceType_i16: |   505   case IceType_i16: | 
|   502     return RegNames16[RegNum]; |   506     return RegNames16[RegNum]; | 
|   503   default: |   507   default: | 
|   504     return RegNames[RegNum]; |   508     return RegNames[RegNum]; | 
|   505   } |   509   } | 
|   506 } |   510 } | 
|   507  |   511  | 
|   508 void TargetX8632::emitVariable(const Variable *Var, const Cfg *Func) const { |   512 void TargetX8632::emitVariable(const Variable *Var) const { | 
|   509   Ostream &Str = Ctx->getStrEmit(); |   513   Ostream &Str = Ctx->getStrEmit(); | 
|   510   assert(Var->getLocalUseNode() == NULL || |  | 
|   511          Var->getLocalUseNode() == Func->getCurrentNode()); |  | 
|   512   if (Var->hasReg()) { |   514   if (Var->hasReg()) { | 
|   513     Str << getRegName(Var->getRegNum(), Var->getType()); |   515     Str << getRegName(Var->getRegNum(), Var->getType()); | 
|   514     return; |   516     return; | 
|   515   } |   517   } | 
|   516   Str << InstX8632::getWidthString(Var->getType()); |   518   Str << InstX8632::getWidthString(Var->getType()); | 
|   517   Str << " [" << getRegName(getFrameOrStackReg(), IceType_i32); |   519   Str << " [" << getRegName(getFrameOrStackReg(), IceType_i32); | 
|   518   int32_t Offset = Var->getStackOffset(); |   520   int32_t Offset = Var->getStackOffset(); | 
|   519   if (!hasFramePointer()) |   521   if (!hasFramePointer()) | 
|   520     Offset += getStackAdjustment(); |   522     Offset += getStackAdjustment(); | 
|   521   if (Offset) { |   523   if (Offset) { | 
| (...skipping 19 matching lines...) Expand all  Loading... | 
|   541     Variable *Arg = Args[I]; |   543     Variable *Arg = Args[I]; | 
|   542     Type Ty = Arg->getType(); |   544     Type Ty = Arg->getType(); | 
|   543     if (!isVectorType(Ty)) |   545     if (!isVectorType(Ty)) | 
|   544       continue; |   546       continue; | 
|   545     // Replace Arg in the argument list with the home register.  Then |   547     // Replace Arg in the argument list with the home register.  Then | 
|   546     // generate an instruction in the prolog to copy the home register |   548     // generate an instruction in the prolog to copy the home register | 
|   547     // to the assigned location of Arg. |   549     // to the assigned location of Arg. | 
|   548     int32_t RegNum = RegX8632::Reg_xmm0 + NumXmmArgs; |   550     int32_t RegNum = RegX8632::Reg_xmm0 + NumXmmArgs; | 
|   549     ++NumXmmArgs; |   551     ++NumXmmArgs; | 
|   550     IceString Name = "home_reg:" + Arg->getName(); |   552     IceString Name = "home_reg:" + Arg->getName(); | 
|   551     const CfgNode *DefNode = NULL; |   553     Variable *RegisterArg = Func->makeVariable(Ty, Name); | 
|   552     Variable *RegisterArg = Func->makeVariable(Ty, DefNode, Name); |  | 
|   553     RegisterArg->setRegNum(RegNum); |   554     RegisterArg->setRegNum(RegNum); | 
|   554     RegisterArg->setIsArg(Func); |   555     RegisterArg->setIsArg(); | 
|   555     Arg->setIsArg(Func, false); |   556     Arg->setIsArg(false); | 
|   556  |   557  | 
|   557     Args[I] = RegisterArg; |   558     Args[I] = RegisterArg; | 
|   558     Context.insert(InstAssign::create(Func, Arg, RegisterArg)); |   559     Context.insert(InstAssign::create(Func, Arg, RegisterArg)); | 
|   559   } |   560   } | 
|   560 } |   561 } | 
|   561  |   562  | 
|   562 void TargetX8632::sortByAlignment(VarList &Dest, const VarList &Source) const { |   563 void TargetX8632::sortByAlignment(VarList &Dest, const VarList &Source) const { | 
|   563   // Sort the variables into buckets according to the log of their width |   564   // Sort the variables into buckets according to the log of their width | 
|   564   // in bytes. |   565   // in bytes. | 
|   565   const SizeT NumBuckets = |   566   const SizeT NumBuckets = | 
| (...skipping 102 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|   668   // block 1 and C is local to block 2, then C may share a slot with A or B. |   669   // block 1 and C is local to block 2, then C may share a slot with A or B. | 
|   669   // |   670   // | 
|   670   // We cannot coalesce stack slots if this function calls a "returns twice" |   671   // We cannot coalesce stack slots if this function calls a "returns twice" | 
|   671   // function. In that case, basic blocks may be revisited, and variables |   672   // function. In that case, basic blocks may be revisited, and variables | 
|   672   // local to those basic blocks are actually live until after the |   673   // local to those basic blocks are actually live until after the | 
|   673   // called function returns a second time. |   674   // called function returns a second time. | 
|   674   const bool SimpleCoalescing = !callsReturnsTwice(); |   675   const bool SimpleCoalescing = !callsReturnsTwice(); | 
|   675   size_t InArgsSizeBytes = 0; |   676   size_t InArgsSizeBytes = 0; | 
|   676   size_t PreservedRegsSizeBytes = 0; |   677   size_t PreservedRegsSizeBytes = 0; | 
|   677   SpillAreaSizeBytes = 0; |   678   SpillAreaSizeBytes = 0; | 
 |   679   const VariablesMetadata *VMetadata = Func->getVMetadata(); | 
|   678   Context.init(Node); |   680   Context.init(Node); | 
|   679   Context.setInsertPoint(Context.getCur()); |   681   Context.setInsertPoint(Context.getCur()); | 
|   680  |   682  | 
|   681   // Determine stack frame offsets for each Variable without a |   683   // Determine stack frame offsets for each Variable without a | 
|   682   // register assignment.  This can be done as one variable per stack |   684   // register assignment.  This can be done as one variable per stack | 
|   683   // slot.  Or, do coalescing by running the register allocator again |   685   // slot.  Or, do coalescing by running the register allocator again | 
|   684   // with an infinite set of registers (as a side effect, this gives |   686   // with an infinite set of registers (as a side effect, this gives | 
|   685   // variables a second chance at physical register assignment). |   687   // variables a second chance at physical register assignment). | 
|   686   // |   688   // | 
|   687   // A middle ground approach is to leverage sparsity and allocate one |   689   // A middle ground approach is to leverage sparsity and allocate one | 
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|   736  |   738  | 
|   737   SortedSpilledVariables.reserve(SpilledVariables.size()); |   739   SortedSpilledVariables.reserve(SpilledVariables.size()); | 
|   738   sortByAlignment(SortedSpilledVariables, SpilledVariables); |   740   sortByAlignment(SortedSpilledVariables, SpilledVariables); | 
|   739   for (VarList::const_iterator I = SortedSpilledVariables.begin(), |   741   for (VarList::const_iterator I = SortedSpilledVariables.begin(), | 
|   740                                E = SortedSpilledVariables.end(); |   742                                E = SortedSpilledVariables.end(); | 
|   741        I != E; ++I) { |   743        I != E; ++I) { | 
|   742     Variable *Var = *I; |   744     Variable *Var = *I; | 
|   743     size_t Increment = typeWidthInBytesOnStack(Var->getType()); |   745     size_t Increment = typeWidthInBytesOnStack(Var->getType()); | 
|   744     if (!SpillAreaAlignmentBytes) |   746     if (!SpillAreaAlignmentBytes) | 
|   745       SpillAreaAlignmentBytes = Increment; |   747       SpillAreaAlignmentBytes = Increment; | 
|   746     if (SimpleCoalescing) { |   748     if (SimpleCoalescing && VMetadata->isTracked(Var)) { | 
|   747       if (Var->isMultiblockLife()) { |   749       if (VMetadata->isMultiBlock(Var)) { | 
|   748         GlobalsSize += Increment; |   750         GlobalsSize += Increment; | 
|   749       } else { |   751       } else { | 
|   750         SizeT NodeIndex = Var->getLocalUseNode()->getIndex(); |   752         SizeT NodeIndex = VMetadata->getLocalUseNode(Var)->getIndex(); | 
|   751         LocalsSize[NodeIndex] += Increment; |   753         LocalsSize[NodeIndex] += Increment; | 
|   752         if (LocalsSize[NodeIndex] > SpillAreaSizeBytes) |   754         if (LocalsSize[NodeIndex] > SpillAreaSizeBytes) | 
|   753           SpillAreaSizeBytes = LocalsSize[NodeIndex]; |   755           SpillAreaSizeBytes = LocalsSize[NodeIndex]; | 
|   754         if (!LocalsSlotsAlignmentBytes) |   756         if (!LocalsSlotsAlignmentBytes) | 
|   755           LocalsSlotsAlignmentBytes = Increment; |   757           LocalsSlotsAlignmentBytes = Increment; | 
|   756       } |   758       } | 
|   757     } else { |   759     } else { | 
|   758       SpillAreaSizeBytes += Increment; |   760       SpillAreaSizeBytes += Increment; | 
|   759     } |   761     } | 
|   760   } |   762   } | 
| (...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|   845  |   847  | 
|   846   // Fill in stack offsets for locals. |   848   // Fill in stack offsets for locals. | 
|   847   size_t GlobalsSpaceUsed = SpillAreaPaddingBytes; |   849   size_t GlobalsSpaceUsed = SpillAreaPaddingBytes; | 
|   848   LocalsSize.assign(LocalsSize.size(), 0); |   850   LocalsSize.assign(LocalsSize.size(), 0); | 
|   849   size_t NextStackOffset = GlobalsSpaceUsed; |   851   size_t NextStackOffset = GlobalsSpaceUsed; | 
|   850   for (VarList::const_iterator I = SortedSpilledVariables.begin(), |   852   for (VarList::const_iterator I = SortedSpilledVariables.begin(), | 
|   851                                E = SortedSpilledVariables.end(); |   853                                E = SortedSpilledVariables.end(); | 
|   852        I != E; ++I) { |   854        I != E; ++I) { | 
|   853     Variable *Var = *I; |   855     Variable *Var = *I; | 
|   854     size_t Increment = typeWidthInBytesOnStack(Var->getType()); |   856     size_t Increment = typeWidthInBytesOnStack(Var->getType()); | 
|   855     if (SimpleCoalescing) { |   857     if (SimpleCoalescing && VMetadata->isTracked(Var)) { | 
|   856       if (Var->isMultiblockLife()) { |   858       if (VMetadata->isMultiBlock(Var)) { | 
|   857         GlobalsSpaceUsed += Increment; |   859         GlobalsSpaceUsed += Increment; | 
|   858         NextStackOffset = GlobalsSpaceUsed; |   860         NextStackOffset = GlobalsSpaceUsed; | 
|   859       } else { |   861       } else { | 
|   860         SizeT NodeIndex = Var->getLocalUseNode()->getIndex(); |   862         SizeT NodeIndex = VMetadata->getLocalUseNode(Var)->getIndex(); | 
|   861         LocalsSize[NodeIndex] += Increment; |   863         LocalsSize[NodeIndex] += Increment; | 
|   862         NextStackOffset = SpillAreaPaddingBytes + |   864         NextStackOffset = SpillAreaPaddingBytes + | 
|   863                           GlobalsAndSubsequentPaddingSize + |   865                           GlobalsAndSubsequentPaddingSize + | 
|   864                           LocalsSize[NodeIndex]; |   866                           LocalsSize[NodeIndex]; | 
|   865       } |   867       } | 
|   866     } else { |   868     } else { | 
|   867       NextStackOffset += Increment; |   869       NextStackOffset += Increment; | 
|   868     } |   870     } | 
|   869     if (IsEbpBasedFrame) |   871     if (IsEbpBasedFrame) | 
|   870       Var->setStackOffset(-NextStackOffset); |   872       Var->setStackOffset(-NextStackOffset); | 
| (...skipping 157 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|  1028   case IceType_f64: |  1030   case IceType_f64: | 
|  1029     break; |  1031     break; | 
|  1030   } |  1032   } | 
|  1031   Variable *Lo = Var->getLo(); |  1033   Variable *Lo = Var->getLo(); | 
|  1032   Variable *Hi = Var->getHi(); |  1034   Variable *Hi = Var->getHi(); | 
|  1033   if (Lo) { |  1035   if (Lo) { | 
|  1034     assert(Hi); |  1036     assert(Hi); | 
|  1035     return; |  1037     return; | 
|  1036   } |  1038   } | 
|  1037   assert(Hi == NULL); |  1039   assert(Hi == NULL); | 
|  1038   Lo = Func->makeVariable(IceType_i32, Context.getNode(), |  1040   Lo = Func->makeVariable(IceType_i32, Var->getName() + "__lo"); | 
|  1039                           Var->getName() + "__lo"); |  1041   Hi = Func->makeVariable(IceType_i32, Var->getName() + "__hi"); | 
|  1040   Hi = Func->makeVariable(IceType_i32, Context.getNode(), |  | 
|  1041                           Var->getName() + "__hi"); |  | 
|  1042   Var->setLoHi(Lo, Hi); |  1042   Var->setLoHi(Lo, Hi); | 
|  1043   if (Var->getIsArg()) { |  1043   if (Var->getIsArg()) { | 
|  1044     Lo->setIsArg(Func); |  1044     Lo->setIsArg(); | 
|  1045     Hi->setIsArg(Func); |  1045     Hi->setIsArg(); | 
|  1046   } |  1046   } | 
|  1047 } |  1047 } | 
|  1048  |  1048  | 
|  1049 Operand *TargetX8632::loOperand(Operand *Operand) { |  1049 Operand *TargetX8632::loOperand(Operand *Operand) { | 
|  1050   assert(Operand->getType() == IceType_i64); |  1050   assert(Operand->getType() == IceType_i64); | 
|  1051   if (Operand->getType() != IceType_i64) |  1051   if (Operand->getType() != IceType_i64) | 
|  1052     return Operand; |  1052     return Operand; | 
|  1053   if (Variable *Var = llvm::dyn_cast<Variable>(Operand)) { |  1053   if (Variable *Var = llvm::dyn_cast<Variable>(Operand)) { | 
|  1054     split64(Var); |  1054     split64(Var); | 
|  1055     return Var->getLo(); |  1055     return Var->getLo(); | 
| (...skipping 1213 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|  2269       (void)DestType; |  2269       (void)DestType; | 
|  2270       assert((DestType == IceType_i32 && SrcType == IceType_f32) || |  2270       assert((DestType == IceType_i32 && SrcType == IceType_f32) || | 
|  2271              (DestType == IceType_f32 && SrcType == IceType_i32)); |  2271              (DestType == IceType_f32 && SrcType == IceType_i32)); | 
|  2272       // a.i32 = bitcast b.f32 ==> |  2272       // a.i32 = bitcast b.f32 ==> | 
|  2273       //   t.f32 = b.f32 |  2273       //   t.f32 = b.f32 | 
|  2274       //   s.f32 = spill t.f32 |  2274       //   s.f32 = spill t.f32 | 
|  2275       //   a.i32 = s.f32 |  2275       //   a.i32 = s.f32 | 
|  2276       Variable *T = NULL; |  2276       Variable *T = NULL; | 
|  2277       // TODO: Should be able to force a spill setup by calling legalize() with |  2277       // TODO: Should be able to force a spill setup by calling legalize() with | 
|  2278       // Legal_Mem and not Legal_Reg or Legal_Imm. |  2278       // Legal_Mem and not Legal_Reg or Legal_Imm. | 
|  2279       SpillVariable *SpillVar = |  2279       SpillVariable *SpillVar = Func->makeVariable<SpillVariable>(SrcType); | 
|  2280           Func->makeVariable<SpillVariable>(SrcType, Context.getNode()); |  | 
|  2281       SpillVar->setLinkedTo(Dest); |  2280       SpillVar->setLinkedTo(Dest); | 
|  2282       Variable *Spill = SpillVar; |  2281       Variable *Spill = SpillVar; | 
|  2283       Spill->setWeight(RegWeight::Zero); |  2282       Spill->setWeight(RegWeight::Zero); | 
|  2284       _mov(T, Src0RM); |  2283       _mov(T, Src0RM); | 
|  2285       _mov(Spill, T); |  2284       _mov(Spill, T); | 
|  2286       _mov(Dest, Spill); |  2285       _mov(Dest, Spill); | 
|  2287     } break; |  2286     } break; | 
|  2288     case IceType_i64: { |  2287     case IceType_i64: { | 
|  2289       Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); |  2288       Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); | 
|  2290       assert(Src0RM->getType() == IceType_f64); |  2289       assert(Src0RM->getType() == IceType_f64); | 
|  2291       // a.i64 = bitcast b.f64 ==> |  2290       // a.i64 = bitcast b.f64 ==> | 
|  2292       //   s.f64 = spill b.f64 |  2291       //   s.f64 = spill b.f64 | 
|  2293       //   t_lo.i32 = lo(s.f64) |  2292       //   t_lo.i32 = lo(s.f64) | 
|  2294       //   a_lo.i32 = t_lo.i32 |  2293       //   a_lo.i32 = t_lo.i32 | 
|  2295       //   t_hi.i32 = hi(s.f64) |  2294       //   t_hi.i32 = hi(s.f64) | 
|  2296       //   a_hi.i32 = t_hi.i32 |  2295       //   a_hi.i32 = t_hi.i32 | 
|  2297       SpillVariable *SpillVar = |  2296       SpillVariable *SpillVar = Func->makeVariable<SpillVariable>(IceType_f64); | 
|  2298           Func->makeVariable<SpillVariable>(IceType_f64, Context.getNode()); |  | 
|  2299       SpillVar->setLinkedTo(llvm::dyn_cast<Variable>(Src0RM)); |  2297       SpillVar->setLinkedTo(llvm::dyn_cast<Variable>(Src0RM)); | 
|  2300       Variable *Spill = SpillVar; |  2298       Variable *Spill = SpillVar; | 
|  2301       Spill->setWeight(RegWeight::Zero); |  2299       Spill->setWeight(RegWeight::Zero); | 
|  2302       _movq(Spill, Src0RM); |  2300       _movq(Spill, Src0RM); | 
|  2303  |  2301  | 
|  2304       Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); |  2302       Variable *DestLo = llvm::cast<Variable>(loOperand(Dest)); | 
|  2305       Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); |  2303       Variable *DestHi = llvm::cast<Variable>(hiOperand(Dest)); | 
|  2306       Variable *T_Lo = makeReg(IceType_i32); |  2304       Variable *T_Lo = makeReg(IceType_i32); | 
|  2307       Variable *T_Hi = makeReg(IceType_i32); |  2305       Variable *T_Hi = makeReg(IceType_i32); | 
|  2308       VariableSplit *SpillLo = |  2306       VariableSplit *SpillLo = | 
|  2309           VariableSplit::create(Func, Spill, VariableSplit::Low); |  2307           VariableSplit::create(Func, Spill, VariableSplit::Low); | 
|  2310       VariableSplit *SpillHi = |  2308       VariableSplit *SpillHi = | 
|  2311           VariableSplit::create(Func, Spill, VariableSplit::High); |  2309           VariableSplit::create(Func, Spill, VariableSplit::High); | 
|  2312  |  2310  | 
|  2313       _mov(T_Lo, SpillLo); |  2311       _mov(T_Lo, SpillLo); | 
|  2314       _mov(DestLo, T_Lo); |  2312       _mov(DestLo, T_Lo); | 
|  2315       _mov(T_Hi, SpillHi); |  2313       _mov(T_Hi, SpillHi); | 
|  2316       _mov(DestHi, T_Hi); |  2314       _mov(DestHi, T_Hi); | 
|  2317     } break; |  2315     } break; | 
|  2318     case IceType_f64: { |  2316     case IceType_f64: { | 
|  2319       Src0 = legalize(Src0); |  2317       Src0 = legalize(Src0); | 
|  2320       assert(Src0->getType() == IceType_i64); |  2318       assert(Src0->getType() == IceType_i64); | 
|  2321       // a.f64 = bitcast b.i64 ==> |  2319       // a.f64 = bitcast b.i64 ==> | 
|  2322       //   t_lo.i32 = b_lo.i32 |  2320       //   t_lo.i32 = b_lo.i32 | 
|  2323       //   FakeDef(s.f64) |  2321       //   FakeDef(s.f64) | 
|  2324       //   lo(s.f64) = t_lo.i32 |  2322       //   lo(s.f64) = t_lo.i32 | 
|  2325       //   t_hi.i32 = b_hi.i32 |  2323       //   t_hi.i32 = b_hi.i32 | 
|  2326       //   hi(s.f64) = t_hi.i32 |  2324       //   hi(s.f64) = t_hi.i32 | 
|  2327       //   a.f64 = s.f64 |  2325       //   a.f64 = s.f64 | 
|  2328       SpillVariable *SpillVar = |  2326       SpillVariable *SpillVar = Func->makeVariable<SpillVariable>(IceType_f64); | 
|  2329           Func->makeVariable<SpillVariable>(IceType_f64, Context.getNode()); |  | 
|  2330       SpillVar->setLinkedTo(Dest); |  2327       SpillVar->setLinkedTo(Dest); | 
|  2331       Variable *Spill = SpillVar; |  2328       Variable *Spill = SpillVar; | 
|  2332       Spill->setWeight(RegWeight::Zero); |  2329       Spill->setWeight(RegWeight::Zero); | 
|  2333  |  2330  | 
|  2334       Variable *T_Lo = NULL, *T_Hi = NULL; |  2331       Variable *T_Lo = NULL, *T_Hi = NULL; | 
|  2335       VariableSplit *SpillLo = |  2332       VariableSplit *SpillLo = | 
|  2336           VariableSplit::create(Func, Spill, VariableSplit::Low); |  2333           VariableSplit::create(Func, Spill, VariableSplit::Low); | 
|  2337       VariableSplit *SpillHi = |  2334       VariableSplit *SpillHi = | 
|  2338           VariableSplit::create(Func, Spill, VariableSplit::High); |  2335           VariableSplit::create(Func, Spill, VariableSplit::High); | 
|  2339       _mov(T_Lo, loOperand(Src0)); |  2336       _mov(T_Lo, loOperand(Src0)); | 
|  2340       // Technically, the Spill is defined after the _store happens, but |  2337       // Technically, the Spill is defined after the _store happens, but | 
|  2341       // SpillLo is considered a "use" of Spill so define Spill before it |  2338       // SpillLo is considered a "use" of Spill so define Spill before it | 
|  2342       // is used. |  2339       // is used. | 
|  2343       Context.insert(InstFakeDef::create(Func, Spill)); |  2340       Context.insert(InstFakeDef::create(Func, Spill)); | 
|  2344       _store(T_Lo, SpillLo); |  2341       _store(T_Lo, SpillLo); | 
|  2345       _mov(T_Hi, hiOperand(Src0)); |  2342       _mov(T_Hi, hiOperand(Src0)); | 
|  2346       _store(T_Hi, SpillHi); |  2343       _store(T_Hi, SpillHi); | 
|  2347       _movq(Dest, Spill); |  2344       _movq(Dest, Spill); | 
|  2348     } break; |  2345     } break; | 
|  2349     case IceType_v8i1: { |  2346     case IceType_v8i1: { | 
|  2350       assert(Src0->getType() == IceType_i8); |  2347       assert(Src0->getType() == IceType_i8); | 
|  2351       InstCall *Call = makeHelperCall("Sz_bitcast_i8_to_v8i1", Dest, 1); |  2348       InstCall *Call = makeHelperCall("Sz_bitcast_i8_to_v8i1", Dest, 1); | 
|  2352       Variable *Src0AsI32 = Func->makeVariable(stackSlotType(), |  2349       Variable *Src0AsI32 = Func->makeVariable(stackSlotType()); | 
|  2353                                                Context.getNode()); |  | 
|  2354       // Arguments to functions are required to be at least 32 bits wide. |  2350       // Arguments to functions are required to be at least 32 bits wide. | 
|  2355       lowerCast(InstCast::create(Func, InstCast::Zext, Src0AsI32, Src0)); |  2351       lowerCast(InstCast::create(Func, InstCast::Zext, Src0AsI32, Src0)); | 
|  2356       Call->addArg(Src0AsI32); |  2352       Call->addArg(Src0AsI32); | 
|  2357       lowerCall(Call); |  2353       lowerCall(Call); | 
|  2358     } break; |  2354     } break; | 
|  2359     case IceType_v16i1: { |  2355     case IceType_v16i1: { | 
|  2360       assert(Src0->getType() == IceType_i16); |  2356       assert(Src0->getType() == IceType_i16); | 
|  2361       InstCall *Call = makeHelperCall("Sz_bitcast_i16_to_v16i1", Dest, 1); |  2357       InstCall *Call = makeHelperCall("Sz_bitcast_i16_to_v16i1", Dest, 1); | 
|  2362       Variable *Src0AsI32 = Func->makeVariable(stackSlotType(), |  2358       Variable *Src0AsI32 = Func->makeVariable(stackSlotType()); | 
|  2363                                                Context.getNode()); |  | 
|  2364       // Arguments to functions are required to be at least 32 bits wide. |  2359       // Arguments to functions are required to be at least 32 bits wide. | 
|  2365       lowerCast(InstCast::create(Func, InstCast::Zext, Src0AsI32, Src0)); |  2360       lowerCast(InstCast::create(Func, InstCast::Zext, Src0AsI32, Src0)); | 
|  2366       Call->addArg(Src0AsI32); |  2361       Call->addArg(Src0AsI32); | 
|  2367       lowerCall(Call); |  2362       lowerCall(Call); | 
|  2368     } break; |  2363     } break; | 
|  2369     case IceType_v8i16: |  2364     case IceType_v8i16: | 
|  2370     case IceType_v16i8: |  2365     case IceType_v16i8: | 
|  2371     case IceType_v4i32: |  2366     case IceType_v4i32: | 
|  2372     case IceType_v4f32: { |  2367     case IceType_v4f32: { | 
|  2373       _movp(Dest, legalizeToVar(Src0)); |  2368       _movp(Dest, legalizeToVar(Src0)); | 
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|  2422       // keep the live range analysis consistent. |  2417       // keep the live range analysis consistent. | 
|  2423       Context.insert(InstFakeDef::create(Func, ExtractedElementR)); |  2418       Context.insert(InstFakeDef::create(Func, ExtractedElementR)); | 
|  2424       _movss(ExtractedElementR, T); |  2419       _movss(ExtractedElementR, T); | 
|  2425     } |  2420     } | 
|  2426   } else { |  2421   } else { | 
|  2427     assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); |  2422     assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); | 
|  2428     // Spill the value to a stack slot and do the extraction in memory. |  2423     // Spill the value to a stack slot and do the extraction in memory. | 
|  2429     // |  2424     // | 
|  2430     // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when |  2425     // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when | 
|  2431     // support for legalizing to mem is implemented. |  2426     // support for legalizing to mem is implemented. | 
|  2432     Variable *Slot = Func->makeVariable(Ty, Context.getNode()); |  2427     Variable *Slot = Func->makeVariable(Ty); | 
|  2433     Slot->setWeight(RegWeight::Zero); |  2428     Slot->setWeight(RegWeight::Zero); | 
|  2434     _movp(Slot, legalizeToVar(SourceVectNotLegalized)); |  2429     _movp(Slot, legalizeToVar(SourceVectNotLegalized)); | 
|  2435  |  2430  | 
|  2436     // Compute the location of the element in memory. |  2431     // Compute the location of the element in memory. | 
|  2437     unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); |  2432     unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); | 
|  2438     OperandX8632Mem *Loc = |  2433     OperandX8632Mem *Loc = | 
|  2439         getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); |  2434         getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); | 
|  2440     _mov(ExtractedElementR, Loc); |  2435     _mov(ExtractedElementR, Loc); | 
|  2441   } |  2436   } | 
|  2442  |  2437  | 
| (...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|  2577       case IceType_v4i1: |  2572       case IceType_v4i1: | 
|  2578         NewTy = IceType_v4i32; |  2573         NewTy = IceType_v4i32; | 
|  2579         break; |  2574         break; | 
|  2580       case IceType_v8i1: |  2575       case IceType_v8i1: | 
|  2581         NewTy = IceType_v8i16; |  2576         NewTy = IceType_v8i16; | 
|  2582         break; |  2577         break; | 
|  2583       case IceType_v16i1: |  2578       case IceType_v16i1: | 
|  2584         NewTy = IceType_v16i8; |  2579         NewTy = IceType_v16i8; | 
|  2585         break; |  2580         break; | 
|  2586       } |  2581       } | 
|  2587       Variable *NewSrc0 = Func->makeVariable(NewTy, Context.getNode()); |  2582       Variable *NewSrc0 = Func->makeVariable(NewTy); | 
|  2588       Variable *NewSrc1 = Func->makeVariable(NewTy, Context.getNode()); |  2583       Variable *NewSrc1 = Func->makeVariable(NewTy); | 
|  2589       lowerCast(InstCast::create(Func, InstCast::Sext, NewSrc0, Src0)); |  2584       lowerCast(InstCast::create(Func, InstCast::Sext, NewSrc0, Src0)); | 
|  2590       lowerCast(InstCast::create(Func, InstCast::Sext, NewSrc1, Src1)); |  2585       lowerCast(InstCast::create(Func, InstCast::Sext, NewSrc1, Src1)); | 
|  2591       Src0 = NewSrc0; |  2586       Src0 = NewSrc0; | 
|  2592       Src1 = NewSrc1; |  2587       Src1 = NewSrc1; | 
|  2593       Ty = NewTy; |  2588       Ty = NewTy; | 
|  2594     } |  2589     } | 
|  2595  |  2590  | 
|  2596     InstIcmp::ICond Condition = Inst->getCondition(); |  2591     InstIcmp::ICond Condition = Inst->getCondition(); | 
|  2597  |  2592  | 
|  2598     Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); |  2593     Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); | 
| (...skipping 154 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|  2753   unsigned Index = ElementIndex->getValue(); |  2748   unsigned Index = ElementIndex->getValue(); | 
|  2754   assert(Index < typeNumElements(SourceVectNotLegalized->getType())); |  2749   assert(Index < typeNumElements(SourceVectNotLegalized->getType())); | 
|  2755  |  2750  | 
|  2756   Type Ty = SourceVectNotLegalized->getType(); |  2751   Type Ty = SourceVectNotLegalized->getType(); | 
|  2757   Type ElementTy = typeElementType(Ty); |  2752   Type ElementTy = typeElementType(Ty); | 
|  2758   Type InVectorElementTy = getInVectorElementType(Ty); |  2753   Type InVectorElementTy = getInVectorElementType(Ty); | 
|  2759  |  2754  | 
|  2760   if (ElementTy == IceType_i1) { |  2755   if (ElementTy == IceType_i1) { | 
|  2761     // Expand the element to the appropriate size for it to be inserted |  2756     // Expand the element to the appropriate size for it to be inserted | 
|  2762     // in the vector. |  2757     // in the vector. | 
|  2763     Variable *Expanded = |  2758     Variable *Expanded = Func->makeVariable(InVectorElementTy); | 
|  2764         Func->makeVariable(InVectorElementTy, Context.getNode()); |  | 
|  2765     InstCast *Cast = InstCast::create(Func, InstCast::Zext, Expanded, |  2759     InstCast *Cast = InstCast::create(Func, InstCast::Zext, Expanded, | 
|  2766                                       ElementToInsertNotLegalized); |  2760                                       ElementToInsertNotLegalized); | 
|  2767     lowerCast(Cast); |  2761     lowerCast(Cast); | 
|  2768     ElementToInsertNotLegalized = Expanded; |  2762     ElementToInsertNotLegalized = Expanded; | 
|  2769   } |  2763   } | 
|  2770  |  2764  | 
|  2771   if (Ty == IceType_v8i16 || Ty == IceType_v8i1 || InstructionSet >= SSE4_1) { |  2765   if (Ty == IceType_v8i16 || Ty == IceType_v8i1 || InstructionSet >= SSE4_1) { | 
|  2772     // Use insertps, pinsrb, pinsrw, or pinsrd. |  2766     // Use insertps, pinsrb, pinsrw, or pinsrd. | 
|  2773     Operand *ElementRM = |  2767     Operand *ElementRM = | 
|  2774         legalize(ElementToInsertNotLegalized, Legal_Reg | Legal_Mem); |  2768         legalize(ElementToInsertNotLegalized, Legal_Reg | Legal_Mem); | 
| (...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|  2846       _shufps(T, ElementR, Mask2Constant); |  2840       _shufps(T, ElementR, Mask2Constant); | 
|  2847       _movp(Inst->getDest(), T); |  2841       _movp(Inst->getDest(), T); | 
|  2848     } |  2842     } | 
|  2849   } else { |  2843   } else { | 
|  2850     assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); |  2844     assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); | 
|  2851     // Spill the value to a stack slot and perform the insertion in |  2845     // Spill the value to a stack slot and perform the insertion in | 
|  2852     // memory. |  2846     // memory. | 
|  2853     // |  2847     // | 
|  2854     // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when |  2848     // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when | 
|  2855     // support for legalizing to mem is implemented. |  2849     // support for legalizing to mem is implemented. | 
|  2856     Variable *Slot = Func->makeVariable(Ty, Context.getNode()); |  2850     Variable *Slot = Func->makeVariable(Ty); | 
|  2857     Slot->setWeight(RegWeight::Zero); |  2851     Slot->setWeight(RegWeight::Zero); | 
|  2858     _movp(Slot, legalizeToVar(SourceVectNotLegalized)); |  2852     _movp(Slot, legalizeToVar(SourceVectNotLegalized)); | 
|  2859  |  2853  | 
|  2860     // Compute the location of the position to insert in memory. |  2854     // Compute the location of the position to insert in memory. | 
|  2861     unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); |  2855     unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); | 
|  2862     OperandX8632Mem *Loc = |  2856     OperandX8632Mem *Loc = | 
|  2863         getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); |  2857         getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); | 
|  2864     _store(legalizeToVar(ElementToInsertNotLegalized), Loc); |  2858     _store(legalizeToVar(ElementToInsertNotLegalized), Loc); | 
|  2865  |  2859  | 
|  2866     Variable *T = makeReg(Ty); |  2860     Variable *T = makeReg(Ty); | 
| (...skipping 249 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|  3116     Call->addArg(Instr->getArg(2)); |  3110     Call->addArg(Instr->getArg(2)); | 
|  3117     lowerCall(Call); |  3111     lowerCall(Call); | 
|  3118     return; |  3112     return; | 
|  3119   } |  3113   } | 
|  3120   case Intrinsics::Memset: { |  3114   case Intrinsics::Memset: { | 
|  3121     // The value operand needs to be extended to a stack slot size |  3115     // The value operand needs to be extended to a stack slot size | 
|  3122     // because the PNaCl ABI requires arguments to be at least 32 bits |  3116     // because the PNaCl ABI requires arguments to be at least 32 bits | 
|  3123     // wide. |  3117     // wide. | 
|  3124     Operand *ValOp = Instr->getArg(1); |  3118     Operand *ValOp = Instr->getArg(1); | 
|  3125     assert(ValOp->getType() == IceType_i8); |  3119     assert(ValOp->getType() == IceType_i8); | 
|  3126     Variable *ValExt = Func->makeVariable(stackSlotType(), Context.getNode()); |  3120     Variable *ValExt = Func->makeVariable(stackSlotType()); | 
|  3127     lowerCast(InstCast::create(Func, InstCast::Zext, ValExt, ValOp)); |  3121     lowerCast(InstCast::create(Func, InstCast::Zext, ValExt, ValOp)); | 
|  3128     InstCall *Call = makeHelperCall("memset", NULL, 3); |  3122     InstCall *Call = makeHelperCall("memset", NULL, 3); | 
|  3129     Call->addArg(Instr->getArg(0)); |  3123     Call->addArg(Instr->getArg(0)); | 
|  3130     Call->addArg(ValExt); |  3124     Call->addArg(ValExt); | 
|  3131     Call->addArg(Instr->getArg(2)); |  3125     Call->addArg(Instr->getArg(2)); | 
|  3132     lowerCall(Call); |  3126     lowerCall(Call); | 
|  3133     return; |  3127     return; | 
|  3134   } |  3128   } | 
|  3135   case Intrinsics::NaClReadTP: { |  3129   case Intrinsics::NaClReadTP: { | 
|  3136     if (Ctx->getFlags().UseSandboxing) { |  3130     if (Ctx->getFlags().UseSandboxing) { | 
| (...skipping 435 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|  3572   else |  3566   else | 
|  3573     Str << "<null>"; |  3567     Str << "<null>"; | 
|  3574   Str << ", Index="; |  3568   Str << ", Index="; | 
|  3575   if (Index) |  3569   if (Index) | 
|  3576     Index->dump(Func); |  3570     Index->dump(Func); | 
|  3577   else |  3571   else | 
|  3578     Str << "<null>"; |  3572     Str << "<null>"; | 
|  3579   Str << ", Shift=" << Shift << ", Offset=" << Offset << "\n"; |  3573   Str << ", Shift=" << Shift << ", Offset=" << Offset << "\n"; | 
|  3580 } |  3574 } | 
|  3581  |  3575  | 
|  3582 bool matchTransitiveAssign(Variable *&Var, const Inst *&Reason) { |  3576 bool matchTransitiveAssign(const VariablesMetadata *VMetadata, Variable *&Var, | 
 |  3577                            const Inst *&Reason) { | 
|  3583   // Var originates from Var=SrcVar ==> |  3578   // Var originates from Var=SrcVar ==> | 
|  3584   //   set Var:=SrcVar |  3579   //   set Var:=SrcVar | 
|  3585   if (Var == NULL) |  3580   if (Var == NULL) | 
|  3586     return false; |  3581     return false; | 
|  3587   if (const Inst *VarAssign = Var->getDefinition()) { |  3582   if (const Inst *VarAssign = VMetadata->getDefinition(Var)) { | 
|  3588     if (llvm::isa<InstAssign>(VarAssign)) { |  3583     if (llvm::isa<InstAssign>(VarAssign)) { | 
|  3589       Operand *SrcOp = VarAssign->getSrc(0); |  3584       Operand *SrcOp = VarAssign->getSrc(0); | 
|  3590       assert(SrcOp); |  3585       assert(SrcOp); | 
|  3591       if (Variable *SrcVar = llvm::dyn_cast<Variable>(SrcOp)) { |  3586       if (Variable *SrcVar = llvm::dyn_cast<Variable>(SrcOp)) { | 
|  3592         if (!SrcVar->getIsMultidef() && |  3587         if (!VMetadata->isMultiDef(SrcVar) && | 
|  3593             // TODO: ensure SrcVar stays single-BB |  3588             // TODO: ensure SrcVar stays single-BB | 
|  3594             true) { |  3589             true) { | 
|  3595           Var = SrcVar; |  3590           Var = SrcVar; | 
|  3596           Reason = VarAssign; |  3591           Reason = VarAssign; | 
|  3597           return true; |  3592           return true; | 
|  3598         } |  3593         } | 
|  3599       } |  3594       } | 
|  3600     } |  3595     } | 
|  3601   } |  3596   } | 
|  3602   return false; |  3597   return false; | 
|  3603 } |  3598 } | 
|  3604  |  3599  | 
|  3605 bool matchCombinedBaseIndex(Variable *&Base, Variable *&Index, uint16_t &Shift, |  3600 bool matchCombinedBaseIndex(const VariablesMetadata *VMetadata, Variable *&Base, | 
 |  3601                             Variable *&Index, uint16_t &Shift, | 
|  3606                             const Inst *&Reason) { |  3602                             const Inst *&Reason) { | 
|  3607   // Index==NULL && Base is Base=Var1+Var2 ==> |  3603   // Index==NULL && Base is Base=Var1+Var2 ==> | 
|  3608   //   set Base=Var1, Index=Var2, Shift=0 |  3604   //   set Base=Var1, Index=Var2, Shift=0 | 
|  3609   if (Base == NULL) |  3605   if (Base == NULL) | 
|  3610     return false; |  3606     return false; | 
|  3611   if (Index != NULL) |  3607   if (Index != NULL) | 
|  3612     return false; |  3608     return false; | 
|  3613   const Inst *BaseInst = Base->getDefinition(); |  3609   const Inst *BaseInst = VMetadata->getDefinition(Base); | 
|  3614   if (BaseInst == NULL) |  3610   if (BaseInst == NULL) | 
|  3615     return false; |  3611     return false; | 
|  3616   if (BaseInst->getSrcSize() < 2) |  3612   if (BaseInst->getSrcSize() < 2) | 
|  3617     return false; |  3613     return false; | 
|  3618   if (Variable *Var1 = llvm::dyn_cast<Variable>(BaseInst->getSrc(0))) { |  3614   if (Variable *Var1 = llvm::dyn_cast<Variable>(BaseInst->getSrc(0))) { | 
|  3619     if (Var1->getIsMultidef()) |  3615     if (VMetadata->isMultiDef(Var1)) | 
|  3620       return false; |  3616       return false; | 
|  3621     if (Variable *Var2 = llvm::dyn_cast<Variable>(BaseInst->getSrc(1))) { |  3617     if (Variable *Var2 = llvm::dyn_cast<Variable>(BaseInst->getSrc(1))) { | 
|  3622       if (Var2->getIsMultidef()) |  3618       if (VMetadata->isMultiDef(Var2)) | 
|  3623         return false; |  3619         return false; | 
|  3624       if (isAdd(BaseInst) && |  3620       if (isAdd(BaseInst) && | 
|  3625           // TODO: ensure Var1 and Var2 stay single-BB |  3621           // TODO: ensure Var1 and Var2 stay single-BB | 
|  3626           true) { |  3622           true) { | 
|  3627         Base = Var1; |  3623         Base = Var1; | 
|  3628         Index = Var2; |  3624         Index = Var2; | 
|  3629         Shift = 0; // should already have been 0 |  3625         Shift = 0; // should already have been 0 | 
|  3630         Reason = BaseInst; |  3626         Reason = BaseInst; | 
|  3631         return true; |  3627         return true; | 
|  3632       } |  3628       } | 
|  3633     } |  3629     } | 
|  3634   } |  3630   } | 
|  3635   return false; |  3631   return false; | 
|  3636 } |  3632 } | 
|  3637  |  3633  | 
|  3638 bool matchShiftedIndex(Variable *&Index, uint16_t &Shift, const Inst *&Reason) { |  3634 bool matchShiftedIndex(const VariablesMetadata *VMetadata, Variable *&Index, | 
 |  3635                        uint16_t &Shift, const Inst *&Reason) { | 
|  3639   // Index is Index=Var*Const && log2(Const)+Shift<=3 ==> |  3636   // Index is Index=Var*Const && log2(Const)+Shift<=3 ==> | 
|  3640   //   Index=Var, Shift+=log2(Const) |  3637   //   Index=Var, Shift+=log2(Const) | 
|  3641   if (Index == NULL) |  3638   if (Index == NULL) | 
|  3642     return false; |  3639     return false; | 
|  3643   const Inst *IndexInst = Index->getDefinition(); |  3640   const Inst *IndexInst = VMetadata->getDefinition(Index); | 
|  3644   if (IndexInst == NULL) |  3641   if (IndexInst == NULL) | 
|  3645     return false; |  3642     return false; | 
|  3646   if (IndexInst->getSrcSize() < 2) |  3643   if (IndexInst->getSrcSize() < 2) | 
|  3647     return false; |  3644     return false; | 
|  3648   if (const InstArithmetic *ArithInst = |  3645   if (const InstArithmetic *ArithInst = | 
|  3649           llvm::dyn_cast<InstArithmetic>(IndexInst)) { |  3646           llvm::dyn_cast<InstArithmetic>(IndexInst)) { | 
|  3650     if (Variable *Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(0))) { |  3647     if (Variable *Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(0))) { | 
|  3651       if (ConstantInteger32 *Const = |  3648       if (ConstantInteger32 *Const = | 
|  3652               llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1))) { |  3649               llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1))) { | 
|  3653         if (ArithInst->getOp() == InstArithmetic::Mul && |  3650         if (ArithInst->getOp() == InstArithmetic::Mul && | 
|  3654             !Var->getIsMultidef() && Const->getType() == IceType_i32) { |  3651             !VMetadata->isMultiDef(Var) && Const->getType() == IceType_i32) { | 
|  3655           uint64_t Mult = Const->getValue(); |  3652           uint64_t Mult = Const->getValue(); | 
|  3656           uint32_t LogMult; |  3653           uint32_t LogMult; | 
|  3657           switch (Mult) { |  3654           switch (Mult) { | 
|  3658           case 1: |  3655           case 1: | 
|  3659             LogMult = 0; |  3656             LogMult = 0; | 
|  3660             break; |  3657             break; | 
|  3661           case 2: |  3658           case 2: | 
|  3662             LogMult = 1; |  3659             LogMult = 1; | 
|  3663             break; |  3660             break; | 
|  3664           case 4: |  3661           case 4: | 
| (...skipping 11 matching lines...) Expand all  Loading... | 
|  3676             Reason = IndexInst; |  3673             Reason = IndexInst; | 
|  3677             return true; |  3674             return true; | 
|  3678           } |  3675           } | 
|  3679         } |  3676         } | 
|  3680       } |  3677       } | 
|  3681     } |  3678     } | 
|  3682   } |  3679   } | 
|  3683   return false; |  3680   return false; | 
|  3684 } |  3681 } | 
|  3685  |  3682  | 
|  3686 bool matchOffsetBase(Variable *&Base, int32_t &Offset, const Inst *&Reason) { |  3683 bool matchOffsetBase(const VariablesMetadata *VMetadata, Variable *&Base, | 
 |  3684                      int32_t &Offset, const Inst *&Reason) { | 
|  3687   // Base is Base=Var+Const || Base is Base=Const+Var ==> |  3685   // Base is Base=Var+Const || Base is Base=Const+Var ==> | 
|  3688   //   set Base=Var, Offset+=Const |  3686   //   set Base=Var, Offset+=Const | 
|  3689   // Base is Base=Var-Const ==> |  3687   // Base is Base=Var-Const ==> | 
|  3690   //   set Base=Var, Offset-=Const |  3688   //   set Base=Var, Offset-=Const | 
|  3691   if (Base == NULL) |  3689   if (Base == NULL) | 
|  3692     return false; |  3690     return false; | 
|  3693   const Inst *BaseInst = Base->getDefinition(); |  3691   const Inst *BaseInst = VMetadata->getDefinition(Base); | 
|  3694   if (BaseInst == NULL) |  3692   if (BaseInst == NULL) | 
|  3695     return false; |  3693     return false; | 
|  3696   if (const InstArithmetic *ArithInst = |  3694   if (const InstArithmetic *ArithInst = | 
|  3697           llvm::dyn_cast<const InstArithmetic>(BaseInst)) { |  3695           llvm::dyn_cast<const InstArithmetic>(BaseInst)) { | 
|  3698     if (ArithInst->getOp() != InstArithmetic::Add && |  3696     if (ArithInst->getOp() != InstArithmetic::Add && | 
|  3699         ArithInst->getOp() != InstArithmetic::Sub) |  3697         ArithInst->getOp() != InstArithmetic::Sub) | 
|  3700       return false; |  3698       return false; | 
|  3701     bool IsAdd = ArithInst->getOp() == InstArithmetic::Add; |  3699     bool IsAdd = ArithInst->getOp() == InstArithmetic::Add; | 
|  3702     Variable *Var = NULL; |  3700     Variable *Var = NULL; | 
|  3703     ConstantInteger32 *Const = NULL; |  3701     ConstantInteger32 *Const = NULL; | 
|  3704     if (Variable *VariableOperand = |  3702     if (Variable *VariableOperand = | 
|  3705             llvm::dyn_cast<Variable>(ArithInst->getSrc(0))) { |  3703             llvm::dyn_cast<Variable>(ArithInst->getSrc(0))) { | 
|  3706       Var = VariableOperand; |  3704       Var = VariableOperand; | 
|  3707       Const = llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1)); |  3705       Const = llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1)); | 
|  3708     } else if (IsAdd) { |  3706     } else if (IsAdd) { | 
|  3709       Const = llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(0)); |  3707       Const = llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(0)); | 
|  3710       Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(1)); |  3708       Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(1)); | 
|  3711     } |  3709     } | 
|  3712     if (Var == NULL || Const == NULL || Var->getIsMultidef()) |  3710     if (Var == NULL || Const == NULL || VMetadata->isMultiDef(Var)) | 
|  3713       return false; |  3711       return false; | 
|  3714     int32_t MoreOffset = IsAdd ? Const->getValue() : -Const->getValue(); |  3712     int32_t MoreOffset = IsAdd ? Const->getValue() : -Const->getValue(); | 
|  3715     if (WouldOverflowAdd(Offset, MoreOffset)) |  3713     if (WouldOverflowAdd(Offset, MoreOffset)) | 
|  3716       return false; |  3714       return false; | 
|  3717     Base = Var; |  3715     Base = Var; | 
|  3718     Offset += MoreOffset; |  3716     Offset += MoreOffset; | 
|  3719     Reason = BaseInst; |  3717     Reason = BaseInst; | 
|  3720     return true; |  3718     return true; | 
|  3721   } |  3719   } | 
|  3722   return false; |  3720   return false; | 
|  3723 } |  3721 } | 
|  3724  |  3722  | 
|  3725 void computeAddressOpt(Cfg *Func, const Inst *Instr, Variable *&Base, |  3723 void computeAddressOpt(Cfg *Func, const Inst *Instr, Variable *&Base, | 
|  3726                        Variable *&Index, uint16_t &Shift, int32_t &Offset) { |  3724                        Variable *&Index, uint16_t &Shift, int32_t &Offset) { | 
|  3727   Func->resetCurrentNode(); |  3725   Func->resetCurrentNode(); | 
|  3728   if (Func->getContext()->isVerbose(IceV_AddrOpt)) { |  3726   if (Func->getContext()->isVerbose(IceV_AddrOpt)) { | 
|  3729     Ostream &Str = Func->getContext()->getStrDump(); |  3727     Ostream &Str = Func->getContext()->getStrDump(); | 
|  3730     Str << "\nStarting computeAddressOpt for instruction:\n  "; |  3728     Str << "\nStarting computeAddressOpt for instruction:\n  "; | 
|  3731     Instr->dumpDecorated(Func); |  3729     Instr->dumpDecorated(Func); | 
|  3732   } |  3730   } | 
|  3733   (void)Offset; // TODO: pattern-match for non-zero offsets. |  3731   (void)Offset; // TODO: pattern-match for non-zero offsets. | 
|  3734   if (Base == NULL) |  3732   if (Base == NULL) | 
|  3735     return; |  3733     return; | 
|  3736   // If the Base has more than one use or is live across multiple |  3734   // If the Base has more than one use or is live across multiple | 
|  3737   // blocks, then don't go further.  Alternatively (?), never consider |  3735   // blocks, then don't go further.  Alternatively (?), never consider | 
|  3738   // a transformation that would change a variable that is currently |  3736   // a transformation that would change a variable that is currently | 
|  3739   // *not* live across basic block boundaries into one that *is*. |  3737   // *not* live across basic block boundaries into one that *is*. | 
|  3740   if (Base->isMultiblockLife() /* || Base->getUseCount() > 1*/) |  3738   if (Func->getVMetadata()->isMultiBlock(Base) /* || Base->getUseCount() > 1*/) | 
|  3741     return; |  3739     return; | 
|  3742  |  3740  | 
 |  3741   const VariablesMetadata *VMetadata = Func->getVMetadata(); | 
|  3743   bool Continue = true; |  3742   bool Continue = true; | 
|  3744   while (Continue) { |  3743   while (Continue) { | 
|  3745     const Inst *Reason = NULL; |  3744     const Inst *Reason = NULL; | 
|  3746     if (matchTransitiveAssign(Base, Reason) || |  3745     if (matchTransitiveAssign(VMetadata, Base, Reason) || | 
|  3747         matchTransitiveAssign(Index, Reason) || |  3746         matchTransitiveAssign(VMetadata, Index, Reason) || | 
|  3748         matchCombinedBaseIndex(Base, Index, Shift, Reason) || |  3747         matchCombinedBaseIndex(VMetadata, Base, Index, Shift, Reason) || | 
|  3749         matchShiftedIndex(Index, Shift, Reason) || |  3748         matchShiftedIndex(VMetadata, Index, Shift, Reason) || | 
|  3750         matchOffsetBase(Base, Offset, Reason)) { |  3749         matchOffsetBase(VMetadata, Base, Offset, Reason)) { | 
|  3751       dumpAddressOpt(Func, Base, Index, Shift, Offset, Reason); |  3750       dumpAddressOpt(Func, Base, Index, Shift, Offset, Reason); | 
|  3752     } else { |  3751     } else { | 
|  3753       Continue = false; |  3752       Continue = false; | 
|  3754     } |  3753     } | 
|  3755  |  3754  | 
|  3756     // Index is Index=Var<<Const && Const+Shift<=3 ==> |  3755     // Index is Index=Var<<Const && Const+Shift<=3 ==> | 
|  3757     //   Index=Var, Shift+=Const |  3756     //   Index=Var, Shift+=Const | 
|  3758  |  3757  | 
|  3759     // Index is Index=Const*Var && log2(Const)+Shift<=3 ==> |  3758     // Index is Index=Const*Var && log2(Const)+Shift<=3 ==> | 
|  3760     //   Index=Var, Shift+=log2(Const) |  3759     //   Index=Var, Shift+=log2(Const) | 
| (...skipping 171 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|  3932     } |  3931     } | 
|  3933     // Lower select without SSE4.1: |  3932     // Lower select without SSE4.1: | 
|  3934     // a=d?b:c ==> |  3933     // a=d?b:c ==> | 
|  3935     //   if elementtype(d) != i1: |  3934     //   if elementtype(d) != i1: | 
|  3936     //      d=sext(d); |  3935     //      d=sext(d); | 
|  3937     //   a=(b&d)|(c&~d); |  3936     //   a=(b&d)|(c&~d); | 
|  3938     Variable *T2 = makeReg(SrcTy); |  3937     Variable *T2 = makeReg(SrcTy); | 
|  3939     // Sign extend the condition operand if applicable. |  3938     // Sign extend the condition operand if applicable. | 
|  3940     if (SrcTy == IceType_v4f32) { |  3939     if (SrcTy == IceType_v4f32) { | 
|  3941       // The sext operation takes only integer arguments. |  3940       // The sext operation takes only integer arguments. | 
|  3942       Variable *T3 = Func->makeVariable(IceType_v4i32, Context.getNode()); |  3941       Variable *T3 = Func->makeVariable(IceType_v4i32); | 
|  3943       lowerCast(InstCast::create(Func, InstCast::Sext, T3, Condition)); |  3942       lowerCast(InstCast::create(Func, InstCast::Sext, T3, Condition)); | 
|  3944       _movp(T, T3); |  3943       _movp(T, T3); | 
|  3945     } else if (typeElementType(SrcTy) != IceType_i1) { |  3944     } else if (typeElementType(SrcTy) != IceType_i1) { | 
|  3946       lowerCast(InstCast::create(Func, InstCast::Sext, T, Condition)); |  3945       lowerCast(InstCast::create(Func, InstCast::Sext, T, Condition)); | 
|  3947     } else { |  3946     } else { | 
|  3948       Operand *ConditionRM = legalize(Condition, Legal_Reg | Legal_Mem); |  3947       Operand *ConditionRM = legalize(Condition, Legal_Reg | Legal_Mem); | 
|  3949       _movp(T, ConditionRM); |  3948       _movp(T, ConditionRM); | 
|  3950     } |  3949     } | 
|  3951     _movp(T2, T); |  3950     _movp(T2, T); | 
|  3952     _pand(T, SrcTRM); |  3951     _pand(T, SrcTRM); | 
| (...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|  4063   assert(isVectorType(Dest->getType())); |  4062   assert(isVectorType(Dest->getType())); | 
|  4064   Type Ty = Dest->getType(); |  4063   Type Ty = Dest->getType(); | 
|  4065   Type ElementTy = typeElementType(Ty); |  4064   Type ElementTy = typeElementType(Ty); | 
|  4066   SizeT NumElements = typeNumElements(Ty); |  4065   SizeT NumElements = typeNumElements(Ty); | 
|  4067  |  4066  | 
|  4068   Operand *T = Ctx->getConstantUndef(Ty); |  4067   Operand *T = Ctx->getConstantUndef(Ty); | 
|  4069   for (SizeT I = 0; I < NumElements; ++I) { |  4068   for (SizeT I = 0; I < NumElements; ++I) { | 
|  4070     Constant *Index = Ctx->getConstantInt32(IceType_i32, I); |  4069     Constant *Index = Ctx->getConstantInt32(IceType_i32, I); | 
|  4071  |  4070  | 
|  4072     // Extract the next two inputs. |  4071     // Extract the next two inputs. | 
|  4073     Variable *Op0 = Func->makeVariable(ElementTy, Context.getNode()); |  4072     Variable *Op0 = Func->makeVariable(ElementTy); | 
|  4074     lowerExtractElement(InstExtractElement::create(Func, Op0, Src0, Index)); |  4073     lowerExtractElement(InstExtractElement::create(Func, Op0, Src0, Index)); | 
|  4075     Variable *Op1 = Func->makeVariable(ElementTy, Context.getNode()); |  4074     Variable *Op1 = Func->makeVariable(ElementTy); | 
|  4076     lowerExtractElement(InstExtractElement::create(Func, Op1, Src1, Index)); |  4075     lowerExtractElement(InstExtractElement::create(Func, Op1, Src1, Index)); | 
|  4077  |  4076  | 
|  4078     // Perform the arithmetic as a scalar operation. |  4077     // Perform the arithmetic as a scalar operation. | 
|  4079     Variable *Res = Func->makeVariable(ElementTy, Context.getNode()); |  4078     Variable *Res = Func->makeVariable(ElementTy); | 
|  4080     lowerArithmetic(InstArithmetic::create(Func, Kind, Res, Op0, Op1)); |  4079     lowerArithmetic(InstArithmetic::create(Func, Kind, Res, Op0, Op1)); | 
|  4081  |  4080  | 
|  4082     // Insert the result into position. |  4081     // Insert the result into position. | 
|  4083     Variable *DestT = Func->makeVariable(Ty, Context.getNode()); |  4082     Variable *DestT = Func->makeVariable(Ty); | 
|  4084     lowerInsertElement(InstInsertElement::create(Func, DestT, T, Res, Index)); |  4083     lowerInsertElement(InstInsertElement::create(Func, DestT, T, Res, Index)); | 
|  4085     T = DestT; |  4084     T = DestT; | 
|  4086     // TODO(stichnot): Use postLower() in -Om1 mode to avoid buildup of |  4085     // TODO(stichnot): Use postLower() in -Om1 mode to avoid buildup of | 
|  4087     // infinite weight temporaries. |  4086     // infinite weight temporaries. | 
|  4088   } |  4087   } | 
|  4089  |  4088  | 
|  4090   lowerAssign(InstAssign::create(Func, Dest, T)); |  4089   lowerAssign(InstAssign::create(Func, Dest, T)); | 
|  4091 } |  4090 } | 
|  4092  |  4091  | 
|  4093 // The following pattern occurs often in lowered C and C++ code: |  4092 // The following pattern occurs often in lowered C and C++ code: | 
| (...skipping 223 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|  4317              llvm::isa<ConstantRelocatable>(Offset)); |  4316              llvm::isa<ConstantRelocatable>(Offset)); | 
|  4318     } |  4317     } | 
|  4319     Mem = OperandX8632Mem::create(Func, Ty, Base, Offset); |  4318     Mem = OperandX8632Mem::create(Func, Ty, Base, Offset); | 
|  4320   } |  4319   } | 
|  4321   return llvm::cast<OperandX8632Mem>(legalize(Mem)); |  4320   return llvm::cast<OperandX8632Mem>(legalize(Mem)); | 
|  4322 } |  4321 } | 
|  4323  |  4322  | 
|  4324 Variable *TargetX8632::makeReg(Type Type, int32_t RegNum) { |  4323 Variable *TargetX8632::makeReg(Type Type, int32_t RegNum) { | 
|  4325   // There aren't any 64-bit integer registers for x86-32. |  4324   // There aren't any 64-bit integer registers for x86-32. | 
|  4326   assert(Type != IceType_i64); |  4325   assert(Type != IceType_i64); | 
|  4327   Variable *Reg = Func->makeVariable(Type, Context.getNode()); |  4326   Variable *Reg = Func->makeVariable(Type); | 
|  4328   if (RegNum == Variable::NoRegister) |  4327   if (RegNum == Variable::NoRegister) | 
|  4329     Reg->setWeightInfinite(); |  4328     Reg->setWeightInfinite(); | 
|  4330   else |  4329   else | 
|  4331     Reg->setRegNum(RegNum); |  4330     Reg->setRegNum(RegNum); | 
|  4332   return Reg; |  4331   return Reg; | 
|  4333 } |  4332 } | 
|  4334  |  4333  | 
|  4335 void TargetX8632::postLower() { |  4334 void TargetX8632::postLower() { | 
|  4336   if (Ctx->getOptLevel() != Opt_m1) |  4335   if (Ctx->getOptLevel() != Opt_m1) | 
|  4337     return; |  4336     return; | 
| (...skipping 214 matching lines...) Expand 10 before | Expand all | Expand 10 after  Loading... | 
|  4552     Str << "\t.align\t" << Align << "\n"; |  4551     Str << "\t.align\t" << Align << "\n"; | 
|  4553     Str << MangledName << ":\n"; |  4552     Str << MangledName << ":\n"; | 
|  4554     for (SizeT i = 0; i < Size; ++i) { |  4553     for (SizeT i = 0; i < Size; ++i) { | 
|  4555       Str << "\t.byte\t" << (((unsigned)Data[i]) & 0xff) << "\n"; |  4554       Str << "\t.byte\t" << (((unsigned)Data[i]) & 0xff) << "\n"; | 
|  4556     } |  4555     } | 
|  4557     Str << "\t.size\t" << MangledName << ", " << Size << "\n"; |  4556     Str << "\t.size\t" << MangledName << ", " << Size << "\n"; | 
|  4558   } |  4557   } | 
|  4559 } |  4558 } | 
|  4560  |  4559  | 
|  4561 } // end of namespace Ice |  4560 } // end of namespace Ice | 
| OLD | NEW |