Lines Matching defs:LU
1173 /// folded in \p LU at isel time.
1175 /// This function returns true if \p LU can accommodate what \p F
1182 const LSRUse &LU, const Formula &F);
1184 // Get the cost of the scaling factor used in F for LU.
1186 const LSRUse &LU, const Formula &F,
1236 const LSRUse &LU,
1500 const LSRUse &LU,
1534 NumBaseParts - (1 + (F.Scale && isAMCompletelyFolded(*TTI, LU, F)));
1538 C.ScaleCost += *getScalingFactorCost(*TTI, LU, F, *L).getValue();
1541 for (const LSRFixup &Fixup : LU.Fixups) {
1553 if (LU.Kind == LSRUse::Address && Offset.isNonZero() &&
1554 !isAMCompletelyFolded(*TTI, LSRUse::Address, LU.AccessTy, F.BaseGV,
1593 if (LU.Kind == LSRUse::ICmpZero && !F.hasZeroEnd() &&
1600 if (LU.Kind != LSRUse::ICmpZero)
1947 const LSRUse &LU, const Formula &F) {
1949 if (LU.Kind == LSRUse::Address && TTI.LSRWithInstrQueries()) {
1950 for (const LSRFixup &Fixup : LU.Fixups)
1951 if (!isAMCompletelyFolded(TTI, LSRUse::Address, LU.AccessTy, F.BaseGV,
1958 return isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1959 LU.AccessTy, F.BaseGV, F.BaseOffset, F.HasBaseReg,
1964 const LSRUse &LU, const Formula &F,
1971 if (!isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1972 LU.AccessTy, F, L))
1975 switch (LU.Kind) {
1980 ScalableMin = (F.BaseOffset + LU.MinOffset).getKnownMinValue();
1981 ScalableMax = (F.BaseOffset + LU.MaxOffset).getKnownMinValue();
1983 FixedMin = (F.BaseOffset + LU.MinOffset).getFixedValue();
1984 FixedMax = (F.BaseOffset + LU.MaxOffset).getFixedValue();
1987 LU.AccessTy.MemTy, F.BaseGV, StackOffset::get(FixedMin, ScalableMin),
1988 F.HasBaseReg, F.Scale, LU.AccessTy.AddrSpace);
1990 LU.AccessTy.MemTy, F.BaseGV, StackOffset::get(FixedMax, ScalableMax),
1991 F.HasBaseReg, F.Scale, LU.AccessTy.AddrSpace);
2216 bool reconcileNewOffset(LSRUse &LU, Immediate NewOffset, bool HasBaseReg,
2222 void DeleteUse(LSRUse &LU, size_t LUIdx);
2226 void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2227 void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2229 bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F);
2233 void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base,
2236 void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
2239 void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base);
2240 void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2243 void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2244 void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2248 void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2249 void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2250 void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2251 void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
2280 const LSRUse &LU) const;
2282 Value *Expand(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2285 void RewriteForPHI(PHINode *PN, const LSRUse &LU, const LSRFixup &LF,
2288 void Rewrite(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2756 bool LSRInstance::reconcileNewOffset(LSRUse &LU, Immediate NewOffset,
2759 Immediate NewMinOffset = LU.MinOffset;
2760 Immediate NewMaxOffset = LU.MaxOffset;
2766 if (LU.Kind != Kind)
2773 if (AccessTy.MemTy != LU.AccessTy.MemTy) {
2780 if (Immediate::isKnownLT(NewOffset, LU.MinOffset)) {
2782 LU.MaxOffset - NewOffset, HasBaseReg))
2785 } else if (Immediate::isKnownGT(NewOffset, LU.MaxOffset)) {
2787 NewOffset - LU.MinOffset, HasBaseReg))
2800 LU.MinOffset = NewMinOffset;
2801 LU.MaxOffset = NewMaxOffset;
2802 LU.AccessTy = NewAccessTy;
2827 LSRUse &LU = Uses[LUIdx];
2828 if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy))
2837 LSRUse &LU = Uses[LUIdx];
2839 LU.MinOffset = Offset;
2840 LU.MaxOffset = Offset;
2845 void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) {
2846 if (&LU != &Uses.back())
2847 std::swap(LU, Uses.back());
2860 for (LSRUse &LU : Uses) {
2866 if (&LU != &OrigLU &&
2867 LU.Kind != LSRUse::ICmpZero &&
2868 LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy &&
2869 LU.WidestFixupType == OrigLU.WidestFixupType &&
2870 LU.HasFormulaWithSameRegs(OrigF)) {
2872 for (const Formula &F : LU.Formulae) {
2881 return &LU;
3619 LSRUse &LU = Uses[LUIdx];
3622 LSRFixup &LF = LU.getNewFixup();
3627 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3633 BaselineCost.RateFormula(F, Regs, VisitedRegs, LU);
3637 if (!LU.WidestFixupType ||
3638 SE.getTypeSizeInBits(LU.WidestFixupType) <
3640 LU.WidestFixupType = LF.OperandValToReplace->getType();
3643 if (LU.Formulae.empty()) {
3644 InsertInitialFormula(S, LU, LUIdx);
3645 CountRegisters(LU.Formulae.back(), LUIdx);
3654 void LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU,
3658 LU.RigidFormula = true;
3662 bool Inserted = InsertFormula(LU, LUIdx, F);
3670 LSRUse &LU, size_t LUIdx) {
3674 bool Inserted = InsertFormula(LU, LUIdx, F);
3688 bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) {
3690 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) &&
3693 if (!LU.InsertFormula(F, *L))
3812 LSRUse &LU = Uses[LUIdx];
3813 LSRFixup &LF = LU.getNewFixup();
3817 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3818 if (!LU.WidestFixupType ||
3819 SE.getTypeSizeInBits(LU.WidestFixupType) <
3821 LU.WidestFixupType = LF.OperandValToReplace->getType();
3822 InsertSupplementalFormula(US, LU, LUIdx);
3823 CountRegisters(LU.Formulae.back(), Uses.size() - 1);
3894 LSRUse &LU, const SCEV *S, const Loop *L,
3896 if (LU.Kind != LSRUse::Address ||
3897 !LU.AccessTy.getType()->isIntOrIntVectorTy())
3916 void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
3925 if (AMK == TTI::AMK_PostIndexed && mayUsePostIncMode(TTI, LU, BaseReg, L, SE))
3945 if (isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
3946 LU.AccessTy, *J, Base.getNumRegs() > 1))
3958 isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
3959 LU.AccessTy, InnerAddOps[0], Base.getNumRegs() > 1))
4002 if (InsertFormula(LU, LUIdx, F))
4009 GenerateReassociations(LU, LUIdx, LU.Formulae.back(),
4015 void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx,
4023 GenerateReassociationsImpl(LU, LUIdx, Base, Depth, i);
4026 GenerateReassociationsImpl(LU, LUIdx, Base, Depth,
4032 void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
4075 (void)InsertFormula(LU, LUIdx, F);
4096 void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
4105 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
4111 (void)InsertFormula(LU, LUIdx, F);
4115 void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx,
4121 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, i);
4123 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, /* Idx */ -1,
4129 LSRUse &LU, unsigned LUIdx, const Formula &Base,
4138 if (isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F)) {
4155 (void)InsertFormula(LU, LUIdx, F);
4169 if (AMK == TTI::AMK_PreIndexed && LU.Kind == LSRUse::Address) {
4195 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
4205 (void)InsertFormula(LU, LUIdx, F);
4209 void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx,
4214 Worklist.push_back(LU.MinOffset);
4215 if (LU.MaxOffset != LU.MinOffset)
4216 Worklist.push_back(LU.MaxOffset);
4219 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, i);
4221 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, /* Idx */ -1,
4227 void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
4229 if (LU.Kind != LSRUse::ICmpZero) return;
4237 if (LU.MinOffset != LU.MaxOffset) return;
4269 Immediate Offset = LU.MinOffset;
4273 if (Offset.getFixedValue() / Factor != LU.MinOffset.getFixedValue())
4284 if (!isLegalUse(TTI, Offset, Offset, LU.Kind, LU.AccessTy, F))
4288 F.BaseOffset = F.BaseOffset.addUnsigned(Offset).subUnsigned(LU.MinOffset);
4321 (void)InsertFormula(LU, LUIdx, F);
4328 void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
4345 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
4349 if (LU.Kind == LSRUse::Basic &&
4350 isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LSRUse::Special,
4351 LU.AccessTy, Base) &&
4352 LU.AllFixupsOutsideLoop)
4353 LU.Kind = LSRUse::Special;
4359 if (LU.Kind == LSRUse::ICmpZero && !Base.HasBaseReg &&
4365 if (AR && (AR->getLoop() == L || LU.AllFixupsOutsideLoop)) {
4381 (AR->getLoop() != L && LU.AllFixupsOutsideLoop)))
4385 if (F.Scale == 1 && LU.AllFixupsOutsideLoop)
4387 (void)InsertFormula(LU, LUIdx, F);
4418 void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
4437 for (auto &LF : LU.Fixups)
4474 (void)InsertFormula(LU, LUIdx, F);
4608 LSRUse &LU = Uses[LUIdx];
4617 for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) {
4618 Formula F = LU.Formulae[L];
4635 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
4658 (void)InsertFormula(LU, LUIdx, NewF);
4671 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset,
4672 LU.Kind, LU.AccessTy, NewF)) {
4674 mayUsePostIncMode(TTI, LU, OrigReg, this->L, SE))
4703 (void)InsertFormula(LU, LUIdx, NewF);
4718 LSRUse &LU = Uses[LUIdx];
4719 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4720 GenerateReassociations(LU, LUIdx, LU.Formulae[i]);
4721 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4722 GenerateCombinations(LU, LUIdx, LU.Formulae[i]);
4725 LSRUse &LU = Uses[LUIdx];
4726 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4727 GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]);
4728 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4729 GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]);
4730 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4731 GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]);
4732 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4733 GenerateScales(LU, LUIdx, LU.Formulae[i]);
4736 LSRUse &LU = Uses[LUIdx];
4737 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4738 GenerateTruncates(LU, LUIdx, LU.Formulae[i]);
4766 LSRUse &LU = Uses[LUIdx];
4767 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
4771 for (size_t FIdx = 0, NumForms = LU.Formulae.size();
4773 Formula &F = LU.Formulae[FIdx];
4784 CostF.RateFormula(F, Regs, VisitedRegs, LU, &LoserRegs);
4813 Formula &Best = LU.Formulae[P.first->second];
4817 CostBest.RateFormula(Best, Regs, VisitedRegs, LU);
4828 LU.DeleteFormula(F);
4836 LU.RecomputeRegs(LUIdx, RegUses);
4854 for (const LSRUse &LU : Uses) {
4855 size_t FSize = LU.Formulae.size();
4879 LSRUse &LU = Uses[LUIdx];
4881 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4882 Formula &F = LU.Formulae[i];
4899 if (LU.HasFormulaWithSameRegs(NewF)) {
4902 LU.DeleteFormula(F);
4915 if (LU.HasFormulaWithSameRegs(NewF)) {
4918 LU.DeleteFormula(F);
4929 LU.RecomputeRegs(LUIdx, RegUses);
4950 LSRUse &LU = Uses[LUIdx];
4951 for (const Formula &F : LU.Formulae) {
4955 LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU);
4960 LU.Kind, LU.AccessTy))
4963 LLVM_DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); dbgs() << '\n');
4965 LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop;
4967 // Transfer the fixups of LU to LUThatHas.
4968 for (LSRFixup &Fixup : LU.Fixups) {
4992 DeleteUse(LU, LUIdx);
5047 LSRUse &LU = Uses[LUIdx];
5048 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
5075 CostFA.RateFormula(FA, Regs, VisitedRegs, LU);
5077 CostFB.RateFormula(FB, Regs, VisitedRegs, LU);
5082 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms;
5084 Formula &F = LU.Formulae[FIdx];
5091 Formula &Best = LU.Formulae[P.first->second];
5101 LU.DeleteFormula(F);
5107 LU.RecomputeRegs(LUIdx, RegUses);
5133 LSRUse &LU = Uses[LUIdx];
5135 if (LU.Kind != LSRUse::Address)
5137 if (!TTI.isIndexedLoadLegal(TTI.MIM_PostInc, LU.AccessTy.getType()) &&
5138 !TTI.isIndexedStoreLegal(TTI.MIM_PostInc, LU.AccessTy.getType()))
5142 for (const Formula &F : LU.Formulae)
5146 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms;
5148 Formula &F = LU.Formulae[FIdx];
5152 LU.DeleteFormula(F);
5159 LU.RecomputeRegs(LUIdx, RegUses);
5228 for (const LSRUse &LU : Uses) {
5229 if (!LU.Regs.count(Reg))
5231 float P = LU.getNotSelectedProbability(Reg);
5245 LSRUse &LU = Uses[LUIdx];
5247 if (LU.Formulae.size() < 2)
5252 float FMinRegNum = LU.Formulae[0].getNumRegs();
5253 float FMinARegNum = LU.Formulae[0].getNumRegs();
5255 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
5256 Formula &F = LU.Formulae[i];
5262 FRegNum += RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg);
5265 RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg);
5270 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg);
5273 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg);
5283 LLVM_DEBUG(dbgs() << " The formula "; LU.Formulae[MinIdx].print(dbgs());
5286 std::swap(LU.Formulae[MinIdx], LU.Formulae[0]);
5287 while (LU.Formulae.size() != 1) {
5288 LLVM_DEBUG(dbgs() << " Deleting "; LU.Formulae.back().print(dbgs());
5290 LU.Formulae.pop_back();
5292 LU.RecomputeRegs(LUIdx, RegUses);
5293 assert(LU.Formulae.size() == 1 && "Should be exactly 1 min regs formula");
5294 Formula &F = LU.Formulae[0];
5382 LSRUse &LU = Uses[LUIdx];
5383 if (!LU.Regs.count(Best)) continue;
5386 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
5387 Formula &F = LU.Formulae[i];
5390 LU.DeleteFormula(F);
5400 LU.RecomputeRegs(LUIdx, RegUses);
5441 const LSRUse &LU = Uses[Workspace.size()];
5449 if (LU.Regs.count(S))
5454 for (const Formula &F : LU.Formulae) {
5461 if (AMK != TTI::AMK_PostIndexed || LU.Kind != LSRUse::Address) {
5482 NewCost.RateFormula(F, NewRegs, VisitedRegs, LU);
5625 BasicBlock::iterator LowestIP, const LSRFixup &LF, const LSRUse &LU) const {
5632 if (LU.Kind == LSRUse::ICmpZero)
5686 Value *LSRInstance::Expand(const LSRUse &LU, const LSRFixup &LF,
5689 if (LU.RigidFormula)
5694 IP = AdjustInsertPositionForExpand(IP, LF, LU);
5735 if (LU.Kind == LSRUse::ICmpZero) {
5754 if (!Ops.empty() && LU.Kind == LSRUse::Address &&
5755 isAMCompletelyFolded(TTI, LU, F)) {
5795 if (LU.Kind == LSRUse::ICmpZero) {
5831 if (LU.Kind == LSRUse::ICmpZero) {
5870 void LSRInstance::RewriteForPHI(PHINode *PN, const LSRUse &LU,
5930 Expand(LU, LF, F, BB->getTerminator()->getIterator(), DeadInsts);
5956 for (LSRUse &LU : Uses)
5957 for (LSRFixup &Fixup : LU.Fixups)
5994 void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF,
6000 RewriteForPHI(PN, LU, LF, F, DeadInsts);
6002 Value *FullV = Expand(LU, LF, F, LF.UserInst->getIterator(), DeadInsts);
6018 if (LU.Kind == LSRUse::ICmpZero)
6032 const LSRUse &LU, Instruction *IVIncInsertPos,
6034 if (LU.Kind != LSRUse::Address)
6253 for (const LSRUse &LU : Uses) {
6254 for (const Formula &F : LU.Formulae)
6255 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
6287 for (const LSRUse &LU : Uses)
6288 for (const LSRFixup &LF : LU.Fixups) {
6297 for (const LSRUse &LU : Uses) {
6299 LU.print(OS);
6301 for (const Formula &F : LU.Formulae) {