Lines Matching defs:LU
1178 /// folded in \p LU at isel time.
1180 /// This function returns true if \p LU can accommodate what \p F
1187 const LSRUse &LU, const Formula &F);
1189 // Get the cost of the scaling factor used in F for LU.
1191 const LSRUse &LU, const Formula &F,
1241 const LSRUse &LU,
1505 const LSRUse &LU,
1539 NumBaseParts - (1 + (F.Scale && isAMCompletelyFolded(*TTI, LU, F)));
1543 C.ScaleCost += *getScalingFactorCost(*TTI, LU, F, *L).getValue();
1546 for (const LSRFixup &Fixup : LU.Fixups) {
1558 if (LU.Kind == LSRUse::Address && Offset.isNonZero() &&
1559 !isAMCompletelyFolded(*TTI, LSRUse::Address, LU.AccessTy, F.BaseGV,
1598 if (LU.Kind == LSRUse::ICmpZero && !F.hasZeroEnd() &&
1605 if (LU.Kind != LSRUse::ICmpZero)
1952 const LSRUse &LU, const Formula &F) {
1954 if (LU.Kind == LSRUse::Address && TTI.LSRWithInstrQueries()) {
1955 for (const LSRFixup &Fixup : LU.Fixups)
1956 if (!isAMCompletelyFolded(TTI, LSRUse::Address, LU.AccessTy, F.BaseGV,
1963 return isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1964 LU.AccessTy, F.BaseGV, F.BaseOffset, F.HasBaseReg,
1969 const LSRUse &LU, const Formula &F,
1976 if (!isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1977 LU.AccessTy, F, L))
1980 switch (LU.Kind) {
1985 ScalableMin = (F.BaseOffset + LU.MinOffset).getKnownMinValue();
1986 ScalableMax = (F.BaseOffset + LU.MaxOffset).getKnownMinValue();
1988 FixedMin = (F.BaseOffset + LU.MinOffset).getFixedValue();
1989 FixedMax = (F.BaseOffset + LU.MaxOffset).getFixedValue();
1992 LU.AccessTy.MemTy, F.BaseGV, StackOffset::get(FixedMin, ScalableMin),
1993 F.HasBaseReg, F.Scale, LU.AccessTy.AddrSpace);
1995 LU.AccessTy.MemTy, F.BaseGV, StackOffset::get(FixedMax, ScalableMax),
1996 F.HasBaseReg, F.Scale, LU.AccessTy.AddrSpace);
2215 bool reconcileNewOffset(LSRUse &LU, Immediate NewOffset, bool HasBaseReg,
2221 void DeleteUse(LSRUse &LU, size_t LUIdx);
2225 void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2226 void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2228 bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F);
2232 void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base,
2235 void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
2238 void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base);
2239 void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2242 void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2243 void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2247 void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2248 void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2249 void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2250 void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
2279 const LSRUse &LU) const;
2281 Value *Expand(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2284 void RewriteForPHI(PHINode *PN, const LSRUse &LU, const LSRFixup &LF,
2287 void Rewrite(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2755 bool LSRInstance::reconcileNewOffset(LSRUse &LU, Immediate NewOffset,
2758 Immediate NewMinOffset = LU.MinOffset;
2759 Immediate NewMaxOffset = LU.MaxOffset;
2765 if (LU.Kind != Kind)
2772 if (AccessTy.MemTy != LU.AccessTy.MemTy) {
2779 if (Immediate::isKnownLT(NewOffset, LU.MinOffset)) {
2781 LU.MaxOffset - NewOffset, HasBaseReg))
2784 } else if (Immediate::isKnownGT(NewOffset, LU.MaxOffset)) {
2786 NewOffset - LU.MinOffset, HasBaseReg))
2799 LU.MinOffset = NewMinOffset;
2800 LU.MaxOffset = NewMaxOffset;
2801 LU.AccessTy = NewAccessTy;
2826 LSRUse &LU = Uses[LUIdx];
2827 if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy))
2836 LSRUse &LU = Uses[LUIdx];
2838 LU.MinOffset = Offset;
2839 LU.MaxOffset = Offset;
2844 void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) {
2845 if (&LU != &Uses.back())
2846 std::swap(LU, Uses.back());
2859 for (LSRUse &LU : Uses) {
2865 if (&LU != &OrigLU &&
2866 LU.Kind != LSRUse::ICmpZero &&
2867 LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy &&
2868 LU.WidestFixupType == OrigLU.WidestFixupType &&
2869 LU.HasFormulaWithSameRegs(OrigF)) {
2871 for (const Formula &F : LU.Formulae) {
2880 return &LU;
3618 LSRUse &LU = Uses[LUIdx];
3621 LSRFixup &LF = LU.getNewFixup();
3626 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3632 BaselineCost.RateFormula(F, Regs, VisitedRegs, LU);
3636 if (!LU.WidestFixupType ||
3637 SE.getTypeSizeInBits(LU.WidestFixupType) <
3639 LU.WidestFixupType = LF.OperandValToReplace->getType();
3642 if (LU.Formulae.empty()) {
3643 InsertInitialFormula(S, LU, LUIdx);
3644 CountRegisters(LU.Formulae.back(), LUIdx);
3653 void LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU,
3657 LU.RigidFormula = true;
3661 bool Inserted = InsertFormula(LU, LUIdx, F);
3669 LSRUse &LU, size_t LUIdx) {
3673 bool Inserted = InsertFormula(LU, LUIdx, F);
3687 bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) {
3689 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) &&
3692 if (!LU.InsertFormula(F, *L))
3811 LSRUse &LU = Uses[LUIdx];
3812 LSRFixup &LF = LU.getNewFixup();
3816 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3817 if (!LU.WidestFixupType ||
3818 SE.getTypeSizeInBits(LU.WidestFixupType) <
3820 LU.WidestFixupType = LF.OperandValToReplace->getType();
3821 InsertSupplementalFormula(US, LU, LUIdx);
3822 CountRegisters(LU.Formulae.back(), Uses.size() - 1);
3893 LSRUse &LU, const SCEV *S, const Loop *L,
3895 if (LU.Kind != LSRUse::Address ||
3896 !LU.AccessTy.getType()->isIntOrIntVectorTy())
3915 void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
3924 if (AMK == TTI::AMK_PostIndexed && mayUsePostIncMode(TTI, LU, BaseReg, L, SE))
3944 if (isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
3945 LU.AccessTy, *J, Base.getNumRegs() > 1))
3957 isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
3958 LU.AccessTy, InnerAddOps[0], Base.getNumRegs() > 1))
4000 if (InsertFormula(LU, LUIdx, F))
4007 GenerateReassociations(LU, LUIdx, LU.Formulae.back(),
4013 void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx,
4021 GenerateReassociationsImpl(LU, LUIdx, Base, Depth, i);
4024 GenerateReassociationsImpl(LU, LUIdx, Base, Depth,
4030 void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
4073 (void)InsertFormula(LU, LUIdx, F);
4094 void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
4103 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
4109 (void)InsertFormula(LU, LUIdx, F);
4113 void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx,
4119 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, i);
4121 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, /* Idx */ -1,
4127 LSRUse &LU, unsigned LUIdx, const Formula &Base,
4136 if (isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F)) {
4153 (void)InsertFormula(LU, LUIdx, F);
4167 if (AMK == TTI::AMK_PreIndexed && LU.Kind == LSRUse::Address) {
4193 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
4203 (void)InsertFormula(LU, LUIdx, F);
4207 void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx,
4212 Worklist.push_back(LU.MinOffset);
4213 if (LU.MaxOffset != LU.MinOffset)
4214 Worklist.push_back(LU.MaxOffset);
4217 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, i);
4219 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, /* Idx */ -1,
4225 void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
4227 if (LU.Kind != LSRUse::ICmpZero) return;
4235 if (LU.MinOffset != LU.MaxOffset) return;
4267 Immediate Offset = LU.MinOffset;
4271 if (Offset.getFixedValue() / Factor != LU.MinOffset.getFixedValue())
4282 if (!isLegalUse(TTI, Offset, Offset, LU.Kind, LU.AccessTy, F))
4286 F.BaseOffset = F.BaseOffset.addUnsigned(Offset).subUnsigned(LU.MinOffset);
4319 (void)InsertFormula(LU, LUIdx, F);
4326 void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
4343 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
4347 if (LU.Kind == LSRUse::Basic &&
4348 isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LSRUse::Special,
4349 LU.AccessTy, Base) &&
4350 LU.AllFixupsOutsideLoop)
4351 LU.Kind = LSRUse::Special;
4357 if (LU.Kind == LSRUse::ICmpZero && !Base.HasBaseReg &&
4363 if (AR && (AR->getLoop() == L || LU.AllFixupsOutsideLoop)) {
4379 (AR->getLoop() != L && LU.AllFixupsOutsideLoop)))
4383 if (F.Scale == 1 && LU.AllFixupsOutsideLoop)
4385 (void)InsertFormula(LU, LUIdx, F);
4416 void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
4435 for (auto &LF : LU.Fixups)
4472 (void)InsertFormula(LU, LUIdx, F);
4606 LSRUse &LU = Uses[LUIdx];
4615 for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) {
4616 Formula F = LU.Formulae[L];
4633 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
4656 (void)InsertFormula(LU, LUIdx, NewF);
4669 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset,
4670 LU.Kind, LU.AccessTy, NewF)) {
4672 mayUsePostIncMode(TTI, LU, OrigReg, this->L, SE))
4701 (void)InsertFormula(LU, LUIdx, NewF);
4716 LSRUse &LU = Uses[LUIdx];
4717 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4718 GenerateReassociations(LU, LUIdx, LU.Formulae[i]);
4719 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4720 GenerateCombinations(LU, LUIdx, LU.Formulae[i]);
4723 LSRUse &LU = Uses[LUIdx];
4724 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4725 GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]);
4726 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4727 GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]);
4728 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4729 GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]);
4730 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4731 GenerateScales(LU, LUIdx, LU.Formulae[i]);
4734 LSRUse &LU = Uses[LUIdx];
4735 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4736 GenerateTruncates(LU, LUIdx, LU.Formulae[i]);
4764 LSRUse &LU = Uses[LUIdx];
4765 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
4769 for (size_t FIdx = 0, NumForms = LU.Formulae.size();
4771 Formula &F = LU.Formulae[FIdx];
4782 CostF.RateFormula(F, Regs, VisitedRegs, LU, &LoserRegs);
4811 Formula &Best = LU.Formulae[P.first->second];
4815 CostBest.RateFormula(Best, Regs, VisitedRegs, LU);
4826 LU.DeleteFormula(F);
4834 LU.RecomputeRegs(LUIdx, RegUses);
4852 for (const LSRUse &LU : Uses) {
4853 size_t FSize = LU.Formulae.size();
4877 LSRUse &LU = Uses[LUIdx];
4879 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4880 Formula &F = LU.Formulae[i];
4897 if (LU.HasFormulaWithSameRegs(NewF)) {
4900 LU.DeleteFormula(F);
4913 if (LU.HasFormulaWithSameRegs(NewF)) {
4916 LU.DeleteFormula(F);
4927 LU.RecomputeRegs(LUIdx, RegUses);
4948 LSRUse &LU = Uses[LUIdx];
4949 for (const Formula &F : LU.Formulae) {
4953 LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU);
4958 LU.Kind, LU.AccessTy))
4961 LLVM_DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); dbgs() << '\n');
4963 LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop;
4965 // Transfer the fixups of LU to LUThatHas.
4966 for (LSRFixup &Fixup : LU.Fixups) {
4990 DeleteUse(LU, LUIdx);
5045 LSRUse &LU = Uses[LUIdx];
5046 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
5073 CostFA.RateFormula(FA, Regs, VisitedRegs, LU);
5075 CostFB.RateFormula(FB, Regs, VisitedRegs, LU);
5080 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms;
5082 Formula &F = LU.Formulae[FIdx];
5089 Formula &Best = LU.Formulae[P.first->second];
5099 LU.DeleteFormula(F);
5105 LU.RecomputeRegs(LUIdx, RegUses);
5131 LSRUse &LU = Uses[LUIdx];
5133 if (LU.Kind != LSRUse::Address)
5135 if (!TTI.isIndexedLoadLegal(TTI.MIM_PostInc, LU.AccessTy.getType()) &&
5136 !TTI.isIndexedStoreLegal(TTI.MIM_PostInc, LU.AccessTy.getType()))
5140 for (const Formula &F : LU.Formulae)
5144 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms;
5146 Formula &F = LU.Formulae[FIdx];
5150 LU.DeleteFormula(F);
5157 LU.RecomputeRegs(LUIdx, RegUses);
5226 for (const LSRUse &LU : Uses) {
5227 if (!LU.Regs.count(Reg))
5229 float P = LU.getNotSelectedProbability(Reg);
5243 LSRUse &LU = Uses[LUIdx];
5245 if (LU.Formulae.size() < 2)
5250 float FMinRegNum = LU.Formulae[0].getNumRegs();
5251 float FMinARegNum = LU.Formulae[0].getNumRegs();
5253 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
5254 Formula &F = LU.Formulae[i];
5260 FRegNum += RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg);
5263 RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg);
5268 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg);
5271 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg);
5281 LLVM_DEBUG(dbgs() << " The formula "; LU.Formulae[MinIdx].print(dbgs());
5284 std::swap(LU.Formulae[MinIdx], LU.Formulae[0]);
5285 while (LU.Formulae.size() != 1) {
5286 LLVM_DEBUG(dbgs() << " Deleting "; LU.Formulae.back().print(dbgs());
5288 LU.Formulae.pop_back();
5290 LU.RecomputeRegs(LUIdx, RegUses);
5291 assert(LU.Formulae.size() == 1 && "Should be exactly 1 min regs formula");
5292 Formula &F = LU.Formulae[0];
5380 LSRUse &LU = Uses[LUIdx];
5381 if (!LU.Regs.count(Best)) continue;
5384 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
5385 Formula &F = LU.Formulae[i];
5388 LU.DeleteFormula(F);
5398 LU.RecomputeRegs(LUIdx, RegUses);
5439 const LSRUse &LU = Uses[Workspace.size()];
5447 if (LU.Regs.count(S))
5452 for (const Formula &F : LU.Formulae) {
5459 if (AMK != TTI::AMK_PostIndexed || LU.Kind != LSRUse::Address) {
5480 NewCost.RateFormula(F, NewRegs, VisitedRegs, LU);
5623 BasicBlock::iterator LowestIP, const LSRFixup &LF, const LSRUse &LU) const {
5630 if (LU.Kind == LSRUse::ICmpZero)
5684 Value *LSRInstance::Expand(const LSRUse &LU, const LSRFixup &LF,
5687 if (LU.RigidFormula)
5692 IP = AdjustInsertPositionForExpand(IP, LF, LU);
5733 if (LU.Kind == LSRUse::ICmpZero) {
5752 if (!Ops.empty() && LU.Kind == LSRUse::Address &&
5753 isAMCompletelyFolded(TTI, LU, F)) {
5793 if (LU.Kind == LSRUse::ICmpZero) {
5829 if (LU.Kind == LSRUse::ICmpZero) {
5869 PHINode *PN, const LSRUse &LU, const LSRFixup &LF, const Formula &F,
5934 Expand(LU, LF, F, BB->getTerminator()->getIterator(), DeadInsts);
5960 for (LSRUse &LU : Uses)
5961 for (LSRFixup &Fixup : LU.Fixups)
6000 void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF,
6006 RewriteForPHI(PN, LU, LF, F, DeadInsts);
6008 Value *FullV = Expand(LU, LF, F, LF.UserInst->getIterator(), DeadInsts);
6024 if (LU.Kind == LSRUse::ICmpZero)
6038 const LSRUse &LU, Instruction *IVIncInsertPos,
6040 if (LU.Kind != LSRUse::Address)
6256 for (const LSRUse &LU : Uses) {
6257 for (const Formula &F : LU.Formulae)
6258 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
6290 for (const LSRUse &LU : Uses)
6291 for (const LSRFixup &LF : LU.Fixups) {
6300 for (const LSRUse &LU : Uses) {
6302 LU.print(OS);
6304 for (const Formula &F : LU.Formulae) {