Lines Matching full:fi
468 static bool checkPHIs(FlattenInfo &FI, const TargetTransformInfo *TTI) {
484 SafeOuterPHIs.insert(FI.OuterInductionPHI);
488 for (PHINode &InnerPHI : FI.InnerLoop->getHeader()->phis()) {
491 if (&InnerPHI == FI.InnerInductionPHI)
493 if (FI.isNarrowInductionPhi(&InnerPHI))
500 InnerPHI.getIncomingValueForBlock(FI.InnerLoop->getLoopPreheader());
502 InnerPHI.getIncomingValueForBlock(FI.InnerLoop->getLoopLatch());
508 if (!OuterPHI || OuterPHI->getParent() != FI.OuterLoop->getHeader()) {
518 OuterPHI->getIncomingValueForBlock(FI.OuterLoop->getLoopLatch()));
536 FI.InnerPHIsToTransform.insert(&InnerPHI);
539 for (PHINode &OuterPHI : FI.OuterLoop->getHeader()->phis()) {
540 if (FI.isNarrowInductionPhi(&OuterPHI))
553 checkOuterLoopInsts(FlattenInfo &FI,
562 for (auto *B : FI.OuterLoop->getBlocks()) {
563 if (FI.InnerLoop->contains(B))
584 Br->getSuccessor(0) == FI.InnerLoop->getHeader())
588 if (match(&I, m_c_Mul(m_Specific(FI.OuterInductionPHI),
589 m_Specific(FI.InnerTripCount))))
620 static bool checkIVUsers(FlattenInfo &FI) {
624 if (!FI.checkInnerInductionPhiUsers(ValidOuterPHIUses))
629 if (!FI.checkOuterInductionPhiUsers(ValidOuterPHIUses))
633 dbgs() << "Found " << FI.LinearIVUses.size()
635 for (Value *V : FI.LinearIVUses) {
644 static OverflowResult checkOverflow(FlattenInfo &FI, DominatorTree *DT,
646 Function *F = FI.OuterLoop->getHeader()->getParent();
656 FI.InnerTripCount, FI.OuterTripCount,
658 FI.OuterLoop->getLoopPreheader()->getTerminator()));
668 if (!isGuaranteedToExecuteForEveryIteration(GEPUserInst, FI.InnerLoop))
688 for (Value *V : FI.LinearIVUses) {
701 static bool CanFlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
705 if (!findLoopComponents(FI.InnerLoop, IterationInstructions,
706 FI.InnerInductionPHI, FI.InnerTripCount,
707 FI.InnerIncrement, FI.InnerBranch, SE, FI.Widened))
709 if (!findLoopComponents(FI.OuterLoop, IterationInstructions,
710 FI.OuterInductionPHI, FI.OuterTripCount,
711 FI.OuterIncrement, FI.OuterBranch, SE, FI.Widened))
716 if (!FI.OuterLoop->isLoopInvariant(FI.InnerTripCount)) {
720 if (!FI.OuterLoop->isLoopInvariant(FI.OuterTripCount)) {
725 if (!checkPHIs(FI, TTI))
729 if (FI.InnerInductionPHI->getType() != FI.OuterInductionPHI->getType())
732 if (!checkOuterLoopInsts(FI, IterationInstructions, TTI))
740 if (!checkIVUsers(FI))
747 static bool DoFlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
751 Function *F = FI.OuterLoop->getHeader()->getParent();
755 OptimizationRemark Remark(DEBUG_TYPE, "Flattened", FI.InnerLoop->getStartLoc(),
756 FI.InnerLoop->getHeader());
762 if (!FI.NewTripCount) {
763 FI.NewTripCount = BinaryOperator::CreateMul(
764 FI.InnerTripCount, FI.OuterTripCount, "flatten.tripcount",
765 FI.OuterLoop->getLoopPreheader()->getTerminator()->getIterator());
767 FI.NewTripCount->dump());
772 FI.InnerInductionPHI->removeIncomingValue(FI.InnerLoop->getLoopLatch());
776 for (PHINode *PHI : FI.InnerPHIsToTransform)
777 PHI->removeIncomingValue(FI.InnerLoop->getLoopLatch());
781 cast<User>(FI.OuterBranch->getCondition())->setOperand(1, FI.NewTripCount);
784 BasicBlock *InnerExitBlock = FI.InnerLoop->getExitBlock();
785 BasicBlock *InnerExitingBlock = FI.InnerLoop->getExitingBlock();
792 DT->deleteEdge(InnerExitingBlock, FI.InnerLoop->getHeader());
794 MSSAU->removeEdge(InnerExitingBlock, FI.InnerLoop->getHeader());
798 IRBuilder<> Builder(FI.OuterInductionPHI->getParent()->getTerminator());
799 for (Value *V : FI.LinearIVUses) {
800 Value *OuterValue = FI.OuterInductionPHI;
801 if (FI.Widened)
802 OuterValue = Builder.CreateTrunc(FI.OuterInductionPHI, V->getType(),
826 SE->forgetLoop(FI.OuterLoop);
829 U->markLoopAsDeleted(*FI.InnerLoop, FI.InnerLoop->getName());
830 LI->erase(FI.InnerLoop);
838 static bool CanWidenIV(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
847 Module *M = FI.InnerLoop->getHeader()->getParent()->getParent();
849 auto *InnerType = FI.InnerInductionPHI->getType();
850 auto *OuterType = FI.OuterInductionPHI->getType();
883 if (!CreateWideIV({FI.InnerInductionPHI, MaxLegalType, false}, Deleted))
888 FI.InnerPHIsToTransform.insert(FI.InnerInductionPHI);
890 if (!CreateWideIV({FI.OuterInductionPHI, MaxLegalType, false}, Deleted))
894 FI.Widened = true;
897 FI.NarrowInnerInductionPHI = FI.InnerInductionPHI;
898 FI.NarrowOuterInductionPHI = FI.OuterInductionPHI;
901 return CanFlattenLoopPair(FI, DT, LI, SE, AC, TTI);
904 static bool FlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
911 << FI.OuterLoop->getHeader()->getName() << " and inner loop "
912 << FI.InnerLoop->getHeader()->getName() << " in "
913 << FI.OuterLoop->getHeader()->getParent()->getName() << "\n");
915 if (!CanFlattenLoopPair(FI, DT, LI, SE, AC, TTI))
919 bool CanFlatten = CanWidenIV(FI, DT, LI, SE, AC, TTI);
932 if (FI.Widened && !CanFlatten)
937 return DoFlattenLoopPair(FI, DT, LI, SE, AC, TTI, U, MSSAU);
943 OverflowResult OR = checkOverflow(FI, DT, AC);
949 Module *M = FI.OuterLoop->getHeader()->getParent()->getParent();
955 FI.OuterTripCount->getType()->getScalarSizeInBits())) {
968 BasicBlock *CheckBlock = FI.OuterLoop->getLoopPreheader();
970 LoopVersioning LVer(LAI, Checks, FI.OuterLoop, LI, DT, SE);
982 Intrinsic::umul_with_overflow, FI.OuterTripCount->getType(),
983 {FI.OuterTripCount, FI.InnerTripCount},
985 FI.NewTripCount = Builder.CreateExtractValue(Call, 0, "flatten.tripcount");
992 return DoFlattenLoopPair(FI, DT, LI, SE, AC, TTI, U, MSSAU);
1017 FlattenInfo FI(OuterLoop, InnerLoop);
1019 FlattenLoopPair(FI, &AR.DT, &AR.LI, &AR.SE, &AR.AC, &AR.TTI, &U,