Lines Matching defs:UP
197 TargetTransformInfo::UnrollingPreferences UP;
200 UP.Threshold =
202 UP.MaxPercentThresholdBoost = 400;
203 UP.OptSizeThreshold = UnrollOptSizeThreshold;
204 UP.PartialThreshold = 150;
205 UP.PartialOptSizeThreshold = UnrollOptSizeThreshold;
206 UP.Count = 0;
207 UP.DefaultUnrollRuntimeCount = 8;
208 UP.MaxCount = std::numeric_limits<unsigned>::max();
209 UP.MaxUpperBound = UnrollMaxUpperBound;
210 UP.FullUnrollMaxCount = std::numeric_limits<unsigned>::max();
211 UP.BEInsns = 2;
212 UP.Partial = false;
213 UP.Runtime = false;
214 UP.AllowRemainder = true;
215 UP.UnrollRemainder = false;
216 UP.AllowExpensiveTripCount = false;
217 UP.Force = false;
218 UP.UpperBound = false;
219 UP.UnrollAndJam = false;
220 UP.UnrollAndJamInnerLoopThreshold = 60;
221 UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;
222 UP.SCEVExpansionBudget = SCEVCheapExpansionBudget;
223 UP.RuntimeUnrollMultiExit = false;
226 TTI.getUnrollingPreferences(L, SE, UP, &ORE);
235 UP.Threshold = UP.OptSizeThreshold;
236 UP.PartialThreshold = UP.PartialOptSizeThreshold;
237 UP.MaxPercentThresholdBoost = 100;
242 UP.Threshold = UnrollThreshold;
244 UP.PartialThreshold = UnrollPartialThreshold;
246 UP.MaxPercentThresholdBoost = UnrollMaxPercentThresholdBoost;
248 UP.MaxCount = UnrollMaxCount;
250 UP.MaxUpperBound = UnrollMaxUpperBound;
252 UP.FullUnrollMaxCount = UnrollFullMaxCount;
254 UP.Partial = UnrollAllowPartial;
256 UP.AllowRemainder = UnrollAllowRemainder;
258 UP.Runtime = UnrollRuntime;
260 UP.UpperBound = false;
262 UP.UnrollRemainder = UnrollUnrollRemainder;
264 UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;
268 UP.Threshold = *UserThreshold;
269 UP.PartialThreshold = *UserThreshold;
272 UP.Count = *UserCount;
274 UP.Partial = *UserAllowPartial;
276 UP.Runtime = *UserRuntime;
278 UP.UpperBound = *UserUpperBound;
280 UP.FullUnrollMaxCount = *UserFullUnrollMaxCount;
282 return UP;
730 const TargetTransformInfo::UnrollingPreferences &UP,
733 assert(LS >= UP.BEInsns && "LoopSize should not be less than BEInsns!");
735 return static_cast<uint64_t>(LS - UP.BEInsns) * CountOverwrite + UP.BEInsns;
737 return static_cast<uint64_t>(LS - UP.BEInsns) * UP.Count + UP.BEInsns;
801 const TargetTransformInfo::UnrollingPreferences &UP) {
807 if (UP.AllowRemainder &&
808 UCE.getUnrolledLoopSize(UP, (unsigned)UnrollCount) < UP.Threshold)
814 if ((UP.AllowRemainder || (TripMultiple % PInfo.PragmaCount == 0)))
831 MaxTripCount <= UP.MaxUpperBound)
842 const TargetTransformInfo::UnrollingPreferences &UP) {
845 if (FullUnrollTripCount > UP.FullUnrollMaxCount)
850 if (UCE.getUnrolledLoopSize(UP) < UP.Threshold)
858 UP.Threshold * UP.MaxPercentThresholdBoost / 100,
859 UP.MaxIterationsCountToAnalyze)) {
861 getFullUnrollBoostingFactor(*Cost, UP.MaxPercentThresholdBoost);
862 if (Cost->UnrolledCost < UP.Threshold * Boost / 100)
871 const TargetTransformInfo::UnrollingPreferences &UP) {
876 if (!UP.Partial) {
881 unsigned count = UP.Count;
884 if (UP.PartialThreshold != NoThreshold) {
886 if (UCE.getUnrolledLoopSize(UP, count) > UP.PartialThreshold)
887 count = (std::max(UP.PartialThreshold, UP.BEInsns + 1) - UP.BEInsns) /
888 (LoopSize - UP.BEInsns);
889 if (count > UP.MaxCount)
890 count = UP.MaxCount;
893 if (UP.AllowRemainder && count <= 1) {
898 count = UP.DefaultUnrollRuntimeCount;
900 UCE.getUnrolledLoopSize(UP, count) > UP.PartialThreshold)
909 if (count > UP.MaxCount)
910 count = UP.MaxCount;
917 // Calculates unroll count and writes it to UP.Count.
930 TargetTransformInfo::UnrollingPreferences &UP,
952 UP.Count = 1;
953 UP.Runtime = false;
960 MaxTripCount, UCE, UP)) {
961 UP.Count = *UnrollFactor;
964 UP.AllowExpensiveTripCount = true;
965 UP.Force = true;
967 UP.Runtime |= (PragmaCount > 0);
974 UP.Threshold = std::max<unsigned>(UP.Threshold, PragmaUnrollThreshold);
975 UP.PartialThreshold =
976 std::max<unsigned>(UP.PartialThreshold, PragmaUnrollThreshold);
982 UP.Count = 0;
984 UP.Count = TripCount;
986 TripCount, UCE, UP)) {
987 UP.Count = *UnrollFactor;
1005 if (!TripCount && MaxTripCount && (UP.UpperBound || MaxOrZero) &&
1006 MaxTripCount <= UP.MaxUpperBound) {
1007 UP.Count = MaxTripCount;
1009 MaxTripCount, UCE, UP)) {
1010 UP.Count = *UnrollFactor;
1017 computePeelCount(L, LoopSize, PP, TripCount, DT, SE, AC, UP.Threshold);
1019 UP.Runtime = false;
1020 UP.Count = 1;
1027 UP.Partial |= ExplicitUnroll;
1031 if (auto UnrollFactor = shouldPartialUnroll(LoopSize, TripCount, UCE, UP)) {
1032 UP.Count = *UnrollFactor;
1035 UP.Count != TripCount)
1045 if (UP.PartialThreshold != NoThreshold) {
1046 if (UP.Count == 0) {
1075 UP.Count = 0;
1080 if (MaxTripCount && !UP.Force && MaxTripCount < UP.MaxUpperBound) {
1081 UP.Count = 0;
1091 UP.AllowExpensiveTripCount = true;
1094 UP.Runtime |= PragmaEnableUnroll || PragmaCount > 0 || UserUnrollCount;
1095 if (!UP.Runtime) {
1099 UP.Count = 0;
1102 if (UP.Count == 0)
1103 UP.Count = UP.DefaultUnrollRuntimeCount;
1107 while (UP.Count != 0 &&
1108 UCE.getUnrolledLoopSize(UP) > UP.PartialThreshold)
1109 UP.Count >>= 1;
1112 unsigned OrigCount = UP.Count;
1115 if (!UP.AllowRemainder && UP.Count != 0 && (TripMultiple % UP.Count) != 0) {
1116 while (UP.Count != 0 && TripMultiple % UP.Count != 0)
1117 UP.Count >>= 1;
1124 << " to " << UP.Count << ".\n");
1128 if (unrollCountPragmaValue(L) > 0 && !UP.AllowRemainder)
1140 << NV("UnrollCount", UP.Count) << " time(s).";
1144 if (UP.Count > UP.MaxCount)
1145 UP.Count = UP.MaxCount;
1147 if (MaxTripCount && UP.Count > MaxTripCount)
1148 UP.Count = MaxTripCount;
1150 LLVM_DEBUG(dbgs() << " runtime unrolling with count: " << UP.Count
1152 if (UP.Count < 2)
1153 UP.Count = 0;
1216 TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences(
1225 if (UP.Threshold == 0 && (!UP.Partial || UP.PartialThreshold == 0) &&
1232 UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns);
1244 UP.Threshold = std::max(UP.Threshold, LoopSize + 1);
1283 UP.AllowRemainder &= UCE.ConvergenceAllowsRuntime;
1299 MaxOrZero, TripMultiple, UCE, UP, PP, UseUpperBound);
1300 if (!UP.Count)
1303 UP.Runtime &= UCE.ConvergenceAllowsRuntime;
1306 assert(UP.Count == 1 && "Cannot perform peel and unroll in the same step");
1329 if (OnlyFullUnroll && (UP.Count < TripCount || UP.Count < MaxTripCount)) {
1335 // At this point, UP.Runtime indicates that run-time unrolling is allowed.
1340 UP.Runtime &= TripCount == 0 && TripMultiple % UP.Count != 0;
1348 ULO.Count = UP.Count;
1349 ULO.Force = UP.Force;
1350 ULO.AllowExpensiveTripCount = UP.AllowExpensiveTripCount;
1351 ULO.UnrollRemainder = UP.UnrollRemainder;
1352 ULO.Runtime = UP.Runtime;
1355 ULO.SCEVExpansionBudget = UP.SCEVExpansionBudget;
1356 ULO.RuntimeUnrollMultiExit = UP.RuntimeUnrollMultiExit;