Lines Matching full:up
197 TargetTransformInfo::UnrollingPreferences UP;
199 // Set up the defaults
200 UP.Threshold =
202 UP.MaxPercentThresholdBoost = 400;
203 UP.OptSizeThreshold = UnrollOptSizeThreshold;
204 UP.PartialThreshold = 150;
205 UP.PartialOptSizeThreshold = UnrollOptSizeThreshold;
206 UP.Count = 0;
207 UP.DefaultUnrollRuntimeCount = 8;
208 UP.MaxCount = std::numeric_limits<unsigned>::max();
209 UP.MaxUpperBound = UnrollMaxUpperBound;
210 UP.FullUnrollMaxCount = std::numeric_limits<unsigned>::max();
211 UP.BEInsns = 2;
212 UP.Partial = false;
213 UP.Runtime = false;
214 UP.AllowRemainder = true;
215 UP.UnrollRemainder = false;
216 UP.AllowExpensiveTripCount = false;
217 UP.Force = false;
218 UP.UpperBound = false;
219 UP.UnrollAndJam = false;
220 UP.UnrollAndJamInnerLoopThreshold = 60;
221 UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;
224 TTI.getUnrollingPreferences(L, SE, UP, &ORE);
233 UP.Threshold = UP.OptSizeThreshold;
234 UP.PartialThreshold = UP.PartialOptSizeThreshold;
235 UP.MaxPercentThresholdBoost = 100;
240 UP.Threshold = UnrollThreshold;
242 UP.PartialThreshold = UnrollPartialThreshold;
244 UP.MaxPercentThresholdBoost = UnrollMaxPercentThresholdBoost;
246 UP.MaxCount = UnrollMaxCount;
248 UP.MaxUpperBound = UnrollMaxUpperBound;
250 UP.FullUnrollMaxCount = UnrollFullMaxCount;
252 UP.Partial = UnrollAllowPartial;
254 UP.AllowRemainder = UnrollAllowRemainder;
256 UP.Runtime = UnrollRuntime;
258 UP.UpperBound = false;
260 UP.UnrollRemainder = UnrollUnrollRemainder;
262 UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;
266 UP.Threshold = *UserThreshold;
267 UP.PartialThreshold = *UserThreshold;
270 UP.Count = *UserCount;
272 UP.Partial = *UserAllowPartial;
274 UP.Runtime = *UserRuntime;
276 UP.UpperBound = *UserUpperBound;
278 UP.FullUnrollMaxCount = *UserFullUnrollMaxCount;
280 return UP;
350 /// given iteration its condition would be resolved to true, we won't add up the
583 // the cost of it and all the instructions leading up to it.
728 const TargetTransformInfo::UnrollingPreferences &UP,
731 assert(LS >= UP.BEInsns && "LoopSize should not be less than BEInsns!");
733 return static_cast<uint64_t>(LS - UP.BEInsns) * CountOverwrite + UP.BEInsns;
735 return static_cast<uint64_t>(LS - UP.BEInsns) * UP.Count + UP.BEInsns;
799 const TargetTransformInfo::UnrollingPreferences &UP) {
805 if (UP.AllowRemainder &&
806 UCE.getUnrolledLoopSize(UP, (unsigned)UnrollCount) < UP.Threshold)
812 if ((UP.AllowRemainder || (TripMultiple % PInfo.PragmaCount == 0)))
829 MaxTripCount <= UP.MaxUpperBound)
840 const TargetTransformInfo::UnrollingPreferences &UP) {
843 if (FullUnrollTripCount > UP.FullUnrollMaxCount)
848 if (UCE.getUnrolledLoopSize(UP) < UP.Threshold)
856 UP.Threshold * UP.MaxPercentThresholdBoost / 100,
857 UP.MaxIterationsCountToAnalyze)) {
859 getFullUnrollBoostingFactor(*Cost, UP.MaxPercentThresholdBoost);
860 if (Cost->UnrolledCost < UP.Threshold * Boost / 100)
869 const TargetTransformInfo::UnrollingPreferences &UP) {
874 if (!UP.Partial) {
879 unsigned count = UP.Count;
882 if (UP.PartialThreshold != NoThreshold) {
884 if (UCE.getUnrolledLoopSize(UP, count) > UP.PartialThreshold)
885 count = (std::max(UP.PartialThreshold, UP.BEInsns + 1) - UP.BEInsns) /
886 (LoopSize - UP.BEInsns);
887 if (count > UP.MaxCount)
888 count = UP.MaxCount;
891 if (UP.AllowRemainder && count <= 1) {
896 count = UP.DefaultUnrollRuntimeCount;
898 UCE.getUnrolledLoopSize(UP, count) > UP.PartialThreshold)
907 if (count > UP.MaxCount)
908 count = UP.MaxCount;
915 // Calculates unroll count and writes it to UP.Count.
928 TargetTransformInfo::UnrollingPreferences &UP,
950 UP.Count = 1;
951 UP.Runtime = false;
958 MaxTripCount, UCE, UP)) {
959 UP.Count = *UnrollFactor;
962 UP.AllowExpensiveTripCount = true;
963 UP.Force = true;
965 UP.Runtime |= (PragmaCount > 0);
972 UP.Threshold = std::max<unsigned>(UP.Threshold, PragmaUnrollThreshold);
973 UP.PartialThreshold =
974 std::max<unsigned>(UP.PartialThreshold, PragmaUnrollThreshold);
980 UP.Count = 0;
982 UP.Count = TripCount;
984 TripCount, UCE, UP)) {
985 UP.Count = *UnrollFactor;
997 // number of loop tests goes up which may end up being worse on targets with
1003 if (!TripCount && MaxTripCount && (UP.UpperBound || MaxOrZero) &&
1004 MaxTripCount <= UP.MaxUpperBound) {
1005 UP.Count = MaxTripCount;
1007 MaxTripCount, UCE, UP)) {
1008 UP.Count = *UnrollFactor;
1015 computePeelCount(L, LoopSize, PP, TripCount, DT, SE, AC, UP.Threshold);
1017 UP.Runtime = false;
1018 UP.Count = 1;
1022 // Before starting partial unrolling, set up.partial to true,
1025 UP.Partial |= ExplicitUnroll;
1029 if (auto UnrollFactor = shouldPartialUnroll(LoopSize, TripCount, UCE, UP)) {
1030 UP.Count = *UnrollFactor;
1033 UP.Count != TripCount)
1043 if (UP.PartialThreshold != NoThreshold) {
1044 if (UP.Count == 0) {
1073 UP.Count = 0;
1078 if (MaxTripCount && !UP.Force && MaxTripCount < UP.MaxUpperBound) {
1079 UP.Count = 0;
1089 UP.AllowExpensiveTripCount = true;
1092 UP.Runtime |= PragmaEnableUnroll || PragmaCount > 0 || UserUnrollCount;
1093 if (!UP.Runtime) {
1097 UP.Count = 0;
1100 if (UP.Count == 0)
1101 UP.Count = UP.DefaultUnrollRuntimeCount;
1105 while (UP.Count != 0 &&
1106 UCE.getUnrolledLoopSize(UP) > UP.PartialThreshold)
1107 UP.Count >>= 1;
1110 unsigned OrigCount = UP.Count;
1113 if (!UP.AllowRemainder && UP.Count != 0 && (TripMultiple % UP.Count) != 0) {
1114 while (UP.Count != 0 && TripMultiple % UP.Count != 0)
1115 UP.Count >>= 1;
1122 << " to " << UP.Count << ".\n");
1126 if (unrollCountPragmaValue(L) > 0 && !UP.AllowRemainder)
1138 << NV("UnrollCount", UP.Count) << " time(s).";
1142 if (UP.Count > UP.MaxCount)
1143 UP.Count = UP.MaxCount;
1145 if (MaxTripCount && UP.Count > MaxTripCount)
1146 UP.Count = MaxTripCount;
1148 LLVM_DEBUG(dbgs() << " runtime unrolling with count: " << UP.Count
1150 if (UP.Count < 2)
1151 UP.Count = 0;
1214 TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences(
1223 if (UP.Threshold == 0 && (!UP.Partial || UP.PartialThreshold == 0) &&
1230 UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns);
1242 UP.Threshold = std::max(UP.Threshold, LoopSize + 1);
1281 UP.AllowRemainder &= UCE.ConvergenceAllowsRuntime;
1297 MaxOrZero, TripMultiple, UCE, UP, PP, UseUpperBound);
1298 if (!UP.Count)
1301 UP.Runtime &= UCE.ConvergenceAllowsRuntime;
1304 assert(UP.Count == 1 && "Cannot perform peel and unroll in the same step");
1317 // If the loop was peeled, we already "used up" the profile information
1327 if (OnlyFullUnroll && (UP.Count < TripCount || UP.Count < MaxTripCount)) {
1333 // At this point, UP.Runtime indicates that run-time unrolling is allowed.
1336 // TODO: This decision should probably be pushed up into
1338 UP.Runtime &= TripCount == 0 && TripMultiple % UP.Count != 0;
1346 ULO.Count = UP.Count;
1347 ULO.Force = UP.Force;
1348 ULO.AllowExpensiveTripCount = UP.AllowExpensiveTripCount;
1349 ULO.UnrollRemainder = UP.UnrollRemainder;
1350 ULO.Runtime = UP.Runtime;
1618 // will simplify all loops, regardless of whether anything end up being