165 static void decide_unroll_stupid (class loop *, int);
166 static void decide_unroll_constant_iterations (class loop *, int);
167 static void decide_unroll_runtime_iterations (class loop *, int);
168 static void unroll_loop_stupid (class loop *);
170 static void unroll_loop_constant_iterations (class loop *);
171 static void unroll_loop_runtime_iterations (class loop *);
172 static struct opt_info *analyze_insns_in_loop (class loop *);
176 static struct var_to_expand *analyze_insn_to_expand_var (class loop*, rtx_insn *);
177 static bool referenced_in_one_insn_in_loop_p (class loop *, rtx, int *);
191 report_unroll (class loop *loop, dump_location_t locus)  in report_unroll()  argument
195   if (loop->lpt_decision.decision == LPT_NONE)  in report_unroll()
204                    loop->lpt_decision.times);  in report_unroll()
205   if (profile_info && loop->header->count.initialized_p ())  in report_unroll()
208                  (int)loop->header->count.to_gcov_type ());  in report_unroll()
217   class loop *loop;  in decide_unrolling()  local
220   FOR_EACH_LOOP (loop, LI_FROM_INNERMOST)  in decide_unrolling()
222       loop->lpt_decision.decision = LPT_NONE;  in decide_unrolling()
223       dump_user_location_t locus = get_loop_location (loop);  in decide_unrolling()
228 			 loop->num, loop->header->index);  in decide_unrolling()
230       if (loop->unroll == 1)  in decide_unrolling()
239       if (optimize_loop_for_size_p (loop))  in decide_unrolling()
247       if (!can_duplicate_loop_p (loop))  in decide_unrolling()
256       if (loop->inner)  in decide_unrolling()
263       loop->ninsns = num_loop_insns (loop);  in decide_unrolling()
264       loop->av_ninsns = average_num_loop_insns (loop);  in decide_unrolling()
267       decide_unroll_constant_iterations (loop, flags);  in decide_unrolling()
268       if (loop->lpt_decision.decision == LPT_NONE)  in decide_unrolling()
269 	decide_unroll_runtime_iterations (loop, flags);  in decide_unrolling()
270       if (loop->lpt_decision.decision == LPT_NONE)  in decide_unrolling()
271 	decide_unroll_stupid (loop, flags);  in decide_unrolling()
273       report_unroll (loop, locus);  in decide_unrolling()
281   class loop *loop;  in unroll_loops()  local
288   FOR_EACH_LOOP (loop, LI_FROM_INNERMOST)  in unroll_loops()
291       switch (loop->lpt_decision.decision)  in unroll_loops()
294 	  unroll_loop_constant_iterations (loop);  in unroll_loops()
298 	  unroll_loop_runtime_iterations (loop);  in unroll_loops()
302 	  unroll_loop_stupid (loop);  in unroll_loops()
324 loop_exit_at_end_p (class loop *loop)  in loop_exit_at_end_p()  argument
326   class niter_desc *desc = get_simple_loop_desc (loop);  in loop_exit_at_end_p()
330   gcc_assert (desc->in_edge->dest != loop->header);  in loop_exit_at_end_p()
332   if (desc->in_edge->dest != loop->latch)  in loop_exit_at_end_p()
336   FOR_BB_INSNS (loop->latch, insn)  in loop_exit_at_end_p()
349 decide_unroll_constant_iterations (class loop *loop, int flags)  in decide_unroll_constant_iterations()  argument
356   if (!(flags & UAP_UNROLL) && !loop->unroll)  in decide_unroll_constant_iterations()
366   nunroll = param_max_unrolled_insns / loop->ninsns;  in decide_unroll_constant_iterations()
368     = param_max_average_unrolled_insns / loop->av_ninsns;  in decide_unroll_constant_iterations()
375     nunroll = targetm.loop_unroll_adjust (nunroll, loop);  in decide_unroll_constant_iterations()
386   desc = get_simple_loop_desc (loop);  in decide_unroll_constant_iterations()
398   if (loop->unroll > 0 && loop->unroll < USHRT_MAX)  in decide_unroll_constant_iterations()
402       if (desc->niter == 0 || (unsigned) loop->unroll > desc->niter - 1)  in decide_unroll_constant_iterations()
409 	  loop->lpt_decision.decision = LPT_UNROLL_CONSTANT;  in decide_unroll_constant_iterations()
410 	  loop->lpt_decision.times = loop->unroll - 1;  in decide_unroll_constant_iterations()
420       || ((get_estimated_loop_iterations (loop, &iterations)  in decide_unroll_constant_iterations()
421 	   || get_likely_max_loop_iterations (loop, &iterations))  in decide_unroll_constant_iterations()
443       if (!loop_exit_at_end_p (loop))  in decide_unroll_constant_iterations()
458   loop->lpt_decision.decision = LPT_UNROLL_CONSTANT;  in decide_unroll_constant_iterations()
459   loop->lpt_decision.times = best_unroll;  in decide_unroll_constant_iterations()
482 unroll_loop_constant_iterations (class loop *loop)  in unroll_loop_constant_iterations()  argument
488   unsigned max_unroll = loop->lpt_decision.times;  in unroll_loop_constant_iterations()
489   class niter_desc *desc = get_simple_loop_desc (loop);  in unroll_loop_constant_iterations()
490   bool exit_at_end = loop_exit_at_end_p (loop);  in unroll_loop_constant_iterations()
507     opt_info = analyze_insns_in_loop (loop);  in unroll_loop_constant_iterations()
526           ok = duplicate_loop_to_header_edge (loop, loop_preheader_edge (loop),  in unroll_loop_constant_iterations()
541 	  loop->nb_iterations_upper_bound -= exit_mod;  in unroll_loop_constant_iterations()
542 	  if (loop->any_estimate  in unroll_loop_constant_iterations()
543 	      && wi::leu_p (exit_mod, loop->nb_iterations_estimate))  in unroll_loop_constant_iterations()
544 	    loop->nb_iterations_estimate -= exit_mod;  in unroll_loop_constant_iterations()
546 	    loop->any_estimate = false;  in unroll_loop_constant_iterations()
547 	  if (loop->any_likely_upper_bound  in unroll_loop_constant_iterations()
548 	      && wi::leu_p (exit_mod, loop->nb_iterations_likely_upper_bound))  in unroll_loop_constant_iterations()
549 	    loop->nb_iterations_likely_upper_bound -= exit_mod;  in unroll_loop_constant_iterations()
551 	    loop->any_likely_upper_bound = false;  in unroll_loop_constant_iterations()
575 	  ok = duplicate_loop_to_header_edge (loop, loop_preheader_edge (loop),  in unroll_loop_constant_iterations()
589 	  loop->nb_iterations_upper_bound -= exit_mod + 1;  in unroll_loop_constant_iterations()
590 	  if (loop->any_estimate  in unroll_loop_constant_iterations()
591 	      && wi::leu_p (exit_mod + 1, loop->nb_iterations_estimate))  in unroll_loop_constant_iterations()
592 	    loop->nb_iterations_estimate -= exit_mod + 1;  in unroll_loop_constant_iterations()
594 	    loop->any_estimate = false;  in unroll_loop_constant_iterations()
595 	  if (loop->any_likely_upper_bound  in unroll_loop_constant_iterations()
596 	      && wi::leu_p (exit_mod + 1, loop->nb_iterations_likely_upper_bound))  in unroll_loop_constant_iterations()
597 	    loop->nb_iterations_likely_upper_bound -= exit_mod + 1;  in unroll_loop_constant_iterations()
599 	    loop->any_likely_upper_bound = false;  in unroll_loop_constant_iterations()
612   ok = duplicate_loop_to_header_edge (loop, loop_latch_edge (loop),  in unroll_loop_constant_iterations()
646   loop->nb_iterations_upper_bound  in unroll_loop_constant_iterations()
647     = wi::udiv_trunc (loop->nb_iterations_upper_bound, max_unroll + 1);  in unroll_loop_constant_iterations()
648   if (loop->any_estimate)  in unroll_loop_constant_iterations()
649     loop->nb_iterations_estimate  in unroll_loop_constant_iterations()
650       = wi::udiv_trunc (loop->nb_iterations_estimate, max_unroll + 1);  in unroll_loop_constant_iterations()
651   if (loop->any_likely_upper_bound)  in unroll_loop_constant_iterations()
652     loop->nb_iterations_likely_upper_bound  in unroll_loop_constant_iterations()
653       = wi::udiv_trunc (loop->nb_iterations_likely_upper_bound, max_unroll + 1);  in unroll_loop_constant_iterations()
663 	     max_unroll, num_loop_insns (loop));  in unroll_loop_constant_iterations()
669 decide_unroll_runtime_iterations (class loop *loop, int flags)  in decide_unroll_runtime_iterations()  argument
676   if (!(flags & UAP_UNROLL) && !loop->unroll)  in decide_unroll_runtime_iterations()
686   nunroll = param_max_unrolled_insns / loop->ninsns;  in decide_unroll_runtime_iterations()
687   nunroll_by_av = param_max_average_unrolled_insns / loop->av_ninsns;  in decide_unroll_runtime_iterations()
694     nunroll = targetm.loop_unroll_adjust (nunroll, loop);  in decide_unroll_runtime_iterations()
696   if (loop->unroll > 0 && loop->unroll < USHRT_MAX)  in decide_unroll_runtime_iterations()
697     nunroll = loop->unroll;  in decide_unroll_runtime_iterations()
708   desc = get_simple_loop_desc (loop);  in decide_unroll_runtime_iterations()
728   if ((get_estimated_loop_iterations (loop, &iterations)  in decide_unroll_runtime_iterations()
729        || get_likely_max_loop_iterations (loop, &iterations))  in decide_unroll_runtime_iterations()
743   loop->lpt_decision.decision = LPT_UNROLL_RUNTIME;  in decide_unroll_runtime_iterations()
744   loop->lpt_decision.times = i - 1;  in decide_unroll_runtime_iterations()
883 unroll_loop_runtime_iterations (class loop *loop)  in unroll_loop_runtime_iterations()  argument
895   unsigned max_unroll = loop->lpt_decision.times;  in unroll_loop_runtime_iterations()
896   class niter_desc *desc = get_simple_loop_desc (loop);  in unroll_loop_runtime_iterations()
897   bool exit_at_end = loop_exit_at_end_p (loop);  in unroll_loop_runtime_iterations()
903     opt_info = analyze_insns_in_loop (loop);  in unroll_loop_runtime_iterations()
908   body = get_loop_body (loop);  in unroll_loop_runtime_iterations()
909   for (i = 0; i < loop->num_nodes; i++)  in unroll_loop_runtime_iterations()
916 	if (!flow_bb_inside_loop_p (loop, bb))  in unroll_loop_runtime_iterations()
972   split_edge_and_insert (loop_preheader_edge (loop), init_code);  in unroll_loop_runtime_iterations()
986       ezc_swtch = loop_preheader_edge (loop)->src;  in unroll_loop_runtime_iterations()
987       ok = duplicate_loop_to_header_edge (loop, loop_preheader_edge (loop),  in unroll_loop_runtime_iterations()
995   swtch = split_edge (loop_preheader_edge (loop));  in unroll_loop_runtime_iterations()
1009       ok = duplicate_loop_to_header_edge (loop, loop_preheader_edge (loop),  in unroll_loop_runtime_iterations()
1019       preheader = split_edge (loop_preheader_edge (loop));  in unroll_loop_runtime_iterations()
1045       preheader = split_edge (loop_preheader_edge (loop));  in unroll_loop_runtime_iterations()
1073   ok = duplicate_loop_to_header_edge (loop, loop_latch_edge (loop),  in unroll_loop_runtime_iterations()
1119   loop->nb_iterations_upper_bound  in unroll_loop_runtime_iterations()
1120     = wi::udiv_trunc (loop->nb_iterations_upper_bound, max_unroll + 1);  in unroll_loop_runtime_iterations()
1121   if (loop->any_estimate)  in unroll_loop_runtime_iterations()
1122     loop->nb_iterations_estimate  in unroll_loop_runtime_iterations()
1123       = wi::udiv_trunc (loop->nb_iterations_estimate, max_unroll + 1);  in unroll_loop_runtime_iterations()
1124   if (loop->any_likely_upper_bound)  in unroll_loop_runtime_iterations()
1125     loop->nb_iterations_likely_upper_bound  in unroll_loop_runtime_iterations()
1126       = wi::udiv_trunc (loop->nb_iterations_likely_upper_bound, max_unroll + 1);  in unroll_loop_runtime_iterations()
1132       --loop->nb_iterations_upper_bound;  in unroll_loop_runtime_iterations()
1133       if (loop->any_estimate  in unroll_loop_runtime_iterations()
1134 	  && loop->nb_iterations_estimate != 0)  in unroll_loop_runtime_iterations()
1135 	--loop->nb_iterations_estimate;  in unroll_loop_runtime_iterations()
1137 	loop->any_estimate = false;  in unroll_loop_runtime_iterations()
1138       if (loop->any_likely_upper_bound  in unroll_loop_runtime_iterations()
1139 	  && loop->nb_iterations_likely_upper_bound != 0)  in unroll_loop_runtime_iterations()
1140 	--loop->nb_iterations_likely_upper_bound;  in unroll_loop_runtime_iterations()
1142 	loop->any_likely_upper_bound = false;  in unroll_loop_runtime_iterations()
1149 	     max_unroll, num_loop_insns (loop));  in unroll_loop_runtime_iterations()
1154 decide_unroll_stupid (class loop *loop, int flags)  in decide_unroll_stupid()  argument
1161   if (!(flags & UAP_UNROLL_ALL) && !loop->unroll)  in decide_unroll_stupid()
1169   nunroll = param_max_unrolled_insns / loop->ninsns;  in decide_unroll_stupid()
1171     = param_max_average_unrolled_insns / loop->av_ninsns;  in decide_unroll_stupid()
1178     nunroll = targetm.loop_unroll_adjust (nunroll, loop);  in decide_unroll_stupid()
1180   if (loop->unroll > 0 && loop->unroll < USHRT_MAX)  in decide_unroll_stupid()
1181     nunroll = loop->unroll;  in decide_unroll_stupid()
1192   desc = get_simple_loop_desc (loop);  in decide_unroll_stupid()
1206   if (num_loop_branches (loop) > 1)  in decide_unroll_stupid()
1214   if ((get_estimated_loop_iterations (loop, &iterations)  in decide_unroll_stupid()
1215        || get_likely_max_loop_iterations (loop, &iterations))  in decide_unroll_stupid()
1229   loop->lpt_decision.decision = LPT_UNROLL_STUPID;  in decide_unroll_stupid()
1230   loop->lpt_decision.times = i - 1;  in decide_unroll_stupid()
1252 unroll_loop_stupid (class loop *loop)  in unroll_loop_stupid()  argument
1254   unsigned nunroll = loop->lpt_decision.times;  in unroll_loop_stupid()
1255   class niter_desc *desc = get_simple_loop_desc (loop);  in unroll_loop_stupid()
1261     opt_info = analyze_insns_in_loop (loop);  in unroll_loop_stupid()
1267   ok = duplicate_loop_to_header_edge (loop, loop_latch_edge (loop),  in unroll_loop_stupid()
1295 	     nunroll, num_loop_insns (loop));  in unroll_loop_stupid()
1303 referenced_in_one_insn_in_loop_p (class loop *loop, rtx reg,  in referenced_in_one_insn_in_loop_p()  argument
1311   body = get_loop_body (loop);  in referenced_in_one_insn_in_loop_p()
1312   for (i = 0; i < loop->num_nodes; i++)  in referenced_in_one_insn_in_loop_p()
1331 reset_debug_uses_in_loop (class loop *loop, rtx reg, int debug_uses)  in reset_debug_uses_in_loop()  argument
1337   body = get_loop_body (loop);  in reset_debug_uses_in_loop()
1338   for (i = 0; debug_uses && i < loop->num_nodes; i++)  in reset_debug_uses_in_loop()
1380 analyze_insn_to_expand_var (class loop *loop, rtx_insn *insn)  in analyze_insn_to_expand_var()  argument
1459   if (!referenced_in_one_insn_in_loop_p (loop, dest, &debug_uses))  in analyze_insn_to_expand_var()
1478     reset_debug_uses_in_loop (loop, dest, debug_uses);  in analyze_insn_to_expand_var()
1573 analyze_insns_in_loop (class loop *loop)  in analyze_insns_in_loop()  argument
1583   vec<edge> edges = get_loop_exit_edges (loop);  in analyze_insns_in_loop()
1587   iv_analysis_loop_init (loop);  in analyze_insns_in_loop()
1589   body = get_loop_body (loop);  in analyze_insns_in_loop()
1594        	= new hash_table<iv_split_hasher> (5 * loop->num_nodes);  in analyze_insns_in_loop()
1600   opt_info->loop_preheader = loop_preheader_edge (loop)->src;  in analyze_insns_in_loop()
1616        	= new hash_table<var_expand_hasher> (5 * loop->num_nodes);  in analyze_insns_in_loop()
1621   for (i = 0; i < loop->num_nodes; i++)  in analyze_insns_in_loop()
1624       if (!dominated_by_p (CDI_DOMINATORS, loop->latch, bb))  in analyze_insns_in_loop()
1646           ves = analyze_insn_to_expand_var (loop, insn);  in analyze_insns_in_loop()