165 static void decide_unroll_stupid (class loop *, int);
166 static void decide_unroll_constant_iterations (class loop *, int);
167 static void decide_unroll_runtime_iterations (class loop *, int);
168 static void unroll_loop_stupid (class loop *);
170 static void unroll_loop_constant_iterations (class loop *);
171 static void unroll_loop_runtime_iterations (class loop *);
172 static struct opt_info *analyze_insns_in_loop (class loop *);
176 static struct var_to_expand *analyze_insn_to_expand_var (class loop*, rtx_insn *);
177 static bool referenced_in_one_insn_in_loop_p (class loop *, rtx, int *);
191 report_unroll (class loop *loop, dump_location_t locus)  in report_unroll()  argument
195   if (loop->lpt_decision.decision == LPT_NONE)  in report_unroll()
204                    loop->lpt_decision.times);  in report_unroll()
205   if (profile_info && loop->header->count.initialized_p ())  in report_unroll()
208                  (int)loop->header->count.to_gcov_type ());  in report_unroll()
218   for (auto loop : loops_list (cfun, LI_FROM_INNERMOST))  in decide_unrolling()  local
220       loop->lpt_decision.decision = LPT_NONE;  in decide_unrolling()
221       dump_user_location_t locus = get_loop_location (loop);  in decide_unrolling()
226 			 loop->num, loop->header->index);  in decide_unrolling()
228       if (loop->unroll == 1)  in decide_unrolling()
237       if (optimize_loop_for_size_p (loop))  in decide_unrolling()
245       if (!can_duplicate_loop_p (loop))  in decide_unrolling()
254       if (loop->inner)  in decide_unrolling()
261       loop->ninsns = num_loop_insns (loop);  in decide_unrolling()
262       loop->av_ninsns = average_num_loop_insns (loop);  in decide_unrolling()
265       decide_unroll_constant_iterations (loop, flags);  in decide_unrolling()
266       if (loop->lpt_decision.decision == LPT_NONE)  in decide_unrolling()
267 	decide_unroll_runtime_iterations (loop, flags);  in decide_unrolling()
268       if (loop->lpt_decision.decision == LPT_NONE)  in decide_unrolling()
269 	decide_unroll_stupid (loop, flags);  in decide_unrolling()
271       report_unroll (loop, locus);  in decide_unrolling()
285   for (auto loop : loops_list (cfun, LI_FROM_INNERMOST))  in unroll_loops()  local
288       switch (loop->lpt_decision.decision)  in unroll_loops()
291 	  unroll_loop_constant_iterations (loop);  in unroll_loops()
295 	  unroll_loop_runtime_iterations (loop);  in unroll_loops()
299 	  unroll_loop_stupid (loop);  in unroll_loops()
321 loop_exit_at_end_p (class loop *loop)  in loop_exit_at_end_p()  argument
323   class niter_desc *desc = get_simple_loop_desc (loop);  in loop_exit_at_end_p()
327   gcc_assert (desc->in_edge->dest != loop->header);  in loop_exit_at_end_p()
329   if (desc->in_edge->dest != loop->latch)  in loop_exit_at_end_p()
333   FOR_BB_INSNS (loop->latch, insn)  in loop_exit_at_end_p()
346 decide_unroll_constant_iterations (class loop *loop, int flags)  in decide_unroll_constant_iterations()  argument
353   if (!(flags & UAP_UNROLL) && !loop->unroll)  in decide_unroll_constant_iterations()
363   nunroll = param_max_unrolled_insns / loop->ninsns;  in decide_unroll_constant_iterations()
365     = param_max_average_unrolled_insns / loop->av_ninsns;  in decide_unroll_constant_iterations()
372     nunroll = targetm.loop_unroll_adjust (nunroll, loop);  in decide_unroll_constant_iterations()
383   desc = get_simple_loop_desc (loop);  in decide_unroll_constant_iterations()
395   if (loop->unroll > 0 && loop->unroll < USHRT_MAX)  in decide_unroll_constant_iterations()
399       if (desc->niter == 0 || (unsigned) loop->unroll > desc->niter - 1)  in decide_unroll_constant_iterations()
406 	  loop->lpt_decision.decision = LPT_UNROLL_CONSTANT;  in decide_unroll_constant_iterations()
407 	  loop->lpt_decision.times = loop->unroll - 1;  in decide_unroll_constant_iterations()
417       || ((get_estimated_loop_iterations (loop, &iterations)  in decide_unroll_constant_iterations()
418 	   || get_likely_max_loop_iterations (loop, &iterations))  in decide_unroll_constant_iterations()
440       if (!loop_exit_at_end_p (loop))  in decide_unroll_constant_iterations()
455   loop->lpt_decision.decision = LPT_UNROLL_CONSTANT;  in decide_unroll_constant_iterations()
456   loop->lpt_decision.times = best_unroll;  in decide_unroll_constant_iterations()
479 unroll_loop_constant_iterations (class loop *loop)  in unroll_loop_constant_iterations()  argument
485   unsigned max_unroll = loop->lpt_decision.times;  in unroll_loop_constant_iterations()
486   class niter_desc *desc = get_simple_loop_desc (loop);  in unroll_loop_constant_iterations()
487   bool exit_at_end = loop_exit_at_end_p (loop);  in unroll_loop_constant_iterations()
504     opt_info = analyze_insns_in_loop (loop);  in unroll_loop_constant_iterations()
524 	    loop, loop_preheader_edge (loop), exit_mod, wont_exit,  in unroll_loop_constant_iterations()
535 	  loop->nb_iterations_upper_bound -= exit_mod;  in unroll_loop_constant_iterations()
536 	  if (loop->any_estimate  in unroll_loop_constant_iterations()
537 	      && wi::leu_p (exit_mod, loop->nb_iterations_estimate))  in unroll_loop_constant_iterations()
538 	    loop->nb_iterations_estimate -= exit_mod;  in unroll_loop_constant_iterations()
540 	    loop->any_estimate = false;  in unroll_loop_constant_iterations()
541 	  if (loop->any_likely_upper_bound  in unroll_loop_constant_iterations()
542 	      && wi::leu_p (exit_mod, loop->nb_iterations_likely_upper_bound))  in unroll_loop_constant_iterations()
543 	    loop->nb_iterations_likely_upper_bound -= exit_mod;  in unroll_loop_constant_iterations()
545 	    loop->any_likely_upper_bound = false;  in unroll_loop_constant_iterations()
570 	    loop, loop_preheader_edge (loop), exit_mod + 1, wont_exit,  in unroll_loop_constant_iterations()
580 	  loop->nb_iterations_upper_bound -= exit_mod + 1;  in unroll_loop_constant_iterations()
581 	  if (loop->any_estimate  in unroll_loop_constant_iterations()
582 	      && wi::leu_p (exit_mod + 1, loop->nb_iterations_estimate))  in unroll_loop_constant_iterations()
583 	    loop->nb_iterations_estimate -= exit_mod + 1;  in unroll_loop_constant_iterations()
585 	    loop->any_estimate = false;  in unroll_loop_constant_iterations()
586 	  if (loop->any_likely_upper_bound  in unroll_loop_constant_iterations()
587 	      && wi::leu_p (exit_mod + 1, loop->nb_iterations_likely_upper_bound))  in unroll_loop_constant_iterations()
588 	    loop->nb_iterations_likely_upper_bound -= exit_mod + 1;  in unroll_loop_constant_iterations()
590 	    loop->any_likely_upper_bound = false;  in unroll_loop_constant_iterations()
604     loop, loop_latch_edge (loop), max_unroll, wont_exit, desc->out_edge,  in unroll_loop_constant_iterations()
633   loop->nb_iterations_upper_bound  in unroll_loop_constant_iterations()
634     = wi::udiv_trunc (loop->nb_iterations_upper_bound, max_unroll + 1);  in unroll_loop_constant_iterations()
635   if (loop->any_estimate)  in unroll_loop_constant_iterations()
636     loop->nb_iterations_estimate  in unroll_loop_constant_iterations()
637       = wi::udiv_trunc (loop->nb_iterations_estimate, max_unroll + 1);  in unroll_loop_constant_iterations()
638   if (loop->any_likely_upper_bound)  in unroll_loop_constant_iterations()
639     loop->nb_iterations_likely_upper_bound  in unroll_loop_constant_iterations()
640       = wi::udiv_trunc (loop->nb_iterations_likely_upper_bound, max_unroll + 1);  in unroll_loop_constant_iterations()
650 	     max_unroll, num_loop_insns (loop));  in unroll_loop_constant_iterations()
656 decide_unroll_runtime_iterations (class loop *loop, int flags)  in decide_unroll_runtime_iterations()  argument
663   if (!(flags & UAP_UNROLL) && !loop->unroll)  in decide_unroll_runtime_iterations()
673   nunroll = param_max_unrolled_insns / loop->ninsns;  in decide_unroll_runtime_iterations()
674   nunroll_by_av = param_max_average_unrolled_insns / loop->av_ninsns;  in decide_unroll_runtime_iterations()
681     nunroll = targetm.loop_unroll_adjust (nunroll, loop);  in decide_unroll_runtime_iterations()
683   if (loop->unroll > 0 && loop->unroll < USHRT_MAX)  in decide_unroll_runtime_iterations()
684     nunroll = loop->unroll;  in decide_unroll_runtime_iterations()
695   desc = get_simple_loop_desc (loop);  in decide_unroll_runtime_iterations()
715   if ((get_estimated_loop_iterations (loop, &iterations)  in decide_unroll_runtime_iterations()
716        || get_likely_max_loop_iterations (loop, &iterations))  in decide_unroll_runtime_iterations()
730   loop->lpt_decision.decision = LPT_UNROLL_RUNTIME;  in decide_unroll_runtime_iterations()
731   loop->lpt_decision.times = i - 1;  in decide_unroll_runtime_iterations()
870 unroll_loop_runtime_iterations (class loop *loop)  in unroll_loop_runtime_iterations()  argument
882   unsigned max_unroll = loop->lpt_decision.times;  in unroll_loop_runtime_iterations()
883   class niter_desc *desc = get_simple_loop_desc (loop);  in unroll_loop_runtime_iterations()
884   bool exit_at_end = loop_exit_at_end_p (loop);  in unroll_loop_runtime_iterations()
890     opt_info = analyze_insns_in_loop (loop);  in unroll_loop_runtime_iterations()
895   body = get_loop_body (loop);  in unroll_loop_runtime_iterations()
896   for (i = 0; i < loop->num_nodes; i++)  in unroll_loop_runtime_iterations()
899 	if (!flow_bb_inside_loop_p (loop, bb))  in unroll_loop_runtime_iterations()
953   split_edge_and_insert (loop_preheader_edge (loop), init_code);  in unroll_loop_runtime_iterations()
967       ezc_swtch = loop_preheader_edge (loop)->src;  in unroll_loop_runtime_iterations()
968       ok = duplicate_loop_body_to_header_edge (loop, loop_preheader_edge (loop),  in unroll_loop_runtime_iterations()
976   swtch = split_edge (loop_preheader_edge (loop));  in unroll_loop_runtime_iterations()
990       ok = duplicate_loop_body_to_header_edge (loop, loop_preheader_edge (loop),  in unroll_loop_runtime_iterations()
1000       preheader = split_edge (loop_preheader_edge (loop));  in unroll_loop_runtime_iterations()
1026       preheader = split_edge (loop_preheader_edge (loop));  in unroll_loop_runtime_iterations()
1055     loop, loop_latch_edge (loop), max_unroll, wont_exit, desc->out_edge,  in unroll_loop_runtime_iterations()
1096   loop->nb_iterations_upper_bound  in unroll_loop_runtime_iterations()
1097     = wi::udiv_trunc (loop->nb_iterations_upper_bound, max_unroll + 1);  in unroll_loop_runtime_iterations()
1098   if (loop->any_estimate)  in unroll_loop_runtime_iterations()
1099     loop->nb_iterations_estimate  in unroll_loop_runtime_iterations()
1100       = wi::udiv_trunc (loop->nb_iterations_estimate, max_unroll + 1);  in unroll_loop_runtime_iterations()
1101   if (loop->any_likely_upper_bound)  in unroll_loop_runtime_iterations()
1102     loop->nb_iterations_likely_upper_bound  in unroll_loop_runtime_iterations()
1103       = wi::udiv_trunc (loop->nb_iterations_likely_upper_bound, max_unroll + 1);  in unroll_loop_runtime_iterations()
1109       --loop->nb_iterations_upper_bound;  in unroll_loop_runtime_iterations()
1110       if (loop->any_estimate  in unroll_loop_runtime_iterations()
1111 	  && loop->nb_iterations_estimate != 0)  in unroll_loop_runtime_iterations()
1112 	--loop->nb_iterations_estimate;  in unroll_loop_runtime_iterations()
1114 	loop->any_estimate = false;  in unroll_loop_runtime_iterations()
1115       if (loop->any_likely_upper_bound  in unroll_loop_runtime_iterations()
1116 	  && loop->nb_iterations_likely_upper_bound != 0)  in unroll_loop_runtime_iterations()
1117 	--loop->nb_iterations_likely_upper_bound;  in unroll_loop_runtime_iterations()
1119 	loop->any_likely_upper_bound = false;  in unroll_loop_runtime_iterations()
1126 	     max_unroll, num_loop_insns (loop));  in unroll_loop_runtime_iterations()
1131 decide_unroll_stupid (class loop *loop, int flags)  in decide_unroll_stupid()  argument
1138   if (!(flags & UAP_UNROLL_ALL) && !loop->unroll)  in decide_unroll_stupid()
1146   nunroll = param_max_unrolled_insns / loop->ninsns;  in decide_unroll_stupid()
1148     = param_max_average_unrolled_insns / loop->av_ninsns;  in decide_unroll_stupid()
1155     nunroll = targetm.loop_unroll_adjust (nunroll, loop);  in decide_unroll_stupid()
1157   if (loop->unroll > 0 && loop->unroll < USHRT_MAX)  in decide_unroll_stupid()
1158     nunroll = loop->unroll;  in decide_unroll_stupid()
1169   desc = get_simple_loop_desc (loop);  in decide_unroll_stupid()
1183   if (num_loop_branches (loop) > 1)  in decide_unroll_stupid()
1191   if ((get_estimated_loop_iterations (loop, &iterations)  in decide_unroll_stupid()
1192        || get_likely_max_loop_iterations (loop, &iterations))  in decide_unroll_stupid()
1206   loop->lpt_decision.decision = LPT_UNROLL_STUPID;  in decide_unroll_stupid()
1207   loop->lpt_decision.times = i - 1;  in decide_unroll_stupid()
1229 unroll_loop_stupid (class loop *loop)  in unroll_loop_stupid()  argument
1231   unsigned nunroll = loop->lpt_decision.times;  in unroll_loop_stupid()
1232   class niter_desc *desc = get_simple_loop_desc (loop);  in unroll_loop_stupid()
1238     opt_info = analyze_insns_in_loop (loop);  in unroll_loop_stupid()
1245     loop, loop_latch_edge (loop), nunroll, wont_exit, NULL, NULL,  in unroll_loop_stupid()
1268 	     nunroll, num_loop_insns (loop));  in unroll_loop_stupid()
1276 referenced_in_one_insn_in_loop_p (class loop *loop, rtx reg,  in referenced_in_one_insn_in_loop_p()  argument
1284   body = get_loop_body (loop);  in referenced_in_one_insn_in_loop_p()
1285   for (i = 0; i < loop->num_nodes; i++)  in referenced_in_one_insn_in_loop_p()
1304 reset_debug_uses_in_loop (class loop *loop, rtx reg, int debug_uses)  in reset_debug_uses_in_loop()  argument
1310   body = get_loop_body (loop);  in reset_debug_uses_in_loop()
1311   for (i = 0; debug_uses && i < loop->num_nodes; i++)  in reset_debug_uses_in_loop()
1353 analyze_insn_to_expand_var (class loop *loop, rtx_insn *insn)  in analyze_insn_to_expand_var()  argument
1432   if (!referenced_in_one_insn_in_loop_p (loop, dest, &debug_uses))  in analyze_insn_to_expand_var()
1451     reset_debug_uses_in_loop (loop, dest, debug_uses);  in analyze_insn_to_expand_var()
1546 analyze_insns_in_loop (class loop *loop)  in analyze_insns_in_loop()  argument
1556   auto_vec<edge> edges = get_loop_exit_edges (loop);  in analyze_insns_in_loop()
1560   iv_analysis_loop_init (loop);  in analyze_insns_in_loop()
1562   body = get_loop_body (loop);  in analyze_insns_in_loop()
1567        	= new hash_table<iv_split_hasher> (5 * loop->num_nodes);  in analyze_insns_in_loop()
1573   opt_info->loop_preheader = loop_preheader_edge (loop)->src;  in analyze_insns_in_loop()
1589        	= new hash_table<var_expand_hasher> (5 * loop->num_nodes);  in analyze_insns_in_loop()
1594   for (i = 0; i < loop->num_nodes; i++)  in analyze_insns_in_loop()
1597       if (!dominated_by_p (CDI_DOMINATORS, loop->latch, bb))  in analyze_insns_in_loop()
1619           ves = analyze_insn_to_expand_var (loop, insn);  in analyze_insns_in_loop()