1 /* Calculate branch probabilities, and basic block execution counts.
2 Copyright (C) 1990-2022 Free Software Foundation, Inc.
3 Contributed by James E. Wilson, UC Berkeley/Cygnus Support;
4 based on some ideas from Dain Samples of UC Berkeley.
5 Further mangling by Bob Manson, Cygnus Support.
6 Converted to use trees by Dale Johannesen, Apple Computer.
7
8 This file is part of GCC.
9
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
13 version.
14
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 for more details.
19
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
23
24 /* Generate basic block profile instrumentation and auxiliary files.
25 Tree-based version. See profile.cc for overview. */
26
27 #include "config.h"
28 #include "system.h"
29 #include "coretypes.h"
30 #include "memmodel.h"
31 #include "backend.h"
32 #include "target.h"
33 #include "tree.h"
34 #include "gimple.h"
35 #include "cfghooks.h"
36 #include "tree-pass.h"
37 #include "ssa.h"
38 #include "cgraph.h"
39 #include "coverage.h"
40 #include "diagnostic-core.h"
41 #include "fold-const.h"
42 #include "varasm.h"
43 #include "tree-nested.h"
44 #include "gimplify.h"
45 #include "gimple-iterator.h"
46 #include "gimplify-me.h"
47 #include "tree-cfg.h"
48 #include "tree-into-ssa.h"
49 #include "value-prof.h"
50 #include "profile.h"
51 #include "tree-cfgcleanup.h"
52 #include "stringpool.h"
53 #include "attribs.h"
54 #include "tree-pretty-print.h"
55 #include "langhooks.h"
56 #include "stor-layout.h"
57 #include "xregex.h"
58 #include "alloc-pool.h"
59 #include "symbol-summary.h"
60 #include "symtab-thunks.h"
61
62 static GTY(()) tree gcov_type_node;
63 static GTY(()) tree tree_interval_profiler_fn;
64 static GTY(()) tree tree_pow2_profiler_fn;
65 static GTY(()) tree tree_topn_values_profiler_fn;
66 static GTY(()) tree tree_indirect_call_profiler_fn;
67 static GTY(()) tree tree_average_profiler_fn;
68 static GTY(()) tree tree_ior_profiler_fn;
69 static GTY(()) tree tree_time_profiler_counter;
70
71
72 static GTY(()) tree ic_tuple_var;
73 static GTY(()) tree ic_tuple_counters_field;
74 static GTY(()) tree ic_tuple_callee_field;
75
76 /* Do initialization work for the edge profiler. */
77
78 /* Add code:
79 __thread gcov* __gcov_indirect_call.counters; // pointer to actual counter
80 __thread void* __gcov_indirect_call.callee; // actual callee address
81 __thread int __gcov_function_counter; // time profiler function counter
82 */
83 static void
init_ic_make_global_vars(void)84 init_ic_make_global_vars (void)
85 {
86 tree gcov_type_ptr;
87
88 gcov_type_ptr = build_pointer_type (get_gcov_type ());
89
90 tree tuple_type = lang_hooks.types.make_type (RECORD_TYPE);
91
92 /* callee */
93 ic_tuple_callee_field = build_decl (BUILTINS_LOCATION, FIELD_DECL, NULL_TREE,
94 ptr_type_node);
95
96 /* counters */
97 ic_tuple_counters_field = build_decl (BUILTINS_LOCATION, FIELD_DECL,
98 NULL_TREE, gcov_type_ptr);
99 DECL_CHAIN (ic_tuple_counters_field) = ic_tuple_callee_field;
100
101 finish_builtin_struct (tuple_type, "indirect_call_tuple",
102 ic_tuple_counters_field, NULL_TREE);
103
104 ic_tuple_var
105 = build_decl (UNKNOWN_LOCATION, VAR_DECL,
106 get_identifier ("__gcov_indirect_call"), tuple_type);
107 TREE_PUBLIC (ic_tuple_var) = 1;
108 DECL_ARTIFICIAL (ic_tuple_var) = 1;
109 DECL_INITIAL (ic_tuple_var) = NULL;
110 DECL_EXTERNAL (ic_tuple_var) = 1;
111 if (targetm.have_tls)
112 set_decl_tls_model (ic_tuple_var, decl_default_tls_model (ic_tuple_var));
113 }
114
115 /* Create the type and function decls for the interface with gcov. */
116
117 void
gimple_init_gcov_profiler(void)118 gimple_init_gcov_profiler (void)
119 {
120 tree interval_profiler_fn_type;
121 tree pow2_profiler_fn_type;
122 tree topn_values_profiler_fn_type;
123 tree gcov_type_ptr;
124 tree ic_profiler_fn_type;
125 tree average_profiler_fn_type;
126 const char *fn_name;
127
128 if (!gcov_type_node)
129 {
130 const char *fn_suffix
131 = flag_profile_update == PROFILE_UPDATE_ATOMIC ? "_atomic" : "";
132
133 gcov_type_node = get_gcov_type ();
134 gcov_type_ptr = build_pointer_type (gcov_type_node);
135
136 /* void (*) (gcov_type *, gcov_type, int, unsigned) */
137 interval_profiler_fn_type
138 = build_function_type_list (void_type_node,
139 gcov_type_ptr, gcov_type_node,
140 integer_type_node,
141 unsigned_type_node, NULL_TREE);
142 fn_name = concat ("__gcov_interval_profiler", fn_suffix, NULL);
143 tree_interval_profiler_fn = build_fn_decl (fn_name,
144 interval_profiler_fn_type);
145 free (CONST_CAST (char *, fn_name));
146 TREE_NOTHROW (tree_interval_profiler_fn) = 1;
147 DECL_ATTRIBUTES (tree_interval_profiler_fn)
148 = tree_cons (get_identifier ("leaf"), NULL,
149 DECL_ATTRIBUTES (tree_interval_profiler_fn));
150
151 /* void (*) (gcov_type *, gcov_type) */
152 pow2_profiler_fn_type
153 = build_function_type_list (void_type_node,
154 gcov_type_ptr, gcov_type_node,
155 NULL_TREE);
156 fn_name = concat ("__gcov_pow2_profiler", fn_suffix, NULL);
157 tree_pow2_profiler_fn = build_fn_decl (fn_name, pow2_profiler_fn_type);
158 free (CONST_CAST (char *, fn_name));
159 TREE_NOTHROW (tree_pow2_profiler_fn) = 1;
160 DECL_ATTRIBUTES (tree_pow2_profiler_fn)
161 = tree_cons (get_identifier ("leaf"), NULL,
162 DECL_ATTRIBUTES (tree_pow2_profiler_fn));
163
164 /* void (*) (gcov_type *, gcov_type) */
165 topn_values_profiler_fn_type
166 = build_function_type_list (void_type_node,
167 gcov_type_ptr, gcov_type_node,
168 NULL_TREE);
169 fn_name = concat ("__gcov_topn_values_profiler", fn_suffix, NULL);
170 tree_topn_values_profiler_fn
171 = build_fn_decl (fn_name, topn_values_profiler_fn_type);
172 free (CONST_CAST (char *, fn_name));
173
174 TREE_NOTHROW (tree_topn_values_profiler_fn) = 1;
175 DECL_ATTRIBUTES (tree_topn_values_profiler_fn)
176 = tree_cons (get_identifier ("leaf"), NULL,
177 DECL_ATTRIBUTES (tree_topn_values_profiler_fn));
178
179 init_ic_make_global_vars ();
180
181 /* void (*) (gcov_type, void *) */
182 ic_profiler_fn_type
183 = build_function_type_list (void_type_node,
184 gcov_type_node,
185 ptr_type_node,
186 NULL_TREE);
187 fn_name = concat ("__gcov_indirect_call_profiler_v4", fn_suffix, NULL);
188 tree_indirect_call_profiler_fn
189 = build_fn_decl (fn_name, ic_profiler_fn_type);
190 free (CONST_CAST (char *, fn_name));
191
192 TREE_NOTHROW (tree_indirect_call_profiler_fn) = 1;
193 DECL_ATTRIBUTES (tree_indirect_call_profiler_fn)
194 = tree_cons (get_identifier ("leaf"), NULL,
195 DECL_ATTRIBUTES (tree_indirect_call_profiler_fn));
196
197 tree_time_profiler_counter
198 = build_decl (UNKNOWN_LOCATION, VAR_DECL,
199 get_identifier ("__gcov_time_profiler_counter"),
200 get_gcov_type ());
201 TREE_PUBLIC (tree_time_profiler_counter) = 1;
202 DECL_EXTERNAL (tree_time_profiler_counter) = 1;
203 TREE_STATIC (tree_time_profiler_counter) = 1;
204 DECL_ARTIFICIAL (tree_time_profiler_counter) = 1;
205 DECL_INITIAL (tree_time_profiler_counter) = NULL;
206
207 /* void (*) (gcov_type *, gcov_type) */
208 average_profiler_fn_type
209 = build_function_type_list (void_type_node,
210 gcov_type_ptr, gcov_type_node, NULL_TREE);
211 fn_name = concat ("__gcov_average_profiler", fn_suffix, NULL);
212 tree_average_profiler_fn = build_fn_decl (fn_name,
213 average_profiler_fn_type);
214 free (CONST_CAST (char *, fn_name));
215 TREE_NOTHROW (tree_average_profiler_fn) = 1;
216 DECL_ATTRIBUTES (tree_average_profiler_fn)
217 = tree_cons (get_identifier ("leaf"), NULL,
218 DECL_ATTRIBUTES (tree_average_profiler_fn));
219 fn_name = concat ("__gcov_ior_profiler", fn_suffix, NULL);
220 tree_ior_profiler_fn = build_fn_decl (fn_name, average_profiler_fn_type);
221 free (CONST_CAST (char *, fn_name));
222 TREE_NOTHROW (tree_ior_profiler_fn) = 1;
223 DECL_ATTRIBUTES (tree_ior_profiler_fn)
224 = tree_cons (get_identifier ("leaf"), NULL,
225 DECL_ATTRIBUTES (tree_ior_profiler_fn));
226
227 /* LTO streamer needs assembler names. Because we create these decls
228 late, we need to initialize them by hand. */
229 DECL_ASSEMBLER_NAME (tree_interval_profiler_fn);
230 DECL_ASSEMBLER_NAME (tree_pow2_profiler_fn);
231 DECL_ASSEMBLER_NAME (tree_topn_values_profiler_fn);
232 DECL_ASSEMBLER_NAME (tree_indirect_call_profiler_fn);
233 DECL_ASSEMBLER_NAME (tree_average_profiler_fn);
234 DECL_ASSEMBLER_NAME (tree_ior_profiler_fn);
235 }
236 }
237
238 /* Output instructions as GIMPLE trees to increment the edge
239 execution count, and insert them on E. We rely on
240 gsi_insert_on_edge to preserve the order. */
241
242 void
gimple_gen_edge_profiler(int edgeno,edge e)243 gimple_gen_edge_profiler (int edgeno, edge e)
244 {
245 tree one;
246
247 one = build_int_cst (gcov_type_node, 1);
248
249 if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
250 {
251 /* __atomic_fetch_add (&counter, 1, MEMMODEL_RELAXED); */
252 tree addr = tree_coverage_counter_addr (GCOV_COUNTER_ARCS, edgeno);
253 tree f = builtin_decl_explicit (TYPE_PRECISION (gcov_type_node) > 32
254 ? BUILT_IN_ATOMIC_FETCH_ADD_8:
255 BUILT_IN_ATOMIC_FETCH_ADD_4);
256 gcall *stmt = gimple_build_call (f, 3, addr, one,
257 build_int_cst (integer_type_node,
258 MEMMODEL_RELAXED));
259 gsi_insert_on_edge (e, stmt);
260 }
261 else
262 {
263 tree ref = tree_coverage_counter_ref (GCOV_COUNTER_ARCS, edgeno);
264 tree gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
265 NULL, "PROF_edge_counter");
266 gassign *stmt1 = gimple_build_assign (gcov_type_tmp_var, ref);
267 gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node,
268 NULL, "PROF_edge_counter");
269 gassign *stmt2 = gimple_build_assign (gcov_type_tmp_var, PLUS_EXPR,
270 gimple_assign_lhs (stmt1), one);
271 gassign *stmt3 = gimple_build_assign (unshare_expr (ref),
272 gimple_assign_lhs (stmt2));
273 gsi_insert_on_edge (e, stmt1);
274 gsi_insert_on_edge (e, stmt2);
275 gsi_insert_on_edge (e, stmt3);
276 }
277 }
278
279 /* Emits code to get VALUE to instrument at GSI, and returns the
280 variable containing the value. */
281
282 static tree
prepare_instrumented_value(gimple_stmt_iterator * gsi,histogram_value value)283 prepare_instrumented_value (gimple_stmt_iterator *gsi, histogram_value value)
284 {
285 tree val = value->hvalue.value;
286 if (POINTER_TYPE_P (TREE_TYPE (val)))
287 val = fold_convert (build_nonstandard_integer_type
288 (TYPE_PRECISION (TREE_TYPE (val)), 1), val);
289 return force_gimple_operand_gsi (gsi, fold_convert (gcov_type_node, val),
290 true, NULL_TREE, true, GSI_SAME_STMT);
291 }
292
293 /* Output instructions as GIMPLE trees to increment the interval histogram
294 counter. VALUE is the expression whose value is profiled. TAG is the
295 tag of the section for counters, BASE is offset of the counter position. */
296
297 void
gimple_gen_interval_profiler(histogram_value value,unsigned tag)298 gimple_gen_interval_profiler (histogram_value value, unsigned tag)
299 {
300 gimple *stmt = value->hvalue.stmt;
301 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
302 tree ref = tree_coverage_counter_ref (tag, 0), ref_ptr;
303 gcall *call;
304 tree val;
305 tree start = build_int_cst_type (integer_type_node,
306 value->hdata.intvl.int_start);
307 tree steps = build_int_cst_type (unsigned_type_node,
308 value->hdata.intvl.steps);
309
310 ref_ptr = force_gimple_operand_gsi (&gsi,
311 build_addr (ref),
312 true, NULL_TREE, true, GSI_SAME_STMT);
313 val = prepare_instrumented_value (&gsi, value);
314 call = gimple_build_call (tree_interval_profiler_fn, 4,
315 ref_ptr, val, start, steps);
316 gsi_insert_before (&gsi, call, GSI_NEW_STMT);
317 }
318
319 /* Output instructions as GIMPLE trees to increment the power of two histogram
320 counter. VALUE is the expression whose value is profiled. TAG is the tag
321 of the section for counters. */
322
323 void
gimple_gen_pow2_profiler(histogram_value value,unsigned tag)324 gimple_gen_pow2_profiler (histogram_value value, unsigned tag)
325 {
326 gimple *stmt = value->hvalue.stmt;
327 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
328 tree ref_ptr = tree_coverage_counter_addr (tag, 0);
329 gcall *call;
330 tree val;
331
332 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
333 true, NULL_TREE, true, GSI_SAME_STMT);
334 val = prepare_instrumented_value (&gsi, value);
335 call = gimple_build_call (tree_pow2_profiler_fn, 2, ref_ptr, val);
336 gsi_insert_before (&gsi, call, GSI_NEW_STMT);
337 }
338
339 /* Output instructions as GIMPLE trees for code to find the most N common
340 values. VALUE is the expression whose value is profiled. TAG is the tag
341 of the section for counters. */
342
343 void
gimple_gen_topn_values_profiler(histogram_value value,unsigned tag)344 gimple_gen_topn_values_profiler (histogram_value value, unsigned tag)
345 {
346 gimple *stmt = value->hvalue.stmt;
347 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
348 tree ref_ptr = tree_coverage_counter_addr (tag, 0);
349 gcall *call;
350 tree val;
351
352 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
353 true, NULL_TREE, true, GSI_SAME_STMT);
354 val = prepare_instrumented_value (&gsi, value);
355 call = gimple_build_call (tree_topn_values_profiler_fn, 2, ref_ptr, val);
356 gsi_insert_before (&gsi, call, GSI_NEW_STMT);
357 }
358
359
360 /* Output instructions as GIMPLE trees for code to find the most
361 common called function in indirect call.
362 VALUE is the call expression whose indirect callee is profiled.
363 TAG is the tag of the section for counters. */
364
365 void
gimple_gen_ic_profiler(histogram_value value,unsigned tag)366 gimple_gen_ic_profiler (histogram_value value, unsigned tag)
367 {
368 tree tmp1;
369 gassign *stmt1, *stmt2, *stmt3;
370 gimple *stmt = value->hvalue.stmt;
371 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
372 tree ref_ptr = tree_coverage_counter_addr (tag, 0);
373
374 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
375 true, NULL_TREE, true, GSI_SAME_STMT);
376
377 /* Insert code:
378
379 stmt1: __gcov_indirect_call.counters = get_relevant_counter_ptr ();
380 stmt2: tmp1 = (void *) (indirect call argument value)
381 stmt3: __gcov_indirect_call.callee = tmp1;
382
383 Example:
384 f_1 = foo;
385 __gcov_indirect_call.counters = &__gcov4.main[0];
386 PROF_9 = f_1;
387 __gcov_indirect_call.callee = PROF_9;
388 _4 = f_1 ();
389 */
390
391 tree gcov_type_ptr = build_pointer_type (get_gcov_type ());
392
393 tree counter_ref = build3 (COMPONENT_REF, gcov_type_ptr,
394 ic_tuple_var, ic_tuple_counters_field, NULL_TREE);
395
396 stmt1 = gimple_build_assign (counter_ref, ref_ptr);
397 tmp1 = make_temp_ssa_name (ptr_type_node, NULL, "PROF");
398 stmt2 = gimple_build_assign (tmp1, unshare_expr (value->hvalue.value));
399 tree callee_ref = build3 (COMPONENT_REF, ptr_type_node,
400 ic_tuple_var, ic_tuple_callee_field, NULL_TREE);
401 stmt3 = gimple_build_assign (callee_ref, tmp1);
402
403 gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
404 gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT);
405 gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT);
406 }
407
408
409 /* Output instructions as GIMPLE trees for code to find the most
410 common called function in indirect call. Insert instructions at the
411 beginning of every possible called function.
412 */
413
414 void
gimple_gen_ic_func_profiler(void)415 gimple_gen_ic_func_profiler (void)
416 {
417 struct cgraph_node * c_node = cgraph_node::get (current_function_decl);
418 gcall *stmt1;
419 tree tree_uid, cur_func, void0;
420
421 /* Disable indirect call profiling for an IFUNC resolver and its
422 callees since it requires TLS which hasn't been set up yet when
423 the dynamic linker is resolving IFUNC symbols. See
424 https://gcc.gnu.org/bugzilla/show_bug.cgi?id=114115
425 */
426 if (c_node->only_called_directly_p ()
427 || c_node->called_by_ifunc_resolver)
428 return;
429
430 gimple_init_gcov_profiler ();
431
432 basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
433 basic_block cond_bb = split_edge (single_succ_edge (entry));
434 basic_block update_bb = split_edge (single_succ_edge (cond_bb));
435
436 /* We need to do an extra split in order to not create an input
437 for a possible PHI node. */
438 split_edge (single_succ_edge (update_bb));
439
440 edge true_edge = single_succ_edge (cond_bb);
441 true_edge->flags = EDGE_TRUE_VALUE;
442
443 profile_probability probability;
444 if (DECL_VIRTUAL_P (current_function_decl))
445 probability = profile_probability::very_likely ();
446 else
447 probability = profile_probability::unlikely ();
448
449 true_edge->probability = probability;
450 edge e = make_edge (cond_bb, single_succ_edge (update_bb)->dest,
451 EDGE_FALSE_VALUE);
452 e->probability = true_edge->probability.invert ();
453
454 /* Insert code:
455
456 if (__gcov_indirect_call.callee != NULL)
457 __gcov_indirect_call_profiler_v3 (profile_id, ¤t_function_decl);
458
459 The function __gcov_indirect_call_profiler_v3 is responsible for
460 resetting __gcov_indirect_call.callee to NULL. */
461
462 gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
463 void0 = build_int_cst (ptr_type_node, 0);
464
465 tree callee_ref = build3 (COMPONENT_REF, ptr_type_node,
466 ic_tuple_var, ic_tuple_callee_field, NULL_TREE);
467
468 tree ref = force_gimple_operand_gsi (&gsi, callee_ref, true, NULL_TREE,
469 true, GSI_SAME_STMT);
470
471 gcond *cond = gimple_build_cond (NE_EXPR, ref,
472 void0, NULL, NULL);
473 gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
474
475 gsi = gsi_after_labels (update_bb);
476
477 cur_func = force_gimple_operand_gsi (&gsi,
478 build_addr (current_function_decl),
479 true, NULL_TREE,
480 true, GSI_SAME_STMT);
481 tree_uid = build_int_cst
482 (gcov_type_node,
483 cgraph_node::get (current_function_decl)->profile_id);
484 stmt1 = gimple_build_call (tree_indirect_call_profiler_fn, 2,
485 tree_uid, cur_func);
486 gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT);
487 }
488
489 /* Output instructions as GIMPLE tree at the beginning for each function.
490 TAG is the tag of the section for counters, BASE is offset of the
491 counter position and GSI is the iterator we place the counter. */
492
493 void
gimple_gen_time_profiler(unsigned tag)494 gimple_gen_time_profiler (unsigned tag)
495 {
496 tree type = get_gcov_type ();
497 basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
498 basic_block cond_bb = split_edge (single_succ_edge (entry));
499 basic_block update_bb = split_edge (single_succ_edge (cond_bb));
500
501 /* We need to do an extra split in order to not create an input
502 for a possible PHI node. */
503 split_edge (single_succ_edge (update_bb));
504
505 edge true_edge = single_succ_edge (cond_bb);
506 true_edge->flags = EDGE_TRUE_VALUE;
507 true_edge->probability = profile_probability::unlikely ();
508 edge e
509 = make_edge (cond_bb, single_succ_edge (update_bb)->dest, EDGE_FALSE_VALUE);
510 e->probability = true_edge->probability.invert ();
511
512 gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
513 tree original_ref = tree_coverage_counter_ref (tag, 0);
514 tree ref = force_gimple_operand_gsi (&gsi, original_ref, true, NULL_TREE,
515 true, GSI_SAME_STMT);
516 tree one = build_int_cst (type, 1);
517
518 /* Emit: if (counters[0] != 0). */
519 gcond *cond = gimple_build_cond (EQ_EXPR, ref, build_int_cst (type, 0),
520 NULL, NULL);
521 gsi_insert_before (&gsi, cond, GSI_NEW_STMT);
522
523 gsi = gsi_start_bb (update_bb);
524
525 /* Emit: counters[0] = ++__gcov_time_profiler_counter. */
526 if (flag_profile_update == PROFILE_UPDATE_ATOMIC)
527 {
528 tree ptr = make_temp_ssa_name (build_pointer_type (type), NULL,
529 "time_profiler_counter_ptr");
530 tree addr = build1 (ADDR_EXPR, TREE_TYPE (ptr),
531 tree_time_profiler_counter);
532 gassign *assign = gimple_build_assign (ptr, NOP_EXPR, addr);
533 gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
534 tree f = builtin_decl_explicit (TYPE_PRECISION (gcov_type_node) > 32
535 ? BUILT_IN_ATOMIC_ADD_FETCH_8:
536 BUILT_IN_ATOMIC_ADD_FETCH_4);
537 gcall *stmt = gimple_build_call (f, 3, ptr, one,
538 build_int_cst (integer_type_node,
539 MEMMODEL_RELAXED));
540 tree result_type = TREE_TYPE (TREE_TYPE (f));
541 tree tmp = make_temp_ssa_name (result_type, NULL, "time_profile");
542 gimple_set_lhs (stmt, tmp);
543 gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
544 tmp = make_temp_ssa_name (type, NULL, "time_profile");
545 assign = gimple_build_assign (tmp, NOP_EXPR,
546 gimple_call_lhs (stmt));
547 gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
548 assign = gimple_build_assign (original_ref, tmp);
549 gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
550 }
551 else
552 {
553 tree tmp = make_temp_ssa_name (type, NULL, "time_profile");
554 gassign *assign = gimple_build_assign (tmp, tree_time_profiler_counter);
555 gsi_insert_before (&gsi, assign, GSI_NEW_STMT);
556
557 tmp = make_temp_ssa_name (type, NULL, "time_profile");
558 assign = gimple_build_assign (tmp, PLUS_EXPR, gimple_assign_lhs (assign),
559 one);
560 gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
561 assign = gimple_build_assign (original_ref, tmp);
562 gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
563 assign = gimple_build_assign (tree_time_profiler_counter, tmp);
564 gsi_insert_after (&gsi, assign, GSI_NEW_STMT);
565 }
566 }
567
568 /* Output instructions as GIMPLE trees to increment the average histogram
569 counter. VALUE is the expression whose value is profiled. TAG is the
570 tag of the section for counters, BASE is offset of the counter position. */
571
572 void
gimple_gen_average_profiler(histogram_value value,unsigned tag)573 gimple_gen_average_profiler (histogram_value value, unsigned tag)
574 {
575 gimple *stmt = value->hvalue.stmt;
576 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
577 tree ref_ptr = tree_coverage_counter_addr (tag, 0);
578 gcall *call;
579 tree val;
580
581 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
582 true, NULL_TREE,
583 true, GSI_SAME_STMT);
584 val = prepare_instrumented_value (&gsi, value);
585 call = gimple_build_call (tree_average_profiler_fn, 2, ref_ptr, val);
586 gsi_insert_before (&gsi, call, GSI_NEW_STMT);
587 }
588
589 /* Output instructions as GIMPLE trees to increment the ior histogram
590 counter. VALUE is the expression whose value is profiled. TAG is the
591 tag of the section for counters, BASE is offset of the counter position. */
592
593 void
gimple_gen_ior_profiler(histogram_value value,unsigned tag)594 gimple_gen_ior_profiler (histogram_value value, unsigned tag)
595 {
596 gimple *stmt = value->hvalue.stmt;
597 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
598 tree ref_ptr = tree_coverage_counter_addr (tag, 0);
599 gcall *call;
600 tree val;
601
602 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr,
603 true, NULL_TREE, true, GSI_SAME_STMT);
604 val = prepare_instrumented_value (&gsi, value);
605 call = gimple_build_call (tree_ior_profiler_fn, 2, ref_ptr, val);
606 gsi_insert_before (&gsi, call, GSI_NEW_STMT);
607 }
608
609 static vec<regex_t> profile_filter_files;
610 static vec<regex_t> profile_exclude_files;
611
612 /* Parse list of provided REGEX (separated with semi-collon) and
613 create expressions (of type regex_t) and save them into V vector.
614 If there is a regular expression parsing error, error message is
615 printed for FLAG_NAME. */
616
617 static void
parse_profile_filter(const char * regex,vec<regex_t> * v,const char * flag_name)618 parse_profile_filter (const char *regex, vec<regex_t> *v,
619 const char *flag_name)
620 {
621 v->create (4);
622 if (regex != NULL)
623 {
624 char *str = xstrdup (regex);
625 for (char *p = strtok (str, ";"); p != NULL; p = strtok (NULL, ";"))
626 {
627 regex_t r;
628 if (regcomp (&r, p, REG_EXTENDED | REG_NOSUB) != 0)
629 {
630 error ("invalid regular expression %qs in %qs",
631 p, flag_name);
632 return;
633 }
634
635 v->safe_push (r);
636 }
637 }
638 }
639
640 /* Parse values of -fprofile-filter-files and -fprofile-exclude-files
641 options. */
642
643 static void
parse_profile_file_filtering()644 parse_profile_file_filtering ()
645 {
646 parse_profile_filter (flag_profile_filter_files, &profile_filter_files,
647 "-fprofile-filter-files");
648 parse_profile_filter (flag_profile_exclude_files, &profile_exclude_files,
649 "-fprofile-exclude-files");
650 }
651
652 /* Parse vectors of regular expressions. */
653
654 static void
release_profile_file_filtering()655 release_profile_file_filtering ()
656 {
657 profile_filter_files.release ();
658 profile_exclude_files.release ();
659 }
660
661 /* Return true when FILENAME should be instrumented based on
662 -fprofile-filter-files and -fprofile-exclude-files options. */
663
664 static bool
include_source_file_for_profile(const char * filename)665 include_source_file_for_profile (const char *filename)
666 {
667 /* First check whether file is included in flag_profile_exclude_files. */
668 for (unsigned i = 0; i < profile_exclude_files.length (); i++)
669 if (regexec (&profile_exclude_files[i],
670 filename, 0, NULL, 0) == REG_NOERROR)
671 return false;
672
673 /* For non-empty flag_profile_filter_files include only files matching a
674 regex in the flag. */
675 if (profile_filter_files.is_empty ())
676 return true;
677
678 for (unsigned i = 0; i < profile_filter_files.length (); i++)
679 if (regexec (&profile_filter_files[i], filename, 0, NULL, 0) == REG_NOERROR)
680 return true;
681
682 return false;
683 }
684
685 #ifndef HAVE_sync_compare_and_swapsi
686 #define HAVE_sync_compare_and_swapsi 0
687 #endif
688 #ifndef HAVE_atomic_compare_and_swapsi
689 #define HAVE_atomic_compare_and_swapsi 0
690 #endif
691
692 #ifndef HAVE_sync_compare_and_swapdi
693 #define HAVE_sync_compare_and_swapdi 0
694 #endif
695 #ifndef HAVE_atomic_compare_and_swapdi
696 #define HAVE_atomic_compare_and_swapdi 0
697 #endif
698
699 /* Profile all functions in the callgraph. */
700
701 static unsigned int
tree_profiling(void)702 tree_profiling (void)
703 {
704 struct cgraph_node *node;
705
706 /* Verify whether we can utilize atomic update operations. */
707 bool can_support_atomic = false;
708 unsigned HOST_WIDE_INT gcov_type_size
709 = tree_to_uhwi (TYPE_SIZE_UNIT (get_gcov_type ()));
710 if (gcov_type_size == 4)
711 can_support_atomic
712 = HAVE_sync_compare_and_swapsi || HAVE_atomic_compare_and_swapsi;
713 else if (gcov_type_size == 8)
714 can_support_atomic
715 = HAVE_sync_compare_and_swapdi || HAVE_atomic_compare_and_swapdi;
716
717 if (flag_profile_update == PROFILE_UPDATE_ATOMIC
718 && !can_support_atomic)
719 {
720 warning (0, "target does not support atomic profile update, "
721 "single mode is selected");
722 flag_profile_update = PROFILE_UPDATE_SINGLE;
723 }
724 else if (flag_profile_update == PROFILE_UPDATE_PREFER_ATOMIC)
725 flag_profile_update = can_support_atomic
726 ? PROFILE_UPDATE_ATOMIC : PROFILE_UPDATE_SINGLE;
727
728 /* This is a small-ipa pass that gets called only once, from
729 cgraphunit.cc:ipa_passes(). */
730 gcc_assert (symtab->state == IPA_SSA);
731
732 init_node_map (true);
733 parse_profile_file_filtering ();
734
735 FOR_EACH_DEFINED_FUNCTION (node)
736 {
737 bool thunk = false;
738 if (!gimple_has_body_p (node->decl) && !node->thunk)
739 continue;
740
741 /* Don't profile functions produced for builtin stuff. */
742 if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
743 continue;
744
745 if (lookup_attribute ("no_profile_instrument_function",
746 DECL_ATTRIBUTES (node->decl)))
747 continue;
748 /* Do not instrument extern inline functions when testing coverage.
749 While this is not perfectly consistent (early inlined extern inlines
750 will get acocunted), testsuite expects that. */
751 if (DECL_EXTERNAL (node->decl)
752 && flag_test_coverage)
753 continue;
754
755 const char *file = LOCATION_FILE (DECL_SOURCE_LOCATION (node->decl));
756 if (!include_source_file_for_profile (file))
757 continue;
758
759 if (node->thunk)
760 {
761 /* We cannot expand variadic thunks to Gimple. */
762 if (stdarg_p (TREE_TYPE (node->decl)))
763 continue;
764 thunk = true;
765 /* When generate profile, expand thunk to gimple so it can be
766 instrumented same way as other functions. */
767 if (profile_arc_flag)
768 expand_thunk (node, false, true);
769 /* Read cgraph profile but keep function as thunk at profile-use
770 time. */
771 else
772 {
773 read_thunk_profile (node);
774 continue;
775 }
776 }
777
778 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
779
780 if (dump_file)
781 dump_function_header (dump_file, cfun->decl, dump_flags);
782
783 /* Local pure-const may imply need to fixup the cfg. */
784 if (gimple_has_body_p (node->decl)
785 && (execute_fixup_cfg () & TODO_cleanup_cfg))
786 cleanup_tree_cfg ();
787
788 branch_prob (thunk);
789
790 if (! flag_branch_probabilities
791 && flag_profile_values)
792 gimple_gen_ic_func_profiler ();
793
794 if (flag_branch_probabilities
795 && !thunk
796 && flag_profile_values
797 && flag_value_profile_transformations
798 && profile_status_for_fn (cfun) == PROFILE_READ)
799 gimple_value_profile_transformations ();
800
801 /* The above could hose dominator info. Currently there is
802 none coming in, this is a safety valve. It should be
803 easy to adjust it, if and when there is some. */
804 free_dominance_info (CDI_DOMINATORS);
805 free_dominance_info (CDI_POST_DOMINATORS);
806 pop_cfun ();
807 }
808
809 release_profile_file_filtering ();
810
811 /* Drop pure/const flags from instrumented functions. */
812 if (profile_arc_flag || flag_test_coverage)
813 FOR_EACH_DEFINED_FUNCTION (node)
814 {
815 if (!gimple_has_body_p (node->decl)
816 || !(!node->clone_of
817 || node->decl != node->clone_of->decl))
818 continue;
819
820 /* Don't profile functions produced for builtin stuff. */
821 if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
822 continue;
823
824 node->set_const_flag (false, false);
825 node->set_pure_flag (false, false);
826 }
827
828 /* Update call statements and rebuild the cgraph. */
829 FOR_EACH_DEFINED_FUNCTION (node)
830 {
831 basic_block bb;
832
833 if (!gimple_has_body_p (node->decl)
834 || !(!node->clone_of
835 || node->decl != node->clone_of->decl))
836 continue;
837
838 /* Don't profile functions produced for builtin stuff. */
839 if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
840 continue;
841
842 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
843
844 FOR_EACH_BB_FN (bb, cfun)
845 {
846 gimple_stmt_iterator gsi;
847 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
848 {
849 gimple *stmt = gsi_stmt (gsi);
850 if (is_gimple_call (stmt))
851 update_stmt (stmt);
852 }
853 }
854
855 /* re-merge split blocks. */
856 cleanup_tree_cfg ();
857 update_ssa (TODO_update_ssa);
858
859 cgraph_edge::rebuild_edges ();
860
861 pop_cfun ();
862 }
863
864 handle_missing_profiles ();
865
866 del_node_map ();
867 return 0;
868 }
869
870 namespace {
871
872 const pass_data pass_data_ipa_tree_profile =
873 {
874 SIMPLE_IPA_PASS, /* type */
875 "profile", /* name */
876 OPTGROUP_NONE, /* optinfo_flags */
877 TV_IPA_PROFILE, /* tv_id */
878 0, /* properties_required */
879 0, /* properties_provided */
880 0, /* properties_destroyed */
881 0, /* todo_flags_start */
882 TODO_dump_symtab, /* todo_flags_finish */
883 };
884
885 class pass_ipa_tree_profile : public simple_ipa_opt_pass
886 {
887 public:
pass_ipa_tree_profile(gcc::context * ctxt)888 pass_ipa_tree_profile (gcc::context *ctxt)
889 : simple_ipa_opt_pass (pass_data_ipa_tree_profile, ctxt)
890 {}
891
892 /* opt_pass methods: */
893 virtual bool gate (function *);
execute(function *)894 virtual unsigned int execute (function *) { return tree_profiling (); }
895
896 }; // class pass_ipa_tree_profile
897
898 bool
gate(function *)899 pass_ipa_tree_profile::gate (function *)
900 {
901 /* When profile instrumentation, use or test coverage shall be performed.
902 But for AutoFDO, this there is no instrumentation, thus this pass is
903 disabled. */
904 return (!in_lto_p && !flag_auto_profile
905 && (flag_branch_probabilities || flag_test_coverage
906 || profile_arc_flag));
907 }
908
909 } // anon namespace
910
911 simple_ipa_opt_pass *
make_pass_ipa_tree_profile(gcc::context * ctxt)912 make_pass_ipa_tree_profile (gcc::context *ctxt)
913 {
914 return new pass_ipa_tree_profile (ctxt);
915 }
916
917 #include "gt-tree-profile.h"
918