1 /* Calculate branch probabilities, and basic block execution counts. 2 Copyright (C) 1990-2019 Free Software Foundation, Inc. 3 Contributed by James E. Wilson, UC Berkeley/Cygnus Support; 4 based on some ideas from Dain Samples of UC Berkeley. 5 Further mangling by Bob Manson, Cygnus Support. 6 Converted to use trees by Dale Johannesen, Apple Computer. 7 8 This file is part of GCC. 9 10 GCC is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free 12 Software Foundation; either version 3, or (at your option) any later 13 version. 14 15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY 16 WARRANTY; without even the implied warranty of MERCHANTABILITY or 17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 18 for more details. 19 20 You should have received a copy of the GNU General Public License 21 along with GCC; see the file COPYING3. If not see 22 <http://www.gnu.org/licenses/>. */ 23 24 /* Generate basic block profile instrumentation and auxiliary files. 25 Tree-based version. See profile.c for overview. */ 26 27 #include "config.h" 28 #include "system.h" 29 #include "coretypes.h" 30 #include "memmodel.h" 31 #include "backend.h" 32 #include "target.h" 33 #include "tree.h" 34 #include "gimple.h" 35 #include "cfghooks.h" 36 #include "tree-pass.h" 37 #include "ssa.h" 38 #include "cgraph.h" 39 #include "coverage.h" 40 #include "diagnostic-core.h" 41 #include "fold-const.h" 42 #include "varasm.h" 43 #include "tree-nested.h" 44 #include "gimplify.h" 45 #include "gimple-iterator.h" 46 #include "gimplify-me.h" 47 #include "tree-cfg.h" 48 #include "tree-into-ssa.h" 49 #include "value-prof.h" 50 #include "profile.h" 51 #include "tree-cfgcleanup.h" 52 #include "params.h" 53 #include "stringpool.h" 54 #include "attribs.h" 55 #include "tree-pretty-print.h" 56 #include "langhooks.h" 57 #include "stor-layout.h" 58 #include "xregex.h" 59 60 static GTY(()) tree gcov_type_node; 61 static GTY(()) tree tree_interval_profiler_fn; 62 static GTY(()) tree tree_pow2_profiler_fn; 63 static GTY(()) tree tree_one_value_profiler_fn; 64 static GTY(()) tree tree_indirect_call_profiler_fn; 65 static GTY(()) tree tree_average_profiler_fn; 66 static GTY(()) tree tree_ior_profiler_fn; 67 static GTY(()) tree tree_time_profiler_counter; 68 69 70 static GTY(()) tree ic_tuple_var; 71 static GTY(()) tree ic_tuple_counters_field; 72 static GTY(()) tree ic_tuple_callee_field; 73 74 /* Do initialization work for the edge profiler. */ 75 76 /* Add code: 77 __thread gcov* __gcov_indirect_call_counters; // pointer to actual counter 78 __thread void* __gcov_indirect_call_callee; // actual callee address 79 __thread int __gcov_function_counter; // time profiler function counter 80 */ 81 static void 82 init_ic_make_global_vars (void) 83 { 84 tree gcov_type_ptr; 85 86 gcov_type_ptr = build_pointer_type (get_gcov_type ()); 87 88 tree tuple_type = lang_hooks.types.make_type (RECORD_TYPE); 89 90 /* callee */ 91 ic_tuple_callee_field = build_decl (BUILTINS_LOCATION, FIELD_DECL, NULL_TREE, 92 ptr_type_node); 93 94 /* counters */ 95 ic_tuple_counters_field = build_decl (BUILTINS_LOCATION, FIELD_DECL, 96 NULL_TREE, gcov_type_ptr); 97 DECL_CHAIN (ic_tuple_counters_field) = ic_tuple_callee_field; 98 99 finish_builtin_struct (tuple_type, "indirect_call_tuple", 100 ic_tuple_counters_field, NULL_TREE); 101 102 ic_tuple_var 103 = build_decl (UNKNOWN_LOCATION, VAR_DECL, 104 get_identifier ( 105 (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) ? 106 "__gcov_indirect_call_topn" : 107 "__gcov_indirect_call")), 108 tuple_type); 109 TREE_PUBLIC (ic_tuple_var) = 1; 110 DECL_ARTIFICIAL (ic_tuple_var) = 1; 111 DECL_INITIAL (ic_tuple_var) = NULL; 112 DECL_EXTERNAL (ic_tuple_var) = 1; 113 if (targetm.have_tls) 114 set_decl_tls_model (ic_tuple_var, decl_default_tls_model (ic_tuple_var)); 115 } 116 117 /* Create the type and function decls for the interface with gcov. */ 118 119 void 120 gimple_init_gcov_profiler (void) 121 { 122 tree interval_profiler_fn_type; 123 tree pow2_profiler_fn_type; 124 tree one_value_profiler_fn_type; 125 tree gcov_type_ptr; 126 tree ic_profiler_fn_type; 127 tree average_profiler_fn_type; 128 const char *profiler_fn_name; 129 const char *fn_name; 130 131 if (!gcov_type_node) 132 { 133 const char *fn_suffix 134 = flag_profile_update == PROFILE_UPDATE_ATOMIC ? "_atomic" : ""; 135 136 gcov_type_node = get_gcov_type (); 137 gcov_type_ptr = build_pointer_type (gcov_type_node); 138 139 /* void (*) (gcov_type *, gcov_type, int, unsigned) */ 140 interval_profiler_fn_type 141 = build_function_type_list (void_type_node, 142 gcov_type_ptr, gcov_type_node, 143 integer_type_node, 144 unsigned_type_node, NULL_TREE); 145 fn_name = concat ("__gcov_interval_profiler", fn_suffix, NULL); 146 tree_interval_profiler_fn = build_fn_decl (fn_name, 147 interval_profiler_fn_type); 148 free (CONST_CAST (char *, fn_name)); 149 TREE_NOTHROW (tree_interval_profiler_fn) = 1; 150 DECL_ATTRIBUTES (tree_interval_profiler_fn) 151 = tree_cons (get_identifier ("leaf"), NULL, 152 DECL_ATTRIBUTES (tree_interval_profiler_fn)); 153 154 /* void (*) (gcov_type *, gcov_type) */ 155 pow2_profiler_fn_type 156 = build_function_type_list (void_type_node, 157 gcov_type_ptr, gcov_type_node, 158 NULL_TREE); 159 fn_name = concat ("__gcov_pow2_profiler", fn_suffix, NULL); 160 tree_pow2_profiler_fn = build_fn_decl (fn_name, pow2_profiler_fn_type); 161 free (CONST_CAST (char *, fn_name)); 162 TREE_NOTHROW (tree_pow2_profiler_fn) = 1; 163 DECL_ATTRIBUTES (tree_pow2_profiler_fn) 164 = tree_cons (get_identifier ("leaf"), NULL, 165 DECL_ATTRIBUTES (tree_pow2_profiler_fn)); 166 167 /* void (*) (gcov_type *, gcov_type) */ 168 one_value_profiler_fn_type 169 = build_function_type_list (void_type_node, 170 gcov_type_ptr, gcov_type_node, 171 NULL_TREE); 172 fn_name = concat ("__gcov_one_value_profiler", fn_suffix, NULL); 173 tree_one_value_profiler_fn = build_fn_decl (fn_name, 174 one_value_profiler_fn_type); 175 free (CONST_CAST (char *, fn_name)); 176 TREE_NOTHROW (tree_one_value_profiler_fn) = 1; 177 DECL_ATTRIBUTES (tree_one_value_profiler_fn) 178 = tree_cons (get_identifier ("leaf"), NULL, 179 DECL_ATTRIBUTES (tree_one_value_profiler_fn)); 180 181 init_ic_make_global_vars (); 182 183 /* void (*) (gcov_type, void *) */ 184 ic_profiler_fn_type 185 = build_function_type_list (void_type_node, 186 gcov_type_node, 187 ptr_type_node, 188 NULL_TREE); 189 profiler_fn_name = "__gcov_indirect_call_profiler_v3"; 190 if (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE)) 191 profiler_fn_name = "__gcov_indirect_call_topn_profiler"; 192 193 tree_indirect_call_profiler_fn 194 = build_fn_decl (profiler_fn_name, ic_profiler_fn_type); 195 196 TREE_NOTHROW (tree_indirect_call_profiler_fn) = 1; 197 DECL_ATTRIBUTES (tree_indirect_call_profiler_fn) 198 = tree_cons (get_identifier ("leaf"), NULL, 199 DECL_ATTRIBUTES (tree_indirect_call_profiler_fn)); 200 201 tree_time_profiler_counter 202 = build_decl (UNKNOWN_LOCATION, VAR_DECL, 203 get_identifier ("__gcov_time_profiler_counter"), 204 get_gcov_type ()); 205 TREE_PUBLIC (tree_time_profiler_counter) = 1; 206 DECL_EXTERNAL (tree_time_profiler_counter) = 1; 207 TREE_STATIC (tree_time_profiler_counter) = 1; 208 DECL_ARTIFICIAL (tree_time_profiler_counter) = 1; 209 DECL_INITIAL (tree_time_profiler_counter) = NULL; 210 211 /* void (*) (gcov_type *, gcov_type) */ 212 average_profiler_fn_type 213 = build_function_type_list (void_type_node, 214 gcov_type_ptr, gcov_type_node, NULL_TREE); 215 fn_name = concat ("__gcov_average_profiler", fn_suffix, NULL); 216 tree_average_profiler_fn = build_fn_decl (fn_name, 217 average_profiler_fn_type); 218 free (CONST_CAST (char *, fn_name)); 219 TREE_NOTHROW (tree_average_profiler_fn) = 1; 220 DECL_ATTRIBUTES (tree_average_profiler_fn) 221 = tree_cons (get_identifier ("leaf"), NULL, 222 DECL_ATTRIBUTES (tree_average_profiler_fn)); 223 fn_name = concat ("__gcov_ior_profiler", fn_suffix, NULL); 224 tree_ior_profiler_fn = build_fn_decl (fn_name, average_profiler_fn_type); 225 free (CONST_CAST (char *, fn_name)); 226 TREE_NOTHROW (tree_ior_profiler_fn) = 1; 227 DECL_ATTRIBUTES (tree_ior_profiler_fn) 228 = tree_cons (get_identifier ("leaf"), NULL, 229 DECL_ATTRIBUTES (tree_ior_profiler_fn)); 230 231 /* LTO streamer needs assembler names. Because we create these decls 232 late, we need to initialize them by hand. */ 233 DECL_ASSEMBLER_NAME (tree_interval_profiler_fn); 234 DECL_ASSEMBLER_NAME (tree_pow2_profiler_fn); 235 DECL_ASSEMBLER_NAME (tree_one_value_profiler_fn); 236 DECL_ASSEMBLER_NAME (tree_indirect_call_profiler_fn); 237 DECL_ASSEMBLER_NAME (tree_average_profiler_fn); 238 DECL_ASSEMBLER_NAME (tree_ior_profiler_fn); 239 } 240 } 241 242 /* Output instructions as GIMPLE trees to increment the edge 243 execution count, and insert them on E. We rely on 244 gsi_insert_on_edge to preserve the order. */ 245 246 void 247 gimple_gen_edge_profiler (int edgeno, edge e) 248 { 249 tree one; 250 251 one = build_int_cst (gcov_type_node, 1); 252 253 if (flag_profile_update == PROFILE_UPDATE_ATOMIC) 254 { 255 /* __atomic_fetch_add (&counter, 1, MEMMODEL_RELAXED); */ 256 tree addr = tree_coverage_counter_addr (GCOV_COUNTER_ARCS, edgeno); 257 tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32 258 ? BUILT_IN_ATOMIC_FETCH_ADD_8: 259 BUILT_IN_ATOMIC_FETCH_ADD_4); 260 gcall *stmt = gimple_build_call (f, 3, addr, one, 261 build_int_cst (integer_type_node, 262 MEMMODEL_RELAXED)); 263 gsi_insert_on_edge (e, stmt); 264 } 265 else 266 { 267 tree ref = tree_coverage_counter_ref (GCOV_COUNTER_ARCS, edgeno); 268 tree gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node, 269 NULL, "PROF_edge_counter"); 270 gassign *stmt1 = gimple_build_assign (gcov_type_tmp_var, ref); 271 gcov_type_tmp_var = make_temp_ssa_name (gcov_type_node, 272 NULL, "PROF_edge_counter"); 273 gassign *stmt2 = gimple_build_assign (gcov_type_tmp_var, PLUS_EXPR, 274 gimple_assign_lhs (stmt1), one); 275 gassign *stmt3 = gimple_build_assign (unshare_expr (ref), 276 gimple_assign_lhs (stmt2)); 277 gsi_insert_on_edge (e, stmt1); 278 gsi_insert_on_edge (e, stmt2); 279 gsi_insert_on_edge (e, stmt3); 280 } 281 } 282 283 /* Emits code to get VALUE to instrument at GSI, and returns the 284 variable containing the value. */ 285 286 static tree 287 prepare_instrumented_value (gimple_stmt_iterator *gsi, histogram_value value) 288 { 289 tree val = value->hvalue.value; 290 if (POINTER_TYPE_P (TREE_TYPE (val))) 291 val = fold_convert (build_nonstandard_integer_type 292 (TYPE_PRECISION (TREE_TYPE (val)), 1), val); 293 return force_gimple_operand_gsi (gsi, fold_convert (gcov_type_node, val), 294 true, NULL_TREE, true, GSI_SAME_STMT); 295 } 296 297 /* Output instructions as GIMPLE trees to increment the interval histogram 298 counter. VALUE is the expression whose value is profiled. TAG is the 299 tag of the section for counters, BASE is offset of the counter position. */ 300 301 void 302 gimple_gen_interval_profiler (histogram_value value, unsigned tag, unsigned base) 303 { 304 gimple *stmt = value->hvalue.stmt; 305 gimple_stmt_iterator gsi = gsi_for_stmt (stmt); 306 tree ref = tree_coverage_counter_ref (tag, base), ref_ptr; 307 gcall *call; 308 tree val; 309 tree start = build_int_cst_type (integer_type_node, 310 value->hdata.intvl.int_start); 311 tree steps = build_int_cst_type (unsigned_type_node, 312 value->hdata.intvl.steps); 313 314 ref_ptr = force_gimple_operand_gsi (&gsi, 315 build_addr (ref), 316 true, NULL_TREE, true, GSI_SAME_STMT); 317 val = prepare_instrumented_value (&gsi, value); 318 call = gimple_build_call (tree_interval_profiler_fn, 4, 319 ref_ptr, val, start, steps); 320 gsi_insert_before (&gsi, call, GSI_NEW_STMT); 321 } 322 323 /* Output instructions as GIMPLE trees to increment the power of two histogram 324 counter. VALUE is the expression whose value is profiled. TAG is the tag 325 of the section for counters, BASE is offset of the counter position. */ 326 327 void 328 gimple_gen_pow2_profiler (histogram_value value, unsigned tag, unsigned base) 329 { 330 gimple *stmt = value->hvalue.stmt; 331 gimple_stmt_iterator gsi = gsi_for_stmt (stmt); 332 tree ref_ptr = tree_coverage_counter_addr (tag, base); 333 gcall *call; 334 tree val; 335 336 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr, 337 true, NULL_TREE, true, GSI_SAME_STMT); 338 val = prepare_instrumented_value (&gsi, value); 339 call = gimple_build_call (tree_pow2_profiler_fn, 2, ref_ptr, val); 340 gsi_insert_before (&gsi, call, GSI_NEW_STMT); 341 } 342 343 /* Output instructions as GIMPLE trees for code to find the most common value. 344 VALUE is the expression whose value is profiled. TAG is the tag of the 345 section for counters, BASE is offset of the counter position. */ 346 347 void 348 gimple_gen_one_value_profiler (histogram_value value, unsigned tag, unsigned base) 349 { 350 gimple *stmt = value->hvalue.stmt; 351 gimple_stmt_iterator gsi = gsi_for_stmt (stmt); 352 tree ref_ptr = tree_coverage_counter_addr (tag, base); 353 gcall *call; 354 tree val; 355 356 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr, 357 true, NULL_TREE, true, GSI_SAME_STMT); 358 val = prepare_instrumented_value (&gsi, value); 359 call = gimple_build_call (tree_one_value_profiler_fn, 2, ref_ptr, val); 360 gsi_insert_before (&gsi, call, GSI_NEW_STMT); 361 } 362 363 364 /* Output instructions as GIMPLE trees for code to find the most 365 common called function in indirect call. 366 VALUE is the call expression whose indirect callee is profiled. 367 TAG is the tag of the section for counters, BASE is offset of the 368 counter position. */ 369 370 void 371 gimple_gen_ic_profiler (histogram_value value, unsigned tag, unsigned base) 372 { 373 tree tmp1; 374 gassign *stmt1, *stmt2, *stmt3; 375 gimple *stmt = value->hvalue.stmt; 376 gimple_stmt_iterator gsi = gsi_for_stmt (stmt); 377 tree ref_ptr = tree_coverage_counter_addr (tag, base); 378 379 if ( (PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) && 380 tag == GCOV_COUNTER_V_INDIR) || 381 (!PARAM_VALUE (PARAM_INDIR_CALL_TOPN_PROFILE) && 382 tag == GCOV_COUNTER_ICALL_TOPNV)) 383 return; 384 385 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr, 386 true, NULL_TREE, true, GSI_SAME_STMT); 387 388 /* Insert code: 389 390 stmt1: __gcov_indirect_call.counters = get_relevant_counter_ptr (); 391 stmt2: tmp1 = (void *) (indirect call argument value) 392 stmt3: __gcov_indirect_call.callee = tmp1; 393 394 Example: 395 f_1 = foo; 396 __gcov_indirect_call.counters = &__gcov4.main[0]; 397 PROF_9 = f_1; 398 __gcov_indirect_call_callee = PROF_9; 399 _4 = f_1 (); 400 */ 401 402 tree gcov_type_ptr = build_pointer_type (get_gcov_type ()); 403 404 tree counter_ref = build3 (COMPONENT_REF, gcov_type_ptr, 405 ic_tuple_var, ic_tuple_counters_field, NULL_TREE); 406 407 stmt1 = gimple_build_assign (counter_ref, ref_ptr); 408 tmp1 = make_temp_ssa_name (ptr_type_node, NULL, "PROF"); 409 stmt2 = gimple_build_assign (tmp1, unshare_expr (value->hvalue.value)); 410 tree callee_ref = build3 (COMPONENT_REF, ptr_type_node, 411 ic_tuple_var, ic_tuple_callee_field, NULL_TREE); 412 stmt3 = gimple_build_assign (callee_ref, tmp1); 413 414 gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT); 415 gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT); 416 gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT); 417 } 418 419 420 /* Output instructions as GIMPLE trees for code to find the most 421 common called function in indirect call. Insert instructions at the 422 beginning of every possible called function. 423 */ 424 425 void 426 gimple_gen_ic_func_profiler (void) 427 { 428 struct cgraph_node * c_node = cgraph_node::get (current_function_decl); 429 gcall *stmt1; 430 tree tree_uid, cur_func, void0; 431 432 if (c_node->only_called_directly_p ()) 433 return; 434 435 gimple_init_gcov_profiler (); 436 437 basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun); 438 basic_block cond_bb = split_edge (single_succ_edge (entry)); 439 basic_block update_bb = split_edge (single_succ_edge (cond_bb)); 440 441 /* We need to do an extra split in order to not create an input 442 for a possible PHI node. */ 443 split_edge (single_succ_edge (update_bb)); 444 445 edge true_edge = single_succ_edge (cond_bb); 446 true_edge->flags = EDGE_TRUE_VALUE; 447 448 profile_probability probability; 449 if (DECL_VIRTUAL_P (current_function_decl)) 450 probability = profile_probability::very_likely (); 451 else 452 probability = profile_probability::unlikely (); 453 454 true_edge->probability = probability; 455 edge e = make_edge (cond_bb, single_succ_edge (update_bb)->dest, 456 EDGE_FALSE_VALUE); 457 e->probability = true_edge->probability.invert (); 458 459 /* Insert code: 460 461 if (__gcov_indirect_call_callee != NULL) 462 __gcov_indirect_call_profiler_v3 (profile_id, ¤t_function_decl); 463 464 The function __gcov_indirect_call_profiler_v3 is responsible for 465 resetting __gcov_indirect_call_callee to NULL. */ 466 467 gimple_stmt_iterator gsi = gsi_start_bb (cond_bb); 468 void0 = build_int_cst (ptr_type_node, 0); 469 470 tree callee_ref = build3 (COMPONENT_REF, ptr_type_node, 471 ic_tuple_var, ic_tuple_callee_field, NULL_TREE); 472 473 tree ref = force_gimple_operand_gsi (&gsi, callee_ref, true, NULL_TREE, 474 true, GSI_SAME_STMT); 475 476 gcond *cond = gimple_build_cond (NE_EXPR, ref, 477 void0, NULL, NULL); 478 gsi_insert_before (&gsi, cond, GSI_NEW_STMT); 479 480 gsi = gsi_after_labels (update_bb); 481 482 cur_func = force_gimple_operand_gsi (&gsi, 483 build_addr (current_function_decl), 484 true, NULL_TREE, 485 true, GSI_SAME_STMT); 486 tree_uid = build_int_cst 487 (gcov_type_node, 488 cgraph_node::get (current_function_decl)->profile_id); 489 stmt1 = gimple_build_call (tree_indirect_call_profiler_fn, 2, 490 tree_uid, cur_func); 491 gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT); 492 } 493 494 /* Output instructions as GIMPLE tree at the beginning for each function. 495 TAG is the tag of the section for counters, BASE is offset of the 496 counter position and GSI is the iterator we place the counter. */ 497 498 void 499 gimple_gen_time_profiler (unsigned tag, unsigned base) 500 { 501 tree type = get_gcov_type (); 502 basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun); 503 basic_block cond_bb = split_edge (single_succ_edge (entry)); 504 basic_block update_bb = split_edge (single_succ_edge (cond_bb)); 505 506 /* We need to do an extra split in order to not create an input 507 for a possible PHI node. */ 508 split_edge (single_succ_edge (update_bb)); 509 510 edge true_edge = single_succ_edge (cond_bb); 511 true_edge->flags = EDGE_TRUE_VALUE; 512 true_edge->probability = profile_probability::unlikely (); 513 edge e 514 = make_edge (cond_bb, single_succ_edge (update_bb)->dest, EDGE_FALSE_VALUE); 515 e->probability = true_edge->probability.invert (); 516 517 gimple_stmt_iterator gsi = gsi_start_bb (cond_bb); 518 tree original_ref = tree_coverage_counter_ref (tag, base); 519 tree ref = force_gimple_operand_gsi (&gsi, original_ref, true, NULL_TREE, 520 true, GSI_SAME_STMT); 521 tree one = build_int_cst (type, 1); 522 523 /* Emit: if (counters[0] != 0). */ 524 gcond *cond = gimple_build_cond (EQ_EXPR, ref, build_int_cst (type, 0), 525 NULL, NULL); 526 gsi_insert_before (&gsi, cond, GSI_NEW_STMT); 527 528 gsi = gsi_start_bb (update_bb); 529 530 /* Emit: counters[0] = ++__gcov_time_profiler_counter. */ 531 if (flag_profile_update == PROFILE_UPDATE_ATOMIC) 532 { 533 tree ptr = make_temp_ssa_name (build_pointer_type (type), NULL, 534 "time_profiler_counter_ptr"); 535 tree addr = build1 (ADDR_EXPR, TREE_TYPE (ptr), 536 tree_time_profiler_counter); 537 gassign *assign = gimple_build_assign (ptr, NOP_EXPR, addr); 538 gsi_insert_before (&gsi, assign, GSI_NEW_STMT); 539 tree f = builtin_decl_explicit (LONG_LONG_TYPE_SIZE > 32 540 ? BUILT_IN_ATOMIC_ADD_FETCH_8: 541 BUILT_IN_ATOMIC_ADD_FETCH_4); 542 gcall *stmt = gimple_build_call (f, 3, ptr, one, 543 build_int_cst (integer_type_node, 544 MEMMODEL_RELAXED)); 545 tree result_type = TREE_TYPE (TREE_TYPE (f)); 546 tree tmp = make_temp_ssa_name (result_type, NULL, "time_profile"); 547 gimple_set_lhs (stmt, tmp); 548 gsi_insert_after (&gsi, stmt, GSI_NEW_STMT); 549 tmp = make_temp_ssa_name (type, NULL, "time_profile"); 550 assign = gimple_build_assign (tmp, NOP_EXPR, 551 gimple_call_lhs (stmt)); 552 gsi_insert_after (&gsi, assign, GSI_NEW_STMT); 553 assign = gimple_build_assign (original_ref, tmp); 554 gsi_insert_after (&gsi, assign, GSI_NEW_STMT); 555 } 556 else 557 { 558 tree tmp = make_temp_ssa_name (type, NULL, "time_profile"); 559 gassign *assign = gimple_build_assign (tmp, tree_time_profiler_counter); 560 gsi_insert_before (&gsi, assign, GSI_NEW_STMT); 561 562 tmp = make_temp_ssa_name (type, NULL, "time_profile"); 563 assign = gimple_build_assign (tmp, PLUS_EXPR, gimple_assign_lhs (assign), 564 one); 565 gsi_insert_after (&gsi, assign, GSI_NEW_STMT); 566 assign = gimple_build_assign (original_ref, tmp); 567 gsi_insert_after (&gsi, assign, GSI_NEW_STMT); 568 assign = gimple_build_assign (tree_time_profiler_counter, tmp); 569 gsi_insert_after (&gsi, assign, GSI_NEW_STMT); 570 } 571 } 572 573 /* Output instructions as GIMPLE trees to increment the average histogram 574 counter. VALUE is the expression whose value is profiled. TAG is the 575 tag of the section for counters, BASE is offset of the counter position. */ 576 577 void 578 gimple_gen_average_profiler (histogram_value value, unsigned tag, unsigned base) 579 { 580 gimple *stmt = value->hvalue.stmt; 581 gimple_stmt_iterator gsi = gsi_for_stmt (stmt); 582 tree ref_ptr = tree_coverage_counter_addr (tag, base); 583 gcall *call; 584 tree val; 585 586 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr, 587 true, NULL_TREE, 588 true, GSI_SAME_STMT); 589 val = prepare_instrumented_value (&gsi, value); 590 call = gimple_build_call (tree_average_profiler_fn, 2, ref_ptr, val); 591 gsi_insert_before (&gsi, call, GSI_NEW_STMT); 592 } 593 594 /* Output instructions as GIMPLE trees to increment the ior histogram 595 counter. VALUE is the expression whose value is profiled. TAG is the 596 tag of the section for counters, BASE is offset of the counter position. */ 597 598 void 599 gimple_gen_ior_profiler (histogram_value value, unsigned tag, unsigned base) 600 { 601 gimple *stmt = value->hvalue.stmt; 602 gimple_stmt_iterator gsi = gsi_for_stmt (stmt); 603 tree ref_ptr = tree_coverage_counter_addr (tag, base); 604 gcall *call; 605 tree val; 606 607 ref_ptr = force_gimple_operand_gsi (&gsi, ref_ptr, 608 true, NULL_TREE, true, GSI_SAME_STMT); 609 val = prepare_instrumented_value (&gsi, value); 610 call = gimple_build_call (tree_ior_profiler_fn, 2, ref_ptr, val); 611 gsi_insert_before (&gsi, call, GSI_NEW_STMT); 612 } 613 614 static vec<regex_t> profile_filter_files; 615 static vec<regex_t> profile_exclude_files; 616 617 /* Parse list of provided REGEX (separated with semi-collon) and 618 create expressions (of type regex_t) and save them into V vector. 619 If there is a regular expression parsing error, error message is 620 printed for FLAG_NAME. */ 621 622 static void 623 parse_profile_filter (const char *regex, vec<regex_t> *v, 624 const char *flag_name) 625 { 626 v->create (4); 627 if (regex != NULL) 628 { 629 char *str = xstrdup (regex); 630 for (char *p = strtok (str, ";"); p != NULL; p = strtok (NULL, ";")) 631 { 632 regex_t r; 633 if (regcomp (&r, p, REG_EXTENDED | REG_NOSUB) != 0) 634 { 635 error ("invalid regular expression %qs in %qs", 636 p, flag_name); 637 return; 638 } 639 640 v->safe_push (r); 641 } 642 } 643 } 644 645 /* Parse values of -fprofile-filter-files and -fprofile-exclude-files 646 options. */ 647 648 static void 649 parse_profile_file_filtering () 650 { 651 parse_profile_filter (flag_profile_filter_files, &profile_filter_files, 652 "-fprofile-filter-files"); 653 parse_profile_filter (flag_profile_exclude_files, &profile_exclude_files, 654 "-fprofile-exclude-files"); 655 } 656 657 /* Parse vectors of regular expressions. */ 658 659 static void 660 release_profile_file_filtering () 661 { 662 profile_filter_files.release (); 663 profile_exclude_files.release (); 664 } 665 666 /* Return true when FILENAME should be instrumented based on 667 -fprofile-filter-files and -fprofile-exclude-files options. */ 668 669 static bool 670 include_source_file_for_profile (const char *filename) 671 { 672 /* First check whether file is included in flag_profile_exclude_files. */ 673 for (unsigned i = 0; i < profile_exclude_files.length (); i++) 674 if (regexec (&profile_exclude_files[i], 675 filename, 0, NULL, 0) == REG_NOERROR) 676 return false; 677 678 /* For non-empty flag_profile_filter_files include only files matching a 679 regex in the flag. */ 680 if (profile_filter_files.is_empty ()) 681 return true; 682 683 for (unsigned i = 0; i < profile_filter_files.length (); i++) 684 if (regexec (&profile_filter_files[i], filename, 0, NULL, 0) == REG_NOERROR) 685 return true; 686 687 return false; 688 } 689 690 #ifndef HAVE_sync_compare_and_swapsi 691 #define HAVE_sync_compare_and_swapsi 0 692 #endif 693 #ifndef HAVE_atomic_compare_and_swapsi 694 #define HAVE_atomic_compare_and_swapsi 0 695 #endif 696 697 #ifndef HAVE_sync_compare_and_swapdi 698 #define HAVE_sync_compare_and_swapdi 0 699 #endif 700 #ifndef HAVE_atomic_compare_and_swapdi 701 #define HAVE_atomic_compare_and_swapdi 0 702 #endif 703 704 /* Profile all functions in the callgraph. */ 705 706 static unsigned int 707 tree_profiling (void) 708 { 709 struct cgraph_node *node; 710 711 /* Verify whether we can utilize atomic update operations. */ 712 bool can_support_atomic = false; 713 unsigned HOST_WIDE_INT gcov_type_size 714 = tree_to_uhwi (TYPE_SIZE_UNIT (get_gcov_type ())); 715 if (gcov_type_size == 4) 716 can_support_atomic 717 = HAVE_sync_compare_and_swapsi || HAVE_atomic_compare_and_swapsi; 718 else if (gcov_type_size == 8) 719 can_support_atomic 720 = HAVE_sync_compare_and_swapdi || HAVE_atomic_compare_and_swapdi; 721 722 if (flag_profile_update == PROFILE_UPDATE_ATOMIC 723 && !can_support_atomic) 724 { 725 warning (0, "target does not support atomic profile update, " 726 "single mode is selected"); 727 flag_profile_update = PROFILE_UPDATE_SINGLE; 728 } 729 else if (flag_profile_update == PROFILE_UPDATE_PREFER_ATOMIC) 730 flag_profile_update = can_support_atomic 731 ? PROFILE_UPDATE_ATOMIC : PROFILE_UPDATE_SINGLE; 732 733 /* This is a small-ipa pass that gets called only once, from 734 cgraphunit.c:ipa_passes(). */ 735 gcc_assert (symtab->state == IPA_SSA); 736 737 init_node_map (true); 738 parse_profile_file_filtering (); 739 740 FOR_EACH_DEFINED_FUNCTION (node) 741 { 742 bool thunk = false; 743 if (!gimple_has_body_p (node->decl) && !node->thunk.thunk_p) 744 continue; 745 746 /* Don't profile functions produced for builtin stuff. */ 747 if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION) 748 continue; 749 750 if (lookup_attribute ("no_profile_instrument_function", 751 DECL_ATTRIBUTES (node->decl))) 752 continue; 753 /* Do not instrument extern inline functions when testing coverage. 754 While this is not perfectly consistent (early inlined extern inlines 755 will get acocunted), testsuite expects that. */ 756 if (DECL_EXTERNAL (node->decl) 757 && flag_test_coverage) 758 continue; 759 760 const char *file = LOCATION_FILE (DECL_SOURCE_LOCATION (node->decl)); 761 if (!include_source_file_for_profile (file)) 762 continue; 763 764 if (node->thunk.thunk_p) 765 { 766 /* We cannot expand variadic thunks to Gimple. */ 767 if (stdarg_p (TREE_TYPE (node->decl))) 768 continue; 769 thunk = true; 770 /* When generate profile, expand thunk to gimple so it can be 771 instrumented same way as other functions. */ 772 if (profile_arc_flag) 773 node->expand_thunk (false, true); 774 /* Read cgraph profile but keep function as thunk at profile-use 775 time. */ 776 else 777 { 778 read_thunk_profile (node); 779 continue; 780 } 781 } 782 783 push_cfun (DECL_STRUCT_FUNCTION (node->decl)); 784 785 if (dump_file) 786 dump_function_header (dump_file, cfun->decl, dump_flags); 787 788 /* Local pure-const may imply need to fixup the cfg. */ 789 if (gimple_has_body_p (node->decl) 790 && (execute_fixup_cfg () & TODO_cleanup_cfg)) 791 cleanup_tree_cfg (); 792 793 branch_prob (thunk); 794 795 if (! flag_branch_probabilities 796 && flag_profile_values) 797 gimple_gen_ic_func_profiler (); 798 799 if (flag_branch_probabilities 800 && !thunk 801 && flag_profile_values 802 && flag_value_profile_transformations) 803 gimple_value_profile_transformations (); 804 805 /* The above could hose dominator info. Currently there is 806 none coming in, this is a safety valve. It should be 807 easy to adjust it, if and when there is some. */ 808 free_dominance_info (CDI_DOMINATORS); 809 free_dominance_info (CDI_POST_DOMINATORS); 810 pop_cfun (); 811 } 812 813 release_profile_file_filtering (); 814 815 /* Drop pure/const flags from instrumented functions. */ 816 if (profile_arc_flag || flag_test_coverage) 817 FOR_EACH_DEFINED_FUNCTION (node) 818 { 819 if (!gimple_has_body_p (node->decl) 820 || !(!node->clone_of 821 || node->decl != node->clone_of->decl)) 822 continue; 823 824 /* Don't profile functions produced for builtin stuff. */ 825 if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION) 826 continue; 827 828 node->set_const_flag (false, false); 829 node->set_pure_flag (false, false); 830 } 831 832 /* Update call statements and rebuild the cgraph. */ 833 FOR_EACH_DEFINED_FUNCTION (node) 834 { 835 basic_block bb; 836 837 if (!gimple_has_body_p (node->decl) 838 || !(!node->clone_of 839 || node->decl != node->clone_of->decl)) 840 continue; 841 842 /* Don't profile functions produced for builtin stuff. */ 843 if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION) 844 continue; 845 846 push_cfun (DECL_STRUCT_FUNCTION (node->decl)); 847 848 FOR_EACH_BB_FN (bb, cfun) 849 { 850 gimple_stmt_iterator gsi; 851 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 852 { 853 gimple *stmt = gsi_stmt (gsi); 854 if (is_gimple_call (stmt)) 855 update_stmt (stmt); 856 } 857 } 858 859 /* re-merge split blocks. */ 860 cleanup_tree_cfg (); 861 update_ssa (TODO_update_ssa); 862 863 cgraph_edge::rebuild_edges (); 864 865 pop_cfun (); 866 } 867 868 handle_missing_profiles (); 869 870 del_node_map (); 871 return 0; 872 } 873 874 namespace { 875 876 const pass_data pass_data_ipa_tree_profile = 877 { 878 SIMPLE_IPA_PASS, /* type */ 879 "profile", /* name */ 880 OPTGROUP_NONE, /* optinfo_flags */ 881 TV_IPA_PROFILE, /* tv_id */ 882 0, /* properties_required */ 883 0, /* properties_provided */ 884 0, /* properties_destroyed */ 885 0, /* todo_flags_start */ 886 TODO_dump_symtab, /* todo_flags_finish */ 887 }; 888 889 class pass_ipa_tree_profile : public simple_ipa_opt_pass 890 { 891 public: 892 pass_ipa_tree_profile (gcc::context *ctxt) 893 : simple_ipa_opt_pass (pass_data_ipa_tree_profile, ctxt) 894 {} 895 896 /* opt_pass methods: */ 897 virtual bool gate (function *); 898 virtual unsigned int execute (function *) { return tree_profiling (); } 899 900 }; // class pass_ipa_tree_profile 901 902 bool 903 pass_ipa_tree_profile::gate (function *) 904 { 905 /* When profile instrumentation, use or test coverage shall be performed. 906 But for AutoFDO, this there is no instrumentation, thus this pass is 907 diabled. */ 908 return (!in_lto_p && !flag_auto_profile 909 && (flag_branch_probabilities || flag_test_coverage 910 || profile_arc_flag)); 911 } 912 913 } // anon namespace 914 915 simple_ipa_opt_pass * 916 make_pass_ipa_tree_profile (gcc::context *ctxt) 917 { 918 return new pass_ipa_tree_profile (ctxt); 919 } 920 921 #include "gt-tree-profile.h" 922