1 /* OMP constructs' SIMD clone supporting code. 2 3 Copyright (C) 2005-2019 Free Software Foundation, Inc. 4 5 This file is part of GCC. 6 7 GCC is free software; you can redistribute it and/or modify it under 8 the terms of the GNU General Public License as published by the Free 9 Software Foundation; either version 3, or (at your option) any later 10 version. 11 12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY 13 WARRANTY; without even the implied warranty of MERCHANTABILITY or 14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 15 for more details. 16 17 You should have received a copy of the GNU General Public License 18 along with GCC; see the file COPYING3. If not see 19 <http://www.gnu.org/licenses/>. */ 20 21 #include "config.h" 22 #include "system.h" 23 #include "coretypes.h" 24 #include "backend.h" 25 #include "target.h" 26 #include "tree.h" 27 #include "gimple.h" 28 #include "cfghooks.h" 29 #include "alloc-pool.h" 30 #include "tree-pass.h" 31 #include "ssa.h" 32 #include "cgraph.h" 33 #include "pretty-print.h" 34 #include "diagnostic-core.h" 35 #include "fold-const.h" 36 #include "stor-layout.h" 37 #include "cfganal.h" 38 #include "gimplify.h" 39 #include "gimple-iterator.h" 40 #include "gimplify-me.h" 41 #include "gimple-walk.h" 42 #include "langhooks.h" 43 #include "tree-cfg.h" 44 #include "tree-into-ssa.h" 45 #include "tree-dfa.h" 46 #include "cfgloop.h" 47 #include "symbol-summary.h" 48 #include "ipa-param-manipulation.h" 49 #include "tree-eh.h" 50 #include "varasm.h" 51 #include "stringpool.h" 52 #include "attribs.h" 53 #include "omp-simd-clone.h" 54 55 /* Return the number of elements in vector type VECTYPE, which is associated 56 with a SIMD clone. At present these always have a constant length. */ 57 58 static unsigned HOST_WIDE_INT 59 simd_clone_subparts (tree vectype) 60 { 61 return TYPE_VECTOR_SUBPARTS (vectype).to_constant (); 62 } 63 64 /* Allocate a fresh `simd_clone' and return it. NARGS is the number 65 of arguments to reserve space for. */ 66 67 static struct cgraph_simd_clone * 68 simd_clone_struct_alloc (int nargs) 69 { 70 struct cgraph_simd_clone *clone_info; 71 size_t len = (sizeof (struct cgraph_simd_clone) 72 + nargs * sizeof (struct cgraph_simd_clone_arg)); 73 clone_info = (struct cgraph_simd_clone *) 74 ggc_internal_cleared_alloc (len); 75 return clone_info; 76 } 77 78 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */ 79 80 static inline void 81 simd_clone_struct_copy (struct cgraph_simd_clone *to, 82 struct cgraph_simd_clone *from) 83 { 84 memcpy (to, from, (sizeof (struct cgraph_simd_clone) 85 + ((from->nargs - from->inbranch) 86 * sizeof (struct cgraph_simd_clone_arg)))); 87 } 88 89 /* Return vector of parameter types of function FNDECL. This uses 90 TYPE_ARG_TYPES if available, otherwise falls back to types of 91 DECL_ARGUMENTS types. */ 92 93 static vec<tree> 94 simd_clone_vector_of_formal_parm_types (tree fndecl) 95 { 96 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl))) 97 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl)); 98 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl); 99 unsigned int i; 100 tree arg; 101 FOR_EACH_VEC_ELT (args, i, arg) 102 args[i] = TREE_TYPE (args[i]); 103 return args; 104 } 105 106 /* Given a simd function in NODE, extract the simd specific 107 information from the OMP clauses passed in CLAUSES, and return 108 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED 109 is set to TRUE if the `inbranch' or `notinbranch' clause specified, 110 otherwise set to FALSE. */ 111 112 static struct cgraph_simd_clone * 113 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses, 114 bool *inbranch_specified) 115 { 116 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl); 117 tree t; 118 int n; 119 *inbranch_specified = false; 120 121 n = args.length (); 122 if (n > 0 && args.last () == void_type_node) 123 n--; 124 125 /* Allocate one more than needed just in case this is an in-branch 126 clone which will require a mask argument. */ 127 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1); 128 clone_info->nargs = n; 129 130 if (!clauses) 131 goto out; 132 133 clauses = TREE_VALUE (clauses); 134 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE) 135 goto out; 136 137 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t)) 138 { 139 switch (OMP_CLAUSE_CODE (t)) 140 { 141 case OMP_CLAUSE_INBRANCH: 142 clone_info->inbranch = 1; 143 *inbranch_specified = true; 144 break; 145 case OMP_CLAUSE_NOTINBRANCH: 146 clone_info->inbranch = 0; 147 *inbranch_specified = true; 148 break; 149 case OMP_CLAUSE_SIMDLEN: 150 clone_info->simdlen 151 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t)); 152 break; 153 case OMP_CLAUSE_LINEAR: 154 { 155 tree decl = OMP_CLAUSE_DECL (t); 156 tree step = OMP_CLAUSE_LINEAR_STEP (t); 157 int argno = TREE_INT_CST_LOW (decl); 158 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t)) 159 { 160 enum cgraph_simd_clone_arg_type arg_type; 161 if (TREE_CODE (args[argno]) == REFERENCE_TYPE) 162 switch (OMP_CLAUSE_LINEAR_KIND (t)) 163 { 164 case OMP_CLAUSE_LINEAR_REF: 165 arg_type 166 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP; 167 break; 168 case OMP_CLAUSE_LINEAR_UVAL: 169 arg_type 170 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP; 171 break; 172 case OMP_CLAUSE_LINEAR_VAL: 173 case OMP_CLAUSE_LINEAR_DEFAULT: 174 arg_type 175 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP; 176 break; 177 default: 178 gcc_unreachable (); 179 } 180 else 181 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP; 182 clone_info->args[argno].arg_type = arg_type; 183 clone_info->args[argno].linear_step = tree_to_shwi (step); 184 gcc_assert (clone_info->args[argno].linear_step >= 0 185 && clone_info->args[argno].linear_step < n); 186 } 187 else 188 { 189 if (POINTER_TYPE_P (args[argno])) 190 step = fold_convert (ssizetype, step); 191 if (!tree_fits_shwi_p (step)) 192 { 193 warning_at (OMP_CLAUSE_LOCATION (t), 0, 194 "ignoring large linear step"); 195 args.release (); 196 return NULL; 197 } 198 else if (integer_zerop (step)) 199 { 200 warning_at (OMP_CLAUSE_LOCATION (t), 0, 201 "ignoring zero linear step"); 202 args.release (); 203 return NULL; 204 } 205 else 206 { 207 enum cgraph_simd_clone_arg_type arg_type; 208 if (TREE_CODE (args[argno]) == REFERENCE_TYPE) 209 switch (OMP_CLAUSE_LINEAR_KIND (t)) 210 { 211 case OMP_CLAUSE_LINEAR_REF: 212 arg_type 213 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP; 214 break; 215 case OMP_CLAUSE_LINEAR_UVAL: 216 arg_type 217 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP; 218 break; 219 case OMP_CLAUSE_LINEAR_VAL: 220 case OMP_CLAUSE_LINEAR_DEFAULT: 221 arg_type 222 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP; 223 break; 224 default: 225 gcc_unreachable (); 226 } 227 else 228 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP; 229 clone_info->args[argno].arg_type = arg_type; 230 clone_info->args[argno].linear_step = tree_to_shwi (step); 231 } 232 } 233 break; 234 } 235 case OMP_CLAUSE_UNIFORM: 236 { 237 tree decl = OMP_CLAUSE_DECL (t); 238 int argno = tree_to_uhwi (decl); 239 clone_info->args[argno].arg_type 240 = SIMD_CLONE_ARG_TYPE_UNIFORM; 241 break; 242 } 243 case OMP_CLAUSE_ALIGNED: 244 { 245 /* Ignore aligned (x) for declare simd, for the ABI we really 246 need an alignment specified. */ 247 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (t) == NULL_TREE) 248 break; 249 tree decl = OMP_CLAUSE_DECL (t); 250 int argno = tree_to_uhwi (decl); 251 clone_info->args[argno].alignment 252 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t)); 253 break; 254 } 255 default: 256 break; 257 } 258 } 259 260 out: 261 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node->decl)))) 262 { 263 warning_at (DECL_SOURCE_LOCATION (node->decl), 0, 264 "ignoring %<#pragma omp declare simd%> on function " 265 "with %<_Atomic%> qualified return type"); 266 args.release (); 267 return NULL; 268 } 269 270 for (unsigned int argno = 0; argno < clone_info->nargs; argno++) 271 if (TYPE_ATOMIC (args[argno]) 272 && clone_info->args[argno].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM) 273 { 274 warning_at (DECL_SOURCE_LOCATION (node->decl), 0, 275 "ignoring %<#pragma omp declare simd%> on function " 276 "with %<_Atomic%> qualified non-%<uniform%> argument"); 277 args.release (); 278 return NULL; 279 } 280 281 args.release (); 282 return clone_info; 283 } 284 285 /* Given a SIMD clone in NODE, calculate the characteristic data 286 type and return the coresponding type. The characteristic data 287 type is computed as described in the Intel Vector ABI. */ 288 289 static tree 290 simd_clone_compute_base_data_type (struct cgraph_node *node, 291 struct cgraph_simd_clone *clone_info) 292 { 293 tree type = integer_type_node; 294 tree fndecl = node->decl; 295 296 /* a) For non-void function, the characteristic data type is the 297 return type. */ 298 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE) 299 type = TREE_TYPE (TREE_TYPE (fndecl)); 300 301 /* b) If the function has any non-uniform, non-linear parameters, 302 then the characteristic data type is the type of the first 303 such parameter. */ 304 else 305 { 306 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl); 307 for (unsigned int i = 0; i < clone_info->nargs; ++i) 308 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR) 309 { 310 type = map[i]; 311 break; 312 } 313 map.release (); 314 } 315 316 /* c) If the characteristic data type determined by a) or b) above 317 is struct, union, or class type which is pass-by-value (except 318 for the type that maps to the built-in complex data type), the 319 characteristic data type is int. */ 320 if (RECORD_OR_UNION_TYPE_P (type) 321 && !aggregate_value_p (type, NULL) 322 && TREE_CODE (type) != COMPLEX_TYPE) 323 return integer_type_node; 324 325 /* d) If none of the above three classes is applicable, the 326 characteristic data type is int. */ 327 328 return type; 329 330 /* e) For Intel Xeon Phi native and offload compilation, if the 331 resulting characteristic data type is 8-bit or 16-bit integer 332 data type, the characteristic data type is int. */ 333 /* Well, we don't handle Xeon Phi yet. */ 334 } 335 336 static tree 337 simd_clone_mangle (struct cgraph_node *node, 338 struct cgraph_simd_clone *clone_info) 339 { 340 char vecsize_mangle = clone_info->vecsize_mangle; 341 char mask = clone_info->inbranch ? 'M' : 'N'; 342 unsigned int simdlen = clone_info->simdlen; 343 unsigned int n; 344 pretty_printer pp; 345 346 gcc_assert (vecsize_mangle && simdlen); 347 348 pp_string (&pp, "_ZGV"); 349 pp_character (&pp, vecsize_mangle); 350 pp_character (&pp, mask); 351 pp_decimal_int (&pp, simdlen); 352 353 for (n = 0; n < clone_info->nargs; ++n) 354 { 355 struct cgraph_simd_clone_arg arg = clone_info->args[n]; 356 357 switch (arg.arg_type) 358 { 359 case SIMD_CLONE_ARG_TYPE_UNIFORM: 360 pp_character (&pp, 'u'); 361 break; 362 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP: 363 pp_character (&pp, 'l'); 364 goto mangle_linear; 365 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP: 366 pp_character (&pp, 'R'); 367 goto mangle_linear; 368 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP: 369 pp_character (&pp, 'L'); 370 goto mangle_linear; 371 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP: 372 pp_character (&pp, 'U'); 373 goto mangle_linear; 374 mangle_linear: 375 gcc_assert (arg.linear_step != 0); 376 if (arg.linear_step > 1) 377 pp_unsigned_wide_integer (&pp, arg.linear_step); 378 else if (arg.linear_step < 0) 379 { 380 pp_character (&pp, 'n'); 381 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT) 382 arg.linear_step)); 383 } 384 break; 385 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP: 386 pp_string (&pp, "ls"); 387 pp_unsigned_wide_integer (&pp, arg.linear_step); 388 break; 389 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP: 390 pp_string (&pp, "Rs"); 391 pp_unsigned_wide_integer (&pp, arg.linear_step); 392 break; 393 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP: 394 pp_string (&pp, "Ls"); 395 pp_unsigned_wide_integer (&pp, arg.linear_step); 396 break; 397 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP: 398 pp_string (&pp, "Us"); 399 pp_unsigned_wide_integer (&pp, arg.linear_step); 400 break; 401 default: 402 pp_character (&pp, 'v'); 403 } 404 if (arg.alignment) 405 { 406 pp_character (&pp, 'a'); 407 pp_decimal_int (&pp, arg.alignment); 408 } 409 } 410 411 pp_underscore (&pp); 412 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl)); 413 if (*str == '*') 414 ++str; 415 pp_string (&pp, str); 416 str = pp_formatted_text (&pp); 417 418 /* If there already is a SIMD clone with the same mangled name, don't 419 add another one. This can happen e.g. for 420 #pragma omp declare simd 421 #pragma omp declare simd simdlen(8) 422 int foo (int, int); 423 if the simdlen is assumed to be 8 for the first one, etc. */ 424 for (struct cgraph_node *clone = node->simd_clones; clone; 425 clone = clone->simdclone->next_clone) 426 if (id_equal (DECL_ASSEMBLER_NAME (clone->decl), str)) 427 return NULL_TREE; 428 429 return get_identifier (str); 430 } 431 432 /* Create a simd clone of OLD_NODE and return it. */ 433 434 static struct cgraph_node * 435 simd_clone_create (struct cgraph_node *old_node) 436 { 437 struct cgraph_node *new_node; 438 if (old_node->definition) 439 { 440 if (!old_node->has_gimple_body_p ()) 441 return NULL; 442 old_node->get_body (); 443 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL, 444 false, NULL, NULL, 445 "simdclone"); 446 } 447 else 448 { 449 tree old_decl = old_node->decl; 450 tree new_decl = copy_node (old_node->decl); 451 DECL_NAME (new_decl) = clone_function_name_numbered (old_decl, 452 "simdclone"); 453 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl)); 454 SET_DECL_RTL (new_decl, NULL); 455 DECL_STATIC_CONSTRUCTOR (new_decl) = 0; 456 DECL_STATIC_DESTRUCTOR (new_decl) = 0; 457 new_node = old_node->create_version_clone (new_decl, vNULL, NULL); 458 if (old_node->in_other_partition) 459 new_node->in_other_partition = 1; 460 } 461 if (new_node == NULL) 462 return new_node; 463 464 DECL_BUILT_IN_CLASS (new_node->decl) = NOT_BUILT_IN; 465 DECL_FUNCTION_CODE (new_node->decl) = (enum built_in_function) 0; 466 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl); 467 DECL_COMDAT (new_node->decl) = DECL_COMDAT (old_node->decl); 468 DECL_WEAK (new_node->decl) = DECL_WEAK (old_node->decl); 469 DECL_EXTERNAL (new_node->decl) = DECL_EXTERNAL (old_node->decl); 470 DECL_VISIBILITY_SPECIFIED (new_node->decl) 471 = DECL_VISIBILITY_SPECIFIED (old_node->decl); 472 DECL_VISIBILITY (new_node->decl) = DECL_VISIBILITY (old_node->decl); 473 DECL_DLLIMPORT_P (new_node->decl) = DECL_DLLIMPORT_P (old_node->decl); 474 if (DECL_ONE_ONLY (old_node->decl)) 475 make_decl_one_only (new_node->decl, DECL_ASSEMBLER_NAME (new_node->decl)); 476 477 /* The method cgraph_version_clone_with_body () will force the new 478 symbol local. Undo this, and inherit external visibility from 479 the old node. */ 480 new_node->local.local = old_node->local.local; 481 new_node->externally_visible = old_node->externally_visible; 482 483 return new_node; 484 } 485 486 /* Adjust the return type of the given function to its appropriate 487 vector counterpart. Returns a simd array to be used throughout the 488 function as a return value. */ 489 490 static tree 491 simd_clone_adjust_return_type (struct cgraph_node *node) 492 { 493 tree fndecl = node->decl; 494 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl)); 495 unsigned int veclen; 496 tree t; 497 498 /* Adjust the function return type. */ 499 if (orig_rettype == void_type_node) 500 return NULL_TREE; 501 t = TREE_TYPE (TREE_TYPE (fndecl)); 502 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t)) 503 veclen = node->simdclone->vecsize_int; 504 else 505 veclen = node->simdclone->vecsize_float; 506 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t)); 507 if (veclen > node->simdclone->simdlen) 508 veclen = node->simdclone->simdlen; 509 if (POINTER_TYPE_P (t)) 510 t = pointer_sized_int_node; 511 if (veclen == node->simdclone->simdlen) 512 t = build_vector_type (t, node->simdclone->simdlen); 513 else 514 { 515 t = build_vector_type (t, veclen); 516 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen); 517 } 518 TREE_TYPE (TREE_TYPE (fndecl)) = t; 519 if (!node->definition) 520 return NULL_TREE; 521 522 t = DECL_RESULT (fndecl); 523 /* Adjust the DECL_RESULT. */ 524 gcc_assert (TREE_TYPE (t) != void_type_node); 525 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl)); 526 relayout_decl (t); 527 528 tree atype = build_array_type_nelts (orig_rettype, 529 node->simdclone->simdlen); 530 if (veclen != node->simdclone->simdlen) 531 return build1 (VIEW_CONVERT_EXPR, atype, t); 532 533 /* Set up a SIMD array to use as the return value. */ 534 tree retval = create_tmp_var_raw (atype, "retval"); 535 gimple_add_tmp_var (retval); 536 return retval; 537 } 538 539 /* Each vector argument has a corresponding array to be used locally 540 as part of the eventual loop. Create such temporary array and 541 return it. 542 543 PREFIX is the prefix to be used for the temporary. 544 545 TYPE is the inner element type. 546 547 SIMDLEN is the number of elements. */ 548 549 static tree 550 create_tmp_simd_array (const char *prefix, tree type, int simdlen) 551 { 552 tree atype = build_array_type_nelts (type, simdlen); 553 tree avar = create_tmp_var_raw (atype, prefix); 554 gimple_add_tmp_var (avar); 555 return avar; 556 } 557 558 /* Modify the function argument types to their corresponding vector 559 counterparts if appropriate. Also, create one array for each simd 560 argument to be used locally when using the function arguments as 561 part of the loop. 562 563 NODE is the function whose arguments are to be adjusted. 564 565 Returns an adjustment vector that will be filled describing how the 566 argument types will be adjusted. */ 567 568 static ipa_parm_adjustment_vec 569 simd_clone_adjust_argument_types (struct cgraph_node *node) 570 { 571 vec<tree> args; 572 ipa_parm_adjustment_vec adjustments; 573 574 if (node->definition) 575 args = ipa_get_vector_of_formal_parms (node->decl); 576 else 577 args = simd_clone_vector_of_formal_parm_types (node->decl); 578 adjustments.create (args.length ()); 579 unsigned i, j, veclen; 580 struct ipa_parm_adjustment adj; 581 struct cgraph_simd_clone *sc = node->simdclone; 582 583 for (i = 0; i < sc->nargs; ++i) 584 { 585 memset (&adj, 0, sizeof (adj)); 586 tree parm = args[i]; 587 tree parm_type = node->definition ? TREE_TYPE (parm) : parm; 588 adj.base_index = i; 589 adj.base = parm; 590 591 sc->args[i].orig_arg = node->definition ? parm : NULL_TREE; 592 sc->args[i].orig_type = parm_type; 593 594 switch (sc->args[i].arg_type) 595 { 596 default: 597 /* No adjustment necessary for scalar arguments. */ 598 adj.op = IPA_PARM_OP_COPY; 599 break; 600 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP: 601 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP: 602 if (node->definition) 603 sc->args[i].simd_array 604 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)), 605 TREE_TYPE (parm_type), 606 sc->simdlen); 607 adj.op = IPA_PARM_OP_COPY; 608 break; 609 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP: 610 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP: 611 case SIMD_CLONE_ARG_TYPE_VECTOR: 612 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type)) 613 veclen = sc->vecsize_int; 614 else 615 veclen = sc->vecsize_float; 616 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type)); 617 if (veclen > sc->simdlen) 618 veclen = sc->simdlen; 619 adj.arg_prefix = "simd"; 620 if (POINTER_TYPE_P (parm_type)) 621 adj.type = build_vector_type (pointer_sized_int_node, veclen); 622 else 623 adj.type = build_vector_type (parm_type, veclen); 624 sc->args[i].vector_type = adj.type; 625 for (j = veclen; j < sc->simdlen; j += veclen) 626 { 627 adjustments.safe_push (adj); 628 if (j == veclen) 629 { 630 memset (&adj, 0, sizeof (adj)); 631 adj.op = IPA_PARM_OP_NEW; 632 adj.arg_prefix = "simd"; 633 adj.base_index = i; 634 adj.type = sc->args[i].vector_type; 635 } 636 } 637 638 if (node->definition) 639 sc->args[i].simd_array 640 = create_tmp_simd_array (DECL_NAME (parm) 641 ? IDENTIFIER_POINTER (DECL_NAME (parm)) 642 : NULL, parm_type, sc->simdlen); 643 } 644 adjustments.safe_push (adj); 645 } 646 647 if (sc->inbranch) 648 { 649 tree base_type = simd_clone_compute_base_data_type (sc->origin, sc); 650 651 memset (&adj, 0, sizeof (adj)); 652 adj.op = IPA_PARM_OP_NEW; 653 adj.arg_prefix = "mask"; 654 655 adj.base_index = i; 656 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type)) 657 veclen = sc->vecsize_int; 658 else 659 veclen = sc->vecsize_float; 660 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type)); 661 if (veclen > sc->simdlen) 662 veclen = sc->simdlen; 663 if (sc->mask_mode != VOIDmode) 664 adj.type 665 = lang_hooks.types.type_for_mode (sc->mask_mode, 1); 666 else if (POINTER_TYPE_P (base_type)) 667 adj.type = build_vector_type (pointer_sized_int_node, veclen); 668 else 669 adj.type = build_vector_type (base_type, veclen); 670 adjustments.safe_push (adj); 671 672 for (j = veclen; j < sc->simdlen; j += veclen) 673 adjustments.safe_push (adj); 674 675 /* We have previously allocated one extra entry for the mask. Use 676 it and fill it. */ 677 sc->nargs++; 678 if (sc->mask_mode != VOIDmode) 679 base_type = boolean_type_node; 680 if (node->definition) 681 { 682 sc->args[i].orig_arg 683 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type); 684 if (sc->mask_mode == VOIDmode) 685 sc->args[i].simd_array 686 = create_tmp_simd_array ("mask", base_type, sc->simdlen); 687 else if (veclen < sc->simdlen) 688 sc->args[i].simd_array 689 = create_tmp_simd_array ("mask", adj.type, sc->simdlen / veclen); 690 else 691 sc->args[i].simd_array = NULL_TREE; 692 } 693 sc->args[i].orig_type = base_type; 694 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK; 695 } 696 697 if (node->definition) 698 ipa_modify_formal_parameters (node->decl, adjustments); 699 else 700 { 701 tree new_arg_types = NULL_TREE, new_reversed; 702 bool last_parm_void = false; 703 if (args.length () > 0 && args.last () == void_type_node) 704 last_parm_void = true; 705 706 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl))); 707 j = adjustments.length (); 708 for (i = 0; i < j; i++) 709 { 710 struct ipa_parm_adjustment *adj = &adjustments[i]; 711 tree ptype; 712 if (adj->op == IPA_PARM_OP_COPY) 713 ptype = args[adj->base_index]; 714 else 715 ptype = adj->type; 716 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types); 717 } 718 new_reversed = nreverse (new_arg_types); 719 if (last_parm_void) 720 { 721 if (new_reversed) 722 TREE_CHAIN (new_arg_types) = void_list_node; 723 else 724 new_reversed = void_list_node; 725 } 726 TYPE_ARG_TYPES (TREE_TYPE (node->decl)) = new_reversed; 727 adjustments.release (); 728 } 729 args.release (); 730 return adjustments; 731 } 732 733 /* Initialize and copy the function arguments in NODE to their 734 corresponding local simd arrays. Returns a fresh gimple_seq with 735 the instruction sequence generated. */ 736 737 static gimple_seq 738 simd_clone_init_simd_arrays (struct cgraph_node *node, 739 ipa_parm_adjustment_vec adjustments) 740 { 741 gimple_seq seq = NULL; 742 unsigned i = 0, j = 0, k; 743 744 for (tree arg = DECL_ARGUMENTS (node->decl); 745 arg; 746 arg = DECL_CHAIN (arg), i++, j++) 747 { 748 if (adjustments[j].op == IPA_PARM_OP_COPY 749 || POINTER_TYPE_P (TREE_TYPE (arg))) 750 continue; 751 752 node->simdclone->args[i].vector_arg = arg; 753 754 tree array = node->simdclone->args[i].simd_array; 755 if (node->simdclone->mask_mode != VOIDmode 756 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK) 757 { 758 if (array == NULL_TREE) 759 continue; 760 unsigned int l 761 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array)))); 762 for (k = 0; k <= l; k++) 763 { 764 if (k) 765 { 766 arg = DECL_CHAIN (arg); 767 j++; 768 } 769 tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)), 770 array, size_int (k), NULL, NULL); 771 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg); 772 gimplify_and_add (t, &seq); 773 } 774 continue; 775 } 776 if (simd_clone_subparts (TREE_TYPE (arg)) == node->simdclone->simdlen) 777 { 778 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array))); 779 tree ptr = build_fold_addr_expr (array); 780 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr, 781 build_int_cst (ptype, 0)); 782 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg); 783 gimplify_and_add (t, &seq); 784 } 785 else 786 { 787 unsigned int simdlen = simd_clone_subparts (TREE_TYPE (arg)); 788 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array))); 789 for (k = 0; k < node->simdclone->simdlen; k += simdlen) 790 { 791 tree ptr = build_fold_addr_expr (array); 792 int elemsize; 793 if (k) 794 { 795 arg = DECL_CHAIN (arg); 796 j++; 797 } 798 tree elemtype = TREE_TYPE (TREE_TYPE (arg)); 799 elemsize = GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype)); 800 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr, 801 build_int_cst (ptype, k * elemsize)); 802 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg); 803 gimplify_and_add (t, &seq); 804 } 805 } 806 } 807 return seq; 808 } 809 810 /* Callback info for ipa_simd_modify_stmt_ops below. */ 811 812 struct modify_stmt_info { 813 ipa_parm_adjustment_vec adjustments; 814 gimple *stmt; 815 /* True if the parent statement was modified by 816 ipa_simd_modify_stmt_ops. */ 817 bool modified; 818 }; 819 820 /* Callback for walk_gimple_op. 821 822 Adjust operands from a given statement as specified in the 823 adjustments vector in the callback data. */ 824 825 static tree 826 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data) 827 { 828 struct walk_stmt_info *wi = (struct walk_stmt_info *) data; 829 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info; 830 tree *orig_tp = tp; 831 if (TREE_CODE (*tp) == ADDR_EXPR) 832 tp = &TREE_OPERAND (*tp, 0); 833 struct ipa_parm_adjustment *cand = NULL; 834 if (TREE_CODE (*tp) == PARM_DECL) 835 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true); 836 else if (TYPE_P (*tp)) 837 *walk_subtrees = 0; 838 839 tree repl = NULL_TREE; 840 if (cand) 841 repl = unshare_expr (cand->new_decl); 842 else 843 { 844 if (tp != orig_tp) 845 { 846 *walk_subtrees = 0; 847 bool modified = info->modified; 848 info->modified = false; 849 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset); 850 if (!info->modified) 851 { 852 info->modified = modified; 853 return NULL_TREE; 854 } 855 info->modified = modified; 856 repl = *tp; 857 } 858 else 859 return NULL_TREE; 860 } 861 862 if (tp != orig_tp) 863 { 864 if (gimple_code (info->stmt) == GIMPLE_PHI 865 && cand 866 && TREE_CODE (*orig_tp) == ADDR_EXPR 867 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL 868 && cand->alias_ptr_type) 869 { 870 gcc_assert (TREE_CODE (cand->alias_ptr_type) == SSA_NAME); 871 *orig_tp = cand->alias_ptr_type; 872 info->modified = true; 873 return NULL_TREE; 874 } 875 876 repl = build_fold_addr_expr (repl); 877 gimple *stmt; 878 if (is_gimple_debug (info->stmt)) 879 { 880 tree vexpr = make_node (DEBUG_EXPR_DECL); 881 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL); 882 DECL_ARTIFICIAL (vexpr) = 1; 883 TREE_TYPE (vexpr) = TREE_TYPE (repl); 884 SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (repl))); 885 repl = vexpr; 886 } 887 else 888 { 889 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl); 890 repl = gimple_assign_lhs (stmt); 891 } 892 gimple_stmt_iterator gsi; 893 if (gimple_code (info->stmt) == GIMPLE_PHI) 894 { 895 gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun))); 896 /* Cache SSA_NAME for next time. */ 897 if (cand 898 && TREE_CODE (*orig_tp) == ADDR_EXPR 899 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL) 900 cand->alias_ptr_type = repl; 901 } 902 else 903 gsi = gsi_for_stmt (info->stmt); 904 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); 905 *orig_tp = repl; 906 } 907 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl))) 908 { 909 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl); 910 *tp = vce; 911 } 912 else 913 *tp = repl; 914 915 info->modified = true; 916 return NULL_TREE; 917 } 918 919 /* Traverse the function body and perform all modifications as 920 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be 921 modified such that the replacement/reduction value will now be an 922 offset into the corresponding simd_array. 923 924 This function will replace all function argument uses with their 925 corresponding simd array elements, and ajust the return values 926 accordingly. */ 927 928 static void 929 ipa_simd_modify_function_body (struct cgraph_node *node, 930 ipa_parm_adjustment_vec adjustments, 931 tree retval_array, tree iter) 932 { 933 basic_block bb; 934 unsigned int i, j, l; 935 936 /* Re-use the adjustments array, but this time use it to replace 937 every function argument use to an offset into the corresponding 938 simd_array. */ 939 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j) 940 { 941 if (!node->simdclone->args[i].vector_arg) 942 continue; 943 944 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg); 945 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg); 946 adjustments[j].new_decl 947 = build4 (ARRAY_REF, 948 basetype, 949 node->simdclone->args[i].simd_array, 950 iter, 951 NULL_TREE, NULL_TREE); 952 if (adjustments[j].op == IPA_PARM_OP_NONE 953 && simd_clone_subparts (vectype) < node->simdclone->simdlen) 954 j += node->simdclone->simdlen / simd_clone_subparts (vectype) - 1; 955 } 956 957 l = adjustments.length (); 958 tree name; 959 960 FOR_EACH_SSA_NAME (i, name, cfun) 961 { 962 if (SSA_NAME_VAR (name) 963 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL) 964 { 965 for (j = 0; j < l; j++) 966 if (SSA_NAME_VAR (name) == adjustments[j].base 967 && adjustments[j].new_decl) 968 { 969 tree base_var; 970 if (adjustments[j].new_ssa_base == NULL_TREE) 971 { 972 base_var 973 = copy_var_decl (adjustments[j].base, 974 DECL_NAME (adjustments[j].base), 975 TREE_TYPE (adjustments[j].base)); 976 adjustments[j].new_ssa_base = base_var; 977 } 978 else 979 base_var = adjustments[j].new_ssa_base; 980 if (SSA_NAME_IS_DEFAULT_DEF (name)) 981 { 982 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 983 gimple_stmt_iterator gsi = gsi_after_labels (bb); 984 tree new_decl = unshare_expr (adjustments[j].new_decl); 985 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE); 986 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var); 987 SSA_NAME_IS_DEFAULT_DEF (name) = 0; 988 gimple *stmt = gimple_build_assign (name, new_decl); 989 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); 990 } 991 else 992 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var); 993 } 994 } 995 } 996 997 struct modify_stmt_info info; 998 info.adjustments = adjustments; 999 1000 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl)) 1001 { 1002 gimple_stmt_iterator gsi; 1003 1004 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 1005 { 1006 gphi *phi = as_a <gphi *> (gsi_stmt (gsi)); 1007 int i, n = gimple_phi_num_args (phi); 1008 info.stmt = phi; 1009 struct walk_stmt_info wi; 1010 memset (&wi, 0, sizeof (wi)); 1011 info.modified = false; 1012 wi.info = &info; 1013 for (i = 0; i < n; ++i) 1014 { 1015 int walk_subtrees = 1; 1016 tree arg = gimple_phi_arg_def (phi, i); 1017 tree op = arg; 1018 ipa_simd_modify_stmt_ops (&op, &walk_subtrees, &wi); 1019 if (op != arg) 1020 { 1021 SET_PHI_ARG_DEF (phi, i, op); 1022 gcc_assert (TREE_CODE (op) == SSA_NAME); 1023 if (gimple_phi_arg_edge (phi, i)->flags & EDGE_ABNORMAL) 1024 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op) = 1; 1025 } 1026 } 1027 } 1028 1029 gsi = gsi_start_bb (bb); 1030 while (!gsi_end_p (gsi)) 1031 { 1032 gimple *stmt = gsi_stmt (gsi); 1033 info.stmt = stmt; 1034 struct walk_stmt_info wi; 1035 1036 memset (&wi, 0, sizeof (wi)); 1037 info.modified = false; 1038 wi.info = &info; 1039 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi); 1040 1041 if (greturn *return_stmt = dyn_cast <greturn *> (stmt)) 1042 { 1043 tree retval = gimple_return_retval (return_stmt); 1044 edge e = find_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun)); 1045 e->flags |= EDGE_FALLTHRU; 1046 if (!retval) 1047 { 1048 gsi_remove (&gsi, true); 1049 continue; 1050 } 1051 1052 /* Replace `return foo' with `retval_array[iter] = foo'. */ 1053 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval), 1054 retval_array, iter, NULL, NULL); 1055 stmt = gimple_build_assign (ref, retval); 1056 gsi_replace (&gsi, stmt, true); 1057 info.modified = true; 1058 } 1059 1060 if (info.modified) 1061 { 1062 update_stmt (stmt); 1063 /* If the above changed the var of a debug bind into something 1064 different, remove the debug stmt. We could also for all the 1065 replaced parameters add VAR_DECLs for debug info purposes, 1066 add debug stmts for those to be the simd array accesses and 1067 replace debug stmt var operand with that var. Debugging of 1068 vectorized loops doesn't work too well, so don't bother for 1069 now. */ 1070 if ((gimple_debug_bind_p (stmt) 1071 && !DECL_P (gimple_debug_bind_get_var (stmt))) 1072 || (gimple_debug_source_bind_p (stmt) 1073 && !DECL_P (gimple_debug_source_bind_get_var (stmt)))) 1074 { 1075 gsi_remove (&gsi, true); 1076 continue; 1077 } 1078 if (maybe_clean_eh_stmt (stmt)) 1079 gimple_purge_dead_eh_edges (gimple_bb (stmt)); 1080 } 1081 gsi_next (&gsi); 1082 } 1083 } 1084 } 1085 1086 /* Helper function of simd_clone_adjust, return linear step addend 1087 of Ith argument. */ 1088 1089 static tree 1090 simd_clone_linear_addend (struct cgraph_node *node, unsigned int i, 1091 tree addtype, basic_block entry_bb) 1092 { 1093 tree ptype = NULL_TREE; 1094 switch (node->simdclone->args[i].arg_type) 1095 { 1096 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP: 1097 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP: 1098 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP: 1099 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP: 1100 return build_int_cst (addtype, node->simdclone->args[i].linear_step); 1101 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP: 1102 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP: 1103 ptype = TREE_TYPE (node->simdclone->args[i].orig_arg); 1104 break; 1105 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP: 1106 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP: 1107 ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg)); 1108 break; 1109 default: 1110 gcc_unreachable (); 1111 } 1112 1113 unsigned int idx = node->simdclone->args[i].linear_step; 1114 tree arg = node->simdclone->args[idx].orig_arg; 1115 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg))); 1116 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb); 1117 gimple *g; 1118 tree ret; 1119 if (is_gimple_reg (arg)) 1120 ret = get_or_create_ssa_default_def (cfun, arg); 1121 else 1122 { 1123 g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg); 1124 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1125 ret = gimple_assign_lhs (g); 1126 } 1127 if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE) 1128 { 1129 g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))), 1130 build_simple_mem_ref (ret)); 1131 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1132 ret = gimple_assign_lhs (g); 1133 } 1134 if (!useless_type_conversion_p (addtype, TREE_TYPE (ret))) 1135 { 1136 g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret); 1137 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1138 ret = gimple_assign_lhs (g); 1139 } 1140 if (POINTER_TYPE_P (ptype)) 1141 { 1142 tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype)); 1143 if (size && TREE_CODE (size) == INTEGER_CST) 1144 { 1145 g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR, 1146 ret, fold_convert (addtype, size)); 1147 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1148 ret = gimple_assign_lhs (g); 1149 } 1150 } 1151 return ret; 1152 } 1153 1154 /* Adjust the argument types in NODE to their appropriate vector 1155 counterparts. */ 1156 1157 static void 1158 simd_clone_adjust (struct cgraph_node *node) 1159 { 1160 push_cfun (DECL_STRUCT_FUNCTION (node->decl)); 1161 1162 TREE_TYPE (node->decl) = build_distinct_type_copy (TREE_TYPE (node->decl)); 1163 targetm.simd_clone.adjust (node); 1164 1165 tree retval = simd_clone_adjust_return_type (node); 1166 ipa_parm_adjustment_vec adjustments 1167 = simd_clone_adjust_argument_types (node); 1168 1169 push_gimplify_context (); 1170 1171 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments); 1172 1173 /* Adjust all uses of vector arguments accordingly. Adjust all 1174 return values accordingly. */ 1175 tree iter = create_tmp_var (unsigned_type_node, "iter"); 1176 tree iter1 = make_ssa_name (iter); 1177 tree iter2 = NULL_TREE; 1178 ipa_simd_modify_function_body (node, adjustments, retval, iter1); 1179 adjustments.release (); 1180 1181 /* Initialize the iteration variable. */ 1182 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 1183 basic_block body_bb = split_block_after_labels (entry_bb)->dest; 1184 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb); 1185 /* Insert the SIMD array and iv initialization at function 1186 entry. */ 1187 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT); 1188 1189 pop_gimplify_context (NULL); 1190 1191 gimple *g; 1192 basic_block incr_bb = NULL; 1193 struct loop *loop = NULL; 1194 1195 /* Create a new BB right before the original exit BB, to hold the 1196 iteration increment and the condition/branch. */ 1197 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)) 1198 { 1199 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src; 1200 incr_bb = create_empty_bb (orig_exit); 1201 incr_bb->count = profile_count::zero (); 1202 add_bb_to_loop (incr_bb, body_bb->loop_father); 1203 while (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)) 1204 { 1205 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0); 1206 redirect_edge_succ (e, incr_bb); 1207 incr_bb->count += e->count (); 1208 } 1209 } 1210 else if (node->simdclone->inbranch) 1211 { 1212 incr_bb = create_empty_bb (entry_bb); 1213 incr_bb->count = profile_count::zero (); 1214 add_bb_to_loop (incr_bb, body_bb->loop_father); 1215 } 1216 1217 if (incr_bb) 1218 { 1219 make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0); 1220 gsi = gsi_last_bb (incr_bb); 1221 iter2 = make_ssa_name (iter); 1222 g = gimple_build_assign (iter2, PLUS_EXPR, iter1, 1223 build_int_cst (unsigned_type_node, 1)); 1224 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1225 1226 /* Mostly annotate the loop for the vectorizer (the rest is done 1227 below). */ 1228 loop = alloc_loop (); 1229 cfun->has_force_vectorize_loops = true; 1230 loop->safelen = node->simdclone->simdlen; 1231 loop->force_vectorize = true; 1232 loop->header = body_bb; 1233 } 1234 1235 /* Branch around the body if the mask applies. */ 1236 if (node->simdclone->inbranch) 1237 { 1238 gsi = gsi_last_bb (loop->header); 1239 tree mask_array 1240 = node->simdclone->args[node->simdclone->nargs - 1].simd_array; 1241 tree mask; 1242 if (node->simdclone->mask_mode != VOIDmode) 1243 { 1244 tree shift_cnt; 1245 if (mask_array == NULL_TREE) 1246 { 1247 tree arg = node->simdclone->args[node->simdclone->nargs 1248 - 1].vector_arg; 1249 mask = get_or_create_ssa_default_def (cfun, arg); 1250 shift_cnt = iter1; 1251 } 1252 else 1253 { 1254 tree maskt = TREE_TYPE (mask_array); 1255 int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt))); 1256 c = node->simdclone->simdlen / (c + 1); 1257 int s = exact_log2 (c); 1258 gcc_assert (s > 0); 1259 c--; 1260 tree idx = make_ssa_name (TREE_TYPE (iter1)); 1261 g = gimple_build_assign (idx, RSHIFT_EXPR, iter1, 1262 build_int_cst (NULL_TREE, s)); 1263 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1264 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array))); 1265 tree aref = build4 (ARRAY_REF, 1266 TREE_TYPE (TREE_TYPE (mask_array)), 1267 mask_array, idx, NULL, NULL); 1268 g = gimple_build_assign (mask, aref); 1269 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1270 shift_cnt = make_ssa_name (TREE_TYPE (iter1)); 1271 g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1, 1272 build_int_cst (TREE_TYPE (iter1), c)); 1273 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1274 } 1275 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)), 1276 RSHIFT_EXPR, mask, shift_cnt); 1277 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1278 mask = gimple_assign_lhs (g); 1279 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)), 1280 BIT_AND_EXPR, mask, 1281 build_int_cst (TREE_TYPE (mask), 1)); 1282 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1283 mask = gimple_assign_lhs (g); 1284 } 1285 else 1286 { 1287 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array))); 1288 tree aref = build4 (ARRAY_REF, 1289 TREE_TYPE (TREE_TYPE (mask_array)), 1290 mask_array, iter1, NULL, NULL); 1291 g = gimple_build_assign (mask, aref); 1292 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1293 int bitsize = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref))); 1294 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref))) 1295 { 1296 aref = build1 (VIEW_CONVERT_EXPR, 1297 build_nonstandard_integer_type (bitsize, 0), 1298 mask); 1299 mask = make_ssa_name (TREE_TYPE (aref)); 1300 g = gimple_build_assign (mask, aref); 1301 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1302 } 1303 } 1304 1305 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)), 1306 NULL, NULL); 1307 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1308 edge e = make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE); 1309 e->probability = profile_probability::unlikely ().guessed (); 1310 incr_bb->count += e->count (); 1311 edge fallthru = FALLTHRU_EDGE (loop->header); 1312 fallthru->flags = EDGE_FALSE_VALUE; 1313 fallthru->probability = profile_probability::likely ().guessed (); 1314 } 1315 1316 basic_block latch_bb = NULL; 1317 basic_block new_exit_bb = NULL; 1318 1319 /* Generate the condition. */ 1320 if (incr_bb) 1321 { 1322 gsi = gsi_last_bb (incr_bb); 1323 g = gimple_build_cond (LT_EXPR, iter2, 1324 build_int_cst (unsigned_type_node, 1325 node->simdclone->simdlen), 1326 NULL, NULL); 1327 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1328 edge e = split_block (incr_bb, gsi_stmt (gsi)); 1329 latch_bb = e->dest; 1330 new_exit_bb = split_block_after_labels (latch_bb)->dest; 1331 loop->latch = latch_bb; 1332 1333 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb); 1334 1335 edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE); 1336 1337 /* FIXME: Do we need to distribute probabilities for the conditional? */ 1338 new_e->probability = profile_probability::guessed_never (); 1339 /* The successor of incr_bb is already pointing to latch_bb; just 1340 change the flags. 1341 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */ 1342 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE; 1343 } 1344 1345 gphi *phi = create_phi_node (iter1, body_bb); 1346 edge preheader_edge = find_edge (entry_bb, body_bb); 1347 edge latch_edge = NULL; 1348 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge, 1349 UNKNOWN_LOCATION); 1350 if (incr_bb) 1351 { 1352 latch_edge = single_succ_edge (latch_bb); 1353 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION); 1354 1355 /* Generate the new return. */ 1356 gsi = gsi_last_bb (new_exit_bb); 1357 if (retval 1358 && TREE_CODE (retval) == VIEW_CONVERT_EXPR 1359 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL) 1360 retval = TREE_OPERAND (retval, 0); 1361 else if (retval) 1362 { 1363 retval = build1 (VIEW_CONVERT_EXPR, 1364 TREE_TYPE (TREE_TYPE (node->decl)), 1365 retval); 1366 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL, 1367 false, GSI_CONTINUE_LINKING); 1368 } 1369 g = gimple_build_return (retval); 1370 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1371 } 1372 1373 /* Handle aligned clauses by replacing default defs of the aligned 1374 uniform args with __builtin_assume_aligned (arg_N(D), alignment) 1375 lhs. Handle linear by adding PHIs. */ 1376 for (unsigned i = 0; i < node->simdclone->nargs; i++) 1377 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM 1378 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg) 1379 || !is_gimple_reg_type 1380 (TREE_TYPE (node->simdclone->args[i].orig_arg)))) 1381 { 1382 tree orig_arg = node->simdclone->args[i].orig_arg; 1383 if (is_gimple_reg_type (TREE_TYPE (orig_arg))) 1384 iter1 = make_ssa_name (TREE_TYPE (orig_arg)); 1385 else 1386 { 1387 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg)); 1388 gimple_add_tmp_var (iter1); 1389 } 1390 gsi = gsi_after_labels (entry_bb); 1391 g = gimple_build_assign (iter1, orig_arg); 1392 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 1393 gsi = gsi_after_labels (body_bb); 1394 g = gimple_build_assign (orig_arg, iter1); 1395 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 1396 } 1397 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM 1398 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg) 1399 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg)) 1400 == REFERENCE_TYPE 1401 && TREE_ADDRESSABLE 1402 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg)))) 1403 { 1404 tree orig_arg = node->simdclone->args[i].orig_arg; 1405 tree def = ssa_default_def (cfun, orig_arg); 1406 if (def && !has_zero_uses (def)) 1407 { 1408 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg))); 1409 gimple_add_tmp_var (iter1); 1410 gsi = gsi_after_labels (entry_bb); 1411 g = gimple_build_assign (iter1, build_simple_mem_ref (def)); 1412 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 1413 gsi = gsi_after_labels (body_bb); 1414 g = gimple_build_assign (build_simple_mem_ref (def), iter1); 1415 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 1416 } 1417 } 1418 else if (node->simdclone->args[i].alignment 1419 && node->simdclone->args[i].arg_type 1420 == SIMD_CLONE_ARG_TYPE_UNIFORM 1421 && (node->simdclone->args[i].alignment 1422 & (node->simdclone->args[i].alignment - 1)) == 0 1423 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg)) 1424 == POINTER_TYPE) 1425 { 1426 unsigned int alignment = node->simdclone->args[i].alignment; 1427 tree orig_arg = node->simdclone->args[i].orig_arg; 1428 tree def = ssa_default_def (cfun, orig_arg); 1429 if (def && !has_zero_uses (def)) 1430 { 1431 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED); 1432 gimple_seq seq = NULL; 1433 bool need_cvt = false; 1434 gcall *call 1435 = gimple_build_call (fn, 2, def, size_int (alignment)); 1436 g = call; 1437 if (!useless_type_conversion_p (TREE_TYPE (orig_arg), 1438 ptr_type_node)) 1439 need_cvt = true; 1440 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg); 1441 gimple_call_set_lhs (g, t); 1442 gimple_seq_add_stmt_without_update (&seq, g); 1443 if (need_cvt) 1444 { 1445 t = make_ssa_name (orig_arg); 1446 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g)); 1447 gimple_seq_add_stmt_without_update (&seq, g); 1448 } 1449 gsi_insert_seq_on_edge_immediate 1450 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq); 1451 1452 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 1453 node->create_edge (cgraph_node::get_create (fn), 1454 call, entry_bb->count); 1455 1456 imm_use_iterator iter; 1457 use_operand_p use_p; 1458 gimple *use_stmt; 1459 tree repl = gimple_get_lhs (g); 1460 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def) 1461 if (is_gimple_debug (use_stmt) || use_stmt == call) 1462 continue; 1463 else 1464 FOR_EACH_IMM_USE_ON_STMT (use_p, iter) 1465 SET_USE (use_p, repl); 1466 } 1467 } 1468 else if ((node->simdclone->args[i].arg_type 1469 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP) 1470 || (node->simdclone->args[i].arg_type 1471 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP) 1472 || (node->simdclone->args[i].arg_type 1473 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP) 1474 || (node->simdclone->args[i].arg_type 1475 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP)) 1476 { 1477 tree orig_arg = node->simdclone->args[i].orig_arg; 1478 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) 1479 || POINTER_TYPE_P (TREE_TYPE (orig_arg))); 1480 tree def = NULL_TREE; 1481 if (TREE_ADDRESSABLE (orig_arg)) 1482 { 1483 def = make_ssa_name (TREE_TYPE (orig_arg)); 1484 iter1 = make_ssa_name (TREE_TYPE (orig_arg)); 1485 if (incr_bb) 1486 iter2 = make_ssa_name (TREE_TYPE (orig_arg)); 1487 gsi = gsi_after_labels (entry_bb); 1488 g = gimple_build_assign (def, orig_arg); 1489 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 1490 } 1491 else 1492 { 1493 def = ssa_default_def (cfun, orig_arg); 1494 if (!def || has_zero_uses (def)) 1495 def = NULL_TREE; 1496 else 1497 { 1498 iter1 = make_ssa_name (orig_arg); 1499 if (incr_bb) 1500 iter2 = make_ssa_name (orig_arg); 1501 } 1502 } 1503 if (def) 1504 { 1505 phi = create_phi_node (iter1, body_bb); 1506 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION); 1507 if (incr_bb) 1508 { 1509 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION); 1510 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) 1511 ? PLUS_EXPR : POINTER_PLUS_EXPR; 1512 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) 1513 ? TREE_TYPE (orig_arg) : sizetype; 1514 tree addcst = simd_clone_linear_addend (node, i, addtype, 1515 entry_bb); 1516 gsi = gsi_last_bb (incr_bb); 1517 g = gimple_build_assign (iter2, code, iter1, addcst); 1518 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1519 } 1520 1521 imm_use_iterator iter; 1522 use_operand_p use_p; 1523 gimple *use_stmt; 1524 if (TREE_ADDRESSABLE (orig_arg)) 1525 { 1526 gsi = gsi_after_labels (body_bb); 1527 g = gimple_build_assign (orig_arg, iter1); 1528 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 1529 } 1530 else 1531 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def) 1532 if (use_stmt == phi) 1533 continue; 1534 else 1535 FOR_EACH_IMM_USE_ON_STMT (use_p, iter) 1536 SET_USE (use_p, iter1); 1537 } 1538 } 1539 else if (node->simdclone->args[i].arg_type 1540 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP 1541 || (node->simdclone->args[i].arg_type 1542 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP)) 1543 { 1544 tree orig_arg = node->simdclone->args[i].orig_arg; 1545 tree def = ssa_default_def (cfun, orig_arg); 1546 gcc_assert (!TREE_ADDRESSABLE (orig_arg) 1547 && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE); 1548 if (def && !has_zero_uses (def)) 1549 { 1550 tree rtype = TREE_TYPE (TREE_TYPE (orig_arg)); 1551 iter1 = make_ssa_name (orig_arg); 1552 if (incr_bb) 1553 iter2 = make_ssa_name (orig_arg); 1554 tree iter3 = make_ssa_name (rtype); 1555 tree iter4 = make_ssa_name (rtype); 1556 tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE; 1557 gsi = gsi_after_labels (entry_bb); 1558 gimple *load 1559 = gimple_build_assign (iter3, build_simple_mem_ref (def)); 1560 gsi_insert_before (&gsi, load, GSI_NEW_STMT); 1561 1562 tree array = node->simdclone->args[i].simd_array; 1563 TREE_ADDRESSABLE (array) = 1; 1564 tree ptr = build_fold_addr_expr (array); 1565 phi = create_phi_node (iter1, body_bb); 1566 add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION); 1567 if (incr_bb) 1568 { 1569 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION); 1570 g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1, 1571 TYPE_SIZE_UNIT (TREE_TYPE (iter3))); 1572 gsi = gsi_last_bb (incr_bb); 1573 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1574 } 1575 1576 phi = create_phi_node (iter4, body_bb); 1577 add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION); 1578 if (incr_bb) 1579 { 1580 add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION); 1581 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3)) 1582 ? PLUS_EXPR : POINTER_PLUS_EXPR; 1583 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3)) 1584 ? TREE_TYPE (iter3) : sizetype; 1585 tree addcst = simd_clone_linear_addend (node, i, addtype, 1586 entry_bb); 1587 g = gimple_build_assign (iter5, code, iter4, addcst); 1588 gsi = gsi_last_bb (incr_bb); 1589 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1590 } 1591 1592 g = gimple_build_assign (build_simple_mem_ref (iter1), iter4); 1593 gsi = gsi_after_labels (body_bb); 1594 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1595 1596 imm_use_iterator iter; 1597 use_operand_p use_p; 1598 gimple *use_stmt; 1599 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def) 1600 if (use_stmt == load) 1601 continue; 1602 else 1603 FOR_EACH_IMM_USE_ON_STMT (use_p, iter) 1604 SET_USE (use_p, iter1); 1605 1606 if (!TYPE_READONLY (rtype) && incr_bb) 1607 { 1608 tree v = make_ssa_name (rtype); 1609 tree aref = build4 (ARRAY_REF, rtype, array, 1610 size_zero_node, NULL_TREE, 1611 NULL_TREE); 1612 gsi = gsi_after_labels (new_exit_bb); 1613 g = gimple_build_assign (v, aref); 1614 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1615 g = gimple_build_assign (build_simple_mem_ref (def), v); 1616 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1617 } 1618 } 1619 } 1620 1621 calculate_dominance_info (CDI_DOMINATORS); 1622 if (loop) 1623 add_loop (loop, loop->header->loop_father); 1624 update_ssa (TODO_update_ssa); 1625 1626 pop_cfun (); 1627 } 1628 1629 /* If the function in NODE is tagged as an elemental SIMD function, 1630 create the appropriate SIMD clones. */ 1631 1632 void 1633 expand_simd_clones (struct cgraph_node *node) 1634 { 1635 tree attr = lookup_attribute ("omp declare simd", 1636 DECL_ATTRIBUTES (node->decl)); 1637 if (attr == NULL_TREE 1638 || node->global.inlined_to 1639 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl))) 1640 return; 1641 1642 /* Ignore 1643 #pragma omp declare simd 1644 extern int foo (); 1645 in C, there we don't know the argument types at all. */ 1646 if (!node->definition 1647 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE) 1648 return; 1649 1650 /* Call this before creating clone_info, as it might ggc_collect. */ 1651 if (node->definition && node->has_gimple_body_p ()) 1652 node->get_body (); 1653 1654 do 1655 { 1656 /* Start with parsing the "omp declare simd" attribute(s). */ 1657 bool inbranch_clause_specified; 1658 struct cgraph_simd_clone *clone_info 1659 = simd_clone_clauses_extract (node, TREE_VALUE (attr), 1660 &inbranch_clause_specified); 1661 if (clone_info == NULL) 1662 continue; 1663 1664 int orig_simdlen = clone_info->simdlen; 1665 tree base_type = simd_clone_compute_base_data_type (node, clone_info); 1666 /* The target can return 0 (no simd clones should be created), 1667 1 (just one ISA of simd clones should be created) or higher 1668 count of ISA variants. In that case, clone_info is initialized 1669 for the first ISA variant. */ 1670 int count 1671 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info, 1672 base_type, 0); 1673 if (count == 0) 1674 continue; 1675 1676 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED, 1677 also create one inbranch and one !inbranch clone of it. */ 1678 for (int i = 0; i < count * 2; i++) 1679 { 1680 struct cgraph_simd_clone *clone = clone_info; 1681 if (inbranch_clause_specified && (i & 1) != 0) 1682 continue; 1683 1684 if (i != 0) 1685 { 1686 clone = simd_clone_struct_alloc (clone_info->nargs 1687 + ((i & 1) != 0)); 1688 simd_clone_struct_copy (clone, clone_info); 1689 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen 1690 and simd_clone_adjust_argument_types did to the first 1691 clone's info. */ 1692 clone->nargs -= clone_info->inbranch; 1693 clone->simdlen = orig_simdlen; 1694 /* And call the target hook again to get the right ISA. */ 1695 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone, 1696 base_type, 1697 i / 2); 1698 if ((i & 1) != 0) 1699 clone->inbranch = 1; 1700 } 1701 1702 /* simd_clone_mangle might fail if such a clone has been created 1703 already. */ 1704 tree id = simd_clone_mangle (node, clone); 1705 if (id == NULL_TREE) 1706 { 1707 if (i == 0) 1708 clone->nargs += clone->inbranch; 1709 continue; 1710 } 1711 1712 /* Only when we are sure we want to create the clone actually 1713 clone the function (or definitions) or create another 1714 extern FUNCTION_DECL (for prototypes without definitions). */ 1715 struct cgraph_node *n = simd_clone_create (node); 1716 if (n == NULL) 1717 { 1718 if (i == 0) 1719 clone->nargs += clone->inbranch; 1720 continue; 1721 } 1722 1723 n->simdclone = clone; 1724 clone->origin = node; 1725 clone->next_clone = NULL; 1726 if (node->simd_clones == NULL) 1727 { 1728 clone->prev_clone = n; 1729 node->simd_clones = n; 1730 } 1731 else 1732 { 1733 clone->prev_clone = node->simd_clones->simdclone->prev_clone; 1734 clone->prev_clone->simdclone->next_clone = n; 1735 node->simd_clones->simdclone->prev_clone = n; 1736 } 1737 symtab->change_decl_assembler_name (n->decl, id); 1738 /* And finally adjust the return type, parameters and for 1739 definitions also function body. */ 1740 if (node->definition) 1741 simd_clone_adjust (n); 1742 else 1743 { 1744 TREE_TYPE (n->decl) 1745 = build_distinct_type_copy (TREE_TYPE (n->decl)); 1746 targetm.simd_clone.adjust (n); 1747 simd_clone_adjust_return_type (n); 1748 simd_clone_adjust_argument_types (n); 1749 } 1750 } 1751 } 1752 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr)))); 1753 } 1754 1755 /* Entry point for IPA simd clone creation pass. */ 1756 1757 static unsigned int 1758 ipa_omp_simd_clone (void) 1759 { 1760 struct cgraph_node *node; 1761 FOR_EACH_FUNCTION (node) 1762 expand_simd_clones (node); 1763 return 0; 1764 } 1765 1766 namespace { 1767 1768 const pass_data pass_data_omp_simd_clone = 1769 { 1770 SIMPLE_IPA_PASS, /* type */ 1771 "simdclone", /* name */ 1772 OPTGROUP_OMP, /* optinfo_flags */ 1773 TV_NONE, /* tv_id */ 1774 ( PROP_ssa | PROP_cfg ), /* properties_required */ 1775 0, /* properties_provided */ 1776 0, /* properties_destroyed */ 1777 0, /* todo_flags_start */ 1778 0, /* todo_flags_finish */ 1779 }; 1780 1781 class pass_omp_simd_clone : public simple_ipa_opt_pass 1782 { 1783 public: 1784 pass_omp_simd_clone(gcc::context *ctxt) 1785 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt) 1786 {} 1787 1788 /* opt_pass methods: */ 1789 virtual bool gate (function *); 1790 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); } 1791 }; 1792 1793 bool 1794 pass_omp_simd_clone::gate (function *) 1795 { 1796 return targetm.simd_clone.compute_vecsize_and_simdlen != NULL; 1797 } 1798 1799 } // anon namespace 1800 1801 simple_ipa_opt_pass * 1802 make_pass_omp_simd_clone (gcc::context *ctxt) 1803 { 1804 return new pass_omp_simd_clone (ctxt); 1805 } 1806