1 /* OMP constructs' SIMD clone supporting code.
2
3 Copyright (C) 2005-2022 Free Software Foundation, Inc.
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
11
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "cfghooks.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
31 #include "ssa.h"
32 #include "cgraph.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
37 #include "cfganal.h"
38 #include "gimplify.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
43 #include "tree-cfg.h"
44 #include "tree-into-ssa.h"
45 #include "tree-dfa.h"
46 #include "cfgloop.h"
47 #include "symbol-summary.h"
48 #include "ipa-param-manipulation.h"
49 #include "tree-eh.h"
50 #include "varasm.h"
51 #include "stringpool.h"
52 #include "attribs.h"
53 #include "omp-simd-clone.h"
54
55 /* Return the number of elements in vector type VECTYPE, which is associated
56 with a SIMD clone. At present these always have a constant length. */
57
58 static unsigned HOST_WIDE_INT
simd_clone_subparts(tree vectype)59 simd_clone_subparts (tree vectype)
60 {
61 return TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
62 }
63
64 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
65 of arguments to reserve space for. */
66
67 static struct cgraph_simd_clone *
simd_clone_struct_alloc(int nargs)68 simd_clone_struct_alloc (int nargs)
69 {
70 struct cgraph_simd_clone *clone_info;
71 size_t len = (sizeof (struct cgraph_simd_clone)
72 + nargs * sizeof (struct cgraph_simd_clone_arg));
73 clone_info = (struct cgraph_simd_clone *)
74 ggc_internal_cleared_alloc (len);
75 return clone_info;
76 }
77
78 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
79
80 static inline void
simd_clone_struct_copy(struct cgraph_simd_clone * to,struct cgraph_simd_clone * from)81 simd_clone_struct_copy (struct cgraph_simd_clone *to,
82 struct cgraph_simd_clone *from)
83 {
84 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
85 + ((from->nargs - from->inbranch)
86 * sizeof (struct cgraph_simd_clone_arg))));
87 }
88
89 /* Fill an empty vector ARGS with parameter types of function FNDECL. This
90 uses TYPE_ARG_TYPES if available, otherwise falls back to types of
91 DECL_ARGUMENTS types. */
92
93 static void
simd_clone_vector_of_formal_parm_types(vec<tree> * args,tree fndecl)94 simd_clone_vector_of_formal_parm_types (vec<tree> *args, tree fndecl)
95 {
96 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
97 {
98 push_function_arg_types (args, TREE_TYPE (fndecl));
99 return;
100 }
101 push_function_arg_decls (args, fndecl);
102 unsigned int i;
103 tree arg;
104 FOR_EACH_VEC_ELT (*args, i, arg)
105 (*args)[i] = TREE_TYPE ((*args)[i]);
106 }
107
108 /* Given a simd function in NODE, extract the simd specific
109 information from the OMP clauses passed in CLAUSES, and return
110 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
111 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
112 otherwise set to FALSE. */
113
114 static struct cgraph_simd_clone *
simd_clone_clauses_extract(struct cgraph_node * node,tree clauses,bool * inbranch_specified)115 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
116 bool *inbranch_specified)
117 {
118 auto_vec<tree> args;
119 simd_clone_vector_of_formal_parm_types (&args, node->decl);
120 tree t;
121 int n;
122 *inbranch_specified = false;
123
124 n = args.length ();
125 if (n > 0 && args.last () == void_type_node)
126 n--;
127
128 /* Allocate one more than needed just in case this is an in-branch
129 clone which will require a mask argument. */
130 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
131 clone_info->nargs = n;
132
133 if (!clauses)
134 goto out;
135
136 clauses = TREE_VALUE (clauses);
137 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
138 goto out;
139
140 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
141 {
142 switch (OMP_CLAUSE_CODE (t))
143 {
144 case OMP_CLAUSE_INBRANCH:
145 clone_info->inbranch = 1;
146 *inbranch_specified = true;
147 break;
148 case OMP_CLAUSE_NOTINBRANCH:
149 clone_info->inbranch = 0;
150 *inbranch_specified = true;
151 break;
152 case OMP_CLAUSE_SIMDLEN:
153 clone_info->simdlen
154 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
155 break;
156 case OMP_CLAUSE_LINEAR:
157 {
158 tree decl = OMP_CLAUSE_DECL (t);
159 tree step = OMP_CLAUSE_LINEAR_STEP (t);
160 int argno = TREE_INT_CST_LOW (decl);
161 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
162 {
163 enum cgraph_simd_clone_arg_type arg_type;
164 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
165 switch (OMP_CLAUSE_LINEAR_KIND (t))
166 {
167 case OMP_CLAUSE_LINEAR_REF:
168 arg_type
169 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
170 break;
171 case OMP_CLAUSE_LINEAR_UVAL:
172 arg_type
173 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
174 break;
175 case OMP_CLAUSE_LINEAR_VAL:
176 case OMP_CLAUSE_LINEAR_DEFAULT:
177 arg_type
178 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
179 break;
180 default:
181 gcc_unreachable ();
182 }
183 else
184 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
185 clone_info->args[argno].arg_type = arg_type;
186 clone_info->args[argno].linear_step = tree_to_shwi (step);
187 gcc_assert (clone_info->args[argno].linear_step >= 0
188 && clone_info->args[argno].linear_step < n);
189 }
190 else
191 {
192 if (POINTER_TYPE_P (args[argno]))
193 step = fold_convert (ssizetype, step);
194 if (!tree_fits_shwi_p (step))
195 {
196 warning_at (OMP_CLAUSE_LOCATION (t), 0,
197 "ignoring large linear step");
198 return NULL;
199 }
200 else if (integer_zerop (step))
201 {
202 warning_at (OMP_CLAUSE_LOCATION (t), 0,
203 "ignoring zero linear step");
204 return NULL;
205 }
206 else
207 {
208 enum cgraph_simd_clone_arg_type arg_type;
209 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
210 switch (OMP_CLAUSE_LINEAR_KIND (t))
211 {
212 case OMP_CLAUSE_LINEAR_REF:
213 arg_type
214 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
215 break;
216 case OMP_CLAUSE_LINEAR_UVAL:
217 arg_type
218 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
219 break;
220 case OMP_CLAUSE_LINEAR_VAL:
221 case OMP_CLAUSE_LINEAR_DEFAULT:
222 arg_type
223 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
224 break;
225 default:
226 gcc_unreachable ();
227 }
228 else
229 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
230 clone_info->args[argno].arg_type = arg_type;
231 clone_info->args[argno].linear_step = tree_to_shwi (step);
232 }
233 }
234 break;
235 }
236 case OMP_CLAUSE_UNIFORM:
237 {
238 tree decl = OMP_CLAUSE_DECL (t);
239 int argno = tree_to_uhwi (decl);
240 clone_info->args[argno].arg_type
241 = SIMD_CLONE_ARG_TYPE_UNIFORM;
242 break;
243 }
244 case OMP_CLAUSE_ALIGNED:
245 {
246 /* Ignore aligned (x) for declare simd, for the ABI we really
247 need an alignment specified. */
248 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (t) == NULL_TREE)
249 break;
250 tree decl = OMP_CLAUSE_DECL (t);
251 int argno = tree_to_uhwi (decl);
252 clone_info->args[argno].alignment
253 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
254 break;
255 }
256 default:
257 break;
258 }
259 }
260
261 out:
262 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node->decl))))
263 {
264 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
265 "ignoring %<#pragma omp declare simd%> on function "
266 "with %<_Atomic%> qualified return type");
267 return NULL;
268 }
269
270 for (unsigned int argno = 0; argno < clone_info->nargs; argno++)
271 if (TYPE_ATOMIC (args[argno])
272 && clone_info->args[argno].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM)
273 {
274 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
275 "ignoring %<#pragma omp declare simd%> on function "
276 "with %<_Atomic%> qualified non-%<uniform%> argument");
277 args.release ();
278 return NULL;
279 }
280
281 return clone_info;
282 }
283
284 /* Given a SIMD clone in NODE, calculate the characteristic data
285 type and return the coresponding type. The characteristic data
286 type is computed as described in the Intel Vector ABI. */
287
288 static tree
simd_clone_compute_base_data_type(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)289 simd_clone_compute_base_data_type (struct cgraph_node *node,
290 struct cgraph_simd_clone *clone_info)
291 {
292 tree type = integer_type_node;
293 tree fndecl = node->decl;
294
295 /* a) For non-void function, the characteristic data type is the
296 return type. */
297 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
298 type = TREE_TYPE (TREE_TYPE (fndecl));
299
300 /* b) If the function has any non-uniform, non-linear parameters,
301 then the characteristic data type is the type of the first
302 such parameter. */
303 else
304 {
305 auto_vec<tree> map;
306 simd_clone_vector_of_formal_parm_types (&map, fndecl);
307 for (unsigned int i = 0; i < clone_info->nargs; ++i)
308 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
309 {
310 type = map[i];
311 break;
312 }
313 }
314
315 /* c) If the characteristic data type determined by a) or b) above
316 is struct, union, or class type which is pass-by-value (except
317 for the type that maps to the built-in complex data type), the
318 characteristic data type is int. */
319 if (RECORD_OR_UNION_TYPE_P (type)
320 && !aggregate_value_p (type, NULL)
321 && TREE_CODE (type) != COMPLEX_TYPE)
322 return integer_type_node;
323
324 /* d) If none of the above three classes is applicable, the
325 characteristic data type is int. */
326
327 return type;
328
329 /* e) For Intel Xeon Phi native and offload compilation, if the
330 resulting characteristic data type is 8-bit or 16-bit integer
331 data type, the characteristic data type is int. */
332 /* Well, we don't handle Xeon Phi yet. */
333 }
334
335 static tree
simd_clone_mangle(struct cgraph_node * node,struct cgraph_simd_clone * clone_info)336 simd_clone_mangle (struct cgraph_node *node,
337 struct cgraph_simd_clone *clone_info)
338 {
339 char vecsize_mangle = clone_info->vecsize_mangle;
340 char mask = clone_info->inbranch ? 'M' : 'N';
341 poly_uint64 simdlen = clone_info->simdlen;
342 unsigned int n;
343 pretty_printer pp;
344
345 gcc_assert (vecsize_mangle && maybe_ne (simdlen, 0U));
346
347 pp_string (&pp, "_ZGV");
348 pp_character (&pp, vecsize_mangle);
349 pp_character (&pp, mask);
350 /* For now, simdlen is always constant, while variable simdlen pp 'n'. */
351 unsigned int len = simdlen.to_constant ();
352 pp_decimal_int (&pp, (len));
353
354 for (n = 0; n < clone_info->nargs; ++n)
355 {
356 struct cgraph_simd_clone_arg arg = clone_info->args[n];
357
358 switch (arg.arg_type)
359 {
360 case SIMD_CLONE_ARG_TYPE_UNIFORM:
361 pp_character (&pp, 'u');
362 break;
363 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
364 pp_character (&pp, 'l');
365 goto mangle_linear;
366 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
367 pp_character (&pp, 'R');
368 goto mangle_linear;
369 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
370 pp_character (&pp, 'L');
371 goto mangle_linear;
372 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
373 pp_character (&pp, 'U');
374 goto mangle_linear;
375 mangle_linear:
376 gcc_assert (arg.linear_step != 0);
377 if (arg.linear_step > 1)
378 pp_unsigned_wide_integer (&pp, arg.linear_step);
379 else if (arg.linear_step < 0)
380 {
381 pp_character (&pp, 'n');
382 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
383 arg.linear_step));
384 }
385 break;
386 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
387 pp_string (&pp, "ls");
388 pp_unsigned_wide_integer (&pp, arg.linear_step);
389 break;
390 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
391 pp_string (&pp, "Rs");
392 pp_unsigned_wide_integer (&pp, arg.linear_step);
393 break;
394 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
395 pp_string (&pp, "Ls");
396 pp_unsigned_wide_integer (&pp, arg.linear_step);
397 break;
398 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
399 pp_string (&pp, "Us");
400 pp_unsigned_wide_integer (&pp, arg.linear_step);
401 break;
402 default:
403 pp_character (&pp, 'v');
404 }
405 if (arg.alignment)
406 {
407 pp_character (&pp, 'a');
408 pp_decimal_int (&pp, arg.alignment);
409 }
410 }
411
412 pp_underscore (&pp);
413 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
414 if (*str == '*')
415 ++str;
416 pp_string (&pp, str);
417 str = pp_formatted_text (&pp);
418
419 /* If there already is a SIMD clone with the same mangled name, don't
420 add another one. This can happen e.g. for
421 #pragma omp declare simd
422 #pragma omp declare simd simdlen(8)
423 int foo (int, int);
424 if the simdlen is assumed to be 8 for the first one, etc. */
425 for (struct cgraph_node *clone = node->simd_clones; clone;
426 clone = clone->simdclone->next_clone)
427 if (id_equal (DECL_ASSEMBLER_NAME (clone->decl), str))
428 return NULL_TREE;
429
430 return get_identifier (str);
431 }
432
433 /* Create a simd clone of OLD_NODE and return it. */
434
435 static struct cgraph_node *
simd_clone_create(struct cgraph_node * old_node)436 simd_clone_create (struct cgraph_node *old_node)
437 {
438 struct cgraph_node *new_node;
439 if (old_node->definition)
440 {
441 if (!old_node->has_gimple_body_p ())
442 return NULL;
443 old_node->get_body ();
444 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
445 NULL, NULL,
446 "simdclone");
447 }
448 else
449 {
450 tree old_decl = old_node->decl;
451 tree new_decl = copy_node (old_node->decl);
452 DECL_NAME (new_decl) = clone_function_name_numbered (old_decl,
453 "simdclone");
454 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
455 SET_DECL_RTL (new_decl, NULL);
456 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
457 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
458 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
459 if (old_node->in_other_partition)
460 new_node->in_other_partition = 1;
461 }
462 if (new_node == NULL)
463 return new_node;
464
465 set_decl_built_in_function (new_node->decl, NOT_BUILT_IN, 0);
466 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
467 DECL_COMDAT (new_node->decl) = DECL_COMDAT (old_node->decl);
468 DECL_WEAK (new_node->decl) = DECL_WEAK (old_node->decl);
469 DECL_EXTERNAL (new_node->decl) = DECL_EXTERNAL (old_node->decl);
470 DECL_VISIBILITY_SPECIFIED (new_node->decl)
471 = DECL_VISIBILITY_SPECIFIED (old_node->decl);
472 DECL_VISIBILITY (new_node->decl) = DECL_VISIBILITY (old_node->decl);
473 DECL_DLLIMPORT_P (new_node->decl) = DECL_DLLIMPORT_P (old_node->decl);
474 if (DECL_ONE_ONLY (old_node->decl))
475 make_decl_one_only (new_node->decl, DECL_ASSEMBLER_NAME (new_node->decl));
476
477 /* The method cgraph_version_clone_with_body () will force the new
478 symbol local. Undo this, and inherit external visibility from
479 the old node. */
480 new_node->local = old_node->local;
481 new_node->externally_visible = old_node->externally_visible;
482 new_node->calls_declare_variant_alt = old_node->calls_declare_variant_alt;
483
484 return new_node;
485 }
486
487 /* Adjust the return type of the given function to its appropriate
488 vector counterpart. Returns a simd array to be used throughout the
489 function as a return value. */
490
491 static tree
simd_clone_adjust_return_type(struct cgraph_node * node)492 simd_clone_adjust_return_type (struct cgraph_node *node)
493 {
494 tree fndecl = node->decl;
495 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
496 poly_uint64 veclen;
497 tree t;
498
499 /* Adjust the function return type. */
500 if (orig_rettype == void_type_node)
501 return NULL_TREE;
502 t = TREE_TYPE (TREE_TYPE (fndecl));
503 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
504 veclen = node->simdclone->vecsize_int;
505 else
506 veclen = node->simdclone->vecsize_float;
507 veclen = exact_div (veclen, GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t)));
508 if (multiple_p (veclen, node->simdclone->simdlen))
509 veclen = node->simdclone->simdlen;
510 if (POINTER_TYPE_P (t))
511 t = pointer_sized_int_node;
512 if (known_eq (veclen, node->simdclone->simdlen))
513 t = build_vector_type (t, node->simdclone->simdlen);
514 else
515 {
516 t = build_vector_type (t, veclen);
517 t = build_array_type_nelts (t, exact_div (node->simdclone->simdlen,
518 veclen));
519 }
520 TREE_TYPE (TREE_TYPE (fndecl)) = t;
521 if (!node->definition)
522 return NULL_TREE;
523
524 t = DECL_RESULT (fndecl);
525 /* Adjust the DECL_RESULT. */
526 gcc_assert (TREE_TYPE (t) != void_type_node);
527 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
528 relayout_decl (t);
529
530 tree atype = build_array_type_nelts (orig_rettype,
531 node->simdclone->simdlen);
532 if (maybe_ne (veclen, node->simdclone->simdlen))
533 return build1 (VIEW_CONVERT_EXPR, atype, t);
534
535 /* Set up a SIMD array to use as the return value. */
536 tree retval = create_tmp_var_raw (atype, "retval");
537 gimple_add_tmp_var (retval);
538 return retval;
539 }
540
541 /* Each vector argument has a corresponding array to be used locally
542 as part of the eventual loop. Create such temporary array and
543 return it.
544
545 PREFIX is the prefix to be used for the temporary.
546
547 TYPE is the inner element type.
548
549 SIMDLEN is the number of elements. */
550
551 static tree
create_tmp_simd_array(const char * prefix,tree type,poly_uint64 simdlen)552 create_tmp_simd_array (const char *prefix, tree type, poly_uint64 simdlen)
553 {
554 tree atype = build_array_type_nelts (type, simdlen);
555 tree avar = create_tmp_var_raw (atype, prefix);
556 gimple_add_tmp_var (avar);
557 return avar;
558 }
559
560 /* Modify the function argument types to their corresponding vector
561 counterparts if appropriate. Also, create one array for each simd
562 argument to be used locally when using the function arguments as
563 part of the loop.
564
565 NODE is the function whose arguments are to be adjusted.
566
567 If NODE does not represent function definition, returns NULL. Otherwise
568 returns an adjustment class that will be filled describing how the argument
569 declarations will be remapped. New arguments which are not to be remapped
570 are marked with USER_FLAG. */
571
572 static ipa_param_body_adjustments *
simd_clone_adjust_argument_types(struct cgraph_node * node)573 simd_clone_adjust_argument_types (struct cgraph_node *node)
574 {
575 auto_vec<tree> args;
576
577 if (node->definition)
578 push_function_arg_decls (&args, node->decl);
579 else
580 simd_clone_vector_of_formal_parm_types (&args, node->decl);
581 struct cgraph_simd_clone *sc = node->simdclone;
582 vec<ipa_adjusted_param, va_gc> *new_params = NULL;
583 vec_safe_reserve (new_params, sc->nargs);
584 unsigned i, j, k;
585 poly_uint64 veclen;
586
587 for (i = 0; i < sc->nargs; ++i)
588 {
589 ipa_adjusted_param adj;
590 memset (&adj, 0, sizeof (adj));
591 tree parm = args[i];
592 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
593 adj.base_index = i;
594 adj.prev_clone_index = i;
595
596 sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
597 sc->args[i].orig_type = parm_type;
598
599 switch (sc->args[i].arg_type)
600 {
601 default:
602 /* No adjustment necessary for scalar arguments. */
603 adj.op = IPA_PARAM_OP_COPY;
604 break;
605 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
606 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
607 if (node->definition)
608 sc->args[i].simd_array
609 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
610 TREE_TYPE (parm_type),
611 sc->simdlen);
612 adj.op = IPA_PARAM_OP_COPY;
613 break;
614 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
615 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
616 case SIMD_CLONE_ARG_TYPE_VECTOR:
617 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
618 veclen = sc->vecsize_int;
619 else
620 veclen = sc->vecsize_float;
621 veclen = exact_div (veclen,
622 GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type)));
623 if (multiple_p (veclen, sc->simdlen))
624 veclen = sc->simdlen;
625 adj.op = IPA_PARAM_OP_NEW;
626 adj.param_prefix_index = IPA_PARAM_PREFIX_SIMD;
627 if (POINTER_TYPE_P (parm_type))
628 adj.type = build_vector_type (pointer_sized_int_node, veclen);
629 else
630 adj.type = build_vector_type (parm_type, veclen);
631 sc->args[i].vector_type = adj.type;
632 k = vector_unroll_factor (sc->simdlen, veclen);
633 for (j = 1; j < k; j++)
634 {
635 vec_safe_push (new_params, adj);
636 if (j == 1)
637 {
638 memset (&adj, 0, sizeof (adj));
639 adj.op = IPA_PARAM_OP_NEW;
640 adj.user_flag = 1;
641 adj.param_prefix_index = IPA_PARAM_PREFIX_SIMD;
642 adj.base_index = i;
643 adj.prev_clone_index = i;
644 adj.type = sc->args[i].vector_type;
645 }
646 }
647
648 if (node->definition)
649 sc->args[i].simd_array
650 = create_tmp_simd_array (DECL_NAME (parm)
651 ? IDENTIFIER_POINTER (DECL_NAME (parm))
652 : NULL, parm_type, sc->simdlen);
653 }
654 vec_safe_push (new_params, adj);
655 }
656
657 if (sc->inbranch)
658 {
659 tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
660 ipa_adjusted_param adj;
661 memset (&adj, 0, sizeof (adj));
662 adj.op = IPA_PARAM_OP_NEW;
663 adj.user_flag = 1;
664 adj.param_prefix_index = IPA_PARAM_PREFIX_MASK;
665
666 adj.base_index = i;
667 adj.prev_clone_index = i;
668 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
669 veclen = sc->vecsize_int;
670 else
671 veclen = sc->vecsize_float;
672 veclen = exact_div (veclen,
673 GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type)));
674 if (multiple_p (veclen, sc->simdlen))
675 veclen = sc->simdlen;
676 if (sc->mask_mode != VOIDmode)
677 adj.type
678 = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
679 else if (POINTER_TYPE_P (base_type))
680 adj.type = build_vector_type (pointer_sized_int_node, veclen);
681 else
682 adj.type = build_vector_type (base_type, veclen);
683 vec_safe_push (new_params, adj);
684
685 k = vector_unroll_factor (sc->simdlen, veclen);
686 for (j = 1; j < k; j++)
687 vec_safe_push (new_params, adj);
688
689 /* We have previously allocated one extra entry for the mask. Use
690 it and fill it. */
691 sc->nargs++;
692 if (sc->mask_mode != VOIDmode)
693 base_type = boolean_type_node;
694 if (node->definition)
695 {
696 sc->args[i].orig_arg
697 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
698 if (sc->mask_mode == VOIDmode)
699 sc->args[i].simd_array
700 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
701 else if (k > 1)
702 sc->args[i].simd_array
703 = create_tmp_simd_array ("mask", adj.type, k);
704 else
705 sc->args[i].simd_array = NULL_TREE;
706 }
707 sc->args[i].orig_type = base_type;
708 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
709 }
710
711 if (node->definition)
712 {
713 ipa_param_body_adjustments *adjustments
714 = new ipa_param_body_adjustments (new_params, node->decl);
715
716 adjustments->modify_formal_parameters ();
717 return adjustments;
718 }
719 else
720 {
721 tree new_arg_types = NULL_TREE, new_reversed;
722 bool last_parm_void = false;
723 if (args.length () > 0 && args.last () == void_type_node)
724 last_parm_void = true;
725
726 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
727 j = vec_safe_length (new_params);
728 for (i = 0; i < j; i++)
729 {
730 struct ipa_adjusted_param *adj = &(*new_params)[i];
731 tree ptype;
732 if (adj->op == IPA_PARAM_OP_COPY)
733 ptype = args[adj->base_index];
734 else
735 ptype = adj->type;
736 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
737 }
738 new_reversed = nreverse (new_arg_types);
739 if (last_parm_void)
740 {
741 if (new_reversed)
742 TREE_CHAIN (new_arg_types) = void_list_node;
743 else
744 new_reversed = void_list_node;
745 }
746 TYPE_ARG_TYPES (TREE_TYPE (node->decl)) = new_reversed;
747 return NULL;
748 }
749 }
750
751 /* Initialize and copy the function arguments in NODE to their
752 corresponding local simd arrays. Returns a fresh gimple_seq with
753 the instruction sequence generated. */
754
755 static gimple_seq
simd_clone_init_simd_arrays(struct cgraph_node * node,ipa_param_body_adjustments * adjustments)756 simd_clone_init_simd_arrays (struct cgraph_node *node,
757 ipa_param_body_adjustments *adjustments)
758 {
759 gimple_seq seq = NULL;
760 unsigned i = 0, j = 0, k;
761
762 for (tree arg = DECL_ARGUMENTS (node->decl);
763 arg;
764 arg = DECL_CHAIN (arg), i++, j++)
765 {
766 if ((*adjustments->m_adj_params)[j].op == IPA_PARAM_OP_COPY
767 || POINTER_TYPE_P (TREE_TYPE (arg)))
768 continue;
769
770 node->simdclone->args[i].vector_arg = arg;
771
772 tree array = node->simdclone->args[i].simd_array;
773 if (node->simdclone->mask_mode != VOIDmode
774 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
775 {
776 if (array == NULL_TREE)
777 continue;
778 unsigned int l
779 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
780 for (k = 0; k <= l; k++)
781 {
782 if (k)
783 {
784 arg = DECL_CHAIN (arg);
785 j++;
786 }
787 tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
788 array, size_int (k), NULL, NULL);
789 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
790 gimplify_and_add (t, &seq);
791 }
792 continue;
793 }
794 if (known_eq (simd_clone_subparts (TREE_TYPE (arg)),
795 node->simdclone->simdlen))
796 {
797 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
798 tree ptr = build_fold_addr_expr (array);
799 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
800 build_int_cst (ptype, 0));
801 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
802 gimplify_and_add (t, &seq);
803 }
804 else
805 {
806 unsigned int simdlen = simd_clone_subparts (TREE_TYPE (arg));
807 unsigned int times = vector_unroll_factor (node->simdclone->simdlen,
808 simdlen);
809 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
810 for (k = 0; k < times; k++)
811 {
812 tree ptr = build_fold_addr_expr (array);
813 int elemsize;
814 if (k)
815 {
816 arg = DECL_CHAIN (arg);
817 j++;
818 }
819 tree elemtype = TREE_TYPE (TREE_TYPE (arg));
820 elemsize = GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype));
821 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
822 build_int_cst (ptype, k * elemsize * simdlen));
823 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
824 gimplify_and_add (t, &seq);
825 }
826 }
827 }
828 return seq;
829 }
830
831 /* Callback info for ipa_simd_modify_stmt_ops below. */
832
833 struct modify_stmt_info {
834 ipa_param_body_adjustments *adjustments;
835 gimple *stmt;
836 gimple *after_stmt;
837 /* True if the parent statement was modified by
838 ipa_simd_modify_stmt_ops. */
839 bool modified;
840 };
841
842 /* Callback for walk_gimple_op.
843
844 Adjust operands from a given statement as specified in the
845 adjustments vector in the callback data. */
846
847 static tree
ipa_simd_modify_stmt_ops(tree * tp,int * walk_subtrees,void * data)848 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
849 {
850 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
851 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
852 tree *orig_tp = tp;
853 if (TREE_CODE (*tp) == ADDR_EXPR)
854 tp = &TREE_OPERAND (*tp, 0);
855
856 if (TREE_CODE (*tp) == BIT_FIELD_REF
857 || TREE_CODE (*tp) == IMAGPART_EXPR
858 || TREE_CODE (*tp) == REALPART_EXPR)
859 tp = &TREE_OPERAND (*tp, 0);
860
861 tree repl = NULL_TREE;
862 ipa_param_body_replacement *pbr = NULL;
863
864 if (TREE_CODE (*tp) == PARM_DECL)
865 {
866 pbr = info->adjustments->get_expr_replacement (*tp, true);
867 if (pbr)
868 repl = pbr->repl;
869 }
870 else if (TYPE_P (*tp))
871 *walk_subtrees = 0;
872
873 if (repl)
874 repl = unshare_expr (repl);
875 else
876 {
877 if (tp != orig_tp)
878 {
879 *walk_subtrees = 0;
880 bool modified = info->modified;
881 info->modified = false;
882 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
883 if (!info->modified)
884 {
885 info->modified = modified;
886 return NULL_TREE;
887 }
888 info->modified = modified;
889 repl = *tp;
890 }
891 else
892 return NULL_TREE;
893 }
894
895 if (tp != orig_tp)
896 {
897 if (gimple_code (info->stmt) == GIMPLE_PHI
898 && pbr
899 && TREE_CODE (*orig_tp) == ADDR_EXPR
900 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL
901 && pbr->dummy)
902 {
903 gcc_assert (TREE_CODE (pbr->dummy) == SSA_NAME);
904 *orig_tp = pbr->dummy;
905 info->modified = true;
906 return NULL_TREE;
907 }
908
909 repl = build_fold_addr_expr (repl);
910 gimple *stmt;
911 if (is_gimple_debug (info->stmt))
912 {
913 tree vexpr = build_debug_expr_decl (TREE_TYPE (repl));
914 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
915 repl = vexpr;
916 }
917 else
918 {
919 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
920 repl = gimple_assign_lhs (stmt);
921 }
922 gimple_stmt_iterator gsi;
923 if (gimple_code (info->stmt) == GIMPLE_PHI)
924 {
925 if (info->after_stmt)
926 gsi = gsi_for_stmt (info->after_stmt);
927 else
928 gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
929 /* Cache SSA_NAME for next time. */
930 if (pbr
931 && TREE_CODE (*orig_tp) == ADDR_EXPR
932 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL)
933 {
934 gcc_assert (!pbr->dummy);
935 pbr->dummy = repl;
936 }
937 }
938 else
939 gsi = gsi_for_stmt (info->stmt);
940 if (info->after_stmt)
941 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
942 else
943 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
944 if (gimple_code (info->stmt) == GIMPLE_PHI)
945 info->after_stmt = stmt;
946 *orig_tp = repl;
947 }
948 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
949 {
950 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
951 *tp = vce;
952 }
953 else
954 *tp = repl;
955
956 info->modified = true;
957 return NULL_TREE;
958 }
959
960 /* Traverse the function body and perform all modifications as
961 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
962 modified such that the replacement/reduction value will now be an
963 offset into the corresponding simd_array.
964
965 This function will replace all function argument uses with their
966 corresponding simd array elements, and ajust the return values
967 accordingly. */
968
969 static void
ipa_simd_modify_function_body(struct cgraph_node * node,ipa_param_body_adjustments * adjustments,tree retval_array,tree iter)970 ipa_simd_modify_function_body (struct cgraph_node *node,
971 ipa_param_body_adjustments *adjustments,
972 tree retval_array, tree iter)
973 {
974 basic_block bb;
975 unsigned int i, j;
976
977
978 /* Register replacements for every function argument use to an offset into
979 the corresponding simd_array. */
980 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
981 {
982 if (!node->simdclone->args[i].vector_arg
983 || (*adjustments->m_adj_params)[j].user_flag)
984 continue;
985
986 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
987 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
988 tree r = build4 (ARRAY_REF, basetype, node->simdclone->args[i].simd_array,
989 iter, NULL_TREE, NULL_TREE);
990 adjustments->register_replacement (&(*adjustments->m_adj_params)[j], r);
991
992 if (multiple_p (node->simdclone->simdlen, simd_clone_subparts (vectype)))
993 j += vector_unroll_factor (node->simdclone->simdlen,
994 simd_clone_subparts (vectype)) - 1;
995 }
996
997 tree name;
998 FOR_EACH_SSA_NAME (i, name, cfun)
999 {
1000 tree base_var;
1001 if (SSA_NAME_VAR (name)
1002 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL
1003 && (base_var
1004 = adjustments->get_replacement_ssa_base (SSA_NAME_VAR (name))))
1005 {
1006 if (SSA_NAME_IS_DEFAULT_DEF (name))
1007 {
1008 tree old_decl = SSA_NAME_VAR (name);
1009 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1010 gimple_stmt_iterator gsi = gsi_after_labels (bb);
1011 tree repl = adjustments->lookup_replacement (old_decl, 0);
1012 gcc_checking_assert (repl);
1013 repl = unshare_expr (repl);
1014 set_ssa_default_def (cfun, old_decl, NULL_TREE);
1015 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
1016 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
1017 gimple *stmt = gimple_build_assign (name, repl);
1018 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
1019 }
1020 else
1021 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
1022 }
1023 }
1024
1025 struct modify_stmt_info info;
1026 info.adjustments = adjustments;
1027
1028 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
1029 {
1030 gimple_stmt_iterator gsi;
1031
1032 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1033 {
1034 gphi *phi = as_a <gphi *> (gsi_stmt (gsi));
1035 int i, n = gimple_phi_num_args (phi);
1036 info.stmt = phi;
1037 info.after_stmt = NULL;
1038 struct walk_stmt_info wi;
1039 memset (&wi, 0, sizeof (wi));
1040 info.modified = false;
1041 wi.info = &info;
1042 for (i = 0; i < n; ++i)
1043 {
1044 int walk_subtrees = 1;
1045 tree arg = gimple_phi_arg_def (phi, i);
1046 tree op = arg;
1047 ipa_simd_modify_stmt_ops (&op, &walk_subtrees, &wi);
1048 if (op != arg)
1049 {
1050 SET_PHI_ARG_DEF (phi, i, op);
1051 gcc_assert (TREE_CODE (op) == SSA_NAME);
1052 if (gimple_phi_arg_edge (phi, i)->flags & EDGE_ABNORMAL)
1053 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op) = 1;
1054 }
1055 }
1056 }
1057
1058 gsi = gsi_start_bb (bb);
1059 while (!gsi_end_p (gsi))
1060 {
1061 gimple *stmt = gsi_stmt (gsi);
1062 info.stmt = stmt;
1063 info.after_stmt = NULL;
1064 struct walk_stmt_info wi;
1065
1066 memset (&wi, 0, sizeof (wi));
1067 info.modified = false;
1068 wi.info = &info;
1069 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
1070
1071 if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
1072 {
1073 tree retval = gimple_return_retval (return_stmt);
1074 edge e = find_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun));
1075 e->flags |= EDGE_FALLTHRU;
1076 if (!retval)
1077 {
1078 gsi_remove (&gsi, true);
1079 continue;
1080 }
1081
1082 /* Replace `return foo' with `retval_array[iter] = foo'. */
1083 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
1084 retval_array, iter, NULL, NULL);
1085 stmt = gimple_build_assign (ref, retval);
1086 gsi_replace (&gsi, stmt, true);
1087 info.modified = true;
1088 }
1089
1090 if (info.modified)
1091 {
1092 update_stmt (stmt);
1093 /* If the above changed the var of a debug bind into something
1094 different, remove the debug stmt. We could also for all the
1095 replaced parameters add VAR_DECLs for debug info purposes,
1096 add debug stmts for those to be the simd array accesses and
1097 replace debug stmt var operand with that var. Debugging of
1098 vectorized loops doesn't work too well, so don't bother for
1099 now. */
1100 if ((gimple_debug_bind_p (stmt)
1101 && !DECL_P (gimple_debug_bind_get_var (stmt)))
1102 || (gimple_debug_source_bind_p (stmt)
1103 && !DECL_P (gimple_debug_source_bind_get_var (stmt))))
1104 {
1105 gsi_remove (&gsi, true);
1106 continue;
1107 }
1108 if (maybe_clean_eh_stmt (stmt))
1109 gimple_purge_dead_eh_edges (gimple_bb (stmt));
1110 }
1111 gsi_next (&gsi);
1112 }
1113 }
1114 }
1115
1116 /* Helper function of simd_clone_adjust, return linear step addend
1117 of Ith argument. */
1118
1119 static tree
simd_clone_linear_addend(struct cgraph_node * node,unsigned int i,tree addtype,basic_block entry_bb)1120 simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
1121 tree addtype, basic_block entry_bb)
1122 {
1123 tree ptype = NULL_TREE;
1124 switch (node->simdclone->args[i].arg_type)
1125 {
1126 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
1127 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
1128 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
1129 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
1130 return build_int_cst (addtype, node->simdclone->args[i].linear_step);
1131 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
1132 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
1133 ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
1134 break;
1135 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
1136 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
1137 ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
1138 break;
1139 default:
1140 gcc_unreachable ();
1141 }
1142
1143 unsigned int idx = node->simdclone->args[i].linear_step;
1144 tree arg = node->simdclone->args[idx].orig_arg;
1145 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
1146 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1147 gimple *g;
1148 tree ret;
1149 if (is_gimple_reg (arg))
1150 ret = get_or_create_ssa_default_def (cfun, arg);
1151 else
1152 {
1153 g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
1154 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1155 ret = gimple_assign_lhs (g);
1156 }
1157 if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
1158 {
1159 g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
1160 build_simple_mem_ref (ret));
1161 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1162 ret = gimple_assign_lhs (g);
1163 }
1164 if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
1165 {
1166 g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
1167 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1168 ret = gimple_assign_lhs (g);
1169 }
1170 if (POINTER_TYPE_P (ptype))
1171 {
1172 tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
1173 if (size && TREE_CODE (size) == INTEGER_CST)
1174 {
1175 g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
1176 ret, fold_convert (addtype, size));
1177 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1178 ret = gimple_assign_lhs (g);
1179 }
1180 }
1181 return ret;
1182 }
1183
1184 /* Adjust the argument types in NODE to their appropriate vector
1185 counterparts. */
1186
1187 static void
simd_clone_adjust(struct cgraph_node * node)1188 simd_clone_adjust (struct cgraph_node *node)
1189 {
1190 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
1191
1192 TREE_TYPE (node->decl) = build_distinct_type_copy (TREE_TYPE (node->decl));
1193 targetm.simd_clone.adjust (node);
1194
1195 tree retval = simd_clone_adjust_return_type (node);
1196 ipa_param_body_adjustments *adjustments
1197 = simd_clone_adjust_argument_types (node);
1198 gcc_assert (adjustments);
1199
1200 push_gimplify_context ();
1201
1202 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
1203
1204 /* Adjust all uses of vector arguments accordingly. Adjust all
1205 return values accordingly. */
1206 tree iter = create_tmp_var (unsigned_type_node, "iter");
1207 tree iter1 = make_ssa_name (iter);
1208 tree iter2 = NULL_TREE;
1209 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
1210 delete adjustments;
1211
1212 /* Initialize the iteration variable. */
1213 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1214 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
1215 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1216 /* Insert the SIMD array and iv initialization at function
1217 entry. */
1218 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
1219
1220 pop_gimplify_context (NULL);
1221
1222 gimple *g;
1223 basic_block incr_bb = NULL;
1224 class loop *loop = NULL;
1225
1226 /* Create a new BB right before the original exit BB, to hold the
1227 iteration increment and the condition/branch. */
1228 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1229 {
1230 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
1231 incr_bb = create_empty_bb (orig_exit);
1232 incr_bb->count = profile_count::zero ();
1233 add_bb_to_loop (incr_bb, body_bb->loop_father);
1234 while (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1235 {
1236 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1237 redirect_edge_succ (e, incr_bb);
1238 incr_bb->count += e->count ();
1239 }
1240 }
1241 else if (node->simdclone->inbranch)
1242 {
1243 incr_bb = create_empty_bb (entry_bb);
1244 incr_bb->count = profile_count::zero ();
1245 add_bb_to_loop (incr_bb, body_bb->loop_father);
1246 }
1247
1248 if (incr_bb)
1249 {
1250 make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1251 gsi = gsi_last_bb (incr_bb);
1252 iter2 = make_ssa_name (iter);
1253 g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
1254 build_int_cst (unsigned_type_node, 1));
1255 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1256
1257 /* Mostly annotate the loop for the vectorizer (the rest is done
1258 below). */
1259 loop = alloc_loop ();
1260 cfun->has_force_vectorize_loops = true;
1261 /* For now, simlen is always constant. */
1262 loop->safelen = node->simdclone->simdlen.to_constant ();
1263 loop->force_vectorize = true;
1264 loop->header = body_bb;
1265 }
1266
1267 /* Branch around the body if the mask applies. */
1268 if (node->simdclone->inbranch)
1269 {
1270 gsi = gsi_last_bb (loop->header);
1271 tree mask_array
1272 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
1273 tree mask;
1274 if (node->simdclone->mask_mode != VOIDmode)
1275 {
1276 tree shift_cnt;
1277 if (mask_array == NULL_TREE)
1278 {
1279 tree arg = node->simdclone->args[node->simdclone->nargs
1280 - 1].vector_arg;
1281 mask = get_or_create_ssa_default_def (cfun, arg);
1282 shift_cnt = iter1;
1283 }
1284 else
1285 {
1286 tree maskt = TREE_TYPE (mask_array);
1287 int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
1288 /* For now, c must be constant here. */
1289 c = exact_div (node->simdclone->simdlen, c + 1).to_constant ();
1290 int s = exact_log2 (c);
1291 gcc_assert (s > 0);
1292 c--;
1293 tree idx = make_ssa_name (TREE_TYPE (iter1));
1294 g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
1295 build_int_cst (NULL_TREE, s));
1296 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1297 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1298 tree aref = build4 (ARRAY_REF,
1299 TREE_TYPE (TREE_TYPE (mask_array)),
1300 mask_array, idx, NULL, NULL);
1301 g = gimple_build_assign (mask, aref);
1302 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1303 shift_cnt = make_ssa_name (TREE_TYPE (iter1));
1304 g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
1305 build_int_cst (TREE_TYPE (iter1), c));
1306 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1307 }
1308 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1309 RSHIFT_EXPR, mask, shift_cnt);
1310 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1311 mask = gimple_assign_lhs (g);
1312 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1313 BIT_AND_EXPR, mask,
1314 build_int_cst (TREE_TYPE (mask), 1));
1315 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1316 mask = gimple_assign_lhs (g);
1317 }
1318 else
1319 {
1320 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1321 tree aref = build4 (ARRAY_REF,
1322 TREE_TYPE (TREE_TYPE (mask_array)),
1323 mask_array, iter1, NULL, NULL);
1324 g = gimple_build_assign (mask, aref);
1325 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1326 int bitsize = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref)));
1327 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
1328 {
1329 aref = build1 (VIEW_CONVERT_EXPR,
1330 build_nonstandard_integer_type (bitsize, 0),
1331 mask);
1332 mask = make_ssa_name (TREE_TYPE (aref));
1333 g = gimple_build_assign (mask, aref);
1334 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1335 }
1336 }
1337
1338 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
1339 NULL, NULL);
1340 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1341 edge e = make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
1342 e->probability = profile_probability::unlikely ().guessed ();
1343 incr_bb->count += e->count ();
1344 edge fallthru = FALLTHRU_EDGE (loop->header);
1345 fallthru->flags = EDGE_FALSE_VALUE;
1346 fallthru->probability = profile_probability::likely ().guessed ();
1347 }
1348
1349 basic_block latch_bb = NULL;
1350 basic_block new_exit_bb = NULL;
1351
1352 /* Generate the condition. */
1353 if (incr_bb)
1354 {
1355 gsi = gsi_last_bb (incr_bb);
1356 g = gimple_build_cond (LT_EXPR, iter2,
1357 build_int_cst (unsigned_type_node,
1358 node->simdclone->simdlen),
1359 NULL, NULL);
1360 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1361 edge e = split_block (incr_bb, gsi_stmt (gsi));
1362 latch_bb = e->dest;
1363 new_exit_bb = split_block_after_labels (latch_bb)->dest;
1364 loop->latch = latch_bb;
1365
1366 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
1367
1368 edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
1369
1370 /* FIXME: Do we need to distribute probabilities for the conditional? */
1371 new_e->probability = profile_probability::guessed_never ();
1372 /* The successor of incr_bb is already pointing to latch_bb; just
1373 change the flags.
1374 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1375 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
1376 }
1377
1378 gphi *phi = create_phi_node (iter1, body_bb);
1379 edge preheader_edge = find_edge (entry_bb, body_bb);
1380 edge latch_edge = NULL;
1381 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
1382 UNKNOWN_LOCATION);
1383 if (incr_bb)
1384 {
1385 latch_edge = single_succ_edge (latch_bb);
1386 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1387
1388 /* Generate the new return. */
1389 gsi = gsi_last_bb (new_exit_bb);
1390 if (retval
1391 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
1392 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
1393 retval = TREE_OPERAND (retval, 0);
1394 else if (retval)
1395 {
1396 retval = build1 (VIEW_CONVERT_EXPR,
1397 TREE_TYPE (TREE_TYPE (node->decl)),
1398 retval);
1399 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
1400 false, GSI_CONTINUE_LINKING);
1401 }
1402 g = gimple_build_return (retval);
1403 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1404 }
1405
1406 /* Handle aligned clauses by replacing default defs of the aligned
1407 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1408 lhs. Handle linear by adding PHIs. */
1409 for (unsigned i = 0; i < node->simdclone->nargs; i++)
1410 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1411 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
1412 || !is_gimple_reg_type
1413 (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1414 {
1415 tree orig_arg = node->simdclone->args[i].orig_arg;
1416 if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
1417 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1418 else
1419 {
1420 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
1421 gimple_add_tmp_var (iter1);
1422 }
1423 gsi = gsi_after_labels (entry_bb);
1424 g = gimple_build_assign (iter1, orig_arg);
1425 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1426 gsi = gsi_after_labels (body_bb);
1427 g = gimple_build_assign (orig_arg, iter1);
1428 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1429 }
1430 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1431 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
1432 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1433 == REFERENCE_TYPE
1434 && TREE_ADDRESSABLE
1435 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1436 {
1437 tree orig_arg = node->simdclone->args[i].orig_arg;
1438 tree def = ssa_default_def (cfun, orig_arg);
1439 if (def && !has_zero_uses (def))
1440 {
1441 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
1442 gimple_add_tmp_var (iter1);
1443 gsi = gsi_after_labels (entry_bb);
1444 g = gimple_build_assign (iter1, build_simple_mem_ref (def));
1445 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1446 gsi = gsi_after_labels (body_bb);
1447 g = gimple_build_assign (build_simple_mem_ref (def), iter1);
1448 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1449 }
1450 }
1451 else if (node->simdclone->args[i].alignment
1452 && node->simdclone->args[i].arg_type
1453 == SIMD_CLONE_ARG_TYPE_UNIFORM
1454 && (node->simdclone->args[i].alignment
1455 & (node->simdclone->args[i].alignment - 1)) == 0
1456 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1457 == POINTER_TYPE)
1458 {
1459 unsigned int alignment = node->simdclone->args[i].alignment;
1460 tree orig_arg = node->simdclone->args[i].orig_arg;
1461 tree def = ssa_default_def (cfun, orig_arg);
1462 if (def && !has_zero_uses (def))
1463 {
1464 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
1465 gimple_seq seq = NULL;
1466 bool need_cvt = false;
1467 gcall *call
1468 = gimple_build_call (fn, 2, def, size_int (alignment));
1469 g = call;
1470 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
1471 ptr_type_node))
1472 need_cvt = true;
1473 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
1474 gimple_call_set_lhs (g, t);
1475 gimple_seq_add_stmt_without_update (&seq, g);
1476 if (need_cvt)
1477 {
1478 t = make_ssa_name (orig_arg);
1479 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
1480 gimple_seq_add_stmt_without_update (&seq, g);
1481 }
1482 gsi_insert_seq_on_edge_immediate
1483 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
1484
1485 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1486 node->create_edge (cgraph_node::get_create (fn),
1487 call, entry_bb->count);
1488
1489 imm_use_iterator iter;
1490 use_operand_p use_p;
1491 gimple *use_stmt;
1492 tree repl = gimple_get_lhs (g);
1493 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1494 if (is_gimple_debug (use_stmt) || use_stmt == call)
1495 continue;
1496 else
1497 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1498 SET_USE (use_p, repl);
1499 }
1500 }
1501 else if ((node->simdclone->args[i].arg_type
1502 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
1503 || (node->simdclone->args[i].arg_type
1504 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
1505 || (node->simdclone->args[i].arg_type
1506 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
1507 || (node->simdclone->args[i].arg_type
1508 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
1509 {
1510 tree orig_arg = node->simdclone->args[i].orig_arg;
1511 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1512 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
1513 tree def = NULL_TREE;
1514 if (TREE_ADDRESSABLE (orig_arg))
1515 {
1516 def = make_ssa_name (TREE_TYPE (orig_arg));
1517 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1518 if (incr_bb)
1519 iter2 = make_ssa_name (TREE_TYPE (orig_arg));
1520 gsi = gsi_after_labels (entry_bb);
1521 g = gimple_build_assign (def, orig_arg);
1522 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1523 }
1524 else
1525 {
1526 def = ssa_default_def (cfun, orig_arg);
1527 if (!def || has_zero_uses (def))
1528 def = NULL_TREE;
1529 else
1530 {
1531 iter1 = make_ssa_name (orig_arg);
1532 if (incr_bb)
1533 iter2 = make_ssa_name (orig_arg);
1534 }
1535 }
1536 if (def)
1537 {
1538 phi = create_phi_node (iter1, body_bb);
1539 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
1540 if (incr_bb)
1541 {
1542 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1543 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1544 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1545 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1546 ? TREE_TYPE (orig_arg) : sizetype;
1547 tree addcst = simd_clone_linear_addend (node, i, addtype,
1548 entry_bb);
1549 gsi = gsi_last_bb (incr_bb);
1550 g = gimple_build_assign (iter2, code, iter1, addcst);
1551 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1552 }
1553
1554 imm_use_iterator iter;
1555 use_operand_p use_p;
1556 gimple *use_stmt;
1557 if (TREE_ADDRESSABLE (orig_arg))
1558 {
1559 gsi = gsi_after_labels (body_bb);
1560 g = gimple_build_assign (orig_arg, iter1);
1561 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1562 }
1563 else
1564 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1565 if (use_stmt == phi)
1566 continue;
1567 else
1568 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1569 SET_USE (use_p, iter1);
1570 }
1571 }
1572 else if (node->simdclone->args[i].arg_type
1573 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1574 || (node->simdclone->args[i].arg_type
1575 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
1576 {
1577 tree orig_arg = node->simdclone->args[i].orig_arg;
1578 tree def = ssa_default_def (cfun, orig_arg);
1579 gcc_assert (!TREE_ADDRESSABLE (orig_arg)
1580 && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
1581 if (def && !has_zero_uses (def))
1582 {
1583 tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
1584 iter1 = make_ssa_name (orig_arg);
1585 if (incr_bb)
1586 iter2 = make_ssa_name (orig_arg);
1587 tree iter3 = make_ssa_name (rtype);
1588 tree iter4 = make_ssa_name (rtype);
1589 tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE;
1590 gsi = gsi_after_labels (entry_bb);
1591 gimple *load
1592 = gimple_build_assign (iter3, build_simple_mem_ref (def));
1593 gsi_insert_before (&gsi, load, GSI_NEW_STMT);
1594
1595 tree array = node->simdclone->args[i].simd_array;
1596 TREE_ADDRESSABLE (array) = 1;
1597 tree ptr = build_fold_addr_expr (array);
1598 phi = create_phi_node (iter1, body_bb);
1599 add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
1600 if (incr_bb)
1601 {
1602 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1603 g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
1604 TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
1605 gsi = gsi_last_bb (incr_bb);
1606 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1607 }
1608
1609 phi = create_phi_node (iter4, body_bb);
1610 add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
1611 if (incr_bb)
1612 {
1613 add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
1614 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1615 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1616 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1617 ? TREE_TYPE (iter3) : sizetype;
1618 tree addcst = simd_clone_linear_addend (node, i, addtype,
1619 entry_bb);
1620 g = gimple_build_assign (iter5, code, iter4, addcst);
1621 gsi = gsi_last_bb (incr_bb);
1622 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1623 }
1624
1625 g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
1626 gsi = gsi_after_labels (body_bb);
1627 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1628
1629 imm_use_iterator iter;
1630 use_operand_p use_p;
1631 gimple *use_stmt;
1632 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1633 if (use_stmt == load)
1634 continue;
1635 else
1636 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1637 SET_USE (use_p, iter1);
1638
1639 if (!TYPE_READONLY (rtype) && incr_bb)
1640 {
1641 tree v = make_ssa_name (rtype);
1642 tree aref = build4 (ARRAY_REF, rtype, array,
1643 size_zero_node, NULL_TREE,
1644 NULL_TREE);
1645 gsi = gsi_after_labels (new_exit_bb);
1646 g = gimple_build_assign (v, aref);
1647 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1648 g = gimple_build_assign (build_simple_mem_ref (def), v);
1649 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1650 }
1651 }
1652 }
1653
1654 calculate_dominance_info (CDI_DOMINATORS);
1655 if (loop)
1656 add_loop (loop, loop->header->loop_father);
1657 update_ssa (TODO_update_ssa);
1658
1659 pop_cfun ();
1660 }
1661
1662 /* If the function in NODE is tagged as an elemental SIMD function,
1663 create the appropriate SIMD clones. */
1664
1665 void
expand_simd_clones(struct cgraph_node * node)1666 expand_simd_clones (struct cgraph_node *node)
1667 {
1668 tree attr = lookup_attribute ("omp declare simd",
1669 DECL_ATTRIBUTES (node->decl));
1670 if (attr == NULL_TREE
1671 || node->inlined_to
1672 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
1673 return;
1674
1675 /* Ignore
1676 #pragma omp declare simd
1677 extern int foo ();
1678 in C, there we don't know the argument types at all. */
1679 if (!node->definition
1680 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
1681 return;
1682
1683 /* Call this before creating clone_info, as it might ggc_collect. */
1684 if (node->definition && node->has_gimple_body_p ())
1685 node->get_body ();
1686
1687 do
1688 {
1689 /* Start with parsing the "omp declare simd" attribute(s). */
1690 bool inbranch_clause_specified;
1691 struct cgraph_simd_clone *clone_info
1692 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
1693 &inbranch_clause_specified);
1694 if (clone_info == NULL)
1695 continue;
1696
1697 poly_uint64 orig_simdlen = clone_info->simdlen;
1698 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
1699 /* The target can return 0 (no simd clones should be created),
1700 1 (just one ISA of simd clones should be created) or higher
1701 count of ISA variants. In that case, clone_info is initialized
1702 for the first ISA variant. */
1703 int count
1704 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
1705 base_type, 0);
1706 if (count == 0)
1707 continue;
1708
1709 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1710 also create one inbranch and one !inbranch clone of it. */
1711 for (int i = 0; i < count * 2; i++)
1712 {
1713 struct cgraph_simd_clone *clone = clone_info;
1714 if (inbranch_clause_specified && (i & 1) != 0)
1715 continue;
1716
1717 if (i != 0)
1718 {
1719 clone = simd_clone_struct_alloc (clone_info->nargs
1720 + ((i & 1) != 0));
1721 simd_clone_struct_copy (clone, clone_info);
1722 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1723 and simd_clone_adjust_argument_types did to the first
1724 clone's info. */
1725 clone->nargs -= clone_info->inbranch;
1726 clone->simdlen = orig_simdlen;
1727 /* And call the target hook again to get the right ISA. */
1728 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
1729 base_type,
1730 i / 2);
1731 if ((i & 1) != 0)
1732 clone->inbranch = 1;
1733 }
1734
1735 /* simd_clone_mangle might fail if such a clone has been created
1736 already. */
1737 tree id = simd_clone_mangle (node, clone);
1738 if (id == NULL_TREE)
1739 {
1740 if (i == 0)
1741 clone->nargs += clone->inbranch;
1742 continue;
1743 }
1744
1745 /* Only when we are sure we want to create the clone actually
1746 clone the function (or definitions) or create another
1747 extern FUNCTION_DECL (for prototypes without definitions). */
1748 struct cgraph_node *n = simd_clone_create (node);
1749 if (n == NULL)
1750 {
1751 if (i == 0)
1752 clone->nargs += clone->inbranch;
1753 continue;
1754 }
1755
1756 n->simdclone = clone;
1757 clone->origin = node;
1758 clone->next_clone = NULL;
1759 if (node->simd_clones == NULL)
1760 {
1761 clone->prev_clone = n;
1762 node->simd_clones = n;
1763 }
1764 else
1765 {
1766 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
1767 clone->prev_clone->simdclone->next_clone = n;
1768 node->simd_clones->simdclone->prev_clone = n;
1769 }
1770 symtab->change_decl_assembler_name (n->decl, id);
1771 /* And finally adjust the return type, parameters and for
1772 definitions also function body. */
1773 if (node->definition)
1774 simd_clone_adjust (n);
1775 else
1776 {
1777 TREE_TYPE (n->decl)
1778 = build_distinct_type_copy (TREE_TYPE (n->decl));
1779 targetm.simd_clone.adjust (n);
1780 simd_clone_adjust_return_type (n);
1781 simd_clone_adjust_argument_types (n);
1782 }
1783 }
1784 }
1785 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
1786 }
1787
1788 /* Entry point for IPA simd clone creation pass. */
1789
1790 static unsigned int
ipa_omp_simd_clone(void)1791 ipa_omp_simd_clone (void)
1792 {
1793 struct cgraph_node *node;
1794 FOR_EACH_FUNCTION (node)
1795 expand_simd_clones (node);
1796 return 0;
1797 }
1798
1799 namespace {
1800
1801 const pass_data pass_data_omp_simd_clone =
1802 {
1803 SIMPLE_IPA_PASS, /* type */
1804 "simdclone", /* name */
1805 OPTGROUP_OMP, /* optinfo_flags */
1806 TV_NONE, /* tv_id */
1807 ( PROP_ssa | PROP_cfg ), /* properties_required */
1808 0, /* properties_provided */
1809 0, /* properties_destroyed */
1810 0, /* todo_flags_start */
1811 0, /* todo_flags_finish */
1812 };
1813
1814 class pass_omp_simd_clone : public simple_ipa_opt_pass
1815 {
1816 public:
pass_omp_simd_clone(gcc::context * ctxt)1817 pass_omp_simd_clone(gcc::context *ctxt)
1818 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
1819 {}
1820
1821 /* opt_pass methods: */
1822 virtual bool gate (function *);
execute(function *)1823 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
1824 };
1825
1826 bool
gate(function *)1827 pass_omp_simd_clone::gate (function *)
1828 {
1829 return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
1830 }
1831
1832 } // anon namespace
1833
1834 simple_ipa_opt_pass *
make_pass_omp_simd_clone(gcc::context * ctxt)1835 make_pass_omp_simd_clone (gcc::context *ctxt)
1836 {
1837 return new pass_omp_simd_clone (ctxt);
1838 }
1839