1 /* ACLE support for AArch64 SVE 2 Copyright (C) 2018-2020 Free Software Foundation, Inc. 3 4 This file is part of GCC. 5 6 GCC is free software; you can redistribute it and/or modify it 7 under the terms of the GNU General Public License as published by 8 the Free Software Foundation; either version 3, or (at your option) 9 any later version. 10 11 GCC is distributed in the hope that it will be useful, but 12 WITHOUT ANY WARRANTY; without even the implied warranty of 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 General Public License for more details. 15 16 You should have received a copy of the GNU General Public License 17 along with GCC; see the file COPYING3. If not see 18 <http://www.gnu.org/licenses/>. */ 19 20 #define IN_TARGET_CODE 1 21 22 #include "config.h" 23 #include "system.h" 24 #include "coretypes.h" 25 #include "tm.h" 26 #include "tree.h" 27 #include "rtl.h" 28 #include "tm_p.h" 29 #include "memmodel.h" 30 #include "insn-codes.h" 31 #include "optabs.h" 32 #include "recog.h" 33 #include "diagnostic.h" 34 #include "expr.h" 35 #include "basic-block.h" 36 #include "function.h" 37 #include "fold-const.h" 38 #include "gimple.h" 39 #include "gimple-iterator.h" 40 #include "gimplify.h" 41 #include "explow.h" 42 #include "emit-rtl.h" 43 #include "tree-vector-builder.h" 44 #include "stor-layout.h" 45 #include "regs.h" 46 #include "alias.h" 47 #include "gimple-fold.h" 48 #include "langhooks.h" 49 #include "stringpool.h" 50 #include "attribs.h" 51 #include "aarch64-sve-builtins.h" 52 #include "aarch64-sve-builtins-base.h" 53 #include "aarch64-sve-builtins-sve2.h" 54 #include "aarch64-sve-builtins-shapes.h" 55 56 namespace aarch64_sve { 57 58 /* Static information about each single-predicate or single-vector 59 ABI and ACLE type. */ 60 struct vector_type_info 61 { 62 /* The name of the type as declared by arm_sve.h. */ 63 const char *acle_name; 64 65 /* The name of the type specified in AAPCS64. The type is always 66 available under this name, even when arm_sve.h isn't included. */ 67 const char *abi_name; 68 69 /* The C++ mangling of ABI_NAME. */ 70 const char *mangled_name; 71 }; 72 73 /* Describes a function decl. */ 74 class GTY(()) registered_function 75 { 76 public: 77 /* The ACLE function that the decl represents. */ 78 function_instance instance GTY ((skip)); 79 80 /* The decl itself. */ 81 tree decl; 82 83 /* The architecture extensions that the function requires, as a set of 84 AARCH64_FL_* flags. */ 85 uint64_t required_extensions; 86 87 /* True if the decl represents an overloaded function that needs to be 88 resolved by function_resolver. */ 89 bool overloaded_p; 90 }; 91 92 /* Hash traits for registered_function. */ 93 struct registered_function_hasher : nofree_ptr_hash <registered_function> 94 { 95 typedef function_instance compare_type; 96 97 static hashval_t hash (value_type); 98 static bool equal (value_type, const compare_type &); 99 }; 100 101 /* Information about each single-predicate or single-vector type. */ 102 static CONSTEXPR const vector_type_info vector_types[] = { 103 #define DEF_SVE_TYPE(ACLE_NAME, NCHARS, ABI_NAME, SCALAR_TYPE) \ 104 { #ACLE_NAME, #ABI_NAME, "u" #NCHARS #ABI_NAME }, 105 #include "aarch64-sve-builtins.def" 106 }; 107 108 /* The function name suffix associated with each predication type. */ 109 static const char *const pred_suffixes[NUM_PREDS + 1] = { 110 "", 111 "", 112 "_m", 113 "_x", 114 "_z", 115 "" 116 }; 117 118 /* Static information about each mode_suffix_index. */ 119 CONSTEXPR const mode_suffix_info mode_suffixes[] = { 120 #define VECTOR_TYPE_none NUM_VECTOR_TYPES 121 #define DEF_SVE_MODE(NAME, BASE, DISPLACEMENT, UNITS) \ 122 { "_" #NAME, VECTOR_TYPE_##BASE, VECTOR_TYPE_##DISPLACEMENT, UNITS_##UNITS }, 123 #include "aarch64-sve-builtins.def" 124 #undef VECTOR_TYPE_none 125 { "", NUM_VECTOR_TYPES, NUM_VECTOR_TYPES, UNITS_none } 126 }; 127 128 /* Static information about each type_suffix_index. */ 129 CONSTEXPR const type_suffix_info type_suffixes[NUM_TYPE_SUFFIXES + 1] = { 130 #define DEF_SVE_TYPE_SUFFIX(NAME, ACLE_TYPE, CLASS, BITS, MODE) \ 131 { "_" #NAME, \ 132 VECTOR_TYPE_##ACLE_TYPE, \ 133 TYPE_##CLASS, \ 134 BITS, \ 135 BITS / BITS_PER_UNIT, \ 136 TYPE_##CLASS == TYPE_signed || TYPE_##CLASS == TYPE_unsigned, \ 137 TYPE_##CLASS == TYPE_unsigned, \ 138 TYPE_##CLASS == TYPE_float, \ 139 TYPE_##CLASS == TYPE_bool, \ 140 0, \ 141 MODE }, 142 #include "aarch64-sve-builtins.def" 143 { "", NUM_VECTOR_TYPES, TYPE_bool, 0, 0, false, false, false, false, 144 0, VOIDmode } 145 }; 146 147 /* Define a TYPES_<combination> macro for each combination of type 148 suffixes that an ACLE function can have, where <combination> is the 149 name used in DEF_SVE_FUNCTION entries. 150 151 Use S (T) for single type suffix T and D (T1, T2) for a pair of type 152 suffixes T1 and T2. Use commas to separate the suffixes. 153 154 Although the order shouldn't matter, the convention is to sort the 155 suffixes lexicographically after dividing suffixes into a type 156 class ("b", "f", etc.) and a numerical bit count. */ 157 158 /* _b8 _b16 _b32 _b64. */ 159 #define TYPES_all_pred(S, D) \ 160 S (b8), S (b16), S (b32), S (b64) 161 162 /* _f16 _f32 _f64. */ 163 #define TYPES_all_float(S, D) \ 164 S (f16), S (f32), S (f64) 165 166 /* _s8 _s16 _s32 _s64. */ 167 #define TYPES_all_signed(S, D) \ 168 S (s8), S (s16), S (s32), S (s64) 169 170 /* _f16 _f32 _f64 171 _s8 _s16 _s32 _s64. */ 172 #define TYPES_all_float_and_signed(S, D) \ 173 TYPES_all_float (S, D), TYPES_all_signed (S, D) 174 175 /* _u8 _u16 _u32 _u64. */ 176 #define TYPES_all_unsigned(S, D) \ 177 S (u8), S (u16), S (u32), S (u64) 178 179 /* _s8 _s16 _s32 _s64 180 _u8 _u16 _u32 _u64. */ 181 #define TYPES_all_integer(S, D) \ 182 TYPES_all_signed (S, D), TYPES_all_unsigned (S, D) 183 184 /* _f16 _f32 _f64 185 _s8 _s16 _s32 _s64 186 _u8 _u16 _u32 _u64. */ 187 #define TYPES_all_arith(S, D) \ 188 TYPES_all_float (S, D), TYPES_all_integer (S, D) 189 190 /* _bf16 191 _f16 _f32 _f64 192 _s8 _s16 _s32 _s64 193 _u8 _u16 _u32 _u64. */ 194 #define TYPES_all_data(S, D) \ 195 S (bf16), TYPES_all_arith (S, D) 196 197 /* _b only. */ 198 #define TYPES_b(S, D) \ 199 S (b) 200 201 /* _u8. */ 202 #define TYPES_b_unsigned(S, D) \ 203 S (u8) 204 205 /* _s8 206 _u8. */ 207 #define TYPES_b_integer(S, D) \ 208 S (s8), TYPES_b_unsigned (S, D) 209 210 /* _s8 _s16 211 _u8 _u16. */ 212 #define TYPES_bh_integer(S, D) \ 213 S (s8), S (s16), S (u8), S (u16) 214 215 /* _u8 _u32. */ 216 #define TYPES_bs_unsigned(S, D) \ 217 S (u8), S (u32) 218 219 /* _s8 _s16 _s32. */ 220 #define TYPES_bhs_signed(S, D) \ 221 S (s8), S (s16), S (s32) 222 223 /* _u8 _u16 _u32. */ 224 #define TYPES_bhs_unsigned(S, D) \ 225 S (u8), S (u16), S (u32) 226 227 /* _s8 _s16 _s32 228 _u8 _u16 _u32. */ 229 #define TYPES_bhs_integer(S, D) \ 230 TYPES_bhs_signed (S, D), TYPES_bhs_unsigned (S, D) 231 232 /* _s16 233 _u16. */ 234 #define TYPES_h_integer(S, D) \ 235 S (s16), S (u16) 236 237 /* _s16 _s32. */ 238 #define TYPES_hs_signed(S, D) \ 239 S (s16), S (s32) 240 241 /* _s16 _s32 242 _u16 _u32. */ 243 #define TYPES_hs_integer(S, D) \ 244 TYPES_hs_signed (S, D), S (u16), S (u32) 245 246 /* _f16 _f32. */ 247 #define TYPES_hs_float(S, D) \ 248 S (f16), S (f32) 249 250 /* _u16 _u64. */ 251 #define TYPES_hd_unsigned(S, D) \ 252 S (u16), S (u64) 253 254 /* _s16 _s32 _s64. */ 255 #define TYPES_hsd_signed(S, D) \ 256 S (s16), S (s32), S (s64) 257 258 /* _s16 _s32 _s64 259 _u16 _u32 _u64. */ 260 #define TYPES_hsd_integer(S, D) \ 261 TYPES_hsd_signed (S, D), S (u16), S (u32), S (u64) 262 263 /* _f32. */ 264 #define TYPES_s_float(S, D) \ 265 S (f32) 266 267 /* _f32 268 _s16 _s32 _s64 269 _u16 _u32 _u64. */ 270 #define TYPES_s_float_hsd_integer(S, D) \ 271 TYPES_s_float (S, D), TYPES_hsd_integer (S, D) 272 273 /* _f32 274 _s32 _s64 275 _u32 _u64. */ 276 #define TYPES_s_float_sd_integer(S, D) \ 277 TYPES_s_float (S, D), TYPES_sd_integer (S, D) 278 279 /* _s32. */ 280 #define TYPES_s_signed(S, D) \ 281 S (s32) 282 283 /* _u32. */ 284 #define TYPES_s_unsigned(S, D) \ 285 S (u32) 286 287 /* _s32 _u32. */ 288 #define TYPES_s_integer(S, D) \ 289 TYPES_s_signed (S, D), TYPES_s_unsigned (S, D) 290 291 /* _s32 _s64. */ 292 #define TYPES_sd_signed(S, D) \ 293 S (s32), S (s64) 294 295 /* _u32 _u64. */ 296 #define TYPES_sd_unsigned(S, D) \ 297 S (u32), S (u64) 298 299 /* _s32 _s64 300 _u32 _u64. */ 301 #define TYPES_sd_integer(S, D) \ 302 TYPES_sd_signed (S, D), TYPES_sd_unsigned (S, D) 303 304 /* _f32 _f64 305 _s32 _s64 306 _u32 _u64. */ 307 #define TYPES_sd_data(S, D) \ 308 S (f32), S (f64), TYPES_sd_integer (S, D) 309 310 /* _f16 _f32 _f64 311 _s32 _s64 312 _u32 _u64. */ 313 #define TYPES_all_float_and_sd_integer(S, D) \ 314 TYPES_all_float (S, D), TYPES_sd_integer (S, D) 315 316 /* _f64. */ 317 #define TYPES_d_float(S, D) \ 318 S (f64) 319 320 /* _u64. */ 321 #define TYPES_d_unsigned(S, D) \ 322 S (u64) 323 324 /* _s64 325 _u64. */ 326 #define TYPES_d_integer(S, D) \ 327 S (s64), TYPES_d_unsigned (S, D) 328 329 /* _f64 330 _s64 331 _u64. */ 332 #define TYPES_d_data(S, D) \ 333 TYPES_d_float (S, D), TYPES_d_integer (S, D) 334 335 /* All the type combinations allowed by svcvt. */ 336 #define TYPES_cvt(S, D) \ 337 D (f16, f32), D (f16, f64), \ 338 D (f16, s16), D (f16, s32), D (f16, s64), \ 339 D (f16, u16), D (f16, u32), D (f16, u64), \ 340 \ 341 D (f32, f16), D (f32, f64), \ 342 D (f32, s32), D (f32, s64), \ 343 D (f32, u32), D (f32, u64), \ 344 \ 345 D (f64, f16), D (f64, f32), \ 346 D (f64, s32), D (f64, s64), \ 347 D (f64, u32), D (f64, u64), \ 348 \ 349 D (s16, f16), \ 350 D (s32, f16), D (s32, f32), D (s32, f64), \ 351 D (s64, f16), D (s64, f32), D (s64, f64), \ 352 \ 353 D (u16, f16), \ 354 D (u32, f16), D (u32, f32), D (u32, f64), \ 355 D (u64, f16), D (u64, f32), D (u64, f64) 356 357 /* _bf16_f32. */ 358 #define TYPES_cvt_bfloat(S, D) \ 359 D (bf16, f32) 360 361 /* _f32_f16 362 _f64_f32. */ 363 #define TYPES_cvt_long(S, D) \ 364 D (f32, f16), D (f64, f32) 365 366 /* _f16_f32. */ 367 #define TYPES_cvt_narrow_s(S, D) \ 368 D (f32, f64) 369 370 /* _f16_f32 371 _f32_f64. */ 372 #define TYPES_cvt_narrow(S, D) \ 373 D (f16, f32), TYPES_cvt_narrow_s (S, D) 374 375 /* { _s32 _s64 } x { _b8 _b16 _b32 _b64 } 376 { _u32 _u64 }. */ 377 #define TYPES_inc_dec_n1(D, A) \ 378 D (A, b8), D (A, b16), D (A, b32), D (A, b64) 379 #define TYPES_inc_dec_n(S, D) \ 380 TYPES_inc_dec_n1 (D, s32), \ 381 TYPES_inc_dec_n1 (D, s64), \ 382 TYPES_inc_dec_n1 (D, u32), \ 383 TYPES_inc_dec_n1 (D, u64) 384 385 /* { _bf16 } { _bf16 } 386 { _f16 _f32 _f64 } { _f16 _f32 _f64 } 387 { _s8 _s16 _s32 _s64 } x { _s8 _s16 _s32 _s64 } 388 { _u8 _u16 _u32 _u64 } { _u8 _u16 _u32 _u64 }. */ 389 #define TYPES_reinterpret1(D, A) \ 390 D (A, bf16), \ 391 D (A, f16), D (A, f32), D (A, f64), \ 392 D (A, s8), D (A, s16), D (A, s32), D (A, s64), \ 393 D (A, u8), D (A, u16), D (A, u32), D (A, u64) 394 #define TYPES_reinterpret(S, D) \ 395 TYPES_reinterpret1 (D, bf16), \ 396 TYPES_reinterpret1 (D, f16), \ 397 TYPES_reinterpret1 (D, f32), \ 398 TYPES_reinterpret1 (D, f64), \ 399 TYPES_reinterpret1 (D, s8), \ 400 TYPES_reinterpret1 (D, s16), \ 401 TYPES_reinterpret1 (D, s32), \ 402 TYPES_reinterpret1 (D, s64), \ 403 TYPES_reinterpret1 (D, u8), \ 404 TYPES_reinterpret1 (D, u16), \ 405 TYPES_reinterpret1 (D, u32), \ 406 TYPES_reinterpret1 (D, u64) 407 408 /* { _b8 _b16 _b32 _b64 } x { _s32 _s64 } 409 { _u32 _u64 } */ 410 #define TYPES_while1(D, bn) \ 411 D (bn, s32), D (bn, s64), D (bn, u32), D (bn, u64) 412 #define TYPES_while(S, D) \ 413 TYPES_while1 (D, b8), \ 414 TYPES_while1 (D, b16), \ 415 TYPES_while1 (D, b32), \ 416 TYPES_while1 (D, b64) 417 418 /* Describe a pair of type suffixes in which only the first is used. */ 419 #define DEF_VECTOR_TYPE(X) { TYPE_SUFFIX_ ## X, NUM_TYPE_SUFFIXES } 420 421 /* Describe a pair of type suffixes in which both are used. */ 422 #define DEF_DOUBLE_TYPE(X, Y) { TYPE_SUFFIX_ ## X, TYPE_SUFFIX_ ## Y } 423 424 /* Create an array that can be used in aarch64-sve-builtins.def to 425 select the type suffixes in TYPES_<NAME>. */ 426 #define DEF_SVE_TYPES_ARRAY(NAME) \ 427 static const type_suffix_pair types_##NAME[] = { \ 428 TYPES_##NAME (DEF_VECTOR_TYPE, DEF_DOUBLE_TYPE), \ 429 { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } \ 430 } 431 432 /* For functions that don't take any type suffixes. */ 433 static const type_suffix_pair types_none[] = { 434 { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES }, 435 { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } 436 }; 437 438 /* Create an array for each TYPES_<combination> macro above. */ 439 DEF_SVE_TYPES_ARRAY (all_pred); 440 DEF_SVE_TYPES_ARRAY (all_float); 441 DEF_SVE_TYPES_ARRAY (all_signed); 442 DEF_SVE_TYPES_ARRAY (all_float_and_signed); 443 DEF_SVE_TYPES_ARRAY (all_unsigned); 444 DEF_SVE_TYPES_ARRAY (all_integer); 445 DEF_SVE_TYPES_ARRAY (all_arith); 446 DEF_SVE_TYPES_ARRAY (all_data); 447 DEF_SVE_TYPES_ARRAY (b); 448 DEF_SVE_TYPES_ARRAY (b_unsigned); 449 DEF_SVE_TYPES_ARRAY (b_integer); 450 DEF_SVE_TYPES_ARRAY (bh_integer); 451 DEF_SVE_TYPES_ARRAY (bs_unsigned); 452 DEF_SVE_TYPES_ARRAY (bhs_signed); 453 DEF_SVE_TYPES_ARRAY (bhs_unsigned); 454 DEF_SVE_TYPES_ARRAY (bhs_integer); 455 DEF_SVE_TYPES_ARRAY (h_integer); 456 DEF_SVE_TYPES_ARRAY (hs_signed); 457 DEF_SVE_TYPES_ARRAY (hs_integer); 458 DEF_SVE_TYPES_ARRAY (hs_float); 459 DEF_SVE_TYPES_ARRAY (hd_unsigned); 460 DEF_SVE_TYPES_ARRAY (hsd_signed); 461 DEF_SVE_TYPES_ARRAY (hsd_integer); 462 DEF_SVE_TYPES_ARRAY (s_float); 463 DEF_SVE_TYPES_ARRAY (s_float_hsd_integer); 464 DEF_SVE_TYPES_ARRAY (s_float_sd_integer); 465 DEF_SVE_TYPES_ARRAY (s_signed); 466 DEF_SVE_TYPES_ARRAY (s_unsigned); 467 DEF_SVE_TYPES_ARRAY (s_integer); 468 DEF_SVE_TYPES_ARRAY (sd_signed); 469 DEF_SVE_TYPES_ARRAY (sd_unsigned); 470 DEF_SVE_TYPES_ARRAY (sd_integer); 471 DEF_SVE_TYPES_ARRAY (sd_data); 472 DEF_SVE_TYPES_ARRAY (all_float_and_sd_integer); 473 DEF_SVE_TYPES_ARRAY (d_float); 474 DEF_SVE_TYPES_ARRAY (d_unsigned); 475 DEF_SVE_TYPES_ARRAY (d_integer); 476 DEF_SVE_TYPES_ARRAY (d_data); 477 DEF_SVE_TYPES_ARRAY (cvt); 478 DEF_SVE_TYPES_ARRAY (cvt_bfloat); 479 DEF_SVE_TYPES_ARRAY (cvt_long); 480 DEF_SVE_TYPES_ARRAY (cvt_narrow_s); 481 DEF_SVE_TYPES_ARRAY (cvt_narrow); 482 DEF_SVE_TYPES_ARRAY (inc_dec_n); 483 DEF_SVE_TYPES_ARRAY (reinterpret); 484 DEF_SVE_TYPES_ARRAY (while); 485 486 /* Used by functions that have no governing predicate. */ 487 static const predication_index preds_none[] = { PRED_none, NUM_PREDS }; 488 489 /* Used by functions that have a governing predicate but do not have an 490 explicit suffix. */ 491 static const predication_index preds_implicit[] = { PRED_implicit, NUM_PREDS }; 492 493 /* Used by functions that allow merging and "don't care" predication, 494 but are not suitable for predicated MOVPRFX. */ 495 static const predication_index preds_mx[] = { 496 PRED_m, PRED_x, NUM_PREDS 497 }; 498 499 /* Used by functions that allow merging, zeroing and "don't care" 500 predication. */ 501 static const predication_index preds_mxz[] = { 502 PRED_m, PRED_x, PRED_z, NUM_PREDS 503 }; 504 505 /* Used by functions that have the mxz predicated forms above, and in addition 506 have an unpredicated form. */ 507 static const predication_index preds_mxz_or_none[] = { 508 PRED_m, PRED_x, PRED_z, PRED_none, NUM_PREDS 509 }; 510 511 /* Used by functions that allow merging and zeroing predication but have 512 no "_x" form. */ 513 static const predication_index preds_mz[] = { PRED_m, PRED_z, NUM_PREDS }; 514 515 /* Used by functions that have an unpredicated form and a _z predicated 516 form. */ 517 static const predication_index preds_z_or_none[] = { 518 PRED_z, PRED_none, NUM_PREDS 519 }; 520 521 /* Used by (mostly predicate) functions that only support "_z" predication. */ 522 static const predication_index preds_z[] = { PRED_z, NUM_PREDS }; 523 524 /* A list of all SVE ACLE functions. */ 525 static CONSTEXPR const function_group_info function_groups[] = { 526 #define DEF_SVE_FUNCTION(NAME, SHAPE, TYPES, PREDS) \ 527 { #NAME, &functions::NAME, &shapes::SHAPE, types_##TYPES, preds_##PREDS, \ 528 REQUIRED_EXTENSIONS | AARCH64_FL_SVE }, 529 #include "aarch64-sve-builtins.def" 530 }; 531 532 /* The scalar type associated with each vector type. */ 533 GTY(()) tree scalar_types[NUM_VECTOR_TYPES]; 534 535 /* The single-predicate and single-vector types, with their built-in 536 "__SV..._t" name. Allow an index of NUM_VECTOR_TYPES, which always 537 yields a null tree. */ 538 static GTY(()) tree abi_vector_types[NUM_VECTOR_TYPES + 1]; 539 540 /* Same, but with the arm_sve.h "sv..._t" name. */ 541 GTY(()) tree acle_vector_types[MAX_TUPLE_SIZE][NUM_VECTOR_TYPES + 1]; 542 543 /* The svpattern enum type. */ 544 GTY(()) tree acle_svpattern; 545 546 /* The svprfop enum type. */ 547 GTY(()) tree acle_svprfop; 548 549 /* The list of all registered function decls, indexed by code. */ 550 static GTY(()) vec<registered_function *, va_gc> *registered_functions; 551 552 /* All registered function decls, hashed on the function_instance 553 that they implement. This is used for looking up implementations of 554 overloaded functions. */ 555 static hash_table<registered_function_hasher> *function_table; 556 557 /* True if we've already complained about attempts to use functions 558 when the required extension is disabled. */ 559 static bool reported_missing_extension_p; 560 561 /* True if we've already complained about attempts to use functions 562 which require registers that are missing. */ 563 static bool reported_missing_registers_p; 564 565 /* Record that TYPE is an ABI-defined SVE type that contains NUM_ZR SVE vectors 566 and NUM_PR SVE predicates. MANGLED_NAME, if nonnull, is the ABI-defined 567 mangling of the type. ACLE_NAME is the <arm_sve.h> name of the type. */ 568 static void 569 add_sve_type_attribute (tree type, unsigned int num_zr, unsigned int num_pr, 570 const char *mangled_name, const char *acle_name) 571 { 572 tree mangled_name_tree 573 = (mangled_name ? get_identifier (mangled_name) : NULL_TREE); 574 575 tree value = tree_cons (NULL_TREE, get_identifier (acle_name), NULL_TREE); 576 value = tree_cons (NULL_TREE, mangled_name_tree, value); 577 value = tree_cons (NULL_TREE, size_int (num_pr), value); 578 value = tree_cons (NULL_TREE, size_int (num_zr), value); 579 TYPE_ATTRIBUTES (type) = tree_cons (get_identifier ("SVE type"), value, 580 TYPE_ATTRIBUTES (type)); 581 } 582 583 /* If TYPE is an ABI-defined SVE type, return its attribute descriptor, 584 otherwise return null. */ 585 static tree 586 lookup_sve_type_attribute (const_tree type) 587 { 588 if (type == error_mark_node) 589 return NULL_TREE; 590 return lookup_attribute ("SVE type", TYPE_ATTRIBUTES (type)); 591 } 592 593 /* Force TYPE to be a sizeless type. */ 594 static void 595 make_type_sizeless (tree type) 596 { 597 TYPE_ATTRIBUTES (type) = tree_cons (get_identifier ("SVE sizeless type"), 598 NULL_TREE, TYPE_ATTRIBUTES (type)); 599 } 600 601 /* Return true if TYPE is a sizeless type. */ 602 static bool 603 sizeless_type_p (const_tree type) 604 { 605 if (type == error_mark_node) 606 return NULL_TREE; 607 return lookup_attribute ("SVE sizeless type", TYPE_ATTRIBUTES (type)); 608 } 609 610 /* Return true if CANDIDATE is equivalent to MODEL_TYPE for overloading 611 purposes. */ 612 static bool 613 matches_type_p (const_tree model_type, const_tree candidate) 614 { 615 if (VECTOR_TYPE_P (model_type)) 616 { 617 if (!VECTOR_TYPE_P (candidate) 618 || maybe_ne (TYPE_VECTOR_SUBPARTS (model_type), 619 TYPE_VECTOR_SUBPARTS (candidate)) 620 || TYPE_MODE (model_type) != TYPE_MODE (candidate)) 621 return false; 622 623 model_type = TREE_TYPE (model_type); 624 candidate = TREE_TYPE (candidate); 625 } 626 return (candidate != error_mark_node 627 && TYPE_MAIN_VARIANT (model_type) == TYPE_MAIN_VARIANT (candidate)); 628 } 629 630 /* If TYPE is a valid SVE element type, return the corresponding type 631 suffix, otherwise return NUM_TYPE_SUFFIXES. */ 632 static type_suffix_index 633 find_type_suffix_for_scalar_type (const_tree type) 634 { 635 /* A linear search should be OK here, since the code isn't hot and 636 the number of types is only small. */ 637 for (unsigned int suffix_i = 0; suffix_i < NUM_TYPE_SUFFIXES; ++suffix_i) 638 if (!type_suffixes[suffix_i].bool_p) 639 { 640 vector_type_index vector_i = type_suffixes[suffix_i].vector_type; 641 if (matches_type_p (scalar_types[vector_i], type)) 642 return type_suffix_index (suffix_i); 643 } 644 return NUM_TYPE_SUFFIXES; 645 } 646 647 /* Report an error against LOCATION that the user has tried to use 648 function FNDECL when extension EXTENSION is disabled. */ 649 static void 650 report_missing_extension (location_t location, tree fndecl, 651 const char *extension) 652 { 653 /* Avoid reporting a slew of messages for a single oversight. */ 654 if (reported_missing_extension_p) 655 return; 656 657 error_at (location, "ACLE function %qD requires ISA extension %qs", 658 fndecl, extension); 659 inform (location, "you can enable %qs using the command-line" 660 " option %<-march%>, or by using the %<target%>" 661 " attribute or pragma", extension); 662 reported_missing_extension_p = true; 663 } 664 665 /* Check whether the registers required by SVE function fndecl are available. 666 Report an error against LOCATION and return false if not. */ 667 static bool 668 check_required_registers (location_t location, tree fndecl) 669 { 670 /* Avoid reporting a slew of messages for a single oversight. */ 671 if (reported_missing_registers_p) 672 return false; 673 674 if (TARGET_GENERAL_REGS_ONLY) 675 { 676 /* SVE registers are not usable when -mgeneral-regs-only option 677 is specified. */ 678 error_at (location, 679 "ACLE function %qD is incompatible with the use of %qs", 680 fndecl, "-mgeneral-regs-only"); 681 reported_missing_registers_p = true; 682 return false; 683 } 684 685 return true; 686 } 687 688 /* Check whether all the AARCH64_FL_* values in REQUIRED_EXTENSIONS are 689 enabled, given that those extensions are required for function FNDECL. 690 Report an error against LOCATION if not. */ 691 static bool 692 check_required_extensions (location_t location, tree fndecl, 693 uint64_t required_extensions) 694 { 695 uint64_t missing_extensions = required_extensions & ~aarch64_isa_flags; 696 if (missing_extensions == 0) 697 return check_required_registers (location, fndecl); 698 699 static const struct { uint64_t flag; const char *name; } extensions[] = { 700 #define AARCH64_OPT_EXTENSION(EXT_NAME, FLAG_CANONICAL, FLAGS_ON, FLAGS_OFF, \ 701 SYNTHETIC, FEATURE_STRING) \ 702 { FLAG_CANONICAL, EXT_NAME }, 703 #include "aarch64-option-extensions.def" 704 }; 705 706 for (unsigned int i = 0; i < ARRAY_SIZE (extensions); ++i) 707 if (missing_extensions & extensions[i].flag) 708 { 709 report_missing_extension (location, fndecl, extensions[i].name); 710 return false; 711 } 712 gcc_unreachable (); 713 } 714 715 /* Report that LOCATION has a call to FNDECL in which argument ARGNO 716 was not an integer constant expression. ARGNO counts from zero. */ 717 static void 718 report_non_ice (location_t location, tree fndecl, unsigned int argno) 719 { 720 error_at (location, "argument %d of %qE must be an integer constant" 721 " expression", argno + 1, fndecl); 722 } 723 724 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has 725 the value ACTUAL, whereas the function requires a value in the range 726 [MIN, MAX]. ARGNO counts from zero. */ 727 static void 728 report_out_of_range (location_t location, tree fndecl, unsigned int argno, 729 HOST_WIDE_INT actual, HOST_WIDE_INT min, 730 HOST_WIDE_INT max) 731 { 732 error_at (location, "passing %wd to argument %d of %qE, which expects" 733 " a value in the range [%wd, %wd]", actual, argno + 1, fndecl, 734 min, max); 735 } 736 737 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has 738 the value ACTUAL, whereas the function requires either VALUE0 or 739 VALUE1. ARGNO counts from zero. */ 740 static void 741 report_neither_nor (location_t location, tree fndecl, unsigned int argno, 742 HOST_WIDE_INT actual, HOST_WIDE_INT value0, 743 HOST_WIDE_INT value1) 744 { 745 error_at (location, "passing %wd to argument %d of %qE, which expects" 746 " either %wd or %wd", actual, argno + 1, fndecl, value0, value1); 747 } 748 749 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has 750 the value ACTUAL, whereas the function requires one of VALUE0..3. 751 ARGNO counts from zero. */ 752 static void 753 report_not_one_of (location_t location, tree fndecl, unsigned int argno, 754 HOST_WIDE_INT actual, HOST_WIDE_INT value0, 755 HOST_WIDE_INT value1, HOST_WIDE_INT value2, 756 HOST_WIDE_INT value3) 757 { 758 error_at (location, "passing %wd to argument %d of %qE, which expects" 759 " %wd, %wd, %wd or %wd", actual, argno + 1, fndecl, value0, value1, 760 value2, value3); 761 } 762 763 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has 764 the value ACTUAL, whereas the function requires a valid value of 765 enum type ENUMTYPE. ARGNO counts from zero. */ 766 static void 767 report_not_enum (location_t location, tree fndecl, unsigned int argno, 768 HOST_WIDE_INT actual, tree enumtype) 769 { 770 error_at (location, "passing %wd to argument %d of %qE, which expects" 771 " a valid %qT value", actual, argno + 1, fndecl, enumtype); 772 } 773 774 /* Return a hash code for a function_instance. */ 775 hashval_t 776 function_instance::hash () const 777 { 778 inchash::hash h; 779 /* BASE uniquely determines BASE_NAME, so we don't need to hash both. */ 780 h.add_ptr (base); 781 h.add_ptr (shape); 782 h.add_int (mode_suffix_id); 783 h.add_int (type_suffix_ids[0]); 784 h.add_int (type_suffix_ids[1]); 785 h.add_int (pred); 786 return h.end (); 787 } 788 789 /* Return a set of CP_* flags that describe what the function could do, 790 taking the command-line flags into account. */ 791 unsigned int 792 function_instance::call_properties () const 793 { 794 unsigned int flags = base->call_properties (*this); 795 796 /* -fno-trapping-math means that we can assume any FP exceptions 797 are not user-visible. */ 798 if (!flag_trapping_math) 799 flags &= ~CP_RAISE_FP_EXCEPTIONS; 800 801 return flags; 802 } 803 804 /* Return true if calls to the function could read some form of 805 global state. */ 806 bool 807 function_instance::reads_global_state_p () const 808 { 809 unsigned int flags = call_properties (); 810 811 /* Preserve any dependence on rounding mode, flush to zero mode, etc. 812 There is currently no way of turning this off; in particular, 813 -fno-rounding-math (which is the default) means that we should make 814 the usual assumptions about rounding mode, which for intrinsics means 815 acting as the instructions do. */ 816 if (flags & CP_READ_FPCR) 817 return true; 818 819 /* Handle direct reads of global state. */ 820 return flags & (CP_READ_MEMORY | CP_READ_FFR); 821 } 822 823 /* Return true if calls to the function could modify some form of 824 global state. */ 825 bool 826 function_instance::modifies_global_state_p () const 827 { 828 unsigned int flags = call_properties (); 829 830 /* Preserve any exception state written back to the FPCR, 831 unless -fno-trapping-math says this is unnecessary. */ 832 if (flags & CP_RAISE_FP_EXCEPTIONS) 833 return true; 834 835 /* Treat prefetches as modifying global state, since that's the 836 only means we have of keeping them in their correct position. */ 837 if (flags & CP_PREFETCH_MEMORY) 838 return true; 839 840 /* Handle direct modifications of global state. */ 841 return flags & (CP_WRITE_MEMORY | CP_WRITE_FFR); 842 } 843 844 /* Return true if calls to the function could raise a signal. */ 845 bool 846 function_instance::could_trap_p () const 847 { 848 unsigned int flags = call_properties (); 849 850 /* Handle functions that could raise SIGFPE. */ 851 if (flags & CP_RAISE_FP_EXCEPTIONS) 852 return true; 853 854 /* Handle functions that could raise SIGBUS or SIGSEGV. */ 855 if (flags & (CP_READ_MEMORY | CP_WRITE_MEMORY)) 856 return true; 857 858 return false; 859 } 860 861 inline hashval_t 862 registered_function_hasher::hash (value_type value) 863 { 864 return value->instance.hash (); 865 } 866 867 inline bool 868 registered_function_hasher::equal (value_type value, const compare_type &key) 869 { 870 return value->instance == key; 871 } 872 873 sve_switcher::sve_switcher () 874 : m_old_isa_flags (aarch64_isa_flags) 875 { 876 /* Changing the ISA flags and have_regs_of_mode should be enough here. 877 We shouldn't need to pay the compile-time cost of a full target 878 switch. */ 879 aarch64_isa_flags = (AARCH64_FL_FP | AARCH64_FL_SIMD | AARCH64_FL_F16 880 | AARCH64_FL_SVE); 881 882 m_old_general_regs_only = TARGET_GENERAL_REGS_ONLY; 883 global_options.x_target_flags &= ~MASK_GENERAL_REGS_ONLY; 884 885 memcpy (m_old_have_regs_of_mode, have_regs_of_mode, 886 sizeof (have_regs_of_mode)); 887 for (int i = 0; i < NUM_MACHINE_MODES; ++i) 888 if (aarch64_sve_mode_p ((machine_mode) i)) 889 have_regs_of_mode[i] = true; 890 } 891 892 sve_switcher::~sve_switcher () 893 { 894 memcpy (have_regs_of_mode, m_old_have_regs_of_mode, 895 sizeof (have_regs_of_mode)); 896 if (m_old_general_regs_only) 897 global_options.x_target_flags |= MASK_GENERAL_REGS_ONLY; 898 aarch64_isa_flags = m_old_isa_flags; 899 } 900 901 function_builder::function_builder () 902 { 903 m_overload_type = build_function_type (void_type_node, void_list_node); 904 m_direct_overloads = lang_GNU_CXX (); 905 gcc_obstack_init (&m_string_obstack); 906 } 907 908 function_builder::~function_builder () 909 { 910 obstack_free (&m_string_obstack, NULL); 911 } 912 913 /* Add NAME to the end of the function name being built. */ 914 void 915 function_builder::append_name (const char *name) 916 { 917 obstack_grow (&m_string_obstack, name, strlen (name)); 918 } 919 920 /* Zero-terminate and complete the function name being built. */ 921 char * 922 function_builder::finish_name () 923 { 924 obstack_1grow (&m_string_obstack, 0); 925 return (char *) obstack_finish (&m_string_obstack); 926 } 927 928 /* Return the overloaded or full function name for INSTANCE; OVERLOADED_P 929 selects which. Allocate the string on m_string_obstack; the caller 930 must use obstack_free to free it after use. */ 931 char * 932 function_builder::get_name (const function_instance &instance, 933 bool overloaded_p) 934 { 935 append_name (instance.base_name); 936 if (overloaded_p) 937 switch (instance.displacement_units ()) 938 { 939 case UNITS_none: 940 break; 941 942 case UNITS_bytes: 943 append_name ("_offset"); 944 break; 945 946 case UNITS_elements: 947 append_name ("_index"); 948 break; 949 950 case UNITS_vectors: 951 append_name ("_vnum"); 952 break; 953 } 954 else 955 append_name (instance.mode_suffix ().string); 956 for (unsigned int i = 0; i < 2; ++i) 957 if (!overloaded_p || instance.shape->explicit_type_suffix_p (i)) 958 append_name (instance.type_suffix (i).string); 959 append_name (pred_suffixes[instance.pred]); 960 return finish_name (); 961 } 962 963 /* Add attribute NAME to ATTRS. */ 964 static tree 965 add_attribute (const char *name, tree attrs) 966 { 967 return tree_cons (get_identifier (name), NULL_TREE, attrs); 968 } 969 970 /* Return the appropriate function attributes for INSTANCE. */ 971 tree 972 function_builder::get_attributes (const function_instance &instance) 973 { 974 tree attrs = NULL_TREE; 975 976 if (!instance.modifies_global_state_p ()) 977 { 978 if (instance.reads_global_state_p ()) 979 attrs = add_attribute ("pure", attrs); 980 else 981 attrs = add_attribute ("const", attrs); 982 } 983 984 if (!flag_non_call_exceptions || !instance.could_trap_p ()) 985 attrs = add_attribute ("nothrow", attrs); 986 987 return add_attribute ("leaf", attrs); 988 } 989 990 /* Add a function called NAME with type FNTYPE and attributes ATTRS. 991 INSTANCE describes what the function does and OVERLOADED_P indicates 992 whether it is overloaded. REQUIRED_EXTENSIONS are the set of 993 architecture extensions that the function requires. */ 994 registered_function & 995 function_builder::add_function (const function_instance &instance, 996 const char *name, tree fntype, tree attrs, 997 uint64_t required_extensions, 998 bool overloaded_p, 999 bool placeholder_p) 1000 { 1001 unsigned int code = vec_safe_length (registered_functions); 1002 code = (code << AARCH64_BUILTIN_SHIFT) | AARCH64_BUILTIN_SVE; 1003 1004 /* We need to be able to generate placeholders to enusre that we have a 1005 consistent numbering scheme for function codes between the C and C++ 1006 frontends, so that everything ties up in LTO. 1007 1008 Currently, tree-streamer-in.c:unpack_ts_function_decl_value_fields 1009 validates that tree nodes returned by TARGET_BUILTIN_DECL are non-NULL and 1010 some node other than error_mark_node. This is a holdover from when builtin 1011 decls were streamed by code rather than by value. 1012 1013 Ultimately, we should be able to remove this validation of BUILT_IN_MD 1014 nodes and remove the target hook. For now, however, we need to appease the 1015 validation and return a non-NULL, non-error_mark_node node, so we 1016 arbitrarily choose integer_zero_node. */ 1017 tree decl = placeholder_p 1018 ? integer_zero_node 1019 : simulate_builtin_function_decl (input_location, name, fntype, 1020 code, NULL, attrs); 1021 1022 registered_function &rfn = *ggc_alloc <registered_function> (); 1023 rfn.instance = instance; 1024 rfn.decl = decl; 1025 rfn.required_extensions = required_extensions; 1026 rfn.overloaded_p = overloaded_p; 1027 vec_safe_push (registered_functions, &rfn); 1028 1029 return rfn; 1030 } 1031 1032 /* Add a built-in function for INSTANCE, with the argument types given 1033 by ARGUMENT_TYPES and the return type given by RETURN_TYPE. 1034 REQUIRED_EXTENSIONS are the set of architecture extensions that the 1035 function requires. FORCE_DIRECT_OVERLOADS is true if there is a 1036 one-to-one mapping between "short" and "full" names, and if standard 1037 overload resolution therefore isn't necessary. */ 1038 void 1039 function_builder::add_unique_function (const function_instance &instance, 1040 tree return_type, 1041 vec<tree> &argument_types, 1042 uint64_t required_extensions, 1043 bool force_direct_overloads) 1044 { 1045 /* Add the function under its full (unique) name. */ 1046 char *name = get_name (instance, false); 1047 tree fntype = build_function_type_array (return_type, 1048 argument_types.length (), 1049 argument_types.address ()); 1050 tree attrs = get_attributes (instance); 1051 registered_function &rfn = add_function (instance, name, fntype, attrs, 1052 required_extensions, false, false); 1053 1054 /* Enter the function into the hash table. */ 1055 hashval_t hash = instance.hash (); 1056 registered_function **rfn_slot 1057 = function_table->find_slot_with_hash (instance, hash, INSERT); 1058 gcc_assert (!*rfn_slot); 1059 *rfn_slot = &rfn; 1060 1061 /* Also add the function under its overloaded alias, if we want 1062 a separate decl for each instance of an overloaded function. */ 1063 char *overload_name = get_name (instance, true); 1064 if (strcmp (name, overload_name) != 0) 1065 { 1066 /* Attribute lists shouldn't be shared. */ 1067 tree attrs = get_attributes (instance); 1068 bool placeholder_p = !(m_direct_overloads || force_direct_overloads); 1069 add_function (instance, overload_name, fntype, attrs, 1070 required_extensions, false, placeholder_p); 1071 } 1072 1073 obstack_free (&m_string_obstack, name); 1074 } 1075 1076 /* Add one function decl for INSTANCE, to be used with manual overload 1077 resolution. REQUIRED_EXTENSIONS are the set of architecture extensions 1078 that the function requires. 1079 1080 For simplicity, deal with duplicate attempts to add the same function, 1081 including cases in which the new function requires more features than 1082 the original one did. In that case we'll check whether the required 1083 features are available as part of resolving the function to the 1084 relevant unique function. */ 1085 void 1086 function_builder::add_overloaded_function (const function_instance &instance, 1087 uint64_t required_extensions) 1088 { 1089 char *name = get_name (instance, true); 1090 if (registered_function **map_value = m_overload_names.get (name)) 1091 { 1092 gcc_assert ((*map_value)->instance == instance 1093 && ((*map_value)->required_extensions 1094 & ~required_extensions) == 0); 1095 obstack_free (&m_string_obstack, name); 1096 } 1097 else 1098 { 1099 registered_function &rfn 1100 = add_function (instance, name, m_overload_type, NULL_TREE, 1101 required_extensions, true, m_direct_overloads); 1102 m_overload_names.put (name, &rfn); 1103 } 1104 } 1105 1106 /* If we are using manual overload resolution, add one function decl 1107 for each overloaded function in GROUP. Take the function base name 1108 from GROUP and the mode from MODE. */ 1109 void 1110 function_builder::add_overloaded_functions (const function_group_info &group, 1111 mode_suffix_index mode) 1112 { 1113 unsigned int explicit_type0 = (*group.shape)->explicit_type_suffix_p (0); 1114 unsigned int explicit_type1 = (*group.shape)->explicit_type_suffix_p (1); 1115 for (unsigned int pi = 0; group.preds[pi] != NUM_PREDS; ++pi) 1116 { 1117 if (!explicit_type0 && !explicit_type1) 1118 { 1119 /* Deal with the common case in which there is one overloaded 1120 function for all type combinations. */ 1121 function_instance instance (group.base_name, *group.base, 1122 *group.shape, mode, types_none[0], 1123 group.preds[pi]); 1124 add_overloaded_function (instance, group.required_extensions); 1125 } 1126 else 1127 for (unsigned int ti = 0; group.types[ti][0] != NUM_TYPE_SUFFIXES; 1128 ++ti) 1129 { 1130 /* Stub out the types that are determined by overload 1131 resolution. */ 1132 type_suffix_pair types = { 1133 explicit_type0 ? group.types[ti][0] : NUM_TYPE_SUFFIXES, 1134 explicit_type1 ? group.types[ti][1] : NUM_TYPE_SUFFIXES 1135 }; 1136 function_instance instance (group.base_name, *group.base, 1137 *group.shape, mode, types, 1138 group.preds[pi]); 1139 add_overloaded_function (instance, group.required_extensions); 1140 } 1141 } 1142 } 1143 1144 /* Register all the functions in GROUP. */ 1145 void 1146 function_builder::register_function_group (const function_group_info &group) 1147 { 1148 (*group.shape)->build (*this, group); 1149 } 1150 1151 function_call_info::function_call_info (location_t location_in, 1152 const function_instance &instance_in, 1153 tree fndecl_in) 1154 : function_instance (instance_in), location (location_in), fndecl (fndecl_in) 1155 { 1156 } 1157 1158 function_resolver::function_resolver (location_t location, 1159 const function_instance &instance, 1160 tree fndecl, vec<tree, va_gc> &arglist) 1161 : function_call_info (location, instance, fndecl), m_arglist (arglist) 1162 { 1163 } 1164 1165 /* Return the vector type associated with type suffix TYPE. */ 1166 tree 1167 function_resolver::get_vector_type (type_suffix_index type) 1168 { 1169 return acle_vector_types[0][type_suffixes[type].vector_type]; 1170 } 1171 1172 /* Return the <stdint.h> name associated with TYPE. Using the <stdint.h> 1173 name should be more user-friendly than the underlying canonical type, 1174 since it makes the signedness and bitwidth explicit. */ 1175 const char * 1176 function_resolver::get_scalar_type_name (type_suffix_index type) 1177 { 1178 return vector_types[type_suffixes[type].vector_type].acle_name + 2; 1179 } 1180 1181 /* Return the type of argument I, or error_mark_node if it isn't 1182 well-formed. */ 1183 tree 1184 function_resolver::get_argument_type (unsigned int i) 1185 { 1186 tree arg = m_arglist[i]; 1187 return arg == error_mark_node ? arg : TREE_TYPE (arg); 1188 } 1189 1190 /* Return true if argument I is some form of scalar value. */ 1191 bool 1192 function_resolver::scalar_argument_p (unsigned int i) 1193 { 1194 tree type = get_argument_type (i); 1195 return (INTEGRAL_TYPE_P (type) 1196 /* Allow pointer types, leaving the frontend to warn where 1197 necessary. */ 1198 || POINTER_TYPE_P (type) 1199 || SCALAR_FLOAT_TYPE_P (type)); 1200 } 1201 1202 /* Report that the function has no form that takes type suffix TYPE. 1203 Return error_mark_node. */ 1204 tree 1205 function_resolver::report_no_such_form (type_suffix_index type) 1206 { 1207 error_at (location, "%qE has no form that takes %qT arguments", 1208 fndecl, get_vector_type (type)); 1209 return error_mark_node; 1210 } 1211 1212 /* Silently check whether there is an instance of the function with the 1213 mode suffix given by MODE and the type suffixes given by TYPE0 and TYPE1. 1214 Return its function decl if so, otherwise return null. */ 1215 tree 1216 function_resolver::lookup_form (mode_suffix_index mode, 1217 type_suffix_index type0, 1218 type_suffix_index type1) 1219 { 1220 type_suffix_pair types = { type0, type1 }; 1221 function_instance instance (base_name, base, shape, mode, types, pred); 1222 registered_function *rfn 1223 = function_table->find_with_hash (instance, instance.hash ()); 1224 return rfn ? rfn->decl : NULL_TREE; 1225 } 1226 1227 /* Resolve the function to one with the mode suffix given by MODE and the 1228 type suffixes given by TYPE0 and TYPE1. Return its function decl on 1229 success, otherwise report an error and return error_mark_node. */ 1230 tree 1231 function_resolver::resolve_to (mode_suffix_index mode, 1232 type_suffix_index type0, 1233 type_suffix_index type1) 1234 { 1235 tree res = lookup_form (mode, type0, type1); 1236 if (!res) 1237 { 1238 if (type1 == NUM_TYPE_SUFFIXES) 1239 return report_no_such_form (type0); 1240 if (type0 == type_suffix_ids[0]) 1241 return report_no_such_form (type1); 1242 /* To be filled in when we have other cases. */ 1243 gcc_unreachable (); 1244 } 1245 return res; 1246 } 1247 1248 /* Require argument ARGNO to be a 32-bit or 64-bit scalar integer type. 1249 Return the associated type suffix on success, otherwise report an 1250 error and return NUM_TYPE_SUFFIXES. */ 1251 type_suffix_index 1252 function_resolver::infer_integer_scalar_type (unsigned int argno) 1253 { 1254 tree actual = get_argument_type (argno); 1255 if (actual == error_mark_node) 1256 return NUM_TYPE_SUFFIXES; 1257 1258 /* Allow enums and booleans to decay to integers, for compatibility 1259 with C++ overloading rules. */ 1260 if (INTEGRAL_TYPE_P (actual)) 1261 { 1262 bool uns_p = TYPE_UNSIGNED (actual); 1263 /* Honor the usual integer promotions, so that resolution works 1264 in the same way as for C++. */ 1265 if (TYPE_PRECISION (actual) < 32) 1266 return TYPE_SUFFIX_s32; 1267 if (TYPE_PRECISION (actual) == 32) 1268 return uns_p ? TYPE_SUFFIX_u32 : TYPE_SUFFIX_s32; 1269 if (TYPE_PRECISION (actual) == 64) 1270 return uns_p ? TYPE_SUFFIX_u64 : TYPE_SUFFIX_s64; 1271 } 1272 1273 error_at (location, "passing %qT to argument %d of %qE, which expects" 1274 " a 32-bit or 64-bit integer type", actual, argno + 1, fndecl); 1275 return NUM_TYPE_SUFFIXES; 1276 } 1277 1278 /* Require argument ARGNO to be a pointer to a scalar type that has a 1279 corresponding type suffix. Return that type suffix on success, 1280 otherwise report an error and return NUM_TYPE_SUFFIXES. 1281 GATHER_SCATTER_P is true if the function is a gather/scatter 1282 operation, and so requires a pointer to 32-bit or 64-bit data. */ 1283 type_suffix_index 1284 function_resolver::infer_pointer_type (unsigned int argno, 1285 bool gather_scatter_p) 1286 { 1287 tree actual = get_argument_type (argno); 1288 if (actual == error_mark_node) 1289 return NUM_TYPE_SUFFIXES; 1290 1291 if (TREE_CODE (actual) != POINTER_TYPE) 1292 { 1293 error_at (location, "passing %qT to argument %d of %qE, which" 1294 " expects a pointer type", actual, argno + 1, fndecl); 1295 if (VECTOR_TYPE_P (actual) && gather_scatter_p) 1296 inform (location, "an explicit type suffix is needed" 1297 " when using a vector of base addresses"); 1298 return NUM_TYPE_SUFFIXES; 1299 } 1300 1301 tree target = TREE_TYPE (actual); 1302 type_suffix_index type = find_type_suffix_for_scalar_type (target); 1303 if (type == NUM_TYPE_SUFFIXES) 1304 { 1305 error_at (location, "passing %qT to argument %d of %qE, but %qT is not" 1306 " a valid SVE element type", actual, argno + 1, fndecl, 1307 build_qualified_type (target, 0)); 1308 return NUM_TYPE_SUFFIXES; 1309 } 1310 unsigned int bits = type_suffixes[type].element_bits; 1311 if (gather_scatter_p && bits != 32 && bits != 64) 1312 { 1313 error_at (location, "passing %qT to argument %d of %qE, which" 1314 " expects a pointer to 32-bit or 64-bit elements", 1315 actual, argno + 1, fndecl); 1316 return NUM_TYPE_SUFFIXES; 1317 } 1318 1319 return type; 1320 } 1321 1322 /* Require argument ARGNO to be a single vector or a tuple of NUM_VECTORS 1323 vectors; NUM_VECTORS is 1 for the former. Return the associated type 1324 suffix on success, using TYPE_SUFFIX_b for predicates. Report an error 1325 and return NUM_TYPE_SUFFIXES on failure. */ 1326 type_suffix_index 1327 function_resolver::infer_vector_or_tuple_type (unsigned int argno, 1328 unsigned int num_vectors) 1329 { 1330 tree actual = get_argument_type (argno); 1331 if (actual == error_mark_node) 1332 return NUM_TYPE_SUFFIXES; 1333 1334 /* A linear search should be OK here, since the code isn't hot and 1335 the number of types is only small. */ 1336 for (unsigned int size_i = 0; size_i < MAX_TUPLE_SIZE; ++size_i) 1337 for (unsigned int suffix_i = 0; suffix_i < NUM_TYPE_SUFFIXES; ++suffix_i) 1338 { 1339 vector_type_index type_i = type_suffixes[suffix_i].vector_type; 1340 tree type = acle_vector_types[size_i][type_i]; 1341 if (type && matches_type_p (type, actual)) 1342 { 1343 if (size_i + 1 == num_vectors) 1344 return type_suffix_index (suffix_i); 1345 1346 if (num_vectors == 1) 1347 error_at (location, "passing %qT to argument %d of %qE, which" 1348 " expects a single SVE vector rather than a tuple", 1349 actual, argno + 1, fndecl); 1350 else if (size_i == 0 && type_i != VECTOR_TYPE_svbool_t) 1351 error_at (location, "passing single vector %qT to argument %d" 1352 " of %qE, which expects a tuple of %d vectors", 1353 actual, argno + 1, fndecl, num_vectors); 1354 else 1355 error_at (location, "passing %qT to argument %d of %qE, which" 1356 " expects a tuple of %d vectors", actual, argno + 1, 1357 fndecl, num_vectors); 1358 return NUM_TYPE_SUFFIXES; 1359 } 1360 } 1361 1362 if (num_vectors == 1) 1363 error_at (location, "passing %qT to argument %d of %qE, which" 1364 " expects an SVE vector type", actual, argno + 1, fndecl); 1365 else 1366 error_at (location, "passing %qT to argument %d of %qE, which" 1367 " expects an SVE tuple type", actual, argno + 1, fndecl); 1368 return NUM_TYPE_SUFFIXES; 1369 } 1370 1371 /* Require argument ARGNO to have some form of vector type. Return the 1372 associated type suffix on success, using TYPE_SUFFIX_b for predicates. 1373 Report an error and return NUM_TYPE_SUFFIXES on failure. */ 1374 type_suffix_index 1375 function_resolver::infer_vector_type (unsigned int argno) 1376 { 1377 return infer_vector_or_tuple_type (argno, 1); 1378 } 1379 1380 /* Like infer_vector_type, but also require the type to be integral. */ 1381 type_suffix_index 1382 function_resolver::infer_integer_vector_type (unsigned int argno) 1383 { 1384 type_suffix_index type = infer_vector_type (argno); 1385 if (type == NUM_TYPE_SUFFIXES) 1386 return type; 1387 1388 if (!type_suffixes[type].integer_p) 1389 { 1390 error_at (location, "passing %qT to argument %d of %qE, which" 1391 " expects a vector of integers", get_argument_type (argno), 1392 argno + 1, fndecl); 1393 return NUM_TYPE_SUFFIXES; 1394 } 1395 1396 return type; 1397 } 1398 1399 /* Like infer_vector_type, but also require the type to be an unsigned 1400 integer. */ 1401 type_suffix_index 1402 function_resolver::infer_unsigned_vector_type (unsigned int argno) 1403 { 1404 type_suffix_index type = infer_vector_type (argno); 1405 if (type == NUM_TYPE_SUFFIXES) 1406 return type; 1407 1408 if (!type_suffixes[type].unsigned_p) 1409 { 1410 error_at (location, "passing %qT to argument %d of %qE, which" 1411 " expects a vector of unsigned integers", 1412 get_argument_type (argno), argno + 1, fndecl); 1413 return NUM_TYPE_SUFFIXES; 1414 } 1415 1416 return type; 1417 } 1418 1419 /* Like infer_vector_type, but also require the element size to be 1420 32 or 64 bits. */ 1421 type_suffix_index 1422 function_resolver::infer_sd_vector_type (unsigned int argno) 1423 { 1424 type_suffix_index type = infer_vector_type (argno); 1425 if (type == NUM_TYPE_SUFFIXES) 1426 return type; 1427 1428 unsigned int bits = type_suffixes[type].element_bits; 1429 if (bits != 32 && bits != 64) 1430 { 1431 error_at (location, "passing %qT to argument %d of %qE, which" 1432 " expects a vector of 32-bit or 64-bit elements", 1433 get_argument_type (argno), argno + 1, fndecl); 1434 return NUM_TYPE_SUFFIXES; 1435 } 1436 1437 return type; 1438 } 1439 1440 /* If the function operates on tuples of vectors, require argument ARGNO to be 1441 a tuple with the appropriate number of vectors, otherwise require it to be 1442 a single vector. Return the associated type suffix on success, using 1443 TYPE_SUFFIX_b for predicates. Report an error and return NUM_TYPE_SUFFIXES 1444 on failure. */ 1445 type_suffix_index 1446 function_resolver::infer_tuple_type (unsigned int argno) 1447 { 1448 return infer_vector_or_tuple_type (argno, vectors_per_tuple ()); 1449 } 1450 1451 /* Require argument ARGNO to be a vector or scalar argument. Return true 1452 if it is, otherwise report an appropriate error. */ 1453 bool 1454 function_resolver::require_vector_or_scalar_type (unsigned int argno) 1455 { 1456 tree actual = get_argument_type (argno); 1457 if (actual == error_mark_node) 1458 return false; 1459 1460 if (!scalar_argument_p (argno) && !VECTOR_TYPE_P (actual)) 1461 { 1462 error_at (location, "passing %qT to argument %d of %qE, which" 1463 " expects a vector or scalar type", actual, argno + 1, fndecl); 1464 return false; 1465 } 1466 1467 return true; 1468 } 1469 1470 /* Require argument ARGNO to have vector type TYPE, in cases where this 1471 requirement holds for all uses of the function. Return true if the 1472 argument has the right form, otherwise report an appropriate error. */ 1473 bool 1474 function_resolver::require_vector_type (unsigned int argno, 1475 vector_type_index type) 1476 { 1477 tree expected = acle_vector_types[0][type]; 1478 tree actual = get_argument_type (argno); 1479 if (actual == error_mark_node) 1480 return false; 1481 1482 if (!matches_type_p (expected, actual)) 1483 { 1484 error_at (location, "passing %qT to argument %d of %qE, which" 1485 " expects %qT", actual, argno + 1, fndecl, expected); 1486 return false; 1487 } 1488 return true; 1489 } 1490 1491 /* Like require_vector_type, but TYPE is inferred from previous arguments 1492 rather than being a fixed part of the function signature. This changes 1493 the nature of the error messages. */ 1494 bool 1495 function_resolver::require_matching_vector_type (unsigned int argno, 1496 type_suffix_index type) 1497 { 1498 type_suffix_index new_type = infer_vector_type (argno); 1499 if (new_type == NUM_TYPE_SUFFIXES) 1500 return false; 1501 1502 if (type != new_type) 1503 { 1504 error_at (location, "passing %qT to argument %d of %qE, but" 1505 " previous arguments had type %qT", 1506 get_vector_type (new_type), argno + 1, fndecl, 1507 get_vector_type (type)); 1508 return false; 1509 } 1510 return true; 1511 } 1512 1513 /* Require argument ARGNO to be a vector type with the following properties: 1514 1515 - the type class must be the same as FIRST_TYPE's if EXPECTED_TCLASS 1516 is SAME_TYPE_CLASS, otherwise it must be EXPECTED_TCLASS itself. 1517 1518 - the element size must be: 1519 1520 - the same as FIRST_TYPE's if EXPECTED_BITS == SAME_SIZE 1521 - half of FIRST_TYPE's if EXPECTED_BITS == HALF_SIZE 1522 - a quarter of FIRST_TYPE's if EXPECTED_BITS == QUARTER_SIZE 1523 - EXPECTED_BITS itself otherwise 1524 1525 Return true if the argument has the required type, otherwise report 1526 an appropriate error. 1527 1528 FIRST_ARGNO is the first argument that is known to have type FIRST_TYPE. 1529 Usually it comes before ARGNO, but sometimes it is more natural to resolve 1530 arguments out of order. 1531 1532 If the required properties depend on FIRST_TYPE then both FIRST_ARGNO and 1533 ARGNO contribute to the resolution process. If the required properties 1534 are fixed, only FIRST_ARGNO contributes to the resolution process. 1535 1536 This function is a bit of a Swiss army knife. The complication comes 1537 from trying to give good error messages when FIRST_ARGNO and ARGNO are 1538 inconsistent, since either of them might be wrong. */ 1539 bool function_resolver:: 1540 require_derived_vector_type (unsigned int argno, 1541 unsigned int first_argno, 1542 type_suffix_index first_type, 1543 type_class_index expected_tclass, 1544 unsigned int expected_bits) 1545 { 1546 /* If the type needs to match FIRST_ARGNO exactly, use the preferred 1547 error message for that case. The VECTOR_TYPE_P test excludes tuple 1548 types, which we handle below instead. */ 1549 bool both_vectors_p = VECTOR_TYPE_P (get_argument_type (first_argno)); 1550 if (both_vectors_p 1551 && expected_tclass == SAME_TYPE_CLASS 1552 && expected_bits == SAME_SIZE) 1553 { 1554 /* There's no need to resolve this case out of order. */ 1555 gcc_assert (argno > first_argno); 1556 return require_matching_vector_type (argno, first_type); 1557 } 1558 1559 /* Use FIRST_TYPE to get the expected type class and element size. */ 1560 type_class_index orig_expected_tclass = expected_tclass; 1561 if (expected_tclass == NUM_TYPE_CLASSES) 1562 expected_tclass = type_suffixes[first_type].tclass; 1563 1564 unsigned int orig_expected_bits = expected_bits; 1565 if (expected_bits == SAME_SIZE) 1566 expected_bits = type_suffixes[first_type].element_bits; 1567 else if (expected_bits == HALF_SIZE) 1568 expected_bits = type_suffixes[first_type].element_bits / 2; 1569 else if (expected_bits == QUARTER_SIZE) 1570 expected_bits = type_suffixes[first_type].element_bits / 4; 1571 1572 /* If the expected type doesn't depend on FIRST_TYPE at all, 1573 just check for the fixed choice of vector type. */ 1574 if (expected_tclass == orig_expected_tclass 1575 && expected_bits == orig_expected_bits) 1576 { 1577 const type_suffix_info &expected_suffix 1578 = type_suffixes[find_type_suffix (expected_tclass, expected_bits)]; 1579 return require_vector_type (argno, expected_suffix.vector_type); 1580 } 1581 1582 /* Require the argument to be some form of SVE vector type, 1583 without being specific about the type of vector we want. */ 1584 type_suffix_index actual_type = infer_vector_type (argno); 1585 if (actual_type == NUM_TYPE_SUFFIXES) 1586 return false; 1587 1588 /* Exit now if we got the right type. */ 1589 bool tclass_ok_p = (type_suffixes[actual_type].tclass == expected_tclass); 1590 bool size_ok_p = (type_suffixes[actual_type].element_bits == expected_bits); 1591 if (tclass_ok_p && size_ok_p) 1592 return true; 1593 1594 /* First look for cases in which the actual type contravenes a fixed 1595 size requirement, without having to refer to FIRST_TYPE. */ 1596 if (!size_ok_p && expected_bits == orig_expected_bits) 1597 { 1598 error_at (location, "passing %qT to argument %d of %qE, which" 1599 " expects a vector of %d-bit elements", 1600 get_vector_type (actual_type), argno + 1, fndecl, 1601 expected_bits); 1602 return false; 1603 } 1604 1605 /* Likewise for a fixed type class requirement. This is only ever 1606 needed for signed and unsigned types, so don't create unnecessary 1607 translation work for other type classes. */ 1608 if (!tclass_ok_p && orig_expected_tclass == TYPE_signed) 1609 { 1610 error_at (location, "passing %qT to argument %d of %qE, which" 1611 " expects a vector of signed integers", 1612 get_vector_type (actual_type), argno + 1, fndecl); 1613 return false; 1614 } 1615 if (!tclass_ok_p && orig_expected_tclass == TYPE_unsigned) 1616 { 1617 error_at (location, "passing %qT to argument %d of %qE, which" 1618 " expects a vector of unsigned integers", 1619 get_vector_type (actual_type), argno + 1, fndecl); 1620 return false; 1621 } 1622 1623 /* Make sure that FIRST_TYPE itself is sensible before using it 1624 as a basis for an error message. */ 1625 if (resolve_to (mode_suffix_id, first_type) == error_mark_node) 1626 return false; 1627 1628 /* If the arguments have consistent type classes, but a link between 1629 the sizes has been broken, try to describe the error in those terms. */ 1630 if (both_vectors_p && tclass_ok_p && orig_expected_bits == SAME_SIZE) 1631 { 1632 if (argno < first_argno) 1633 { 1634 std::swap (argno, first_argno); 1635 std::swap (actual_type, first_type); 1636 } 1637 error_at (location, "arguments %d and %d of %qE must have the" 1638 " same element size, but the values passed here have type" 1639 " %qT and %qT respectively", first_argno + 1, argno + 1, 1640 fndecl, get_vector_type (first_type), 1641 get_vector_type (actual_type)); 1642 return false; 1643 } 1644 1645 /* Likewise in reverse: look for cases in which the sizes are consistent 1646 but a link between the type classes has been broken. */ 1647 if (both_vectors_p 1648 && size_ok_p 1649 && orig_expected_tclass == SAME_TYPE_CLASS 1650 && type_suffixes[first_type].integer_p 1651 && type_suffixes[actual_type].integer_p) 1652 { 1653 if (argno < first_argno) 1654 { 1655 std::swap (argno, first_argno); 1656 std::swap (actual_type, first_type); 1657 } 1658 error_at (location, "arguments %d and %d of %qE must have the" 1659 " same signedness, but the values passed here have type" 1660 " %qT and %qT respectively", first_argno + 1, argno + 1, 1661 fndecl, get_vector_type (first_type), 1662 get_vector_type (actual_type)); 1663 return false; 1664 } 1665 1666 /* The two arguments are wildly inconsistent. */ 1667 type_suffix_index expected_type 1668 = find_type_suffix (expected_tclass, expected_bits); 1669 error_at (location, "passing %qT instead of the expected %qT to argument" 1670 " %d of %qE, after passing %qT to argument %d", 1671 get_vector_type (actual_type), get_vector_type (expected_type), 1672 argno + 1, fndecl, get_argument_type (first_argno), 1673 first_argno + 1); 1674 return false; 1675 } 1676 1677 /* Require argument ARGNO to match argument FIRST_ARGNO, which was inferred 1678 to be a pointer to a scalar element of type TYPE. */ 1679 bool 1680 function_resolver::require_matching_pointer_type (unsigned int argno, 1681 unsigned int first_argno, 1682 type_suffix_index type) 1683 { 1684 type_suffix_index new_type = infer_pointer_type (argno); 1685 if (new_type == NUM_TYPE_SUFFIXES) 1686 return false; 1687 1688 if (type != new_type) 1689 { 1690 error_at (location, "passing %qT to argument %d of %qE, but" 1691 " argument %d had type %qT", get_argument_type (argno), 1692 argno + 1, fndecl, first_argno + 1, 1693 get_argument_type (first_argno)); 1694 return false; 1695 } 1696 return true; 1697 } 1698 1699 /* Require argument ARGNO to be a (possibly variable) scalar, using EXPECTED 1700 as the name of its expected type. Return true if the argument has the 1701 right form, otherwise report an appropriate error. */ 1702 bool 1703 function_resolver::require_scalar_type (unsigned int argno, 1704 const char *expected) 1705 { 1706 if (!scalar_argument_p (argno)) 1707 { 1708 error_at (location, "passing %qT to argument %d of %qE, which" 1709 " expects %qs", get_argument_type (argno), argno + 1, 1710 fndecl, expected); 1711 return false; 1712 } 1713 return true; 1714 } 1715 1716 /* Require argument ARGNO to be some form of pointer, without being specific 1717 about its target type. Return true if the argument has the right form, 1718 otherwise report an appropriate error. */ 1719 bool 1720 function_resolver::require_pointer_type (unsigned int argno) 1721 { 1722 if (!scalar_argument_p (argno)) 1723 { 1724 error_at (location, "passing %qT to argument %d of %qE, which" 1725 " expects a scalar pointer", get_argument_type (argno), 1726 argno + 1, fndecl); 1727 return false; 1728 } 1729 return true; 1730 } 1731 1732 /* Argument FIRST_ARGNO is a scalar with type EXPECTED_TYPE, and argument 1733 ARGNO should be consistent with it. Return true if it is, otherwise 1734 report an appropriate error. */ 1735 bool function_resolver:: 1736 require_matching_integer_scalar_type (unsigned int argno, 1737 unsigned int first_argno, 1738 type_suffix_index expected_type) 1739 { 1740 type_suffix_index actual_type = infer_integer_scalar_type (argno); 1741 if (actual_type == NUM_TYPE_SUFFIXES) 1742 return false; 1743 1744 if (actual_type == expected_type) 1745 return true; 1746 1747 error_at (location, "call to %qE is ambiguous; argument %d has type" 1748 " %qs but argument %d has type %qs", fndecl, 1749 first_argno + 1, get_scalar_type_name (expected_type), 1750 argno + 1, get_scalar_type_name (actual_type)); 1751 return false; 1752 } 1753 1754 /* Require argument ARGNO to be a (possibly variable) scalar, expecting it 1755 to have the following properties: 1756 1757 - the type class must be the same as for type suffix 0 if EXPECTED_TCLASS 1758 is SAME_TYPE_CLASS, otherwise it must be EXPECTED_TCLASS itself. 1759 1760 - the element size must be the same as for type suffix 0 if EXPECTED_BITS 1761 is SAME_TYPE_SIZE, otherwise it must be EXPECTED_BITS itself. 1762 1763 Return true if the argument is valid, otherwise report an appropriate error. 1764 1765 Note that we don't check whether the scalar type actually has the required 1766 properties, since that's subject to implicit promotions and conversions. 1767 Instead we just use the expected properties to tune the error message. */ 1768 bool function_resolver:: 1769 require_derived_scalar_type (unsigned int argno, 1770 type_class_index expected_tclass, 1771 unsigned int expected_bits) 1772 { 1773 gcc_assert (expected_tclass == SAME_TYPE_CLASS 1774 || expected_tclass == TYPE_signed 1775 || expected_tclass == TYPE_unsigned); 1776 1777 /* If the expected type doesn't depend on the type suffix at all, 1778 just check for the fixed choice of scalar type. */ 1779 if (expected_tclass != SAME_TYPE_CLASS && expected_bits != SAME_SIZE) 1780 { 1781 type_suffix_index expected_type 1782 = find_type_suffix (expected_tclass, expected_bits); 1783 return require_scalar_type (argno, get_scalar_type_name (expected_type)); 1784 } 1785 1786 if (scalar_argument_p (argno)) 1787 return true; 1788 1789 if (expected_tclass == SAME_TYPE_CLASS) 1790 /* It doesn't really matter whether the element is expected to be 1791 the same size as type suffix 0. */ 1792 error_at (location, "passing %qT to argument %d of %qE, which" 1793 " expects a scalar element", get_argument_type (argno), 1794 argno + 1, fndecl); 1795 else 1796 /* It doesn't seem useful to distinguish between signed and unsigned 1797 scalars here. */ 1798 error_at (location, "passing %qT to argument %d of %qE, which" 1799 " expects a scalar integer", get_argument_type (argno), 1800 argno + 1, fndecl); 1801 return false; 1802 } 1803 1804 /* Require argument ARGNO to be suitable for an integer constant expression. 1805 Return true if it is, otherwise report an appropriate error. 1806 1807 function_checker checks whether the argument is actually constant and 1808 has a suitable range. The reason for distinguishing immediate arguments 1809 here is because it provides more consistent error messages than 1810 require_scalar_type would. */ 1811 bool 1812 function_resolver::require_integer_immediate (unsigned int argno) 1813 { 1814 if (!scalar_argument_p (argno)) 1815 { 1816 report_non_ice (location, fndecl, argno); 1817 return false; 1818 } 1819 return true; 1820 } 1821 1822 /* Require argument ARGNO to be a vector base in a gather-style address. 1823 Return its type on success, otherwise return NUM_VECTOR_TYPES. */ 1824 vector_type_index 1825 function_resolver::infer_vector_base_type (unsigned int argno) 1826 { 1827 type_suffix_index type = infer_vector_type (argno); 1828 if (type == NUM_TYPE_SUFFIXES) 1829 return NUM_VECTOR_TYPES; 1830 1831 if (type == TYPE_SUFFIX_u32 || type == TYPE_SUFFIX_u64) 1832 return type_suffixes[type].vector_type; 1833 1834 error_at (location, "passing %qT to argument %d of %qE, which" 1835 " expects %qs or %qs", get_argument_type (argno), 1836 argno + 1, fndecl, "svuint32_t", "svuint64_t"); 1837 return NUM_VECTOR_TYPES; 1838 } 1839 1840 /* Require argument ARGNO to be a vector displacement in a gather-style 1841 address. Return its type on success, otherwise return NUM_VECTOR_TYPES. */ 1842 vector_type_index 1843 function_resolver::infer_vector_displacement_type (unsigned int argno) 1844 { 1845 type_suffix_index type = infer_integer_vector_type (argno); 1846 if (type == NUM_TYPE_SUFFIXES) 1847 return NUM_VECTOR_TYPES; 1848 1849 if (type_suffixes[type].integer_p 1850 && (type_suffixes[type].element_bits == 32 1851 || type_suffixes[type].element_bits == 64)) 1852 return type_suffixes[type].vector_type; 1853 1854 error_at (location, "passing %qT to argument %d of %qE, which" 1855 " expects a vector of 32-bit or 64-bit integers", 1856 get_argument_type (argno), argno + 1, fndecl); 1857 return NUM_VECTOR_TYPES; 1858 } 1859 1860 /* Require argument ARGNO to be a vector displacement in a gather-style 1861 address. There are three possible uses: 1862 1863 - for loading into elements of type TYPE (when LOAD_P is true) 1864 - for storing from elements of type TYPE (when LOAD_P is false) 1865 - for prefetching data (when TYPE is NUM_TYPE_SUFFIXES) 1866 1867 The overloaded function's mode suffix determines the units of the 1868 displacement (bytes for "_offset", elements for "_index"). 1869 1870 Return the associated mode on success, otherwise report an error 1871 and return MODE_none. */ 1872 mode_suffix_index 1873 function_resolver::resolve_sv_displacement (unsigned int argno, 1874 type_suffix_index type, 1875 bool load_p) 1876 { 1877 if (type == NUM_TYPE_SUFFIXES) 1878 { 1879 /* For prefetches, the base is a void pointer and the displacement 1880 can be any valid offset or index type. */ 1881 vector_type_index displacement_vector_type 1882 = infer_vector_displacement_type (argno); 1883 if (displacement_vector_type == NUM_VECTOR_TYPES) 1884 return MODE_none; 1885 1886 mode_suffix_index mode = find_mode_suffix (NUM_VECTOR_TYPES, 1887 displacement_vector_type, 1888 displacement_units ()); 1889 gcc_assert (mode != MODE_none); 1890 return mode; 1891 } 1892 1893 unsigned int required_bits = type_suffixes[type].element_bits; 1894 if (required_bits == 32 1895 && displacement_units () == UNITS_elements 1896 && !lookup_form (MODE_s32index, type) 1897 && !lookup_form (MODE_u32index, type)) 1898 { 1899 if (lookup_form (MODE_u32base_index, type)) 1900 { 1901 if (type_suffix_ids[0] == NUM_TYPE_SUFFIXES) 1902 { 1903 gcc_assert (!load_p); 1904 error_at (location, "when storing %qT, %qE requires a vector" 1905 " base and a scalar index", get_vector_type (type), 1906 fndecl); 1907 } 1908 else 1909 error_at (location, "%qE requires a vector base and a scalar" 1910 " index", fndecl); 1911 } 1912 else 1913 error_at (location, "%qE does not support 32-bit vector type %qT", 1914 fndecl, get_vector_type (type)); 1915 return MODE_none; 1916 } 1917 1918 /* Check for some form of vector type, without naming any in particular 1919 as being expected. */ 1920 type_suffix_index displacement_type = infer_vector_type (argno); 1921 if (displacement_type == NUM_TYPE_SUFFIXES) 1922 return MODE_none; 1923 1924 /* If the displacement type is consistent with the data vector type, 1925 try to find the associated mode suffix. This will fall through 1926 for non-integral displacement types. */ 1927 if (type_suffixes[displacement_type].element_bits == required_bits) 1928 { 1929 vector_type_index displacement_vector_type 1930 = type_suffixes[displacement_type].vector_type; 1931 mode_suffix_index mode = find_mode_suffix (NUM_VECTOR_TYPES, 1932 displacement_vector_type, 1933 displacement_units ()); 1934 if (mode != MODE_none) 1935 { 1936 if (mode == MODE_s32offset 1937 && !lookup_form (mode, type) 1938 && lookup_form (MODE_u32offset, type)) 1939 { 1940 if (type_suffix_ids[0] == NUM_TYPE_SUFFIXES) 1941 error_at (location, "%qE does not support 32-bit sign-extended" 1942 " offsets", fndecl); 1943 else 1944 error_at (location, "%qE does not support sign-extended" 1945 " offsets", fndecl); 1946 return MODE_none; 1947 } 1948 return mode; 1949 } 1950 } 1951 1952 if (type_suffix_ids[0] == NUM_TYPE_SUFFIXES) 1953 { 1954 /* TYPE has been inferred rather than specified by the user, 1955 so mention it in the error messages. */ 1956 if (load_p) 1957 error_at (location, "passing %qT to argument %d of %qE, which when" 1958 " loading %qT expects a vector of %d-bit integers", 1959 get_argument_type (argno), argno + 1, fndecl, 1960 get_vector_type (type), required_bits); 1961 else 1962 error_at (location, "passing %qT to argument %d of %qE, which when" 1963 " storing %qT expects a vector of %d-bit integers", 1964 get_argument_type (argno), argno + 1, fndecl, 1965 get_vector_type (type), required_bits); 1966 } 1967 else 1968 /* TYPE is part of the function name. */ 1969 error_at (location, "passing %qT to argument %d of %qE, which" 1970 " expects a vector of %d-bit integers", 1971 get_argument_type (argno), argno + 1, fndecl, required_bits); 1972 return MODE_none; 1973 } 1974 1975 /* Require the arguments starting at ARGNO to form a gather-style address. 1976 There are three possible uses: 1977 1978 - for loading into elements of type TYPE (when LOAD_P is true) 1979 - for storing from elements of type TYPE (when LOAD_P is false) 1980 - for prefetching data (when TYPE is NUM_TYPE_SUFFIXES) 1981 1982 The three possible addresses are: 1983 1984 - a vector base with no displacement 1985 - a vector base and a scalar displacement 1986 - a scalar (pointer) base and a vector displacement 1987 1988 The overloaded function's mode suffix determines whether there is 1989 a displacement, and if so, what units it uses: 1990 1991 - MODE_none: no displacement 1992 - MODE_offset: the displacement is measured in bytes 1993 - MODE_index: the displacement is measured in elements 1994 1995 Return the mode of the non-overloaded function on success, otherwise 1996 report an error and return MODE_none. */ 1997 mode_suffix_index 1998 function_resolver::resolve_gather_address (unsigned int argno, 1999 type_suffix_index type, 2000 bool load_p) 2001 { 2002 tree actual = get_argument_type (argno); 2003 if (actual == error_mark_node) 2004 return MODE_none; 2005 2006 if (displacement_units () != UNITS_none) 2007 { 2008 /* Some form of displacement is needed. First handle a scalar 2009 pointer base and a vector displacement. */ 2010 if (scalar_argument_p (argno)) 2011 /* Don't check the pointer type here, since there's only one valid 2012 choice. Leave that to the frontend. */ 2013 return resolve_sv_displacement (argno + 1, type, load_p); 2014 2015 if (!VECTOR_TYPE_P (actual)) 2016 { 2017 error_at (location, "passing %qT to argument %d of %qE," 2018 " which expects a vector or pointer base address", 2019 actual, argno + 1, fndecl); 2020 return MODE_none; 2021 } 2022 } 2023 2024 /* Check for the correct choice of vector base type. */ 2025 vector_type_index base_vector_type; 2026 if (type == NUM_TYPE_SUFFIXES) 2027 { 2028 /* Since prefetches have no type suffix, there is a free choice 2029 between 32-bit and 64-bit base addresses. */ 2030 base_vector_type = infer_vector_base_type (argno); 2031 if (base_vector_type == NUM_VECTOR_TYPES) 2032 return MODE_none; 2033 } 2034 else 2035 { 2036 /* Check for some form of vector type, without saying which type 2037 we expect. */ 2038 type_suffix_index base_type = infer_vector_type (argno); 2039 if (base_type == NUM_TYPE_SUFFIXES) 2040 return MODE_none; 2041 2042 /* Check whether the type is the right one. */ 2043 unsigned int required_bits = type_suffixes[type].element_bits; 2044 gcc_assert (required_bits == 32 || required_bits == 64); 2045 type_suffix_index required_type = (required_bits == 32 2046 ? TYPE_SUFFIX_u32 2047 : TYPE_SUFFIX_u64); 2048 if (required_type != base_type) 2049 { 2050 error_at (location, "passing %qT to argument %d of %qE," 2051 " which expects %qT", actual, argno + 1, fndecl, 2052 get_vector_type (required_type)); 2053 return MODE_none; 2054 } 2055 base_vector_type = type_suffixes[base_type].vector_type; 2056 } 2057 2058 /* Check the scalar displacement, if any. */ 2059 if (displacement_units () != UNITS_none 2060 && !require_scalar_type (argno + 1, "int64_t")) 2061 return MODE_none; 2062 2063 /* Find the appropriate mode suffix. The checks above should have 2064 weeded out all erroneous cases. */ 2065 for (unsigned int mode_i = 0; mode_i < ARRAY_SIZE (mode_suffixes); ++mode_i) 2066 { 2067 const mode_suffix_info &mode = mode_suffixes[mode_i]; 2068 if (mode.base_vector_type == base_vector_type 2069 && mode.displacement_vector_type == NUM_VECTOR_TYPES 2070 && mode.displacement_units == displacement_units ()) 2071 return mode_suffix_index (mode_i); 2072 } 2073 2074 gcc_unreachable (); 2075 } 2076 2077 /* Require arguments ARGNO and ARGNO + 1 to form an ADR-style address, 2078 i.e. one with a vector of base addresses and a vector of displacements. 2079 The overloaded function's mode suffix determines the units of the 2080 displacement (bytes for "_offset", elements for "_index"). 2081 2082 Return the associated mode suffix on success, otherwise report 2083 an error and return MODE_none. */ 2084 mode_suffix_index 2085 function_resolver::resolve_adr_address (unsigned int argno) 2086 { 2087 vector_type_index base_type = infer_vector_base_type (argno); 2088 if (base_type == NUM_VECTOR_TYPES) 2089 return MODE_none; 2090 2091 vector_type_index displacement_type 2092 = infer_vector_displacement_type (argno + 1); 2093 if (displacement_type == NUM_VECTOR_TYPES) 2094 return MODE_none; 2095 2096 mode_suffix_index mode = find_mode_suffix (base_type, displacement_type, 2097 displacement_units ()); 2098 if (mode == MODE_none) 2099 { 2100 if (mode_suffix_id == MODE_offset) 2101 error_at (location, "cannot combine a base of type %qT with" 2102 " an offset of type %qT", 2103 get_argument_type (argno), get_argument_type (argno + 1)); 2104 else 2105 error_at (location, "cannot combine a base of type %qT with" 2106 " an index of type %qT", 2107 get_argument_type (argno), get_argument_type (argno + 1)); 2108 } 2109 return mode; 2110 } 2111 2112 /* Require the function to have exactly EXPECTED arguments. Return true 2113 if it does, otherwise report an appropriate error. */ 2114 bool 2115 function_resolver::check_num_arguments (unsigned int expected) 2116 { 2117 if (m_arglist.length () < expected) 2118 error_at (location, "too few arguments to function %qE", fndecl); 2119 else if (m_arglist.length () > expected) 2120 error_at (location, "too many arguments to function %qE", fndecl); 2121 return m_arglist.length () == expected; 2122 } 2123 2124 /* If the function is predicated, check that the first argument is a 2125 suitable governing predicate. Also check that there are NOPS further 2126 arguments after any governing predicate, but don't check what they are. 2127 2128 Return true on success, otherwise report a suitable error. 2129 When returning true: 2130 2131 - set I to the number of the first unchecked argument. 2132 - set NARGS to the total number of arguments. */ 2133 bool 2134 function_resolver::check_gp_argument (unsigned int nops, 2135 unsigned int &i, unsigned int &nargs) 2136 { 2137 i = 0; 2138 if (pred != PRED_none) 2139 { 2140 /* Unary merge operations should use resolve_unary instead. */ 2141 gcc_assert (nops != 1 || pred != PRED_m); 2142 nargs = nops + 1; 2143 if (!check_num_arguments (nargs) 2144 || !require_vector_type (i, VECTOR_TYPE_svbool_t)) 2145 return false; 2146 i += 1; 2147 } 2148 else 2149 { 2150 nargs = nops; 2151 if (!check_num_arguments (nargs)) 2152 return false; 2153 } 2154 2155 return true; 2156 } 2157 2158 /* Finish resolving a function whose final argument can be a vector 2159 or a scalar, with the function having an implicit "_n" suffix 2160 in the latter case. This "_n" form might only exist for certain 2161 type suffixes. 2162 2163 ARGNO is the index of the final argument. The inferred type suffix 2164 was obtained from argument FIRST_ARGNO, which has type FIRST_TYPE. 2165 EXPECTED_TCLASS and EXPECTED_BITS describe the expected properties 2166 of the final vector or scalar argument, in the same way as for 2167 require_derived_vector_type. INFERRED_TYPE is the inferred type 2168 suffix itself, or NUM_TYPE_SUFFIXES if it's the same as FIRST_TYPE. 2169 2170 Return the function decl of the resolved function on success, 2171 otherwise report a suitable error and return error_mark_node. */ 2172 tree function_resolver:: 2173 finish_opt_n_resolution (unsigned int argno, unsigned int first_argno, 2174 type_suffix_index first_type, 2175 type_class_index expected_tclass, 2176 unsigned int expected_bits, 2177 type_suffix_index inferred_type) 2178 { 2179 if (inferred_type == NUM_TYPE_SUFFIXES) 2180 inferred_type = first_type; 2181 tree scalar_form = lookup_form (MODE_n, inferred_type); 2182 2183 /* Allow the final argument to be scalar, if an _n form exists. */ 2184 if (scalar_argument_p (argno)) 2185 { 2186 if (scalar_form) 2187 return scalar_form; 2188 2189 /* Check the vector form normally. If that succeeds, raise an 2190 error about having no corresponding _n form. */ 2191 tree res = resolve_to (mode_suffix_id, inferred_type); 2192 if (res != error_mark_node) 2193 error_at (location, "passing %qT to argument %d of %qE, but its" 2194 " %qT form does not accept scalars", 2195 get_argument_type (argno), argno + 1, fndecl, 2196 get_vector_type (first_type)); 2197 return error_mark_node; 2198 } 2199 2200 /* If an _n form does exist, provide a more accurate message than 2201 require_derived_vector_type would for arguments that are neither 2202 vectors nor scalars. */ 2203 if (scalar_form && !require_vector_or_scalar_type (argno)) 2204 return error_mark_node; 2205 2206 /* Check for the correct vector type. */ 2207 if (!require_derived_vector_type (argno, first_argno, first_type, 2208 expected_tclass, expected_bits)) 2209 return error_mark_node; 2210 2211 return resolve_to (mode_suffix_id, inferred_type); 2212 } 2213 2214 /* Resolve a (possibly predicated) unary function. If the function uses 2215 merge predication or if TREAT_AS_MERGE_P is true, there is an extra 2216 vector argument before the governing predicate that specifies the 2217 values of inactive elements. This argument has the following 2218 properties: 2219 2220 - the type class must be the same as for active elements if MERGE_TCLASS 2221 is SAME_TYPE_CLASS, otherwise it must be MERGE_TCLASS itself. 2222 2223 - the element size must be the same as for active elements if MERGE_BITS 2224 is SAME_TYPE_SIZE, otherwise it must be MERGE_BITS itself. 2225 2226 Return the function decl of the resolved function on success, 2227 otherwise report a suitable error and return error_mark_node. */ 2228 tree 2229 function_resolver::resolve_unary (type_class_index merge_tclass, 2230 unsigned int merge_bits, 2231 bool treat_as_merge_p) 2232 { 2233 type_suffix_index type; 2234 if (pred == PRED_m || treat_as_merge_p) 2235 { 2236 if (!check_num_arguments (3)) 2237 return error_mark_node; 2238 if (merge_tclass == SAME_TYPE_CLASS && merge_bits == SAME_SIZE) 2239 { 2240 /* The inactive elements are the same as the active elements, 2241 so we can use normal left-to-right resolution. */ 2242 if ((type = infer_vector_type (0)) == NUM_TYPE_SUFFIXES 2243 || !require_vector_type (1, VECTOR_TYPE_svbool_t) 2244 || !require_matching_vector_type (2, type)) 2245 return error_mark_node; 2246 } 2247 else 2248 { 2249 /* The inactive element type is a function of the active one, 2250 so resolve the active one first. */ 2251 if (!require_vector_type (1, VECTOR_TYPE_svbool_t) 2252 || (type = infer_vector_type (2)) == NUM_TYPE_SUFFIXES 2253 || !require_derived_vector_type (0, 2, type, merge_tclass, 2254 merge_bits)) 2255 return error_mark_node; 2256 } 2257 } 2258 else 2259 { 2260 /* We just need to check the predicate (if any) and the single 2261 vector argument. */ 2262 unsigned int i, nargs; 2263 if (!check_gp_argument (1, i, nargs) 2264 || (type = infer_vector_type (i)) == NUM_TYPE_SUFFIXES) 2265 return error_mark_node; 2266 } 2267 2268 /* Handle convert-like functions in which the first type suffix is 2269 explicit. */ 2270 if (type_suffix_ids[0] != NUM_TYPE_SUFFIXES) 2271 return resolve_to (mode_suffix_id, type_suffix_ids[0], type); 2272 2273 return resolve_to (mode_suffix_id, type); 2274 } 2275 2276 /* Resolve a (possibly predicated) function that takes NOPS like-typed 2277 vector arguments followed by NIMM integer immediates. Return the 2278 function decl of the resolved function on success, otherwise report 2279 a suitable error and return error_mark_node. */ 2280 tree 2281 function_resolver::resolve_uniform (unsigned int nops, unsigned int nimm) 2282 { 2283 unsigned int i, nargs; 2284 type_suffix_index type; 2285 if (!check_gp_argument (nops + nimm, i, nargs) 2286 || (type = infer_vector_type (i)) == NUM_TYPE_SUFFIXES) 2287 return error_mark_node; 2288 2289 i += 1; 2290 for (; i < nargs - nimm; ++i) 2291 if (!require_matching_vector_type (i, type)) 2292 return error_mark_node; 2293 2294 for (; i < nargs; ++i) 2295 if (!require_integer_immediate (i)) 2296 return error_mark_node; 2297 2298 return resolve_to (mode_suffix_id, type); 2299 } 2300 2301 /* Resolve a (possibly predicated) function that offers a choice between 2302 taking: 2303 2304 - NOPS like-typed vector arguments or 2305 - NOPS - 1 like-typed vector arguments followed by a scalar argument 2306 2307 Return the function decl of the resolved function on success, 2308 otherwise report a suitable error and return error_mark_node. */ 2309 tree 2310 function_resolver::resolve_uniform_opt_n (unsigned int nops) 2311 { 2312 unsigned int i, nargs; 2313 type_suffix_index type; 2314 if (!check_gp_argument (nops, i, nargs) 2315 || (type = infer_vector_type (i)) == NUM_TYPE_SUFFIXES) 2316 return error_mark_node; 2317 2318 unsigned int first_arg = i++; 2319 for (; i < nargs - 1; ++i) 2320 if (!require_matching_vector_type (i, type)) 2321 return error_mark_node; 2322 2323 return finish_opt_n_resolution (i, first_arg, type); 2324 } 2325 2326 /* If the call is erroneous, report an appropriate error and return 2327 error_mark_node. Otherwise, if the function is overloaded, return 2328 the decl of the non-overloaded function. Return NULL_TREE otherwise, 2329 indicating that the call should be processed in the normal way. */ 2330 tree 2331 function_resolver::resolve () 2332 { 2333 return shape->resolve (*this); 2334 } 2335 2336 function_checker::function_checker (location_t location, 2337 const function_instance &instance, 2338 tree fndecl, tree fntype, 2339 unsigned int nargs, tree *args) 2340 : function_call_info (location, instance, fndecl), 2341 m_fntype (fntype), m_nargs (nargs), m_args (args), 2342 /* We don't have to worry about unary _m operations here, since they 2343 never have arguments that need checking. */ 2344 m_base_arg (pred != PRED_none ? 1 : 0) 2345 { 2346 } 2347 2348 /* Return true if argument ARGNO exists. which it might not for 2349 erroneous calls. It is safe to wave through checks if this 2350 function returns false. */ 2351 bool 2352 function_checker::argument_exists_p (unsigned int argno) 2353 { 2354 gcc_assert (argno < (unsigned int) type_num_arguments (m_fntype)); 2355 return argno < m_nargs; 2356 } 2357 2358 /* Check that argument ARGNO is an integer constant expression and 2359 store its value in VALUE_OUT if so. The caller should first 2360 check that argument ARGNO exists. */ 2361 bool 2362 function_checker::require_immediate (unsigned int argno, 2363 HOST_WIDE_INT &value_out) 2364 { 2365 gcc_assert (argno < m_nargs); 2366 tree arg = m_args[argno]; 2367 2368 /* The type and range are unsigned, so read the argument as an 2369 unsigned rather than signed HWI. */ 2370 if (!tree_fits_uhwi_p (arg)) 2371 { 2372 report_non_ice (location, fndecl, argno); 2373 return false; 2374 } 2375 2376 /* ...but treat VALUE_OUT as signed for error reporting, since printing 2377 -1 is more user-friendly than the maximum uint64_t value. */ 2378 value_out = tree_to_uhwi (arg); 2379 return true; 2380 } 2381 2382 /* Check that argument REL_ARGNO is an integer constant expression that 2383 has the value VALUE0 or VALUE1. REL_ARGNO counts from the end of the 2384 predication arguments. */ 2385 bool 2386 function_checker::require_immediate_either_or (unsigned int rel_argno, 2387 HOST_WIDE_INT value0, 2388 HOST_WIDE_INT value1) 2389 { 2390 unsigned int argno = m_base_arg + rel_argno; 2391 if (!argument_exists_p (argno)) 2392 return true; 2393 2394 HOST_WIDE_INT actual; 2395 if (!require_immediate (argno, actual)) 2396 return false; 2397 2398 if (actual != value0 && actual != value1) 2399 { 2400 report_neither_nor (location, fndecl, argno, actual, 90, 270); 2401 return false; 2402 } 2403 2404 return true; 2405 } 2406 2407 /* Check that argument REL_ARGNO is an integer constant expression that has 2408 a valid value for enumeration type TYPE. REL_ARGNO counts from the end 2409 of the predication arguments. */ 2410 bool 2411 function_checker::require_immediate_enum (unsigned int rel_argno, tree type) 2412 { 2413 unsigned int argno = m_base_arg + rel_argno; 2414 if (!argument_exists_p (argno)) 2415 return true; 2416 2417 HOST_WIDE_INT actual; 2418 if (!require_immediate (argno, actual)) 2419 return false; 2420 2421 for (tree entry = TYPE_VALUES (type); entry; entry = TREE_CHAIN (entry)) 2422 { 2423 /* The value is an INTEGER_CST for C and a CONST_DECL wrapper 2424 around an INTEGER_CST for C++. */ 2425 tree value = TREE_VALUE (entry); 2426 if (TREE_CODE (value) == CONST_DECL) 2427 value = DECL_INITIAL (value); 2428 if (wi::to_widest (value) == actual) 2429 return true; 2430 } 2431 2432 report_not_enum (location, fndecl, argno, actual, type); 2433 return false; 2434 } 2435 2436 /* Check that argument REL_ARGNO is suitable for indexing argument 2437 REL_ARGNO - 1, in groups of GROUP_SIZE elements. REL_ARGNO counts 2438 from the end of the predication arguments. */ 2439 bool 2440 function_checker::require_immediate_lane_index (unsigned int rel_argno, 2441 unsigned int group_size) 2442 { 2443 unsigned int argno = m_base_arg + rel_argno; 2444 if (!argument_exists_p (argno)) 2445 return true; 2446 2447 /* Get the type of the previous argument. tree_argument_type wants a 2448 1-based number, whereas ARGNO is 0-based. */ 2449 machine_mode mode = TYPE_MODE (type_argument_type (m_fntype, argno)); 2450 gcc_assert (VECTOR_MODE_P (mode)); 2451 unsigned int nlanes = 128 / (group_size * GET_MODE_UNIT_BITSIZE (mode)); 2452 return require_immediate_range (rel_argno, 0, nlanes - 1); 2453 } 2454 2455 /* Check that argument REL_ARGNO is an integer constant expression that 2456 has one of the given values. */ 2457 bool 2458 function_checker::require_immediate_one_of (unsigned int rel_argno, 2459 HOST_WIDE_INT value0, 2460 HOST_WIDE_INT value1, 2461 HOST_WIDE_INT value2, 2462 HOST_WIDE_INT value3) 2463 { 2464 unsigned int argno = m_base_arg + rel_argno; 2465 if (!argument_exists_p (argno)) 2466 return true; 2467 2468 HOST_WIDE_INT actual; 2469 if (!require_immediate (argno, actual)) 2470 return false; 2471 2472 if (actual != value0 2473 && actual != value1 2474 && actual != value2 2475 && actual != value3) 2476 { 2477 report_not_one_of (location, fndecl, argno, actual, 2478 value0, value1, value2, value3); 2479 return false; 2480 } 2481 2482 return true; 2483 } 2484 2485 /* Check that argument REL_ARGNO is an integer constant expression in the 2486 range [MIN, MAX]. REL_ARGNO counts from the end of the predication 2487 arguments. */ 2488 bool 2489 function_checker::require_immediate_range (unsigned int rel_argno, 2490 HOST_WIDE_INT min, 2491 HOST_WIDE_INT max) 2492 { 2493 unsigned int argno = m_base_arg + rel_argno; 2494 if (!argument_exists_p (argno)) 2495 return true; 2496 2497 /* Required because of the tree_to_uhwi -> HOST_WIDE_INT conversion 2498 in require_immediate. */ 2499 gcc_assert (min >= 0 && min <= max); 2500 HOST_WIDE_INT actual; 2501 if (!require_immediate (argno, actual)) 2502 return false; 2503 2504 if (!IN_RANGE (actual, min, max)) 2505 { 2506 report_out_of_range (location, fndecl, argno, actual, min, max); 2507 return false; 2508 } 2509 2510 return true; 2511 } 2512 2513 /* Perform semantic checks on the call. Return true if the call is valid, 2514 otherwise report a suitable error. */ 2515 bool 2516 function_checker::check () 2517 { 2518 function_args_iterator iter; 2519 tree type; 2520 unsigned int i = 0; 2521 FOREACH_FUNCTION_ARGS (m_fntype, type, iter) 2522 { 2523 if (type == void_type_node || i >= m_nargs) 2524 break; 2525 2526 if (i >= m_base_arg 2527 && TREE_CODE (type) == ENUMERAL_TYPE 2528 && !require_immediate_enum (i - m_base_arg, type)) 2529 return false; 2530 2531 i += 1; 2532 } 2533 2534 return shape->check (*this); 2535 } 2536 2537 gimple_folder::gimple_folder (const function_instance &instance, tree fndecl, 2538 gimple_stmt_iterator *gsi_in, gcall *call_in) 2539 : function_call_info (gimple_location (call_in), instance, fndecl), 2540 gsi (gsi_in), call (call_in), lhs (gimple_call_lhs (call_in)) 2541 { 2542 } 2543 2544 /* VALUE might be a vector of type VECTYPE or a single scalar element. 2545 Duplicate it into a vector of type VECTYPE in the latter case, adding any 2546 new statements to STMTS. */ 2547 tree 2548 gimple_folder::force_vector (gimple_seq &stmts, tree vectype, tree value) 2549 { 2550 if (!VECTOR_TYPE_P (TREE_TYPE (value))) 2551 value = gimple_build_vector_from_val (&stmts, vectype, value); 2552 return value; 2553 } 2554 2555 /* Convert predicate argument ARGNO so that it has the type appropriate for 2556 an operation on VECTYPE. Add any new statements to STMTS. */ 2557 tree 2558 gimple_folder::convert_pred (gimple_seq &stmts, tree vectype, 2559 unsigned int argno) 2560 { 2561 tree pred = gimple_call_arg (call, argno); 2562 if (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (pred)), 2563 TYPE_VECTOR_SUBPARTS (vectype))) 2564 return pred; 2565 2566 return gimple_build (&stmts, VIEW_CONVERT_EXPR, 2567 truth_type_for (vectype), pred); 2568 } 2569 2570 /* Return a pointer to the address in a contiguous load or store, 2571 given that each memory vector has type VECTYPE. Add any new 2572 statements to STMTS. */ 2573 tree 2574 gimple_folder::fold_contiguous_base (gimple_seq &stmts, tree vectype) 2575 { 2576 tree base = gimple_call_arg (call, 1); 2577 if (mode_suffix_id == MODE_vnum) 2578 { 2579 tree offset = gimple_call_arg (call, 2); 2580 offset = gimple_convert (&stmts, sizetype, offset); 2581 offset = gimple_build (&stmts, MULT_EXPR, sizetype, offset, 2582 TYPE_SIZE_UNIT (vectype)); 2583 base = gimple_build (&stmts, POINTER_PLUS_EXPR, TREE_TYPE (base), 2584 base, offset); 2585 } 2586 return base; 2587 } 2588 2589 /* Return the alignment and TBAA argument to an internal load or store 2590 function like IFN_MASK_LOAD or IFN_MASK_STORE, given that it accesses 2591 memory elements of type TYPE. */ 2592 tree 2593 gimple_folder::load_store_cookie (tree type) 2594 { 2595 return build_int_cst (build_pointer_type (type), TYPE_ALIGN (type)); 2596 } 2597 2598 /* Fold the call to a call to INSTANCE, with the same arguments. */ 2599 gimple * 2600 gimple_folder::redirect_call (const function_instance &instance) 2601 { 2602 registered_function *rfn 2603 = function_table->find_with_hash (instance, instance.hash ()); 2604 if (!rfn) 2605 return NULL; 2606 2607 gimple_call_set_fndecl (call, rfn->decl); 2608 return call; 2609 } 2610 2611 /* Fold the call to a PTRUE, taking the element size from type suffix 0. */ 2612 gimple * 2613 gimple_folder::fold_to_ptrue () 2614 { 2615 tree svbool_type = TREE_TYPE (lhs); 2616 tree bool_type = TREE_TYPE (svbool_type); 2617 unsigned int element_bytes = type_suffix (0).element_bytes; 2618 2619 /* The return type is svbool_t for all type suffixes, thus for b8 we 2620 want { 1, 1, 1, 1, ... }, for b16 we want { 1, 0, 1, 0, ... }, etc. */ 2621 tree_vector_builder builder (svbool_type, element_bytes, 1); 2622 builder.quick_push (build_all_ones_cst (bool_type)); 2623 for (unsigned int i = 1; i < element_bytes; ++i) 2624 builder.quick_push (build_zero_cst (bool_type)); 2625 return gimple_build_assign (lhs, builder.build ()); 2626 } 2627 2628 /* Fold the call to a PFALSE. */ 2629 gimple * 2630 gimple_folder::fold_to_pfalse () 2631 { 2632 return gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs))); 2633 } 2634 2635 /* Fold an operation to a constant predicate in which the first VL 2636 elements are set and the rest are clear. Take the element size 2637 from type suffix 0. */ 2638 gimple * 2639 gimple_folder::fold_to_vl_pred (unsigned int vl) 2640 { 2641 tree vectype = TREE_TYPE (lhs); 2642 tree element_type = TREE_TYPE (vectype); 2643 tree minus_one = build_all_ones_cst (element_type); 2644 tree zero = build_zero_cst (element_type); 2645 unsigned int element_bytes = type_suffix (0).element_bytes; 2646 2647 /* Construct COUNT elements that contain the ptrue followed by 2648 a repeating sequence of COUNT elements. */ 2649 unsigned int count = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vectype)); 2650 gcc_assert (vl * element_bytes <= count); 2651 tree_vector_builder builder (vectype, count, 2); 2652 for (unsigned int i = 0; i < count * 2; ++i) 2653 { 2654 bool bit = (i & (element_bytes - 1)) == 0 && i < vl * element_bytes; 2655 builder.quick_push (bit ? minus_one : zero); 2656 } 2657 return gimple_build_assign (lhs, builder.build ()); 2658 } 2659 2660 /* Try to fold the call. Return the new statement on success and null 2661 on failure. */ 2662 gimple * 2663 gimple_folder::fold () 2664 { 2665 /* Don't fold anything when SVE is disabled; emit an error during 2666 expansion instead. */ 2667 if (!TARGET_SVE) 2668 return NULL; 2669 2670 /* Punt if the function has a return type and no result location is 2671 provided. The attributes should allow target-independent code to 2672 remove the calls if appropriate. */ 2673 if (!lhs && TREE_TYPE (gimple_call_fntype (call)) != void_type_node) 2674 return NULL; 2675 2676 return base->fold (*this); 2677 } 2678 2679 function_expander::function_expander (const function_instance &instance, 2680 tree fndecl, tree call_expr_in, 2681 rtx possible_target_in) 2682 : function_call_info (EXPR_LOCATION (call_expr_in), instance, fndecl), 2683 call_expr (call_expr_in), possible_target (possible_target_in) 2684 { 2685 } 2686 2687 /* Return the handler of direct optab OP for type suffix SUFFIX_I. */ 2688 insn_code 2689 function_expander::direct_optab_handler (optab op, unsigned int suffix_i) 2690 { 2691 return ::direct_optab_handler (op, vector_mode (suffix_i)); 2692 } 2693 2694 /* Choose between signed and unsigned direct optabs SIGNED_OP and 2695 UNSIGNED_OP based on the signedness of type suffix SUFFIX_I, then 2696 pick the appropriate optab handler for the mode. Use MODE as the 2697 mode if given, otherwise use the mode of type suffix SUFFIX_I. */ 2698 insn_code 2699 function_expander::direct_optab_handler_for_sign (optab signed_op, 2700 optab unsigned_op, 2701 unsigned int suffix_i, 2702 machine_mode mode) 2703 { 2704 if (mode == VOIDmode) 2705 mode = vector_mode (suffix_i); 2706 optab op = type_suffix (suffix_i).unsigned_p ? unsigned_op : signed_op; 2707 return ::direct_optab_handler (op, mode); 2708 } 2709 2710 /* Return true if X overlaps any input. */ 2711 bool 2712 function_expander::overlaps_input_p (rtx x) 2713 { 2714 for (unsigned int i = 0; i < args.length (); ++i) 2715 if (reg_overlap_mentioned_p (x, args[i])) 2716 return true; 2717 return false; 2718 } 2719 2720 /* Convert ptr_mode value X to Pmode. */ 2721 rtx 2722 function_expander::convert_to_pmode (rtx x) 2723 { 2724 if (ptr_mode == SImode) 2725 x = simplify_gen_unary (ZERO_EXTEND, DImode, x, SImode); 2726 return x; 2727 } 2728 2729 /* Return the base address for a contiguous load or store function. 2730 MEM_MODE is the mode of the addressed memory. */ 2731 rtx 2732 function_expander::get_contiguous_base (machine_mode mem_mode) 2733 { 2734 rtx base = convert_to_pmode (args[1]); 2735 if (mode_suffix_id == MODE_vnum) 2736 { 2737 /* Use the size of the memory mode for extending loads and truncating 2738 stores. Use the size of a full vector for non-extending loads 2739 and non-truncating stores (including svld[234] and svst[234]). */ 2740 poly_int64 size = ordered_min (GET_MODE_SIZE (mem_mode), 2741 BYTES_PER_SVE_VECTOR); 2742 rtx offset = gen_int_mode (size, Pmode); 2743 offset = simplify_gen_binary (MULT, Pmode, args[2], offset); 2744 base = simplify_gen_binary (PLUS, Pmode, base, offset); 2745 } 2746 return base; 2747 } 2748 2749 /* For a function that does the equivalent of: 2750 2751 OUTPUT = COND ? FN (INPUTS) : FALLBACK; 2752 2753 return the value of FALLBACK. 2754 2755 MODE is the mode of OUTPUT. NOPS is the number of operands in INPUTS. 2756 MERGE_ARGNO is the argument that provides FALLBACK for _m functions, 2757 or DEFAULT_MERGE_ARGNO if we should apply the usual rules. 2758 2759 ARGNO is the caller's index into args. If the returned value is 2760 argument 0 (as for unary _m operations), increment ARGNO past the 2761 returned argument. */ 2762 rtx 2763 function_expander::get_fallback_value (machine_mode mode, unsigned int nops, 2764 unsigned int merge_argno, 2765 unsigned int &argno) 2766 { 2767 if (pred == PRED_z) 2768 return CONST0_RTX (mode); 2769 2770 gcc_assert (pred == PRED_m || pred == PRED_x); 2771 if (merge_argno == DEFAULT_MERGE_ARGNO) 2772 merge_argno = nops == 1 && pred == PRED_m ? 0 : 1; 2773 2774 if (merge_argno == 0) 2775 return args[argno++]; 2776 2777 return args[merge_argno]; 2778 } 2779 2780 /* Return a REG rtx that can be used for the result of the function, 2781 using the preferred target if suitable. */ 2782 rtx 2783 function_expander::get_reg_target () 2784 { 2785 machine_mode target_mode = TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl))); 2786 if (!possible_target || GET_MODE (possible_target) != target_mode) 2787 possible_target = gen_reg_rtx (target_mode); 2788 return possible_target; 2789 } 2790 2791 /* As for get_reg_target, but make sure that the returned REG does not 2792 overlap any inputs. */ 2793 rtx 2794 function_expander::get_nonoverlapping_reg_target () 2795 { 2796 if (possible_target && overlaps_input_p (possible_target)) 2797 possible_target = NULL_RTX; 2798 return get_reg_target (); 2799 } 2800 2801 /* Add an output operand to the instruction we're building, which has 2802 code ICODE. Bind the output to the preferred target rtx if possible. */ 2803 void 2804 function_expander::add_output_operand (insn_code icode) 2805 { 2806 unsigned int opno = m_ops.length (); 2807 machine_mode mode = insn_data[icode].operand[opno].mode; 2808 m_ops.safe_grow (opno + 1); 2809 create_output_operand (&m_ops.last (), possible_target, mode); 2810 } 2811 2812 /* Add an input operand to the instruction we're building, which has 2813 code ICODE. Calculate the value of the operand as follows: 2814 2815 - If the operand is a vector and X is not, broadcast X to fill a 2816 vector of the appropriate mode. 2817 2818 - Otherwise, if the operand is a predicate, coerce X to have the 2819 mode that the instruction expects. In this case X is known to be 2820 VNx16BImode (the mode of svbool_t). 2821 2822 - Otherwise use X directly. The expand machinery checks that X has 2823 the right mode for the instruction. */ 2824 void 2825 function_expander::add_input_operand (insn_code icode, rtx x) 2826 { 2827 unsigned int opno = m_ops.length (); 2828 const insn_operand_data &operand = insn_data[icode].operand[opno]; 2829 machine_mode mode = operand.mode; 2830 if (mode == VOIDmode) 2831 { 2832 /* The only allowable use of VOIDmode is the wildcard 2833 aarch64_any_register_operand, which is used to avoid 2834 combinatorial explosion in the reinterpret patterns. */ 2835 gcc_assert (operand.predicate == aarch64_any_register_operand); 2836 mode = GET_MODE (x); 2837 } 2838 else if (!VECTOR_MODE_P (GET_MODE (x)) && VECTOR_MODE_P (mode)) 2839 x = expand_vector_broadcast (mode, x); 2840 else if (GET_MODE_CLASS (mode) == MODE_VECTOR_BOOL) 2841 { 2842 gcc_assert (GET_MODE (x) == VNx16BImode); 2843 x = gen_lowpart (mode, x); 2844 } 2845 m_ops.safe_grow (m_ops.length () + 1); 2846 create_input_operand (&m_ops.last (), x, mode); 2847 } 2848 2849 /* Add an integer operand with value X to the instruction. */ 2850 void 2851 function_expander::add_integer_operand (HOST_WIDE_INT x) 2852 { 2853 m_ops.safe_grow (m_ops.length () + 1); 2854 create_integer_operand (&m_ops.last (), x); 2855 } 2856 2857 /* Add a memory operand with mode MODE and address ADDR. */ 2858 void 2859 function_expander::add_mem_operand (machine_mode mode, rtx addr) 2860 { 2861 /* Exception for OImode for the ld1ro intrinsics. 2862 They act on 256 bit octaword data, and it's just easier to use a scalar 2863 mode to represent that than add a new vector mode solely for the purpose 2864 of this intrinsic. */ 2865 gcc_assert (VECTOR_MODE_P (mode) || mode == OImode); 2866 rtx mem = gen_rtx_MEM (mode, memory_address (mode, addr)); 2867 /* The memory is only guaranteed to be element-aligned. */ 2868 set_mem_align (mem, GET_MODE_ALIGNMENT (GET_MODE_INNER (mode))); 2869 add_fixed_operand (mem); 2870 } 2871 2872 /* Add an address operand with value X. The static operand data says 2873 what mode and form the address must have. */ 2874 void 2875 function_expander::add_address_operand (rtx x) 2876 { 2877 m_ops.safe_grow (m_ops.length () + 1); 2878 create_address_operand (&m_ops.last (), x); 2879 } 2880 2881 /* Add an operand that must be X. The only way of legitimizing an 2882 invalid X is to reload the address of a MEM. */ 2883 void 2884 function_expander::add_fixed_operand (rtx x) 2885 { 2886 m_ops.safe_grow (m_ops.length () + 1); 2887 create_fixed_operand (&m_ops.last (), x); 2888 } 2889 2890 /* Generate instruction ICODE, given that its operands have already 2891 been added to M_OPS. Return the value of the first operand. */ 2892 rtx 2893 function_expander::generate_insn (insn_code icode) 2894 { 2895 expand_insn (icode, m_ops.length (), m_ops.address ()); 2896 return function_returns_void_p () ? const0_rtx : m_ops[0].value; 2897 } 2898 2899 /* Convert the arguments to a gather/scatter function into the 2900 associated md operands. Argument ARGNO is the scalar or vector base and 2901 argument ARGNO + 1 is the scalar or vector displacement (if applicable). 2902 The md pattern expects: 2903 2904 - a scalar base 2905 - a vector displacement 2906 2907 If SCALED_P is true, it also expects: 2908 2909 - a const_int that is 1 if the displacement is zero-extended from 32 bits 2910 - a scaling multiplier (1 for bytes, 2 for .h indices, etc.). 2911 2912 If SCALED_P is false, the displacement is implicitly zero-extended 2913 and the scaling multiplier is implicitly 1. */ 2914 void 2915 function_expander::prepare_gather_address_operands (unsigned int argno, 2916 bool scaled_p) 2917 { 2918 machine_mode mem_mode = memory_vector_mode (); 2919 tree vector_type = base_vector_type (); 2920 units_index units = displacement_units (); 2921 int shift_idx = -1; 2922 if (units == UNITS_none) 2923 { 2924 /* Vector base, no displacement. Convert to an integer zero base 2925 and a vector byte offset. */ 2926 args.quick_insert (argno, const0_rtx); 2927 units = UNITS_bytes; 2928 } 2929 else if (vector_type) 2930 { 2931 /* Vector base, scalar displacement. Convert to a scalar base and 2932 a vector byte offset. */ 2933 std::swap (args[argno], args[argno + 1]); 2934 if (units == UNITS_elements) 2935 shift_idx = argno; 2936 } 2937 else 2938 { 2939 /* Scalar base, vector displacement. This is the order that the md 2940 pattern wants. */ 2941 args[argno] = convert_to_pmode (args[argno]); 2942 vector_type = displacement_vector_type (); 2943 if (units == UNITS_elements && !scaled_p) 2944 shift_idx = argno + 1; 2945 } 2946 tree scalar_displacement_type = TREE_TYPE (vector_type); 2947 2948 if (shift_idx >= 0) 2949 { 2950 machine_mode arg_mode = GET_MODE (args[shift_idx]); 2951 if (arg_mode == VOIDmode) 2952 arg_mode = DImode; 2953 unsigned int elt_bytes = GET_MODE_UNIT_SIZE (mem_mode); 2954 rtx shift = gen_int_mode (exact_log2 (elt_bytes), DImode); 2955 args[shift_idx] = simplify_gen_binary (ASHIFT, arg_mode, 2956 args[shift_idx], shift); 2957 units = UNITS_bytes; 2958 } 2959 2960 bool uxtw_p = (TYPE_PRECISION (scalar_displacement_type) == 64 2961 || TYPE_UNSIGNED (scalar_displacement_type)); 2962 unsigned int scale = (units == UNITS_bytes 2963 ? 1 : GET_MODE_UNIT_SIZE (mem_mode)); 2964 2965 if (scaled_p) 2966 { 2967 args.quick_insert (argno + 2, GEN_INT (uxtw_p)); 2968 args.quick_insert (argno + 3, GEN_INT (scale)); 2969 } 2970 else 2971 gcc_assert (uxtw_p && scale == 1); 2972 } 2973 2974 /* The final argument is an immediate svprfop value. Add two fake arguments 2975 to represent the rw and locality operands of a PREFETCH rtx. */ 2976 void 2977 function_expander::prepare_prefetch_operands () 2978 { 2979 unsigned int prfop = INTVAL (args.last ()); 2980 /* Bit 3 of the prfop selects stores over loads. */ 2981 args.quick_push (GEN_INT ((prfop & 8) != 0)); 2982 /* Bits 1 and 2 specify the locality; 0-based for svprfop but 2983 1-based for PREFETCH. */ 2984 args.quick_push (GEN_INT (((prfop >> 1) & 3) + 1)); 2985 } 2986 2987 /* Add a dummy argument to indicate whether predicate argument ARGNO 2988 is all-true when interpreted in mode PRED_MODE. The hint goes 2989 immediately after ARGNO. */ 2990 void 2991 function_expander::add_ptrue_hint (unsigned int argno, machine_mode pred_mode) 2992 { 2993 rtx pred = gen_lowpart (pred_mode, args[argno]); 2994 int hint = (pred == CONSTM1_RTX (pred_mode) 2995 ? SVE_KNOWN_PTRUE : SVE_MAYBE_NOT_PTRUE); 2996 args.quick_insert (argno + 1, gen_int_mode (hint, SImode)); 2997 } 2998 2999 /* Rotate inputs args[START:END] one position to the left, so that 3000 args[START] becomes args[END - 1]. */ 3001 void 3002 function_expander::rotate_inputs_left (unsigned int start, unsigned int end) 3003 { 3004 rtx new_last = args[start]; 3005 for (unsigned int i = start; i < end - 1; ++i) 3006 args[i] = args[i + 1]; 3007 args[end - 1] = new_last; 3008 } 3009 3010 /* Return true if the negation of argument ARGNO can be folded away, 3011 replacing it with the negated value if so. MODE is the associated 3012 vector mode, but the argument could be a single element. The main 3013 case this handles is constant arguments. */ 3014 bool 3015 function_expander::try_negating_argument (unsigned int argno, 3016 machine_mode mode) 3017 { 3018 rtx x = args[argno]; 3019 if (!VECTOR_MODE_P (GET_MODE (x))) 3020 mode = GET_MODE_INNER (mode); 3021 3022 x = simplify_unary_operation (NEG, mode, x, mode); 3023 if (!x) 3024 return false; 3025 3026 args[argno] = x; 3027 return true; 3028 } 3029 3030 /* Implement the call using instruction ICODE, with a 1:1 mapping between 3031 arguments and input operands. */ 3032 rtx 3033 function_expander::use_exact_insn (insn_code icode) 3034 { 3035 unsigned int nops = insn_data[icode].n_operands; 3036 if (!function_returns_void_p ()) 3037 { 3038 add_output_operand (icode); 3039 nops -= 1; 3040 } 3041 for (unsigned int i = 0; i < nops; ++i) 3042 add_input_operand (icode, args[i]); 3043 return generate_insn (icode); 3044 } 3045 3046 /* Implement the call using instruction ICODE, which does not use a 3047 governing predicate. We must therefore drop the GP from an _x call. */ 3048 rtx 3049 function_expander::use_unpred_insn (insn_code icode) 3050 { 3051 /* We can't drop the predicate for _z and _m. */ 3052 gcc_assert (pred == PRED_x || pred == PRED_none); 3053 /* Discount the output operand. */ 3054 unsigned int nops = insn_data[icode].n_operands - 1; 3055 /* Drop the predicate argument in the case of _x predication. */ 3056 unsigned int bias = (pred == PRED_x ? 1 : 0); 3057 unsigned int i = 0; 3058 3059 add_output_operand (icode); 3060 for (; i < nops; ++i) 3061 add_input_operand (icode, args[i + bias]); 3062 3063 return generate_insn (icode); 3064 } 3065 3066 /* Implement the call using instruction ICODE, which is a predicated 3067 operation that returns arbitrary values for inactive lanes. */ 3068 rtx 3069 function_expander::use_pred_x_insn (insn_code icode) 3070 { 3071 /* At present we never need to handle PRED_none, which would involve 3072 creating a new predicate rather than using one supplied by the user. */ 3073 gcc_assert (pred == PRED_x); 3074 /* Discount the output operand. */ 3075 unsigned int nops = args.length () - 1; 3076 3077 bool has_float_operand_p = FLOAT_MODE_P (insn_data[icode].operand[0].mode); 3078 3079 /* Add the normal operands. */ 3080 add_output_operand (icode); 3081 add_input_operand (icode, args[0]); 3082 for (unsigned int i = 0; i < nops; ++i) 3083 { 3084 add_input_operand (icode, args[i + 1]); 3085 if (FLOAT_MODE_P (GET_MODE (args[i + 1]))) 3086 has_float_operand_p = true; 3087 } 3088 3089 if (has_float_operand_p) 3090 { 3091 /* Add a flag that indicates whether unpredicated instructions 3092 are allowed. */ 3093 rtx pred = m_ops[1].value; 3094 if (flag_trapping_math && pred != CONST1_RTX (GET_MODE (pred))) 3095 add_integer_operand (SVE_STRICT_GP); 3096 else 3097 add_integer_operand (SVE_RELAXED_GP); 3098 } 3099 3100 return generate_insn (icode); 3101 } 3102 3103 /* Implement the call using instruction ICODE, which does the equivalent of: 3104 3105 OUTPUT = COND ? FN (INPUTS) : FALLBACK; 3106 3107 The instruction operands are in the order above: OUTPUT, COND, INPUTS 3108 and FALLBACK. MERGE_ARGNO is the argument that provides FALLBACK for _m 3109 functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */ 3110 rtx 3111 function_expander::use_cond_insn (insn_code icode, unsigned int merge_argno) 3112 { 3113 /* At present we never need to handle PRED_none, which would involve 3114 creating a new predicate rather than using one supplied by the user. */ 3115 gcc_assert (pred != PRED_none); 3116 /* Discount the output, predicate and fallback value. */ 3117 unsigned int nops = insn_data[icode].n_operands - 3; 3118 machine_mode mode = insn_data[icode].operand[0].mode; 3119 3120 unsigned int opno = 0; 3121 rtx fallback_arg = get_fallback_value (mode, nops, merge_argno, opno); 3122 rtx pred = args[opno++]; 3123 3124 add_output_operand (icode); 3125 add_input_operand (icode, pred); 3126 for (unsigned int i = 0; i < nops; ++i) 3127 add_input_operand (icode, args[opno + i]); 3128 add_input_operand (icode, fallback_arg); 3129 return generate_insn (icode); 3130 } 3131 3132 /* Implement the call using instruction ICODE, which is a select-like 3133 operation with the following operands: 3134 3135 0: output 3136 1: true value 3137 2: false value 3138 3: predicate 3139 3140 MERGE_ARGNO is the argument that provides the "false" value for _m 3141 functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */ 3142 rtx 3143 function_expander::use_vcond_mask_insn (insn_code icode, 3144 unsigned int merge_argno) 3145 { 3146 machine_mode mode = vector_mode (0); 3147 3148 unsigned int opno = 0; 3149 rtx false_arg = get_fallback_value (mode, 1, merge_argno, opno); 3150 rtx pred_arg = args[opno++]; 3151 rtx true_arg = args[opno++]; 3152 3153 add_output_operand (icode); 3154 add_input_operand (icode, true_arg); 3155 add_input_operand (icode, false_arg); 3156 add_input_operand (icode, pred_arg); 3157 return generate_insn (icode); 3158 } 3159 3160 /* Implement the call using instruction ICODE, which loads memory operand 1 3161 into register operand 0 under the control of predicate operand 2. 3162 Extending loads have a further predicate (operand 3) that nominally 3163 controls the extension. */ 3164 rtx 3165 function_expander::use_contiguous_load_insn (insn_code icode) 3166 { 3167 machine_mode mem_mode = memory_vector_mode (); 3168 3169 add_output_operand (icode); 3170 add_mem_operand (mem_mode, get_contiguous_base (mem_mode)); 3171 add_input_operand (icode, args[0]); 3172 if (GET_MODE_UNIT_BITSIZE (mem_mode) < type_suffix (0).element_bits) 3173 add_input_operand (icode, CONSTM1_RTX (VNx16BImode)); 3174 return generate_insn (icode); 3175 } 3176 3177 /* Implement the call using instruction ICODE, which prefetches from 3178 address operand 1 under the control of predicate operand 0. 3179 Operands 2, 3 and 4 respectively specify the svprfop value, 3180 the PREFETCH rw flag and the PREFETCH locality. */ 3181 rtx 3182 function_expander::use_contiguous_prefetch_insn (insn_code icode) 3183 { 3184 add_input_operand (icode, args[0]); 3185 add_address_operand (get_contiguous_base (VNx16QImode)); 3186 for (unsigned int i = args.length () - 3; i < args.length (); ++i) 3187 add_input_operand (icode, args[i]); 3188 return generate_insn (icode); 3189 } 3190 3191 /* Implement the call using instruction ICODE, which stores register operand 1 3192 into memory operand 0 under the control of predicate operand 2. */ 3193 rtx 3194 function_expander::use_contiguous_store_insn (insn_code icode) 3195 { 3196 machine_mode mem_mode = memory_vector_mode (); 3197 3198 add_mem_operand (mem_mode, get_contiguous_base (mem_mode)); 3199 add_input_operand (icode, args.last ()); 3200 add_input_operand (icode, args[0]); 3201 return generate_insn (icode); 3202 } 3203 3204 /* Implement the call using one of the following strategies, chosen in order: 3205 3206 (1) "aarch64_pred_<optab><mode>_z" for PRED_z predicate functions 3207 3208 (2) "aarch64_pred_<optab><mode>" for PRED_x functions 3209 3210 (3) a normal unpredicated optab for PRED_none and PRED_x functions, 3211 dropping the predicate in the latter case 3212 3213 (4) an unpredicated "aarch64_sve_<code_optab><mode>" for PRED_none and 3214 PRED_x functions, again dropping the predicate for PRED_x 3215 3216 (5) "cond_<optab><mode>" otherwise 3217 3218 where <optab> corresponds to: 3219 3220 - CODE_FOR_SINT for signed integers 3221 - CODE_FOR_UINT for unsigned integers 3222 - UNSPEC_FOR_FP for floating-point values 3223 3224 and where <code_optab> is like <optab>, but uses CODE_FOR_SINT instead 3225 of UNSPEC_FOR_FP for floating-point values. 3226 3227 MERGE_ARGNO is the argument that provides the values of inactive lanes for 3228 _m functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */ 3229 rtx 3230 function_expander::map_to_rtx_codes (rtx_code code_for_sint, 3231 rtx_code code_for_uint, 3232 int unspec_for_fp, 3233 unsigned int merge_argno) 3234 { 3235 machine_mode mode = vector_mode (0); 3236 rtx_code code = (type_suffix (0).unsigned_p ? code_for_uint : code_for_sint); 3237 insn_code icode; 3238 3239 /* Handle predicate logic operations, which always use _z predication. */ 3240 if (type_suffix (0).tclass == TYPE_bool) 3241 { 3242 gcc_assert (pred == PRED_z && code_for_uint == code_for_sint); 3243 return use_exact_insn (code_for_aarch64_pred_z (code, mode)); 3244 } 3245 3246 /* First try using UNSPEC_PRED_X patterns for _x predication, 3247 if available. */ 3248 if (pred == PRED_x) 3249 { 3250 if (type_suffix (0).integer_p) 3251 icode = maybe_code_for_aarch64_pred (code, mode); 3252 else 3253 icode = maybe_code_for_aarch64_pred (unspec_for_fp, mode); 3254 if (icode != CODE_FOR_nothing) 3255 return use_pred_x_insn (icode); 3256 } 3257 3258 /* Otherwise expand PRED_none and PRED_x operations without a predicate. 3259 Floating-point operations conventionally use the signed rtx code. */ 3260 if (pred == PRED_none || pred == PRED_x) 3261 { 3262 icode = direct_optab_handler (code_to_optab (code), 0); 3263 if (icode == CODE_FOR_nothing) 3264 icode = code_for_aarch64_sve (code, mode); 3265 return use_unpred_insn (icode); 3266 } 3267 3268 /* Don't use cond_*_optabs here, since not all codes have one yet. */ 3269 if (type_suffix (0).integer_p) 3270 icode = code_for_cond (code, mode); 3271 else 3272 icode = code_for_cond (unspec_for_fp, mode); 3273 return use_cond_insn (icode, merge_argno); 3274 } 3275 3276 /* Implement the call using one of the following strategies, chosen in order: 3277 3278 (1) "aarch64_pred_<optab><mode>" for PRED_x functions; this is a 3279 predicated pattern 3280 3281 (2) "aarch64_sve_<optab><mode>" for PRED_none and PRED_x functions; 3282 this is an unpredicated pattern 3283 3284 (3) "cond_<optab><mode>" otherwise 3285 3286 where <optab> corresponds to: 3287 3288 - UNSPEC_FOR_SINT for signed integers 3289 - UNSPEC_FOR_UINT for unsigned integers 3290 - UNSPEC_FOR_FP for floating-point values 3291 3292 MERGE_ARGNO is the argument that provides the values of inactive lanes for 3293 _m functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */ 3294 rtx 3295 function_expander::map_to_unspecs (int unspec_for_sint, int unspec_for_uint, 3296 int unspec_for_fp, unsigned int merge_argno) 3297 { 3298 machine_mode mode = vector_mode (0); 3299 int unspec = (!type_suffix (0).integer_p ? unspec_for_fp 3300 : type_suffix (0).unsigned_p ? unspec_for_uint 3301 : unspec_for_sint); 3302 3303 if (pred == PRED_x) 3304 { 3305 insn_code icode = maybe_code_for_aarch64_pred (unspec, mode); 3306 if (icode != CODE_FOR_nothing) 3307 return use_pred_x_insn (icode); 3308 } 3309 3310 if (pred == PRED_none || pred == PRED_x) 3311 { 3312 insn_code icode = maybe_code_for_aarch64_sve (unspec, mode); 3313 if (icode != CODE_FOR_nothing) 3314 return use_unpred_insn (icode); 3315 } 3316 3317 insn_code icode = code_for_cond (unspec, vector_mode (0)); 3318 return use_cond_insn (icode, merge_argno); 3319 } 3320 3321 /* Expand the call and return its lhs. */ 3322 rtx 3323 function_expander::expand () 3324 { 3325 unsigned int nargs = call_expr_nargs (call_expr); 3326 args.reserve (nargs); 3327 for (unsigned int i = 0; i < nargs; ++i) 3328 args.quick_push (expand_normal (CALL_EXPR_ARG (call_expr, i))); 3329 3330 return base->expand (*this); 3331 } 3332 3333 /* Register the built-in SVE ABI types, such as __SVBool_t. */ 3334 static void 3335 register_builtin_types () 3336 { 3337 #define DEF_SVE_TYPE(ACLE_NAME, NCHARS, ABI_NAME, SCALAR_TYPE) \ 3338 scalar_types[VECTOR_TYPE_ ## ACLE_NAME] = SCALAR_TYPE; 3339 #include "aarch64-sve-builtins.def" 3340 3341 for (unsigned int i = 0; i < NUM_VECTOR_TYPES; ++i) 3342 { 3343 tree eltype = scalar_types[i]; 3344 tree vectype; 3345 unsigned int num_zr = 0, num_pr = 0; 3346 if (eltype == boolean_type_node) 3347 { 3348 vectype = build_truth_vector_type_for_mode (BYTES_PER_SVE_VECTOR, 3349 VNx16BImode); 3350 gcc_assert (TYPE_MODE (vectype) == VNx16BImode 3351 && TYPE_MODE (vectype) == TYPE_MODE_RAW (vectype) 3352 && TYPE_ALIGN (vectype) == 16 3353 && known_eq (wi::to_poly_offset (TYPE_SIZE (vectype)), 3354 BYTES_PER_SVE_VECTOR)); 3355 num_pr = 1; 3356 } 3357 else 3358 { 3359 scalar_mode elmode = SCALAR_TYPE_MODE (eltype); 3360 unsigned int elbytes = GET_MODE_SIZE (elmode); 3361 poly_uint64 nunits = exact_div (BYTES_PER_SVE_VECTOR, elbytes); 3362 machine_mode mode 3363 = aarch64_sve_data_mode (elmode, nunits).require (); 3364 vectype = build_vector_type_for_mode (eltype, mode); 3365 gcc_assert (VECTOR_MODE_P (TYPE_MODE (vectype)) 3366 && TYPE_MODE (vectype) == mode 3367 && TYPE_MODE_RAW (vectype) == mode 3368 && TYPE_ALIGN (vectype) == 128 3369 && known_eq (wi::to_poly_offset (TYPE_SIZE (vectype)), 3370 BITS_PER_SVE_VECTOR)); 3371 num_zr = 1; 3372 } 3373 vectype = build_distinct_type_copy (vectype); 3374 gcc_assert (vectype == TYPE_MAIN_VARIANT (vectype)); 3375 SET_TYPE_STRUCTURAL_EQUALITY (vectype); 3376 TYPE_ARTIFICIAL (vectype) = 1; 3377 TYPE_INDIVISIBLE_P (vectype) = 1; 3378 add_sve_type_attribute (vectype, num_zr, num_pr, 3379 vector_types[i].mangled_name, 3380 vector_types[i].acle_name); 3381 make_type_sizeless (vectype); 3382 abi_vector_types[i] = vectype; 3383 lang_hooks.types.register_builtin_type (vectype, 3384 vector_types[i].abi_name); 3385 } 3386 } 3387 3388 /* Initialize all compiler built-ins related to SVE that should be 3389 defined at start-up. */ 3390 void 3391 init_builtins () 3392 { 3393 sve_switcher sve; 3394 register_builtin_types (); 3395 if (in_lto_p) 3396 handle_arm_sve_h (); 3397 } 3398 3399 /* Register vector type TYPE under its arm_sve.h name. */ 3400 static void 3401 register_vector_type (vector_type_index type) 3402 { 3403 tree vectype = abi_vector_types[type]; 3404 tree id = get_identifier (vector_types[type].acle_name); 3405 tree decl = build_decl (input_location, TYPE_DECL, id, vectype); 3406 decl = lang_hooks.decls.pushdecl (decl); 3407 3408 /* Record the new ACLE type if pushdecl succeeded without error. Use 3409 the ABI type otherwise, so that the type we record at least has the 3410 right form, even if it doesn't have the right name. This should give 3411 better error recovery behavior than installing error_mark_node or 3412 installing an incorrect type. */ 3413 if (decl 3414 && TREE_CODE (decl) == TYPE_DECL 3415 && TYPE_MAIN_VARIANT (TREE_TYPE (decl)) == vectype) 3416 vectype = TREE_TYPE (decl); 3417 acle_vector_types[0][type] = vectype; 3418 } 3419 3420 /* Register the tuple type that contains NUM_VECTORS vectors of type TYPE. */ 3421 static void 3422 register_tuple_type (unsigned int num_vectors, vector_type_index type) 3423 { 3424 tree tuple_type = lang_hooks.types.make_type (RECORD_TYPE); 3425 3426 /* Work out the structure name. */ 3427 char buffer[sizeof ("svbfloat16x4_t")]; 3428 const char *vector_type_name = vector_types[type].acle_name; 3429 snprintf (buffer, sizeof (buffer), "%.*sx%d_t", 3430 (int) strlen (vector_type_name) - 2, vector_type_name, 3431 num_vectors); 3432 3433 /* The contents of the type are opaque, so we can define them in any 3434 way that maps to the correct ABI type. 3435 3436 Here we choose to use the same layout as for arm_neon.h, but with 3437 "__val" instead of "val": 3438 3439 struct svfooxN_t { svfoo_t __val[N]; }; 3440 3441 (It wouldn't be possible to write that directly in C or C++ for 3442 sizeless types, but that's not a problem for this function.) 3443 3444 Using arrays simplifies the handling of svget and svset for variable 3445 arguments. */ 3446 tree vector_type = acle_vector_types[0][type]; 3447 tree array_type = build_array_type_nelts (vector_type, num_vectors); 3448 gcc_assert (VECTOR_MODE_P (TYPE_MODE (array_type)) 3449 && TYPE_MODE_RAW (array_type) == TYPE_MODE (array_type) 3450 && TYPE_ALIGN (array_type) == 128); 3451 3452 tree field = build_decl (input_location, FIELD_DECL, 3453 get_identifier ("__val"), array_type); 3454 DECL_FIELD_CONTEXT (field) = tuple_type; 3455 TYPE_FIELDS (tuple_type) = field; 3456 add_sve_type_attribute (tuple_type, num_vectors, 0, NULL, buffer); 3457 make_type_sizeless (tuple_type); 3458 layout_type (tuple_type); 3459 gcc_assert (VECTOR_MODE_P (TYPE_MODE (tuple_type)) 3460 && TYPE_MODE_RAW (tuple_type) == TYPE_MODE (tuple_type) 3461 && TYPE_ALIGN (tuple_type) == 128); 3462 3463 tree decl = build_decl (input_location, TYPE_DECL, 3464 get_identifier (buffer), tuple_type); 3465 TYPE_NAME (tuple_type) = decl; 3466 TYPE_STUB_DECL (tuple_type) = decl; 3467 lang_hooks.decls.pushdecl (decl); 3468 /* ??? Undo the effect of set_underlying_type for C. The C frontend 3469 doesn't recognize DECL as a built-in because (as intended) the decl has 3470 a real location instead of BUILTINS_LOCATION. The frontend therefore 3471 treats the decl like a normal C "typedef struct foo foo;", expecting 3472 the type for tag "struct foo" to have a dummy unnamed TYPE_DECL instead 3473 of the named one we attached above. It then sets DECL_ORIGINAL_TYPE 3474 on the supposedly unnamed decl, creating a circularity that upsets 3475 dwarf2out. 3476 3477 We don't want to follow the normal C model and create "struct foo" 3478 tags for tuple types since (a) the types are supposed to be opaque 3479 and (b) they couldn't be defined as a real struct anyway. Treating 3480 the TYPE_DECLs as "typedef struct foo foo;" without creating 3481 "struct foo" would lead to confusing error messages. */ 3482 DECL_ORIGINAL_TYPE (decl) = NULL_TREE; 3483 3484 acle_vector_types[num_vectors - 1][type] = tuple_type; 3485 } 3486 3487 /* Register the svpattern enum. */ 3488 static void 3489 register_svpattern () 3490 { 3491 auto_vec<string_int_pair, 32> values; 3492 #define PUSH(UPPER, LOWER, VALUE) \ 3493 values.quick_push (string_int_pair ("SV_" #UPPER, VALUE)); 3494 AARCH64_FOR_SVPATTERN (PUSH) 3495 #undef PUSH 3496 3497 acle_svpattern = lang_hooks.types.simulate_enum_decl (input_location, 3498 "svpattern", values); 3499 } 3500 3501 /* Register the svprfop enum. */ 3502 static void 3503 register_svprfop () 3504 { 3505 auto_vec<string_int_pair, 16> values; 3506 #define PUSH(UPPER, LOWER, VALUE) \ 3507 values.quick_push (string_int_pair ("SV_" #UPPER, VALUE)); 3508 AARCH64_FOR_SVPRFOP (PUSH) 3509 #undef PUSH 3510 3511 acle_svprfop = lang_hooks.types.simulate_enum_decl (input_location, 3512 "svprfop", values); 3513 } 3514 3515 /* Implement #pragma GCC aarch64 "arm_sve.h". */ 3516 void 3517 handle_arm_sve_h () 3518 { 3519 if (function_table) 3520 { 3521 error ("duplicate definition of %qs", "arm_sve.h"); 3522 return; 3523 } 3524 3525 sve_switcher sve; 3526 3527 /* Define the vector and tuple types. */ 3528 for (unsigned int type_i = 0; type_i < NUM_VECTOR_TYPES; ++type_i) 3529 { 3530 vector_type_index type = vector_type_index (type_i); 3531 register_vector_type (type); 3532 if (type != VECTOR_TYPE_svbool_t) 3533 for (unsigned int count = 2; count <= MAX_TUPLE_SIZE; ++count) 3534 register_tuple_type (count, type); 3535 } 3536 3537 /* Define the enums. */ 3538 register_svpattern (); 3539 register_svprfop (); 3540 3541 /* Define the functions. */ 3542 function_table = new hash_table<registered_function_hasher> (1023); 3543 function_builder builder; 3544 for (unsigned int i = 0; i < ARRAY_SIZE (function_groups); ++i) 3545 builder.register_function_group (function_groups[i]); 3546 } 3547 3548 /* Return the function decl with SVE function subcode CODE, or error_mark_node 3549 if no such function exists. */ 3550 tree 3551 builtin_decl (unsigned int code, bool) 3552 { 3553 if (code >= vec_safe_length (registered_functions)) 3554 return error_mark_node; 3555 return (*registered_functions)[code]->decl; 3556 } 3557 3558 /* If we're implementing manual overloading, check whether the SVE 3559 function with subcode CODE is overloaded, and if so attempt to 3560 determine the corresponding non-overloaded function. The call 3561 occurs at location LOCATION and has the arguments given by ARGLIST. 3562 3563 If the call is erroneous, report an appropriate error and return 3564 error_mark_node. Otherwise, if the function is overloaded, return 3565 the decl of the non-overloaded function. Return NULL_TREE otherwise, 3566 indicating that the call should be processed in the normal way. */ 3567 tree 3568 resolve_overloaded_builtin (location_t location, unsigned int code, 3569 vec<tree, va_gc> *arglist) 3570 { 3571 if (code >= vec_safe_length (registered_functions)) 3572 return NULL_TREE; 3573 3574 registered_function &rfn = *(*registered_functions)[code]; 3575 if (rfn.overloaded_p) 3576 return function_resolver (location, rfn.instance, rfn.decl, 3577 *arglist).resolve (); 3578 return NULL_TREE; 3579 } 3580 3581 /* Perform any semantic checks needed for a call to the SVE function 3582 with subcode CODE, such as testing for integer constant expressions. 3583 The call occurs at location LOCATION and has NARGS arguments, 3584 given by ARGS. FNDECL is the original function decl, before 3585 overload resolution. 3586 3587 Return true if the call is valid, otherwise report a suitable error. */ 3588 bool 3589 check_builtin_call (location_t location, vec<location_t>, unsigned int code, 3590 tree fndecl, unsigned int nargs, tree *args) 3591 { 3592 const registered_function &rfn = *(*registered_functions)[code]; 3593 if (!check_required_extensions (location, rfn.decl, rfn.required_extensions)) 3594 return false; 3595 return function_checker (location, rfn.instance, fndecl, 3596 TREE_TYPE (rfn.decl), nargs, args).check (); 3597 } 3598 3599 /* Attempt to fold STMT, given that it's a call to the SVE function 3600 with subcode CODE. Return the new statement on success and null 3601 on failure. Insert any other new statements at GSI. */ 3602 gimple * 3603 gimple_fold_builtin (unsigned int code, gimple_stmt_iterator *gsi, gcall *stmt) 3604 { 3605 registered_function &rfn = *(*registered_functions)[code]; 3606 return gimple_folder (rfn.instance, rfn.decl, gsi, stmt).fold (); 3607 } 3608 3609 /* Expand a call to the SVE function with subcode CODE. EXP is the call 3610 expression and TARGET is the preferred location for the result. 3611 Return the value of the lhs. */ 3612 rtx 3613 expand_builtin (unsigned int code, tree exp, rtx target) 3614 { 3615 registered_function &rfn = *(*registered_functions)[code]; 3616 if (!check_required_extensions (EXPR_LOCATION (exp), rfn.decl, 3617 rfn.required_extensions)) 3618 return target; 3619 return function_expander (rfn.instance, rfn.decl, exp, target).expand (); 3620 } 3621 3622 /* If TYPE is a built-in type defined by the SVE ABI, return the mangled name, 3623 otherwise return NULL. */ 3624 const char * 3625 mangle_builtin_type (const_tree type) 3626 { 3627 /* ??? The C++ frontend normally strips qualifiers and attributes before 3628 calling this hook, adding separate mangling for attributes that affect 3629 type identity. Fortunately the type copy will have the same TYPE_NAME 3630 as the original, so we can get the attributes from there. */ 3631 if (TYPE_NAME (type) && TREE_CODE (TYPE_NAME (type)) == TYPE_DECL) 3632 type = TREE_TYPE (TYPE_NAME (type)); 3633 if (tree attr = lookup_sve_type_attribute (type)) 3634 if (tree id = TREE_VALUE (chain_index (2, TREE_VALUE (attr)))) 3635 return IDENTIFIER_POINTER (id); 3636 return NULL; 3637 } 3638 3639 /* Return true if TYPE is a built-in SVE type defined by the ABI or ACLE. */ 3640 bool 3641 builtin_type_p (const_tree type) 3642 { 3643 return lookup_sve_type_attribute (type); 3644 } 3645 3646 /* Return true if TYPE is a built-in SVE type defined by the ABI or ACLE. 3647 If so, store the number of constituent SVE vectors in *NUM_ZR and the 3648 number of constituent SVE predicates in *NUM_PR. */ 3649 bool 3650 builtin_type_p (const_tree type, unsigned int *num_zr, unsigned int *num_pr) 3651 { 3652 if (tree attr = lookup_sve_type_attribute (type)) 3653 { 3654 tree num_zr_node = TREE_VALUE (attr); 3655 tree num_pr_node = TREE_CHAIN (num_zr_node); 3656 *num_zr = tree_to_uhwi (TREE_VALUE (num_zr_node)); 3657 *num_pr = tree_to_uhwi (TREE_VALUE (num_pr_node)); 3658 return true; 3659 } 3660 return false; 3661 } 3662 3663 /* ATTRS is the attribute list for a sizeless SVE type. Return the 3664 attributes of the associated fixed-length SVE type, taking the 3665 "SVE type" attributes from NEW_SVE_TYPE_ARGS. */ 3666 static tree 3667 get_arm_sve_vector_bits_attributes (tree old_attrs, tree new_sve_type_args) 3668 { 3669 tree new_attrs = NULL_TREE; 3670 tree *ptr = &new_attrs; 3671 for (tree attr = old_attrs; attr; attr = TREE_CHAIN (attr)) 3672 { 3673 tree name = get_attribute_name (attr); 3674 if (is_attribute_p ("SVE sizeless type", name)) 3675 continue; 3676 3677 tree args = TREE_VALUE (attr); 3678 if (is_attribute_p ("SVE type", name)) 3679 args = new_sve_type_args; 3680 *ptr = tree_cons (TREE_PURPOSE (attr), args, NULL_TREE); 3681 ptr = &TREE_CHAIN (*ptr); 3682 } 3683 return new_attrs; 3684 } 3685 3686 /* An attribute callback for the "arm_sve_vector_bits" attribute. */ 3687 tree 3688 handle_arm_sve_vector_bits_attribute (tree *node, tree, tree args, int, 3689 bool *no_add_attrs) 3690 { 3691 *no_add_attrs = true; 3692 3693 tree type = *node; 3694 tree attr = lookup_sve_type_attribute (type); 3695 if (!attr) 3696 { 3697 error ("%qs applied to non-SVE type %qT", "arm_sve_vector_bits", type); 3698 return NULL_TREE; 3699 } 3700 3701 if (!VECTOR_TYPE_P (type)) 3702 { 3703 error ("%qs applied to non-vector type %qT", 3704 "arm_sve_vector_bits", type); 3705 return NULL_TREE; 3706 } 3707 3708 if (!sizeless_type_p (type)) 3709 { 3710 error ("%qs applied to type %qT, which already has a size", 3711 "arm_sve_vector_bits", type); 3712 return NULL_TREE; 3713 } 3714 3715 tree size = TREE_VALUE (args); 3716 if (TREE_CODE (size) != INTEGER_CST) 3717 { 3718 error ("%qs requires an integer constant expression", 3719 "arm_sve_vector_bits"); 3720 return NULL_TREE; 3721 } 3722 3723 unsigned HOST_WIDE_INT value = tree_to_uhwi (size); 3724 if (maybe_ne (value, BITS_PER_SVE_VECTOR)) 3725 { 3726 warning (OPT_Wattributes, "unsupported SVE vector size"); 3727 return NULL_TREE; 3728 } 3729 3730 /* Construct a new list of "SVE type" attribute arguments. */ 3731 tree new_sve_type_args = copy_list (TREE_VALUE (attr)); 3732 3733 /* Mangle the type as an instance of the imaginary template: 3734 3735 __SVE_VLS<typename, unsigned> 3736 3737 where the first parameter is the SVE type and where the second 3738 parameter is the SVE vector length in bits. */ 3739 tree mangled_name_node = chain_index (2, new_sve_type_args); 3740 const char *old_mangled_name 3741 = IDENTIFIER_POINTER (TREE_VALUE (mangled_name_node)); 3742 char *new_mangled_name 3743 = xasprintf ("9__SVE_VLSI%sLj%dEE", old_mangled_name, (int) value); 3744 TREE_VALUE (mangled_name_node) = get_identifier (new_mangled_name); 3745 free (new_mangled_name); 3746 3747 /* FIXME: The type ought to be a distinct copy in all cases, but 3748 currently that makes the C frontend reject conversions between 3749 svbool_t and its fixed-length variants. Using a type variant 3750 avoids that but means that we treat some ambiguous combinations 3751 as valid. */ 3752 tree new_type; 3753 tree base_type = TYPE_MAIN_VARIANT (type); 3754 if (lang_GNU_C () && VECTOR_BOOLEAN_TYPE_P (type)) 3755 new_type = build_variant_type_copy (base_type); 3756 else 3757 new_type = build_distinct_type_copy (base_type); 3758 3759 /* Construct a TYPE_DECL for the new type. This serves two purposes: 3760 3761 - It ensures we don't print the original TYPE_DECL in error messages. 3762 Printing the original name would be confusing because there are 3763 situations in which the distinction between the original type and 3764 the new type matters. For example: 3765 3766 __SVInt8_t __attribute__((arm_sve_vector_bits(512))) *a; 3767 __SVInt8_t *b; 3768 3769 a = b; 3770 3771 is invalid in C++, but without this, we'd print both types in 3772 the same way. 3773 3774 - Having a separate TYPE_DECL is necessary to ensure that C++ 3775 mangling works correctly. See mangle_builtin_type for details. 3776 3777 The name of the decl is something like: 3778 3779 svint8_t __attribute__((arm_sve_vector_bits(512))) 3780 3781 This is a compromise. It would be more accurate to use something like: 3782 3783 __SVInt8_t __attribute__((arm_sve_vector_bits(512))) 3784 3785 but the <arm_sve.h> name is likely to be more meaningful. */ 3786 tree acle_name_node = TREE_CHAIN (mangled_name_node); 3787 const char *old_type_name = IDENTIFIER_POINTER (TREE_VALUE (acle_name_node)); 3788 char *new_type_name 3789 = xasprintf ("%s __attribute__((arm_sve_vector_bits(%d)))", 3790 old_type_name, (int) value); 3791 tree decl = build_decl (BUILTINS_LOCATION, TYPE_DECL, 3792 get_identifier (new_type_name), new_type); 3793 DECL_ARTIFICIAL (decl) = 1; 3794 TYPE_NAME (new_type) = decl; 3795 free (new_type_name); 3796 3797 /* Allow the GNU vector extensions to be applied to vectors. 3798 The extensions aren't yet defined for packed predicates, 3799 so continue to treat them as abstract entities for now. */ 3800 if (!VECTOR_BOOLEAN_TYPE_P (new_type)) 3801 TYPE_INDIVISIBLE_P (new_type) = 0; 3802 3803 /* The new type is a normal sized type; it doesn't have the same 3804 restrictions as sizeless types. */ 3805 TYPE_ATTRIBUTES (new_type) 3806 = get_arm_sve_vector_bits_attributes (TYPE_ATTRIBUTES (new_type), 3807 new_sve_type_args); 3808 3809 /* Apply the relevant attributes, qualifiers and alignment of TYPE, 3810 if they differ from the original (sizeless) BASE_TYPE. */ 3811 if (TYPE_ATTRIBUTES (base_type) != TYPE_ATTRIBUTES (type) 3812 || TYPE_QUALS (base_type) != TYPE_QUALS (type)) 3813 { 3814 tree attrs 3815 = get_arm_sve_vector_bits_attributes (TYPE_ATTRIBUTES (type), 3816 new_sve_type_args); 3817 new_type = build_type_attribute_qual_variant (new_type, attrs, 3818 TYPE_QUALS (type)); 3819 } 3820 if (TYPE_ALIGN (base_type) != TYPE_ALIGN (type)) 3821 new_type = build_aligned_type (new_type, TYPE_ALIGN (type)); 3822 3823 *node = new_type; 3824 return NULL_TREE; 3825 } 3826 3827 /* Implement TARGET_VERIFY_TYPE_CONTEXT for SVE types. */ 3828 bool 3829 verify_type_context (location_t loc, type_context_kind context, 3830 const_tree type, bool silent_p) 3831 { 3832 if (!sizeless_type_p (type)) 3833 return true; 3834 3835 switch (context) 3836 { 3837 case TCTX_SIZEOF: 3838 case TCTX_STATIC_STORAGE: 3839 if (!silent_p) 3840 error_at (loc, "SVE type %qT does not have a fixed size", type); 3841 return false; 3842 3843 case TCTX_ALIGNOF: 3844 if (!silent_p) 3845 error_at (loc, "SVE type %qT does not have a defined alignment", type); 3846 return false; 3847 3848 case TCTX_THREAD_STORAGE: 3849 if (!silent_p) 3850 error_at (loc, "variables of type %qT cannot have thread-local" 3851 " storage duration", type); 3852 return false; 3853 3854 case TCTX_POINTER_ARITH: 3855 if (!silent_p) 3856 error_at (loc, "arithmetic on pointer to SVE type %qT", type); 3857 return false; 3858 3859 case TCTX_FIELD: 3860 if (silent_p) 3861 ; 3862 else if (lang_GNU_CXX ()) 3863 error_at (loc, "member variables cannot have SVE type %qT", type); 3864 else 3865 error_at (loc, "fields cannot have SVE type %qT", type); 3866 return false; 3867 3868 case TCTX_ARRAY_ELEMENT: 3869 if (!silent_p) 3870 error_at (loc, "array elements cannot have SVE type %qT", type); 3871 return false; 3872 3873 case TCTX_ALLOCATION: 3874 if (!silent_p) 3875 error_at (loc, "cannot allocate objects with SVE type %qT", type); 3876 return false; 3877 3878 case TCTX_DEALLOCATION: 3879 if (!silent_p) 3880 error_at (loc, "cannot delete objects with SVE type %qT", type); 3881 return false; 3882 3883 case TCTX_EXCEPTIONS: 3884 if (!silent_p) 3885 error_at (loc, "cannot throw or catch SVE type %qT", type); 3886 return false; 3887 3888 case TCTX_CAPTURE_BY_COPY: 3889 if (!silent_p) 3890 error_at (loc, "capture by copy of SVE type %qT", type); 3891 return false; 3892 } 3893 gcc_unreachable (); 3894 } 3895 3896 } 3897 3898 using namespace aarch64_sve; 3899 3900 inline void 3901 gt_ggc_mx (function_instance *) 3902 { 3903 } 3904 3905 inline void 3906 gt_pch_nx (function_instance *) 3907 { 3908 } 3909 3910 inline void 3911 gt_pch_nx (function_instance *, void (*) (void *, void *), void *) 3912 { 3913 } 3914 3915 #include "gt-aarch64-sve-builtins.h" 3916