1 /* ACLE support for AArch64 SVE 2 Copyright (C) 2018-2020 Free Software Foundation, Inc. 3 4 This file is part of GCC. 5 6 GCC is free software; you can redistribute it and/or modify it 7 under the terms of the GNU General Public License as published by 8 the Free Software Foundation; either version 3, or (at your option) 9 any later version. 10 11 GCC is distributed in the hope that it will be useful, but 12 WITHOUT ANY WARRANTY; without even the implied warranty of 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 General Public License for more details. 15 16 You should have received a copy of the GNU General Public License 17 along with GCC; see the file COPYING3. If not see 18 <http://www.gnu.org/licenses/>. */ 19 20 #define IN_TARGET_CODE 1 21 22 #include "config.h" 23 #include "system.h" 24 #include "coretypes.h" 25 #include "tm.h" 26 #include "tree.h" 27 #include "rtl.h" 28 #include "tm_p.h" 29 #include "memmodel.h" 30 #include "insn-codes.h" 31 #include "optabs.h" 32 #include "recog.h" 33 #include "diagnostic.h" 34 #include "expr.h" 35 #include "basic-block.h" 36 #include "function.h" 37 #include "fold-const.h" 38 #include "gimple.h" 39 #include "gimple-iterator.h" 40 #include "gimplify.h" 41 #include "explow.h" 42 #include "emit-rtl.h" 43 #include "tree-vector-builder.h" 44 #include "stor-layout.h" 45 #include "regs.h" 46 #include "alias.h" 47 #include "gimple-fold.h" 48 #include "langhooks.h" 49 #include "stringpool.h" 50 #include "attribs.h" 51 #include "aarch64-sve-builtins.h" 52 #include "aarch64-sve-builtins-base.h" 53 #include "aarch64-sve-builtins-sve2.h" 54 #include "aarch64-sve-builtins-shapes.h" 55 56 namespace aarch64_sve { 57 58 /* Static information about each single-predicate or single-vector 59 ABI and ACLE type. */ 60 struct vector_type_info 61 { 62 /* The name of the type as declared by arm_sve.h. */ 63 const char *acle_name; 64 65 /* The name of the type specified in AAPCS64. The type is always 66 available under this name, even when arm_sve.h isn't included. */ 67 const char *abi_name; 68 69 /* The C++ mangling of ABI_NAME. */ 70 const char *mangled_name; 71 }; 72 73 /* Describes a function decl. */ 74 class GTY(()) registered_function 75 { 76 public: 77 /* The ACLE function that the decl represents. */ 78 function_instance instance GTY ((skip)); 79 80 /* The decl itself. */ 81 tree decl; 82 83 /* The architecture extensions that the function requires, as a set of 84 AARCH64_FL_* flags. */ 85 uint64_t required_extensions; 86 87 /* True if the decl represents an overloaded function that needs to be 88 resolved by function_resolver. */ 89 bool overloaded_p; 90 }; 91 92 /* Hash traits for registered_function. */ 93 struct registered_function_hasher : nofree_ptr_hash <registered_function> 94 { 95 typedef function_instance compare_type; 96 97 static hashval_t hash (value_type); 98 static bool equal (value_type, const compare_type &); 99 }; 100 101 /* Information about each single-predicate or single-vector type. */ 102 static CONSTEXPR const vector_type_info vector_types[] = { 103 #define DEF_SVE_TYPE(ACLE_NAME, NCHARS, ABI_NAME, SCALAR_TYPE) \ 104 { #ACLE_NAME, #ABI_NAME, "u" #NCHARS #ABI_NAME }, 105 #include "aarch64-sve-builtins.def" 106 }; 107 108 /* The function name suffix associated with each predication type. */ 109 static const char *const pred_suffixes[NUM_PREDS + 1] = { 110 "", 111 "", 112 "_m", 113 "_x", 114 "_z", 115 "" 116 }; 117 118 /* Static information about each mode_suffix_index. */ 119 CONSTEXPR const mode_suffix_info mode_suffixes[] = { 120 #define VECTOR_TYPE_none NUM_VECTOR_TYPES 121 #define DEF_SVE_MODE(NAME, BASE, DISPLACEMENT, UNITS) \ 122 { "_" #NAME, VECTOR_TYPE_##BASE, VECTOR_TYPE_##DISPLACEMENT, UNITS_##UNITS }, 123 #include "aarch64-sve-builtins.def" 124 #undef VECTOR_TYPE_none 125 { "", NUM_VECTOR_TYPES, NUM_VECTOR_TYPES, UNITS_none } 126 }; 127 128 /* Static information about each type_suffix_index. */ 129 CONSTEXPR const type_suffix_info type_suffixes[NUM_TYPE_SUFFIXES + 1] = { 130 #define DEF_SVE_TYPE_SUFFIX(NAME, ACLE_TYPE, CLASS, BITS, MODE) \ 131 { "_" #NAME, \ 132 VECTOR_TYPE_##ACLE_TYPE, \ 133 TYPE_##CLASS, \ 134 BITS, \ 135 BITS / BITS_PER_UNIT, \ 136 TYPE_##CLASS == TYPE_signed || TYPE_##CLASS == TYPE_unsigned, \ 137 TYPE_##CLASS == TYPE_unsigned, \ 138 TYPE_##CLASS == TYPE_float, \ 139 TYPE_##CLASS == TYPE_bool, \ 140 0, \ 141 MODE }, 142 #include "aarch64-sve-builtins.def" 143 { "", NUM_VECTOR_TYPES, TYPE_bool, 0, 0, false, false, false, false, 144 0, VOIDmode } 145 }; 146 147 /* Define a TYPES_<combination> macro for each combination of type 148 suffixes that an ACLE function can have, where <combination> is the 149 name used in DEF_SVE_FUNCTION entries. 150 151 Use S (T) for single type suffix T and D (T1, T2) for a pair of type 152 suffixes T1 and T2. Use commas to separate the suffixes. 153 154 Although the order shouldn't matter, the convention is to sort the 155 suffixes lexicographically after dividing suffixes into a type 156 class ("b", "f", etc.) and a numerical bit count. */ 157 158 /* _b8 _b16 _b32 _b64. */ 159 #define TYPES_all_pred(S, D) \ 160 S (b8), S (b16), S (b32), S (b64) 161 162 /* _f16 _f32 _f64. */ 163 #define TYPES_all_float(S, D) \ 164 S (f16), S (f32), S (f64) 165 166 /* _s8 _s16 _s32 _s64. */ 167 #define TYPES_all_signed(S, D) \ 168 S (s8), S (s16), S (s32), S (s64) 169 170 /* _f16 _f32 _f64 171 _s8 _s16 _s32 _s64. */ 172 #define TYPES_all_float_and_signed(S, D) \ 173 TYPES_all_float (S, D), TYPES_all_signed (S, D) 174 175 /* _u8 _u16 _u32 _u64. */ 176 #define TYPES_all_unsigned(S, D) \ 177 S (u8), S (u16), S (u32), S (u64) 178 179 /* _s8 _s16 _s32 _s64 180 _u8 _u16 _u32 _u64. */ 181 #define TYPES_all_integer(S, D) \ 182 TYPES_all_signed (S, D), TYPES_all_unsigned (S, D) 183 184 /* _f16 _f32 _f64 185 _s8 _s16 _s32 _s64 186 _u8 _u16 _u32 _u64. */ 187 #define TYPES_all_arith(S, D) \ 188 TYPES_all_float (S, D), TYPES_all_integer (S, D) 189 190 /* _bf16 191 _f16 _f32 _f64 192 _s8 _s16 _s32 _s64 193 _u8 _u16 _u32 _u64. */ 194 #define TYPES_all_data(S, D) \ 195 S (bf16), TYPES_all_arith (S, D) 196 197 /* _b only. */ 198 #define TYPES_b(S, D) \ 199 S (b) 200 201 /* _u8. */ 202 #define TYPES_b_unsigned(S, D) \ 203 S (u8) 204 205 /* _s8 206 _u8. */ 207 #define TYPES_b_integer(S, D) \ 208 S (s8), TYPES_b_unsigned (S, D) 209 210 /* _s8 _s16 211 _u8 _u16. */ 212 #define TYPES_bh_integer(S, D) \ 213 S (s8), S (s16), S (u8), S (u16) 214 215 /* _u8 _u32. */ 216 #define TYPES_bs_unsigned(S, D) \ 217 S (u8), S (u32) 218 219 /* _s8 _s16 _s32. */ 220 #define TYPES_bhs_signed(S, D) \ 221 S (s8), S (s16), S (s32) 222 223 /* _u8 _u16 _u32. */ 224 #define TYPES_bhs_unsigned(S, D) \ 225 S (u8), S (u16), S (u32) 226 227 /* _s8 _s16 _s32 228 _u8 _u16 _u32. */ 229 #define TYPES_bhs_integer(S, D) \ 230 TYPES_bhs_signed (S, D), TYPES_bhs_unsigned (S, D) 231 232 /* _s16 233 _u16. */ 234 #define TYPES_h_integer(S, D) \ 235 S (s16), S (u16) 236 237 /* _s16 _s32. */ 238 #define TYPES_hs_signed(S, D) \ 239 S (s16), S (s32) 240 241 /* _s16 _s32 242 _u16 _u32. */ 243 #define TYPES_hs_integer(S, D) \ 244 TYPES_hs_signed (S, D), S (u16), S (u32) 245 246 /* _f16 _f32. */ 247 #define TYPES_hs_float(S, D) \ 248 S (f16), S (f32) 249 250 /* _u16 _u64. */ 251 #define TYPES_hd_unsigned(S, D) \ 252 S (u16), S (u64) 253 254 /* _s16 _s32 _s64. */ 255 #define TYPES_hsd_signed(S, D) \ 256 S (s16), S (s32), S (s64) 257 258 /* _s16 _s32 _s64 259 _u16 _u32 _u64. */ 260 #define TYPES_hsd_integer(S, D) \ 261 TYPES_hsd_signed (S, D), S (u16), S (u32), S (u64) 262 263 /* _f32. */ 264 #define TYPES_s_float(S, D) \ 265 S (f32) 266 267 /* _f32 268 _s16 _s32 _s64 269 _u16 _u32 _u64. */ 270 #define TYPES_s_float_hsd_integer(S, D) \ 271 TYPES_s_float (S, D), TYPES_hsd_integer (S, D) 272 273 /* _f32 274 _s32 _s64 275 _u32 _u64. */ 276 #define TYPES_s_float_sd_integer(S, D) \ 277 TYPES_s_float (S, D), TYPES_sd_integer (S, D) 278 279 /* _s32. */ 280 #define TYPES_s_signed(S, D) \ 281 S (s32) 282 283 /* _u32. */ 284 #define TYPES_s_unsigned(S, D) \ 285 S (u32) 286 287 /* _s32 _u32. */ 288 #define TYPES_s_integer(S, D) \ 289 TYPES_s_signed (S, D), TYPES_s_unsigned (S, D) 290 291 /* _s32 _s64. */ 292 #define TYPES_sd_signed(S, D) \ 293 S (s32), S (s64) 294 295 /* _u32 _u64. */ 296 #define TYPES_sd_unsigned(S, D) \ 297 S (u32), S (u64) 298 299 /* _s32 _s64 300 _u32 _u64. */ 301 #define TYPES_sd_integer(S, D) \ 302 TYPES_sd_signed (S, D), TYPES_sd_unsigned (S, D) 303 304 /* _f32 _f64 305 _s32 _s64 306 _u32 _u64. */ 307 #define TYPES_sd_data(S, D) \ 308 S (f32), S (f64), TYPES_sd_integer (S, D) 309 310 /* _f16 _f32 _f64 311 _s32 _s64 312 _u32 _u64. */ 313 #define TYPES_all_float_and_sd_integer(S, D) \ 314 TYPES_all_float (S, D), TYPES_sd_integer (S, D) 315 316 /* _f64. */ 317 #define TYPES_d_float(S, D) \ 318 S (f64) 319 320 /* _u64. */ 321 #define TYPES_d_unsigned(S, D) \ 322 S (u64) 323 324 /* _s64 325 _u64. */ 326 #define TYPES_d_integer(S, D) \ 327 S (s64), TYPES_d_unsigned (S, D) 328 329 /* _f64 330 _s64 331 _u64. */ 332 #define TYPES_d_data(S, D) \ 333 TYPES_d_float (S, D), TYPES_d_integer (S, D) 334 335 /* All the type combinations allowed by svcvt. */ 336 #define TYPES_cvt(S, D) \ 337 D (f16, f32), D (f16, f64), \ 338 D (f16, s16), D (f16, s32), D (f16, s64), \ 339 D (f16, u16), D (f16, u32), D (f16, u64), \ 340 \ 341 D (f32, f16), D (f32, f64), \ 342 D (f32, s32), D (f32, s64), \ 343 D (f32, u32), D (f32, u64), \ 344 \ 345 D (f64, f16), D (f64, f32), \ 346 D (f64, s32), D (f64, s64), \ 347 D (f64, u32), D (f64, u64), \ 348 \ 349 D (s16, f16), \ 350 D (s32, f16), D (s32, f32), D (s32, f64), \ 351 D (s64, f16), D (s64, f32), D (s64, f64), \ 352 \ 353 D (u16, f16), \ 354 D (u32, f16), D (u32, f32), D (u32, f64), \ 355 D (u64, f16), D (u64, f32), D (u64, f64) 356 357 /* _bf16_f32. */ 358 #define TYPES_cvt_bfloat(S, D) \ 359 D (bf16, f32) 360 361 /* _f32_f16 362 _f64_f32. */ 363 #define TYPES_cvt_long(S, D) \ 364 D (f32, f16), D (f64, f32) 365 366 /* _f16_f32. */ 367 #define TYPES_cvt_narrow_s(S, D) \ 368 D (f32, f64) 369 370 /* _f16_f32 371 _f32_f64. */ 372 #define TYPES_cvt_narrow(S, D) \ 373 D (f16, f32), TYPES_cvt_narrow_s (S, D) 374 375 /* { _s32 _s64 } x { _b8 _b16 _b32 _b64 } 376 { _u32 _u64 }. */ 377 #define TYPES_inc_dec_n1(D, A) \ 378 D (A, b8), D (A, b16), D (A, b32), D (A, b64) 379 #define TYPES_inc_dec_n(S, D) \ 380 TYPES_inc_dec_n1 (D, s32), \ 381 TYPES_inc_dec_n1 (D, s64), \ 382 TYPES_inc_dec_n1 (D, u32), \ 383 TYPES_inc_dec_n1 (D, u64) 384 385 /* { _bf16 } { _bf16 } 386 { _f16 _f32 _f64 } { _f16 _f32 _f64 } 387 { _s8 _s16 _s32 _s64 } x { _s8 _s16 _s32 _s64 } 388 { _u8 _u16 _u32 _u64 } { _u8 _u16 _u32 _u64 }. */ 389 #define TYPES_reinterpret1(D, A) \ 390 D (A, bf16), \ 391 D (A, f16), D (A, f32), D (A, f64), \ 392 D (A, s8), D (A, s16), D (A, s32), D (A, s64), \ 393 D (A, u8), D (A, u16), D (A, u32), D (A, u64) 394 #define TYPES_reinterpret(S, D) \ 395 TYPES_reinterpret1 (D, bf16), \ 396 TYPES_reinterpret1 (D, f16), \ 397 TYPES_reinterpret1 (D, f32), \ 398 TYPES_reinterpret1 (D, f64), \ 399 TYPES_reinterpret1 (D, s8), \ 400 TYPES_reinterpret1 (D, s16), \ 401 TYPES_reinterpret1 (D, s32), \ 402 TYPES_reinterpret1 (D, s64), \ 403 TYPES_reinterpret1 (D, u8), \ 404 TYPES_reinterpret1 (D, u16), \ 405 TYPES_reinterpret1 (D, u32), \ 406 TYPES_reinterpret1 (D, u64) 407 408 /* { _b8 _b16 _b32 _b64 } x { _s32 _s64 } 409 { _u32 _u64 } */ 410 #define TYPES_while1(D, bn) \ 411 D (bn, s32), D (bn, s64), D (bn, u32), D (bn, u64) 412 #define TYPES_while(S, D) \ 413 TYPES_while1 (D, b8), \ 414 TYPES_while1 (D, b16), \ 415 TYPES_while1 (D, b32), \ 416 TYPES_while1 (D, b64) 417 418 /* Describe a pair of type suffixes in which only the first is used. */ 419 #define DEF_VECTOR_TYPE(X) { TYPE_SUFFIX_ ## X, NUM_TYPE_SUFFIXES } 420 421 /* Describe a pair of type suffixes in which both are used. */ 422 #define DEF_DOUBLE_TYPE(X, Y) { TYPE_SUFFIX_ ## X, TYPE_SUFFIX_ ## Y } 423 424 /* Create an array that can be used in aarch64-sve-builtins.def to 425 select the type suffixes in TYPES_<NAME>. */ 426 #define DEF_SVE_TYPES_ARRAY(NAME) \ 427 static const type_suffix_pair types_##NAME[] = { \ 428 TYPES_##NAME (DEF_VECTOR_TYPE, DEF_DOUBLE_TYPE), \ 429 { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } \ 430 } 431 432 /* For functions that don't take any type suffixes. */ 433 static const type_suffix_pair types_none[] = { 434 { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES }, 435 { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } 436 }; 437 438 /* Create an array for each TYPES_<combination> macro above. */ 439 DEF_SVE_TYPES_ARRAY (all_pred); 440 DEF_SVE_TYPES_ARRAY (all_float); 441 DEF_SVE_TYPES_ARRAY (all_signed); 442 DEF_SVE_TYPES_ARRAY (all_float_and_signed); 443 DEF_SVE_TYPES_ARRAY (all_unsigned); 444 DEF_SVE_TYPES_ARRAY (all_integer); 445 DEF_SVE_TYPES_ARRAY (all_arith); 446 DEF_SVE_TYPES_ARRAY (all_data); 447 DEF_SVE_TYPES_ARRAY (b); 448 DEF_SVE_TYPES_ARRAY (b_unsigned); 449 DEF_SVE_TYPES_ARRAY (b_integer); 450 DEF_SVE_TYPES_ARRAY (bh_integer); 451 DEF_SVE_TYPES_ARRAY (bs_unsigned); 452 DEF_SVE_TYPES_ARRAY (bhs_signed); 453 DEF_SVE_TYPES_ARRAY (bhs_unsigned); 454 DEF_SVE_TYPES_ARRAY (bhs_integer); 455 DEF_SVE_TYPES_ARRAY (h_integer); 456 DEF_SVE_TYPES_ARRAY (hs_signed); 457 DEF_SVE_TYPES_ARRAY (hs_integer); 458 DEF_SVE_TYPES_ARRAY (hs_float); 459 DEF_SVE_TYPES_ARRAY (hd_unsigned); 460 DEF_SVE_TYPES_ARRAY (hsd_signed); 461 DEF_SVE_TYPES_ARRAY (hsd_integer); 462 DEF_SVE_TYPES_ARRAY (s_float); 463 DEF_SVE_TYPES_ARRAY (s_float_hsd_integer); 464 DEF_SVE_TYPES_ARRAY (s_float_sd_integer); 465 DEF_SVE_TYPES_ARRAY (s_signed); 466 DEF_SVE_TYPES_ARRAY (s_unsigned); 467 DEF_SVE_TYPES_ARRAY (s_integer); 468 DEF_SVE_TYPES_ARRAY (sd_signed); 469 DEF_SVE_TYPES_ARRAY (sd_unsigned); 470 DEF_SVE_TYPES_ARRAY (sd_integer); 471 DEF_SVE_TYPES_ARRAY (sd_data); 472 DEF_SVE_TYPES_ARRAY (all_float_and_sd_integer); 473 DEF_SVE_TYPES_ARRAY (d_float); 474 DEF_SVE_TYPES_ARRAY (d_unsigned); 475 DEF_SVE_TYPES_ARRAY (d_integer); 476 DEF_SVE_TYPES_ARRAY (d_data); 477 DEF_SVE_TYPES_ARRAY (cvt); 478 DEF_SVE_TYPES_ARRAY (cvt_bfloat); 479 DEF_SVE_TYPES_ARRAY (cvt_long); 480 DEF_SVE_TYPES_ARRAY (cvt_narrow_s); 481 DEF_SVE_TYPES_ARRAY (cvt_narrow); 482 DEF_SVE_TYPES_ARRAY (inc_dec_n); 483 DEF_SVE_TYPES_ARRAY (reinterpret); 484 DEF_SVE_TYPES_ARRAY (while); 485 486 /* Used by functions that have no governing predicate. */ 487 static const predication_index preds_none[] = { PRED_none, NUM_PREDS }; 488 489 /* Used by functions that have a governing predicate but do not have an 490 explicit suffix. */ 491 static const predication_index preds_implicit[] = { PRED_implicit, NUM_PREDS }; 492 493 /* Used by functions that allow merging and "don't care" predication, 494 but are not suitable for predicated MOVPRFX. */ 495 static const predication_index preds_mx[] = { 496 PRED_m, PRED_x, NUM_PREDS 497 }; 498 499 /* Used by functions that allow merging, zeroing and "don't care" 500 predication. */ 501 static const predication_index preds_mxz[] = { 502 PRED_m, PRED_x, PRED_z, NUM_PREDS 503 }; 504 505 /* Used by functions that have the mxz predicated forms above, and in addition 506 have an unpredicated form. */ 507 static const predication_index preds_mxz_or_none[] = { 508 PRED_m, PRED_x, PRED_z, PRED_none, NUM_PREDS 509 }; 510 511 /* Used by functions that allow merging and zeroing predication but have 512 no "_x" form. */ 513 static const predication_index preds_mz[] = { PRED_m, PRED_z, NUM_PREDS }; 514 515 /* Used by functions that have an unpredicated form and a _z predicated 516 form. */ 517 static const predication_index preds_z_or_none[] = { 518 PRED_z, PRED_none, NUM_PREDS 519 }; 520 521 /* Used by (mostly predicate) functions that only support "_z" predication. */ 522 static const predication_index preds_z[] = { PRED_z, NUM_PREDS }; 523 524 /* A list of all SVE ACLE functions. */ 525 static CONSTEXPR const function_group_info function_groups[] = { 526 #define DEF_SVE_FUNCTION(NAME, SHAPE, TYPES, PREDS) \ 527 { #NAME, &functions::NAME, &shapes::SHAPE, types_##TYPES, preds_##PREDS, \ 528 REQUIRED_EXTENSIONS | AARCH64_FL_SVE }, 529 #include "aarch64-sve-builtins.def" 530 }; 531 532 /* The scalar type associated with each vector type. */ 533 extern GTY(()) tree scalar_types[NUM_VECTOR_TYPES]; 534 tree scalar_types[NUM_VECTOR_TYPES]; 535 536 /* The single-predicate and single-vector types, with their built-in 537 "__SV..._t" name. Allow an index of NUM_VECTOR_TYPES, which always 538 yields a null tree. */ 539 static GTY(()) tree abi_vector_types[NUM_VECTOR_TYPES + 1]; 540 541 /* Same, but with the arm_sve.h "sv..._t" name. */ 542 extern GTY(()) tree acle_vector_types[MAX_TUPLE_SIZE][NUM_VECTOR_TYPES + 1]; 543 tree acle_vector_types[MAX_TUPLE_SIZE][NUM_VECTOR_TYPES + 1]; 544 545 /* The svpattern enum type. */ 546 extern GTY(()) tree acle_svpattern; 547 tree acle_svpattern; 548 549 /* The svprfop enum type. */ 550 extern GTY(()) tree acle_svprfop; 551 tree acle_svprfop; 552 553 /* The list of all registered function decls, indexed by code. */ 554 static GTY(()) vec<registered_function *, va_gc> *registered_functions; 555 556 /* All registered function decls, hashed on the function_instance 557 that they implement. This is used for looking up implementations of 558 overloaded functions. */ 559 static hash_table<registered_function_hasher> *function_table; 560 561 /* True if we've already complained about attempts to use functions 562 when the required extension is disabled. */ 563 static bool reported_missing_extension_p; 564 565 /* True if we've already complained about attempts to use functions 566 which require registers that are missing. */ 567 static bool reported_missing_registers_p; 568 569 /* Record that TYPE is an ABI-defined SVE type that contains NUM_ZR SVE vectors 570 and NUM_PR SVE predicates. MANGLED_NAME, if nonnull, is the ABI-defined 571 mangling of the type. ACLE_NAME is the <arm_sve.h> name of the type. */ 572 static void 573 add_sve_type_attribute (tree type, unsigned int num_zr, unsigned int num_pr, 574 const char *mangled_name, const char *acle_name) 575 { 576 tree mangled_name_tree 577 = (mangled_name ? get_identifier (mangled_name) : NULL_TREE); 578 579 tree value = tree_cons (NULL_TREE, get_identifier (acle_name), NULL_TREE); 580 value = tree_cons (NULL_TREE, mangled_name_tree, value); 581 value = tree_cons (NULL_TREE, size_int (num_pr), value); 582 value = tree_cons (NULL_TREE, size_int (num_zr), value); 583 TYPE_ATTRIBUTES (type) = tree_cons (get_identifier ("SVE type"), value, 584 TYPE_ATTRIBUTES (type)); 585 } 586 587 /* If TYPE is an ABI-defined SVE type, return its attribute descriptor, 588 otherwise return null. */ 589 static tree 590 lookup_sve_type_attribute (const_tree type) 591 { 592 if (type == error_mark_node) 593 return NULL_TREE; 594 return lookup_attribute ("SVE type", TYPE_ATTRIBUTES (type)); 595 } 596 597 /* Force TYPE to be a sizeless type. */ 598 static void 599 make_type_sizeless (tree type) 600 { 601 TYPE_ATTRIBUTES (type) = tree_cons (get_identifier ("SVE sizeless type"), 602 NULL_TREE, TYPE_ATTRIBUTES (type)); 603 } 604 605 /* Return true if TYPE is a sizeless type. */ 606 static bool 607 sizeless_type_p (const_tree type) 608 { 609 if (type == error_mark_node) 610 return NULL_TREE; 611 return lookup_attribute ("SVE sizeless type", TYPE_ATTRIBUTES (type)); 612 } 613 614 /* Return true if CANDIDATE is equivalent to MODEL_TYPE for overloading 615 purposes. */ 616 static bool 617 matches_type_p (const_tree model_type, const_tree candidate) 618 { 619 if (VECTOR_TYPE_P (model_type)) 620 { 621 if (!VECTOR_TYPE_P (candidate) 622 || maybe_ne (TYPE_VECTOR_SUBPARTS (model_type), 623 TYPE_VECTOR_SUBPARTS (candidate)) 624 || TYPE_MODE (model_type) != TYPE_MODE (candidate)) 625 return false; 626 627 model_type = TREE_TYPE (model_type); 628 candidate = TREE_TYPE (candidate); 629 } 630 return (candidate != error_mark_node 631 && TYPE_MAIN_VARIANT (model_type) == TYPE_MAIN_VARIANT (candidate)); 632 } 633 634 /* If TYPE is a valid SVE element type, return the corresponding type 635 suffix, otherwise return NUM_TYPE_SUFFIXES. */ 636 static type_suffix_index 637 find_type_suffix_for_scalar_type (const_tree type) 638 { 639 /* A linear search should be OK here, since the code isn't hot and 640 the number of types is only small. */ 641 for (unsigned int suffix_i = 0; suffix_i < NUM_TYPE_SUFFIXES; ++suffix_i) 642 if (!type_suffixes[suffix_i].bool_p) 643 { 644 vector_type_index vector_i = type_suffixes[suffix_i].vector_type; 645 if (matches_type_p (scalar_types[vector_i], type)) 646 return type_suffix_index (suffix_i); 647 } 648 return NUM_TYPE_SUFFIXES; 649 } 650 651 /* Report an error against LOCATION that the user has tried to use 652 function FNDECL when extension EXTENSION is disabled. */ 653 static void 654 report_missing_extension (location_t location, tree fndecl, 655 const char *extension) 656 { 657 /* Avoid reporting a slew of messages for a single oversight. */ 658 if (reported_missing_extension_p) 659 return; 660 661 error_at (location, "ACLE function %qD requires ISA extension %qs", 662 fndecl, extension); 663 inform (location, "you can enable %qs using the command-line" 664 " option %<-march%>, or by using the %<target%>" 665 " attribute or pragma", extension); 666 reported_missing_extension_p = true; 667 } 668 669 /* Check whether the registers required by SVE function fndecl are available. 670 Report an error against LOCATION and return false if not. */ 671 static bool 672 check_required_registers (location_t location, tree fndecl) 673 { 674 /* Avoid reporting a slew of messages for a single oversight. */ 675 if (reported_missing_registers_p) 676 return false; 677 678 if (TARGET_GENERAL_REGS_ONLY) 679 { 680 /* SVE registers are not usable when -mgeneral-regs-only option 681 is specified. */ 682 error_at (location, 683 "ACLE function %qD is incompatible with the use of %qs", 684 fndecl, "-mgeneral-regs-only"); 685 reported_missing_registers_p = true; 686 return false; 687 } 688 689 return true; 690 } 691 692 /* Check whether all the AARCH64_FL_* values in REQUIRED_EXTENSIONS are 693 enabled, given that those extensions are required for function FNDECL. 694 Report an error against LOCATION if not. */ 695 static bool 696 check_required_extensions (location_t location, tree fndecl, 697 uint64_t required_extensions) 698 { 699 uint64_t missing_extensions = required_extensions & ~aarch64_isa_flags; 700 if (missing_extensions == 0) 701 return check_required_registers (location, fndecl); 702 703 static const struct { uint64_t flag; const char *name; } extensions[] = { 704 #define AARCH64_OPT_EXTENSION(EXT_NAME, FLAG_CANONICAL, FLAGS_ON, FLAGS_OFF, \ 705 SYNTHETIC, FEATURE_STRING) \ 706 { FLAG_CANONICAL, EXT_NAME }, 707 #include "aarch64-option-extensions.def" 708 }; 709 710 for (unsigned int i = 0; i < ARRAY_SIZE (extensions); ++i) 711 if (missing_extensions & extensions[i].flag) 712 { 713 report_missing_extension (location, fndecl, extensions[i].name); 714 return false; 715 } 716 gcc_unreachable (); 717 } 718 719 /* Report that LOCATION has a call to FNDECL in which argument ARGNO 720 was not an integer constant expression. ARGNO counts from zero. */ 721 static void 722 report_non_ice (location_t location, tree fndecl, unsigned int argno) 723 { 724 error_at (location, "argument %d of %qE must be an integer constant" 725 " expression", argno + 1, fndecl); 726 } 727 728 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has 729 the value ACTUAL, whereas the function requires a value in the range 730 [MIN, MAX]. ARGNO counts from zero. */ 731 static void 732 report_out_of_range (location_t location, tree fndecl, unsigned int argno, 733 HOST_WIDE_INT actual, HOST_WIDE_INT min, 734 HOST_WIDE_INT max) 735 { 736 error_at (location, "passing %wd to argument %d of %qE, which expects" 737 " a value in the range [%wd, %wd]", actual, argno + 1, fndecl, 738 min, max); 739 } 740 741 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has 742 the value ACTUAL, whereas the function requires either VALUE0 or 743 VALUE1. ARGNO counts from zero. */ 744 static void 745 report_neither_nor (location_t location, tree fndecl, unsigned int argno, 746 HOST_WIDE_INT actual, HOST_WIDE_INT value0, 747 HOST_WIDE_INT value1) 748 { 749 error_at (location, "passing %wd to argument %d of %qE, which expects" 750 " either %wd or %wd", actual, argno + 1, fndecl, value0, value1); 751 } 752 753 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has 754 the value ACTUAL, whereas the function requires one of VALUE0..3. 755 ARGNO counts from zero. */ 756 static void 757 report_not_one_of (location_t location, tree fndecl, unsigned int argno, 758 HOST_WIDE_INT actual, HOST_WIDE_INT value0, 759 HOST_WIDE_INT value1, HOST_WIDE_INT value2, 760 HOST_WIDE_INT value3) 761 { 762 error_at (location, "passing %wd to argument %d of %qE, which expects" 763 " %wd, %wd, %wd or %wd", actual, argno + 1, fndecl, value0, value1, 764 value2, value3); 765 } 766 767 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has 768 the value ACTUAL, whereas the function requires a valid value of 769 enum type ENUMTYPE. ARGNO counts from zero. */ 770 static void 771 report_not_enum (location_t location, tree fndecl, unsigned int argno, 772 HOST_WIDE_INT actual, tree enumtype) 773 { 774 error_at (location, "passing %wd to argument %d of %qE, which expects" 775 " a valid %qT value", actual, argno + 1, fndecl, enumtype); 776 } 777 778 /* Return a hash code for a function_instance. */ 779 hashval_t 780 function_instance::hash () const 781 { 782 inchash::hash h; 783 /* BASE uniquely determines BASE_NAME, so we don't need to hash both. */ 784 h.add_ptr (base); 785 h.add_ptr (shape); 786 h.add_int (mode_suffix_id); 787 h.add_int (type_suffix_ids[0]); 788 h.add_int (type_suffix_ids[1]); 789 h.add_int (pred); 790 return h.end (); 791 } 792 793 /* Return a set of CP_* flags that describe what the function could do, 794 taking the command-line flags into account. */ 795 unsigned int 796 function_instance::call_properties () const 797 { 798 unsigned int flags = base->call_properties (*this); 799 800 /* -fno-trapping-math means that we can assume any FP exceptions 801 are not user-visible. */ 802 if (!flag_trapping_math) 803 flags &= ~CP_RAISE_FP_EXCEPTIONS; 804 805 return flags; 806 } 807 808 /* Return true if calls to the function could read some form of 809 global state. */ 810 bool 811 function_instance::reads_global_state_p () const 812 { 813 unsigned int flags = call_properties (); 814 815 /* Preserve any dependence on rounding mode, flush to zero mode, etc. 816 There is currently no way of turning this off; in particular, 817 -fno-rounding-math (which is the default) means that we should make 818 the usual assumptions about rounding mode, which for intrinsics means 819 acting as the instructions do. */ 820 if (flags & CP_READ_FPCR) 821 return true; 822 823 /* Handle direct reads of global state. */ 824 return flags & (CP_READ_MEMORY | CP_READ_FFR); 825 } 826 827 /* Return true if calls to the function could modify some form of 828 global state. */ 829 bool 830 function_instance::modifies_global_state_p () const 831 { 832 unsigned int flags = call_properties (); 833 834 /* Preserve any exception state written back to the FPCR, 835 unless -fno-trapping-math says this is unnecessary. */ 836 if (flags & CP_RAISE_FP_EXCEPTIONS) 837 return true; 838 839 /* Treat prefetches as modifying global state, since that's the 840 only means we have of keeping them in their correct position. */ 841 if (flags & CP_PREFETCH_MEMORY) 842 return true; 843 844 /* Handle direct modifications of global state. */ 845 return flags & (CP_WRITE_MEMORY | CP_WRITE_FFR); 846 } 847 848 /* Return true if calls to the function could raise a signal. */ 849 bool 850 function_instance::could_trap_p () const 851 { 852 unsigned int flags = call_properties (); 853 854 /* Handle functions that could raise SIGFPE. */ 855 if (flags & CP_RAISE_FP_EXCEPTIONS) 856 return true; 857 858 /* Handle functions that could raise SIGBUS or SIGSEGV. */ 859 if (flags & (CP_READ_MEMORY | CP_WRITE_MEMORY)) 860 return true; 861 862 return false; 863 } 864 865 inline hashval_t 866 registered_function_hasher::hash (value_type value) 867 { 868 return value->instance.hash (); 869 } 870 871 inline bool 872 registered_function_hasher::equal (value_type value, const compare_type &key) 873 { 874 return value->instance == key; 875 } 876 877 sve_switcher::sve_switcher () 878 : m_old_isa_flags (aarch64_isa_flags) 879 { 880 /* Changing the ISA flags and have_regs_of_mode should be enough here. 881 We shouldn't need to pay the compile-time cost of a full target 882 switch. */ 883 aarch64_isa_flags = (AARCH64_FL_FP | AARCH64_FL_SIMD | AARCH64_FL_F16 884 | AARCH64_FL_SVE); 885 886 m_old_general_regs_only = TARGET_GENERAL_REGS_ONLY; 887 global_options.x_target_flags &= ~MASK_GENERAL_REGS_ONLY; 888 889 memcpy (m_old_have_regs_of_mode, have_regs_of_mode, 890 sizeof (have_regs_of_mode)); 891 for (int i = 0; i < NUM_MACHINE_MODES; ++i) 892 if (aarch64_sve_mode_p ((machine_mode) i)) 893 have_regs_of_mode[i] = true; 894 } 895 896 sve_switcher::~sve_switcher () 897 { 898 memcpy (have_regs_of_mode, m_old_have_regs_of_mode, 899 sizeof (have_regs_of_mode)); 900 if (m_old_general_regs_only) 901 global_options.x_target_flags |= MASK_GENERAL_REGS_ONLY; 902 aarch64_isa_flags = m_old_isa_flags; 903 } 904 905 function_builder::function_builder () 906 { 907 m_overload_type = build_function_type (void_type_node, void_list_node); 908 m_direct_overloads = lang_GNU_CXX (); 909 gcc_obstack_init (&m_string_obstack); 910 } 911 912 function_builder::~function_builder () 913 { 914 obstack_free (&m_string_obstack, NULL); 915 } 916 917 /* Add NAME to the end of the function name being built. */ 918 void 919 function_builder::append_name (const char *name) 920 { 921 obstack_grow (&m_string_obstack, name, strlen (name)); 922 } 923 924 /* Zero-terminate and complete the function name being built. */ 925 char * 926 function_builder::finish_name () 927 { 928 obstack_1grow (&m_string_obstack, 0); 929 return (char *) obstack_finish (&m_string_obstack); 930 } 931 932 /* Return the overloaded or full function name for INSTANCE; OVERLOADED_P 933 selects which. Allocate the string on m_string_obstack; the caller 934 must use obstack_free to free it after use. */ 935 char * 936 function_builder::get_name (const function_instance &instance, 937 bool overloaded_p) 938 { 939 append_name (instance.base_name); 940 if (overloaded_p) 941 switch (instance.displacement_units ()) 942 { 943 case UNITS_none: 944 break; 945 946 case UNITS_bytes: 947 append_name ("_offset"); 948 break; 949 950 case UNITS_elements: 951 append_name ("_index"); 952 break; 953 954 case UNITS_vectors: 955 append_name ("_vnum"); 956 break; 957 } 958 else 959 append_name (instance.mode_suffix ().string); 960 for (unsigned int i = 0; i < 2; ++i) 961 if (!overloaded_p || instance.shape->explicit_type_suffix_p (i)) 962 append_name (instance.type_suffix (i).string); 963 append_name (pred_suffixes[instance.pred]); 964 return finish_name (); 965 } 966 967 /* Add attribute NAME to ATTRS. */ 968 static tree 969 add_attribute (const char *name, tree attrs) 970 { 971 return tree_cons (get_identifier (name), NULL_TREE, attrs); 972 } 973 974 /* Return the appropriate function attributes for INSTANCE. */ 975 tree 976 function_builder::get_attributes (const function_instance &instance) 977 { 978 tree attrs = NULL_TREE; 979 980 if (!instance.modifies_global_state_p ()) 981 { 982 if (instance.reads_global_state_p ()) 983 attrs = add_attribute ("pure", attrs); 984 else 985 attrs = add_attribute ("const", attrs); 986 } 987 988 if (!flag_non_call_exceptions || !instance.could_trap_p ()) 989 attrs = add_attribute ("nothrow", attrs); 990 991 return add_attribute ("leaf", attrs); 992 } 993 994 /* Add a function called NAME with type FNTYPE and attributes ATTRS. 995 INSTANCE describes what the function does and OVERLOADED_P indicates 996 whether it is overloaded. REQUIRED_EXTENSIONS are the set of 997 architecture extensions that the function requires. */ 998 registered_function & 999 function_builder::add_function (const function_instance &instance, 1000 const char *name, tree fntype, tree attrs, 1001 uint64_t required_extensions, 1002 bool overloaded_p, 1003 bool placeholder_p) 1004 { 1005 unsigned int code = vec_safe_length (registered_functions); 1006 code = (code << AARCH64_BUILTIN_SHIFT) | AARCH64_BUILTIN_SVE; 1007 1008 /* We need to be able to generate placeholders to enusre that we have a 1009 consistent numbering scheme for function codes between the C and C++ 1010 frontends, so that everything ties up in LTO. 1011 1012 Currently, tree-streamer-in.c:unpack_ts_function_decl_value_fields 1013 validates that tree nodes returned by TARGET_BUILTIN_DECL are non-NULL and 1014 some node other than error_mark_node. This is a holdover from when builtin 1015 decls were streamed by code rather than by value. 1016 1017 Ultimately, we should be able to remove this validation of BUILT_IN_MD 1018 nodes and remove the target hook. For now, however, we need to appease the 1019 validation and return a non-NULL, non-error_mark_node node, so we 1020 arbitrarily choose integer_zero_node. */ 1021 tree decl = placeholder_p 1022 ? integer_zero_node 1023 : simulate_builtin_function_decl (input_location, name, fntype, 1024 code, NULL, attrs); 1025 1026 registered_function &rfn = *ggc_alloc <registered_function> (); 1027 rfn.instance = instance; 1028 rfn.decl = decl; 1029 rfn.required_extensions = required_extensions; 1030 rfn.overloaded_p = overloaded_p; 1031 vec_safe_push (registered_functions, &rfn); 1032 1033 return rfn; 1034 } 1035 1036 /* Add a built-in function for INSTANCE, with the argument types given 1037 by ARGUMENT_TYPES and the return type given by RETURN_TYPE. 1038 REQUIRED_EXTENSIONS are the set of architecture extensions that the 1039 function requires. FORCE_DIRECT_OVERLOADS is true if there is a 1040 one-to-one mapping between "short" and "full" names, and if standard 1041 overload resolution therefore isn't necessary. */ 1042 void 1043 function_builder::add_unique_function (const function_instance &instance, 1044 tree return_type, 1045 vec<tree> &argument_types, 1046 uint64_t required_extensions, 1047 bool force_direct_overloads) 1048 { 1049 /* Add the function under its full (unique) name. */ 1050 char *name = get_name (instance, false); 1051 tree fntype = build_function_type_array (return_type, 1052 argument_types.length (), 1053 argument_types.address ()); 1054 tree attrs = get_attributes (instance); 1055 registered_function &rfn = add_function (instance, name, fntype, attrs, 1056 required_extensions, false, false); 1057 1058 /* Enter the function into the hash table. */ 1059 hashval_t hash = instance.hash (); 1060 registered_function **rfn_slot 1061 = function_table->find_slot_with_hash (instance, hash, INSERT); 1062 gcc_assert (!*rfn_slot); 1063 *rfn_slot = &rfn; 1064 1065 /* Also add the function under its overloaded alias, if we want 1066 a separate decl for each instance of an overloaded function. */ 1067 char *overload_name = get_name (instance, true); 1068 if (strcmp (name, overload_name) != 0) 1069 { 1070 /* Attribute lists shouldn't be shared. */ 1071 tree attrs = get_attributes (instance); 1072 bool placeholder_p = !(m_direct_overloads || force_direct_overloads); 1073 add_function (instance, overload_name, fntype, attrs, 1074 required_extensions, false, placeholder_p); 1075 } 1076 1077 obstack_free (&m_string_obstack, name); 1078 } 1079 1080 /* Add one function decl for INSTANCE, to be used with manual overload 1081 resolution. REQUIRED_EXTENSIONS are the set of architecture extensions 1082 that the function requires. 1083 1084 For simplicity, deal with duplicate attempts to add the same function, 1085 including cases in which the new function requires more features than 1086 the original one did. In that case we'll check whether the required 1087 features are available as part of resolving the function to the 1088 relevant unique function. */ 1089 void 1090 function_builder::add_overloaded_function (const function_instance &instance, 1091 uint64_t required_extensions) 1092 { 1093 char *name = get_name (instance, true); 1094 if (registered_function **map_value = m_overload_names.get (name)) 1095 { 1096 gcc_assert ((*map_value)->instance == instance 1097 && ((*map_value)->required_extensions 1098 & ~required_extensions) == 0); 1099 obstack_free (&m_string_obstack, name); 1100 } 1101 else 1102 { 1103 registered_function &rfn 1104 = add_function (instance, name, m_overload_type, NULL_TREE, 1105 required_extensions, true, m_direct_overloads); 1106 m_overload_names.put (name, &rfn); 1107 } 1108 } 1109 1110 /* If we are using manual overload resolution, add one function decl 1111 for each overloaded function in GROUP. Take the function base name 1112 from GROUP and the mode from MODE. */ 1113 void 1114 function_builder::add_overloaded_functions (const function_group_info &group, 1115 mode_suffix_index mode) 1116 { 1117 unsigned int explicit_type0 = (*group.shape)->explicit_type_suffix_p (0); 1118 unsigned int explicit_type1 = (*group.shape)->explicit_type_suffix_p (1); 1119 for (unsigned int pi = 0; group.preds[pi] != NUM_PREDS; ++pi) 1120 { 1121 if (!explicit_type0 && !explicit_type1) 1122 { 1123 /* Deal with the common case in which there is one overloaded 1124 function for all type combinations. */ 1125 function_instance instance (group.base_name, *group.base, 1126 *group.shape, mode, types_none[0], 1127 group.preds[pi]); 1128 add_overloaded_function (instance, group.required_extensions); 1129 } 1130 else 1131 for (unsigned int ti = 0; group.types[ti][0] != NUM_TYPE_SUFFIXES; 1132 ++ti) 1133 { 1134 /* Stub out the types that are determined by overload 1135 resolution. */ 1136 type_suffix_pair types = { 1137 explicit_type0 ? group.types[ti][0] : NUM_TYPE_SUFFIXES, 1138 explicit_type1 ? group.types[ti][1] : NUM_TYPE_SUFFIXES 1139 }; 1140 function_instance instance (group.base_name, *group.base, 1141 *group.shape, mode, types, 1142 group.preds[pi]); 1143 add_overloaded_function (instance, group.required_extensions); 1144 } 1145 } 1146 } 1147 1148 /* Register all the functions in GROUP. */ 1149 void 1150 function_builder::register_function_group (const function_group_info &group) 1151 { 1152 (*group.shape)->build (*this, group); 1153 } 1154 1155 function_call_info::function_call_info (location_t location_in, 1156 const function_instance &instance_in, 1157 tree fndecl_in) 1158 : function_instance (instance_in), location (location_in), fndecl (fndecl_in) 1159 { 1160 } 1161 1162 function_resolver::function_resolver (location_t location, 1163 const function_instance &instance, 1164 tree fndecl, vec<tree, va_gc> &arglist) 1165 : function_call_info (location, instance, fndecl), m_arglist (arglist) 1166 { 1167 } 1168 1169 /* Return the vector type associated with type suffix TYPE. */ 1170 tree 1171 function_resolver::get_vector_type (type_suffix_index type) 1172 { 1173 return acle_vector_types[0][type_suffixes[type].vector_type]; 1174 } 1175 1176 /* Return the <stdint.h> name associated with TYPE. Using the <stdint.h> 1177 name should be more user-friendly than the underlying canonical type, 1178 since it makes the signedness and bitwidth explicit. */ 1179 const char * 1180 function_resolver::get_scalar_type_name (type_suffix_index type) 1181 { 1182 return vector_types[type_suffixes[type].vector_type].acle_name + 2; 1183 } 1184 1185 /* Return the type of argument I, or error_mark_node if it isn't 1186 well-formed. */ 1187 tree 1188 function_resolver::get_argument_type (unsigned int i) 1189 { 1190 tree arg = m_arglist[i]; 1191 return arg == error_mark_node ? arg : TREE_TYPE (arg); 1192 } 1193 1194 /* Return true if argument I is some form of scalar value. */ 1195 bool 1196 function_resolver::scalar_argument_p (unsigned int i) 1197 { 1198 tree type = get_argument_type (i); 1199 return (INTEGRAL_TYPE_P (type) 1200 /* Allow pointer types, leaving the frontend to warn where 1201 necessary. */ 1202 || POINTER_TYPE_P (type) 1203 || SCALAR_FLOAT_TYPE_P (type)); 1204 } 1205 1206 /* Report that the function has no form that takes type suffix TYPE. 1207 Return error_mark_node. */ 1208 tree 1209 function_resolver::report_no_such_form (type_suffix_index type) 1210 { 1211 error_at (location, "%qE has no form that takes %qT arguments", 1212 fndecl, get_vector_type (type)); 1213 return error_mark_node; 1214 } 1215 1216 /* Silently check whether there is an instance of the function with the 1217 mode suffix given by MODE and the type suffixes given by TYPE0 and TYPE1. 1218 Return its function decl if so, otherwise return null. */ 1219 tree 1220 function_resolver::lookup_form (mode_suffix_index mode, 1221 type_suffix_index type0, 1222 type_suffix_index type1) 1223 { 1224 type_suffix_pair types = { type0, type1 }; 1225 function_instance instance (base_name, base, shape, mode, types, pred); 1226 registered_function *rfn 1227 = function_table->find_with_hash (instance, instance.hash ()); 1228 return rfn ? rfn->decl : NULL_TREE; 1229 } 1230 1231 /* Resolve the function to one with the mode suffix given by MODE and the 1232 type suffixes given by TYPE0 and TYPE1. Return its function decl on 1233 success, otherwise report an error and return error_mark_node. */ 1234 tree 1235 function_resolver::resolve_to (mode_suffix_index mode, 1236 type_suffix_index type0, 1237 type_suffix_index type1) 1238 { 1239 tree res = lookup_form (mode, type0, type1); 1240 if (!res) 1241 { 1242 if (type1 == NUM_TYPE_SUFFIXES) 1243 return report_no_such_form (type0); 1244 if (type0 == type_suffix_ids[0]) 1245 return report_no_such_form (type1); 1246 /* To be filled in when we have other cases. */ 1247 gcc_unreachable (); 1248 } 1249 return res; 1250 } 1251 1252 /* Require argument ARGNO to be a 32-bit or 64-bit scalar integer type. 1253 Return the associated type suffix on success, otherwise report an 1254 error and return NUM_TYPE_SUFFIXES. */ 1255 type_suffix_index 1256 function_resolver::infer_integer_scalar_type (unsigned int argno) 1257 { 1258 tree actual = get_argument_type (argno); 1259 if (actual == error_mark_node) 1260 return NUM_TYPE_SUFFIXES; 1261 1262 /* Allow enums and booleans to decay to integers, for compatibility 1263 with C++ overloading rules. */ 1264 if (INTEGRAL_TYPE_P (actual)) 1265 { 1266 bool uns_p = TYPE_UNSIGNED (actual); 1267 /* Honor the usual integer promotions, so that resolution works 1268 in the same way as for C++. */ 1269 if (TYPE_PRECISION (actual) < 32) 1270 return TYPE_SUFFIX_s32; 1271 if (TYPE_PRECISION (actual) == 32) 1272 return uns_p ? TYPE_SUFFIX_u32 : TYPE_SUFFIX_s32; 1273 if (TYPE_PRECISION (actual) == 64) 1274 return uns_p ? TYPE_SUFFIX_u64 : TYPE_SUFFIX_s64; 1275 } 1276 1277 error_at (location, "passing %qT to argument %d of %qE, which expects" 1278 " a 32-bit or 64-bit integer type", actual, argno + 1, fndecl); 1279 return NUM_TYPE_SUFFIXES; 1280 } 1281 1282 /* Require argument ARGNO to be a pointer to a scalar type that has a 1283 corresponding type suffix. Return that type suffix on success, 1284 otherwise report an error and return NUM_TYPE_SUFFIXES. 1285 GATHER_SCATTER_P is true if the function is a gather/scatter 1286 operation, and so requires a pointer to 32-bit or 64-bit data. */ 1287 type_suffix_index 1288 function_resolver::infer_pointer_type (unsigned int argno, 1289 bool gather_scatter_p) 1290 { 1291 tree actual = get_argument_type (argno); 1292 if (actual == error_mark_node) 1293 return NUM_TYPE_SUFFIXES; 1294 1295 if (TREE_CODE (actual) != POINTER_TYPE) 1296 { 1297 error_at (location, "passing %qT to argument %d of %qE, which" 1298 " expects a pointer type", actual, argno + 1, fndecl); 1299 if (VECTOR_TYPE_P (actual) && gather_scatter_p) 1300 inform (location, "an explicit type suffix is needed" 1301 " when using a vector of base addresses"); 1302 return NUM_TYPE_SUFFIXES; 1303 } 1304 1305 tree target = TREE_TYPE (actual); 1306 type_suffix_index type = find_type_suffix_for_scalar_type (target); 1307 if (type == NUM_TYPE_SUFFIXES) 1308 { 1309 error_at (location, "passing %qT to argument %d of %qE, but %qT is not" 1310 " a valid SVE element type", actual, argno + 1, fndecl, 1311 build_qualified_type (target, 0)); 1312 return NUM_TYPE_SUFFIXES; 1313 } 1314 unsigned int bits = type_suffixes[type].element_bits; 1315 if (gather_scatter_p && bits != 32 && bits != 64) 1316 { 1317 error_at (location, "passing %qT to argument %d of %qE, which" 1318 " expects a pointer to 32-bit or 64-bit elements", 1319 actual, argno + 1, fndecl); 1320 return NUM_TYPE_SUFFIXES; 1321 } 1322 1323 return type; 1324 } 1325 1326 /* Require argument ARGNO to be a single vector or a tuple of NUM_VECTORS 1327 vectors; NUM_VECTORS is 1 for the former. Return the associated type 1328 suffix on success, using TYPE_SUFFIX_b for predicates. Report an error 1329 and return NUM_TYPE_SUFFIXES on failure. */ 1330 type_suffix_index 1331 function_resolver::infer_vector_or_tuple_type (unsigned int argno, 1332 unsigned int num_vectors) 1333 { 1334 tree actual = get_argument_type (argno); 1335 if (actual == error_mark_node) 1336 return NUM_TYPE_SUFFIXES; 1337 1338 /* A linear search should be OK here, since the code isn't hot and 1339 the number of types is only small. */ 1340 for (unsigned int size_i = 0; size_i < MAX_TUPLE_SIZE; ++size_i) 1341 for (unsigned int suffix_i = 0; suffix_i < NUM_TYPE_SUFFIXES; ++suffix_i) 1342 { 1343 vector_type_index type_i = type_suffixes[suffix_i].vector_type; 1344 tree type = acle_vector_types[size_i][type_i]; 1345 if (type && matches_type_p (type, actual)) 1346 { 1347 if (size_i + 1 == num_vectors) 1348 return type_suffix_index (suffix_i); 1349 1350 if (num_vectors == 1) 1351 error_at (location, "passing %qT to argument %d of %qE, which" 1352 " expects a single SVE vector rather than a tuple", 1353 actual, argno + 1, fndecl); 1354 else if (size_i == 0 && type_i != VECTOR_TYPE_svbool_t) 1355 error_at (location, "passing single vector %qT to argument %d" 1356 " of %qE, which expects a tuple of %d vectors", 1357 actual, argno + 1, fndecl, num_vectors); 1358 else 1359 error_at (location, "passing %qT to argument %d of %qE, which" 1360 " expects a tuple of %d vectors", actual, argno + 1, 1361 fndecl, num_vectors); 1362 return NUM_TYPE_SUFFIXES; 1363 } 1364 } 1365 1366 if (num_vectors == 1) 1367 error_at (location, "passing %qT to argument %d of %qE, which" 1368 " expects an SVE vector type", actual, argno + 1, fndecl); 1369 else 1370 error_at (location, "passing %qT to argument %d of %qE, which" 1371 " expects an SVE tuple type", actual, argno + 1, fndecl); 1372 return NUM_TYPE_SUFFIXES; 1373 } 1374 1375 /* Require argument ARGNO to have some form of vector type. Return the 1376 associated type suffix on success, using TYPE_SUFFIX_b for predicates. 1377 Report an error and return NUM_TYPE_SUFFIXES on failure. */ 1378 type_suffix_index 1379 function_resolver::infer_vector_type (unsigned int argno) 1380 { 1381 return infer_vector_or_tuple_type (argno, 1); 1382 } 1383 1384 /* Like infer_vector_type, but also require the type to be integral. */ 1385 type_suffix_index 1386 function_resolver::infer_integer_vector_type (unsigned int argno) 1387 { 1388 type_suffix_index type = infer_vector_type (argno); 1389 if (type == NUM_TYPE_SUFFIXES) 1390 return type; 1391 1392 if (!type_suffixes[type].integer_p) 1393 { 1394 error_at (location, "passing %qT to argument %d of %qE, which" 1395 " expects a vector of integers", get_argument_type (argno), 1396 argno + 1, fndecl); 1397 return NUM_TYPE_SUFFIXES; 1398 } 1399 1400 return type; 1401 } 1402 1403 /* Like infer_vector_type, but also require the type to be an unsigned 1404 integer. */ 1405 type_suffix_index 1406 function_resolver::infer_unsigned_vector_type (unsigned int argno) 1407 { 1408 type_suffix_index type = infer_vector_type (argno); 1409 if (type == NUM_TYPE_SUFFIXES) 1410 return type; 1411 1412 if (!type_suffixes[type].unsigned_p) 1413 { 1414 error_at (location, "passing %qT to argument %d of %qE, which" 1415 " expects a vector of unsigned integers", 1416 get_argument_type (argno), argno + 1, fndecl); 1417 return NUM_TYPE_SUFFIXES; 1418 } 1419 1420 return type; 1421 } 1422 1423 /* Like infer_vector_type, but also require the element size to be 1424 32 or 64 bits. */ 1425 type_suffix_index 1426 function_resolver::infer_sd_vector_type (unsigned int argno) 1427 { 1428 type_suffix_index type = infer_vector_type (argno); 1429 if (type == NUM_TYPE_SUFFIXES) 1430 return type; 1431 1432 unsigned int bits = type_suffixes[type].element_bits; 1433 if (bits != 32 && bits != 64) 1434 { 1435 error_at (location, "passing %qT to argument %d of %qE, which" 1436 " expects a vector of 32-bit or 64-bit elements", 1437 get_argument_type (argno), argno + 1, fndecl); 1438 return NUM_TYPE_SUFFIXES; 1439 } 1440 1441 return type; 1442 } 1443 1444 /* If the function operates on tuples of vectors, require argument ARGNO to be 1445 a tuple with the appropriate number of vectors, otherwise require it to be 1446 a single vector. Return the associated type suffix on success, using 1447 TYPE_SUFFIX_b for predicates. Report an error and return NUM_TYPE_SUFFIXES 1448 on failure. */ 1449 type_suffix_index 1450 function_resolver::infer_tuple_type (unsigned int argno) 1451 { 1452 return infer_vector_or_tuple_type (argno, vectors_per_tuple ()); 1453 } 1454 1455 /* Require argument ARGNO to be a vector or scalar argument. Return true 1456 if it is, otherwise report an appropriate error. */ 1457 bool 1458 function_resolver::require_vector_or_scalar_type (unsigned int argno) 1459 { 1460 tree actual = get_argument_type (argno); 1461 if (actual == error_mark_node) 1462 return false; 1463 1464 if (!scalar_argument_p (argno) && !VECTOR_TYPE_P (actual)) 1465 { 1466 error_at (location, "passing %qT to argument %d of %qE, which" 1467 " expects a vector or scalar type", actual, argno + 1, fndecl); 1468 return false; 1469 } 1470 1471 return true; 1472 } 1473 1474 /* Require argument ARGNO to have vector type TYPE, in cases where this 1475 requirement holds for all uses of the function. Return true if the 1476 argument has the right form, otherwise report an appropriate error. */ 1477 bool 1478 function_resolver::require_vector_type (unsigned int argno, 1479 vector_type_index type) 1480 { 1481 tree expected = acle_vector_types[0][type]; 1482 tree actual = get_argument_type (argno); 1483 if (actual == error_mark_node) 1484 return false; 1485 1486 if (!matches_type_p (expected, actual)) 1487 { 1488 error_at (location, "passing %qT to argument %d of %qE, which" 1489 " expects %qT", actual, argno + 1, fndecl, expected); 1490 return false; 1491 } 1492 return true; 1493 } 1494 1495 /* Like require_vector_type, but TYPE is inferred from previous arguments 1496 rather than being a fixed part of the function signature. This changes 1497 the nature of the error messages. */ 1498 bool 1499 function_resolver::require_matching_vector_type (unsigned int argno, 1500 type_suffix_index type) 1501 { 1502 type_suffix_index new_type = infer_vector_type (argno); 1503 if (new_type == NUM_TYPE_SUFFIXES) 1504 return false; 1505 1506 if (type != new_type) 1507 { 1508 error_at (location, "passing %qT to argument %d of %qE, but" 1509 " previous arguments had type %qT", 1510 get_vector_type (new_type), argno + 1, fndecl, 1511 get_vector_type (type)); 1512 return false; 1513 } 1514 return true; 1515 } 1516 1517 /* Require argument ARGNO to be a vector type with the following properties: 1518 1519 - the type class must be the same as FIRST_TYPE's if EXPECTED_TCLASS 1520 is SAME_TYPE_CLASS, otherwise it must be EXPECTED_TCLASS itself. 1521 1522 - the element size must be: 1523 1524 - the same as FIRST_TYPE's if EXPECTED_BITS == SAME_SIZE 1525 - half of FIRST_TYPE's if EXPECTED_BITS == HALF_SIZE 1526 - a quarter of FIRST_TYPE's if EXPECTED_BITS == QUARTER_SIZE 1527 - EXPECTED_BITS itself otherwise 1528 1529 Return true if the argument has the required type, otherwise report 1530 an appropriate error. 1531 1532 FIRST_ARGNO is the first argument that is known to have type FIRST_TYPE. 1533 Usually it comes before ARGNO, but sometimes it is more natural to resolve 1534 arguments out of order. 1535 1536 If the required properties depend on FIRST_TYPE then both FIRST_ARGNO and 1537 ARGNO contribute to the resolution process. If the required properties 1538 are fixed, only FIRST_ARGNO contributes to the resolution process. 1539 1540 This function is a bit of a Swiss army knife. The complication comes 1541 from trying to give good error messages when FIRST_ARGNO and ARGNO are 1542 inconsistent, since either of them might be wrong. */ 1543 bool function_resolver:: 1544 require_derived_vector_type (unsigned int argno, 1545 unsigned int first_argno, 1546 type_suffix_index first_type, 1547 type_class_index expected_tclass, 1548 unsigned int expected_bits) 1549 { 1550 /* If the type needs to match FIRST_ARGNO exactly, use the preferred 1551 error message for that case. The VECTOR_TYPE_P test excludes tuple 1552 types, which we handle below instead. */ 1553 bool both_vectors_p = VECTOR_TYPE_P (get_argument_type (first_argno)); 1554 if (both_vectors_p 1555 && expected_tclass == SAME_TYPE_CLASS 1556 && expected_bits == SAME_SIZE) 1557 { 1558 /* There's no need to resolve this case out of order. */ 1559 gcc_assert (argno > first_argno); 1560 return require_matching_vector_type (argno, first_type); 1561 } 1562 1563 /* Use FIRST_TYPE to get the expected type class and element size. */ 1564 type_class_index orig_expected_tclass = expected_tclass; 1565 if (expected_tclass == NUM_TYPE_CLASSES) 1566 expected_tclass = type_suffixes[first_type].tclass; 1567 1568 unsigned int orig_expected_bits = expected_bits; 1569 if (expected_bits == SAME_SIZE) 1570 expected_bits = type_suffixes[first_type].element_bits; 1571 else if (expected_bits == HALF_SIZE) 1572 expected_bits = type_suffixes[first_type].element_bits / 2; 1573 else if (expected_bits == QUARTER_SIZE) 1574 expected_bits = type_suffixes[first_type].element_bits / 4; 1575 1576 /* If the expected type doesn't depend on FIRST_TYPE at all, 1577 just check for the fixed choice of vector type. */ 1578 if (expected_tclass == orig_expected_tclass 1579 && expected_bits == orig_expected_bits) 1580 { 1581 const type_suffix_info &expected_suffix 1582 = type_suffixes[find_type_suffix (expected_tclass, expected_bits)]; 1583 return require_vector_type (argno, expected_suffix.vector_type); 1584 } 1585 1586 /* Require the argument to be some form of SVE vector type, 1587 without being specific about the type of vector we want. */ 1588 type_suffix_index actual_type = infer_vector_type (argno); 1589 if (actual_type == NUM_TYPE_SUFFIXES) 1590 return false; 1591 1592 /* Exit now if we got the right type. */ 1593 bool tclass_ok_p = (type_suffixes[actual_type].tclass == expected_tclass); 1594 bool size_ok_p = (type_suffixes[actual_type].element_bits == expected_bits); 1595 if (tclass_ok_p && size_ok_p) 1596 return true; 1597 1598 /* First look for cases in which the actual type contravenes a fixed 1599 size requirement, without having to refer to FIRST_TYPE. */ 1600 if (!size_ok_p && expected_bits == orig_expected_bits) 1601 { 1602 error_at (location, "passing %qT to argument %d of %qE, which" 1603 " expects a vector of %d-bit elements", 1604 get_vector_type (actual_type), argno + 1, fndecl, 1605 expected_bits); 1606 return false; 1607 } 1608 1609 /* Likewise for a fixed type class requirement. This is only ever 1610 needed for signed and unsigned types, so don't create unnecessary 1611 translation work for other type classes. */ 1612 if (!tclass_ok_p && orig_expected_tclass == TYPE_signed) 1613 { 1614 error_at (location, "passing %qT to argument %d of %qE, which" 1615 " expects a vector of signed integers", 1616 get_vector_type (actual_type), argno + 1, fndecl); 1617 return false; 1618 } 1619 if (!tclass_ok_p && orig_expected_tclass == TYPE_unsigned) 1620 { 1621 error_at (location, "passing %qT to argument %d of %qE, which" 1622 " expects a vector of unsigned integers", 1623 get_vector_type (actual_type), argno + 1, fndecl); 1624 return false; 1625 } 1626 1627 /* Make sure that FIRST_TYPE itself is sensible before using it 1628 as a basis for an error message. */ 1629 if (resolve_to (mode_suffix_id, first_type) == error_mark_node) 1630 return false; 1631 1632 /* If the arguments have consistent type classes, but a link between 1633 the sizes has been broken, try to describe the error in those terms. */ 1634 if (both_vectors_p && tclass_ok_p && orig_expected_bits == SAME_SIZE) 1635 { 1636 if (argno < first_argno) 1637 { 1638 std::swap (argno, first_argno); 1639 std::swap (actual_type, first_type); 1640 } 1641 error_at (location, "arguments %d and %d of %qE must have the" 1642 " same element size, but the values passed here have type" 1643 " %qT and %qT respectively", first_argno + 1, argno + 1, 1644 fndecl, get_vector_type (first_type), 1645 get_vector_type (actual_type)); 1646 return false; 1647 } 1648 1649 /* Likewise in reverse: look for cases in which the sizes are consistent 1650 but a link between the type classes has been broken. */ 1651 if (both_vectors_p 1652 && size_ok_p 1653 && orig_expected_tclass == SAME_TYPE_CLASS 1654 && type_suffixes[first_type].integer_p 1655 && type_suffixes[actual_type].integer_p) 1656 { 1657 if (argno < first_argno) 1658 { 1659 std::swap (argno, first_argno); 1660 std::swap (actual_type, first_type); 1661 } 1662 error_at (location, "arguments %d and %d of %qE must have the" 1663 " same signedness, but the values passed here have type" 1664 " %qT and %qT respectively", first_argno + 1, argno + 1, 1665 fndecl, get_vector_type (first_type), 1666 get_vector_type (actual_type)); 1667 return false; 1668 } 1669 1670 /* The two arguments are wildly inconsistent. */ 1671 type_suffix_index expected_type 1672 = find_type_suffix (expected_tclass, expected_bits); 1673 error_at (location, "passing %qT instead of the expected %qT to argument" 1674 " %d of %qE, after passing %qT to argument %d", 1675 get_vector_type (actual_type), get_vector_type (expected_type), 1676 argno + 1, fndecl, get_argument_type (first_argno), 1677 first_argno + 1); 1678 return false; 1679 } 1680 1681 /* Require argument ARGNO to match argument FIRST_ARGNO, which was inferred 1682 to be a pointer to a scalar element of type TYPE. */ 1683 bool 1684 function_resolver::require_matching_pointer_type (unsigned int argno, 1685 unsigned int first_argno, 1686 type_suffix_index type) 1687 { 1688 type_suffix_index new_type = infer_pointer_type (argno); 1689 if (new_type == NUM_TYPE_SUFFIXES) 1690 return false; 1691 1692 if (type != new_type) 1693 { 1694 error_at (location, "passing %qT to argument %d of %qE, but" 1695 " argument %d had type %qT", get_argument_type (argno), 1696 argno + 1, fndecl, first_argno + 1, 1697 get_argument_type (first_argno)); 1698 return false; 1699 } 1700 return true; 1701 } 1702 1703 /* Require argument ARGNO to be a (possibly variable) scalar, using EXPECTED 1704 as the name of its expected type. Return true if the argument has the 1705 right form, otherwise report an appropriate error. */ 1706 bool 1707 function_resolver::require_scalar_type (unsigned int argno, 1708 const char *expected) 1709 { 1710 if (!scalar_argument_p (argno)) 1711 { 1712 error_at (location, "passing %qT to argument %d of %qE, which" 1713 " expects %qs", get_argument_type (argno), argno + 1, 1714 fndecl, expected); 1715 return false; 1716 } 1717 return true; 1718 } 1719 1720 /* Require argument ARGNO to be some form of pointer, without being specific 1721 about its target type. Return true if the argument has the right form, 1722 otherwise report an appropriate error. */ 1723 bool 1724 function_resolver::require_pointer_type (unsigned int argno) 1725 { 1726 if (!scalar_argument_p (argno)) 1727 { 1728 error_at (location, "passing %qT to argument %d of %qE, which" 1729 " expects a scalar pointer", get_argument_type (argno), 1730 argno + 1, fndecl); 1731 return false; 1732 } 1733 return true; 1734 } 1735 1736 /* Argument FIRST_ARGNO is a scalar with type EXPECTED_TYPE, and argument 1737 ARGNO should be consistent with it. Return true if it is, otherwise 1738 report an appropriate error. */ 1739 bool function_resolver:: 1740 require_matching_integer_scalar_type (unsigned int argno, 1741 unsigned int first_argno, 1742 type_suffix_index expected_type) 1743 { 1744 type_suffix_index actual_type = infer_integer_scalar_type (argno); 1745 if (actual_type == NUM_TYPE_SUFFIXES) 1746 return false; 1747 1748 if (actual_type == expected_type) 1749 return true; 1750 1751 error_at (location, "call to %qE is ambiguous; argument %d has type" 1752 " %qs but argument %d has type %qs", fndecl, 1753 first_argno + 1, get_scalar_type_name (expected_type), 1754 argno + 1, get_scalar_type_name (actual_type)); 1755 return false; 1756 } 1757 1758 /* Require argument ARGNO to be a (possibly variable) scalar, expecting it 1759 to have the following properties: 1760 1761 - the type class must be the same as for type suffix 0 if EXPECTED_TCLASS 1762 is SAME_TYPE_CLASS, otherwise it must be EXPECTED_TCLASS itself. 1763 1764 - the element size must be the same as for type suffix 0 if EXPECTED_BITS 1765 is SAME_TYPE_SIZE, otherwise it must be EXPECTED_BITS itself. 1766 1767 Return true if the argument is valid, otherwise report an appropriate error. 1768 1769 Note that we don't check whether the scalar type actually has the required 1770 properties, since that's subject to implicit promotions and conversions. 1771 Instead we just use the expected properties to tune the error message. */ 1772 bool function_resolver:: 1773 require_derived_scalar_type (unsigned int argno, 1774 type_class_index expected_tclass, 1775 unsigned int expected_bits) 1776 { 1777 gcc_assert (expected_tclass == SAME_TYPE_CLASS 1778 || expected_tclass == TYPE_signed 1779 || expected_tclass == TYPE_unsigned); 1780 1781 /* If the expected type doesn't depend on the type suffix at all, 1782 just check for the fixed choice of scalar type. */ 1783 if (expected_tclass != SAME_TYPE_CLASS && expected_bits != SAME_SIZE) 1784 { 1785 type_suffix_index expected_type 1786 = find_type_suffix (expected_tclass, expected_bits); 1787 return require_scalar_type (argno, get_scalar_type_name (expected_type)); 1788 } 1789 1790 if (scalar_argument_p (argno)) 1791 return true; 1792 1793 if (expected_tclass == SAME_TYPE_CLASS) 1794 /* It doesn't really matter whether the element is expected to be 1795 the same size as type suffix 0. */ 1796 error_at (location, "passing %qT to argument %d of %qE, which" 1797 " expects a scalar element", get_argument_type (argno), 1798 argno + 1, fndecl); 1799 else 1800 /* It doesn't seem useful to distinguish between signed and unsigned 1801 scalars here. */ 1802 error_at (location, "passing %qT to argument %d of %qE, which" 1803 " expects a scalar integer", get_argument_type (argno), 1804 argno + 1, fndecl); 1805 return false; 1806 } 1807 1808 /* Require argument ARGNO to be suitable for an integer constant expression. 1809 Return true if it is, otherwise report an appropriate error. 1810 1811 function_checker checks whether the argument is actually constant and 1812 has a suitable range. The reason for distinguishing immediate arguments 1813 here is because it provides more consistent error messages than 1814 require_scalar_type would. */ 1815 bool 1816 function_resolver::require_integer_immediate (unsigned int argno) 1817 { 1818 if (!scalar_argument_p (argno)) 1819 { 1820 report_non_ice (location, fndecl, argno); 1821 return false; 1822 } 1823 return true; 1824 } 1825 1826 /* Require argument ARGNO to be a vector base in a gather-style address. 1827 Return its type on success, otherwise return NUM_VECTOR_TYPES. */ 1828 vector_type_index 1829 function_resolver::infer_vector_base_type (unsigned int argno) 1830 { 1831 type_suffix_index type = infer_vector_type (argno); 1832 if (type == NUM_TYPE_SUFFIXES) 1833 return NUM_VECTOR_TYPES; 1834 1835 if (type == TYPE_SUFFIX_u32 || type == TYPE_SUFFIX_u64) 1836 return type_suffixes[type].vector_type; 1837 1838 error_at (location, "passing %qT to argument %d of %qE, which" 1839 " expects %qs or %qs", get_argument_type (argno), 1840 argno + 1, fndecl, "svuint32_t", "svuint64_t"); 1841 return NUM_VECTOR_TYPES; 1842 } 1843 1844 /* Require argument ARGNO to be a vector displacement in a gather-style 1845 address. Return its type on success, otherwise return NUM_VECTOR_TYPES. */ 1846 vector_type_index 1847 function_resolver::infer_vector_displacement_type (unsigned int argno) 1848 { 1849 type_suffix_index type = infer_integer_vector_type (argno); 1850 if (type == NUM_TYPE_SUFFIXES) 1851 return NUM_VECTOR_TYPES; 1852 1853 if (type_suffixes[type].integer_p 1854 && (type_suffixes[type].element_bits == 32 1855 || type_suffixes[type].element_bits == 64)) 1856 return type_suffixes[type].vector_type; 1857 1858 error_at (location, "passing %qT to argument %d of %qE, which" 1859 " expects a vector of 32-bit or 64-bit integers", 1860 get_argument_type (argno), argno + 1, fndecl); 1861 return NUM_VECTOR_TYPES; 1862 } 1863 1864 /* Require argument ARGNO to be a vector displacement in a gather-style 1865 address. There are three possible uses: 1866 1867 - for loading into elements of type TYPE (when LOAD_P is true) 1868 - for storing from elements of type TYPE (when LOAD_P is false) 1869 - for prefetching data (when TYPE is NUM_TYPE_SUFFIXES) 1870 1871 The overloaded function's mode suffix determines the units of the 1872 displacement (bytes for "_offset", elements for "_index"). 1873 1874 Return the associated mode on success, otherwise report an error 1875 and return MODE_none. */ 1876 mode_suffix_index 1877 function_resolver::resolve_sv_displacement (unsigned int argno, 1878 type_suffix_index type, 1879 bool load_p) 1880 { 1881 if (type == NUM_TYPE_SUFFIXES) 1882 { 1883 /* For prefetches, the base is a void pointer and the displacement 1884 can be any valid offset or index type. */ 1885 vector_type_index displacement_vector_type 1886 = infer_vector_displacement_type (argno); 1887 if (displacement_vector_type == NUM_VECTOR_TYPES) 1888 return MODE_none; 1889 1890 mode_suffix_index mode = find_mode_suffix (NUM_VECTOR_TYPES, 1891 displacement_vector_type, 1892 displacement_units ()); 1893 gcc_assert (mode != MODE_none); 1894 return mode; 1895 } 1896 1897 unsigned int required_bits = type_suffixes[type].element_bits; 1898 if (required_bits == 32 1899 && displacement_units () == UNITS_elements 1900 && !lookup_form (MODE_s32index, type) 1901 && !lookup_form (MODE_u32index, type)) 1902 { 1903 if (lookup_form (MODE_u32base_index, type)) 1904 { 1905 if (type_suffix_ids[0] == NUM_TYPE_SUFFIXES) 1906 { 1907 gcc_assert (!load_p); 1908 error_at (location, "when storing %qT, %qE requires a vector" 1909 " base and a scalar index", get_vector_type (type), 1910 fndecl); 1911 } 1912 else 1913 error_at (location, "%qE requires a vector base and a scalar" 1914 " index", fndecl); 1915 } 1916 else 1917 error_at (location, "%qE does not support 32-bit vector type %qT", 1918 fndecl, get_vector_type (type)); 1919 return MODE_none; 1920 } 1921 1922 /* Check for some form of vector type, without naming any in particular 1923 as being expected. */ 1924 type_suffix_index displacement_type = infer_vector_type (argno); 1925 if (displacement_type == NUM_TYPE_SUFFIXES) 1926 return MODE_none; 1927 1928 /* If the displacement type is consistent with the data vector type, 1929 try to find the associated mode suffix. This will fall through 1930 for non-integral displacement types. */ 1931 if (type_suffixes[displacement_type].element_bits == required_bits) 1932 { 1933 vector_type_index displacement_vector_type 1934 = type_suffixes[displacement_type].vector_type; 1935 mode_suffix_index mode = find_mode_suffix (NUM_VECTOR_TYPES, 1936 displacement_vector_type, 1937 displacement_units ()); 1938 if (mode != MODE_none) 1939 { 1940 if (mode == MODE_s32offset 1941 && !lookup_form (mode, type) 1942 && lookup_form (MODE_u32offset, type)) 1943 { 1944 if (type_suffix_ids[0] == NUM_TYPE_SUFFIXES) 1945 error_at (location, "%qE does not support 32-bit sign-extended" 1946 " offsets", fndecl); 1947 else 1948 error_at (location, "%qE does not support sign-extended" 1949 " offsets", fndecl); 1950 return MODE_none; 1951 } 1952 return mode; 1953 } 1954 } 1955 1956 if (type_suffix_ids[0] == NUM_TYPE_SUFFIXES) 1957 { 1958 /* TYPE has been inferred rather than specified by the user, 1959 so mention it in the error messages. */ 1960 if (load_p) 1961 error_at (location, "passing %qT to argument %d of %qE, which when" 1962 " loading %qT expects a vector of %d-bit integers", 1963 get_argument_type (argno), argno + 1, fndecl, 1964 get_vector_type (type), required_bits); 1965 else 1966 error_at (location, "passing %qT to argument %d of %qE, which when" 1967 " storing %qT expects a vector of %d-bit integers", 1968 get_argument_type (argno), argno + 1, fndecl, 1969 get_vector_type (type), required_bits); 1970 } 1971 else 1972 /* TYPE is part of the function name. */ 1973 error_at (location, "passing %qT to argument %d of %qE, which" 1974 " expects a vector of %d-bit integers", 1975 get_argument_type (argno), argno + 1, fndecl, required_bits); 1976 return MODE_none; 1977 } 1978 1979 /* Require the arguments starting at ARGNO to form a gather-style address. 1980 There are three possible uses: 1981 1982 - for loading into elements of type TYPE (when LOAD_P is true) 1983 - for storing from elements of type TYPE (when LOAD_P is false) 1984 - for prefetching data (when TYPE is NUM_TYPE_SUFFIXES) 1985 1986 The three possible addresses are: 1987 1988 - a vector base with no displacement 1989 - a vector base and a scalar displacement 1990 - a scalar (pointer) base and a vector displacement 1991 1992 The overloaded function's mode suffix determines whether there is 1993 a displacement, and if so, what units it uses: 1994 1995 - MODE_none: no displacement 1996 - MODE_offset: the displacement is measured in bytes 1997 - MODE_index: the displacement is measured in elements 1998 1999 Return the mode of the non-overloaded function on success, otherwise 2000 report an error and return MODE_none. */ 2001 mode_suffix_index 2002 function_resolver::resolve_gather_address (unsigned int argno, 2003 type_suffix_index type, 2004 bool load_p) 2005 { 2006 tree actual = get_argument_type (argno); 2007 if (actual == error_mark_node) 2008 return MODE_none; 2009 2010 if (displacement_units () != UNITS_none) 2011 { 2012 /* Some form of displacement is needed. First handle a scalar 2013 pointer base and a vector displacement. */ 2014 if (scalar_argument_p (argno)) 2015 /* Don't check the pointer type here, since there's only one valid 2016 choice. Leave that to the frontend. */ 2017 return resolve_sv_displacement (argno + 1, type, load_p); 2018 2019 if (!VECTOR_TYPE_P (actual)) 2020 { 2021 error_at (location, "passing %qT to argument %d of %qE," 2022 " which expects a vector or pointer base address", 2023 actual, argno + 1, fndecl); 2024 return MODE_none; 2025 } 2026 } 2027 2028 /* Check for the correct choice of vector base type. */ 2029 vector_type_index base_vector_type; 2030 if (type == NUM_TYPE_SUFFIXES) 2031 { 2032 /* Since prefetches have no type suffix, there is a free choice 2033 between 32-bit and 64-bit base addresses. */ 2034 base_vector_type = infer_vector_base_type (argno); 2035 if (base_vector_type == NUM_VECTOR_TYPES) 2036 return MODE_none; 2037 } 2038 else 2039 { 2040 /* Check for some form of vector type, without saying which type 2041 we expect. */ 2042 type_suffix_index base_type = infer_vector_type (argno); 2043 if (base_type == NUM_TYPE_SUFFIXES) 2044 return MODE_none; 2045 2046 /* Check whether the type is the right one. */ 2047 unsigned int required_bits = type_suffixes[type].element_bits; 2048 gcc_assert (required_bits == 32 || required_bits == 64); 2049 type_suffix_index required_type = (required_bits == 32 2050 ? TYPE_SUFFIX_u32 2051 : TYPE_SUFFIX_u64); 2052 if (required_type != base_type) 2053 { 2054 error_at (location, "passing %qT to argument %d of %qE," 2055 " which expects %qT", actual, argno + 1, fndecl, 2056 get_vector_type (required_type)); 2057 return MODE_none; 2058 } 2059 base_vector_type = type_suffixes[base_type].vector_type; 2060 } 2061 2062 /* Check the scalar displacement, if any. */ 2063 if (displacement_units () != UNITS_none 2064 && !require_scalar_type (argno + 1, "int64_t")) 2065 return MODE_none; 2066 2067 /* Find the appropriate mode suffix. The checks above should have 2068 weeded out all erroneous cases. */ 2069 for (unsigned int mode_i = 0; mode_i < ARRAY_SIZE (mode_suffixes); ++mode_i) 2070 { 2071 const mode_suffix_info &mode = mode_suffixes[mode_i]; 2072 if (mode.base_vector_type == base_vector_type 2073 && mode.displacement_vector_type == NUM_VECTOR_TYPES 2074 && mode.displacement_units == displacement_units ()) 2075 return mode_suffix_index (mode_i); 2076 } 2077 2078 gcc_unreachable (); 2079 } 2080 2081 /* Require arguments ARGNO and ARGNO + 1 to form an ADR-style address, 2082 i.e. one with a vector of base addresses and a vector of displacements. 2083 The overloaded function's mode suffix determines the units of the 2084 displacement (bytes for "_offset", elements for "_index"). 2085 2086 Return the associated mode suffix on success, otherwise report 2087 an error and return MODE_none. */ 2088 mode_suffix_index 2089 function_resolver::resolve_adr_address (unsigned int argno) 2090 { 2091 vector_type_index base_type = infer_vector_base_type (argno); 2092 if (base_type == NUM_VECTOR_TYPES) 2093 return MODE_none; 2094 2095 vector_type_index displacement_type 2096 = infer_vector_displacement_type (argno + 1); 2097 if (displacement_type == NUM_VECTOR_TYPES) 2098 return MODE_none; 2099 2100 mode_suffix_index mode = find_mode_suffix (base_type, displacement_type, 2101 displacement_units ()); 2102 if (mode == MODE_none) 2103 { 2104 if (mode_suffix_id == MODE_offset) 2105 error_at (location, "cannot combine a base of type %qT with" 2106 " an offset of type %qT", 2107 get_argument_type (argno), get_argument_type (argno + 1)); 2108 else 2109 error_at (location, "cannot combine a base of type %qT with" 2110 " an index of type %qT", 2111 get_argument_type (argno), get_argument_type (argno + 1)); 2112 } 2113 return mode; 2114 } 2115 2116 /* Require the function to have exactly EXPECTED arguments. Return true 2117 if it does, otherwise report an appropriate error. */ 2118 bool 2119 function_resolver::check_num_arguments (unsigned int expected) 2120 { 2121 if (m_arglist.length () < expected) 2122 error_at (location, "too few arguments to function %qE", fndecl); 2123 else if (m_arglist.length () > expected) 2124 error_at (location, "too many arguments to function %qE", fndecl); 2125 return m_arglist.length () == expected; 2126 } 2127 2128 /* If the function is predicated, check that the first argument is a 2129 suitable governing predicate. Also check that there are NOPS further 2130 arguments after any governing predicate, but don't check what they are. 2131 2132 Return true on success, otherwise report a suitable error. 2133 When returning true: 2134 2135 - set I to the number of the first unchecked argument. 2136 - set NARGS to the total number of arguments. */ 2137 bool 2138 function_resolver::check_gp_argument (unsigned int nops, 2139 unsigned int &i, unsigned int &nargs) 2140 { 2141 i = 0; 2142 if (pred != PRED_none) 2143 { 2144 /* Unary merge operations should use resolve_unary instead. */ 2145 gcc_assert (nops != 1 || pred != PRED_m); 2146 nargs = nops + 1; 2147 if (!check_num_arguments (nargs) 2148 || !require_vector_type (i, VECTOR_TYPE_svbool_t)) 2149 return false; 2150 i += 1; 2151 } 2152 else 2153 { 2154 nargs = nops; 2155 if (!check_num_arguments (nargs)) 2156 return false; 2157 } 2158 2159 return true; 2160 } 2161 2162 /* Finish resolving a function whose final argument can be a vector 2163 or a scalar, with the function having an implicit "_n" suffix 2164 in the latter case. This "_n" form might only exist for certain 2165 type suffixes. 2166 2167 ARGNO is the index of the final argument. The inferred type suffix 2168 was obtained from argument FIRST_ARGNO, which has type FIRST_TYPE. 2169 EXPECTED_TCLASS and EXPECTED_BITS describe the expected properties 2170 of the final vector or scalar argument, in the same way as for 2171 require_derived_vector_type. INFERRED_TYPE is the inferred type 2172 suffix itself, or NUM_TYPE_SUFFIXES if it's the same as FIRST_TYPE. 2173 2174 Return the function decl of the resolved function on success, 2175 otherwise report a suitable error and return error_mark_node. */ 2176 tree function_resolver:: 2177 finish_opt_n_resolution (unsigned int argno, unsigned int first_argno, 2178 type_suffix_index first_type, 2179 type_class_index expected_tclass, 2180 unsigned int expected_bits, 2181 type_suffix_index inferred_type) 2182 { 2183 if (inferred_type == NUM_TYPE_SUFFIXES) 2184 inferred_type = first_type; 2185 tree scalar_form = lookup_form (MODE_n, inferred_type); 2186 2187 /* Allow the final argument to be scalar, if an _n form exists. */ 2188 if (scalar_argument_p (argno)) 2189 { 2190 if (scalar_form) 2191 return scalar_form; 2192 2193 /* Check the vector form normally. If that succeeds, raise an 2194 error about having no corresponding _n form. */ 2195 tree res = resolve_to (mode_suffix_id, inferred_type); 2196 if (res != error_mark_node) 2197 error_at (location, "passing %qT to argument %d of %qE, but its" 2198 " %qT form does not accept scalars", 2199 get_argument_type (argno), argno + 1, fndecl, 2200 get_vector_type (first_type)); 2201 return error_mark_node; 2202 } 2203 2204 /* If an _n form does exist, provide a more accurate message than 2205 require_derived_vector_type would for arguments that are neither 2206 vectors nor scalars. */ 2207 if (scalar_form && !require_vector_or_scalar_type (argno)) 2208 return error_mark_node; 2209 2210 /* Check for the correct vector type. */ 2211 if (!require_derived_vector_type (argno, first_argno, first_type, 2212 expected_tclass, expected_bits)) 2213 return error_mark_node; 2214 2215 return resolve_to (mode_suffix_id, inferred_type); 2216 } 2217 2218 /* Resolve a (possibly predicated) unary function. If the function uses 2219 merge predication or if TREAT_AS_MERGE_P is true, there is an extra 2220 vector argument before the governing predicate that specifies the 2221 values of inactive elements. This argument has the following 2222 properties: 2223 2224 - the type class must be the same as for active elements if MERGE_TCLASS 2225 is SAME_TYPE_CLASS, otherwise it must be MERGE_TCLASS itself. 2226 2227 - the element size must be the same as for active elements if MERGE_BITS 2228 is SAME_TYPE_SIZE, otherwise it must be MERGE_BITS itself. 2229 2230 Return the function decl of the resolved function on success, 2231 otherwise report a suitable error and return error_mark_node. */ 2232 tree 2233 function_resolver::resolve_unary (type_class_index merge_tclass, 2234 unsigned int merge_bits, 2235 bool treat_as_merge_p) 2236 { 2237 type_suffix_index type; 2238 if (pred == PRED_m || treat_as_merge_p) 2239 { 2240 if (!check_num_arguments (3)) 2241 return error_mark_node; 2242 if (merge_tclass == SAME_TYPE_CLASS && merge_bits == SAME_SIZE) 2243 { 2244 /* The inactive elements are the same as the active elements, 2245 so we can use normal left-to-right resolution. */ 2246 if ((type = infer_vector_type (0)) == NUM_TYPE_SUFFIXES 2247 || !require_vector_type (1, VECTOR_TYPE_svbool_t) 2248 || !require_matching_vector_type (2, type)) 2249 return error_mark_node; 2250 } 2251 else 2252 { 2253 /* The inactive element type is a function of the active one, 2254 so resolve the active one first. */ 2255 if (!require_vector_type (1, VECTOR_TYPE_svbool_t) 2256 || (type = infer_vector_type (2)) == NUM_TYPE_SUFFIXES 2257 || !require_derived_vector_type (0, 2, type, merge_tclass, 2258 merge_bits)) 2259 return error_mark_node; 2260 } 2261 } 2262 else 2263 { 2264 /* We just need to check the predicate (if any) and the single 2265 vector argument. */ 2266 unsigned int i, nargs; 2267 if (!check_gp_argument (1, i, nargs) 2268 || (type = infer_vector_type (i)) == NUM_TYPE_SUFFIXES) 2269 return error_mark_node; 2270 } 2271 2272 /* Handle convert-like functions in which the first type suffix is 2273 explicit. */ 2274 if (type_suffix_ids[0] != NUM_TYPE_SUFFIXES) 2275 return resolve_to (mode_suffix_id, type_suffix_ids[0], type); 2276 2277 return resolve_to (mode_suffix_id, type); 2278 } 2279 2280 /* Resolve a (possibly predicated) function that takes NOPS like-typed 2281 vector arguments followed by NIMM integer immediates. Return the 2282 function decl of the resolved function on success, otherwise report 2283 a suitable error and return error_mark_node. */ 2284 tree 2285 function_resolver::resolve_uniform (unsigned int nops, unsigned int nimm) 2286 { 2287 unsigned int i, nargs; 2288 type_suffix_index type; 2289 if (!check_gp_argument (nops + nimm, i, nargs) 2290 || (type = infer_vector_type (i)) == NUM_TYPE_SUFFIXES) 2291 return error_mark_node; 2292 2293 i += 1; 2294 for (; i < nargs - nimm; ++i) 2295 if (!require_matching_vector_type (i, type)) 2296 return error_mark_node; 2297 2298 for (; i < nargs; ++i) 2299 if (!require_integer_immediate (i)) 2300 return error_mark_node; 2301 2302 return resolve_to (mode_suffix_id, type); 2303 } 2304 2305 /* Resolve a (possibly predicated) function that offers a choice between 2306 taking: 2307 2308 - NOPS like-typed vector arguments or 2309 - NOPS - 1 like-typed vector arguments followed by a scalar argument 2310 2311 Return the function decl of the resolved function on success, 2312 otherwise report a suitable error and return error_mark_node. */ 2313 tree 2314 function_resolver::resolve_uniform_opt_n (unsigned int nops) 2315 { 2316 unsigned int i, nargs; 2317 type_suffix_index type; 2318 if (!check_gp_argument (nops, i, nargs) 2319 || (type = infer_vector_type (i)) == NUM_TYPE_SUFFIXES) 2320 return error_mark_node; 2321 2322 unsigned int first_arg = i++; 2323 for (; i < nargs - 1; ++i) 2324 if (!require_matching_vector_type (i, type)) 2325 return error_mark_node; 2326 2327 return finish_opt_n_resolution (i, first_arg, type); 2328 } 2329 2330 /* If the call is erroneous, report an appropriate error and return 2331 error_mark_node. Otherwise, if the function is overloaded, return 2332 the decl of the non-overloaded function. Return NULL_TREE otherwise, 2333 indicating that the call should be processed in the normal way. */ 2334 tree 2335 function_resolver::resolve () 2336 { 2337 return shape->resolve (*this); 2338 } 2339 2340 function_checker::function_checker (location_t location, 2341 const function_instance &instance, 2342 tree fndecl, tree fntype, 2343 unsigned int nargs, tree *args) 2344 : function_call_info (location, instance, fndecl), 2345 m_fntype (fntype), m_nargs (nargs), m_args (args), 2346 /* We don't have to worry about unary _m operations here, since they 2347 never have arguments that need checking. */ 2348 m_base_arg (pred != PRED_none ? 1 : 0) 2349 { 2350 } 2351 2352 /* Return true if argument ARGNO exists. which it might not for 2353 erroneous calls. It is safe to wave through checks if this 2354 function returns false. */ 2355 bool 2356 function_checker::argument_exists_p (unsigned int argno) 2357 { 2358 gcc_assert (argno < (unsigned int) type_num_arguments (m_fntype)); 2359 return argno < m_nargs; 2360 } 2361 2362 /* Check that argument ARGNO is an integer constant expression and 2363 store its value in VALUE_OUT if so. The caller should first 2364 check that argument ARGNO exists. */ 2365 bool 2366 function_checker::require_immediate (unsigned int argno, 2367 HOST_WIDE_INT &value_out) 2368 { 2369 gcc_assert (argno < m_nargs); 2370 tree arg = m_args[argno]; 2371 2372 /* The type and range are unsigned, so read the argument as an 2373 unsigned rather than signed HWI. */ 2374 if (!tree_fits_uhwi_p (arg)) 2375 { 2376 report_non_ice (location, fndecl, argno); 2377 return false; 2378 } 2379 2380 /* ...but treat VALUE_OUT as signed for error reporting, since printing 2381 -1 is more user-friendly than the maximum uint64_t value. */ 2382 value_out = tree_to_uhwi (arg); 2383 return true; 2384 } 2385 2386 /* Check that argument REL_ARGNO is an integer constant expression that 2387 has the value VALUE0 or VALUE1. REL_ARGNO counts from the end of the 2388 predication arguments. */ 2389 bool 2390 function_checker::require_immediate_either_or (unsigned int rel_argno, 2391 HOST_WIDE_INT value0, 2392 HOST_WIDE_INT value1) 2393 { 2394 unsigned int argno = m_base_arg + rel_argno; 2395 if (!argument_exists_p (argno)) 2396 return true; 2397 2398 HOST_WIDE_INT actual; 2399 if (!require_immediate (argno, actual)) 2400 return false; 2401 2402 if (actual != value0 && actual != value1) 2403 { 2404 report_neither_nor (location, fndecl, argno, actual, 90, 270); 2405 return false; 2406 } 2407 2408 return true; 2409 } 2410 2411 /* Check that argument REL_ARGNO is an integer constant expression that has 2412 a valid value for enumeration type TYPE. REL_ARGNO counts from the end 2413 of the predication arguments. */ 2414 bool 2415 function_checker::require_immediate_enum (unsigned int rel_argno, tree type) 2416 { 2417 unsigned int argno = m_base_arg + rel_argno; 2418 if (!argument_exists_p (argno)) 2419 return true; 2420 2421 HOST_WIDE_INT actual; 2422 if (!require_immediate (argno, actual)) 2423 return false; 2424 2425 for (tree entry = TYPE_VALUES (type); entry; entry = TREE_CHAIN (entry)) 2426 { 2427 /* The value is an INTEGER_CST for C and a CONST_DECL wrapper 2428 around an INTEGER_CST for C++. */ 2429 tree value = TREE_VALUE (entry); 2430 if (TREE_CODE (value) == CONST_DECL) 2431 value = DECL_INITIAL (value); 2432 if (wi::to_widest (value) == actual) 2433 return true; 2434 } 2435 2436 report_not_enum (location, fndecl, argno, actual, type); 2437 return false; 2438 } 2439 2440 /* Check that argument REL_ARGNO is suitable for indexing argument 2441 REL_ARGNO - 1, in groups of GROUP_SIZE elements. REL_ARGNO counts 2442 from the end of the predication arguments. */ 2443 bool 2444 function_checker::require_immediate_lane_index (unsigned int rel_argno, 2445 unsigned int group_size) 2446 { 2447 unsigned int argno = m_base_arg + rel_argno; 2448 if (!argument_exists_p (argno)) 2449 return true; 2450 2451 /* Get the type of the previous argument. tree_argument_type wants a 2452 1-based number, whereas ARGNO is 0-based. */ 2453 machine_mode mode = TYPE_MODE (type_argument_type (m_fntype, argno)); 2454 gcc_assert (VECTOR_MODE_P (mode)); 2455 unsigned int nlanes = 128 / (group_size * GET_MODE_UNIT_BITSIZE (mode)); 2456 return require_immediate_range (rel_argno, 0, nlanes - 1); 2457 } 2458 2459 /* Check that argument REL_ARGNO is an integer constant expression that 2460 has one of the given values. */ 2461 bool 2462 function_checker::require_immediate_one_of (unsigned int rel_argno, 2463 HOST_WIDE_INT value0, 2464 HOST_WIDE_INT value1, 2465 HOST_WIDE_INT value2, 2466 HOST_WIDE_INT value3) 2467 { 2468 unsigned int argno = m_base_arg + rel_argno; 2469 if (!argument_exists_p (argno)) 2470 return true; 2471 2472 HOST_WIDE_INT actual; 2473 if (!require_immediate (argno, actual)) 2474 return false; 2475 2476 if (actual != value0 2477 && actual != value1 2478 && actual != value2 2479 && actual != value3) 2480 { 2481 report_not_one_of (location, fndecl, argno, actual, 2482 value0, value1, value2, value3); 2483 return false; 2484 } 2485 2486 return true; 2487 } 2488 2489 /* Check that argument REL_ARGNO is an integer constant expression in the 2490 range [MIN, MAX]. REL_ARGNO counts from the end of the predication 2491 arguments. */ 2492 bool 2493 function_checker::require_immediate_range (unsigned int rel_argno, 2494 HOST_WIDE_INT min, 2495 HOST_WIDE_INT max) 2496 { 2497 unsigned int argno = m_base_arg + rel_argno; 2498 if (!argument_exists_p (argno)) 2499 return true; 2500 2501 /* Required because of the tree_to_uhwi -> HOST_WIDE_INT conversion 2502 in require_immediate. */ 2503 gcc_assert (min >= 0 && min <= max); 2504 HOST_WIDE_INT actual; 2505 if (!require_immediate (argno, actual)) 2506 return false; 2507 2508 if (!IN_RANGE (actual, min, max)) 2509 { 2510 report_out_of_range (location, fndecl, argno, actual, min, max); 2511 return false; 2512 } 2513 2514 return true; 2515 } 2516 2517 /* Perform semantic checks on the call. Return true if the call is valid, 2518 otherwise report a suitable error. */ 2519 bool 2520 function_checker::check () 2521 { 2522 function_args_iterator iter; 2523 tree type; 2524 unsigned int i = 0; 2525 FOREACH_FUNCTION_ARGS (m_fntype, type, iter) 2526 { 2527 if (type == void_type_node || i >= m_nargs) 2528 break; 2529 2530 if (i >= m_base_arg 2531 && TREE_CODE (type) == ENUMERAL_TYPE 2532 && !require_immediate_enum (i - m_base_arg, type)) 2533 return false; 2534 2535 i += 1; 2536 } 2537 2538 return shape->check (*this); 2539 } 2540 2541 gimple_folder::gimple_folder (const function_instance &instance, tree fndecl, 2542 gimple_stmt_iterator *gsi_in, gcall *call_in) 2543 : function_call_info (gimple_location (call_in), instance, fndecl), 2544 gsi (gsi_in), call (call_in), lhs (gimple_call_lhs (call_in)) 2545 { 2546 } 2547 2548 /* VALUE might be a vector of type VECTYPE or a single scalar element. 2549 Duplicate it into a vector of type VECTYPE in the latter case, adding any 2550 new statements to STMTS. */ 2551 tree 2552 gimple_folder::force_vector (gimple_seq &stmts, tree vectype, tree value) 2553 { 2554 if (!VECTOR_TYPE_P (TREE_TYPE (value))) 2555 value = gimple_build_vector_from_val (&stmts, vectype, value); 2556 return value; 2557 } 2558 2559 /* Convert predicate argument ARGNO so that it has the type appropriate for 2560 an operation on VECTYPE. Add any new statements to STMTS. */ 2561 tree 2562 gimple_folder::convert_pred (gimple_seq &stmts, tree vectype, 2563 unsigned int argno) 2564 { 2565 tree pred = gimple_call_arg (call, argno); 2566 if (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (pred)), 2567 TYPE_VECTOR_SUBPARTS (vectype))) 2568 return pred; 2569 2570 return gimple_build (&stmts, VIEW_CONVERT_EXPR, 2571 truth_type_for (vectype), pred); 2572 } 2573 2574 /* Return a pointer to the address in a contiguous load or store, 2575 given that each memory vector has type VECTYPE. Add any new 2576 statements to STMTS. */ 2577 tree 2578 gimple_folder::fold_contiguous_base (gimple_seq &stmts, tree vectype) 2579 { 2580 tree base = gimple_call_arg (call, 1); 2581 if (mode_suffix_id == MODE_vnum) 2582 { 2583 tree offset = gimple_call_arg (call, 2); 2584 offset = gimple_convert (&stmts, sizetype, offset); 2585 offset = gimple_build (&stmts, MULT_EXPR, sizetype, offset, 2586 TYPE_SIZE_UNIT (vectype)); 2587 base = gimple_build (&stmts, POINTER_PLUS_EXPR, TREE_TYPE (base), 2588 base, offset); 2589 } 2590 return base; 2591 } 2592 2593 /* Return the alignment and TBAA argument to an internal load or store 2594 function like IFN_MASK_LOAD or IFN_MASK_STORE, given that it accesses 2595 memory elements of type TYPE. */ 2596 tree 2597 gimple_folder::load_store_cookie (tree type) 2598 { 2599 return build_int_cst (build_pointer_type (type), TYPE_ALIGN (type)); 2600 } 2601 2602 /* Fold the call to a call to INSTANCE, with the same arguments. */ 2603 gimple * 2604 gimple_folder::redirect_call (const function_instance &instance) 2605 { 2606 registered_function *rfn 2607 = function_table->find_with_hash (instance, instance.hash ()); 2608 if (!rfn) 2609 return NULL; 2610 2611 gimple_call_set_fndecl (call, rfn->decl); 2612 return call; 2613 } 2614 2615 /* Fold the call to a PTRUE, taking the element size from type suffix 0. */ 2616 gimple * 2617 gimple_folder::fold_to_ptrue () 2618 { 2619 tree svbool_type = TREE_TYPE (lhs); 2620 tree bool_type = TREE_TYPE (svbool_type); 2621 unsigned int element_bytes = type_suffix (0).element_bytes; 2622 2623 /* The return type is svbool_t for all type suffixes, thus for b8 we 2624 want { 1, 1, 1, 1, ... }, for b16 we want { 1, 0, 1, 0, ... }, etc. */ 2625 tree_vector_builder builder (svbool_type, element_bytes, 1); 2626 builder.quick_push (build_all_ones_cst (bool_type)); 2627 for (unsigned int i = 1; i < element_bytes; ++i) 2628 builder.quick_push (build_zero_cst (bool_type)); 2629 return gimple_build_assign (lhs, builder.build ()); 2630 } 2631 2632 /* Fold the call to a PFALSE. */ 2633 gimple * 2634 gimple_folder::fold_to_pfalse () 2635 { 2636 return gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs))); 2637 } 2638 2639 /* Fold an operation to a constant predicate in which the first VL 2640 elements are set and the rest are clear. Take the element size 2641 from type suffix 0. */ 2642 gimple * 2643 gimple_folder::fold_to_vl_pred (unsigned int vl) 2644 { 2645 tree vectype = TREE_TYPE (lhs); 2646 tree element_type = TREE_TYPE (vectype); 2647 tree minus_one = build_all_ones_cst (element_type); 2648 tree zero = build_zero_cst (element_type); 2649 unsigned int element_bytes = type_suffix (0).element_bytes; 2650 2651 /* Construct COUNT elements that contain the ptrue followed by 2652 a repeating sequence of COUNT elements. */ 2653 unsigned int count = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vectype)); 2654 gcc_assert (vl * element_bytes <= count); 2655 tree_vector_builder builder (vectype, count, 2); 2656 for (unsigned int i = 0; i < count * 2; ++i) 2657 { 2658 bool bit = (i & (element_bytes - 1)) == 0 && i < vl * element_bytes; 2659 builder.quick_push (bit ? minus_one : zero); 2660 } 2661 return gimple_build_assign (lhs, builder.build ()); 2662 } 2663 2664 /* Try to fold the call. Return the new statement on success and null 2665 on failure. */ 2666 gimple * 2667 gimple_folder::fold () 2668 { 2669 /* Don't fold anything when SVE is disabled; emit an error during 2670 expansion instead. */ 2671 if (!TARGET_SVE) 2672 return NULL; 2673 2674 /* Punt if the function has a return type and no result location is 2675 provided. The attributes should allow target-independent code to 2676 remove the calls if appropriate. */ 2677 if (!lhs && TREE_TYPE (gimple_call_fntype (call)) != void_type_node) 2678 return NULL; 2679 2680 return base->fold (*this); 2681 } 2682 2683 function_expander::function_expander (const function_instance &instance, 2684 tree fndecl, tree call_expr_in, 2685 rtx possible_target_in) 2686 : function_call_info (EXPR_LOCATION (call_expr_in), instance, fndecl), 2687 call_expr (call_expr_in), possible_target (possible_target_in) 2688 { 2689 } 2690 2691 /* Return the handler of direct optab OP for type suffix SUFFIX_I. */ 2692 insn_code 2693 function_expander::direct_optab_handler (optab op, unsigned int suffix_i) 2694 { 2695 return ::direct_optab_handler (op, vector_mode (suffix_i)); 2696 } 2697 2698 /* Choose between signed and unsigned direct optabs SIGNED_OP and 2699 UNSIGNED_OP based on the signedness of type suffix SUFFIX_I, then 2700 pick the appropriate optab handler for the mode. Use MODE as the 2701 mode if given, otherwise use the mode of type suffix SUFFIX_I. */ 2702 insn_code 2703 function_expander::direct_optab_handler_for_sign (optab signed_op, 2704 optab unsigned_op, 2705 unsigned int suffix_i, 2706 machine_mode mode) 2707 { 2708 if (mode == VOIDmode) 2709 mode = vector_mode (suffix_i); 2710 optab op = type_suffix (suffix_i).unsigned_p ? unsigned_op : signed_op; 2711 return ::direct_optab_handler (op, mode); 2712 } 2713 2714 /* Return true if X overlaps any input. */ 2715 bool 2716 function_expander::overlaps_input_p (rtx x) 2717 { 2718 for (unsigned int i = 0; i < args.length (); ++i) 2719 if (reg_overlap_mentioned_p (x, args[i])) 2720 return true; 2721 return false; 2722 } 2723 2724 /* Convert ptr_mode value X to Pmode. */ 2725 rtx 2726 function_expander::convert_to_pmode (rtx x) 2727 { 2728 if (ptr_mode == SImode) 2729 x = simplify_gen_unary (ZERO_EXTEND, DImode, x, SImode); 2730 return x; 2731 } 2732 2733 /* Return the base address for a contiguous load or store function. 2734 MEM_MODE is the mode of the addressed memory. */ 2735 rtx 2736 function_expander::get_contiguous_base (machine_mode mem_mode) 2737 { 2738 rtx base = convert_to_pmode (args[1]); 2739 if (mode_suffix_id == MODE_vnum) 2740 { 2741 /* Use the size of the memory mode for extending loads and truncating 2742 stores. Use the size of a full vector for non-extending loads 2743 and non-truncating stores (including svld[234] and svst[234]). */ 2744 poly_int64 size = ordered_min (GET_MODE_SIZE (mem_mode), 2745 BYTES_PER_SVE_VECTOR); 2746 rtx offset = gen_int_mode (size, Pmode); 2747 offset = simplify_gen_binary (MULT, Pmode, args[2], offset); 2748 base = simplify_gen_binary (PLUS, Pmode, base, offset); 2749 } 2750 return base; 2751 } 2752 2753 /* For a function that does the equivalent of: 2754 2755 OUTPUT = COND ? FN (INPUTS) : FALLBACK; 2756 2757 return the value of FALLBACK. 2758 2759 MODE is the mode of OUTPUT. NOPS is the number of operands in INPUTS. 2760 MERGE_ARGNO is the argument that provides FALLBACK for _m functions, 2761 or DEFAULT_MERGE_ARGNO if we should apply the usual rules. 2762 2763 ARGNO is the caller's index into args. If the returned value is 2764 argument 0 (as for unary _m operations), increment ARGNO past the 2765 returned argument. */ 2766 rtx 2767 function_expander::get_fallback_value (machine_mode mode, unsigned int nops, 2768 unsigned int merge_argno, 2769 unsigned int &argno) 2770 { 2771 if (pred == PRED_z) 2772 return CONST0_RTX (mode); 2773 2774 gcc_assert (pred == PRED_m || pred == PRED_x); 2775 if (merge_argno == DEFAULT_MERGE_ARGNO) 2776 merge_argno = nops == 1 && pred == PRED_m ? 0 : 1; 2777 2778 if (merge_argno == 0) 2779 return args[argno++]; 2780 2781 return args[merge_argno]; 2782 } 2783 2784 /* Return a REG rtx that can be used for the result of the function, 2785 using the preferred target if suitable. */ 2786 rtx 2787 function_expander::get_reg_target () 2788 { 2789 machine_mode target_mode = TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl))); 2790 if (!possible_target || GET_MODE (possible_target) != target_mode) 2791 possible_target = gen_reg_rtx (target_mode); 2792 return possible_target; 2793 } 2794 2795 /* As for get_reg_target, but make sure that the returned REG does not 2796 overlap any inputs. */ 2797 rtx 2798 function_expander::get_nonoverlapping_reg_target () 2799 { 2800 if (possible_target && overlaps_input_p (possible_target)) 2801 possible_target = NULL_RTX; 2802 return get_reg_target (); 2803 } 2804 2805 /* Add an output operand to the instruction we're building, which has 2806 code ICODE. Bind the output to the preferred target rtx if possible. */ 2807 void 2808 function_expander::add_output_operand (insn_code icode) 2809 { 2810 unsigned int opno = m_ops.length (); 2811 machine_mode mode = insn_data[icode].operand[opno].mode; 2812 m_ops.safe_grow (opno + 1); 2813 create_output_operand (&m_ops.last (), possible_target, mode); 2814 } 2815 2816 /* Add an input operand to the instruction we're building, which has 2817 code ICODE. Calculate the value of the operand as follows: 2818 2819 - If the operand is a vector and X is not, broadcast X to fill a 2820 vector of the appropriate mode. 2821 2822 - Otherwise, if the operand is a predicate, coerce X to have the 2823 mode that the instruction expects. In this case X is known to be 2824 VNx16BImode (the mode of svbool_t). 2825 2826 - Otherwise use X directly. The expand machinery checks that X has 2827 the right mode for the instruction. */ 2828 void 2829 function_expander::add_input_operand (insn_code icode, rtx x) 2830 { 2831 unsigned int opno = m_ops.length (); 2832 const insn_operand_data &operand = insn_data[icode].operand[opno]; 2833 machine_mode mode = operand.mode; 2834 if (mode == VOIDmode) 2835 { 2836 /* The only allowable use of VOIDmode is the wildcard 2837 aarch64_any_register_operand, which is used to avoid 2838 combinatorial explosion in the reinterpret patterns. */ 2839 gcc_assert (operand.predicate == aarch64_any_register_operand); 2840 mode = GET_MODE (x); 2841 } 2842 else if (!VECTOR_MODE_P (GET_MODE (x)) && VECTOR_MODE_P (mode)) 2843 x = expand_vector_broadcast (mode, x); 2844 else if (GET_MODE_CLASS (mode) == MODE_VECTOR_BOOL) 2845 { 2846 gcc_assert (GET_MODE (x) == VNx16BImode); 2847 x = gen_lowpart (mode, x); 2848 } 2849 m_ops.safe_grow (m_ops.length () + 1); 2850 create_input_operand (&m_ops.last (), x, mode); 2851 } 2852 2853 /* Add an integer operand with value X to the instruction. */ 2854 void 2855 function_expander::add_integer_operand (HOST_WIDE_INT x) 2856 { 2857 m_ops.safe_grow (m_ops.length () + 1); 2858 create_integer_operand (&m_ops.last (), x); 2859 } 2860 2861 /* Add a memory operand with mode MODE and address ADDR. */ 2862 void 2863 function_expander::add_mem_operand (machine_mode mode, rtx addr) 2864 { 2865 /* Exception for OImode for the ld1ro intrinsics. 2866 They act on 256 bit octaword data, and it's just easier to use a scalar 2867 mode to represent that than add a new vector mode solely for the purpose 2868 of this intrinsic. */ 2869 gcc_assert (VECTOR_MODE_P (mode) || mode == OImode); 2870 rtx mem = gen_rtx_MEM (mode, memory_address (mode, addr)); 2871 /* The memory is only guaranteed to be element-aligned. */ 2872 set_mem_align (mem, GET_MODE_ALIGNMENT (GET_MODE_INNER (mode))); 2873 add_fixed_operand (mem); 2874 } 2875 2876 /* Add an address operand with value X. The static operand data says 2877 what mode and form the address must have. */ 2878 void 2879 function_expander::add_address_operand (rtx x) 2880 { 2881 m_ops.safe_grow (m_ops.length () + 1); 2882 create_address_operand (&m_ops.last (), x); 2883 } 2884 2885 /* Add an operand that must be X. The only way of legitimizing an 2886 invalid X is to reload the address of a MEM. */ 2887 void 2888 function_expander::add_fixed_operand (rtx x) 2889 { 2890 m_ops.safe_grow (m_ops.length () + 1); 2891 create_fixed_operand (&m_ops.last (), x); 2892 } 2893 2894 /* Generate instruction ICODE, given that its operands have already 2895 been added to M_OPS. Return the value of the first operand. */ 2896 rtx 2897 function_expander::generate_insn (insn_code icode) 2898 { 2899 expand_insn (icode, m_ops.length (), m_ops.address ()); 2900 return function_returns_void_p () ? const0_rtx : m_ops[0].value; 2901 } 2902 2903 /* Convert the arguments to a gather/scatter function into the 2904 associated md operands. Argument ARGNO is the scalar or vector base and 2905 argument ARGNO + 1 is the scalar or vector displacement (if applicable). 2906 The md pattern expects: 2907 2908 - a scalar base 2909 - a vector displacement 2910 2911 If SCALED_P is true, it also expects: 2912 2913 - a const_int that is 1 if the displacement is zero-extended from 32 bits 2914 - a scaling multiplier (1 for bytes, 2 for .h indices, etc.). 2915 2916 If SCALED_P is false, the displacement is implicitly zero-extended 2917 and the scaling multiplier is implicitly 1. */ 2918 void 2919 function_expander::prepare_gather_address_operands (unsigned int argno, 2920 bool scaled_p) 2921 { 2922 machine_mode mem_mode = memory_vector_mode (); 2923 tree vector_type = base_vector_type (); 2924 units_index units = displacement_units (); 2925 int shift_idx = -1; 2926 if (units == UNITS_none) 2927 { 2928 /* Vector base, no displacement. Convert to an integer zero base 2929 and a vector byte offset. */ 2930 args.quick_insert (argno, const0_rtx); 2931 units = UNITS_bytes; 2932 } 2933 else if (vector_type) 2934 { 2935 /* Vector base, scalar displacement. Convert to a scalar base and 2936 a vector byte offset. */ 2937 std::swap (args[argno], args[argno + 1]); 2938 if (units == UNITS_elements) 2939 shift_idx = argno; 2940 } 2941 else 2942 { 2943 /* Scalar base, vector displacement. This is the order that the md 2944 pattern wants. */ 2945 args[argno] = convert_to_pmode (args[argno]); 2946 vector_type = displacement_vector_type (); 2947 if (units == UNITS_elements && !scaled_p) 2948 shift_idx = argno + 1; 2949 } 2950 tree scalar_displacement_type = TREE_TYPE (vector_type); 2951 2952 if (shift_idx >= 0) 2953 { 2954 machine_mode arg_mode = GET_MODE (args[shift_idx]); 2955 if (arg_mode == VOIDmode) 2956 arg_mode = DImode; 2957 unsigned int elt_bytes = GET_MODE_UNIT_SIZE (mem_mode); 2958 rtx shift = gen_int_mode (exact_log2 (elt_bytes), DImode); 2959 args[shift_idx] = simplify_gen_binary (ASHIFT, arg_mode, 2960 args[shift_idx], shift); 2961 units = UNITS_bytes; 2962 } 2963 2964 bool uxtw_p = (TYPE_PRECISION (scalar_displacement_type) == 64 2965 || TYPE_UNSIGNED (scalar_displacement_type)); 2966 unsigned int scale = (units == UNITS_bytes 2967 ? 1 : GET_MODE_UNIT_SIZE (mem_mode)); 2968 2969 if (scaled_p) 2970 { 2971 args.quick_insert (argno + 2, GEN_INT (uxtw_p)); 2972 args.quick_insert (argno + 3, GEN_INT (scale)); 2973 } 2974 else 2975 gcc_assert (uxtw_p && scale == 1); 2976 } 2977 2978 /* The final argument is an immediate svprfop value. Add two fake arguments 2979 to represent the rw and locality operands of a PREFETCH rtx. */ 2980 void 2981 function_expander::prepare_prefetch_operands () 2982 { 2983 unsigned int prfop = INTVAL (args.last ()); 2984 /* Bit 3 of the prfop selects stores over loads. */ 2985 args.quick_push (GEN_INT ((prfop & 8) != 0)); 2986 /* Bits 1 and 2 specify the locality; 0-based for svprfop but 2987 1-based for PREFETCH. */ 2988 args.quick_push (GEN_INT (((prfop >> 1) & 3) + 1)); 2989 } 2990 2991 /* Add a dummy argument to indicate whether predicate argument ARGNO 2992 is all-true when interpreted in mode PRED_MODE. The hint goes 2993 immediately after ARGNO. */ 2994 void 2995 function_expander::add_ptrue_hint (unsigned int argno, machine_mode pred_mode) 2996 { 2997 rtx pred = gen_lowpart (pred_mode, args[argno]); 2998 int hint = (pred == CONSTM1_RTX (pred_mode) 2999 ? SVE_KNOWN_PTRUE : SVE_MAYBE_NOT_PTRUE); 3000 args.quick_insert (argno + 1, gen_int_mode (hint, SImode)); 3001 } 3002 3003 /* Rotate inputs args[START:END] one position to the left, so that 3004 args[START] becomes args[END - 1]. */ 3005 void 3006 function_expander::rotate_inputs_left (unsigned int start, unsigned int end) 3007 { 3008 rtx new_last = args[start]; 3009 for (unsigned int i = start; i < end - 1; ++i) 3010 args[i] = args[i + 1]; 3011 args[end - 1] = new_last; 3012 } 3013 3014 /* Return true if the negation of argument ARGNO can be folded away, 3015 replacing it with the negated value if so. MODE is the associated 3016 vector mode, but the argument could be a single element. The main 3017 case this handles is constant arguments. */ 3018 bool 3019 function_expander::try_negating_argument (unsigned int argno, 3020 machine_mode mode) 3021 { 3022 rtx x = args[argno]; 3023 if (!VECTOR_MODE_P (GET_MODE (x))) 3024 mode = GET_MODE_INNER (mode); 3025 3026 x = simplify_unary_operation (NEG, mode, x, mode); 3027 if (!x) 3028 return false; 3029 3030 args[argno] = x; 3031 return true; 3032 } 3033 3034 /* Implement the call using instruction ICODE, with a 1:1 mapping between 3035 arguments and input operands. */ 3036 rtx 3037 function_expander::use_exact_insn (insn_code icode) 3038 { 3039 unsigned int nops = insn_data[icode].n_operands; 3040 if (!function_returns_void_p ()) 3041 { 3042 add_output_operand (icode); 3043 nops -= 1; 3044 } 3045 for (unsigned int i = 0; i < nops; ++i) 3046 add_input_operand (icode, args[i]); 3047 return generate_insn (icode); 3048 } 3049 3050 /* Implement the call using instruction ICODE, which does not use a 3051 governing predicate. We must therefore drop the GP from an _x call. */ 3052 rtx 3053 function_expander::use_unpred_insn (insn_code icode) 3054 { 3055 /* We can't drop the predicate for _z and _m. */ 3056 gcc_assert (pred == PRED_x || pred == PRED_none); 3057 /* Discount the output operand. */ 3058 unsigned int nops = insn_data[icode].n_operands - 1; 3059 /* Drop the predicate argument in the case of _x predication. */ 3060 unsigned int bias = (pred == PRED_x ? 1 : 0); 3061 unsigned int i = 0; 3062 3063 add_output_operand (icode); 3064 for (; i < nops; ++i) 3065 add_input_operand (icode, args[i + bias]); 3066 3067 return generate_insn (icode); 3068 } 3069 3070 /* Implement the call using instruction ICODE, which is a predicated 3071 operation that returns arbitrary values for inactive lanes. */ 3072 rtx 3073 function_expander::use_pred_x_insn (insn_code icode) 3074 { 3075 /* At present we never need to handle PRED_none, which would involve 3076 creating a new predicate rather than using one supplied by the user. */ 3077 gcc_assert (pred == PRED_x); 3078 /* Discount the output operand. */ 3079 unsigned int nops = args.length () - 1; 3080 3081 bool has_float_operand_p = FLOAT_MODE_P (insn_data[icode].operand[0].mode); 3082 3083 /* Add the normal operands. */ 3084 add_output_operand (icode); 3085 add_input_operand (icode, args[0]); 3086 for (unsigned int i = 0; i < nops; ++i) 3087 { 3088 add_input_operand (icode, args[i + 1]); 3089 if (FLOAT_MODE_P (GET_MODE (args[i + 1]))) 3090 has_float_operand_p = true; 3091 } 3092 3093 if (has_float_operand_p) 3094 { 3095 /* Add a flag that indicates whether unpredicated instructions 3096 are allowed. */ 3097 rtx pred = m_ops[1].value; 3098 if (flag_trapping_math && pred != CONST1_RTX (GET_MODE (pred))) 3099 add_integer_operand (SVE_STRICT_GP); 3100 else 3101 add_integer_operand (SVE_RELAXED_GP); 3102 } 3103 3104 return generate_insn (icode); 3105 } 3106 3107 /* Implement the call using instruction ICODE, which does the equivalent of: 3108 3109 OUTPUT = COND ? FN (INPUTS) : FALLBACK; 3110 3111 The instruction operands are in the order above: OUTPUT, COND, INPUTS 3112 and FALLBACK. MERGE_ARGNO is the argument that provides FALLBACK for _m 3113 functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */ 3114 rtx 3115 function_expander::use_cond_insn (insn_code icode, unsigned int merge_argno) 3116 { 3117 /* At present we never need to handle PRED_none, which would involve 3118 creating a new predicate rather than using one supplied by the user. */ 3119 gcc_assert (pred != PRED_none); 3120 /* Discount the output, predicate and fallback value. */ 3121 unsigned int nops = insn_data[icode].n_operands - 3; 3122 machine_mode mode = insn_data[icode].operand[0].mode; 3123 3124 unsigned int opno = 0; 3125 rtx fallback_arg = get_fallback_value (mode, nops, merge_argno, opno); 3126 rtx pred = args[opno++]; 3127 3128 add_output_operand (icode); 3129 add_input_operand (icode, pred); 3130 for (unsigned int i = 0; i < nops; ++i) 3131 add_input_operand (icode, args[opno + i]); 3132 add_input_operand (icode, fallback_arg); 3133 return generate_insn (icode); 3134 } 3135 3136 /* Implement the call using instruction ICODE, which is a select-like 3137 operation with the following operands: 3138 3139 0: output 3140 1: true value 3141 2: false value 3142 3: predicate 3143 3144 MERGE_ARGNO is the argument that provides the "false" value for _m 3145 functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */ 3146 rtx 3147 function_expander::use_vcond_mask_insn (insn_code icode, 3148 unsigned int merge_argno) 3149 { 3150 machine_mode mode = vector_mode (0); 3151 3152 unsigned int opno = 0; 3153 rtx false_arg = get_fallback_value (mode, 1, merge_argno, opno); 3154 rtx pred_arg = args[opno++]; 3155 rtx true_arg = args[opno++]; 3156 3157 add_output_operand (icode); 3158 add_input_operand (icode, true_arg); 3159 add_input_operand (icode, false_arg); 3160 add_input_operand (icode, pred_arg); 3161 return generate_insn (icode); 3162 } 3163 3164 /* Implement the call using instruction ICODE, which loads memory operand 1 3165 into register operand 0 under the control of predicate operand 2. 3166 Extending loads have a further predicate (operand 3) that nominally 3167 controls the extension. */ 3168 rtx 3169 function_expander::use_contiguous_load_insn (insn_code icode) 3170 { 3171 machine_mode mem_mode = memory_vector_mode (); 3172 3173 add_output_operand (icode); 3174 add_mem_operand (mem_mode, get_contiguous_base (mem_mode)); 3175 add_input_operand (icode, args[0]); 3176 if (GET_MODE_UNIT_BITSIZE (mem_mode) < type_suffix (0).element_bits) 3177 add_input_operand (icode, CONSTM1_RTX (VNx16BImode)); 3178 return generate_insn (icode); 3179 } 3180 3181 /* Implement the call using instruction ICODE, which prefetches from 3182 address operand 1 under the control of predicate operand 0. 3183 Operands 2, 3 and 4 respectively specify the svprfop value, 3184 the PREFETCH rw flag and the PREFETCH locality. */ 3185 rtx 3186 function_expander::use_contiguous_prefetch_insn (insn_code icode) 3187 { 3188 add_input_operand (icode, args[0]); 3189 add_address_operand (get_contiguous_base (VNx16QImode)); 3190 for (unsigned int i = args.length () - 3; i < args.length (); ++i) 3191 add_input_operand (icode, args[i]); 3192 return generate_insn (icode); 3193 } 3194 3195 /* Implement the call using instruction ICODE, which stores register operand 1 3196 into memory operand 0 under the control of predicate operand 2. */ 3197 rtx 3198 function_expander::use_contiguous_store_insn (insn_code icode) 3199 { 3200 machine_mode mem_mode = memory_vector_mode (); 3201 3202 add_mem_operand (mem_mode, get_contiguous_base (mem_mode)); 3203 add_input_operand (icode, args.last ()); 3204 add_input_operand (icode, args[0]); 3205 return generate_insn (icode); 3206 } 3207 3208 /* Implement the call using one of the following strategies, chosen in order: 3209 3210 (1) "aarch64_pred_<optab><mode>_z" for PRED_z predicate functions 3211 3212 (2) "aarch64_pred_<optab><mode>" for PRED_x functions 3213 3214 (3) a normal unpredicated optab for PRED_none and PRED_x functions, 3215 dropping the predicate in the latter case 3216 3217 (4) an unpredicated "aarch64_sve_<code_optab><mode>" for PRED_none and 3218 PRED_x functions, again dropping the predicate for PRED_x 3219 3220 (5) "cond_<optab><mode>" otherwise 3221 3222 where <optab> corresponds to: 3223 3224 - CODE_FOR_SINT for signed integers 3225 - CODE_FOR_UINT for unsigned integers 3226 - UNSPEC_FOR_FP for floating-point values 3227 3228 and where <code_optab> is like <optab>, but uses CODE_FOR_SINT instead 3229 of UNSPEC_FOR_FP for floating-point values. 3230 3231 MERGE_ARGNO is the argument that provides the values of inactive lanes for 3232 _m functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */ 3233 rtx 3234 function_expander::map_to_rtx_codes (rtx_code code_for_sint, 3235 rtx_code code_for_uint, 3236 int unspec_for_fp, 3237 unsigned int merge_argno) 3238 { 3239 machine_mode mode = vector_mode (0); 3240 rtx_code code = (type_suffix (0).unsigned_p ? code_for_uint : code_for_sint); 3241 insn_code icode; 3242 3243 /* Handle predicate logic operations, which always use _z predication. */ 3244 if (type_suffix (0).tclass == TYPE_bool) 3245 { 3246 gcc_assert (pred == PRED_z && code_for_uint == code_for_sint); 3247 return use_exact_insn (code_for_aarch64_pred_z (code, mode)); 3248 } 3249 3250 /* First try using UNSPEC_PRED_X patterns for _x predication, 3251 if available. */ 3252 if (pred == PRED_x) 3253 { 3254 if (type_suffix (0).integer_p) 3255 icode = maybe_code_for_aarch64_pred (code, mode); 3256 else 3257 icode = maybe_code_for_aarch64_pred (unspec_for_fp, mode); 3258 if (icode != CODE_FOR_nothing) 3259 return use_pred_x_insn (icode); 3260 } 3261 3262 /* Otherwise expand PRED_none and PRED_x operations without a predicate. 3263 Floating-point operations conventionally use the signed rtx code. */ 3264 if (pred == PRED_none || pred == PRED_x) 3265 { 3266 icode = direct_optab_handler (code_to_optab (code), 0); 3267 if (icode == CODE_FOR_nothing) 3268 icode = code_for_aarch64_sve (code, mode); 3269 return use_unpred_insn (icode); 3270 } 3271 3272 /* Don't use cond_*_optabs here, since not all codes have one yet. */ 3273 if (type_suffix (0).integer_p) 3274 icode = code_for_cond (code, mode); 3275 else 3276 icode = code_for_cond (unspec_for_fp, mode); 3277 return use_cond_insn (icode, merge_argno); 3278 } 3279 3280 /* Implement the call using one of the following strategies, chosen in order: 3281 3282 (1) "aarch64_pred_<optab><mode>" for PRED_x functions; this is a 3283 predicated pattern 3284 3285 (2) "aarch64_sve_<optab><mode>" for PRED_none and PRED_x functions; 3286 this is an unpredicated pattern 3287 3288 (3) "cond_<optab><mode>" otherwise 3289 3290 where <optab> corresponds to: 3291 3292 - UNSPEC_FOR_SINT for signed integers 3293 - UNSPEC_FOR_UINT for unsigned integers 3294 - UNSPEC_FOR_FP for floating-point values 3295 3296 MERGE_ARGNO is the argument that provides the values of inactive lanes for 3297 _m functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */ 3298 rtx 3299 function_expander::map_to_unspecs (int unspec_for_sint, int unspec_for_uint, 3300 int unspec_for_fp, unsigned int merge_argno) 3301 { 3302 machine_mode mode = vector_mode (0); 3303 int unspec = (!type_suffix (0).integer_p ? unspec_for_fp 3304 : type_suffix (0).unsigned_p ? unspec_for_uint 3305 : unspec_for_sint); 3306 3307 if (pred == PRED_x) 3308 { 3309 insn_code icode = maybe_code_for_aarch64_pred (unspec, mode); 3310 if (icode != CODE_FOR_nothing) 3311 return use_pred_x_insn (icode); 3312 } 3313 3314 if (pred == PRED_none || pred == PRED_x) 3315 { 3316 insn_code icode = maybe_code_for_aarch64_sve (unspec, mode); 3317 if (icode != CODE_FOR_nothing) 3318 return use_unpred_insn (icode); 3319 } 3320 3321 insn_code icode = code_for_cond (unspec, vector_mode (0)); 3322 return use_cond_insn (icode, merge_argno); 3323 } 3324 3325 /* Expand the call and return its lhs. */ 3326 rtx 3327 function_expander::expand () 3328 { 3329 unsigned int nargs = call_expr_nargs (call_expr); 3330 args.reserve (nargs); 3331 for (unsigned int i = 0; i < nargs; ++i) 3332 args.quick_push (expand_normal (CALL_EXPR_ARG (call_expr, i))); 3333 3334 return base->expand (*this); 3335 } 3336 3337 /* Register the built-in SVE ABI types, such as __SVBool_t. */ 3338 static void 3339 register_builtin_types () 3340 { 3341 #define DEF_SVE_TYPE(ACLE_NAME, NCHARS, ABI_NAME, SCALAR_TYPE) \ 3342 scalar_types[VECTOR_TYPE_ ## ACLE_NAME] = SCALAR_TYPE; 3343 #include "aarch64-sve-builtins.def" 3344 3345 for (unsigned int i = 0; i < NUM_VECTOR_TYPES; ++i) 3346 { 3347 tree eltype = scalar_types[i]; 3348 tree vectype; 3349 unsigned int num_zr = 0, num_pr = 0; 3350 if (eltype == boolean_type_node) 3351 { 3352 vectype = build_truth_vector_type_for_mode (BYTES_PER_SVE_VECTOR, 3353 VNx16BImode); 3354 gcc_assert (TYPE_MODE (vectype) == VNx16BImode 3355 && TYPE_MODE (vectype) == TYPE_MODE_RAW (vectype) 3356 && TYPE_ALIGN (vectype) == 16 3357 && known_eq (wi::to_poly_offset (TYPE_SIZE (vectype)), 3358 BYTES_PER_SVE_VECTOR)); 3359 num_pr = 1; 3360 } 3361 else 3362 { 3363 scalar_mode elmode = SCALAR_TYPE_MODE (eltype); 3364 unsigned int elbytes = GET_MODE_SIZE (elmode); 3365 poly_uint64 nunits = exact_div (BYTES_PER_SVE_VECTOR, elbytes); 3366 machine_mode mode 3367 = aarch64_sve_data_mode (elmode, nunits).require (); 3368 vectype = build_vector_type_for_mode (eltype, mode); 3369 gcc_assert (VECTOR_MODE_P (TYPE_MODE (vectype)) 3370 && TYPE_MODE (vectype) == mode 3371 && TYPE_MODE_RAW (vectype) == mode 3372 && TYPE_ALIGN (vectype) == 128 3373 && known_eq (wi::to_poly_offset (TYPE_SIZE (vectype)), 3374 BITS_PER_SVE_VECTOR)); 3375 num_zr = 1; 3376 } 3377 vectype = build_distinct_type_copy (vectype); 3378 gcc_assert (vectype == TYPE_MAIN_VARIANT (vectype)); 3379 SET_TYPE_STRUCTURAL_EQUALITY (vectype); 3380 TYPE_ARTIFICIAL (vectype) = 1; 3381 TYPE_INDIVISIBLE_P (vectype) = 1; 3382 add_sve_type_attribute (vectype, num_zr, num_pr, 3383 vector_types[i].mangled_name, 3384 vector_types[i].acle_name); 3385 make_type_sizeless (vectype); 3386 abi_vector_types[i] = vectype; 3387 lang_hooks.types.register_builtin_type (vectype, 3388 vector_types[i].abi_name); 3389 } 3390 } 3391 3392 /* Initialize all compiler built-ins related to SVE that should be 3393 defined at start-up. */ 3394 void 3395 init_builtins () 3396 { 3397 sve_switcher sve; 3398 register_builtin_types (); 3399 if (in_lto_p) 3400 handle_arm_sve_h (); 3401 } 3402 3403 /* Register vector type TYPE under its arm_sve.h name. */ 3404 static void 3405 register_vector_type (vector_type_index type) 3406 { 3407 tree vectype = abi_vector_types[type]; 3408 tree id = get_identifier (vector_types[type].acle_name); 3409 tree decl = build_decl (input_location, TYPE_DECL, id, vectype); 3410 decl = lang_hooks.decls.pushdecl (decl); 3411 3412 /* Record the new ACLE type if pushdecl succeeded without error. Use 3413 the ABI type otherwise, so that the type we record at least has the 3414 right form, even if it doesn't have the right name. This should give 3415 better error recovery behavior than installing error_mark_node or 3416 installing an incorrect type. */ 3417 if (decl 3418 && TREE_CODE (decl) == TYPE_DECL 3419 && TYPE_MAIN_VARIANT (TREE_TYPE (decl)) == vectype) 3420 vectype = TREE_TYPE (decl); 3421 acle_vector_types[0][type] = vectype; 3422 } 3423 3424 /* Register the tuple type that contains NUM_VECTORS vectors of type TYPE. */ 3425 static void 3426 register_tuple_type (unsigned int num_vectors, vector_type_index type) 3427 { 3428 tree tuple_type = lang_hooks.types.make_type (RECORD_TYPE); 3429 3430 /* Work out the structure name. */ 3431 char buffer[sizeof ("svbfloat16x4_t")]; 3432 const char *vector_type_name = vector_types[type].acle_name; 3433 snprintf (buffer, sizeof (buffer), "%.*sx%d_t", 3434 (int) strlen (vector_type_name) - 2, vector_type_name, 3435 num_vectors); 3436 3437 /* The contents of the type are opaque, so we can define them in any 3438 way that maps to the correct ABI type. 3439 3440 Here we choose to use the same layout as for arm_neon.h, but with 3441 "__val" instead of "val": 3442 3443 struct svfooxN_t { svfoo_t __val[N]; }; 3444 3445 (It wouldn't be possible to write that directly in C or C++ for 3446 sizeless types, but that's not a problem for this function.) 3447 3448 Using arrays simplifies the handling of svget and svset for variable 3449 arguments. */ 3450 tree vector_type = acle_vector_types[0][type]; 3451 tree array_type = build_array_type_nelts (vector_type, num_vectors); 3452 gcc_assert (VECTOR_MODE_P (TYPE_MODE (array_type)) 3453 && TYPE_MODE_RAW (array_type) == TYPE_MODE (array_type) 3454 && TYPE_ALIGN (array_type) == 128); 3455 3456 tree field = build_decl (input_location, FIELD_DECL, 3457 get_identifier ("__val"), array_type); 3458 DECL_FIELD_CONTEXT (field) = tuple_type; 3459 TYPE_FIELDS (tuple_type) = field; 3460 add_sve_type_attribute (tuple_type, num_vectors, 0, NULL, buffer); 3461 make_type_sizeless (tuple_type); 3462 layout_type (tuple_type); 3463 gcc_assert (VECTOR_MODE_P (TYPE_MODE (tuple_type)) 3464 && TYPE_MODE_RAW (tuple_type) == TYPE_MODE (tuple_type) 3465 && TYPE_ALIGN (tuple_type) == 128); 3466 3467 tree decl = build_decl (input_location, TYPE_DECL, 3468 get_identifier (buffer), tuple_type); 3469 TYPE_NAME (tuple_type) = decl; 3470 TYPE_STUB_DECL (tuple_type) = decl; 3471 lang_hooks.decls.pushdecl (decl); 3472 /* ??? Undo the effect of set_underlying_type for C. The C frontend 3473 doesn't recognize DECL as a built-in because (as intended) the decl has 3474 a real location instead of BUILTINS_LOCATION. The frontend therefore 3475 treats the decl like a normal C "typedef struct foo foo;", expecting 3476 the type for tag "struct foo" to have a dummy unnamed TYPE_DECL instead 3477 of the named one we attached above. It then sets DECL_ORIGINAL_TYPE 3478 on the supposedly unnamed decl, creating a circularity that upsets 3479 dwarf2out. 3480 3481 We don't want to follow the normal C model and create "struct foo" 3482 tags for tuple types since (a) the types are supposed to be opaque 3483 and (b) they couldn't be defined as a real struct anyway. Treating 3484 the TYPE_DECLs as "typedef struct foo foo;" without creating 3485 "struct foo" would lead to confusing error messages. */ 3486 DECL_ORIGINAL_TYPE (decl) = NULL_TREE; 3487 3488 acle_vector_types[num_vectors - 1][type] = tuple_type; 3489 } 3490 3491 /* Register the svpattern enum. */ 3492 static void 3493 register_svpattern () 3494 { 3495 auto_vec<string_int_pair, 32> values; 3496 #define PUSH(UPPER, LOWER, VALUE) \ 3497 values.quick_push (string_int_pair ("SV_" #UPPER, VALUE)); 3498 AARCH64_FOR_SVPATTERN (PUSH) 3499 #undef PUSH 3500 3501 acle_svpattern = lang_hooks.types.simulate_enum_decl (input_location, 3502 "svpattern", values); 3503 } 3504 3505 /* Register the svprfop enum. */ 3506 static void 3507 register_svprfop () 3508 { 3509 auto_vec<string_int_pair, 16> values; 3510 #define PUSH(UPPER, LOWER, VALUE) \ 3511 values.quick_push (string_int_pair ("SV_" #UPPER, VALUE)); 3512 AARCH64_FOR_SVPRFOP (PUSH) 3513 #undef PUSH 3514 3515 acle_svprfop = lang_hooks.types.simulate_enum_decl (input_location, 3516 "svprfop", values); 3517 } 3518 3519 /* Implement #pragma GCC aarch64 "arm_sve.h". */ 3520 void 3521 handle_arm_sve_h () 3522 { 3523 if (function_table) 3524 { 3525 error ("duplicate definition of %qs", "arm_sve.h"); 3526 return; 3527 } 3528 3529 sve_switcher sve; 3530 3531 /* Define the vector and tuple types. */ 3532 for (unsigned int type_i = 0; type_i < NUM_VECTOR_TYPES; ++type_i) 3533 { 3534 vector_type_index type = vector_type_index (type_i); 3535 register_vector_type (type); 3536 if (type != VECTOR_TYPE_svbool_t) 3537 for (unsigned int count = 2; count <= MAX_TUPLE_SIZE; ++count) 3538 register_tuple_type (count, type); 3539 } 3540 3541 /* Define the enums. */ 3542 register_svpattern (); 3543 register_svprfop (); 3544 3545 /* Define the functions. */ 3546 function_table = new hash_table<registered_function_hasher> (1023); 3547 function_builder builder; 3548 for (unsigned int i = 0; i < ARRAY_SIZE (function_groups); ++i) 3549 builder.register_function_group (function_groups[i]); 3550 } 3551 3552 /* Return the function decl with SVE function subcode CODE, or error_mark_node 3553 if no such function exists. */ 3554 tree 3555 builtin_decl (unsigned int code, bool) 3556 { 3557 if (code >= vec_safe_length (registered_functions)) 3558 return error_mark_node; 3559 return (*registered_functions)[code]->decl; 3560 } 3561 3562 /* If we're implementing manual overloading, check whether the SVE 3563 function with subcode CODE is overloaded, and if so attempt to 3564 determine the corresponding non-overloaded function. The call 3565 occurs at location LOCATION and has the arguments given by ARGLIST. 3566 3567 If the call is erroneous, report an appropriate error and return 3568 error_mark_node. Otherwise, if the function is overloaded, return 3569 the decl of the non-overloaded function. Return NULL_TREE otherwise, 3570 indicating that the call should be processed in the normal way. */ 3571 tree 3572 resolve_overloaded_builtin (location_t location, unsigned int code, 3573 vec<tree, va_gc> *arglist) 3574 { 3575 if (code >= vec_safe_length (registered_functions)) 3576 return NULL_TREE; 3577 3578 registered_function &rfn = *(*registered_functions)[code]; 3579 if (rfn.overloaded_p) 3580 return function_resolver (location, rfn.instance, rfn.decl, 3581 *arglist).resolve (); 3582 return NULL_TREE; 3583 } 3584 3585 /* Perform any semantic checks needed for a call to the SVE function 3586 with subcode CODE, such as testing for integer constant expressions. 3587 The call occurs at location LOCATION and has NARGS arguments, 3588 given by ARGS. FNDECL is the original function decl, before 3589 overload resolution. 3590 3591 Return true if the call is valid, otherwise report a suitable error. */ 3592 bool 3593 check_builtin_call (location_t location, vec<location_t>, unsigned int code, 3594 tree fndecl, unsigned int nargs, tree *args) 3595 { 3596 const registered_function &rfn = *(*registered_functions)[code]; 3597 if (!check_required_extensions (location, rfn.decl, rfn.required_extensions)) 3598 return false; 3599 return function_checker (location, rfn.instance, fndecl, 3600 TREE_TYPE (rfn.decl), nargs, args).check (); 3601 } 3602 3603 /* Attempt to fold STMT, given that it's a call to the SVE function 3604 with subcode CODE. Return the new statement on success and null 3605 on failure. Insert any other new statements at GSI. */ 3606 gimple * 3607 gimple_fold_builtin (unsigned int code, gimple_stmt_iterator *gsi, gcall *stmt) 3608 { 3609 registered_function &rfn = *(*registered_functions)[code]; 3610 return gimple_folder (rfn.instance, rfn.decl, gsi, stmt).fold (); 3611 } 3612 3613 /* Expand a call to the SVE function with subcode CODE. EXP is the call 3614 expression and TARGET is the preferred location for the result. 3615 Return the value of the lhs. */ 3616 rtx 3617 expand_builtin (unsigned int code, tree exp, rtx target) 3618 { 3619 registered_function &rfn = *(*registered_functions)[code]; 3620 if (!check_required_extensions (EXPR_LOCATION (exp), rfn.decl, 3621 rfn.required_extensions)) 3622 return target; 3623 return function_expander (rfn.instance, rfn.decl, exp, target).expand (); 3624 } 3625 3626 /* If TYPE is a built-in type defined by the SVE ABI, return the mangled name, 3627 otherwise return NULL. */ 3628 const char * 3629 mangle_builtin_type (const_tree type) 3630 { 3631 /* ??? The C++ frontend normally strips qualifiers and attributes before 3632 calling this hook, adding separate mangling for attributes that affect 3633 type identity. Fortunately the type copy will have the same TYPE_NAME 3634 as the original, so we can get the attributes from there. */ 3635 if (TYPE_NAME (type) && TREE_CODE (TYPE_NAME (type)) == TYPE_DECL) 3636 type = TREE_TYPE (TYPE_NAME (type)); 3637 if (tree attr = lookup_sve_type_attribute (type)) 3638 if (tree id = TREE_VALUE (chain_index (2, TREE_VALUE (attr)))) 3639 return IDENTIFIER_POINTER (id); 3640 return NULL; 3641 } 3642 3643 /* Return true if TYPE is a built-in SVE type defined by the ABI or ACLE. */ 3644 bool 3645 builtin_type_p (const_tree type) 3646 { 3647 return lookup_sve_type_attribute (type); 3648 } 3649 3650 /* Return true if TYPE is a built-in SVE type defined by the ABI or ACLE. 3651 If so, store the number of constituent SVE vectors in *NUM_ZR and the 3652 number of constituent SVE predicates in *NUM_PR. */ 3653 bool 3654 builtin_type_p (const_tree type, unsigned int *num_zr, unsigned int *num_pr) 3655 { 3656 if (tree attr = lookup_sve_type_attribute (type)) 3657 { 3658 tree num_zr_node = TREE_VALUE (attr); 3659 tree num_pr_node = TREE_CHAIN (num_zr_node); 3660 *num_zr = tree_to_uhwi (TREE_VALUE (num_zr_node)); 3661 *num_pr = tree_to_uhwi (TREE_VALUE (num_pr_node)); 3662 return true; 3663 } 3664 return false; 3665 } 3666 3667 /* ATTRS is the attribute list for a sizeless SVE type. Return the 3668 attributes of the associated fixed-length SVE type, taking the 3669 "SVE type" attributes from NEW_SVE_TYPE_ARGS. */ 3670 static tree 3671 get_arm_sve_vector_bits_attributes (tree old_attrs, tree new_sve_type_args) 3672 { 3673 tree new_attrs = NULL_TREE; 3674 tree *ptr = &new_attrs; 3675 for (tree attr = old_attrs; attr; attr = TREE_CHAIN (attr)) 3676 { 3677 tree name = get_attribute_name (attr); 3678 if (is_attribute_p ("SVE sizeless type", name)) 3679 continue; 3680 3681 tree args = TREE_VALUE (attr); 3682 if (is_attribute_p ("SVE type", name)) 3683 args = new_sve_type_args; 3684 *ptr = tree_cons (TREE_PURPOSE (attr), args, NULL_TREE); 3685 ptr = &TREE_CHAIN (*ptr); 3686 } 3687 return new_attrs; 3688 } 3689 3690 /* An attribute callback for the "arm_sve_vector_bits" attribute. */ 3691 tree 3692 handle_arm_sve_vector_bits_attribute (tree *node, tree, tree args, int, 3693 bool *no_add_attrs) 3694 { 3695 *no_add_attrs = true; 3696 3697 tree type = *node; 3698 tree attr = lookup_sve_type_attribute (type); 3699 if (!attr) 3700 { 3701 error ("%qs applied to non-SVE type %qT", "arm_sve_vector_bits", type); 3702 return NULL_TREE; 3703 } 3704 3705 if (!VECTOR_TYPE_P (type)) 3706 { 3707 error ("%qs applied to non-vector type %qT", 3708 "arm_sve_vector_bits", type); 3709 return NULL_TREE; 3710 } 3711 3712 if (!sizeless_type_p (type)) 3713 { 3714 error ("%qs applied to type %qT, which already has a size", 3715 "arm_sve_vector_bits", type); 3716 return NULL_TREE; 3717 } 3718 3719 tree size = TREE_VALUE (args); 3720 if (TREE_CODE (size) != INTEGER_CST) 3721 { 3722 error ("%qs requires an integer constant expression", 3723 "arm_sve_vector_bits"); 3724 return NULL_TREE; 3725 } 3726 3727 unsigned HOST_WIDE_INT value = tree_to_uhwi (size); 3728 if (maybe_ne (value, BITS_PER_SVE_VECTOR)) 3729 { 3730 warning (OPT_Wattributes, "unsupported SVE vector size"); 3731 return NULL_TREE; 3732 } 3733 3734 /* Construct a new list of "SVE type" attribute arguments. */ 3735 tree new_sve_type_args = copy_list (TREE_VALUE (attr)); 3736 3737 /* Mangle the type as an instance of the imaginary template: 3738 3739 __SVE_VLS<typename, unsigned> 3740 3741 where the first parameter is the SVE type and where the second 3742 parameter is the SVE vector length in bits. */ 3743 tree mangled_name_node = chain_index (2, new_sve_type_args); 3744 const char *old_mangled_name 3745 = IDENTIFIER_POINTER (TREE_VALUE (mangled_name_node)); 3746 char *new_mangled_name 3747 = xasprintf ("9__SVE_VLSI%sLj%dEE", old_mangled_name, (int) value); 3748 TREE_VALUE (mangled_name_node) = get_identifier (new_mangled_name); 3749 free (new_mangled_name); 3750 3751 /* FIXME: The type ought to be a distinct copy in all cases, but 3752 currently that makes the C frontend reject conversions between 3753 svbool_t and its fixed-length variants. Using a type variant 3754 avoids that but means that we treat some ambiguous combinations 3755 as valid. */ 3756 tree new_type; 3757 tree base_type = TYPE_MAIN_VARIANT (type); 3758 if (lang_GNU_C () && VECTOR_BOOLEAN_TYPE_P (type)) 3759 new_type = build_variant_type_copy (base_type); 3760 else 3761 new_type = build_distinct_type_copy (base_type); 3762 3763 /* Construct a TYPE_DECL for the new type. This serves two purposes: 3764 3765 - It ensures we don't print the original TYPE_DECL in error messages. 3766 Printing the original name would be confusing because there are 3767 situations in which the distinction between the original type and 3768 the new type matters. For example: 3769 3770 __SVInt8_t __attribute__((arm_sve_vector_bits(512))) *a; 3771 __SVInt8_t *b; 3772 3773 a = b; 3774 3775 is invalid in C++, but without this, we'd print both types in 3776 the same way. 3777 3778 - Having a separate TYPE_DECL is necessary to ensure that C++ 3779 mangling works correctly. See mangle_builtin_type for details. 3780 3781 The name of the decl is something like: 3782 3783 svint8_t __attribute__((arm_sve_vector_bits(512))) 3784 3785 This is a compromise. It would be more accurate to use something like: 3786 3787 __SVInt8_t __attribute__((arm_sve_vector_bits(512))) 3788 3789 but the <arm_sve.h> name is likely to be more meaningful. */ 3790 tree acle_name_node = TREE_CHAIN (mangled_name_node); 3791 const char *old_type_name = IDENTIFIER_POINTER (TREE_VALUE (acle_name_node)); 3792 char *new_type_name 3793 = xasprintf ("%s __attribute__((arm_sve_vector_bits(%d)))", 3794 old_type_name, (int) value); 3795 tree decl = build_decl (BUILTINS_LOCATION, TYPE_DECL, 3796 get_identifier (new_type_name), new_type); 3797 DECL_ARTIFICIAL (decl) = 1; 3798 TYPE_NAME (new_type) = decl; 3799 free (new_type_name); 3800 3801 /* Allow the GNU vector extensions to be applied to vectors. 3802 The extensions aren't yet defined for packed predicates, 3803 so continue to treat them as abstract entities for now. */ 3804 if (!VECTOR_BOOLEAN_TYPE_P (new_type)) 3805 TYPE_INDIVISIBLE_P (new_type) = 0; 3806 3807 /* The new type is a normal sized type; it doesn't have the same 3808 restrictions as sizeless types. */ 3809 TYPE_ATTRIBUTES (new_type) 3810 = get_arm_sve_vector_bits_attributes (TYPE_ATTRIBUTES (new_type), 3811 new_sve_type_args); 3812 3813 /* Apply the relevant attributes, qualifiers and alignment of TYPE, 3814 if they differ from the original (sizeless) BASE_TYPE. */ 3815 if (TYPE_ATTRIBUTES (base_type) != TYPE_ATTRIBUTES (type) 3816 || TYPE_QUALS (base_type) != TYPE_QUALS (type)) 3817 { 3818 tree attrs 3819 = get_arm_sve_vector_bits_attributes (TYPE_ATTRIBUTES (type), 3820 new_sve_type_args); 3821 new_type = build_type_attribute_qual_variant (new_type, attrs, 3822 TYPE_QUALS (type)); 3823 } 3824 if (TYPE_ALIGN (base_type) != TYPE_ALIGN (type)) 3825 new_type = build_aligned_type (new_type, TYPE_ALIGN (type)); 3826 3827 *node = new_type; 3828 return NULL_TREE; 3829 } 3830 3831 /* Implement TARGET_VERIFY_TYPE_CONTEXT for SVE types. */ 3832 bool 3833 verify_type_context (location_t loc, type_context_kind context, 3834 const_tree type, bool silent_p) 3835 { 3836 if (!sizeless_type_p (type)) 3837 return true; 3838 3839 switch (context) 3840 { 3841 case TCTX_SIZEOF: 3842 case TCTX_STATIC_STORAGE: 3843 if (!silent_p) 3844 error_at (loc, "SVE type %qT does not have a fixed size", type); 3845 return false; 3846 3847 case TCTX_ALIGNOF: 3848 if (!silent_p) 3849 error_at (loc, "SVE type %qT does not have a defined alignment", type); 3850 return false; 3851 3852 case TCTX_THREAD_STORAGE: 3853 if (!silent_p) 3854 error_at (loc, "variables of type %qT cannot have thread-local" 3855 " storage duration", type); 3856 return false; 3857 3858 case TCTX_POINTER_ARITH: 3859 if (!silent_p) 3860 error_at (loc, "arithmetic on pointer to SVE type %qT", type); 3861 return false; 3862 3863 case TCTX_FIELD: 3864 if (silent_p) 3865 ; 3866 else if (lang_GNU_CXX ()) 3867 error_at (loc, "member variables cannot have SVE type %qT", type); 3868 else 3869 error_at (loc, "fields cannot have SVE type %qT", type); 3870 return false; 3871 3872 case TCTX_ARRAY_ELEMENT: 3873 if (!silent_p) 3874 error_at (loc, "array elements cannot have SVE type %qT", type); 3875 return false; 3876 3877 case TCTX_ALLOCATION: 3878 if (!silent_p) 3879 error_at (loc, "cannot allocate objects with SVE type %qT", type); 3880 return false; 3881 3882 case TCTX_DEALLOCATION: 3883 if (!silent_p) 3884 error_at (loc, "cannot delete objects with SVE type %qT", type); 3885 return false; 3886 3887 case TCTX_EXCEPTIONS: 3888 if (!silent_p) 3889 error_at (loc, "cannot throw or catch SVE type %qT", type); 3890 return false; 3891 3892 case TCTX_CAPTURE_BY_COPY: 3893 if (!silent_p) 3894 error_at (loc, "capture by copy of SVE type %qT", type); 3895 return false; 3896 } 3897 gcc_unreachable (); 3898 } 3899 3900 } 3901 3902 using namespace aarch64_sve; 3903 3904 inline void 3905 gt_ggc_mx (function_instance *) 3906 { 3907 } 3908 3909 inline void 3910 gt_pch_nx (function_instance *) 3911 { 3912 } 3913 3914 inline void 3915 gt_pch_nx (function_instance *, void (*) (void *, void *), void *) 3916 { 3917 } 3918 3919 #include "gt-aarch64-sve-builtins.h" 3920