xref: /netbsd-src/external/gpl3/gcc.old/dist/gcc/config/aarch64/aarch64-builtins.c (revision 8feb0f0b7eaff0608f8350bbfa3098827b4bb91b)
1 /* Builtins' description for AArch64 SIMD architecture.
2    Copyright (C) 2011-2020 Free Software Foundation, Inc.
3    Contributed by ARM Ltd.
4 
5    This file is part of GCC.
6 
7    GCC is free software; you can redistribute it and/or modify it
8    under the terms of the GNU General Public License as published by
9    the Free Software Foundation; either version 3, or (at your option)
10    any later version.
11 
12    GCC is distributed in the hope that it will be useful, but
13    WITHOUT ANY WARRANTY; without even the implied warranty of
14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15    General Public License for more details.
16 
17    You should have received a copy of the GNU General Public License
18    along with GCC; see the file COPYING3.  If not see
19    <http://www.gnu.org/licenses/>.  */
20 
21 #define IN_TARGET_CODE 1
22 
23 #include "config.h"
24 #include "system.h"
25 #include "coretypes.h"
26 #include "tm.h"
27 #include "function.h"
28 #include "basic-block.h"
29 #include "rtl.h"
30 #include "tree.h"
31 #include "gimple.h"
32 #include "memmodel.h"
33 #include "tm_p.h"
34 #include "expmed.h"
35 #include "optabs.h"
36 #include "recog.h"
37 #include "diagnostic-core.h"
38 #include "fold-const.h"
39 #include "stor-layout.h"
40 #include "explow.h"
41 #include "expr.h"
42 #include "langhooks.h"
43 #include "gimple-iterator.h"
44 #include "case-cfn-macros.h"
45 #include "emit-rtl.h"
46 #include "stringpool.h"
47 #include "attribs.h"
48 
49 #define v8qi_UP  E_V8QImode
50 #define v4hi_UP  E_V4HImode
51 #define v4hf_UP  E_V4HFmode
52 #define v2si_UP  E_V2SImode
53 #define v2sf_UP  E_V2SFmode
54 #define v1df_UP  E_V1DFmode
55 #define di_UP    E_DImode
56 #define df_UP    E_DFmode
57 #define v16qi_UP E_V16QImode
58 #define v8hi_UP  E_V8HImode
59 #define v8hf_UP  E_V8HFmode
60 #define v4si_UP  E_V4SImode
61 #define v4sf_UP  E_V4SFmode
62 #define v2di_UP  E_V2DImode
63 #define v2df_UP  E_V2DFmode
64 #define ti_UP	 E_TImode
65 #define oi_UP	 E_OImode
66 #define ci_UP	 E_CImode
67 #define xi_UP	 E_XImode
68 #define si_UP    E_SImode
69 #define sf_UP    E_SFmode
70 #define hi_UP    E_HImode
71 #define hf_UP    E_HFmode
72 #define qi_UP    E_QImode
73 #define bf_UP    E_BFmode
74 #define v4bf_UP  E_V4BFmode
75 #define v8bf_UP  E_V8BFmode
76 #define UP(X) X##_UP
77 
78 #define SIMD_MAX_BUILTIN_ARGS 5
79 
80 enum aarch64_type_qualifiers
81 {
82   /* T foo.  */
83   qualifier_none = 0x0,
84   /* unsigned T foo.  */
85   qualifier_unsigned = 0x1, /* 1 << 0  */
86   /* const T foo.  */
87   qualifier_const = 0x2, /* 1 << 1  */
88   /* T *foo.  */
89   qualifier_pointer = 0x4, /* 1 << 2  */
90   /* Used when expanding arguments if an operand could
91      be an immediate.  */
92   qualifier_immediate = 0x8, /* 1 << 3  */
93   qualifier_maybe_immediate = 0x10, /* 1 << 4  */
94   /* void foo (...).  */
95   qualifier_void = 0x20, /* 1 << 5  */
96   /* Some patterns may have internal operands, this qualifier is an
97      instruction to the initialisation code to skip this operand.  */
98   qualifier_internal = 0x40, /* 1 << 6  */
99   /* Some builtins should use the T_*mode* encoded in a simd_builtin_datum
100      rather than using the type of the operand.  */
101   qualifier_map_mode = 0x80, /* 1 << 7  */
102   /* qualifier_pointer | qualifier_map_mode  */
103   qualifier_pointer_map_mode = 0x84,
104   /* qualifier_const | qualifier_pointer | qualifier_map_mode  */
105   qualifier_const_pointer_map_mode = 0x86,
106   /* Polynomial types.  */
107   qualifier_poly = 0x100,
108   /* Lane indices - must be in range, and flipped for bigendian.  */
109   qualifier_lane_index = 0x200,
110   /* Lane indices for single lane structure loads and stores.  */
111   qualifier_struct_load_store_lane_index = 0x400,
112   /* Lane indices selected in pairs. - must be in range, and flipped for
113      bigendian.  */
114   qualifier_lane_pair_index = 0x800,
115   /* Lane indices selected in quadtuplets. - must be in range, and flipped for
116      bigendian.  */
117   qualifier_lane_quadtup_index = 0x1000,
118 };
119 
120 typedef struct
121 {
122   const char *name;
123   machine_mode mode;
124   const enum insn_code code;
125   unsigned int fcode;
126   enum aarch64_type_qualifiers *qualifiers;
127 } aarch64_simd_builtin_datum;
128 
129 static enum aarch64_type_qualifiers
130 aarch64_types_unop_qualifiers[SIMD_MAX_BUILTIN_ARGS]
131   = { qualifier_none, qualifier_none };
132 #define TYPES_UNOP (aarch64_types_unop_qualifiers)
133 static enum aarch64_type_qualifiers
134 aarch64_types_unopu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
135   = { qualifier_unsigned, qualifier_unsigned };
136 #define TYPES_UNOPU (aarch64_types_unopu_qualifiers)
137 static enum aarch64_type_qualifiers
138 aarch64_types_unopus_qualifiers[SIMD_MAX_BUILTIN_ARGS]
139   = { qualifier_unsigned, qualifier_none };
140 #define TYPES_UNOPUS (aarch64_types_unopus_qualifiers)
141 static enum aarch64_type_qualifiers
142 aarch64_types_binop_qualifiers[SIMD_MAX_BUILTIN_ARGS]
143   = { qualifier_none, qualifier_none, qualifier_maybe_immediate };
144 #define TYPES_BINOP (aarch64_types_binop_qualifiers)
145 static enum aarch64_type_qualifiers
146 aarch64_types_binopu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
147   = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned };
148 #define TYPES_BINOPU (aarch64_types_binopu_qualifiers)
149 static enum aarch64_type_qualifiers
150 aarch64_types_binop_uus_qualifiers[SIMD_MAX_BUILTIN_ARGS]
151   = { qualifier_unsigned, qualifier_unsigned, qualifier_none };
152 #define TYPES_BINOP_UUS (aarch64_types_binop_uus_qualifiers)
153 static enum aarch64_type_qualifiers
154 aarch64_types_binop_ssu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
155   = { qualifier_none, qualifier_none, qualifier_unsigned };
156 #define TYPES_BINOP_SSU (aarch64_types_binop_ssu_qualifiers)
157 static enum aarch64_type_qualifiers
158 aarch64_types_binop_uss_qualifiers[SIMD_MAX_BUILTIN_ARGS]
159   = { qualifier_unsigned, qualifier_none, qualifier_none };
160 #define TYPES_BINOP_USS (aarch64_types_binop_uss_qualifiers)
161 static enum aarch64_type_qualifiers
162 aarch64_types_binopp_qualifiers[SIMD_MAX_BUILTIN_ARGS]
163   = { qualifier_poly, qualifier_poly, qualifier_poly };
164 #define TYPES_BINOPP (aarch64_types_binopp_qualifiers)
165 
166 static enum aarch64_type_qualifiers
167 aarch64_types_ternop_qualifiers[SIMD_MAX_BUILTIN_ARGS]
168   = { qualifier_none, qualifier_none, qualifier_none, qualifier_none };
169 #define TYPES_TERNOP (aarch64_types_ternop_qualifiers)
170 static enum aarch64_type_qualifiers
171 aarch64_types_ternop_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
172   = { qualifier_none, qualifier_none, qualifier_none, qualifier_lane_index };
173 #define TYPES_TERNOP_LANE (aarch64_types_ternop_lane_qualifiers)
174 static enum aarch64_type_qualifiers
175 aarch64_types_ternopu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
176   = { qualifier_unsigned, qualifier_unsigned,
177       qualifier_unsigned, qualifier_unsigned };
178 #define TYPES_TERNOPU (aarch64_types_ternopu_qualifiers)
179 static enum aarch64_type_qualifiers
180 aarch64_types_ternopu_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
181   = { qualifier_unsigned, qualifier_unsigned,
182       qualifier_unsigned, qualifier_lane_index };
183 #define TYPES_TERNOPU_LANE (aarch64_types_ternopu_lane_qualifiers)
184 static enum aarch64_type_qualifiers
185 aarch64_types_ternopu_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
186   = { qualifier_unsigned, qualifier_unsigned,
187       qualifier_unsigned, qualifier_immediate };
188 #define TYPES_TERNOPUI (aarch64_types_ternopu_imm_qualifiers)
189 static enum aarch64_type_qualifiers
190 aarch64_types_ternop_ssus_qualifiers[SIMD_MAX_BUILTIN_ARGS]
191   = { qualifier_none, qualifier_none, qualifier_unsigned, qualifier_none };
192 #define TYPES_TERNOP_SSUS (aarch64_types_ternop_ssus_qualifiers)
193 
194 
195 static enum aarch64_type_qualifiers
196 aarch64_types_quadop_lane_pair_qualifiers[SIMD_MAX_BUILTIN_ARGS]
197   = { qualifier_none, qualifier_none, qualifier_none,
198       qualifier_none, qualifier_lane_pair_index };
199 #define TYPES_QUADOP_LANE_PAIR (aarch64_types_quadop_lane_pair_qualifiers)
200 static enum aarch64_type_qualifiers
201 aarch64_types_quadop_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
202   = { qualifier_none, qualifier_none, qualifier_none,
203       qualifier_none, qualifier_lane_index };
204 #define TYPES_QUADOP_LANE (aarch64_types_quadop_lane_qualifiers)
205 static enum aarch64_type_qualifiers
206 aarch64_types_quadopu_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
207   = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
208       qualifier_unsigned, qualifier_lane_index };
209 #define TYPES_QUADOPU_LANE (aarch64_types_quadopu_lane_qualifiers)
210 
211 static enum aarch64_type_qualifiers
212 aarch64_types_quadopssus_lane_quadtup_qualifiers[SIMD_MAX_BUILTIN_ARGS]
213   = { qualifier_none, qualifier_none, qualifier_unsigned,
214       qualifier_none, qualifier_lane_quadtup_index };
215 #define TYPES_QUADOPSSUS_LANE_QUADTUP \
216 	(aarch64_types_quadopssus_lane_quadtup_qualifiers)
217 static enum aarch64_type_qualifiers
218 aarch64_types_quadopsssu_lane_quadtup_qualifiers[SIMD_MAX_BUILTIN_ARGS]
219   = { qualifier_none, qualifier_none, qualifier_none,
220       qualifier_unsigned, qualifier_lane_quadtup_index };
221 #define TYPES_QUADOPSSSU_LANE_QUADTUP \
222 	(aarch64_types_quadopsssu_lane_quadtup_qualifiers)
223 
224 static enum aarch64_type_qualifiers
225 aarch64_types_quadopu_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
226   = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
227       qualifier_unsigned, qualifier_immediate };
228 #define TYPES_QUADOPUI (aarch64_types_quadopu_imm_qualifiers)
229 
230 static enum aarch64_type_qualifiers
231 aarch64_types_binop_imm_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
232   = { qualifier_poly, qualifier_none, qualifier_immediate };
233 #define TYPES_GETREGP (aarch64_types_binop_imm_p_qualifiers)
234 static enum aarch64_type_qualifiers
235 aarch64_types_binop_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
236   = { qualifier_none, qualifier_none, qualifier_immediate };
237 #define TYPES_GETREG (aarch64_types_binop_imm_qualifiers)
238 #define TYPES_SHIFTIMM (aarch64_types_binop_imm_qualifiers)
239 static enum aarch64_type_qualifiers
240 aarch64_types_shift_to_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS]
241   = { qualifier_unsigned, qualifier_none, qualifier_immediate };
242 #define TYPES_SHIFTIMM_USS (aarch64_types_shift_to_unsigned_qualifiers)
243 static enum aarch64_type_qualifiers
244 aarch64_types_fcvt_from_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS]
245   = { qualifier_none, qualifier_unsigned, qualifier_immediate };
246 #define TYPES_FCVTIMM_SUS (aarch64_types_fcvt_from_unsigned_qualifiers)
247 static enum aarch64_type_qualifiers
248 aarch64_types_unsigned_shift_qualifiers[SIMD_MAX_BUILTIN_ARGS]
249   = { qualifier_unsigned, qualifier_unsigned, qualifier_immediate };
250 #define TYPES_USHIFTIMM (aarch64_types_unsigned_shift_qualifiers)
251 
252 static enum aarch64_type_qualifiers
253 aarch64_types_ternop_s_imm_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
254   = { qualifier_none, qualifier_none, qualifier_poly, qualifier_immediate};
255 #define TYPES_SETREGP (aarch64_types_ternop_s_imm_p_qualifiers)
256 static enum aarch64_type_qualifiers
257 aarch64_types_ternop_s_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
258   = { qualifier_none, qualifier_none, qualifier_none, qualifier_immediate};
259 #define TYPES_SETREG (aarch64_types_ternop_s_imm_qualifiers)
260 #define TYPES_SHIFTINSERT (aarch64_types_ternop_s_imm_qualifiers)
261 #define TYPES_SHIFTACC (aarch64_types_ternop_s_imm_qualifiers)
262 
263 static enum aarch64_type_qualifiers
264 aarch64_types_ternop_p_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
265   = { qualifier_poly, qualifier_poly, qualifier_poly, qualifier_immediate};
266 #define TYPES_SHIFTINSERTP (aarch64_types_ternop_p_imm_qualifiers)
267 
268 static enum aarch64_type_qualifiers
269 aarch64_types_unsigned_shiftacc_qualifiers[SIMD_MAX_BUILTIN_ARGS]
270   = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
271       qualifier_immediate };
272 #define TYPES_USHIFTACC (aarch64_types_unsigned_shiftacc_qualifiers)
273 
274 
275 static enum aarch64_type_qualifiers
276 aarch64_types_combine_qualifiers[SIMD_MAX_BUILTIN_ARGS]
277   = { qualifier_none, qualifier_none, qualifier_none };
278 #define TYPES_COMBINE (aarch64_types_combine_qualifiers)
279 
280 static enum aarch64_type_qualifiers
281 aarch64_types_combine_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
282   = { qualifier_poly, qualifier_poly, qualifier_poly };
283 #define TYPES_COMBINEP (aarch64_types_combine_p_qualifiers)
284 
285 static enum aarch64_type_qualifiers
286 aarch64_types_load1_qualifiers[SIMD_MAX_BUILTIN_ARGS]
287   = { qualifier_none, qualifier_const_pointer_map_mode };
288 #define TYPES_LOAD1 (aarch64_types_load1_qualifiers)
289 #define TYPES_LOADSTRUCT (aarch64_types_load1_qualifiers)
290 static enum aarch64_type_qualifiers
291 aarch64_types_loadstruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
292   = { qualifier_none, qualifier_const_pointer_map_mode,
293       qualifier_none, qualifier_struct_load_store_lane_index };
294 #define TYPES_LOADSTRUCT_LANE (aarch64_types_loadstruct_lane_qualifiers)
295 
296 static enum aarch64_type_qualifiers
297 aarch64_types_bsl_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
298   = { qualifier_poly, qualifier_unsigned,
299       qualifier_poly, qualifier_poly };
300 #define TYPES_BSL_P (aarch64_types_bsl_p_qualifiers)
301 static enum aarch64_type_qualifiers
302 aarch64_types_bsl_s_qualifiers[SIMD_MAX_BUILTIN_ARGS]
303   = { qualifier_none, qualifier_unsigned,
304       qualifier_none, qualifier_none };
305 #define TYPES_BSL_S (aarch64_types_bsl_s_qualifiers)
306 static enum aarch64_type_qualifiers
307 aarch64_types_bsl_u_qualifiers[SIMD_MAX_BUILTIN_ARGS]
308   = { qualifier_unsigned, qualifier_unsigned,
309       qualifier_unsigned, qualifier_unsigned };
310 #define TYPES_BSL_U (aarch64_types_bsl_u_qualifiers)
311 
312 /* The first argument (return type) of a store should be void type,
313    which we represent with qualifier_void.  Their first operand will be
314    a DImode pointer to the location to store to, so we must use
315    qualifier_map_mode | qualifier_pointer to build a pointer to the
316    element type of the vector.  */
317 static enum aarch64_type_qualifiers
318 aarch64_types_store1_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
319   = { qualifier_void, qualifier_pointer_map_mode, qualifier_poly };
320 #define TYPES_STORE1P (aarch64_types_store1_p_qualifiers)
321 static enum aarch64_type_qualifiers
322 aarch64_types_store1_qualifiers[SIMD_MAX_BUILTIN_ARGS]
323   = { qualifier_void, qualifier_pointer_map_mode, qualifier_none };
324 #define TYPES_STORE1 (aarch64_types_store1_qualifiers)
325 #define TYPES_STORESTRUCT (aarch64_types_store1_qualifiers)
326 static enum aarch64_type_qualifiers
327 aarch64_types_storestruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
328   = { qualifier_void, qualifier_pointer_map_mode,
329       qualifier_none, qualifier_struct_load_store_lane_index };
330 #define TYPES_STORESTRUCT_LANE (aarch64_types_storestruct_lane_qualifiers)
331 
332 #define CF0(N, X) CODE_FOR_aarch64_##N##X
333 #define CF1(N, X) CODE_FOR_##N##X##1
334 #define CF2(N, X) CODE_FOR_##N##X##2
335 #define CF3(N, X) CODE_FOR_##N##X##3
336 #define CF4(N, X) CODE_FOR_##N##X##4
337 #define CF10(N, X) CODE_FOR_##N##X
338 
339 #define VAR1(T, N, MAP, A) \
340   {#N #A, UP (A), CF##MAP (N, A), 0, TYPES_##T},
341 #define VAR2(T, N, MAP, A, B) \
342   VAR1 (T, N, MAP, A) \
343   VAR1 (T, N, MAP, B)
344 #define VAR3(T, N, MAP, A, B, C) \
345   VAR2 (T, N, MAP, A, B) \
346   VAR1 (T, N, MAP, C)
347 #define VAR4(T, N, MAP, A, B, C, D) \
348   VAR3 (T, N, MAP, A, B, C) \
349   VAR1 (T, N, MAP, D)
350 #define VAR5(T, N, MAP, A, B, C, D, E) \
351   VAR4 (T, N, MAP, A, B, C, D) \
352   VAR1 (T, N, MAP, E)
353 #define VAR6(T, N, MAP, A, B, C, D, E, F) \
354   VAR5 (T, N, MAP, A, B, C, D, E) \
355   VAR1 (T, N, MAP, F)
356 #define VAR7(T, N, MAP, A, B, C, D, E, F, G) \
357   VAR6 (T, N, MAP, A, B, C, D, E, F) \
358   VAR1 (T, N, MAP, G)
359 #define VAR8(T, N, MAP, A, B, C, D, E, F, G, H) \
360   VAR7 (T, N, MAP, A, B, C, D, E, F, G) \
361   VAR1 (T, N, MAP, H)
362 #define VAR9(T, N, MAP, A, B, C, D, E, F, G, H, I) \
363   VAR8 (T, N, MAP, A, B, C, D, E, F, G, H) \
364   VAR1 (T, N, MAP, I)
365 #define VAR10(T, N, MAP, A, B, C, D, E, F, G, H, I, J) \
366   VAR9 (T, N, MAP, A, B, C, D, E, F, G, H, I) \
367   VAR1 (T, N, MAP, J)
368 #define VAR11(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K) \
369   VAR10 (T, N, MAP, A, B, C, D, E, F, G, H, I, J) \
370   VAR1 (T, N, MAP, K)
371 #define VAR12(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K, L) \
372   VAR11 (T, N, MAP, A, B, C, D, E, F, G, H, I, J, K) \
373   VAR1 (T, N, MAP, L)
374 #define VAR13(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M) \
375   VAR12 (T, N, MAP, A, B, C, D, E, F, G, H, I, J, K, L) \
376   VAR1 (T, N, MAP, M)
377 #define VAR14(T, X, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M, N) \
378   VAR13 (T, X, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M) \
379   VAR1 (T, X, MAP, N)
380 #define VAR15(T, X, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) \
381   VAR14 (T, X, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M, N) \
382   VAR1 (T, X, MAP, O)
383 #define VAR16(T, X, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) \
384   VAR15 (T, X, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) \
385   VAR1 (T, X, MAP, P)
386 
387 #include "aarch64-builtin-iterators.h"
388 
389 static aarch64_simd_builtin_datum aarch64_simd_builtin_data[] = {
390 #include "aarch64-simd-builtins.def"
391 };
392 
393 /* There's only 8 CRC32 builtins.  Probably not worth their own .def file.  */
394 #define AARCH64_CRC32_BUILTINS \
395   CRC32_BUILTIN (crc32b, QI) \
396   CRC32_BUILTIN (crc32h, HI) \
397   CRC32_BUILTIN (crc32w, SI) \
398   CRC32_BUILTIN (crc32x, DI) \
399   CRC32_BUILTIN (crc32cb, QI) \
400   CRC32_BUILTIN (crc32ch, HI) \
401   CRC32_BUILTIN (crc32cw, SI) \
402   CRC32_BUILTIN (crc32cx, DI)
403 
404 /* The next 8 FCMLA instrinsics require some special handling compared the
405    normal simd intrinsics.  */
406 #define AARCH64_SIMD_FCMLA_LANEQ_BUILTINS \
407   FCMLA_LANEQ_BUILTIN (0, v2sf, fcmla, V2SF, false) \
408   FCMLA_LANEQ_BUILTIN (90, v2sf, fcmla, V2SF, false) \
409   FCMLA_LANEQ_BUILTIN (180, v2sf, fcmla, V2SF, false) \
410   FCMLA_LANEQ_BUILTIN (270, v2sf, fcmla, V2SF, false) \
411   FCMLA_LANEQ_BUILTIN (0, v4hf, fcmla_laneq, V4HF, true) \
412   FCMLA_LANEQ_BUILTIN (90, v4hf, fcmla_laneq, V4HF, true) \
413   FCMLA_LANEQ_BUILTIN (180, v4hf, fcmla_laneq, V4HF, true) \
414   FCMLA_LANEQ_BUILTIN (270, v4hf, fcmla_laneq, V4HF, true) \
415 
416 typedef struct
417 {
418   const char *name;
419   machine_mode mode;
420   const enum insn_code icode;
421   unsigned int fcode;
422 } aarch64_crc_builtin_datum;
423 
424 /* Hold information about how to expand the FCMLA_LANEQ builtins.  */
425 typedef struct
426 {
427   const char *name;
428   machine_mode mode;
429   const enum insn_code icode;
430   unsigned int fcode;
431   bool lane;
432 } aarch64_fcmla_laneq_builtin_datum;
433 
434 #define CRC32_BUILTIN(N, M) \
435   AARCH64_BUILTIN_##N,
436 
437 #define FCMLA_LANEQ_BUILTIN(I, N, X, M, T) \
438   AARCH64_SIMD_BUILTIN_FCMLA_LANEQ##I##_##M,
439 
440 #undef VAR1
441 #define VAR1(T, N, MAP, A) \
442   AARCH64_SIMD_BUILTIN_##T##_##N##A,
443 
444 enum aarch64_builtins
445 {
446   AARCH64_BUILTIN_MIN,
447 
448   AARCH64_BUILTIN_GET_FPCR,
449   AARCH64_BUILTIN_SET_FPCR,
450   AARCH64_BUILTIN_GET_FPSR,
451   AARCH64_BUILTIN_SET_FPSR,
452 
453   AARCH64_BUILTIN_RSQRT_DF,
454   AARCH64_BUILTIN_RSQRT_SF,
455   AARCH64_BUILTIN_RSQRT_V2DF,
456   AARCH64_BUILTIN_RSQRT_V2SF,
457   AARCH64_BUILTIN_RSQRT_V4SF,
458   AARCH64_SIMD_BUILTIN_BASE,
459   AARCH64_SIMD_BUILTIN_LANE_CHECK,
460 #include "aarch64-simd-builtins.def"
461   /* The first enum element which is based on an insn_data pattern.  */
462   AARCH64_SIMD_PATTERN_START = AARCH64_SIMD_BUILTIN_LANE_CHECK + 1,
463   AARCH64_SIMD_BUILTIN_MAX = AARCH64_SIMD_PATTERN_START
464 			      + ARRAY_SIZE (aarch64_simd_builtin_data) - 1,
465   AARCH64_CRC32_BUILTIN_BASE,
466   AARCH64_CRC32_BUILTINS
467   AARCH64_CRC32_BUILTIN_MAX,
468   /* ARMv8.3-A Pointer Authentication Builtins.  */
469   AARCH64_PAUTH_BUILTIN_AUTIA1716,
470   AARCH64_PAUTH_BUILTIN_PACIA1716,
471   AARCH64_PAUTH_BUILTIN_AUTIB1716,
472   AARCH64_PAUTH_BUILTIN_PACIB1716,
473   AARCH64_PAUTH_BUILTIN_XPACLRI,
474   /* Special cased Armv8.3-A Complex FMA by Lane quad Builtins.  */
475   AARCH64_SIMD_FCMLA_LANEQ_BUILTIN_BASE,
476   AARCH64_SIMD_FCMLA_LANEQ_BUILTINS
477   /* Builtin for Arm8.3-a Javascript conversion instruction.  */
478   AARCH64_JSCVT,
479   /* TME builtins.  */
480   AARCH64_TME_BUILTIN_TSTART,
481   AARCH64_TME_BUILTIN_TCOMMIT,
482   AARCH64_TME_BUILTIN_TTEST,
483   AARCH64_TME_BUILTIN_TCANCEL,
484   /* Armv8.5-a RNG instruction builtins.  */
485   AARCH64_BUILTIN_RNG_RNDR,
486   AARCH64_BUILTIN_RNG_RNDRRS,
487   /* MEMTAG builtins.  */
488   AARCH64_MEMTAG_BUILTIN_START,
489   AARCH64_MEMTAG_BUILTIN_IRG,
490   AARCH64_MEMTAG_BUILTIN_GMI,
491   AARCH64_MEMTAG_BUILTIN_SUBP,
492   AARCH64_MEMTAG_BUILTIN_INC_TAG,
493   AARCH64_MEMTAG_BUILTIN_SET_TAG,
494   AARCH64_MEMTAG_BUILTIN_GET_TAG,
495   AARCH64_MEMTAG_BUILTIN_END,
496   AARCH64_BUILTIN_MAX
497 };
498 
499 #undef CRC32_BUILTIN
500 #define CRC32_BUILTIN(N, M) \
501   {"__builtin_aarch64_"#N, E_##M##mode, CODE_FOR_aarch64_##N, AARCH64_BUILTIN_##N},
502 
503 static aarch64_crc_builtin_datum aarch64_crc_builtin_data[] = {
504   AARCH64_CRC32_BUILTINS
505 };
506 
507 
508 #undef FCMLA_LANEQ_BUILTIN
509 #define FCMLA_LANEQ_BUILTIN(I, N, X, M, T) \
510   {"__builtin_aarch64_fcmla_laneq"#I#N, E_##M##mode, CODE_FOR_aarch64_##X##I##N, \
511    AARCH64_SIMD_BUILTIN_FCMLA_LANEQ##I##_##M, T},
512 
513 /* This structure contains how to manage the mapping form the builtin to the
514    instruction to generate in the backend and how to invoke the instruction.  */
515 static aarch64_fcmla_laneq_builtin_datum aarch64_fcmla_lane_builtin_data[] = {
516   AARCH64_SIMD_FCMLA_LANEQ_BUILTINS
517 };
518 
519 #undef CRC32_BUILTIN
520 
521 static GTY(()) tree aarch64_builtin_decls[AARCH64_BUILTIN_MAX];
522 
523 #define NUM_DREG_TYPES 6
524 #define NUM_QREG_TYPES 6
525 
526 /* Internal scalar builtin types.  These types are used to support
527    neon intrinsic builtins.  They are _not_ user-visible types.  Therefore
528    the mangling for these types are implementation defined.  */
529 const char *aarch64_scalar_builtin_types[] = {
530   "__builtin_aarch64_simd_qi",
531   "__builtin_aarch64_simd_hi",
532   "__builtin_aarch64_simd_si",
533   "__builtin_aarch64_simd_hf",
534   "__builtin_aarch64_simd_sf",
535   "__builtin_aarch64_simd_di",
536   "__builtin_aarch64_simd_df",
537   "__builtin_aarch64_simd_poly8",
538   "__builtin_aarch64_simd_poly16",
539   "__builtin_aarch64_simd_poly64",
540   "__builtin_aarch64_simd_poly128",
541   "__builtin_aarch64_simd_ti",
542   "__builtin_aarch64_simd_uqi",
543   "__builtin_aarch64_simd_uhi",
544   "__builtin_aarch64_simd_usi",
545   "__builtin_aarch64_simd_udi",
546   "__builtin_aarch64_simd_ei",
547   "__builtin_aarch64_simd_oi",
548   "__builtin_aarch64_simd_ci",
549   "__builtin_aarch64_simd_xi",
550   "__builtin_aarch64_simd_bf",
551   NULL
552 };
553 
554 #define ENTRY(E, M, Q, G) E,
555 enum aarch64_simd_type
556 {
557 #include "aarch64-simd-builtin-types.def"
558   ARM_NEON_H_TYPES_LAST
559 };
560 #undef ENTRY
561 
562 struct aarch64_simd_type_info
563 {
564   enum aarch64_simd_type type;
565 
566   /* Internal type name.  */
567   const char *name;
568 
569   /* Internal type name(mangled).  The mangled names conform to the
570      AAPCS64 (see "Procedure Call Standard for the ARM 64-bit Architecture",
571      Appendix A).  To qualify for emission with the mangled names defined in
572      that document, a vector type must not only be of the correct mode but also
573      be of the correct internal AdvSIMD vector type (e.g. __Int8x8_t); these
574      types are registered by aarch64_init_simd_builtin_types ().  In other
575      words, vector types defined in other ways e.g. via vector_size attribute
576      will get default mangled names.  */
577   const char *mangle;
578 
579   /* Internal type.  */
580   tree itype;
581 
582   /* Element type.  */
583   tree eltype;
584 
585   /* Machine mode the internal type maps to.  */
586   enum machine_mode mode;
587 
588   /* Qualifiers.  */
589   enum aarch64_type_qualifiers q;
590 };
591 
592 #define ENTRY(E, M, Q, G)  \
593   {E, "__" #E, #G "__" #E, NULL_TREE, NULL_TREE, E_##M##mode, qualifier_##Q},
594 static struct aarch64_simd_type_info aarch64_simd_types [] = {
595 #include "aarch64-simd-builtin-types.def"
596 };
597 #undef ENTRY
598 
599 static tree aarch64_simd_intOI_type_node = NULL_TREE;
600 static tree aarch64_simd_intCI_type_node = NULL_TREE;
601 static tree aarch64_simd_intXI_type_node = NULL_TREE;
602 
603 /* The user-visible __fp16 type, and a pointer to that type.  Used
604    across the back-end.  */
605 tree aarch64_fp16_type_node = NULL_TREE;
606 tree aarch64_fp16_ptr_type_node = NULL_TREE;
607 
608 /* Back-end node type for brain float (bfloat) types.  */
609 tree aarch64_bf16_type_node = NULL_TREE;
610 tree aarch64_bf16_ptr_type_node = NULL_TREE;
611 
612 /* Wrapper around add_builtin_function.  NAME is the name of the built-in
613    function, TYPE is the function type, and CODE is the function subcode
614    (relative to AARCH64_BUILTIN_GENERAL).  */
615 static tree
aarch64_general_add_builtin(const char * name,tree type,unsigned int code)616 aarch64_general_add_builtin (const char *name, tree type, unsigned int code)
617 {
618   code = (code << AARCH64_BUILTIN_SHIFT) | AARCH64_BUILTIN_GENERAL;
619   return add_builtin_function (name, type, code, BUILT_IN_MD,
620 			       NULL, NULL_TREE);
621 }
622 
623 static const char *
aarch64_mangle_builtin_scalar_type(const_tree type)624 aarch64_mangle_builtin_scalar_type (const_tree type)
625 {
626   int i = 0;
627 
628   while (aarch64_scalar_builtin_types[i] != NULL)
629     {
630       const char *name = aarch64_scalar_builtin_types[i];
631 
632       if (TREE_CODE (TYPE_NAME (type)) == TYPE_DECL
633 	  && DECL_NAME (TYPE_NAME (type))
634 	  && !strcmp (IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type))), name))
635 	return aarch64_scalar_builtin_types[i];
636       i++;
637     }
638   return NULL;
639 }
640 
641 static const char *
aarch64_mangle_builtin_vector_type(const_tree type)642 aarch64_mangle_builtin_vector_type (const_tree type)
643 {
644   int i;
645   int nelts = sizeof (aarch64_simd_types) / sizeof (aarch64_simd_types[0]);
646 
647   for (i = 0; i < nelts; i++)
648     if (aarch64_simd_types[i].mode ==  TYPE_MODE (type)
649 	&& TYPE_NAME (type)
650 	&& TREE_CODE (TYPE_NAME (type)) == TYPE_DECL
651 	&& DECL_NAME (TYPE_NAME (type))
652 	&& !strcmp
653 	     (IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type))),
654 	      aarch64_simd_types[i].name))
655       return aarch64_simd_types[i].mangle;
656 
657   return NULL;
658 }
659 
660 const char *
aarch64_general_mangle_builtin_type(const_tree type)661 aarch64_general_mangle_builtin_type (const_tree type)
662 {
663   const char *mangle;
664   /* Walk through all the AArch64 builtins types tables to filter out the
665      incoming type.  */
666   if ((mangle = aarch64_mangle_builtin_vector_type (type))
667       || (mangle = aarch64_mangle_builtin_scalar_type (type)))
668     return mangle;
669 
670   return NULL;
671 }
672 
673 static tree
aarch64_simd_builtin_std_type(machine_mode mode,enum aarch64_type_qualifiers q)674 aarch64_simd_builtin_std_type (machine_mode mode,
675 			       enum aarch64_type_qualifiers q)
676 {
677 #define QUAL_TYPE(M)  \
678   ((q == qualifier_none) ? int##M##_type_node : unsigned_int##M##_type_node);
679   switch (mode)
680     {
681     case E_QImode:
682       return QUAL_TYPE (QI);
683     case E_HImode:
684       return QUAL_TYPE (HI);
685     case E_SImode:
686       return QUAL_TYPE (SI);
687     case E_DImode:
688       return QUAL_TYPE (DI);
689     case E_TImode:
690       return QUAL_TYPE (TI);
691     case E_OImode:
692       return aarch64_simd_intOI_type_node;
693     case E_CImode:
694       return aarch64_simd_intCI_type_node;
695     case E_XImode:
696       return aarch64_simd_intXI_type_node;
697     case E_HFmode:
698       return aarch64_fp16_type_node;
699     case E_SFmode:
700       return float_type_node;
701     case E_DFmode:
702       return double_type_node;
703     case E_BFmode:
704       return aarch64_bf16_type_node;
705     default:
706       gcc_unreachable ();
707     }
708 #undef QUAL_TYPE
709 }
710 
711 static tree
aarch64_lookup_simd_builtin_type(machine_mode mode,enum aarch64_type_qualifiers q)712 aarch64_lookup_simd_builtin_type (machine_mode mode,
713 				  enum aarch64_type_qualifiers q)
714 {
715   int i;
716   int nelts = sizeof (aarch64_simd_types) / sizeof (aarch64_simd_types[0]);
717 
718   /* Non-poly scalar modes map to standard types not in the table.  */
719   if (q != qualifier_poly && !VECTOR_MODE_P (mode))
720     return aarch64_simd_builtin_std_type (mode, q);
721 
722   for (i = 0; i < nelts; i++)
723     if (aarch64_simd_types[i].mode == mode
724 	&& aarch64_simd_types[i].q == q)
725       return aarch64_simd_types[i].itype;
726 
727   return NULL_TREE;
728 }
729 
730 static tree
aarch64_simd_builtin_type(machine_mode mode,bool unsigned_p,bool poly_p)731 aarch64_simd_builtin_type (machine_mode mode,
732 			   bool unsigned_p, bool poly_p)
733 {
734   if (poly_p)
735     return aarch64_lookup_simd_builtin_type (mode, qualifier_poly);
736   else if (unsigned_p)
737     return aarch64_lookup_simd_builtin_type (mode, qualifier_unsigned);
738   else
739     return aarch64_lookup_simd_builtin_type (mode, qualifier_none);
740 }
741 
742 static void
aarch64_init_simd_builtin_types(void)743 aarch64_init_simd_builtin_types (void)
744 {
745   int i;
746   int nelts = sizeof (aarch64_simd_types) / sizeof (aarch64_simd_types[0]);
747   tree tdecl;
748 
749   /* Init all the element types built by the front-end.  */
750   aarch64_simd_types[Int8x8_t].eltype = intQI_type_node;
751   aarch64_simd_types[Int8x16_t].eltype = intQI_type_node;
752   aarch64_simd_types[Int16x4_t].eltype = intHI_type_node;
753   aarch64_simd_types[Int16x8_t].eltype = intHI_type_node;
754   aarch64_simd_types[Int32x2_t].eltype = intSI_type_node;
755   aarch64_simd_types[Int32x4_t].eltype = intSI_type_node;
756   aarch64_simd_types[Int64x1_t].eltype = intDI_type_node;
757   aarch64_simd_types[Int64x2_t].eltype = intDI_type_node;
758   aarch64_simd_types[Uint8x8_t].eltype = unsigned_intQI_type_node;
759   aarch64_simd_types[Uint8x16_t].eltype = unsigned_intQI_type_node;
760   aarch64_simd_types[Uint16x4_t].eltype = unsigned_intHI_type_node;
761   aarch64_simd_types[Uint16x8_t].eltype = unsigned_intHI_type_node;
762   aarch64_simd_types[Uint32x2_t].eltype = unsigned_intSI_type_node;
763   aarch64_simd_types[Uint32x4_t].eltype = unsigned_intSI_type_node;
764   aarch64_simd_types[Uint64x1_t].eltype = unsigned_intDI_type_node;
765   aarch64_simd_types[Uint64x2_t].eltype = unsigned_intDI_type_node;
766 
767   /* Poly types are a world of their own.  */
768   aarch64_simd_types[Poly8_t].eltype = aarch64_simd_types[Poly8_t].itype =
769     build_distinct_type_copy (unsigned_intQI_type_node);
770   /* Prevent front-ends from transforming Poly8_t arrays into string
771      literals.  */
772   TYPE_STRING_FLAG (aarch64_simd_types[Poly8_t].eltype) = false;
773 
774   aarch64_simd_types[Poly16_t].eltype = aarch64_simd_types[Poly16_t].itype =
775     build_distinct_type_copy (unsigned_intHI_type_node);
776   aarch64_simd_types[Poly64_t].eltype = aarch64_simd_types[Poly64_t].itype =
777     build_distinct_type_copy (unsigned_intDI_type_node);
778   aarch64_simd_types[Poly128_t].eltype = aarch64_simd_types[Poly128_t].itype =
779     build_distinct_type_copy (unsigned_intTI_type_node);
780   /* Init poly vector element types with scalar poly types.  */
781   aarch64_simd_types[Poly8x8_t].eltype = aarch64_simd_types[Poly8_t].itype;
782   aarch64_simd_types[Poly8x16_t].eltype = aarch64_simd_types[Poly8_t].itype;
783   aarch64_simd_types[Poly16x4_t].eltype = aarch64_simd_types[Poly16_t].itype;
784   aarch64_simd_types[Poly16x8_t].eltype = aarch64_simd_types[Poly16_t].itype;
785   aarch64_simd_types[Poly64x1_t].eltype = aarch64_simd_types[Poly64_t].itype;
786   aarch64_simd_types[Poly64x2_t].eltype = aarch64_simd_types[Poly64_t].itype;
787 
788   /* Continue with standard types.  */
789   aarch64_simd_types[Float16x4_t].eltype = aarch64_fp16_type_node;
790   aarch64_simd_types[Float16x8_t].eltype = aarch64_fp16_type_node;
791   aarch64_simd_types[Float32x2_t].eltype = float_type_node;
792   aarch64_simd_types[Float32x4_t].eltype = float_type_node;
793   aarch64_simd_types[Float64x1_t].eltype = double_type_node;
794   aarch64_simd_types[Float64x2_t].eltype = double_type_node;
795 
796   /* Init Bfloat vector types with underlying __bf16 type.  */
797   aarch64_simd_types[Bfloat16x4_t].eltype = aarch64_bf16_type_node;
798   aarch64_simd_types[Bfloat16x8_t].eltype = aarch64_bf16_type_node;
799 
800   for (i = 0; i < nelts; i++)
801     {
802       tree eltype = aarch64_simd_types[i].eltype;
803       machine_mode mode = aarch64_simd_types[i].mode;
804 
805       if (aarch64_simd_types[i].itype == NULL)
806 	{
807 	  tree type = build_vector_type (eltype, GET_MODE_NUNITS (mode));
808 	  type = build_distinct_type_copy (type);
809 	  SET_TYPE_STRUCTURAL_EQUALITY (type);
810 
811 	  TYPE_ATTRIBUTES (type)
812 	    = tree_cons (get_identifier ("Advanced SIMD type"),
813 			 NULL_TREE, TYPE_ATTRIBUTES (type));
814 	  aarch64_simd_types[i].itype = type;
815 	}
816 
817       tdecl = add_builtin_type (aarch64_simd_types[i].name,
818 				aarch64_simd_types[i].itype);
819       TYPE_NAME (aarch64_simd_types[i].itype) = tdecl;
820     }
821 
822 #define AARCH64_BUILD_SIGNED_TYPE(mode)  \
823   make_signed_type (GET_MODE_PRECISION (mode));
824   aarch64_simd_intOI_type_node = AARCH64_BUILD_SIGNED_TYPE (OImode);
825   aarch64_simd_intCI_type_node = AARCH64_BUILD_SIGNED_TYPE (CImode);
826   aarch64_simd_intXI_type_node = AARCH64_BUILD_SIGNED_TYPE (XImode);
827 #undef AARCH64_BUILD_SIGNED_TYPE
828 
829   tdecl = add_builtin_type
830 	    ("__builtin_aarch64_simd_oi" , aarch64_simd_intOI_type_node);
831   TYPE_NAME (aarch64_simd_intOI_type_node) = tdecl;
832   tdecl = add_builtin_type
833 	    ("__builtin_aarch64_simd_ci" , aarch64_simd_intCI_type_node);
834   TYPE_NAME (aarch64_simd_intCI_type_node) = tdecl;
835   tdecl = add_builtin_type
836 	    ("__builtin_aarch64_simd_xi" , aarch64_simd_intXI_type_node);
837   TYPE_NAME (aarch64_simd_intXI_type_node) = tdecl;
838 }
839 
840 static void
aarch64_init_simd_builtin_scalar_types(void)841 aarch64_init_simd_builtin_scalar_types (void)
842 {
843   /* Define typedefs for all the standard scalar types.  */
844   (*lang_hooks.types.register_builtin_type) (intQI_type_node,
845 					     "__builtin_aarch64_simd_qi");
846   (*lang_hooks.types.register_builtin_type) (intHI_type_node,
847 					     "__builtin_aarch64_simd_hi");
848   (*lang_hooks.types.register_builtin_type) (aarch64_fp16_type_node,
849 					     "__builtin_aarch64_simd_hf");
850   (*lang_hooks.types.register_builtin_type) (intSI_type_node,
851 					     "__builtin_aarch64_simd_si");
852   (*lang_hooks.types.register_builtin_type) (float_type_node,
853 					     "__builtin_aarch64_simd_sf");
854   (*lang_hooks.types.register_builtin_type) (intDI_type_node,
855 					     "__builtin_aarch64_simd_di");
856   (*lang_hooks.types.register_builtin_type) (double_type_node,
857 					     "__builtin_aarch64_simd_df");
858   (*lang_hooks.types.register_builtin_type) (unsigned_intQI_type_node,
859 					     "__builtin_aarch64_simd_poly8");
860   (*lang_hooks.types.register_builtin_type) (unsigned_intHI_type_node,
861 					     "__builtin_aarch64_simd_poly16");
862   (*lang_hooks.types.register_builtin_type) (unsigned_intDI_type_node,
863 					     "__builtin_aarch64_simd_poly64");
864   (*lang_hooks.types.register_builtin_type) (unsigned_intTI_type_node,
865 					     "__builtin_aarch64_simd_poly128");
866   (*lang_hooks.types.register_builtin_type) (intTI_type_node,
867 					     "__builtin_aarch64_simd_ti");
868   (*lang_hooks.types.register_builtin_type) (aarch64_bf16_type_node,
869 					     "__builtin_aarch64_simd_bf");
870   /* Unsigned integer types for various mode sizes.  */
871   (*lang_hooks.types.register_builtin_type) (unsigned_intQI_type_node,
872 					     "__builtin_aarch64_simd_uqi");
873   (*lang_hooks.types.register_builtin_type) (unsigned_intHI_type_node,
874 					     "__builtin_aarch64_simd_uhi");
875   (*lang_hooks.types.register_builtin_type) (unsigned_intSI_type_node,
876 					     "__builtin_aarch64_simd_usi");
877   (*lang_hooks.types.register_builtin_type) (unsigned_intDI_type_node,
878 					     "__builtin_aarch64_simd_udi");
879 }
880 
881 static bool aarch64_simd_builtins_initialized_p = false;
882 
883 /* Due to the architecture not providing lane variant of the lane instructions
884    for fcmla we can't use the standard simd builtin expansion code, but we
885    still want the majority of the validation that would normally be done.  */
886 
887 void
aarch64_init_fcmla_laneq_builtins(void)888 aarch64_init_fcmla_laneq_builtins (void)
889 {
890   unsigned int i = 0;
891 
892   for (i = 0; i < ARRAY_SIZE (aarch64_fcmla_lane_builtin_data); ++i)
893     {
894       aarch64_fcmla_laneq_builtin_datum* d
895 	= &aarch64_fcmla_lane_builtin_data[i];
896       tree argtype = aarch64_lookup_simd_builtin_type (d->mode, qualifier_none);
897       machine_mode quadmode = GET_MODE_2XWIDER_MODE (d->mode).require ();
898       tree quadtype
899 	= aarch64_lookup_simd_builtin_type (quadmode, qualifier_none);
900       tree lanetype
901 	= aarch64_simd_builtin_std_type (SImode, qualifier_lane_pair_index);
902       tree ftype = build_function_type_list (argtype, argtype, argtype,
903 					     quadtype, lanetype, NULL_TREE);
904       tree fndecl = aarch64_general_add_builtin (d->name, ftype, d->fcode);
905 
906       aarch64_builtin_decls[d->fcode] = fndecl;
907     }
908 }
909 
910 void
aarch64_init_simd_builtins(void)911 aarch64_init_simd_builtins (void)
912 {
913   unsigned int i, fcode = AARCH64_SIMD_PATTERN_START;
914 
915   if (aarch64_simd_builtins_initialized_p)
916     return;
917 
918   aarch64_simd_builtins_initialized_p = true;
919 
920   aarch64_init_simd_builtin_types ();
921 
922   /* Strong-typing hasn't been implemented for all AdvSIMD builtin intrinsics.
923      Therefore we need to preserve the old __builtin scalar types.  It can be
924      removed once all the intrinsics become strongly typed using the qualifier
925      system.  */
926   aarch64_init_simd_builtin_scalar_types ();
927 
928   tree lane_check_fpr = build_function_type_list (void_type_node,
929 						  size_type_node,
930 						  size_type_node,
931 						  intSI_type_node,
932 						  NULL);
933   aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_LANE_CHECK]
934     = aarch64_general_add_builtin ("__builtin_aarch64_im_lane_boundsi",
935 				   lane_check_fpr,
936 				   AARCH64_SIMD_BUILTIN_LANE_CHECK);
937 
938   for (i = 0; i < ARRAY_SIZE (aarch64_simd_builtin_data); i++, fcode++)
939     {
940       bool print_type_signature_p = false;
941       char type_signature[SIMD_MAX_BUILTIN_ARGS + 1] = { 0 };
942       aarch64_simd_builtin_datum *d = &aarch64_simd_builtin_data[i];
943       char namebuf[60];
944       tree ftype = NULL;
945       tree fndecl = NULL;
946 
947       d->fcode = fcode;
948 
949       /* We must track two variables here.  op_num is
950 	 the operand number as in the RTL pattern.  This is
951 	 required to access the mode (e.g. V4SF mode) of the
952 	 argument, from which the base type can be derived.
953 	 arg_num is an index in to the qualifiers data, which
954 	 gives qualifiers to the type (e.g. const unsigned).
955 	 The reason these two variables may differ by one is the
956 	 void return type.  While all return types take the 0th entry
957 	 in the qualifiers array, there is no operand for them in the
958 	 RTL pattern.  */
959       int op_num = insn_data[d->code].n_operands - 1;
960       int arg_num = d->qualifiers[0] & qualifier_void
961 		      ? op_num + 1
962 		      : op_num;
963       tree return_type = void_type_node, args = void_list_node;
964       tree eltype;
965 
966       /* Build a function type directly from the insn_data for this
967 	 builtin.  The build_function_type () function takes care of
968 	 removing duplicates for us.  */
969       for (; op_num >= 0; arg_num--, op_num--)
970 	{
971 	  machine_mode op_mode = insn_data[d->code].operand[op_num].mode;
972 	  enum aarch64_type_qualifiers qualifiers = d->qualifiers[arg_num];
973 
974 	  if (qualifiers & qualifier_unsigned)
975 	    {
976 	      type_signature[op_num] = 'u';
977 	      print_type_signature_p = true;
978 	    }
979 	  else if (qualifiers & qualifier_poly)
980 	    {
981 	      type_signature[op_num] = 'p';
982 	      print_type_signature_p = true;
983 	    }
984 	  else
985 	    type_signature[op_num] = 's';
986 
987 	  /* Skip an internal operand for vget_{low, high}.  */
988 	  if (qualifiers & qualifier_internal)
989 	    continue;
990 
991 	  /* Some builtins have different user-facing types
992 	     for certain arguments, encoded in d->mode.  */
993 	  if (qualifiers & qualifier_map_mode)
994 	      op_mode = d->mode;
995 
996 	  /* For pointers, we want a pointer to the basic type
997 	     of the vector.  */
998 	  if (qualifiers & qualifier_pointer && VECTOR_MODE_P (op_mode))
999 	    op_mode = GET_MODE_INNER (op_mode);
1000 
1001 	  eltype = aarch64_simd_builtin_type
1002 		     (op_mode,
1003 		      (qualifiers & qualifier_unsigned) != 0,
1004 		      (qualifiers & qualifier_poly) != 0);
1005 	  gcc_assert (eltype != NULL);
1006 
1007 	  /* Add qualifiers.  */
1008 	  if (qualifiers & qualifier_const)
1009 	    eltype = build_qualified_type (eltype, TYPE_QUAL_CONST);
1010 
1011 	  if (qualifiers & qualifier_pointer)
1012 	      eltype = build_pointer_type (eltype);
1013 
1014 	  /* If we have reached arg_num == 0, we are at a non-void
1015 	     return type.  Otherwise, we are still processing
1016 	     arguments.  */
1017 	  if (arg_num == 0)
1018 	    return_type = eltype;
1019 	  else
1020 	    args = tree_cons (NULL_TREE, eltype, args);
1021 	}
1022 
1023       ftype = build_function_type (return_type, args);
1024 
1025       gcc_assert (ftype != NULL);
1026 
1027       if (print_type_signature_p)
1028 	snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s_%s",
1029 		  d->name, type_signature);
1030       else
1031 	snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s",
1032 		  d->name);
1033 
1034       fndecl = aarch64_general_add_builtin (namebuf, ftype, fcode);
1035       aarch64_builtin_decls[fcode] = fndecl;
1036     }
1037 
1038    /* Initialize the remaining fcmla_laneq intrinsics.  */
1039    aarch64_init_fcmla_laneq_builtins ();
1040 }
1041 
1042 static void
aarch64_init_crc32_builtins()1043 aarch64_init_crc32_builtins ()
1044 {
1045   tree usi_type = aarch64_simd_builtin_std_type (SImode, qualifier_unsigned);
1046   unsigned int i = 0;
1047 
1048   for (i = 0; i < ARRAY_SIZE (aarch64_crc_builtin_data); ++i)
1049     {
1050       aarch64_crc_builtin_datum* d = &aarch64_crc_builtin_data[i];
1051       tree argtype = aarch64_simd_builtin_std_type (d->mode,
1052 						    qualifier_unsigned);
1053       tree ftype = build_function_type_list (usi_type, usi_type, argtype, NULL_TREE);
1054       tree fndecl = aarch64_general_add_builtin (d->name, ftype, d->fcode);
1055 
1056       aarch64_builtin_decls[d->fcode] = fndecl;
1057     }
1058 }
1059 
1060 /* Add builtins for reciprocal square root.  */
1061 
1062 void
aarch64_init_builtin_rsqrt(void)1063 aarch64_init_builtin_rsqrt (void)
1064 {
1065   tree fndecl = NULL;
1066   tree ftype = NULL;
1067 
1068   tree V2SF_type_node = build_vector_type (float_type_node, 2);
1069   tree V2DF_type_node = build_vector_type (double_type_node, 2);
1070   tree V4SF_type_node = build_vector_type (float_type_node, 4);
1071 
1072   struct builtin_decls_data
1073   {
1074     tree type_node;
1075     const char *builtin_name;
1076     int function_code;
1077   };
1078 
1079   builtin_decls_data bdda[] =
1080   {
1081     { double_type_node, "__builtin_aarch64_rsqrt_df", AARCH64_BUILTIN_RSQRT_DF },
1082     { float_type_node, "__builtin_aarch64_rsqrt_sf", AARCH64_BUILTIN_RSQRT_SF },
1083     { V2DF_type_node, "__builtin_aarch64_rsqrt_v2df", AARCH64_BUILTIN_RSQRT_V2DF },
1084     { V2SF_type_node, "__builtin_aarch64_rsqrt_v2sf", AARCH64_BUILTIN_RSQRT_V2SF },
1085     { V4SF_type_node, "__builtin_aarch64_rsqrt_v4sf", AARCH64_BUILTIN_RSQRT_V4SF }
1086   };
1087 
1088   builtin_decls_data *bdd = bdda;
1089   builtin_decls_data *bdd_end = bdd + (sizeof (bdda) / sizeof (builtin_decls_data));
1090 
1091   for (; bdd < bdd_end; bdd++)
1092   {
1093     ftype = build_function_type_list (bdd->type_node, bdd->type_node, NULL_TREE);
1094     fndecl = aarch64_general_add_builtin (bdd->builtin_name,
1095 					  ftype, bdd->function_code);
1096     aarch64_builtin_decls[bdd->function_code] = fndecl;
1097   }
1098 }
1099 
1100 /* Initialize the backend types that support the user-visible __fp16
1101    type, also initialize a pointer to that type, to be used when
1102    forming HFAs.  */
1103 
1104 static void
aarch64_init_fp16_types(void)1105 aarch64_init_fp16_types (void)
1106 {
1107   aarch64_fp16_type_node = make_node (REAL_TYPE);
1108   TYPE_PRECISION (aarch64_fp16_type_node) = 16;
1109   layout_type (aarch64_fp16_type_node);
1110 
1111   (*lang_hooks.types.register_builtin_type) (aarch64_fp16_type_node, "__fp16");
1112   aarch64_fp16_ptr_type_node = build_pointer_type (aarch64_fp16_type_node);
1113 }
1114 
1115 /* Initialize the backend REAL_TYPE type supporting bfloat types.  */
1116 static void
aarch64_init_bf16_types(void)1117 aarch64_init_bf16_types (void)
1118 {
1119   aarch64_bf16_type_node = make_node (REAL_TYPE);
1120   TYPE_PRECISION (aarch64_bf16_type_node) = 16;
1121   SET_TYPE_MODE (aarch64_bf16_type_node, BFmode);
1122   layout_type (aarch64_bf16_type_node);
1123 
1124   lang_hooks.types.register_builtin_type (aarch64_bf16_type_node, "__bf16");
1125   aarch64_bf16_ptr_type_node = build_pointer_type (aarch64_bf16_type_node);
1126 }
1127 
1128 /* Pointer authentication builtins that will become NOP on legacy platform.
1129    Currently, these builtins are for internal use only (libgcc EH unwinder).  */
1130 
1131 void
aarch64_init_pauth_hint_builtins(void)1132 aarch64_init_pauth_hint_builtins (void)
1133 {
1134   /* Pointer Authentication builtins.  */
1135   tree ftype_pointer_auth
1136     = build_function_type_list (ptr_type_node, ptr_type_node,
1137 				unsigned_intDI_type_node, NULL_TREE);
1138   tree ftype_pointer_strip
1139     = build_function_type_list (ptr_type_node, ptr_type_node, NULL_TREE);
1140 
1141   aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_AUTIA1716]
1142     = aarch64_general_add_builtin ("__builtin_aarch64_autia1716",
1143 				   ftype_pointer_auth,
1144 				   AARCH64_PAUTH_BUILTIN_AUTIA1716);
1145   aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_PACIA1716]
1146     = aarch64_general_add_builtin ("__builtin_aarch64_pacia1716",
1147 				   ftype_pointer_auth,
1148 				   AARCH64_PAUTH_BUILTIN_PACIA1716);
1149   aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_AUTIB1716]
1150     = aarch64_general_add_builtin ("__builtin_aarch64_autib1716",
1151 				   ftype_pointer_auth,
1152 				   AARCH64_PAUTH_BUILTIN_AUTIB1716);
1153   aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_PACIB1716]
1154     = aarch64_general_add_builtin ("__builtin_aarch64_pacib1716",
1155 				   ftype_pointer_auth,
1156 				   AARCH64_PAUTH_BUILTIN_PACIB1716);
1157   aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_XPACLRI]
1158     = aarch64_general_add_builtin ("__builtin_aarch64_xpaclri",
1159 				   ftype_pointer_strip,
1160 				   AARCH64_PAUTH_BUILTIN_XPACLRI);
1161 }
1162 
1163 /* Initialize the transactional memory extension (TME) builtins.  */
1164 static void
aarch64_init_tme_builtins(void)1165 aarch64_init_tme_builtins (void)
1166 {
1167   tree ftype_uint64_void
1168     = build_function_type_list (uint64_type_node, NULL);
1169   tree ftype_void_void
1170     = build_function_type_list (void_type_node, NULL);
1171   tree ftype_void_uint64
1172     = build_function_type_list (void_type_node, uint64_type_node, NULL);
1173 
1174   aarch64_builtin_decls[AARCH64_TME_BUILTIN_TSTART]
1175     = aarch64_general_add_builtin ("__builtin_aarch64_tstart",
1176 				   ftype_uint64_void,
1177 				   AARCH64_TME_BUILTIN_TSTART);
1178   aarch64_builtin_decls[AARCH64_TME_BUILTIN_TTEST]
1179     = aarch64_general_add_builtin ("__builtin_aarch64_ttest",
1180 				   ftype_uint64_void,
1181 				   AARCH64_TME_BUILTIN_TTEST);
1182   aarch64_builtin_decls[AARCH64_TME_BUILTIN_TCOMMIT]
1183     = aarch64_general_add_builtin ("__builtin_aarch64_tcommit",
1184 				   ftype_void_void,
1185 				   AARCH64_TME_BUILTIN_TCOMMIT);
1186   aarch64_builtin_decls[AARCH64_TME_BUILTIN_TCANCEL]
1187     = aarch64_general_add_builtin ("__builtin_aarch64_tcancel",
1188 				   ftype_void_uint64,
1189 				   AARCH64_TME_BUILTIN_TCANCEL);
1190 }
1191 
1192 /* Add builtins for Random Number instructions.  */
1193 
1194 static void
aarch64_init_rng_builtins(void)1195 aarch64_init_rng_builtins (void)
1196 {
1197   tree unsigned_ptr_type = build_pointer_type (unsigned_intDI_type_node);
1198   tree ftype
1199     = build_function_type_list (integer_type_node, unsigned_ptr_type, NULL);
1200   aarch64_builtin_decls[AARCH64_BUILTIN_RNG_RNDR]
1201     = aarch64_general_add_builtin ("__builtin_aarch64_rndr", ftype,
1202 				   AARCH64_BUILTIN_RNG_RNDR);
1203   aarch64_builtin_decls[AARCH64_BUILTIN_RNG_RNDRRS]
1204     = aarch64_general_add_builtin ("__builtin_aarch64_rndrrs", ftype,
1205 				   AARCH64_BUILTIN_RNG_RNDRRS);
1206 }
1207 
1208 /* Initialize the memory tagging extension (MTE) builtins.  */
1209 struct aarch64_mte
1210 {
1211   tree ftype;
1212   enum insn_code icode;
1213 } aarch64_memtag_builtin_data[AARCH64_MEMTAG_BUILTIN_END -
1214 			      AARCH64_MEMTAG_BUILTIN_START - 1];
1215 
1216 static void
aarch64_init_memtag_builtins(void)1217 aarch64_init_memtag_builtins (void)
1218 {
1219   tree fntype = NULL;
1220 
1221 #define AARCH64_INIT_MEMTAG_BUILTINS_DECL(F, N, I, T) \
1222   aarch64_builtin_decls[AARCH64_MEMTAG_BUILTIN_##F] \
1223     = aarch64_general_add_builtin ("__builtin_aarch64_memtag_"#N, \
1224 				   T, AARCH64_MEMTAG_BUILTIN_##F); \
1225   aarch64_memtag_builtin_data[AARCH64_MEMTAG_BUILTIN_##F - \
1226 			      AARCH64_MEMTAG_BUILTIN_START - 1].ftype = T; \
1227   aarch64_memtag_builtin_data[AARCH64_MEMTAG_BUILTIN_##F - \
1228 			      AARCH64_MEMTAG_BUILTIN_START - 1].icode = CODE_FOR_##I;
1229 
1230   fntype = build_function_type_list (ptr_type_node, ptr_type_node,
1231 				     uint64_type_node, NULL);
1232   AARCH64_INIT_MEMTAG_BUILTINS_DECL (IRG, irg, irg, fntype);
1233 
1234   fntype = build_function_type_list (uint64_type_node, ptr_type_node,
1235 				     uint64_type_node, NULL);
1236   AARCH64_INIT_MEMTAG_BUILTINS_DECL (GMI, gmi, gmi, fntype);
1237 
1238   fntype = build_function_type_list (ptrdiff_type_node, ptr_type_node,
1239 				     ptr_type_node, NULL);
1240   AARCH64_INIT_MEMTAG_BUILTINS_DECL (SUBP, subp, subp, fntype);
1241 
1242   fntype = build_function_type_list (ptr_type_node, ptr_type_node,
1243 				     unsigned_type_node, NULL);
1244   AARCH64_INIT_MEMTAG_BUILTINS_DECL (INC_TAG, inc_tag, addg, fntype);
1245 
1246   fntype = build_function_type_list (void_type_node, ptr_type_node, NULL);
1247   AARCH64_INIT_MEMTAG_BUILTINS_DECL (SET_TAG, set_tag, stg, fntype);
1248 
1249   fntype = build_function_type_list (ptr_type_node, ptr_type_node, NULL);
1250   AARCH64_INIT_MEMTAG_BUILTINS_DECL (GET_TAG, get_tag, ldg, fntype);
1251 
1252 #undef AARCH64_INIT_MEMTAG_BUILTINS_DECL
1253 }
1254 
1255 /* Initialize all builtins in the AARCH64_BUILTIN_GENERAL group.  */
1256 
1257 void
aarch64_general_init_builtins(void)1258 aarch64_general_init_builtins (void)
1259 {
1260   tree ftype_set_fpr
1261     = build_function_type_list (void_type_node, unsigned_type_node, NULL);
1262   tree ftype_get_fpr
1263     = build_function_type_list (unsigned_type_node, NULL);
1264 
1265   aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR]
1266     = aarch64_general_add_builtin ("__builtin_aarch64_get_fpcr",
1267 				   ftype_get_fpr,
1268 				   AARCH64_BUILTIN_GET_FPCR);
1269   aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR]
1270     = aarch64_general_add_builtin ("__builtin_aarch64_set_fpcr",
1271 				   ftype_set_fpr,
1272 				   AARCH64_BUILTIN_SET_FPCR);
1273   aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR]
1274     = aarch64_general_add_builtin ("__builtin_aarch64_get_fpsr",
1275 				   ftype_get_fpr,
1276 				   AARCH64_BUILTIN_GET_FPSR);
1277   aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR]
1278     = aarch64_general_add_builtin ("__builtin_aarch64_set_fpsr",
1279 				   ftype_set_fpr,
1280 				   AARCH64_BUILTIN_SET_FPSR);
1281 
1282   aarch64_init_fp16_types ();
1283 
1284   aarch64_init_bf16_types ();
1285 
1286   if (TARGET_SIMD)
1287     aarch64_init_simd_builtins ();
1288 
1289   aarch64_init_crc32_builtins ();
1290   aarch64_init_builtin_rsqrt ();
1291   aarch64_init_rng_builtins ();
1292 
1293   tree ftype_jcvt
1294     = build_function_type_list (intSI_type_node, double_type_node, NULL);
1295   aarch64_builtin_decls[AARCH64_JSCVT]
1296     = aarch64_general_add_builtin ("__builtin_aarch64_jcvtzs", ftype_jcvt,
1297 				   AARCH64_JSCVT);
1298 
1299   /* Initialize pointer authentication builtins which are backed by instructions
1300      in NOP encoding space.
1301 
1302      NOTE: these builtins are supposed to be used by libgcc unwinder only, as
1303      there is no support on return address signing under ILP32, we don't
1304      register them.  */
1305   if (!TARGET_ILP32)
1306     aarch64_init_pauth_hint_builtins ();
1307 
1308   if (TARGET_TME)
1309     aarch64_init_tme_builtins ();
1310 
1311   if (TARGET_MEMTAG)
1312     aarch64_init_memtag_builtins ();
1313 }
1314 
1315 /* Implement TARGET_BUILTIN_DECL for the AARCH64_BUILTIN_GENERAL group.  */
1316 tree
aarch64_general_builtin_decl(unsigned code,bool)1317 aarch64_general_builtin_decl (unsigned code, bool)
1318 {
1319   if (code >= AARCH64_BUILTIN_MAX)
1320     return error_mark_node;
1321 
1322   return aarch64_builtin_decls[code];
1323 }
1324 
1325 typedef enum
1326 {
1327   SIMD_ARG_COPY_TO_REG,
1328   SIMD_ARG_CONSTANT,
1329   SIMD_ARG_LANE_INDEX,
1330   SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX,
1331   SIMD_ARG_LANE_PAIR_INDEX,
1332   SIMD_ARG_LANE_QUADTUP_INDEX,
1333   SIMD_ARG_STOP
1334 } builtin_simd_arg;
1335 
1336 
1337 static rtx
aarch64_simd_expand_args(rtx target,int icode,int have_retval,tree exp,builtin_simd_arg * args,machine_mode builtin_mode)1338 aarch64_simd_expand_args (rtx target, int icode, int have_retval,
1339 			  tree exp, builtin_simd_arg *args,
1340 			  machine_mode builtin_mode)
1341 {
1342   rtx pat;
1343   rtx op[SIMD_MAX_BUILTIN_ARGS + 1]; /* First element for result operand.  */
1344   int opc = 0;
1345 
1346   if (have_retval)
1347     {
1348       machine_mode tmode = insn_data[icode].operand[0].mode;
1349       if (!target
1350 	  || GET_MODE (target) != tmode
1351 	  || !(*insn_data[icode].operand[0].predicate) (target, tmode))
1352 	target = gen_reg_rtx (tmode);
1353       op[opc++] = target;
1354     }
1355 
1356   for (;;)
1357     {
1358       builtin_simd_arg thisarg = args[opc - have_retval];
1359 
1360       if (thisarg == SIMD_ARG_STOP)
1361 	break;
1362       else
1363 	{
1364 	  tree arg = CALL_EXPR_ARG (exp, opc - have_retval);
1365 	  machine_mode mode = insn_data[icode].operand[opc].mode;
1366 	  op[opc] = expand_normal (arg);
1367 
1368 	  switch (thisarg)
1369 	    {
1370 	    case SIMD_ARG_COPY_TO_REG:
1371 	      if (POINTER_TYPE_P (TREE_TYPE (arg)))
1372 		op[opc] = convert_memory_address (Pmode, op[opc]);
1373 	      /*gcc_assert (GET_MODE (op[opc]) == mode); */
1374 	      if (!(*insn_data[icode].operand[opc].predicate)
1375 		  (op[opc], mode))
1376 		op[opc] = copy_to_mode_reg (mode, op[opc]);
1377 	      break;
1378 
1379 	    case SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX:
1380 	      gcc_assert (opc > 1);
1381 	      if (CONST_INT_P (op[opc]))
1382 		{
1383 		  unsigned int nunits
1384 		    = GET_MODE_NUNITS (builtin_mode).to_constant ();
1385 		  aarch64_simd_lane_bounds (op[opc], 0, nunits, exp);
1386 		  /* Keep to GCC-vector-extension lane indices in the RTL.  */
1387 		  op[opc] = aarch64_endian_lane_rtx (builtin_mode,
1388 						     INTVAL (op[opc]));
1389 		}
1390 	      goto constant_arg;
1391 
1392 	    case SIMD_ARG_LANE_INDEX:
1393 	      /* Must be a previous operand into which this is an index.  */
1394 	      gcc_assert (opc > 0);
1395 	      if (CONST_INT_P (op[opc]))
1396 		{
1397 		  machine_mode vmode = insn_data[icode].operand[opc - 1].mode;
1398 		  unsigned int nunits
1399 		    = GET_MODE_NUNITS (vmode).to_constant ();
1400 		  aarch64_simd_lane_bounds (op[opc], 0, nunits, exp);
1401 		  /* Keep to GCC-vector-extension lane indices in the RTL.  */
1402 		  op[opc] = aarch64_endian_lane_rtx (vmode, INTVAL (op[opc]));
1403 		}
1404 	      /* If the lane index isn't a constant then error out.  */
1405 	      goto constant_arg;
1406 
1407 	    case SIMD_ARG_LANE_PAIR_INDEX:
1408 	      /* Must be a previous operand into which this is an index and
1409 		 index is restricted to nunits / 2.  */
1410 	      gcc_assert (opc > 0);
1411 	      if (CONST_INT_P (op[opc]))
1412 		{
1413 		  machine_mode vmode = insn_data[icode].operand[opc - 1].mode;
1414 		  unsigned int nunits
1415 		    = GET_MODE_NUNITS (vmode).to_constant ();
1416 		  aarch64_simd_lane_bounds (op[opc], 0, nunits / 2, exp);
1417 		  /* Keep to GCC-vector-extension lane indices in the RTL.  */
1418 		  int lane = INTVAL (op[opc]);
1419 		  op[opc] = gen_int_mode (ENDIAN_LANE_N (nunits / 2, lane),
1420 					  SImode);
1421 		}
1422 	      /* If the lane index isn't a constant then error out.  */
1423 	      goto constant_arg;
1424 	    case SIMD_ARG_LANE_QUADTUP_INDEX:
1425 	      /* Must be a previous operand into which this is an index and
1426 		 index is restricted to nunits / 4.  */
1427 	      gcc_assert (opc > 0);
1428 	      if (CONST_INT_P (op[opc]))
1429 		{
1430 		  machine_mode vmode = insn_data[icode].operand[opc - 1].mode;
1431 		  unsigned int nunits
1432 		    = GET_MODE_NUNITS (vmode).to_constant ();
1433 		  aarch64_simd_lane_bounds (op[opc], 0, nunits / 4, exp);
1434 		  /* Keep to GCC-vector-extension lane indices in the RTL.  */
1435 		  int lane = INTVAL (op[opc]);
1436 		  op[opc] = gen_int_mode (ENDIAN_LANE_N (nunits / 4, lane),
1437 					  SImode);
1438 		}
1439 	      /* If the lane index isn't a constant then error out.  */
1440 	      goto constant_arg;
1441 	    case SIMD_ARG_CONSTANT:
1442 constant_arg:
1443 	      if (!(*insn_data[icode].operand[opc].predicate)
1444 		  (op[opc], mode))
1445 	      {
1446 		error ("%Kargument %d must be a constant immediate",
1447 		       exp, opc + 1 - have_retval);
1448 		return const0_rtx;
1449 	      }
1450 	      break;
1451 
1452 	    case SIMD_ARG_STOP:
1453 	      gcc_unreachable ();
1454 	    }
1455 
1456 	  opc++;
1457 	}
1458     }
1459 
1460   switch (opc)
1461     {
1462     case 1:
1463       pat = GEN_FCN (icode) (op[0]);
1464       break;
1465 
1466     case 2:
1467       pat = GEN_FCN (icode) (op[0], op[1]);
1468       break;
1469 
1470     case 3:
1471       pat = GEN_FCN (icode) (op[0], op[1], op[2]);
1472       break;
1473 
1474     case 4:
1475       pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3]);
1476       break;
1477 
1478     case 5:
1479       pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4]);
1480       break;
1481 
1482     case 6:
1483       pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5]);
1484       break;
1485 
1486     default:
1487       gcc_unreachable ();
1488     }
1489 
1490   if (!pat)
1491     return NULL_RTX;
1492 
1493   emit_insn (pat);
1494 
1495   return target;
1496 }
1497 
1498 /* Expand an AArch64 AdvSIMD builtin(intrinsic).  */
1499 rtx
aarch64_simd_expand_builtin(int fcode,tree exp,rtx target)1500 aarch64_simd_expand_builtin (int fcode, tree exp, rtx target)
1501 {
1502   if (fcode == AARCH64_SIMD_BUILTIN_LANE_CHECK)
1503     {
1504       rtx totalsize = expand_normal (CALL_EXPR_ARG (exp, 0));
1505       rtx elementsize = expand_normal (CALL_EXPR_ARG (exp, 1));
1506       if (CONST_INT_P (totalsize) && CONST_INT_P (elementsize)
1507 	  && UINTVAL (elementsize) != 0
1508 	  && UINTVAL (totalsize) != 0)
1509 	{
1510 	  rtx lane_idx = expand_normal (CALL_EXPR_ARG (exp, 2));
1511           if (CONST_INT_P (lane_idx))
1512 	    aarch64_simd_lane_bounds (lane_idx, 0,
1513 				      UINTVAL (totalsize)
1514 				       / UINTVAL (elementsize),
1515 				      exp);
1516           else
1517 	    error ("%Klane index must be a constant immediate", exp);
1518 	}
1519       else
1520 	error ("%Ktotal size and element size must be a non-zero constant immediate", exp);
1521       /* Don't generate any RTL.  */
1522       return const0_rtx;
1523     }
1524   aarch64_simd_builtin_datum *d =
1525 		&aarch64_simd_builtin_data[fcode - AARCH64_SIMD_PATTERN_START];
1526   enum insn_code icode = d->code;
1527   builtin_simd_arg args[SIMD_MAX_BUILTIN_ARGS + 1];
1528   int num_args = insn_data[d->code].n_operands;
1529   int is_void = 0;
1530   int k;
1531 
1532   is_void = !!(d->qualifiers[0] & qualifier_void);
1533 
1534   num_args += is_void;
1535 
1536   for (k = 1; k < num_args; k++)
1537     {
1538       /* We have four arrays of data, each indexed in a different fashion.
1539 	 qualifiers - element 0 always describes the function return type.
1540 	 operands - element 0 is either the operand for return value (if
1541 	   the function has a non-void return type) or the operand for the
1542 	   first argument.
1543 	 expr_args - element 0 always holds the first argument.
1544 	 args - element 0 is always used for the return type.  */
1545       int qualifiers_k = k;
1546       int operands_k = k - is_void;
1547       int expr_args_k = k - 1;
1548 
1549       if (d->qualifiers[qualifiers_k] & qualifier_lane_index)
1550 	args[k] = SIMD_ARG_LANE_INDEX;
1551       else if (d->qualifiers[qualifiers_k] & qualifier_lane_pair_index)
1552 	args[k] = SIMD_ARG_LANE_PAIR_INDEX;
1553       else if (d->qualifiers[qualifiers_k] & qualifier_lane_quadtup_index)
1554 	args[k] = SIMD_ARG_LANE_QUADTUP_INDEX;
1555       else if (d->qualifiers[qualifiers_k] & qualifier_struct_load_store_lane_index)
1556 	args[k] = SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX;
1557       else if (d->qualifiers[qualifiers_k] & qualifier_immediate)
1558 	args[k] = SIMD_ARG_CONSTANT;
1559       else if (d->qualifiers[qualifiers_k] & qualifier_maybe_immediate)
1560 	{
1561 	  rtx arg
1562 	    = expand_normal (CALL_EXPR_ARG (exp,
1563 					    (expr_args_k)));
1564 	  /* Handle constants only if the predicate allows it.  */
1565 	  bool op_const_int_p =
1566 	    (CONST_INT_P (arg)
1567 	     && (*insn_data[icode].operand[operands_k].predicate)
1568 		(arg, insn_data[icode].operand[operands_k].mode));
1569 	  args[k] = op_const_int_p ? SIMD_ARG_CONSTANT : SIMD_ARG_COPY_TO_REG;
1570 	}
1571       else
1572 	args[k] = SIMD_ARG_COPY_TO_REG;
1573 
1574     }
1575   args[k] = SIMD_ARG_STOP;
1576 
1577   /* The interface to aarch64_simd_expand_args expects a 0 if
1578      the function is void, and a 1 if it is not.  */
1579   return aarch64_simd_expand_args
1580 	  (target, icode, !is_void, exp, &args[1], d->mode);
1581 }
1582 
1583 rtx
aarch64_crc32_expand_builtin(int fcode,tree exp,rtx target)1584 aarch64_crc32_expand_builtin (int fcode, tree exp, rtx target)
1585 {
1586   rtx pat;
1587   aarch64_crc_builtin_datum *d
1588     = &aarch64_crc_builtin_data[fcode - (AARCH64_CRC32_BUILTIN_BASE + 1)];
1589   enum insn_code icode = d->icode;
1590   tree arg0 = CALL_EXPR_ARG (exp, 0);
1591   tree arg1 = CALL_EXPR_ARG (exp, 1);
1592   rtx op0 = expand_normal (arg0);
1593   rtx op1 = expand_normal (arg1);
1594   machine_mode tmode = insn_data[icode].operand[0].mode;
1595   machine_mode mode0 = insn_data[icode].operand[1].mode;
1596   machine_mode mode1 = insn_data[icode].operand[2].mode;
1597 
1598   if (! target
1599       || GET_MODE (target) != tmode
1600       || ! (*insn_data[icode].operand[0].predicate) (target, tmode))
1601     target = gen_reg_rtx (tmode);
1602 
1603   gcc_assert ((GET_MODE (op0) == mode0 || GET_MODE (op0) == VOIDmode)
1604 	      && (GET_MODE (op1) == mode1 || GET_MODE (op1) == VOIDmode));
1605 
1606   if (! (*insn_data[icode].operand[1].predicate) (op0, mode0))
1607     op0 = copy_to_mode_reg (mode0, op0);
1608   if (! (*insn_data[icode].operand[2].predicate) (op1, mode1))
1609     op1 = copy_to_mode_reg (mode1, op1);
1610 
1611   pat = GEN_FCN (icode) (target, op0, op1);
1612   if (!pat)
1613     return NULL_RTX;
1614 
1615   emit_insn (pat);
1616   return target;
1617 }
1618 
1619 /* Function to expand reciprocal square root builtins.  */
1620 
1621 static rtx
aarch64_expand_builtin_rsqrt(int fcode,tree exp,rtx target)1622 aarch64_expand_builtin_rsqrt (int fcode, tree exp, rtx target)
1623 {
1624   tree arg0 = CALL_EXPR_ARG (exp, 0);
1625   rtx op0 = expand_normal (arg0);
1626 
1627   rtx (*gen) (rtx, rtx);
1628 
1629   switch (fcode)
1630     {
1631       case AARCH64_BUILTIN_RSQRT_DF:
1632 	gen = gen_rsqrtdf2;
1633 	break;
1634       case AARCH64_BUILTIN_RSQRT_SF:
1635 	gen = gen_rsqrtsf2;
1636 	break;
1637       case AARCH64_BUILTIN_RSQRT_V2DF:
1638 	gen = gen_rsqrtv2df2;
1639 	break;
1640       case AARCH64_BUILTIN_RSQRT_V2SF:
1641 	gen = gen_rsqrtv2sf2;
1642 	break;
1643       case AARCH64_BUILTIN_RSQRT_V4SF:
1644 	gen = gen_rsqrtv4sf2;
1645 	break;
1646       default: gcc_unreachable ();
1647     }
1648 
1649   if (!target)
1650     target = gen_reg_rtx (GET_MODE (op0));
1651 
1652   emit_insn (gen (target, op0));
1653 
1654   return target;
1655 }
1656 
1657 /* Expand a FCMLA lane expression EXP with code FCODE and
1658    result going to TARGET if that is convenient.  */
1659 
1660 rtx
aarch64_expand_fcmla_builtin(tree exp,rtx target,int fcode)1661 aarch64_expand_fcmla_builtin (tree exp, rtx target, int fcode)
1662 {
1663   int bcode = fcode - AARCH64_SIMD_FCMLA_LANEQ_BUILTIN_BASE - 1;
1664   aarch64_fcmla_laneq_builtin_datum* d
1665     = &aarch64_fcmla_lane_builtin_data[bcode];
1666   machine_mode quadmode = GET_MODE_2XWIDER_MODE (d->mode).require ();
1667   rtx op0 = force_reg (d->mode, expand_normal (CALL_EXPR_ARG (exp, 0)));
1668   rtx op1 = force_reg (d->mode, expand_normal (CALL_EXPR_ARG (exp, 1)));
1669   rtx op2 = force_reg (quadmode, expand_normal (CALL_EXPR_ARG (exp, 2)));
1670   tree tmp = CALL_EXPR_ARG (exp, 3);
1671   rtx lane_idx = expand_expr (tmp, NULL_RTX, VOIDmode, EXPAND_INITIALIZER);
1672 
1673   /* Validate that the lane index is a constant.  */
1674   if (!CONST_INT_P (lane_idx))
1675     {
1676       error ("%Kargument %d must be a constant immediate", exp, 4);
1677       return const0_rtx;
1678     }
1679 
1680   /* Validate that the index is within the expected range.  */
1681   int nunits = GET_MODE_NUNITS (quadmode).to_constant ();
1682   aarch64_simd_lane_bounds (lane_idx, 0, nunits / 2, exp);
1683 
1684   /* Generate the correct register and mode.  */
1685   int lane = INTVAL (lane_idx);
1686 
1687   if (lane < nunits / 4)
1688     op2 = simplify_gen_subreg (d->mode, op2, quadmode,
1689 			       subreg_lowpart_offset (d->mode, quadmode));
1690   else
1691     {
1692       /* Select the upper 64 bits, either a V2SF or V4HF, this however
1693 	 is quite messy, as the operation required even though simple
1694 	 doesn't have a simple RTL pattern, and seems it's quite hard to
1695 	 define using a single RTL pattern.  The target generic version
1696 	 gen_highpart_mode generates code that isn't optimal.  */
1697       rtx temp1 = gen_reg_rtx (d->mode);
1698       rtx temp2 = gen_reg_rtx (DImode);
1699       temp1 = simplify_gen_subreg (d->mode, op2, quadmode,
1700 				   subreg_lowpart_offset (d->mode, quadmode));
1701       temp1 = simplify_gen_subreg (V2DImode, temp1, d->mode, 0);
1702       if (BYTES_BIG_ENDIAN)
1703 	emit_insn (gen_aarch64_get_lanev2di (temp2, temp1, const0_rtx));
1704       else
1705 	emit_insn (gen_aarch64_get_lanev2di (temp2, temp1, const1_rtx));
1706       op2 = simplify_gen_subreg (d->mode, temp2, GET_MODE (temp2), 0);
1707 
1708       /* And recalculate the index.  */
1709       lane -= nunits / 4;
1710     }
1711 
1712   /* Keep to GCC-vector-extension lane indices in the RTL, only nunits / 4
1713      (max nunits in range check) are valid.  Which means only 0-1, so we
1714      only need to know the order in a V2mode.  */
1715   lane_idx = aarch64_endian_lane_rtx (V2DImode, lane);
1716 
1717   if (!target)
1718     target = gen_reg_rtx (d->mode);
1719   else
1720     target = force_reg (d->mode, target);
1721 
1722   rtx pat = NULL_RTX;
1723 
1724   if (d->lane)
1725     pat = GEN_FCN (d->icode) (target, op0, op1, op2, lane_idx);
1726   else
1727     pat = GEN_FCN (d->icode) (target, op0, op1, op2);
1728 
1729   if (!pat)
1730     return NULL_RTX;
1731 
1732   emit_insn (pat);
1733   return target;
1734 }
1735 
1736 /* Function to expand an expression EXP which calls one of the Transactional
1737    Memory Extension (TME) builtins FCODE with the result going to TARGET.  */
1738 static rtx
aarch64_expand_builtin_tme(int fcode,tree exp,rtx target)1739 aarch64_expand_builtin_tme (int fcode, tree exp, rtx target)
1740 {
1741   switch (fcode)
1742     {
1743     case AARCH64_TME_BUILTIN_TSTART:
1744       target = gen_reg_rtx (DImode);
1745       emit_insn (GEN_FCN (CODE_FOR_tstart) (target));
1746       break;
1747 
1748     case AARCH64_TME_BUILTIN_TTEST:
1749       target = gen_reg_rtx (DImode);
1750       emit_insn (GEN_FCN (CODE_FOR_ttest) (target));
1751       break;
1752 
1753     case AARCH64_TME_BUILTIN_TCOMMIT:
1754       emit_insn (GEN_FCN (CODE_FOR_tcommit) ());
1755       break;
1756 
1757     case AARCH64_TME_BUILTIN_TCANCEL:
1758       {
1759 	tree arg0 = CALL_EXPR_ARG (exp, 0);
1760 	rtx op0 = expand_normal (arg0);
1761 	if (CONST_INT_P (op0) && UINTVAL (op0) <= 65536)
1762 	  emit_insn (GEN_FCN (CODE_FOR_tcancel) (op0));
1763 	else
1764 	  {
1765 	    error ("%Kargument must be a 16-bit constant immediate", exp);
1766 	    return const0_rtx;
1767 	  }
1768       }
1769       break;
1770 
1771     default :
1772       gcc_unreachable ();
1773     }
1774     return target;
1775 }
1776 
1777 /* Expand a random number builtin EXP with code FCODE, putting the result
1778    int TARGET.  If IGNORE is true the return value is ignored.  */
1779 
1780 rtx
aarch64_expand_rng_builtin(tree exp,rtx target,int fcode,int ignore)1781 aarch64_expand_rng_builtin (tree exp, rtx target, int fcode, int ignore)
1782 {
1783   rtx pat;
1784   enum insn_code icode;
1785   if (fcode == AARCH64_BUILTIN_RNG_RNDR)
1786     icode = CODE_FOR_aarch64_rndr;
1787   else if (fcode == AARCH64_BUILTIN_RNG_RNDRRS)
1788     icode = CODE_FOR_aarch64_rndrrs;
1789   else
1790     gcc_unreachable ();
1791 
1792   rtx rand = gen_reg_rtx (DImode);
1793   pat = GEN_FCN (icode) (rand);
1794   if (!pat)
1795     return NULL_RTX;
1796 
1797   tree arg0 = CALL_EXPR_ARG (exp, 0);
1798   rtx res_addr = expand_normal (arg0);
1799   res_addr = convert_memory_address (Pmode, res_addr);
1800   rtx res_mem = gen_rtx_MEM (DImode, res_addr);
1801   emit_insn (pat);
1802   emit_move_insn (res_mem, rand);
1803   /* If the status result is unused don't generate the CSET code.  */
1804   if (ignore)
1805     return target;
1806 
1807   rtx cc_reg = gen_rtx_REG (CC_Zmode, CC_REGNUM);
1808   rtx cmp_rtx = gen_rtx_fmt_ee (EQ, SImode, cc_reg, const0_rtx);
1809   emit_insn (gen_aarch64_cstoresi (target, cmp_rtx, cc_reg));
1810   return target;
1811 }
1812 
1813 /* Expand an expression EXP that calls a MEMTAG built-in FCODE
1814    with result going to TARGET.  */
1815 static rtx
aarch64_expand_builtin_memtag(int fcode,tree exp,rtx target)1816 aarch64_expand_builtin_memtag (int fcode, tree exp, rtx target)
1817 {
1818   if (TARGET_ILP32)
1819     {
1820       error ("Memory Tagging Extension does not support %<-mabi=ilp32%>");
1821       return const0_rtx;
1822     }
1823 
1824   rtx pat = NULL;
1825   enum insn_code icode = aarch64_memtag_builtin_data[fcode -
1826 			   AARCH64_MEMTAG_BUILTIN_START - 1].icode;
1827 
1828   rtx op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
1829   machine_mode mode0 = GET_MODE (op0);
1830   op0 = force_reg (mode0 == VOIDmode ? DImode : mode0, op0);
1831   op0 = convert_to_mode (DImode, op0, true);
1832 
1833   switch (fcode)
1834     {
1835       case AARCH64_MEMTAG_BUILTIN_IRG:
1836       case AARCH64_MEMTAG_BUILTIN_GMI:
1837       case AARCH64_MEMTAG_BUILTIN_SUBP:
1838       case AARCH64_MEMTAG_BUILTIN_INC_TAG:
1839 	{
1840 	  if (! target
1841 	      || GET_MODE (target) != DImode
1842 	      || ! (*insn_data[icode].operand[0].predicate) (target, DImode))
1843 	    target = gen_reg_rtx (DImode);
1844 
1845 	  if (fcode == AARCH64_MEMTAG_BUILTIN_INC_TAG)
1846 	    {
1847 	      rtx op1 = expand_normal (CALL_EXPR_ARG (exp, 1));
1848 
1849 	      if ((*insn_data[icode].operand[3].predicate) (op1, QImode))
1850 		{
1851 		  pat = GEN_FCN (icode) (target, op0, const0_rtx, op1);
1852 		  break;
1853 		}
1854 	      error ("%Kargument %d must be a constant immediate "
1855 		     "in range [0,15]", exp, 2);
1856 	      return const0_rtx;
1857 	    }
1858 	  else
1859 	    {
1860 	      rtx op1 = expand_normal (CALL_EXPR_ARG (exp, 1));
1861 	      machine_mode mode1 = GET_MODE (op1);
1862 	      op1 = force_reg (mode1 == VOIDmode ? DImode : mode1, op1);
1863 	      op1 = convert_to_mode (DImode, op1, true);
1864 	      pat = GEN_FCN (icode) (target, op0, op1);
1865 	    }
1866 	  break;
1867 	}
1868       case AARCH64_MEMTAG_BUILTIN_GET_TAG:
1869 	target = op0;
1870 	pat = GEN_FCN (icode) (target, op0, const0_rtx);
1871 	break;
1872       case AARCH64_MEMTAG_BUILTIN_SET_TAG:
1873 	pat = GEN_FCN (icode) (op0, op0, const0_rtx);
1874 	break;
1875       default:
1876 	gcc_unreachable();
1877     }
1878 
1879   if (!pat)
1880     return NULL_RTX;
1881 
1882   emit_insn (pat);
1883   return target;
1884 }
1885 
1886 /* Expand an expression EXP that calls built-in function FCODE,
1887    with result going to TARGET if that's convenient.  IGNORE is true
1888    if the result of the builtin is ignored.  */
1889 rtx
aarch64_general_expand_builtin(unsigned int fcode,tree exp,rtx target,int ignore)1890 aarch64_general_expand_builtin (unsigned int fcode, tree exp, rtx target,
1891 				int ignore)
1892 {
1893   int icode;
1894   rtx pat, op0;
1895   tree arg0;
1896 
1897   switch (fcode)
1898     {
1899     case AARCH64_BUILTIN_GET_FPCR:
1900     case AARCH64_BUILTIN_SET_FPCR:
1901     case AARCH64_BUILTIN_GET_FPSR:
1902     case AARCH64_BUILTIN_SET_FPSR:
1903       if ((fcode == AARCH64_BUILTIN_GET_FPCR)
1904 	  || (fcode == AARCH64_BUILTIN_GET_FPSR))
1905 	{
1906 	  icode = (fcode == AARCH64_BUILTIN_GET_FPSR) ?
1907 	    CODE_FOR_get_fpsr : CODE_FOR_get_fpcr;
1908 	  target = gen_reg_rtx (SImode);
1909 	  pat = GEN_FCN (icode) (target);
1910 	}
1911       else
1912 	{
1913 	  target = NULL_RTX;
1914 	  icode = (fcode == AARCH64_BUILTIN_SET_FPSR) ?
1915 	    CODE_FOR_set_fpsr : CODE_FOR_set_fpcr;
1916 	  arg0 = CALL_EXPR_ARG (exp, 0);
1917 	  op0 = force_reg (SImode, expand_normal (arg0));
1918 	  pat = GEN_FCN (icode) (op0);
1919 	}
1920       emit_insn (pat);
1921       return target;
1922 
1923     case AARCH64_PAUTH_BUILTIN_AUTIA1716:
1924     case AARCH64_PAUTH_BUILTIN_PACIA1716:
1925     case AARCH64_PAUTH_BUILTIN_AUTIB1716:
1926     case AARCH64_PAUTH_BUILTIN_PACIB1716:
1927     case AARCH64_PAUTH_BUILTIN_XPACLRI:
1928       arg0 = CALL_EXPR_ARG (exp, 0);
1929       op0 = force_reg (Pmode, expand_normal (arg0));
1930 
1931       if (!target)
1932 	target = gen_reg_rtx (Pmode);
1933       else
1934 	target = force_reg (Pmode, target);
1935 
1936       emit_move_insn (target, op0);
1937 
1938       if (fcode == AARCH64_PAUTH_BUILTIN_XPACLRI)
1939 	{
1940 	  rtx lr = gen_rtx_REG (Pmode, R30_REGNUM);
1941 	  icode = CODE_FOR_xpaclri;
1942 	  emit_move_insn (lr, op0);
1943 	  emit_insn (GEN_FCN (icode) ());
1944 	  emit_move_insn (target, lr);
1945 	}
1946       else
1947 	{
1948 	  tree arg1 = CALL_EXPR_ARG (exp, 1);
1949 	  rtx op1 = force_reg (Pmode, expand_normal (arg1));
1950 	  switch (fcode)
1951 	    {
1952 	    case AARCH64_PAUTH_BUILTIN_AUTIA1716:
1953 	      icode = CODE_FOR_autia1716;
1954 	      break;
1955 	    case AARCH64_PAUTH_BUILTIN_AUTIB1716:
1956 	      icode = CODE_FOR_autib1716;
1957 	      break;
1958 	    case AARCH64_PAUTH_BUILTIN_PACIA1716:
1959 	      icode = CODE_FOR_pacia1716;
1960 	      break;
1961 	    case AARCH64_PAUTH_BUILTIN_PACIB1716:
1962 	      icode = CODE_FOR_pacib1716;
1963 	      break;
1964 	    default:
1965 	      icode = 0;
1966 	      gcc_unreachable ();
1967 	    }
1968 
1969 	  rtx x16_reg = gen_rtx_REG (Pmode, R16_REGNUM);
1970 	  rtx x17_reg = gen_rtx_REG (Pmode, R17_REGNUM);
1971 	  emit_move_insn (x17_reg, op0);
1972 	  emit_move_insn (x16_reg, op1);
1973 	  emit_insn (GEN_FCN (icode) ());
1974 	  emit_move_insn (target, x17_reg);
1975 	}
1976 
1977       return target;
1978 
1979     case AARCH64_JSCVT:
1980       {
1981 	expand_operand ops[2];
1982 	create_output_operand (&ops[0], target, SImode);
1983 	op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
1984 	create_input_operand (&ops[1], op0, DFmode);
1985 	expand_insn (CODE_FOR_aarch64_fjcvtzs, 2, ops);
1986 	return ops[0].value;
1987       }
1988 
1989     case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ0_V2SF:
1990     case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ90_V2SF:
1991     case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ180_V2SF:
1992     case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ270_V2SF:
1993     case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ0_V4HF:
1994     case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ90_V4HF:
1995     case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ180_V4HF:
1996     case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ270_V4HF:
1997       return aarch64_expand_fcmla_builtin (exp, target, fcode);
1998     case AARCH64_BUILTIN_RNG_RNDR:
1999     case AARCH64_BUILTIN_RNG_RNDRRS:
2000       return aarch64_expand_rng_builtin (exp, target, fcode, ignore);
2001     }
2002 
2003   if (fcode >= AARCH64_SIMD_BUILTIN_BASE && fcode <= AARCH64_SIMD_BUILTIN_MAX)
2004     return aarch64_simd_expand_builtin (fcode, exp, target);
2005   else if (fcode >= AARCH64_CRC32_BUILTIN_BASE && fcode <= AARCH64_CRC32_BUILTIN_MAX)
2006     return aarch64_crc32_expand_builtin (fcode, exp, target);
2007 
2008   if (fcode == AARCH64_BUILTIN_RSQRT_DF
2009       || fcode == AARCH64_BUILTIN_RSQRT_SF
2010       || fcode == AARCH64_BUILTIN_RSQRT_V2DF
2011       || fcode == AARCH64_BUILTIN_RSQRT_V2SF
2012       || fcode == AARCH64_BUILTIN_RSQRT_V4SF)
2013     return aarch64_expand_builtin_rsqrt (fcode, exp, target);
2014 
2015   if (fcode == AARCH64_TME_BUILTIN_TSTART
2016       || fcode == AARCH64_TME_BUILTIN_TCOMMIT
2017       || fcode == AARCH64_TME_BUILTIN_TTEST
2018       || fcode == AARCH64_TME_BUILTIN_TCANCEL)
2019     return aarch64_expand_builtin_tme (fcode, exp, target);
2020 
2021   if (fcode >= AARCH64_MEMTAG_BUILTIN_START
2022       && fcode <= AARCH64_MEMTAG_BUILTIN_END)
2023     return aarch64_expand_builtin_memtag (fcode, exp, target);
2024 
2025   gcc_unreachable ();
2026 }
2027 
2028 tree
aarch64_builtin_vectorized_function(unsigned int fn,tree type_out,tree type_in)2029 aarch64_builtin_vectorized_function (unsigned int fn, tree type_out,
2030 				     tree type_in)
2031 {
2032   machine_mode in_mode, out_mode;
2033 
2034   if (TREE_CODE (type_out) != VECTOR_TYPE
2035       || TREE_CODE (type_in) != VECTOR_TYPE)
2036     return NULL_TREE;
2037 
2038   out_mode = TYPE_MODE (type_out);
2039   in_mode = TYPE_MODE (type_in);
2040 
2041 #undef AARCH64_CHECK_BUILTIN_MODE
2042 #define AARCH64_CHECK_BUILTIN_MODE(C, N) 1
2043 #define AARCH64_FIND_FRINT_VARIANT(N) \
2044   (AARCH64_CHECK_BUILTIN_MODE (2, D) \
2045     ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v2df] \
2046     : (AARCH64_CHECK_BUILTIN_MODE (4, S) \
2047 	? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v4sf] \
2048 	: (AARCH64_CHECK_BUILTIN_MODE (2, S) \
2049 	   ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v2sf] \
2050 	   : NULL_TREE)))
2051   switch (fn)
2052     {
2053 #undef AARCH64_CHECK_BUILTIN_MODE
2054 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
2055   (out_mode == V##C##N##Fmode && in_mode == V##C##N##Fmode)
2056     CASE_CFN_FLOOR:
2057       return AARCH64_FIND_FRINT_VARIANT (floor);
2058     CASE_CFN_CEIL:
2059       return AARCH64_FIND_FRINT_VARIANT (ceil);
2060     CASE_CFN_TRUNC:
2061       return AARCH64_FIND_FRINT_VARIANT (btrunc);
2062     CASE_CFN_ROUND:
2063       return AARCH64_FIND_FRINT_VARIANT (round);
2064     CASE_CFN_NEARBYINT:
2065       return AARCH64_FIND_FRINT_VARIANT (nearbyint);
2066     CASE_CFN_SQRT:
2067       return AARCH64_FIND_FRINT_VARIANT (sqrt);
2068 #undef AARCH64_CHECK_BUILTIN_MODE
2069 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
2070   (out_mode == V##C##SImode && in_mode == V##C##N##Imode)
2071     CASE_CFN_CLZ:
2072       {
2073 	if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2074 	  return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_clzv4si];
2075 	return NULL_TREE;
2076       }
2077     CASE_CFN_CTZ:
2078       {
2079 	if (AARCH64_CHECK_BUILTIN_MODE (2, S))
2080 	  return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_ctzv2si];
2081 	else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2082 	  return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_ctzv4si];
2083 	return NULL_TREE;
2084       }
2085 #undef AARCH64_CHECK_BUILTIN_MODE
2086 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
2087   (out_mode == V##C##N##Imode && in_mode == V##C##N##Fmode)
2088     CASE_CFN_IFLOOR:
2089     CASE_CFN_LFLOOR:
2090     CASE_CFN_LLFLOOR:
2091       {
2092 	enum aarch64_builtins builtin;
2093 	if (AARCH64_CHECK_BUILTIN_MODE (2, D))
2094 	  builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv2dfv2di;
2095 	else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2096 	  builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv4sfv4si;
2097 	else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
2098 	  builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv2sfv2si;
2099 	else
2100 	  return NULL_TREE;
2101 
2102 	return aarch64_builtin_decls[builtin];
2103       }
2104     CASE_CFN_ICEIL:
2105     CASE_CFN_LCEIL:
2106     CASE_CFN_LLCEIL:
2107       {
2108 	enum aarch64_builtins builtin;
2109 	if (AARCH64_CHECK_BUILTIN_MODE (2, D))
2110 	  builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv2dfv2di;
2111 	else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2112 	  builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv4sfv4si;
2113 	else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
2114 	  builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv2sfv2si;
2115 	else
2116 	  return NULL_TREE;
2117 
2118 	return aarch64_builtin_decls[builtin];
2119       }
2120     CASE_CFN_IROUND:
2121     CASE_CFN_LROUND:
2122     CASE_CFN_LLROUND:
2123       {
2124 	enum aarch64_builtins builtin;
2125 	if (AARCH64_CHECK_BUILTIN_MODE (2, D))
2126 	  builtin =	AARCH64_SIMD_BUILTIN_UNOP_lroundv2dfv2di;
2127 	else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2128 	  builtin =	AARCH64_SIMD_BUILTIN_UNOP_lroundv4sfv4si;
2129 	else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
2130 	  builtin =	AARCH64_SIMD_BUILTIN_UNOP_lroundv2sfv2si;
2131 	else
2132 	  return NULL_TREE;
2133 
2134 	return aarch64_builtin_decls[builtin];
2135       }
2136     default:
2137       return NULL_TREE;
2138     }
2139 
2140   return NULL_TREE;
2141 }
2142 
2143 /* Return builtin for reciprocal square root.  */
2144 
2145 tree
aarch64_general_builtin_rsqrt(unsigned int fn)2146 aarch64_general_builtin_rsqrt (unsigned int fn)
2147 {
2148   if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2df)
2149     return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V2DF];
2150   if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2sf)
2151     return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V2SF];
2152   if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv4sf)
2153     return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V4SF];
2154   return NULL_TREE;
2155 }
2156 
2157 #undef VAR1
2158 #define VAR1(T, N, MAP, A) \
2159   case AARCH64_SIMD_BUILTIN_##T##_##N##A:
2160 
2161 /* Try to fold a call to the built-in function with subcode FCODE.  The
2162    function is passed the N_ARGS arguments in ARGS and it returns a value
2163    of type TYPE.  Return the new expression on success and NULL_TREE on
2164    failure.  */
2165 tree
aarch64_general_fold_builtin(unsigned int fcode,tree type,unsigned int n_args ATTRIBUTE_UNUSED,tree * args)2166 aarch64_general_fold_builtin (unsigned int fcode, tree type,
2167 			      unsigned int n_args ATTRIBUTE_UNUSED, tree *args)
2168 {
2169   switch (fcode)
2170     {
2171       BUILTIN_VDQF (UNOP, abs, 2)
2172 	return fold_build1 (ABS_EXPR, type, args[0]);
2173       VAR1 (UNOP, floatv2si, 2, v2sf)
2174       VAR1 (UNOP, floatv4si, 2, v4sf)
2175       VAR1 (UNOP, floatv2di, 2, v2df)
2176 	return fold_build1 (FLOAT_EXPR, type, args[0]);
2177       default:
2178 	break;
2179     }
2180 
2181   return NULL_TREE;
2182 }
2183 
2184 /* Try to fold STMT, given that it's a call to the built-in function with
2185    subcode FCODE.  Return the new statement on success and null on
2186    failure.  */
2187 gimple *
aarch64_general_gimple_fold_builtin(unsigned int fcode,gcall * stmt)2188 aarch64_general_gimple_fold_builtin (unsigned int fcode, gcall *stmt)
2189 {
2190   gimple *new_stmt = NULL;
2191   unsigned nargs = gimple_call_num_args (stmt);
2192   tree *args = (nargs > 0
2193 		? gimple_call_arg_ptr (stmt, 0)
2194 		: &error_mark_node);
2195 
2196   /* We use gimple's IFN_REDUC_(PLUS|MIN|MAX)s for float, signed int
2197      and unsigned int; it will distinguish according to the types of
2198      the arguments to the __builtin.  */
2199   switch (fcode)
2200     {
2201       BUILTIN_VALL (UNOP, reduc_plus_scal_, 10)
2202 	new_stmt = gimple_build_call_internal (IFN_REDUC_PLUS,
2203 					       1, args[0]);
2204 	gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
2205 	break;
2206       BUILTIN_VDQIF (UNOP, reduc_smax_scal_, 10)
2207       BUILTIN_VDQ_BHSI (UNOPU, reduc_umax_scal_, 10)
2208 	new_stmt = gimple_build_call_internal (IFN_REDUC_MAX,
2209 					       1, args[0]);
2210 	gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
2211 	break;
2212       BUILTIN_VDQIF (UNOP, reduc_smin_scal_, 10)
2213       BUILTIN_VDQ_BHSI (UNOPU, reduc_umin_scal_, 10)
2214 	new_stmt = gimple_build_call_internal (IFN_REDUC_MIN,
2215 					       1, args[0]);
2216 	gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
2217 	break;
2218       BUILTIN_GPF (BINOP, fmulx, 0)
2219 	{
2220 	  gcc_assert (nargs == 2);
2221 	  bool a0_cst_p = TREE_CODE (args[0]) == REAL_CST;
2222 	  bool a1_cst_p = TREE_CODE (args[1]) == REAL_CST;
2223 	  if (a0_cst_p || a1_cst_p)
2224 	    {
2225 	      if (a0_cst_p && a1_cst_p)
2226 		{
2227 		  tree t0 = TREE_TYPE (args[0]);
2228 		  real_value a0 = (TREE_REAL_CST (args[0]));
2229 		  real_value a1 = (TREE_REAL_CST (args[1]));
2230 		  if (real_equal (&a1, &dconst0))
2231 		    std::swap (a0, a1);
2232 		  /* According to real_equal (), +0 equals -0.  */
2233 		  if (real_equal (&a0, &dconst0) && real_isinf (&a1))
2234 		    {
2235 		      real_value res = dconst2;
2236 		      res.sign = a0.sign ^ a1.sign;
2237 		      new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
2238 						      REAL_CST,
2239 						      build_real (t0, res));
2240 		    }
2241 		  else
2242 		    new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
2243 						    MULT_EXPR,
2244 						    args[0], args[1]);
2245 		}
2246 	      else /* a0_cst_p ^ a1_cst_p.  */
2247 		{
2248 		  real_value const_part = a0_cst_p
2249 		    ? TREE_REAL_CST (args[0]) : TREE_REAL_CST (args[1]);
2250 		  if (!real_equal (&const_part, &dconst0)
2251 		      && !real_isinf (&const_part))
2252 		    new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
2253 						    MULT_EXPR, args[0],
2254 						    args[1]);
2255 		}
2256 	    }
2257 	  if (new_stmt)
2258 	    {
2259 	      gimple_set_vuse (new_stmt, gimple_vuse (stmt));
2260 	      gimple_set_vdef (new_stmt, gimple_vdef (stmt));
2261 	    }
2262 	  break;
2263 	}
2264     default:
2265       break;
2266     }
2267   return new_stmt;
2268 }
2269 
2270 void
aarch64_atomic_assign_expand_fenv(tree * hold,tree * clear,tree * update)2271 aarch64_atomic_assign_expand_fenv (tree *hold, tree *clear, tree *update)
2272 {
2273   const unsigned AARCH64_FE_INVALID = 1;
2274   const unsigned AARCH64_FE_DIVBYZERO = 2;
2275   const unsigned AARCH64_FE_OVERFLOW = 4;
2276   const unsigned AARCH64_FE_UNDERFLOW = 8;
2277   const unsigned AARCH64_FE_INEXACT = 16;
2278   const unsigned HOST_WIDE_INT AARCH64_FE_ALL_EXCEPT = (AARCH64_FE_INVALID
2279 							| AARCH64_FE_DIVBYZERO
2280 							| AARCH64_FE_OVERFLOW
2281 							| AARCH64_FE_UNDERFLOW
2282 							| AARCH64_FE_INEXACT);
2283   const unsigned HOST_WIDE_INT AARCH64_FE_EXCEPT_SHIFT = 8;
2284   tree fenv_cr, fenv_sr, get_fpcr, set_fpcr, mask_cr, mask_sr;
2285   tree ld_fenv_cr, ld_fenv_sr, masked_fenv_cr, masked_fenv_sr, hold_fnclex_cr;
2286   tree hold_fnclex_sr, new_fenv_var, reload_fenv, restore_fnenv, get_fpsr, set_fpsr;
2287   tree update_call, atomic_feraiseexcept, hold_fnclex, masked_fenv, ld_fenv;
2288 
2289   /* Generate the equivalence of :
2290        unsigned int fenv_cr;
2291        fenv_cr = __builtin_aarch64_get_fpcr ();
2292 
2293        unsigned int fenv_sr;
2294        fenv_sr = __builtin_aarch64_get_fpsr ();
2295 
2296        Now set all exceptions to non-stop
2297        unsigned int mask_cr
2298 		= ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT);
2299        unsigned int masked_cr;
2300        masked_cr = fenv_cr & mask_cr;
2301 
2302        And clear all exception flags
2303        unsigned int maske_sr = ~AARCH64_FE_ALL_EXCEPT;
2304        unsigned int masked_cr;
2305        masked_sr = fenv_sr & mask_sr;
2306 
2307        __builtin_aarch64_set_cr (masked_cr);
2308        __builtin_aarch64_set_sr (masked_sr);  */
2309 
2310   fenv_cr = create_tmp_var_raw (unsigned_type_node);
2311   fenv_sr = create_tmp_var_raw (unsigned_type_node);
2312 
2313   get_fpcr = aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR];
2314   set_fpcr = aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR];
2315   get_fpsr = aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR];
2316   set_fpsr = aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR];
2317 
2318   mask_cr = build_int_cst (unsigned_type_node,
2319 			   ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT));
2320   mask_sr = build_int_cst (unsigned_type_node,
2321 			   ~(AARCH64_FE_ALL_EXCEPT));
2322 
2323   ld_fenv_cr = build4 (TARGET_EXPR, unsigned_type_node,
2324 		       fenv_cr, build_call_expr (get_fpcr, 0),
2325 		       NULL_TREE, NULL_TREE);
2326   ld_fenv_sr = build4 (TARGET_EXPR, unsigned_type_node,
2327 		       fenv_sr, build_call_expr (get_fpsr, 0),
2328 		       NULL_TREE, NULL_TREE);
2329 
2330   masked_fenv_cr = build2 (BIT_AND_EXPR, unsigned_type_node, fenv_cr, mask_cr);
2331   masked_fenv_sr = build2 (BIT_AND_EXPR, unsigned_type_node, fenv_sr, mask_sr);
2332 
2333   hold_fnclex_cr = build_call_expr (set_fpcr, 1, masked_fenv_cr);
2334   hold_fnclex_sr = build_call_expr (set_fpsr, 1, masked_fenv_sr);
2335 
2336   hold_fnclex = build2 (COMPOUND_EXPR, void_type_node, hold_fnclex_cr,
2337 			hold_fnclex_sr);
2338   masked_fenv = build2 (COMPOUND_EXPR, void_type_node, masked_fenv_cr,
2339 			masked_fenv_sr);
2340   ld_fenv = build2 (COMPOUND_EXPR, void_type_node, ld_fenv_cr, ld_fenv_sr);
2341 
2342   *hold = build2 (COMPOUND_EXPR, void_type_node,
2343 		  build2 (COMPOUND_EXPR, void_type_node, masked_fenv, ld_fenv),
2344 		  hold_fnclex);
2345 
2346   /* Store the value of masked_fenv to clear the exceptions:
2347      __builtin_aarch64_set_fpsr (masked_fenv_sr);  */
2348 
2349   *clear = build_call_expr (set_fpsr, 1, masked_fenv_sr);
2350 
2351   /* Generate the equivalent of :
2352        unsigned int new_fenv_var;
2353        new_fenv_var = __builtin_aarch64_get_fpsr ();
2354 
2355        __builtin_aarch64_set_fpsr (fenv_sr);
2356 
2357        __atomic_feraiseexcept (new_fenv_var);  */
2358 
2359   new_fenv_var = create_tmp_var_raw (unsigned_type_node);
2360   reload_fenv = build4 (TARGET_EXPR, unsigned_type_node,
2361 			new_fenv_var, build_call_expr (get_fpsr, 0),
2362 			NULL_TREE, NULL_TREE);
2363   restore_fnenv = build_call_expr (set_fpsr, 1, fenv_sr);
2364   atomic_feraiseexcept = builtin_decl_implicit (BUILT_IN_ATOMIC_FERAISEEXCEPT);
2365   update_call = build_call_expr (atomic_feraiseexcept, 1,
2366 				 fold_convert (integer_type_node, new_fenv_var));
2367   *update = build2 (COMPOUND_EXPR, void_type_node,
2368 		    build2 (COMPOUND_EXPR, void_type_node,
2369 			    reload_fenv, restore_fnenv), update_call);
2370 }
2371 
2372 /* Resolve overloaded MEMTAG build-in functions.  */
2373 #define AARCH64_BUILTIN_SUBCODE(F) \
2374   (DECL_MD_FUNCTION_CODE (F) >> AARCH64_BUILTIN_SHIFT)
2375 
2376 static tree
aarch64_resolve_overloaded_memtag(location_t loc,tree fndecl,void * pass_params)2377 aarch64_resolve_overloaded_memtag (location_t loc,
2378 				   tree fndecl, void *pass_params)
2379 {
2380   vec<tree, va_gc> *params = static_cast<vec<tree, va_gc> *> (pass_params);
2381   unsigned param_num = params ? params->length() : 0;
2382   unsigned int fcode = AARCH64_BUILTIN_SUBCODE (fndecl);
2383   tree inittype = aarch64_memtag_builtin_data[
2384 		    fcode - AARCH64_MEMTAG_BUILTIN_START - 1].ftype;
2385   unsigned arg_num = list_length (TYPE_ARG_TYPES (inittype)) - 1;
2386 
2387   if (param_num != arg_num)
2388     {
2389       TREE_TYPE (fndecl) = inittype;
2390       return NULL_TREE;
2391     }
2392   tree retype = NULL;
2393 
2394   if (fcode == AARCH64_MEMTAG_BUILTIN_SUBP)
2395     {
2396       tree t0 = TREE_TYPE ((*params)[0]);
2397       tree t1 = TREE_TYPE ((*params)[1]);
2398 
2399       if (t0 == error_mark_node || TREE_CODE (t0) != POINTER_TYPE)
2400 	t0 = ptr_type_node;
2401       if (t1 == error_mark_node || TREE_CODE (t1) != POINTER_TYPE)
2402 	t1 = ptr_type_node;
2403 
2404       if (TYPE_MODE (t0) != DImode)
2405 	warning_at (loc, 1, "expected 64-bit address but argument 1 is %d-bit",
2406 	    (int)tree_to_shwi (DECL_SIZE ((*params)[0])));
2407 
2408       if (TYPE_MODE (t1) != DImode)
2409 	warning_at (loc, 1, "expected 64-bit address but argument 2 is %d-bit",
2410 	    (int)tree_to_shwi (DECL_SIZE ((*params)[1])));
2411 
2412       retype = build_function_type_list (ptrdiff_type_node, t0, t1, NULL);
2413     }
2414   else
2415     {
2416       tree t0 = TREE_TYPE ((*params)[0]);
2417 
2418       if (t0 == error_mark_node || TREE_CODE (t0) != POINTER_TYPE)
2419 	{
2420 	  TREE_TYPE (fndecl) = inittype;
2421 	  return NULL_TREE;
2422 	}
2423 
2424       if (TYPE_MODE (t0) != DImode)
2425 	warning_at (loc, 1, "expected 64-bit address but argument 1 is %d-bit",
2426 	    (int)tree_to_shwi (DECL_SIZE ((*params)[0])));
2427 
2428       switch (fcode)
2429 	{
2430 	case AARCH64_MEMTAG_BUILTIN_IRG:
2431 	  retype = build_function_type_list (t0, t0, uint64_type_node, NULL);
2432 	  break;
2433 	case AARCH64_MEMTAG_BUILTIN_GMI:
2434 	  retype = build_function_type_list (uint64_type_node, t0,
2435 	      uint64_type_node, NULL);
2436 	  break;
2437 	case AARCH64_MEMTAG_BUILTIN_INC_TAG:
2438 	  retype = build_function_type_list (t0, t0, unsigned_type_node, NULL);
2439 	  break;
2440 	case AARCH64_MEMTAG_BUILTIN_SET_TAG:
2441 	  retype = build_function_type_list (void_type_node, t0, NULL);
2442 	  break;
2443 	case AARCH64_MEMTAG_BUILTIN_GET_TAG:
2444 	  retype = build_function_type_list (t0, t0, NULL);
2445 	  break;
2446 	default:
2447 	  return NULL_TREE;
2448 	}
2449     }
2450 
2451   if (!retype || retype == error_mark_node)
2452     TREE_TYPE (fndecl) = inittype;
2453   else
2454     TREE_TYPE (fndecl) = retype;
2455 
2456   return NULL_TREE;
2457 }
2458 
2459 /* Called at aarch64_resolve_overloaded_builtin in aarch64-c.c.  */
2460 tree
aarch64_resolve_overloaded_builtin_general(location_t loc,tree function,void * pass_params)2461 aarch64_resolve_overloaded_builtin_general (location_t loc, tree function,
2462 					    void *pass_params)
2463 {
2464   unsigned int fcode = AARCH64_BUILTIN_SUBCODE (function);
2465 
2466   if (fcode >= AARCH64_MEMTAG_BUILTIN_START
2467       && fcode <= AARCH64_MEMTAG_BUILTIN_END)
2468     return aarch64_resolve_overloaded_memtag(loc, function, pass_params);
2469 
2470   return NULL_TREE;
2471 }
2472 
2473 #undef AARCH64_CHECK_BUILTIN_MODE
2474 #undef AARCH64_FIND_FRINT_VARIANT
2475 #undef CF0
2476 #undef CF1
2477 #undef CF2
2478 #undef CF3
2479 #undef CF4
2480 #undef CF10
2481 #undef VAR1
2482 #undef VAR2
2483 #undef VAR3
2484 #undef VAR4
2485 #undef VAR5
2486 #undef VAR6
2487 #undef VAR7
2488 #undef VAR8
2489 #undef VAR9
2490 #undef VAR10
2491 #undef VAR11
2492 
2493 #include "gt-aarch64-builtins.h"
2494