xref: /netbsd-src/external/gpl3/gcc/dist/gcc/internal-fn.cc (revision 0a3071956a3a9fdebdbf7f338cf2d439b45fc728)
1 /* Internal functions.
2    Copyright (C) 2011-2022 Free Software Foundation, Inc.
3 
4 This file is part of GCC.
5 
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
10 
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14 for more details.
15 
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3.  If not see
18 <http://www.gnu.org/licenses/>.  */
19 
20 #include "config.h"
21 #include "system.h"
22 #include "coretypes.h"
23 #include "backend.h"
24 #include "target.h"
25 #include "rtl.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "predict.h"
29 #include "stringpool.h"
30 #include "tree-vrp.h"
31 #include "tree-ssanames.h"
32 #include "expmed.h"
33 #include "memmodel.h"
34 #include "optabs.h"
35 #include "emit-rtl.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
40 #include "dojump.h"
41 #include "expr.h"
42 #include "stringpool.h"
43 #include "attribs.h"
44 #include "asan.h"
45 #include "ubsan.h"
46 #include "recog.h"
47 #include "builtins.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
52 #include "explow.h"
53 #include "rtl-iter.h"
54 #include "gimple-range.h"
55 #include "tree-ssa-live.h"
56 #include "tree-outof-ssa.h"
57 
58 /* For lang_hooks.types.type_for_mode.  */
59 #include "langhooks.h"
60 
61 /* The names of each internal function, indexed by function number.  */
62 const char *const internal_fn_name_array[] = {
63 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
64 #include "internal-fn.def"
65   "<invalid-fn>"
66 };
67 
68 /* The ECF_* flags of each internal function, indexed by function number.  */
69 const int internal_fn_flags_array[] = {
70 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
71 #include "internal-fn.def"
72   0
73 };
74 
75 /* Return the internal function called NAME, or IFN_LAST if there's
76    no such function.  */
77 
78 internal_fn
lookup_internal_fn(const char * name)79 lookup_internal_fn (const char *name)
80 {
81   typedef hash_map<nofree_string_hash, internal_fn> name_to_fn_map_type;
82   static name_to_fn_map_type *name_to_fn_map;
83 
84   if (!name_to_fn_map)
85     {
86       name_to_fn_map = new name_to_fn_map_type (IFN_LAST);
87       for (unsigned int i = 0; i < IFN_LAST; ++i)
88 	name_to_fn_map->put (internal_fn_name (internal_fn (i)),
89 			     internal_fn (i));
90     }
91   internal_fn *entry = name_to_fn_map->get (name);
92   return entry ? *entry : IFN_LAST;
93 }
94 
95 /* Fnspec of each internal function, indexed by function number.  */
96 const_tree internal_fn_fnspec_array[IFN_LAST + 1];
97 
98 void
init_internal_fns()99 init_internal_fns ()
100 {
101 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
102   if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
103     build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
104 #include "internal-fn.def"
105   internal_fn_fnspec_array[IFN_LAST] = 0;
106 }
107 
108 /* Create static initializers for the information returned by
109    direct_internal_fn.  */
110 #define not_direct { -2, -2, false }
111 #define mask_load_direct { -1, 2, false }
112 #define load_lanes_direct { -1, -1, false }
113 #define mask_load_lanes_direct { -1, -1, false }
114 #define gather_load_direct { 3, 1, false }
115 #define len_load_direct { -1, -1, false }
116 #define mask_store_direct { 3, 2, false }
117 #define store_lanes_direct { 0, 0, false }
118 #define mask_store_lanes_direct { 0, 0, false }
119 #define vec_cond_mask_direct { 1, 0, false }
120 #define vec_cond_direct { 2, 0, false }
121 #define scatter_store_direct { 3, 1, false }
122 #define len_store_direct { 3, 3, false }
123 #define vec_set_direct { 3, 3, false }
124 #define unary_direct { 0, 0, true }
125 #define binary_direct { 0, 0, true }
126 #define ternary_direct { 0, 0, true }
127 #define cond_unary_direct { 1, 1, true }
128 #define cond_binary_direct { 1, 1, true }
129 #define cond_ternary_direct { 1, 1, true }
130 #define while_direct { 0, 2, false }
131 #define fold_extract_direct { 2, 2, false }
132 #define fold_left_direct { 1, 1, false }
133 #define mask_fold_left_direct { 1, 1, false }
134 #define check_ptrs_direct { 0, 0, false }
135 
136 const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
137 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
138 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
139 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
140 				     UNSIGNED_OPTAB, TYPE) TYPE##_direct,
141 #include "internal-fn.def"
142   not_direct
143 };
144 
145 /* ARRAY_TYPE is an array of vector modes.  Return the associated insn
146    for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none.  */
147 
148 static enum insn_code
get_multi_vector_move(tree array_type,convert_optab optab)149 get_multi_vector_move (tree array_type, convert_optab optab)
150 {
151   machine_mode imode;
152   machine_mode vmode;
153 
154   gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
155   imode = TYPE_MODE (array_type);
156   vmode = TYPE_MODE (TREE_TYPE (array_type));
157 
158   return convert_optab_handler (optab, imode, vmode);
159 }
160 
161 /* Expand LOAD_LANES call STMT using optab OPTAB.  */
162 
163 static void
expand_load_lanes_optab_fn(internal_fn,gcall * stmt,convert_optab optab)164 expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
165 {
166   class expand_operand ops[2];
167   tree type, lhs, rhs;
168   rtx target, mem;
169 
170   lhs = gimple_call_lhs (stmt);
171   rhs = gimple_call_arg (stmt, 0);
172   type = TREE_TYPE (lhs);
173 
174   target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
175   mem = expand_normal (rhs);
176 
177   gcc_assert (MEM_P (mem));
178   PUT_MODE (mem, TYPE_MODE (type));
179 
180   create_output_operand (&ops[0], target, TYPE_MODE (type));
181   create_fixed_operand (&ops[1], mem);
182   expand_insn (get_multi_vector_move (type, optab), 2, ops);
183   if (!rtx_equal_p (target, ops[0].value))
184     emit_move_insn (target, ops[0].value);
185 }
186 
187 /* Expand STORE_LANES call STMT using optab OPTAB.  */
188 
189 static void
expand_store_lanes_optab_fn(internal_fn,gcall * stmt,convert_optab optab)190 expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
191 {
192   class expand_operand ops[2];
193   tree type, lhs, rhs;
194   rtx target, reg;
195 
196   lhs = gimple_call_lhs (stmt);
197   rhs = gimple_call_arg (stmt, 0);
198   type = TREE_TYPE (rhs);
199 
200   target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
201   reg = expand_normal (rhs);
202 
203   gcc_assert (MEM_P (target));
204   PUT_MODE (target, TYPE_MODE (type));
205 
206   create_fixed_operand (&ops[0], target);
207   create_input_operand (&ops[1], reg, TYPE_MODE (type));
208   expand_insn (get_multi_vector_move (type, optab), 2, ops);
209 }
210 
211 static void
expand_ANNOTATE(internal_fn,gcall *)212 expand_ANNOTATE (internal_fn, gcall *)
213 {
214   gcc_unreachable ();
215 }
216 
217 /* This should get expanded in omp_device_lower pass.  */
218 
219 static void
expand_GOMP_USE_SIMT(internal_fn,gcall *)220 expand_GOMP_USE_SIMT (internal_fn, gcall *)
221 {
222   gcc_unreachable ();
223 }
224 
225 /* This should get expanded in omp_device_lower pass.  */
226 
227 static void
expand_GOMP_SIMT_ENTER(internal_fn,gcall *)228 expand_GOMP_SIMT_ENTER (internal_fn, gcall *)
229 {
230   gcc_unreachable ();
231 }
232 
233 /* Allocate per-lane storage and begin non-uniform execution region.  */
234 
235 static void
expand_GOMP_SIMT_ENTER_ALLOC(internal_fn,gcall * stmt)236 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn, gcall *stmt)
237 {
238   rtx target;
239   tree lhs = gimple_call_lhs (stmt);
240   if (lhs)
241     target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
242   else
243     target = gen_reg_rtx (Pmode);
244   rtx size = expand_normal (gimple_call_arg (stmt, 0));
245   rtx align = expand_normal (gimple_call_arg (stmt, 1));
246   class expand_operand ops[3];
247   create_output_operand (&ops[0], target, Pmode);
248   create_input_operand (&ops[1], size, Pmode);
249   create_input_operand (&ops[2], align, Pmode);
250   gcc_assert (targetm.have_omp_simt_enter ());
251   expand_insn (targetm.code_for_omp_simt_enter, 3, ops);
252   if (!rtx_equal_p (target, ops[0].value))
253     emit_move_insn (target, ops[0].value);
254 }
255 
256 /* Deallocate per-lane storage and leave non-uniform execution region.  */
257 
258 static void
expand_GOMP_SIMT_EXIT(internal_fn,gcall * stmt)259 expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
260 {
261   gcc_checking_assert (!gimple_call_lhs (stmt));
262   rtx arg = expand_normal (gimple_call_arg (stmt, 0));
263   class expand_operand ops[1];
264   create_input_operand (&ops[0], arg, Pmode);
265   gcc_assert (targetm.have_omp_simt_exit ());
266   expand_insn (targetm.code_for_omp_simt_exit, 1, ops);
267 }
268 
269 /* Lane index on SIMT targets: thread index in the warp on NVPTX.  On targets
270    without SIMT execution this should be expanded in omp_device_lower pass.  */
271 
272 static void
expand_GOMP_SIMT_LANE(internal_fn,gcall * stmt)273 expand_GOMP_SIMT_LANE (internal_fn, gcall *stmt)
274 {
275   tree lhs = gimple_call_lhs (stmt);
276   if (!lhs)
277     return;
278 
279   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
280   gcc_assert (targetm.have_omp_simt_lane ());
281   emit_insn (targetm.gen_omp_simt_lane (target));
282 }
283 
284 /* This should get expanded in omp_device_lower pass.  */
285 
286 static void
expand_GOMP_SIMT_VF(internal_fn,gcall *)287 expand_GOMP_SIMT_VF (internal_fn, gcall *)
288 {
289   gcc_unreachable ();
290 }
291 
292 /* Lane index of the first SIMT lane that supplies a non-zero argument.
293    This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
294    lane that executed the last iteration for handling OpenMP lastprivate.  */
295 
296 static void
expand_GOMP_SIMT_LAST_LANE(internal_fn,gcall * stmt)297 expand_GOMP_SIMT_LAST_LANE (internal_fn, gcall *stmt)
298 {
299   tree lhs = gimple_call_lhs (stmt);
300   if (!lhs)
301     return;
302 
303   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
304   rtx cond = expand_normal (gimple_call_arg (stmt, 0));
305   machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
306   class expand_operand ops[2];
307   create_output_operand (&ops[0], target, mode);
308   create_input_operand (&ops[1], cond, mode);
309   gcc_assert (targetm.have_omp_simt_last_lane ());
310   expand_insn (targetm.code_for_omp_simt_last_lane, 2, ops);
311   if (!rtx_equal_p (target, ops[0].value))
312     emit_move_insn (target, ops[0].value);
313 }
314 
315 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered".  */
316 
317 static void
expand_GOMP_SIMT_ORDERED_PRED(internal_fn,gcall * stmt)318 expand_GOMP_SIMT_ORDERED_PRED (internal_fn, gcall *stmt)
319 {
320   tree lhs = gimple_call_lhs (stmt);
321   if (!lhs)
322     return;
323 
324   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
325   rtx ctr = expand_normal (gimple_call_arg (stmt, 0));
326   machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
327   class expand_operand ops[2];
328   create_output_operand (&ops[0], target, mode);
329   create_input_operand (&ops[1], ctr, mode);
330   gcc_assert (targetm.have_omp_simt_ordered ());
331   expand_insn (targetm.code_for_omp_simt_ordered, 2, ops);
332   if (!rtx_equal_p (target, ops[0].value))
333     emit_move_insn (target, ops[0].value);
334 }
335 
336 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
337    any lane supplies a non-zero argument.  */
338 
339 static void
expand_GOMP_SIMT_VOTE_ANY(internal_fn,gcall * stmt)340 expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
341 {
342   tree lhs = gimple_call_lhs (stmt);
343   if (!lhs)
344     return;
345 
346   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
347   rtx cond = expand_normal (gimple_call_arg (stmt, 0));
348   machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
349   class expand_operand ops[2];
350   create_output_operand (&ops[0], target, mode);
351   create_input_operand (&ops[1], cond, mode);
352   gcc_assert (targetm.have_omp_simt_vote_any ());
353   expand_insn (targetm.code_for_omp_simt_vote_any, 2, ops);
354   if (!rtx_equal_p (target, ops[0].value))
355     emit_move_insn (target, ops[0].value);
356 }
357 
358 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
359    is destination lane index XOR given offset.  */
360 
361 static void
expand_GOMP_SIMT_XCHG_BFLY(internal_fn,gcall * stmt)362 expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
363 {
364   tree lhs = gimple_call_lhs (stmt);
365   if (!lhs)
366     return;
367 
368   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
369   rtx src = expand_normal (gimple_call_arg (stmt, 0));
370   rtx idx = expand_normal (gimple_call_arg (stmt, 1));
371   machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
372   class expand_operand ops[3];
373   create_output_operand (&ops[0], target, mode);
374   create_input_operand (&ops[1], src, mode);
375   create_input_operand (&ops[2], idx, SImode);
376   gcc_assert (targetm.have_omp_simt_xchg_bfly ());
377   expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops);
378   if (!rtx_equal_p (target, ops[0].value))
379     emit_move_insn (target, ops[0].value);
380 }
381 
382 /* Exchange between SIMT lanes according to given source lane index.  */
383 
384 static void
expand_GOMP_SIMT_XCHG_IDX(internal_fn,gcall * stmt)385 expand_GOMP_SIMT_XCHG_IDX (internal_fn, gcall *stmt)
386 {
387   tree lhs = gimple_call_lhs (stmt);
388   if (!lhs)
389     return;
390 
391   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
392   rtx src = expand_normal (gimple_call_arg (stmt, 0));
393   rtx idx = expand_normal (gimple_call_arg (stmt, 1));
394   machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
395   class expand_operand ops[3];
396   create_output_operand (&ops[0], target, mode);
397   create_input_operand (&ops[1], src, mode);
398   create_input_operand (&ops[2], idx, SImode);
399   gcc_assert (targetm.have_omp_simt_xchg_idx ());
400   expand_insn (targetm.code_for_omp_simt_xchg_idx, 3, ops);
401   if (!rtx_equal_p (target, ops[0].value))
402     emit_move_insn (target, ops[0].value);
403 }
404 
405 /* This should get expanded in adjust_simduid_builtins.  */
406 
407 static void
expand_GOMP_SIMD_LANE(internal_fn,gcall *)408 expand_GOMP_SIMD_LANE (internal_fn, gcall *)
409 {
410   gcc_unreachable ();
411 }
412 
413 /* This should get expanded in adjust_simduid_builtins.  */
414 
415 static void
expand_GOMP_SIMD_VF(internal_fn,gcall *)416 expand_GOMP_SIMD_VF (internal_fn, gcall *)
417 {
418   gcc_unreachable ();
419 }
420 
421 /* This should get expanded in adjust_simduid_builtins.  */
422 
423 static void
expand_GOMP_SIMD_LAST_LANE(internal_fn,gcall *)424 expand_GOMP_SIMD_LAST_LANE (internal_fn, gcall *)
425 {
426   gcc_unreachable ();
427 }
428 
429 /* This should get expanded in adjust_simduid_builtins.  */
430 
431 static void
expand_GOMP_SIMD_ORDERED_START(internal_fn,gcall *)432 expand_GOMP_SIMD_ORDERED_START (internal_fn, gcall *)
433 {
434   gcc_unreachable ();
435 }
436 
437 /* This should get expanded in adjust_simduid_builtins.  */
438 
439 static void
expand_GOMP_SIMD_ORDERED_END(internal_fn,gcall *)440 expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
441 {
442   gcc_unreachable ();
443 }
444 
445 /* This should get expanded in the sanopt pass.  */
446 
447 static void
expand_UBSAN_NULL(internal_fn,gcall *)448 expand_UBSAN_NULL (internal_fn, gcall *)
449 {
450   gcc_unreachable ();
451 }
452 
453 /* This should get expanded in the sanopt pass.  */
454 
455 static void
expand_UBSAN_BOUNDS(internal_fn,gcall *)456 expand_UBSAN_BOUNDS (internal_fn, gcall *)
457 {
458   gcc_unreachable ();
459 }
460 
461 /* This should get expanded in the sanopt pass.  */
462 
463 static void
expand_UBSAN_VPTR(internal_fn,gcall *)464 expand_UBSAN_VPTR (internal_fn, gcall *)
465 {
466   gcc_unreachable ();
467 }
468 
469 /* This should get expanded in the sanopt pass.  */
470 
471 static void
expand_UBSAN_PTR(internal_fn,gcall *)472 expand_UBSAN_PTR (internal_fn, gcall *)
473 {
474   gcc_unreachable ();
475 }
476 
477 /* This should get expanded in the sanopt pass.  */
478 
479 static void
expand_UBSAN_OBJECT_SIZE(internal_fn,gcall *)480 expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
481 {
482   gcc_unreachable ();
483 }
484 
485 /* This should get expanded in the sanopt pass.  */
486 
487 static void
expand_HWASAN_CHECK(internal_fn,gcall *)488 expand_HWASAN_CHECK (internal_fn, gcall *)
489 {
490   gcc_unreachable ();
491 }
492 
493 /* For hwasan stack tagging:
494    Clear tags on the dynamically allocated space.
495    For use after an object dynamically allocated on the stack goes out of
496    scope.  */
497 static void
expand_HWASAN_ALLOCA_UNPOISON(internal_fn,gcall * gc)498 expand_HWASAN_ALLOCA_UNPOISON (internal_fn, gcall *gc)
499 {
500   gcc_assert (Pmode == ptr_mode);
501   tree restored_position = gimple_call_arg (gc, 0);
502   rtx restored_rtx = expand_expr (restored_position, NULL_RTX, VOIDmode,
503 				  EXPAND_NORMAL);
504   rtx func = init_one_libfunc ("__hwasan_tag_memory");
505   rtx off = expand_simple_binop (Pmode, MINUS, restored_rtx,
506 				 stack_pointer_rtx, NULL_RTX, 0,
507 				 OPTAB_WIDEN);
508   emit_library_call_value (func, NULL_RTX, LCT_NORMAL, VOIDmode,
509 			   virtual_stack_dynamic_rtx, Pmode,
510 			   HWASAN_STACK_BACKGROUND, QImode,
511 			   off, Pmode);
512 }
513 
514 /* For hwasan stack tagging:
515    Return a tag to be used for a dynamic allocation.  */
516 static void
expand_HWASAN_CHOOSE_TAG(internal_fn,gcall * gc)517 expand_HWASAN_CHOOSE_TAG (internal_fn, gcall *gc)
518 {
519   tree tag = gimple_call_lhs (gc);
520   rtx target = expand_expr (tag, NULL_RTX, VOIDmode, EXPAND_NORMAL);
521   machine_mode mode = GET_MODE (target);
522   gcc_assert (mode == QImode);
523 
524   rtx base_tag = targetm.memtag.extract_tag (hwasan_frame_base (), NULL_RTX);
525   gcc_assert (base_tag);
526   rtx tag_offset = gen_int_mode (hwasan_current_frame_tag (), QImode);
527   rtx chosen_tag = expand_simple_binop (QImode, PLUS, base_tag, tag_offset,
528 					target, /* unsignedp = */1,
529 					OPTAB_WIDEN);
530   chosen_tag = hwasan_truncate_to_tag_size (chosen_tag, target);
531 
532   /* Really need to put the tag into the `target` RTX.  */
533   if (chosen_tag != target)
534     {
535       rtx temp = chosen_tag;
536       gcc_assert (GET_MODE (chosen_tag) == mode);
537       emit_move_insn (target, temp);
538     }
539 
540   hwasan_increment_frame_tag ();
541 }
542 
543 /* For hwasan stack tagging:
544    Tag a region of space in the shadow stack according to the base pointer of
545    an object on the stack.  N.b. the length provided in the internal call is
546    required to be aligned to HWASAN_TAG_GRANULE_SIZE.  */
547 static void
expand_HWASAN_MARK(internal_fn,gcall * gc)548 expand_HWASAN_MARK (internal_fn, gcall *gc)
549 {
550   gcc_assert (ptr_mode == Pmode);
551   HOST_WIDE_INT flag = tree_to_shwi (gimple_call_arg (gc, 0));
552   bool is_poison = ((asan_mark_flags)flag) == ASAN_MARK_POISON;
553 
554   tree base = gimple_call_arg (gc, 1);
555   gcc_checking_assert (TREE_CODE (base) == ADDR_EXPR);
556   rtx base_rtx = expand_normal (base);
557 
558   rtx tag = is_poison ? HWASAN_STACK_BACKGROUND
559     : targetm.memtag.extract_tag (base_rtx, NULL_RTX);
560   rtx address = targetm.memtag.untagged_pointer (base_rtx, NULL_RTX);
561 
562   tree len = gimple_call_arg (gc, 2);
563   rtx r_len = expand_normal (len);
564 
565   rtx func = init_one_libfunc ("__hwasan_tag_memory");
566   emit_library_call (func, LCT_NORMAL, VOIDmode, address, Pmode,
567 		     tag, QImode, r_len, Pmode);
568 }
569 
570 /* For hwasan stack tagging:
571    Store a tag into a pointer.  */
572 static void
expand_HWASAN_SET_TAG(internal_fn,gcall * gc)573 expand_HWASAN_SET_TAG (internal_fn, gcall *gc)
574 {
575   gcc_assert (ptr_mode == Pmode);
576   tree g_target = gimple_call_lhs (gc);
577   tree g_ptr = gimple_call_arg (gc, 0);
578   tree g_tag = gimple_call_arg (gc, 1);
579 
580   rtx ptr = expand_normal (g_ptr);
581   rtx tag = expand_expr (g_tag, NULL_RTX, QImode, EXPAND_NORMAL);
582   rtx target = expand_normal (g_target);
583 
584   rtx untagged = targetm.memtag.untagged_pointer (ptr, target);
585   rtx tagged_value = targetm.memtag.set_tag (untagged, tag, target);
586   if (tagged_value != target)
587     emit_move_insn (target, tagged_value);
588 }
589 
590 /* This should get expanded in the sanopt pass.  */
591 
592 static void
expand_ASAN_CHECK(internal_fn,gcall *)593 expand_ASAN_CHECK (internal_fn, gcall *)
594 {
595   gcc_unreachable ();
596 }
597 
598 /* This should get expanded in the sanopt pass.  */
599 
600 static void
expand_ASAN_MARK(internal_fn,gcall *)601 expand_ASAN_MARK (internal_fn, gcall *)
602 {
603   gcc_unreachable ();
604 }
605 
606 /* This should get expanded in the sanopt pass.  */
607 
608 static void
expand_ASAN_POISON(internal_fn,gcall *)609 expand_ASAN_POISON (internal_fn, gcall *)
610 {
611   gcc_unreachable ();
612 }
613 
614 /* This should get expanded in the sanopt pass.  */
615 
616 static void
expand_ASAN_POISON_USE(internal_fn,gcall *)617 expand_ASAN_POISON_USE (internal_fn, gcall *)
618 {
619   gcc_unreachable ();
620 }
621 
622 /* This should get expanded in the tsan pass.  */
623 
624 static void
expand_TSAN_FUNC_EXIT(internal_fn,gcall *)625 expand_TSAN_FUNC_EXIT (internal_fn, gcall *)
626 {
627   gcc_unreachable ();
628 }
629 
630 /* This should get expanded in the lower pass.  */
631 
632 static void
expand_FALLTHROUGH(internal_fn,gcall * call)633 expand_FALLTHROUGH (internal_fn, gcall *call)
634 {
635   error_at (gimple_location (call),
636 	    "invalid use of attribute %<fallthrough%>");
637 }
638 
639 /* Return minimum precision needed to represent all values
640    of ARG in SIGNed integral type.  */
641 
642 static int
get_min_precision(tree arg,signop sign)643 get_min_precision (tree arg, signop sign)
644 {
645   int prec = TYPE_PRECISION (TREE_TYPE (arg));
646   int cnt = 0;
647   signop orig_sign = sign;
648   if (TREE_CODE (arg) == INTEGER_CST)
649     {
650       int p;
651       if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
652 	{
653 	  widest_int w = wi::to_widest (arg);
654 	  w = wi::ext (w, prec, sign);
655 	  p = wi::min_precision (w, sign);
656 	}
657       else
658 	p = wi::min_precision (wi::to_wide (arg), sign);
659       return MIN (p, prec);
660     }
661   while (CONVERT_EXPR_P (arg)
662 	 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
663 	 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
664     {
665       arg = TREE_OPERAND (arg, 0);
666       if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
667 	{
668 	  if (TYPE_UNSIGNED (TREE_TYPE (arg)))
669 	    sign = UNSIGNED;
670 	  else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
671 	    return prec + (orig_sign != sign);
672 	  prec = TYPE_PRECISION (TREE_TYPE (arg));
673 	}
674       if (++cnt > 30)
675 	return prec + (orig_sign != sign);
676     }
677   if (CONVERT_EXPR_P (arg)
678       && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
679       && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) > prec)
680     {
681       /* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
682 	 If y_2's min precision is smaller than prec, return that.  */
683       int oprec = get_min_precision (TREE_OPERAND (arg, 0), sign);
684       if (oprec < prec)
685 	return oprec + (orig_sign != sign);
686     }
687   if (TREE_CODE (arg) != SSA_NAME)
688     return prec + (orig_sign != sign);
689   value_range r;
690   while (!get_global_range_query ()->range_of_expr (r, arg)
691 	 || r.kind () != VR_RANGE)
692     {
693       gimple *g = SSA_NAME_DEF_STMT (arg);
694       if (is_gimple_assign (g)
695 	  && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
696 	{
697 	  tree t = gimple_assign_rhs1 (g);
698 	  if (INTEGRAL_TYPE_P (TREE_TYPE (t))
699 	      && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
700 	    {
701 	      arg = t;
702 	      if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
703 		{
704 		  if (TYPE_UNSIGNED (TREE_TYPE (arg)))
705 		    sign = UNSIGNED;
706 		  else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
707 		    return prec + (orig_sign != sign);
708 		  prec = TYPE_PRECISION (TREE_TYPE (arg));
709 		}
710 	      if (++cnt > 30)
711 		return prec + (orig_sign != sign);
712 	      continue;
713 	    }
714 	}
715       return prec + (orig_sign != sign);
716     }
717   if (sign == TYPE_SIGN (TREE_TYPE (arg)))
718     {
719       int p1 = wi::min_precision (r.lower_bound (), sign);
720       int p2 = wi::min_precision (r.upper_bound (), sign);
721       p1 = MAX (p1, p2);
722       prec = MIN (prec, p1);
723     }
724   else if (sign == UNSIGNED && !wi::neg_p (r.lower_bound (), SIGNED))
725     {
726       int p = wi::min_precision (r.upper_bound (), UNSIGNED);
727       prec = MIN (prec, p);
728     }
729   return prec + (orig_sign != sign);
730 }
731 
732 /* Helper for expand_*_overflow.  Set the __imag__ part to true
733    (1 except for signed:1 type, in which case store -1).  */
734 
735 static void
expand_arith_set_overflow(tree lhs,rtx target)736 expand_arith_set_overflow (tree lhs, rtx target)
737 {
738   if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs))) == 1
739       && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs))))
740     write_complex_part (target, constm1_rtx, true);
741   else
742     write_complex_part (target, const1_rtx, true);
743 }
744 
745 /* Helper for expand_*_overflow.  Store RES into the __real__ part
746    of TARGET.  If RES has larger MODE than __real__ part of TARGET,
747    set the __imag__ part to 1 if RES doesn't fit into it.  Similarly
748    if LHS has smaller precision than its mode.  */
749 
750 static void
expand_arith_overflow_result_store(tree lhs,rtx target,scalar_int_mode mode,rtx res)751 expand_arith_overflow_result_store (tree lhs, rtx target,
752 				    scalar_int_mode mode, rtx res)
753 {
754   scalar_int_mode tgtmode
755     = as_a <scalar_int_mode> (GET_MODE_INNER (GET_MODE (target)));
756   rtx lres = res;
757   if (tgtmode != mode)
758     {
759       rtx_code_label *done_label = gen_label_rtx ();
760       int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
761       lres = convert_modes (tgtmode, mode, res, uns);
762       gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
763       do_compare_rtx_and_jump (res, convert_modes (mode, tgtmode, lres, uns),
764 			       EQ, true, mode, NULL_RTX, NULL, done_label,
765 			       profile_probability::very_likely ());
766       expand_arith_set_overflow (lhs, target);
767       emit_label (done_label);
768     }
769   int prec = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs)));
770   int tgtprec = GET_MODE_PRECISION (tgtmode);
771   if (prec < tgtprec)
772     {
773       rtx_code_label *done_label = gen_label_rtx ();
774       int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
775       res = lres;
776       if (uns)
777 	{
778 	  rtx mask
779 	    = immed_wide_int_const (wi::shifted_mask (0, prec, false, tgtprec),
780 				    tgtmode);
781 	  lres = expand_simple_binop (tgtmode, AND, res, mask, NULL_RTX,
782 				      true, OPTAB_LIB_WIDEN);
783 	}
784       else
785 	{
786 	  lres = expand_shift (LSHIFT_EXPR, tgtmode, res, tgtprec - prec,
787 			       NULL_RTX, 1);
788 	  lres = expand_shift (RSHIFT_EXPR, tgtmode, lres, tgtprec - prec,
789 			       NULL_RTX, 0);
790 	}
791       do_compare_rtx_and_jump (res, lres,
792 			       EQ, true, tgtmode, NULL_RTX, NULL, done_label,
793 			       profile_probability::very_likely ());
794       expand_arith_set_overflow (lhs, target);
795       emit_label (done_label);
796     }
797   write_complex_part (target, lres, false);
798 }
799 
800 /* Helper for expand_*_overflow.  Store RES into TARGET.  */
801 
802 static void
expand_ubsan_result_store(rtx target,rtx res)803 expand_ubsan_result_store (rtx target, rtx res)
804 {
805   if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
806     /* If this is a scalar in a register that is stored in a wider mode
807        than the declared mode, compute the result into its declared mode
808        and then convert to the wider mode.  Our value is the computed
809        expression.  */
810     convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
811   else
812     emit_move_insn (target, res);
813 }
814 
815 /* Add sub/add overflow checking to the statement STMT.
816    CODE says whether the operation is +, or -.  */
817 
818 void
expand_addsub_overflow(location_t loc,tree_code code,tree lhs,tree arg0,tree arg1,bool unsr_p,bool uns0_p,bool uns1_p,bool is_ubsan,tree * datap)819 expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
820 			tree arg0, tree arg1, bool unsr_p, bool uns0_p,
821 			bool uns1_p, bool is_ubsan, tree *datap)
822 {
823   rtx res, target = NULL_RTX;
824   tree fn;
825   rtx_code_label *done_label = gen_label_rtx ();
826   rtx_code_label *do_error = gen_label_rtx ();
827   do_pending_stack_adjust ();
828   rtx op0 = expand_normal (arg0);
829   rtx op1 = expand_normal (arg1);
830   scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
831   int prec = GET_MODE_PRECISION (mode);
832   rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
833   bool do_xor = false;
834 
835   if (is_ubsan)
836     gcc_assert (!unsr_p && !uns0_p && !uns1_p);
837 
838   if (lhs)
839     {
840       target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
841       if (!is_ubsan)
842 	write_complex_part (target, const0_rtx, true);
843     }
844 
845   /* We assume both operands and result have the same precision
846      here (GET_MODE_BITSIZE (mode)), S stands for signed type
847      with that precision, U for unsigned type with that precision,
848      sgn for unsigned most significant bit in that precision.
849      s1 is signed first operand, u1 is unsigned first operand,
850      s2 is signed second operand, u2 is unsigned second operand,
851      sr is signed result, ur is unsigned result and the following
852      rules say how to compute result (which is always result of
853      the operands as if both were unsigned, cast to the right
854      signedness) and how to compute whether operation overflowed.
855 
856      s1 + s2 -> sr
857 	res = (S) ((U) s1 + (U) s2)
858 	ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
859      s1 - s2 -> sr
860 	res = (S) ((U) s1 - (U) s2)
861 	ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
862      u1 + u2 -> ur
863 	res = u1 + u2
864 	ovf = res < u1 (or jump on carry, but RTL opts will handle it)
865      u1 - u2 -> ur
866 	res = u1 - u2
867 	ovf = res > u1 (or jump on carry, but RTL opts will handle it)
868      s1 + u2 -> sr
869 	res = (S) ((U) s1 + u2)
870 	ovf = ((U) res ^ sgn) < u2
871      s1 + u2 -> ur
872 	t1 = (S) (u2 ^ sgn)
873 	t2 = s1 + t1
874 	res = (U) t2 ^ sgn
875 	ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
876      s1 - u2 -> sr
877 	res = (S) ((U) s1 - u2)
878 	ovf = u2 > ((U) s1 ^ sgn)
879      s1 - u2 -> ur
880 	res = (U) s1 - u2
881 	ovf = s1 < 0 || u2 > (U) s1
882      u1 - s2 -> sr
883 	res = u1 - (U) s2
884  	ovf = u1 >= ((U) s2 ^ sgn)
885      u1 - s2 -> ur
886 	t1 = u1 ^ sgn
887 	t2 = t1 - (U) s2
888 	res = t2 ^ sgn
889 	ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
890      s1 + s2 -> ur
891 	res = (U) s1 + (U) s2
892 	ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
893      u1 + u2 -> sr
894 	res = (S) (u1 + u2)
895 	ovf = (U) res < u2 || res < 0
896      u1 - u2 -> sr
897 	res = (S) (u1 - u2)
898 	ovf = u1 >= u2 ? res < 0 : res >= 0
899      s1 - s2 -> ur
900 	res = (U) s1 - (U) s2
901 	ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0)  */
902 
903   if (code == PLUS_EXPR && uns0_p && !uns1_p)
904     {
905       /* PLUS_EXPR is commutative, if operand signedness differs,
906 	 canonicalize to the first operand being signed and second
907 	 unsigned to simplify following code.  */
908       std::swap (op0, op1);
909       std::swap (arg0, arg1);
910       uns0_p = false;
911       uns1_p = true;
912     }
913 
914   /* u1 +- u2 -> ur  */
915   if (uns0_p && uns1_p && unsr_p)
916     {
917       insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab
918                                        : usubv4_optab, mode);
919       if (icode != CODE_FOR_nothing)
920 	{
921 	  class expand_operand ops[4];
922 	  rtx_insn *last = get_last_insn ();
923 
924 	  res = gen_reg_rtx (mode);
925 	  create_output_operand (&ops[0], res, mode);
926 	  create_input_operand (&ops[1], op0, mode);
927 	  create_input_operand (&ops[2], op1, mode);
928 	  create_fixed_operand (&ops[3], do_error);
929 	  if (maybe_expand_insn (icode, 4, ops))
930 	    {
931 	      last = get_last_insn ();
932 	      if (profile_status_for_fn (cfun) != PROFILE_ABSENT
933 		  && JUMP_P (last)
934 		  && any_condjump_p (last)
935 		  && !find_reg_note (last, REG_BR_PROB, 0))
936 		add_reg_br_prob_note (last,
937 				      profile_probability::very_unlikely ());
938 	      emit_jump (done_label);
939 	      goto do_error_label;
940 	    }
941 
942 	  delete_insns_since (last);
943 	}
944 
945       /* Compute the operation.  On RTL level, the addition is always
946 	 unsigned.  */
947       res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
948 			  op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
949       rtx tem = op0;
950       /* For PLUS_EXPR, the operation is commutative, so we can pick
951 	 operand to compare against.  For prec <= BITS_PER_WORD, I think
952 	 preferring REG operand is better over CONST_INT, because
953 	 the CONST_INT might enlarge the instruction or CSE would need
954 	 to figure out we'd already loaded it into a register before.
955 	 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
956 	 as then the multi-word comparison can be perhaps simplified.  */
957       if (code == PLUS_EXPR
958 	  && (prec <= BITS_PER_WORD
959 	      ? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
960 	      : CONST_SCALAR_INT_P (op1)))
961 	tem = op1;
962       do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
963 			       true, mode, NULL_RTX, NULL, done_label,
964 			       profile_probability::very_likely ());
965       goto do_error_label;
966     }
967 
968   /* s1 +- u2 -> sr  */
969   if (!uns0_p && uns1_p && !unsr_p)
970     {
971       /* Compute the operation.  On RTL level, the addition is always
972 	 unsigned.  */
973       res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
974 			  op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
975       rtx tem = expand_binop (mode, add_optab,
976 			      code == PLUS_EXPR ? res : op0, sgn,
977 			      NULL_RTX, false, OPTAB_LIB_WIDEN);
978       do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
979 			       done_label, profile_probability::very_likely ());
980       goto do_error_label;
981     }
982 
983   /* s1 + u2 -> ur  */
984   if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
985     {
986       op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
987 			  OPTAB_LIB_WIDEN);
988       /* As we've changed op1, we have to avoid using the value range
989 	 for the original argument.  */
990       arg1 = error_mark_node;
991       do_xor = true;
992       goto do_signed;
993     }
994 
995   /* u1 - s2 -> ur  */
996   if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
997     {
998       op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
999 			  OPTAB_LIB_WIDEN);
1000       /* As we've changed op0, we have to avoid using the value range
1001 	 for the original argument.  */
1002       arg0 = error_mark_node;
1003       do_xor = true;
1004       goto do_signed;
1005     }
1006 
1007   /* s1 - u2 -> ur  */
1008   if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
1009     {
1010       /* Compute the operation.  On RTL level, the addition is always
1011 	 unsigned.  */
1012       res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1013 			  OPTAB_LIB_WIDEN);
1014       int pos_neg = get_range_pos_neg (arg0);
1015       if (pos_neg == 2)
1016 	/* If ARG0 is known to be always negative, this is always overflow.  */
1017 	emit_jump (do_error);
1018       else if (pos_neg == 3)
1019 	/* If ARG0 is not known to be always positive, check at runtime.  */
1020 	do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
1021 				 NULL, do_error, profile_probability::very_unlikely ());
1022       do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
1023 			       done_label, profile_probability::very_likely ());
1024       goto do_error_label;
1025     }
1026 
1027   /* u1 - s2 -> sr  */
1028   if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
1029     {
1030       /* Compute the operation.  On RTL level, the addition is always
1031 	 unsigned.  */
1032       res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1033 			  OPTAB_LIB_WIDEN);
1034       rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
1035 			      OPTAB_LIB_WIDEN);
1036       do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
1037 			       done_label, profile_probability::very_likely ());
1038       goto do_error_label;
1039     }
1040 
1041   /* u1 + u2 -> sr  */
1042   if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
1043     {
1044       /* Compute the operation.  On RTL level, the addition is always
1045 	 unsigned.  */
1046       res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
1047 			  OPTAB_LIB_WIDEN);
1048       do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
1049 			       NULL, do_error, profile_probability::very_unlikely ());
1050       rtx tem = op1;
1051       /* The operation is commutative, so we can pick operand to compare
1052 	 against.  For prec <= BITS_PER_WORD, I think preferring REG operand
1053 	 is better over CONST_INT, because the CONST_INT might enlarge the
1054 	 instruction or CSE would need to figure out we'd already loaded it
1055 	 into a register before.  For prec > BITS_PER_WORD, I think CONST_INT
1056 	 might be more beneficial, as then the multi-word comparison can be
1057 	 perhaps simplified.  */
1058       if (prec <= BITS_PER_WORD
1059 	  ? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
1060 	  : CONST_SCALAR_INT_P (op0))
1061 	tem = op0;
1062       do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
1063 			       done_label, profile_probability::very_likely ());
1064       goto do_error_label;
1065     }
1066 
1067   /* s1 +- s2 -> ur  */
1068   if (!uns0_p && !uns1_p && unsr_p)
1069     {
1070       /* Compute the operation.  On RTL level, the addition is always
1071 	 unsigned.  */
1072       res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1073 			  op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1074       int pos_neg = get_range_pos_neg (arg1);
1075       if (code == PLUS_EXPR)
1076 	{
1077 	  int pos_neg0 = get_range_pos_neg (arg0);
1078 	  if (pos_neg0 != 3 && pos_neg == 3)
1079 	    {
1080 	      std::swap (op0, op1);
1081 	      pos_neg = pos_neg0;
1082 	    }
1083 	}
1084       rtx tem;
1085       if (pos_neg != 3)
1086 	{
1087 	  tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
1088 				    ? and_optab : ior_optab,
1089 			      op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
1090 	  do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
1091 				   NULL, done_label, profile_probability::very_likely ());
1092 	}
1093       else
1094 	{
1095 	  rtx_code_label *do_ior_label = gen_label_rtx ();
1096 	  do_compare_rtx_and_jump (op1, const0_rtx,
1097 				   code == MINUS_EXPR ? GE : LT, false, mode,
1098 				   NULL_RTX, NULL, do_ior_label,
1099 				   profile_probability::even ());
1100 	  tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
1101 			      OPTAB_LIB_WIDEN);
1102 	  do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1103 				   NULL, done_label, profile_probability::very_likely ());
1104 	  emit_jump (do_error);
1105 	  emit_label (do_ior_label);
1106 	  tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
1107 			      OPTAB_LIB_WIDEN);
1108 	  do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1109 				   NULL, done_label, profile_probability::very_likely ());
1110 	}
1111       goto do_error_label;
1112     }
1113 
1114   /* u1 - u2 -> sr  */
1115   if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
1116     {
1117       /* Compute the operation.  On RTL level, the addition is always
1118 	 unsigned.  */
1119       res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1120 			  OPTAB_LIB_WIDEN);
1121       rtx_code_label *op0_geu_op1 = gen_label_rtx ();
1122       do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
1123 			       op0_geu_op1, profile_probability::even ());
1124       do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
1125 			       NULL, done_label, profile_probability::very_likely ());
1126       emit_jump (do_error);
1127       emit_label (op0_geu_op1);
1128       do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
1129 			       NULL, done_label, profile_probability::very_likely ());
1130       goto do_error_label;
1131     }
1132 
1133   gcc_assert (!uns0_p && !uns1_p && !unsr_p);
1134 
1135   /* s1 +- s2 -> sr  */
1136  do_signed:
1137   {
1138     insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab
1139 				     : subv4_optab, mode);
1140     if (icode != CODE_FOR_nothing)
1141       {
1142 	class expand_operand ops[4];
1143 	rtx_insn *last = get_last_insn ();
1144 
1145 	res = gen_reg_rtx (mode);
1146 	create_output_operand (&ops[0], res, mode);
1147 	create_input_operand (&ops[1], op0, mode);
1148 	create_input_operand (&ops[2], op1, mode);
1149 	create_fixed_operand (&ops[3], do_error);
1150 	if (maybe_expand_insn (icode, 4, ops))
1151 	  {
1152 	    last = get_last_insn ();
1153 	    if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1154 		&& JUMP_P (last)
1155 		&& any_condjump_p (last)
1156 		&& !find_reg_note (last, REG_BR_PROB, 0))
1157 	      add_reg_br_prob_note (last,
1158 				    profile_probability::very_unlikely ());
1159 	    emit_jump (done_label);
1160 	    goto do_error_label;
1161 	  }
1162 
1163 	delete_insns_since (last);
1164       }
1165 
1166     /* Compute the operation.  On RTL level, the addition is always
1167        unsigned.  */
1168     res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1169 			op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1170 
1171     /* If we can prove that one of the arguments (for MINUS_EXPR only
1172        the second operand, as subtraction is not commutative) is always
1173        non-negative or always negative, we can do just one comparison
1174        and conditional jump.  */
1175     int pos_neg = get_range_pos_neg (arg1);
1176     if (code == PLUS_EXPR)
1177       {
1178 	int pos_neg0 = get_range_pos_neg (arg0);
1179 	if (pos_neg0 != 3 && pos_neg == 3)
1180 	  {
1181 	    std::swap (op0, op1);
1182 	    pos_neg = pos_neg0;
1183 	  }
1184       }
1185 
1186     /* Addition overflows if and only if the two operands have the same sign,
1187        and the result has the opposite sign.  Subtraction overflows if and
1188        only if the two operands have opposite sign, and the subtrahend has
1189        the same sign as the result.  Here 0 is counted as positive.  */
1190     if (pos_neg == 3)
1191       {
1192 	/* Compute op0 ^ op1 (operands have opposite sign).  */
1193         rtx op_xor = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1194 				   OPTAB_LIB_WIDEN);
1195 
1196 	/* Compute res ^ op1 (result and 2nd operand have opposite sign).  */
1197 	rtx res_xor = expand_binop (mode, xor_optab, res, op1, NULL_RTX, false,
1198 				    OPTAB_LIB_WIDEN);
1199 
1200 	rtx tem;
1201 	if (code == PLUS_EXPR)
1202 	  {
1203 	    /* Compute (res ^ op1) & ~(op0 ^ op1).  */
1204 	    tem = expand_unop (mode, one_cmpl_optab, op_xor, NULL_RTX, false);
1205 	    tem = expand_binop (mode, and_optab, res_xor, tem, NULL_RTX, false,
1206 				OPTAB_LIB_WIDEN);
1207 	  }
1208 	else
1209 	  {
1210 	    /* Compute (op0 ^ op1) & ~(res ^ op1).  */
1211 	    tem = expand_unop (mode, one_cmpl_optab, res_xor, NULL_RTX, false);
1212 	    tem = expand_binop (mode, and_optab, op_xor, tem, NULL_RTX, false,
1213 				OPTAB_LIB_WIDEN);
1214 	  }
1215 
1216 	/* No overflow if the result has bit sign cleared.  */
1217 	do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1218 				 NULL, done_label, profile_probability::very_likely ());
1219       }
1220 
1221     /* Compare the result of the operation with the first operand.
1222        No overflow for addition if second operand is positive and result
1223        is larger or second operand is negative and result is smaller.
1224        Likewise for subtraction with sign of second operand flipped.  */
1225     else
1226       do_compare_rtx_and_jump (res, op0,
1227 			       (pos_neg == 1) ^ (code == MINUS_EXPR) ? GE : LE,
1228 			       false, mode, NULL_RTX, NULL, done_label,
1229 			       profile_probability::very_likely ());
1230   }
1231 
1232  do_error_label:
1233   emit_label (do_error);
1234   if (is_ubsan)
1235     {
1236       /* Expand the ubsan builtin call.  */
1237       push_temp_slots ();
1238       fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
1239 					 arg0, arg1, datap);
1240       expand_normal (fn);
1241       pop_temp_slots ();
1242       do_pending_stack_adjust ();
1243     }
1244   else if (lhs)
1245     expand_arith_set_overflow (lhs, target);
1246 
1247   /* We're done.  */
1248   emit_label (done_label);
1249 
1250   if (lhs)
1251     {
1252       if (is_ubsan)
1253 	expand_ubsan_result_store (target, res);
1254       else
1255 	{
1256 	  if (do_xor)
1257 	    res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
1258 				OPTAB_LIB_WIDEN);
1259 
1260 	  expand_arith_overflow_result_store (lhs, target, mode, res);
1261 	}
1262     }
1263 }
1264 
1265 /* Add negate overflow checking to the statement STMT.  */
1266 
1267 static void
expand_neg_overflow(location_t loc,tree lhs,tree arg1,bool is_ubsan,tree * datap)1268 expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan,
1269 		     tree *datap)
1270 {
1271   rtx res, op1;
1272   tree fn;
1273   rtx_code_label *done_label, *do_error;
1274   rtx target = NULL_RTX;
1275 
1276   done_label = gen_label_rtx ();
1277   do_error = gen_label_rtx ();
1278 
1279   do_pending_stack_adjust ();
1280   op1 = expand_normal (arg1);
1281 
1282   scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1));
1283   if (lhs)
1284     {
1285       target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1286       if (!is_ubsan)
1287 	write_complex_part (target, const0_rtx, true);
1288     }
1289 
1290   enum insn_code icode = optab_handler (negv3_optab, mode);
1291   if (icode != CODE_FOR_nothing)
1292     {
1293       class expand_operand ops[3];
1294       rtx_insn *last = get_last_insn ();
1295 
1296       res = gen_reg_rtx (mode);
1297       create_output_operand (&ops[0], res, mode);
1298       create_input_operand (&ops[1], op1, mode);
1299       create_fixed_operand (&ops[2], do_error);
1300       if (maybe_expand_insn (icode, 3, ops))
1301 	{
1302 	  last = get_last_insn ();
1303 	  if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1304 	      && JUMP_P (last)
1305 	      && any_condjump_p (last)
1306 	      && !find_reg_note (last, REG_BR_PROB, 0))
1307 	    add_reg_br_prob_note (last,
1308 				  profile_probability::very_unlikely ());
1309 	  emit_jump (done_label);
1310         }
1311       else
1312 	{
1313 	  delete_insns_since (last);
1314 	  icode = CODE_FOR_nothing;
1315 	}
1316     }
1317 
1318   if (icode == CODE_FOR_nothing)
1319     {
1320       /* Compute the operation.  On RTL level, the addition is always
1321 	 unsigned.  */
1322       res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1323 
1324       /* Compare the operand with the most negative value.  */
1325       rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
1326       do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
1327 			       done_label, profile_probability::very_likely ());
1328     }
1329 
1330   emit_label (do_error);
1331   if (is_ubsan)
1332     {
1333       /* Expand the ubsan builtin call.  */
1334       push_temp_slots ();
1335       fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
1336 					 arg1, NULL_TREE, datap);
1337       expand_normal (fn);
1338       pop_temp_slots ();
1339       do_pending_stack_adjust ();
1340     }
1341   else if (lhs)
1342     expand_arith_set_overflow (lhs, target);
1343 
1344   /* We're done.  */
1345   emit_label (done_label);
1346 
1347   if (lhs)
1348     {
1349       if (is_ubsan)
1350 	expand_ubsan_result_store (target, res);
1351       else
1352 	expand_arith_overflow_result_store (lhs, target, mode, res);
1353     }
1354 }
1355 
1356 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1357    mode MODE can be expanded without using a libcall.  */
1358 
1359 static bool
can_widen_mult_without_libcall(scalar_int_mode wmode,scalar_int_mode mode,rtx op0,rtx op1,bool uns)1360 can_widen_mult_without_libcall (scalar_int_mode wmode, scalar_int_mode mode,
1361 				rtx op0, rtx op1, bool uns)
1362 {
1363   if (find_widening_optab_handler (umul_widen_optab, wmode, mode)
1364       != CODE_FOR_nothing)
1365     return true;
1366 
1367   if (find_widening_optab_handler (smul_widen_optab, wmode, mode)
1368       != CODE_FOR_nothing)
1369     return true;
1370 
1371   rtx_insn *last = get_last_insn ();
1372   if (CONSTANT_P (op0))
1373     op0 = convert_modes (wmode, mode, op0, uns);
1374   else
1375     op0 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 1);
1376   if (CONSTANT_P (op1))
1377     op1 = convert_modes (wmode, mode, op1, uns);
1378   else
1379     op1 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 2);
1380   rtx ret = expand_mult (wmode, op0, op1, NULL_RTX, uns, true);
1381   delete_insns_since (last);
1382   return ret != NULL_RTX;
1383 }
1384 
1385 /* Add mul overflow checking to the statement STMT.  */
1386 
1387 static void
expand_mul_overflow(location_t loc,tree lhs,tree arg0,tree arg1,bool unsr_p,bool uns0_p,bool uns1_p,bool is_ubsan,tree * datap)1388 expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
1389 		     bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan,
1390 		     tree *datap)
1391 {
1392   rtx res, op0, op1;
1393   tree fn, type;
1394   rtx_code_label *done_label, *do_error;
1395   rtx target = NULL_RTX;
1396   signop sign;
1397   enum insn_code icode;
1398   int save_flag_trapv = flag_trapv;
1399 
1400   /* We don't want any __mulv?i3 etc. calls from the expansion of
1401      these internal functions, so disable -ftrapv temporarily.  */
1402   flag_trapv = 0;
1403   done_label = gen_label_rtx ();
1404   do_error = gen_label_rtx ();
1405 
1406   do_pending_stack_adjust ();
1407   op0 = expand_normal (arg0);
1408   op1 = expand_normal (arg1);
1409 
1410   scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
1411   bool uns = unsr_p;
1412   if (lhs)
1413     {
1414       target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1415       if (!is_ubsan)
1416 	write_complex_part (target, const0_rtx, true);
1417     }
1418 
1419   if (is_ubsan)
1420     gcc_assert (!unsr_p && !uns0_p && !uns1_p);
1421 
1422   /* We assume both operands and result have the same precision
1423      here (GET_MODE_BITSIZE (mode)), S stands for signed type
1424      with that precision, U for unsigned type with that precision,
1425      sgn for unsigned most significant bit in that precision.
1426      s1 is signed first operand, u1 is unsigned first operand,
1427      s2 is signed second operand, u2 is unsigned second operand,
1428      sr is signed result, ur is unsigned result and the following
1429      rules say how to compute result (which is always result of
1430      the operands as if both were unsigned, cast to the right
1431      signedness) and how to compute whether operation overflowed.
1432      main_ovf (false) stands for jump on signed multiplication
1433      overflow or the main algorithm with uns == false.
1434      main_ovf (true) stands for jump on unsigned multiplication
1435      overflow or the main algorithm with uns == true.
1436 
1437      s1 * s2 -> sr
1438 	res = (S) ((U) s1 * (U) s2)
1439 	ovf = main_ovf (false)
1440      u1 * u2 -> ur
1441 	res = u1 * u2
1442 	ovf = main_ovf (true)
1443      s1 * u2 -> ur
1444 	res = (U) s1 * u2
1445 	ovf = (s1 < 0 && u2) || main_ovf (true)
1446      u1 * u2 -> sr
1447 	res = (S) (u1 * u2)
1448 	ovf = res < 0 || main_ovf (true)
1449      s1 * u2 -> sr
1450 	res = (S) ((U) s1 * u2)
1451 	ovf = (S) u2 >= 0 ? main_ovf (false)
1452 			  : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1453      s1 * s2 -> ur
1454 	t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1455 	t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1456 	res = t1 * t2
1457 	ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true)  */
1458 
1459   if (uns0_p && !uns1_p)
1460     {
1461       /* Multiplication is commutative, if operand signedness differs,
1462 	 canonicalize to the first operand being signed and second
1463 	 unsigned to simplify following code.  */
1464       std::swap (op0, op1);
1465       std::swap (arg0, arg1);
1466       uns0_p = false;
1467       uns1_p = true;
1468     }
1469 
1470   int pos_neg0 = get_range_pos_neg (arg0);
1471   int pos_neg1 = get_range_pos_neg (arg1);
1472 
1473   /* s1 * u2 -> ur  */
1474   if (!uns0_p && uns1_p && unsr_p)
1475     {
1476       switch (pos_neg0)
1477 	{
1478 	case 1:
1479 	  /* If s1 is non-negative, just perform normal u1 * u2 -> ur.  */
1480 	  goto do_main;
1481 	case 2:
1482 	  /* If s1 is negative, avoid the main code, just multiply and
1483 	     signal overflow if op1 is not 0.  */
1484 	  struct separate_ops ops;
1485 	  ops.code = MULT_EXPR;
1486 	  ops.type = TREE_TYPE (arg1);
1487 	  ops.op0 = make_tree (ops.type, op0);
1488 	  ops.op1 = make_tree (ops.type, op1);
1489 	  ops.op2 = NULL_TREE;
1490 	  ops.location = loc;
1491 	  res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1492 	  do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1493 				   NULL, done_label, profile_probability::very_likely ());
1494 	  goto do_error_label;
1495 	case 3:
1496 	  if (get_min_precision (arg1, UNSIGNED)
1497 	      + get_min_precision (arg0, SIGNED) <= GET_MODE_PRECISION (mode))
1498 	    {
1499 	      /* If the first operand is sign extended from narrower type, the
1500 		 second operand is zero extended from narrower type and
1501 		 the sum of the two precisions is smaller or equal to the
1502 		 result precision: if the first argument is at runtime
1503 		 non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1504 		 and there will be no overflow, if the first argument is
1505 		 negative and the second argument zero, the result will be
1506 		 0 and there will be no overflow, if the first argument is
1507 		 negative and the second argument positive, the result when
1508 		 treated as signed will be negative (minimum -0x7f80 or
1509 		 -0x7f..f80..0) there will be always overflow.  So, do
1510 		 res = (U) (s1 * u2)
1511 		 ovf = (S) res < 0  */
1512 	      struct separate_ops ops;
1513 	      ops.code = MULT_EXPR;
1514 	      ops.type
1515 		= build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1516 						  1);
1517 	      ops.op0 = make_tree (ops.type, op0);
1518 	      ops.op1 = make_tree (ops.type, op1);
1519 	      ops.op2 = NULL_TREE;
1520 	      ops.location = loc;
1521 	      res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1522 	      do_compare_rtx_and_jump (res, const0_rtx, GE, false,
1523 				       mode, NULL_RTX, NULL, done_label,
1524 				       profile_probability::very_likely ());
1525 	      goto do_error_label;
1526 	    }
1527 	  rtx_code_label *do_main_label;
1528 	  do_main_label = gen_label_rtx ();
1529 	  do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
1530 				   NULL, do_main_label, profile_probability::very_likely ());
1531 	  do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1532 				   NULL, do_main_label, profile_probability::very_likely ());
1533 	  expand_arith_set_overflow (lhs, target);
1534 	  emit_label (do_main_label);
1535 	  goto do_main;
1536 	default:
1537 	  gcc_unreachable ();
1538 	}
1539     }
1540 
1541   /* u1 * u2 -> sr  */
1542   if (uns0_p && uns1_p && !unsr_p)
1543     {
1544       if ((pos_neg0 | pos_neg1) == 1)
1545 	{
1546 	  /* If both arguments are zero extended from narrower types,
1547 	     the MSB will be clear on both and so we can pretend it is
1548 	     a normal s1 * s2 -> sr multiplication.  */
1549 	  uns0_p = false;
1550 	  uns1_p = false;
1551 	}
1552       else
1553 	uns = true;
1554       /* Rest of handling of this case after res is computed.  */
1555       goto do_main;
1556     }
1557 
1558   /* s1 * u2 -> sr  */
1559   if (!uns0_p && uns1_p && !unsr_p)
1560     {
1561       switch (pos_neg1)
1562 	{
1563 	case 1:
1564 	  goto do_main;
1565 	case 2:
1566 	  /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1567 	     avoid the main code, just multiply and signal overflow
1568 	     unless 0 * u2 or -1 * ((U) Smin).  */
1569 	  struct separate_ops ops;
1570 	  ops.code = MULT_EXPR;
1571 	  ops.type = TREE_TYPE (arg1);
1572 	  ops.op0 = make_tree (ops.type, op0);
1573 	  ops.op1 = make_tree (ops.type, op1);
1574 	  ops.op2 = NULL_TREE;
1575 	  ops.location = loc;
1576 	  res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1577 	  do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1578 				   NULL, done_label, profile_probability::very_likely ());
1579 	  do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
1580 				   NULL, do_error, profile_probability::very_unlikely ());
1581 	  int prec;
1582 	  prec = GET_MODE_PRECISION (mode);
1583 	  rtx sgn;
1584 	  sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1585 	  do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
1586 				   NULL, done_label, profile_probability::very_likely ());
1587 	  goto do_error_label;
1588 	case 3:
1589 	  /* Rest of handling of this case after res is computed.  */
1590 	  goto do_main;
1591 	default:
1592 	  gcc_unreachable ();
1593 	}
1594     }
1595 
1596   /* s1 * s2 -> ur  */
1597   if (!uns0_p && !uns1_p && unsr_p)
1598     {
1599       rtx tem;
1600       switch (pos_neg0 | pos_neg1)
1601 	{
1602 	case 1: /* Both operands known to be non-negative.  */
1603 	  goto do_main;
1604 	case 2: /* Both operands known to be negative.  */
1605 	  op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
1606 	  op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1607 	  /* Avoid looking at arg0/arg1 ranges, as we've changed
1608 	     the arguments.  */
1609 	  arg0 = error_mark_node;
1610 	  arg1 = error_mark_node;
1611 	  goto do_main;
1612 	case 3:
1613 	  if ((pos_neg0 ^ pos_neg1) == 3)
1614 	    {
1615 	      /* If one operand is known to be negative and the other
1616 		 non-negative, this overflows always, unless the non-negative
1617 		 one is 0.  Just do normal multiply and set overflow
1618 		 unless one of the operands is 0.  */
1619 	      struct separate_ops ops;
1620 	      ops.code = MULT_EXPR;
1621 	      ops.type
1622 		= build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1623 						  1);
1624 	      ops.op0 = make_tree (ops.type, op0);
1625 	      ops.op1 = make_tree (ops.type, op1);
1626 	      ops.op2 = NULL_TREE;
1627 	      ops.location = loc;
1628 	      res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1629 	      do_compare_rtx_and_jump (pos_neg0 == 1 ? op0 : op1, const0_rtx, EQ,
1630 				       true, mode, NULL_RTX, NULL, done_label,
1631 				       profile_probability::very_likely ());
1632 	      goto do_error_label;
1633 	    }
1634 	  if (get_min_precision (arg0, SIGNED)
1635 	      + get_min_precision (arg1, SIGNED) <= GET_MODE_PRECISION (mode))
1636 	    {
1637 	      /* If both operands are sign extended from narrower types and
1638 		 the sum of the two precisions is smaller or equal to the
1639 		 result precision: if both arguments are at runtime
1640 		 non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1641 		 and there will be no overflow, if both arguments are negative,
1642 		 maximum result will be 0x40..00 and there will be no overflow
1643 		 either, if one argument is positive and the other argument
1644 		 negative, the result when treated as signed will be negative
1645 		 and there will be always overflow, and if one argument is
1646 		 zero and the other negative the result will be zero and no
1647 		 overflow.  So, do
1648 		 res = (U) (s1 * s2)
1649 		 ovf = (S) res < 0  */
1650 	      struct separate_ops ops;
1651 	      ops.code = MULT_EXPR;
1652 	      ops.type
1653 		= build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1654 						  1);
1655 	      ops.op0 = make_tree (ops.type, op0);
1656 	      ops.op1 = make_tree (ops.type, op1);
1657 	      ops.op2 = NULL_TREE;
1658 	      ops.location = loc;
1659 	      res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1660 	      do_compare_rtx_and_jump (res, const0_rtx, GE, false,
1661 				       mode, NULL_RTX, NULL, done_label,
1662 				       profile_probability::very_likely ());
1663 	      goto do_error_label;
1664 	    }
1665 	  /* The general case, do all the needed comparisons at runtime.  */
1666 	  rtx_code_label *do_main_label, *after_negate_label;
1667 	  rtx rop0, rop1;
1668 	  rop0 = gen_reg_rtx (mode);
1669 	  rop1 = gen_reg_rtx (mode);
1670 	  emit_move_insn (rop0, op0);
1671 	  emit_move_insn (rop1, op1);
1672 	  op0 = rop0;
1673 	  op1 = rop1;
1674 	  do_main_label = gen_label_rtx ();
1675 	  after_negate_label = gen_label_rtx ();
1676 	  tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
1677 			      OPTAB_LIB_WIDEN);
1678 	  do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1679 				   NULL, after_negate_label, profile_probability::very_likely ());
1680 	  /* Both arguments negative here, negate them and continue with
1681 	     normal unsigned overflow checking multiplication.  */
1682 	  emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
1683 					    NULL_RTX, false));
1684 	  emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
1685 					    NULL_RTX, false));
1686 	  /* Avoid looking at arg0/arg1 ranges, as we might have changed
1687 	     the arguments.  */
1688 	  arg0 = error_mark_node;
1689 	  arg1 = error_mark_node;
1690 	  emit_jump (do_main_label);
1691 	  emit_label (after_negate_label);
1692 	  tem = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1693 			      OPTAB_LIB_WIDEN);
1694 	  do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1695 				   NULL, do_main_label,
1696 				   profile_probability::very_likely ());
1697 	  /* One argument is negative here, the other positive.  This
1698 	     overflows always, unless one of the arguments is 0.  But
1699 	     if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1700 	     is, thus we can keep do_main code oring in overflow as is.  */
1701 	  if (pos_neg0 != 2)
1702 	    do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1703 				     NULL, do_main_label,
1704 				     profile_probability::very_unlikely ());
1705 	  if (pos_neg1 != 2)
1706 	    do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1707 				     NULL, do_main_label,
1708 				     profile_probability::very_unlikely ());
1709 	  expand_arith_set_overflow (lhs, target);
1710 	  emit_label (do_main_label);
1711 	  goto do_main;
1712 	default:
1713 	  gcc_unreachable ();
1714 	}
1715     }
1716 
1717  do_main:
1718   type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
1719   sign = uns ? UNSIGNED : SIGNED;
1720   icode = optab_handler (uns ? umulv4_optab : mulv4_optab, mode);
1721   if (uns
1722       && (integer_pow2p (arg0) || integer_pow2p (arg1))
1723       && (optimize_insn_for_speed_p () || icode == CODE_FOR_nothing))
1724     {
1725       /* Optimize unsigned multiplication by power of 2 constant
1726 	 using 2 shifts, one for result, one to extract the shifted
1727 	 out bits to see if they are all zero.
1728 	 Don't do this if optimizing for size and we have umulv4_optab,
1729 	 in that case assume multiplication will be shorter.
1730 	 This is heuristics based on the single target that provides
1731 	 umulv4 right now (i?86/x86_64), if further targets add it, this
1732 	 might need to be revisited.
1733 	 Cases where both operands are constant should be folded already
1734 	 during GIMPLE, and cases where one operand is constant but not
1735 	 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1736 	 below can be done without multiplication, just by shifts and adds,
1737 	 or we'd need to divide the result (and hope it actually doesn't
1738 	 really divide nor multiply) and compare the result of the division
1739 	 with the original operand.  */
1740       rtx opn0 = op0;
1741       rtx opn1 = op1;
1742       tree argn0 = arg0;
1743       tree argn1 = arg1;
1744       if (integer_pow2p (arg0))
1745 	{
1746 	  std::swap (opn0, opn1);
1747 	  std::swap (argn0, argn1);
1748 	}
1749       int cnt = tree_log2 (argn1);
1750       if (cnt >= 0 && cnt < GET_MODE_PRECISION (mode))
1751 	{
1752 	  rtx upper = const0_rtx;
1753 	  res = expand_shift (LSHIFT_EXPR, mode, opn0, cnt, NULL_RTX, uns);
1754 	  if (cnt != 0)
1755 	    upper = expand_shift (RSHIFT_EXPR, mode, opn0,
1756 				  GET_MODE_PRECISION (mode) - cnt,
1757 				  NULL_RTX, uns);
1758 	  do_compare_rtx_and_jump (upper, const0_rtx, EQ, true, mode,
1759 				   NULL_RTX, NULL, done_label,
1760 				   profile_probability::very_likely ());
1761 	  goto do_error_label;
1762 	}
1763     }
1764   if (icode != CODE_FOR_nothing)
1765     {
1766       class expand_operand ops[4];
1767       rtx_insn *last = get_last_insn ();
1768 
1769       res = gen_reg_rtx (mode);
1770       create_output_operand (&ops[0], res, mode);
1771       create_input_operand (&ops[1], op0, mode);
1772       create_input_operand (&ops[2], op1, mode);
1773       create_fixed_operand (&ops[3], do_error);
1774       if (maybe_expand_insn (icode, 4, ops))
1775 	{
1776 	  last = get_last_insn ();
1777 	  if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1778 	      && JUMP_P (last)
1779 	      && any_condjump_p (last)
1780 	      && !find_reg_note (last, REG_BR_PROB, 0))
1781 	    add_reg_br_prob_note (last,
1782 				  profile_probability::very_unlikely ());
1783 	  emit_jump (done_label);
1784         }
1785       else
1786 	{
1787 	  delete_insns_since (last);
1788 	  icode = CODE_FOR_nothing;
1789 	}
1790     }
1791 
1792   if (icode == CODE_FOR_nothing)
1793     {
1794       struct separate_ops ops;
1795       int prec = GET_MODE_PRECISION (mode);
1796       scalar_int_mode hmode, wmode;
1797       ops.op0 = make_tree (type, op0);
1798       ops.op1 = make_tree (type, op1);
1799       ops.op2 = NULL_TREE;
1800       ops.location = loc;
1801 
1802       /* Optimize unsigned overflow check where we don't use the
1803 	 multiplication result, just whether overflow happened.
1804 	 If we can do MULT_HIGHPART_EXPR, that followed by
1805 	 comparison of the result against zero is cheapest.
1806 	 We'll still compute res, but it should be DCEd later.  */
1807       use_operand_p use;
1808       gimple *use_stmt;
1809       if (!is_ubsan
1810 	  && lhs
1811 	  && uns
1812 	  && !(uns0_p && uns1_p && !unsr_p)
1813 	  && can_mult_highpart_p (mode, uns) == 1
1814 	  && single_imm_use (lhs, &use, &use_stmt)
1815 	  && is_gimple_assign (use_stmt)
1816 	  && gimple_assign_rhs_code (use_stmt) == IMAGPART_EXPR)
1817 	goto highpart;
1818 
1819       if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
1820 	  && targetm.scalar_mode_supported_p (wmode)
1821 	  && can_widen_mult_without_libcall (wmode, mode, op0, op1, uns))
1822 	{
1823 	twoxwider:
1824 	  ops.code = WIDEN_MULT_EXPR;
1825 	  ops.type
1826 	    = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), uns);
1827 
1828 	  res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
1829 	  rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec,
1830 				     NULL_RTX, uns);
1831 	  hipart = convert_modes (mode, wmode, hipart, uns);
1832 	  res = convert_modes (mode, wmode, res, uns);
1833 	  if (uns)
1834 	    /* For the unsigned multiplication, there was overflow if
1835 	       HIPART is non-zero.  */
1836 	    do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
1837 				     NULL_RTX, NULL, done_label,
1838 				     profile_probability::very_likely ());
1839 	  else
1840 	    {
1841 	      /* RES is used more than once, place it in a pseudo.  */
1842 	      res = force_reg (mode, res);
1843 
1844 	      rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
1845 					  NULL_RTX, 0);
1846 	      /* RES is low half of the double width result, HIPART
1847 		 the high half.  There was overflow if
1848 		 HIPART is different from RES < 0 ? -1 : 0.  */
1849 	      do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
1850 				       NULL_RTX, NULL, done_label,
1851 				       profile_probability::very_likely ());
1852 	    }
1853 	}
1854       else if (can_mult_highpart_p (mode, uns) == 1)
1855 	{
1856 	highpart:
1857 	  ops.code = MULT_HIGHPART_EXPR;
1858 	  ops.type = type;
1859 
1860 	  rtx hipart = expand_expr_real_2 (&ops, NULL_RTX, mode,
1861 					   EXPAND_NORMAL);
1862 	  ops.code = MULT_EXPR;
1863 	  res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1864 	  if (uns)
1865 	    /* For the unsigned multiplication, there was overflow if
1866 	       HIPART is non-zero.  */
1867 	    do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
1868 				     NULL_RTX, NULL, done_label,
1869 				     profile_probability::very_likely ());
1870 	  else
1871 	    {
1872 	      rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
1873 					  NULL_RTX, 0);
1874 	      /* RES is low half of the double width result, HIPART
1875 		 the high half.  There was overflow if
1876 		 HIPART is different from RES < 0 ? -1 : 0.  */
1877 	      do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
1878 				       NULL_RTX, NULL, done_label,
1879 				       profile_probability::very_likely ());
1880 	    }
1881 
1882 	}
1883       else if (int_mode_for_size (prec / 2, 1).exists (&hmode)
1884 	       && 2 * GET_MODE_PRECISION (hmode) == prec)
1885 	{
1886 	  rtx_code_label *large_op0 = gen_label_rtx ();
1887 	  rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
1888 	  rtx_code_label *one_small_one_large = gen_label_rtx ();
1889 	  rtx_code_label *both_ops_large = gen_label_rtx ();
1890 	  rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
1891 	  rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
1892 	  rtx_code_label *do_overflow = gen_label_rtx ();
1893 	  rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
1894 
1895 	  unsigned int hprec = GET_MODE_PRECISION (hmode);
1896 	  rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
1897 				      NULL_RTX, uns);
1898 	  hipart0 = convert_modes (hmode, mode, hipart0, uns);
1899 	  rtx lopart0 = convert_modes (hmode, mode, op0, uns);
1900 	  rtx signbit0 = const0_rtx;
1901 	  if (!uns)
1902 	    signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
1903 				     NULL_RTX, 0);
1904 	  rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
1905 				      NULL_RTX, uns);
1906 	  hipart1 = convert_modes (hmode, mode, hipart1, uns);
1907 	  rtx lopart1 = convert_modes (hmode, mode, op1, uns);
1908 	  rtx signbit1 = const0_rtx;
1909 	  if (!uns)
1910 	    signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
1911 				     NULL_RTX, 0);
1912 
1913 	  res = gen_reg_rtx (mode);
1914 
1915 	  /* True if op0 resp. op1 are known to be in the range of
1916 	     halfstype.  */
1917 	  bool op0_small_p = false;
1918 	  bool op1_small_p = false;
1919 	  /* True if op0 resp. op1 are known to have all zeros or all ones
1920 	     in the upper half of bits, but are not known to be
1921 	     op{0,1}_small_p.  */
1922 	  bool op0_medium_p = false;
1923 	  bool op1_medium_p = false;
1924 	  /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1925 	     nonnegative, 1 if unknown.  */
1926 	  int op0_sign = 1;
1927 	  int op1_sign = 1;
1928 
1929 	  if (pos_neg0 == 1)
1930 	    op0_sign = 0;
1931 	  else if (pos_neg0 == 2)
1932 	    op0_sign = -1;
1933 	  if (pos_neg1 == 1)
1934 	    op1_sign = 0;
1935 	  else if (pos_neg1 == 2)
1936 	    op1_sign = -1;
1937 
1938 	  unsigned int mprec0 = prec;
1939 	  if (arg0 != error_mark_node)
1940 	    mprec0 = get_min_precision (arg0, sign);
1941 	  if (mprec0 <= hprec)
1942 	    op0_small_p = true;
1943 	  else if (!uns && mprec0 <= hprec + 1)
1944 	    op0_medium_p = true;
1945 	  unsigned int mprec1 = prec;
1946 	  if (arg1 != error_mark_node)
1947 	    mprec1 = get_min_precision (arg1, sign);
1948 	  if (mprec1 <= hprec)
1949 	    op1_small_p = true;
1950 	  else if (!uns && mprec1 <= hprec + 1)
1951 	    op1_medium_p = true;
1952 
1953 	  int smaller_sign = 1;
1954 	  int larger_sign = 1;
1955 	  if (op0_small_p)
1956 	    {
1957 	      smaller_sign = op0_sign;
1958 	      larger_sign = op1_sign;
1959 	    }
1960 	  else if (op1_small_p)
1961 	    {
1962 	      smaller_sign = op1_sign;
1963 	      larger_sign = op0_sign;
1964 	    }
1965 	  else if (op0_sign == op1_sign)
1966 	    {
1967 	      smaller_sign = op0_sign;
1968 	      larger_sign = op0_sign;
1969 	    }
1970 
1971 	  if (!op0_small_p)
1972 	    do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
1973 				     NULL_RTX, NULL, large_op0,
1974 				     profile_probability::unlikely ());
1975 
1976 	  if (!op1_small_p)
1977 	    do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
1978 				     NULL_RTX, NULL, small_op0_large_op1,
1979 				     profile_probability::unlikely ());
1980 
1981 	  /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1982 	     hmode to mode, the multiplication will never overflow.  We can
1983 	     do just one hmode x hmode => mode widening multiplication.  */
1984 	  tree halfstype = build_nonstandard_integer_type (hprec, uns);
1985 	  ops.op0 = make_tree (halfstype, lopart0);
1986 	  ops.op1 = make_tree (halfstype, lopart1);
1987 	  ops.code = WIDEN_MULT_EXPR;
1988 	  ops.type = type;
1989 	  rtx thisres
1990 	    = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1991 	  emit_move_insn (res, thisres);
1992 	  emit_jump (done_label);
1993 
1994 	  emit_label (small_op0_large_op1);
1995 
1996 	  /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1997 	     but op1 is not, just swap the arguments and handle it as op1
1998 	     sign/zero extended, op0 not.  */
1999 	  rtx larger = gen_reg_rtx (mode);
2000 	  rtx hipart = gen_reg_rtx (hmode);
2001 	  rtx lopart = gen_reg_rtx (hmode);
2002 	  emit_move_insn (larger, op1);
2003 	  emit_move_insn (hipart, hipart1);
2004 	  emit_move_insn (lopart, lopart0);
2005 	  emit_jump (one_small_one_large);
2006 
2007 	  emit_label (large_op0);
2008 
2009 	  if (!op1_small_p)
2010 	    do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
2011 				     NULL_RTX, NULL, both_ops_large,
2012 				     profile_probability::unlikely ());
2013 
2014 	  /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
2015 	     but op0 is not, prepare larger, hipart and lopart pseudos and
2016 	     handle it together with small_op0_large_op1.  */
2017 	  emit_move_insn (larger, op0);
2018 	  emit_move_insn (hipart, hipart0);
2019 	  emit_move_insn (lopart, lopart1);
2020 
2021 	  emit_label (one_small_one_large);
2022 
2023 	  /* lopart is the low part of the operand that is sign extended
2024 	     to mode, larger is the other operand, hipart is the
2025 	     high part of larger and lopart0 and lopart1 are the low parts
2026 	     of both operands.
2027 	     We perform lopart0 * lopart1 and lopart * hipart widening
2028 	     multiplications.  */
2029 	  tree halfutype = build_nonstandard_integer_type (hprec, 1);
2030 	  ops.op0 = make_tree (halfutype, lopart0);
2031 	  ops.op1 = make_tree (halfutype, lopart1);
2032 	  rtx lo0xlo1
2033 	    = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2034 
2035 	  ops.op0 = make_tree (halfutype, lopart);
2036 	  ops.op1 = make_tree (halfutype, hipart);
2037 	  rtx loxhi = gen_reg_rtx (mode);
2038 	  rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2039 	  emit_move_insn (loxhi, tem);
2040 
2041 	  if (!uns)
2042 	    {
2043 	      /* if (hipart < 0) loxhi -= lopart << (bitsize / 2);  */
2044 	      if (larger_sign == 0)
2045 		emit_jump (after_hipart_neg);
2046 	      else if (larger_sign != -1)
2047 		do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
2048 					 NULL_RTX, NULL, after_hipart_neg,
2049 					 profile_probability::even ());
2050 
2051 	      tem = convert_modes (mode, hmode, lopart, 1);
2052 	      tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
2053 	      tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
2054 					 1, OPTAB_WIDEN);
2055 	      emit_move_insn (loxhi, tem);
2056 
2057 	      emit_label (after_hipart_neg);
2058 
2059 	      /* if (lopart < 0) loxhi -= larger;  */
2060 	      if (smaller_sign == 0)
2061 		emit_jump (after_lopart_neg);
2062 	      else if (smaller_sign != -1)
2063 		do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
2064 					 NULL_RTX, NULL, after_lopart_neg,
2065 					 profile_probability::even ());
2066 
2067 	      tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
2068 					 1, OPTAB_WIDEN);
2069 	      emit_move_insn (loxhi, tem);
2070 
2071 	      emit_label (after_lopart_neg);
2072 	    }
2073 
2074 	  /* loxhi += (uns) lo0xlo1 >> (bitsize / 2);  */
2075 	  tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
2076 	  tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
2077 				     1, OPTAB_WIDEN);
2078 	  emit_move_insn (loxhi, tem);
2079 
2080 	  /* if (loxhi >> (bitsize / 2)
2081 		 == (hmode) loxhi >> (bitsize / 2 - 1))  (if !uns)
2082 	     if (loxhi >> (bitsize / 2) == 0		 (if uns).  */
2083 	  rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
2084 					  NULL_RTX, 0);
2085 	  hipartloxhi = convert_modes (hmode, mode, hipartloxhi, 0);
2086 	  rtx signbitloxhi = const0_rtx;
2087 	  if (!uns)
2088 	    signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
2089 					 convert_modes (hmode, mode,
2090 							loxhi, 0),
2091 					 hprec - 1, NULL_RTX, 0);
2092 
2093 	  do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
2094 				   NULL_RTX, NULL, do_overflow,
2095 				   profile_probability::very_unlikely ());
2096 
2097 	  /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1;  */
2098 	  rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
2099 					   NULL_RTX, 1);
2100 	  tem = convert_modes (mode, hmode,
2101 			       convert_modes (hmode, mode, lo0xlo1, 1), 1);
2102 
2103 	  tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
2104 				     1, OPTAB_WIDEN);
2105 	  if (tem != res)
2106 	    emit_move_insn (res, tem);
2107 	  emit_jump (done_label);
2108 
2109 	  emit_label (both_ops_large);
2110 
2111 	  /* If both operands are large (not sign (!uns) or zero (uns)
2112 	     extended from hmode), then perform the full multiplication
2113 	     which will be the result of the operation.
2114 	     The only cases which don't overflow are for signed multiplication
2115 	     some cases where both hipart0 and highpart1 are 0 or -1.
2116 	     For unsigned multiplication when high parts are both non-zero
2117 	     this overflows always.  */
2118 	  ops.code = MULT_EXPR;
2119 	  ops.op0 = make_tree (type, op0);
2120 	  ops.op1 = make_tree (type, op1);
2121 	  tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2122 	  emit_move_insn (res, tem);
2123 
2124 	  if (!uns)
2125 	    {
2126 	      if (!op0_medium_p)
2127 		{
2128 		  tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
2129 					     NULL_RTX, 1, OPTAB_WIDEN);
2130 		  do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
2131 					   NULL_RTX, NULL, do_error,
2132 					   profile_probability::very_unlikely ());
2133 		}
2134 
2135 	      if (!op1_medium_p)
2136 		{
2137 		  tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
2138 					     NULL_RTX, 1, OPTAB_WIDEN);
2139 		  do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
2140 					   NULL_RTX, NULL, do_error,
2141 					   profile_probability::very_unlikely ());
2142 		}
2143 
2144 	      /* At this point hipart{0,1} are both in [-1, 0].  If they are
2145 		 the same, overflow happened if res is non-positive, if they
2146 		 are different, overflow happened if res is positive.  */
2147 	      if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
2148 		emit_jump (hipart_different);
2149 	      else if (op0_sign == 1 || op1_sign == 1)
2150 		do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
2151 					 NULL_RTX, NULL, hipart_different,
2152 					 profile_probability::even ());
2153 
2154 	      do_compare_rtx_and_jump (res, const0_rtx, LE, false, mode,
2155 				       NULL_RTX, NULL, do_error,
2156 				       profile_probability::very_unlikely ());
2157 	      emit_jump (done_label);
2158 
2159 	      emit_label (hipart_different);
2160 
2161 	      do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
2162 				       NULL_RTX, NULL, do_error,
2163 				       profile_probability::very_unlikely ());
2164 	      emit_jump (done_label);
2165 	    }
2166 
2167 	  emit_label (do_overflow);
2168 
2169 	  /* Overflow, do full multiplication and fallthru into do_error.  */
2170 	  ops.op0 = make_tree (type, op0);
2171 	  ops.op1 = make_tree (type, op1);
2172 	  tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2173 	  emit_move_insn (res, tem);
2174 	}
2175       else if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
2176 	       && targetm.scalar_mode_supported_p (wmode))
2177 	/* Even emitting a libcall is better than not detecting overflow
2178 	   at all.  */
2179 	goto twoxwider;
2180       else
2181 	{
2182 	  gcc_assert (!is_ubsan);
2183 	  ops.code = MULT_EXPR;
2184 	  ops.type = type;
2185 	  res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2186 	  emit_jump (done_label);
2187 	}
2188     }
2189 
2190  do_error_label:
2191   emit_label (do_error);
2192   if (is_ubsan)
2193     {
2194       /* Expand the ubsan builtin call.  */
2195       push_temp_slots ();
2196       fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0),
2197 					 arg0, arg1, datap);
2198       expand_normal (fn);
2199       pop_temp_slots ();
2200       do_pending_stack_adjust ();
2201     }
2202   else if (lhs)
2203     expand_arith_set_overflow (lhs, target);
2204 
2205   /* We're done.  */
2206   emit_label (done_label);
2207 
2208   /* u1 * u2 -> sr  */
2209   if (uns0_p && uns1_p && !unsr_p)
2210     {
2211       rtx_code_label *all_done_label = gen_label_rtx ();
2212       do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
2213 			       NULL, all_done_label, profile_probability::very_likely ());
2214       expand_arith_set_overflow (lhs, target);
2215       emit_label (all_done_label);
2216     }
2217 
2218   /* s1 * u2 -> sr  */
2219   if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
2220     {
2221       rtx_code_label *all_done_label = gen_label_rtx ();
2222       rtx_code_label *set_noovf = gen_label_rtx ();
2223       do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
2224 			       NULL, all_done_label, profile_probability::very_likely ());
2225       expand_arith_set_overflow (lhs, target);
2226       do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
2227 			       NULL, set_noovf, profile_probability::very_likely ());
2228       do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
2229 			       NULL, all_done_label, profile_probability::very_unlikely ());
2230       do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
2231 			       all_done_label, profile_probability::very_unlikely ());
2232       emit_label (set_noovf);
2233       write_complex_part (target, const0_rtx, true);
2234       emit_label (all_done_label);
2235     }
2236 
2237   if (lhs)
2238     {
2239       if (is_ubsan)
2240 	expand_ubsan_result_store (target, res);
2241       else
2242 	expand_arith_overflow_result_store (lhs, target, mode, res);
2243     }
2244   flag_trapv = save_flag_trapv;
2245 }
2246 
2247 /* Expand UBSAN_CHECK_* internal function if it has vector operands.  */
2248 
2249 static void
expand_vector_ubsan_overflow(location_t loc,enum tree_code code,tree lhs,tree arg0,tree arg1)2250 expand_vector_ubsan_overflow (location_t loc, enum tree_code code, tree lhs,
2251 			      tree arg0, tree arg1)
2252 {
2253   poly_uint64 cnt = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
2254   rtx_code_label *loop_lab = NULL;
2255   rtx cntvar = NULL_RTX;
2256   tree cntv = NULL_TREE;
2257   tree eltype = TREE_TYPE (TREE_TYPE (arg0));
2258   tree sz = TYPE_SIZE (eltype);
2259   tree data = NULL_TREE;
2260   tree resv = NULL_TREE;
2261   rtx lhsr = NULL_RTX;
2262   rtx resvr = NULL_RTX;
2263   unsigned HOST_WIDE_INT const_cnt = 0;
2264   bool use_loop_p = (!cnt.is_constant (&const_cnt) || const_cnt > 4);
2265   int save_flag_trapv = flag_trapv;
2266 
2267   /* We don't want any __mulv?i3 etc. calls from the expansion of
2268      these internal functions, so disable -ftrapv temporarily.  */
2269   flag_trapv = 0;
2270   if (lhs)
2271     {
2272       optab op;
2273       lhsr = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2274       if (!VECTOR_MODE_P (GET_MODE (lhsr))
2275 	  || (op = optab_for_tree_code (code, TREE_TYPE (arg0),
2276 					optab_default)) == unknown_optab
2277 	  || (optab_handler (op, TYPE_MODE (TREE_TYPE (arg0)))
2278 	      == CODE_FOR_nothing))
2279 	{
2280 	  if (MEM_P (lhsr))
2281 	    resv = make_tree (TREE_TYPE (lhs), lhsr);
2282 	  else
2283 	    {
2284 	      resvr = assign_temp (TREE_TYPE (lhs), 1, 1);
2285 	      resv = make_tree (TREE_TYPE (lhs), resvr);
2286 	    }
2287 	}
2288     }
2289   if (use_loop_p)
2290     {
2291       do_pending_stack_adjust ();
2292       loop_lab = gen_label_rtx ();
2293       cntvar = gen_reg_rtx (TYPE_MODE (sizetype));
2294       cntv = make_tree (sizetype, cntvar);
2295       emit_move_insn (cntvar, const0_rtx);
2296       emit_label (loop_lab);
2297     }
2298   if (TREE_CODE (arg0) != VECTOR_CST)
2299     {
2300       rtx arg0r = expand_normal (arg0);
2301       arg0 = make_tree (TREE_TYPE (arg0), arg0r);
2302     }
2303   if (TREE_CODE (arg1) != VECTOR_CST)
2304     {
2305       rtx arg1r = expand_normal (arg1);
2306       arg1 = make_tree (TREE_TYPE (arg1), arg1r);
2307     }
2308   for (unsigned int i = 0; i < (use_loop_p ? 1 : const_cnt); i++)
2309     {
2310       tree op0, op1, res = NULL_TREE;
2311       if (use_loop_p)
2312 	{
2313 	  tree atype = build_array_type_nelts (eltype, cnt);
2314 	  op0 = uniform_vector_p (arg0);
2315 	  if (op0 == NULL_TREE)
2316 	    {
2317 	      op0 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg0);
2318 	      op0 = build4_loc (loc, ARRAY_REF, eltype, op0, cntv,
2319 				NULL_TREE, NULL_TREE);
2320 	    }
2321 	  op1 = uniform_vector_p (arg1);
2322 	  if (op1 == NULL_TREE)
2323 	    {
2324 	      op1 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg1);
2325 	      op1 = build4_loc (loc, ARRAY_REF, eltype, op1, cntv,
2326 				NULL_TREE, NULL_TREE);
2327 	    }
2328 	  if (resv)
2329 	    {
2330 	      res = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, resv);
2331 	      res = build4_loc (loc, ARRAY_REF, eltype, res, cntv,
2332 				NULL_TREE, NULL_TREE);
2333 	    }
2334 	}
2335       else
2336 	{
2337 	  tree bitpos = bitsize_int (tree_to_uhwi (sz) * i);
2338 	  op0 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg0, sz, bitpos);
2339 	  op1 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg1, sz, bitpos);
2340 	  if (resv)
2341 	    res = fold_build3_loc (loc, BIT_FIELD_REF, eltype, resv, sz,
2342 				   bitpos);
2343 	}
2344       switch (code)
2345 	{
2346 	case PLUS_EXPR:
2347 	  expand_addsub_overflow (loc, PLUS_EXPR, res, op0, op1,
2348 				  false, false, false, true, &data);
2349 	  break;
2350 	case MINUS_EXPR:
2351 	  if (use_loop_p ? integer_zerop (arg0) : integer_zerop (op0))
2352 	    expand_neg_overflow (loc, res, op1, true, &data);
2353 	  else
2354 	    expand_addsub_overflow (loc, MINUS_EXPR, res, op0, op1,
2355 				    false, false, false, true, &data);
2356 	  break;
2357 	case MULT_EXPR:
2358 	  expand_mul_overflow (loc, res, op0, op1, false, false, false,
2359 			       true, &data);
2360 	  break;
2361 	default:
2362 	  gcc_unreachable ();
2363 	}
2364     }
2365   if (use_loop_p)
2366     {
2367       struct separate_ops ops;
2368       ops.code = PLUS_EXPR;
2369       ops.type = TREE_TYPE (cntv);
2370       ops.op0 = cntv;
2371       ops.op1 = build_int_cst (TREE_TYPE (cntv), 1);
2372       ops.op2 = NULL_TREE;
2373       ops.location = loc;
2374       rtx ret = expand_expr_real_2 (&ops, cntvar, TYPE_MODE (sizetype),
2375 				    EXPAND_NORMAL);
2376       if (ret != cntvar)
2377 	emit_move_insn (cntvar, ret);
2378       rtx cntrtx = gen_int_mode (cnt, TYPE_MODE (sizetype));
2379       do_compare_rtx_and_jump (cntvar, cntrtx, NE, false,
2380 			       TYPE_MODE (sizetype), NULL_RTX, NULL, loop_lab,
2381 			       profile_probability::very_likely ());
2382     }
2383   if (lhs && resv == NULL_TREE)
2384     {
2385       struct separate_ops ops;
2386       ops.code = code;
2387       ops.type = TREE_TYPE (arg0);
2388       ops.op0 = arg0;
2389       ops.op1 = arg1;
2390       ops.op2 = NULL_TREE;
2391       ops.location = loc;
2392       rtx ret = expand_expr_real_2 (&ops, lhsr, TYPE_MODE (TREE_TYPE (arg0)),
2393 				    EXPAND_NORMAL);
2394       if (ret != lhsr)
2395 	emit_move_insn (lhsr, ret);
2396     }
2397   else if (resvr)
2398     emit_move_insn (lhsr, resvr);
2399   flag_trapv = save_flag_trapv;
2400 }
2401 
2402 /* Expand UBSAN_CHECK_ADD call STMT.  */
2403 
2404 static void
expand_UBSAN_CHECK_ADD(internal_fn,gcall * stmt)2405 expand_UBSAN_CHECK_ADD (internal_fn, gcall *stmt)
2406 {
2407   location_t loc = gimple_location (stmt);
2408   tree lhs = gimple_call_lhs (stmt);
2409   tree arg0 = gimple_call_arg (stmt, 0);
2410   tree arg1 = gimple_call_arg (stmt, 1);
2411   if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2412     expand_vector_ubsan_overflow (loc, PLUS_EXPR, lhs, arg0, arg1);
2413   else
2414     expand_addsub_overflow (loc, PLUS_EXPR, lhs, arg0, arg1,
2415 			    false, false, false, true, NULL);
2416 }
2417 
2418 /* Expand UBSAN_CHECK_SUB call STMT.  */
2419 
2420 static void
expand_UBSAN_CHECK_SUB(internal_fn,gcall * stmt)2421 expand_UBSAN_CHECK_SUB (internal_fn, gcall *stmt)
2422 {
2423   location_t loc = gimple_location (stmt);
2424   tree lhs = gimple_call_lhs (stmt);
2425   tree arg0 = gimple_call_arg (stmt, 0);
2426   tree arg1 = gimple_call_arg (stmt, 1);
2427   if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2428     expand_vector_ubsan_overflow (loc, MINUS_EXPR, lhs, arg0, arg1);
2429   else if (integer_zerop (arg0))
2430     expand_neg_overflow (loc, lhs, arg1, true, NULL);
2431   else
2432     expand_addsub_overflow (loc, MINUS_EXPR, lhs, arg0, arg1,
2433 			    false, false, false, true, NULL);
2434 }
2435 
2436 /* Expand UBSAN_CHECK_MUL call STMT.  */
2437 
2438 static void
expand_UBSAN_CHECK_MUL(internal_fn,gcall * stmt)2439 expand_UBSAN_CHECK_MUL (internal_fn, gcall *stmt)
2440 {
2441   location_t loc = gimple_location (stmt);
2442   tree lhs = gimple_call_lhs (stmt);
2443   tree arg0 = gimple_call_arg (stmt, 0);
2444   tree arg1 = gimple_call_arg (stmt, 1);
2445   if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2446     expand_vector_ubsan_overflow (loc, MULT_EXPR, lhs, arg0, arg1);
2447   else
2448     expand_mul_overflow (loc, lhs, arg0, arg1, false, false, false, true,
2449 			 NULL);
2450 }
2451 
2452 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion.  */
2453 
2454 static void
expand_arith_overflow(enum tree_code code,gimple * stmt)2455 expand_arith_overflow (enum tree_code code, gimple *stmt)
2456 {
2457   tree lhs = gimple_call_lhs (stmt);
2458   if (lhs == NULL_TREE)
2459     return;
2460   tree arg0 = gimple_call_arg (stmt, 0);
2461   tree arg1 = gimple_call_arg (stmt, 1);
2462   tree type = TREE_TYPE (TREE_TYPE (lhs));
2463   int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
2464   int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
2465   int unsr_p = TYPE_UNSIGNED (type);
2466   int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
2467   int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
2468   int precres = TYPE_PRECISION (type);
2469   location_t loc = gimple_location (stmt);
2470   if (!uns0_p && get_range_pos_neg (arg0) == 1)
2471     uns0_p = true;
2472   if (!uns1_p && get_range_pos_neg (arg1) == 1)
2473     uns1_p = true;
2474   int pr = get_min_precision (arg0, uns0_p ? UNSIGNED : SIGNED);
2475   prec0 = MIN (prec0, pr);
2476   pr = get_min_precision (arg1, uns1_p ? UNSIGNED : SIGNED);
2477   prec1 = MIN (prec1, pr);
2478   int save_flag_trapv = flag_trapv;
2479 
2480   /* We don't want any __mulv?i3 etc. calls from the expansion of
2481      these internal functions, so disable -ftrapv temporarily.  */
2482   flag_trapv = 0;
2483   /* If uns0_p && uns1_p, precop is minimum needed precision
2484      of unsigned type to hold the exact result, otherwise
2485      precop is minimum needed precision of signed type to
2486      hold the exact result.  */
2487   int precop;
2488   if (code == MULT_EXPR)
2489     precop = prec0 + prec1 + (uns0_p != uns1_p);
2490   else
2491     {
2492       if (uns0_p == uns1_p)
2493 	precop = MAX (prec0, prec1) + 1;
2494       else if (uns0_p)
2495 	precop = MAX (prec0 + 1, prec1) + 1;
2496       else
2497 	precop = MAX (prec0, prec1 + 1) + 1;
2498     }
2499   int orig_precres = precres;
2500 
2501   do
2502     {
2503       if ((uns0_p && uns1_p)
2504 	  ? ((precop + !unsr_p) <= precres
2505 	     /* u1 - u2 -> ur can overflow, no matter what precision
2506 		the result has.  */
2507 	     && (code != MINUS_EXPR || !unsr_p))
2508 	  : (!unsr_p && precop <= precres))
2509 	{
2510 	  /* The infinity precision result will always fit into result.  */
2511 	  rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2512 	  write_complex_part (target, const0_rtx, true);
2513 	  scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
2514 	  struct separate_ops ops;
2515 	  ops.code = code;
2516 	  ops.type = type;
2517 	  ops.op0 = fold_convert_loc (loc, type, arg0);
2518 	  ops.op1 = fold_convert_loc (loc, type, arg1);
2519 	  ops.op2 = NULL_TREE;
2520 	  ops.location = loc;
2521 	  rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2522 	  expand_arith_overflow_result_store (lhs, target, mode, tem);
2523 	  flag_trapv = save_flag_trapv;
2524 	  return;
2525 	}
2526 
2527       /* For operations with low precision, if target doesn't have them, start
2528 	 with precres widening right away, otherwise do it only if the most
2529 	 simple cases can't be used.  */
2530       const int min_precision = targetm.min_arithmetic_precision ();
2531       if (orig_precres == precres && precres < min_precision)
2532 	;
2533       else if ((uns0_p && uns1_p && unsr_p && prec0 <= precres
2534 		&& prec1 <= precres)
2535 	  || ((!uns0_p || !uns1_p) && !unsr_p
2536 	      && prec0 + uns0_p <= precres
2537 	      && prec1 + uns1_p <= precres))
2538 	{
2539 	  arg0 = fold_convert_loc (loc, type, arg0);
2540 	  arg1 = fold_convert_loc (loc, type, arg1);
2541 	  switch (code)
2542 	    {
2543 	    case MINUS_EXPR:
2544 	      if (integer_zerop (arg0) && !unsr_p)
2545 		{
2546 		  expand_neg_overflow (loc, lhs, arg1, false, NULL);
2547 		  flag_trapv = save_flag_trapv;
2548 		  return;
2549 		}
2550 	      /* FALLTHRU */
2551 	    case PLUS_EXPR:
2552 	      expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2553 				      unsr_p, unsr_p, false, NULL);
2554 	      flag_trapv = save_flag_trapv;
2555 	      return;
2556 	    case MULT_EXPR:
2557 	      expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2558 				   unsr_p, unsr_p, false, NULL);
2559 	      flag_trapv = save_flag_trapv;
2560 	      return;
2561 	    default:
2562 	      gcc_unreachable ();
2563 	    }
2564 	}
2565 
2566       /* For sub-word operations, retry with a wider type first.  */
2567       if (orig_precres == precres && precop <= BITS_PER_WORD)
2568 	{
2569 	  int p = MAX (min_precision, precop);
2570 	  scalar_int_mode m = smallest_int_mode_for_size (p);
2571 	  tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
2572 							uns0_p && uns1_p
2573 							&& unsr_p);
2574 	  p = TYPE_PRECISION (optype);
2575 	  if (p > precres)
2576 	    {
2577 	      precres = p;
2578 	      unsr_p = TYPE_UNSIGNED (optype);
2579 	      type = optype;
2580 	      continue;
2581 	    }
2582 	}
2583 
2584       if (prec0 <= precres && prec1 <= precres)
2585 	{
2586 	  tree types[2];
2587 	  if (unsr_p)
2588 	    {
2589 	      types[0] = build_nonstandard_integer_type (precres, 0);
2590 	      types[1] = type;
2591 	    }
2592 	  else
2593 	    {
2594 	      types[0] = type;
2595 	      types[1] = build_nonstandard_integer_type (precres, 1);
2596 	    }
2597 	  arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
2598 	  arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
2599 	  if (code != MULT_EXPR)
2600 	    expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2601 				    uns0_p, uns1_p, false, NULL);
2602 	  else
2603 	    expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2604 				 uns0_p, uns1_p, false, NULL);
2605 	  flag_trapv = save_flag_trapv;
2606 	  return;
2607 	}
2608 
2609       /* Retry with a wider type.  */
2610       if (orig_precres == precres)
2611 	{
2612 	  int p = MAX (prec0, prec1);
2613 	  scalar_int_mode m = smallest_int_mode_for_size (p);
2614 	  tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
2615 							uns0_p && uns1_p
2616 							&& unsr_p);
2617 	  p = TYPE_PRECISION (optype);
2618 	  if (p > precres)
2619 	    {
2620 	      precres = p;
2621 	      unsr_p = TYPE_UNSIGNED (optype);
2622 	      type = optype;
2623 	      continue;
2624 	    }
2625 	}
2626 
2627       gcc_unreachable ();
2628     }
2629   while (1);
2630 }
2631 
2632 /* Expand ADD_OVERFLOW STMT.  */
2633 
2634 static void
expand_ADD_OVERFLOW(internal_fn,gcall * stmt)2635 expand_ADD_OVERFLOW (internal_fn, gcall *stmt)
2636 {
2637   expand_arith_overflow (PLUS_EXPR, stmt);
2638 }
2639 
2640 /* Expand SUB_OVERFLOW STMT.  */
2641 
2642 static void
expand_SUB_OVERFLOW(internal_fn,gcall * stmt)2643 expand_SUB_OVERFLOW (internal_fn, gcall *stmt)
2644 {
2645   expand_arith_overflow (MINUS_EXPR, stmt);
2646 }
2647 
2648 /* Expand MUL_OVERFLOW STMT.  */
2649 
2650 static void
expand_MUL_OVERFLOW(internal_fn,gcall * stmt)2651 expand_MUL_OVERFLOW (internal_fn, gcall *stmt)
2652 {
2653   expand_arith_overflow (MULT_EXPR, stmt);
2654 }
2655 
2656 /* This should get folded in tree-vectorizer.cc.  */
2657 
2658 static void
expand_LOOP_VECTORIZED(internal_fn,gcall *)2659 expand_LOOP_VECTORIZED (internal_fn, gcall *)
2660 {
2661   gcc_unreachable ();
2662 }
2663 
2664 /* This should get folded in tree-vectorizer.cc.  */
2665 
2666 static void
expand_LOOP_DIST_ALIAS(internal_fn,gcall *)2667 expand_LOOP_DIST_ALIAS (internal_fn, gcall *)
2668 {
2669   gcc_unreachable ();
2670 }
2671 
2672 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2673    Use argument INDEX + 1 to derive the second (TBAA) operand.  */
2674 
2675 static tree
expand_call_mem_ref(tree type,gcall * stmt,int index)2676 expand_call_mem_ref (tree type, gcall *stmt, int index)
2677 {
2678   tree addr = gimple_call_arg (stmt, index);
2679   tree alias_ptr_type = TREE_TYPE (gimple_call_arg (stmt, index + 1));
2680   unsigned int align = tree_to_shwi (gimple_call_arg (stmt, index + 1));
2681   if (TYPE_ALIGN (type) != align)
2682     type = build_aligned_type (type, align);
2683 
2684   tree tmp = addr;
2685   if (TREE_CODE (tmp) == SSA_NAME)
2686     {
2687       gimple *def = get_gimple_for_ssa_name (tmp);
2688       if (def && gimple_assign_single_p (def))
2689 	tmp = gimple_assign_rhs1 (def);
2690     }
2691 
2692   if (TREE_CODE (tmp) == ADDR_EXPR)
2693     {
2694       tree mem = TREE_OPERAND (tmp, 0);
2695       if (TREE_CODE (mem) == TARGET_MEM_REF
2696 	  && types_compatible_p (TREE_TYPE (mem), type))
2697 	{
2698 	  tree offset = TMR_OFFSET (mem);
2699 	  if (type != TREE_TYPE (mem)
2700 	      || alias_ptr_type != TREE_TYPE (offset)
2701 	      || !integer_zerop (offset))
2702 	    {
2703 	      mem = copy_node (mem);
2704 	      TMR_OFFSET (mem) = wide_int_to_tree (alias_ptr_type,
2705 						   wi::to_poly_wide (offset));
2706 	      TREE_TYPE (mem) = type;
2707 	    }
2708 	  return mem;
2709 	}
2710     }
2711 
2712   return fold_build2 (MEM_REF, type, addr, build_int_cst (alias_ptr_type, 0));
2713 }
2714 
2715 /* Expand MASK_LOAD{,_LANES} or LEN_LOAD call STMT using optab OPTAB.  */
2716 
2717 static void
expand_partial_load_optab_fn(internal_fn,gcall * stmt,convert_optab optab)2718 expand_partial_load_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2719 {
2720   class expand_operand ops[4];
2721   tree type, lhs, rhs, maskt, biast;
2722   rtx mem, target, mask, bias;
2723   insn_code icode;
2724 
2725   maskt = gimple_call_arg (stmt, 2);
2726   lhs = gimple_call_lhs (stmt);
2727   if (lhs == NULL_TREE)
2728     return;
2729   type = TREE_TYPE (lhs);
2730   rhs = expand_call_mem_ref (type, stmt, 0);
2731 
2732   if (optab == vec_mask_load_lanes_optab)
2733     icode = get_multi_vector_move (type, optab);
2734   else if (optab == len_load_optab)
2735     icode = direct_optab_handler (optab, TYPE_MODE (type));
2736   else
2737     icode = convert_optab_handler (optab, TYPE_MODE (type),
2738 				   TYPE_MODE (TREE_TYPE (maskt)));
2739 
2740   mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2741   gcc_assert (MEM_P (mem));
2742   mask = expand_normal (maskt);
2743   target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2744   create_output_operand (&ops[0], target, TYPE_MODE (type));
2745   create_fixed_operand (&ops[1], mem);
2746   if (optab == len_load_optab)
2747     {
2748       create_convert_operand_from (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)),
2749 				   TYPE_UNSIGNED (TREE_TYPE (maskt)));
2750       biast = gimple_call_arg (stmt, 3);
2751       bias = expand_normal (biast);
2752       create_input_operand (&ops[3], bias, QImode);
2753       expand_insn (icode, 4, ops);
2754     }
2755   else
2756     {
2757       create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
2758       expand_insn (icode, 3, ops);
2759     }
2760 
2761   if (!rtx_equal_p (target, ops[0].value))
2762     emit_move_insn (target, ops[0].value);
2763 }
2764 
2765 #define expand_mask_load_optab_fn expand_partial_load_optab_fn
2766 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2767 #define expand_len_load_optab_fn expand_partial_load_optab_fn
2768 
2769 /* Expand MASK_STORE{,_LANES} or LEN_STORE call STMT using optab OPTAB.  */
2770 
2771 static void
expand_partial_store_optab_fn(internal_fn,gcall * stmt,convert_optab optab)2772 expand_partial_store_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2773 {
2774   class expand_operand ops[4];
2775   tree type, lhs, rhs, maskt, biast;
2776   rtx mem, reg, mask, bias;
2777   insn_code icode;
2778 
2779   maskt = gimple_call_arg (stmt, 2);
2780   rhs = gimple_call_arg (stmt, 3);
2781   type = TREE_TYPE (rhs);
2782   lhs = expand_call_mem_ref (type, stmt, 0);
2783 
2784   if (optab == vec_mask_store_lanes_optab)
2785     icode = get_multi_vector_move (type, optab);
2786   else if (optab == len_store_optab)
2787     icode = direct_optab_handler (optab, TYPE_MODE (type));
2788   else
2789     icode = convert_optab_handler (optab, TYPE_MODE (type),
2790 				   TYPE_MODE (TREE_TYPE (maskt)));
2791 
2792   mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2793   gcc_assert (MEM_P (mem));
2794   mask = expand_normal (maskt);
2795   reg = expand_normal (rhs);
2796   create_fixed_operand (&ops[0], mem);
2797   create_input_operand (&ops[1], reg, TYPE_MODE (type));
2798   if (optab == len_store_optab)
2799     {
2800       create_convert_operand_from (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)),
2801 				   TYPE_UNSIGNED (TREE_TYPE (maskt)));
2802       biast = gimple_call_arg (stmt, 4);
2803       bias = expand_normal (biast);
2804       create_input_operand (&ops[3], bias, QImode);
2805       expand_insn (icode, 4, ops);
2806     }
2807   else
2808     {
2809       create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
2810       expand_insn (icode, 3, ops);
2811     }
2812 }
2813 
2814 #define expand_mask_store_optab_fn expand_partial_store_optab_fn
2815 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
2816 #define expand_len_store_optab_fn expand_partial_store_optab_fn
2817 
2818 /* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
2819    The expansion of STMT happens based on OPTAB table associated.  */
2820 
2821 static void
expand_vec_cond_optab_fn(internal_fn,gcall * stmt,convert_optab optab)2822 expand_vec_cond_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2823 {
2824   class expand_operand ops[6];
2825   insn_code icode;
2826   tree lhs = gimple_call_lhs (stmt);
2827   tree op0a = gimple_call_arg (stmt, 0);
2828   tree op0b = gimple_call_arg (stmt, 1);
2829   tree op1 = gimple_call_arg (stmt, 2);
2830   tree op2 = gimple_call_arg (stmt, 3);
2831   enum tree_code tcode = (tree_code) int_cst_value (gimple_call_arg (stmt, 4));
2832 
2833   tree vec_cond_type = TREE_TYPE (lhs);
2834   tree op_mode = TREE_TYPE (op0a);
2835   bool unsignedp = TYPE_UNSIGNED (op_mode);
2836 
2837   machine_mode mode = TYPE_MODE (vec_cond_type);
2838   machine_mode cmp_op_mode = TYPE_MODE (op_mode);
2839 
2840   icode = convert_optab_handler (optab, mode, cmp_op_mode);
2841   rtx comparison
2842     = vector_compare_rtx (VOIDmode, tcode, op0a, op0b, unsignedp, icode, 4);
2843   rtx rtx_op1 = expand_normal (op1);
2844   rtx rtx_op2 = expand_normal (op2);
2845 
2846   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2847   create_output_operand (&ops[0], target, mode);
2848   create_input_operand (&ops[1], rtx_op1, mode);
2849   create_input_operand (&ops[2], rtx_op2, mode);
2850   create_fixed_operand (&ops[3], comparison);
2851   create_fixed_operand (&ops[4], XEXP (comparison, 0));
2852   create_fixed_operand (&ops[5], XEXP (comparison, 1));
2853   expand_insn (icode, 6, ops);
2854   if (!rtx_equal_p (ops[0].value, target))
2855     emit_move_insn (target, ops[0].value);
2856 }
2857 
2858 /* Expand VCOND_MASK optab internal function.
2859    The expansion of STMT happens based on OPTAB table associated.  */
2860 
2861 static void
expand_vec_cond_mask_optab_fn(internal_fn,gcall * stmt,convert_optab optab)2862 expand_vec_cond_mask_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2863 {
2864   class expand_operand ops[4];
2865 
2866   tree lhs = gimple_call_lhs (stmt);
2867   tree op0 = gimple_call_arg (stmt, 0);
2868   tree op1 = gimple_call_arg (stmt, 1);
2869   tree op2 = gimple_call_arg (stmt, 2);
2870   tree vec_cond_type = TREE_TYPE (lhs);
2871 
2872   machine_mode mode = TYPE_MODE (vec_cond_type);
2873   machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
2874   enum insn_code icode = convert_optab_handler (optab, mode, mask_mode);
2875   rtx mask, rtx_op1, rtx_op2;
2876 
2877   gcc_assert (icode != CODE_FOR_nothing);
2878 
2879   mask = expand_normal (op0);
2880   rtx_op1 = expand_normal (op1);
2881   rtx_op2 = expand_normal (op2);
2882 
2883   mask = force_reg (mask_mode, mask);
2884   rtx_op1 = force_reg (mode, rtx_op1);
2885 
2886   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2887   create_output_operand (&ops[0], target, mode);
2888   create_input_operand (&ops[1], rtx_op1, mode);
2889   create_input_operand (&ops[2], rtx_op2, mode);
2890   create_input_operand (&ops[3], mask, mask_mode);
2891   expand_insn (icode, 4, ops);
2892   if (!rtx_equal_p (ops[0].value, target))
2893     emit_move_insn (target, ops[0].value);
2894 }
2895 
2896 /* Expand VEC_SET internal functions.  */
2897 
2898 static void
expand_vec_set_optab_fn(internal_fn,gcall * stmt,convert_optab optab)2899 expand_vec_set_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2900 {
2901   tree lhs = gimple_call_lhs (stmt);
2902   tree op0 = gimple_call_arg (stmt, 0);
2903   tree op1 = gimple_call_arg (stmt, 1);
2904   tree op2 = gimple_call_arg (stmt, 2);
2905   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2906   rtx src = expand_normal (op0);
2907 
2908   machine_mode outermode = TYPE_MODE (TREE_TYPE (op0));
2909   scalar_mode innermode = GET_MODE_INNER (outermode);
2910 
2911   rtx value = expand_normal (op1);
2912   rtx pos = expand_normal (op2);
2913 
2914   class expand_operand ops[3];
2915   enum insn_code icode = optab_handler (optab, outermode);
2916 
2917   if (icode != CODE_FOR_nothing)
2918     {
2919       rtx temp = gen_reg_rtx (outermode);
2920       emit_move_insn (temp, src);
2921 
2922       create_fixed_operand (&ops[0], temp);
2923       create_input_operand (&ops[1], value, innermode);
2924       create_convert_operand_from (&ops[2], pos, TYPE_MODE (TREE_TYPE (op2)),
2925 				   true);
2926       if (maybe_expand_insn (icode, 3, ops))
2927 	{
2928 	  emit_move_insn (target, temp);
2929 	  return;
2930 	}
2931     }
2932   gcc_unreachable ();
2933 }
2934 
2935 static void
expand_ABNORMAL_DISPATCHER(internal_fn,gcall *)2936 expand_ABNORMAL_DISPATCHER (internal_fn, gcall *)
2937 {
2938 }
2939 
2940 static void
expand_BUILTIN_EXPECT(internal_fn,gcall * stmt)2941 expand_BUILTIN_EXPECT (internal_fn, gcall *stmt)
2942 {
2943   /* When guessing was done, the hints should be already stripped away.  */
2944   gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
2945 
2946   rtx target;
2947   tree lhs = gimple_call_lhs (stmt);
2948   if (lhs)
2949     target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2950   else
2951     target = const0_rtx;
2952   rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL);
2953   if (lhs && val != target)
2954     emit_move_insn (target, val);
2955 }
2956 
2957 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg.  So this dummy function
2958    should never be called.  */
2959 
2960 static void
expand_VA_ARG(internal_fn,gcall *)2961 expand_VA_ARG (internal_fn, gcall *)
2962 {
2963   gcc_unreachable ();
2964 }
2965 
2966 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector.  So this
2967    dummy function should never be called.  */
2968 
2969 static void
expand_VEC_CONVERT(internal_fn,gcall *)2970 expand_VEC_CONVERT (internal_fn, gcall *)
2971 {
2972   gcc_unreachable ();
2973 }
2974 
2975 /* Expand IFN_RAWMEMCHAR internal function.  */
2976 
2977 void
expand_RAWMEMCHR(internal_fn,gcall * stmt)2978 expand_RAWMEMCHR (internal_fn, gcall *stmt)
2979 {
2980   expand_operand ops[3];
2981 
2982   tree lhs = gimple_call_lhs (stmt);
2983   if (!lhs)
2984     return;
2985   machine_mode lhs_mode = TYPE_MODE (TREE_TYPE (lhs));
2986   rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2987   create_output_operand (&ops[0], lhs_rtx, lhs_mode);
2988 
2989   tree mem = gimple_call_arg (stmt, 0);
2990   rtx mem_rtx = get_memory_rtx (mem, NULL);
2991   create_fixed_operand (&ops[1], mem_rtx);
2992 
2993   tree pattern = gimple_call_arg (stmt, 1);
2994   machine_mode mode = TYPE_MODE (TREE_TYPE (pattern));
2995   rtx pattern_rtx = expand_normal (pattern);
2996   create_input_operand (&ops[2], pattern_rtx, mode);
2997 
2998   insn_code icode = direct_optab_handler (rawmemchr_optab, mode);
2999 
3000   expand_insn (icode, 3, ops);
3001   if (!rtx_equal_p (lhs_rtx, ops[0].value))
3002     emit_move_insn (lhs_rtx, ops[0].value);
3003 }
3004 
3005 /* Expand the IFN_UNIQUE function according to its first argument.  */
3006 
3007 static void
expand_UNIQUE(internal_fn,gcall * stmt)3008 expand_UNIQUE (internal_fn, gcall *stmt)
3009 {
3010   rtx pattern = NULL_RTX;
3011   enum ifn_unique_kind kind
3012     = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
3013 
3014   switch (kind)
3015     {
3016     default:
3017       gcc_unreachable ();
3018 
3019     case IFN_UNIQUE_UNSPEC:
3020       if (targetm.have_unique ())
3021 	pattern = targetm.gen_unique ();
3022       break;
3023 
3024     case IFN_UNIQUE_OACC_FORK:
3025     case IFN_UNIQUE_OACC_JOIN:
3026       if (targetm.have_oacc_fork () && targetm.have_oacc_join ())
3027 	{
3028 	  tree lhs = gimple_call_lhs (stmt);
3029 	  rtx target = const0_rtx;
3030 
3031 	  if (lhs)
3032 	    target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3033 
3034 	  rtx data_dep = expand_normal (gimple_call_arg (stmt, 1));
3035 	  rtx axis = expand_normal (gimple_call_arg (stmt, 2));
3036 
3037 	  if (kind == IFN_UNIQUE_OACC_FORK)
3038 	    pattern = targetm.gen_oacc_fork (target, data_dep, axis);
3039 	  else
3040 	    pattern = targetm.gen_oacc_join (target, data_dep, axis);
3041 	}
3042       else
3043 	gcc_unreachable ();
3044       break;
3045     }
3046 
3047   if (pattern)
3048     emit_insn (pattern);
3049 }
3050 
3051 /* Expand the IFN_DEFERRED_INIT function:
3052    LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL);
3053 
3054    Initialize the LHS with zero/pattern according to its second argument
3055    INIT_TYPE:
3056    if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize;
3057    if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern
3058      to initialize;
3059    The LHS variable is initialized including paddings.
3060    The reasons to choose 0xFE for pattern initialization are:
3061      1. It is a non-canonical virtual address on x86_64, and at the
3062 	high end of the i386 kernel address space.
3063      2. It is a very large float value (-1.694739530317379e+38).
3064      3. It is also an unusual number for integers.  */
3065 #define INIT_PATTERN_VALUE  0xFE
3066 static void
expand_DEFERRED_INIT(internal_fn,gcall * stmt)3067 expand_DEFERRED_INIT (internal_fn, gcall *stmt)
3068 {
3069   tree lhs = gimple_call_lhs (stmt);
3070   tree var_size = gimple_call_arg (stmt, 0);
3071   enum auto_init_type init_type
3072     = (enum auto_init_type) TREE_INT_CST_LOW (gimple_call_arg (stmt, 1));
3073   bool reg_lhs = true;
3074 
3075   tree var_type = TREE_TYPE (lhs);
3076   gcc_assert (init_type > AUTO_INIT_UNINITIALIZED);
3077 
3078   if (TREE_CODE (lhs) == SSA_NAME)
3079     reg_lhs = true;
3080   else
3081     {
3082       tree lhs_base = lhs;
3083       while (handled_component_p (lhs_base))
3084 	lhs_base = TREE_OPERAND (lhs_base, 0);
3085       reg_lhs = (mem_ref_refers_to_non_mem_p (lhs_base)
3086 		 || non_mem_decl_p (lhs_base));
3087       /* If this expands to a register and the underlying decl is wrapped in
3088 	 a MEM_REF that just serves as an access type change expose the decl
3089 	 if it is of correct size.  This avoids a situation as in PR103271
3090 	 if the target does not support a direct move to the registers mode.  */
3091       if (reg_lhs
3092 	  && TREE_CODE (lhs_base) == MEM_REF
3093 	  && TREE_CODE (TREE_OPERAND (lhs_base, 0)) == ADDR_EXPR
3094 	  && DECL_P (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))
3095 	  && integer_zerop (TREE_OPERAND (lhs_base, 1))
3096 	  && tree_fits_uhwi_p (var_size)
3097 	  && tree_int_cst_equal
3098 	       (var_size,
3099 		DECL_SIZE_UNIT (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))))
3100 	{
3101 	  lhs = TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0);
3102 	  var_type = TREE_TYPE (lhs);
3103 	}
3104     }
3105 
3106   if (!reg_lhs)
3107     {
3108       /* If the variable is not in register, expand to a memset
3109 	 to initialize it.  */
3110       mark_addressable (lhs);
3111       tree var_addr = build_fold_addr_expr (lhs);
3112 
3113       tree value = (init_type == AUTO_INIT_PATTERN)
3114 		    ? build_int_cst (integer_type_node,
3115 				     INIT_PATTERN_VALUE)
3116 		    : integer_zero_node;
3117       tree m_call = build_call_expr (builtin_decl_implicit (BUILT_IN_MEMSET),
3118 				     3, var_addr, value, var_size);
3119       /* Expand this memset call.  */
3120       expand_builtin_memset (m_call, NULL_RTX, TYPE_MODE (var_type));
3121     }
3122   else
3123     {
3124       /* If this variable is in a register use expand_assignment.
3125 	 For boolean scalars force zero-init.  */
3126       tree init;
3127       scalar_int_mode var_mode;
3128       if (TREE_CODE (TREE_TYPE (lhs)) != BOOLEAN_TYPE
3129 	  && tree_fits_uhwi_p (var_size)
3130 	  && (init_type == AUTO_INIT_PATTERN
3131 	      || !is_gimple_reg_type (var_type))
3132 	  && int_mode_for_size (tree_to_uhwi (var_size) * BITS_PER_UNIT,
3133 				0).exists (&var_mode)
3134 	  && have_insn_for (SET, var_mode))
3135 	{
3136 	  unsigned HOST_WIDE_INT total_bytes = tree_to_uhwi (var_size);
3137 	  unsigned char *buf = XALLOCAVEC (unsigned char, total_bytes);
3138 	  memset (buf, (init_type == AUTO_INIT_PATTERN
3139 			? INIT_PATTERN_VALUE : 0), total_bytes);
3140 	  tree itype = build_nonstandard_integer_type
3141 			 (total_bytes * BITS_PER_UNIT, 1);
3142 	  wide_int w = wi::from_buffer (buf, total_bytes);
3143 	  init = wide_int_to_tree (itype, w);
3144 	  /* Pun the LHS to make sure its type has constant size
3145 	     unless it is an SSA name where that's already known.  */
3146 	  if (TREE_CODE (lhs) != SSA_NAME)
3147 	    lhs = build1 (VIEW_CONVERT_EXPR, itype, lhs);
3148 	  else
3149 	    init = fold_build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), init);
3150 	}
3151       else
3152 	/* Use zero-init also for variable-length sizes.  */
3153 	init = build_zero_cst (var_type);
3154 
3155       expand_assignment (lhs, init, false);
3156     }
3157 }
3158 
3159 /* The size of an OpenACC compute dimension.  */
3160 
3161 static void
expand_GOACC_DIM_SIZE(internal_fn,gcall * stmt)3162 expand_GOACC_DIM_SIZE (internal_fn, gcall *stmt)
3163 {
3164   tree lhs = gimple_call_lhs (stmt);
3165 
3166   if (!lhs)
3167     return;
3168 
3169   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3170   if (targetm.have_oacc_dim_size ())
3171     {
3172       rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
3173 			     VOIDmode, EXPAND_NORMAL);
3174       emit_insn (targetm.gen_oacc_dim_size (target, dim));
3175     }
3176   else
3177     emit_move_insn (target, GEN_INT (1));
3178 }
3179 
3180 /* The position of an OpenACC execution engine along one compute axis.  */
3181 
3182 static void
expand_GOACC_DIM_POS(internal_fn,gcall * stmt)3183 expand_GOACC_DIM_POS (internal_fn, gcall *stmt)
3184 {
3185   tree lhs = gimple_call_lhs (stmt);
3186 
3187   if (!lhs)
3188     return;
3189 
3190   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3191   if (targetm.have_oacc_dim_pos ())
3192     {
3193       rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
3194 			     VOIDmode, EXPAND_NORMAL);
3195       emit_insn (targetm.gen_oacc_dim_pos (target, dim));
3196     }
3197   else
3198     emit_move_insn (target, const0_rtx);
3199 }
3200 
3201 /* This is expanded by oacc_device_lower pass.  */
3202 
3203 static void
expand_GOACC_LOOP(internal_fn,gcall *)3204 expand_GOACC_LOOP (internal_fn, gcall *)
3205 {
3206   gcc_unreachable ();
3207 }
3208 
3209 /* This is expanded by oacc_device_lower pass.  */
3210 
3211 static void
expand_GOACC_REDUCTION(internal_fn,gcall *)3212 expand_GOACC_REDUCTION (internal_fn, gcall *)
3213 {
3214   gcc_unreachable ();
3215 }
3216 
3217 /* This is expanded by oacc_device_lower pass.  */
3218 
3219 static void
expand_GOACC_TILE(internal_fn,gcall *)3220 expand_GOACC_TILE (internal_fn, gcall *)
3221 {
3222   gcc_unreachable ();
3223 }
3224 
3225 /* Set errno to EDOM.  */
3226 
3227 static void
expand_SET_EDOM(internal_fn,gcall *)3228 expand_SET_EDOM (internal_fn, gcall *)
3229 {
3230 #ifdef TARGET_EDOM
3231 #ifdef GEN_ERRNO_RTX
3232   rtx errno_rtx = GEN_ERRNO_RTX;
3233 #else
3234   rtx errno_rtx = gen_rtx_MEM (word_mode, gen_rtx_SYMBOL_REF (Pmode, "errno"));
3235 #endif
3236   emit_move_insn (errno_rtx,
3237 		  gen_int_mode (TARGET_EDOM, GET_MODE (errno_rtx)));
3238 #else
3239   gcc_unreachable ();
3240 #endif
3241 }
3242 
3243 /* Expand atomic bit test and set.  */
3244 
3245 static void
expand_ATOMIC_BIT_TEST_AND_SET(internal_fn,gcall * call)3246 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn, gcall *call)
3247 {
3248   expand_ifn_atomic_bit_test_and (call);
3249 }
3250 
3251 /* Expand atomic bit test and complement.  */
3252 
3253 static void
expand_ATOMIC_BIT_TEST_AND_COMPLEMENT(internal_fn,gcall * call)3254 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn, gcall *call)
3255 {
3256   expand_ifn_atomic_bit_test_and (call);
3257 }
3258 
3259 /* Expand atomic bit test and reset.  */
3260 
3261 static void
expand_ATOMIC_BIT_TEST_AND_RESET(internal_fn,gcall * call)3262 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn, gcall *call)
3263 {
3264   expand_ifn_atomic_bit_test_and (call);
3265 }
3266 
3267 /* Expand atomic bit test and set.  */
3268 
3269 static void
expand_ATOMIC_COMPARE_EXCHANGE(internal_fn,gcall * call)3270 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn, gcall *call)
3271 {
3272   expand_ifn_atomic_compare_exchange (call);
3273 }
3274 
3275 /* Expand atomic add fetch and cmp with 0.  */
3276 
3277 static void
expand_ATOMIC_ADD_FETCH_CMP_0(internal_fn,gcall * call)3278 expand_ATOMIC_ADD_FETCH_CMP_0 (internal_fn, gcall *call)
3279 {
3280   expand_ifn_atomic_op_fetch_cmp_0 (call);
3281 }
3282 
3283 /* Expand atomic sub fetch and cmp with 0.  */
3284 
3285 static void
expand_ATOMIC_SUB_FETCH_CMP_0(internal_fn,gcall * call)3286 expand_ATOMIC_SUB_FETCH_CMP_0 (internal_fn, gcall *call)
3287 {
3288   expand_ifn_atomic_op_fetch_cmp_0 (call);
3289 }
3290 
3291 /* Expand atomic and fetch and cmp with 0.  */
3292 
3293 static void
expand_ATOMIC_AND_FETCH_CMP_0(internal_fn,gcall * call)3294 expand_ATOMIC_AND_FETCH_CMP_0 (internal_fn, gcall *call)
3295 {
3296   expand_ifn_atomic_op_fetch_cmp_0 (call);
3297 }
3298 
3299 /* Expand atomic or fetch and cmp with 0.  */
3300 
3301 static void
expand_ATOMIC_OR_FETCH_CMP_0(internal_fn,gcall * call)3302 expand_ATOMIC_OR_FETCH_CMP_0 (internal_fn, gcall *call)
3303 {
3304   expand_ifn_atomic_op_fetch_cmp_0 (call);
3305 }
3306 
3307 /* Expand atomic xor fetch and cmp with 0.  */
3308 
3309 static void
expand_ATOMIC_XOR_FETCH_CMP_0(internal_fn,gcall * call)3310 expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn, gcall *call)
3311 {
3312   expand_ifn_atomic_op_fetch_cmp_0 (call);
3313 }
3314 
3315 /* Expand LAUNDER to assignment, lhs = arg0.  */
3316 
3317 static void
expand_LAUNDER(internal_fn,gcall * call)3318 expand_LAUNDER (internal_fn, gcall *call)
3319 {
3320   tree lhs = gimple_call_lhs (call);
3321 
3322   if (!lhs)
3323     return;
3324 
3325   expand_assignment (lhs, gimple_call_arg (call, 0), false);
3326 }
3327 
3328 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB.  */
3329 
3330 static void
expand_scatter_store_optab_fn(internal_fn,gcall * stmt,direct_optab optab)3331 expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
3332 {
3333   internal_fn ifn = gimple_call_internal_fn (stmt);
3334   int rhs_index = internal_fn_stored_value_index (ifn);
3335   int mask_index = internal_fn_mask_index (ifn);
3336   tree base = gimple_call_arg (stmt, 0);
3337   tree offset = gimple_call_arg (stmt, 1);
3338   tree scale = gimple_call_arg (stmt, 2);
3339   tree rhs = gimple_call_arg (stmt, rhs_index);
3340 
3341   rtx base_rtx = expand_normal (base);
3342   rtx offset_rtx = expand_normal (offset);
3343   HOST_WIDE_INT scale_int = tree_to_shwi (scale);
3344   rtx rhs_rtx = expand_normal (rhs);
3345 
3346   class expand_operand ops[6];
3347   int i = 0;
3348   create_address_operand (&ops[i++], base_rtx);
3349   create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
3350   create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
3351   create_integer_operand (&ops[i++], scale_int);
3352   create_input_operand (&ops[i++], rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
3353   if (mask_index >= 0)
3354     {
3355       tree mask = gimple_call_arg (stmt, mask_index);
3356       rtx mask_rtx = expand_normal (mask);
3357       create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
3358     }
3359 
3360   insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs)),
3361 					   TYPE_MODE (TREE_TYPE (offset)));
3362   expand_insn (icode, i, ops);
3363 }
3364 
3365 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB.  */
3366 
3367 static void
expand_gather_load_optab_fn(internal_fn,gcall * stmt,direct_optab optab)3368 expand_gather_load_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
3369 {
3370   tree lhs = gimple_call_lhs (stmt);
3371   tree base = gimple_call_arg (stmt, 0);
3372   tree offset = gimple_call_arg (stmt, 1);
3373   tree scale = gimple_call_arg (stmt, 2);
3374 
3375   rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3376   rtx base_rtx = expand_normal (base);
3377   rtx offset_rtx = expand_normal (offset);
3378   HOST_WIDE_INT scale_int = tree_to_shwi (scale);
3379 
3380   int i = 0;
3381   class expand_operand ops[6];
3382   create_output_operand (&ops[i++], lhs_rtx, TYPE_MODE (TREE_TYPE (lhs)));
3383   create_address_operand (&ops[i++], base_rtx);
3384   create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
3385   create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
3386   create_integer_operand (&ops[i++], scale_int);
3387   if (optab == mask_gather_load_optab)
3388     {
3389       tree mask = gimple_call_arg (stmt, 4);
3390       rtx mask_rtx = expand_normal (mask);
3391       create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
3392     }
3393   insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)),
3394 					   TYPE_MODE (TREE_TYPE (offset)));
3395   expand_insn (icode, i, ops);
3396   if (!rtx_equal_p (lhs_rtx, ops[0].value))
3397     emit_move_insn (lhs_rtx, ops[0].value);
3398 }
3399 
3400 /* Helper for expand_DIVMOD.  Return true if the sequence starting with
3401    INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes.  */
3402 
3403 static bool
contains_call_div_mod(rtx_insn * insn)3404 contains_call_div_mod (rtx_insn *insn)
3405 {
3406   subrtx_iterator::array_type array;
3407   for (; insn; insn = NEXT_INSN (insn))
3408     if (CALL_P (insn))
3409       return true;
3410     else if (INSN_P (insn))
3411       FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
3412 	switch (GET_CODE (*iter))
3413 	  {
3414 	  case CALL:
3415 	  case DIV:
3416 	  case UDIV:
3417 	  case MOD:
3418 	  case UMOD:
3419 	    return true;
3420 	  default:
3421 	    break;
3422 	  }
3423   return false;
3424  }
3425 
3426 /* Expand DIVMOD() using:
3427  a) optab handler for udivmod/sdivmod if it is available.
3428  b) If optab_handler doesn't exist, generate call to
3429     target-specific divmod libfunc.  */
3430 
3431 static void
expand_DIVMOD(internal_fn,gcall * call_stmt)3432 expand_DIVMOD (internal_fn, gcall *call_stmt)
3433 {
3434   tree lhs = gimple_call_lhs (call_stmt);
3435   tree arg0 = gimple_call_arg (call_stmt, 0);
3436   tree arg1 = gimple_call_arg (call_stmt, 1);
3437 
3438   gcc_assert (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE);
3439   tree type = TREE_TYPE (TREE_TYPE (lhs));
3440   machine_mode mode = TYPE_MODE (type);
3441   bool unsignedp = TYPE_UNSIGNED (type);
3442   optab tab = (unsignedp) ? udivmod_optab : sdivmod_optab;
3443 
3444   rtx op0 = expand_normal (arg0);
3445   rtx op1 = expand_normal (arg1);
3446   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3447 
3448   rtx quotient = NULL_RTX, remainder = NULL_RTX;
3449   rtx_insn *insns = NULL;
3450 
3451   if (TREE_CODE (arg1) == INTEGER_CST)
3452     {
3453       /* For DIVMOD by integral constants, there could be efficient code
3454 	 expanded inline e.g. using shifts and plus/minus.  Try to expand
3455 	 the division and modulo and if it emits any library calls or any
3456 	 {,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3457 	 divmod libcall.  */
3458       scalar_int_mode int_mode;
3459       if (remainder == NULL_RTX
3460 	  && optimize
3461 	  && CONST_INT_P (op1)
3462 	  && !pow2p_hwi (INTVAL (op1))
3463 	  && is_int_mode (TYPE_MODE (type), &int_mode)
3464 	  && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3465 	  && optab_handler (and_optab, word_mode) != CODE_FOR_nothing
3466 	  && optab_handler (add_optab, word_mode) != CODE_FOR_nothing
3467 	  && optimize_insn_for_speed_p ())
3468 	{
3469 	  rtx_insn *last = get_last_insn ();
3470 	  remainder = NULL_RTX;
3471 	  quotient = expand_doubleword_divmod (int_mode, op0, op1, &remainder,
3472 					       TYPE_UNSIGNED (type));
3473 	  if (quotient != NULL_RTX)
3474 	    {
3475 	      if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
3476 		{
3477 		  rtx_insn *move = emit_move_insn (quotient, quotient);
3478 		  set_dst_reg_note (move, REG_EQUAL,
3479 				    gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
3480 						    ? UDIV : DIV, int_mode,
3481 						    copy_rtx (op0), op1),
3482 				    quotient);
3483 		  move = emit_move_insn (remainder, remainder);
3484 		  set_dst_reg_note (move, REG_EQUAL,
3485 				    gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
3486 						    ? UMOD : MOD, int_mode,
3487 						    copy_rtx (op0), op1),
3488 				    quotient);
3489 		}
3490 	    }
3491 	  else
3492 	    delete_insns_since (last);
3493 	}
3494 
3495       if (remainder == NULL_RTX)
3496 	{
3497 	  struct separate_ops ops;
3498 	  ops.code = TRUNC_DIV_EXPR;
3499 	  ops.type = type;
3500 	  ops.op0 = make_tree (ops.type, op0);
3501 	  ops.op1 = arg1;
3502 	  ops.op2 = NULL_TREE;
3503 	  ops.location = gimple_location (call_stmt);
3504 	  start_sequence ();
3505 	  quotient = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
3506 	  if (contains_call_div_mod (get_insns ()))
3507 	    quotient = NULL_RTX;
3508 	  else
3509 	    {
3510 	      ops.code = TRUNC_MOD_EXPR;
3511 	      remainder = expand_expr_real_2 (&ops, NULL_RTX, mode,
3512 					      EXPAND_NORMAL);
3513 	      if (contains_call_div_mod (get_insns ()))
3514 		remainder = NULL_RTX;
3515 	    }
3516 	  if (remainder)
3517 	    insns = get_insns ();
3518 	  end_sequence ();
3519 	}
3520     }
3521 
3522   if (remainder)
3523     emit_insn (insns);
3524 
3525   /* Check if optab_handler exists for divmod_optab for given mode.  */
3526   else if (optab_handler (tab, mode) != CODE_FOR_nothing)
3527     {
3528       quotient = gen_reg_rtx (mode);
3529       remainder = gen_reg_rtx (mode);
3530       expand_twoval_binop (tab, op0, op1, quotient, remainder, unsignedp);
3531     }
3532 
3533   /* Generate call to divmod libfunc if it exists.  */
3534   else if (rtx libfunc = optab_libfunc (tab, mode))
3535     targetm.expand_divmod_libfunc (libfunc, mode, op0, op1,
3536 				   &quotient, &remainder);
3537 
3538   else
3539     gcc_unreachable ();
3540 
3541   /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR.  */
3542   expand_expr (build2 (COMPLEX_EXPR, TREE_TYPE (lhs),
3543 		       make_tree (TREE_TYPE (arg0), quotient),
3544 		       make_tree (TREE_TYPE (arg1), remainder)),
3545 	       target, VOIDmode, EXPAND_NORMAL);
3546 }
3547 
3548 /* Expand a NOP.  */
3549 
3550 static void
expand_NOP(internal_fn,gcall *)3551 expand_NOP (internal_fn, gcall *)
3552 {
3553   /* Nothing.  But it shouldn't really prevail.  */
3554 }
3555 
3556 /* Coroutines, all should have been processed at this stage.  */
3557 
3558 static void
expand_CO_FRAME(internal_fn,gcall *)3559 expand_CO_FRAME (internal_fn, gcall *)
3560 {
3561   gcc_unreachable ();
3562 }
3563 
3564 static void
expand_CO_YIELD(internal_fn,gcall *)3565 expand_CO_YIELD (internal_fn, gcall *)
3566 {
3567   gcc_unreachable ();
3568 }
3569 
3570 static void
expand_CO_SUSPN(internal_fn,gcall *)3571 expand_CO_SUSPN (internal_fn, gcall *)
3572 {
3573   gcc_unreachable ();
3574 }
3575 
3576 static void
expand_CO_ACTOR(internal_fn,gcall *)3577 expand_CO_ACTOR (internal_fn, gcall *)
3578 {
3579   gcc_unreachable ();
3580 }
3581 
3582 /* Expand a call to FN using the operands in STMT.  FN has a single
3583    output operand and NARGS input operands.  */
3584 
3585 static void
expand_direct_optab_fn(internal_fn fn,gcall * stmt,direct_optab optab,unsigned int nargs)3586 expand_direct_optab_fn (internal_fn fn, gcall *stmt, direct_optab optab,
3587 			unsigned int nargs)
3588 {
3589   expand_operand *ops = XALLOCAVEC (expand_operand, nargs + 1);
3590 
3591   tree_pair types = direct_internal_fn_types (fn, stmt);
3592   insn_code icode = direct_optab_handler (optab, TYPE_MODE (types.first));
3593   gcc_assert (icode != CODE_FOR_nothing);
3594 
3595   tree lhs = gimple_call_lhs (stmt);
3596   rtx lhs_rtx = NULL_RTX;
3597   if (lhs)
3598     lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3599 
3600   /* Do not assign directly to a promoted subreg, since there is no
3601      guarantee that the instruction will leave the upper bits of the
3602      register in the state required by SUBREG_PROMOTED_SIGN.  */
3603   rtx dest = lhs_rtx;
3604   if (dest && GET_CODE (dest) == SUBREG && SUBREG_PROMOTED_VAR_P (dest))
3605     dest = NULL_RTX;
3606 
3607   create_output_operand (&ops[0], dest, insn_data[icode].operand[0].mode);
3608 
3609   for (unsigned int i = 0; i < nargs; ++i)
3610     {
3611       tree rhs = gimple_call_arg (stmt, i);
3612       tree rhs_type = TREE_TYPE (rhs);
3613       rtx rhs_rtx = expand_normal (rhs);
3614       if (INTEGRAL_TYPE_P (rhs_type))
3615 	create_convert_operand_from (&ops[i + 1], rhs_rtx,
3616 				     TYPE_MODE (rhs_type),
3617 				     TYPE_UNSIGNED (rhs_type));
3618       else
3619 	create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type));
3620     }
3621 
3622   expand_insn (icode, nargs + 1, ops);
3623   if (lhs_rtx && !rtx_equal_p (lhs_rtx, ops[0].value))
3624     {
3625       /* If the return value has an integral type, convert the instruction
3626 	 result to that type.  This is useful for things that return an
3627 	 int regardless of the size of the input.  If the instruction result
3628 	 is smaller than required, assume that it is signed.
3629 
3630 	 If the return value has a nonintegral type, its mode must match
3631 	 the instruction result.  */
3632       if (GET_CODE (lhs_rtx) == SUBREG && SUBREG_PROMOTED_VAR_P (lhs_rtx))
3633 	{
3634 	  /* If this is a scalar in a register that is stored in a wider
3635 	     mode than the declared mode, compute the result into its
3636 	     declared mode and then convert to the wider mode.  */
3637 	  gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
3638 	  rtx tmp = convert_to_mode (GET_MODE (lhs_rtx), ops[0].value, 0);
3639 	  convert_move (SUBREG_REG (lhs_rtx), tmp,
3640 			SUBREG_PROMOTED_SIGN (lhs_rtx));
3641 	}
3642       else if (GET_MODE (lhs_rtx) == GET_MODE (ops[0].value))
3643 	emit_move_insn (lhs_rtx, ops[0].value);
3644       else
3645 	{
3646 	  gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
3647 	  convert_move (lhs_rtx, ops[0].value, 0);
3648 	}
3649     }
3650 }
3651 
3652 /* Expand WHILE_ULT call STMT using optab OPTAB.  */
3653 
3654 static void
expand_while_optab_fn(internal_fn,gcall * stmt,convert_optab optab)3655 expand_while_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3656 {
3657   expand_operand ops[3];
3658   tree rhs_type[2];
3659 
3660   tree lhs = gimple_call_lhs (stmt);
3661   tree lhs_type = TREE_TYPE (lhs);
3662   rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3663   create_output_operand (&ops[0], lhs_rtx, TYPE_MODE (lhs_type));
3664 
3665   for (unsigned int i = 0; i < 2; ++i)
3666     {
3667       tree rhs = gimple_call_arg (stmt, i);
3668       rhs_type[i] = TREE_TYPE (rhs);
3669       rtx rhs_rtx = expand_normal (rhs);
3670       create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type[i]));
3671     }
3672 
3673   insn_code icode = convert_optab_handler (optab, TYPE_MODE (rhs_type[0]),
3674 					   TYPE_MODE (lhs_type));
3675 
3676   expand_insn (icode, 3, ops);
3677   if (!rtx_equal_p (lhs_rtx, ops[0].value))
3678     emit_move_insn (lhs_rtx, ops[0].value);
3679 }
3680 
3681 /* Expanders for optabs that can use expand_direct_optab_fn.  */
3682 
3683 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
3684   expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3685 
3686 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
3687   expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3688 
3689 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3690   expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3691 
3692 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3693   expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3694 
3695 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3696   expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3697 
3698 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3699   expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3700 
3701 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3702   expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3703 
3704 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3705   expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3706 
3707 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3708   expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3709 
3710 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3711   expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3712 
3713 /* RETURN_TYPE and ARGS are a return type and argument list that are
3714    in principle compatible with FN (which satisfies direct_internal_fn_p).
3715    Return the types that should be used to determine whether the
3716    target supports FN.  */
3717 
3718 tree_pair
direct_internal_fn_types(internal_fn fn,tree return_type,tree * args)3719 direct_internal_fn_types (internal_fn fn, tree return_type, tree *args)
3720 {
3721   const direct_internal_fn_info &info = direct_internal_fn (fn);
3722   tree type0 = (info.type0 < 0 ? return_type : TREE_TYPE (args[info.type0]));
3723   tree type1 = (info.type1 < 0 ? return_type : TREE_TYPE (args[info.type1]));
3724   return tree_pair (type0, type1);
3725 }
3726 
3727 /* CALL is a call whose return type and arguments are in principle
3728    compatible with FN (which satisfies direct_internal_fn_p).  Return the
3729    types that should be used to determine whether the target supports FN.  */
3730 
3731 tree_pair
direct_internal_fn_types(internal_fn fn,gcall * call)3732 direct_internal_fn_types (internal_fn fn, gcall *call)
3733 {
3734   const direct_internal_fn_info &info = direct_internal_fn (fn);
3735   tree op0 = (info.type0 < 0
3736 	      ? gimple_call_lhs (call)
3737 	      : gimple_call_arg (call, info.type0));
3738   tree op1 = (info.type1 < 0
3739 	      ? gimple_call_lhs (call)
3740 	      : gimple_call_arg (call, info.type1));
3741   return tree_pair (TREE_TYPE (op0), TREE_TYPE (op1));
3742 }
3743 
3744 /* Return true if OPTAB is supported for TYPES (whose modes should be
3745    the same) when the optimization type is OPT_TYPE.  Used for simple
3746    direct optabs.  */
3747 
3748 static bool
direct_optab_supported_p(direct_optab optab,tree_pair types,optimization_type opt_type)3749 direct_optab_supported_p (direct_optab optab, tree_pair types,
3750 			  optimization_type opt_type)
3751 {
3752   machine_mode mode = TYPE_MODE (types.first);
3753   gcc_checking_assert (mode == TYPE_MODE (types.second));
3754   return direct_optab_handler (optab, mode, opt_type) != CODE_FOR_nothing;
3755 }
3756 
3757 /* Return true if OPTAB is supported for TYPES, where the first type
3758    is the destination and the second type is the source.  Used for
3759    convert optabs.  */
3760 
3761 static bool
convert_optab_supported_p(convert_optab optab,tree_pair types,optimization_type opt_type)3762 convert_optab_supported_p (convert_optab optab, tree_pair types,
3763 			   optimization_type opt_type)
3764 {
3765   return (convert_optab_handler (optab, TYPE_MODE (types.first),
3766 				 TYPE_MODE (types.second), opt_type)
3767 	  != CODE_FOR_nothing);
3768 }
3769 
3770 /* Return true if load/store lanes optab OPTAB is supported for
3771    array type TYPES.first when the optimization type is OPT_TYPE.  */
3772 
3773 static bool
multi_vector_optab_supported_p(convert_optab optab,tree_pair types,optimization_type opt_type)3774 multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
3775 				optimization_type opt_type)
3776 {
3777   gcc_assert (TREE_CODE (types.first) == ARRAY_TYPE);
3778   machine_mode imode = TYPE_MODE (types.first);
3779   machine_mode vmode = TYPE_MODE (TREE_TYPE (types.first));
3780   return (convert_optab_handler (optab, imode, vmode, opt_type)
3781 	  != CODE_FOR_nothing);
3782 }
3783 
3784 #define direct_unary_optab_supported_p direct_optab_supported_p
3785 #define direct_binary_optab_supported_p direct_optab_supported_p
3786 #define direct_ternary_optab_supported_p direct_optab_supported_p
3787 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
3788 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
3789 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
3790 #define direct_mask_load_optab_supported_p convert_optab_supported_p
3791 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
3792 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
3793 #define direct_gather_load_optab_supported_p convert_optab_supported_p
3794 #define direct_len_load_optab_supported_p direct_optab_supported_p
3795 #define direct_mask_store_optab_supported_p convert_optab_supported_p
3796 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
3797 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
3798 #define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
3799 #define direct_vec_cond_optab_supported_p convert_optab_supported_p
3800 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
3801 #define direct_len_store_optab_supported_p direct_optab_supported_p
3802 #define direct_while_optab_supported_p convert_optab_supported_p
3803 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
3804 #define direct_fold_left_optab_supported_p direct_optab_supported_p
3805 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
3806 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
3807 #define direct_vec_set_optab_supported_p direct_optab_supported_p
3808 
3809 /* Return the optab used by internal function FN.  */
3810 
3811 static optab
direct_internal_fn_optab(internal_fn fn,tree_pair types)3812 direct_internal_fn_optab (internal_fn fn, tree_pair types)
3813 {
3814   switch (fn)
3815     {
3816 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3817     case IFN_##CODE: break;
3818 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3819     case IFN_##CODE: return OPTAB##_optab;
3820 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3821 				     UNSIGNED_OPTAB, TYPE)		\
3822     case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR)		\
3823 			     ? UNSIGNED_OPTAB ## _optab			\
3824 			     : SIGNED_OPTAB ## _optab);
3825 #include "internal-fn.def"
3826 
3827     case IFN_LAST:
3828       break;
3829     }
3830   gcc_unreachable ();
3831 }
3832 
3833 /* Return the optab used by internal function FN.  */
3834 
3835 static optab
direct_internal_fn_optab(internal_fn fn)3836 direct_internal_fn_optab (internal_fn fn)
3837 {
3838   switch (fn)
3839     {
3840 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3841     case IFN_##CODE: break;
3842 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3843     case IFN_##CODE: return OPTAB##_optab;
3844 #include "internal-fn.def"
3845 
3846     case IFN_LAST:
3847       break;
3848     }
3849   gcc_unreachable ();
3850 }
3851 
3852 /* Return true if FN is supported for the types in TYPES when the
3853    optimization type is OPT_TYPE.  The types are those associated with
3854    the "type0" and "type1" fields of FN's direct_internal_fn_info
3855    structure.  */
3856 
3857 bool
direct_internal_fn_supported_p(internal_fn fn,tree_pair types,optimization_type opt_type)3858 direct_internal_fn_supported_p (internal_fn fn, tree_pair types,
3859 				optimization_type opt_type)
3860 {
3861   switch (fn)
3862     {
3863 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3864     case IFN_##CODE: break;
3865 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3866     case IFN_##CODE: \
3867       return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
3868 						opt_type);
3869 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3870 				     UNSIGNED_OPTAB, TYPE)		\
3871     case IFN_##CODE:							\
3872       {									\
3873 	optab which_optab = (TYPE_UNSIGNED (types.SELECTOR)		\
3874 			     ? UNSIGNED_OPTAB ## _optab			\
3875 			     : SIGNED_OPTAB ## _optab);			\
3876 	return direct_##TYPE##_optab_supported_p (which_optab, types,	\
3877 						  opt_type);		\
3878       }
3879 #include "internal-fn.def"
3880 
3881     case IFN_LAST:
3882       break;
3883     }
3884   gcc_unreachable ();
3885 }
3886 
3887 /* Return true if FN is supported for type TYPE when the optimization
3888    type is OPT_TYPE.  The caller knows that the "type0" and "type1"
3889    fields of FN's direct_internal_fn_info structure are the same.  */
3890 
3891 bool
direct_internal_fn_supported_p(internal_fn fn,tree type,optimization_type opt_type)3892 direct_internal_fn_supported_p (internal_fn fn, tree type,
3893 				optimization_type opt_type)
3894 {
3895   const direct_internal_fn_info &info = direct_internal_fn (fn);
3896   gcc_checking_assert (info.type0 == info.type1);
3897   return direct_internal_fn_supported_p (fn, tree_pair (type, type), opt_type);
3898 }
3899 
3900 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
3901    given that STMT is a call to a direct internal function.  */
3902 
3903 bool
direct_internal_fn_supported_p(gcall * stmt,optimization_type opt_type)3904 direct_internal_fn_supported_p (gcall *stmt, optimization_type opt_type)
3905 {
3906   internal_fn fn = gimple_call_internal_fn (stmt);
3907   tree_pair types = direct_internal_fn_types (fn, stmt);
3908   return direct_internal_fn_supported_p (fn, types, opt_type);
3909 }
3910 
3911 /* Return true if FN is a binary operation and if FN is commutative.  */
3912 
3913 bool
commutative_binary_fn_p(internal_fn fn)3914 commutative_binary_fn_p (internal_fn fn)
3915 {
3916   switch (fn)
3917     {
3918     case IFN_AVG_FLOOR:
3919     case IFN_AVG_CEIL:
3920     case IFN_MULH:
3921     case IFN_MULHS:
3922     case IFN_MULHRS:
3923     case IFN_FMIN:
3924     case IFN_FMAX:
3925     case IFN_COMPLEX_MUL:
3926     case IFN_UBSAN_CHECK_ADD:
3927     case IFN_UBSAN_CHECK_MUL:
3928     case IFN_ADD_OVERFLOW:
3929     case IFN_MUL_OVERFLOW:
3930       return true;
3931 
3932     default:
3933       return false;
3934     }
3935 }
3936 
3937 /* Return true if FN is a ternary operation and if its first two arguments
3938    are commutative.  */
3939 
3940 bool
commutative_ternary_fn_p(internal_fn fn)3941 commutative_ternary_fn_p (internal_fn fn)
3942 {
3943   switch (fn)
3944     {
3945     case IFN_FMA:
3946     case IFN_FMS:
3947     case IFN_FNMA:
3948     case IFN_FNMS:
3949       return true;
3950 
3951     default:
3952       return false;
3953     }
3954 }
3955 
3956 /* Return true if FN is an associative binary operation.  */
3957 
3958 bool
associative_binary_fn_p(internal_fn fn)3959 associative_binary_fn_p (internal_fn fn)
3960 {
3961   switch (fn)
3962     {
3963     case IFN_FMIN:
3964     case IFN_FMAX:
3965       return true;
3966 
3967     default:
3968       return false;
3969     }
3970 }
3971 
3972 /* If FN is commutative in two consecutive arguments, return the
3973    index of the first, otherwise return -1.  */
3974 
3975 int
first_commutative_argument(internal_fn fn)3976 first_commutative_argument (internal_fn fn)
3977 {
3978   switch (fn)
3979     {
3980     case IFN_COND_ADD:
3981     case IFN_COND_MUL:
3982     case IFN_COND_MIN:
3983     case IFN_COND_MAX:
3984     case IFN_COND_FMIN:
3985     case IFN_COND_FMAX:
3986     case IFN_COND_AND:
3987     case IFN_COND_IOR:
3988     case IFN_COND_XOR:
3989     case IFN_COND_FMA:
3990     case IFN_COND_FMS:
3991     case IFN_COND_FNMA:
3992     case IFN_COND_FNMS:
3993       return 1;
3994 
3995     default:
3996       if (commutative_binary_fn_p (fn)
3997 	  || commutative_ternary_fn_p (fn))
3998 	return 0;
3999       return -1;
4000     }
4001 }
4002 
4003 /* Return true if IFN_SET_EDOM is supported.  */
4004 
4005 bool
set_edom_supported_p(void)4006 set_edom_supported_p (void)
4007 {
4008 #ifdef TARGET_EDOM
4009   return true;
4010 #else
4011   return false;
4012 #endif
4013 }
4014 
4015 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4016   static void						\
4017   expand_##CODE (internal_fn fn, gcall *stmt)		\
4018   {							\
4019     expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab);	\
4020   }
4021 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4022 				     UNSIGNED_OPTAB, TYPE)		\
4023   static void								\
4024   expand_##CODE (internal_fn fn, gcall *stmt)				\
4025   {									\
4026     tree_pair types = direct_internal_fn_types (fn, stmt);		\
4027     optab which_optab = direct_internal_fn_optab (fn, types);		\
4028     expand_##TYPE##_optab_fn (fn, stmt, which_optab);			\
4029   }
4030 #include "internal-fn.def"
4031 
4032 /* Routines to expand each internal function, indexed by function number.
4033    Each routine has the prototype:
4034 
4035        expand_<NAME> (gcall *stmt)
4036 
4037    where STMT is the statement that performs the call. */
4038 static void (*const internal_fn_expanders[]) (internal_fn, gcall *) = {
4039 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
4040 #include "internal-fn.def"
4041   0
4042 };
4043 
4044 /* Invoke T(CODE, IFN) for each conditional function IFN that maps to a
4045    tree code CODE.  */
4046 #define FOR_EACH_CODE_MAPPING(T) \
4047   T (PLUS_EXPR, IFN_COND_ADD) \
4048   T (MINUS_EXPR, IFN_COND_SUB) \
4049   T (MULT_EXPR, IFN_COND_MUL) \
4050   T (TRUNC_DIV_EXPR, IFN_COND_DIV) \
4051   T (TRUNC_MOD_EXPR, IFN_COND_MOD) \
4052   T (RDIV_EXPR, IFN_COND_RDIV) \
4053   T (MIN_EXPR, IFN_COND_MIN) \
4054   T (MAX_EXPR, IFN_COND_MAX) \
4055   T (BIT_AND_EXPR, IFN_COND_AND) \
4056   T (BIT_IOR_EXPR, IFN_COND_IOR) \
4057   T (BIT_XOR_EXPR, IFN_COND_XOR) \
4058   T (LSHIFT_EXPR, IFN_COND_SHL) \
4059   T (RSHIFT_EXPR, IFN_COND_SHR) \
4060   T (NEGATE_EXPR, IFN_COND_NEG)
4061 
4062 /* Return a function that only performs CODE when a certain condition is met
4063    and that uses a given fallback value otherwise.  For example, if CODE is
4064    a binary operation associated with conditional function FN:
4065 
4066      LHS = FN (COND, A, B, ELSE)
4067 
4068    is equivalent to the C expression:
4069 
4070      LHS = COND ? A CODE B : ELSE;
4071 
4072    operating elementwise if the operands are vectors.
4073 
4074    Return IFN_LAST if no such function exists.  */
4075 
4076 internal_fn
get_conditional_internal_fn(tree_code code)4077 get_conditional_internal_fn (tree_code code)
4078 {
4079   switch (code)
4080     {
4081 #define CASE(CODE, IFN) case CODE: return IFN;
4082       FOR_EACH_CODE_MAPPING(CASE)
4083 #undef CASE
4084     default:
4085       return IFN_LAST;
4086     }
4087 }
4088 
4089 /* If IFN implements the conditional form of a tree code, return that
4090    tree code, otherwise return ERROR_MARK.  */
4091 
4092 tree_code
conditional_internal_fn_code(internal_fn ifn)4093 conditional_internal_fn_code (internal_fn ifn)
4094 {
4095   switch (ifn)
4096     {
4097 #define CASE(CODE, IFN) case IFN: return CODE;
4098       FOR_EACH_CODE_MAPPING(CASE)
4099 #undef CASE
4100     default:
4101       return ERROR_MARK;
4102     }
4103 }
4104 
4105 /* Invoke T(IFN) for each internal function IFN that also has an
4106    IFN_COND_* form.  */
4107 #define FOR_EACH_COND_FN_PAIR(T) \
4108   T (FMAX) \
4109   T (FMIN) \
4110   T (FMA) \
4111   T (FMS) \
4112   T (FNMA) \
4113   T (FNMS)
4114 
4115 /* Return a function that only performs internal function FN when a
4116    certain condition is met and that uses a given fallback value otherwise.
4117    In other words, the returned function FN' is such that:
4118 
4119      LHS = FN' (COND, A1, ... An, ELSE)
4120 
4121    is equivalent to the C expression:
4122 
4123      LHS = COND ? FN (A1, ..., An) : ELSE;
4124 
4125    operating elementwise if the operands are vectors.
4126 
4127    Return IFN_LAST if no such function exists.  */
4128 
4129 internal_fn
get_conditional_internal_fn(internal_fn fn)4130 get_conditional_internal_fn (internal_fn fn)
4131 {
4132   switch (fn)
4133     {
4134 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
4135       FOR_EACH_COND_FN_PAIR(CASE)
4136 #undef CASE
4137     default:
4138       return IFN_LAST;
4139     }
4140 }
4141 
4142 /* If IFN implements the conditional form of an unconditional internal
4143    function, return that unconditional function, otherwise return IFN_LAST.  */
4144 
4145 internal_fn
get_unconditional_internal_fn(internal_fn ifn)4146 get_unconditional_internal_fn (internal_fn ifn)
4147 {
4148   switch (ifn)
4149     {
4150 #define CASE(NAME) case IFN_COND_##NAME: return IFN_##NAME;
4151       FOR_EACH_COND_FN_PAIR(CASE)
4152 #undef CASE
4153     default:
4154       return IFN_LAST;
4155     }
4156 }
4157 
4158 /* Return true if STMT can be interpreted as a conditional tree code
4159    operation of the form:
4160 
4161      LHS = COND ? OP (RHS1, ...) : ELSE;
4162 
4163    operating elementwise if the operands are vectors.  This includes
4164    the case of an all-true COND, so that the operation always happens.
4165 
4166    When returning true, set:
4167 
4168    - *COND_OUT to the condition COND, or to NULL_TREE if the condition
4169      is known to be all-true
4170    - *CODE_OUT to the tree code
4171    - OPS[I] to operand I of *CODE_OUT
4172    - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
4173      condition is known to be all true.  */
4174 
4175 bool
can_interpret_as_conditional_op_p(gimple * stmt,tree * cond_out,tree_code * code_out,tree (& ops)[3],tree * else_out)4176 can_interpret_as_conditional_op_p (gimple *stmt, tree *cond_out,
4177 				   tree_code *code_out,
4178 				   tree (&ops)[3], tree *else_out)
4179 {
4180   if (gassign *assign = dyn_cast <gassign *> (stmt))
4181     {
4182       *cond_out = NULL_TREE;
4183       *code_out = gimple_assign_rhs_code (assign);
4184       ops[0] = gimple_assign_rhs1 (assign);
4185       ops[1] = gimple_assign_rhs2 (assign);
4186       ops[2] = gimple_assign_rhs3 (assign);
4187       *else_out = NULL_TREE;
4188       return true;
4189     }
4190   if (gcall *call = dyn_cast <gcall *> (stmt))
4191     if (gimple_call_internal_p (call))
4192       {
4193 	internal_fn ifn = gimple_call_internal_fn (call);
4194 	tree_code code = conditional_internal_fn_code (ifn);
4195 	if (code != ERROR_MARK)
4196 	  {
4197 	    *cond_out = gimple_call_arg (call, 0);
4198 	    *code_out = code;
4199 	    unsigned int nops = gimple_call_num_args (call) - 2;
4200 	    for (unsigned int i = 0; i < 3; ++i)
4201 	      ops[i] = i < nops ? gimple_call_arg (call, i + 1) : NULL_TREE;
4202 	    *else_out = gimple_call_arg (call, nops + 1);
4203 	    if (integer_truep (*cond_out))
4204 	      {
4205 		*cond_out = NULL_TREE;
4206 		*else_out = NULL_TREE;
4207 	      }
4208 	    return true;
4209 	  }
4210       }
4211   return false;
4212 }
4213 
4214 /* Return true if IFN is some form of load from memory.  */
4215 
4216 bool
internal_load_fn_p(internal_fn fn)4217 internal_load_fn_p (internal_fn fn)
4218 {
4219   switch (fn)
4220     {
4221     case IFN_MASK_LOAD:
4222     case IFN_LOAD_LANES:
4223     case IFN_MASK_LOAD_LANES:
4224     case IFN_GATHER_LOAD:
4225     case IFN_MASK_GATHER_LOAD:
4226     case IFN_LEN_LOAD:
4227       return true;
4228 
4229     default:
4230       return false;
4231     }
4232 }
4233 
4234 /* Return true if IFN is some form of store to memory.  */
4235 
4236 bool
internal_store_fn_p(internal_fn fn)4237 internal_store_fn_p (internal_fn fn)
4238 {
4239   switch (fn)
4240     {
4241     case IFN_MASK_STORE:
4242     case IFN_STORE_LANES:
4243     case IFN_MASK_STORE_LANES:
4244     case IFN_SCATTER_STORE:
4245     case IFN_MASK_SCATTER_STORE:
4246     case IFN_LEN_STORE:
4247       return true;
4248 
4249     default:
4250       return false;
4251     }
4252 }
4253 
4254 /* Return true if IFN is some form of gather load or scatter store.  */
4255 
4256 bool
internal_gather_scatter_fn_p(internal_fn fn)4257 internal_gather_scatter_fn_p (internal_fn fn)
4258 {
4259   switch (fn)
4260     {
4261     case IFN_GATHER_LOAD:
4262     case IFN_MASK_GATHER_LOAD:
4263     case IFN_SCATTER_STORE:
4264     case IFN_MASK_SCATTER_STORE:
4265       return true;
4266 
4267     default:
4268       return false;
4269     }
4270 }
4271 
4272 /* If FN takes a vector mask argument, return the index of that argument,
4273    otherwise return -1.  */
4274 
4275 int
internal_fn_mask_index(internal_fn fn)4276 internal_fn_mask_index (internal_fn fn)
4277 {
4278   switch (fn)
4279     {
4280     case IFN_MASK_LOAD:
4281     case IFN_MASK_LOAD_LANES:
4282     case IFN_MASK_STORE:
4283     case IFN_MASK_STORE_LANES:
4284       return 2;
4285 
4286     case IFN_MASK_GATHER_LOAD:
4287     case IFN_MASK_SCATTER_STORE:
4288       return 4;
4289 
4290     default:
4291       return (conditional_internal_fn_code (fn) != ERROR_MARK
4292 	      || get_unconditional_internal_fn (fn) != IFN_LAST ? 0 : -1);
4293     }
4294 }
4295 
4296 /* If FN takes a value that should be stored to memory, return the index
4297    of that argument, otherwise return -1.  */
4298 
4299 int
internal_fn_stored_value_index(internal_fn fn)4300 internal_fn_stored_value_index (internal_fn fn)
4301 {
4302   switch (fn)
4303     {
4304     case IFN_MASK_STORE:
4305     case IFN_MASK_STORE_LANES:
4306     case IFN_SCATTER_STORE:
4307     case IFN_MASK_SCATTER_STORE:
4308     case IFN_LEN_STORE:
4309       return 3;
4310 
4311     default:
4312       return -1;
4313     }
4314 }
4315 
4316 /* Return true if the target supports gather load or scatter store function
4317    IFN.  For loads, VECTOR_TYPE is the vector type of the load result,
4318    while for stores it is the vector type of the stored data argument.
4319    MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
4320    or stored.  OFFSET_VECTOR_TYPE is the vector type that holds the
4321    offset from the shared base address of each loaded or stored element.
4322    SCALE is the amount by which these offsets should be multiplied
4323    *after* they have been extended to address width.  */
4324 
4325 bool
internal_gather_scatter_fn_supported_p(internal_fn ifn,tree vector_type,tree memory_element_type,tree offset_vector_type,int scale)4326 internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
4327 					tree memory_element_type,
4328 					tree offset_vector_type, int scale)
4329 {
4330   if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)),
4331 			   TYPE_SIZE (memory_element_type)))
4332     return false;
4333   if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type),
4334 		TYPE_VECTOR_SUBPARTS (offset_vector_type)))
4335     return false;
4336   optab optab = direct_internal_fn_optab (ifn);
4337   insn_code icode = convert_optab_handler (optab, TYPE_MODE (vector_type),
4338 					   TYPE_MODE (offset_vector_type));
4339   int output_ops = internal_load_fn_p (ifn) ? 1 : 0;
4340   bool unsigned_p = TYPE_UNSIGNED (TREE_TYPE (offset_vector_type));
4341   return (icode != CODE_FOR_nothing
4342 	  && insn_operand_matches (icode, 2 + output_ops, GEN_INT (unsigned_p))
4343 	  && insn_operand_matches (icode, 3 + output_ops, GEN_INT (scale)));
4344 }
4345 
4346 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
4347    for pointers of type TYPE when the accesses have LENGTH bytes and their
4348    common byte alignment is ALIGN.  */
4349 
4350 bool
internal_check_ptrs_fn_supported_p(internal_fn ifn,tree type,poly_uint64 length,unsigned int align)4351 internal_check_ptrs_fn_supported_p (internal_fn ifn, tree type,
4352 				    poly_uint64 length, unsigned int align)
4353 {
4354   machine_mode mode = TYPE_MODE (type);
4355   optab optab = direct_internal_fn_optab (ifn);
4356   insn_code icode = direct_optab_handler (optab, mode);
4357   if (icode == CODE_FOR_nothing)
4358     return false;
4359   rtx length_rtx = immed_wide_int_const (length, mode);
4360   return (insn_operand_matches (icode, 3, length_rtx)
4361 	  && insn_operand_matches (icode, 4, GEN_INT (align)));
4362 }
4363 
4364 /* Return the supported bias for IFN which is either IFN_LEN_LOAD
4365    or IFN_LEN_STORE.  For now we only support the biases of 0 and -1
4366    (in case 0 is not an allowable length for len_load or len_store).
4367    If none of the biases match what the backend provides, return
4368    VECT_PARTIAL_BIAS_UNSUPPORTED.  */
4369 
4370 signed char
internal_len_load_store_bias(internal_fn ifn,machine_mode mode)4371 internal_len_load_store_bias (internal_fn ifn, machine_mode mode)
4372 {
4373   optab optab = direct_internal_fn_optab (ifn);
4374   insn_code icode = direct_optab_handler (optab, mode);
4375 
4376   if (icode != CODE_FOR_nothing)
4377     {
4378       /* For now we only support biases of 0 or -1.  Try both of them.  */
4379       if (insn_operand_matches (icode, 3, GEN_INT (0)))
4380 	return 0;
4381       if (insn_operand_matches (icode, 3, GEN_INT (-1)))
4382 	return -1;
4383     }
4384 
4385   return VECT_PARTIAL_BIAS_UNSUPPORTED;
4386 }
4387 
4388 /* Expand STMT as though it were a call to internal function FN.  */
4389 
4390 void
expand_internal_call(internal_fn fn,gcall * stmt)4391 expand_internal_call (internal_fn fn, gcall *stmt)
4392 {
4393   internal_fn_expanders[fn] (fn, stmt);
4394 }
4395 
4396 /* Expand STMT, which is a call to internal function FN.  */
4397 
4398 void
expand_internal_call(gcall * stmt)4399 expand_internal_call (gcall *stmt)
4400 {
4401   expand_internal_call (gimple_call_internal_fn (stmt), stmt);
4402 }
4403 
4404 /* If TYPE is a vector type, return true if IFN is a direct internal
4405    function that is supported for that type.  If TYPE is a scalar type,
4406    return true if IFN is a direct internal function that is supported for
4407    the target's preferred vector version of TYPE.  */
4408 
4409 bool
vectorized_internal_fn_supported_p(internal_fn ifn,tree type)4410 vectorized_internal_fn_supported_p (internal_fn ifn, tree type)
4411 {
4412   if (VECTOR_MODE_P (TYPE_MODE (type)))
4413     return direct_internal_fn_supported_p (ifn, type, OPTIMIZE_FOR_SPEED);
4414 
4415   scalar_mode smode;
4416   if (VECTOR_TYPE_P (type)
4417       || !is_a <scalar_mode> (TYPE_MODE (type), &smode))
4418     return false;
4419 
4420   machine_mode vmode = targetm.vectorize.preferred_simd_mode (smode);
4421   if (VECTOR_MODE_P (vmode))
4422     {
4423       tree vectype = build_vector_type_for_mode (type, vmode);
4424       if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
4425 	return true;
4426     }
4427 
4428   auto_vector_modes vector_modes;
4429   targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
4430   for (machine_mode base_mode : vector_modes)
4431     if (related_vector_mode (base_mode, smode).exists (&vmode))
4432       {
4433 	tree vectype = build_vector_type_for_mode (type, vmode);
4434 	if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
4435 	  return true;
4436       }
4437 
4438   return false;
4439 }
4440 
4441 void
expand_SHUFFLEVECTOR(internal_fn,gcall *)4442 expand_SHUFFLEVECTOR (internal_fn, gcall *)
4443 {
4444   gcc_unreachable ();
4445 }
4446 
4447 void
expand_PHI(internal_fn,gcall *)4448 expand_PHI (internal_fn, gcall *)
4449 {
4450   gcc_unreachable ();
4451 }
4452 
4453 void
expand_SPACESHIP(internal_fn,gcall * stmt)4454 expand_SPACESHIP (internal_fn, gcall *stmt)
4455 {
4456   tree lhs = gimple_call_lhs (stmt);
4457   tree rhs1 = gimple_call_arg (stmt, 0);
4458   tree rhs2 = gimple_call_arg (stmt, 1);
4459   tree type = TREE_TYPE (rhs1);
4460 
4461   do_pending_stack_adjust ();
4462 
4463   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
4464   rtx op1 = expand_normal (rhs1);
4465   rtx op2 = expand_normal (rhs2);
4466 
4467   class expand_operand ops[3];
4468   create_output_operand (&ops[0], target, TYPE_MODE (TREE_TYPE (lhs)));
4469   create_input_operand (&ops[1], op1, TYPE_MODE (type));
4470   create_input_operand (&ops[2], op2, TYPE_MODE (type));
4471   insn_code icode = optab_handler (spaceship_optab, TYPE_MODE (type));
4472   expand_insn (icode, 3, ops);
4473   if (!rtx_equal_p (target, ops[0].value))
4474     emit_move_insn (target, ops[0].value);
4475 }
4476