1 /* Schedule GIMPLE vector statements.
2 Copyright (C) 2020-2022 Free Software Foundation, Inc.
3
4 This file is part of GCC.
5
6 GCC is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
9 later version.
10
11 GCC is distributed in the hope that it will be useful, but WITHOUT
12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
19
20 #include "config.h"
21 #include "system.h"
22 #include "coretypes.h"
23 #include "backend.h"
24 #include "rtl.h"
25 #include "tree.h"
26 #include "gimple.h"
27 #include "tree-pass.h"
28 #include "ssa.h"
29 #include "expmed.h"
30 #include "optabs-tree.h"
31 #include "tree-eh.h"
32 #include "gimple-iterator.h"
33 #include "gimplify-me.h"
34 #include "gimplify.h"
35 #include "tree-cfg.h"
36 #include "bitmap.h"
37 #include "tree-ssa-dce.h"
38 #include "memmodel.h"
39 #include "optabs.h"
40 #include "gimple-fold.h"
41 #include "internal-fn.h"
42
43 /* Expand all ARRAY_REF(VIEW_CONVERT_EXPR) gimple assignments into calls to
44 internal function based on vector type of selected expansion.
45 i.e.:
46 VIEW_CONVERT_EXPR<int[4]>(u)[_1] = i_4(D);
47 =>
48 _7 = u;
49 _8 = .VEC_SET (_7, i_4(D), _1);
50 u = _8; */
51
52 static bool
gimple_expand_vec_set_expr(struct function * fun,gimple_stmt_iterator * gsi)53 gimple_expand_vec_set_expr (struct function *fun, gimple_stmt_iterator *gsi)
54 {
55 enum tree_code code;
56 gcall *new_stmt = NULL;
57 gassign *ass_stmt = NULL;
58 bool cfg_changed = false;
59
60 /* Only consider code == GIMPLE_ASSIGN. */
61 gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi));
62 if (!stmt)
63 return false;
64
65 tree lhs = gimple_assign_lhs (stmt);
66 code = TREE_CODE (lhs);
67 if (code != ARRAY_REF)
68 return false;
69
70 tree val = gimple_assign_rhs1 (stmt);
71 tree op0 = TREE_OPERAND (lhs, 0);
72 if (TREE_CODE (op0) == VIEW_CONVERT_EXPR && DECL_P (TREE_OPERAND (op0, 0))
73 && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (op0, 0)))
74 && TYPE_MODE (TREE_TYPE (lhs))
75 == TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0)))))
76 {
77 tree pos = TREE_OPERAND (lhs, 1);
78 tree view_op0 = TREE_OPERAND (op0, 0);
79 machine_mode outermode = TYPE_MODE (TREE_TYPE (view_op0));
80 if (auto_var_in_fn_p (view_op0, fun->decl)
81 && !TREE_ADDRESSABLE (view_op0) && can_vec_set_var_idx_p (outermode))
82 {
83 location_t loc = gimple_location (stmt);
84 tree var_src = make_ssa_name (TREE_TYPE (view_op0));
85 tree var_dst = make_ssa_name (TREE_TYPE (view_op0));
86
87 ass_stmt = gimple_build_assign (var_src, view_op0);
88 gimple_set_vuse (ass_stmt, gimple_vuse (stmt));
89 gimple_set_location (ass_stmt, loc);
90 gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
91
92 new_stmt
93 = gimple_build_call_internal (IFN_VEC_SET, 3, var_src, val, pos);
94 gimple_call_set_lhs (new_stmt, var_dst);
95 gimple_set_location (new_stmt, loc);
96 gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
97
98 ass_stmt = gimple_build_assign (view_op0, var_dst);
99 gimple_set_location (ass_stmt, loc);
100 gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
101
102 basic_block bb = gimple_bb (stmt);
103 gimple_move_vops (ass_stmt, stmt);
104 if (gsi_remove (gsi, true)
105 && gimple_purge_dead_eh_edges (bb))
106 cfg_changed = true;
107 *gsi = gsi_for_stmt (ass_stmt);
108 }
109 }
110
111 return cfg_changed;
112 }
113
114 /* Expand all VEC_COND_EXPR gimple assignments into calls to internal
115 function based on type of selected expansion. */
116
117 static gimple *
gimple_expand_vec_cond_expr(struct function * fun,gimple_stmt_iterator * gsi,hash_map<tree,unsigned int> * vec_cond_ssa_name_uses)118 gimple_expand_vec_cond_expr (struct function *fun, gimple_stmt_iterator *gsi,
119 hash_map<tree, unsigned int> *vec_cond_ssa_name_uses)
120 {
121 tree lhs, op0a = NULL_TREE, op0b = NULL_TREE;
122 enum tree_code code;
123 enum tree_code tcode;
124 machine_mode cmp_op_mode;
125 bool unsignedp;
126 enum insn_code icode;
127 imm_use_iterator imm_iter;
128
129 /* Only consider code == GIMPLE_ASSIGN. */
130 gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi));
131 if (!stmt)
132 return NULL;
133
134 code = gimple_assign_rhs_code (stmt);
135 if (code != VEC_COND_EXPR)
136 return NULL;
137
138 tree op0 = gimple_assign_rhs1 (stmt);
139 tree op1 = gimple_assign_rhs2 (stmt);
140 tree op2 = gimple_assign_rhs3 (stmt);
141 lhs = gimple_assign_lhs (stmt);
142 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
143
144 /* Lower mask typed, non-vector mode VEC_COND_EXPRs to bitwise operations.
145 Those can end up generated by folding and at least for integer mode masks
146 we cannot expect vcond expanders to exist. We lower a ? b : c
147 to (b & a) | (c & ~a). */
148 if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (lhs))
149 && !VECTOR_MODE_P (mode))
150 {
151 gcc_assert (types_compatible_p (TREE_TYPE (op0), TREE_TYPE (op1)));
152 gimple_seq stmts = NULL;
153 tree type = TREE_TYPE (lhs);
154 location_t loc = gimple_location (stmt);
155 tree tem0 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op1, op0);
156 tree tem1 = gimple_build (&stmts, loc, BIT_NOT_EXPR, type, op0);
157 tree tem2 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op2, tem1);
158 tree tem3 = gimple_build (&stmts, loc, BIT_IOR_EXPR, type, tem0, tem2);
159 gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
160 return gimple_build_assign (lhs, tem3);
161 }
162
163 bool can_compute_op0 = true;
164 gcc_assert (!COMPARISON_CLASS_P (op0));
165 if (TREE_CODE (op0) == SSA_NAME)
166 {
167 unsigned int used_vec_cond_exprs = 0;
168 unsigned int *slot = vec_cond_ssa_name_uses->get (op0);
169 if (slot)
170 used_vec_cond_exprs = *slot;
171 else
172 {
173 gimple *use_stmt;
174 FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, op0)
175 {
176 gassign *assign = dyn_cast<gassign *> (use_stmt);
177 if (assign != NULL
178 && gimple_assign_rhs_code (assign) == VEC_COND_EXPR
179 && gimple_assign_rhs1 (assign) == op0)
180 used_vec_cond_exprs++;
181 }
182 vec_cond_ssa_name_uses->put (op0, used_vec_cond_exprs);
183 }
184
185 gassign *def_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op0));
186 if (def_stmt)
187 {
188 tcode = gimple_assign_rhs_code (def_stmt);
189 op0a = gimple_assign_rhs1 (def_stmt);
190 op0b = gimple_assign_rhs2 (def_stmt);
191
192 tree op0_type = TREE_TYPE (op0);
193 tree op0a_type = TREE_TYPE (op0a);
194 if (TREE_CODE_CLASS (tcode) == tcc_comparison)
195 can_compute_op0 = expand_vec_cmp_expr_p (op0a_type, op0_type,
196 tcode);
197
198 /* Try to fold x CMP y ? -1 : 0 to x CMP y. */
199 if (can_compute_op0
200 && integer_minus_onep (op1)
201 && integer_zerop (op2)
202 && TYPE_MODE (TREE_TYPE (lhs)) == TYPE_MODE (TREE_TYPE (op0)))
203 {
204 tree conv_op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), op0);
205 gassign *new_stmt = gimple_build_assign (lhs, conv_op);
206 gsi_replace (gsi, new_stmt, true);
207 return new_stmt;
208 }
209
210 /* When the compare has EH we do not want to forward it when
211 it has multiple uses and in general because of the complication
212 with EH redirection. */
213 if (stmt_can_throw_internal (fun, def_stmt))
214 tcode = TREE_CODE (op0);
215
216 /* If we can compute op0 and have multiple uses, keep the SSA
217 name and use vcond_mask. */
218 else if (can_compute_op0
219 && used_vec_cond_exprs >= 2
220 && (get_vcond_mask_icode (mode, TYPE_MODE (op0_type))
221 != CODE_FOR_nothing))
222 tcode = TREE_CODE (op0);
223 }
224 else
225 tcode = TREE_CODE (op0);
226 }
227 else
228 tcode = TREE_CODE (op0);
229
230 if (TREE_CODE_CLASS (tcode) != tcc_comparison)
231 {
232 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
233 if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
234 != CODE_FOR_nothing)
235 return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
236 /* Fake op0 < 0. */
237 else
238 {
239 gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
240 == MODE_VECTOR_INT);
241 op0a = op0;
242 op0b = build_zero_cst (TREE_TYPE (op0));
243 tcode = LT_EXPR;
244 }
245 }
246 cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
247 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
248
249 gcc_assert (known_eq (GET_MODE_NUNITS (mode),
250 GET_MODE_NUNITS (cmp_op_mode)));
251
252 icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
253 /* Some targets do not have vcondeq and only vcond with NE/EQ
254 but not vcondu, so make sure to also try vcond here as
255 vcond_icode_p would canonicalize the optab query to. */
256 if (icode == CODE_FOR_nothing
257 && (tcode == NE_EXPR || tcode == EQ_EXPR)
258 && ((icode = get_vcond_icode (mode, cmp_op_mode, !unsignedp))
259 != CODE_FOR_nothing))
260 unsignedp = !unsignedp;
261 if (icode == CODE_FOR_nothing)
262 {
263 if (tcode == LT_EXPR
264 && op0a == op0)
265 {
266 /* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR
267 into a constant when only get_vcond_eq_icode is supported.
268 Try changing it to NE_EXPR. */
269 tcode = NE_EXPR;
270 }
271 if ((tcode == EQ_EXPR || tcode == NE_EXPR)
272 && direct_internal_fn_supported_p (IFN_VCONDEQ, TREE_TYPE (lhs),
273 TREE_TYPE (op0a),
274 OPTIMIZE_FOR_BOTH))
275 {
276 tree tcode_tree = build_int_cst (integer_type_node, tcode);
277 return gimple_build_call_internal (IFN_VCONDEQ, 5, op0a, op0b, op1,
278 op2, tcode_tree);
279 }
280
281 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))
282 && can_compute_op0
283 && (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
284 != CODE_FOR_nothing));
285 return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
286 }
287
288 tree tcode_tree = build_int_cst (integer_type_node, tcode);
289 return gimple_build_call_internal (unsignedp ? IFN_VCONDU : IFN_VCOND,
290 5, op0a, op0b, op1, op2, tcode_tree);
291 }
292
293
294
295 /* Iterate all gimple statements and try to expand
296 VEC_COND_EXPR assignments. */
297
298 static unsigned int
gimple_expand_vec_exprs(struct function * fun)299 gimple_expand_vec_exprs (struct function *fun)
300 {
301 gimple_stmt_iterator gsi;
302 basic_block bb;
303 hash_map<tree, unsigned int> vec_cond_ssa_name_uses;
304 auto_bitmap dce_ssa_names;
305 bool cfg_changed = false;
306
307 FOR_EACH_BB_FN (bb, fun)
308 {
309 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
310 {
311 gimple *g = gimple_expand_vec_cond_expr (fun, &gsi,
312 &vec_cond_ssa_name_uses);
313 if (g != NULL)
314 {
315 tree lhs = gimple_assign_lhs (gsi_stmt (gsi));
316 gimple_set_lhs (g, lhs);
317 gsi_replace (&gsi, g, false);
318 }
319
320 cfg_changed |= gimple_expand_vec_set_expr (fun, &gsi);
321 if (gsi_end_p (gsi))
322 break;
323 }
324 }
325
326 for (hash_map<tree, unsigned int>::iterator it = vec_cond_ssa_name_uses.begin ();
327 it != vec_cond_ssa_name_uses.end (); ++it)
328 bitmap_set_bit (dce_ssa_names, SSA_NAME_VERSION ((*it).first));
329
330 simple_dce_from_worklist (dce_ssa_names);
331
332 return cfg_changed ? TODO_cleanup_cfg : 0;
333 }
334
335 namespace {
336
337 const pass_data pass_data_gimple_isel =
338 {
339 GIMPLE_PASS, /* type */
340 "isel", /* name */
341 OPTGROUP_VEC, /* optinfo_flags */
342 TV_NONE, /* tv_id */
343 PROP_cfg, /* properties_required */
344 0, /* properties_provided */
345 0, /* properties_destroyed */
346 0, /* todo_flags_start */
347 TODO_update_ssa, /* todo_flags_finish */
348 };
349
350 class pass_gimple_isel : public gimple_opt_pass
351 {
352 public:
pass_gimple_isel(gcc::context * ctxt)353 pass_gimple_isel (gcc::context *ctxt)
354 : gimple_opt_pass (pass_data_gimple_isel, ctxt)
355 {}
356
357 /* opt_pass methods: */
gate(function *)358 virtual bool gate (function *)
359 {
360 return true;
361 }
362
execute(function * fun)363 virtual unsigned int execute (function *fun)
364 {
365 return gimple_expand_vec_exprs (fun);
366 }
367
368 }; // class pass_gimple_isel
369
370 } // anon namespace
371
372 gimple_opt_pass *
make_pass_gimple_isel(gcc::context * ctxt)373 make_pass_gimple_isel (gcc::context *ctxt)
374 {
375 return new pass_gimple_isel (ctxt);
376 }
377
378