xref: /llvm-project/llvm/test/Analysis/CostModel/AArch64/mem-op-cost-model.ll (revision 38fffa630ee80163dc65e759392ad29798905679)
1; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 2
2; Check memory cost model action for fixed vector SVE and Neon
3; Vector bits size lower than 256 bits end up assuming Neon cost model
4; CHECK-NEON has same performance as CHECK-SVE-128
5
6; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=+neon  < %s | FileCheck %s --check-prefix=CHECK-NEON
7; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-sve-vector-bits-min=128 < %s | FileCheck %s --check-prefix=CHECK-SVE-128
8; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s --check-prefix=CHECK-SVE-256
9; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s --check-prefix=CHECK-SVE-512
10
11target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
12
13define <16 x i8> @load16(ptr %ptr) {
14; CHECK: function 'load16'
15; CHECK-NEON-LABEL: 'load16'
16; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load <16 x i8>, ptr %ptr, align 16
17; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %out
18;
19; CHECK-SVE-128-LABEL: 'load16'
20; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load <16 x i8>, ptr %ptr, align 16
21; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %out
22;
23; CHECK-SVE-256-LABEL: 'load16'
24; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load <16 x i8>, ptr %ptr, align 16
25; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %out
26;
27; CHECK-SVE-512-LABEL: 'load16'
28; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load <16 x i8>, ptr %ptr, align 16
29; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %out
30;
31  %out = load <16 x i8>, ptr %ptr
32  ret <16 x i8> %out
33}
34
35define void @store16(ptr %ptr, <16 x i8> %val) {
36; CHECK: function 'store16'
37; CHECK-NEON-LABEL: 'store16'
38; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %val, ptr %ptr, align 16
39; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
40;
41; CHECK-SVE-128-LABEL: 'store16'
42; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %val, ptr %ptr, align 16
43; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
44;
45; CHECK-SVE-256-LABEL: 'store16'
46; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %val, ptr %ptr, align 16
47; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
48;
49; CHECK-SVE-512-LABEL: 'store16'
50; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %val, ptr %ptr, align 16
51; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
52;
53  store <16 x i8> %val, ptr %ptr
54  ret void
55}
56
57define <8 x i8> @load8(ptr %ptr) {
58; CHECK: function 'load8'
59; CHECK-NEON-LABEL: 'load8'
60; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load <8 x i8>, ptr %ptr, align 8
61; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %out
62;
63; CHECK-SVE-128-LABEL: 'load8'
64; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load <8 x i8>, ptr %ptr, align 8
65; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %out
66;
67; CHECK-SVE-256-LABEL: 'load8'
68; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load <8 x i8>, ptr %ptr, align 8
69; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %out
70;
71; CHECK-SVE-512-LABEL: 'load8'
72; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load <8 x i8>, ptr %ptr, align 8
73; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %out
74;
75  %out = load <8 x i8>, ptr %ptr
76  ret <8 x i8> %out
77}
78
79define void @store8(ptr %ptr, <8 x i8> %val) {
80; CHECK: function 'store8'
81; CHECK-NEON-LABEL: 'store8'
82; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %val, ptr %ptr, align 8
83; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
84;
85; CHECK-SVE-128-LABEL: 'store8'
86; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %val, ptr %ptr, align 8
87; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
88;
89; CHECK-SVE-256-LABEL: 'store8'
90; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %val, ptr %ptr, align 8
91; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
92;
93; CHECK-SVE-512-LABEL: 'store8'
94; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %val, ptr %ptr, align 8
95; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
96;
97  store <8 x i8> %val, ptr %ptr
98  ret void
99}
100
101define <4 x i8> @load4(ptr %ptr) {
102; CHECK: function 'load4'
103; CHECK-NEON-LABEL: 'load4'
104; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %out = load <4 x i8>, ptr %ptr, align 4
105; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %out
106;
107; CHECK-SVE-128-LABEL: 'load4'
108; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %out = load <4 x i8>, ptr %ptr, align 4
109; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %out
110;
111; CHECK-SVE-256-LABEL: 'load4'
112; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load <4 x i8>, ptr %ptr, align 4
113; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %out
114;
115; CHECK-SVE-512-LABEL: 'load4'
116; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load <4 x i8>, ptr %ptr, align 4
117; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %out
118;
119  %out = load <4 x i8>, ptr %ptr
120  ret <4 x i8> %out
121}
122
123define void @store4(ptr %ptr, <4 x i8> %val) {
124; CHECK: function 'store4'
125; CHECK-NEON-LABEL: 'store4'
126; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> %val, ptr %ptr, align 4
127; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
128;
129; CHECK-SVE-128-LABEL: 'store4'
130; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> %val, ptr %ptr, align 4
131; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
132;
133; CHECK-SVE-256-LABEL: 'store4'
134; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %val, ptr %ptr, align 4
135; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
136;
137; CHECK-SVE-512-LABEL: 'store4'
138; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %val, ptr %ptr, align 4
139; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
140;
141  store <4 x i8> %val, ptr %ptr
142  ret void
143}
144
145define <16 x i16> @load_256(ptr %ptr) {
146; CHECK: function 'load_256'
147; CHECK-NEON-LABEL: 'load_256'
148; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %out = load <16 x i16>, ptr %ptr, align 32
149; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %out
150;
151; CHECK-SVE-128-LABEL: 'load_256'
152; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %out = load <16 x i16>, ptr %ptr, align 32
153; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %out
154;
155; CHECK-SVE-256-LABEL: 'load_256'
156; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load <16 x i16>, ptr %ptr, align 32
157; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %out
158;
159; CHECK-SVE-512-LABEL: 'load_256'
160; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load <16 x i16>, ptr %ptr, align 32
161; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %out
162;
163  %out = load <16 x i16>, ptr %ptr
164  ret <16 x i16> %out
165}
166
167define <8 x i64> @load_512(ptr %ptr) {
168; CHECK: function 'load_512'
169; CHECK-NEON-LABEL: 'load_512'
170; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %out = load <8 x i64>, ptr %ptr, align 64
171; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %out
172;
173; CHECK-SVE-128-LABEL: 'load_512'
174; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %out = load <8 x i64>, ptr %ptr, align 64
175; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %out
176;
177; CHECK-SVE-256-LABEL: 'load_512'
178; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %out = load <8 x i64>, ptr %ptr, align 64
179; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %out
180;
181; CHECK-SVE-512-LABEL: 'load_512'
182; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %out = load <8 x i64>, ptr %ptr, align 64
183; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %out
184;
185  %out = load <8 x i64>, ptr %ptr
186  ret <8 x i64> %out
187}
188
189declare <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr>, i32 immarg, <4 x i1>, <4 x i8>)
190define <4 x i8> @gather_load_4xi8_constant_mask(<4 x ptr> %ptrs) {
191; CHECK:         gather_load_4xi8_constant_mask
192; CHECK-NEON-LABEL: 'gather_load_4xi8_constant_mask'
193; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i8> undef)
194; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv
195;
196; CHECK-SVE-128-LABEL: 'gather_load_4xi8_constant_mask'
197; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i8> undef)
198; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv
199;
200; CHECK-SVE-256-LABEL: 'gather_load_4xi8_constant_mask'
201; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i8> undef)
202; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv
203;
204; CHECK-SVE-512-LABEL: 'gather_load_4xi8_constant_mask'
205; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i8> undef)
206; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv
207;
208  %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
209  ret <4 x i8> %lv
210}
211
212define <4 x i8> @gather_load_4xi8_variable_mask(<4 x ptr> %ptrs, <4 x i1> %cond) {
213; CHECK:         gather_load_4xi8_variable_mask
214; CHECK-NEON-LABEL: 'gather_load_4xi8_variable_mask'
215; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef)
216; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv
217;
218; CHECK-SVE-128-LABEL: 'gather_load_4xi8_variable_mask'
219; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef)
220; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv
221;
222; CHECK-SVE-256-LABEL: 'gather_load_4xi8_variable_mask'
223; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef)
224; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv
225;
226; CHECK-SVE-512-LABEL: 'gather_load_4xi8_variable_mask'
227; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef)
228; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv
229;
230  %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef)
231  ret <4 x i8> %lv
232}
233
234declare void @llvm.masked.scatter.v4i8.v4p0(<4 x i8>, <4 x ptr>, i32 immarg, <4 x i1>)
235define void @scatter_store_4xi8_constant_mask(<4 x i8> %val, <4 x ptr> %ptrs) {
236; CHECK:         scatter_store_4xi8_constant_mask
237; CHECK-NEON-LABEL: 'scatter_store_4xi8_constant_mask'
238; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true))
239; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
240;
241; CHECK-SVE-128-LABEL: 'scatter_store_4xi8_constant_mask'
242; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true))
243; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
244;
245; CHECK-SVE-256-LABEL: 'scatter_store_4xi8_constant_mask'
246; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true))
247; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
248;
249; CHECK-SVE-512-LABEL: 'scatter_store_4xi8_constant_mask'
250; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true))
251; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
252;
253  call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
254  ret void
255}
256
257define void @scatter_store_4xi8_variable_mask(<4 x i8> %val, <4 x ptr> %ptrs, <4 x i1> %cond) {
258; CHECK:         scatter_store_4xi8_variable_mask
259; CHECK-NEON-LABEL: 'scatter_store_4xi8_variable_mask'
260; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
261; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
262;
263; CHECK-SVE-128-LABEL: 'scatter_store_4xi8_variable_mask'
264; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
265; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
266;
267; CHECK-SVE-256-LABEL: 'scatter_store_4xi8_variable_mask'
268; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
269; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
270;
271; CHECK-SVE-512-LABEL: 'scatter_store_4xi8_variable_mask'
272; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
273; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
274;
275  call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
276  ret void
277}
278
279declare <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr>, i32 immarg, <4 x i1>, <4 x i32>)
280define <4 x i32> @gather_load_4xi32_constant_mask(<4 x ptr> %ptrs) {
281; CHECK:         gather_load_4xi32_constant_mask
282; CHECK-NEON-LABEL: 'gather_load_4xi32_constant_mask'
283; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i32> undef)
284; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv
285;
286; CHECK-SVE-128-LABEL: 'gather_load_4xi32_constant_mask'
287; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i32> undef)
288; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv
289;
290; CHECK-SVE-256-LABEL: 'gather_load_4xi32_constant_mask'
291; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i32> undef)
292; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv
293;
294; CHECK-SVE-512-LABEL: 'gather_load_4xi32_constant_mask'
295; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i32> undef)
296; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv
297;
298  %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
299  ret <4 x i32> %lv
300}
301
302define <4 x i32> @gather_load_4xi32_variable_mask(<4 x ptr> %ptrs, <4 x i1> %cond) {
303; CHECK:         gather_load_4xi32_variable_mask
304; CHECK-NEON-LABEL: 'gather_load_4xi32_variable_mask'
305; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef)
306; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv
307;
308; CHECK-SVE-128-LABEL: 'gather_load_4xi32_variable_mask'
309; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef)
310; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv
311;
312; CHECK-SVE-256-LABEL: 'gather_load_4xi32_variable_mask'
313; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef)
314; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv
315;
316; CHECK-SVE-512-LABEL: 'gather_load_4xi32_variable_mask'
317; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef)
318; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv
319;
320  %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef)
321  ret <4 x i32> %lv
322}
323
324declare void @llvm.masked.scatter.v4i32.v4p0(<4 x i32>, <4 x ptr>, i32 immarg, <4 x i1>)
325define void @scatter_store_4xi32_constant_mask(<4 x i32> %val, <4 x ptr> %ptrs) {
326; CHECK:         scatter_store_4xi32_constant_mask
327; CHECK-NEON-LABEL: 'scatter_store_4xi32_constant_mask'
328; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true))
329; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
330;
331; CHECK-SVE-128-LABEL: 'scatter_store_4xi32_constant_mask'
332; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true))
333; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
334;
335; CHECK-SVE-256-LABEL: 'scatter_store_4xi32_constant_mask'
336; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true))
337; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
338;
339; CHECK-SVE-512-LABEL: 'scatter_store_4xi32_constant_mask'
340; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true))
341; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
342;
343  call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
344  ret void
345}
346
347define void @scatter_store_4xi32_variable_mask(<4 x i32> %val, <4 x ptr> %ptrs, <4 x i1> %cond) {
348; CHECK:         scatter_store_4xi32_variable_mask
349; CHECK-NEON-LABEL: 'scatter_store_4xi32_variable_mask'
350; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
351; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
352;
353; CHECK-SVE-128-LABEL: 'scatter_store_4xi32_variable_mask'
354; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
355; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
356;
357; CHECK-SVE-256-LABEL: 'scatter_store_4xi32_variable_mask'
358; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
359; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
360;
361; CHECK-SVE-512-LABEL: 'scatter_store_4xi32_variable_mask'
362; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
363; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
364;
365  call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)
366  ret void
367}
368
369declare <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr>, i32, <256 x i1>, <256 x i16>)
370define void @sve_gather_vls(<256 x i1> %v256i1mask) {
371; CHECK-LABEL: 'sve_scatter_vls'
372; CHECK-NEON-LABEL: 'sve_gather_vls'
373; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1792 for instruction: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
374; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
375;
376; CHECK-SVE-128-LABEL: 'sve_gather_vls'
377; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 1792 for instruction: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
378; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
379;
380; CHECK-SVE-256-LABEL: 'sve_gather_vls'
381; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
382; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
383;
384; CHECK-SVE-512-LABEL: 'sve_gather_vls'
385; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
386; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
387;
388entry:
389  %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)
390  ret void
391}
392
393declare <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr>, i32, <256 x i1>, <256 x float>)
394define void @sve_gather_vls_float(<256 x i1> %v256i1mask) {
395; CHECK-LABEL: 'sve_gather_vls_float'
396; CHECK-NEON-LABEL: 'sve_gather_vls_float'
397; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1664 for instruction: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
398; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
399;
400; CHECK-SVE-128-LABEL: 'sve_gather_vls_float'
401; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 1664 for instruction: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
402; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
403;
404; CHECK-SVE-256-LABEL: 'sve_gather_vls_float'
405; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
406; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
407;
408; CHECK-SVE-512-LABEL: 'sve_gather_vls_float'
409; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
410; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
411;
412entry:
413  %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)
414  ret void
415}
416
417declare void @llvm.masked.scatter.v256i8.v256p0(<256 x i8>, <256 x ptr>, i32, <256 x i1>)
418define void @sve_scatter_vls(<256 x i1> %v256i1mask){
419; CHECK-LABEL: 'sve_scatter_vls'
420; CHECK-NEON-LABEL: 'sve_scatter_vls'
421; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1792 for instruction: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask)
422; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
423;
424; CHECK-SVE-128-LABEL: 'sve_scatter_vls'
425; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 1792 for instruction: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask)
426; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
427;
428; CHECK-SVE-256-LABEL: 'sve_scatter_vls'
429; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask)
430; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
431;
432; CHECK-SVE-512-LABEL: 'sve_scatter_vls'
433; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask)
434; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
435;
436entry:
437  call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask)
438  ret void
439}
440
441declare void @llvm.masked.scatter.v512f16.v512p0(<512 x half>, <512 x ptr>, i32, <512 x i1>)
442define void @sve_scatter_vls_float(<512 x i1> %v512i1mask){
443; CHECK-LABEL: 'sve_scatter_vls_float'
444; CHECK-NEON-LABEL: 'sve_scatter_vls_float'
445; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 3456 for instruction: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask)
446; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
447;
448; CHECK-SVE-128-LABEL: 'sve_scatter_vls_float'
449; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 3456 for instruction: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask)
450; CHECK-SVE-128-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
451;
452; CHECK-SVE-256-LABEL: 'sve_scatter_vls_float'
453; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 5120 for instruction: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask)
454; CHECK-SVE-256-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
455;
456; CHECK-SVE-512-LABEL: 'sve_scatter_vls_float'
457; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 5120 for instruction: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask)
458; CHECK-SVE-512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
459;
460  call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask)
461  ret void
462}
463