1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
3
4;
5; LD1Q: vector base + unscaled offset
6;   e.g. ld1q { z0.q }, p0/z, [z0.d, x0]
7;
8define <vscale x 16 x i8> @ld1q_gather_u64base_i8(<vscale x 1 x i1> %pg, <vscale x 2 x i64> %base, i64 %offset) {
9; CHECK-LABEL: ld1q_gather_u64base_i8:
10; CHECK:       // %bb.0:
11; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
12; CHECK-NEXT:    ret
13  %load = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv16i8.nxv2i64(<vscale x 1 x i1> %pg,
14                                                                        <vscale x 2 x i64> %base,
15                                                                        i64 %offset)
16  ret <vscale x 16 x i8> %load
17}
18
19define <vscale x 8 x i16> @ld1q_gather_u64base_i16(<vscale x 1 x i1> %pg, <vscale x 2 x i64> %base, i64 %offset) {
20; CHECK-LABEL: ld1q_gather_u64base_i16:
21; CHECK:       // %bb.0:
22; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
23; CHECK-NEXT:    ret
24  %load = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv8i16.nxv2i64(<vscale x 1 x i1> %pg,
25                                                                        <vscale x 2 x i64> %base,
26                                                                        i64 %offset)
27  ret <vscale x 8 x i16> %load
28}
29
30define <vscale x 4 x i32> @ld1q_gather_u64base_i32(<vscale x 1 x i1> %pg, <vscale x 2 x i64> %base, i64 %offset) {
31; CHECK-LABEL: ld1q_gather_u64base_i32:
32; CHECK:       // %bb.0:
33; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
34; CHECK-NEXT:    ret
35  %load = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv4i32.nxv2i64(<vscale x 1 x i1> %pg,
36                                                                        <vscale x 2 x i64> %base,
37                                                                        i64 %offset)
38  ret <vscale x 4 x i32> %load
39}
40
41define <vscale x 2 x i64> @ld1q_gather_u64base_i64(<vscale x 1 x i1> %pg, <vscale x 2 x i64> %base, i64 %offset) {
42; CHECK-LABEL: ld1q_gather_u64base_i64:
43; CHECK:       // %bb.0:
44; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
45; CHECK-NEXT:    ret
46  %load = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 1 x i1> %pg,
47                                                                        <vscale x 2 x i64> %base,
48                                                                        i64 %offset)
49  ret <vscale x 2 x i64> %load
50}
51
52define <vscale x 8 x half> @ld1q_gather_u64base_f16(<vscale x 1 x i1> %pg, <vscale x 2 x i64> %base, i64 %offset) {
53; CHECK-LABEL: ld1q_gather_u64base_f16:
54; CHECK:       // %bb.0:
55; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
56; CHECK-NEXT:    ret
57  %load = call <vscale x 8 x half> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv8f16.nxv2i64(<vscale x 1 x i1> %pg,
58                                                                         <vscale x 2 x i64> %base,
59                                                                         i64 %offset)
60  ret <vscale x 8 x half> %load
61}
62
63define <vscale x 4 x float> @ld1q_gather_u64base_f32(<vscale x 1 x i1> %pg, <vscale x 2 x i64> %base, i64 %offset) {
64; CHECK-LABEL: ld1q_gather_u64base_f32:
65; CHECK:       // %bb.0:
66; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
67; CHECK-NEXT:    ret
68  %load = call <vscale x 4 x float> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv4f32.nxv2i64(<vscale x 1 x i1> %pg,
69                                                                          <vscale x 2 x i64> %base,
70                                                                          i64 %offset)
71  ret <vscale x 4 x float> %load
72}
73
74
75define <vscale x 2 x double> @ld1q_gather_u64base_f64(<vscale x 1 x i1> %pg, <vscale x 2 x i64> %base, i64 %offset) {
76; CHECK-LABEL: ld1q_gather_u64base_f64:
77; CHECK:       // %bb.0:
78; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
79; CHECK-NEXT:    ret
80  %load = call <vscale x 2 x double> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv2f64.nxv2i64(<vscale x 1 x i1> %pg,
81                                                                           <vscale x 2 x i64> %base,
82                                                                           i64 %offset)
83  ret <vscale x 2 x double> %load
84}
85
86define <vscale x 8 x bfloat> @ld1q_gather_u64base_bf16(<vscale x 1 x i1> %pg, <vscale x 2 x i64> %base, i64 %offset) {
87; CHECK-LABEL: ld1q_gather_u64base_bf16:
88; CHECK:       // %bb.0:
89; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
90; CHECK-NEXT:    ret
91  %load = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv8bf16.nxv2i64(<vscale x 1 x i1> %pg,
92                                                                            <vscale x 2 x i64> %base,
93                                                                            i64 %offset)
94  ret <vscale x 8 x bfloat> %load
95}
96
97define <vscale x 16 x i8> @test_svdl1q_gather_u64offset_s8(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off) {
98; CHECK-LABEL: test_svdl1q_gather_u64offset_s8:
99; CHECK:       // %bb.0: // %entry
100; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
101; CHECK-NEXT:    ret
102entry:
103  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv16i8(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off)
104  ret <vscale x 16 x i8> %0
105}
106
107define <vscale x 16 x i8> @test_svdl1q_gather_u64offset_u8(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off) {
108; CHECK-LABEL: test_svdl1q_gather_u64offset_u8:
109; CHECK:       // %bb.0: // %entry
110; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
111; CHECK-NEXT:    ret
112entry:
113  %0 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv16i8(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off)
114  ret <vscale x 16 x i8> %0
115}
116
117define <vscale x 8 x i16> @test_svdl1q_gather_u64offset_s16(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off) {
118; CHECK-LABEL: test_svdl1q_gather_u64offset_s16:
119; CHECK:       // %bb.0: // %entry
120; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
121; CHECK-NEXT:    ret
122entry:
123  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv8i16(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off)
124  ret <vscale x 8 x i16> %0
125}
126
127define <vscale x 8 x i16> @test_svdl1q_gather_u64offset_u16(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off) {
128; CHECK-LABEL: test_svdl1q_gather_u64offset_u16:
129; CHECK:       // %bb.0: // %entry
130; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
131; CHECK-NEXT:    ret
132entry:
133  %0 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv8i16(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off)
134  ret <vscale x 8 x i16> %0
135}
136
137define <vscale x 4 x i32> @test_svdl1q_gather_u64offset_s32(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off) {
138; CHECK-LABEL: test_svdl1q_gather_u64offset_s32:
139; CHECK:       // %bb.0: // %entry
140; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
141; CHECK-NEXT:    ret
142entry:
143  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv4i32(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off)
144  ret <vscale x 4 x i32> %0
145}
146
147define <vscale x 4 x i32> @test_svdl1q_gather_u64offset_u32(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off) {
148; CHECK-LABEL: test_svdl1q_gather_u64offset_u32:
149; CHECK:       // %bb.0: // %entry
150; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
151; CHECK-NEXT:    ret
152entry:
153  %0 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv4i32(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off)
154  ret <vscale x 4 x i32> %0
155}
156
157define <vscale x 2 x i64> @test_svdl1q_gather_u64offset_s64(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off) {
158; CHECK-LABEL: test_svdl1q_gather_u64offset_s64:
159; CHECK:       // %bb.0: // %entry
160; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
161; CHECK-NEXT:    ret
162entry:
163  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv2i64(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off)
164  ret <vscale x 2 x i64> %0
165}
166
167define <vscale x 2 x i64> @test_svdl1q_gather_u64offset_u64(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off) {
168; CHECK-LABEL: test_svdl1q_gather_u64offset_u64:
169; CHECK:       // %bb.0: // %entry
170; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
171; CHECK-NEXT:    ret
172entry:
173  %0 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv2i64(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off)
174  ret <vscale x 2 x i64> %0
175}
176
177define <vscale x 8 x bfloat> @test_svdl1q_gather_u64offset_bf16(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off) {
178; CHECK-LABEL: test_svdl1q_gather_u64offset_bf16:
179; CHECK:       // %bb.0: // %entry
180; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
181; CHECK-NEXT:    ret
182entry:
183  %0 = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv8bf16(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off)
184  ret <vscale x 8 x bfloat> %0
185}
186
187define <vscale x 8 x half> @test_svdl1q_gather_u64offset_f16(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off) {
188; CHECK-LABEL: test_svdl1q_gather_u64offset_f16:
189; CHECK:       // %bb.0: // %entry
190; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
191; CHECK-NEXT:    ret
192entry:
193  %0 = tail call <vscale x 8 x half> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv8f16(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off)
194  ret <vscale x 8 x half> %0
195}
196
197define <vscale x 4 x float> @test_svdl1q_gather_u64offset_f32(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off) {
198; CHECK-LABEL: test_svdl1q_gather_u64offset_f32:
199; CHECK:       // %bb.0: // %entry
200; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
201; CHECK-NEXT:    ret
202entry:
203  %0 = tail call <vscale x 4 x float> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv4f32(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off)
204  ret <vscale x 4 x float> %0
205}
206
207define <vscale x 2 x double> @test_svdl1q_gather_u64offset_f64(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off) {
208; CHECK-LABEL: test_svdl1q_gather_u64offset_f64:
209; CHECK:       // %bb.0: // %entry
210; CHECK-NEXT:    ld1q { z0.q }, p0/z, [z0.d, x0]
211; CHECK-NEXT:    ret
212entry:
213  %0 = tail call <vscale x 2 x double> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv2f64(<vscale x 1 x i1> %pg, ptr %base, <vscale x 2 x i64> %off)
214  ret <vscale x 2 x double> %0
215}
216
217declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv16i8.nxv2i64(<vscale x 1 x i1>, <vscale x 2 x i64>, i64)
218declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv8i16.nxv2i64(<vscale x 1 x i1>, <vscale x 2 x i64>, i64)
219declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv4i32.nxv2i64(<vscale x 1 x i1>, <vscale x 2 x i64>, i64)
220declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv2i64.nxv2i64(<vscale x 1 x i1>, <vscale x 2 x i64>, i64)
221declare <vscale x 8 x half> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv8f16.nxv2i64(<vscale x 1 x i1>, <vscale x 2 x i64>, i64)
222declare <vscale x 4 x float> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv4f32.nxv2i64(<vscale x 1 x i1>, <vscale x 2 x i64>, i64)
223declare <vscale x 2 x double> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv2f64.nxv2i64(<vscale x 1 x i1>, <vscale x 2 x i64>, i64)
224declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1q.gather.scalar.offset.nxv8bf16.nxv2i64(<vscale x 1 x i1>, <vscale x 2 x i64>, i64)
225declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv16i8(<vscale x 1 x i1>, ptr, <vscale x 2 x i64>)
226declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv8i16(<vscale x 1 x i1>, ptr, <vscale x 2 x i64>)
227declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv4i32(<vscale x 1 x i1>, ptr, <vscale x 2 x i64>)
228declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv2i64(<vscale x 1 x i1>, ptr, <vscale x 2 x i64>)
229declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv8bf16(<vscale x 1 x i1>, ptr, <vscale x 2 x i64>)
230declare <vscale x 8 x half> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv8f16(<vscale x 1 x i1>, ptr, <vscale x 2 x i64>)
231declare <vscale x 4 x float> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv4f32(<vscale x 1 x i1>, ptr, <vscale x 2 x i64>)
232declare <vscale x 2 x double> @llvm.aarch64.sve.ld1q.gather.vector.offset.nxv2f64(<vscale x 1 x i1>, ptr, <vscale x 2 x i64>)
233