xref: /llvm-project/llvm/test/CodeGen/AArch64/sve-fixed-length-concat.ll (revision cc82f1290a1e2157a6c0530d78d8cc84d2b8553d) 
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -aarch64-sve-vector-bits-min=256  < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
3; RUN: llc -aarch64-sve-vector-bits-min=512  < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
4; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
5
6target triple = "aarch64-unknown-linux-gnu"
7
8;
9; i8
10;
11
12; Don't use SVE for 64-bit vectors.
13define <8 x i8> @concat_v8i8(<4 x i8> %op1, <4 x i8> %op2) vscale_range(2,0) #0 {
14; CHECK-LABEL: concat_v8i8:
15; CHECK:       // %bb.0:
16; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v1.8b
17; CHECK-NEXT:    ret
18  %res = shufflevector <4 x i8> %op1, <4 x i8> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
19  ret <8 x i8> %res
20}
21
22; Don't use SVE for 128-bit vectors.
23define <16 x i8> @concat_v16i8(<8 x i8> %op1, <8 x i8> %op2) vscale_range(2,0) #0 {
24; CHECK-LABEL: concat_v16i8:
25; CHECK:       // %bb.0:
26; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
27; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
28; CHECK-NEXT:    mov v0.d[1], v1.d[0]
29; CHECK-NEXT:    ret
30  %res = shufflevector <8 x i8> %op1, <8 x i8> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
31                                                                 i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
32  ret <16 x i8> %res
33}
34
35define void @concat_v32i8(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 {
36; CHECK-LABEL: concat_v32i8:
37; CHECK:       // %bb.0:
38; CHECK-NEXT:    ptrue p0.b, vl16
39; CHECK-NEXT:    ldr q0, [x0]
40; CHECK-NEXT:    ldr q1, [x1]
41; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
42; CHECK-NEXT:    ptrue p0.b, vl32
43; CHECK-NEXT:    st1b { z0.b }, p0, [x2]
44; CHECK-NEXT:    ret
45  %op1 = load <16 x i8>, ptr %a
46  %op2 = load <16 x i8>, ptr %b
47  %res = shufflevector <16 x i8> %op1, <16 x i8> %op2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
48                                                                   i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
49                                                                   i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
50                                                                   i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
51  store <32 x i8> %res, ptr %c
52  ret void
53}
54
55define void @concat_v64i8(ptr %a, ptr %b, ptr %c) #0 {
56; VBITS_GE_256-LABEL: concat_v64i8:
57; VBITS_GE_256:       // %bb.0:
58; VBITS_GE_256-NEXT:    ptrue p0.b, vl32
59; VBITS_GE_256-NEXT:    mov w8, #32 // =0x20
60; VBITS_GE_256-NEXT:    ld1b { z0.b }, p0/z, [x1]
61; VBITS_GE_256-NEXT:    ld1b { z1.b }, p0/z, [x0]
62; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x2, x8]
63; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x2]
64; VBITS_GE_256-NEXT:    ret
65;
66; VBITS_GE_512-LABEL: concat_v64i8:
67; VBITS_GE_512:       // %bb.0:
68; VBITS_GE_512-NEXT:    ptrue p0.b, vl32
69; VBITS_GE_512-NEXT:    ld1b { z0.b }, p0/z, [x0]
70; VBITS_GE_512-NEXT:    ld1b { z1.b }, p0/z, [x1]
71; VBITS_GE_512-NEXT:    splice z0.b, p0, z0.b, z1.b
72; VBITS_GE_512-NEXT:    ptrue p0.b, vl64
73; VBITS_GE_512-NEXT:    st1b { z0.b }, p0, [x2]
74; VBITS_GE_512-NEXT:    ret
75  %op1 = load <32 x i8>, ptr %a
76  %op2 = load <32 x i8>, ptr %b
77  %res = shufflevector <32 x i8> %op1, <32 x i8> %op2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
78                                                                   i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
79                                                                   i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
80                                                                   i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31,
81                                                                   i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39,
82                                                                   i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47,
83                                                                   i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55,
84                                                                   i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
85  store <64 x i8> %res, ptr %c
86  ret void
87}
88
89define void @concat_v128i8(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 {
90; CHECK-LABEL: concat_v128i8:
91; CHECK:       // %bb.0:
92; CHECK-NEXT:    ptrue p0.b, vl64
93; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
94; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x1]
95; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
96; CHECK-NEXT:    ptrue p0.b, vl128
97; CHECK-NEXT:    st1b { z0.b }, p0, [x2]
98; CHECK-NEXT:    ret
99  %op1 = load <64 x i8>, ptr %a
100  %op2 = load <64 x i8>, ptr %b
101  %res = shufflevector <64 x i8> %op1, <64 x i8> %op2, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
102                                                                    i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
103                                                                    i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
104                                                                    i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31,
105                                                                    i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39,
106                                                                    i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47,
107                                                                    i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55,
108                                                                    i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63,
109                                                                    i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71,
110                                                                    i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79,
111                                                                    i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87,
112                                                                    i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95,
113                                                                    i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103,
114                                                                    i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111,
115                                                                    i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119,
116                                                                    i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
117  store <128 x i8> %res, ptr %c
118  ret void
119}
120
121define void @concat_v256i8(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 {
122; CHECK-LABEL: concat_v256i8:
123; CHECK:       // %bb.0:
124; CHECK-NEXT:    ptrue p0.b, vl128
125; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
126; CHECK-NEXT:    ld1b { z1.b }, p0/z, [x1]
127; CHECK-NEXT:    splice z0.b, p0, z0.b, z1.b
128; CHECK-NEXT:    ptrue p0.b, vl256
129; CHECK-NEXT:    st1b { z0.b }, p0, [x2]
130; CHECK-NEXT:    ret
131  %op1 = load <128 x i8>, ptr %a
132  %op2 = load <128 x i8>, ptr %b
133  %res = shufflevector <128 x i8> %op1, <128 x i8> %op2, <256 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
134                                                                      i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
135                                                                      i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
136                                                                      i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31,
137                                                                      i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39,
138                                                                      i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47,
139                                                                      i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55,
140                                                                      i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63,
141                                                                      i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71,
142                                                                      i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79,
143                                                                      i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87,
144                                                                      i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95,
145                                                                      i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103,
146                                                                      i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111,
147                                                                      i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119,
148                                                                      i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127,
149                                                                      i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135,
150                                                                      i32 136, i32 137, i32 138, i32 139, i32 140, i32 141, i32 142, i32 143,
151                                                                      i32 144, i32 145, i32 146, i32 147, i32 148, i32 149, i32 150, i32 151,
152                                                                      i32 152, i32 153, i32 154, i32 155, i32 156, i32 157, i32 158, i32 159,
153                                                                      i32 160, i32 161, i32 162, i32 163, i32 164, i32 165, i32 166, i32 167,
154                                                                      i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175,
155                                                                      i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183,
156                                                                      i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191,
157                                                                      i32 192, i32 193, i32 194, i32 195, i32 196, i32 197, i32 198, i32 199,
158                                                                      i32 200, i32 201, i32 202, i32 203, i32 204, i32 205, i32 206, i32 207,
159                                                                      i32 208, i32 209, i32 210, i32 211, i32 212, i32 213, i32 214, i32 215,
160                                                                      i32 216, i32 217, i32 218, i32 219, i32 220, i32 221, i32 222, i32 223,
161                                                                      i32 224, i32 225, i32 226, i32 227, i32 228, i32 229, i32 230, i32 231,
162                                                                      i32 232, i32 233, i32 234, i32 235, i32 236, i32 237, i32 238, i32 239,
163                                                                      i32 240, i32 241, i32 242, i32 243, i32 244, i32 245, i32 246, i32 247,
164                                                                      i32 248, i32 249, i32 250, i32 251, i32 252, i32 253, i32 254, i32 255>
165  store <256 x i8> %res, ptr %c
166  ret void
167}
168
169;
170; i16
171;
172
173; Don't use SVE for 64-bit vectors.
174define <4 x i16> @concat_v4i16(<2 x i16> %op1, <2 x i16> %op2) vscale_range(2,0) #0 {
175; CHECK-LABEL: concat_v4i16:
176; CHECK:       // %bb.0:
177; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
178; CHECK-NEXT:    ret
179  %res = shufflevector <2 x i16> %op1, <2 x i16> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
180  ret <4 x i16> %res
181}
182
183; Don't use SVE for 128-bit vectors.
184define <8 x i16> @concat_v8i16(<4 x i16> %op1, <4 x i16> %op2) vscale_range(2,0) #0 {
185; CHECK-LABEL: concat_v8i16:
186; CHECK:       // %bb.0:
187; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
188; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
189; CHECK-NEXT:    mov v0.d[1], v1.d[0]
190; CHECK-NEXT:    ret
191  %res = shufflevector <4 x i16> %op1, <4 x i16> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
192  ret <8 x i16> %res
193}
194
195define void @concat_v16i16(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 {
196; CHECK-LABEL: concat_v16i16:
197; CHECK:       // %bb.0:
198; CHECK-NEXT:    ptrue p0.h, vl8
199; CHECK-NEXT:    ldr q0, [x0]
200; CHECK-NEXT:    ldr q1, [x1]
201; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
202; CHECK-NEXT:    ptrue p0.h, vl16
203; CHECK-NEXT:    st1h { z0.h }, p0, [x2]
204; CHECK-NEXT:    ret
205  %op1 = load <8 x i16>, ptr %a
206  %op2 = load <8 x i16>, ptr %b
207  %res = shufflevector <8 x i16> %op1, <8 x i16> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
208                                                                   i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
209  store <16 x i16> %res, ptr %c
210  ret void
211}
212
213define void @concat_v32i16(ptr %a, ptr %b, ptr %c) #0 {
214; VBITS_GE_256-LABEL: concat_v32i16:
215; VBITS_GE_256:       // %bb.0:
216; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
217; VBITS_GE_256-NEXT:    mov x8, #16 // =0x10
218; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x1]
219; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x0]
220; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x2, x8, lsl #1]
221; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x2]
222; VBITS_GE_256-NEXT:    ret
223;
224; VBITS_GE_512-LABEL: concat_v32i16:
225; VBITS_GE_512:       // %bb.0:
226; VBITS_GE_512-NEXT:    ptrue p0.h, vl16
227; VBITS_GE_512-NEXT:    ld1h { z0.h }, p0/z, [x0]
228; VBITS_GE_512-NEXT:    ld1h { z1.h }, p0/z, [x1]
229; VBITS_GE_512-NEXT:    splice z0.h, p0, z0.h, z1.h
230; VBITS_GE_512-NEXT:    ptrue p0.h, vl32
231; VBITS_GE_512-NEXT:    st1h { z0.h }, p0, [x2]
232; VBITS_GE_512-NEXT:    ret
233  %op1 = load <16 x i16>, ptr %a
234  %op2 = load <16 x i16>, ptr %b
235  %res = shufflevector <16 x i16> %op1, <16 x i16> %op2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
236                                                                     i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
237                                                                     i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
238                                                                     i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
239  store <32 x i16> %res, ptr %c
240  ret void
241}
242
243define void @concat_v64i16(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 {
244; CHECK-LABEL: concat_v64i16:
245; CHECK:       // %bb.0:
246; CHECK-NEXT:    ptrue p0.h, vl32
247; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
248; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
249; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
250; CHECK-NEXT:    ptrue p0.h, vl64
251; CHECK-NEXT:    st1h { z0.h }, p0, [x2]
252; CHECK-NEXT:    ret
253  %op1 = load <32 x i16>, ptr %a
254  %op2 = load <32 x i16>, ptr %b
255  %res = shufflevector <32 x i16> %op1, <32 x i16> %op2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
256                                                                     i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
257                                                                     i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
258                                                                     i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31,
259                                                                     i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39,
260                                                                     i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47,
261                                                                     i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55,
262                                                                     i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
263  store <64 x i16> %res, ptr %c
264  ret void
265}
266
267define void @concat_v128i16(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 {
268; CHECK-LABEL: concat_v128i16:
269; CHECK:       // %bb.0:
270; CHECK-NEXT:    ptrue p0.h, vl64
271; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
272; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
273; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
274; CHECK-NEXT:    ptrue p0.h, vl128
275; CHECK-NEXT:    st1h { z0.h }, p0, [x2]
276; CHECK-NEXT:    ret
277  %op1 = load <64 x i16>, ptr %a
278  %op2 = load <64 x i16>, ptr %b
279  %res = shufflevector <64 x i16> %op1, <64 x i16> %op2, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
280                                                                      i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
281                                                                      i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
282                                                                      i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31,
283                                                                      i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39,
284                                                                      i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47,
285                                                                      i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55,
286                                                                      i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63,
287                                                                      i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71,
288                                                                      i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79,
289                                                                      i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87,
290                                                                      i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95,
291                                                                      i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103,
292                                                                      i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111,
293                                                                      i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119,
294                                                                      i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
295  store <128 x i16> %res, ptr %c
296  ret void
297}
298
299;
300; i32
301;
302
303; Don't use SVE for 64-bit vectors.
304define <2 x i32> @concat_v2i32(<1 x i32> %op1, <1 x i32> %op2) vscale_range(2,0) #0 {
305; CHECK-LABEL: concat_v2i32:
306; CHECK:       // %bb.0:
307; CHECK-NEXT:    zip1 v0.2s, v0.2s, v1.2s
308; CHECK-NEXT:    ret
309  %res = shufflevector <1 x i32> %op1, <1 x i32> %op2, <2 x i32> <i32 0, i32 1>
310  ret <2 x i32> %res
311}
312
313; Don't use SVE for 128-bit vectors.
314define <4 x i32> @concat_v4i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(2,0) #0 {
315; CHECK-LABEL: concat_v4i32:
316; CHECK:       // %bb.0:
317; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
318; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
319; CHECK-NEXT:    mov v0.d[1], v1.d[0]
320; CHECK-NEXT:    ret
321  %res = shufflevector <2 x i32> %op1, <2 x i32> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
322  ret <4 x i32> %res
323}
324
325define void @concat_v8i32(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 {
326; CHECK-LABEL: concat_v8i32:
327; CHECK:       // %bb.0:
328; CHECK-NEXT:    ptrue p0.s, vl4
329; CHECK-NEXT:    ldr q0, [x0]
330; CHECK-NEXT:    ldr q1, [x1]
331; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
332; CHECK-NEXT:    ptrue p0.s, vl8
333; CHECK-NEXT:    st1w { z0.s }, p0, [x2]
334; CHECK-NEXT:    ret
335  %op1 = load <4 x i32>, ptr %a
336  %op2 = load <4 x i32>, ptr %b
337  %res = shufflevector <4 x i32> %op1, <4 x i32> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
338  store <8 x i32> %res, ptr %c
339  ret void
340}
341
342define void @concat_v16i32(ptr %a, ptr %b, ptr %c) #0 {
343; VBITS_GE_256-LABEL: concat_v16i32:
344; VBITS_GE_256:       // %bb.0:
345; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
346; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
347; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x1]
348; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x0]
349; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x2, x8, lsl #2]
350; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x2]
351; VBITS_GE_256-NEXT:    ret
352;
353; VBITS_GE_512-LABEL: concat_v16i32:
354; VBITS_GE_512:       // %bb.0:
355; VBITS_GE_512-NEXT:    ptrue p0.s, vl8
356; VBITS_GE_512-NEXT:    ld1w { z0.s }, p0/z, [x0]
357; VBITS_GE_512-NEXT:    ld1w { z1.s }, p0/z, [x1]
358; VBITS_GE_512-NEXT:    splice z0.s, p0, z0.s, z1.s
359; VBITS_GE_512-NEXT:    ptrue p0.s, vl16
360; VBITS_GE_512-NEXT:    st1w { z0.s }, p0, [x2]
361; VBITS_GE_512-NEXT:    ret
362  %op1 = load <8 x i32>, ptr %a
363  %op2 = load <8 x i32>, ptr %b
364  %res = shufflevector <8 x i32> %op1, <8 x i32> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
365                                                                   i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
366  store <16 x i32> %res, ptr %c
367  ret void
368}
369
370define void @concat_v32i32(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 {
371; CHECK-LABEL: concat_v32i32:
372; CHECK:       // %bb.0:
373; CHECK-NEXT:    ptrue p0.s, vl16
374; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
375; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
376; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
377; CHECK-NEXT:    ptrue p0.s, vl32
378; CHECK-NEXT:    st1w { z0.s }, p0, [x2]
379; CHECK-NEXT:    ret
380  %op1 = load <16 x i32>, ptr %a
381  %op2 = load <16 x i32>, ptr %b
382  %res = shufflevector <16 x i32> %op1, <16 x i32> %op2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
383                                                                     i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
384                                                                     i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
385                                                                     i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
386  store <32 x i32> %res, ptr %c
387  ret void
388}
389
390define void @concat_v64i32(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 {
391; CHECK-LABEL: concat_v64i32:
392; CHECK:       // %bb.0:
393; CHECK-NEXT:    ptrue p0.s, vl32
394; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
395; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
396; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
397; CHECK-NEXT:    ptrue p0.s, vl64
398; CHECK-NEXT:    st1w { z0.s }, p0, [x2]
399; CHECK-NEXT:    ret
400  %op1 = load <32 x i32>, ptr %a
401  %op2 = load <32 x i32>, ptr %b
402  %res = shufflevector <32 x i32> %op1, <32 x i32> %op2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
403                                                                     i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
404                                                                     i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
405                                                                     i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31,
406                                                                     i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39,
407                                                                     i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47,
408                                                                     i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55,
409                                                                     i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
410  store <64 x i32> %res, ptr %c
411  ret void
412}
413
414;
415; i64
416;
417
418; Don't use SVE for 128-bit vectors.
419define <2 x i64> @concat_v2i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(2,0) #0 {
420; CHECK-LABEL: concat_v2i64:
421; CHECK:       // %bb.0:
422; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
423; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
424; CHECK-NEXT:    mov v0.d[1], v1.d[0]
425; CHECK-NEXT:    ret
426  %res = shufflevector <1 x i64> %op1, <1 x i64> %op2, <2 x i32> <i32 0, i32 1>
427  ret <2 x i64> %res
428}
429
430define void @concat_v4i64(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 {
431; CHECK-LABEL: concat_v4i64:
432; CHECK:       // %bb.0:
433; CHECK-NEXT:    ptrue p0.d, vl2
434; CHECK-NEXT:    ldr q0, [x0]
435; CHECK-NEXT:    ldr q1, [x1]
436; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
437; CHECK-NEXT:    ptrue p0.d, vl4
438; CHECK-NEXT:    st1d { z0.d }, p0, [x2]
439; CHECK-NEXT:    ret
440  %op1 = load <2 x i64>, ptr %a
441  %op2 = load <2 x i64>, ptr %b
442  %res = shufflevector <2 x i64> %op1, <2 x i64> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
443  store <4 x i64> %res, ptr %c
444  ret void
445}
446
447define void @concat_v8i64(ptr %a, ptr %b, ptr %c) #0 {
448; VBITS_GE_256-LABEL: concat_v8i64:
449; VBITS_GE_256:       // %bb.0:
450; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
451; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
452; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x1]
453; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x0]
454; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x2, x8, lsl #3]
455; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x2]
456; VBITS_GE_256-NEXT:    ret
457;
458; VBITS_GE_512-LABEL: concat_v8i64:
459; VBITS_GE_512:       // %bb.0:
460; VBITS_GE_512-NEXT:    ptrue p0.d, vl4
461; VBITS_GE_512-NEXT:    ld1d { z0.d }, p0/z, [x0]
462; VBITS_GE_512-NEXT:    ld1d { z1.d }, p0/z, [x1]
463; VBITS_GE_512-NEXT:    splice z0.d, p0, z0.d, z1.d
464; VBITS_GE_512-NEXT:    ptrue p0.d, vl8
465; VBITS_GE_512-NEXT:    st1d { z0.d }, p0, [x2]
466; VBITS_GE_512-NEXT:    ret
467  %op1 = load <4 x i64>, ptr %a
468  %op2 = load <4 x i64>, ptr %b
469  %res = shufflevector <4 x i64> %op1, <4 x i64> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
470  store <8 x i64> %res, ptr %c
471  ret void
472}
473
474define void @concat_v16i64(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 {
475; CHECK-LABEL: concat_v16i64:
476; CHECK:       // %bb.0:
477; CHECK-NEXT:    ptrue p0.d, vl8
478; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
479; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
480; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
481; CHECK-NEXT:    ptrue p0.d, vl16
482; CHECK-NEXT:    st1d { z0.d }, p0, [x2]
483; CHECK-NEXT:    ret
484  %op1 = load <8 x i64>, ptr %a
485  %op2 = load <8 x i64>, ptr %b
486  %res = shufflevector <8 x i64> %op1, <8 x i64> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
487                                                                   i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
488  store <16 x i64> %res, ptr %c
489  ret void
490}
491
492define void @concat_v32i64(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 {
493; CHECK-LABEL: concat_v32i64:
494; CHECK:       // %bb.0:
495; CHECK-NEXT:    ptrue p0.d, vl16
496; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
497; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
498; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
499; CHECK-NEXT:    ptrue p0.d, vl32
500; CHECK-NEXT:    st1d { z0.d }, p0, [x2]
501; CHECK-NEXT:    ret
502  %op1 = load <16 x i64>, ptr %a
503  %op2 = load <16 x i64>, ptr %b
504  %res = shufflevector <16 x i64> %op1, <16 x i64> %op2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
505                                                                     i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
506                                                                     i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
507                                                                     i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
508  store <32 x i64> %res, ptr %c
509  ret void
510}
511
512;
513; f16
514;
515
516; Don't use SVE for 64-bit vectors.
517define <4 x half> @concat_v4f16(<2 x half> %op1, <2 x half> %op2) vscale_range(2,0) #0 {
518; CHECK-LABEL: concat_v4f16:
519; CHECK:       // %bb.0:
520; CHECK-NEXT:    zip1 v0.2s, v0.2s, v1.2s
521; CHECK-NEXT:    ret
522  %res = shufflevector <2 x half> %op1, <2 x half> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
523  ret <4 x half> %res
524}
525
526; Don't use SVE for 128-bit vectors.
527define <8 x half> @concat_v8f16(<4 x half> %op1, <4 x half> %op2) vscale_range(2,0) #0 {
528; CHECK-LABEL: concat_v8f16:
529; CHECK:       // %bb.0:
530; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
531; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
532; CHECK-NEXT:    mov v0.d[1], v1.d[0]
533; CHECK-NEXT:    ret
534  %res = shufflevector <4 x half> %op1, <4 x half> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
535  ret <8 x half> %res
536}
537
538define void @concat_v16f16(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 {
539; CHECK-LABEL: concat_v16f16:
540; CHECK:       // %bb.0:
541; CHECK-NEXT:    ptrue p0.h, vl8
542; CHECK-NEXT:    ldr q0, [x0]
543; CHECK-NEXT:    ldr q1, [x1]
544; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
545; CHECK-NEXT:    ptrue p0.h, vl16
546; CHECK-NEXT:    st1h { z0.h }, p0, [x2]
547; CHECK-NEXT:    ret
548  %op1 = load <8 x half>, ptr %a
549  %op2 = load <8 x half>, ptr %b
550  %res = shufflevector <8 x half> %op1, <8 x half> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
551                                                                     i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
552  store <16 x half> %res, ptr %c
553  ret void
554}
555
556define void @concat_v32f16(ptr %a, ptr %b, ptr %c) #0 {
557; VBITS_GE_256-LABEL: concat_v32f16:
558; VBITS_GE_256:       // %bb.0:
559; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
560; VBITS_GE_256-NEXT:    mov x8, #16 // =0x10
561; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x1]
562; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x0]
563; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x2, x8, lsl #1]
564; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x2]
565; VBITS_GE_256-NEXT:    ret
566;
567; VBITS_GE_512-LABEL: concat_v32f16:
568; VBITS_GE_512:       // %bb.0:
569; VBITS_GE_512-NEXT:    ptrue p0.h, vl16
570; VBITS_GE_512-NEXT:    ld1h { z0.h }, p0/z, [x0]
571; VBITS_GE_512-NEXT:    ld1h { z1.h }, p0/z, [x1]
572; VBITS_GE_512-NEXT:    splice z0.h, p0, z0.h, z1.h
573; VBITS_GE_512-NEXT:    ptrue p0.h, vl32
574; VBITS_GE_512-NEXT:    st1h { z0.h }, p0, [x2]
575; VBITS_GE_512-NEXT:    ret
576  %op1 = load <16 x half>, ptr %a
577  %op2 = load <16 x half>, ptr %b
578  %res = shufflevector <16 x half> %op1, <16 x half> %op2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
579                                                                       i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
580                                                                       i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
581                                                                       i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
582  store <32 x half> %res, ptr %c
583  ret void
584}
585
586define void @concat_v64f16(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 {
587; CHECK-LABEL: concat_v64f16:
588; CHECK:       // %bb.0:
589; CHECK-NEXT:    ptrue p0.h, vl32
590; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
591; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
592; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
593; CHECK-NEXT:    ptrue p0.h, vl64
594; CHECK-NEXT:    st1h { z0.h }, p0, [x2]
595; CHECK-NEXT:    ret
596  %op1 = load <32 x half>, ptr %a
597  %op2 = load <32 x half>, ptr %b
598  %res = shufflevector <32 x half> %op1, <32 x half> %op2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
599                                                                       i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
600                                                                       i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
601                                                                       i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31,
602                                                                       i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39,
603                                                                       i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47,
604                                                                       i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55,
605                                                                       i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
606  store <64 x half> %res, ptr %c
607  ret void
608}
609
610define void @concat_v128f16(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 {
611; CHECK-LABEL: concat_v128f16:
612; CHECK:       // %bb.0:
613; CHECK-NEXT:    ptrue p0.h, vl64
614; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
615; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
616; CHECK-NEXT:    splice z0.h, p0, z0.h, z1.h
617; CHECK-NEXT:    ptrue p0.h, vl128
618; CHECK-NEXT:    st1h { z0.h }, p0, [x2]
619; CHECK-NEXT:    ret
620  %op1 = load <64 x half>, ptr %a
621  %op2 = load <64 x half>, ptr %b
622  %res = shufflevector <64 x half> %op1, <64 x half> %op2, <128 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
623                                                                        i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
624                                                                        i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
625                                                                        i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31,
626                                                                        i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39,
627                                                                        i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47,
628                                                                        i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55,
629                                                                        i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63,
630                                                                        i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71,
631                                                                        i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79,
632                                                                        i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87,
633                                                                        i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95,
634                                                                        i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103,
635                                                                        i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111,
636                                                                        i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119,
637                                                                        i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
638  store <128 x half> %res, ptr %c
639  ret void
640}
641
642;
643; i32
644;
645
646; Don't use SVE for 64-bit vectors.
647define <2 x float> @concat_v2f32(<1 x float> %op1, <1 x float> %op2) vscale_range(2,0) #0 {
648; CHECK-LABEL: concat_v2f32:
649; CHECK:       // %bb.0:
650; CHECK-NEXT:    zip1 v0.2s, v0.2s, v1.2s
651; CHECK-NEXT:    ret
652  %res = shufflevector <1 x float> %op1, <1 x float> %op2, <2 x i32> <i32 0, i32 1>
653  ret <2 x float> %res
654}
655
656; Don't use SVE for 128-bit vectors.
657define <4 x float> @concat_v4f32(<2 x float> %op1, <2 x float> %op2) vscale_range(2,0) #0 {
658; CHECK-LABEL: concat_v4f32:
659; CHECK:       // %bb.0:
660; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
661; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
662; CHECK-NEXT:    mov v0.d[1], v1.d[0]
663; CHECK-NEXT:    ret
664  %res = shufflevector <2 x float> %op1, <2 x float> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
665  ret <4 x float> %res
666}
667
668define void @concat_v8f32(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 {
669; CHECK-LABEL: concat_v8f32:
670; CHECK:       // %bb.0:
671; CHECK-NEXT:    ptrue p0.s, vl4
672; CHECK-NEXT:    ldr q0, [x0]
673; CHECK-NEXT:    ldr q1, [x1]
674; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
675; CHECK-NEXT:    ptrue p0.s, vl8
676; CHECK-NEXT:    st1w { z0.s }, p0, [x2]
677; CHECK-NEXT:    ret
678  %op1 = load <4 x float>, ptr %a
679  %op2 = load <4 x float>, ptr %b
680  %res = shufflevector <4 x float> %op1, <4 x float> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
681  store <8 x float> %res, ptr %c
682  ret void
683}
684
685define void @concat_v16f32(ptr %a, ptr %b, ptr %c) #0 {
686; VBITS_GE_256-LABEL: concat_v16f32:
687; VBITS_GE_256:       // %bb.0:
688; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
689; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
690; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x1]
691; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x0]
692; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x2, x8, lsl #2]
693; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x2]
694; VBITS_GE_256-NEXT:    ret
695;
696; VBITS_GE_512-LABEL: concat_v16f32:
697; VBITS_GE_512:       // %bb.0:
698; VBITS_GE_512-NEXT:    ptrue p0.s, vl8
699; VBITS_GE_512-NEXT:    ld1w { z0.s }, p0/z, [x0]
700; VBITS_GE_512-NEXT:    ld1w { z1.s }, p0/z, [x1]
701; VBITS_GE_512-NEXT:    splice z0.s, p0, z0.s, z1.s
702; VBITS_GE_512-NEXT:    ptrue p0.s, vl16
703; VBITS_GE_512-NEXT:    st1w { z0.s }, p0, [x2]
704; VBITS_GE_512-NEXT:    ret
705  %op1 = load <8 x float>, ptr %a
706  %op2 = load <8 x float>, ptr %b
707  %res = shufflevector <8 x float> %op1, <8 x float> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
708                                                                       i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
709  store <16 x float> %res, ptr %c
710  ret void
711}
712
713define void @concat_v32f32(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 {
714; CHECK-LABEL: concat_v32f32:
715; CHECK:       // %bb.0:
716; CHECK-NEXT:    ptrue p0.s, vl16
717; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
718; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
719; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
720; CHECK-NEXT:    ptrue p0.s, vl32
721; CHECK-NEXT:    st1w { z0.s }, p0, [x2]
722; CHECK-NEXT:    ret
723  %op1 = load <16 x float>, ptr %a
724  %op2 = load <16 x float>, ptr %b
725  %res = shufflevector <16 x float> %op1, <16 x float> %op2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
726                                                                         i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
727                                                                         i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
728                                                                         i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
729  store <32 x float> %res, ptr %c
730  ret void
731}
732
733define void @concat_v64f32(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 {
734; CHECK-LABEL: concat_v64f32:
735; CHECK:       // %bb.0:
736; CHECK-NEXT:    ptrue p0.s, vl32
737; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
738; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x1]
739; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
740; CHECK-NEXT:    ptrue p0.s, vl64
741; CHECK-NEXT:    st1w { z0.s }, p0, [x2]
742; CHECK-NEXT:    ret
743  %op1 = load <32 x float>, ptr %a
744  %op2 = load <32 x float>, ptr %b
745  %res = shufflevector <32 x float> %op1, <32 x float> %op2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
746                                                                         i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
747                                                                         i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
748                                                                         i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31,
749                                                                         i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39,
750                                                                         i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47,
751                                                                         i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55,
752                                                                         i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
753  store <64 x float> %res, ptr %c
754  ret void
755}
756
757;
758; f64
759;
760
761; Don't use SVE for 128-bit vectors.
762define <2 x double> @concat_v2f64(<1 x double> %op1, <1 x double> %op2) vscale_range(2,0) #0 {
763; CHECK-LABEL: concat_v2f64:
764; CHECK:       // %bb.0:
765; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
766; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
767; CHECK-NEXT:    mov v0.d[1], v1.d[0]
768; CHECK-NEXT:    ret
769  %res = shufflevector <1 x double> %op1, <1 x double> %op2, <2 x i32> <i32 0, i32 1>
770  ret <2 x double> %res
771}
772
773define void @concat_v4f64(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 {
774; CHECK-LABEL: concat_v4f64:
775; CHECK:       // %bb.0:
776; CHECK-NEXT:    ptrue p0.d, vl2
777; CHECK-NEXT:    ldr q0, [x0]
778; CHECK-NEXT:    ldr q1, [x1]
779; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
780; CHECK-NEXT:    ptrue p0.d, vl4
781; CHECK-NEXT:    st1d { z0.d }, p0, [x2]
782; CHECK-NEXT:    ret
783  %op1 = load <2 x double>, ptr %a
784  %op2 = load <2 x double>, ptr %b
785  %res = shufflevector <2 x double> %op1, <2 x double> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
786  store <4 x double> %res, ptr %c
787  ret void
788}
789
790define void @concat_v8f64(ptr %a, ptr %b, ptr %c) #0 {
791; VBITS_GE_256-LABEL: concat_v8f64:
792; VBITS_GE_256:       // %bb.0:
793; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
794; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
795; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x1]
796; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x0]
797; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x2, x8, lsl #3]
798; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x2]
799; VBITS_GE_256-NEXT:    ret
800;
801; VBITS_GE_512-LABEL: concat_v8f64:
802; VBITS_GE_512:       // %bb.0:
803; VBITS_GE_512-NEXT:    ptrue p0.d, vl4
804; VBITS_GE_512-NEXT:    ld1d { z0.d }, p0/z, [x0]
805; VBITS_GE_512-NEXT:    ld1d { z1.d }, p0/z, [x1]
806; VBITS_GE_512-NEXT:    splice z0.d, p0, z0.d, z1.d
807; VBITS_GE_512-NEXT:    ptrue p0.d, vl8
808; VBITS_GE_512-NEXT:    st1d { z0.d }, p0, [x2]
809; VBITS_GE_512-NEXT:    ret
810  %op1 = load <4 x double>, ptr %a
811  %op2 = load <4 x double>, ptr %b
812  %res = shufflevector <4 x double> %op1, <4 x double> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
813  store <8 x double> %res, ptr %c
814  ret void
815}
816
817define void @concat_v16f64(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 {
818; CHECK-LABEL: concat_v16f64:
819; CHECK:       // %bb.0:
820; CHECK-NEXT:    ptrue p0.d, vl8
821; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
822; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
823; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
824; CHECK-NEXT:    ptrue p0.d, vl16
825; CHECK-NEXT:    st1d { z0.d }, p0, [x2]
826; CHECK-NEXT:    ret
827  %op1 = load <8 x double>, ptr %a
828  %op2 = load <8 x double>, ptr %b
829  %res = shufflevector <8 x double> %op1, <8 x double> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
830                                                                         i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
831  store <16 x double> %res, ptr %c
832  ret void
833}
834
835define void @concat_v32f64(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 {
836; CHECK-LABEL: concat_v32f64:
837; CHECK:       // %bb.0:
838; CHECK-NEXT:    ptrue p0.d, vl16
839; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
840; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x1]
841; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
842; CHECK-NEXT:    ptrue p0.d, vl32
843; CHECK-NEXT:    st1d { z0.d }, p0, [x2]
844; CHECK-NEXT:    ret
845  %op1 = load <16 x double>, ptr %a
846  %op2 = load <16 x double>, ptr %b
847  %res = shufflevector <16 x double> %op1, <16 x double> %op2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
848                                                                           i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
849                                                                           i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
850                                                                           i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
851  store <32 x double> %res, ptr %c
852  ret void
853}
854
855;
856; undef
857;
858
859define void @concat_v32i8_undef(ptr %a, ptr %b) vscale_range(2,0) #0 {
860; CHECK-LABEL: concat_v32i8_undef:
861; CHECK:       // %bb.0:
862; CHECK-NEXT:    ptrue p0.b, vl32
863; CHECK-NEXT:    ldr q0, [x0]
864; CHECK-NEXT:    st1b { z0.b }, p0, [x1]
865; CHECK-NEXT:    ret
866  %op1 = load <16 x i8>, ptr %a
867  %res = shufflevector <16 x i8> %op1, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
868                                                                    i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
869                                                                    i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
870                                                                    i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
871  store <32 x i8> %res, ptr %b
872  ret void
873}
874
875define void @concat_v16i16_undef(ptr %a, ptr %b) vscale_range(2,0) #0 {
876; CHECK-LABEL: concat_v16i16_undef:
877; CHECK:       // %bb.0:
878; CHECK-NEXT:    ptrue p0.h, vl16
879; CHECK-NEXT:    ldr q0, [x0]
880; CHECK-NEXT:    st1h { z0.h }, p0, [x1]
881; CHECK-NEXT:    ret
882  %op1 = load <8 x i16>, ptr %a
883  %res = shufflevector <8 x i16> %op1, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
884                                                                    i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
885  store <16 x i16> %res, ptr %b
886  ret void
887}
888
889define void @concat_v8i32_undef(ptr %a, ptr %b) vscale_range(2,0) #0 {
890; CHECK-LABEL: concat_v8i32_undef:
891; CHECK:       // %bb.0:
892; CHECK-NEXT:    ptrue p0.s, vl8
893; CHECK-NEXT:    ldr q0, [x0]
894; CHECK-NEXT:    st1w { z0.s }, p0, [x1]
895; CHECK-NEXT:    ret
896  %op1 = load <4 x i32>, ptr %a
897  %res = shufflevector <4 x i32> %op1, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
898  store <8 x i32> %res, ptr %b
899  ret void
900}
901
902define void @concat_v4i64_undef(ptr %a, ptr %b) vscale_range(2,0) #0 {
903; CHECK-LABEL: concat_v4i64_undef:
904; CHECK:       // %bb.0:
905; CHECK-NEXT:    ptrue p0.d, vl4
906; CHECK-NEXT:    ldr q0, [x0]
907; CHECK-NEXT:    st1d { z0.d }, p0, [x1]
908; CHECK-NEXT:    ret
909  %op1 = load <2 x i64>, ptr %a
910  %res = shufflevector <2 x i64> %op1, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
911  store <4 x i64> %res, ptr %b
912  ret void
913}
914
915;
916; > 2 operands
917;
918
919define void @concat_v32i8_4op(ptr %a, ptr %b) vscale_range(2,0) #0 {
920; CHECK-LABEL: concat_v32i8_4op:
921; CHECK:       // %bb.0:
922; CHECK-NEXT:    ptrue p0.b, vl32
923; CHECK-NEXT:    ldr d0, [x0]
924; CHECK-NEXT:    st1b { z0.b }, p0, [x1]
925; CHECK-NEXT:    ret
926  %op1 = load <8 x i8>, ptr %a
927  %shuffle = shufflevector <8 x i8> %op1, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
928                                                                      i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
929  %res = shufflevector <16 x i8> %shuffle, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
930                                                                        i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
931                                                                        i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
932                                                                        i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
933  store <32 x i8> %res, ptr %b
934  ret void
935}
936
937define void @concat_v16i16_4op(ptr %a, ptr %b) vscale_range(2,0) #0 {
938; CHECK-LABEL: concat_v16i16_4op:
939; CHECK:       // %bb.0:
940; CHECK-NEXT:    ptrue p0.h, vl16
941; CHECK-NEXT:    ldr d0, [x0]
942; CHECK-NEXT:    st1h { z0.h }, p0, [x1]
943; CHECK-NEXT:    ret
944  %op1 = load <4 x i16>, ptr %a
945  %shuffle = shufflevector <4 x i16> %op1, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
946  %res = shufflevector <8 x i16> %shuffle, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
947                                                                        i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
948  store <16 x i16> %res, ptr %b
949  ret void
950}
951
952define void @concat_v8i32_4op(ptr %a, ptr %b) vscale_range(2,0) #0 {
953; CHECK-LABEL: concat_v8i32_4op:
954; CHECK:       // %bb.0:
955; CHECK-NEXT:    ptrue p0.s, vl8
956; CHECK-NEXT:    ldr d0, [x0]
957; CHECK-NEXT:    st1w { z0.s }, p0, [x1]
958; CHECK-NEXT:    ret
959  %op1 = load <2 x i32>, ptr %a
960  %shuffle = shufflevector <2 x i32> %op1, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
961  %res = shufflevector <4 x i32> %shuffle, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
962  store <8 x i32> %res, ptr %b
963  ret void
964}
965
966define void @concat_v4i64_4op(ptr %a, ptr %b) vscale_range(2,0) #0 {
967; CHECK-LABEL: concat_v4i64_4op:
968; CHECK:       // %bb.0:
969; CHECK-NEXT:    ptrue p0.d, vl4
970; CHECK-NEXT:    ldr d0, [x0]
971; CHECK-NEXT:    st1d { z0.d }, p0, [x1]
972; CHECK-NEXT:    ret
973  %op1 = load <1 x i64>, ptr %a
974  %shuffle = shufflevector <1 x i64> %op1, <1 x i64> undef, <2 x i32> <i32 0, i32 1>
975  %res = shufflevector <2 x i64> %shuffle, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
976  store <4 x i64> %res, ptr %b
977  ret void
978}
979
980attributes #0 = { "target-features"="+sve" }
981