xref: /llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll (revision 8e0cd7382adacd8bc1741dc26bc0be6bdf8e238a) 
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mattr=+sve -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=CHECK,SVE
3; RUN: llc -mattr=+sve2 -force-streaming-compatible  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
4; RUN: llc -mattr=+sme -force-streaming  < %s | FileCheck %s --check-prefixes=CHECK,SVE2
5; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
6
7target triple = "aarch64-unknown-linux-gnu"
8
9;
10; ADD
11;
12define <4 x i8> @add_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
13; CHECK-LABEL: add_v4i8:
14; CHECK:       // %bb.0:
15; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
16; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
17; CHECK-NEXT:    add z0.h, z0.h, z1.h
18; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
19; CHECK-NEXT:    ret
20;
21; NONEON-NOSVE-LABEL: add_v4i8:
22; NONEON-NOSVE:       // %bb.0:
23; NONEON-NOSVE-NEXT:    sub sp, sp, #32
24; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
25; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
26; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
27; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
28; NONEON-NOSVE-NEXT:    add w8, w9, w8
29; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]
30; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
31; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
32; NONEON-NOSVE-NEXT:    add w8, w9, w8
33; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
34; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
35; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
36; NONEON-NOSVE-NEXT:    add w8, w9, w8
37; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
38; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
39; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
40; NONEON-NOSVE-NEXT:    add w8, w9, w8
41; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
42; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
43; NONEON-NOSVE-NEXT:    add sp, sp, #32
44; NONEON-NOSVE-NEXT:    ret
45  %res = add <4 x i8> %op1, %op2
46  ret <4 x i8> %res
47}
48
49define <8 x i8> @add_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
50; CHECK-LABEL: add_v8i8:
51; CHECK:       // %bb.0:
52; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
53; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
54; CHECK-NEXT:    add z0.b, z0.b, z1.b
55; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
56; CHECK-NEXT:    ret
57;
58; NONEON-NOSVE-LABEL: add_v8i8:
59; NONEON-NOSVE:       // %bb.0:
60; NONEON-NOSVE-NEXT:    sub sp, sp, #32
61; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
62; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
63; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
64; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]
65; NONEON-NOSVE-NEXT:    add w8, w9, w8
66; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
67; NONEON-NOSVE-NEXT:    strb w8, [sp, #31]
68; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
69; NONEON-NOSVE-NEXT:    add w8, w9, w8
70; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]
71; NONEON-NOSVE-NEXT:    strb w8, [sp, #30]
72; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
73; NONEON-NOSVE-NEXT:    add w8, w9, w8
74; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
75; NONEON-NOSVE-NEXT:    strb w8, [sp, #29]
76; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
77; NONEON-NOSVE-NEXT:    add w8, w9, w8
78; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]
79; NONEON-NOSVE-NEXT:    strb w8, [sp, #28]
80; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
81; NONEON-NOSVE-NEXT:    add w8, w9, w8
82; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
83; NONEON-NOSVE-NEXT:    strb w8, [sp, #27]
84; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
85; NONEON-NOSVE-NEXT:    add w8, w9, w8
86; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
87; NONEON-NOSVE-NEXT:    strb w8, [sp, #26]
88; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
89; NONEON-NOSVE-NEXT:    add w8, w9, w8
90; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]
91; NONEON-NOSVE-NEXT:    strb w8, [sp, #25]
92; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
93; NONEON-NOSVE-NEXT:    add w8, w9, w8
94; NONEON-NOSVE-NEXT:    strb w8, [sp, #24]
95; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
96; NONEON-NOSVE-NEXT:    add sp, sp, #32
97; NONEON-NOSVE-NEXT:    ret
98  %res = add <8 x i8> %op1, %op2
99  ret <8 x i8> %res
100}
101
102define <16 x i8> @add_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
103; CHECK-LABEL: add_v16i8:
104; CHECK:       // %bb.0:
105; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
106; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
107; CHECK-NEXT:    add z0.b, z0.b, z1.b
108; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
109; CHECK-NEXT:    ret
110;
111; NONEON-NOSVE-LABEL: add_v16i8:
112; NONEON-NOSVE:       // %bb.0:
113; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
114; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
115; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]
116; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]
117; NONEON-NOSVE-NEXT:    add w8, w9, w8
118; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
119; NONEON-NOSVE-NEXT:    strb w8, [sp, #47]
120; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
121; NONEON-NOSVE-NEXT:    add w8, w9, w8
122; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]
123; NONEON-NOSVE-NEXT:    strb w8, [sp, #46]
124; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]
125; NONEON-NOSVE-NEXT:    add w8, w9, w8
126; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
127; NONEON-NOSVE-NEXT:    strb w8, [sp, #45]
128; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]
129; NONEON-NOSVE-NEXT:    add w8, w9, w8
130; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]
131; NONEON-NOSVE-NEXT:    strb w8, [sp, #44]
132; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]
133; NONEON-NOSVE-NEXT:    add w8, w9, w8
134; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
135; NONEON-NOSVE-NEXT:    strb w8, [sp, #43]
136; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]
137; NONEON-NOSVE-NEXT:    add w8, w9, w8
138; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
139; NONEON-NOSVE-NEXT:    strb w8, [sp, #42]
140; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]
141; NONEON-NOSVE-NEXT:    add w8, w9, w8
142; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]
143; NONEON-NOSVE-NEXT:    strb w8, [sp, #41]
144; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]
145; NONEON-NOSVE-NEXT:    add w8, w9, w8
146; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]
147; NONEON-NOSVE-NEXT:    strb w8, [sp, #40]
148; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
149; NONEON-NOSVE-NEXT:    add w8, w9, w8
150; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #6]
151; NONEON-NOSVE-NEXT:    strb w8, [sp, #39]
152; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
153; NONEON-NOSVE-NEXT:    add w8, w9, w8
154; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #5]
155; NONEON-NOSVE-NEXT:    strb w8, [sp, #38]
156; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
157; NONEON-NOSVE-NEXT:    add w8, w9, w8
158; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #4]
159; NONEON-NOSVE-NEXT:    strb w8, [sp, #37]
160; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
161; NONEON-NOSVE-NEXT:    add w8, w9, w8
162; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]
163; NONEON-NOSVE-NEXT:    strb w8, [sp, #36]
164; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
165; NONEON-NOSVE-NEXT:    add w8, w9, w8
166; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #2]
167; NONEON-NOSVE-NEXT:    strb w8, [sp, #35]
168; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
169; NONEON-NOSVE-NEXT:    add w8, w9, w8
170; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1]
171; NONEON-NOSVE-NEXT:    strb w8, [sp, #34]
172; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
173; NONEON-NOSVE-NEXT:    add w8, w9, w8
174; NONEON-NOSVE-NEXT:    ldrb w9, [sp]
175; NONEON-NOSVE-NEXT:    strb w8, [sp, #33]
176; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
177; NONEON-NOSVE-NEXT:    add w8, w9, w8
178; NONEON-NOSVE-NEXT:    strb w8, [sp, #32]
179; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
180; NONEON-NOSVE-NEXT:    add sp, sp, #48
181; NONEON-NOSVE-NEXT:    ret
182  %res = add <16 x i8> %op1, %op2
183  ret <16 x i8> %res
184}
185
186define void @add_v32i8(ptr %a, ptr %b) {
187; CHECK-LABEL: add_v32i8:
188; CHECK:       // %bb.0:
189; CHECK-NEXT:    ldp q0, q3, [x1]
190; CHECK-NEXT:    ldp q1, q2, [x0]
191; CHECK-NEXT:    add z0.b, z1.b, z0.b
192; CHECK-NEXT:    add z1.b, z2.b, z3.b
193; CHECK-NEXT:    stp q0, q1, [x0]
194; CHECK-NEXT:    ret
195;
196; NONEON-NOSVE-LABEL: add_v32i8:
197; NONEON-NOSVE:       // %bb.0:
198; NONEON-NOSVE-NEXT:    sub sp, sp, #96
199; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
200; NONEON-NOSVE-NEXT:    ldp q3, q0, [x1]
201; NONEON-NOSVE-NEXT:    ldp q2, q1, [x0]
202; NONEON-NOSVE-NEXT:    stp q2, q3, [sp]
203; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #32]
204; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #63]
205; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #47]
206; NONEON-NOSVE-NEXT:    add w8, w9, w8
207; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #46]
208; NONEON-NOSVE-NEXT:    strb w8, [sp, #95]
209; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #62]
210; NONEON-NOSVE-NEXT:    add w8, w9, w8
211; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #45]
212; NONEON-NOSVE-NEXT:    strb w8, [sp, #94]
213; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #61]
214; NONEON-NOSVE-NEXT:    add w8, w9, w8
215; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #44]
216; NONEON-NOSVE-NEXT:    strb w8, [sp, #93]
217; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #60]
218; NONEON-NOSVE-NEXT:    add w8, w9, w8
219; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #43]
220; NONEON-NOSVE-NEXT:    strb w8, [sp, #92]
221; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #59]
222; NONEON-NOSVE-NEXT:    add w8, w9, w8
223; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #42]
224; NONEON-NOSVE-NEXT:    strb w8, [sp, #91]
225; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #58]
226; NONEON-NOSVE-NEXT:    add w8, w9, w8
227; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #41]
228; NONEON-NOSVE-NEXT:    strb w8, [sp, #90]
229; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #57]
230; NONEON-NOSVE-NEXT:    add w8, w9, w8
231; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
232; NONEON-NOSVE-NEXT:    strb w8, [sp, #89]
233; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #56]
234; NONEON-NOSVE-NEXT:    add w8, w9, w8
235; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #39]
236; NONEON-NOSVE-NEXT:    strb w8, [sp, #88]
237; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #55]
238; NONEON-NOSVE-NEXT:    add w8, w9, w8
239; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #38]
240; NONEON-NOSVE-NEXT:    strb w8, [sp, #87]
241; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #54]
242; NONEON-NOSVE-NEXT:    add w8, w9, w8
243; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #37]
244; NONEON-NOSVE-NEXT:    strb w8, [sp, #86]
245; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #53]
246; NONEON-NOSVE-NEXT:    add w8, w9, w8
247; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #36]
248; NONEON-NOSVE-NEXT:    strb w8, [sp, #85]
249; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #52]
250; NONEON-NOSVE-NEXT:    add w8, w9, w8
251; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #35]
252; NONEON-NOSVE-NEXT:    strb w8, [sp, #84]
253; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #51]
254; NONEON-NOSVE-NEXT:    add w8, w9, w8
255; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #34]
256; NONEON-NOSVE-NEXT:    strb w8, [sp, #83]
257; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #50]
258; NONEON-NOSVE-NEXT:    add w8, w9, w8
259; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #33]
260; NONEON-NOSVE-NEXT:    strb w8, [sp, #82]
261; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #49]
262; NONEON-NOSVE-NEXT:    add w8, w9, w8
263; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #32]
264; NONEON-NOSVE-NEXT:    strb w8, [sp, #81]
265; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #48]
266; NONEON-NOSVE-NEXT:    add w8, w9, w8
267; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]
268; NONEON-NOSVE-NEXT:    strb w8, [sp, #80]
269; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]
270; NONEON-NOSVE-NEXT:    add w8, w9, w8
271; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
272; NONEON-NOSVE-NEXT:    strb w8, [sp, #79]
273; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
274; NONEON-NOSVE-NEXT:    add w8, w9, w8
275; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]
276; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
277; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]
278; NONEON-NOSVE-NEXT:    add w8, w9, w8
279; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
280; NONEON-NOSVE-NEXT:    strb w8, [sp, #77]
281; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]
282; NONEON-NOSVE-NEXT:    add w8, w9, w8
283; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]
284; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
285; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]
286; NONEON-NOSVE-NEXT:    add w8, w9, w8
287; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
288; NONEON-NOSVE-NEXT:    strb w8, [sp, #75]
289; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]
290; NONEON-NOSVE-NEXT:    add w8, w9, w8
291; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
292; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
293; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]
294; NONEON-NOSVE-NEXT:    add w8, w9, w8
295; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]
296; NONEON-NOSVE-NEXT:    strb w8, [sp, #73]
297; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]
298; NONEON-NOSVE-NEXT:    add w8, w9, w8
299; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]
300; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
301; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
302; NONEON-NOSVE-NEXT:    add w8, w9, w8
303; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #6]
304; NONEON-NOSVE-NEXT:    strb w8, [sp, #71]
305; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
306; NONEON-NOSVE-NEXT:    add w8, w9, w8
307; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #5]
308; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
309; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
310; NONEON-NOSVE-NEXT:    add w8, w9, w8
311; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #4]
312; NONEON-NOSVE-NEXT:    strb w8, [sp, #69]
313; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
314; NONEON-NOSVE-NEXT:    add w8, w9, w8
315; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]
316; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
317; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
318; NONEON-NOSVE-NEXT:    add w8, w9, w8
319; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #2]
320; NONEON-NOSVE-NEXT:    strb w8, [sp, #67]
321; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
322; NONEON-NOSVE-NEXT:    add w8, w9, w8
323; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1]
324; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
325; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
326; NONEON-NOSVE-NEXT:    add w8, w9, w8
327; NONEON-NOSVE-NEXT:    ldrb w9, [sp]
328; NONEON-NOSVE-NEXT:    strb w8, [sp, #65]
329; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
330; NONEON-NOSVE-NEXT:    add w8, w9, w8
331; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
332; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
333; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
334; NONEON-NOSVE-NEXT:    add sp, sp, #96
335; NONEON-NOSVE-NEXT:    ret
336  %op1 = load <32 x i8>, ptr %a
337  %op2 = load <32 x i8>, ptr %b
338  %res = add <32 x i8> %op1, %op2
339  store <32 x i8> %res, ptr %a
340  ret void
341}
342
343define <2 x i16> @add_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
344; CHECK-LABEL: add_v2i16:
345; CHECK:       // %bb.0:
346; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
347; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
348; CHECK-NEXT:    add z0.s, z0.s, z1.s
349; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
350; CHECK-NEXT:    ret
351;
352; NONEON-NOSVE-LABEL: add_v2i16:
353; NONEON-NOSVE:       // %bb.0:
354; NONEON-NOSVE-NEXT:    sub sp, sp, #32
355; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
356; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
357; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #8]
358; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
359; NONEON-NOSVE-NEXT:    add w8, w10, w8
360; NONEON-NOSVE-NEXT:    str w8, [sp, #28]
361; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
362; NONEON-NOSVE-NEXT:    add w8, w9, w8
363; NONEON-NOSVE-NEXT:    str w8, [sp, #24]
364; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
365; NONEON-NOSVE-NEXT:    add sp, sp, #32
366; NONEON-NOSVE-NEXT:    ret
367  %res = add <2 x i16> %op1, %op2
368  ret <2 x i16> %res
369}
370
371define <4 x i16> @add_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
372; CHECK-LABEL: add_v4i16:
373; CHECK:       // %bb.0:
374; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
375; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
376; CHECK-NEXT:    add z0.h, z0.h, z1.h
377; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
378; CHECK-NEXT:    ret
379;
380; NONEON-NOSVE-LABEL: add_v4i16:
381; NONEON-NOSVE:       // %bb.0:
382; NONEON-NOSVE-NEXT:    sub sp, sp, #32
383; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
384; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
385; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
386; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
387; NONEON-NOSVE-NEXT:    add w8, w9, w8
388; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]
389; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
390; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
391; NONEON-NOSVE-NEXT:    add w8, w9, w8
392; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
393; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
394; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
395; NONEON-NOSVE-NEXT:    add w8, w9, w8
396; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
397; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
398; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
399; NONEON-NOSVE-NEXT:    add w8, w9, w8
400; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
401; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
402; NONEON-NOSVE-NEXT:    add sp, sp, #32
403; NONEON-NOSVE-NEXT:    ret
404  %res = add <4 x i16> %op1, %op2
405  ret <4 x i16> %res
406}
407
408define <8 x i16> @add_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
409; CHECK-LABEL: add_v8i16:
410; CHECK:       // %bb.0:
411; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
412; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
413; CHECK-NEXT:    add z0.h, z0.h, z1.h
414; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
415; CHECK-NEXT:    ret
416;
417; NONEON-NOSVE-LABEL: add_v8i16:
418; NONEON-NOSVE:       // %bb.0:
419; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
420; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
421; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #30]
422; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
423; NONEON-NOSVE-NEXT:    add w8, w9, w8
424; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]
425; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
426; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
427; NONEON-NOSVE-NEXT:    add w8, w9, w8
428; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
429; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
430; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #26]
431; NONEON-NOSVE-NEXT:    add w8, w9, w8
432; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
433; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
434; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
435; NONEON-NOSVE-NEXT:    add w8, w9, w8
436; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #6]
437; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
438; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
439; NONEON-NOSVE-NEXT:    add w8, w9, w8
440; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]
441; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]
442; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
443; NONEON-NOSVE-NEXT:    add w8, w9, w8
444; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #2]
445; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]
446; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
447; NONEON-NOSVE-NEXT:    add w8, w9, w8
448; NONEON-NOSVE-NEXT:    ldrh w9, [sp]
449; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]
450; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
451; NONEON-NOSVE-NEXT:    add w8, w9, w8
452; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
453; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
454; NONEON-NOSVE-NEXT:    add sp, sp, #48
455; NONEON-NOSVE-NEXT:    ret
456  %res = add <8 x i16> %op1, %op2
457  ret <8 x i16> %res
458}
459
460define void @add_v16i16(ptr %a, ptr %b) {
461; CHECK-LABEL: add_v16i16:
462; CHECK:       // %bb.0:
463; CHECK-NEXT:    ldp q0, q3, [x1]
464; CHECK-NEXT:    ldp q1, q2, [x0]
465; CHECK-NEXT:    add z0.h, z1.h, z0.h
466; CHECK-NEXT:    add z1.h, z2.h, z3.h
467; CHECK-NEXT:    stp q0, q1, [x0]
468; CHECK-NEXT:    ret
469;
470; NONEON-NOSVE-LABEL: add_v16i16:
471; NONEON-NOSVE:       // %bb.0:
472; NONEON-NOSVE-NEXT:    sub sp, sp, #96
473; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
474; NONEON-NOSVE-NEXT:    ldp q3, q0, [x1]
475; NONEON-NOSVE-NEXT:    ldp q2, q1, [x0]
476; NONEON-NOSVE-NEXT:    stp q2, q3, [sp]
477; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #32]
478; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #62]
479; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #46]
480; NONEON-NOSVE-NEXT:    add w8, w9, w8
481; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #44]
482; NONEON-NOSVE-NEXT:    strh w8, [sp, #94]
483; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #60]
484; NONEON-NOSVE-NEXT:    add w8, w9, w8
485; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #42]
486; NONEON-NOSVE-NEXT:    strh w8, [sp, #92]
487; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #58]
488; NONEON-NOSVE-NEXT:    add w8, w9, w8
489; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #40]
490; NONEON-NOSVE-NEXT:    strh w8, [sp, #90]
491; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #56]
492; NONEON-NOSVE-NEXT:    add w8, w9, w8
493; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #38]
494; NONEON-NOSVE-NEXT:    strh w8, [sp, #88]
495; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #54]
496; NONEON-NOSVE-NEXT:    add w8, w9, w8
497; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #36]
498; NONEON-NOSVE-NEXT:    strh w8, [sp, #86]
499; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #52]
500; NONEON-NOSVE-NEXT:    add w8, w9, w8
501; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #34]
502; NONEON-NOSVE-NEXT:    strh w8, [sp, #84]
503; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #50]
504; NONEON-NOSVE-NEXT:    add w8, w9, w8
505; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #32]
506; NONEON-NOSVE-NEXT:    strh w8, [sp, #82]
507; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #48]
508; NONEON-NOSVE-NEXT:    add w8, w9, w8
509; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
510; NONEON-NOSVE-NEXT:    strh w8, [sp, #80]
511; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #30]
512; NONEON-NOSVE-NEXT:    add w8, w9, w8
513; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]
514; NONEON-NOSVE-NEXT:    strh w8, [sp, #78]
515; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
516; NONEON-NOSVE-NEXT:    add w8, w9, w8
517; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
518; NONEON-NOSVE-NEXT:    strh w8, [sp, #76]
519; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #26]
520; NONEON-NOSVE-NEXT:    add w8, w9, w8
521; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
522; NONEON-NOSVE-NEXT:    strh w8, [sp, #74]
523; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
524; NONEON-NOSVE-NEXT:    add w8, w9, w8
525; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #6]
526; NONEON-NOSVE-NEXT:    strh w8, [sp, #72]
527; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
528; NONEON-NOSVE-NEXT:    add w8, w9, w8
529; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]
530; NONEON-NOSVE-NEXT:    strh w8, [sp, #70]
531; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
532; NONEON-NOSVE-NEXT:    add w8, w9, w8
533; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #2]
534; NONEON-NOSVE-NEXT:    strh w8, [sp, #68]
535; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
536; NONEON-NOSVE-NEXT:    add w8, w9, w8
537; NONEON-NOSVE-NEXT:    ldrh w9, [sp]
538; NONEON-NOSVE-NEXT:    strh w8, [sp, #66]
539; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
540; NONEON-NOSVE-NEXT:    add w8, w9, w8
541; NONEON-NOSVE-NEXT:    strh w8, [sp, #64]
542; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
543; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
544; NONEON-NOSVE-NEXT:    add sp, sp, #96
545; NONEON-NOSVE-NEXT:    ret
546  %op1 = load <16 x i16>, ptr %a
547  %op2 = load <16 x i16>, ptr %b
548  %res = add <16 x i16> %op1, %op2
549  store <16 x i16> %res, ptr %a
550  ret void
551}
552
553define <2 x i32> @add_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
554; CHECK-LABEL: add_v2i32:
555; CHECK:       // %bb.0:
556; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
557; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
558; CHECK-NEXT:    add z0.s, z0.s, z1.s
559; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
560; CHECK-NEXT:    ret
561;
562; NONEON-NOSVE-LABEL: add_v2i32:
563; NONEON-NOSVE:       // %bb.0:
564; NONEON-NOSVE-NEXT:    sub sp, sp, #32
565; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
566; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
567; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #8]
568; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
569; NONEON-NOSVE-NEXT:    add w8, w10, w8
570; NONEON-NOSVE-NEXT:    str w8, [sp, #28]
571; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
572; NONEON-NOSVE-NEXT:    add w8, w9, w8
573; NONEON-NOSVE-NEXT:    str w8, [sp, #24]
574; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
575; NONEON-NOSVE-NEXT:    add sp, sp, #32
576; NONEON-NOSVE-NEXT:    ret
577  %res = add <2 x i32> %op1, %op2
578  ret <2 x i32> %res
579}
580
581define <4 x i32> @add_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
582; CHECK-LABEL: add_v4i32:
583; CHECK:       // %bb.0:
584; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
585; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
586; CHECK-NEXT:    add z0.s, z0.s, z1.s
587; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
588; CHECK-NEXT:    ret
589;
590; NONEON-NOSVE-LABEL: add_v4i32:
591; NONEON-NOSVE:       // %bb.0:
592; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
593; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
594; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #8]
595; NONEON-NOSVE-NEXT:    ldr w8, [sp, #28]
596; NONEON-NOSVE-NEXT:    add w8, w10, w8
597; NONEON-NOSVE-NEXT:    str w8, [sp, #44]
598; NONEON-NOSVE-NEXT:    ldr w8, [sp, #24]
599; NONEON-NOSVE-NEXT:    add w8, w9, w8
600; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp]
601; NONEON-NOSVE-NEXT:    str w8, [sp, #40]
602; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
603; NONEON-NOSVE-NEXT:    add w8, w10, w8
604; NONEON-NOSVE-NEXT:    str w8, [sp, #36]
605; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
606; NONEON-NOSVE-NEXT:    add w8, w9, w8
607; NONEON-NOSVE-NEXT:    str w8, [sp, #32]
608; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
609; NONEON-NOSVE-NEXT:    add sp, sp, #48
610; NONEON-NOSVE-NEXT:    ret
611  %res = add <4 x i32> %op1, %op2
612  ret <4 x i32> %res
613}
614
615define void @add_v8i32(ptr %a, ptr %b) {
616; CHECK-LABEL: add_v8i32:
617; CHECK:       // %bb.0:
618; CHECK-NEXT:    ldp q0, q3, [x1]
619; CHECK-NEXT:    ldp q1, q2, [x0]
620; CHECK-NEXT:    add z0.s, z1.s, z0.s
621; CHECK-NEXT:    add z1.s, z2.s, z3.s
622; CHECK-NEXT:    stp q0, q1, [x0]
623; CHECK-NEXT:    ret
624;
625; NONEON-NOSVE-LABEL: add_v8i32:
626; NONEON-NOSVE:       // %bb.0:
627; NONEON-NOSVE-NEXT:    sub sp, sp, #96
628; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
629; NONEON-NOSVE-NEXT:    ldp q3, q0, [x1]
630; NONEON-NOSVE-NEXT:    ldp q2, q1, [x0]
631; NONEON-NOSVE-NEXT:    stp q2, q3, [sp]
632; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #32]
633; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #40]
634; NONEON-NOSVE-NEXT:    ldr w8, [sp, #60]
635; NONEON-NOSVE-NEXT:    add w8, w10, w8
636; NONEON-NOSVE-NEXT:    str w8, [sp, #92]
637; NONEON-NOSVE-NEXT:    ldr w8, [sp, #56]
638; NONEON-NOSVE-NEXT:    add w8, w9, w8
639; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #32]
640; NONEON-NOSVE-NEXT:    str w8, [sp, #88]
641; NONEON-NOSVE-NEXT:    ldr w8, [sp, #52]
642; NONEON-NOSVE-NEXT:    add w8, w10, w8
643; NONEON-NOSVE-NEXT:    str w8, [sp, #84]
644; NONEON-NOSVE-NEXT:    ldr w8, [sp, #48]
645; NONEON-NOSVE-NEXT:    add w8, w9, w8
646; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #8]
647; NONEON-NOSVE-NEXT:    str w8, [sp, #80]
648; NONEON-NOSVE-NEXT:    ldr w8, [sp, #28]
649; NONEON-NOSVE-NEXT:    add w8, w10, w8
650; NONEON-NOSVE-NEXT:    str w8, [sp, #76]
651; NONEON-NOSVE-NEXT:    ldr w8, [sp, #24]
652; NONEON-NOSVE-NEXT:    add w8, w9, w8
653; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp]
654; NONEON-NOSVE-NEXT:    str w8, [sp, #72]
655; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
656; NONEON-NOSVE-NEXT:    add w8, w10, w8
657; NONEON-NOSVE-NEXT:    str w8, [sp, #68]
658; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
659; NONEON-NOSVE-NEXT:    add w8, w9, w8
660; NONEON-NOSVE-NEXT:    str w8, [sp, #64]
661; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
662; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
663; NONEON-NOSVE-NEXT:    add sp, sp, #96
664; NONEON-NOSVE-NEXT:    ret
665  %op1 = load <8 x i32>, ptr %a
666  %op2 = load <8 x i32>, ptr %b
667  %res = add <8 x i32> %op1, %op2
668  store <8 x i32> %res, ptr %a
669  ret void
670}
671
672define <1 x i64> @add_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
673; CHECK-LABEL: add_v1i64:
674; CHECK:       // %bb.0:
675; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
676; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
677; CHECK-NEXT:    add z0.d, z0.d, z1.d
678; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
679; CHECK-NEXT:    ret
680;
681; NONEON-NOSVE-LABEL: add_v1i64:
682; NONEON-NOSVE:       // %bb.0:
683; NONEON-NOSVE-NEXT:    sub sp, sp, #16
684; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
685; NONEON-NOSVE-NEXT:    fmov x8, d1
686; NONEON-NOSVE-NEXT:    fmov x9, d0
687; NONEON-NOSVE-NEXT:    add x8, x9, x8
688; NONEON-NOSVE-NEXT:    str x8, [sp, #8]
689; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
690; NONEON-NOSVE-NEXT:    add sp, sp, #16
691; NONEON-NOSVE-NEXT:    ret
692  %res = add <1 x i64> %op1, %op2
693  ret <1 x i64> %res
694}
695
696define <2 x i64> @add_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
697; CHECK-LABEL: add_v2i64:
698; CHECK:       // %bb.0:
699; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
700; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
701; CHECK-NEXT:    add z0.d, z0.d, z1.d
702; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
703; CHECK-NEXT:    ret
704;
705; NONEON-NOSVE-LABEL: add_v2i64:
706; NONEON-NOSVE:       // %bb.0:
707; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
708; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
709; NONEON-NOSVE-NEXT:    ldp x9, x10, [sp]
710; NONEON-NOSVE-NEXT:    ldr x8, [sp, #24]
711; NONEON-NOSVE-NEXT:    add x8, x10, x8
712; NONEON-NOSVE-NEXT:    str x8, [sp, #40]
713; NONEON-NOSVE-NEXT:    ldr x8, [sp, #16]
714; NONEON-NOSVE-NEXT:    add x8, x9, x8
715; NONEON-NOSVE-NEXT:    str x8, [sp, #32]
716; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
717; NONEON-NOSVE-NEXT:    add sp, sp, #48
718; NONEON-NOSVE-NEXT:    ret
719  %res = add <2 x i64> %op1, %op2
720  ret <2 x i64> %res
721}
722
723define void @add_v4i64(ptr %a, ptr %b) {
724; CHECK-LABEL: add_v4i64:
725; CHECK:       // %bb.0:
726; CHECK-NEXT:    ldp q0, q3, [x1]
727; CHECK-NEXT:    ldp q1, q2, [x0]
728; CHECK-NEXT:    add z0.d, z1.d, z0.d
729; CHECK-NEXT:    add z1.d, z2.d, z3.d
730; CHECK-NEXT:    stp q0, q1, [x0]
731; CHECK-NEXT:    ret
732;
733; NONEON-NOSVE-LABEL: add_v4i64:
734; NONEON-NOSVE:       // %bb.0:
735; NONEON-NOSVE-NEXT:    sub sp, sp, #96
736; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
737; NONEON-NOSVE-NEXT:    ldp q3, q0, [x1]
738; NONEON-NOSVE-NEXT:    ldp q2, q1, [x0]
739; NONEON-NOSVE-NEXT:    stp q2, q3, [sp]
740; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #32]
741; NONEON-NOSVE-NEXT:    ldp x9, x10, [sp, #32]
742; NONEON-NOSVE-NEXT:    ldr x8, [sp, #56]
743; NONEON-NOSVE-NEXT:    add x8, x10, x8
744; NONEON-NOSVE-NEXT:    str x8, [sp, #88]
745; NONEON-NOSVE-NEXT:    ldr x8, [sp, #48]
746; NONEON-NOSVE-NEXT:    add x8, x9, x8
747; NONEON-NOSVE-NEXT:    ldp x9, x10, [sp]
748; NONEON-NOSVE-NEXT:    str x8, [sp, #80]
749; NONEON-NOSVE-NEXT:    ldr x8, [sp, #24]
750; NONEON-NOSVE-NEXT:    add x8, x10, x8
751; NONEON-NOSVE-NEXT:    str x8, [sp, #72]
752; NONEON-NOSVE-NEXT:    ldr x8, [sp, #16]
753; NONEON-NOSVE-NEXT:    add x8, x9, x8
754; NONEON-NOSVE-NEXT:    str x8, [sp, #64]
755; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
756; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
757; NONEON-NOSVE-NEXT:    add sp, sp, #96
758; NONEON-NOSVE-NEXT:    ret
759  %op1 = load <4 x i64>, ptr %a
760  %op2 = load <4 x i64>, ptr %b
761  %res = add <4 x i64> %op1, %op2
762  store <4 x i64> %res, ptr %a
763  ret void
764}
765
766;
767; MUL
768;
769
770define <4 x i8> @mul_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
771; SVE-LABEL: mul_v4i8:
772; SVE:       // %bb.0:
773; SVE-NEXT:    ptrue p0.h, vl4
774; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
775; SVE-NEXT:    // kill: def $d1 killed $d1 def $z1
776; SVE-NEXT:    mul z0.h, p0/m, z0.h, z1.h
777; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
778; SVE-NEXT:    ret
779;
780; SVE2-LABEL: mul_v4i8:
781; SVE2:       // %bb.0:
782; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
783; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
784; SVE2-NEXT:    mul z0.h, z0.h, z1.h
785; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
786; SVE2-NEXT:    ret
787;
788; NONEON-NOSVE-LABEL: mul_v4i8:
789; NONEON-NOSVE:       // %bb.0:
790; NONEON-NOSVE-NEXT:    sub sp, sp, #32
791; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
792; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
793; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
794; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
795; NONEON-NOSVE-NEXT:    mul w8, w9, w8
796; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]
797; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
798; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
799; NONEON-NOSVE-NEXT:    mul w8, w9, w8
800; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
801; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
802; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
803; NONEON-NOSVE-NEXT:    mul w8, w9, w8
804; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
805; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
806; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
807; NONEON-NOSVE-NEXT:    mul w8, w9, w8
808; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
809; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
810; NONEON-NOSVE-NEXT:    add sp, sp, #32
811; NONEON-NOSVE-NEXT:    ret
812  %res = mul <4 x i8> %op1, %op2
813  ret <4 x i8> %res
814}
815
816define <8 x i8> @mul_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
817; SVE-LABEL: mul_v8i8:
818; SVE:       // %bb.0:
819; SVE-NEXT:    ptrue p0.b, vl8
820; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
821; SVE-NEXT:    // kill: def $d1 killed $d1 def $z1
822; SVE-NEXT:    mul z0.b, p0/m, z0.b, z1.b
823; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
824; SVE-NEXT:    ret
825;
826; SVE2-LABEL: mul_v8i8:
827; SVE2:       // %bb.0:
828; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
829; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
830; SVE2-NEXT:    mul z0.b, z0.b, z1.b
831; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
832; SVE2-NEXT:    ret
833;
834; NONEON-NOSVE-LABEL: mul_v8i8:
835; NONEON-NOSVE:       // %bb.0:
836; NONEON-NOSVE-NEXT:    sub sp, sp, #32
837; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
838; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
839; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
840; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]
841; NONEON-NOSVE-NEXT:    mul w8, w9, w8
842; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
843; NONEON-NOSVE-NEXT:    strb w8, [sp, #31]
844; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
845; NONEON-NOSVE-NEXT:    mul w8, w9, w8
846; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]
847; NONEON-NOSVE-NEXT:    strb w8, [sp, #30]
848; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
849; NONEON-NOSVE-NEXT:    mul w8, w9, w8
850; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
851; NONEON-NOSVE-NEXT:    strb w8, [sp, #29]
852; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
853; NONEON-NOSVE-NEXT:    mul w8, w9, w8
854; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]
855; NONEON-NOSVE-NEXT:    strb w8, [sp, #28]
856; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
857; NONEON-NOSVE-NEXT:    mul w8, w9, w8
858; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
859; NONEON-NOSVE-NEXT:    strb w8, [sp, #27]
860; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
861; NONEON-NOSVE-NEXT:    mul w8, w9, w8
862; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
863; NONEON-NOSVE-NEXT:    strb w8, [sp, #26]
864; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
865; NONEON-NOSVE-NEXT:    mul w8, w9, w8
866; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]
867; NONEON-NOSVE-NEXT:    strb w8, [sp, #25]
868; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
869; NONEON-NOSVE-NEXT:    mul w8, w9, w8
870; NONEON-NOSVE-NEXT:    strb w8, [sp, #24]
871; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
872; NONEON-NOSVE-NEXT:    add sp, sp, #32
873; NONEON-NOSVE-NEXT:    ret
874  %res = mul <8 x i8> %op1, %op2
875  ret <8 x i8> %res
876}
877
878define <16 x i8> @mul_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
879; SVE-LABEL: mul_v16i8:
880; SVE:       // %bb.0:
881; SVE-NEXT:    ptrue p0.b, vl16
882; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
883; SVE-NEXT:    // kill: def $q1 killed $q1 def $z1
884; SVE-NEXT:    mul z0.b, p0/m, z0.b, z1.b
885; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
886; SVE-NEXT:    ret
887;
888; SVE2-LABEL: mul_v16i8:
889; SVE2:       // %bb.0:
890; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
891; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
892; SVE2-NEXT:    mul z0.b, z0.b, z1.b
893; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
894; SVE2-NEXT:    ret
895;
896; NONEON-NOSVE-LABEL: mul_v16i8:
897; NONEON-NOSVE:       // %bb.0:
898; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
899; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
900; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]
901; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]
902; NONEON-NOSVE-NEXT:    mul w8, w9, w8
903; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
904; NONEON-NOSVE-NEXT:    strb w8, [sp, #47]
905; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
906; NONEON-NOSVE-NEXT:    mul w8, w9, w8
907; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]
908; NONEON-NOSVE-NEXT:    strb w8, [sp, #46]
909; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]
910; NONEON-NOSVE-NEXT:    mul w8, w9, w8
911; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
912; NONEON-NOSVE-NEXT:    strb w8, [sp, #45]
913; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]
914; NONEON-NOSVE-NEXT:    mul w8, w9, w8
915; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]
916; NONEON-NOSVE-NEXT:    strb w8, [sp, #44]
917; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]
918; NONEON-NOSVE-NEXT:    mul w8, w9, w8
919; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
920; NONEON-NOSVE-NEXT:    strb w8, [sp, #43]
921; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]
922; NONEON-NOSVE-NEXT:    mul w8, w9, w8
923; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
924; NONEON-NOSVE-NEXT:    strb w8, [sp, #42]
925; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]
926; NONEON-NOSVE-NEXT:    mul w8, w9, w8
927; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]
928; NONEON-NOSVE-NEXT:    strb w8, [sp, #41]
929; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]
930; NONEON-NOSVE-NEXT:    mul w8, w9, w8
931; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]
932; NONEON-NOSVE-NEXT:    strb w8, [sp, #40]
933; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
934; NONEON-NOSVE-NEXT:    mul w8, w9, w8
935; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #6]
936; NONEON-NOSVE-NEXT:    strb w8, [sp, #39]
937; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
938; NONEON-NOSVE-NEXT:    mul w8, w9, w8
939; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #5]
940; NONEON-NOSVE-NEXT:    strb w8, [sp, #38]
941; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
942; NONEON-NOSVE-NEXT:    mul w8, w9, w8
943; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #4]
944; NONEON-NOSVE-NEXT:    strb w8, [sp, #37]
945; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
946; NONEON-NOSVE-NEXT:    mul w8, w9, w8
947; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]
948; NONEON-NOSVE-NEXT:    strb w8, [sp, #36]
949; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
950; NONEON-NOSVE-NEXT:    mul w8, w9, w8
951; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #2]
952; NONEON-NOSVE-NEXT:    strb w8, [sp, #35]
953; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
954; NONEON-NOSVE-NEXT:    mul w8, w9, w8
955; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1]
956; NONEON-NOSVE-NEXT:    strb w8, [sp, #34]
957; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
958; NONEON-NOSVE-NEXT:    mul w8, w9, w8
959; NONEON-NOSVE-NEXT:    ldrb w9, [sp]
960; NONEON-NOSVE-NEXT:    strb w8, [sp, #33]
961; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
962; NONEON-NOSVE-NEXT:    mul w8, w9, w8
963; NONEON-NOSVE-NEXT:    strb w8, [sp, #32]
964; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
965; NONEON-NOSVE-NEXT:    add sp, sp, #48
966; NONEON-NOSVE-NEXT:    ret
967  %res = mul <16 x i8> %op1, %op2
968  ret <16 x i8> %res
969}
970
971define void @mul_v32i8(ptr %a, ptr %b) {
972; SVE-LABEL: mul_v32i8:
973; SVE:       // %bb.0:
974; SVE-NEXT:    ldp q0, q3, [x1]
975; SVE-NEXT:    ptrue p0.b, vl16
976; SVE-NEXT:    ldp q1, q2, [x0]
977; SVE-NEXT:    mul z0.b, p0/m, z0.b, z1.b
978; SVE-NEXT:    movprfx z1, z2
979; SVE-NEXT:    mul z1.b, p0/m, z1.b, z3.b
980; SVE-NEXT:    stp q0, q1, [x0]
981; SVE-NEXT:    ret
982;
983; SVE2-LABEL: mul_v32i8:
984; SVE2:       // %bb.0:
985; SVE2-NEXT:    ldp q0, q3, [x1]
986; SVE2-NEXT:    ldp q1, q2, [x0]
987; SVE2-NEXT:    mul z0.b, z1.b, z0.b
988; SVE2-NEXT:    mul z1.b, z2.b, z3.b
989; SVE2-NEXT:    stp q0, q1, [x0]
990; SVE2-NEXT:    ret
991;
992; NONEON-NOSVE-LABEL: mul_v32i8:
993; NONEON-NOSVE:       // %bb.0:
994; NONEON-NOSVE-NEXT:    sub sp, sp, #96
995; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
996; NONEON-NOSVE-NEXT:    ldp q3, q0, [x1]
997; NONEON-NOSVE-NEXT:    ldp q2, q1, [x0]
998; NONEON-NOSVE-NEXT:    stp q2, q3, [sp]
999; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #32]
1000; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #63]
1001; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #47]
1002; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1003; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #46]
1004; NONEON-NOSVE-NEXT:    strb w8, [sp, #95]
1005; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #62]
1006; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1007; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #45]
1008; NONEON-NOSVE-NEXT:    strb w8, [sp, #94]
1009; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #61]
1010; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1011; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #44]
1012; NONEON-NOSVE-NEXT:    strb w8, [sp, #93]
1013; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #60]
1014; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1015; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #43]
1016; NONEON-NOSVE-NEXT:    strb w8, [sp, #92]
1017; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #59]
1018; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1019; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #42]
1020; NONEON-NOSVE-NEXT:    strb w8, [sp, #91]
1021; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #58]
1022; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1023; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #41]
1024; NONEON-NOSVE-NEXT:    strb w8, [sp, #90]
1025; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #57]
1026; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1027; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
1028; NONEON-NOSVE-NEXT:    strb w8, [sp, #89]
1029; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #56]
1030; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1031; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #39]
1032; NONEON-NOSVE-NEXT:    strb w8, [sp, #88]
1033; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #55]
1034; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1035; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #38]
1036; NONEON-NOSVE-NEXT:    strb w8, [sp, #87]
1037; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #54]
1038; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1039; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #37]
1040; NONEON-NOSVE-NEXT:    strb w8, [sp, #86]
1041; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #53]
1042; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1043; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #36]
1044; NONEON-NOSVE-NEXT:    strb w8, [sp, #85]
1045; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #52]
1046; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1047; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #35]
1048; NONEON-NOSVE-NEXT:    strb w8, [sp, #84]
1049; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #51]
1050; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1051; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #34]
1052; NONEON-NOSVE-NEXT:    strb w8, [sp, #83]
1053; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #50]
1054; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1055; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #33]
1056; NONEON-NOSVE-NEXT:    strb w8, [sp, #82]
1057; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #49]
1058; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1059; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #32]
1060; NONEON-NOSVE-NEXT:    strb w8, [sp, #81]
1061; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #48]
1062; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1063; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]
1064; NONEON-NOSVE-NEXT:    strb w8, [sp, #80]
1065; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]
1066; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1067; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
1068; NONEON-NOSVE-NEXT:    strb w8, [sp, #79]
1069; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
1070; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1071; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]
1072; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
1073; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]
1074; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1075; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
1076; NONEON-NOSVE-NEXT:    strb w8, [sp, #77]
1077; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]
1078; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1079; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]
1080; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
1081; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]
1082; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1083; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
1084; NONEON-NOSVE-NEXT:    strb w8, [sp, #75]
1085; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]
1086; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1087; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
1088; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
1089; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]
1090; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1091; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]
1092; NONEON-NOSVE-NEXT:    strb w8, [sp, #73]
1093; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]
1094; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1095; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]
1096; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
1097; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
1098; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1099; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #6]
1100; NONEON-NOSVE-NEXT:    strb w8, [sp, #71]
1101; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
1102; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1103; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #5]
1104; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
1105; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
1106; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1107; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #4]
1108; NONEON-NOSVE-NEXT:    strb w8, [sp, #69]
1109; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
1110; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1111; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]
1112; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
1113; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
1114; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1115; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #2]
1116; NONEON-NOSVE-NEXT:    strb w8, [sp, #67]
1117; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
1118; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1119; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1]
1120; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
1121; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
1122; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1123; NONEON-NOSVE-NEXT:    ldrb w9, [sp]
1124; NONEON-NOSVE-NEXT:    strb w8, [sp, #65]
1125; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
1126; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1127; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
1128; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
1129; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
1130; NONEON-NOSVE-NEXT:    add sp, sp, #96
1131; NONEON-NOSVE-NEXT:    ret
1132  %op1 = load <32 x i8>, ptr %a
1133  %op2 = load <32 x i8>, ptr %b
1134  %res = mul <32 x i8> %op1, %op2
1135  store <32 x i8> %res, ptr %a
1136  ret void
1137}
1138
1139define <2 x i16> @mul_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
1140; SVE-LABEL: mul_v2i16:
1141; SVE:       // %bb.0:
1142; SVE-NEXT:    ptrue p0.s, vl2
1143; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
1144; SVE-NEXT:    // kill: def $d1 killed $d1 def $z1
1145; SVE-NEXT:    mul z0.s, p0/m, z0.s, z1.s
1146; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
1147; SVE-NEXT:    ret
1148;
1149; SVE2-LABEL: mul_v2i16:
1150; SVE2:       // %bb.0:
1151; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
1152; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
1153; SVE2-NEXT:    mul z0.s, z0.s, z1.s
1154; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
1155; SVE2-NEXT:    ret
1156;
1157; NONEON-NOSVE-LABEL: mul_v2i16:
1158; NONEON-NOSVE:       // %bb.0:
1159; NONEON-NOSVE-NEXT:    sub sp, sp, #32
1160; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
1161; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
1162; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #8]
1163; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
1164; NONEON-NOSVE-NEXT:    mul w11, w10, w8
1165; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
1166; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1167; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #24]
1168; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
1169; NONEON-NOSVE-NEXT:    add sp, sp, #32
1170; NONEON-NOSVE-NEXT:    ret
1171  %res = mul <2 x i16> %op1, %op2
1172  ret <2 x i16> %res
1173}
1174
1175define <4 x i16> @mul_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
1176; SVE-LABEL: mul_v4i16:
1177; SVE:       // %bb.0:
1178; SVE-NEXT:    ptrue p0.h, vl4
1179; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
1180; SVE-NEXT:    // kill: def $d1 killed $d1 def $z1
1181; SVE-NEXT:    mul z0.h, p0/m, z0.h, z1.h
1182; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
1183; SVE-NEXT:    ret
1184;
1185; SVE2-LABEL: mul_v4i16:
1186; SVE2:       // %bb.0:
1187; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
1188; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
1189; SVE2-NEXT:    mul z0.h, z0.h, z1.h
1190; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
1191; SVE2-NEXT:    ret
1192;
1193; NONEON-NOSVE-LABEL: mul_v4i16:
1194; NONEON-NOSVE:       // %bb.0:
1195; NONEON-NOSVE-NEXT:    sub sp, sp, #32
1196; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
1197; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
1198; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
1199; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
1200; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1201; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]
1202; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
1203; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
1204; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1205; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
1206; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
1207; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
1208; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1209; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
1210; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
1211; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
1212; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1213; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
1214; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
1215; NONEON-NOSVE-NEXT:    add sp, sp, #32
1216; NONEON-NOSVE-NEXT:    ret
1217  %res = mul <4 x i16> %op1, %op2
1218  ret <4 x i16> %res
1219}
1220
1221define <8 x i16> @mul_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
1222; SVE-LABEL: mul_v8i16:
1223; SVE:       // %bb.0:
1224; SVE-NEXT:    ptrue p0.h, vl8
1225; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
1226; SVE-NEXT:    // kill: def $q1 killed $q1 def $z1
1227; SVE-NEXT:    mul z0.h, p0/m, z0.h, z1.h
1228; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
1229; SVE-NEXT:    ret
1230;
1231; SVE2-LABEL: mul_v8i16:
1232; SVE2:       // %bb.0:
1233; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
1234; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
1235; SVE2-NEXT:    mul z0.h, z0.h, z1.h
1236; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
1237; SVE2-NEXT:    ret
1238;
1239; NONEON-NOSVE-LABEL: mul_v8i16:
1240; NONEON-NOSVE:       // %bb.0:
1241; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
1242; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
1243; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #30]
1244; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
1245; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1246; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]
1247; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
1248; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
1249; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1250; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
1251; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
1252; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #26]
1253; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1254; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
1255; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
1256; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
1257; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1258; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #6]
1259; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
1260; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
1261; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1262; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]
1263; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]
1264; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
1265; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1266; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #2]
1267; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]
1268; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
1269; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1270; NONEON-NOSVE-NEXT:    ldrh w9, [sp]
1271; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]
1272; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
1273; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1274; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
1275; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
1276; NONEON-NOSVE-NEXT:    add sp, sp, #48
1277; NONEON-NOSVE-NEXT:    ret
1278  %res = mul <8 x i16> %op1, %op2
1279  ret <8 x i16> %res
1280}
1281
1282define void @mul_v16i16(ptr %a, ptr %b) {
1283; SVE-LABEL: mul_v16i16:
1284; SVE:       // %bb.0:
1285; SVE-NEXT:    ldp q0, q3, [x1]
1286; SVE-NEXT:    ptrue p0.h, vl8
1287; SVE-NEXT:    ldp q1, q2, [x0]
1288; SVE-NEXT:    mul z0.h, p0/m, z0.h, z1.h
1289; SVE-NEXT:    movprfx z1, z2
1290; SVE-NEXT:    mul z1.h, p0/m, z1.h, z3.h
1291; SVE-NEXT:    stp q0, q1, [x0]
1292; SVE-NEXT:    ret
1293;
1294; SVE2-LABEL: mul_v16i16:
1295; SVE2:       // %bb.0:
1296; SVE2-NEXT:    ldp q0, q3, [x1]
1297; SVE2-NEXT:    ldp q1, q2, [x0]
1298; SVE2-NEXT:    mul z0.h, z1.h, z0.h
1299; SVE2-NEXT:    mul z1.h, z2.h, z3.h
1300; SVE2-NEXT:    stp q0, q1, [x0]
1301; SVE2-NEXT:    ret
1302;
1303; NONEON-NOSVE-LABEL: mul_v16i16:
1304; NONEON-NOSVE:       // %bb.0:
1305; NONEON-NOSVE-NEXT:    sub sp, sp, #96
1306; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
1307; NONEON-NOSVE-NEXT:    ldp q3, q0, [x1]
1308; NONEON-NOSVE-NEXT:    ldp q2, q1, [x0]
1309; NONEON-NOSVE-NEXT:    stp q2, q3, [sp]
1310; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #32]
1311; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #62]
1312; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #46]
1313; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1314; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #44]
1315; NONEON-NOSVE-NEXT:    strh w8, [sp, #94]
1316; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #60]
1317; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1318; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #42]
1319; NONEON-NOSVE-NEXT:    strh w8, [sp, #92]
1320; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #58]
1321; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1322; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #40]
1323; NONEON-NOSVE-NEXT:    strh w8, [sp, #90]
1324; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #56]
1325; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1326; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #38]
1327; NONEON-NOSVE-NEXT:    strh w8, [sp, #88]
1328; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #54]
1329; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1330; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #36]
1331; NONEON-NOSVE-NEXT:    strh w8, [sp, #86]
1332; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #52]
1333; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1334; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #34]
1335; NONEON-NOSVE-NEXT:    strh w8, [sp, #84]
1336; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #50]
1337; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1338; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #32]
1339; NONEON-NOSVE-NEXT:    strh w8, [sp, #82]
1340; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #48]
1341; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1342; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
1343; NONEON-NOSVE-NEXT:    strh w8, [sp, #80]
1344; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #30]
1345; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1346; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]
1347; NONEON-NOSVE-NEXT:    strh w8, [sp, #78]
1348; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
1349; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1350; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
1351; NONEON-NOSVE-NEXT:    strh w8, [sp, #76]
1352; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #26]
1353; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1354; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
1355; NONEON-NOSVE-NEXT:    strh w8, [sp, #74]
1356; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
1357; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1358; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #6]
1359; NONEON-NOSVE-NEXT:    strh w8, [sp, #72]
1360; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
1361; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1362; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]
1363; NONEON-NOSVE-NEXT:    strh w8, [sp, #70]
1364; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
1365; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1366; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #2]
1367; NONEON-NOSVE-NEXT:    strh w8, [sp, #68]
1368; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
1369; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1370; NONEON-NOSVE-NEXT:    ldrh w9, [sp]
1371; NONEON-NOSVE-NEXT:    strh w8, [sp, #66]
1372; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
1373; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1374; NONEON-NOSVE-NEXT:    strh w8, [sp, #64]
1375; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
1376; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
1377; NONEON-NOSVE-NEXT:    add sp, sp, #96
1378; NONEON-NOSVE-NEXT:    ret
1379  %op1 = load <16 x i16>, ptr %a
1380  %op2 = load <16 x i16>, ptr %b
1381  %res = mul <16 x i16> %op1, %op2
1382  store <16 x i16> %res, ptr %a
1383  ret void
1384}
1385
1386define <2 x i32> @mul_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
1387; SVE-LABEL: mul_v2i32:
1388; SVE:       // %bb.0:
1389; SVE-NEXT:    ptrue p0.s, vl2
1390; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
1391; SVE-NEXT:    // kill: def $d1 killed $d1 def $z1
1392; SVE-NEXT:    mul z0.s, p0/m, z0.s, z1.s
1393; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
1394; SVE-NEXT:    ret
1395;
1396; SVE2-LABEL: mul_v2i32:
1397; SVE2:       // %bb.0:
1398; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
1399; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
1400; SVE2-NEXT:    mul z0.s, z0.s, z1.s
1401; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
1402; SVE2-NEXT:    ret
1403;
1404; NONEON-NOSVE-LABEL: mul_v2i32:
1405; NONEON-NOSVE:       // %bb.0:
1406; NONEON-NOSVE-NEXT:    sub sp, sp, #32
1407; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
1408; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
1409; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #8]
1410; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
1411; NONEON-NOSVE-NEXT:    mul w11, w10, w8
1412; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
1413; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1414; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #24]
1415; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
1416; NONEON-NOSVE-NEXT:    add sp, sp, #32
1417; NONEON-NOSVE-NEXT:    ret
1418  %res = mul <2 x i32> %op1, %op2
1419  ret <2 x i32> %res
1420}
1421
1422define <4 x i32> @mul_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
1423; SVE-LABEL: mul_v4i32:
1424; SVE:       // %bb.0:
1425; SVE-NEXT:    ptrue p0.s, vl4
1426; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
1427; SVE-NEXT:    // kill: def $q1 killed $q1 def $z1
1428; SVE-NEXT:    mul z0.s, p0/m, z0.s, z1.s
1429; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
1430; SVE-NEXT:    ret
1431;
1432; SVE2-LABEL: mul_v4i32:
1433; SVE2:       // %bb.0:
1434; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
1435; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
1436; SVE2-NEXT:    mul z0.s, z0.s, z1.s
1437; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
1438; SVE2-NEXT:    ret
1439;
1440; NONEON-NOSVE-LABEL: mul_v4i32:
1441; NONEON-NOSVE:       // %bb.0:
1442; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
1443; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
1444; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #8]
1445; NONEON-NOSVE-NEXT:    ldr w8, [sp, #28]
1446; NONEON-NOSVE-NEXT:    mul w11, w10, w8
1447; NONEON-NOSVE-NEXT:    ldr w8, [sp, #24]
1448; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1449; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp]
1450; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #40]
1451; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
1452; NONEON-NOSVE-NEXT:    mul w11, w10, w8
1453; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
1454; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1455; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #32]
1456; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
1457; NONEON-NOSVE-NEXT:    add sp, sp, #48
1458; NONEON-NOSVE-NEXT:    ret
1459  %res = mul <4 x i32> %op1, %op2
1460  ret <4 x i32> %res
1461}
1462
1463define void @mul_v8i32(ptr %a, ptr %b) {
1464; SVE-LABEL: mul_v8i32:
1465; SVE:       // %bb.0:
1466; SVE-NEXT:    ldp q0, q3, [x1]
1467; SVE-NEXT:    ptrue p0.s, vl4
1468; SVE-NEXT:    ldp q1, q2, [x0]
1469; SVE-NEXT:    mul z0.s, p0/m, z0.s, z1.s
1470; SVE-NEXT:    movprfx z1, z2
1471; SVE-NEXT:    mul z1.s, p0/m, z1.s, z3.s
1472; SVE-NEXT:    stp q0, q1, [x0]
1473; SVE-NEXT:    ret
1474;
1475; SVE2-LABEL: mul_v8i32:
1476; SVE2:       // %bb.0:
1477; SVE2-NEXT:    ldp q0, q3, [x1]
1478; SVE2-NEXT:    ldp q1, q2, [x0]
1479; SVE2-NEXT:    mul z0.s, z1.s, z0.s
1480; SVE2-NEXT:    mul z1.s, z2.s, z3.s
1481; SVE2-NEXT:    stp q0, q1, [x0]
1482; SVE2-NEXT:    ret
1483;
1484; NONEON-NOSVE-LABEL: mul_v8i32:
1485; NONEON-NOSVE:       // %bb.0:
1486; NONEON-NOSVE-NEXT:    sub sp, sp, #96
1487; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
1488; NONEON-NOSVE-NEXT:    ldp q3, q0, [x1]
1489; NONEON-NOSVE-NEXT:    ldp q2, q1, [x0]
1490; NONEON-NOSVE-NEXT:    stp q2, q3, [sp]
1491; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #32]
1492; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #40]
1493; NONEON-NOSVE-NEXT:    ldr w8, [sp, #60]
1494; NONEON-NOSVE-NEXT:    mul w11, w10, w8
1495; NONEON-NOSVE-NEXT:    ldr w8, [sp, #56]
1496; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1497; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #32]
1498; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #88]
1499; NONEON-NOSVE-NEXT:    ldr w8, [sp, #52]
1500; NONEON-NOSVE-NEXT:    mul w11, w10, w8
1501; NONEON-NOSVE-NEXT:    ldr w8, [sp, #48]
1502; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1503; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #8]
1504; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #80]
1505; NONEON-NOSVE-NEXT:    ldr w8, [sp, #28]
1506; NONEON-NOSVE-NEXT:    mul w11, w10, w8
1507; NONEON-NOSVE-NEXT:    ldr w8, [sp, #24]
1508; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1509; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp]
1510; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #72]
1511; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
1512; NONEON-NOSVE-NEXT:    mul w11, w10, w8
1513; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
1514; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1515; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #64]
1516; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
1517; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
1518; NONEON-NOSVE-NEXT:    add sp, sp, #96
1519; NONEON-NOSVE-NEXT:    ret
1520  %op1 = load <8 x i32>, ptr %a
1521  %op2 = load <8 x i32>, ptr %b
1522  %res = mul <8 x i32> %op1, %op2
1523  store <8 x i32> %res, ptr %a
1524  ret void
1525}
1526
1527define <1 x i64> @mul_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
1528; SVE-LABEL: mul_v1i64:
1529; SVE:       // %bb.0:
1530; SVE-NEXT:    ptrue p0.d, vl1
1531; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
1532; SVE-NEXT:    // kill: def $d1 killed $d1 def $z1
1533; SVE-NEXT:    mul z0.d, p0/m, z0.d, z1.d
1534; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
1535; SVE-NEXT:    ret
1536;
1537; SVE2-LABEL: mul_v1i64:
1538; SVE2:       // %bb.0:
1539; SVE2-NEXT:    // kill: def $d0 killed $d0 def $z0
1540; SVE2-NEXT:    // kill: def $d1 killed $d1 def $z1
1541; SVE2-NEXT:    mul z0.d, z0.d, z1.d
1542; SVE2-NEXT:    // kill: def $d0 killed $d0 killed $z0
1543; SVE2-NEXT:    ret
1544;
1545; NONEON-NOSVE-LABEL: mul_v1i64:
1546; NONEON-NOSVE:       // %bb.0:
1547; NONEON-NOSVE-NEXT:    sub sp, sp, #16
1548; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
1549; NONEON-NOSVE-NEXT:    fmov x8, d1
1550; NONEON-NOSVE-NEXT:    fmov x9, d0
1551; NONEON-NOSVE-NEXT:    mul x8, x9, x8
1552; NONEON-NOSVE-NEXT:    str x8, [sp, #8]
1553; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
1554; NONEON-NOSVE-NEXT:    add sp, sp, #16
1555; NONEON-NOSVE-NEXT:    ret
1556  %res = mul <1 x i64> %op1, %op2
1557  ret <1 x i64> %res
1558}
1559
1560define <2 x i64> @mul_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
1561; SVE-LABEL: mul_v2i64:
1562; SVE:       // %bb.0:
1563; SVE-NEXT:    ptrue p0.d, vl2
1564; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
1565; SVE-NEXT:    // kill: def $q1 killed $q1 def $z1
1566; SVE-NEXT:    mul z0.d, p0/m, z0.d, z1.d
1567; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
1568; SVE-NEXT:    ret
1569;
1570; SVE2-LABEL: mul_v2i64:
1571; SVE2:       // %bb.0:
1572; SVE2-NEXT:    // kill: def $q0 killed $q0 def $z0
1573; SVE2-NEXT:    // kill: def $q1 killed $q1 def $z1
1574; SVE2-NEXT:    mul z0.d, z0.d, z1.d
1575; SVE2-NEXT:    // kill: def $q0 killed $q0 killed $z0
1576; SVE2-NEXT:    ret
1577;
1578; NONEON-NOSVE-LABEL: mul_v2i64:
1579; NONEON-NOSVE:       // %bb.0:
1580; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
1581; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
1582; NONEON-NOSVE-NEXT:    ldp x9, x10, [sp]
1583; NONEON-NOSVE-NEXT:    ldr x8, [sp, #24]
1584; NONEON-NOSVE-NEXT:    mul x11, x10, x8
1585; NONEON-NOSVE-NEXT:    ldr x8, [sp, #16]
1586; NONEON-NOSVE-NEXT:    mul x8, x9, x8
1587; NONEON-NOSVE-NEXT:    stp x8, x11, [sp, #32]
1588; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
1589; NONEON-NOSVE-NEXT:    add sp, sp, #48
1590; NONEON-NOSVE-NEXT:    ret
1591  %res = mul <2 x i64> %op1, %op2
1592  ret <2 x i64> %res
1593}
1594
1595define void @mul_v4i64(ptr %a, ptr %b) {
1596; SVE-LABEL: mul_v4i64:
1597; SVE:       // %bb.0:
1598; SVE-NEXT:    ldp q0, q3, [x1]
1599; SVE-NEXT:    ptrue p0.d, vl2
1600; SVE-NEXT:    ldp q1, q2, [x0]
1601; SVE-NEXT:    mul z0.d, p0/m, z0.d, z1.d
1602; SVE-NEXT:    movprfx z1, z2
1603; SVE-NEXT:    mul z1.d, p0/m, z1.d, z3.d
1604; SVE-NEXT:    stp q0, q1, [x0]
1605; SVE-NEXT:    ret
1606;
1607; SVE2-LABEL: mul_v4i64:
1608; SVE2:       // %bb.0:
1609; SVE2-NEXT:    ldp q0, q3, [x1]
1610; SVE2-NEXT:    ldp q1, q2, [x0]
1611; SVE2-NEXT:    mul z0.d, z1.d, z0.d
1612; SVE2-NEXT:    mul z1.d, z2.d, z3.d
1613; SVE2-NEXT:    stp q0, q1, [x0]
1614; SVE2-NEXT:    ret
1615;
1616; NONEON-NOSVE-LABEL: mul_v4i64:
1617; NONEON-NOSVE:       // %bb.0:
1618; NONEON-NOSVE-NEXT:    sub sp, sp, #96
1619; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
1620; NONEON-NOSVE-NEXT:    ldp q3, q0, [x1]
1621; NONEON-NOSVE-NEXT:    ldp q2, q1, [x0]
1622; NONEON-NOSVE-NEXT:    stp q2, q3, [sp]
1623; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #32]
1624; NONEON-NOSVE-NEXT:    ldp x9, x10, [sp, #32]
1625; NONEON-NOSVE-NEXT:    ldr x8, [sp, #56]
1626; NONEON-NOSVE-NEXT:    mul x11, x10, x8
1627; NONEON-NOSVE-NEXT:    ldr x8, [sp, #48]
1628; NONEON-NOSVE-NEXT:    mul x8, x9, x8
1629; NONEON-NOSVE-NEXT:    ldp x9, x10, [sp]
1630; NONEON-NOSVE-NEXT:    stp x8, x11, [sp, #80]
1631; NONEON-NOSVE-NEXT:    ldr x8, [sp, #24]
1632; NONEON-NOSVE-NEXT:    mul x11, x10, x8
1633; NONEON-NOSVE-NEXT:    ldr x8, [sp, #16]
1634; NONEON-NOSVE-NEXT:    mul x8, x9, x8
1635; NONEON-NOSVE-NEXT:    stp x8, x11, [sp, #64]
1636; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
1637; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
1638; NONEON-NOSVE-NEXT:    add sp, sp, #96
1639; NONEON-NOSVE-NEXT:    ret
1640  %op1 = load <4 x i64>, ptr %a
1641  %op2 = load <4 x i64>, ptr %b
1642  %res = mul <4 x i64> %op1, %op2
1643  store <4 x i64> %res, ptr %a
1644  ret void
1645}
1646
1647;
1648; SUB
1649;
1650
1651define <4 x i8> @sub_v4i8(<4 x i8> %op1, <4 x i8> %op2) {
1652; CHECK-LABEL: sub_v4i8:
1653; CHECK:       // %bb.0:
1654; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
1655; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
1656; CHECK-NEXT:    sub z0.h, z0.h, z1.h
1657; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
1658; CHECK-NEXT:    ret
1659;
1660; NONEON-NOSVE-LABEL: sub_v4i8:
1661; NONEON-NOSVE:       // %bb.0:
1662; NONEON-NOSVE-NEXT:    sub sp, sp, #32
1663; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
1664; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
1665; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
1666; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
1667; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1668; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]
1669; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
1670; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
1671; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1672; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
1673; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
1674; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
1675; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1676; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
1677; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
1678; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
1679; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1680; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
1681; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
1682; NONEON-NOSVE-NEXT:    add sp, sp, #32
1683; NONEON-NOSVE-NEXT:    ret
1684  %res = sub <4 x i8> %op1, %op2
1685  ret <4 x i8> %res
1686}
1687
1688define <8 x i8> @sub_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
1689; CHECK-LABEL: sub_v8i8:
1690; CHECK:       // %bb.0:
1691; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
1692; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
1693; CHECK-NEXT:    sub z0.b, z0.b, z1.b
1694; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
1695; CHECK-NEXT:    ret
1696;
1697; NONEON-NOSVE-LABEL: sub_v8i8:
1698; NONEON-NOSVE:       // %bb.0:
1699; NONEON-NOSVE-NEXT:    sub sp, sp, #32
1700; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
1701; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
1702; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
1703; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]
1704; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1705; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
1706; NONEON-NOSVE-NEXT:    strb w8, [sp, #31]
1707; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
1708; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1709; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]
1710; NONEON-NOSVE-NEXT:    strb w8, [sp, #30]
1711; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
1712; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1713; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
1714; NONEON-NOSVE-NEXT:    strb w8, [sp, #29]
1715; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
1716; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1717; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]
1718; NONEON-NOSVE-NEXT:    strb w8, [sp, #28]
1719; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
1720; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1721; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
1722; NONEON-NOSVE-NEXT:    strb w8, [sp, #27]
1723; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
1724; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1725; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
1726; NONEON-NOSVE-NEXT:    strb w8, [sp, #26]
1727; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
1728; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1729; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]
1730; NONEON-NOSVE-NEXT:    strb w8, [sp, #25]
1731; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
1732; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1733; NONEON-NOSVE-NEXT:    strb w8, [sp, #24]
1734; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
1735; NONEON-NOSVE-NEXT:    add sp, sp, #32
1736; NONEON-NOSVE-NEXT:    ret
1737  %res = sub <8 x i8> %op1, %op2
1738  ret <8 x i8> %res
1739}
1740
1741define <16 x i8> @sub_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
1742; CHECK-LABEL: sub_v16i8:
1743; CHECK:       // %bb.0:
1744; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
1745; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
1746; CHECK-NEXT:    sub z0.b, z0.b, z1.b
1747; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
1748; CHECK-NEXT:    ret
1749;
1750; NONEON-NOSVE-LABEL: sub_v16i8:
1751; NONEON-NOSVE:       // %bb.0:
1752; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
1753; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
1754; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]
1755; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]
1756; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1757; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
1758; NONEON-NOSVE-NEXT:    strb w8, [sp, #47]
1759; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
1760; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1761; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]
1762; NONEON-NOSVE-NEXT:    strb w8, [sp, #46]
1763; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]
1764; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1765; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
1766; NONEON-NOSVE-NEXT:    strb w8, [sp, #45]
1767; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]
1768; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1769; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]
1770; NONEON-NOSVE-NEXT:    strb w8, [sp, #44]
1771; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]
1772; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1773; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
1774; NONEON-NOSVE-NEXT:    strb w8, [sp, #43]
1775; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]
1776; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1777; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
1778; NONEON-NOSVE-NEXT:    strb w8, [sp, #42]
1779; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]
1780; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1781; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]
1782; NONEON-NOSVE-NEXT:    strb w8, [sp, #41]
1783; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]
1784; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1785; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]
1786; NONEON-NOSVE-NEXT:    strb w8, [sp, #40]
1787; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
1788; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1789; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #6]
1790; NONEON-NOSVE-NEXT:    strb w8, [sp, #39]
1791; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
1792; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1793; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #5]
1794; NONEON-NOSVE-NEXT:    strb w8, [sp, #38]
1795; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
1796; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1797; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #4]
1798; NONEON-NOSVE-NEXT:    strb w8, [sp, #37]
1799; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
1800; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1801; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]
1802; NONEON-NOSVE-NEXT:    strb w8, [sp, #36]
1803; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
1804; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1805; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #2]
1806; NONEON-NOSVE-NEXT:    strb w8, [sp, #35]
1807; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
1808; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1809; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1]
1810; NONEON-NOSVE-NEXT:    strb w8, [sp, #34]
1811; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
1812; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1813; NONEON-NOSVE-NEXT:    ldrb w9, [sp]
1814; NONEON-NOSVE-NEXT:    strb w8, [sp, #33]
1815; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
1816; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1817; NONEON-NOSVE-NEXT:    strb w8, [sp, #32]
1818; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
1819; NONEON-NOSVE-NEXT:    add sp, sp, #48
1820; NONEON-NOSVE-NEXT:    ret
1821  %res = sub <16 x i8> %op1, %op2
1822  ret <16 x i8> %res
1823}
1824
1825define void @sub_v32i8(ptr %a, ptr %b) {
1826; CHECK-LABEL: sub_v32i8:
1827; CHECK:       // %bb.0:
1828; CHECK-NEXT:    ldp q0, q3, [x1]
1829; CHECK-NEXT:    ldp q1, q2, [x0]
1830; CHECK-NEXT:    sub z0.b, z1.b, z0.b
1831; CHECK-NEXT:    sub z1.b, z2.b, z3.b
1832; CHECK-NEXT:    stp q0, q1, [x0]
1833; CHECK-NEXT:    ret
1834;
1835; NONEON-NOSVE-LABEL: sub_v32i8:
1836; NONEON-NOSVE:       // %bb.0:
1837; NONEON-NOSVE-NEXT:    sub sp, sp, #96
1838; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
1839; NONEON-NOSVE-NEXT:    ldp q3, q0, [x1]
1840; NONEON-NOSVE-NEXT:    ldp q2, q1, [x0]
1841; NONEON-NOSVE-NEXT:    stp q2, q3, [sp]
1842; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #32]
1843; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #63]
1844; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #47]
1845; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1846; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #46]
1847; NONEON-NOSVE-NEXT:    strb w8, [sp, #95]
1848; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #62]
1849; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1850; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #45]
1851; NONEON-NOSVE-NEXT:    strb w8, [sp, #94]
1852; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #61]
1853; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1854; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #44]
1855; NONEON-NOSVE-NEXT:    strb w8, [sp, #93]
1856; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #60]
1857; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1858; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #43]
1859; NONEON-NOSVE-NEXT:    strb w8, [sp, #92]
1860; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #59]
1861; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1862; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #42]
1863; NONEON-NOSVE-NEXT:    strb w8, [sp, #91]
1864; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #58]
1865; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1866; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #41]
1867; NONEON-NOSVE-NEXT:    strb w8, [sp, #90]
1868; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #57]
1869; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1870; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
1871; NONEON-NOSVE-NEXT:    strb w8, [sp, #89]
1872; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #56]
1873; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1874; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #39]
1875; NONEON-NOSVE-NEXT:    strb w8, [sp, #88]
1876; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #55]
1877; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1878; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #38]
1879; NONEON-NOSVE-NEXT:    strb w8, [sp, #87]
1880; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #54]
1881; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1882; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #37]
1883; NONEON-NOSVE-NEXT:    strb w8, [sp, #86]
1884; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #53]
1885; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1886; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #36]
1887; NONEON-NOSVE-NEXT:    strb w8, [sp, #85]
1888; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #52]
1889; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1890; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #35]
1891; NONEON-NOSVE-NEXT:    strb w8, [sp, #84]
1892; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #51]
1893; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1894; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #34]
1895; NONEON-NOSVE-NEXT:    strb w8, [sp, #83]
1896; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #50]
1897; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1898; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #33]
1899; NONEON-NOSVE-NEXT:    strb w8, [sp, #82]
1900; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #49]
1901; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1902; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #32]
1903; NONEON-NOSVE-NEXT:    strb w8, [sp, #81]
1904; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #48]
1905; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1906; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]
1907; NONEON-NOSVE-NEXT:    strb w8, [sp, #80]
1908; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]
1909; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1910; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
1911; NONEON-NOSVE-NEXT:    strb w8, [sp, #79]
1912; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
1913; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1914; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]
1915; NONEON-NOSVE-NEXT:    strb w8, [sp, #78]
1916; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]
1917; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1918; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
1919; NONEON-NOSVE-NEXT:    strb w8, [sp, #77]
1920; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]
1921; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1922; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]
1923; NONEON-NOSVE-NEXT:    strb w8, [sp, #76]
1924; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]
1925; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1926; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
1927; NONEON-NOSVE-NEXT:    strb w8, [sp, #75]
1928; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]
1929; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1930; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
1931; NONEON-NOSVE-NEXT:    strb w8, [sp, #74]
1932; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]
1933; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1934; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]
1935; NONEON-NOSVE-NEXT:    strb w8, [sp, #73]
1936; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]
1937; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1938; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]
1939; NONEON-NOSVE-NEXT:    strb w8, [sp, #72]
1940; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
1941; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1942; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #6]
1943; NONEON-NOSVE-NEXT:    strb w8, [sp, #71]
1944; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
1945; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1946; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #5]
1947; NONEON-NOSVE-NEXT:    strb w8, [sp, #70]
1948; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
1949; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1950; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #4]
1951; NONEON-NOSVE-NEXT:    strb w8, [sp, #69]
1952; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
1953; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1954; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]
1955; NONEON-NOSVE-NEXT:    strb w8, [sp, #68]
1956; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
1957; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1958; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #2]
1959; NONEON-NOSVE-NEXT:    strb w8, [sp, #67]
1960; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
1961; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1962; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1]
1963; NONEON-NOSVE-NEXT:    strb w8, [sp, #66]
1964; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
1965; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1966; NONEON-NOSVE-NEXT:    ldrb w9, [sp]
1967; NONEON-NOSVE-NEXT:    strb w8, [sp, #65]
1968; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
1969; NONEON-NOSVE-NEXT:    sub w8, w9, w8
1970; NONEON-NOSVE-NEXT:    strb w8, [sp, #64]
1971; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
1972; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
1973; NONEON-NOSVE-NEXT:    add sp, sp, #96
1974; NONEON-NOSVE-NEXT:    ret
1975  %op1 = load <32 x i8>, ptr %a
1976  %op2 = load <32 x i8>, ptr %b
1977  %res = sub <32 x i8> %op1, %op2
1978  store <32 x i8> %res, ptr %a
1979  ret void
1980}
1981
1982define <2 x i16> @sub_v2i16(<2 x i16> %op1, <2 x i16> %op2) {
1983; CHECK-LABEL: sub_v2i16:
1984; CHECK:       // %bb.0:
1985; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
1986; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
1987; CHECK-NEXT:    sub z0.s, z0.s, z1.s
1988; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
1989; CHECK-NEXT:    ret
1990;
1991; NONEON-NOSVE-LABEL: sub_v2i16:
1992; NONEON-NOSVE:       // %bb.0:
1993; NONEON-NOSVE-NEXT:    sub sp, sp, #32
1994; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
1995; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
1996; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #8]
1997; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
1998; NONEON-NOSVE-NEXT:    sub w8, w10, w8
1999; NONEON-NOSVE-NEXT:    str w8, [sp, #28]
2000; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
2001; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2002; NONEON-NOSVE-NEXT:    str w8, [sp, #24]
2003; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
2004; NONEON-NOSVE-NEXT:    add sp, sp, #32
2005; NONEON-NOSVE-NEXT:    ret
2006  %res = sub <2 x i16> %op1, %op2
2007  ret <2 x i16> %res
2008}
2009
2010define <4 x i16> @sub_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
2011; CHECK-LABEL: sub_v4i16:
2012; CHECK:       // %bb.0:
2013; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2014; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
2015; CHECK-NEXT:    sub z0.h, z0.h, z1.h
2016; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2017; CHECK-NEXT:    ret
2018;
2019; NONEON-NOSVE-LABEL: sub_v4i16:
2020; NONEON-NOSVE:       // %bb.0:
2021; NONEON-NOSVE-NEXT:    sub sp, sp, #32
2022; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
2023; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
2024; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
2025; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
2026; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2027; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]
2028; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
2029; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
2030; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2031; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
2032; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
2033; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
2034; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2035; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
2036; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
2037; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
2038; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2039; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
2040; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
2041; NONEON-NOSVE-NEXT:    add sp, sp, #32
2042; NONEON-NOSVE-NEXT:    ret
2043  %res = sub <4 x i16> %op1, %op2
2044  ret <4 x i16> %res
2045}
2046
2047define <8 x i16> @sub_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
2048; CHECK-LABEL: sub_v8i16:
2049; CHECK:       // %bb.0:
2050; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
2051; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
2052; CHECK-NEXT:    sub z0.h, z0.h, z1.h
2053; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2054; CHECK-NEXT:    ret
2055;
2056; NONEON-NOSVE-LABEL: sub_v8i16:
2057; NONEON-NOSVE:       // %bb.0:
2058; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
2059; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
2060; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #30]
2061; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
2062; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2063; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]
2064; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
2065; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
2066; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2067; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
2068; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
2069; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #26]
2070; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2071; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
2072; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
2073; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
2074; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2075; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #6]
2076; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
2077; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
2078; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2079; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]
2080; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]
2081; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
2082; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2083; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #2]
2084; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]
2085; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
2086; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2087; NONEON-NOSVE-NEXT:    ldrh w9, [sp]
2088; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]
2089; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
2090; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2091; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
2092; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
2093; NONEON-NOSVE-NEXT:    add sp, sp, #48
2094; NONEON-NOSVE-NEXT:    ret
2095  %res = sub <8 x i16> %op1, %op2
2096  ret <8 x i16> %res
2097}
2098
2099define void @sub_v16i16(ptr %a, ptr %b) {
2100; CHECK-LABEL: sub_v16i16:
2101; CHECK:       // %bb.0:
2102; CHECK-NEXT:    ldp q0, q3, [x1]
2103; CHECK-NEXT:    ldp q1, q2, [x0]
2104; CHECK-NEXT:    sub z0.h, z1.h, z0.h
2105; CHECK-NEXT:    sub z1.h, z2.h, z3.h
2106; CHECK-NEXT:    stp q0, q1, [x0]
2107; CHECK-NEXT:    ret
2108;
2109; NONEON-NOSVE-LABEL: sub_v16i16:
2110; NONEON-NOSVE:       // %bb.0:
2111; NONEON-NOSVE-NEXT:    sub sp, sp, #96
2112; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
2113; NONEON-NOSVE-NEXT:    ldp q3, q0, [x1]
2114; NONEON-NOSVE-NEXT:    ldp q2, q1, [x0]
2115; NONEON-NOSVE-NEXT:    stp q2, q3, [sp]
2116; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #32]
2117; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #62]
2118; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #46]
2119; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2120; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #44]
2121; NONEON-NOSVE-NEXT:    strh w8, [sp, #94]
2122; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #60]
2123; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2124; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #42]
2125; NONEON-NOSVE-NEXT:    strh w8, [sp, #92]
2126; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #58]
2127; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2128; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #40]
2129; NONEON-NOSVE-NEXT:    strh w8, [sp, #90]
2130; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #56]
2131; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2132; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #38]
2133; NONEON-NOSVE-NEXT:    strh w8, [sp, #88]
2134; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #54]
2135; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2136; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #36]
2137; NONEON-NOSVE-NEXT:    strh w8, [sp, #86]
2138; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #52]
2139; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2140; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #34]
2141; NONEON-NOSVE-NEXT:    strh w8, [sp, #84]
2142; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #50]
2143; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2144; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #32]
2145; NONEON-NOSVE-NEXT:    strh w8, [sp, #82]
2146; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #48]
2147; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2148; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
2149; NONEON-NOSVE-NEXT:    strh w8, [sp, #80]
2150; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #30]
2151; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2152; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]
2153; NONEON-NOSVE-NEXT:    strh w8, [sp, #78]
2154; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
2155; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2156; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
2157; NONEON-NOSVE-NEXT:    strh w8, [sp, #76]
2158; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #26]
2159; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2160; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
2161; NONEON-NOSVE-NEXT:    strh w8, [sp, #74]
2162; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
2163; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2164; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #6]
2165; NONEON-NOSVE-NEXT:    strh w8, [sp, #72]
2166; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
2167; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2168; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]
2169; NONEON-NOSVE-NEXT:    strh w8, [sp, #70]
2170; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
2171; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2172; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #2]
2173; NONEON-NOSVE-NEXT:    strh w8, [sp, #68]
2174; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
2175; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2176; NONEON-NOSVE-NEXT:    ldrh w9, [sp]
2177; NONEON-NOSVE-NEXT:    strh w8, [sp, #66]
2178; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
2179; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2180; NONEON-NOSVE-NEXT:    strh w8, [sp, #64]
2181; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
2182; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
2183; NONEON-NOSVE-NEXT:    add sp, sp, #96
2184; NONEON-NOSVE-NEXT:    ret
2185  %op1 = load <16 x i16>, ptr %a
2186  %op2 = load <16 x i16>, ptr %b
2187  %res = sub <16 x i16> %op1, %op2
2188  store <16 x i16> %res, ptr %a
2189  ret void
2190}
2191
2192define <2 x i32> @sub_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
2193; CHECK-LABEL: sub_v2i32:
2194; CHECK:       // %bb.0:
2195; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2196; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
2197; CHECK-NEXT:    sub z0.s, z0.s, z1.s
2198; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2199; CHECK-NEXT:    ret
2200;
2201; NONEON-NOSVE-LABEL: sub_v2i32:
2202; NONEON-NOSVE:       // %bb.0:
2203; NONEON-NOSVE-NEXT:    sub sp, sp, #32
2204; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
2205; NONEON-NOSVE-NEXT:    stp d0, d1, [sp, #8]
2206; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #8]
2207; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
2208; NONEON-NOSVE-NEXT:    sub w8, w10, w8
2209; NONEON-NOSVE-NEXT:    str w8, [sp, #28]
2210; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
2211; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2212; NONEON-NOSVE-NEXT:    str w8, [sp, #24]
2213; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
2214; NONEON-NOSVE-NEXT:    add sp, sp, #32
2215; NONEON-NOSVE-NEXT:    ret
2216  %res = sub <2 x i32> %op1, %op2
2217  ret <2 x i32> %res
2218}
2219
2220define <4 x i32> @sub_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
2221; CHECK-LABEL: sub_v4i32:
2222; CHECK:       // %bb.0:
2223; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
2224; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
2225; CHECK-NEXT:    sub z0.s, z0.s, z1.s
2226; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2227; CHECK-NEXT:    ret
2228;
2229; NONEON-NOSVE-LABEL: sub_v4i32:
2230; NONEON-NOSVE:       // %bb.0:
2231; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
2232; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
2233; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #8]
2234; NONEON-NOSVE-NEXT:    ldr w8, [sp, #28]
2235; NONEON-NOSVE-NEXT:    sub w8, w10, w8
2236; NONEON-NOSVE-NEXT:    str w8, [sp, #44]
2237; NONEON-NOSVE-NEXT:    ldr w8, [sp, #24]
2238; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2239; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp]
2240; NONEON-NOSVE-NEXT:    str w8, [sp, #40]
2241; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
2242; NONEON-NOSVE-NEXT:    sub w8, w10, w8
2243; NONEON-NOSVE-NEXT:    str w8, [sp, #36]
2244; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
2245; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2246; NONEON-NOSVE-NEXT:    str w8, [sp, #32]
2247; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
2248; NONEON-NOSVE-NEXT:    add sp, sp, #48
2249; NONEON-NOSVE-NEXT:    ret
2250  %res = sub <4 x i32> %op1, %op2
2251  ret <4 x i32> %res
2252}
2253
2254define void @sub_v8i32(ptr %a, ptr %b) {
2255; CHECK-LABEL: sub_v8i32:
2256; CHECK:       // %bb.0:
2257; CHECK-NEXT:    ldp q0, q3, [x1]
2258; CHECK-NEXT:    ldp q1, q2, [x0]
2259; CHECK-NEXT:    sub z0.s, z1.s, z0.s
2260; CHECK-NEXT:    sub z1.s, z2.s, z3.s
2261; CHECK-NEXT:    stp q0, q1, [x0]
2262; CHECK-NEXT:    ret
2263;
2264; NONEON-NOSVE-LABEL: sub_v8i32:
2265; NONEON-NOSVE:       // %bb.0:
2266; NONEON-NOSVE-NEXT:    sub sp, sp, #96
2267; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
2268; NONEON-NOSVE-NEXT:    ldp q3, q0, [x1]
2269; NONEON-NOSVE-NEXT:    ldp q2, q1, [x0]
2270; NONEON-NOSVE-NEXT:    stp q2, q3, [sp]
2271; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #32]
2272; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #40]
2273; NONEON-NOSVE-NEXT:    ldr w8, [sp, #60]
2274; NONEON-NOSVE-NEXT:    sub w8, w10, w8
2275; NONEON-NOSVE-NEXT:    str w8, [sp, #92]
2276; NONEON-NOSVE-NEXT:    ldr w8, [sp, #56]
2277; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2278; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #32]
2279; NONEON-NOSVE-NEXT:    str w8, [sp, #88]
2280; NONEON-NOSVE-NEXT:    ldr w8, [sp, #52]
2281; NONEON-NOSVE-NEXT:    sub w8, w10, w8
2282; NONEON-NOSVE-NEXT:    str w8, [sp, #84]
2283; NONEON-NOSVE-NEXT:    ldr w8, [sp, #48]
2284; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2285; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp, #8]
2286; NONEON-NOSVE-NEXT:    str w8, [sp, #80]
2287; NONEON-NOSVE-NEXT:    ldr w8, [sp, #28]
2288; NONEON-NOSVE-NEXT:    sub w8, w10, w8
2289; NONEON-NOSVE-NEXT:    str w8, [sp, #76]
2290; NONEON-NOSVE-NEXT:    ldr w8, [sp, #24]
2291; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2292; NONEON-NOSVE-NEXT:    ldp w9, w10, [sp]
2293; NONEON-NOSVE-NEXT:    str w8, [sp, #72]
2294; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
2295; NONEON-NOSVE-NEXT:    sub w8, w10, w8
2296; NONEON-NOSVE-NEXT:    str w8, [sp, #68]
2297; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
2298; NONEON-NOSVE-NEXT:    sub w8, w9, w8
2299; NONEON-NOSVE-NEXT:    str w8, [sp, #64]
2300; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
2301; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
2302; NONEON-NOSVE-NEXT:    add sp, sp, #96
2303; NONEON-NOSVE-NEXT:    ret
2304  %op1 = load <8 x i32>, ptr %a
2305  %op2 = load <8 x i32>, ptr %b
2306  %res = sub <8 x i32> %op1, %op2
2307  store <8 x i32> %res, ptr %a
2308  ret void
2309}
2310
2311define <1 x i64> @sub_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
2312; CHECK-LABEL: sub_v1i64:
2313; CHECK:       // %bb.0:
2314; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2315; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
2316; CHECK-NEXT:    sub z0.d, z0.d, z1.d
2317; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2318; CHECK-NEXT:    ret
2319;
2320; NONEON-NOSVE-LABEL: sub_v1i64:
2321; NONEON-NOSVE:       // %bb.0:
2322; NONEON-NOSVE-NEXT:    sub sp, sp, #16
2323; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
2324; NONEON-NOSVE-NEXT:    fmov x8, d1
2325; NONEON-NOSVE-NEXT:    fmov x9, d0
2326; NONEON-NOSVE-NEXT:    sub x8, x9, x8
2327; NONEON-NOSVE-NEXT:    str x8, [sp, #8]
2328; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
2329; NONEON-NOSVE-NEXT:    add sp, sp, #16
2330; NONEON-NOSVE-NEXT:    ret
2331  %res = sub <1 x i64> %op1, %op2
2332  ret <1 x i64> %res
2333}
2334
2335define <2 x i64> @sub_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
2336; CHECK-LABEL: sub_v2i64:
2337; CHECK:       // %bb.0:
2338; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
2339; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
2340; CHECK-NEXT:    sub z0.d, z0.d, z1.d
2341; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2342; CHECK-NEXT:    ret
2343;
2344; NONEON-NOSVE-LABEL: sub_v2i64:
2345; NONEON-NOSVE:       // %bb.0:
2346; NONEON-NOSVE-NEXT:    stp q0, q1, [sp, #-48]!
2347; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
2348; NONEON-NOSVE-NEXT:    ldp x9, x10, [sp]
2349; NONEON-NOSVE-NEXT:    ldr x8, [sp, #24]
2350; NONEON-NOSVE-NEXT:    sub x8, x10, x8
2351; NONEON-NOSVE-NEXT:    str x8, [sp, #40]
2352; NONEON-NOSVE-NEXT:    ldr x8, [sp, #16]
2353; NONEON-NOSVE-NEXT:    sub x8, x9, x8
2354; NONEON-NOSVE-NEXT:    str x8, [sp, #32]
2355; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
2356; NONEON-NOSVE-NEXT:    add sp, sp, #48
2357; NONEON-NOSVE-NEXT:    ret
2358  %res = sub <2 x i64> %op1, %op2
2359  ret <2 x i64> %res
2360}
2361
2362define void @sub_v4i64(ptr %a, ptr %b) {
2363; CHECK-LABEL: sub_v4i64:
2364; CHECK:       // %bb.0:
2365; CHECK-NEXT:    ldp q0, q3, [x1]
2366; CHECK-NEXT:    ldp q1, q2, [x0]
2367; CHECK-NEXT:    sub z0.d, z1.d, z0.d
2368; CHECK-NEXT:    sub z1.d, z2.d, z3.d
2369; CHECK-NEXT:    stp q0, q1, [x0]
2370; CHECK-NEXT:    ret
2371;
2372; NONEON-NOSVE-LABEL: sub_v4i64:
2373; NONEON-NOSVE:       // %bb.0:
2374; NONEON-NOSVE-NEXT:    sub sp, sp, #96
2375; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
2376; NONEON-NOSVE-NEXT:    ldp q3, q0, [x1]
2377; NONEON-NOSVE-NEXT:    ldp q2, q1, [x0]
2378; NONEON-NOSVE-NEXT:    stp q2, q3, [sp]
2379; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #32]
2380; NONEON-NOSVE-NEXT:    ldp x9, x10, [sp, #32]
2381; NONEON-NOSVE-NEXT:    ldr x8, [sp, #56]
2382; NONEON-NOSVE-NEXT:    sub x8, x10, x8
2383; NONEON-NOSVE-NEXT:    str x8, [sp, #88]
2384; NONEON-NOSVE-NEXT:    ldr x8, [sp, #48]
2385; NONEON-NOSVE-NEXT:    sub x8, x9, x8
2386; NONEON-NOSVE-NEXT:    ldp x9, x10, [sp]
2387; NONEON-NOSVE-NEXT:    str x8, [sp, #80]
2388; NONEON-NOSVE-NEXT:    ldr x8, [sp, #24]
2389; NONEON-NOSVE-NEXT:    sub x8, x10, x8
2390; NONEON-NOSVE-NEXT:    str x8, [sp, #72]
2391; NONEON-NOSVE-NEXT:    ldr x8, [sp, #16]
2392; NONEON-NOSVE-NEXT:    sub x8, x9, x8
2393; NONEON-NOSVE-NEXT:    str x8, [sp, #64]
2394; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #64]
2395; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
2396; NONEON-NOSVE-NEXT:    add sp, sp, #96
2397; NONEON-NOSVE-NEXT:    ret
2398  %op1 = load <4 x i64>, ptr %a
2399  %op2 = load <4 x i64>, ptr %b
2400  %res = sub <4 x i64> %op1, %op2
2401  store <4 x i64> %res, ptr %a
2402  ret void
2403}
2404
2405;
2406; ABS
2407;
2408
2409define <4 x i8> @abs_v4i8(<4 x i8> %op1) {
2410; CHECK-LABEL: abs_v4i8:
2411; CHECK:       // %bb.0:
2412; CHECK-NEXT:    ptrue p0.h, vl4
2413; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2414; CHECK-NEXT:    sxtb z0.h, p0/m, z0.h
2415; CHECK-NEXT:    abs z0.h, p0/m, z0.h
2416; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2417; CHECK-NEXT:    ret
2418;
2419; NONEON-NOSVE-LABEL: abs_v4i8:
2420; NONEON-NOSVE:       // %bb.0:
2421; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
2422; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
2423; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #6]
2424; NONEON-NOSVE-NEXT:    ldrsb w9, [sp, #4]
2425; NONEON-NOSVE-NEXT:    ldrsb w10, [sp, #2]
2426; NONEON-NOSVE-NEXT:    ldrsb w11, [sp]
2427; NONEON-NOSVE-NEXT:    cmp w8, #0
2428; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2429; NONEON-NOSVE-NEXT:    cmp w9, #0
2430; NONEON-NOSVE-NEXT:    strh w8, [sp, #14]
2431; NONEON-NOSVE-NEXT:    cneg w8, w9, mi
2432; NONEON-NOSVE-NEXT:    cmp w10, #0
2433; NONEON-NOSVE-NEXT:    strh w8, [sp, #12]
2434; NONEON-NOSVE-NEXT:    cneg w8, w10, mi
2435; NONEON-NOSVE-NEXT:    cmp w11, #0
2436; NONEON-NOSVE-NEXT:    strh w8, [sp, #10]
2437; NONEON-NOSVE-NEXT:    cneg w8, w11, mi
2438; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]
2439; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
2440; NONEON-NOSVE-NEXT:    add sp, sp, #16
2441; NONEON-NOSVE-NEXT:    ret
2442  %res = call <4 x i8> @llvm.abs.v4i8(<4 x i8> %op1, i1 false)
2443  ret <4 x i8> %res
2444}
2445
2446define <8 x i8> @abs_v8i8(<8 x i8> %op1) {
2447; CHECK-LABEL: abs_v8i8:
2448; CHECK:       // %bb.0:
2449; CHECK-NEXT:    ptrue p0.b, vl8
2450; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2451; CHECK-NEXT:    abs z0.b, p0/m, z0.b
2452; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2453; CHECK-NEXT:    ret
2454;
2455; NONEON-NOSVE-LABEL: abs_v8i8:
2456; NONEON-NOSVE:       // %bb.0:
2457; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
2458; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
2459; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #7]
2460; NONEON-NOSVE-NEXT:    cmp w8, #0
2461; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2462; NONEON-NOSVE-NEXT:    strb w8, [sp, #15]
2463; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #6]
2464; NONEON-NOSVE-NEXT:    cmp w8, #0
2465; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2466; NONEON-NOSVE-NEXT:    strb w8, [sp, #14]
2467; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #5]
2468; NONEON-NOSVE-NEXT:    cmp w8, #0
2469; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2470; NONEON-NOSVE-NEXT:    strb w8, [sp, #13]
2471; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #4]
2472; NONEON-NOSVE-NEXT:    cmp w8, #0
2473; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2474; NONEON-NOSVE-NEXT:    strb w8, [sp, #12]
2475; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #3]
2476; NONEON-NOSVE-NEXT:    cmp w8, #0
2477; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2478; NONEON-NOSVE-NEXT:    strb w8, [sp, #11]
2479; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #2]
2480; NONEON-NOSVE-NEXT:    cmp w8, #0
2481; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2482; NONEON-NOSVE-NEXT:    strb w8, [sp, #10]
2483; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #1]
2484; NONEON-NOSVE-NEXT:    cmp w8, #0
2485; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2486; NONEON-NOSVE-NEXT:    strb w8, [sp, #9]
2487; NONEON-NOSVE-NEXT:    ldrsb w8, [sp]
2488; NONEON-NOSVE-NEXT:    cmp w8, #0
2489; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2490; NONEON-NOSVE-NEXT:    strb w8, [sp, #8]
2491; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
2492; NONEON-NOSVE-NEXT:    add sp, sp, #16
2493; NONEON-NOSVE-NEXT:    ret
2494  %res = call <8 x i8> @llvm.abs.v8i8(<8 x i8> %op1, i1 false)
2495  ret <8 x i8> %res
2496}
2497
2498define <16 x i8> @abs_v16i8(<16 x i8> %op1) {
2499; CHECK-LABEL: abs_v16i8:
2500; CHECK:       // %bb.0:
2501; CHECK-NEXT:    ptrue p0.b, vl16
2502; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
2503; CHECK-NEXT:    abs z0.b, p0/m, z0.b
2504; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2505; CHECK-NEXT:    ret
2506;
2507; NONEON-NOSVE-LABEL: abs_v16i8:
2508; NONEON-NOSVE:       // %bb.0:
2509; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
2510; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
2511; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #15]
2512; NONEON-NOSVE-NEXT:    cmp w8, #0
2513; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2514; NONEON-NOSVE-NEXT:    strb w8, [sp, #31]
2515; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #14]
2516; NONEON-NOSVE-NEXT:    cmp w8, #0
2517; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2518; NONEON-NOSVE-NEXT:    strb w8, [sp, #30]
2519; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #13]
2520; NONEON-NOSVE-NEXT:    cmp w8, #0
2521; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2522; NONEON-NOSVE-NEXT:    strb w8, [sp, #29]
2523; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #12]
2524; NONEON-NOSVE-NEXT:    cmp w8, #0
2525; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2526; NONEON-NOSVE-NEXT:    strb w8, [sp, #28]
2527; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #11]
2528; NONEON-NOSVE-NEXT:    cmp w8, #0
2529; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2530; NONEON-NOSVE-NEXT:    strb w8, [sp, #27]
2531; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #10]
2532; NONEON-NOSVE-NEXT:    cmp w8, #0
2533; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2534; NONEON-NOSVE-NEXT:    strb w8, [sp, #26]
2535; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #9]
2536; NONEON-NOSVE-NEXT:    cmp w8, #0
2537; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2538; NONEON-NOSVE-NEXT:    strb w8, [sp, #25]
2539; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #8]
2540; NONEON-NOSVE-NEXT:    cmp w8, #0
2541; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2542; NONEON-NOSVE-NEXT:    strb w8, [sp, #24]
2543; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #7]
2544; NONEON-NOSVE-NEXT:    cmp w8, #0
2545; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2546; NONEON-NOSVE-NEXT:    strb w8, [sp, #23]
2547; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #6]
2548; NONEON-NOSVE-NEXT:    cmp w8, #0
2549; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2550; NONEON-NOSVE-NEXT:    strb w8, [sp, #22]
2551; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #5]
2552; NONEON-NOSVE-NEXT:    cmp w8, #0
2553; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2554; NONEON-NOSVE-NEXT:    strb w8, [sp, #21]
2555; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #4]
2556; NONEON-NOSVE-NEXT:    cmp w8, #0
2557; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2558; NONEON-NOSVE-NEXT:    strb w8, [sp, #20]
2559; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #3]
2560; NONEON-NOSVE-NEXT:    cmp w8, #0
2561; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2562; NONEON-NOSVE-NEXT:    strb w8, [sp, #19]
2563; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #2]
2564; NONEON-NOSVE-NEXT:    cmp w8, #0
2565; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2566; NONEON-NOSVE-NEXT:    strb w8, [sp, #18]
2567; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #1]
2568; NONEON-NOSVE-NEXT:    cmp w8, #0
2569; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2570; NONEON-NOSVE-NEXT:    strb w8, [sp, #17]
2571; NONEON-NOSVE-NEXT:    ldrsb w8, [sp]
2572; NONEON-NOSVE-NEXT:    cmp w8, #0
2573; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2574; NONEON-NOSVE-NEXT:    strb w8, [sp, #16]
2575; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
2576; NONEON-NOSVE-NEXT:    add sp, sp, #32
2577; NONEON-NOSVE-NEXT:    ret
2578  %res = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %op1, i1 false)
2579  ret <16 x i8> %res
2580}
2581
2582define void @abs_v32i8(ptr %a) {
2583; CHECK-LABEL: abs_v32i8:
2584; CHECK:       // %bb.0:
2585; CHECK-NEXT:    ldp q0, q1, [x0]
2586; CHECK-NEXT:    ptrue p0.b, vl16
2587; CHECK-NEXT:    abs z0.b, p0/m, z0.b
2588; CHECK-NEXT:    abs z1.b, p0/m, z1.b
2589; CHECK-NEXT:    stp q0, q1, [x0]
2590; CHECK-NEXT:    ret
2591;
2592; NONEON-NOSVE-LABEL: abs_v32i8:
2593; NONEON-NOSVE:       // %bb.0:
2594; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
2595; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
2596; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
2597; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #31]
2598; NONEON-NOSVE-NEXT:    cmp w8, #0
2599; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2600; NONEON-NOSVE-NEXT:    strb w8, [sp, #63]
2601; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #30]
2602; NONEON-NOSVE-NEXT:    cmp w8, #0
2603; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2604; NONEON-NOSVE-NEXT:    strb w8, [sp, #62]
2605; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #29]
2606; NONEON-NOSVE-NEXT:    cmp w8, #0
2607; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2608; NONEON-NOSVE-NEXT:    strb w8, [sp, #61]
2609; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #28]
2610; NONEON-NOSVE-NEXT:    cmp w8, #0
2611; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2612; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
2613; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #27]
2614; NONEON-NOSVE-NEXT:    cmp w8, #0
2615; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2616; NONEON-NOSVE-NEXT:    strb w8, [sp, #59]
2617; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #26]
2618; NONEON-NOSVE-NEXT:    cmp w8, #0
2619; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2620; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
2621; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #25]
2622; NONEON-NOSVE-NEXT:    cmp w8, #0
2623; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2624; NONEON-NOSVE-NEXT:    strb w8, [sp, #57]
2625; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #24]
2626; NONEON-NOSVE-NEXT:    cmp w8, #0
2627; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2628; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
2629; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #23]
2630; NONEON-NOSVE-NEXT:    cmp w8, #0
2631; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2632; NONEON-NOSVE-NEXT:    strb w8, [sp, #55]
2633; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #22]
2634; NONEON-NOSVE-NEXT:    cmp w8, #0
2635; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2636; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
2637; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #21]
2638; NONEON-NOSVE-NEXT:    cmp w8, #0
2639; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2640; NONEON-NOSVE-NEXT:    strb w8, [sp, #53]
2641; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #20]
2642; NONEON-NOSVE-NEXT:    cmp w8, #0
2643; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2644; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
2645; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #19]
2646; NONEON-NOSVE-NEXT:    cmp w8, #0
2647; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2648; NONEON-NOSVE-NEXT:    strb w8, [sp, #51]
2649; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #18]
2650; NONEON-NOSVE-NEXT:    cmp w8, #0
2651; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2652; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
2653; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #17]
2654; NONEON-NOSVE-NEXT:    cmp w8, #0
2655; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2656; NONEON-NOSVE-NEXT:    strb w8, [sp, #49]
2657; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #16]
2658; NONEON-NOSVE-NEXT:    cmp w8, #0
2659; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2660; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
2661; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #15]
2662; NONEON-NOSVE-NEXT:    cmp w8, #0
2663; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2664; NONEON-NOSVE-NEXT:    strb w8, [sp, #47]
2665; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #14]
2666; NONEON-NOSVE-NEXT:    cmp w8, #0
2667; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2668; NONEON-NOSVE-NEXT:    strb w8, [sp, #46]
2669; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #13]
2670; NONEON-NOSVE-NEXT:    cmp w8, #0
2671; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2672; NONEON-NOSVE-NEXT:    strb w8, [sp, #45]
2673; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #12]
2674; NONEON-NOSVE-NEXT:    cmp w8, #0
2675; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2676; NONEON-NOSVE-NEXT:    strb w8, [sp, #44]
2677; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #11]
2678; NONEON-NOSVE-NEXT:    cmp w8, #0
2679; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2680; NONEON-NOSVE-NEXT:    strb w8, [sp, #43]
2681; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #10]
2682; NONEON-NOSVE-NEXT:    cmp w8, #0
2683; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2684; NONEON-NOSVE-NEXT:    strb w8, [sp, #42]
2685; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #9]
2686; NONEON-NOSVE-NEXT:    cmp w8, #0
2687; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2688; NONEON-NOSVE-NEXT:    strb w8, [sp, #41]
2689; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #8]
2690; NONEON-NOSVE-NEXT:    cmp w8, #0
2691; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2692; NONEON-NOSVE-NEXT:    strb w8, [sp, #40]
2693; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #7]
2694; NONEON-NOSVE-NEXT:    cmp w8, #0
2695; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2696; NONEON-NOSVE-NEXT:    strb w8, [sp, #39]
2697; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #6]
2698; NONEON-NOSVE-NEXT:    cmp w8, #0
2699; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2700; NONEON-NOSVE-NEXT:    strb w8, [sp, #38]
2701; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #5]
2702; NONEON-NOSVE-NEXT:    cmp w8, #0
2703; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2704; NONEON-NOSVE-NEXT:    strb w8, [sp, #37]
2705; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #4]
2706; NONEON-NOSVE-NEXT:    cmp w8, #0
2707; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2708; NONEON-NOSVE-NEXT:    strb w8, [sp, #36]
2709; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #3]
2710; NONEON-NOSVE-NEXT:    cmp w8, #0
2711; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2712; NONEON-NOSVE-NEXT:    strb w8, [sp, #35]
2713; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #2]
2714; NONEON-NOSVE-NEXT:    cmp w8, #0
2715; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2716; NONEON-NOSVE-NEXT:    strb w8, [sp, #34]
2717; NONEON-NOSVE-NEXT:    ldrsb w8, [sp, #1]
2718; NONEON-NOSVE-NEXT:    cmp w8, #0
2719; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2720; NONEON-NOSVE-NEXT:    strb w8, [sp, #33]
2721; NONEON-NOSVE-NEXT:    ldrsb w8, [sp]
2722; NONEON-NOSVE-NEXT:    cmp w8, #0
2723; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2724; NONEON-NOSVE-NEXT:    strb w8, [sp, #32]
2725; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
2726; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
2727; NONEON-NOSVE-NEXT:    add sp, sp, #64
2728; NONEON-NOSVE-NEXT:    ret
2729  %op1 = load <32 x i8>, ptr %a
2730  %res = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %op1, i1 false)
2731  store <32 x i8> %res, ptr %a
2732  ret void
2733}
2734
2735define <2 x i16> @abs_v2i16(<2 x i16> %op1) {
2736; CHECK-LABEL: abs_v2i16:
2737; CHECK:       // %bb.0:
2738; CHECK-NEXT:    ptrue p0.s, vl2
2739; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2740; CHECK-NEXT:    sxth z0.s, p0/m, z0.s
2741; CHECK-NEXT:    abs z0.s, p0/m, z0.s
2742; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2743; CHECK-NEXT:    ret
2744;
2745; NONEON-NOSVE-LABEL: abs_v2i16:
2746; NONEON-NOSVE:       // %bb.0:
2747; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
2748; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
2749; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #4]
2750; NONEON-NOSVE-NEXT:    ldrsh w9, [sp]
2751; NONEON-NOSVE-NEXT:    cmp w8, #0
2752; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2753; NONEON-NOSVE-NEXT:    cmp w9, #0
2754; NONEON-NOSVE-NEXT:    cneg w9, w9, mi
2755; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #8]
2756; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
2757; NONEON-NOSVE-NEXT:    add sp, sp, #16
2758; NONEON-NOSVE-NEXT:    ret
2759  %res = call <2 x i16> @llvm.abs.v2i16(<2 x i16> %op1, i1 false)
2760  ret <2 x i16> %res
2761}
2762
2763define <4 x i16> @abs_v4i16(<4 x i16> %op1) {
2764; CHECK-LABEL: abs_v4i16:
2765; CHECK:       // %bb.0:
2766; CHECK-NEXT:    ptrue p0.h, vl4
2767; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2768; CHECK-NEXT:    abs z0.h, p0/m, z0.h
2769; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2770; CHECK-NEXT:    ret
2771;
2772; NONEON-NOSVE-LABEL: abs_v4i16:
2773; NONEON-NOSVE:       // %bb.0:
2774; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
2775; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
2776; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #6]
2777; NONEON-NOSVE-NEXT:    cmp w8, #0
2778; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2779; NONEON-NOSVE-NEXT:    strh w8, [sp, #14]
2780; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #4]
2781; NONEON-NOSVE-NEXT:    cmp w8, #0
2782; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2783; NONEON-NOSVE-NEXT:    strh w8, [sp, #12]
2784; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #2]
2785; NONEON-NOSVE-NEXT:    cmp w8, #0
2786; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2787; NONEON-NOSVE-NEXT:    strh w8, [sp, #10]
2788; NONEON-NOSVE-NEXT:    ldrsh w8, [sp]
2789; NONEON-NOSVE-NEXT:    cmp w8, #0
2790; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2791; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]
2792; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
2793; NONEON-NOSVE-NEXT:    add sp, sp, #16
2794; NONEON-NOSVE-NEXT:    ret
2795  %res = call <4 x i16> @llvm.abs.v4i16(<4 x i16> %op1, i1 false)
2796  ret <4 x i16> %res
2797}
2798
2799define <8 x i16> @abs_v8i16(<8 x i16> %op1) {
2800; CHECK-LABEL: abs_v8i16:
2801; CHECK:       // %bb.0:
2802; CHECK-NEXT:    ptrue p0.h, vl8
2803; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
2804; CHECK-NEXT:    abs z0.h, p0/m, z0.h
2805; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2806; CHECK-NEXT:    ret
2807;
2808; NONEON-NOSVE-LABEL: abs_v8i16:
2809; NONEON-NOSVE:       // %bb.0:
2810; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
2811; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
2812; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #14]
2813; NONEON-NOSVE-NEXT:    cmp w8, #0
2814; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2815; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
2816; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #12]
2817; NONEON-NOSVE-NEXT:    cmp w8, #0
2818; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2819; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
2820; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #10]
2821; NONEON-NOSVE-NEXT:    cmp w8, #0
2822; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2823; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
2824; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #8]
2825; NONEON-NOSVE-NEXT:    cmp w8, #0
2826; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2827; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
2828; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #6]
2829; NONEON-NOSVE-NEXT:    cmp w8, #0
2830; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2831; NONEON-NOSVE-NEXT:    strh w8, [sp, #22]
2832; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #4]
2833; NONEON-NOSVE-NEXT:    cmp w8, #0
2834; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2835; NONEON-NOSVE-NEXT:    strh w8, [sp, #20]
2836; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #2]
2837; NONEON-NOSVE-NEXT:    cmp w8, #0
2838; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2839; NONEON-NOSVE-NEXT:    strh w8, [sp, #18]
2840; NONEON-NOSVE-NEXT:    ldrsh w8, [sp]
2841; NONEON-NOSVE-NEXT:    cmp w8, #0
2842; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2843; NONEON-NOSVE-NEXT:    strh w8, [sp, #16]
2844; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
2845; NONEON-NOSVE-NEXT:    add sp, sp, #32
2846; NONEON-NOSVE-NEXT:    ret
2847  %res = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %op1, i1 false)
2848  ret <8 x i16> %res
2849}
2850
2851define void @abs_v16i16(ptr %a) {
2852; CHECK-LABEL: abs_v16i16:
2853; CHECK:       // %bb.0:
2854; CHECK-NEXT:    ldp q0, q1, [x0]
2855; CHECK-NEXT:    ptrue p0.h, vl8
2856; CHECK-NEXT:    abs z0.h, p0/m, z0.h
2857; CHECK-NEXT:    abs z1.h, p0/m, z1.h
2858; CHECK-NEXT:    stp q0, q1, [x0]
2859; CHECK-NEXT:    ret
2860;
2861; NONEON-NOSVE-LABEL: abs_v16i16:
2862; NONEON-NOSVE:       // %bb.0:
2863; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
2864; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
2865; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
2866; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #30]
2867; NONEON-NOSVE-NEXT:    cmp w8, #0
2868; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2869; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]
2870; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #28]
2871; NONEON-NOSVE-NEXT:    cmp w8, #0
2872; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2873; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
2874; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #26]
2875; NONEON-NOSVE-NEXT:    cmp w8, #0
2876; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2877; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
2878; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #24]
2879; NONEON-NOSVE-NEXT:    cmp w8, #0
2880; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2881; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
2882; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #22]
2883; NONEON-NOSVE-NEXT:    cmp w8, #0
2884; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2885; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
2886; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #20]
2887; NONEON-NOSVE-NEXT:    cmp w8, #0
2888; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2889; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
2890; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #18]
2891; NONEON-NOSVE-NEXT:    cmp w8, #0
2892; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2893; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
2894; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #16]
2895; NONEON-NOSVE-NEXT:    cmp w8, #0
2896; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2897; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
2898; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #14]
2899; NONEON-NOSVE-NEXT:    cmp w8, #0
2900; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2901; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
2902; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #12]
2903; NONEON-NOSVE-NEXT:    cmp w8, #0
2904; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2905; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
2906; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #10]
2907; NONEON-NOSVE-NEXT:    cmp w8, #0
2908; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2909; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
2910; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #8]
2911; NONEON-NOSVE-NEXT:    cmp w8, #0
2912; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2913; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
2914; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #6]
2915; NONEON-NOSVE-NEXT:    cmp w8, #0
2916; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2917; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]
2918; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #4]
2919; NONEON-NOSVE-NEXT:    cmp w8, #0
2920; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2921; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]
2922; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #2]
2923; NONEON-NOSVE-NEXT:    cmp w8, #0
2924; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2925; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]
2926; NONEON-NOSVE-NEXT:    ldrsh w8, [sp]
2927; NONEON-NOSVE-NEXT:    cmp w8, #0
2928; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2929; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
2930; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
2931; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
2932; NONEON-NOSVE-NEXT:    add sp, sp, #64
2933; NONEON-NOSVE-NEXT:    ret
2934  %op1 = load <16 x i16>, ptr %a
2935  %res = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %op1, i1 false)
2936  store <16 x i16> %res, ptr %a
2937  ret void
2938}
2939
2940define <2 x i32> @abs_v2i32(<2 x i32> %op1) {
2941; CHECK-LABEL: abs_v2i32:
2942; CHECK:       // %bb.0:
2943; CHECK-NEXT:    ptrue p0.s, vl2
2944; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2945; CHECK-NEXT:    abs z0.s, p0/m, z0.s
2946; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2947; CHECK-NEXT:    ret
2948;
2949; NONEON-NOSVE-LABEL: abs_v2i32:
2950; NONEON-NOSVE:       // %bb.0:
2951; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
2952; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
2953; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]
2954; NONEON-NOSVE-NEXT:    cmp w8, #0
2955; NONEON-NOSVE-NEXT:    cneg w9, w8, mi
2956; NONEON-NOSVE-NEXT:    ldr w8, [sp]
2957; NONEON-NOSVE-NEXT:    cmp w8, #0
2958; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2959; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8]
2960; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
2961; NONEON-NOSVE-NEXT:    add sp, sp, #16
2962; NONEON-NOSVE-NEXT:    ret
2963  %res = call <2 x i32> @llvm.abs.v2i32(<2 x i32> %op1, i1 false)
2964  ret <2 x i32> %res
2965}
2966
2967define <4 x i32> @abs_v4i32(<4 x i32> %op1) {
2968; CHECK-LABEL: abs_v4i32:
2969; CHECK:       // %bb.0:
2970; CHECK-NEXT:    ptrue p0.s, vl4
2971; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
2972; CHECK-NEXT:    abs z0.s, p0/m, z0.s
2973; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2974; CHECK-NEXT:    ret
2975;
2976; NONEON-NOSVE-LABEL: abs_v4i32:
2977; NONEON-NOSVE:       // %bb.0:
2978; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
2979; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
2980; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12]
2981; NONEON-NOSVE-NEXT:    cmp w8, #0
2982; NONEON-NOSVE-NEXT:    cneg w9, w8, mi
2983; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]
2984; NONEON-NOSVE-NEXT:    cmp w8, #0
2985; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2986; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
2987; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]
2988; NONEON-NOSVE-NEXT:    cmp w8, #0
2989; NONEON-NOSVE-NEXT:    cneg w9, w8, mi
2990; NONEON-NOSVE-NEXT:    ldr w8, [sp]
2991; NONEON-NOSVE-NEXT:    cmp w8, #0
2992; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
2993; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
2994; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
2995; NONEON-NOSVE-NEXT:    add sp, sp, #32
2996; NONEON-NOSVE-NEXT:    ret
2997  %res = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %op1, i1 false)
2998  ret <4 x i32> %res
2999}
3000
3001define void @abs_v8i32(ptr %a) {
3002; CHECK-LABEL: abs_v8i32:
3003; CHECK:       // %bb.0:
3004; CHECK-NEXT:    ldp q0, q1, [x0]
3005; CHECK-NEXT:    ptrue p0.s, vl4
3006; CHECK-NEXT:    abs z0.s, p0/m, z0.s
3007; CHECK-NEXT:    abs z1.s, p0/m, z1.s
3008; CHECK-NEXT:    stp q0, q1, [x0]
3009; CHECK-NEXT:    ret
3010;
3011; NONEON-NOSVE-LABEL: abs_v8i32:
3012; NONEON-NOSVE:       // %bb.0:
3013; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
3014; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
3015; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
3016; NONEON-NOSVE-NEXT:    ldr w8, [sp, #28]
3017; NONEON-NOSVE-NEXT:    cmp w8, #0
3018; NONEON-NOSVE-NEXT:    cneg w9, w8, mi
3019; NONEON-NOSVE-NEXT:    ldr w8, [sp, #24]
3020; NONEON-NOSVE-NEXT:    cmp w8, #0
3021; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
3022; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
3023; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
3024; NONEON-NOSVE-NEXT:    cmp w8, #0
3025; NONEON-NOSVE-NEXT:    cneg w9, w8, mi
3026; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
3027; NONEON-NOSVE-NEXT:    cmp w8, #0
3028; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
3029; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
3030; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12]
3031; NONEON-NOSVE-NEXT:    cmp w8, #0
3032; NONEON-NOSVE-NEXT:    cneg w9, w8, mi
3033; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]
3034; NONEON-NOSVE-NEXT:    cmp w8, #0
3035; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
3036; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
3037; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]
3038; NONEON-NOSVE-NEXT:    cmp w8, #0
3039; NONEON-NOSVE-NEXT:    cneg w9, w8, mi
3040; NONEON-NOSVE-NEXT:    ldr w8, [sp]
3041; NONEON-NOSVE-NEXT:    cmp w8, #0
3042; NONEON-NOSVE-NEXT:    cneg w8, w8, mi
3043; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
3044; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
3045; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
3046; NONEON-NOSVE-NEXT:    add sp, sp, #64
3047; NONEON-NOSVE-NEXT:    ret
3048  %op1 = load <8 x i32>, ptr %a
3049  %res = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %op1, i1 false)
3050  store <8 x i32> %res, ptr %a
3051  ret void
3052}
3053
3054define <1 x i64> @abs_v1i64(<1 x i64> %op1) {
3055; CHECK-LABEL: abs_v1i64:
3056; CHECK:       // %bb.0:
3057; CHECK-NEXT:    ptrue p0.d, vl1
3058; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
3059; CHECK-NEXT:    abs z0.d, p0/m, z0.d
3060; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
3061; CHECK-NEXT:    ret
3062;
3063; NONEON-NOSVE-LABEL: abs_v1i64:
3064; NONEON-NOSVE:       // %bb.0:
3065; NONEON-NOSVE-NEXT:    sub sp, sp, #16
3066; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
3067; NONEON-NOSVE-NEXT:    fmov x8, d0
3068; NONEON-NOSVE-NEXT:    cmp x8, #0
3069; NONEON-NOSVE-NEXT:    cneg x8, x8, mi
3070; NONEON-NOSVE-NEXT:    str x8, [sp, #8]
3071; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
3072; NONEON-NOSVE-NEXT:    add sp, sp, #16
3073; NONEON-NOSVE-NEXT:    ret
3074  %res = call <1 x i64> @llvm.abs.v1i64(<1 x i64> %op1, i1 false)
3075  ret <1 x i64> %res
3076}
3077
3078define <2 x i64> @abs_v2i64(<2 x i64> %op1) {
3079; CHECK-LABEL: abs_v2i64:
3080; CHECK:       // %bb.0:
3081; CHECK-NEXT:    ptrue p0.d, vl2
3082; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
3083; CHECK-NEXT:    abs z0.d, p0/m, z0.d
3084; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
3085; CHECK-NEXT:    ret
3086;
3087; NONEON-NOSVE-LABEL: abs_v2i64:
3088; NONEON-NOSVE:       // %bb.0:
3089; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
3090; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
3091; NONEON-NOSVE-NEXT:    ldr x8, [sp, #8]
3092; NONEON-NOSVE-NEXT:    cmp x8, #0
3093; NONEON-NOSVE-NEXT:    cneg x9, x8, mi
3094; NONEON-NOSVE-NEXT:    ldr x8, [sp]
3095; NONEON-NOSVE-NEXT:    cmp x8, #0
3096; NONEON-NOSVE-NEXT:    cneg x8, x8, mi
3097; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #16]
3098; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
3099; NONEON-NOSVE-NEXT:    add sp, sp, #32
3100; NONEON-NOSVE-NEXT:    ret
3101  %res = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %op1, i1 false)
3102  ret <2 x i64> %res
3103}
3104
3105define void @abs_v4i64(ptr %a) {
3106; CHECK-LABEL: abs_v4i64:
3107; CHECK:       // %bb.0:
3108; CHECK-NEXT:    ldp q0, q1, [x0]
3109; CHECK-NEXT:    ptrue p0.d, vl2
3110; CHECK-NEXT:    abs z0.d, p0/m, z0.d
3111; CHECK-NEXT:    abs z1.d, p0/m, z1.d
3112; CHECK-NEXT:    stp q0, q1, [x0]
3113; CHECK-NEXT:    ret
3114;
3115; NONEON-NOSVE-LABEL: abs_v4i64:
3116; NONEON-NOSVE:       // %bb.0:
3117; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
3118; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
3119; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
3120; NONEON-NOSVE-NEXT:    ldr x8, [sp, #24]
3121; NONEON-NOSVE-NEXT:    cmp x8, #0
3122; NONEON-NOSVE-NEXT:    cneg x9, x8, mi
3123; NONEON-NOSVE-NEXT:    ldr x8, [sp, #16]
3124; NONEON-NOSVE-NEXT:    cmp x8, #0
3125; NONEON-NOSVE-NEXT:    cneg x8, x8, mi
3126; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #48]
3127; NONEON-NOSVE-NEXT:    ldr x8, [sp, #8]
3128; NONEON-NOSVE-NEXT:    cmp x8, #0
3129; NONEON-NOSVE-NEXT:    cneg x9, x8, mi
3130; NONEON-NOSVE-NEXT:    ldr x8, [sp]
3131; NONEON-NOSVE-NEXT:    cmp x8, #0
3132; NONEON-NOSVE-NEXT:    cneg x8, x8, mi
3133; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #32]
3134; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
3135; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
3136; NONEON-NOSVE-NEXT:    add sp, sp, #64
3137; NONEON-NOSVE-NEXT:    ret
3138  %op1 = load <4 x i64>, ptr %a
3139  %res = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %op1, i1 false)
3140  store <4 x i64> %res, ptr %a
3141  ret void
3142}
3143
3144declare <4 x i8> @llvm.abs.v4i8(<4 x i8>, i1)
3145declare <8 x i8> @llvm.abs.v8i8(<8 x i8>, i1)
3146declare <16 x i8> @llvm.abs.v16i8(<16 x i8>, i1)
3147declare <32 x i8> @llvm.abs.v32i8(<32 x i8>, i1)
3148declare <4 x i16> @llvm.abs.v4i16(<4 x i16>, i1)
3149declare <2 x i16> @llvm.abs.v2i16(<2 x i16>, i1)
3150declare <8 x i16> @llvm.abs.v8i16(<8 x i16>, i1)
3151declare <16 x i16> @llvm.abs.v16i16(<16 x i16>, i1)
3152declare <2 x i32> @llvm.abs.v2i32(<2 x i32>, i1)
3153declare <4 x i32> @llvm.abs.v4i32(<4 x i32>, i1)
3154declare <8 x i32> @llvm.abs.v8i32(<8 x i32>, i1)
3155declare <1 x i64> @llvm.abs.v1i64(<1 x i64>, i1)
3156declare <2 x i64> @llvm.abs.v2i64(<2 x i64>, i1)
3157declare <4 x i64> @llvm.abs.v4i64(<4 x i64>, i1)
3158
3159