xref: /llvm-project/llvm/test/CodeGen/AArch64/fmla.ll (revision 61510b51c33464a6bc15e4cf5b1ee07e2e0ec1c9) 
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
2; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16
3; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16
4; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16
5; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16
6
7define double @fma_f64(double %a, double %b, double %c) {
8; CHECK-LABEL: fma_f64:
9; CHECK:       // %bb.0: // %entry
10; CHECK-NEXT:    fmadd d0, d0, d1, d2
11; CHECK-NEXT:    ret
12entry:
13  %d = call double @llvm.fma.f64(double %a, double %b, double %c)
14  ret double %d
15}
16
17define float @fma_f32(float %a, float %b, float %c) {
18; CHECK-LABEL: fma_f32:
19; CHECK:       // %bb.0: // %entry
20; CHECK-NEXT:    fmadd s0, s0, s1, s2
21; CHECK-NEXT:    ret
22entry:
23  %d = call float @llvm.fma.f32(float %a, float %b, float %c)
24  ret float %d
25}
26
27define half @fma_f16(half %a, half %b, half %c) {
28; CHECK-SD-NOFP16-LABEL: fma_f16:
29; CHECK-SD-NOFP16:       // %bb.0: // %entry
30; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
31; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
32; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
33; CHECK-SD-NOFP16-NEXT:    fmadd s0, s0, s1, s2
34; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
35; CHECK-SD-NOFP16-NEXT:    ret
36;
37; CHECK-SD-FP16-LABEL: fma_f16:
38; CHECK-SD-FP16:       // %bb.0: // %entry
39; CHECK-SD-FP16-NEXT:    fmadd h0, h0, h1, h2
40; CHECK-SD-FP16-NEXT:    ret
41;
42; CHECK-GI-NOFP16-LABEL: fma_f16:
43; CHECK-GI-NOFP16:       // %bb.0: // %entry
44; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
45; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
46; CHECK-GI-NOFP16-NEXT:    fcvt s2, h2
47; CHECK-GI-NOFP16-NEXT:    fmadd s0, s0, s1, s2
48; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
49; CHECK-GI-NOFP16-NEXT:    ret
50;
51; CHECK-GI-FP16-LABEL: fma_f16:
52; CHECK-GI-FP16:       // %bb.0: // %entry
53; CHECK-GI-FP16-NEXT:    fmadd h0, h0, h1, h2
54; CHECK-GI-FP16-NEXT:    ret
55entry:
56  %d = call half @llvm.fma.f16(half %a, half %b, half %c)
57  ret half %d
58}
59
60define <2 x double> @fma_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
61; CHECK-LABEL: fma_v2f64:
62; CHECK:       // %bb.0: // %entry
63; CHECK-NEXT:    fmla v2.2d, v1.2d, v0.2d
64; CHECK-NEXT:    mov v0.16b, v2.16b
65; CHECK-NEXT:    ret
66entry:
67  %d = call <2 x double> @llvm.fma.v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c)
68  ret <2 x double> %d
69}
70
71define <3 x double> @fma_v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c) {
72; CHECK-SD-LABEL: fma_v3f64:
73; CHECK-SD:       // %bb.0: // %entry
74; CHECK-SD-NEXT:    // kill: def $d6 killed $d6 def $q6
75; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
76; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
77; CHECK-SD-NEXT:    // kill: def $d7 killed $d7 def $q7
78; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
79; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
80; CHECK-SD-NEXT:    // kill: def $d5 killed $d5 def $q5
81; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
82; CHECK-SD-NEXT:    mov v3.d[1], v4.d[0]
83; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
84; CHECK-SD-NEXT:    mov v6.d[1], v7.d[0]
85; CHECK-SD-NEXT:    fmla v6.2d, v3.2d, v0.2d
86; CHECK-SD-NEXT:    ldr d3, [sp]
87; CHECK-SD-NEXT:    fmla v3.2d, v5.2d, v2.2d
88; CHECK-SD-NEXT:    fmov d0, d6
89; CHECK-SD-NEXT:    ext v1.16b, v6.16b, v6.16b, #8
90; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
91; CHECK-SD-NEXT:    fmov d2, d3
92; CHECK-SD-NEXT:    ret
93;
94; CHECK-GI-LABEL: fma_v3f64:
95; CHECK-GI:       // %bb.0: // %entry
96; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
97; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
98; CHECK-GI-NEXT:    // kill: def $d6 killed $d6 def $q6
99; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
100; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
101; CHECK-GI-NEXT:    // kill: def $d7 killed $d7 def $q7
102; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
103; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
104; CHECK-GI-NEXT:    mov v6.d[1], v7.d[0]
105; CHECK-GI-NEXT:    fmla v6.2d, v3.2d, v0.2d
106; CHECK-GI-NEXT:    ldr d0, [sp]
107; CHECK-GI-NEXT:    fmadd d2, d2, d5, d0
108; CHECK-GI-NEXT:    mov d1, v6.d[1]
109; CHECK-GI-NEXT:    fmov d0, d6
110; CHECK-GI-NEXT:    ret
111entry:
112  %d = call <3 x double> @llvm.fma.v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c)
113  ret <3 x double> %d
114}
115
116define <4 x double> @fma_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c) {
117; CHECK-LABEL: fma_v4f64:
118; CHECK:       // %bb.0: // %entry
119; CHECK-NEXT:    fmla v4.2d, v2.2d, v0.2d
120; CHECK-NEXT:    fmla v5.2d, v3.2d, v1.2d
121; CHECK-NEXT:    mov v0.16b, v4.16b
122; CHECK-NEXT:    mov v1.16b, v5.16b
123; CHECK-NEXT:    ret
124entry:
125  %d = call <4 x double> @llvm.fma.v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c)
126  ret <4 x double> %d
127}
128
129define <2 x float> @fma_v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
130; CHECK-LABEL: fma_v2f32:
131; CHECK:       // %bb.0: // %entry
132; CHECK-NEXT:    fmla v2.2s, v1.2s, v0.2s
133; CHECK-NEXT:    fmov d0, d2
134; CHECK-NEXT:    ret
135entry:
136  %d = call <2 x float> @llvm.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c)
137  ret <2 x float> %d
138}
139
140define <3 x float> @fma_v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c) {
141; CHECK-LABEL: fma_v3f32:
142; CHECK:       // %bb.0: // %entry
143; CHECK-NEXT:    fmla v2.4s, v1.4s, v0.4s
144; CHECK-NEXT:    mov v0.16b, v2.16b
145; CHECK-NEXT:    ret
146entry:
147  %d = call <3 x float> @llvm.fma.v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c)
148  ret <3 x float> %d
149}
150
151define <4 x float> @fma_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
152; CHECK-LABEL: fma_v4f32:
153; CHECK:       // %bb.0: // %entry
154; CHECK-NEXT:    fmla v2.4s, v1.4s, v0.4s
155; CHECK-NEXT:    mov v0.16b, v2.16b
156; CHECK-NEXT:    ret
157entry:
158  %d = call <4 x float> @llvm.fma.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c)
159  ret <4 x float> %d
160}
161
162define <8 x float> @fma_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) {
163; CHECK-LABEL: fma_v8f32:
164; CHECK:       // %bb.0: // %entry
165; CHECK-NEXT:    fmla v4.4s, v2.4s, v0.4s
166; CHECK-NEXT:    fmla v5.4s, v3.4s, v1.4s
167; CHECK-NEXT:    mov v0.16b, v4.16b
168; CHECK-NEXT:    mov v1.16b, v5.16b
169; CHECK-NEXT:    ret
170entry:
171  %d = call <8 x float> @llvm.fma.v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c)
172  ret <8 x float> %d
173}
174
175define <7 x half> @fma_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {
176; CHECK-SD-NOFP16-LABEL: fma_v7f16:
177; CHECK-SD-NOFP16:       // %bb.0: // %entry
178; CHECK-SD-NOFP16-NEXT:    mov h3, v2.h[1]
179; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[1]
180; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[1]
181; CHECK-SD-NOFP16-NEXT:    fcvt s6, h2
182; CHECK-SD-NOFP16-NEXT:    fcvt s7, h1
183; CHECK-SD-NOFP16-NEXT:    fcvt s16, h0
184; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[2]
185; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[2]
186; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
187; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
188; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
189; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
190; CHECK-SD-NOFP16-NEXT:    fmadd s6, s16, s7, s6
191; CHECK-SD-NOFP16-NEXT:    fcvt s7, h17
192; CHECK-SD-NOFP16-NEXT:    fcvt s16, h18
193; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
194; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[3]
195; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[3]
196; CHECK-SD-NOFP16-NEXT:    fmadd s4, s5, s4, s3
197; CHECK-SD-NOFP16-NEXT:    mov h5, v2.h[3]
198; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
199; CHECK-SD-NOFP16-NEXT:    fmadd s6, s17, s16, s7
200; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[4]
201; CHECK-SD-NOFP16-NEXT:    fcvt s7, h18
202; CHECK-SD-NOFP16-NEXT:    fcvt s16, h19
203; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[4]
204; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
205; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
206; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[4]
207; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
208; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
209; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
210; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v4.h[0]
211; CHECK-SD-NOFP16-NEXT:    mov h4, v2.h[5]
212; CHECK-SD-NOFP16-NEXT:    fmadd s5, s16, s7, s5
213; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
214; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
215; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
216; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
217; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[6]
218; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
219; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
220; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
221; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
222; CHECK-SD-NOFP16-NEXT:    fmadd s17, s19, s18, s17
223; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
224; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[6]
225; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
226; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
227; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
228; CHECK-SD-NOFP16-NEXT:    fmadd s4, s16, s7, s4
229; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
230; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
231; CHECK-SD-NOFP16-NEXT:    fcvt s6, h18
232; CHECK-SD-NOFP16-NEXT:    fcvt s7, h19
233; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
234; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
235; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
236; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
237; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
238; CHECK-SD-NOFP16-NEXT:    fmadd s5, s7, s6, s5
239; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v16.h[0]
240; CHECK-SD-NOFP16-NEXT:    fmadd s0, s0, s1, s2
241; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v4.h[0]
242; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
243; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
244; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v4.h[0]
245; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v0.h[0]
246; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v3.16b
247; CHECK-SD-NOFP16-NEXT:    ret
248;
249; CHECK-SD-FP16-LABEL: fma_v7f16:
250; CHECK-SD-FP16:       // %bb.0: // %entry
251; CHECK-SD-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
252; CHECK-SD-FP16-NEXT:    mov v0.16b, v2.16b
253; CHECK-SD-FP16-NEXT:    ret
254;
255; CHECK-GI-NOFP16-LABEL: fma_v7f16:
256; CHECK-GI-NOFP16:       // %bb.0: // %entry
257; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
258; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v1.4h
259; CHECK-GI-NOFP16-NEXT:    fcvtl v5.4s, v2.4h
260; CHECK-GI-NOFP16-NEXT:    mov v6.h[0], v0.h[4]
261; CHECK-GI-NOFP16-NEXT:    fmla v5.4s, v4.4s, v3.4s
262; CHECK-GI-NOFP16-NEXT:    mov v3.h[0], v1.h[4]
263; CHECK-GI-NOFP16-NEXT:    mov v4.h[0], v2.h[4]
264; CHECK-GI-NOFP16-NEXT:    mov v6.h[1], v0.h[5]
265; CHECK-GI-NOFP16-NEXT:    mov v3.h[1], v1.h[5]
266; CHECK-GI-NOFP16-NEXT:    mov v4.h[1], v2.h[5]
267; CHECK-GI-NOFP16-NEXT:    fcvtn v5.4h, v5.4s
268; CHECK-GI-NOFP16-NEXT:    mov v6.h[2], v0.h[6]
269; CHECK-GI-NOFP16-NEXT:    mov v3.h[2], v1.h[6]
270; CHECK-GI-NOFP16-NEXT:    mov v4.h[2], v2.h[6]
271; CHECK-GI-NOFP16-NEXT:    mov v0.h[0], v5.h[0]
272; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v6.4h
273; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v3.4h
274; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v4.4h
275; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], v5.h[1]
276; CHECK-GI-NOFP16-NEXT:    fmla v3.4s, v2.4s, v1.4s
277; CHECK-GI-NOFP16-NEXT:    mov v0.h[2], v5.h[2]
278; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
279; CHECK-GI-NOFP16-NEXT:    mov v0.h[3], v5.h[3]
280; CHECK-GI-NOFP16-NEXT:    mov v0.h[4], v1.h[0]
281; CHECK-GI-NOFP16-NEXT:    mov v0.h[5], v1.h[1]
282; CHECK-GI-NOFP16-NEXT:    mov v0.h[6], v1.h[2]
283; CHECK-GI-NOFP16-NEXT:    ret
284;
285; CHECK-GI-FP16-LABEL: fma_v7f16:
286; CHECK-GI-FP16:       // %bb.0: // %entry
287; CHECK-GI-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
288; CHECK-GI-FP16-NEXT:    mov v0.16b, v2.16b
289; CHECK-GI-FP16-NEXT:    ret
290entry:
291  %d = call <7 x half> @llvm.fma.v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c)
292  ret <7 x half> %d
293}
294
295define <4 x half> @fma_v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
296; CHECK-SD-NOFP16-LABEL: fma_v4f16:
297; CHECK-SD-NOFP16:       // %bb.0: // %entry
298; CHECK-SD-NOFP16-NEXT:    // kill: def $d2 killed $d2 def $q2
299; CHECK-SD-NOFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
300; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
301; CHECK-SD-NOFP16-NEXT:    mov h3, v2.h[1]
302; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[1]
303; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[1]
304; CHECK-SD-NOFP16-NEXT:    fcvt s6, h2
305; CHECK-SD-NOFP16-NEXT:    fcvt s7, h1
306; CHECK-SD-NOFP16-NEXT:    fcvt s16, h0
307; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[2]
308; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[2]
309; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
310; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[3]
311; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[3]
312; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
313; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
314; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
315; CHECK-SD-NOFP16-NEXT:    fmadd s6, s16, s7, s6
316; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[3]
317; CHECK-SD-NOFP16-NEXT:    fcvt s7, h19
318; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
319; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
320; CHECK-SD-NOFP16-NEXT:    fmadd s3, s5, s4, s3
321; CHECK-SD-NOFP16-NEXT:    fcvt s4, h17
322; CHECK-SD-NOFP16-NEXT:    fcvt s5, h18
323; CHECK-SD-NOFP16-NEXT:    fcvt h0, s6
324; CHECK-SD-NOFP16-NEXT:    fmadd s4, s7, s5, s4
325; CHECK-SD-NOFP16-NEXT:    fcvt h3, s3
326; CHECK-SD-NOFP16-NEXT:    fcvt s5, h16
327; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v3.h[0]
328; CHECK-SD-NOFP16-NEXT:    fcvt h3, s4
329; CHECK-SD-NOFP16-NEXT:    fmadd s1, s5, s1, s2
330; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], v3.h[0]
331; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
332; CHECK-SD-NOFP16-NEXT:    mov v0.h[3], v1.h[0]
333; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
334; CHECK-SD-NOFP16-NEXT:    ret
335;
336; CHECK-SD-FP16-LABEL: fma_v4f16:
337; CHECK-SD-FP16:       // %bb.0: // %entry
338; CHECK-SD-FP16-NEXT:    fmla v2.4h, v1.4h, v0.4h
339; CHECK-SD-FP16-NEXT:    fmov d0, d2
340; CHECK-SD-FP16-NEXT:    ret
341;
342; CHECK-GI-NOFP16-LABEL: fma_v4f16:
343; CHECK-GI-NOFP16:       // %bb.0: // %entry
344; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
345; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
346; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
347; CHECK-GI-NOFP16-NEXT:    fmla v2.4s, v1.4s, v0.4s
348; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
349; CHECK-GI-NOFP16-NEXT:    ret
350;
351; CHECK-GI-FP16-LABEL: fma_v4f16:
352; CHECK-GI-FP16:       // %bb.0: // %entry
353; CHECK-GI-FP16-NEXT:    fmla v2.4h, v1.4h, v0.4h
354; CHECK-GI-FP16-NEXT:    fmov d0, d2
355; CHECK-GI-FP16-NEXT:    ret
356entry:
357  %d = call <4 x half> @llvm.fma.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c)
358  ret <4 x half> %d
359}
360
361define <8 x half> @fma_v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
362; CHECK-SD-NOFP16-LABEL: fma_v8f16:
363; CHECK-SD-NOFP16:       // %bb.0: // %entry
364; CHECK-SD-NOFP16-NEXT:    mov h3, v2.h[1]
365; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[1]
366; CHECK-SD-NOFP16-NEXT:    mov h5, v0.h[1]
367; CHECK-SD-NOFP16-NEXT:    fcvt s6, h2
368; CHECK-SD-NOFP16-NEXT:    fcvt s7, h1
369; CHECK-SD-NOFP16-NEXT:    fcvt s16, h0
370; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[2]
371; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[2]
372; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[2]
373; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
374; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
375; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
376; CHECK-SD-NOFP16-NEXT:    fmadd s6, s16, s7, s6
377; CHECK-SD-NOFP16-NEXT:    fcvt s7, h17
378; CHECK-SD-NOFP16-NEXT:    fcvt s16, h18
379; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
380; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[3]
381; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[3]
382; CHECK-SD-NOFP16-NEXT:    fmadd s4, s5, s4, s3
383; CHECK-SD-NOFP16-NEXT:    mov h5, v2.h[3]
384; CHECK-SD-NOFP16-NEXT:    fcvt h3, s6
385; CHECK-SD-NOFP16-NEXT:    fmadd s6, s17, s16, s7
386; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[4]
387; CHECK-SD-NOFP16-NEXT:    fcvt s7, h18
388; CHECK-SD-NOFP16-NEXT:    fcvt s16, h19
389; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[4]
390; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
391; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
392; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[4]
393; CHECK-SD-NOFP16-NEXT:    fcvt h6, s6
394; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
395; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
396; CHECK-SD-NOFP16-NEXT:    mov v3.h[1], v4.h[0]
397; CHECK-SD-NOFP16-NEXT:    mov h4, v2.h[5]
398; CHECK-SD-NOFP16-NEXT:    fmadd s5, s16, s7, s5
399; CHECK-SD-NOFP16-NEXT:    mov h7, v1.h[5]
400; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[5]
401; CHECK-SD-NOFP16-NEXT:    fcvt s19, h19
402; CHECK-SD-NOFP16-NEXT:    mov v3.h[2], v6.h[0]
403; CHECK-SD-NOFP16-NEXT:    mov h6, v2.h[6]
404; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
405; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
406; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
407; CHECK-SD-NOFP16-NEXT:    fcvt h5, s5
408; CHECK-SD-NOFP16-NEXT:    fmadd s17, s19, s18, s17
409; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[6]
410; CHECK-SD-NOFP16-NEXT:    mov h19, v0.h[6]
411; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
412; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
413; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
414; CHECK-SD-NOFP16-NEXT:    fmadd s4, s16, s7, s4
415; CHECK-SD-NOFP16-NEXT:    mov v3.h[3], v5.h[0]
416; CHECK-SD-NOFP16-NEXT:    fcvt s5, h6
417; CHECK-SD-NOFP16-NEXT:    fcvt s6, h18
418; CHECK-SD-NOFP16-NEXT:    fcvt s7, h19
419; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
420; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
421; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
422; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
423; CHECK-SD-NOFP16-NEXT:    fcvt h4, s4
424; CHECK-SD-NOFP16-NEXT:    fmadd s5, s7, s6, s5
425; CHECK-SD-NOFP16-NEXT:    mov v3.h[4], v16.h[0]
426; CHECK-SD-NOFP16-NEXT:    fmadd s0, s0, s1, s2
427; CHECK-SD-NOFP16-NEXT:    mov v3.h[5], v4.h[0]
428; CHECK-SD-NOFP16-NEXT:    fcvt h4, s5
429; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
430; CHECK-SD-NOFP16-NEXT:    mov v3.h[6], v4.h[0]
431; CHECK-SD-NOFP16-NEXT:    mov v3.h[7], v0.h[0]
432; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v3.16b
433; CHECK-SD-NOFP16-NEXT:    ret
434;
435; CHECK-SD-FP16-LABEL: fma_v8f16:
436; CHECK-SD-FP16:       // %bb.0: // %entry
437; CHECK-SD-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
438; CHECK-SD-FP16-NEXT:    mov v0.16b, v2.16b
439; CHECK-SD-FP16-NEXT:    ret
440;
441; CHECK-GI-NOFP16-LABEL: fma_v8f16:
442; CHECK-GI-NOFP16:       // %bb.0: // %entry
443; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
444; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v1.4h
445; CHECK-GI-NOFP16-NEXT:    fcvtl v5.4s, v2.4h
446; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
447; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
448; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
449; CHECK-GI-NOFP16-NEXT:    fmla v5.4s, v4.4s, v3.4s
450; CHECK-GI-NOFP16-NEXT:    fmla v2.4s, v1.4s, v0.4s
451; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v5.4s
452; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
453; CHECK-GI-NOFP16-NEXT:    ret
454;
455; CHECK-GI-FP16-LABEL: fma_v8f16:
456; CHECK-GI-FP16:       // %bb.0: // %entry
457; CHECK-GI-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
458; CHECK-GI-FP16-NEXT:    mov v0.16b, v2.16b
459; CHECK-GI-FP16-NEXT:    ret
460entry:
461  %d = call <8 x half> @llvm.fma.v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c)
462  ret <8 x half> %d
463}
464
465define <16 x half> @fma_v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c) {
466; CHECK-SD-NOFP16-LABEL: fma_v16f16:
467; CHECK-SD-NOFP16:       // %bb.0: // %entry
468; CHECK-SD-NOFP16-NEXT:    mov h6, v4.h[1]
469; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[1]
470; CHECK-SD-NOFP16-NEXT:    mov h16, v0.h[1]
471; CHECK-SD-NOFP16-NEXT:    fcvt s17, h4
472; CHECK-SD-NOFP16-NEXT:    fcvt s18, h2
473; CHECK-SD-NOFP16-NEXT:    fcvt s19, h0
474; CHECK-SD-NOFP16-NEXT:    mov h20, v4.h[2]
475; CHECK-SD-NOFP16-NEXT:    mov h21, v2.h[2]
476; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[2]
477; CHECK-SD-NOFP16-NEXT:    mov h23, v4.h[3]
478; CHECK-SD-NOFP16-NEXT:    mov h24, v2.h[3]
479; CHECK-SD-NOFP16-NEXT:    mov h25, v0.h[3]
480; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
481; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
482; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
483; CHECK-SD-NOFP16-NEXT:    fmadd s17, s19, s18, s17
484; CHECK-SD-NOFP16-NEXT:    mov h26, v1.h[1]
485; CHECK-SD-NOFP16-NEXT:    fcvt s27, h5
486; CHECK-SD-NOFP16-NEXT:    fcvt s18, h20
487; CHECK-SD-NOFP16-NEXT:    fcvt s19, h21
488; CHECK-SD-NOFP16-NEXT:    fcvt s20, h22
489; CHECK-SD-NOFP16-NEXT:    fcvt s21, h23
490; CHECK-SD-NOFP16-NEXT:    fcvt s22, h24
491; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
492; CHECK-SD-NOFP16-NEXT:    fmadd s7, s16, s7, s6
493; CHECK-SD-NOFP16-NEXT:    mov h24, v5.h[1]
494; CHECK-SD-NOFP16-NEXT:    mov h25, v3.h[1]
495; CHECK-SD-NOFP16-NEXT:    fcvt h6, s17
496; CHECK-SD-NOFP16-NEXT:    fcvt s28, h3
497; CHECK-SD-NOFP16-NEXT:    fcvt s29, h1
498; CHECK-SD-NOFP16-NEXT:    fmadd s19, s20, s19, s18
499; CHECK-SD-NOFP16-NEXT:    fcvt s26, h26
500; CHECK-SD-NOFP16-NEXT:    mov h16, v4.h[4]
501; CHECK-SD-NOFP16-NEXT:    fmadd s21, s23, s22, s21
502; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[2]
503; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[2]
504; CHECK-SD-NOFP16-NEXT:    fcvt h20, s7
505; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
506; CHECK-SD-NOFP16-NEXT:    fcvt s25, h25
507; CHECK-SD-NOFP16-NEXT:    mov h17, v2.h[4]
508; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[4]
509; CHECK-SD-NOFP16-NEXT:    mov h7, v4.h[5]
510; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
511; CHECK-SD-NOFP16-NEXT:    mov h30, v2.h[5]
512; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
513; CHECK-SD-NOFP16-NEXT:    fcvt h21, s21
514; CHECK-SD-NOFP16-NEXT:    mov h31, v1.h[4]
515; CHECK-SD-NOFP16-NEXT:    fmadd s24, s26, s25, s24
516; CHECK-SD-NOFP16-NEXT:    fmadd s25, s29, s28, s27
517; CHECK-SD-NOFP16-NEXT:    mov v6.h[1], v20.h[0]
518; CHECK-SD-NOFP16-NEXT:    mov h20, v5.h[2]
519; CHECK-SD-NOFP16-NEXT:    mov h26, v5.h[3]
520; CHECK-SD-NOFP16-NEXT:    mov h27, v3.h[3]
521; CHECK-SD-NOFP16-NEXT:    mov h28, v1.h[3]
522; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
523; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
524; CHECK-SD-NOFP16-NEXT:    fcvt s29, h7
525; CHECK-SD-NOFP16-NEXT:    fcvt s30, h30
526; CHECK-SD-NOFP16-NEXT:    mov v6.h[2], v19.h[0]
527; CHECK-SD-NOFP16-NEXT:    fcvt h24, s24
528; CHECK-SD-NOFP16-NEXT:    fcvt h7, s25
529; CHECK-SD-NOFP16-NEXT:    fcvt s19, h20
530; CHECK-SD-NOFP16-NEXT:    fcvt s20, h22
531; CHECK-SD-NOFP16-NEXT:    fcvt s22, h23
532; CHECK-SD-NOFP16-NEXT:    fmadd s16, s18, s17, s16
533; CHECK-SD-NOFP16-NEXT:    mov h23, v0.h[5]
534; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
535; CHECK-SD-NOFP16-NEXT:    fcvt s26, h27
536; CHECK-SD-NOFP16-NEXT:    fcvt s27, h28
537; CHECK-SD-NOFP16-NEXT:    mov h18, v4.h[6]
538; CHECK-SD-NOFP16-NEXT:    mov v6.h[3], v21.h[0]
539; CHECK-SD-NOFP16-NEXT:    mov v7.h[1], v24.h[0]
540; CHECK-SD-NOFP16-NEXT:    mov h24, v5.h[5]
541; CHECK-SD-NOFP16-NEXT:    fmadd s19, s22, s20, s19
542; CHECK-SD-NOFP16-NEXT:    mov h20, v5.h[4]
543; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[4]
544; CHECK-SD-NOFP16-NEXT:    fcvt s23, h23
545; CHECK-SD-NOFP16-NEXT:    mov h28, v0.h[6]
546; CHECK-SD-NOFP16-NEXT:    fcvt h16, s16
547; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
548; CHECK-SD-NOFP16-NEXT:    mov h4, v4.h[7]
549; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[7]
550; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
551; CHECK-SD-NOFP16-NEXT:    fcvt s21, h22
552; CHECK-SD-NOFP16-NEXT:    fcvt s22, h31
553; CHECK-SD-NOFP16-NEXT:    fmadd s17, s23, s30, s29
554; CHECK-SD-NOFP16-NEXT:    fmadd s23, s27, s26, s25
555; CHECK-SD-NOFP16-NEXT:    fcvt h19, s19
556; CHECK-SD-NOFP16-NEXT:    mov h25, v3.h[5]
557; CHECK-SD-NOFP16-NEXT:    mov h26, v1.h[5]
558; CHECK-SD-NOFP16-NEXT:    mov h27, v2.h[6]
559; CHECK-SD-NOFP16-NEXT:    mov h29, v1.h[6]
560; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[7]
561; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[7]
562; CHECK-SD-NOFP16-NEXT:    fmadd s20, s22, s21, s20
563; CHECK-SD-NOFP16-NEXT:    mov h21, v5.h[6]
564; CHECK-SD-NOFP16-NEXT:    mov h22, v3.h[6]
565; CHECK-SD-NOFP16-NEXT:    mov v7.h[2], v19.h[0]
566; CHECK-SD-NOFP16-NEXT:    fcvt h19, s23
567; CHECK-SD-NOFP16-NEXT:    fcvt s23, h24
568; CHECK-SD-NOFP16-NEXT:    fcvt s24, h25
569; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
570; CHECK-SD-NOFP16-NEXT:    fcvt s26, h27
571; CHECK-SD-NOFP16-NEXT:    fcvt s27, h28
572; CHECK-SD-NOFP16-NEXT:    fcvt s28, h29
573; CHECK-SD-NOFP16-NEXT:    mov h5, v5.h[7]
574; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
575; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
576; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[7]
577; CHECK-SD-NOFP16-NEXT:    mov v7.h[3], v19.h[0]
578; CHECK-SD-NOFP16-NEXT:    fcvt h19, s20
579; CHECK-SD-NOFP16-NEXT:    mov v6.h[4], v16.h[0]
580; CHECK-SD-NOFP16-NEXT:    fmadd s20, s25, s24, s23
581; CHECK-SD-NOFP16-NEXT:    fcvt h16, s17
582; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
583; CHECK-SD-NOFP16-NEXT:    fmadd s18, s27, s26, s18
584; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
585; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
586; CHECK-SD-NOFP16-NEXT:    fmadd s21, s28, s22, s21
587; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
588; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
589; CHECK-SD-NOFP16-NEXT:    mov v7.h[4], v19.h[0]
590; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
591; CHECK-SD-NOFP16-NEXT:    fcvt h17, s20
592; CHECK-SD-NOFP16-NEXT:    mov v6.h[5], v16.h[0]
593; CHECK-SD-NOFP16-NEXT:    fmadd s0, s0, s2, s4
594; CHECK-SD-NOFP16-NEXT:    fcvt h2, s18
595; CHECK-SD-NOFP16-NEXT:    fcvt h4, s21
596; CHECK-SD-NOFP16-NEXT:    fmadd s1, s1, s3, s5
597; CHECK-SD-NOFP16-NEXT:    mov v7.h[5], v17.h[0]
598; CHECK-SD-NOFP16-NEXT:    mov v6.h[6], v2.h[0]
599; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
600; CHECK-SD-NOFP16-NEXT:    fcvt h1, s1
601; CHECK-SD-NOFP16-NEXT:    mov v7.h[6], v4.h[0]
602; CHECK-SD-NOFP16-NEXT:    mov v6.h[7], v0.h[0]
603; CHECK-SD-NOFP16-NEXT:    mov v7.h[7], v1.h[0]
604; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v6.16b
605; CHECK-SD-NOFP16-NEXT:    mov v1.16b, v7.16b
606; CHECK-SD-NOFP16-NEXT:    ret
607;
608; CHECK-SD-FP16-LABEL: fma_v16f16:
609; CHECK-SD-FP16:       // %bb.0: // %entry
610; CHECK-SD-FP16-NEXT:    fmla v4.8h, v2.8h, v0.8h
611; CHECK-SD-FP16-NEXT:    fmla v5.8h, v3.8h, v1.8h
612; CHECK-SD-FP16-NEXT:    mov v0.16b, v4.16b
613; CHECK-SD-FP16-NEXT:    mov v1.16b, v5.16b
614; CHECK-SD-FP16-NEXT:    ret
615;
616; CHECK-GI-NOFP16-LABEL: fma_v16f16:
617; CHECK-GI-NOFP16:       // %bb.0: // %entry
618; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v0.4h
619; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v1.4h
620; CHECK-GI-NOFP16-NEXT:    fcvtl v16.4s, v2.4h
621; CHECK-GI-NOFP16-NEXT:    fcvtl v17.4s, v3.4h
622; CHECK-GI-NOFP16-NEXT:    fcvtl v18.4s, v4.4h
623; CHECK-GI-NOFP16-NEXT:    fcvtl v19.4s, v5.4h
624; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
625; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
626; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
627; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
628; CHECK-GI-NOFP16-NEXT:    fcvtl2 v4.4s, v4.8h
629; CHECK-GI-NOFP16-NEXT:    fcvtl2 v5.4s, v5.8h
630; CHECK-GI-NOFP16-NEXT:    fmla v18.4s, v16.4s, v6.4s
631; CHECK-GI-NOFP16-NEXT:    fmla v19.4s, v17.4s, v7.4s
632; CHECK-GI-NOFP16-NEXT:    fmla v4.4s, v2.4s, v0.4s
633; CHECK-GI-NOFP16-NEXT:    fmla v5.4s, v3.4s, v1.4s
634; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v18.4s
635; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v19.4s
636; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v4.4s
637; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v5.4s
638; CHECK-GI-NOFP16-NEXT:    ret
639;
640; CHECK-GI-FP16-LABEL: fma_v16f16:
641; CHECK-GI-FP16:       // %bb.0: // %entry
642; CHECK-GI-FP16-NEXT:    fmla v4.8h, v2.8h, v0.8h
643; CHECK-GI-FP16-NEXT:    fmla v5.8h, v3.8h, v1.8h
644; CHECK-GI-FP16-NEXT:    mov v0.16b, v4.16b
645; CHECK-GI-FP16-NEXT:    mov v1.16b, v5.16b
646; CHECK-GI-FP16-NEXT:    ret
647entry:
648  %d = call <16 x half> @llvm.fma.v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c)
649  ret <16 x half> %d
650}
651
652define double @fmuladd_f64(double %a, double %b, double %c) {
653; CHECK-LABEL: fmuladd_f64:
654; CHECK:       // %bb.0: // %entry
655; CHECK-NEXT:    fmadd d0, d0, d1, d2
656; CHECK-NEXT:    ret
657entry:
658  %d = call double @llvm.fmuladd.f64(double %a, double %b, double %c)
659  ret double %d
660}
661
662define float @fmuladd_f32(float %a, float %b, float %c) {
663; CHECK-LABEL: fmuladd_f32:
664; CHECK:       // %bb.0: // %entry
665; CHECK-NEXT:    fmadd s0, s0, s1, s2
666; CHECK-NEXT:    ret
667entry:
668  %d = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
669  ret float %d
670}
671
672define half @fmuladd_f16(half %a, half %b, half %c) {
673; CHECK-SD-NOFP16-LABEL: fmuladd_f16:
674; CHECK-SD-NOFP16:       // %bb.0: // %entry
675; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
676; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
677; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
678; CHECK-SD-NOFP16-NEXT:    fcvt s1, h2
679; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
680; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
681; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
682; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
683; CHECK-SD-NOFP16-NEXT:    ret
684;
685; CHECK-SD-FP16-LABEL: fmuladd_f16:
686; CHECK-SD-FP16:       // %bb.0: // %entry
687; CHECK-SD-FP16-NEXT:    fmadd h0, h0, h1, h2
688; CHECK-SD-FP16-NEXT:    ret
689;
690; CHECK-GI-NOFP16-LABEL: fmuladd_f16:
691; CHECK-GI-NOFP16:       // %bb.0: // %entry
692; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
693; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
694; CHECK-GI-NOFP16-NEXT:    fmul s0, s0, s1
695; CHECK-GI-NOFP16-NEXT:    fcvt s1, h2
696; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
697; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
698; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s1
699; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
700; CHECK-GI-NOFP16-NEXT:    ret
701;
702; CHECK-GI-FP16-LABEL: fmuladd_f16:
703; CHECK-GI-FP16:       // %bb.0: // %entry
704; CHECK-GI-FP16-NEXT:    fmadd h0, h0, h1, h2
705; CHECK-GI-FP16-NEXT:    ret
706entry:
707  %d = call half @llvm.fmuladd.f16(half %a, half %b, half %c)
708  ret half %d
709}
710
711define <2 x double> @fmuladd_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
712; CHECK-LABEL: fmuladd_v2f64:
713; CHECK:       // %bb.0: // %entry
714; CHECK-NEXT:    fmla v2.2d, v1.2d, v0.2d
715; CHECK-NEXT:    mov v0.16b, v2.16b
716; CHECK-NEXT:    ret
717entry:
718  %d = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c)
719  ret <2 x double> %d
720}
721
722define <3 x double> @fmuladd_v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c) {
723; CHECK-SD-LABEL: fmuladd_v3f64:
724; CHECK-SD:       // %bb.0: // %entry
725; CHECK-SD-NEXT:    // kill: def $d6 killed $d6 def $q6
726; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
727; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
728; CHECK-SD-NEXT:    // kill: def $d7 killed $d7 def $q7
729; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
730; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
731; CHECK-SD-NEXT:    // kill: def $d5 killed $d5 def $q5
732; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
733; CHECK-SD-NEXT:    mov v3.d[1], v4.d[0]
734; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
735; CHECK-SD-NEXT:    mov v6.d[1], v7.d[0]
736; CHECK-SD-NEXT:    fmla v6.2d, v3.2d, v0.2d
737; CHECK-SD-NEXT:    ldr d3, [sp]
738; CHECK-SD-NEXT:    fmla v3.2d, v5.2d, v2.2d
739; CHECK-SD-NEXT:    fmov d0, d6
740; CHECK-SD-NEXT:    ext v1.16b, v6.16b, v6.16b, #8
741; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
742; CHECK-SD-NEXT:    fmov d2, d3
743; CHECK-SD-NEXT:    ret
744;
745; CHECK-GI-LABEL: fmuladd_v3f64:
746; CHECK-GI:       // %bb.0: // %entry
747; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
748; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
749; CHECK-GI-NEXT:    // kill: def $d6 killed $d6 def $q6
750; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
751; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
752; CHECK-GI-NEXT:    // kill: def $d7 killed $d7 def $q7
753; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
754; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
755; CHECK-GI-NEXT:    mov v6.d[1], v7.d[0]
756; CHECK-GI-NEXT:    fmla v6.2d, v3.2d, v0.2d
757; CHECK-GI-NEXT:    ldr d0, [sp]
758; CHECK-GI-NEXT:    fmadd d2, d2, d5, d0
759; CHECK-GI-NEXT:    mov d1, v6.d[1]
760; CHECK-GI-NEXT:    fmov d0, d6
761; CHECK-GI-NEXT:    ret
762entry:
763  %d = call <3 x double> @llvm.fmuladd.v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c)
764  ret <3 x double> %d
765}
766
767define <4 x double> @fmuladd_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c) {
768; CHECK-LABEL: fmuladd_v4f64:
769; CHECK:       // %bb.0: // %entry
770; CHECK-NEXT:    fmla v4.2d, v2.2d, v0.2d
771; CHECK-NEXT:    fmla v5.2d, v3.2d, v1.2d
772; CHECK-NEXT:    mov v0.16b, v4.16b
773; CHECK-NEXT:    mov v1.16b, v5.16b
774; CHECK-NEXT:    ret
775entry:
776  %d = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c)
777  ret <4 x double> %d
778}
779
780define <2 x float> @fmuladd_v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
781; CHECK-LABEL: fmuladd_v2f32:
782; CHECK:       // %bb.0: // %entry
783; CHECK-NEXT:    fmla v2.2s, v1.2s, v0.2s
784; CHECK-NEXT:    fmov d0, d2
785; CHECK-NEXT:    ret
786entry:
787  %d = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c)
788  ret <2 x float> %d
789}
790
791define <3 x float> @fmuladd_v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c) {
792; CHECK-LABEL: fmuladd_v3f32:
793; CHECK:       // %bb.0: // %entry
794; CHECK-NEXT:    fmla v2.4s, v1.4s, v0.4s
795; CHECK-NEXT:    mov v0.16b, v2.16b
796; CHECK-NEXT:    ret
797entry:
798  %d = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c)
799  ret <3 x float> %d
800}
801
802define <4 x float> @fmuladd_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
803; CHECK-LABEL: fmuladd_v4f32:
804; CHECK:       // %bb.0: // %entry
805; CHECK-NEXT:    fmla v2.4s, v1.4s, v0.4s
806; CHECK-NEXT:    mov v0.16b, v2.16b
807; CHECK-NEXT:    ret
808entry:
809  %d = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c)
810  ret <4 x float> %d
811}
812
813define <8 x float> @fmuladd_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) {
814; CHECK-LABEL: fmuladd_v8f32:
815; CHECK:       // %bb.0: // %entry
816; CHECK-NEXT:    fmla v4.4s, v2.4s, v0.4s
817; CHECK-NEXT:    fmla v5.4s, v3.4s, v1.4s
818; CHECK-NEXT:    mov v0.16b, v4.16b
819; CHECK-NEXT:    mov v1.16b, v5.16b
820; CHECK-NEXT:    ret
821entry:
822  %d = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c)
823  ret <8 x float> %d
824}
825
826define <7 x half> @fmuladd_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {
827; CHECK-SD-NOFP16-LABEL: fmuladd_v7f16:
828; CHECK-SD-NOFP16:       // %bb.0: // %entry
829; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
830; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
831; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
832; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
833; CHECK-SD-NOFP16-NEXT:    fmul v3.4s, v4.4s, v3.4s
834; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
835; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
836; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
837; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
838; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
839; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v1.4h
840; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
841; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v3.4s
842; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v1.4s, v2.4s
843; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
844; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
845; CHECK-SD-NOFP16-NEXT:    ret
846;
847; CHECK-SD-FP16-LABEL: fmuladd_v7f16:
848; CHECK-SD-FP16:       // %bb.0: // %entry
849; CHECK-SD-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
850; CHECK-SD-FP16-NEXT:    mov v0.16b, v2.16b
851; CHECK-SD-FP16-NEXT:    ret
852;
853; CHECK-GI-NOFP16-LABEL: fmuladd_v7f16:
854; CHECK-GI-NOFP16:       // %bb.0: // %entry
855; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
856; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v1.4h
857; CHECK-GI-NOFP16-NEXT:    mov v5.h[0], v0.h[4]
858; CHECK-GI-NOFP16-NEXT:    fmul v3.4s, v3.4s, v4.4s
859; CHECK-GI-NOFP16-NEXT:    mov v4.h[0], v1.h[4]
860; CHECK-GI-NOFP16-NEXT:    mov v5.h[1], v0.h[5]
861; CHECK-GI-NOFP16-NEXT:    mov v4.h[1], v1.h[5]
862; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v3.4s
863; CHECK-GI-NOFP16-NEXT:    mov v5.h[2], v0.h[6]
864; CHECK-GI-NOFP16-NEXT:    mov v4.h[2], v1.h[6]
865; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v3.4h
866; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
867; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v5.4h
868; CHECK-GI-NOFP16-NEXT:    mov v5.h[0], v2.h[4]
869; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v4.4h
870; CHECK-GI-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
871; CHECK-GI-NOFP16-NEXT:    mov v5.h[1], v2.h[5]
872; CHECK-GI-NOFP16-NEXT:    fmul v1.4s, v3.4s, v4.4s
873; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v0.4s
874; CHECK-GI-NOFP16-NEXT:    mov v5.h[2], v2.h[6]
875; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
876; CHECK-GI-NOFP16-NEXT:    mov v0.h[0], v3.h[0]
877; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v5.4h
878; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
879; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], v3.h[1]
880; CHECK-GI-NOFP16-NEXT:    fadd v1.4s, v1.4s, v2.4s
881; CHECK-GI-NOFP16-NEXT:    mov v0.h[2], v3.h[2]
882; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
883; CHECK-GI-NOFP16-NEXT:    mov v0.h[3], v3.h[3]
884; CHECK-GI-NOFP16-NEXT:    mov v0.h[4], v1.h[0]
885; CHECK-GI-NOFP16-NEXT:    mov v0.h[5], v1.h[1]
886; CHECK-GI-NOFP16-NEXT:    mov v0.h[6], v1.h[2]
887; CHECK-GI-NOFP16-NEXT:    ret
888;
889; CHECK-GI-FP16-LABEL: fmuladd_v7f16:
890; CHECK-GI-FP16:       // %bb.0: // %entry
891; CHECK-GI-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
892; CHECK-GI-FP16-NEXT:    mov v0.16b, v2.16b
893; CHECK-GI-FP16-NEXT:    ret
894entry:
895  %d = call <7 x half> @llvm.fmuladd.v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c)
896  ret <7 x half> %d
897}
898
899define <4 x half> @fmuladd_v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
900; CHECK-SD-NOFP16-LABEL: fmuladd_v4f16:
901; CHECK-SD-NOFP16:       // %bb.0: // %entry
902; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
903; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
904; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
905; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
906; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
907; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
908; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
909; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
910; CHECK-SD-NOFP16-NEXT:    ret
911;
912; CHECK-SD-FP16-LABEL: fmuladd_v4f16:
913; CHECK-SD-FP16:       // %bb.0: // %entry
914; CHECK-SD-FP16-NEXT:    fmla v2.4h, v1.4h, v0.4h
915; CHECK-SD-FP16-NEXT:    fmov d0, d2
916; CHECK-SD-FP16-NEXT:    ret
917;
918; CHECK-GI-NOFP16-LABEL: fmuladd_v4f16:
919; CHECK-GI-NOFP16:       // %bb.0: // %entry
920; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
921; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
922; CHECK-GI-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
923; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
924; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
925; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
926; CHECK-GI-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
927; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
928; CHECK-GI-NOFP16-NEXT:    ret
929;
930; CHECK-GI-FP16-LABEL: fmuladd_v4f16:
931; CHECK-GI-FP16:       // %bb.0: // %entry
932; CHECK-GI-FP16-NEXT:    fmla v2.4h, v1.4h, v0.4h
933; CHECK-GI-FP16-NEXT:    fmov d0, d2
934; CHECK-GI-FP16-NEXT:    ret
935entry:
936  %d = call <4 x half> @llvm.fmuladd.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c)
937  ret <4 x half> %d
938}
939
940define <8 x half> @fmuladd_v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
941; CHECK-SD-NOFP16-LABEL: fmuladd_v8f16:
942; CHECK-SD-NOFP16:       // %bb.0: // %entry
943; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
944; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
945; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
946; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
947; CHECK-SD-NOFP16-NEXT:    fmul v3.4s, v4.4s, v3.4s
948; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
949; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
950; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
951; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
952; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
953; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v1.4h
954; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
955; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v3.4s
956; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v1.4s, v2.4s
957; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
958; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
959; CHECK-SD-NOFP16-NEXT:    ret
960;
961; CHECK-SD-FP16-LABEL: fmuladd_v8f16:
962; CHECK-SD-FP16:       // %bb.0: // %entry
963; CHECK-SD-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
964; CHECK-SD-FP16-NEXT:    mov v0.16b, v2.16b
965; CHECK-SD-FP16-NEXT:    ret
966;
967; CHECK-GI-NOFP16-LABEL: fmuladd_v8f16:
968; CHECK-GI-NOFP16:       // %bb.0: // %entry
969; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
970; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v1.4h
971; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
972; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
973; CHECK-GI-NOFP16-NEXT:    fmul v3.4s, v3.4s, v4.4s
974; CHECK-GI-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
975; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
976; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
977; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
978; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
979; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
980; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
981; CHECK-GI-NOFP16-NEXT:    fadd v1.4s, v1.4s, v3.4s
982; CHECK-GI-NOFP16-NEXT:    fadd v2.4s, v0.4s, v2.4s
983; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
984; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
985; CHECK-GI-NOFP16-NEXT:    ret
986;
987; CHECK-GI-FP16-LABEL: fmuladd_v8f16:
988; CHECK-GI-FP16:       // %bb.0: // %entry
989; CHECK-GI-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
990; CHECK-GI-FP16-NEXT:    mov v0.16b, v2.16b
991; CHECK-GI-FP16-NEXT:    ret
992entry:
993  %d = call <8 x half> @llvm.fmuladd.v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c)
994  ret <8 x half> %d
995}
996
997define <16 x half> @fmuladd_v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c) {
998; CHECK-SD-NOFP16-LABEL: fmuladd_v16f16:
999; CHECK-SD-NOFP16:       // %bb.0: // %entry
1000; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v2.4h
1001; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v0.4h
1002; CHECK-SD-NOFP16-NEXT:    fcvtl v16.4s, v3.4h
1003; CHECK-SD-NOFP16-NEXT:    fcvtl v17.4s, v1.4h
1004; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
1005; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
1006; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
1007; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
1008; CHECK-SD-NOFP16-NEXT:    fmul v6.4s, v7.4s, v6.4s
1009; CHECK-SD-NOFP16-NEXT:    fmul v7.4s, v17.4s, v16.4s
1010; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v2.4s
1011; CHECK-SD-NOFP16-NEXT:    fmul v1.4s, v1.4s, v3.4s
1012; CHECK-SD-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
1013; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v5.4h
1014; CHECK-SD-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
1015; CHECK-SD-NOFP16-NEXT:    fcvtn2 v2.8h, v0.4s
1016; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v4.4h
1017; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v4.8h
1018; CHECK-SD-NOFP16-NEXT:    fcvtn2 v3.8h, v1.4s
1019; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
1020; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
1021; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v3.4h
1022; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v1.4s, v0.4s
1023; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v3.8h
1024; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v5.8h
1025; CHECK-SD-NOFP16-NEXT:    fadd v5.4s, v7.4s, v6.4s
1026; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v2.4s, v4.4s
1027; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
1028; CHECK-SD-NOFP16-NEXT:    fadd v3.4s, v1.4s, v3.4s
1029; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
1030; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
1031; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
1032; CHECK-SD-NOFP16-NEXT:    ret
1033;
1034; CHECK-SD-FP16-LABEL: fmuladd_v16f16:
1035; CHECK-SD-FP16:       // %bb.0: // %entry
1036; CHECK-SD-FP16-NEXT:    fmla v4.8h, v2.8h, v0.8h
1037; CHECK-SD-FP16-NEXT:    fmla v5.8h, v3.8h, v1.8h
1038; CHECK-SD-FP16-NEXT:    mov v0.16b, v4.16b
1039; CHECK-SD-FP16-NEXT:    mov v1.16b, v5.16b
1040; CHECK-SD-FP16-NEXT:    ret
1041;
1042; CHECK-GI-NOFP16-LABEL: fmuladd_v16f16:
1043; CHECK-GI-NOFP16:       // %bb.0: // %entry
1044; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v0.4h
1045; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v2.4h
1046; CHECK-GI-NOFP16-NEXT:    fcvtl v16.4s, v1.4h
1047; CHECK-GI-NOFP16-NEXT:    fcvtl v17.4s, v3.4h
1048; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
1049; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
1050; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
1051; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
1052; CHECK-GI-NOFP16-NEXT:    fmul v6.4s, v6.4s, v7.4s
1053; CHECK-GI-NOFP16-NEXT:    fmul v7.4s, v16.4s, v17.4s
1054; CHECK-GI-NOFP16-NEXT:    fmul v0.4s, v0.4s, v2.4s
1055; CHECK-GI-NOFP16-NEXT:    fmul v1.4s, v1.4s, v3.4s
1056; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
1057; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v4.4h
1058; CHECK-GI-NOFP16-NEXT:    fcvtl2 v4.4s, v4.8h
1059; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
1060; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v5.4h
1061; CHECK-GI-NOFP16-NEXT:    fcvtl2 v5.4s, v5.8h
1062; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
1063; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
1064; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
1065; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
1066; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
1067; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
1068; CHECK-GI-NOFP16-NEXT:    fadd v2.4s, v2.4s, v6.4s
1069; CHECK-GI-NOFP16-NEXT:    fadd v3.4s, v3.4s, v7.4s
1070; CHECK-GI-NOFP16-NEXT:    fadd v4.4s, v0.4s, v4.4s
1071; CHECK-GI-NOFP16-NEXT:    fadd v5.4s, v1.4s, v5.4s
1072; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
1073; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
1074; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v4.4s
1075; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v5.4s
1076; CHECK-GI-NOFP16-NEXT:    ret
1077;
1078; CHECK-GI-FP16-LABEL: fmuladd_v16f16:
1079; CHECK-GI-FP16:       // %bb.0: // %entry
1080; CHECK-GI-FP16-NEXT:    fmla v4.8h, v2.8h, v0.8h
1081; CHECK-GI-FP16-NEXT:    fmla v5.8h, v3.8h, v1.8h
1082; CHECK-GI-FP16-NEXT:    mov v0.16b, v4.16b
1083; CHECK-GI-FP16-NEXT:    mov v1.16b, v5.16b
1084; CHECK-GI-FP16-NEXT:    ret
1085entry:
1086  %d = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c)
1087  ret <16 x half> %d
1088}
1089
1090define double @fmul_f64(double %a, double %b, double %c) {
1091; CHECK-LABEL: fmul_f64:
1092; CHECK:       // %bb.0: // %entry
1093; CHECK-NEXT:    fmadd d0, d0, d1, d2
1094; CHECK-NEXT:    ret
1095entry:
1096  %d = fmul fast double %a, %b
1097  %e = fadd fast double %d, %c
1098  ret double %e
1099}
1100
1101define float @fmul_f32(float %a, float %b, float %c) {
1102; CHECK-LABEL: fmul_f32:
1103; CHECK:       // %bb.0: // %entry
1104; CHECK-NEXT:    fmadd s0, s0, s1, s2
1105; CHECK-NEXT:    ret
1106entry:
1107  %d = fmul fast float %a, %b
1108  %e = fadd fast float %d, %c
1109  ret float %e
1110}
1111
1112define half @fmul_f16(half %a, half %b, half %c) {
1113; CHECK-SD-NOFP16-LABEL: fmul_f16:
1114; CHECK-SD-NOFP16:       // %bb.0: // %entry
1115; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
1116; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
1117; CHECK-SD-NOFP16-NEXT:    fmul s0, s0, s1
1118; CHECK-SD-NOFP16-NEXT:    fcvt s1, h2
1119; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
1120; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
1121; CHECK-SD-NOFP16-NEXT:    fadd s0, s0, s1
1122; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
1123; CHECK-SD-NOFP16-NEXT:    ret
1124;
1125; CHECK-SD-FP16-LABEL: fmul_f16:
1126; CHECK-SD-FP16:       // %bb.0: // %entry
1127; CHECK-SD-FP16-NEXT:    fmadd h0, h0, h1, h2
1128; CHECK-SD-FP16-NEXT:    ret
1129;
1130; CHECK-GI-NOFP16-LABEL: fmul_f16:
1131; CHECK-GI-NOFP16:       // %bb.0: // %entry
1132; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
1133; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
1134; CHECK-GI-NOFP16-NEXT:    fmul s0, s0, s1
1135; CHECK-GI-NOFP16-NEXT:    fcvt s1, h2
1136; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
1137; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
1138; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s1
1139; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
1140; CHECK-GI-NOFP16-NEXT:    ret
1141;
1142; CHECK-GI-FP16-LABEL: fmul_f16:
1143; CHECK-GI-FP16:       // %bb.0: // %entry
1144; CHECK-GI-FP16-NEXT:    fmadd h0, h0, h1, h2
1145; CHECK-GI-FP16-NEXT:    ret
1146entry:
1147  %d = fmul fast half %a, %b
1148  %e = fadd fast half %d, %c
1149  ret half %e
1150}
1151
1152define <2 x double> @fmul_v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
1153; CHECK-SD-LABEL: fmul_v2f64:
1154; CHECK-SD:       // %bb.0: // %entry
1155; CHECK-SD-NEXT:    fmla v2.2d, v1.2d, v0.2d
1156; CHECK-SD-NEXT:    mov v0.16b, v2.16b
1157; CHECK-SD-NEXT:    ret
1158;
1159; CHECK-GI-LABEL: fmul_v2f64:
1160; CHECK-GI:       // %bb.0: // %entry
1161; CHECK-GI-NEXT:    fmla v2.2d, v0.2d, v1.2d
1162; CHECK-GI-NEXT:    mov v0.16b, v2.16b
1163; CHECK-GI-NEXT:    ret
1164entry:
1165  %d = fmul fast <2 x double> %a, %b
1166  %e = fadd fast <2 x double> %d, %c
1167  ret <2 x double> %e
1168}
1169
1170define <3 x double> @fmul_v3f64(<3 x double> %a, <3 x double> %b, <3 x double> %c) {
1171; CHECK-SD-LABEL: fmul_v3f64:
1172; CHECK-SD:       // %bb.0: // %entry
1173; CHECK-SD-NEXT:    // kill: def $d6 killed $d6 def $q6
1174; CHECK-SD-NEXT:    // kill: def $d3 killed $d3 def $q3
1175; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
1176; CHECK-SD-NEXT:    // kill: def $d7 killed $d7 def $q7
1177; CHECK-SD-NEXT:    // kill: def $d4 killed $d4 def $q4
1178; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
1179; CHECK-SD-NEXT:    // kill: def $d5 killed $d5 def $q5
1180; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
1181; CHECK-SD-NEXT:    mov v3.d[1], v4.d[0]
1182; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
1183; CHECK-SD-NEXT:    mov v6.d[1], v7.d[0]
1184; CHECK-SD-NEXT:    fmla v6.2d, v3.2d, v0.2d
1185; CHECK-SD-NEXT:    ldr d3, [sp]
1186; CHECK-SD-NEXT:    fmla v3.2d, v5.2d, v2.2d
1187; CHECK-SD-NEXT:    fmov d0, d6
1188; CHECK-SD-NEXT:    ext v1.16b, v6.16b, v6.16b, #8
1189; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
1190; CHECK-SD-NEXT:    fmov d2, d3
1191; CHECK-SD-NEXT:    ret
1192;
1193; CHECK-GI-LABEL: fmul_v3f64:
1194; CHECK-GI:       // %bb.0: // %entry
1195; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
1196; CHECK-GI-NEXT:    // kill: def $d3 killed $d3 def $q3
1197; CHECK-GI-NEXT:    // kill: def $d6 killed $d6 def $q6
1198; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
1199; CHECK-GI-NEXT:    // kill: def $d4 killed $d4 def $q4
1200; CHECK-GI-NEXT:    // kill: def $d7 killed $d7 def $q7
1201; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
1202; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
1203; CHECK-GI-NEXT:    mov v6.d[1], v7.d[0]
1204; CHECK-GI-NEXT:    fmla v6.2d, v0.2d, v3.2d
1205; CHECK-GI-NEXT:    ldr d0, [sp]
1206; CHECK-GI-NEXT:    fmadd d2, d2, d5, d0
1207; CHECK-GI-NEXT:    mov d1, v6.d[1]
1208; CHECK-GI-NEXT:    fmov d0, d6
1209; CHECK-GI-NEXT:    ret
1210entry:
1211  %d = fmul fast <3 x double> %a, %b
1212  %e = fadd fast <3 x double> %d, %c
1213  ret <3 x double> %e
1214}
1215
1216define <4 x double> @fmul_v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %c) {
1217; CHECK-SD-LABEL: fmul_v4f64:
1218; CHECK-SD:       // %bb.0: // %entry
1219; CHECK-SD-NEXT:    fmla v4.2d, v2.2d, v0.2d
1220; CHECK-SD-NEXT:    fmla v5.2d, v3.2d, v1.2d
1221; CHECK-SD-NEXT:    mov v0.16b, v4.16b
1222; CHECK-SD-NEXT:    mov v1.16b, v5.16b
1223; CHECK-SD-NEXT:    ret
1224;
1225; CHECK-GI-LABEL: fmul_v4f64:
1226; CHECK-GI:       // %bb.0: // %entry
1227; CHECK-GI-NEXT:    fmla v4.2d, v0.2d, v2.2d
1228; CHECK-GI-NEXT:    fmla v5.2d, v1.2d, v3.2d
1229; CHECK-GI-NEXT:    mov v0.16b, v4.16b
1230; CHECK-GI-NEXT:    mov v1.16b, v5.16b
1231; CHECK-GI-NEXT:    ret
1232entry:
1233  %d = fmul fast <4 x double> %a, %b
1234  %e = fadd fast <4 x double> %d, %c
1235  ret <4 x double> %e
1236}
1237
1238define <2 x float> @fmul_v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
1239; CHECK-SD-LABEL: fmul_v2f32:
1240; CHECK-SD:       // %bb.0: // %entry
1241; CHECK-SD-NEXT:    fmla v2.2s, v1.2s, v0.2s
1242; CHECK-SD-NEXT:    fmov d0, d2
1243; CHECK-SD-NEXT:    ret
1244;
1245; CHECK-GI-LABEL: fmul_v2f32:
1246; CHECK-GI:       // %bb.0: // %entry
1247; CHECK-GI-NEXT:    fmla v2.2s, v0.2s, v1.2s
1248; CHECK-GI-NEXT:    fmov d0, d2
1249; CHECK-GI-NEXT:    ret
1250entry:
1251  %d = fmul fast <2 x float> %a, %b
1252  %e = fadd fast <2 x float> %d, %c
1253  ret <2 x float> %e
1254}
1255
1256define <3 x float> @fmul_v3f32(<3 x float> %a, <3 x float> %b, <3 x float> %c) {
1257; CHECK-SD-LABEL: fmul_v3f32:
1258; CHECK-SD:       // %bb.0: // %entry
1259; CHECK-SD-NEXT:    fmla v2.4s, v1.4s, v0.4s
1260; CHECK-SD-NEXT:    mov v0.16b, v2.16b
1261; CHECK-SD-NEXT:    ret
1262;
1263; CHECK-GI-LABEL: fmul_v3f32:
1264; CHECK-GI:       // %bb.0: // %entry
1265; CHECK-GI-NEXT:    fmla v2.4s, v0.4s, v1.4s
1266; CHECK-GI-NEXT:    mov v0.16b, v2.16b
1267; CHECK-GI-NEXT:    ret
1268entry:
1269  %d = fmul fast <3 x float> %a, %b
1270  %e = fadd fast <3 x float> %d, %c
1271  ret <3 x float> %e
1272}
1273
1274define <4 x float> @fmul_v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
1275; CHECK-SD-LABEL: fmul_v4f32:
1276; CHECK-SD:       // %bb.0: // %entry
1277; CHECK-SD-NEXT:    fmla v2.4s, v1.4s, v0.4s
1278; CHECK-SD-NEXT:    mov v0.16b, v2.16b
1279; CHECK-SD-NEXT:    ret
1280;
1281; CHECK-GI-LABEL: fmul_v4f32:
1282; CHECK-GI:       // %bb.0: // %entry
1283; CHECK-GI-NEXT:    fmla v2.4s, v0.4s, v1.4s
1284; CHECK-GI-NEXT:    mov v0.16b, v2.16b
1285; CHECK-GI-NEXT:    ret
1286entry:
1287  %d = fmul fast <4 x float> %a, %b
1288  %e = fadd fast <4 x float> %d, %c
1289  ret <4 x float> %e
1290}
1291
1292define <8 x float> @fmul_v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c) {
1293; CHECK-SD-LABEL: fmul_v8f32:
1294; CHECK-SD:       // %bb.0: // %entry
1295; CHECK-SD-NEXT:    fmla v4.4s, v2.4s, v0.4s
1296; CHECK-SD-NEXT:    fmla v5.4s, v3.4s, v1.4s
1297; CHECK-SD-NEXT:    mov v0.16b, v4.16b
1298; CHECK-SD-NEXT:    mov v1.16b, v5.16b
1299; CHECK-SD-NEXT:    ret
1300;
1301; CHECK-GI-LABEL: fmul_v8f32:
1302; CHECK-GI:       // %bb.0: // %entry
1303; CHECK-GI-NEXT:    fmla v4.4s, v0.4s, v2.4s
1304; CHECK-GI-NEXT:    fmla v5.4s, v1.4s, v3.4s
1305; CHECK-GI-NEXT:    mov v0.16b, v4.16b
1306; CHECK-GI-NEXT:    mov v1.16b, v5.16b
1307; CHECK-GI-NEXT:    ret
1308entry:
1309  %d = fmul fast <8 x float> %a, %b
1310  %e = fadd fast <8 x float> %d, %c
1311  ret <8 x float> %e
1312}
1313
1314define <7 x half> @fmul_v7f16(<7 x half> %a, <7 x half> %b, <7 x half> %c) {
1315; CHECK-SD-NOFP16-LABEL: fmul_v7f16:
1316; CHECK-SD-NOFP16:       // %bb.0: // %entry
1317; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
1318; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
1319; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
1320; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
1321; CHECK-SD-NOFP16-NEXT:    fmul v3.4s, v4.4s, v3.4s
1322; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
1323; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
1324; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
1325; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
1326; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
1327; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v1.4h
1328; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
1329; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v3.4s
1330; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v1.4s, v2.4s
1331; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
1332; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
1333; CHECK-SD-NOFP16-NEXT:    ret
1334;
1335; CHECK-SD-FP16-LABEL: fmul_v7f16:
1336; CHECK-SD-FP16:       // %bb.0: // %entry
1337; CHECK-SD-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
1338; CHECK-SD-FP16-NEXT:    mov v0.16b, v2.16b
1339; CHECK-SD-FP16-NEXT:    ret
1340;
1341; CHECK-GI-NOFP16-LABEL: fmul_v7f16:
1342; CHECK-GI-NOFP16:       // %bb.0: // %entry
1343; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
1344; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v1.4h
1345; CHECK-GI-NOFP16-NEXT:    mov v5.h[0], v0.h[4]
1346; CHECK-GI-NOFP16-NEXT:    fmul v3.4s, v3.4s, v4.4s
1347; CHECK-GI-NOFP16-NEXT:    mov v4.h[0], v1.h[4]
1348; CHECK-GI-NOFP16-NEXT:    mov v5.h[1], v0.h[5]
1349; CHECK-GI-NOFP16-NEXT:    mov v4.h[1], v1.h[5]
1350; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v3.4s
1351; CHECK-GI-NOFP16-NEXT:    mov v5.h[2], v0.h[6]
1352; CHECK-GI-NOFP16-NEXT:    mov v4.h[2], v1.h[6]
1353; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v3.4h
1354; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
1355; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v5.4h
1356; CHECK-GI-NOFP16-NEXT:    mov v5.h[0], v2.h[4]
1357; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v4.4h
1358; CHECK-GI-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
1359; CHECK-GI-NOFP16-NEXT:    mov v5.h[1], v2.h[5]
1360; CHECK-GI-NOFP16-NEXT:    fmul v1.4s, v3.4s, v4.4s
1361; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v0.4s
1362; CHECK-GI-NOFP16-NEXT:    mov v5.h[2], v2.h[6]
1363; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
1364; CHECK-GI-NOFP16-NEXT:    mov v0.h[0], v3.h[0]
1365; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v5.4h
1366; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
1367; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], v3.h[1]
1368; CHECK-GI-NOFP16-NEXT:    fadd v1.4s, v1.4s, v2.4s
1369; CHECK-GI-NOFP16-NEXT:    mov v0.h[2], v3.h[2]
1370; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
1371; CHECK-GI-NOFP16-NEXT:    mov v0.h[3], v3.h[3]
1372; CHECK-GI-NOFP16-NEXT:    mov v0.h[4], v1.h[0]
1373; CHECK-GI-NOFP16-NEXT:    mov v0.h[5], v1.h[1]
1374; CHECK-GI-NOFP16-NEXT:    mov v0.h[6], v1.h[2]
1375; CHECK-GI-NOFP16-NEXT:    ret
1376;
1377; CHECK-GI-FP16-LABEL: fmul_v7f16:
1378; CHECK-GI-FP16:       // %bb.0: // %entry
1379; CHECK-GI-FP16-NEXT:    fmla v2.8h, v0.8h, v1.8h
1380; CHECK-GI-FP16-NEXT:    mov v0.16b, v2.16b
1381; CHECK-GI-FP16-NEXT:    ret
1382entry:
1383  %d = fmul fast <7 x half> %a, %b
1384  %e = fadd fast <7 x half> %d, %c
1385  ret <7 x half> %e
1386}
1387
1388define <4 x half> @fmul_v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
1389; CHECK-SD-NOFP16-LABEL: fmul_v4f16:
1390; CHECK-SD-NOFP16:       // %bb.0: // %entry
1391; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
1392; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
1393; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
1394; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
1395; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
1396; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
1397; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
1398; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
1399; CHECK-SD-NOFP16-NEXT:    ret
1400;
1401; CHECK-SD-FP16-LABEL: fmul_v4f16:
1402; CHECK-SD-FP16:       // %bb.0: // %entry
1403; CHECK-SD-FP16-NEXT:    fmla v2.4h, v1.4h, v0.4h
1404; CHECK-SD-FP16-NEXT:    fmov d0, d2
1405; CHECK-SD-FP16-NEXT:    ret
1406;
1407; CHECK-GI-NOFP16-LABEL: fmul_v4f16:
1408; CHECK-GI-NOFP16:       // %bb.0: // %entry
1409; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
1410; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
1411; CHECK-GI-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
1412; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
1413; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
1414; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
1415; CHECK-GI-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
1416; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
1417; CHECK-GI-NOFP16-NEXT:    ret
1418;
1419; CHECK-GI-FP16-LABEL: fmul_v4f16:
1420; CHECK-GI-FP16:       // %bb.0: // %entry
1421; CHECK-GI-FP16-NEXT:    fmla v2.4h, v0.4h, v1.4h
1422; CHECK-GI-FP16-NEXT:    fmov d0, d2
1423; CHECK-GI-FP16-NEXT:    ret
1424entry:
1425  %d = fmul fast <4 x half> %a, %b
1426  %e = fadd fast <4 x half> %d, %c
1427  ret <4 x half> %e
1428}
1429
1430define <8 x half> @fmul_v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
1431; CHECK-SD-NOFP16-LABEL: fmul_v8f16:
1432; CHECK-SD-NOFP16:       // %bb.0: // %entry
1433; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
1434; CHECK-SD-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
1435; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
1436; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
1437; CHECK-SD-NOFP16-NEXT:    fmul v3.4s, v4.4s, v3.4s
1438; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
1439; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
1440; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
1441; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
1442; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v0.4s
1443; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v1.4h
1444; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
1445; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v0.4s, v3.4s
1446; CHECK-SD-NOFP16-NEXT:    fadd v1.4s, v1.4s, v2.4s
1447; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
1448; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
1449; CHECK-SD-NOFP16-NEXT:    ret
1450;
1451; CHECK-SD-FP16-LABEL: fmul_v8f16:
1452; CHECK-SD-FP16:       // %bb.0: // %entry
1453; CHECK-SD-FP16-NEXT:    fmla v2.8h, v1.8h, v0.8h
1454; CHECK-SD-FP16-NEXT:    mov v0.16b, v2.16b
1455; CHECK-SD-FP16-NEXT:    ret
1456;
1457; CHECK-GI-NOFP16-LABEL: fmul_v8f16:
1458; CHECK-GI-NOFP16:       // %bb.0: // %entry
1459; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v0.4h
1460; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v1.4h
1461; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
1462; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
1463; CHECK-GI-NOFP16-NEXT:    fmul v3.4s, v3.4s, v4.4s
1464; CHECK-GI-NOFP16-NEXT:    fmul v0.4s, v0.4s, v1.4s
1465; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
1466; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v2.4h
1467; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
1468; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
1469; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
1470; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
1471; CHECK-GI-NOFP16-NEXT:    fadd v1.4s, v1.4s, v3.4s
1472; CHECK-GI-NOFP16-NEXT:    fadd v2.4s, v0.4s, v2.4s
1473; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
1474; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
1475; CHECK-GI-NOFP16-NEXT:    ret
1476;
1477; CHECK-GI-FP16-LABEL: fmul_v8f16:
1478; CHECK-GI-FP16:       // %bb.0: // %entry
1479; CHECK-GI-FP16-NEXT:    fmla v2.8h, v0.8h, v1.8h
1480; CHECK-GI-FP16-NEXT:    mov v0.16b, v2.16b
1481; CHECK-GI-FP16-NEXT:    ret
1482entry:
1483  %d = fmul fast <8 x half> %a, %b
1484  %e = fadd fast <8 x half> %d, %c
1485  ret <8 x half> %e
1486}
1487
1488define <16 x half> @fmul_v16f16(<16 x half> %a, <16 x half> %b, <16 x half> %c) {
1489; CHECK-SD-NOFP16-LABEL: fmul_v16f16:
1490; CHECK-SD-NOFP16:       // %bb.0: // %entry
1491; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v2.4h
1492; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v0.4h
1493; CHECK-SD-NOFP16-NEXT:    fcvtl v16.4s, v3.4h
1494; CHECK-SD-NOFP16-NEXT:    fcvtl v17.4s, v1.4h
1495; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
1496; CHECK-SD-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
1497; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
1498; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
1499; CHECK-SD-NOFP16-NEXT:    fmul v6.4s, v7.4s, v6.4s
1500; CHECK-SD-NOFP16-NEXT:    fmul v7.4s, v17.4s, v16.4s
1501; CHECK-SD-NOFP16-NEXT:    fmul v0.4s, v0.4s, v2.4s
1502; CHECK-SD-NOFP16-NEXT:    fmul v1.4s, v1.4s, v3.4s
1503; CHECK-SD-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
1504; CHECK-SD-NOFP16-NEXT:    fcvtl v6.4s, v5.4h
1505; CHECK-SD-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
1506; CHECK-SD-NOFP16-NEXT:    fcvtn2 v2.8h, v0.4s
1507; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v4.4h
1508; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v4.8h
1509; CHECK-SD-NOFP16-NEXT:    fcvtn2 v3.8h, v1.4s
1510; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v2.4h
1511; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
1512; CHECK-SD-NOFP16-NEXT:    fcvtl v7.4s, v3.4h
1513; CHECK-SD-NOFP16-NEXT:    fadd v0.4s, v1.4s, v0.4s
1514; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v3.8h
1515; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v5.8h
1516; CHECK-SD-NOFP16-NEXT:    fadd v5.4s, v7.4s, v6.4s
1517; CHECK-SD-NOFP16-NEXT:    fadd v2.4s, v2.4s, v4.4s
1518; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
1519; CHECK-SD-NOFP16-NEXT:    fadd v3.4s, v1.4s, v3.4s
1520; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
1521; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
1522; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
1523; CHECK-SD-NOFP16-NEXT:    ret
1524;
1525; CHECK-SD-FP16-LABEL: fmul_v16f16:
1526; CHECK-SD-FP16:       // %bb.0: // %entry
1527; CHECK-SD-FP16-NEXT:    fmla v4.8h, v2.8h, v0.8h
1528; CHECK-SD-FP16-NEXT:    fmla v5.8h, v3.8h, v1.8h
1529; CHECK-SD-FP16-NEXT:    mov v0.16b, v4.16b
1530; CHECK-SD-FP16-NEXT:    mov v1.16b, v5.16b
1531; CHECK-SD-FP16-NEXT:    ret
1532;
1533; CHECK-GI-NOFP16-LABEL: fmul_v16f16:
1534; CHECK-GI-NOFP16:       // %bb.0: // %entry
1535; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v0.4h
1536; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v2.4h
1537; CHECK-GI-NOFP16-NEXT:    fcvtl v16.4s, v1.4h
1538; CHECK-GI-NOFP16-NEXT:    fcvtl v17.4s, v3.4h
1539; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
1540; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
1541; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
1542; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
1543; CHECK-GI-NOFP16-NEXT:    fmul v6.4s, v6.4s, v7.4s
1544; CHECK-GI-NOFP16-NEXT:    fmul v7.4s, v16.4s, v17.4s
1545; CHECK-GI-NOFP16-NEXT:    fmul v0.4s, v0.4s, v2.4s
1546; CHECK-GI-NOFP16-NEXT:    fmul v1.4s, v1.4s, v3.4s
1547; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
1548; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v4.4h
1549; CHECK-GI-NOFP16-NEXT:    fcvtl2 v4.4s, v4.8h
1550; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
1551; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v5.4h
1552; CHECK-GI-NOFP16-NEXT:    fcvtl2 v5.4s, v5.8h
1553; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
1554; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
1555; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
1556; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
1557; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
1558; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
1559; CHECK-GI-NOFP16-NEXT:    fadd v2.4s, v2.4s, v6.4s
1560; CHECK-GI-NOFP16-NEXT:    fadd v3.4s, v3.4s, v7.4s
1561; CHECK-GI-NOFP16-NEXT:    fadd v4.4s, v0.4s, v4.4s
1562; CHECK-GI-NOFP16-NEXT:    fadd v5.4s, v1.4s, v5.4s
1563; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
1564; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
1565; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v4.4s
1566; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v5.4s
1567; CHECK-GI-NOFP16-NEXT:    ret
1568;
1569; CHECK-GI-FP16-LABEL: fmul_v16f16:
1570; CHECK-GI-FP16:       // %bb.0: // %entry
1571; CHECK-GI-FP16-NEXT:    fmla v4.8h, v0.8h, v2.8h
1572; CHECK-GI-FP16-NEXT:    fmla v5.8h, v1.8h, v3.8h
1573; CHECK-GI-FP16-NEXT:    mov v0.16b, v4.16b
1574; CHECK-GI-FP16-NEXT:    mov v1.16b, v5.16b
1575; CHECK-GI-FP16-NEXT:    ret
1576entry:
1577  %d = fmul fast <16 x half> %a, %b
1578  %e = fadd fast <16 x half> %d, %c
1579  ret <16 x half> %e
1580}
1581
1582declare <16 x half> @llvm.fma.v16f16(<16 x half>, <16 x half>, <16 x half>)
1583declare <16 x half> @llvm.fmuladd.v16f16(<16 x half>, <16 x half>, <16 x half>)
1584declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)
1585declare <2 x double> @llvm.fmuladd.v2f64(<2 x double>, <2 x double>, <2 x double>)
1586declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)
1587declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>)
1588declare <3 x double> @llvm.fma.v3f64(<3 x double>, <3 x double>, <3 x double>)
1589declare <3 x double> @llvm.fmuladd.v3f64(<3 x double>, <3 x double>, <3 x double>)
1590declare <3 x float> @llvm.fma.v3f32(<3 x float>, <3 x float>, <3 x float>)
1591declare <3 x float> @llvm.fmuladd.v3f32(<3 x float>, <3 x float>, <3 x float>)
1592declare <4 x double> @llvm.fma.v4f64(<4 x double>, <4 x double>, <4 x double>)
1593declare <4 x double> @llvm.fmuladd.v4f64(<4 x double>, <4 x double>, <4 x double>)
1594declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
1595declare <4 x float> @llvm.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>)
1596declare <4 x half> @llvm.fma.v4f16(<4 x half>, <4 x half>, <4 x half>)
1597declare <4 x half> @llvm.fmuladd.v4f16(<4 x half>, <4 x half>, <4 x half>)
1598declare <7 x half> @llvm.fma.v7f16(<7 x half>, <7 x half>, <7 x half>)
1599declare <7 x half> @llvm.fmuladd.v7f16(<7 x half>, <7 x half>, <7 x half>)
1600declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>)
1601declare <8 x float> @llvm.fmuladd.v8f32(<8 x float>, <8 x float>, <8 x float>)
1602declare <8 x half> @llvm.fma.v8f16(<8 x half>, <8 x half>, <8 x half>)
1603declare <8 x half> @llvm.fmuladd.v8f16(<8 x half>, <8 x half>, <8 x half>)
1604declare double @llvm.fma.f64(double, double, double)
1605declare double @llvm.fmuladd.f64(double, double, double)
1606declare float @llvm.fma.f32(float, float, float)
1607declare float @llvm.fmuladd.f32(float, float, float)
1608declare half @llvm.fma.f16(half, half, half)
1609declare half @llvm.fmuladd.f16(half, half, half)
1610