xref: /llvm-project/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll (revision 6548b6354d1d990e1c98736f5e7c3de876bedc8e) 
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck --check-prefix=SI %s
3; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX9 %s
4; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX10 %s
5; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX11 %s
6; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck --check-prefix=GFX12 %s
7
8define { i64, i1 } @umulo_i64_v_v(i64 %x, i64 %y) {
9; SI-LABEL: umulo_i64_v_v:
10; SI:       ; %bb.0: ; %bb
11; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
12; SI-NEXT:    v_mul_hi_u32 v4, v1, v2
13; SI-NEXT:    v_mul_lo_u32 v5, v1, v2
14; SI-NEXT:    v_mul_hi_u32 v6, v0, v3
15; SI-NEXT:    v_mul_lo_u32 v7, v0, v3
16; SI-NEXT:    v_mul_hi_u32 v8, v0, v2
17; SI-NEXT:    v_mul_hi_u32 v9, v1, v3
18; SI-NEXT:    v_mul_lo_u32 v3, v1, v3
19; SI-NEXT:    v_mul_lo_u32 v0, v0, v2
20; SI-NEXT:    v_add_i32_e32 v1, vcc, v8, v7
21; SI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v6, vcc
22; SI-NEXT:    v_add_i32_e32 v6, vcc, v1, v5
23; SI-NEXT:    v_add_i32_e64 v1, s[4:5], v1, v5
24; SI-NEXT:    v_addc_u32_e32 v2, vcc, v2, v4, vcc
25; SI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v9, vcc
26; SI-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
27; SI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
28; SI-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
29; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
30; SI-NEXT:    s_setpc_b64 s[30:31]
31;
32; GFX9-LABEL: umulo_i64_v_v:
33; GFX9:       ; %bb.0: ; %bb
34; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
35; GFX9-NEXT:    v_mov_b32_e32 v5, v0
36; GFX9-NEXT:    v_mov_b32_e32 v4, v1
37; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0
38; GFX9-NEXT:    v_mad_u64_u32 v[5:6], s[4:5], v5, v3, 0
39; GFX9-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v4, v2, 0
40; GFX9-NEXT:    v_mov_b32_e32 v2, v1
41; GFX9-NEXT:    v_add_co_u32_e32 v9, vcc, v2, v5
42; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v4, v3, 0
43; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
44; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v9, v7
45; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v6, v8, vcc
46; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
47; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v4, v2
48; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
49; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[2:3]
50; GFX9-NEXT:    v_add3_u32 v1, v1, v5, v7
51; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
52; GFX9-NEXT:    s_setpc_b64 s[30:31]
53;
54; GFX10-LABEL: umulo_i64_v_v:
55; GFX10:       ; %bb.0: ; %bb
56; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
57; GFX10-NEXT:    v_mov_b32_e32 v4, v0
58; GFX10-NEXT:    v_mov_b32_e32 v5, v1
59; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v2, 0
60; GFX10-NEXT:    v_mad_u64_u32 v[6:7], s4, v4, v3, 0
61; GFX10-NEXT:    v_mad_u64_u32 v[8:9], s4, v5, v2, 0
62; GFX10-NEXT:    v_mad_u64_u32 v[2:3], s4, v5, v3, 0
63; GFX10-NEXT:    v_mov_b32_e32 v4, v1
64; GFX10-NEXT:    v_add3_u32 v1, v1, v6, v8
65; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v6
66; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v7, vcc_lo
67; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v8
68; GFX10-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v5, v9, vcc_lo
69; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
70; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v4, v2
71; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
72; GFX10-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
73; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
74; GFX10-NEXT:    s_setpc_b64 s[30:31]
75;
76; GFX11-LABEL: umulo_i64_v_v:
77; GFX11:       ; %bb.0: ; %bb
78; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
79; GFX11-NEXT:    v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
80; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
81; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v2, 0
82; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, v4, v3, 0
83; GFX11-NEXT:    v_mad_u64_u32 v[8:9], null, v5, v2, 0
84; GFX11-NEXT:    v_mad_u64_u32 v[10:11], null, v5, v3, 0
85; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
86; GFX11-NEXT:    v_mov_b32_e32 v4, v1
87; GFX11-NEXT:    v_add3_u32 v1, v1, v6, v8
88; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
89; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v4, v6
90; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v7, vcc_lo
91; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v8
92; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
93; GFX11-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v3, v9, vcc_lo
94; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v11, vcc_lo
95; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v10
96; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
97; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
98; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
99; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
100; GFX11-NEXT:    s_setpc_b64 s[30:31]
101;
102; GFX12-LABEL: umulo_i64_v_v:
103; GFX12:       ; %bb.0: ; %bb
104; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
105; GFX12-NEXT:    s_wait_expcnt 0x0
106; GFX12-NEXT:    s_wait_samplecnt 0x0
107; GFX12-NEXT:    s_wait_bvhcnt 0x0
108; GFX12-NEXT:    s_wait_kmcnt 0x0
109; GFX12-NEXT:    v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
110; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
111; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v4, v2, 0
112; GFX12-NEXT:    v_mad_co_u64_u32 v[6:7], null, v4, v3, 0
113; GFX12-NEXT:    v_mad_co_u64_u32 v[8:9], null, v5, v2, 0
114; GFX12-NEXT:    v_mad_co_u64_u32 v[2:3], null, v5, v3, 0
115; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
116; GFX12-NEXT:    v_mov_b32_e32 v4, v1
117; GFX12-NEXT:    v_add3_u32 v1, v1, v6, v8
118; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
119; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v6
120; GFX12-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v7, vcc_lo
121; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v8
122; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
123; GFX12-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v5, v9, vcc_lo
124; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
125; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v4, v2
126; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
127; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
128; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
129; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
130; GFX12-NEXT:    s_setpc_b64 s[30:31]
131bb:
132  %umulo = tail call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %x, i64 %y)
133  ret { i64, i1 } %umulo
134}
135
136define { i64, i1 } @smulo_i64_v_v(i64 %x, i64 %y) {
137; SI-LABEL: smulo_i64_v_v:
138; SI:       ; %bb.0: ; %bb
139; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
140; SI-NEXT:    v_mul_hi_u32 v6, v1, v2
141; SI-NEXT:    v_mul_lo_u32 v5, v1, v2
142; SI-NEXT:    v_mul_hi_u32 v7, v0, v3
143; SI-NEXT:    v_mul_lo_u32 v8, v0, v3
144; SI-NEXT:    v_mul_hi_u32 v9, v0, v2
145; SI-NEXT:    v_mul_hi_i32 v10, v1, v3
146; SI-NEXT:    v_mul_lo_u32 v11, v1, v3
147; SI-NEXT:    v_mul_lo_u32 v4, v0, v2
148; SI-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
149; SI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
150; SI-NEXT:    v_add_i32_e32 v9, vcc, v8, v5
151; SI-NEXT:    v_add_i32_e64 v5, s[4:5], v8, v5
152; SI-NEXT:    v_addc_u32_e32 v8, vcc, v7, v6, vcc
153; SI-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
154; SI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v10, vcc
155; SI-NEXT:    v_mov_b32_e32 v7, v6
156; SI-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
157; SI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
158; SI-NEXT:    v_sub_i32_e32 v2, vcc, v8, v2
159; SI-NEXT:    v_subbrev_u32_e32 v10, vcc, 0, v9, vcc
160; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v1
161; SI-NEXT:    v_cndmask_b32_e32 v1, v9, v10, vcc
162; SI-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
163; SI-NEXT:    v_sub_i32_e32 v0, vcc, v2, v0
164; SI-NEXT:    v_subbrev_u32_e32 v8, vcc, 0, v1, vcc
165; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v3
166; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v8, vcc
167; SI-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
168; SI-NEXT:    v_cmp_ne_u64_e32 vcc, v[0:1], v[6:7]
169; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
170; SI-NEXT:    v_mov_b32_e32 v0, v4
171; SI-NEXT:    v_mov_b32_e32 v1, v5
172; SI-NEXT:    s_setpc_b64 s[30:31]
173;
174; GFX9-LABEL: smulo_i64_v_v:
175; GFX9:       ; %bb.0: ; %bb
176; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
177; GFX9-NEXT:    v_mov_b32_e32 v5, v0
178; GFX9-NEXT:    v_mov_b32_e32 v4, v1
179; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, v2, 0
180; GFX9-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v5, v3, 0
181; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v4, v2, 0
182; GFX9-NEXT:    v_mov_b32_e32 v10, v1
183; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, v10, v6
184; GFX9-NEXT:    v_mad_i64_i32 v[10:11], s[4:5], v4, v3, 0
185; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
186; GFX9-NEXT:    v_add_co_u32_e32 v12, vcc, v12, v8
187; GFX9-NEXT:    v_addc_co_u32_e32 v7, vcc, v7, v9, vcc
188; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v11, vcc
189; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v10
190; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
191; GFX9-NEXT:    v_sub_co_u32_e32 v2, vcc, v7, v2
192; GFX9-NEXT:    v_subbrev_co_u32_e32 v10, vcc, 0, v9, vcc
193; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
194; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v2, vcc
195; GFX9-NEXT:    v_cndmask_b32_e32 v4, v9, v10, vcc
196; GFX9-NEXT:    v_sub_co_u32_e32 v5, vcc, v2, v5
197; GFX9-NEXT:    v_subbrev_co_u32_e32 v7, vcc, 0, v4, vcc
198; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v3
199; GFX9-NEXT:    v_add3_u32 v1, v1, v6, v8
200; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v7, vcc
201; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
202; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
203; GFX9-NEXT:    v_mov_b32_e32 v5, v4
204; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, v[2:3], v[4:5]
205; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
206; GFX9-NEXT:    s_setpc_b64 s[30:31]
207;
208; GFX10-LABEL: smulo_i64_v_v:
209; GFX10:       ; %bb.0: ; %bb
210; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
211; GFX10-NEXT:    v_mov_b32_e32 v4, v0
212; GFX10-NEXT:    v_mov_b32_e32 v5, v1
213; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v2, 0
214; GFX10-NEXT:    v_mad_u64_u32 v[6:7], s4, v4, v3, 0
215; GFX10-NEXT:    v_mad_u64_u32 v[8:9], s4, v5, v2, 0
216; GFX10-NEXT:    v_mad_i64_i32 v[10:11], s4, v5, v3, 0
217; GFX10-NEXT:    v_mov_b32_e32 v12, v1
218; GFX10-NEXT:    v_add3_u32 v1, v1, v6, v8
219; GFX10-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v6
220; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
221; GFX10-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v8
222; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v7, v9, vcc_lo
223; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v11, vcc_lo
224; GFX10-NEXT:    v_add_co_u32 v7, vcc_lo, v7, v10
225; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v9, vcc_lo
226; GFX10-NEXT:    v_sub_co_u32 v2, vcc_lo, v7, v2
227; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v10, vcc_lo, 0, v9, vcc_lo
228; GFX10-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v5
229; GFX10-NEXT:    v_cndmask_b32_e32 v6, v7, v2, vcc_lo
230; GFX10-NEXT:    v_cndmask_b32_e32 v5, v9, v10, vcc_lo
231; GFX10-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
232; GFX10-NEXT:    v_sub_co_u32 v4, vcc_lo, v6, v4
233; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v7, vcc_lo, 0, v5, vcc_lo
234; GFX10-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v3
235; GFX10-NEXT:    v_mov_b32_e32 v3, v2
236; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc_lo
237; GFX10-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc_lo
238; GFX10-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[2:3]
239; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
240; GFX10-NEXT:    s_setpc_b64 s[30:31]
241;
242; GFX11-LABEL: smulo_i64_v_v:
243; GFX11:       ; %bb.0: ; %bb
244; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
245; GFX11-NEXT:    v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
246; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
247; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v2, 0
248; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, v4, v3, 0
249; GFX11-NEXT:    v_mad_u64_u32 v[8:9], null, v5, v2, 0
250; GFX11-NEXT:    v_mad_i64_i32 v[10:11], null, v5, v3, 0
251; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
252; GFX11-NEXT:    v_mov_b32_e32 v12, v1
253; GFX11-NEXT:    v_add3_u32 v1, v1, v6, v8
254; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
255; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v6
256; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
257; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v8
258; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
259; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v7, v9, vcc_lo
260; GFX11-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v11, vcc_lo
261; GFX11-NEXT:    v_add_co_u32 v7, vcc_lo, v7, v10
262; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
263; GFX11-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v9, vcc_lo
264; GFX11-NEXT:    v_sub_co_u32 v2, vcc_lo, v7, v2
265; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
266; GFX11-NEXT:    v_subrev_co_ci_u32_e32 v10, vcc_lo, 0, v9, vcc_lo
267; GFX11-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v5
268; GFX11-NEXT:    v_cndmask_b32_e32 v6, v7, v2, vcc_lo
269; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
270; GFX11-NEXT:    v_cndmask_b32_e32 v5, v9, v10, vcc_lo
271; GFX11-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
272; GFX11-NEXT:    v_sub_co_u32 v4, vcc_lo, v6, v4
273; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
274; GFX11-NEXT:    v_subrev_co_ci_u32_e32 v7, vcc_lo, 0, v5, vcc_lo
275; GFX11-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v3
276; GFX11-NEXT:    v_mov_b32_e32 v3, v2
277; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
278; GFX11-NEXT:    v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v5, v5, v7
279; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[2:3]
280; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
281; GFX11-NEXT:    s_setpc_b64 s[30:31]
282;
283; GFX12-LABEL: smulo_i64_v_v:
284; GFX12:       ; %bb.0: ; %bb
285; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
286; GFX12-NEXT:    s_wait_expcnt 0x0
287; GFX12-NEXT:    s_wait_samplecnt 0x0
288; GFX12-NEXT:    s_wait_bvhcnt 0x0
289; GFX12-NEXT:    s_wait_kmcnt 0x0
290; GFX12-NEXT:    v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
291; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
292; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v4, v2, 0
293; GFX12-NEXT:    v_mad_co_u64_u32 v[6:7], null, v4, v3, 0
294; GFX12-NEXT:    v_mad_co_u64_u32 v[8:9], null, v5, v2, 0
295; GFX12-NEXT:    v_mad_co_i64_i32 v[10:11], null, v5, v3, 0
296; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
297; GFX12-NEXT:    v_mov_b32_e32 v12, v1
298; GFX12-NEXT:    v_add3_u32 v1, v1, v6, v8
299; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
300; GFX12-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v6
301; GFX12-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
302; GFX12-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v8
303; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
304; GFX12-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v7, v9, vcc_lo
305; GFX12-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v11, vcc_lo
306; GFX12-NEXT:    v_add_co_u32 v7, vcc_lo, v7, v10
307; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
308; GFX12-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v9, vcc_lo
309; GFX12-NEXT:    v_sub_co_u32 v2, vcc_lo, v7, v2
310; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
311; GFX12-NEXT:    v_subrev_co_ci_u32_e32 v10, vcc_lo, 0, v9, vcc_lo
312; GFX12-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v5
313; GFX12-NEXT:    v_cndmask_b32_e32 v6, v7, v2, vcc_lo
314; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
315; GFX12-NEXT:    v_cndmask_b32_e32 v5, v9, v10, vcc_lo
316; GFX12-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
317; GFX12-NEXT:    v_sub_co_u32 v4, vcc_lo, v6, v4
318; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
319; GFX12-NEXT:    v_subrev_co_ci_u32_e32 v7, vcc_lo, 0, v5, vcc_lo
320; GFX12-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v3
321; GFX12-NEXT:    v_mov_b32_e32 v3, v2
322; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
323; GFX12-NEXT:    v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v5, v5, v7
324; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[2:3]
325; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
326; GFX12-NEXT:    s_setpc_b64 s[30:31]
327bb:
328  %smulo = tail call { i64, i1 } @llvm.smul.with.overflow.i64(i64 %x, i64 %y)
329  ret { i64, i1 } %smulo
330}
331
332define amdgpu_kernel void @umulo_i64_s(i64 %x, i64 %y) {
333; SI-LABEL: umulo_i64_s:
334; SI:       ; %bb.0: ; %bb
335; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
336; SI-NEXT:    s_mov_b32 s7, 0xf000
337; SI-NEXT:    s_waitcnt lgkmcnt(0)
338; SI-NEXT:    v_mov_b32_e32 v0, s2
339; SI-NEXT:    v_mul_hi_u32 v1, s1, v0
340; SI-NEXT:    s_mul_i32 s4, s1, s2
341; SI-NEXT:    v_mov_b32_e32 v2, s3
342; SI-NEXT:    v_mul_hi_u32 v3, s0, v2
343; SI-NEXT:    s_mul_i32 s5, s0, s3
344; SI-NEXT:    v_mul_hi_u32 v0, s0, v0
345; SI-NEXT:    v_mul_hi_u32 v2, s1, v2
346; SI-NEXT:    s_mul_i32 s1, s1, s3
347; SI-NEXT:    s_mul_i32 s2, s0, s2
348; SI-NEXT:    v_add_i32_e32 v4, vcc, s5, v0
349; SI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
350; SI-NEXT:    v_add_i32_e32 v4, vcc, s4, v4
351; SI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
352; SI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
353; SI-NEXT:    v_add_i32_e32 v3, vcc, s5, v0
354; SI-NEXT:    v_add_i32_e32 v0, vcc, s1, v1
355; SI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
356; SI-NEXT:    v_add_i32_e32 v2, vcc, s4, v3
357; SI-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
358; SI-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
359; SI-NEXT:    s_and_b64 s[0:1], vcc, exec
360; SI-NEXT:    s_cselect_b32 s0, 0, s2
361; SI-NEXT:    s_mov_b32 s6, -1
362; SI-NEXT:    v_mov_b32_e32 v0, s0
363; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
364; SI-NEXT:    s_endpgm
365;
366; GFX9-LABEL: umulo_i64_s:
367; GFX9:       ; %bb.0: ; %bb
368; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
369; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
370; GFX9-NEXT:    s_mul_i32 s7, s0, s3
371; GFX9-NEXT:    s_mul_hi_u32 s8, s0, s2
372; GFX9-NEXT:    s_mul_hi_u32 s5, s0, s3
373; GFX9-NEXT:    s_add_u32 s9, s8, s7
374; GFX9-NEXT:    s_mul_i32 s6, s1, s2
375; GFX9-NEXT:    s_addc_u32 s5, 0, s5
376; GFX9-NEXT:    s_mul_hi_u32 s4, s1, s2
377; GFX9-NEXT:    s_add_u32 s9, s9, s6
378; GFX9-NEXT:    s_mul_hi_u32 s10, s1, s3
379; GFX9-NEXT:    s_addc_u32 s4, s5, s4
380; GFX9-NEXT:    s_addc_u32 s5, s10, 0
381; GFX9-NEXT:    s_mul_i32 s1, s1, s3
382; GFX9-NEXT:    s_add_u32 s4, s4, s1
383; GFX9-NEXT:    s_addc_u32 s5, 0, s5
384; GFX9-NEXT:    s_add_i32 s1, s8, s7
385; GFX9-NEXT:    s_add_i32 s1, s1, s6
386; GFX9-NEXT:    s_mul_i32 s0, s0, s2
387; GFX9-NEXT:    s_cmp_lg_u64 s[4:5], 0
388; GFX9-NEXT:    s_cselect_b32 s1, 0, s1
389; GFX9-NEXT:    s_cselect_b32 s0, 0, s0
390; GFX9-NEXT:    v_mov_b32_e32 v0, s0
391; GFX9-NEXT:    v_mov_b32_e32 v1, s1
392; GFX9-NEXT:    global_store_dwordx2 v[0:1], v[0:1], off
393; GFX9-NEXT:    s_endpgm
394;
395; GFX10-LABEL: umulo_i64_s:
396; GFX10:       ; %bb.0: ; %bb
397; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
398; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
399; GFX10-NEXT:    s_mul_i32 s7, s0, s3
400; GFX10-NEXT:    s_mul_hi_u32 s8, s0, s2
401; GFX10-NEXT:    s_mul_hi_u32 s5, s0, s3
402; GFX10-NEXT:    s_mul_hi_u32 s4, s1, s2
403; GFX10-NEXT:    s_mul_i32 s6, s1, s2
404; GFX10-NEXT:    s_mul_hi_u32 s9, s1, s3
405; GFX10-NEXT:    s_mul_i32 s1, s1, s3
406; GFX10-NEXT:    s_add_u32 s3, s8, s7
407; GFX10-NEXT:    s_addc_u32 s5, 0, s5
408; GFX10-NEXT:    s_add_u32 s3, s3, s6
409; GFX10-NEXT:    s_addc_u32 s3, s5, s4
410; GFX10-NEXT:    s_addc_u32 s5, s9, 0
411; GFX10-NEXT:    s_add_u32 s4, s3, s1
412; GFX10-NEXT:    s_addc_u32 s5, 0, s5
413; GFX10-NEXT:    s_add_i32 s1, s8, s7
414; GFX10-NEXT:    s_mul_i32 s0, s0, s2
415; GFX10-NEXT:    s_add_i32 s1, s1, s6
416; GFX10-NEXT:    s_cmp_lg_u64 s[4:5], 0
417; GFX10-NEXT:    s_cselect_b32 s0, 0, s0
418; GFX10-NEXT:    s_cselect_b32 s1, 0, s1
419; GFX10-NEXT:    v_mov_b32_e32 v0, s0
420; GFX10-NEXT:    v_mov_b32_e32 v1, s1
421; GFX10-NEXT:    global_store_dwordx2 v[0:1], v[0:1], off
422; GFX10-NEXT:    s_endpgm
423;
424; GFX11-LABEL: umulo_i64_s:
425; GFX11:       ; %bb.0: ; %bb
426; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
427; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
428; GFX11-NEXT:    s_mul_i32 s7, s0, s3
429; GFX11-NEXT:    s_mul_hi_u32 s8, s0, s2
430; GFX11-NEXT:    s_mul_hi_u32 s5, s0, s3
431; GFX11-NEXT:    s_mul_hi_u32 s4, s1, s2
432; GFX11-NEXT:    s_mul_i32 s6, s1, s2
433; GFX11-NEXT:    s_mul_hi_u32 s9, s1, s3
434; GFX11-NEXT:    s_mul_i32 s1, s1, s3
435; GFX11-NEXT:    s_add_u32 s3, s8, s7
436; GFX11-NEXT:    s_addc_u32 s5, 0, s5
437; GFX11-NEXT:    s_add_u32 s3, s3, s6
438; GFX11-NEXT:    s_addc_u32 s3, s5, s4
439; GFX11-NEXT:    s_addc_u32 s5, s9, 0
440; GFX11-NEXT:    s_add_u32 s4, s3, s1
441; GFX11-NEXT:    s_addc_u32 s5, 0, s5
442; GFX11-NEXT:    s_add_i32 s1, s8, s7
443; GFX11-NEXT:    s_mul_i32 s0, s0, s2
444; GFX11-NEXT:    s_add_i32 s1, s1, s6
445; GFX11-NEXT:    s_cmp_lg_u64 s[4:5], 0
446; GFX11-NEXT:    s_cselect_b32 s0, 0, s0
447; GFX11-NEXT:    s_cselect_b32 s1, 0, s1
448; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
449; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
450; GFX11-NEXT:    global_store_b64 v[0:1], v[0:1], off
451; GFX11-NEXT:    s_endpgm
452;
453; GFX12-LABEL: umulo_i64_s:
454; GFX12:       ; %bb.0: ; %bb
455; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
456; GFX12-NEXT:    s_mov_b32 s5, 0
457; GFX12-NEXT:    s_wait_kmcnt 0x0
458; GFX12-NEXT:    s_mul_hi_u32 s7, s0, s3
459; GFX12-NEXT:    s_mul_i32 s6, s0, s3
460; GFX12-NEXT:    s_mul_hi_u32 s4, s0, s2
461; GFX12-NEXT:    s_mul_i32 s10, s1, s2
462; GFX12-NEXT:    s_add_nc_u64 s[6:7], s[4:5], s[6:7]
463; GFX12-NEXT:    s_mul_hi_u32 s9, s1, s2
464; GFX12-NEXT:    s_mul_hi_u32 s11, s1, s3
465; GFX12-NEXT:    s_add_co_u32 s4, s6, s10
466; GFX12-NEXT:    s_add_co_ci_u32 s4, s7, s9
467; GFX12-NEXT:    s_mul_i32 s8, s1, s3
468; GFX12-NEXT:    s_add_co_ci_u32 s9, s11, 0
469; GFX12-NEXT:    s_mul_u64 s[0:1], s[0:1], s[2:3]
470; GFX12-NEXT:    s_add_nc_u64 s[4:5], s[4:5], s[8:9]
471; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
472; GFX12-NEXT:    s_cmp_lg_u64 s[4:5], 0
473; GFX12-NEXT:    s_cselect_b32 s0, 0, s0
474; GFX12-NEXT:    s_cselect_b32 s1, 0, s1
475; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
476; GFX12-NEXT:    global_store_b64 v[0:1], v[0:1], off
477; GFX12-NEXT:    s_endpgm
478bb:
479  %umulo = tail call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %x, i64 %y)
480  %mul = extractvalue { i64, i1 } %umulo, 0
481  %overflow = extractvalue { i64, i1 } %umulo, 1
482  %res = select i1 %overflow, i64 0, i64 %mul
483  store i64 %res, ptr addrspace(1) undef
484  ret void
485}
486
487define amdgpu_kernel void @smulo_i64_s(i64 %x, i64 %y) {
488; SI-LABEL: smulo_i64_s:
489; SI:       ; %bb.0: ; %bb
490; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
491; SI-NEXT:    s_mov_b32 s7, 0xf000
492; SI-NEXT:    s_waitcnt lgkmcnt(0)
493; SI-NEXT:    v_mov_b32_e32 v0, s2
494; SI-NEXT:    v_mul_hi_u32 v1, s1, v0
495; SI-NEXT:    s_mul_i32 s4, s1, s2
496; SI-NEXT:    v_mov_b32_e32 v2, s3
497; SI-NEXT:    v_mul_hi_u32 v3, s0, v2
498; SI-NEXT:    s_mul_i32 s5, s0, s3
499; SI-NEXT:    v_mul_hi_u32 v0, s0, v0
500; SI-NEXT:    v_mul_hi_i32 v2, s1, v2
501; SI-NEXT:    s_mul_i32 s6, s1, s3
502; SI-NEXT:    s_mul_i32 s8, s0, s2
503; SI-NEXT:    v_readfirstlane_b32 s9, v1
504; SI-NEXT:    v_readfirstlane_b32 s10, v3
505; SI-NEXT:    v_readfirstlane_b32 s11, v0
506; SI-NEXT:    v_readfirstlane_b32 s12, v2
507; SI-NEXT:    v_add_i32_e32 v0, vcc, s5, v0
508; SI-NEXT:    s_add_u32 s5, s11, s5
509; SI-NEXT:    v_add_i32_e32 v2, vcc, s4, v0
510; SI-NEXT:    s_addc_u32 s10, 0, s10
511; SI-NEXT:    v_ashrrev_i32_e32 v0, 31, v2
512; SI-NEXT:    s_add_u32 s4, s5, s4
513; SI-NEXT:    v_mov_b32_e32 v1, v0
514; SI-NEXT:    s_addc_u32 s4, s10, s9
515; SI-NEXT:    s_addc_u32 s5, s12, 0
516; SI-NEXT:    s_add_u32 s4, s4, s6
517; SI-NEXT:    s_addc_u32 s5, 0, s5
518; SI-NEXT:    s_sub_u32 s2, s4, s2
519; SI-NEXT:    s_subb_u32 s6, s5, 0
520; SI-NEXT:    s_cmp_lt_i32 s1, 0
521; SI-NEXT:    s_cselect_b32 s1, s6, s5
522; SI-NEXT:    s_cselect_b32 s2, s2, s4
523; SI-NEXT:    s_sub_u32 s0, s2, s0
524; SI-NEXT:    s_subb_u32 s4, s1, 0
525; SI-NEXT:    s_cmp_lt_i32 s3, 0
526; SI-NEXT:    s_cselect_b32 s1, s4, s1
527; SI-NEXT:    s_cselect_b32 s0, s0, s2
528; SI-NEXT:    v_cmp_ne_u64_e32 vcc, s[0:1], v[0:1]
529; SI-NEXT:    v_cndmask_b32_e64 v1, v2, 0, vcc
530; SI-NEXT:    s_and_b64 s[0:1], vcc, exec
531; SI-NEXT:    s_cselect_b32 s0, 0, s8
532; SI-NEXT:    s_mov_b32 s6, -1
533; SI-NEXT:    v_mov_b32_e32 v0, s0
534; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
535; SI-NEXT:    s_endpgm
536;
537; GFX9-LABEL: smulo_i64_s:
538; GFX9:       ; %bb.0: ; %bb
539; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
540; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
541; GFX9-NEXT:    s_mul_i32 s7, s0, s3
542; GFX9-NEXT:    s_mul_hi_u32 s8, s0, s2
543; GFX9-NEXT:    s_mul_hi_u32 s5, s0, s3
544; GFX9-NEXT:    s_add_u32 s9, s8, s7
545; GFX9-NEXT:    s_mul_i32 s6, s1, s2
546; GFX9-NEXT:    s_addc_u32 s5, 0, s5
547; GFX9-NEXT:    s_mul_hi_u32 s4, s1, s2
548; GFX9-NEXT:    s_add_u32 s9, s9, s6
549; GFX9-NEXT:    s_mul_hi_i32 s10, s1, s3
550; GFX9-NEXT:    s_addc_u32 s4, s5, s4
551; GFX9-NEXT:    s_addc_u32 s5, s10, 0
552; GFX9-NEXT:    s_mul_i32 s9, s1, s3
553; GFX9-NEXT:    s_add_u32 s4, s4, s9
554; GFX9-NEXT:    s_addc_u32 s5, 0, s5
555; GFX9-NEXT:    s_sub_u32 s9, s4, s2
556; GFX9-NEXT:    s_subb_u32 s10, s5, 0
557; GFX9-NEXT:    s_cmp_lt_i32 s1, 0
558; GFX9-NEXT:    s_cselect_b32 s4, s9, s4
559; GFX9-NEXT:    s_cselect_b32 s1, s10, s5
560; GFX9-NEXT:    s_sub_u32 s9, s4, s0
561; GFX9-NEXT:    s_subb_u32 s5, s1, 0
562; GFX9-NEXT:    s_cmp_lt_i32 s3, 0
563; GFX9-NEXT:    s_cselect_b32 s5, s5, s1
564; GFX9-NEXT:    s_cselect_b32 s4, s9, s4
565; GFX9-NEXT:    s_add_i32 s1, s8, s7
566; GFX9-NEXT:    s_add_i32 s1, s1, s6
567; GFX9-NEXT:    s_ashr_i32 s6, s1, 31
568; GFX9-NEXT:    s_mov_b32 s7, s6
569; GFX9-NEXT:    s_mul_i32 s0, s0, s2
570; GFX9-NEXT:    s_cmp_lg_u64 s[4:5], s[6:7]
571; GFX9-NEXT:    s_cselect_b32 s1, 0, s1
572; GFX9-NEXT:    s_cselect_b32 s0, 0, s0
573; GFX9-NEXT:    v_mov_b32_e32 v0, s0
574; GFX9-NEXT:    v_mov_b32_e32 v1, s1
575; GFX9-NEXT:    global_store_dwordx2 v[0:1], v[0:1], off
576; GFX9-NEXT:    s_endpgm
577;
578; GFX10-LABEL: smulo_i64_s:
579; GFX10:       ; %bb.0: ; %bb
580; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
581; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
582; GFX10-NEXT:    s_mul_i32 s7, s0, s3
583; GFX10-NEXT:    s_mul_hi_u32 s8, s0, s2
584; GFX10-NEXT:    s_mul_hi_u32 s5, s0, s3
585; GFX10-NEXT:    s_mul_i32 s6, s1, s2
586; GFX10-NEXT:    s_add_u32 s11, s8, s7
587; GFX10-NEXT:    s_mul_hi_u32 s4, s1, s2
588; GFX10-NEXT:    s_addc_u32 s5, 0, s5
589; GFX10-NEXT:    s_mul_hi_i32 s9, s1, s3
590; GFX10-NEXT:    s_add_u32 s11, s11, s6
591; GFX10-NEXT:    s_mul_i32 s10, s1, s3
592; GFX10-NEXT:    s_addc_u32 s4, s5, s4
593; GFX10-NEXT:    s_addc_u32 s5, s9, 0
594; GFX10-NEXT:    s_add_u32 s4, s4, s10
595; GFX10-NEXT:    s_addc_u32 s5, 0, s5
596; GFX10-NEXT:    s_sub_u32 s9, s4, s2
597; GFX10-NEXT:    s_subb_u32 s10, s5, 0
598; GFX10-NEXT:    s_cmp_lt_i32 s1, 0
599; GFX10-NEXT:    s_cselect_b32 s1, s9, s4
600; GFX10-NEXT:    s_cselect_b32 s4, s10, s5
601; GFX10-NEXT:    s_sub_u32 s9, s1, s0
602; GFX10-NEXT:    s_subb_u32 s5, s4, 0
603; GFX10-NEXT:    s_cmp_lt_i32 s3, 0
604; GFX10-NEXT:    s_mul_i32 s0, s0, s2
605; GFX10-NEXT:    s_cselect_b32 s5, s5, s4
606; GFX10-NEXT:    s_cselect_b32 s4, s9, s1
607; GFX10-NEXT:    s_add_i32 s1, s8, s7
608; GFX10-NEXT:    s_add_i32 s1, s1, s6
609; GFX10-NEXT:    s_ashr_i32 s6, s1, 31
610; GFX10-NEXT:    s_mov_b32 s7, s6
611; GFX10-NEXT:    s_cmp_lg_u64 s[4:5], s[6:7]
612; GFX10-NEXT:    s_cselect_b32 s0, 0, s0
613; GFX10-NEXT:    s_cselect_b32 s1, 0, s1
614; GFX10-NEXT:    v_mov_b32_e32 v0, s0
615; GFX10-NEXT:    v_mov_b32_e32 v1, s1
616; GFX10-NEXT:    global_store_dwordx2 v[0:1], v[0:1], off
617; GFX10-NEXT:    s_endpgm
618;
619; GFX11-LABEL: smulo_i64_s:
620; GFX11:       ; %bb.0: ; %bb
621; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
622; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
623; GFX11-NEXT:    s_mul_i32 s7, s0, s3
624; GFX11-NEXT:    s_mul_hi_u32 s8, s0, s2
625; GFX11-NEXT:    s_mul_hi_u32 s5, s0, s3
626; GFX11-NEXT:    s_mul_i32 s6, s1, s2
627; GFX11-NEXT:    s_add_u32 s11, s8, s7
628; GFX11-NEXT:    s_mul_hi_u32 s4, s1, s2
629; GFX11-NEXT:    s_addc_u32 s5, 0, s5
630; GFX11-NEXT:    s_mul_hi_i32 s9, s1, s3
631; GFX11-NEXT:    s_add_u32 s11, s11, s6
632; GFX11-NEXT:    s_mul_i32 s10, s1, s3
633; GFX11-NEXT:    s_addc_u32 s4, s5, s4
634; GFX11-NEXT:    s_addc_u32 s5, s9, 0
635; GFX11-NEXT:    s_add_u32 s4, s4, s10
636; GFX11-NEXT:    s_addc_u32 s5, 0, s5
637; GFX11-NEXT:    s_sub_u32 s9, s4, s2
638; GFX11-NEXT:    s_subb_u32 s10, s5, 0
639; GFX11-NEXT:    s_cmp_lt_i32 s1, 0
640; GFX11-NEXT:    s_cselect_b32 s1, s9, s4
641; GFX11-NEXT:    s_cselect_b32 s4, s10, s5
642; GFX11-NEXT:    s_sub_u32 s9, s1, s0
643; GFX11-NEXT:    s_subb_u32 s5, s4, 0
644; GFX11-NEXT:    s_cmp_lt_i32 s3, 0
645; GFX11-NEXT:    s_mul_i32 s0, s0, s2
646; GFX11-NEXT:    s_cselect_b32 s5, s5, s4
647; GFX11-NEXT:    s_cselect_b32 s4, s9, s1
648; GFX11-NEXT:    s_add_i32 s1, s8, s7
649; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
650; GFX11-NEXT:    s_add_i32 s1, s1, s6
651; GFX11-NEXT:    s_ashr_i32 s6, s1, 31
652; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
653; GFX11-NEXT:    s_mov_b32 s7, s6
654; GFX11-NEXT:    s_cmp_lg_u64 s[4:5], s[6:7]
655; GFX11-NEXT:    s_cselect_b32 s0, 0, s0
656; GFX11-NEXT:    s_cselect_b32 s1, 0, s1
657; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
658; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
659; GFX11-NEXT:    global_store_b64 v[0:1], v[0:1], off
660; GFX11-NEXT:    s_endpgm
661;
662; GFX12-LABEL: smulo_i64_s:
663; GFX12:       ; %bb.0: ; %bb
664; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
665; GFX12-NEXT:    s_mov_b32 s5, 0
666; GFX12-NEXT:    s_wait_kmcnt 0x0
667; GFX12-NEXT:    s_mul_hi_u32 s7, s0, s3
668; GFX12-NEXT:    s_mul_i32 s6, s0, s3
669; GFX12-NEXT:    s_mul_hi_u32 s4, s0, s2
670; GFX12-NEXT:    s_mul_i32 s10, s1, s2
671; GFX12-NEXT:    s_add_nc_u64 s[6:7], s[4:5], s[6:7]
672; GFX12-NEXT:    s_mul_hi_u32 s9, s1, s2
673; GFX12-NEXT:    s_mul_hi_i32 s11, s1, s3
674; GFX12-NEXT:    s_add_co_u32 s4, s6, s10
675; GFX12-NEXT:    s_add_co_ci_u32 s4, s7, s9
676; GFX12-NEXT:    s_mul_i32 s8, s1, s3
677; GFX12-NEXT:    s_add_co_ci_u32 s9, s11, 0
678; GFX12-NEXT:    s_cmp_lt_i32 s1, 0
679; GFX12-NEXT:    s_add_nc_u64 s[6:7], s[4:5], s[8:9]
680; GFX12-NEXT:    s_mov_b32 s4, s2
681; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
682; GFX12-NEXT:    s_sub_nc_u64 s[8:9], s[6:7], s[4:5]
683; GFX12-NEXT:    s_mov_b32 s4, s0
684; GFX12-NEXT:    s_cselect_b32 s7, s9, s7
685; GFX12-NEXT:    s_cselect_b32 s6, s8, s6
686; GFX12-NEXT:    s_cmp_lt_i32 s3, 0
687; GFX12-NEXT:    s_sub_nc_u64 s[4:5], s[6:7], s[4:5]
688; GFX12-NEXT:    s_mul_u64 s[0:1], s[0:1], s[2:3]
689; GFX12-NEXT:    s_cselect_b32 s3, s5, s7
690; GFX12-NEXT:    s_cselect_b32 s2, s4, s6
691; GFX12-NEXT:    s_ashr_i32 s4, s1, 31
692; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
693; GFX12-NEXT:    s_mov_b32 s5, s4
694; GFX12-NEXT:    s_cmp_lg_u64 s[2:3], s[4:5]
695; GFX12-NEXT:    s_cselect_b32 s0, 0, s0
696; GFX12-NEXT:    s_cselect_b32 s1, 0, s1
697; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
698; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
699; GFX12-NEXT:    global_store_b64 v[0:1], v[0:1], off
700; GFX12-NEXT:    s_endpgm
701bb:
702  %umulo = tail call { i64, i1 } @llvm.smul.with.overflow.i64(i64 %x, i64 %y)
703  %mul = extractvalue { i64, i1 } %umulo, 0
704  %overflow = extractvalue { i64, i1 } %umulo, 1
705  %res = select i1 %overflow, i64 0, i64 %mul
706  store i64 %res, ptr addrspace(1) undef
707  ret void
708}
709
710define { i64, i1 } @smulo_i64_v_4(i64 %i) {
711; SI-LABEL: smulo_i64_v_4:
712; SI:       ; %bb.0: ; %bb
713; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
714; SI-NEXT:    v_lshl_b64 v[5:6], v[0:1], 2
715; SI-NEXT:    v_alignbit_b32 v4, v1, v0, 30
716; SI-NEXT:    v_ashr_i64 v[2:3], v[5:6], 2
717; SI-NEXT:    v_cmp_ne_u64_e32 vcc, v[2:3], v[0:1]
718; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
719; SI-NEXT:    v_mov_b32_e32 v0, v5
720; SI-NEXT:    v_mov_b32_e32 v1, v4
721; SI-NEXT:    s_setpc_b64 s[30:31]
722;
723; GFX9-LABEL: smulo_i64_v_4:
724; GFX9:       ; %bb.0: ; %bb
725; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
726; GFX9-NEXT:    v_lshlrev_b64 v[4:5], 2, v[0:1]
727; GFX9-NEXT:    v_alignbit_b32 v3, v1, v0, 30
728; GFX9-NEXT:    v_ashrrev_i64 v[5:6], 2, v[4:5]
729; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, v[5:6], v[0:1]
730; GFX9-NEXT:    v_mov_b32_e32 v0, v4
731; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
732; GFX9-NEXT:    v_mov_b32_e32 v1, v3
733; GFX9-NEXT:    s_setpc_b64 s[30:31]
734;
735; GFX10-LABEL: smulo_i64_v_4:
736; GFX10:       ; %bb.0: ; %bb
737; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
738; GFX10-NEXT:    v_lshlrev_b64 v[4:5], 2, v[0:1]
739; GFX10-NEXT:    v_alignbit_b32 v3, v1, v0, 30
740; GFX10-NEXT:    v_ashrrev_i64 v[5:6], 2, v[4:5]
741; GFX10-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[5:6], v[0:1]
742; GFX10-NEXT:    v_mov_b32_e32 v0, v4
743; GFX10-NEXT:    v_mov_b32_e32 v1, v3
744; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
745; GFX10-NEXT:    s_setpc_b64 s[30:31]
746;
747; GFX11-LABEL: smulo_i64_v_4:
748; GFX11:       ; %bb.0: ; %bb
749; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
750; GFX11-NEXT:    v_lshlrev_b64 v[4:5], 2, v[0:1]
751; GFX11-NEXT:    v_alignbit_b32 v3, v1, v0, 30
752; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
753; GFX11-NEXT:    v_ashrrev_i64 v[5:6], 2, v[4:5]
754; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[5:6], v[0:1]
755; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3)
756; GFX11-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v3
757; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
758; GFX11-NEXT:    s_setpc_b64 s[30:31]
759;
760; GFX12-LABEL: smulo_i64_v_4:
761; GFX12:       ; %bb.0: ; %bb
762; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
763; GFX12-NEXT:    s_wait_expcnt 0x0
764; GFX12-NEXT:    s_wait_samplecnt 0x0
765; GFX12-NEXT:    s_wait_bvhcnt 0x0
766; GFX12-NEXT:    s_wait_kmcnt 0x0
767; GFX12-NEXT:    v_lshlrev_b64_e32 v[4:5], 2, v[0:1]
768; GFX12-NEXT:    v_alignbit_b32 v3, v1, v0, 30
769; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
770; GFX12-NEXT:    v_ashrrev_i64 v[5:6], 2, v[4:5]
771; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[5:6], v[0:1]
772; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
773; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v3
774; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
775; GFX12-NEXT:    s_setpc_b64 s[30:31]
776bb:
777  %umulo = tail call { i64, i1 } @llvm.smul.with.overflow.i64(i64 %i, i64 4)
778  ret { i64, i1 } %umulo
779}
780
781define { i64, i1 } @umulo_i64_v_4(i64 %i) {
782; SI-LABEL: umulo_i64_v_4:
783; SI:       ; %bb.0: ; %bb
784; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
785; SI-NEXT:    v_and_b32_e32 v7, 0x3fffffff, v1
786; SI-NEXT:    v_mov_b32_e32 v6, v0
787; SI-NEXT:    v_lshl_b64 v[4:5], v[0:1], 2
788; SI-NEXT:    v_alignbit_b32 v3, v1, v0, 30
789; SI-NEXT:    v_cmp_ne_u64_e32 vcc, v[6:7], v[0:1]
790; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
791; SI-NEXT:    v_mov_b32_e32 v0, v4
792; SI-NEXT:    v_mov_b32_e32 v1, v3
793; SI-NEXT:    s_setpc_b64 s[30:31]
794;
795; GFX9-LABEL: umulo_i64_v_4:
796; GFX9:       ; %bb.0: ; %bb
797; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
798; GFX9-NEXT:    v_and_b32_e32 v7, 0x3fffffff, v1
799; GFX9-NEXT:    v_mov_b32_e32 v6, v0
800; GFX9-NEXT:    v_lshlrev_b64 v[4:5], 2, v[0:1]
801; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, v[6:7], v[0:1]
802; GFX9-NEXT:    v_alignbit_b32 v3, v1, v0, 30
803; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
804; GFX9-NEXT:    v_mov_b32_e32 v0, v4
805; GFX9-NEXT:    v_mov_b32_e32 v1, v3
806; GFX9-NEXT:    s_setpc_b64 s[30:31]
807;
808; GFX10-LABEL: umulo_i64_v_4:
809; GFX10:       ; %bb.0: ; %bb
810; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
811; GFX10-NEXT:    v_and_b32_e32 v7, 0x3fffffff, v1
812; GFX10-NEXT:    v_mov_b32_e32 v6, v0
813; GFX10-NEXT:    v_lshlrev_b64 v[4:5], 2, v[0:1]
814; GFX10-NEXT:    v_alignbit_b32 v3, v1, v0, 30
815; GFX10-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[6:7], v[0:1]
816; GFX10-NEXT:    v_mov_b32_e32 v0, v4
817; GFX10-NEXT:    v_mov_b32_e32 v1, v3
818; GFX10-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
819; GFX10-NEXT:    s_setpc_b64 s[30:31]
820;
821; GFX11-LABEL: umulo_i64_v_4:
822; GFX11:       ; %bb.0: ; %bb
823; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
824; GFX11-NEXT:    v_dual_mov_b32 v6, v0 :: v_dual_and_b32 v7, 0x3fffffff, v1
825; GFX11-NEXT:    v_lshlrev_b64 v[4:5], 2, v[0:1]
826; GFX11-NEXT:    v_alignbit_b32 v3, v1, v0, 30
827; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
828; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[6:7], v[0:1]
829; GFX11-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v3
830; GFX11-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
831; GFX11-NEXT:    s_setpc_b64 s[30:31]
832;
833; GFX12-LABEL: umulo_i64_v_4:
834; GFX12:       ; %bb.0: ; %bb
835; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
836; GFX12-NEXT:    s_wait_expcnt 0x0
837; GFX12-NEXT:    s_wait_samplecnt 0x0
838; GFX12-NEXT:    s_wait_bvhcnt 0x0
839; GFX12-NEXT:    s_wait_kmcnt 0x0
840; GFX12-NEXT:    v_dual_mov_b32 v6, v0 :: v_dual_and_b32 v7, 0x3fffffff, v1
841; GFX12-NEXT:    v_lshlrev_b64_e32 v[4:5], 2, v[0:1]
842; GFX12-NEXT:    v_alignbit_b32 v3, v1, v0, 30
843; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
844; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[6:7], v[0:1]
845; GFX12-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v3
846; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
847; GFX12-NEXT:    s_setpc_b64 s[30:31]
848bb:
849  %umulo = tail call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %i, i64 4)
850  ret { i64, i1 } %umulo
851}
852
853declare { i64, i1 } @llvm.umul.with.overflow.i64(i64, i64)
854declare { i64, i1 } @llvm.smul.with.overflow.i64(i64, i64)
855