xref: /llvm-project/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.dec.ll (revision 9e9907f1cfa424366fba58d9520f9305b537cec9)
1; RUN: llc -mtriple=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,CIVI %s
2; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,CIVI %s
3; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
4
5declare i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) nocapture, i32, i32, i32, i1) #2
6declare i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) nocapture, i32, i32, i32, i1) #2
7declare i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr nocapture, i32, i32, i32, i1) #2
8
9declare i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) nocapture, i64, i32, i32, i1) #2
10declare i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) nocapture, i64, i32, i32, i1) #2
11declare i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr nocapture, i64, i32, i32, i1) #2
12
13declare i32 @llvm.amdgcn.workitem.id.x() #1
14
15; GCN-LABEL: {{^}}lds_atomic_dec_ret_i32:
16; CIVI-DAG: s_mov_b32 m0
17; GFX9-NOT: m0
18
19; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42
20; GCN: ds_dec_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[K]]
21define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {
22  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 false)
23  store i32 %result, ptr addrspace(1) %out
24  ret void
25}
26
27; GCN-LABEL: {{^}}lds_atomic_dec_ret_i32_offset:
28; CIVI-DAG: s_mov_b32 m0
29; GFX9-NOT: m0
30
31; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42
32; GCN: ds_dec_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[K]] offset:16
33define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {
34  %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4
35  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %gep, i32 42, i32 0, i32 0, i1 false)
36  store i32 %result, ptr addrspace(1) %out
37  ret void
38}
39
40; GCN-LABEL: {{^}}lds_atomic_dec_noret_i32:
41; CIVI-DAG: s_mov_b32 m0
42; GFX9-NOT: m0
43
44; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],
45; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
46; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
47; GCN: ds_dec_u32 [[VPTR]], [[DATA]]
48define amdgpu_kernel void @lds_atomic_dec_noret_i32(ptr addrspace(3) %ptr) nounwind {
49  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %ptr, i32 42, i32 0, i32 0, i1 false)
50  ret void
51}
52
53; GCN-LABEL: {{^}}lds_atomic_dec_noret_i32_offset:
54; CIVI-DAG: s_mov_b32 m0
55; GFX9-NOT: m0
56
57; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42
58; GCN: ds_dec_u32 v{{[0-9]+}}, [[K]] offset:16
59define amdgpu_kernel void @lds_atomic_dec_noret_i32_offset(ptr addrspace(3) %ptr) nounwind {
60  %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4
61  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %gep, i32 42, i32 0, i32 0, i1 false)
62  ret void
63}
64
65; GCN-LABEL: {{^}}global_atomic_dec_ret_i32:
66; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42
67; CIVI: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 glc{{$}}
68; GFX9-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
69; GFX9: global_atomic_dec v{{[0-9]+}}, [[ZERO]], [[K]], s{{\[[0-9]+:[0-9]+\]}} glc{{$}}
70define amdgpu_kernel void @global_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {
71  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %ptr, i32 42, i32 0, i32 0, i1 false)
72  store i32 %result, ptr addrspace(1) %out
73  ret void
74}
75
76; GCN-LABEL: {{^}}global_atomic_dec_ret_i32_offset:
77; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42
78; CIVI: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16 glc{{$}}
79
80; GFX9-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
81; GFX9: global_atomic_dec v{{[0-9]+}}, [[ZERO]], [[K]], s{{\[[0-9]+:[0-9]+\]}} offset:16 glc{{$}}
82define amdgpu_kernel void @global_atomic_dec_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {
83  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
84  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false)
85  store i32 %result, ptr addrspace(1) %out
86  ret void
87}
88
89; GCN-LABEL: {{^}}global_atomic_dec_noret_i32:
90; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42
91; CIVI: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
92
93; GFX9-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
94; GFX9: global_atomic_dec [[ZERO]], [[K]], s{{\[[0-9]+:[0-9]+\]$}}
95define amdgpu_kernel void @global_atomic_dec_noret_i32(ptr addrspace(1) %ptr) nounwind {
96  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %ptr, i32 42, i32 0, i32 0, i1 false)
97  ret void
98}
99
100; GCN-LABEL: {{^}}global_atomic_dec_noret_i32_offset:
101; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42
102; CIVI: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16{{$}}
103
104; GFX9-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
105; GFX9: global_atomic_dec [[ZERO]], [[K]], s{{\[[0-9]+:[0-9]+\]}} offset:16{{$}}
106define amdgpu_kernel void @global_atomic_dec_noret_i32_offset(ptr addrspace(1) %ptr) nounwind {
107  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
108  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false)
109  ret void
110}
111
112; GCN-LABEL: {{^}}global_atomic_dec_ret_i32_offset_addr64:
113; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
114; CI: buffer_atomic_dec [[K]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:20 glc{{$}}
115; VI: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}}
116define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_addr64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {
117  %id = call i32 @llvm.amdgcn.workitem.id.x()
118  %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id
119  %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id
120  %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5
121  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false)
122  store i32 %result, ptr addrspace(1) %out.gep
123  ret void
124}
125
126; GCN-LABEL: {{^}}global_atomic_dec_noret_i32_offset_addr64:
127; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
128; CI: buffer_atomic_dec [[K]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:20{{$}}
129; VI: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}}
130define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_addr64(ptr addrspace(1) %ptr) #0 {
131  %id = call i32 @llvm.amdgcn.workitem.id.x()
132  %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id
133  %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5
134  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %gep, i32 42, i32 0, i32 0, i1 false)
135  ret void
136}
137
138; GCN-LABEL: {{^}}flat_atomic_dec_ret_i32:
139; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
140; GCN: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}}
141define amdgpu_kernel void @flat_atomic_dec_ret_i32(ptr %out, ptr %ptr) #0 {
142  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr, i32 42, i32 0, i32 0, i1 false)
143  store i32 %result, ptr %out
144  ret void
145}
146
147; GCN-LABEL: {{^}}flat_atomic_dec_ret_i32_offset:
148; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
149; CIVI: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}}
150; GFX9: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:16 glc{{$}}
151define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset(ptr %out, ptr %ptr) #0 {
152  %gep = getelementptr i32, ptr %ptr, i32 4
153  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false)
154  store i32 %result, ptr %out
155  ret void
156}
157
158; GCN-LABEL: {{^}}flat_atomic_dec_noret_i32:
159; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
160; GCN: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}}
161define amdgpu_kernel void @flat_atomic_dec_noret_i32(ptr %ptr) nounwind {
162  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr, i32 42, i32 0, i32 0, i1 false)
163  ret void
164}
165
166; GCN-LABEL: {{^}}flat_atomic_dec_noret_i32_offset:
167; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
168; CIVI: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}}
169; GFX9: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:16{{$}}
170define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset(ptr %ptr) nounwind {
171  %gep = getelementptr i32, ptr %ptr, i32 4
172  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false)
173  ret void
174}
175
176; GCN-LABEL: {{^}}flat_atomic_dec_ret_i32_offset_addr64:
177; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
178; CIVI: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}}
179; GFX9: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:20 glc{{$}}
180define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(ptr %out, ptr %ptr) #0 {
181  %id = call i32 @llvm.amdgcn.workitem.id.x()
182  %gep.tid = getelementptr i32, ptr %ptr, i32 %id
183  %out.gep = getelementptr i32, ptr %out, i32 %id
184  %gep = getelementptr i32, ptr %gep.tid, i32 5
185  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false)
186  store i32 %result, ptr %out.gep
187  ret void
188}
189
190; GCN-LABEL: {{^}}flat_atomic_dec_noret_i32_offset_addr64:
191; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
192; CIVI: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}}
193; GFX9: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:20{{$}}
194define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #0 {
195  %id = call i32 @llvm.amdgcn.workitem.id.x()
196  %gep.tid = getelementptr i32, ptr %ptr, i32 %id
197  %gep = getelementptr i32, ptr %gep.tid, i32 5
198  %result = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false)
199  ret void
200}
201
202; GCN-LABEL: {{^}}flat_atomic_dec_ret_i64:
203; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
204; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
205; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v[[[KLO]]:[[KHI]]] glc{{$}}
206define amdgpu_kernel void @flat_atomic_dec_ret_i64(ptr %out, ptr %ptr) #0 {
207  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %ptr, i64 42, i32 0, i32 0, i1 false)
208  store i64 %result, ptr %out
209  ret void
210}
211
212; GCN-LABEL: {{^}}flat_atomic_dec_ret_i64_offset:
213; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
214; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
215; CIVI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v[[[KLO]]:[[KHI]]] glc{{$}}
216; GFX9: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v[[[KLO]]:[[KHI]]] offset:32 glc{{$}}
217define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset(ptr %out, ptr %ptr) #0 {
218  %gep = getelementptr i64, ptr %ptr, i32 4
219  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false)
220  store i64 %result, ptr %out
221  ret void
222}
223
224; GCN-LABEL: {{^}}flat_atomic_dec_noret_i64:
225; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
226; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
227; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v[[[KLO]]:[[KHI]]{{\]$}}
228define amdgpu_kernel void @flat_atomic_dec_noret_i64(ptr %ptr) nounwind {
229  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %ptr, i64 42, i32 0, i32 0, i1 false)
230  ret void
231}
232
233; GCN-LABEL: {{^}}flat_atomic_dec_noret_i64_offset:
234; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
235; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
236; CIVI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v[[[KLO]]:[[KHI]]{{\]$}}
237; GFX9: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v[[[KLO]]:[[KHI]]] offset:32{{$}}
238define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset(ptr %ptr) nounwind {
239  %gep = getelementptr i64, ptr %ptr, i32 4
240  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false)
241  ret void
242}
243
244; GCN-LABEL: {{^}}flat_atomic_dec_ret_i64_offset_addr64:
245; GCN: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
246; GCN: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
247; CIVI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v[[[KLO]]:[[KHI]]] glc{{$}}
248; GFX9: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v[[[KLO]]:[[KHI]]] offset:40 glc{{$}}
249define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(ptr %out, ptr %ptr) #0 {
250  %id = call i32 @llvm.amdgcn.workitem.id.x()
251  %gep.tid = getelementptr i64, ptr %ptr, i32 %id
252  %out.gep = getelementptr i64, ptr %out, i32 %id
253  %gep = getelementptr i64, ptr %gep.tid, i32 5
254  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false)
255  store i64 %result, ptr %out.gep
256  ret void
257}
258
259; GCN-LABEL: {{^}}flat_atomic_dec_noret_i64_offset_addr64:
260; GCN: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
261; GCN: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
262; CIVI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v[[[KLO]]:[[KHI]]{{\]$}}
263; GFX9: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v[[[KLO]]:[[KHI]]] offset:40{{$}}
264define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #0 {
265  %id = call i32 @llvm.amdgcn.workitem.id.x()
266  %gep.tid = getelementptr i64, ptr %ptr, i32 %id
267  %gep = getelementptr i64, ptr %gep.tid, i32 5
268  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false)
269  ret void
270}
271
272@lds0 = addrspace(3) global [512 x i32] undef
273
274; GCN-LABEL: {{^}}atomic_dec_shl_base_lds_0:
275; CIVI-DAG: s_mov_b32 m0
276; GFX9-NOT: m0
277
278; GCN-DAG:  v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
279; GCN: ds_dec_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
280define amdgpu_kernel void @atomic_dec_shl_base_lds_0(ptr addrspace(1) %out, ptr addrspace(1) %add_use) #0 {
281  %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
282  %idx.0 = add nsw i32 %tid.x, 2
283  %arrayidx0 = getelementptr inbounds [512 x i32], ptr addrspace(3) @lds0, i32 0, i32 %idx.0
284  %val0 = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %arrayidx0, i32 9, i32 0, i32 0, i1 false)
285  store i32 %idx.0, ptr addrspace(1) %add_use
286  store i32 %val0, ptr addrspace(1) %out
287  ret void
288}
289
290; GCN-LABEL: {{^}}lds_atomic_dec_ret_i64:
291; CIVI-DAG: s_mov_b32 m0
292; GFX9-NOT: m0
293
294; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
295; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
296; GCN: ds_dec_rtn_u64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, v[[[KLO]]:[[KHI]]]{{$}}
297define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {
298  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %ptr, i64 42, i32 0, i32 0, i1 false)
299  store i64 %result, ptr addrspace(1) %out
300  ret void
301}
302
303; GCN-LABEL: {{^}}lds_atomic_dec_ret_i64_offset:
304; CIVI-DAG: s_mov_b32 m0
305; GFX9-NOT: m0
306
307; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
308; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
309; GCN: ds_dec_rtn_u64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, v[[[KLO]]:[[KHI]]] offset:32
310define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {
311  %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4
312  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %gep, i64 42, i32 0, i32 0, i1 false)
313  store i64 %result, ptr addrspace(1) %out
314  ret void
315}
316
317; GCN-LABEL: {{^}}lds_atomic_dec_noret_i64:
318; CIVI-DAG: s_mov_b32 m0
319; GFX9-NOT: m0
320
321; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
322; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
323; GCN: ds_dec_u64 v{{[0-9]+}}, v[[[KLO]]:[[KHI]]]{{$}}
324define amdgpu_kernel void @lds_atomic_dec_noret_i64(ptr addrspace(3) %ptr) nounwind {
325  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %ptr, i64 42, i32 0, i32 0, i1 false)
326  ret void
327}
328
329; GCN-LABEL: {{^}}lds_atomic_dec_noret_i64_offset:
330; CIVI-DAG: s_mov_b32 m0
331; GFX9-NOT: m0
332
333; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
334; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
335; GCN: ds_dec_u64 v{{[0-9]+}}, v[[[KLO]]:[[KHI]]] offset:32{{$}}
336define amdgpu_kernel void @lds_atomic_dec_noret_i64_offset(ptr addrspace(3) %ptr) nounwind {
337  %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4
338  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %gep, i64 42, i32 0, i32 0, i1 false)
339  ret void
340}
341
342; GCN-LABEL: {{^}}global_atomic_dec_ret_i64:
343; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
344; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
345; GFX9: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}}
346; CIVI: buffer_atomic_dec_x2 v[[[KLO]]:[[KHI]]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 glc{{$}}
347
348; GFX9: global_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v[[ZERO]], v[[[KLO]]:[[KHI]]], s{{\[[0-9]+:[0-9]+\]}} glc{{$}}
349define amdgpu_kernel void @global_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {
350  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %ptr, i64 42, i32 0, i32 0, i1 false)
351  store i64 %result, ptr addrspace(1) %out
352  ret void
353}
354
355; GCN-LABEL: {{^}}global_atomic_dec_ret_i64_offset:
356; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
357; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
358; GFX9: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}}
359; CIVI: buffer_atomic_dec_x2 v[[[KLO]]:[[KHI]]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:32 glc{{$}}
360; GFX9: global_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v[[ZERO]], v[[[KLO]]:[[KHI]]], s{{\[[0-9]+:[0-9]+\]}} offset:32 glc{{$}}
361define amdgpu_kernel void @global_atomic_dec_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {
362  %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4
363  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false)
364  store i64 %result, ptr addrspace(1) %out
365  ret void
366}
367
368; GCN-LABEL: {{^}}global_atomic_dec_noret_i64:
369; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
370; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
371; GFX9: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}}
372; CIVI: buffer_atomic_dec_x2 v[[[KLO]]:[[KHI]]], off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
373; GFX9: global_atomic_dec_x2 v[[ZERO]], v[[[KLO]]:[[KHI]]], s{{\[[0-9]+:[0-9]+\]$}}
374define amdgpu_kernel void @global_atomic_dec_noret_i64(ptr addrspace(1) %ptr) nounwind {
375  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %ptr, i64 42, i32 0, i32 0, i1 false)
376  ret void
377}
378
379; GCN-LABEL: {{^}}global_atomic_dec_noret_i64_offset:
380; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
381; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
382; GFX9: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}}
383; CIVI: buffer_atomic_dec_x2 v[[[KLO]]:[[KHI]]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:32{{$}}
384; GFX9: global_atomic_dec_x2 v[[ZERO]], v[[[KLO]]:[[KHI]]], s{{\[[0-9]+:[0-9]+\]}} offset:32{{$}}
385define amdgpu_kernel void @global_atomic_dec_noret_i64_offset(ptr addrspace(1) %ptr) nounwind {
386  %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4
387  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false)
388  ret void
389}
390
391; GCN-LABEL: {{^}}global_atomic_dec_ret_i64_offset_addr64:
392; GCN: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
393; CI: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}}
394; GCN: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
395; CI: buffer_atomic_dec_x2 v[[[KLO]]:[[KHI]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:40 glc{{$}}
396; VI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v[[[KLO]]:[[KHI]]] glc{{$}}
397define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_addr64(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {
398  %id = call i32 @llvm.amdgcn.workitem.id.x()
399  %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id
400  %out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %id
401  %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 5
402  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false)
403  store i64 %result, ptr addrspace(1) %out.gep
404  ret void
405}
406
407; GCN-LABEL: {{^}}global_atomic_dec_noret_i64_offset_addr64:
408; GCN: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
409; CI: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}}
410; GCN: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
411; CI: buffer_atomic_dec_x2 v[[[KLO]]:[[KHI]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:40{{$}}
412; VI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v[[[KLO]]:[[KHI]]]{{$}}
413define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(ptr addrspace(1) %ptr) #0 {
414  %id = call i32 @llvm.amdgcn.workitem.id.x()
415  %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id
416  %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 5
417  %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %gep, i64 42, i32 0, i32 0, i1 false)
418  ret void
419}
420
421@lds1 = addrspace(3) global [512 x i64] undef, align 8
422
423; GCN-LABEL: {{^}}atomic_dec_shl_base_lds_0_i64:
424; CIVI-DAG: s_mov_b32 m0
425; GFX9-NOT: m0
426
427; GCN-DAG: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 3, {{v[0-9]+}}
428; GCN: ds_dec_rtn_u64 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]], v{{\[[0-9]+:[0-9]+\]}} offset:16
429define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(ptr addrspace(1) %out, ptr addrspace(1) %add_use) #0 {
430  %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
431  %idx.0 = add nsw i32 %tid.x, 2
432  %arrayidx0 = getelementptr inbounds [512 x i64], ptr addrspace(3) @lds1, i32 0, i32 %idx.0
433  %val0 = call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %arrayidx0, i64 9, i32 0, i32 0, i1 false)
434  store i32 %idx.0, ptr addrspace(1) %add_use
435  store i64 %val0, ptr addrspace(1) %out
436  ret void
437}
438
439attributes #0 = { nounwind }
440attributes #1 = { nounwind readnone }
441attributes #2 = { nounwind argmemonly }
442