xref: /llvm-project/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8-system.ll (revision 31af5e9001508dd2e58d2232e900adba01896736) 
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes=atomic-expand %s | FileCheck %s
3
4define i8 @test_atomicrmw_xchg_i8_global_system(ptr addrspace(1) %ptr, i8 %value) {
5; CHECK-LABEL: @test_atomicrmw_xchg_i8_global_system(
6; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
7; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
8; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
9; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
10; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
11; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
12; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
13; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
14; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
15; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
16; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
17; CHECK:       atomicrmw.start:
18; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
19; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
20; CHECK-NEXT:    [[TMP6:%.*]] = or i32 [[TMP5]], [[VALOPERAND_SHIFTED]]
21; CHECK-NEXT:    [[TMP7:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP6]] seq_cst seq_cst, align 4
22; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
23; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
24; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
25; CHECK:       atomicrmw.end:
26; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
27; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
28; CHECK-NEXT:    ret i8 [[EXTRACTED]]
29;
30  %res = atomicrmw xchg ptr addrspace(1) %ptr, i8 %value seq_cst
31  ret i8 %res
32}
33
34define i8 @test_atomicrmw_add_i8_global_system(ptr addrspace(1) %ptr, i8 %value) {
35; CHECK-LABEL: @test_atomicrmw_add_i8_global_system(
36; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
37; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
38; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
39; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
40; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
41; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
42; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
43; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
44; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
45; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
46; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
47; CHECK:       atomicrmw.start:
48; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
49; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
50; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]
51; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
52; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]
53; CHECK-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] seq_cst seq_cst, align 4
54; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
55; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
56; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
57; CHECK:       atomicrmw.end:
58; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
59; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
60; CHECK-NEXT:    ret i8 [[EXTRACTED]]
61;
62  %res = atomicrmw add ptr addrspace(1) %ptr, i8 %value seq_cst
63  ret i8 %res
64}
65
66define i8 @test_atomicrmw_add_i8_global_system_align2(ptr addrspace(1) %ptr, i8 %value) {
67; CHECK-LABEL: @test_atomicrmw_add_i8_global_system_align2(
68; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
69; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
70; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
71; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
72; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
73; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
74; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
75; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
76; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
77; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
78; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
79; CHECK:       atomicrmw.start:
80; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
81; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
82; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]
83; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
84; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]
85; CHECK-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] seq_cst seq_cst, align 4
86; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
87; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
88; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
89; CHECK:       atomicrmw.end:
90; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
91; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
92; CHECK-NEXT:    ret i8 [[EXTRACTED]]
93;
94  %res = atomicrmw add ptr addrspace(1) %ptr, i8 %value seq_cst, align 2
95  ret i8 %res
96}
97
98define i8 @test_atomicrmw_add_i8_global_system_align4(ptr addrspace(1) %ptr, i8 %value) {
99; CHECK-LABEL: @test_atomicrmw_add_i8_global_system_align4(
100; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[VALUE:%.*]] to i32
101; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4
102; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
103; CHECK:       atomicrmw.start:
104; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
105; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], [[TMP1]]
106; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[NEW]], 255
107; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[LOADED]], -256
108; CHECK-NEXT:    [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP3]]
109; CHECK-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[TMP5]] seq_cst seq_cst, align 4
110; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
111; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
112; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
113; CHECK:       atomicrmw.end:
114; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i8
115; CHECK-NEXT:    ret i8 [[EXTRACTED]]
116;
117  %res = atomicrmw add ptr addrspace(1) %ptr, i8 %value seq_cst, align 4
118  ret i8 %res
119}
120
121define i8 @test_atomicrmw_sub_i8_global_system(ptr addrspace(1) %ptr, i8 %value) {
122; CHECK-LABEL: @test_atomicrmw_sub_i8_global_system(
123; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
124; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
125; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
126; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
127; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
128; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
129; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
130; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
131; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
132; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
133; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
134; CHECK:       atomicrmw.start:
135; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
136; CHECK-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
137; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]
138; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
139; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]
140; CHECK-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] seq_cst seq_cst, align 4
141; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
142; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
143; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
144; CHECK:       atomicrmw.end:
145; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
146; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
147; CHECK-NEXT:    ret i8 [[EXTRACTED]]
148;
149  %res = atomicrmw sub ptr addrspace(1) %ptr, i8 %value seq_cst
150  ret i8 %res
151}
152
153define i8 @test_atomicrmw_and_i8_global_system(ptr addrspace(1) %ptr, i8 %value) {
154; CHECK-LABEL: @test_atomicrmw_and_i8_global_system(
155; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
156; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
157; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
158; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
159; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
160; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
161; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
162; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
163; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
164; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
165; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] seq_cst, align 4
166; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
167; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
168; CHECK-NEXT:    ret i8 [[EXTRACTED]]
169;
170  %res = atomicrmw and ptr addrspace(1) %ptr, i8 %value seq_cst
171  ret i8 %res
172}
173
174define i8 @test_atomicrmw_nand_i8_global_system(ptr addrspace(1) %ptr, i8 %value) {
175; CHECK-LABEL: @test_atomicrmw_nand_i8_global_system(
176; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
177; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
178; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
179; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
180; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
181; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
182; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
183; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
184; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
185; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
186; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
187; CHECK:       atomicrmw.start:
188; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
189; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
190; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[TMP5]], -1
191; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[NEW]], [[MASK]]
192; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
193; CHECK-NEXT:    [[TMP8:%.*]] = or i32 [[TMP7]], [[TMP6]]
194; CHECK-NEXT:    [[TMP9:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP8]] seq_cst seq_cst, align 4
195; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP9]], 1
196; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP9]], 0
197; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
198; CHECK:       atomicrmw.end:
199; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
200; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
201; CHECK-NEXT:    ret i8 [[EXTRACTED]]
202;
203  %res = atomicrmw nand ptr addrspace(1) %ptr, i8 %value seq_cst
204  ret i8 %res
205}
206
207define i8 @test_atomicrmw_or_i8_global_system(ptr addrspace(1) %ptr, i8 %value) {
208; CHECK-LABEL: @test_atomicrmw_or_i8_global_system(
209; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
210; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
211; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
212; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
213; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
214; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
215; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
216; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
217; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
218; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw or ptr addrspace(1) [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] seq_cst, align 4
219; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
220; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
221; CHECK-NEXT:    ret i8 [[EXTRACTED]]
222;
223  %res = atomicrmw or ptr addrspace(1) %ptr, i8 %value seq_cst
224  ret i8 %res
225}
226
227define i8 @test_atomicrmw_xor_i8_global_system(ptr addrspace(1) %ptr, i8 %value) {
228; CHECK-LABEL: @test_atomicrmw_xor_i8_global_system(
229; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
230; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
231; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
232; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
233; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
234; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
235; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
236; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
237; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
238; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw xor ptr addrspace(1) [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] seq_cst, align 4
239; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
240; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
241; CHECK-NEXT:    ret i8 [[EXTRACTED]]
242;
243  %res = atomicrmw xor ptr addrspace(1) %ptr, i8 %value seq_cst
244  ret i8 %res
245}
246
247define i8 @test_atomicrmw_max_i8_global_system(ptr addrspace(1) %ptr, i8 %value) {
248; CHECK-LABEL: @test_atomicrmw_max_i8_global_system(
249; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
250; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
251; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
252; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
253; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
254; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
255; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
256; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
257; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
258; CHECK:       atomicrmw.start:
259; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
260; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]
261; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
262; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt i8 [[EXTRACTED]], [[VALUE:%.*]]
263; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP4]], i8 [[EXTRACTED]], i8 [[VALUE]]
264; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
265; CHECK-NEXT:    [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]
266; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
267; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]
268; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
269; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
270; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
271; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
272; CHECK:       atomicrmw.end:
273; CHECK-NEXT:    [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
274; CHECK-NEXT:    [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i8
275; CHECK-NEXT:    ret i8 [[EXTRACTED3]]
276;
277  %res = atomicrmw max ptr addrspace(1) %ptr, i8 %value seq_cst
278  ret i8 %res
279}
280
281define i8 @test_atomicrmw_min_i8_global_system(ptr addrspace(1) %ptr, i8 %value) {
282; CHECK-LABEL: @test_atomicrmw_min_i8_global_system(
283; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
284; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
285; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
286; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
287; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
288; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
289; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
290; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
291; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
292; CHECK:       atomicrmw.start:
293; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
294; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]
295; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
296; CHECK-NEXT:    [[TMP4:%.*]] = icmp sle i8 [[EXTRACTED]], [[VALUE:%.*]]
297; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP4]], i8 [[EXTRACTED]], i8 [[VALUE]]
298; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
299; CHECK-NEXT:    [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]
300; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
301; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]
302; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
303; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
304; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
305; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
306; CHECK:       atomicrmw.end:
307; CHECK-NEXT:    [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
308; CHECK-NEXT:    [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i8
309; CHECK-NEXT:    ret i8 [[EXTRACTED3]]
310;
311  %res = atomicrmw min ptr addrspace(1) %ptr, i8 %value seq_cst
312  ret i8 %res
313}
314
315define i8 @test_atomicrmw_umax_i8_global_system(ptr addrspace(1) %ptr, i8 %value) {
316; CHECK-LABEL: @test_atomicrmw_umax_i8_global_system(
317; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
318; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
319; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
320; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
321; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
322; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
323; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
324; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
325; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
326; CHECK:       atomicrmw.start:
327; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
328; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]
329; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
330; CHECK-NEXT:    [[TMP4:%.*]] = icmp ugt i8 [[EXTRACTED]], [[VALUE:%.*]]
331; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP4]], i8 [[EXTRACTED]], i8 [[VALUE]]
332; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
333; CHECK-NEXT:    [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]
334; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
335; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]
336; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
337; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
338; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
339; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
340; CHECK:       atomicrmw.end:
341; CHECK-NEXT:    [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
342; CHECK-NEXT:    [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i8
343; CHECK-NEXT:    ret i8 [[EXTRACTED3]]
344;
345  %res = atomicrmw umax ptr addrspace(1) %ptr, i8 %value seq_cst
346  ret i8 %res
347}
348
349define i8 @test_atomicrmw_umin_i8_global_system(ptr addrspace(1) %ptr, i8 %value) {
350; CHECK-LABEL: @test_atomicrmw_umin_i8_global_system(
351; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
352; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
353; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
354; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
355; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
356; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
357; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
358; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
359; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
360; CHECK:       atomicrmw.start:
361; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
362; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]
363; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
364; CHECK-NEXT:    [[TMP4:%.*]] = icmp ule i8 [[EXTRACTED]], [[VALUE:%.*]]
365; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP4]], i8 [[EXTRACTED]], i8 [[VALUE]]
366; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
367; CHECK-NEXT:    [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]
368; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
369; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]
370; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
371; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
372; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
373; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
374; CHECK:       atomicrmw.end:
375; CHECK-NEXT:    [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
376; CHECK-NEXT:    [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i8
377; CHECK-NEXT:    ret i8 [[EXTRACTED3]]
378;
379  %res = atomicrmw umin ptr addrspace(1) %ptr, i8 %value seq_cst
380  ret i8 %res
381}
382
383define i8 @test_cmpxchg_i8_global_system(ptr addrspace(1) %out, i8 %in, i8 %old) {
384; CHECK-LABEL: @test_cmpxchg_i8_global_system(
385; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr addrspace(1) [[OUT:%.*]], i64 4
386; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[GEP]], i64 -4)
387; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[GEP]] to i64
388; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
389; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
390; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
391; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
392; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
393; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[IN:%.*]] to i32
394; CHECK-NEXT:    [[TMP4:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
395; CHECK-NEXT:    [[TMP5:%.*]] = zext i8 [[OLD:%.*]] to i32
396; CHECK-NEXT:    [[TMP6:%.*]] = shl i32 [[TMP5]], [[SHIFTAMT]]
397; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
398; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP7]], [[INV_MASK]]
399; CHECK-NEXT:    br label [[PARTWORD_CMPXCHG_LOOP:%.*]]
400; CHECK:       partword.cmpxchg.loop:
401; CHECK-NEXT:    [[TMP9:%.*]] = phi i32 [ [[TMP8]], [[TMP0:%.*]] ], [ [[TMP15:%.*]], [[PARTWORD_CMPXCHG_FAILURE:%.*]] ]
402; CHECK-NEXT:    [[TMP10:%.*]] = or i32 [[TMP9]], [[TMP4]]
403; CHECK-NEXT:    [[TMP11:%.*]] = or i32 [[TMP9]], [[TMP6]]
404; CHECK-NEXT:    [[TMP12:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[TMP11]], i32 [[TMP10]] seq_cst seq_cst, align 4
405; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { i32, i1 } [[TMP12]], 0
406; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { i32, i1 } [[TMP12]], 1
407; CHECK-NEXT:    br i1 [[TMP14]], label [[PARTWORD_CMPXCHG_END:%.*]], label [[PARTWORD_CMPXCHG_FAILURE]]
408; CHECK:       partword.cmpxchg.failure:
409; CHECK-NEXT:    [[TMP15]] = and i32 [[TMP13]], [[INV_MASK]]
410; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne i32 [[TMP9]], [[TMP15]]
411; CHECK-NEXT:    br i1 [[TMP16]], label [[PARTWORD_CMPXCHG_LOOP]], label [[PARTWORD_CMPXCHG_END]]
412; CHECK:       partword.cmpxchg.end:
413; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP13]], [[SHIFTAMT]]
414; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
415; CHECK-NEXT:    [[TMP17:%.*]] = insertvalue { i8, i1 } poison, i8 [[EXTRACTED]], 0
416; CHECK-NEXT:    [[TMP18:%.*]] = insertvalue { i8, i1 } [[TMP17]], i1 [[TMP14]], 1
417; CHECK-NEXT:    [[EXTRACT:%.*]] = extractvalue { i8, i1 } [[TMP18]], 0
418; CHECK-NEXT:    ret i8 [[EXTRACT]]
419;
420  %gep = getelementptr i8, ptr addrspace(1) %out, i64 4
421  %res = cmpxchg ptr addrspace(1) %gep, i8 %old, i8 %in seq_cst seq_cst
422  %extract = extractvalue {i8, i1} %res, 0
423  ret i8 %extract
424}
425
426define i8 @test_atomicrmw_inc_i8_global_system(ptr addrspace(1) %ptr, i8 %value) {
427; CHECK-LABEL: @test_atomicrmw_inc_i8_global_system(
428; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
429; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
430; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
431; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
432; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
433; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
434; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
435; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
436; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
437; CHECK:       atomicrmw.start:
438; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
439; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]
440; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
441; CHECK-NEXT:    [[TMP4:%.*]] = add i8 [[EXTRACTED]], 1
442; CHECK-NEXT:    [[TMP5:%.*]] = icmp uge i8 [[EXTRACTED]], [[VALUE:%.*]]
443; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i8 0, i8 [[TMP4]]
444; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
445; CHECK-NEXT:    [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]
446; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
447; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]
448; CHECK-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
449; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
450; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
451; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
452; CHECK:       atomicrmw.end:
453; CHECK-NEXT:    [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
454; CHECK-NEXT:    [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i8
455; CHECK-NEXT:    ret i8 [[EXTRACTED3]]
456;
457  %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i8 %value seq_cst
458  ret i8 %res
459}
460
461define i8 @test_atomicrmw_inc_i8_global_system_align2(ptr addrspace(1) %ptr, i8 %value) {
462; CHECK-LABEL: @test_atomicrmw_inc_i8_global_system_align2(
463; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
464; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
465; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
466; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
467; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
468; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
469; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
470; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
471; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
472; CHECK:       atomicrmw.start:
473; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
474; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]
475; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
476; CHECK-NEXT:    [[TMP4:%.*]] = add i8 [[EXTRACTED]], 1
477; CHECK-NEXT:    [[TMP5:%.*]] = icmp uge i8 [[EXTRACTED]], [[VALUE:%.*]]
478; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i8 0, i8 [[TMP4]]
479; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
480; CHECK-NEXT:    [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]
481; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
482; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]
483; CHECK-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
484; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
485; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
486; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
487; CHECK:       atomicrmw.end:
488; CHECK-NEXT:    [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
489; CHECK-NEXT:    [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i8
490; CHECK-NEXT:    ret i8 [[EXTRACTED3]]
491;
492  %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i8 %value seq_cst, align 2
493  ret i8 %res
494}
495
496define i8 @test_atomicrmw_inc_i8_global_system_align4(ptr addrspace(1) %ptr, i8 %value) {
497; CHECK-LABEL: @test_atomicrmw_inc_i8_global_system_align4(
498; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4
499; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
500; CHECK:       atomicrmw.start:
501; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
502; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i8
503; CHECK-NEXT:    [[TMP2:%.*]] = add i8 [[EXTRACTED]], 1
504; CHECK-NEXT:    [[TMP3:%.*]] = icmp uge i8 [[EXTRACTED]], [[VALUE:%.*]]
505; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 0, i8 [[TMP2]]
506; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
507; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], -256
508; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]
509; CHECK-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
510; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
511; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
512; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
513; CHECK:       atomicrmw.end:
514; CHECK-NEXT:    [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i8
515; CHECK-NEXT:    ret i8 [[EXTRACTED1]]
516;
517  %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i8 %value seq_cst, align 4
518  ret i8 %res
519}
520
521define i8 @test_atomicrmw_inc_i8_flat_system(ptr %ptr, i8 %value) {
522; CHECK-LABEL: @test_atomicrmw_inc_i8_flat_system(
523; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)
524; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64
525; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
526; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
527; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
528; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
529; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
530; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 4
531; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
532; CHECK:       atomicrmw.start:
533; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
534; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]
535; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
536; CHECK-NEXT:    [[TMP4:%.*]] = add i8 [[EXTRACTED]], 1
537; CHECK-NEXT:    [[TMP5:%.*]] = icmp uge i8 [[EXTRACTED]], [[VALUE:%.*]]
538; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i8 0, i8 [[TMP4]]
539; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
540; CHECK-NEXT:    [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]
541; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
542; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]
543; CHECK-NEXT:    [[TMP6:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
544; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
545; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
546; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
547; CHECK:       atomicrmw.end:
548; CHECK-NEXT:    [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
549; CHECK-NEXT:    [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i8
550; CHECK-NEXT:    ret i8 [[EXTRACTED3]]
551;
552  %res = atomicrmw uinc_wrap ptr %ptr, i8 %value seq_cst
553  ret i8 %res
554}
555
556define i8 @test_atomicrmw_inc_i8_flat_system_align2(ptr %ptr, i8 %value) {
557; CHECK-LABEL: @test_atomicrmw_inc_i8_flat_system_align2(
558; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)
559; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64
560; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
561; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
562; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
563; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
564; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
565; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 4
566; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
567; CHECK:       atomicrmw.start:
568; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
569; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]
570; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
571; CHECK-NEXT:    [[TMP4:%.*]] = add i8 [[EXTRACTED]], 1
572; CHECK-NEXT:    [[TMP5:%.*]] = icmp uge i8 [[EXTRACTED]], [[VALUE:%.*]]
573; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i8 0, i8 [[TMP4]]
574; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
575; CHECK-NEXT:    [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]
576; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
577; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]
578; CHECK-NEXT:    [[TMP6:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
579; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
580; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
581; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
582; CHECK:       atomicrmw.end:
583; CHECK-NEXT:    [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
584; CHECK-NEXT:    [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i8
585; CHECK-NEXT:    ret i8 [[EXTRACTED3]]
586;
587  %res = atomicrmw uinc_wrap ptr %ptr, i8 %value seq_cst, align 2
588  ret i8 %res
589}
590
591define i8 @test_atomicrmw_inc_i8_flat_system_align4(ptr %ptr, i8 %value) {
592; CHECK-LABEL: @test_atomicrmw_inc_i8_flat_system_align4(
593; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[PTR:%.*]], align 4
594; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
595; CHECK:       atomicrmw.start:
596; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
597; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i8
598; CHECK-NEXT:    [[TMP2:%.*]] = add i8 [[EXTRACTED]], 1
599; CHECK-NEXT:    [[TMP3:%.*]] = icmp uge i8 [[EXTRACTED]], [[VALUE:%.*]]
600; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 0, i8 [[TMP2]]
601; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
602; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], -256
603; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]
604; CHECK-NEXT:    [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
605; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
606; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
607; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
608; CHECK:       atomicrmw.end:
609; CHECK-NEXT:    [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i8
610; CHECK-NEXT:    ret i8 [[EXTRACTED1]]
611;
612  %res = atomicrmw uinc_wrap ptr %ptr, i8 %value seq_cst, align 4
613  ret i8 %res
614}
615
616define i8 @test_atomicrmw_dec_i8_global_system(ptr addrspace(1) %ptr, i8 %value) {
617; CHECK-LABEL: @test_atomicrmw_dec_i8_global_system(
618; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
619; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
620; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
621; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
622; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
623; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
624; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
625; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
626; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
627; CHECK:       atomicrmw.start:
628; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
629; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]
630; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
631; CHECK-NEXT:    [[TMP4:%.*]] = sub i8 [[EXTRACTED]], 1
632; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i8 [[EXTRACTED]], 0
633; CHECK-NEXT:    [[TMP6:%.*]] = icmp ugt i8 [[EXTRACTED]], [[VALUE:%.*]]
634; CHECK-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
635; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP7]], i8 [[VALUE]], i8 [[TMP4]]
636; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
637; CHECK-NEXT:    [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]
638; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
639; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]
640; CHECK-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
641; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
642; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
643; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
644; CHECK:       atomicrmw.end:
645; CHECK-NEXT:    [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
646; CHECK-NEXT:    [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i8
647; CHECK-NEXT:    ret i8 [[EXTRACTED3]]
648;
649  %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i8 %value seq_cst
650  ret i8 %res
651}
652
653define i8 @test_atomicrmw_dec_i8_global_system_align2(ptr addrspace(1) %ptr, i8 %value) {
654; CHECK-LABEL: @test_atomicrmw_dec_i8_global_system_align2(
655; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
656; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
657; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
658; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
659; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
660; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
661; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
662; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
663; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
664; CHECK:       atomicrmw.start:
665; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
666; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]
667; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
668; CHECK-NEXT:    [[TMP4:%.*]] = sub i8 [[EXTRACTED]], 1
669; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i8 [[EXTRACTED]], 0
670; CHECK-NEXT:    [[TMP6:%.*]] = icmp ugt i8 [[EXTRACTED]], [[VALUE:%.*]]
671; CHECK-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
672; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP7]], i8 [[VALUE]], i8 [[TMP4]]
673; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
674; CHECK-NEXT:    [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]
675; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
676; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]
677; CHECK-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
678; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
679; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
680; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
681; CHECK:       atomicrmw.end:
682; CHECK-NEXT:    [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
683; CHECK-NEXT:    [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i8
684; CHECK-NEXT:    ret i8 [[EXTRACTED3]]
685;
686  %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i8 %value seq_cst, align 2
687  ret i8 %res
688}
689
690define i8 @test_atomicrmw_dec_i8_global_system_align4(ptr addrspace(1) %ptr, i8 %value) {
691; CHECK-LABEL: @test_atomicrmw_dec_i8_global_system_align4(
692; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4
693; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
694; CHECK:       atomicrmw.start:
695; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
696; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i8
697; CHECK-NEXT:    [[TMP2:%.*]] = sub i8 [[EXTRACTED]], 1
698; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i8 [[EXTRACTED]], 0
699; CHECK-NEXT:    [[TMP4:%.*]] = icmp ugt i8 [[EXTRACTED]], [[VALUE:%.*]]
700; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
701; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i8 [[VALUE]], i8 [[TMP2]]
702; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
703; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], -256
704; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]
705; CHECK-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
706; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
707; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
708; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
709; CHECK:       atomicrmw.end:
710; CHECK-NEXT:    [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i8
711; CHECK-NEXT:    ret i8 [[EXTRACTED1]]
712;
713  %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i8 %value seq_cst, align 4
714  ret i8 %res
715}
716
717define i8 @test_atomicrmw_dec_i8_flat_system(ptr %ptr, i8 %value) {
718; CHECK-LABEL: @test_atomicrmw_dec_i8_flat_system(
719; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)
720; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64
721; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
722; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
723; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
724; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
725; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
726; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 4
727; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
728; CHECK:       atomicrmw.start:
729; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
730; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]
731; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
732; CHECK-NEXT:    [[TMP4:%.*]] = sub i8 [[EXTRACTED]], 1
733; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i8 [[EXTRACTED]], 0
734; CHECK-NEXT:    [[TMP6:%.*]] = icmp ugt i8 [[EXTRACTED]], [[VALUE:%.*]]
735; CHECK-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
736; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP7]], i8 [[VALUE]], i8 [[TMP4]]
737; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
738; CHECK-NEXT:    [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]
739; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
740; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]
741; CHECK-NEXT:    [[TMP8:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
742; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
743; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
744; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
745; CHECK:       atomicrmw.end:
746; CHECK-NEXT:    [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
747; CHECK-NEXT:    [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i8
748; CHECK-NEXT:    ret i8 [[EXTRACTED3]]
749;
750  %res = atomicrmw udec_wrap ptr %ptr, i8 %value seq_cst
751  ret i8 %res
752}
753
754define i8 @test_atomicrmw_dec_i8_flat_system_align2(ptr %ptr, i8 %value) {
755; CHECK-LABEL: @test_atomicrmw_dec_i8_flat_system_align2(
756; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 -4)
757; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[PTR]] to i64
758; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
759; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
760; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
761; CHECK-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
762; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
763; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[ALIGNEDADDR]], align 4
764; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
765; CHECK:       atomicrmw.start:
766; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP3]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
767; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[LOADED]], [[SHIFTAMT]]
768; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
769; CHECK-NEXT:    [[TMP4:%.*]] = sub i8 [[EXTRACTED]], 1
770; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i8 [[EXTRACTED]], 0
771; CHECK-NEXT:    [[TMP6:%.*]] = icmp ugt i8 [[EXTRACTED]], [[VALUE:%.*]]
772; CHECK-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
773; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP7]], i8 [[VALUE]], i8 [[TMP4]]
774; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
775; CHECK-NEXT:    [[SHIFTED1:%.*]] = shl nuw i32 [[EXTENDED]], [[SHIFTAMT]]
776; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
777; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[SHIFTED1]]
778; CHECK-NEXT:    [[TMP8:%.*]] = cmpxchg ptr [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
779; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
780; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
781; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
782; CHECK:       atomicrmw.end:
783; CHECK-NEXT:    [[SHIFTED2:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
784; CHECK-NEXT:    [[EXTRACTED3:%.*]] = trunc i32 [[SHIFTED2]] to i8
785; CHECK-NEXT:    ret i8 [[EXTRACTED3]]
786;
787  %res = atomicrmw udec_wrap ptr %ptr, i8 %value seq_cst, align 2
788  ret i8 %res
789}
790
791define i8 @test_atomicrmw_dec_i8_flat_system_align4(ptr %ptr, i8 %value) {
792; CHECK-LABEL: @test_atomicrmw_dec_i8_flat_system_align4(
793; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[PTR:%.*]], align 4
794; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
795; CHECK:       atomicrmw.start:
796; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
797; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[LOADED]] to i8
798; CHECK-NEXT:    [[TMP2:%.*]] = sub i8 [[EXTRACTED]], 1
799; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i8 [[EXTRACTED]], 0
800; CHECK-NEXT:    [[TMP4:%.*]] = icmp ugt i8 [[EXTRACTED]], [[VALUE:%.*]]
801; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
802; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i8 [[VALUE]], i8 [[TMP2]]
803; CHECK-NEXT:    [[EXTENDED:%.*]] = zext i8 [[NEW]] to i32
804; CHECK-NEXT:    [[UNMASKED:%.*]] = and i32 [[LOADED]], -256
805; CHECK-NEXT:    [[INSERTED:%.*]] = or i32 [[UNMASKED]], [[EXTENDED]]
806; CHECK-NEXT:    [[TMP6:%.*]] = cmpxchg ptr [[PTR]], i32 [[LOADED]], i32 [[INSERTED]] seq_cst seq_cst, align 4
807; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
808; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
809; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
810; CHECK:       atomicrmw.end:
811; CHECK-NEXT:    [[EXTRACTED1:%.*]] = trunc i32 [[NEWLOADED]] to i8
812; CHECK-NEXT:    ret i8 [[EXTRACTED1]]
813;
814  %res = atomicrmw udec_wrap ptr %ptr, i8 %value seq_cst, align 4
815  ret i8 %res
816}
817