xref: /llvm-project/llvm/test/CodeGen/AArch64/atomic-ops.ll (revision 7b3bbd83c0c24087072ec5b22a76799ab31f87d5) 
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -aarch64-enable-sink-fold=true < %s | FileCheck %s --check-prefixes=CHECK,INLINE_ATOMICS
3; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+outline-atomics -aarch64-enable-sink-fold=true < %s | FileCheck %s --check-prefixes=CHECK,OUTLINE_ATOMICS
4
5@var8 = dso_local global i8 0
6@var16 = dso_local global i16 0
7@var32 = dso_local global i32 0
8@var64 = dso_local global i64 0
9
10define dso_local i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
11; INLINE_ATOMICS-LABEL: test_atomic_load_add_i8:
12; INLINE_ATOMICS:       // %bb.0:
13; INLINE_ATOMICS-NEXT:    adrp x9, var8
14; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var8
15; INLINE_ATOMICS-NEXT:  .LBB0_1: // %atomicrmw.start
16; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
17; INLINE_ATOMICS-NEXT:    ldaxrb w8, [x9]
18; INLINE_ATOMICS-NEXT:    add w10, w8, w0
19; INLINE_ATOMICS-NEXT:    stlxrb w11, w10, [x9]
20; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB0_1
21; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
22; INLINE_ATOMICS-NEXT:    mov w0, w8
23; INLINE_ATOMICS-NEXT:    ret
24;
25; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i8:
26; OUTLINE_ATOMICS:       // %bb.0:
27; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
28; OUTLINE_ATOMICS-NEXT:    adrp x1, var8
29; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var8
30; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd1_acq_rel
31; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
32; OUTLINE_ATOMICS-NEXT:    ret
33   %old = atomicrmw add ptr @var8, i8 %offset seq_cst
34   ret i8 %old
35}
36
37define dso_local i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
38; INLINE_ATOMICS-LABEL: test_atomic_load_add_i16:
39; INLINE_ATOMICS:       // %bb.0:
40; INLINE_ATOMICS-NEXT:    adrp x9, var16
41; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var16
42; INLINE_ATOMICS-NEXT:  .LBB1_1: // %atomicrmw.start
43; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
44; INLINE_ATOMICS-NEXT:    ldaxrh w8, [x9]
45; INLINE_ATOMICS-NEXT:    add w10, w8, w0
46; INLINE_ATOMICS-NEXT:    stxrh w11, w10, [x9]
47; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB1_1
48; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
49; INLINE_ATOMICS-NEXT:    mov w0, w8
50; INLINE_ATOMICS-NEXT:    ret
51;
52; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i16:
53; OUTLINE_ATOMICS:       // %bb.0:
54; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
55; OUTLINE_ATOMICS-NEXT:    adrp x1, var16
56; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var16
57; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd2_acq
58; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
59; OUTLINE_ATOMICS-NEXT:    ret
60   %old = atomicrmw add ptr @var16, i16 %offset acquire
61   ret i16 %old
62}
63
64define dso_local i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
65; INLINE_ATOMICS-LABEL: test_atomic_load_add_i32:
66; INLINE_ATOMICS:       // %bb.0:
67; INLINE_ATOMICS-NEXT:    adrp x9, var32
68; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var32
69; INLINE_ATOMICS-NEXT:  .LBB2_1: // %atomicrmw.start
70; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
71; INLINE_ATOMICS-NEXT:    ldxr w8, [x9]
72; INLINE_ATOMICS-NEXT:    add w10, w8, w0
73; INLINE_ATOMICS-NEXT:    stlxr w11, w10, [x9]
74; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB2_1
75; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
76; INLINE_ATOMICS-NEXT:    mov w0, w8
77; INLINE_ATOMICS-NEXT:    ret
78;
79; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i32:
80; OUTLINE_ATOMICS:       // %bb.0:
81; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
82; OUTLINE_ATOMICS-NEXT:    adrp x1, var32
83; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var32
84; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd4_rel
85; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
86; OUTLINE_ATOMICS-NEXT:    ret
87   %old = atomicrmw add ptr @var32, i32 %offset release
88   ret i32 %old
89}
90
91define dso_local i64 @test_atomic_load_add_i64(i64 %offset) nounwind {
92; INLINE_ATOMICS-LABEL: test_atomic_load_add_i64:
93; INLINE_ATOMICS:       // %bb.0:
94; INLINE_ATOMICS-NEXT:    adrp x9, var64
95; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var64
96; INLINE_ATOMICS-NEXT:  .LBB3_1: // %atomicrmw.start
97; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
98; INLINE_ATOMICS-NEXT:    ldxr x8, [x9]
99; INLINE_ATOMICS-NEXT:    add x10, x8, x0
100; INLINE_ATOMICS-NEXT:    stxr w11, x10, [x9]
101; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB3_1
102; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
103; INLINE_ATOMICS-NEXT:    mov x0, x8
104; INLINE_ATOMICS-NEXT:    ret
105;
106; OUTLINE_ATOMICS-LABEL: test_atomic_load_add_i64:
107; OUTLINE_ATOMICS:       // %bb.0:
108; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
109; OUTLINE_ATOMICS-NEXT:    adrp x1, var64
110; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var64
111; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd8_relax
112; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
113; OUTLINE_ATOMICS-NEXT:    ret
114   %old = atomicrmw add ptr @var64, i64 %offset monotonic
115   ret i64 %old
116}
117
118define dso_local i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
119; INLINE_ATOMICS-LABEL: test_atomic_load_sub_i8:
120; INLINE_ATOMICS:       // %bb.0:
121; INLINE_ATOMICS-NEXT:    adrp x9, var8
122; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var8
123; INLINE_ATOMICS-NEXT:  .LBB4_1: // %atomicrmw.start
124; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
125; INLINE_ATOMICS-NEXT:    ldxrb w8, [x9]
126; INLINE_ATOMICS-NEXT:    sub w10, w8, w0
127; INLINE_ATOMICS-NEXT:    stxrb w11, w10, [x9]
128; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB4_1
129; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
130; INLINE_ATOMICS-NEXT:    mov w0, w8
131; INLINE_ATOMICS-NEXT:    ret
132;
133; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i8:
134; OUTLINE_ATOMICS:       // %bb.0:
135; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
136; OUTLINE_ATOMICS-NEXT:    neg w0, w0
137; OUTLINE_ATOMICS-NEXT:    adrp x1, var8
138; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var8
139; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd1_relax
140; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
141; OUTLINE_ATOMICS-NEXT:    ret
142   %old = atomicrmw sub ptr @var8, i8 %offset monotonic
143   ret i8 %old
144}
145
146define dso_local i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
147; INLINE_ATOMICS-LABEL: test_atomic_load_sub_i16:
148; INLINE_ATOMICS:       // %bb.0:
149; INLINE_ATOMICS-NEXT:    adrp x9, var16
150; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var16
151; INLINE_ATOMICS-NEXT:  .LBB5_1: // %atomicrmw.start
152; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
153; INLINE_ATOMICS-NEXT:    ldxrh w8, [x9]
154; INLINE_ATOMICS-NEXT:    sub w10, w8, w0
155; INLINE_ATOMICS-NEXT:    stlxrh w11, w10, [x9]
156; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB5_1
157; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
158; INLINE_ATOMICS-NEXT:    mov w0, w8
159; INLINE_ATOMICS-NEXT:    ret
160;
161; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i16:
162; OUTLINE_ATOMICS:       // %bb.0:
163; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
164; OUTLINE_ATOMICS-NEXT:    neg w0, w0
165; OUTLINE_ATOMICS-NEXT:    adrp x1, var16
166; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var16
167; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd2_rel
168; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
169; OUTLINE_ATOMICS-NEXT:    ret
170   %old = atomicrmw sub ptr @var16, i16 %offset release
171   ret i16 %old
172}
173
174define dso_local i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
175; INLINE_ATOMICS-LABEL: test_atomic_load_sub_i32:
176; INLINE_ATOMICS:       // %bb.0:
177; INLINE_ATOMICS-NEXT:    adrp x9, var32
178; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var32
179; INLINE_ATOMICS-NEXT:  .LBB6_1: // %atomicrmw.start
180; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
181; INLINE_ATOMICS-NEXT:    ldaxr w8, [x9]
182; INLINE_ATOMICS-NEXT:    sub w10, w8, w0
183; INLINE_ATOMICS-NEXT:    stxr w11, w10, [x9]
184; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB6_1
185; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
186; INLINE_ATOMICS-NEXT:    mov w0, w8
187; INLINE_ATOMICS-NEXT:    ret
188;
189; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i32:
190; OUTLINE_ATOMICS:       // %bb.0:
191; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
192; OUTLINE_ATOMICS-NEXT:    neg w0, w0
193; OUTLINE_ATOMICS-NEXT:    adrp x1, var32
194; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var32
195; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd4_acq
196; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
197; OUTLINE_ATOMICS-NEXT:    ret
198   %old = atomicrmw sub ptr @var32, i32 %offset acquire
199   ret i32 %old
200}
201
202define dso_local i64 @test_atomic_load_sub_i64(i64 %offset) nounwind {
203; INLINE_ATOMICS-LABEL: test_atomic_load_sub_i64:
204; INLINE_ATOMICS:       // %bb.0:
205; INLINE_ATOMICS-NEXT:    adrp x9, var64
206; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var64
207; INLINE_ATOMICS-NEXT:  .LBB7_1: // %atomicrmw.start
208; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
209; INLINE_ATOMICS-NEXT:    ldaxr x8, [x9]
210; INLINE_ATOMICS-NEXT:    sub x10, x8, x0
211; INLINE_ATOMICS-NEXT:    stlxr w11, x10, [x9]
212; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB7_1
213; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
214; INLINE_ATOMICS-NEXT:    mov x0, x8
215; INLINE_ATOMICS-NEXT:    ret
216;
217; OUTLINE_ATOMICS-LABEL: test_atomic_load_sub_i64:
218; OUTLINE_ATOMICS:       // %bb.0:
219; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
220; OUTLINE_ATOMICS-NEXT:    neg x0, x0
221; OUTLINE_ATOMICS-NEXT:    adrp x1, var64
222; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var64
223; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldadd8_acq_rel
224; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
225; OUTLINE_ATOMICS-NEXT:    ret
226   %old = atomicrmw sub ptr @var64, i64 %offset seq_cst
227   ret i64 %old
228}
229
230define dso_local i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
231; INLINE_ATOMICS-LABEL: test_atomic_load_and_i8:
232; INLINE_ATOMICS:       // %bb.0:
233; INLINE_ATOMICS-NEXT:    adrp x9, var8
234; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var8
235; INLINE_ATOMICS-NEXT:  .LBB8_1: // %atomicrmw.start
236; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
237; INLINE_ATOMICS-NEXT:    ldxrb w8, [x9]
238; INLINE_ATOMICS-NEXT:    and w10, w8, w0
239; INLINE_ATOMICS-NEXT:    stlxrb w11, w10, [x9]
240; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB8_1
241; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
242; INLINE_ATOMICS-NEXT:    mov w0, w8
243; INLINE_ATOMICS-NEXT:    ret
244;
245; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i8:
246; OUTLINE_ATOMICS:       // %bb.0:
247; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
248; OUTLINE_ATOMICS-NEXT:    adrp x1, var8
249; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var8
250; OUTLINE_ATOMICS-NEXT:    mvn w0, w0
251; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldclr1_rel
252; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
253; OUTLINE_ATOMICS-NEXT:    ret
254   %old = atomicrmw and ptr @var8, i8 %offset release
255   ret i8 %old
256}
257
258define dso_local i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
259; INLINE_ATOMICS-LABEL: test_atomic_load_and_i16:
260; INLINE_ATOMICS:       // %bb.0:
261; INLINE_ATOMICS-NEXT:    adrp x9, var16
262; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var16
263; INLINE_ATOMICS-NEXT:  .LBB9_1: // %atomicrmw.start
264; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
265; INLINE_ATOMICS-NEXT:    ldxrh w8, [x9]
266; INLINE_ATOMICS-NEXT:    and w10, w8, w0
267; INLINE_ATOMICS-NEXT:    stxrh w11, w10, [x9]
268; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB9_1
269; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
270; INLINE_ATOMICS-NEXT:    mov w0, w8
271; INLINE_ATOMICS-NEXT:    ret
272;
273; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i16:
274; OUTLINE_ATOMICS:       // %bb.0:
275; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
276; OUTLINE_ATOMICS-NEXT:    adrp x1, var16
277; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var16
278; OUTLINE_ATOMICS-NEXT:    mvn w0, w0
279; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldclr2_relax
280; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
281; OUTLINE_ATOMICS-NEXT:    ret
282   %old = atomicrmw and ptr @var16, i16 %offset monotonic
283   ret i16 %old
284}
285
286define dso_local i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
287; INLINE_ATOMICS-LABEL: test_atomic_load_and_i32:
288; INLINE_ATOMICS:       // %bb.0:
289; INLINE_ATOMICS-NEXT:    adrp x9, var32
290; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var32
291; INLINE_ATOMICS-NEXT:  .LBB10_1: // %atomicrmw.start
292; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
293; INLINE_ATOMICS-NEXT:    ldaxr w8, [x9]
294; INLINE_ATOMICS-NEXT:    and w10, w8, w0
295; INLINE_ATOMICS-NEXT:    stlxr w11, w10, [x9]
296; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB10_1
297; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
298; INLINE_ATOMICS-NEXT:    mov w0, w8
299; INLINE_ATOMICS-NEXT:    ret
300;
301; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i32:
302; OUTLINE_ATOMICS:       // %bb.0:
303; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
304; OUTLINE_ATOMICS-NEXT:    adrp x1, var32
305; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var32
306; OUTLINE_ATOMICS-NEXT:    mvn w0, w0
307; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldclr4_acq_rel
308; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
309; OUTLINE_ATOMICS-NEXT:    ret
310   %old = atomicrmw and ptr @var32, i32 %offset seq_cst
311   ret i32 %old
312}
313
314define dso_local i64 @test_atomic_load_and_i64(i64 %offset) nounwind {
315; INLINE_ATOMICS-LABEL: test_atomic_load_and_i64:
316; INLINE_ATOMICS:       // %bb.0:
317; INLINE_ATOMICS-NEXT:    adrp x9, var64
318; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var64
319; INLINE_ATOMICS-NEXT:  .LBB11_1: // %atomicrmw.start
320; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
321; INLINE_ATOMICS-NEXT:    ldaxr x8, [x9]
322; INLINE_ATOMICS-NEXT:    and x10, x8, x0
323; INLINE_ATOMICS-NEXT:    stxr w11, x10, [x9]
324; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB11_1
325; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
326; INLINE_ATOMICS-NEXT:    mov x0, x8
327; INLINE_ATOMICS-NEXT:    ret
328;
329; OUTLINE_ATOMICS-LABEL: test_atomic_load_and_i64:
330; OUTLINE_ATOMICS:       // %bb.0:
331; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
332; OUTLINE_ATOMICS-NEXT:    adrp x1, var64
333; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var64
334; OUTLINE_ATOMICS-NEXT:    mvn x0, x0
335; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldclr8_acq
336; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
337; OUTLINE_ATOMICS-NEXT:    ret
338   %old = atomicrmw and ptr @var64, i64 %offset acquire
339   ret i64 %old
340}
341
342define dso_local i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
343; INLINE_ATOMICS-LABEL: test_atomic_load_or_i8:
344; INLINE_ATOMICS:       // %bb.0:
345; INLINE_ATOMICS-NEXT:    adrp x9, var8
346; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var8
347; INLINE_ATOMICS-NEXT:  .LBB12_1: // %atomicrmw.start
348; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
349; INLINE_ATOMICS-NEXT:    ldaxrb w8, [x9]
350; INLINE_ATOMICS-NEXT:    orr w10, w8, w0
351; INLINE_ATOMICS-NEXT:    stlxrb w11, w10, [x9]
352; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB12_1
353; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
354; INLINE_ATOMICS-NEXT:    mov w0, w8
355; INLINE_ATOMICS-NEXT:    ret
356;
357; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i8:
358; OUTLINE_ATOMICS:       // %bb.0:
359; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
360; OUTLINE_ATOMICS-NEXT:    adrp x1, var8
361; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var8
362; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldset1_acq_rel
363; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
364; OUTLINE_ATOMICS-NEXT:    ret
365   %old = atomicrmw or ptr @var8, i8 %offset seq_cst
366   ret i8 %old
367}
368
369define dso_local i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
370; INLINE_ATOMICS-LABEL: test_atomic_load_or_i16:
371; INLINE_ATOMICS:       // %bb.0:
372; INLINE_ATOMICS-NEXT:    adrp x9, var16
373; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var16
374; INLINE_ATOMICS-NEXT:  .LBB13_1: // %atomicrmw.start
375; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
376; INLINE_ATOMICS-NEXT:    ldxrh w8, [x9]
377; INLINE_ATOMICS-NEXT:    orr w10, w8, w0
378; INLINE_ATOMICS-NEXT:    stxrh w11, w10, [x9]
379; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB13_1
380; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
381; INLINE_ATOMICS-NEXT:    mov w0, w8
382; INLINE_ATOMICS-NEXT:    ret
383;
384; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i16:
385; OUTLINE_ATOMICS:       // %bb.0:
386; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
387; OUTLINE_ATOMICS-NEXT:    adrp x1, var16
388; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var16
389; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldset2_relax
390; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
391; OUTLINE_ATOMICS-NEXT:    ret
392   %old = atomicrmw or ptr @var16, i16 %offset monotonic
393   ret i16 %old
394}
395
396define dso_local i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
397; INLINE_ATOMICS-LABEL: test_atomic_load_or_i32:
398; INLINE_ATOMICS:       // %bb.0:
399; INLINE_ATOMICS-NEXT:    adrp x9, var32
400; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var32
401; INLINE_ATOMICS-NEXT:  .LBB14_1: // %atomicrmw.start
402; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
403; INLINE_ATOMICS-NEXT:    ldaxr w8, [x9]
404; INLINE_ATOMICS-NEXT:    orr w10, w8, w0
405; INLINE_ATOMICS-NEXT:    stxr w11, w10, [x9]
406; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB14_1
407; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
408; INLINE_ATOMICS-NEXT:    mov w0, w8
409; INLINE_ATOMICS-NEXT:    ret
410;
411; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i32:
412; OUTLINE_ATOMICS:       // %bb.0:
413; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
414; OUTLINE_ATOMICS-NEXT:    adrp x1, var32
415; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var32
416; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldset4_acq
417; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
418; OUTLINE_ATOMICS-NEXT:    ret
419   %old = atomicrmw or ptr @var32, i32 %offset acquire
420   ret i32 %old
421}
422
423define dso_local i64 @test_atomic_load_or_i64(i64 %offset) nounwind {
424; INLINE_ATOMICS-LABEL: test_atomic_load_or_i64:
425; INLINE_ATOMICS:       // %bb.0:
426; INLINE_ATOMICS-NEXT:    adrp x9, var64
427; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var64
428; INLINE_ATOMICS-NEXT:  .LBB15_1: // %atomicrmw.start
429; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
430; INLINE_ATOMICS-NEXT:    ldxr x8, [x9]
431; INLINE_ATOMICS-NEXT:    orr x10, x8, x0
432; INLINE_ATOMICS-NEXT:    stlxr w11, x10, [x9]
433; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB15_1
434; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
435; INLINE_ATOMICS-NEXT:    mov x0, x8
436; INLINE_ATOMICS-NEXT:    ret
437;
438; OUTLINE_ATOMICS-LABEL: test_atomic_load_or_i64:
439; OUTLINE_ATOMICS:       // %bb.0:
440; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
441; OUTLINE_ATOMICS-NEXT:    adrp x1, var64
442; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var64
443; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldset8_rel
444; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
445; OUTLINE_ATOMICS-NEXT:    ret
446   %old = atomicrmw or ptr @var64, i64 %offset release
447   ret i64 %old
448}
449
450define dso_local i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
451; INLINE_ATOMICS-LABEL: test_atomic_load_xor_i8:
452; INLINE_ATOMICS:       // %bb.0:
453; INLINE_ATOMICS-NEXT:    adrp x9, var8
454; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var8
455; INLINE_ATOMICS-NEXT:  .LBB16_1: // %atomicrmw.start
456; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
457; INLINE_ATOMICS-NEXT:    ldaxrb w8, [x9]
458; INLINE_ATOMICS-NEXT:    eor w10, w8, w0
459; INLINE_ATOMICS-NEXT:    stxrb w11, w10, [x9]
460; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB16_1
461; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
462; INLINE_ATOMICS-NEXT:    mov w0, w8
463; INLINE_ATOMICS-NEXT:    ret
464;
465; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i8:
466; OUTLINE_ATOMICS:       // %bb.0:
467; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
468; OUTLINE_ATOMICS-NEXT:    adrp x1, var8
469; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var8
470; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldeor1_acq
471; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
472; OUTLINE_ATOMICS-NEXT:    ret
473   %old = atomicrmw xor ptr @var8, i8 %offset acquire
474   ret i8 %old
475}
476
477define dso_local i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
478; INLINE_ATOMICS-LABEL: test_atomic_load_xor_i16:
479; INLINE_ATOMICS:       // %bb.0:
480; INLINE_ATOMICS-NEXT:    adrp x9, var16
481; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var16
482; INLINE_ATOMICS-NEXT:  .LBB17_1: // %atomicrmw.start
483; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
484; INLINE_ATOMICS-NEXT:    ldxrh w8, [x9]
485; INLINE_ATOMICS-NEXT:    eor w10, w8, w0
486; INLINE_ATOMICS-NEXT:    stlxrh w11, w10, [x9]
487; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB17_1
488; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
489; INLINE_ATOMICS-NEXT:    mov w0, w8
490; INLINE_ATOMICS-NEXT:    ret
491;
492; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i16:
493; OUTLINE_ATOMICS:       // %bb.0:
494; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
495; OUTLINE_ATOMICS-NEXT:    adrp x1, var16
496; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var16
497; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldeor2_rel
498; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
499; OUTLINE_ATOMICS-NEXT:    ret
500   %old = atomicrmw xor ptr @var16, i16 %offset release
501   ret i16 %old
502}
503
504define dso_local i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
505; INLINE_ATOMICS-LABEL: test_atomic_load_xor_i32:
506; INLINE_ATOMICS:       // %bb.0:
507; INLINE_ATOMICS-NEXT:    adrp x9, var32
508; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var32
509; INLINE_ATOMICS-NEXT:  .LBB18_1: // %atomicrmw.start
510; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
511; INLINE_ATOMICS-NEXT:    ldaxr w8, [x9]
512; INLINE_ATOMICS-NEXT:    eor w10, w8, w0
513; INLINE_ATOMICS-NEXT:    stlxr w11, w10, [x9]
514; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB18_1
515; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
516; INLINE_ATOMICS-NEXT:    mov w0, w8
517; INLINE_ATOMICS-NEXT:    ret
518;
519; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i32:
520; OUTLINE_ATOMICS:       // %bb.0:
521; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
522; OUTLINE_ATOMICS-NEXT:    adrp x1, var32
523; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var32
524; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldeor4_acq_rel
525; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
526; OUTLINE_ATOMICS-NEXT:    ret
527   %old = atomicrmw xor ptr @var32, i32 %offset seq_cst
528   ret i32 %old
529}
530
531define dso_local i64 @test_atomic_load_xor_i64(i64 %offset) nounwind {
532; INLINE_ATOMICS-LABEL: test_atomic_load_xor_i64:
533; INLINE_ATOMICS:       // %bb.0:
534; INLINE_ATOMICS-NEXT:    adrp x9, var64
535; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var64
536; INLINE_ATOMICS-NEXT:  .LBB19_1: // %atomicrmw.start
537; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
538; INLINE_ATOMICS-NEXT:    ldxr x8, [x9]
539; INLINE_ATOMICS-NEXT:    eor x10, x8, x0
540; INLINE_ATOMICS-NEXT:    stxr w11, x10, [x9]
541; INLINE_ATOMICS-NEXT:    cbnz w11, .LBB19_1
542; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
543; INLINE_ATOMICS-NEXT:    mov x0, x8
544; INLINE_ATOMICS-NEXT:    ret
545;
546; OUTLINE_ATOMICS-LABEL: test_atomic_load_xor_i64:
547; OUTLINE_ATOMICS:       // %bb.0:
548; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
549; OUTLINE_ATOMICS-NEXT:    adrp x1, var64
550; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var64
551; OUTLINE_ATOMICS-NEXT:    bl __aarch64_ldeor8_relax
552; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
553; OUTLINE_ATOMICS-NEXT:    ret
554   %old = atomicrmw xor ptr @var64, i64 %offset monotonic
555   ret i64 %old
556}
557
558define dso_local i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
559; INLINE_ATOMICS-LABEL: test_atomic_load_xchg_i8:
560; INLINE_ATOMICS:       // %bb.0:
561; INLINE_ATOMICS-NEXT:    // kill: def $w0 killed $w0 def $x0
562; INLINE_ATOMICS-NEXT:    adrp x9, var8
563; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var8
564; INLINE_ATOMICS-NEXT:  .LBB20_1: // %atomicrmw.start
565; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
566; INLINE_ATOMICS-NEXT:    ldxrb w8, [x9]
567; INLINE_ATOMICS-NEXT:    stxrb w10, w0, [x9]
568; INLINE_ATOMICS-NEXT:    cbnz w10, .LBB20_1
569; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
570; INLINE_ATOMICS-NEXT:    mov w0, w8
571; INLINE_ATOMICS-NEXT:    ret
572;
573; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i8:
574; OUTLINE_ATOMICS:       // %bb.0:
575; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
576; OUTLINE_ATOMICS-NEXT:    adrp x1, var8
577; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var8
578; OUTLINE_ATOMICS-NEXT:    bl __aarch64_swp1_relax
579; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
580; OUTLINE_ATOMICS-NEXT:    ret
581   %old = atomicrmw xchg ptr @var8, i8 %offset monotonic
582   ret i8 %old
583}
584
585define dso_local i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
586; INLINE_ATOMICS-LABEL: test_atomic_load_xchg_i16:
587; INLINE_ATOMICS:       // %bb.0:
588; INLINE_ATOMICS-NEXT:    // kill: def $w0 killed $w0 def $x0
589; INLINE_ATOMICS-NEXT:    adrp x9, var16
590; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var16
591; INLINE_ATOMICS-NEXT:  .LBB21_1: // %atomicrmw.start
592; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
593; INLINE_ATOMICS-NEXT:    ldaxrh w8, [x9]
594; INLINE_ATOMICS-NEXT:    stlxrh w10, w0, [x9]
595; INLINE_ATOMICS-NEXT:    cbnz w10, .LBB21_1
596; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
597; INLINE_ATOMICS-NEXT:    mov w0, w8
598; INLINE_ATOMICS-NEXT:    ret
599;
600; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i16:
601; OUTLINE_ATOMICS:       // %bb.0:
602; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
603; OUTLINE_ATOMICS-NEXT:    adrp x1, var16
604; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var16
605; OUTLINE_ATOMICS-NEXT:    bl __aarch64_swp2_acq_rel
606; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
607; OUTLINE_ATOMICS-NEXT:    ret
608   %old = atomicrmw xchg ptr @var16, i16 %offset seq_cst
609   ret i16 %old
610}
611
612define dso_local i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
613; INLINE_ATOMICS-LABEL: test_atomic_load_xchg_i32:
614; INLINE_ATOMICS:       // %bb.0:
615; INLINE_ATOMICS-NEXT:    mov w8, w0
616; INLINE_ATOMICS-NEXT:    adrp x9, var32
617; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var32
618; INLINE_ATOMICS-NEXT:  .LBB22_1: // %atomicrmw.start
619; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
620; INLINE_ATOMICS-NEXT:    ldxr w0, [x9]
621; INLINE_ATOMICS-NEXT:    stlxr w10, w8, [x9]
622; INLINE_ATOMICS-NEXT:    cbnz w10, .LBB22_1
623; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
624; INLINE_ATOMICS-NEXT:    // kill: def $w0 killed $w0 killed $x0
625; INLINE_ATOMICS-NEXT:    ret
626;
627; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i32:
628; OUTLINE_ATOMICS:       // %bb.0:
629; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
630; OUTLINE_ATOMICS-NEXT:    adrp x1, var32
631; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var32
632; OUTLINE_ATOMICS-NEXT:    bl __aarch64_swp4_rel
633; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
634; OUTLINE_ATOMICS-NEXT:    ret
635   %old = atomicrmw xchg ptr @var32, i32 %offset release
636   ret i32 %old
637}
638
639define dso_local i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind {
640; INLINE_ATOMICS-LABEL: test_atomic_load_xchg_i64:
641; INLINE_ATOMICS:       // %bb.0:
642; INLINE_ATOMICS-NEXT:    adrp x9, var64
643; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var64
644; INLINE_ATOMICS-NEXT:  .LBB23_1: // %atomicrmw.start
645; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
646; INLINE_ATOMICS-NEXT:    ldaxr x8, [x9]
647; INLINE_ATOMICS-NEXT:    stxr w10, x0, [x9]
648; INLINE_ATOMICS-NEXT:    cbnz w10, .LBB23_1
649; INLINE_ATOMICS-NEXT:  // %bb.2: // %atomicrmw.end
650; INLINE_ATOMICS-NEXT:    mov x0, x8
651; INLINE_ATOMICS-NEXT:    ret
652;
653; OUTLINE_ATOMICS-LABEL: test_atomic_load_xchg_i64:
654; OUTLINE_ATOMICS:       // %bb.0:
655; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
656; OUTLINE_ATOMICS-NEXT:    adrp x1, var64
657; OUTLINE_ATOMICS-NEXT:    add x1, x1, :lo12:var64
658; OUTLINE_ATOMICS-NEXT:    bl __aarch64_swp8_acq
659; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
660; OUTLINE_ATOMICS-NEXT:    ret
661   %old = atomicrmw xchg ptr @var64, i64 %offset acquire
662   ret i64 %old
663}
664
665
666define dso_local i8 @test_atomic_load_min_i8(i8 %offset) nounwind {
667; CHECK-LABEL: test_atomic_load_min_i8:
668; CHECK:       // %bb.0:
669; CHECK-NEXT:    adrp x9, var8
670; CHECK-NEXT:    add x9, x9, :lo12:var8
671; CHECK-NEXT:  .LBB24_1: // %atomicrmw.start
672; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
673; CHECK-NEXT:    ldaxrb w10, [x9]
674; CHECK-NEXT:    sxtb w8, w10
675; CHECK-NEXT:    cmp w8, w0, sxtb
676; CHECK-NEXT:    csel w10, w10, w0, le
677; CHECK-NEXT:    stxrb w11, w10, [x9]
678; CHECK-NEXT:    cbnz w11, .LBB24_1
679; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
680; CHECK-NEXT:    mov w0, w8
681; CHECK-NEXT:    ret
682   %old = atomicrmw min ptr @var8, i8 %offset acquire
683   ret i8 %old
684}
685
686define dso_local i16 @test_atomic_load_min_i16(i16 %offset) nounwind {
687; CHECK-LABEL: test_atomic_load_min_i16:
688; CHECK:       // %bb.0:
689; CHECK-NEXT:    adrp x9, var16
690; CHECK-NEXT:    add x9, x9, :lo12:var16
691; CHECK-NEXT:  .LBB25_1: // %atomicrmw.start
692; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
693; CHECK-NEXT:    ldxrh w10, [x9]
694; CHECK-NEXT:    sxth w8, w10
695; CHECK-NEXT:    cmp w8, w0, sxth
696; CHECK-NEXT:    csel w10, w10, w0, le
697; CHECK-NEXT:    stlxrh w11, w10, [x9]
698; CHECK-NEXT:    cbnz w11, .LBB25_1
699; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
700; CHECK-NEXT:    mov w0, w8
701; CHECK-NEXT:    ret
702   %old = atomicrmw min ptr @var16, i16 %offset release
703   ret i16 %old
704}
705
706define dso_local i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
707; CHECK-LABEL: test_atomic_load_min_i32:
708; CHECK:       // %bb.0:
709; CHECK-NEXT:    adrp x9, var32
710; CHECK-NEXT:    add x9, x9, :lo12:var32
711; CHECK-NEXT:  .LBB26_1: // %atomicrmw.start
712; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
713; CHECK-NEXT:    ldxr w8, [x9]
714; CHECK-NEXT:    cmp w8, w0
715; CHECK-NEXT:    csel w10, w8, w0, le
716; CHECK-NEXT:    stxr w11, w10, [x9]
717; CHECK-NEXT:    cbnz w11, .LBB26_1
718; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
719; CHECK-NEXT:    mov w0, w8
720; CHECK-NEXT:    ret
721   %old = atomicrmw min ptr @var32, i32 %offset monotonic
722   ret i32 %old
723}
724
725define dso_local i64 @test_atomic_load_min_i64(i64 %offset) nounwind {
726; CHECK-LABEL: test_atomic_load_min_i64:
727; CHECK:       // %bb.0:
728; CHECK-NEXT:    adrp x9, var64
729; CHECK-NEXT:    add x9, x9, :lo12:var64
730; CHECK-NEXT:  .LBB27_1: // %atomicrmw.start
731; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
732; CHECK-NEXT:    ldaxr x8, [x9]
733; CHECK-NEXT:    cmp x8, x0
734; CHECK-NEXT:    csel x10, x8, x0, le
735; CHECK-NEXT:    stlxr w11, x10, [x9]
736; CHECK-NEXT:    cbnz w11, .LBB27_1
737; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
738; CHECK-NEXT:    mov x0, x8
739; CHECK-NEXT:    ret
740   %old = atomicrmw min ptr @var64, i64 %offset seq_cst
741   ret i64 %old
742}
743
744define dso_local i8 @test_atomic_load_max_i8(i8 %offset) nounwind {
745; CHECK-LABEL: test_atomic_load_max_i8:
746; CHECK:       // %bb.0:
747; CHECK-NEXT:    adrp x9, var8
748; CHECK-NEXT:    add x9, x9, :lo12:var8
749; CHECK-NEXT:  .LBB28_1: // %atomicrmw.start
750; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
751; CHECK-NEXT:    ldaxrb w10, [x9]
752; CHECK-NEXT:    sxtb w8, w10
753; CHECK-NEXT:    cmp w8, w0, sxtb
754; CHECK-NEXT:    csel w10, w10, w0, gt
755; CHECK-NEXT:    stlxrb w11, w10, [x9]
756; CHECK-NEXT:    cbnz w11, .LBB28_1
757; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
758; CHECK-NEXT:    mov w0, w8
759; CHECK-NEXT:    ret
760   %old = atomicrmw max ptr @var8, i8 %offset seq_cst
761   ret i8 %old
762}
763
764define dso_local i16 @test_atomic_load_max_i16(i16 %offset) nounwind {
765; CHECK-LABEL: test_atomic_load_max_i16:
766; CHECK:       // %bb.0:
767; CHECK-NEXT:    adrp x9, var16
768; CHECK-NEXT:    add x9, x9, :lo12:var16
769; CHECK-NEXT:  .LBB29_1: // %atomicrmw.start
770; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
771; CHECK-NEXT:    ldaxrh w10, [x9]
772; CHECK-NEXT:    sxth w8, w10
773; CHECK-NEXT:    cmp w8, w0, sxth
774; CHECK-NEXT:    csel w10, w10, w0, gt
775; CHECK-NEXT:    stxrh w11, w10, [x9]
776; CHECK-NEXT:    cbnz w11, .LBB29_1
777; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
778; CHECK-NEXT:    mov w0, w8
779; CHECK-NEXT:    ret
780   %old = atomicrmw max ptr @var16, i16 %offset acquire
781   ret i16 %old
782}
783
784define dso_local i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
785; CHECK-LABEL: test_atomic_load_max_i32:
786; CHECK:       // %bb.0:
787; CHECK-NEXT:    adrp x9, var32
788; CHECK-NEXT:    add x9, x9, :lo12:var32
789; CHECK-NEXT:  .LBB30_1: // %atomicrmw.start
790; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
791; CHECK-NEXT:    ldxr w8, [x9]
792; CHECK-NEXT:    cmp w8, w0
793; CHECK-NEXT:    csel w10, w8, w0, gt
794; CHECK-NEXT:    stlxr w11, w10, [x9]
795; CHECK-NEXT:    cbnz w11, .LBB30_1
796; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
797; CHECK-NEXT:    mov w0, w8
798; CHECK-NEXT:    ret
799   %old = atomicrmw max ptr @var32, i32 %offset release
800   ret i32 %old
801}
802
803define dso_local i64 @test_atomic_load_max_i64(i64 %offset) nounwind {
804; CHECK-LABEL: test_atomic_load_max_i64:
805; CHECK:       // %bb.0:
806; CHECK-NEXT:    adrp x9, var64
807; CHECK-NEXT:    add x9, x9, :lo12:var64
808; CHECK-NEXT:  .LBB31_1: // %atomicrmw.start
809; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
810; CHECK-NEXT:    ldxr x8, [x9]
811; CHECK-NEXT:    cmp x8, x0
812; CHECK-NEXT:    csel x10, x8, x0, gt
813; CHECK-NEXT:    stxr w11, x10, [x9]
814; CHECK-NEXT:    cbnz w11, .LBB31_1
815; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
816; CHECK-NEXT:    mov x0, x8
817; CHECK-NEXT:    ret
818   %old = atomicrmw max ptr @var64, i64 %offset monotonic
819   ret i64 %old
820}
821
822define dso_local i8 @test_atomic_load_umin_i8(i8 %offset) nounwind {
823; CHECK-LABEL: test_atomic_load_umin_i8:
824; CHECK:       // %bb.0:
825; CHECK-NEXT:    adrp x8, var8
826; CHECK-NEXT:    add x8, x8, :lo12:var8
827; CHECK-NEXT:    and w9, w0, #0xff
828; CHECK-NEXT:  .LBB32_1: // %atomicrmw.start
829; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
830; CHECK-NEXT:    ldxrb w0, [x8]
831; CHECK-NEXT:    cmp w0, w9
832; CHECK-NEXT:    csel w10, w0, w9, ls
833; CHECK-NEXT:    stxrb w11, w10, [x8]
834; CHECK-NEXT:    cbnz w11, .LBB32_1
835; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
836; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
837; CHECK-NEXT:    ret
838   %old = atomicrmw umin ptr @var8, i8 %offset monotonic
839   ret i8 %old
840}
841
842define dso_local i16 @test_atomic_load_umin_i16(i16 %offset) nounwind {
843; CHECK-LABEL: test_atomic_load_umin_i16:
844; CHECK:       // %bb.0:
845; CHECK-NEXT:    adrp x8, var16
846; CHECK-NEXT:    add x8, x8, :lo12:var16
847; CHECK-NEXT:    and w9, w0, #0xffff
848; CHECK-NEXT:  .LBB33_1: // %atomicrmw.start
849; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
850; CHECK-NEXT:    ldaxrh w0, [x8]
851; CHECK-NEXT:    cmp w0, w9
852; CHECK-NEXT:    csel w10, w0, w9, ls
853; CHECK-NEXT:    stxrh w11, w10, [x8]
854; CHECK-NEXT:    cbnz w11, .LBB33_1
855; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
856; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
857; CHECK-NEXT:    ret
858   %old = atomicrmw umin ptr @var16, i16 %offset acquire
859   ret i16 %old
860}
861
862define dso_local i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
863; CHECK-LABEL: test_atomic_load_umin_i32:
864; CHECK:       // %bb.0:
865; CHECK-NEXT:    adrp x9, var32
866; CHECK-NEXT:    add x9, x9, :lo12:var32
867; CHECK-NEXT:  .LBB34_1: // %atomicrmw.start
868; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
869; CHECK-NEXT:    ldaxr w8, [x9]
870; CHECK-NEXT:    cmp w8, w0
871; CHECK-NEXT:    csel w10, w8, w0, ls
872; CHECK-NEXT:    stlxr w11, w10, [x9]
873; CHECK-NEXT:    cbnz w11, .LBB34_1
874; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
875; CHECK-NEXT:    mov w0, w8
876; CHECK-NEXT:    ret
877   %old = atomicrmw umin ptr @var32, i32 %offset seq_cst
878   ret i32 %old
879}
880
881define dso_local i64 @test_atomic_load_umin_i64(i64 %offset) nounwind {
882; CHECK-LABEL: test_atomic_load_umin_i64:
883; CHECK:       // %bb.0:
884; CHECK-NEXT:    adrp x9, var64
885; CHECK-NEXT:    add x9, x9, :lo12:var64
886; CHECK-NEXT:  .LBB35_1: // %atomicrmw.start
887; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
888; CHECK-NEXT:    ldaxr x8, [x9]
889; CHECK-NEXT:    cmp x8, x0
890; CHECK-NEXT:    csel x10, x8, x0, ls
891; CHECK-NEXT:    stlxr w11, x10, [x9]
892; CHECK-NEXT:    cbnz w11, .LBB35_1
893; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
894; CHECK-NEXT:    mov x0, x8
895; CHECK-NEXT:    ret
896   %old = atomicrmw umin ptr @var64, i64 %offset acq_rel
897   ret i64 %old
898}
899
900define dso_local i8 @test_atomic_load_umax_i8(i8 %offset) nounwind {
901; CHECK-LABEL: test_atomic_load_umax_i8:
902; CHECK:       // %bb.0:
903; CHECK-NEXT:    adrp x8, var8
904; CHECK-NEXT:    add x8, x8, :lo12:var8
905; CHECK-NEXT:    and w9, w0, #0xff
906; CHECK-NEXT:  .LBB36_1: // %atomicrmw.start
907; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
908; CHECK-NEXT:    ldaxrb w0, [x8]
909; CHECK-NEXT:    cmp w0, w9
910; CHECK-NEXT:    csel w10, w0, w9, hi
911; CHECK-NEXT:    stlxrb w11, w10, [x8]
912; CHECK-NEXT:    cbnz w11, .LBB36_1
913; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
914; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
915; CHECK-NEXT:    ret
916   %old = atomicrmw umax ptr @var8, i8 %offset acq_rel
917   ret i8 %old
918}
919
920define dso_local i16 @test_atomic_load_umax_i16(i16 %offset) nounwind {
921; CHECK-LABEL: test_atomic_load_umax_i16:
922; CHECK:       // %bb.0:
923; CHECK-NEXT:    adrp x8, var16
924; CHECK-NEXT:    add x8, x8, :lo12:var16
925; CHECK-NEXT:    and w9, w0, #0xffff
926; CHECK-NEXT:  .LBB37_1: // %atomicrmw.start
927; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
928; CHECK-NEXT:    ldxrh w0, [x8]
929; CHECK-NEXT:    cmp w0, w9
930; CHECK-NEXT:    csel w10, w0, w9, hi
931; CHECK-NEXT:    stxrh w11, w10, [x8]
932; CHECK-NEXT:    cbnz w11, .LBB37_1
933; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
934; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
935; CHECK-NEXT:    ret
936   %old = atomicrmw umax ptr @var16, i16 %offset monotonic
937   ret i16 %old
938}
939
940define dso_local i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
941; CHECK-LABEL: test_atomic_load_umax_i32:
942; CHECK:       // %bb.0:
943; CHECK-NEXT:    adrp x9, var32
944; CHECK-NEXT:    add x9, x9, :lo12:var32
945; CHECK-NEXT:  .LBB38_1: // %atomicrmw.start
946; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
947; CHECK-NEXT:    ldaxr w8, [x9]
948; CHECK-NEXT:    cmp w8, w0
949; CHECK-NEXT:    csel w10, w8, w0, hi
950; CHECK-NEXT:    stlxr w11, w10, [x9]
951; CHECK-NEXT:    cbnz w11, .LBB38_1
952; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
953; CHECK-NEXT:    mov w0, w8
954; CHECK-NEXT:    ret
955   %old = atomicrmw umax ptr @var32, i32 %offset seq_cst
956   ret i32 %old
957}
958
959define dso_local i64 @test_atomic_load_umax_i64(i64 %offset) nounwind {
960; CHECK-LABEL: test_atomic_load_umax_i64:
961; CHECK:       // %bb.0:
962; CHECK-NEXT:    adrp x9, var64
963; CHECK-NEXT:    add x9, x9, :lo12:var64
964; CHECK-NEXT:  .LBB39_1: // %atomicrmw.start
965; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
966; CHECK-NEXT:    ldxr x8, [x9]
967; CHECK-NEXT:    cmp x8, x0
968; CHECK-NEXT:    csel x10, x8, x0, hi
969; CHECK-NEXT:    stlxr w11, x10, [x9]
970; CHECK-NEXT:    cbnz w11, .LBB39_1
971; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
972; CHECK-NEXT:    mov x0, x8
973; CHECK-NEXT:    ret
974   %old = atomicrmw umax ptr @var64, i64 %offset release
975   ret i64 %old
976}
977
978define dso_local i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind {
979; INLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i8:
980; INLINE_ATOMICS:       // %bb.0:
981; INLINE_ATOMICS-NEXT:    // kill: def $w1 killed $w1 def $x1
982; INLINE_ATOMICS-NEXT:    and w8, w0, #0xff
983; INLINE_ATOMICS-NEXT:    adrp x9, var8
984; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var8
985; INLINE_ATOMICS-NEXT:  .LBB40_1: // %cmpxchg.start
986; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
987; INLINE_ATOMICS-NEXT:    ldaxrb w0, [x9]
988; INLINE_ATOMICS-NEXT:    cmp w0, w8
989; INLINE_ATOMICS-NEXT:    b.ne .LBB40_4
990; INLINE_ATOMICS-NEXT:  // %bb.2: // %cmpxchg.trystore
991; INLINE_ATOMICS-NEXT:    // in Loop: Header=BB40_1 Depth=1
992; INLINE_ATOMICS-NEXT:    stxrb w10, w1, [x9]
993; INLINE_ATOMICS-NEXT:    cbnz w10, .LBB40_1
994; INLINE_ATOMICS-NEXT:  // %bb.3: // %cmpxchg.end
995; INLINE_ATOMICS-NEXT:    // kill: def $w0 killed $w0 killed $x0
996; INLINE_ATOMICS-NEXT:    ret
997; INLINE_ATOMICS-NEXT:  .LBB40_4: // %cmpxchg.nostore
998; INLINE_ATOMICS-NEXT:    clrex
999; INLINE_ATOMICS-NEXT:    // kill: def $w0 killed $w0 killed $x0
1000; INLINE_ATOMICS-NEXT:    ret
1001;
1002; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i8:
1003; OUTLINE_ATOMICS:       // %bb.0:
1004; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
1005; OUTLINE_ATOMICS-NEXT:    adrp x2, var8
1006; OUTLINE_ATOMICS-NEXT:    add x2, x2, :lo12:var8
1007; OUTLINE_ATOMICS-NEXT:    bl __aarch64_cas1_acq
1008; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
1009; OUTLINE_ATOMICS-NEXT:    ret
1010   %pair = cmpxchg ptr @var8, i8 %wanted, i8 %new acquire acquire
1011   %old = extractvalue { i8, i1 } %pair, 0
1012   ret i8 %old
1013}
1014
1015define dso_local i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind {
1016; INLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i16:
1017; INLINE_ATOMICS:       // %bb.0:
1018; INLINE_ATOMICS-NEXT:    // kill: def $w1 killed $w1 def $x1
1019; INLINE_ATOMICS-NEXT:    and w8, w0, #0xffff
1020; INLINE_ATOMICS-NEXT:    adrp x9, var16
1021; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var16
1022; INLINE_ATOMICS-NEXT:  .LBB41_1: // %cmpxchg.start
1023; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
1024; INLINE_ATOMICS-NEXT:    ldaxrh w0, [x9]
1025; INLINE_ATOMICS-NEXT:    cmp w0, w8
1026; INLINE_ATOMICS-NEXT:    b.ne .LBB41_4
1027; INLINE_ATOMICS-NEXT:  // %bb.2: // %cmpxchg.trystore
1028; INLINE_ATOMICS-NEXT:    // in Loop: Header=BB41_1 Depth=1
1029; INLINE_ATOMICS-NEXT:    stlxrh w10, w1, [x9]
1030; INLINE_ATOMICS-NEXT:    cbnz w10, .LBB41_1
1031; INLINE_ATOMICS-NEXT:  // %bb.3: // %cmpxchg.end
1032; INLINE_ATOMICS-NEXT:    // kill: def $w0 killed $w0 killed $x0
1033; INLINE_ATOMICS-NEXT:    ret
1034; INLINE_ATOMICS-NEXT:  .LBB41_4: // %cmpxchg.nostore
1035; INLINE_ATOMICS-NEXT:    clrex
1036; INLINE_ATOMICS-NEXT:    // kill: def $w0 killed $w0 killed $x0
1037; INLINE_ATOMICS-NEXT:    ret
1038;
1039; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i16:
1040; OUTLINE_ATOMICS:       // %bb.0:
1041; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
1042; OUTLINE_ATOMICS-NEXT:    adrp x2, var16
1043; OUTLINE_ATOMICS-NEXT:    add x2, x2, :lo12:var16
1044; OUTLINE_ATOMICS-NEXT:    bl __aarch64_cas2_acq_rel
1045; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
1046; OUTLINE_ATOMICS-NEXT:    ret
1047   %pair = cmpxchg ptr @var16, i16 %wanted, i16 %new seq_cst seq_cst
1048   %old = extractvalue { i16, i1 } %pair, 0
1049   ret i16 %old
1050}
1051
1052define dso_local i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
1053; INLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i32:
1054; INLINE_ATOMICS:       // %bb.0:
1055; INLINE_ATOMICS-NEXT:    mov w8, w0
1056; INLINE_ATOMICS-NEXT:    adrp x9, var32
1057; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var32
1058; INLINE_ATOMICS-NEXT:  .LBB42_1: // %cmpxchg.start
1059; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
1060; INLINE_ATOMICS-NEXT:    ldxr w0, [x9]
1061; INLINE_ATOMICS-NEXT:    cmp w0, w8
1062; INLINE_ATOMICS-NEXT:    b.ne .LBB42_4
1063; INLINE_ATOMICS-NEXT:  // %bb.2: // %cmpxchg.trystore
1064; INLINE_ATOMICS-NEXT:    // in Loop: Header=BB42_1 Depth=1
1065; INLINE_ATOMICS-NEXT:    stlxr w10, w1, [x9]
1066; INLINE_ATOMICS-NEXT:    cbnz w10, .LBB42_1
1067; INLINE_ATOMICS-NEXT:  // %bb.3: // %cmpxchg.end
1068; INLINE_ATOMICS-NEXT:    // kill: def $w0 killed $w0 killed $x0
1069; INLINE_ATOMICS-NEXT:    ret
1070; INLINE_ATOMICS-NEXT:  .LBB42_4: // %cmpxchg.nostore
1071; INLINE_ATOMICS-NEXT:    clrex
1072; INLINE_ATOMICS-NEXT:    // kill: def $w0 killed $w0 killed $x0
1073; INLINE_ATOMICS-NEXT:    ret
1074;
1075; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i32:
1076; OUTLINE_ATOMICS:       // %bb.0:
1077; OUTLINE_ATOMICS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
1078; OUTLINE_ATOMICS-NEXT:    adrp x2, var32
1079; OUTLINE_ATOMICS-NEXT:    add x2, x2, :lo12:var32
1080; OUTLINE_ATOMICS-NEXT:    bl __aarch64_cas4_rel
1081; OUTLINE_ATOMICS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
1082; OUTLINE_ATOMICS-NEXT:    ret
1083   %pair = cmpxchg ptr @var32, i32 %wanted, i32 %new release monotonic
1084   %old = extractvalue { i32, i1 } %pair, 0
1085   ret i32 %old
1086}
1087
1088define dso_local void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
1089; INLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i64:
1090; INLINE_ATOMICS:       // %bb.0:
1091; INLINE_ATOMICS-NEXT:    adrp x9, var64
1092; INLINE_ATOMICS-NEXT:    add x9, x9, :lo12:var64
1093; INLINE_ATOMICS-NEXT:  .LBB43_1: // %cmpxchg.start
1094; INLINE_ATOMICS-NEXT:    // =>This Inner Loop Header: Depth=1
1095; INLINE_ATOMICS-NEXT:    ldxr x8, [x9]
1096; INLINE_ATOMICS-NEXT:    cmp x8, x0
1097; INLINE_ATOMICS-NEXT:    b.ne .LBB43_3
1098; INLINE_ATOMICS-NEXT:  // %bb.2: // %cmpxchg.trystore
1099; INLINE_ATOMICS-NEXT:    // in Loop: Header=BB43_1 Depth=1
1100; INLINE_ATOMICS-NEXT:    stxr w10, x1, [x9]
1101; INLINE_ATOMICS-NEXT:    cbnz w10, .LBB43_1
1102; INLINE_ATOMICS-NEXT:    b .LBB43_4
1103; INLINE_ATOMICS-NEXT:  .LBB43_3: // %cmpxchg.nostore
1104; INLINE_ATOMICS-NEXT:    clrex
1105; INLINE_ATOMICS-NEXT:  .LBB43_4: // %cmpxchg.end
1106; INLINE_ATOMICS-NEXT:    adrp x9, var64
1107; INLINE_ATOMICS-NEXT:    str x8, [x9, :lo12:var64]
1108; INLINE_ATOMICS-NEXT:    ret
1109;
1110; OUTLINE_ATOMICS-LABEL: test_atomic_cmpxchg_i64:
1111; OUTLINE_ATOMICS:       // %bb.0:
1112; OUTLINE_ATOMICS-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
1113; OUTLINE_ATOMICS-NEXT:    adrp x19, var64
1114; OUTLINE_ATOMICS-NEXT:    add x19, x19, :lo12:var64
1115; OUTLINE_ATOMICS-NEXT:    mov x2, x19
1116; OUTLINE_ATOMICS-NEXT:    bl __aarch64_cas8_relax
1117; OUTLINE_ATOMICS-NEXT:    str x0, [x19]
1118; OUTLINE_ATOMICS-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
1119; OUTLINE_ATOMICS-NEXT:    ret
1120   %pair = cmpxchg ptr @var64, i64 %wanted, i64 %new monotonic monotonic
1121   %old = extractvalue { i64, i1 } %pair, 0
1122   store i64 %old, ptr @var64
1123   ret void
1124}
1125
1126define dso_local i8 @test_atomic_load_monotonic_i8() nounwind {
1127; CHECK-LABEL: test_atomic_load_monotonic_i8:
1128; CHECK:       // %bb.0:
1129; CHECK-NEXT:    adrp x8, var8
1130; CHECK-NEXT:    ldrb w0, [x8, :lo12:var8]
1131; CHECK-NEXT:    ret
1132  %val = load atomic i8, ptr @var8 monotonic, align 1
1133  ret i8 %val
1134}
1135
1136define dso_local i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {
1137; CHECK-LABEL: test_atomic_load_monotonic_regoff_i8:
1138; CHECK:       // %bb.0:
1139; CHECK-NEXT:    ldrb w0, [x0, x1]
1140; CHECK-NEXT:    ret
1141  %addr_int = add i64 %base, %off
1142  %addr = inttoptr i64 %addr_int to ptr
1143  %val = load atomic i8, ptr %addr monotonic, align 1
1144  ret i8 %val
1145}
1146
1147define dso_local i8 @test_atomic_load_acquire_i8() nounwind {
1148; CHECK-LABEL: test_atomic_load_acquire_i8:
1149; CHECK:       // %bb.0:
1150; CHECK-NEXT:    adrp x8, var8
1151; CHECK-NEXT:    add x8, x8, :lo12:var8
1152; CHECK-NEXT:    ldarb w0, [x8]
1153; CHECK-NEXT:    ret
1154  %val = load atomic i8, ptr @var8 acquire, align 1
1155  ret i8 %val
1156}
1157
1158define dso_local i8 @test_atomic_load_seq_cst_i8() nounwind {
1159; CHECK-LABEL: test_atomic_load_seq_cst_i8:
1160; CHECK:       // %bb.0:
1161; CHECK-NEXT:    adrp x8, var8
1162; CHECK-NEXT:    add x8, x8, :lo12:var8
1163; CHECK-NEXT:    ldarb w0, [x8]
1164; CHECK-NEXT:    ret
1165  %val = load atomic i8, ptr @var8 seq_cst, align 1
1166  ret i8 %val
1167}
1168
1169define dso_local i16 @test_atomic_load_monotonic_i16() nounwind {
1170; CHECK-LABEL: test_atomic_load_monotonic_i16:
1171; CHECK:       // %bb.0:
1172; CHECK-NEXT:    adrp x8, var16
1173; CHECK-NEXT:    ldrh w0, [x8, :lo12:var16]
1174; CHECK-NEXT:    ret
1175  %val = load atomic i16, ptr @var16 monotonic, align 2
1176  ret i16 %val
1177}
1178
1179define dso_local i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind {
1180; CHECK-LABEL: test_atomic_load_monotonic_regoff_i32:
1181; CHECK:       // %bb.0:
1182; CHECK-NEXT:    ldr w0, [x0, x1]
1183; CHECK-NEXT:    ret
1184  %addr_int = add i64 %base, %off
1185  %addr = inttoptr i64 %addr_int to ptr
1186  %val = load atomic i32, ptr %addr monotonic, align 4
1187  ret i32 %val
1188}
1189
1190define dso_local i64 @test_atomic_load_seq_cst_i64() nounwind {
1191; CHECK-LABEL: test_atomic_load_seq_cst_i64:
1192; CHECK:       // %bb.0:
1193; CHECK-NEXT:    adrp x8, var64
1194; CHECK-NEXT:    add x8, x8, :lo12:var64
1195; CHECK-NEXT:    ldar x0, [x8]
1196; CHECK-NEXT:    ret
1197  %val = load atomic i64, ptr @var64 seq_cst, align 8
1198  ret i64 %val
1199}
1200
1201define dso_local void @test_atomic_store_monotonic_i8(i8 %val) nounwind {
1202; CHECK-LABEL: test_atomic_store_monotonic_i8:
1203; CHECK:       // %bb.0:
1204; CHECK-NEXT:    adrp x8, var8
1205; CHECK-NEXT:    strb w0, [x8, :lo12:var8]
1206; CHECK-NEXT:    ret
1207  store atomic i8 %val, ptr @var8 monotonic, align 1
1208  ret void
1209}
1210
1211define dso_local void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind {
1212; CHECK-LABEL: test_atomic_store_monotonic_regoff_i8:
1213; CHECK:       // %bb.0:
1214; CHECK-NEXT:    strb w2, [x0, x1]
1215; CHECK-NEXT:    ret
1216  %addr_int = add i64 %base, %off
1217  %addr = inttoptr i64 %addr_int to ptr
1218  store atomic i8 %val, ptr %addr monotonic, align 1
1219  ret void
1220}
1221define dso_local void @test_atomic_store_release_i8(i8 %val) nounwind {
1222; CHECK-LABEL: test_atomic_store_release_i8:
1223; CHECK:       // %bb.0:
1224; CHECK-NEXT:    adrp x8, var8
1225; CHECK-NEXT:    add x8, x8, :lo12:var8
1226; CHECK-NEXT:    stlrb w0, [x8]
1227; CHECK-NEXT:    ret
1228  store atomic i8 %val, ptr @var8 release, align 1
1229  ret void
1230}
1231
1232define dso_local void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {
1233; CHECK-LABEL: test_atomic_store_seq_cst_i8:
1234; CHECK:       // %bb.0:
1235; CHECK-NEXT:    adrp x8, var8
1236; CHECK-NEXT:    add x8, x8, :lo12:var8
1237; CHECK-NEXT:    stlrb w0, [x8]
1238; CHECK-NEXT:    ret
1239  store atomic i8 %val, ptr @var8 seq_cst, align 1
1240  ret void
1241}
1242
1243define dso_local void @test_atomic_store_monotonic_i16(i16 %val) nounwind {
1244; CHECK-LABEL: test_atomic_store_monotonic_i16:
1245; CHECK:       // %bb.0:
1246; CHECK-NEXT:    adrp x8, var16
1247; CHECK-NEXT:    strh w0, [x8, :lo12:var16]
1248; CHECK-NEXT:    ret
1249  store atomic i16 %val, ptr @var16 monotonic, align 2
1250  ret void
1251}
1252
1253define dso_local void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind {
1254; CHECK-LABEL: test_atomic_store_monotonic_regoff_i32:
1255; CHECK:       // %bb.0:
1256; CHECK-NEXT:    str w2, [x0, x1]
1257; CHECK-NEXT:    ret
1258  %addr_int = add i64 %base, %off
1259  %addr = inttoptr i64 %addr_int to ptr
1260  store atomic i32 %val, ptr %addr monotonic, align 4
1261  ret void
1262}
1263
1264define dso_local void @test_atomic_store_release_i64(i64 %val) nounwind {
1265; CHECK-LABEL: test_atomic_store_release_i64:
1266; CHECK:       // %bb.0:
1267; CHECK-NEXT:    adrp x8, var64
1268; CHECK-NEXT:    add x8, x8, :lo12:var64
1269; CHECK-NEXT:    stlr x0, [x8]
1270; CHECK-NEXT:    ret
1271  store atomic i64 %val, ptr @var64 release, align 8
1272  ret void
1273}
1274