xref: /llvm-project/llvm/test/CodeGen/ARM/atomic-ops-v8.ll (revision e0ed0333f0fed2e73f805afd58b61176a87aa3ad)
1; RUN: llc -mtriple=armv8-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE --check-prefix=CHECK-ARM --check-prefix=CHECK-ARM-LE
2; RUN: llc -mtriple=armebv8-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE --check-prefix=CHECK-ARM --check-prefix=CHECK-ARM-BE
3; RUN: llc -mtriple=thumbv8-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE --check-prefix=CHECK-THUMB --check-prefix=CHECK-THUMB-LE
4; RUN: llc -mtriple=thumbebv8-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE --check-prefix=CHECK-THUMB --check-prefix=CHECK-THUMB-BE
5
6@var8 = global i8 0
7@var16 = global i16 0
8@var32 = global i32 0
9@var64 = global i64 0
10
11define i8 @test_atomic_load_add_i8(i8 %offset) nounwind {
12; CHECK-LABEL: test_atomic_load_add_i8:
13   %old = atomicrmw add ptr @var8, i8 %offset seq_cst
14; CHECK-NOT: dmb
15; CHECK-NOT: mcr
16; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8
17; CHECK: movt r[[ADDR]], :upper16:var8
18
19; CHECK: .LBB{{[0-9]+}}_1:
20; CHECK: ldaexb r[[OLD:[0-9]+]], [r[[ADDR]]]
21  ; r0 below is a reasonable guess but could change: it certainly comes into the
22  ;  function there.
23; CHECK-NEXT: add{{s?}} [[NEW:r[0-9]+]], r[[OLD]], r0
24; CHECK-NEXT: stlexb [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
25; CHECK-NEXT: cmp [[STATUS]], #0
26; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
27; CHECK-NOT: dmb
28; CHECK-NOT: mcr
29
30; CHECK: mov r0, r[[OLD]]
31   ret i8 %old
32}
33
34define i16 @test_atomic_load_add_i16(i16 %offset) nounwind {
35; CHECK-LABEL: test_atomic_load_add_i16:
36   %old = atomicrmw add ptr @var16, i16 %offset acquire
37; CHECK-NOT: dmb
38; CHECK-NOT: mcr
39; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var16
40; CHECK: movt r[[ADDR]], :upper16:var16
41
42; CHECK: .LBB{{[0-9]+}}_1:
43; CHECK: ldaexh r[[OLD:[0-9]+]], [r[[ADDR]]]
44  ; r0 below is a reasonable guess but could change: it certainly comes into the
45  ;  function there.
46; CHECK-NEXT: add{{s?}} [[NEW:r[0-9]+]], r[[OLD]], r0
47; CHECK-NEXT: strexh [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
48; CHECK-NEXT: cmp [[STATUS]], #0
49; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
50; CHECK-NOT: dmb
51; CHECK-NOT: mcr
52
53; CHECK: mov r0, r[[OLD]]
54   ret i16 %old
55}
56
57define i32 @test_atomic_load_add_i32(i32 %offset) nounwind {
58; CHECK-LABEL: test_atomic_load_add_i32:
59   %old = atomicrmw add ptr @var32, i32 %offset release
60; CHECK-NOT: dmb
61; CHECK-NOT: mcr
62; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var32
63; CHECK: movt r[[ADDR]], :upper16:var32
64
65; CHECK: .LBB{{[0-9]+}}_1:
66; CHECK: ldrex r[[OLD:[0-9]+]], [r[[ADDR]]]
67  ; r0 below is a reasonable guess but could change: it certainly comes into the
68  ;  function there.
69; CHECK-NEXT: add{{s?}} [[NEW:r[0-9]+]], r[[OLD]], r0
70; CHECK-NEXT: stlex [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
71; CHECK-NEXT: cmp [[STATUS]], #0
72; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
73; CHECK-NOT: dmb
74; CHECK-NOT: mcr
75
76; CHECK: mov r0, r[[OLD]]
77   ret i32 %old
78}
79
80define void @test_atomic_load_add_i64(i64 %offset) nounwind {
81; CHECK-LABEL: test_atomic_load_add_i64:
82   %old = atomicrmw add ptr @var64, i64 %offset monotonic
83; CHECK-NOT: dmb
84; CHECK-NOT: mcr
85; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
86; CHECK: movt r[[ADDR]], :upper16:var64
87
88; CHECK: .LBB{{[0-9]+}}_1:
89; CHECK: ldrexd r[[OLD1:[0-9]+]], r[[OLD2:[0-9]+]], [r[[ADDR]]]
90  ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
91  ; function there.
92; CHECK-LE-NEXT: adds{{(\.w)?}} [[NEW1:r[0-9]+|lr]], r[[OLD1]], r0
93; CHECK-LE-NEXT: adc{{(\.w)?}}  [[NEW2:r[0-9]+]], r[[OLD2]], r1
94; CHECK-BE-NEXT: adds{{(\.w)?}} [[NEW2:r[0-9]+|lr]], r[[OLD2]], r1
95; CHECK-BE-NEXT: adc{{(\.w)?}}  [[NEW1:r[0-9]+]], r[[OLD1]], r0
96; CHECK-NEXT: strexd [[STATUS:r[0-9]+]], [[NEW1]], [[NEW2]], [r[[ADDR]]]
97; CHECK-NEXT: cmp [[STATUS]], #0
98; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
99; CHECK-NOT: dmb
100; CHECK-NOT: mcr
101
102; CHECK: strd r[[OLD1]], r[[OLD2]], [r[[ADDR]]]
103  store i64 %old, ptr @var64
104   ret void
105}
106
107define i8 @test_atomic_load_sub_i8(i8 %offset) nounwind {
108; CHECK-LABEL: test_atomic_load_sub_i8:
109   %old = atomicrmw sub ptr @var8, i8 %offset monotonic
110; CHECK-NOT: dmb
111; CHECK-NOT: mcr
112; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8
113; CHECK: movt r[[ADDR]], :upper16:var8
114
115; CHECK: .LBB{{[0-9]+}}_1:
116; CHECK: ldrexb r[[OLD:[0-9]+]], [r[[ADDR]]]
117  ; r0 below is a reasonable guess but could change: it certainly comes into the
118  ;  function there.
119; CHECK-NEXT: sub{{s?}} [[NEW:r[0-9]+]], r[[OLD]], r0
120; CHECK-NEXT: strexb [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
121; CHECK-NEXT: cmp [[STATUS]], #0
122; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
123; CHECK-NOT: dmb
124; CHECK-NOT: mcr
125
126; CHECK: mov r0, r[[OLD]]
127   ret i8 %old
128}
129
130define i16 @test_atomic_load_sub_i16(i16 %offset) nounwind {
131; CHECK-LABEL: test_atomic_load_sub_i16:
132   %old = atomicrmw sub ptr @var16, i16 %offset release
133; CHECK-NOT: dmb
134; CHECK-NOT: mcr
135; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var16
136; CHECK: movt r[[ADDR]], :upper16:var16
137
138; CHECK: .LBB{{[0-9]+}}_1:
139; CHECK: ldrexh r[[OLD:[0-9]+]], [r[[ADDR]]]
140  ; r0 below is a reasonable guess but could change: it certainly comes into the
141  ;  function there.
142; CHECK-NEXT: sub{{s?}} [[NEW:r[0-9]+]], r[[OLD]], r0
143; CHECK-NEXT: stlexh [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
144; CHECK-NEXT: cmp [[STATUS]], #0
145; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
146; CHECK-NOT: dmb
147; CHECK-NOT: mcr
148
149; CHECK: mov r0, r[[OLD]]
150   ret i16 %old
151}
152
153define i32 @test_atomic_load_sub_i32(i32 %offset) nounwind {
154; CHECK-LABEL: test_atomic_load_sub_i32:
155   %old = atomicrmw sub ptr @var32, i32 %offset acquire
156; CHECK-NOT: dmb
157; CHECK-NOT: mcr
158; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var32
159; CHECK: movt r[[ADDR]], :upper16:var32
160
161; CHECK: .LBB{{[0-9]+}}_1:
162; CHECK: ldaex r[[OLD:[0-9]+]], [r[[ADDR]]]
163  ; r0 below is a reasonable guess but could change: it certainly comes into the
164  ;  function there.
165; CHECK-NEXT: sub{{s?}} [[NEW:r[0-9]+]], r[[OLD]], r0
166; CHECK-NEXT: strex [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
167; CHECK-NEXT: cmp [[STATUS]], #0
168; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
169; CHECK-NOT: dmb
170; CHECK-NOT: mcr
171
172; CHECK: mov r0, r[[OLD]]
173   ret i32 %old
174}
175
176define void @test_atomic_load_sub_i64(i64 %offset) nounwind {
177; CHECK-LABEL: test_atomic_load_sub_i64:
178   %old = atomicrmw sub ptr @var64, i64 %offset seq_cst
179; CHECK-NOT: dmb
180; CHECK-NOT: mcr
181; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
182; CHECK: movt r[[ADDR]], :upper16:var64
183
184; CHECK: .LBB{{[0-9]+}}_1:
185; CHECK: ldaexd r[[OLD1:[0-9]+]], r[[OLD2:[0-9]+]], [r[[ADDR]]]
186  ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
187  ; function there.
188; CHECK-LE-NEXT: subs{{(\.w)?}} [[NEW1:r[0-9]+|lr]], r[[OLD1]], r0
189; CHECK-LE-NEXT: sbc{{(\.w)?}}  [[NEW2:r[0-9]+]], r[[OLD2]], r1
190; CHECK-BE-NEXT: subs{{(\.w)?}} [[NEW2:r[0-9]+|lr]], r[[OLD2]], r1
191; CHECK-BE-NEXT: sbc{{(\.w)?}}  [[NEW1:r[0-9]+]], r[[OLD1]], r0
192; CHECK-NEXT: stlexd [[STATUS:r[0-9]+]], [[NEW1]], [[NEW2]], [r[[ADDR]]]
193; CHECK-NEXT: cmp [[STATUS]], #0
194; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
195; CHECK-NOT: dmb
196; CHECK-NOT: mcr
197
198; CHECK: strd r[[OLD1]], r[[OLD2]], [r[[ADDR]]]
199   store i64 %old, ptr @var64
200   ret void
201}
202
203define i8 @test_atomic_load_and_i8(i8 %offset) nounwind {
204; CHECK-LABEL: test_atomic_load_and_i8:
205   %old = atomicrmw and ptr @var8, i8 %offset release
206; CHECK-NOT: dmb
207; CHECK-NOT: mcr
208; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8
209; CHECK: movt r[[ADDR]], :upper16:var8
210
211; CHECK: .LBB{{[0-9]+}}_1:
212; CHECK: ldrexb r[[OLD:[0-9]+]], [r[[ADDR]]]
213  ; r0 below is a reasonable guess but could change: it certainly comes into the
214  ;  function there.
215; CHECK-NEXT: and{{(\.w)?}} [[NEW:r[0-9]+]], r[[OLD]], r0
216; CHECK-NEXT: stlexb [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
217; CHECK-NEXT: cmp [[STATUS]], #0
218; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
219; CHECK-NOT: dmb
220; CHECK-NOT: mcr
221
222; CHECK: mov r0, r[[OLD]]
223   ret i8 %old
224}
225
226define i16 @test_atomic_load_and_i16(i16 %offset) nounwind {
227; CHECK-LABEL: test_atomic_load_and_i16:
228   %old = atomicrmw and ptr @var16, i16 %offset monotonic
229; CHECK-NOT: dmb
230; CHECK-NOT: mcr
231; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var16
232; CHECK: movt r[[ADDR]], :upper16:var16
233
234; CHECK: .LBB{{[0-9]+}}_1:
235; CHECK: ldrexh r[[OLD:[0-9]+]], [r[[ADDR]]]
236  ; r0 below is a reasonable guess but could change: it certainly comes into the
237  ;  function there.
238; CHECK-NEXT: and{{(\.w)?}} [[NEW:r[0-9]+]], r[[OLD]], r0
239; CHECK-NEXT: strexh [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
240; CHECK-NEXT: cmp [[STATUS]], #0
241; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
242; CHECK-NOT: dmb
243; CHECK-NOT: mcr
244
245; CHECK: mov r0, r[[OLD]]
246   ret i16 %old
247}
248
249define i32 @test_atomic_load_and_i32(i32 %offset) nounwind {
250; CHECK-LABEL: test_atomic_load_and_i32:
251   %old = atomicrmw and ptr @var32, i32 %offset seq_cst
252; CHECK-NOT: dmb
253; CHECK-NOT: mcr
254; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var32
255; CHECK: movt r[[ADDR]], :upper16:var32
256
257; CHECK: .LBB{{[0-9]+}}_1:
258; CHECK: ldaex r[[OLD:[0-9]+]], [r[[ADDR]]]
259  ; r0 below is a reasonable guess but could change: it certainly comes into the
260  ;  function there.
261; CHECK-NEXT: and{{(\.w)?}} [[NEW:r[0-9]+]], r[[OLD]], r0
262; CHECK-NEXT: stlex [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
263; CHECK-NEXT: cmp [[STATUS]], #0
264; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
265; CHECK-NOT: dmb
266; CHECK-NOT: mcr
267
268; CHECK: mov r0, r[[OLD]]
269   ret i32 %old
270}
271
272define void @test_atomic_load_and_i64(i64 %offset) nounwind {
273; CHECK-LABEL: test_atomic_load_and_i64:
274   %old = atomicrmw and ptr @var64, i64 %offset acquire
275; CHECK-NOT: dmb
276; CHECK-NOT: mcr
277; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
278; CHECK: movt r[[ADDR]], :upper16:var64
279
280; CHECK: .LBB{{[0-9]+}}_1:
281; CHECK: ldaexd r[[OLD1:[0-9]+]], r[[OLD2:[0-9]+]], [r[[ADDR]]]
282  ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
283  ; function there.
284; CHECK-LE-DAG: and{{(\.w)?}} [[NEW1:r[0-9]+|lr]], r[[OLD1]], r0
285; CHECK-LE-DAG: and{{(\.w)?}} [[NEW2:r[0-9]+|lr]], r[[OLD2]], r1
286; CHECK-BE-DAG: and{{(\.w)?}} [[NEW2:r[0-9]+|lr]], r[[OLD2]], r1
287; CHECK-BE-DAG: and{{(\.w)?}} [[NEW1:r[0-9]+|lr]], r[[OLD1]], r0
288; CHECK: strexd [[STATUS:r[0-9]+]], [[NEW1]], [[NEW2]], [r[[ADDR]]]
289; CHECK-NEXT: cmp [[STATUS]], #0
290; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
291; CHECK-NOT: dmb
292; CHECK-NOT: mcr
293
294; CHECK: strd r[[OLD1]], r[[OLD2]], [r[[ADDR]]]
295   store i64 %old, ptr @var64
296   ret void
297}
298
299define i8 @test_atomic_load_or_i8(i8 %offset) nounwind {
300; CHECK-LABEL: test_atomic_load_or_i8:
301   %old = atomicrmw or ptr @var8, i8 %offset seq_cst
302; CHECK-NOT: dmb
303; CHECK-NOT: mcr
304; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8
305; CHECK: movt r[[ADDR]], :upper16:var8
306
307; CHECK: .LBB{{[0-9]+}}_1:
308; CHECK: ldaexb r[[OLD:[0-9]+]], [r[[ADDR]]]
309  ; r0 below is a reasonable guess but could change: it certainly comes into the
310  ;  function there.
311; CHECK-NEXT: orr{{(\.w)?}} [[NEW:r[0-9]+]], r[[OLD]], r0
312; CHECK-NEXT: stlexb [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
313; CHECK-NEXT: cmp [[STATUS]], #0
314; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
315; CHECK-NOT: dmb
316; CHECK-NOT: mcr
317
318; CHECK: mov r0, r[[OLD]]
319   ret i8 %old
320}
321
322define i16 @test_atomic_load_or_i16(i16 %offset) nounwind {
323; CHECK-LABEL: test_atomic_load_or_i16:
324   %old = atomicrmw or ptr @var16, i16 %offset monotonic
325; CHECK-NOT: dmb
326; CHECK-NOT: mcr
327; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var16
328; CHECK: movt r[[ADDR]], :upper16:var16
329
330; CHECK: .LBB{{[0-9]+}}_1:
331; CHECK: ldrexh r[[OLD:[0-9]+]], [r[[ADDR]]]
332  ; r0 below is a reasonable guess but could change: it certainly comes into the
333  ;  function there.
334; CHECK-NEXT: orr{{(\.w)?}} [[NEW:r[0-9]+]], r[[OLD]], r0
335; CHECK-NEXT: strexh [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
336; CHECK-NEXT: cmp [[STATUS]], #0
337; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
338; CHECK-NOT: dmb
339; CHECK-NOT: mcr
340
341; CHECK: mov r0, r[[OLD]]
342   ret i16 %old
343}
344
345define i32 @test_atomic_load_or_i32(i32 %offset) nounwind {
346; CHECK-LABEL: test_atomic_load_or_i32:
347   %old = atomicrmw or ptr @var32, i32 %offset acquire
348; CHECK-NOT: dmb
349; CHECK-NOT: mcr
350; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var32
351; CHECK: movt r[[ADDR]], :upper16:var32
352
353; CHECK: .LBB{{[0-9]+}}_1:
354; CHECK: ldaex r[[OLD:[0-9]+]], [r[[ADDR]]]
355  ; r0 below is a reasonable guess but could change: it certainly comes into the
356  ;  function there.
357; CHECK-NEXT: orr{{(\.w)?}} [[NEW:r[0-9]+]], r[[OLD]], r0
358; CHECK-NEXT: strex [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
359; CHECK-NEXT: cmp [[STATUS]], #0
360; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
361; CHECK-NOT: dmb
362; CHECK-NOT: mcr
363
364; CHECK: mov r0, r[[OLD]]
365   ret i32 %old
366}
367
368define void @test_atomic_load_or_i64(i64 %offset) nounwind {
369; CHECK-LABEL: test_atomic_load_or_i64:
370   %old = atomicrmw or ptr @var64, i64 %offset release
371; CHECK-NOT: dmb
372; CHECK-NOT: mcr
373; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
374; CHECK: movt r[[ADDR]], :upper16:var64
375
376; CHECK: .LBB{{[0-9]+}}_1:
377; CHECK: ldrexd r[[OLD1:[0-9]+]], r[[OLD2:[0-9]+]], [r[[ADDR]]]
378  ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
379  ; function there.
380; CHECK-LE-DAG: orr{{(\.w)?}} [[NEW1:r[0-9]+|lr]], r[[OLD1]], r0
381; CHECK-LE-DAG: orr{{(\.w)?}} [[NEW2:r[0-9]+|lr]], r[[OLD2]], r1
382; CHECK-BE-DAG: orr{{(\.w)?}} [[NEW2:r[0-9]+|lr]], r[[OLD2]], r1
383; CHECK-BE-DAG: orr{{(\.w)?}} [[NEW1:r[0-9]+|lr]], r[[OLD1]], r0
384; CHECK: stlexd [[STATUS:r[0-9]+]], [[NEW1]], [[NEW2]], [r[[ADDR]]]
385; CHECK-NEXT: cmp [[STATUS]], #0
386; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
387; CHECK-NOT: dmb
388; CHECK-NOT: mcr
389
390; CHECK: strd r[[OLD1]], r[[OLD2]], [r[[ADDR]]]
391   store i64 %old, ptr @var64
392   ret void
393}
394
395define i8 @test_atomic_load_xor_i8(i8 %offset) nounwind {
396; CHECK-LABEL: test_atomic_load_xor_i8:
397   %old = atomicrmw xor ptr @var8, i8 %offset acquire
398; CHECK-NOT: dmb
399; CHECK-NOT: mcr
400; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8
401; CHECK: movt r[[ADDR]], :upper16:var8
402
403; CHECK: .LBB{{[0-9]+}}_1:
404; CHECK: ldaexb r[[OLD:[0-9]+]], [r[[ADDR]]]
405  ; r0 below is a reasonable guess but could change: it certainly comes into the
406  ;  function there.
407; CHECK-NEXT: eor{{(\.w)?}} [[NEW:r[0-9]+]], r[[OLD]], r0
408; CHECK-NEXT: strexb [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
409; CHECK-NEXT: cmp [[STATUS]], #0
410; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
411; CHECK-NOT: dmb
412; CHECK-NOT: mcr
413
414; CHECK: mov r0, r[[OLD]]
415   ret i8 %old
416}
417
418define i16 @test_atomic_load_xor_i16(i16 %offset) nounwind {
419; CHECK-LABEL: test_atomic_load_xor_i16:
420   %old = atomicrmw xor ptr @var16, i16 %offset release
421; CHECK-NOT: dmb
422; CHECK-NOT: mcr
423; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var16
424; CHECK: movt r[[ADDR]], :upper16:var16
425
426; CHECK: .LBB{{[0-9]+}}_1:
427; CHECK: ldrexh r[[OLD:[0-9]+]], [r[[ADDR]]]
428  ; r0 below is a reasonable guess but could change: it certainly comes into the
429  ;  function there.
430; CHECK-NEXT: eor{{(\.w)?}} [[NEW:r[0-9]+]], r[[OLD]], r0
431; CHECK-NEXT: stlexh [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
432; CHECK-NEXT: cmp [[STATUS]], #0
433; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
434; CHECK-NOT: dmb
435; CHECK-NOT: mcr
436
437; CHECK: mov r0, r[[OLD]]
438   ret i16 %old
439}
440
441define i32 @test_atomic_load_xor_i32(i32 %offset) nounwind {
442; CHECK-LABEL: test_atomic_load_xor_i32:
443   %old = atomicrmw xor ptr @var32, i32 %offset seq_cst
444; CHECK-NOT: dmb
445; CHECK-NOT: mcr
446; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var32
447; CHECK: movt r[[ADDR]], :upper16:var32
448
449; CHECK: .LBB{{[0-9]+}}_1:
450; CHECK: ldaex r[[OLD:[0-9]+]], [r[[ADDR]]]
451  ; r0 below is a reasonable guess but could change: it certainly comes into the
452  ;  function there.
453; CHECK-NEXT: eor{{(\.w)?}} [[NEW:r[0-9]+]], r[[OLD]], r0
454; CHECK-NEXT: stlex [[STATUS:r[0-9]+]], [[NEW]], [r[[ADDR]]]
455; CHECK-NEXT: cmp [[STATUS]], #0
456; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
457; CHECK-NOT: dmb
458; CHECK-NOT: mcr
459
460; CHECK: mov r0, r[[OLD]]
461   ret i32 %old
462}
463
464define void @test_atomic_load_xor_i64(i64 %offset) nounwind {
465; CHECK-LABEL: test_atomic_load_xor_i64:
466   %old = atomicrmw xor ptr @var64, i64 %offset monotonic
467; CHECK-NOT: dmb
468; CHECK-NOT: mcr
469; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
470; CHECK: movt r[[ADDR]], :upper16:var64
471
472; CHECK: .LBB{{[0-9]+}}_1:
473; CHECK: ldrexd r[[OLD1:[0-9]+]], r[[OLD2:[0-9]+]], [r[[ADDR]]]
474  ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
475  ; function there.
476; CHECK-LE-DAG: eor{{(\.w)?}} [[NEW1:r[0-9]+|lr]], r[[OLD1]], r0
477; CHECK-LE-DAG: eor{{(\.w)?}} [[NEW2:r[0-9]+|lr]], r[[OLD2]], r1
478; CHECK-BE-DAG: eor{{(\.w)?}} [[NEW2:r[0-9]+|lr]], r[[OLD2]], r1
479; CHECK-BE-DAG: eor{{(\.w)?}} [[NEW1:r[0-9]+|lr]], r[[OLD1]], r0
480; CHECK: strexd [[STATUS:r[0-9]+]], [[NEW1]], [[NEW2]], [r[[ADDR]]]
481; CHECK-NEXT: cmp [[STATUS]], #0
482; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
483; CHECK-NOT: dmb
484; CHECK-NOT: mcr
485
486; CHECK: strd r[[OLD1]], r[[OLD2]], [r[[ADDR]]]
487   store i64 %old, ptr @var64
488   ret void
489}
490
491define i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind {
492; CHECK-LABEL: test_atomic_load_xchg_i8:
493   %old = atomicrmw xchg ptr @var8, i8 %offset monotonic
494; CHECK-NOT: dmb
495; CHECK-NOT: mcr
496; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8
497; CHECK: movt r[[ADDR]], :upper16:var8
498
499; CHECK: .LBB{{[0-9]+}}_1:
500; CHECK: ldrexb r[[OLD:[0-9]+]], [r[[ADDR]]]
501  ; r0 below is a reasonable guess but could change: it certainly comes into the
502  ;  function there.
503; CHECK-NEXT: strexb [[STATUS:r[0-9]+]], r0, [r[[ADDR]]]
504; CHECK-NEXT: cmp [[STATUS]], #0
505; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
506; CHECK-NOT: dmb
507; CHECK-NOT: mcr
508
509; CHECK: mov r0, r[[OLD]]
510   ret i8 %old
511}
512
513define i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind {
514; CHECK-LABEL: test_atomic_load_xchg_i16:
515   %old = atomicrmw xchg ptr @var16, i16 %offset seq_cst
516; CHECK-NOT: dmb
517; CHECK-NOT: mcr
518; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var16
519; CHECK: movt r[[ADDR]], :upper16:var16
520
521; CHECK: .LBB{{[0-9]+}}_1:
522; CHECK: ldaexh r[[OLD:[0-9]+]], [r[[ADDR]]]
523  ; r0 below is a reasonable guess but could change: it certainly comes into the
524  ;  function there.
525; CHECK-NEXT: stlexh [[STATUS:r[0-9]+]], r0, [r[[ADDR]]]
526; CHECK-NEXT: cmp [[STATUS]], #0
527; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
528; CHECK-NOT: dmb
529; CHECK-NOT: mcr
530
531; CHECK: mov r0, r[[OLD]]
532   ret i16 %old
533}
534
535define i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind {
536; CHECK-LABEL: test_atomic_load_xchg_i32:
537   %old = atomicrmw xchg ptr @var32, i32 %offset release
538; CHECK-NOT: dmb
539; CHECK-NOT: mcr
540; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var32
541; CHECK: movt r[[ADDR]], :upper16:var32
542
543; CHECK: .LBB{{[0-9]+}}_1:
544; CHECK: ldrex r[[OLD:[0-9]+]], [r[[ADDR]]]
545  ; r0 below is a reasonable guess but could change: it certainly comes into the
546  ;  function there.
547; CHECK-NEXT: stlex [[STATUS:r[0-9]+]], r0, [r[[ADDR]]]
548; CHECK-NEXT: cmp [[STATUS]], #0
549; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
550; CHECK-NOT: dmb
551; CHECK-NOT: mcr
552
553; CHECK: mov r0, r[[OLD]]
554   ret i32 %old
555}
556
557define void @test_atomic_load_xchg_i64(i64 %offset) nounwind {
558; CHECK-LABEL: test_atomic_load_xchg_i64:
559   %old = atomicrmw xchg ptr @var64, i64 %offset acquire
560; CHECK-NOT: dmb
561; CHECK-NOT: mcr
562; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
563; CHECK: movt r[[ADDR]], :upper16:var64
564
565; CHECK: .LBB{{[0-9]+}}_1:
566; CHECK: ldaexd [[OLD1:r[0-9]+]], [[OLD2:r[0-9]+|lr]], [r[[ADDR]]]
567  ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
568  ; function there.
569; CHECK-NEXT: strexd [[STATUS:r[0-9]+]], r0, r1, [r[[ADDR]]]
570; CHECK-NEXT: cmp [[STATUS]], #0
571; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
572; CHECK-NOT: dmb
573; CHECK-NOT: mcr
574
575; CHECK: strd [[OLD1]], [[OLD2]], [r[[ADDR]]]
576   store i64 %old, ptr @var64
577   ret void
578}
579
580define i8 @test_atomic_load_min_i8(i8 signext %offset) nounwind {
581; CHECK-LABEL: test_atomic_load_min_i8:
582   %old = atomicrmw min ptr @var8, i8 %offset acquire
583; CHECK-NOT: dmb
584; CHECK-NOT: mcr
585; CHECK-DAG: movw [[ADDR:r[0-9]+|lr]], :lower16:var8
586; CHECK-DAG: movt [[ADDR]], :upper16:var8
587
588; CHECK: .LBB{{[0-9]+}}_1:
589; CHECK: ldaexb r[[OLD:[0-9]+]], {{.*}}[[ADDR]]
590; CHECK-NEXT: sxtb r[[OLDX:[0-9]+]], r[[OLD]]
591  ; r0 below is a reasonable guess but could change: it certainly comes into the
592  ;  function there.
593; CHECK-NEXT: cmp r[[OLDX]], r0
594; Thumb mode: it le
595; CHECK:      movle r[[OLDX]], r[[OLD]]
596; CHECK-NEXT: strexb [[STATUS:r[0-9]+]], r[[OLDX]], {{.*}}[[ADDR]]]
597; CHECK-NEXT: cmp [[STATUS]], #0
598; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
599; CHECK-NOT: dmb
600; CHECK-NOT: mcr
601
602; CHECK: mov r0, r[[OLD]]
603   ret i8 %old
604}
605
606define i16 @test_atomic_load_min_i16(i16 signext %offset) nounwind {
607; CHECK-LABEL: test_atomic_load_min_i16:
608   %old = atomicrmw min ptr @var16, i16 %offset release
609; CHECK-NOT: dmb
610; CHECK-NOT: mcr
611; CHECK: movw [[ADDR:r[0-9]+|lr]], :lower16:var16
612; CHECK: movt [[ADDR]], :upper16:var16
613
614; CHECK: .LBB{{[0-9]+}}_1:
615; CHECK: ldrexh r[[OLD:[0-9]+]], {{.*}}[[ADDR]]
616; CHECK-NEXT: sxth r[[OLDX:[0-9]+]], r[[OLD]]
617  ; r0 below is a reasonable guess but could change: it certainly comes into the
618  ;  function there.
619; CHECK-NEXT: cmp r[[OLDX]], r0
620; Thumb mode: it le
621; CHECK:      movle r[[OLDX]], r[[OLD]]
622; CHECK-NEXT: stlexh [[STATUS:r[0-9]+]], r[[OLDX]], {{.*}}[[ADDR]]
623; CHECK-NEXT: cmp [[STATUS]], #0
624; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
625; CHECK-NOT: dmb
626; CHECK-NOT: mcr
627
628; CHECK: mov r0, r[[OLD]]
629   ret i16 %old
630}
631
632define i32 @test_atomic_load_min_i32(i32 %offset) nounwind {
633; CHECK-LABEL: test_atomic_load_min_i32:
634   %old = atomicrmw min ptr @var32, i32 %offset monotonic
635; CHECK-NOT: dmb
636; CHECK-NOT: mcr
637; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var32
638; CHECK: movt r[[ADDR]], :upper16:var32
639
640; CHECK: .LBB{{[0-9]+}}_1:
641; CHECK: ldrex r[[OLD:[0-9]+]], [r[[ADDR]]]
642  ; r0 below is a reasonable guess but could change: it certainly comes into the
643  ;  function there.
644; CHECK-NEXT: mov r[[NEW:[0-9]+]], r0
645; CHECK-NEXT: cmp r[[OLD]], r0
646; Thumb mode: it le
647; CHECK:      movle r[[NEW]], r[[OLD]]
648; CHECK-NEXT: strex [[STATUS:r[0-9]+]], r[[NEW]], [r[[ADDR]]]
649; CHECK-NEXT: cmp [[STATUS]], #0
650; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
651; CHECK-NOT: dmb
652; CHECK-NOT: mcr
653
654; CHECK: mov r0, r[[OLD]]
655   ret i32 %old
656}
657
658define void @test_atomic_load_min_i64(i64 %offset) nounwind {
659; CHECK-LABEL: test_atomic_load_min_i64:
660   %old = atomicrmw min ptr @var64, i64 %offset seq_cst
661; CHECK-NOT: dmb
662; CHECK-NOT: mcr
663; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
664; CHECK: movt r[[ADDR]], :upper16:var64
665
666; CHECK: .LBB{{[0-9]+}}_1:
667; CHECK: ldaexd [[OLD1:r[0-9]+|lr]], [[OLD2:r[0-9]+|lr]], [r[[ADDR]]]
668  ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
669  ; function there.
670; CHECK-ARM: mov [[MINHI:r[0-9]+]], r1
671; CHECK-ARM-LE: subs {{[^,]+}}, r0, [[OLD1]]
672; CHECK-ARM-LE: sbcs {{[^,]+}}, r1, [[OLD2]]
673; CHECK-ARM-BE: subs {{[^,]+}}, r1, [[OLD2]]
674; CHECK-ARM-BE: sbcs {{[^,]+}}, r0, [[OLD1]]
675; CHECK-ARM: movge [[MINHI]], [[OLD2]]
676; CHECK-ARM: mov [[MINLO:r[0-9]+]], r0
677; CHECK-ARM: movge [[MINLO]], [[OLD1]]
678; CHECK-ARM: stlexd [[STATUS:r[0-9]+]], [[MINLO]], [[MINHI]], [r[[ADDR]]]
679; CHECK-THUMB: stlexd [[STATUS:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}, [r[[ADDR]]]
680; CHECK-NEXT: cmp [[STATUS]], #0
681; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
682; CHECK-NOT: dmb
683; CHECK-NOT: mcr
684
685; CHECK-ARM: strd [[OLD1]], [[OLD2]], [r[[ADDR]]]
686   store i64 %old, ptr @var64
687   ret void
688}
689
690define i8 @test_atomic_load_max_i8(i8 signext %offset) nounwind {
691; CHECK-LABEL: test_atomic_load_max_i8:
692   %old = atomicrmw max ptr @var8, i8 %offset seq_cst
693; CHECK-NOT: dmb
694; CHECK-NOT: mcr
695; CHECK: movw [[ADDR:r[0-9]+|lr]], :lower16:var8
696; CHECK: movt [[ADDR]], :upper16:var8
697
698; CHECK: .LBB{{[0-9]+}}_1:
699; CHECK: ldaexb r[[OLD:[0-9]+]], {{.*}}[[ADDR]]
700; CHECK-NEXT: sxtb r[[OLDX:[0-9]+]], r[[OLD]]
701  ; r0 below is a reasonable guess but could change: it certainly comes into the
702  ;  function there.
703; CHECK-NEXT: cmp r[[OLDX]], r0
704; Thumb mode: it gt
705; CHECK:      movgt r[[OLDX]], r[[OLD]]
706; CHECK-NEXT: stlexb [[STATUS:r[0-9]+]], r[[OLDX]], {{.*}}[[ADDR]]
707; CHECK-NEXT: cmp [[STATUS]], #0
708; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
709; CHECK-NOT: dmb
710; CHECK-NOT: mcr
711
712; CHECK: mov r0, r[[OLD]]
713   ret i8 %old
714}
715
716define i16 @test_atomic_load_max_i16(i16 signext %offset) nounwind {
717; CHECK-LABEL: test_atomic_load_max_i16:
718   %old = atomicrmw max ptr @var16, i16 %offset acquire
719; CHECK-NOT: dmb
720; CHECK-NOT: mcr
721; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var16
722; CHECK: movt r[[ADDR]], :upper16:var16
723
724; CHECK: .LBB{{[0-9]+}}_1:
725; CHECK: ldaexh r[[OLD:[0-9]+]], [r[[ADDR]]]
726; CHECK-NEXT: sxth r[[OLDX:[0-9]+]], r[[OLD]]
727  ; r0 below is a reasonable guess but could change: it certainly comes into the
728  ;  function there.
729; CHECK-NEXT: cmp r[[OLDX]], r0
730; Thumb mode: it gt
731; CHECK:      movgt r[[OLDX]], r[[OLD]]
732; CHECK-NEXT: strexh [[STATUS:r[0-9]+]], r[[OLDX]], [r[[ADDR]]]
733; CHECK-NEXT: cmp [[STATUS]], #0
734; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
735; CHECK-NOT: dmb
736; CHECK-NOT: mcr
737
738; CHECK: mov r0, r[[OLD]]
739   ret i16 %old
740}
741
742define i32 @test_atomic_load_max_i32(i32 %offset) nounwind {
743; CHECK-LABEL: test_atomic_load_max_i32:
744   %old = atomicrmw max ptr @var32, i32 %offset release
745; CHECK-NOT: dmb
746; CHECK-NOT: mcr
747; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var32
748; CHECK: movt r[[ADDR]], :upper16:var32
749
750; CHECK: .LBB{{[0-9]+}}_1:
751; CHECK: ldrex r[[OLD:[0-9]+]], [r[[ADDR]]]
752  ; r0 below is a reasonable guess but could change: it certainly comes into the
753  ;  function there.
754; CHECK-NEXT: mov r[[NEW:[0-9]+]], r0
755; CHECK-NEXT: cmp r[[OLD]], r0
756; Thumb mode: it gt
757; CHECK:      movgt r[[NEW]], r[[OLD]]
758; CHECK-NEXT: stlex [[STATUS:r[0-9]+]], r[[NEW]], [r[[ADDR]]]
759; CHECK-NEXT: cmp [[STATUS]], #0
760; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
761; CHECK-NOT: dmb
762; CHECK-NOT: mcr
763
764; CHECK: mov r0, r[[OLD]]
765   ret i32 %old
766}
767
768define void @test_atomic_load_max_i64(i64 %offset) nounwind {
769; CHECK-LABEL: test_atomic_load_max_i64:
770   %old = atomicrmw max ptr @var64, i64 %offset monotonic
771; CHECK-NOT: dmb
772; CHECK-NOT: mcr
773; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
774; CHECK: movt r[[ADDR]], :upper16:var64
775
776; CHECK: .LBB{{[0-9]+}}_1:
777; CHECK: ldrexd [[OLD1:r[0-9]+]], [[OLD2:r[0-9]+|lr]], [r[[ADDR]]]
778  ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
779  ; function there.
780; CHECK-ARM: mov [[MINHI:r[0-9]+]], r1
781; CHECK-ARM-LE: subs {{[^,]+}}, r0, [[OLD1]]
782; CHECK-ARM-LE: sbcs {{[^,]+}}, r1, [[OLD2]]
783; CHECK-ARM-BE: subs {{[^,]+}}, r1, [[OLD2]]
784; CHECK-ARM-BE: sbcs {{[^,]+}}, r0, [[OLD1]]
785; CHECK-ARM: movlt [[MINHI]], [[OLD2]]
786; CHECK-ARM: mov [[MINLO:r[0-9]+]], r0
787; CHECK-ARM: movlt [[MINLO]], [[OLD1]]
788; CHECK-ARM: strexd [[STATUS:r[0-9]+]], [[MINLO]], [[MINHI]], [r[[ADDR]]]
789; CHECK-THUMB: strexd [[STATUS:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}, [r[[ADDR]]]
790; CHECK-NEXT: cmp [[STATUS]], #0
791; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
792; CHECK-NOT: dmb
793; CHECK-NOT: mcr
794
795; CHECK-ARM: strd [[OLD1]], [[OLD2]], [r[[ADDR]]]
796   store i64 %old, ptr @var64
797   ret void
798}
799
800define i8 @test_atomic_load_umin_i8(i8 zeroext %offset) nounwind {
801; CHECK-LABEL: test_atomic_load_umin_i8:
802   %old = atomicrmw umin ptr @var8, i8 %offset monotonic
803; CHECK-NOT: dmb
804; CHECK-NOT: mcr
805; CHECK: movw [[ADDR:r[0-9]+|lr]], :lower16:var8
806; CHECK: movt [[ADDR]], :upper16:var8
807
808; CHECK: .LBB{{[0-9]+}}_1:
809; CHECK: ldrexb r[[OLD:[0-9]+]], {{.*}}[[ADDR]]
810  ; r0 below is a reasonable guess but could change: it certainly comes into the
811  ;  function there.
812; CHECK-NEXT: mov r[[NEW:[0-9]+]], r0
813; CHECK-NEXT: cmp r[[OLD]], r0
814; Thumb mode: it ls
815; CHECK:      movls r[[NEW]], r[[OLD]]
816; CHECK-NEXT: strexb [[STATUS:r[0-9]+]], r[[NEW]], {{.*}}[[ADDR]]
817; CHECK-NEXT: cmp [[STATUS]], #0
818; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
819; CHECK-NOT: dmb
820; CHECK-NOT: mcr
821
822; CHECK: mov r0, r[[OLD]]
823   ret i8 %old
824}
825
826define i16 @test_atomic_load_umin_i16(i16 zeroext %offset) nounwind {
827; CHECK-LABEL: test_atomic_load_umin_i16:
828   %old = atomicrmw umin ptr @var16, i16 %offset acquire
829; CHECK-NOT: dmb
830; CHECK-NOT: mcr
831; CHECK: movw [[ADDR:r[0-9]+|lr]], :lower16:var16
832; CHECK: movt [[ADDR]], :upper16:var16
833
834; CHECK: .LBB{{[0-9]+}}_1:
835; CHECK: ldaexh r[[OLD:[0-9]+]], {{.*}}[[ADDR]]
836  ; r0 below is a reasonable guess but could change: it certainly comes into the
837  ;  function there.
838; CHECK-NEXT: mov r[[NEW:[0-9]+]], r0
839; CHECK-NEXT: cmp r[[OLD]], r0
840; Thumb mode: it ls
841; CHECK:      movls r[[NEW]], r[[OLD]]
842; CHECK-NEXT: strexh [[STATUS:r[0-9]+]], r[[NEW]], {{.*}}[[ADDR]]
843; CHECK-NEXT: cmp [[STATUS]], #0
844; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
845; CHECK-NOT: dmb
846; CHECK-NOT: mcr
847
848; CHECK: mov r0, r[[OLD]]
849   ret i16 %old
850}
851
852define i32 @test_atomic_load_umin_i32(i32 %offset) nounwind {
853; CHECK-LABEL: test_atomic_load_umin_i32:
854   %old = atomicrmw umin ptr @var32, i32 %offset seq_cst
855; CHECK-NOT: dmb
856; CHECK-NOT: mcr
857; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var32
858; CHECK: movt r[[ADDR]], :upper16:var32
859
860; CHECK: .LBB{{[0-9]+}}_1:
861; CHECK: ldaex r[[OLD:[0-9]+]], [r[[ADDR]]]
862  ; r0 below is a reasonable guess but could change: it certainly comes into the
863  ;  function there.
864; CHECK-NEXT: mov r[[NEW:[0-9]+]], r0
865; CHECK-NEXT: cmp r[[OLD]], r0
866; Thumb mode: it ls
867; CHECK:      movls r[[NEW]], r[[OLD]]
868; CHECK-NEXT: stlex [[STATUS:r[0-9]+]], r[[NEW]], [r[[ADDR]]]
869; CHECK-NEXT: cmp [[STATUS]], #0
870; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
871; CHECK-NOT: dmb
872; CHECK-NOT: mcr
873
874; CHECK: mov r0, r[[OLD]]
875   ret i32 %old
876}
877
878define void @test_atomic_load_umin_i64(i64 %offset) nounwind {
879; CHECK-LABEL: test_atomic_load_umin_i64:
880   %old = atomicrmw umin ptr @var64, i64 %offset seq_cst
881; CHECK-NOT: dmb
882; CHECK-NOT: mcr
883; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
884; CHECK: movt r[[ADDR]], :upper16:var64
885
886; CHECK: .LBB{{[0-9]+}}_1:
887; CHECK: ldaexd [[OLD1:r[0-9]+|lr]], [[OLD2:r[0-9]+|lr]], [r[[ADDR]]]
888  ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
889  ; function there.
890; CHECK-ARM: mov [[MINHI:r[0-9]+]], r1
891; CHECK-ARM-LE: subs {{[^,]+}}, r0, [[OLD1]]
892; CHECK-ARM-LE: sbcs {{[^,]+}}, r1, [[OLD2]]
893; CHECK-ARM-BE: subs {{[^,]+}}, r1, [[OLD2]]
894; CHECK-ARM-BE: sbcs {{[^,]+}}, r0, [[OLD1]]
895; CHECK-ARM: movhs [[MINHI]], [[OLD2]]
896; CHECK-ARM: mov [[MINLO:r[0-9]+]], r0
897; CHECK-ARM: movhs [[MINLO]], [[OLD1]]
898; CHECK-ARM: stlexd [[STATUS:r[0-9]+]], [[MINLO]], [[MINHI]], [r[[ADDR]]]
899; CHECK-THUMB: stlexd [[STATUS:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}, [r[[ADDR]]]
900; CHECK-NEXT: cmp [[STATUS]], #0
901; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
902; CHECK-NOT: dmb
903; CHECK-NOT: mcr
904
905; CHECK-ARM: strd [[OLD1]], [[OLD2]], [r[[ADDR]]]
906   store i64 %old, ptr @var64
907   ret void
908}
909
910define i8 @test_atomic_load_umax_i8(i8 zeroext %offset) nounwind {
911; CHECK-LABEL: test_atomic_load_umax_i8:
912   %old = atomicrmw umax ptr @var8, i8 %offset acq_rel
913; CHECK-NOT: dmb
914; CHECK-NOT: mcr
915; CHECK: movw [[ADDR:r[0-9]+|lr]], :lower16:var8
916; CHECK: movt [[ADDR]], :upper16:var8
917
918; CHECK: .LBB{{[0-9]+}}_1:
919; CHECK: ldaexb r[[OLD:[0-9]+]], {{.*}}[[ADDR]]
920  ; r0 below is a reasonable guess but could change: it certainly comes into the
921  ;  function there.
922; CHECK-NEXT: mov r[[NEW:[0-9]+]], r0
923; CHECK-NEXT: cmp r[[OLD]], r0
924; Thumb mode: it hi
925; CHECK:      movhi r[[NEW]], r[[OLD]]
926; CHECK-NEXT: stlexb [[STATUS:r[0-9]+]], r[[NEW]], {{.*}}[[ADDR]]
927; CHECK-NEXT: cmp [[STATUS]], #0
928; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
929; CHECK-NOT: dmb
930; CHECK-NOT: mcr
931
932; CHECK: mov r0, r[[OLD]]
933   ret i8 %old
934}
935
936define i16 @test_atomic_load_umax_i16(i16 zeroext %offset) nounwind {
937; CHECK-LABEL: test_atomic_load_umax_i16:
938   %old = atomicrmw umax ptr @var16, i16 %offset monotonic
939; CHECK-NOT: dmb
940; CHECK-NOT: mcr
941; CHECK: movw [[ADDR:r[0-9]+|lr]], :lower16:var16
942; CHECK: movt [[ADDR]], :upper16:var16
943
944; CHECK: .LBB{{[0-9]+}}_1:
945; CHECK: ldrexh r[[OLD:[0-9]+]], {{.*}}[[ADDR]]
946  ; r0 below is a reasonable guess but could change: it certainly comes into the
947  ;  function there.
948; CHECK-NEXT: mov r[[NEW:[0-9]+]], r0
949; CHECK-NEXT: cmp r[[OLD]], r0
950; Thumb mode: it hi
951; CHECK:      movhi r[[NEW]], r[[OLD]]
952; CHECK-NEXT: strexh [[STATUS:r[0-9]+]], r[[NEW]], {{.*}}[[ADDR]]
953; CHECK-NEXT: cmp [[STATUS]], #0
954; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
955; CHECK-NOT: dmb
956; CHECK-NOT: mcr
957
958; CHECK: mov r0, r[[OLD]]
959   ret i16 %old
960}
961
962define i32 @test_atomic_load_umax_i32(i32 %offset) nounwind {
963; CHECK-LABEL: test_atomic_load_umax_i32:
964   %old = atomicrmw umax ptr @var32, i32 %offset seq_cst
965; CHECK-NOT: dmb
966; CHECK-NOT: mcr
967; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var32
968; CHECK: movt r[[ADDR]], :upper16:var32
969
970; CHECK: .LBB{{[0-9]+}}_1:
971; CHECK: ldaex r[[OLD:[0-9]+]], [r[[ADDR]]]
972  ; r0 below is a reasonable guess but could change: it certainly comes into the
973  ;  function there.
974; CHECK-NEXT: mov r[[NEW:[0-9]+]], r0
975; CHECK-NEXT: cmp r[[OLD]], r0
976; Thumb mode: it hi
977; CHECK:      movhi r[[NEW]], r[[OLD]]
978; CHECK-NEXT: stlex [[STATUS:r[0-9]+]], r[[NEW]], [r[[ADDR]]]
979; CHECK-NEXT: cmp [[STATUS]], #0
980; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
981; CHECK-NOT: dmb
982; CHECK-NOT: mcr
983
984; CHECK: mov r0, r[[OLD]]
985   ret i32 %old
986}
987
988define void @test_atomic_load_umax_i64(i64 %offset) nounwind {
989; CHECK-LABEL: test_atomic_load_umax_i64:
990   %old = atomicrmw umax ptr @var64, i64 %offset seq_cst
991; CHECK-NOT: dmb
992; CHECK-NOT: mcr
993; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
994; CHECK: movt r[[ADDR]], :upper16:var64
995
996; CHECK: .LBB{{[0-9]+}}_1:
997; CHECK: ldaexd [[OLD1:r[0-9]+|lr]], [[OLD2:r[0-9]+|lr]], [r[[ADDR]]]
998  ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
999  ; function there.
1000; CHECK-ARM: mov [[MINHI:r[0-9]+]], r1
1001; CHECK-ARM-LE: subs {{[^,]+}}, r0, [[OLD1]]
1002; CHECK-ARM-LE: sbcs {{[^,]+}}, r1, [[OLD2]]
1003; CHECK-ARM-BE: subs {{[^,]+}}, r1, [[OLD2]]
1004; CHECK-ARM-BE: sbcs {{[^,]+}}, r0, [[OLD1]]
1005; CHECK-ARM: movlo [[MINHI]], [[OLD2]]
1006; CHECK-ARM: mov [[MINLO:r[0-9]+]], r0
1007; CHECK-ARM: movlo [[MINLO]], [[OLD1]]
1008; CHECK-ARM: stlexd [[STATUS:r[0-9]+]], [[MINLO]], [[MINHI]], [r[[ADDR]]]
1009; CHECK-THUMB: stlexd [[STATUS:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}, [r[[ADDR]]]
1010; CHECK-NEXT: cmp [[STATUS]], #0
1011; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
1012; CHECK-NOT: dmb
1013; CHECK-NOT: mcr
1014
1015; CHECK-ARM: strd [[OLD1]], [[OLD2]], [r[[ADDR]]]
1016   store i64 %old, ptr @var64
1017   ret void
1018}
1019
1020define i8 @test_atomic_cmpxchg_i8(i8 zeroext %wanted, i8 zeroext %new) nounwind {
1021; CHECK-LABEL: test_atomic_cmpxchg_i8:
1022   %pair = cmpxchg ptr @var8, i8 %wanted, i8 %new acquire acquire
1023   %old = extractvalue { i8, i1 } %pair, 0
1024; CHECK-NOT: dmb
1025; CHECK-NOT: mcr
1026; CHECK-DAG: movw r[[ADDR:[0-9]+]], :lower16:var8
1027; CHECK-DAG: movt r[[ADDR]], :upper16:var8
1028; CHECK-THUMB-DAG: mov r[[WANTED:[0-9]+]], r0
1029
1030; CHECK: .LBB{{[0-9]+}}_1:
1031; CHECK: ldaexb r[[OLD:[0-9]+]], [r[[ADDR]]]
1032  ; r0 below is a reasonable guess but could change: it certainly comes into the
1033  ;  function there.
1034; CHECK-ARM-NEXT:   cmp r[[OLD]], r0
1035; CHECK-THUMB-NEXT: cmp r[[OLD]], r[[WANTED]]
1036; CHECK-NEXT: bne .LBB{{[0-9]+}}_{{[0-9]}}
1037; CHECK-NEXT: %bb.2:
1038  ; As above, r1 is a reasonable guess.
1039; CHECK: strexb [[STATUS:r[0-9]+]], r1, [r[[ADDR]]]
1040; CHECK-NEXT: cmp [[STATUS]], #0
1041; CHECK-ARM-NEXT: bne .LBB{{[0-9]+}}_{{[0-9]}}
1042; CHECK-THUMB-NEXT: it eq
1043; CHECK-THUMB-NEXT: bxeq lr
1044; CHECK-ARM: mov r0, r[[OLD]]
1045; CHECK-ARM: clrex
1046; CHECK: bx lr
1047; CHECK-NOT: dmb
1048; CHECK-NOT: mcr
1049   ret i8 %old
1050}
1051
1052define i16 @test_atomic_cmpxchg_i16(i16 zeroext %wanted, i16 zeroext %new) nounwind {
1053; CHECK-LABEL: test_atomic_cmpxchg_i16:
1054   %pair = cmpxchg ptr @var16, i16 %wanted, i16 %new seq_cst seq_cst
1055   %old = extractvalue { i16, i1 } %pair, 0
1056; CHECK-NOT: dmb
1057; CHECK-NOT: mcr
1058; CHECK-DAG: movw r[[ADDR:[0-9]+]], :lower16:var16
1059; CHECK-DAG: movt r[[ADDR]], :upper16:var16
1060; CHECK-THUMB-DAG: mov r[[WANTED:[0-9]+]], r0
1061
1062; CHECK: .LBB{{[0-9]+}}_1:
1063; CHECK: ldaexh r[[OLD:[0-9]+]], [r[[ADDR]]]
1064  ; r0 below is a reasonable guess but could change: it certainly comes into the
1065  ;  function there.
1066; CHECK-ARM-NEXT:   cmp r[[OLD]], r0
1067; CHECK-THUMB-NEXT: cmp r[[OLD]], r[[WANTED]]
1068; CHECK-NEXT: bne .LBB{{[0-9]+}}_{{[0-9]}}
1069; CHECK-NEXT: %bb.2:
1070  ; As above, r1 is a reasonable guess.
1071; CHECK: stlexh [[STATUS:r[0-9]+]], r1, [r[[ADDR]]]
1072; CHECK-NEXT: cmp [[STATUS]], #0
1073; CHECK-ARM-NEXT: bne .LBB{{[0-9]+}}_{{[0-9]}}
1074; CHECK-THUMB-NEXT: it eq
1075; CHECK-THUMB-NEXT: bxeq lr
1076; CHECK-ARM: mov r0, r[[OLD]]
1077; CHECK: bx lr
1078; CHECK-ARM-NEXT: .LBB{{[0-9]+}}_{{[0-9]}}
1079; CHECK-ARM-NEXT: clrex
1080; CHECK-NOT: dmb
1081; CHECK-NOT: mcr
1082
1083; CHECK-ARM: mov r0, r[[OLD]]
1084; CHECK-ARM-NEXT: bx lr
1085   ret i16 %old
1086}
1087
1088define void @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind {
1089; CHECK-LABEL: test_atomic_cmpxchg_i32:
1090   %pair = cmpxchg ptr @var32, i32 %wanted, i32 %new release monotonic
1091   %old = extractvalue { i32, i1 } %pair, 0
1092   store i32 %old, ptr @var32
1093; CHECK-NOT: dmb
1094; CHECK-NOT: mcr
1095; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var32
1096; CHECK: movt r[[ADDR]], :upper16:var32
1097
1098; CHECK: .LBB{{[0-9]+}}_1:
1099; CHECK: ldrex r[[OLD:[0-9]+]], [r[[ADDR]]]
1100  ; r0 below is a reasonable guess but could change: it certainly comes into the
1101  ;  function there.
1102; CHECK-NEXT: cmp r[[OLD]], r0
1103; CHECK-NEXT: bne .LBB{{[0-9]+}}_4
1104; CHECK-NEXT: %bb.2:
1105  ; As above, r1 is a reasonable guess.
1106; CHECK: stlex [[STATUS:r[0-9]+]], r1, [r[[ADDR]]]
1107; CHECK-NEXT: cmp [[STATUS]], #0
1108; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
1109; CHECK: str{{(.w)?}} r[[OLD]],
1110; CHECK-NEXT: bx lr
1111; CHECK-NEXT: .LBB{{[0-9]+}}_4:
1112; CHECK-NEXT: clrex
1113; CHECK-NOT: dmb
1114; CHECK-NOT: mcr
1115
1116; CHECK: str{{(.w)?}} r[[OLD]],
1117; CHECK-ARM-NEXT: bx lr
1118   ret void
1119}
1120
1121define void @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind {
1122; CHECK-LABEL: test_atomic_cmpxchg_i64:
1123   %pair = cmpxchg ptr @var64, i64 %wanted, i64 %new monotonic monotonic
1124   %old = extractvalue { i64, i1 } %pair, 0
1125; CHECK-NOT: dmb
1126; CHECK-NOT: mcr
1127; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
1128; CHECK: movt r[[ADDR]], :upper16:var64
1129
1130; CHECK: .LBB{{[0-9]+}}_1:
1131; CHECK: ldrexd [[OLD1:r[0-9]+|lr]], [[OLD2:r[0-9]+|lr]], [r[[ADDR]]]
1132  ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
1133  ; function there.
1134; CHECK-LE-DAG: eor{{(\.w)?}} [[MISMATCH_LO:r[0-9]+|lr]], [[OLD1]], r0
1135; CHECK-LE-DAG: eor{{(\.w)?}} [[MISMATCH_HI:r[0-9]+|lr]], [[OLD2]], r1
1136; CHECK-ARM-LE: orrs{{(\.w)?}} {{r[0-9]+}}, [[MISMATCH_LO]], [[MISMATCH_HI]]
1137; CHECK-THUMB-LE: orrs{{(\.w)?}} {{(r[0-9]+, )?}}[[MISMATCH_HI]], [[MISMATCH_LO]]
1138; CHECK-BE-DAG: eor{{(\.w)?}} [[MISMATCH_HI:r[0-9]+|lr]], [[OLD2]], r1
1139; CHECK-BE-DAG: eor{{(\.w)?}} [[MISMATCH_LO:r[0-9]+|lr]], [[OLD1]], r0
1140; CHECK-ARM-BE: orrs{{(\.w)?}} {{r[0-9]+}}, [[MISMATCH_HI]], [[MISMATCH_LO]]
1141; CHECK-THUMB-BE: orrs{{(\.w)?}} {{(r[0-9]+, )?}}[[MISMATCH_LO]], [[MISMATCH_HI]]
1142; CHECK-NEXT: bne .LBB{{[0-9]+}}_4
1143; CHECK-NEXT: %bb.2:
1144  ; As above, r2, r3 is a reasonable guess.
1145; CHECK: strexd [[STATUS:r[0-9]+]], r2, r3, [r[[ADDR]]]
1146; CHECK-NEXT: cmp [[STATUS]], #0
1147; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
1148; CHECK: strd [[OLD1]], [[OLD2]], [r[[ADDR]]]
1149; CHECK-NEXT: pop
1150; CHECK-NEXT: .LBB{{[0-9]+}}_4:
1151; CHECK-NEXT: clrex
1152; CHECK-NOT: dmb
1153; CHECK-NOT: mcr
1154
1155; CHECK-ARM: strd [[OLD1]], [[OLD2]], [r[[ADDR]]]
1156   store i64 %old, ptr @var64
1157   ret void
1158}
1159
1160define i8 @test_atomic_load_monotonic_i8() nounwind {
1161; CHECK-LABEL: test_atomic_load_monotonic_i8:
1162  %val = load atomic i8, ptr @var8 monotonic, align 1
1163; CHECK-NOT: dmb
1164; CHECK-NOT: mcr
1165; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8
1166; CHECK: movt r[[ADDR]], :upper16:var8
1167; CHECK: ldrb r0, [r[[ADDR]]]
1168; CHECK-NOT: dmb
1169; CHECK-NOT: mcr
1170
1171  ret i8 %val
1172}
1173
1174define i8 @test_atomic_load_monotonic_regoff_i8(i64 %base, i64 %off) nounwind {
1175; CHECK-LABEL: test_atomic_load_monotonic_regoff_i8:
1176  %addr_int = add i64 %base, %off
1177  %addr = inttoptr i64 %addr_int to ptr
1178
1179  %val = load atomic i8, ptr %addr monotonic, align 1
1180; CHECK-NOT: dmb
1181; CHECK-NOT: mcr
1182; CHECK-LE: ldrb r0, [r0, r2]
1183; CHECK-BE: ldrb r0, [r1, r3]
1184; CHECK-NOT: dmb
1185; CHECK-NOT: mcr
1186
1187  ret i8 %val
1188}
1189
1190define i8 @test_atomic_load_acquire_i8() nounwind {
1191; CHECK-LABEL: test_atomic_load_acquire_i8:
1192  %val = load atomic i8, ptr @var8 acquire, align 1
1193; CHECK-NOT: dmb
1194; CHECK-NOT: mcr
1195; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8
1196; CHECK-NOT: dmb
1197; CHECK-NOT: mcr
1198; CHECK: movt r[[ADDR]], :upper16:var8
1199; CHECK-NOT: dmb
1200; CHECK-NOT: mcr
1201; CHECK: ldab r0, [r[[ADDR]]]
1202; CHECK-NOT: dmb
1203; CHECK-NOT: mcr
1204  ret i8 %val
1205}
1206
1207define i8 @test_atomic_load_seq_cst_i8() nounwind {
1208; CHECK-LABEL: test_atomic_load_seq_cst_i8:
1209  %val = load atomic i8, ptr @var8 seq_cst, align 1
1210; CHECK-NOT: dmb
1211; CHECK-NOT: mcr
1212; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8
1213; CHECK-NOT: dmb
1214; CHECK-NOT: mcr
1215; CHECK: movt r[[ADDR]], :upper16:var8
1216; CHECK-NOT: dmb
1217; CHECK-NOT: mcr
1218; CHECK: ldab r0, [r[[ADDR]]]
1219; CHECK-NOT: dmb
1220; CHECK-NOT: mcr
1221  ret i8 %val
1222}
1223
1224define i16 @test_atomic_load_monotonic_i16() nounwind {
1225; CHECK-LABEL: test_atomic_load_monotonic_i16:
1226  %val = load atomic i16, ptr @var16 monotonic, align 2
1227; CHECK-NOT: dmb
1228; CHECK-NOT: mcr
1229; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var16
1230; CHECK-NOT: dmb
1231; CHECK-NOT: mcr
1232; CHECK: movt r[[ADDR]], :upper16:var16
1233; CHECK-NOT: dmb
1234; CHECK-NOT: mcr
1235; CHECK: ldrh r0, [r[[ADDR]]]
1236; CHECK-NOT: dmb
1237; CHECK-NOT: mcr
1238
1239  ret i16 %val
1240}
1241
1242define i32 @test_atomic_load_monotonic_regoff_i32(i64 %base, i64 %off) nounwind {
1243; CHECK-LABEL: test_atomic_load_monotonic_regoff_i32:
1244  %addr_int = add i64 %base, %off
1245  %addr = inttoptr i64 %addr_int to ptr
1246
1247  %val = load atomic i32, ptr %addr monotonic, align 4
1248; CHECK-NOT: dmb
1249; CHECK-NOT: mcr
1250; CHECK-LE: ldr r0, [r0, r2]
1251; CHECK-BE: ldr r0, [r1, r3]
1252; CHECK-NOT: dmb
1253; CHECK-NOT: mcr
1254
1255  ret i32 %val
1256}
1257
1258define i64 @test_atomic_load_seq_cst_i64() nounwind {
1259; CHECK-LABEL: test_atomic_load_seq_cst_i64:
1260  %val = load atomic i64, ptr @var64 seq_cst, align 8
1261; CHECK-NOT: dmb
1262; CHECK-NOT: mcr
1263; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
1264; CHECK-NOT: dmb
1265; CHECK-NOT: mcr
1266; CHECK: movt r[[ADDR]], :upper16:var64
1267; CHECK-NOT: dmb
1268; CHECK-NOT: mcr
1269; CHECK: ldaexd r0, r1, [r[[ADDR]]]
1270; CHECK-NOT: dmb
1271; CHECK-NOT: mcr
1272  ret i64 %val
1273}
1274
1275define void @test_atomic_store_monotonic_i8(i8 %val) nounwind {
1276; CHECK-LABEL: test_atomic_store_monotonic_i8:
1277  store atomic i8 %val, ptr @var8 monotonic, align 1
1278; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8
1279; CHECK: movt r[[ADDR]], :upper16:var8
1280; CHECK: strb r0, [r[[ADDR]]]
1281
1282  ret void
1283}
1284
1285define void @test_atomic_store_monotonic_regoff_i8(i64 %base, i64 %off, i8 %val) nounwind {
1286; CHECK-LABEL: test_atomic_store_monotonic_regoff_i8:
1287
1288  %addr_int = add i64 %base, %off
1289  %addr = inttoptr i64 %addr_int to ptr
1290
1291  store atomic i8 %val, ptr %addr monotonic, align 1
1292; CHECK-LE: ldr{{b?(\.w)?}} [[VAL:r[0-9]+]], [sp]
1293; CHECK-LE: strb [[VAL]], [r0, r2]
1294; CHECK-BE: ldrb{{(\.w)?}} [[VAL:r[0-9]+]], [sp, #3]
1295; CHECK-BE: strb [[VAL]], [r1, r3]
1296
1297  ret void
1298}
1299
1300define void @test_atomic_store_release_i8(i8 %val) nounwind {
1301; CHECK-LABEL: test_atomic_store_release_i8:
1302  store atomic i8 %val, ptr @var8 release, align 1
1303; CHECK-NOT: dmb
1304; CHECK-NOT: mcr
1305; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8
1306; CHECK-NOT: dmb
1307; CHECK-NOT: mcr
1308; CHECK: movt r[[ADDR]], :upper16:var8
1309; CHECK-NOT: dmb
1310; CHECK-NOT: mcr
1311; CHECK: stlb r0, [r[[ADDR]]]
1312; CHECK-NOT: dmb
1313; CHECK-NOT: mcr
1314  ret void
1315}
1316
1317define void @test_atomic_store_seq_cst_i8(i8 %val) nounwind {
1318; CHECK-LABEL: test_atomic_store_seq_cst_i8:
1319  store atomic i8 %val, ptr @var8 seq_cst, align 1
1320; CHECK-NOT: dmb
1321; CHECK-NOT: mcr
1322; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var8
1323; CHECK-NOT: dmb
1324; CHECK-NOT: mcr
1325; CHECK: movt r[[ADDR]], :upper16:var8
1326; CHECK-NOT: dmb
1327; CHECK-NOT: mcr
1328; CHECK: stlb r0, [r[[ADDR]]]
1329; CHECK-NOT: dmb
1330; CHECK-NOT: mcr
1331  ret void
1332}
1333
1334define void @test_atomic_store_monotonic_i16(i16 %val) nounwind {
1335; CHECK-LABEL: test_atomic_store_monotonic_i16:
1336  store atomic i16 %val, ptr @var16 monotonic, align 2
1337; CHECK-NOT: dmb
1338; CHECK-NOT: mcr
1339; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var16
1340; CHECK-NOT: dmb
1341; CHECK-NOT: mcr
1342; CHECK: movt r[[ADDR]], :upper16:var16
1343; CHECK-NOT: dmb
1344; CHECK-NOT: mcr
1345; CHECK: strh r0, [r[[ADDR]]]
1346; CHECK-NOT: dmb
1347; CHECK-NOT: mcr
1348  ret void
1349}
1350
1351define void @test_atomic_store_monotonic_regoff_i32(i64 %base, i64 %off, i32 %val) nounwind {
1352; CHECK-LABEL: test_atomic_store_monotonic_regoff_i32:
1353
1354  %addr_int = add i64 %base, %off
1355  %addr = inttoptr i64 %addr_int to ptr
1356
1357  store atomic i32 %val, ptr %addr monotonic, align 4
1358; CHECK-NOT: dmb
1359; CHECK-NOT: mcr
1360; CHECK: ldr [[VAL:r[0-9]+]], [sp]
1361; CHECK-NOT: dmb
1362; CHECK-NOT: mcr
1363; CHECK-LE: str [[VAL]], [r0, r2]
1364; CHECK-BE: str [[VAL]], [r1, r3]
1365; CHECK-NOT: dmb
1366; CHECK-NOT: mcr
1367
1368  ret void
1369}
1370
1371define void @test_atomic_store_release_i64(i64 %val) nounwind {
1372; CHECK-LABEL: test_atomic_store_release_i64:
1373  store atomic i64 %val, ptr @var64 release, align 8
1374; CHECK-NOT: dmb
1375; CHECK-NOT: mcr
1376; CHECK: movw [[ADDR:r[0-9]+|lr]], :lower16:var64
1377; CHECK: movt [[ADDR]], :upper16:var64
1378
1379; CHECK: .LBB{{[0-9]+}}_1:
1380  ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
1381  ; function there.
1382; CHECK: stlexd [[STATUS:r[0-9]+]], r0, r1, {{.*}}[[ADDR]]
1383; CHECK-NEXT: cmp [[STATUS]], #0
1384; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
1385; CHECK-NOT: dmb
1386; CHECK-NOT: mcr
1387
1388  ret void
1389}
1390
1391define i32 @not.barriers(ptr %var, i1 %cond) {
1392; CHECK-LABEL: not.barriers:
1393  br i1 %cond, label %atomic_ver, label %simple_ver
1394simple_ver:
1395  %oldval = load i32, ptr %var
1396  %newval = add nsw i32 %oldval, -1
1397  store i32 %newval, ptr %var
1398  br label %somewhere
1399atomic_ver:
1400  fence seq_cst
1401  %val = atomicrmw add ptr %var, i32 -1 monotonic
1402  fence seq_cst
1403  br label %somewhere
1404; CHECK: dmb
1405; CHECK: ldrex
1406; CHECK: dmb
1407  ; The key point here is that the second dmb isn't immediately followed by the
1408  ; simple_ver basic block, which LLVM attempted to do when DMB had been marked
1409  ; with isBarrier. For now, look for something that looks like "somewhere".
1410; CHECK-NEXT: {{mov|bx}}
1411somewhere:
1412  %combined = phi i32 [ %val, %atomic_ver ], [ %newval, %simple_ver]
1413  ret i32 %combined
1414}
1415