1; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+lse -aarch64-enable-sink-fold=true < %s | FileCheck %s 2; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+lse -mattr=+outline-atomics -aarch64-enable-sink-fold=true < %s | FileCheck %s 3; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+outline-atomics -aarch64-enable-sink-fold=true < %s | FileCheck %s --check-prefix=OUTLINE-ATOMICS 4; RUN: llc -mtriple=aarch64_be-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+lse -aarch64-enable-sink-fold=true < %s | FileCheck %s 5; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-post-ra -verify-machineinstrs -mattr=+lse -aarch64-enable-sink-fold=true < %s | FileCheck %s --check-prefix=CHECK-REG 6 7; Point of CHECK-REG is to make sure UNPREDICTABLE instructions aren't created 8; (i.e. reusing a register for status & data in store exclusive). 9; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], w[[NEW]], [x{{[0-9]+}}] 10; CHECK-REG-NOT: stlxrb w[[NEW:[0-9]+]], x[[NEW]], [x{{[0-9]+}}] 11 12@var8 = dso_local global i8 0 13@var16 = dso_local global i16 0 14@var32 = dso_local global i32 0 15@var64 = dso_local global i64 0 16@var128 = dso_local global i128 0 17 18define dso_local i8 @test_atomic_load_add_i8(i8 %offset) nounwind { 19; CHECK-LABEL: test_atomic_load_add_i8: 20; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i8: 21; OUTLINE-ATOMICS: // %bb.0: 22; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 23; OUTLINE-ATOMICS-NEXT: adrp x1, var8 24; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 25; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel 26; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 27; OUTLINE-ATOMICS-NEXT: ret 28 %old = atomicrmw add ptr @var8, i8 %offset seq_cst 29; CHECK-NOT: dmb 30; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 31; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 32 33; CHECK: ldaddalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 34; CHECK-NOT: dmb 35 36 ret i8 %old 37} 38 39define dso_local i16 @test_atomic_load_add_i16(i16 %offset) nounwind { 40; CHECK-LABEL: test_atomic_load_add_i16: 41; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i16: 42; OUTLINE-ATOMICS: // %bb.0: 43; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 44; OUTLINE-ATOMICS-NEXT: adrp x1, var16 45; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 46; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel 47; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 48; OUTLINE-ATOMICS-NEXT: ret 49 %old = atomicrmw add ptr @var16, i16 %offset seq_cst 50; CHECK-NOT: dmb 51; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 52; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 53 54; CHECK: ldaddalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 55; CHECK-NOT: dmb 56 57 ret i16 %old 58} 59 60define dso_local i32 @test_atomic_load_add_i32(i32 %offset) nounwind { 61; CHECK-LABEL: test_atomic_load_add_i32: 62; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32: 63; OUTLINE-ATOMICS: // %bb.0: 64; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 65; OUTLINE-ATOMICS-NEXT: adrp x1, var32 66; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 67; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel 68; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 69; OUTLINE-ATOMICS-NEXT: ret 70 %old = atomicrmw add ptr @var32, i32 %offset seq_cst 71; CHECK-NOT: dmb 72; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 73; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 74 75; CHECK: ldaddal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 76; CHECK-NOT: dmb 77 78 ret i32 %old 79} 80 81define dso_local i64 @test_atomic_load_add_i64(i64 %offset) nounwind { 82; CHECK-LABEL: test_atomic_load_add_i64: 83; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64: 84; OUTLINE-ATOMICS: // %bb.0: 85; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 86; OUTLINE-ATOMICS-NEXT: adrp x1, var64 87; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 88; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel 89; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 90; OUTLINE-ATOMICS-NEXT: ret 91 %old = atomicrmw add ptr @var64, i64 %offset seq_cst 92; CHECK-NOT: dmb 93; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 94; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 95 96; CHECK: ldaddal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 97; CHECK-NOT: dmb 98 99 ret i64 %old 100} 101 102define dso_local void @test_atomic_load_add_i32_noret(i32 %offset) nounwind { 103; CHECK-LABEL: test_atomic_load_add_i32_noret: 104; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_noret: 105; OUTLINE-ATOMICS: // %bb.0: 106; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 107; OUTLINE-ATOMICS-NEXT: adrp x1, var32 108; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 109; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel 110; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 111; OUTLINE-ATOMICS-NEXT: ret 112 atomicrmw add ptr @var32, i32 %offset seq_cst 113; CHECK-NOT: dmb 114; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 115; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 116 117; CHECK: ldaddal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 118; CHECK-NOT: dmb 119 ret void 120} 121 122define dso_local void @test_atomic_load_add_i64_noret(i64 %offset) nounwind { 123; CHECK-LABEL: test_atomic_load_add_i64_noret: 124; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_noret: 125; OUTLINE-ATOMICS: // %bb.0: 126; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 127; OUTLINE-ATOMICS-NEXT: adrp x1, var64 128; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 129; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel 130; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 131; OUTLINE-ATOMICS-NEXT: ret 132 atomicrmw add ptr @var64, i64 %offset seq_cst 133; CHECK-NOT: dmb 134; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 135; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 136 137; CHECK: ldaddal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 138; CHECK-NOT: dmb 139 ret void 140} 141 142define dso_local i8 @test_atomic_load_or_i8(i8 %offset) nounwind { 143; CHECK-LABEL: test_atomic_load_or_i8: 144; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i8: 145; OUTLINE-ATOMICS: // %bb.0: 146; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 147; OUTLINE-ATOMICS-NEXT: adrp x1, var8 148; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 149; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset1_acq_rel 150; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 151; OUTLINE-ATOMICS-NEXT: ret 152 %old = atomicrmw or ptr @var8, i8 %offset seq_cst 153; CHECK-NOT: dmb 154; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 155; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 156 157; CHECK: ldsetalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 158; CHECK-NOT: dmb 159 160 ret i8 %old 161} 162 163define dso_local i16 @test_atomic_load_or_i16(i16 %offset) nounwind { 164; CHECK-LABEL: test_atomic_load_or_i16: 165; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i16: 166; OUTLINE-ATOMICS: // %bb.0: 167; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 168; OUTLINE-ATOMICS-NEXT: adrp x1, var16 169; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 170; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset2_acq_rel 171; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 172; OUTLINE-ATOMICS-NEXT: ret 173 %old = atomicrmw or ptr @var16, i16 %offset seq_cst 174; CHECK-NOT: dmb 175; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 176; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 177 178; CHECK: ldsetalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 179; CHECK-NOT: dmb 180 181 ret i16 %old 182} 183 184define dso_local i32 @test_atomic_load_or_i32(i32 %offset) nounwind { 185; CHECK-LABEL: test_atomic_load_or_i32: 186; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32: 187; OUTLINE-ATOMICS: // %bb.0: 188; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 189; OUTLINE-ATOMICS-NEXT: adrp x1, var32 190; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 191; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq_rel 192; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 193; OUTLINE-ATOMICS-NEXT: ret 194 %old = atomicrmw or ptr @var32, i32 %offset seq_cst 195; CHECK-NOT: dmb 196; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 197; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 198 199; CHECK: ldsetal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 200; CHECK-NOT: dmb 201 202 ret i32 %old 203} 204 205define dso_local i64 @test_atomic_load_or_i64(i64 %offset) nounwind { 206; CHECK-LABEL: test_atomic_load_or_i64: 207; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64: 208; OUTLINE-ATOMICS: // %bb.0: 209; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 210; OUTLINE-ATOMICS-NEXT: adrp x1, var64 211; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 212; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq_rel 213; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 214; OUTLINE-ATOMICS-NEXT: ret 215 %old = atomicrmw or ptr @var64, i64 %offset seq_cst 216; CHECK-NOT: dmb 217; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 218; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 219 220; CHECK: ldsetal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 221; CHECK-NOT: dmb 222 223 ret i64 %old 224} 225 226define dso_local void @test_atomic_load_or_i32_noret(i32 %offset) nounwind { 227; CHECK-LABEL: test_atomic_load_or_i32_noret: 228; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_noret: 229; OUTLINE-ATOMICS: // %bb.0: 230; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 231; OUTLINE-ATOMICS-NEXT: adrp x1, var32 232; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 233; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq_rel 234; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 235; OUTLINE-ATOMICS-NEXT: ret 236 atomicrmw or ptr @var32, i32 %offset seq_cst 237; CHECK-NOT: dmb 238; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 239; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 240 241; CHECK: ldsetal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 242; CHECK-NOT: dmb 243 ret void 244} 245 246define dso_local void @test_atomic_load_or_i64_noret(i64 %offset) nounwind { 247; CHECK-LABEL: test_atomic_load_or_i64_noret: 248; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_noret: 249; OUTLINE-ATOMICS: // %bb.0: 250; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 251; OUTLINE-ATOMICS-NEXT: adrp x1, var64 252; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 253; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq_rel 254; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 255; OUTLINE-ATOMICS-NEXT: ret 256 atomicrmw or ptr @var64, i64 %offset seq_cst 257; CHECK-NOT: dmb 258; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 259; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 260 261; CHECK: ldsetal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 262; CHECK-NOT: dmb 263 ret void 264} 265 266define dso_local i8 @test_atomic_load_xor_i8(i8 %offset) nounwind { 267; CHECK-LABEL: test_atomic_load_xor_i8: 268; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i8: 269; OUTLINE-ATOMICS: // %bb.0: 270; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 271; OUTLINE-ATOMICS-NEXT: adrp x1, var8 272; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 273; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor1_acq_rel 274; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 275; OUTLINE-ATOMICS-NEXT: ret 276 %old = atomicrmw xor ptr @var8, i8 %offset seq_cst 277; CHECK-NOT: dmb 278; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 279; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 280 281; CHECK: ldeoralb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 282; CHECK-NOT: dmb 283 284 ret i8 %old 285} 286 287define dso_local i16 @test_atomic_load_xor_i16(i16 %offset) nounwind { 288; CHECK-LABEL: test_atomic_load_xor_i16: 289; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i16: 290; OUTLINE-ATOMICS: // %bb.0: 291; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 292; OUTLINE-ATOMICS-NEXT: adrp x1, var16 293; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 294; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor2_acq_rel 295; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 296; OUTLINE-ATOMICS-NEXT: ret 297 %old = atomicrmw xor ptr @var16, i16 %offset seq_cst 298; CHECK-NOT: dmb 299; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 300; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 301 302; CHECK: ldeoralh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 303; CHECK-NOT: dmb 304 305 ret i16 %old 306} 307 308define dso_local i32 @test_atomic_load_xor_i32(i32 %offset) nounwind { 309; CHECK-LABEL: test_atomic_load_xor_i32: 310; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32: 311; OUTLINE-ATOMICS: // %bb.0: 312; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 313; OUTLINE-ATOMICS-NEXT: adrp x1, var32 314; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 315; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel 316; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 317; OUTLINE-ATOMICS-NEXT: ret 318 %old = atomicrmw xor ptr @var32, i32 %offset seq_cst 319; CHECK-NOT: dmb 320; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 321; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 322 323; CHECK: ldeoral w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 324; CHECK-NOT: dmb 325 326 ret i32 %old 327} 328 329define dso_local i64 @test_atomic_load_xor_i64(i64 %offset) nounwind { 330; CHECK-LABEL: test_atomic_load_xor_i64: 331; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64: 332; OUTLINE-ATOMICS: // %bb.0: 333; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 334; OUTLINE-ATOMICS-NEXT: adrp x1, var64 335; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 336; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq_rel 337; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 338; OUTLINE-ATOMICS-NEXT: ret 339 %old = atomicrmw xor ptr @var64, i64 %offset seq_cst 340; CHECK-NOT: dmb 341; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 342; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 343 344; CHECK: ldeoral x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 345; CHECK-NOT: dmb 346 347 ret i64 %old 348} 349 350define dso_local void @test_atomic_load_xor_i32_noret(i32 %offset) nounwind { 351; CHECK-LABEL: test_atomic_load_xor_i32_noret: 352; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_noret: 353; OUTLINE-ATOMICS: // %bb.0: 354; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 355; OUTLINE-ATOMICS-NEXT: adrp x1, var32 356; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 357; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel 358; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 359; OUTLINE-ATOMICS-NEXT: ret 360 atomicrmw xor ptr @var32, i32 %offset seq_cst 361; CHECK-NOT: dmb 362; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 363; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 364 365; CHECK: ldeoral w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 366; CHECK-NOT: dmb 367 ret void 368} 369 370define dso_local void @test_atomic_load_xor_i64_noret(i64 %offset) nounwind { 371; CHECK-LABEL: test_atomic_load_xor_i64_noret: 372; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_noret: 373; OUTLINE-ATOMICS: // %bb.0: 374; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 375; OUTLINE-ATOMICS-NEXT: adrp x1, var64 376; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 377; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq_rel 378; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 379; OUTLINE-ATOMICS-NEXT: ret 380 atomicrmw xor ptr @var64, i64 %offset seq_cst 381; CHECK-NOT: dmb 382; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 383; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 384 385; CHECK: ldeoral x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 386; CHECK-NOT: dmb 387 ret void 388} 389 390define dso_local i8 @test_atomic_load_min_i8(i8 %offset) nounwind { 391; CHECK-LABEL: test_atomic_load_min_i8: 392; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8: 393; OUTLINE-ATOMICS: // %bb.0: 394; OUTLINE-ATOMICS-NEXT: adrp x9, var8 395; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 396; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 397; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 398; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] 399; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 400; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb 401; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le 402; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] 403; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 404; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 405; OUTLINE-ATOMICS-NEXT: mov w0, w8 406; OUTLINE-ATOMICS-NEXT: ret 407 %old = atomicrmw min ptr @var8, i8 %offset seq_cst 408; CHECK-NOT: dmb 409; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 410; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 411 412; CHECK: ldsminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 413; CHECK-NOT: dmb 414 415 ret i8 %old 416} 417 418define dso_local i16 @test_atomic_load_min_i16(i16 %offset) nounwind { 419; CHECK-LABEL: test_atomic_load_min_i16: 420; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16: 421; OUTLINE-ATOMICS: // %bb.0: 422; OUTLINE-ATOMICS-NEXT: adrp x9, var16 423; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 424; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 425; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 426; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] 427; OUTLINE-ATOMICS-NEXT: sxth w8, w10 428; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth 429; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le 430; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] 431; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 432; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 433; OUTLINE-ATOMICS-NEXT: mov w0, w8 434; OUTLINE-ATOMICS-NEXT: ret 435 %old = atomicrmw min ptr @var16, i16 %offset seq_cst 436; CHECK-NOT: dmb 437; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 438; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 439 440; CHECK: ldsminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 441; CHECK-NOT: dmb 442 443 ret i16 %old 444} 445 446define dso_local i32 @test_atomic_load_min_i32(i32 %offset) nounwind { 447; CHECK-LABEL: test_atomic_load_min_i32: 448; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32: 449; OUTLINE-ATOMICS: // %bb.0: 450; OUTLINE-ATOMICS-NEXT: adrp x9, var32 451; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 452; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 453; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 454; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 455; OUTLINE-ATOMICS-NEXT: cmp w8, w0 456; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le 457; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 458; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 459; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 460; OUTLINE-ATOMICS-NEXT: mov w0, w8 461; OUTLINE-ATOMICS-NEXT: ret 462 %old = atomicrmw min ptr @var32, i32 %offset seq_cst 463; CHECK-NOT: dmb 464; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 465; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 466 467; CHECK: ldsminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 468; CHECK-NOT: dmb 469 470 ret i32 %old 471} 472 473define dso_local i64 @test_atomic_load_min_i64(i64 %offset) nounwind { 474; CHECK-LABEL: test_atomic_load_min_i64: 475; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64: 476; OUTLINE-ATOMICS: // %bb.0: 477; OUTLINE-ATOMICS-NEXT: adrp x9, var64 478; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 479; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 480; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 481; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 482; OUTLINE-ATOMICS-NEXT: cmp x8, x0 483; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le 484; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 485; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 486; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 487; OUTLINE-ATOMICS-NEXT: mov x0, x8 488; OUTLINE-ATOMICS-NEXT: ret 489 %old = atomicrmw min ptr @var64, i64 %offset seq_cst 490; CHECK-NOT: dmb 491; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 492; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 493 494; CHECK: ldsminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 495; CHECK-NOT: dmb 496 497 ret i64 %old 498} 499 500define dso_local void @test_atomic_load_min_i32_noret(i32 %offset) nounwind { 501; CHECK-LABEL: test_atomic_load_min_i32_noret: 502; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret: 503; OUTLINE-ATOMICS: // %bb.0: 504; OUTLINE-ATOMICS-NEXT: adrp x8, var32 505; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 506; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 507; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 508; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 509; OUTLINE-ATOMICS-NEXT: cmp w9, w0 510; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le 511; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 512; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 513; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 514; OUTLINE-ATOMICS-NEXT: ret 515 atomicrmw min ptr @var32, i32 %offset seq_cst 516; CHECK-NOT: dmb 517; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 518; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 519 520; CHECK: ldsminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 521; CHECK-NOT: dmb 522 ret void 523} 524 525define dso_local void @test_atomic_load_min_i64_noret(i64 %offset) nounwind { 526; CHECK-LABEL: test_atomic_load_min_i64_noret: 527; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret: 528; OUTLINE-ATOMICS: // %bb.0: 529; OUTLINE-ATOMICS-NEXT: adrp x8, var64 530; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 531; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 532; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 533; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 534; OUTLINE-ATOMICS-NEXT: cmp x9, x0 535; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le 536; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 537; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 538; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 539; OUTLINE-ATOMICS-NEXT: ret 540 atomicrmw min ptr @var64, i64 %offset seq_cst 541; CHECK-NOT: dmb 542; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 543; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 544 545; CHECK: ldsminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 546; CHECK-NOT: dmb 547 ret void 548} 549 550define dso_local i8 @test_atomic_load_umin_i8(i8 %offset) nounwind { 551; CHECK-LABEL: test_atomic_load_umin_i8: 552; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8: 553; OUTLINE-ATOMICS: // %bb.0: 554; OUTLINE-ATOMICS-NEXT: adrp x8, var8 555; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var8 556; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xff 557; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 558; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 559; OUTLINE-ATOMICS-NEXT: ldaxrb w0, [x8] 560; OUTLINE-ATOMICS-NEXT: cmp w0, w9 561; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, ls 562; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x8] 563; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 564; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 565; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 566; OUTLINE-ATOMICS-NEXT: ret 567 %old = atomicrmw umin ptr @var8, i8 %offset seq_cst 568; CHECK-NOT: dmb 569; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 570; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 571 572; CHECK: lduminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 573; CHECK-NOT: dmb 574 575 ret i8 %old 576} 577 578define dso_local i16 @test_atomic_load_umin_i16(i16 %offset) nounwind { 579; CHECK-LABEL: test_atomic_load_umin_i16: 580; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16: 581; OUTLINE-ATOMICS: // %bb.0: 582; OUTLINE-ATOMICS-NEXT: adrp x8, var16 583; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var16 584; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xffff 585; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 586; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 587; OUTLINE-ATOMICS-NEXT: ldaxrh w0, [x8] 588; OUTLINE-ATOMICS-NEXT: cmp w0, w9 589; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, ls 590; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x8] 591; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 592; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 593; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 594; OUTLINE-ATOMICS-NEXT: ret 595 %old = atomicrmw umin ptr @var16, i16 %offset seq_cst 596; CHECK-NOT: dmb 597; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 598; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 599 600; CHECK: lduminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 601; CHECK-NOT: dmb 602 603 ret i16 %old 604} 605 606define dso_local i32 @test_atomic_load_umin_i32(i32 %offset) nounwind { 607; CHECK-LABEL: test_atomic_load_umin_i32: 608; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32: 609; OUTLINE-ATOMICS: // %bb.0: 610; OUTLINE-ATOMICS-NEXT: adrp x9, var32 611; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 612; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 613; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 614; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 615; OUTLINE-ATOMICS-NEXT: cmp w8, w0 616; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls 617; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 618; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 619; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 620; OUTLINE-ATOMICS-NEXT: mov w0, w8 621; OUTLINE-ATOMICS-NEXT: ret 622 %old = atomicrmw umin ptr @var32, i32 %offset seq_cst 623; CHECK-NOT: dmb 624; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 625; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 626 627; CHECK: lduminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 628; CHECK-NOT: dmb 629 630 ret i32 %old 631} 632 633define dso_local i64 @test_atomic_load_umin_i64(i64 %offset) nounwind { 634; CHECK-LABEL: test_atomic_load_umin_i64: 635; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64: 636; OUTLINE-ATOMICS: // %bb.0: 637; OUTLINE-ATOMICS-NEXT: adrp x9, var64 638; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 639; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 640; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 641; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 642; OUTLINE-ATOMICS-NEXT: cmp x8, x0 643; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls 644; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 645; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 646; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 647; OUTLINE-ATOMICS-NEXT: mov x0, x8 648; OUTLINE-ATOMICS-NEXT: ret 649 %old = atomicrmw umin ptr @var64, i64 %offset seq_cst 650; CHECK-NOT: dmb 651; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 652; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 653 654; CHECK: lduminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 655; CHECK-NOT: dmb 656 657 ret i64 %old 658} 659 660define dso_local void @test_atomic_load_umin_i32_noret(i32 %offset) nounwind { 661; CHECK-LABEL: test_atomic_load_umin_i32_noret: 662; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret: 663; OUTLINE-ATOMICS: // %bb.0: 664; OUTLINE-ATOMICS-NEXT: adrp x8, var32 665; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 666; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 667; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 668; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 669; OUTLINE-ATOMICS-NEXT: cmp w9, w0 670; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls 671; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 672; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 673; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 674; OUTLINE-ATOMICS-NEXT: ret 675 atomicrmw umin ptr @var32, i32 %offset seq_cst 676; CHECK-NOT: dmb 677; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 678; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 679 680; CHECK: lduminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 681; CHECK-NOT: dmb 682 ret void 683} 684 685define dso_local void @test_atomic_load_umin_i64_noret(i64 %offset) nounwind { 686; CHECK-LABEL: test_atomic_load_umin_i64_noret: 687; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret: 688; OUTLINE-ATOMICS: // %bb.0: 689; OUTLINE-ATOMICS-NEXT: adrp x8, var64 690; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 691; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 692; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 693; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 694; OUTLINE-ATOMICS-NEXT: cmp x9, x0 695; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls 696; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 697; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 698; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 699; OUTLINE-ATOMICS-NEXT: ret 700 atomicrmw umin ptr @var64, i64 %offset seq_cst 701; CHECK-NOT: dmb 702; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 703; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 704 705; CHECK: lduminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 706; CHECK-NOT: dmb 707 ret void 708} 709 710define dso_local i8 @test_atomic_load_max_i8(i8 %offset) nounwind { 711; CHECK-LABEL: test_atomic_load_max_i8: 712; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8: 713; OUTLINE-ATOMICS: // %bb.0: 714; OUTLINE-ATOMICS-NEXT: adrp x9, var8 715; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 716; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 717; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 718; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] 719; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 720; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb 721; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt 722; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] 723; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 724; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 725; OUTLINE-ATOMICS-NEXT: mov w0, w8 726; OUTLINE-ATOMICS-NEXT: ret 727 %old = atomicrmw max ptr @var8, i8 %offset seq_cst 728; CHECK-NOT: dmb 729; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 730; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 731 732; CHECK: ldsmaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 733; CHECK-NOT: dmb 734 735 ret i8 %old 736} 737 738define dso_local i16 @test_atomic_load_max_i16(i16 %offset) nounwind { 739; CHECK-LABEL: test_atomic_load_max_i16: 740; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16: 741; OUTLINE-ATOMICS: // %bb.0: 742; OUTLINE-ATOMICS-NEXT: adrp x9, var16 743; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 744; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 745; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 746; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] 747; OUTLINE-ATOMICS-NEXT: sxth w8, w10 748; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth 749; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt 750; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] 751; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 752; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 753; OUTLINE-ATOMICS-NEXT: mov w0, w8 754; OUTLINE-ATOMICS-NEXT: ret 755 %old = atomicrmw max ptr @var16, i16 %offset seq_cst 756; CHECK-NOT: dmb 757; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 758; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 759 760; CHECK: ldsmaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 761; CHECK-NOT: dmb 762 763 ret i16 %old 764} 765 766define dso_local i32 @test_atomic_load_max_i32(i32 %offset) nounwind { 767; CHECK-LABEL: test_atomic_load_max_i32: 768; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32: 769; OUTLINE-ATOMICS: // %bb.0: 770; OUTLINE-ATOMICS-NEXT: adrp x9, var32 771; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 772; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 773; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 774; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 775; OUTLINE-ATOMICS-NEXT: cmp w8, w0 776; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt 777; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 778; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 779; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 780; OUTLINE-ATOMICS-NEXT: mov w0, w8 781; OUTLINE-ATOMICS-NEXT: ret 782 %old = atomicrmw max ptr @var32, i32 %offset seq_cst 783; CHECK-NOT: dmb 784; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 785; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 786 787; CHECK: ldsmaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 788; CHECK-NOT: dmb 789 790 ret i32 %old 791} 792 793define dso_local i64 @test_atomic_load_max_i64(i64 %offset) nounwind { 794; CHECK-LABEL: test_atomic_load_max_i64: 795; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64: 796; OUTLINE-ATOMICS: // %bb.0: 797; OUTLINE-ATOMICS-NEXT: adrp x9, var64 798; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 799; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 800; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 801; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 802; OUTLINE-ATOMICS-NEXT: cmp x8, x0 803; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt 804; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 805; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 806; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 807; OUTLINE-ATOMICS-NEXT: mov x0, x8 808; OUTLINE-ATOMICS-NEXT: ret 809 %old = atomicrmw max ptr @var64, i64 %offset seq_cst 810; CHECK-NOT: dmb 811; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 812; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 813 814; CHECK: ldsmaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 815; CHECK-NOT: dmb 816 817 ret i64 %old 818} 819 820define dso_local void @test_atomic_load_max_i32_noret(i32 %offset) nounwind { 821; CHECK-LABEL: test_atomic_load_max_i32_noret: 822; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret: 823; OUTLINE-ATOMICS: // %bb.0: 824; OUTLINE-ATOMICS-NEXT: adrp x8, var32 825; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 826; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 827; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 828; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 829; OUTLINE-ATOMICS-NEXT: cmp w9, w0 830; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt 831; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 832; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 833; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 834; OUTLINE-ATOMICS-NEXT: ret 835 atomicrmw max ptr @var32, i32 %offset seq_cst 836; CHECK-NOT: dmb 837; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 838; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 839 840; CHECK: ldsmaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 841; CHECK-NOT: dmb 842 ret void 843} 844 845define dso_local void @test_atomic_load_max_i64_noret(i64 %offset) nounwind { 846; CHECK-LABEL: test_atomic_load_max_i64_noret: 847; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret: 848; OUTLINE-ATOMICS: // %bb.0: 849; OUTLINE-ATOMICS-NEXT: adrp x8, var64 850; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 851; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 852; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 853; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 854; OUTLINE-ATOMICS-NEXT: cmp x9, x0 855; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt 856; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 857; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 858; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 859; OUTLINE-ATOMICS-NEXT: ret 860 atomicrmw max ptr @var64, i64 %offset seq_cst 861; CHECK-NOT: dmb 862; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 863; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 864 865; CHECK: ldsmaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 866; CHECK-NOT: dmb 867 ret void 868} 869 870define dso_local i8 @test_atomic_load_umax_i8(i8 %offset) nounwind { 871; CHECK-LABEL: test_atomic_load_umax_i8: 872; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8: 873; OUTLINE-ATOMICS: // %bb.0: 874; OUTLINE-ATOMICS-NEXT: adrp x8, var8 875; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var8 876; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xff 877; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 878; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 879; OUTLINE-ATOMICS-NEXT: ldaxrb w0, [x8] 880; OUTLINE-ATOMICS-NEXT: cmp w0, w9 881; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, hi 882; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x8] 883; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 884; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 885; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 886; OUTLINE-ATOMICS-NEXT: ret 887 %old = atomicrmw umax ptr @var8, i8 %offset seq_cst 888; CHECK-NOT: dmb 889; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 890; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 891 892; CHECK: ldumaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 893; CHECK-NOT: dmb 894 895 ret i8 %old 896} 897 898define dso_local i16 @test_atomic_load_umax_i16(i16 %offset) nounwind { 899; CHECK-LABEL: test_atomic_load_umax_i16: 900; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16: 901; OUTLINE-ATOMICS: // %bb.0: 902; OUTLINE-ATOMICS-NEXT: adrp x8, var16 903; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var16 904; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xffff 905; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 906; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 907; OUTLINE-ATOMICS-NEXT: ldaxrh w0, [x8] 908; OUTLINE-ATOMICS-NEXT: cmp w0, w9 909; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, hi 910; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x8] 911; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 912; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 913; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 914; OUTLINE-ATOMICS-NEXT: ret 915 %old = atomicrmw umax ptr @var16, i16 %offset seq_cst 916; CHECK-NOT: dmb 917; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 918; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 919 920; CHECK: ldumaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 921; CHECK-NOT: dmb 922 923 ret i16 %old 924} 925 926define dso_local i32 @test_atomic_load_umax_i32(i32 %offset) nounwind { 927; CHECK-LABEL: test_atomic_load_umax_i32: 928; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32: 929; OUTLINE-ATOMICS: // %bb.0: 930; OUTLINE-ATOMICS-NEXT: adrp x9, var32 931; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 932; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 933; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 934; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 935; OUTLINE-ATOMICS-NEXT: cmp w8, w0 936; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi 937; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 938; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 939; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 940; OUTLINE-ATOMICS-NEXT: mov w0, w8 941; OUTLINE-ATOMICS-NEXT: ret 942 %old = atomicrmw umax ptr @var32, i32 %offset seq_cst 943; CHECK-NOT: dmb 944; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 945; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 946 947; CHECK: ldumaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 948; CHECK-NOT: dmb 949 950 ret i32 %old 951} 952 953define dso_local i64 @test_atomic_load_umax_i64(i64 %offset) nounwind { 954; CHECK-LABEL: test_atomic_load_umax_i64: 955; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64: 956; OUTLINE-ATOMICS: // %bb.0: 957; OUTLINE-ATOMICS-NEXT: adrp x9, var64 958; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 959; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 960; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 961; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 962; OUTLINE-ATOMICS-NEXT: cmp x8, x0 963; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi 964; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 965; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 966; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 967; OUTLINE-ATOMICS-NEXT: mov x0, x8 968; OUTLINE-ATOMICS-NEXT: ret 969 %old = atomicrmw umax ptr @var64, i64 %offset seq_cst 970; CHECK-NOT: dmb 971; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 972; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 973 974; CHECK: ldumaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 975; CHECK-NOT: dmb 976 977 ret i64 %old 978} 979 980define dso_local void @test_atomic_load_umax_i32_noret(i32 %offset) nounwind { 981; CHECK-LABEL: test_atomic_load_umax_i32_noret: 982; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret: 983; OUTLINE-ATOMICS: // %bb.0: 984; OUTLINE-ATOMICS-NEXT: adrp x8, var32 985; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 986; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 987; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 988; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 989; OUTLINE-ATOMICS-NEXT: cmp w9, w0 990; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi 991; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 992; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 993; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 994; OUTLINE-ATOMICS-NEXT: ret 995 atomicrmw umax ptr @var32, i32 %offset seq_cst 996; CHECK-NOT: dmb 997; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 998; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 999 1000; CHECK: ldumaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 1001; CHECK-NOT: dmb 1002 ret void 1003} 1004 1005define dso_local void @test_atomic_load_umax_i64_noret(i64 %offset) nounwind { 1006; CHECK-LABEL: test_atomic_load_umax_i64_noret: 1007; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret: 1008; OUTLINE-ATOMICS: // %bb.0: 1009; OUTLINE-ATOMICS-NEXT: adrp x8, var64 1010; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 1011; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 1012; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 1013; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 1014; OUTLINE-ATOMICS-NEXT: cmp x9, x0 1015; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi 1016; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 1017; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 1018; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 1019; OUTLINE-ATOMICS-NEXT: ret 1020 atomicrmw umax ptr @var64, i64 %offset seq_cst 1021; CHECK-NOT: dmb 1022; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 1023; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 1024 1025; CHECK: ldumaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 1026; CHECK-NOT: dmb 1027 ret void 1028} 1029 1030define dso_local i8 @test_atomic_load_xchg_i8(i8 %offset) nounwind { 1031; CHECK-LABEL: test_atomic_load_xchg_i8: 1032; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i8: 1033; OUTLINE-ATOMICS: // %bb.0: 1034; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1035; OUTLINE-ATOMICS-NEXT: adrp x1, var8 1036; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 1037; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp1_acq_rel 1038; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1039; OUTLINE-ATOMICS-NEXT: ret 1040 %old = atomicrmw xchg ptr @var8, i8 %offset seq_cst 1041; CHECK-NOT: dmb 1042; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 1043; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 1044 1045; CHECK: swpalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1046; CHECK-NOT: dmb 1047 1048 ret i8 %old 1049} 1050 1051define dso_local i16 @test_atomic_load_xchg_i16(i16 %offset) nounwind { 1052; CHECK-LABEL: test_atomic_load_xchg_i16: 1053; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i16: 1054; OUTLINE-ATOMICS: // %bb.0: 1055; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1056; OUTLINE-ATOMICS-NEXT: adrp x1, var16 1057; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 1058; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp2_acq_rel 1059; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1060; OUTLINE-ATOMICS-NEXT: ret 1061 %old = atomicrmw xchg ptr @var16, i16 %offset seq_cst 1062; CHECK-NOT: dmb 1063; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 1064; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 1065 1066; CHECK: swpalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1067; CHECK-NOT: dmb 1068 1069 ret i16 %old 1070} 1071 1072define dso_local i32 @test_atomic_load_xchg_i32(i32 %offset) nounwind { 1073; CHECK-LABEL: test_atomic_load_xchg_i32: 1074; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32: 1075; OUTLINE-ATOMICS: // %bb.0: 1076; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1077; OUTLINE-ATOMICS-NEXT: adrp x1, var32 1078; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 1079; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq_rel 1080; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1081; OUTLINE-ATOMICS-NEXT: ret 1082 %old = atomicrmw xchg ptr @var32, i32 %offset seq_cst 1083; CHECK-NOT: dmb 1084; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 1085; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 1086 1087; CHECK: swpal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1088; CHECK-NOT: dmb 1089 1090 ret i32 %old 1091} 1092 1093define dso_local i64 @test_atomic_load_xchg_i64(i64 %offset) nounwind { 1094; CHECK-LABEL: test_atomic_load_xchg_i64: 1095; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64: 1096; OUTLINE-ATOMICS: // %bb.0: 1097; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1098; OUTLINE-ATOMICS-NEXT: adrp x1, var64 1099; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 1100; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq_rel 1101; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1102; OUTLINE-ATOMICS-NEXT: ret 1103 %old = atomicrmw xchg ptr @var64, i64 %offset seq_cst 1104; CHECK-NOT: dmb 1105; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 1106; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 1107 1108; CHECK: swpal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 1109; CHECK-NOT: dmb 1110 1111 ret i64 %old 1112} 1113 1114define dso_local void @test_atomic_load_xchg_i32_noret(i32 %offset) nounwind { 1115; CHECK-LABEL: test_atomic_load_xchg_i32_noret: 1116; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_noret: 1117; OUTLINE-ATOMICS: // %bb.0: 1118; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1119; OUTLINE-ATOMICS-NEXT: adrp x1, var32 1120; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 1121; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq_rel 1122; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1123; OUTLINE-ATOMICS-NEXT: ret 1124 atomicrmw xchg ptr @var32, i32 %offset seq_cst 1125; CHECK-NOT: dmb 1126; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 1127; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 1128 1129; CHECK: swpal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1130; CHECK-NOT: dmb 1131 1132 ret void 1133} 1134 1135define dso_local void @test_atomic_load_xchg_i64_noret(i64 %offset) nounwind { 1136; CHECK-LABEL: test_atomic_load_xchg_i64_noret: 1137; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_noret: 1138; OUTLINE-ATOMICS: // %bb.0: 1139; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1140; OUTLINE-ATOMICS-NEXT: adrp x1, var64 1141; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 1142; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq_rel 1143; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1144; OUTLINE-ATOMICS-NEXT: ret 1145 atomicrmw xchg ptr @var64, i64 %offset seq_cst 1146; CHECK-NOT: dmb 1147; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 1148; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 1149 1150; CHECK: swpal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 1151; CHECK-NOT: dmb 1152 1153 ret void 1154} 1155 1156define dso_local i8 @test_atomic_cmpxchg_i8(i8 %wanted, i8 %new) nounwind { 1157; CHECK-LABEL: test_atomic_cmpxchg_i8: 1158; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i8: 1159; OUTLINE-ATOMICS: // %bb.0: 1160; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1161; OUTLINE-ATOMICS-NEXT: adrp x2, var8 1162; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var8 1163; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas1_acq 1164; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1165; OUTLINE-ATOMICS-NEXT: ret 1166 %pair = cmpxchg ptr @var8, i8 %wanted, i8 %new acquire acquire 1167 %old = extractvalue { i8, i1 } %pair, 0 1168 1169; CHECK-NOT: dmb 1170; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 1171; CHECK-NEXT: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 1172; CHECK-NEXT: casab w0, w1, [x[[ADDR]]] 1173; CHECK-NEXT: ret 1174 1175 ret i8 %old 1176} 1177 1178define dso_local i1 @test_atomic_cmpxchg_i8_1(i8 %wanted, i8 %new) nounwind { 1179; CHECK-LABEL: test_atomic_cmpxchg_i8_1: 1180; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i8_1: 1181; OUTLINE-ATOMICS: // %bb.0: 1182; OUTLINE-ATOMICS-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill 1183; OUTLINE-ATOMICS-NEXT: mov w19, w0 1184; OUTLINE-ATOMICS-NEXT: adrp x2, var8 1185; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var8 1186; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas1_acq 1187; OUTLINE-ATOMICS-NEXT: cmp w0, w19, uxtb 1188; OUTLINE-ATOMICS-NEXT: cset w0, eq 1189; OUTLINE-ATOMICS-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload 1190; OUTLINE-ATOMICS-NEXT: ret 1191 %pair = cmpxchg ptr @var8, i8 %wanted, i8 %new acquire acquire 1192 %success = extractvalue { i8, i1 } %pair, 1 1193 1194; CHECK-NOT: dmb 1195; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 1196; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 1197 1198; CHECK: casab w[[NEW:[0-9]+]], w1, [x[[ADDR]]] 1199; CHECK-NEXT: cmp w[[NEW]], w0, uxtb 1200; CHECK-NEXT: cset w0, eq 1201; CHECK-NEXT: ret 1202 ret i1 %success 1203} 1204 1205define dso_local i16 @test_atomic_cmpxchg_i16(i16 %wanted, i16 %new) nounwind { 1206; CHECK-LABEL: test_atomic_cmpxchg_i16: 1207; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i16: 1208; OUTLINE-ATOMICS: // %bb.0: 1209; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1210; OUTLINE-ATOMICS-NEXT: adrp x2, var16 1211; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var16 1212; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas2_acq 1213; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1214; OUTLINE-ATOMICS-NEXT: ret 1215 %pair = cmpxchg ptr @var16, i16 %wanted, i16 %new acquire acquire 1216 %old = extractvalue { i16, i1 } %pair, 0 1217 1218; CHECK-NOT: dmb 1219; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 1220; CHECK-NEXT: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 1221; CHECK-NEXT: casah w0, w1, [x[[ADDR]]] 1222; CHECK-NEXT: ret 1223 1224 ret i16 %old 1225} 1226 1227define dso_local i1 @test_atomic_cmpxchg_i16_1(i16 %wanted, i16 %new) nounwind { 1228; CHECK-LABEL: test_atomic_cmpxchg_i16_1: 1229; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i16_1: 1230; OUTLINE-ATOMICS: // %bb.0: 1231; OUTLINE-ATOMICS-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill 1232; OUTLINE-ATOMICS-NEXT: mov w19, w0 1233; OUTLINE-ATOMICS-NEXT: adrp x2, var16 1234; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var16 1235; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas2_acq 1236; OUTLINE-ATOMICS-NEXT: cmp w0, w19, uxth 1237; OUTLINE-ATOMICS-NEXT: cset w0, eq 1238; OUTLINE-ATOMICS-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload 1239; OUTLINE-ATOMICS-NEXT: ret 1240 %pair = cmpxchg ptr @var16, i16 %wanted, i16 %new acquire acquire 1241 %success = extractvalue { i16, i1 } %pair, 1 1242 1243; CHECK-NOT: dmb 1244; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 1245; CHECK-NEXT: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 1246 1247; CHECK: casah w[[NEW:[0-9]+]], w1, [x[[ADDR]]] 1248; CHECK-NEXT: cmp w[[NEW]], w0, uxth 1249; CHECK-NEXT: cset w0, eq 1250; CHECK-NEXT: ret 1251 1252 ret i1 %success 1253} 1254 1255define dso_local i32 @test_atomic_cmpxchg_i32(i32 %wanted, i32 %new) nounwind { 1256; CHECK-LABEL: test_atomic_cmpxchg_i32: 1257; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i32: 1258; OUTLINE-ATOMICS: // %bb.0: 1259; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1260; OUTLINE-ATOMICS-NEXT: adrp x2, var32 1261; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var32 1262; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas4_acq 1263; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1264; OUTLINE-ATOMICS-NEXT: ret 1265 %pair = cmpxchg ptr @var32, i32 %wanted, i32 %new acquire acquire 1266 %old = extractvalue { i32, i1 } %pair, 0 1267 1268; CHECK-NOT: dmb 1269; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 1270; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 1271 1272; CHECK: casa w0, w1, [x[[ADDR]]] 1273; CHECK-NOT: dmb 1274 1275 ret i32 %old 1276} 1277 1278define dso_local i32 @test_atomic_cmpxchg_i32_monotonic_acquire(i32 %wanted, i32 %new) nounwind { 1279; CHECK-LABEL: test_atomic_cmpxchg_i32_monotonic_acquire: 1280; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i32_monotonic_acquire: 1281; OUTLINE-ATOMICS: // %bb.0: 1282; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1283; OUTLINE-ATOMICS-NEXT: adrp x2, var32 1284; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var32 1285; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas4_acq 1286; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1287; OUTLINE-ATOMICS-NEXT: ret 1288 %pair = cmpxchg ptr @var32, i32 %wanted, i32 %new monotonic acquire 1289 %old = extractvalue { i32, i1 } %pair, 0 1290 1291; CHECK-NOT: dmb 1292; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 1293; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 1294 1295; CHECK: casa w0, w1, [x[[ADDR]]] 1296; CHECK-NOT: dmb 1297 1298 ret i32 %old 1299} 1300 1301define dso_local i64 @test_atomic_cmpxchg_i64(i64 %wanted, i64 %new) nounwind { 1302; CHECK-LABEL: test_atomic_cmpxchg_i64: 1303; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i64: 1304; OUTLINE-ATOMICS: // %bb.0: 1305; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1306; OUTLINE-ATOMICS-NEXT: adrp x2, var64 1307; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var64 1308; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas8_acq 1309; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1310; OUTLINE-ATOMICS-NEXT: ret 1311 %pair = cmpxchg ptr @var64, i64 %wanted, i64 %new acquire acquire 1312 %old = extractvalue { i64, i1 } %pair, 0 1313 1314; CHECK-NOT: dmb 1315; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 1316; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 1317 1318; CHECK: casa x0, x1, [x[[ADDR]]] 1319; CHECK-NOT: dmb 1320 1321 ret i64 %old 1322} 1323 1324define dso_local i128 @test_atomic_cmpxchg_i128(i128 %wanted, i128 %new) nounwind { 1325; CHECK-LABEL: test_atomic_cmpxchg_i128: 1326; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i128: 1327; OUTLINE-ATOMICS: // %bb.0: 1328; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1329; OUTLINE-ATOMICS-NEXT: adrp x4, var128 1330; OUTLINE-ATOMICS-NEXT: add x4, x4, :lo12:var128 1331; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas16_acq 1332; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1333; OUTLINE-ATOMICS-NEXT: ret 1334 %pair = cmpxchg ptr @var128, i128 %wanted, i128 %new acquire acquire 1335 %old = extractvalue { i128, i1 } %pair, 0 1336 1337; CHECK-NOT: dmb 1338; CHECK: adrp [[TMPADDR:x[0-9]+]], var128 1339; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var128 1340 1341; CHECK: caspa x0, x1, x2, x3, [x[[ADDR]]] 1342; CHECK-NOT: dmb 1343 1344 ret i128 %old 1345} 1346 1347define dso_local i128 @test_atomic_cmpxchg_i128_monotonic_seqcst(i128 %wanted, i128 %new) nounwind { 1348; CHECK-LABEL: test_atomic_cmpxchg_i128_monotonic_seqcst: 1349; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i128_monotonic_seqcst: 1350; OUTLINE-ATOMICS: // %bb.0: 1351; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1352; OUTLINE-ATOMICS-NEXT: adrp x4, var128 1353; OUTLINE-ATOMICS-NEXT: add x4, x4, :lo12:var128 1354; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas16_acq_rel 1355; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1356; OUTLINE-ATOMICS-NEXT: ret 1357 %pair = cmpxchg ptr @var128, i128 %wanted, i128 %new monotonic seq_cst 1358 %old = extractvalue { i128, i1 } %pair, 0 1359 1360; CHECK-NOT: dmb 1361; CHECK: adrp [[TMPADDR:x[0-9]+]], var128 1362; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var128 1363 1364; CHECK: caspal x0, x1, x2, x3, [x[[ADDR]]] 1365; CHECK-NOT: dmb 1366 1367 ret i128 %old 1368} 1369 1370define dso_local i128 @test_atomic_cmpxchg_i128_release_acquire(i128 %wanted, i128 %new) nounwind { 1371; CHECK-LABEL: test_atomic_cmpxchg_i128_release_acquire: 1372; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i128_release_acquire: 1373; OUTLINE-ATOMICS: // %bb.0: 1374; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1375; OUTLINE-ATOMICS-NEXT: adrp x4, var128 1376; OUTLINE-ATOMICS-NEXT: add x4, x4, :lo12:var128 1377; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas16_acq_rel 1378; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1379; OUTLINE-ATOMICS-NEXT: ret 1380 %pair = cmpxchg ptr @var128, i128 %wanted, i128 %new release acquire 1381 %old = extractvalue { i128, i1 } %pair, 0 1382 1383; CHECK-NOT: dmb 1384; CHECK: adrp [[TMPADDR:x[0-9]+]], var128 1385; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var128 1386 1387; CHECK: caspal x0, x1, x2, x3, [x[[ADDR]]] 1388; CHECK-NOT: dmb 1389 1390 ret i128 %old 1391} 1392 1393define dso_local i8 @test_atomic_load_sub_i8(i8 %offset) nounwind { 1394; CHECK-LABEL: test_atomic_load_sub_i8: 1395; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8: 1396; OUTLINE-ATOMICS: // %bb.0: 1397; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1398; OUTLINE-ATOMICS-NEXT: neg w0, w0 1399; OUTLINE-ATOMICS-NEXT: adrp x1, var8 1400; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 1401; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel 1402; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1403; OUTLINE-ATOMICS-NEXT: ret 1404 %old = atomicrmw sub ptr @var8, i8 %offset seq_cst 1405; CHECK-NOT: dmb 1406; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 1407; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 1408; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 1409 1410; CHECK: ldaddalb w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1411; CHECK-NOT: dmb 1412 1413 ret i8 %old 1414} 1415 1416define dso_local i16 @test_atomic_load_sub_i16(i16 %offset) nounwind { 1417; CHECK-LABEL: test_atomic_load_sub_i16: 1418; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16: 1419; OUTLINE-ATOMICS: // %bb.0: 1420; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1421; OUTLINE-ATOMICS-NEXT: neg w0, w0 1422; OUTLINE-ATOMICS-NEXT: adrp x1, var16 1423; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 1424; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel 1425; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1426; OUTLINE-ATOMICS-NEXT: ret 1427 %old = atomicrmw sub ptr @var16, i16 %offset seq_cst 1428; CHECK-NOT: dmb 1429; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 1430; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 1431; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 1432 1433; CHECK: ldaddalh w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1434; CHECK-NOT: dmb 1435 1436 ret i16 %old 1437} 1438 1439define dso_local i32 @test_atomic_load_sub_i32(i32 %offset) nounwind { 1440; CHECK-LABEL: test_atomic_load_sub_i32: 1441; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32: 1442; OUTLINE-ATOMICS: // %bb.0: 1443; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1444; OUTLINE-ATOMICS-NEXT: neg w0, w0 1445; OUTLINE-ATOMICS-NEXT: adrp x1, var32 1446; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 1447; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel 1448; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1449; OUTLINE-ATOMICS-NEXT: ret 1450 %old = atomicrmw sub ptr @var32, i32 %offset seq_cst 1451; CHECK-NOT: dmb 1452; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 1453; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 1454; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 1455 1456; CHECK: ldaddal w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1457; CHECK-NOT: dmb 1458 1459 ret i32 %old 1460} 1461 1462define dso_local i64 @test_atomic_load_sub_i64(i64 %offset) nounwind { 1463; CHECK-LABEL: test_atomic_load_sub_i64: 1464; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64: 1465; OUTLINE-ATOMICS: // %bb.0: 1466; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1467; OUTLINE-ATOMICS-NEXT: neg x0, x0 1468; OUTLINE-ATOMICS-NEXT: adrp x1, var64 1469; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 1470; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel 1471; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1472; OUTLINE-ATOMICS-NEXT: ret 1473 %old = atomicrmw sub ptr @var64, i64 %offset seq_cst 1474; CHECK-NOT: dmb 1475; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] 1476; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 1477; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 1478 1479; CHECK: ldaddal x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]] 1480; CHECK-NOT: dmb 1481 1482 ret i64 %old 1483} 1484 1485define dso_local void @test_atomic_load_sub_i32_noret(i32 %offset) nounwind { 1486; CHECK-LABEL: test_atomic_load_sub_i32_noret: 1487; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_noret: 1488; OUTLINE-ATOMICS: // %bb.0: 1489; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1490; OUTLINE-ATOMICS-NEXT: neg w0, w0 1491; OUTLINE-ATOMICS-NEXT: adrp x1, var32 1492; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 1493; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel 1494; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1495; OUTLINE-ATOMICS-NEXT: ret 1496 atomicrmw sub ptr @var32, i32 %offset seq_cst 1497; CHECK-NOT: dmb 1498; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 1499; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 1500; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 1501 1502; CHECK: ldaddal w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1503; CHECK-NOT: dmb 1504 1505 ret void 1506} 1507 1508define dso_local void @test_atomic_load_sub_i64_noret(i64 %offset) nounwind { 1509; CHECK-LABEL: test_atomic_load_sub_i64_noret: 1510; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_noret: 1511; OUTLINE-ATOMICS: // %bb.0: 1512; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1513; OUTLINE-ATOMICS-NEXT: neg x0, x0 1514; OUTLINE-ATOMICS-NEXT: adrp x1, var64 1515; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 1516; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel 1517; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1518; OUTLINE-ATOMICS-NEXT: ret 1519 atomicrmw sub ptr @var64, i64 %offset seq_cst 1520; CHECK-NOT: dmb 1521; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] 1522; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 1523; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 1524 1525; CHECK: ldaddal x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]] 1526; CHECK-NOT: dmb 1527 1528 ret void 1529} 1530 1531define dso_local i8 @test_atomic_load_sub_i8_neg_imm() nounwind { 1532; CHECK-LABEL: test_atomic_load_sub_i8_neg_imm: 1533; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8_neg_imm: 1534; OUTLINE-ATOMICS: // %bb.0: 1535; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1536; OUTLINE-ATOMICS-NEXT: adrp x1, var8 1537; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 1538; OUTLINE-ATOMICS-NEXT: mov w0, #1 1539; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel 1540; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1541; OUTLINE-ATOMICS-NEXT: ret 1542 %old = atomicrmw sub ptr @var8, i8 -1 seq_cst 1543 1544; CHECK-NOT: dmb 1545; CHECK: mov w[[IMM:[0-9]+]], #1 1546; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 1547; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 1548; CHECK: ldaddalb w[[IMM]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1549; CHECK-NOT: dmb 1550 1551 ret i8 %old 1552} 1553 1554define dso_local i16 @test_atomic_load_sub_i16_neg_imm() nounwind { 1555; CHECK-LABEL: test_atomic_load_sub_i16_neg_imm: 1556; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16_neg_imm: 1557; OUTLINE-ATOMICS: // %bb.0: 1558; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1559; OUTLINE-ATOMICS-NEXT: adrp x1, var16 1560; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 1561; OUTLINE-ATOMICS-NEXT: mov w0, #1 1562; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel 1563; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1564; OUTLINE-ATOMICS-NEXT: ret 1565 %old = atomicrmw sub ptr @var16, i16 -1 seq_cst 1566 1567; CHECK-NOT: dmb 1568; CHECK: mov w[[IMM:[0-9]+]], #1 1569; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 1570; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 1571; CHECK: ldaddalh w[[IMM]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1572; CHECK-NOT: dmb 1573 1574 ret i16 %old 1575} 1576 1577define dso_local i32 @test_atomic_load_sub_i32_neg_imm() nounwind { 1578; CHECK-LABEL: test_atomic_load_sub_i32_neg_imm: 1579; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_neg_imm: 1580; OUTLINE-ATOMICS: // %bb.0: 1581; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1582; OUTLINE-ATOMICS-NEXT: adrp x1, var32 1583; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 1584; OUTLINE-ATOMICS-NEXT: mov w0, #1 1585; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel 1586; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1587; OUTLINE-ATOMICS-NEXT: ret 1588 %old = atomicrmw sub ptr @var32, i32 -1 seq_cst 1589 1590; CHECK-NOT: dmb 1591; CHECK: mov w[[IMM:[0-9]+]], #1 1592; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 1593; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 1594; CHECK: ldaddal w[[IMM]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1595; CHECK-NOT: dmb 1596 1597 ret i32 %old 1598} 1599 1600define dso_local i64 @test_atomic_load_sub_i64_neg_imm() nounwind { 1601; CHECK-LABEL: test_atomic_load_sub_i64_neg_imm: 1602; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_neg_imm: 1603; OUTLINE-ATOMICS: // %bb.0: 1604; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1605; OUTLINE-ATOMICS-NEXT: adrp x1, var64 1606; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 1607; OUTLINE-ATOMICS-NEXT: mov w0, #1 1608; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel 1609; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1610; OUTLINE-ATOMICS-NEXT: ret 1611 %old = atomicrmw sub ptr @var64, i64 -1 seq_cst 1612 1613; CHECK-NOT: dmb 1614; CHECK: mov w[[IMM:[0-9]+]], #1 1615; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 1616; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 1617; CHECK: ldaddal x[[IMM]], x[[NEW:[0-9]+]], [x[[ADDR]]] 1618; CHECK-NOT: dmb 1619 1620 ret i64 %old 1621} 1622 1623define dso_local i8 @test_atomic_load_sub_i8_neg_arg(i8 %offset) nounwind { 1624; CHECK-LABEL: test_atomic_load_sub_i8_neg_arg: 1625; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8_neg_arg: 1626; OUTLINE-ATOMICS: // %bb.0: 1627; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1628; OUTLINE-ATOMICS-NEXT: adrp x1, var8 1629; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 1630; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel 1631; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1632; OUTLINE-ATOMICS-NEXT: ret 1633 %neg = sub i8 0, %offset 1634 %old = atomicrmw sub ptr @var8, i8 %neg seq_cst 1635 1636; CHECK-NOT: dmb 1637; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 1638; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 1639; CHECK: ldaddalb w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 1640; CHECK-NOT: dmb 1641 1642 ret i8 %old 1643} 1644 1645define dso_local i16 @test_atomic_load_sub_i16_neg_arg(i16 %offset) nounwind { 1646; CHECK-LABEL: test_atomic_load_sub_i16_neg_arg: 1647; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16_neg_arg: 1648; OUTLINE-ATOMICS: // %bb.0: 1649; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1650; OUTLINE-ATOMICS-NEXT: adrp x1, var16 1651; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 1652; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel 1653; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1654; OUTLINE-ATOMICS-NEXT: ret 1655 %neg = sub i16 0, %offset 1656 %old = atomicrmw sub ptr @var16, i16 %neg seq_cst 1657 1658; CHECK-NOT: dmb 1659; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 1660; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 1661; CHECK: ldaddalh w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 1662; CHECK-NOT: dmb 1663 1664 ret i16 %old 1665} 1666 1667define dso_local i32 @test_atomic_load_sub_i32_neg_arg(i32 %offset) nounwind { 1668; CHECK-LABEL: test_atomic_load_sub_i32_neg_arg: 1669; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_neg_arg: 1670; OUTLINE-ATOMICS: // %bb.0: 1671; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1672; OUTLINE-ATOMICS-NEXT: adrp x1, var32 1673; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 1674; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel 1675; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1676; OUTLINE-ATOMICS-NEXT: ret 1677 %neg = sub i32 0, %offset 1678 %old = atomicrmw sub ptr @var32, i32 %neg seq_cst 1679 1680; CHECK-NOT: dmb 1681; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 1682; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 1683; CHECK: ldaddal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 1684; CHECK-NOT: dmb 1685 1686 ret i32 %old 1687} 1688 1689define dso_local i64 @test_atomic_load_sub_i64_neg_arg(i64 %offset) nounwind { 1690; CHECK-LABEL: test_atomic_load_sub_i64_neg_arg: 1691; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_neg_arg: 1692; OUTLINE-ATOMICS: // %bb.0: 1693; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1694; OUTLINE-ATOMICS-NEXT: adrp x1, var64 1695; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 1696; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel 1697; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1698; OUTLINE-ATOMICS-NEXT: ret 1699 %neg = sub i64 0, %offset 1700 %old = atomicrmw sub ptr @var64, i64 %neg seq_cst 1701 1702; CHECK-NOT: dmb 1703; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 1704; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 1705; CHECK: ldaddal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 1706; CHECK-NOT: dmb 1707 1708 ret i64 %old 1709} 1710 1711define dso_local i8 @test_atomic_load_and_i8(i8 %offset) nounwind { 1712; CHECK-LABEL: test_atomic_load_and_i8: 1713; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8: 1714; OUTLINE-ATOMICS: // %bb.0: 1715; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1716; OUTLINE-ATOMICS-NEXT: adrp x1, var8 1717; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 1718; OUTLINE-ATOMICS-NEXT: mvn w0, w0 1719; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_acq_rel 1720; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1721; OUTLINE-ATOMICS-NEXT: ret 1722 %old = atomicrmw and ptr @var8, i8 %offset seq_cst 1723; CHECK-NOT: dmb 1724; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 1725; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 1726; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 1727 1728; CHECK: ldclralb w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1729; CHECK-NOT: dmb 1730 ret i8 %old 1731} 1732 1733define dso_local i16 @test_atomic_load_and_i16(i16 %offset) nounwind { 1734; CHECK-LABEL: test_atomic_load_and_i16: 1735; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16: 1736; OUTLINE-ATOMICS: // %bb.0: 1737; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1738; OUTLINE-ATOMICS-NEXT: adrp x1, var16 1739; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 1740; OUTLINE-ATOMICS-NEXT: mvn w0, w0 1741; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_acq_rel 1742; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1743; OUTLINE-ATOMICS-NEXT: ret 1744 %old = atomicrmw and ptr @var16, i16 %offset seq_cst 1745; CHECK-NOT: dmb 1746; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 1747; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 1748; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 1749 1750; CHECK: ldclralh w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1751; CHECK-NOT: dmb 1752 ret i16 %old 1753} 1754 1755define dso_local i32 @test_atomic_load_and_i32(i32 %offset) nounwind { 1756; CHECK-LABEL: test_atomic_load_and_i32: 1757; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32: 1758; OUTLINE-ATOMICS: // %bb.0: 1759; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1760; OUTLINE-ATOMICS-NEXT: adrp x1, var32 1761; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 1762; OUTLINE-ATOMICS-NEXT: mvn w0, w0 1763; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel 1764; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1765; OUTLINE-ATOMICS-NEXT: ret 1766 %old = atomicrmw and ptr @var32, i32 %offset seq_cst 1767; CHECK-NOT: dmb 1768; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 1769; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 1770; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 1771 1772; CHECK: ldclral w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1773; CHECK-NOT: dmb 1774 ret i32 %old 1775} 1776 1777define dso_local i64 @test_atomic_load_and_i64(i64 %offset) nounwind { 1778; CHECK-LABEL: test_atomic_load_and_i64: 1779; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64: 1780; OUTLINE-ATOMICS: // %bb.0: 1781; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1782; OUTLINE-ATOMICS-NEXT: adrp x1, var64 1783; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 1784; OUTLINE-ATOMICS-NEXT: mvn x0, x0 1785; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel 1786; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1787; OUTLINE-ATOMICS-NEXT: ret 1788 %old = atomicrmw and ptr @var64, i64 %offset seq_cst 1789; CHECK-NOT: dmb 1790; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] 1791; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 1792; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 1793 1794; CHECK: ldclral x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]] 1795; CHECK-NOT: dmb 1796 ret i64 %old 1797} 1798 1799define dso_local i8 @test_atomic_load_and_i8_inv_imm() nounwind { 1800; CHECK-LABEL: test_atomic_load_and_i8_inv_imm: 1801; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8_inv_imm: 1802; OUTLINE-ATOMICS: // %bb.0: 1803; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1804; OUTLINE-ATOMICS-NEXT: adrp x1, var8 1805; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 1806; OUTLINE-ATOMICS-NEXT: mov w0, #1 1807; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_acq_rel 1808; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1809; OUTLINE-ATOMICS-NEXT: ret 1810 %old = atomicrmw and ptr @var8, i8 -2 seq_cst 1811; CHECK-NOT: dmb 1812; CHECK: mov w[[CONST:[0-9]+]], #1 1813; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 1814; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 1815; CHECK: ldclralb w[[CONST]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1816; CHECK-NOT: dmb 1817 ret i8 %old 1818} 1819 1820define dso_local i16 @test_atomic_load_and_i16_inv_imm() nounwind { 1821; CHECK-LABEL: test_atomic_load_and_i16_inv_imm: 1822; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16_inv_imm: 1823; OUTLINE-ATOMICS: // %bb.0: 1824; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1825; OUTLINE-ATOMICS-NEXT: adrp x1, var16 1826; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 1827; OUTLINE-ATOMICS-NEXT: mov w0, #1 1828; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_acq_rel 1829; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1830; OUTLINE-ATOMICS-NEXT: ret 1831 %old = atomicrmw and ptr @var16, i16 -2 seq_cst 1832; CHECK-NOT: dmb 1833; CHECK: mov w[[CONST:[0-9]+]], #1 1834; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 1835; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 1836; CHECK: ldclralh w[[CONST]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1837; CHECK-NOT: dmb 1838 ret i16 %old 1839} 1840 1841define dso_local i32 @test_atomic_load_and_i32_inv_imm() nounwind { 1842; CHECK-LABEL: test_atomic_load_and_i32_inv_imm: 1843; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_inv_imm: 1844; OUTLINE-ATOMICS: // %bb.0: 1845; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1846; OUTLINE-ATOMICS-NEXT: adrp x1, var32 1847; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 1848; OUTLINE-ATOMICS-NEXT: mov w0, #1 1849; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel 1850; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1851; OUTLINE-ATOMICS-NEXT: ret 1852 %old = atomicrmw and ptr @var32, i32 -2 seq_cst 1853; CHECK-NOT: dmb 1854; CHECK: mov w[[CONST:[0-9]+]], #1 1855; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 1856; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 1857; CHECK: ldclral w[[CONST]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1858; CHECK-NOT: dmb 1859 ret i32 %old 1860} 1861 1862define dso_local i64 @test_atomic_load_and_i64_inv_imm() nounwind { 1863; CHECK-LABEL: test_atomic_load_and_i64_inv_imm: 1864; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_inv_imm: 1865; OUTLINE-ATOMICS: // %bb.0: 1866; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1867; OUTLINE-ATOMICS-NEXT: adrp x1, var64 1868; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 1869; OUTLINE-ATOMICS-NEXT: mov w0, #1 1870; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel 1871; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1872; OUTLINE-ATOMICS-NEXT: ret 1873 %old = atomicrmw and ptr @var64, i64 -2 seq_cst 1874; CHECK-NOT: dmb 1875; CHECK: mov w[[CONST:[0-9]+]], #1 1876; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 1877; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 1878; CHECK: ldclral x[[CONST]], x[[NEW:[0-9]+]], [x[[ADDR]]] 1879; CHECK-NOT: dmb 1880 ret i64 %old 1881} 1882 1883define dso_local i8 @test_atomic_load_and_i8_inv_arg(i8 %offset) nounwind { 1884; CHECK-LABEL: test_atomic_load_and_i8_inv_arg: 1885; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8_inv_arg: 1886; OUTLINE-ATOMICS: // %bb.0: 1887; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1888; OUTLINE-ATOMICS-NEXT: adrp x1, var8 1889; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 1890; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_acq_rel 1891; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1892; OUTLINE-ATOMICS-NEXT: ret 1893 %inv = xor i8 %offset, -1 1894 %old = atomicrmw and ptr @var8, i8 %inv seq_cst 1895; CHECK-NOT: dmb 1896; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 1897; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 1898; CHECK: ldclralb w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 1899; CHECK-NOT: dmb 1900 ret i8 %old 1901} 1902 1903define dso_local i16 @test_atomic_load_and_i16_inv_arg(i16 %offset) nounwind { 1904; CHECK-LABEL: test_atomic_load_and_i16_inv_arg: 1905; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16_inv_arg: 1906; OUTLINE-ATOMICS: // %bb.0: 1907; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1908; OUTLINE-ATOMICS-NEXT: adrp x1, var16 1909; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 1910; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_acq_rel 1911; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1912; OUTLINE-ATOMICS-NEXT: ret 1913 %inv = xor i16 %offset, -1 1914 %old = atomicrmw and ptr @var16, i16 %inv seq_cst 1915; CHECK-NOT: dmb 1916; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 1917; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 1918; CHECK: ldclralh w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 1919; CHECK-NOT: dmb 1920 ret i16 %old 1921} 1922 1923define dso_local i32 @test_atomic_load_and_i32_inv_arg(i32 %offset) nounwind { 1924; CHECK-LABEL: test_atomic_load_and_i32_inv_arg: 1925; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_inv_arg: 1926; OUTLINE-ATOMICS: // %bb.0: 1927; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1928; OUTLINE-ATOMICS-NEXT: adrp x1, var32 1929; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 1930; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel 1931; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1932; OUTLINE-ATOMICS-NEXT: ret 1933 %inv = xor i32 %offset, -1 1934 %old = atomicrmw and ptr @var32, i32 %inv seq_cst 1935; CHECK-NOT: dmb 1936; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 1937; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 1938; CHECK: ldclral w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 1939; CHECK-NOT: dmb 1940 ret i32 %old 1941} 1942 1943define dso_local i64 @test_atomic_load_and_i64_inv_arg(i64 %offset) nounwind { 1944; CHECK-LABEL: test_atomic_load_and_i64_inv_arg: 1945; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_inv_arg: 1946; OUTLINE-ATOMICS: // %bb.0: 1947; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1948; OUTLINE-ATOMICS-NEXT: adrp x1, var64 1949; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 1950; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel 1951; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1952; OUTLINE-ATOMICS-NEXT: ret 1953 %inv = xor i64 %offset, -1 1954 %old = atomicrmw and ptr @var64, i64 %inv seq_cst 1955; CHECK-NOT: dmb 1956; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 1957; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 1958; CHECK: ldclral x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 1959; CHECK-NOT: dmb 1960 ret i64 %old 1961} 1962 1963define dso_local void @test_atomic_load_and_i32_noret(i32 %offset) nounwind { 1964; CHECK-LABEL: test_atomic_load_and_i32_noret: 1965; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_noret: 1966; OUTLINE-ATOMICS: // %bb.0: 1967; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1968; OUTLINE-ATOMICS-NEXT: adrp x1, var32 1969; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 1970; OUTLINE-ATOMICS-NEXT: mvn w0, w0 1971; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel 1972; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1973; OUTLINE-ATOMICS-NEXT: ret 1974 atomicrmw and ptr @var32, i32 %offset seq_cst 1975; CHECK-NOT: dmb 1976; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 1977; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 1978; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 1979 1980; CHECK: ldclral w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 1981; CHECK-NOT: dmb 1982 ret void 1983} 1984 1985define dso_local void @test_atomic_load_and_i64_noret(i64 %offset) nounwind { 1986; CHECK-LABEL: test_atomic_load_and_i64_noret: 1987; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_noret: 1988; OUTLINE-ATOMICS: // %bb.0: 1989; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 1990; OUTLINE-ATOMICS-NEXT: adrp x1, var64 1991; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 1992; OUTLINE-ATOMICS-NEXT: mvn x0, x0 1993; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel 1994; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 1995; OUTLINE-ATOMICS-NEXT: ret 1996 atomicrmw and ptr @var64, i64 %offset seq_cst 1997; CHECK-NOT: dmb 1998; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] 1999; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2000; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2001 2002; CHECK: ldclral x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]] 2003; CHECK-NOT: dmb 2004 ret void 2005} 2006 2007define dso_local i8 @test_atomic_load_add_i8_acq_rel(i8 %offset) nounwind { 2008; CHECK-LABEL: test_atomic_load_add_i8_acq_rel: 2009; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i8_acq_rel: 2010; OUTLINE-ATOMICS: // %bb.0: 2011; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2012; OUTLINE-ATOMICS-NEXT: adrp x1, var8 2013; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 2014; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel 2015; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2016; OUTLINE-ATOMICS-NEXT: ret 2017 %old = atomicrmw add ptr @var8, i8 %offset acq_rel 2018; CHECK-NOT: dmb 2019; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 2020; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 2021 2022; CHECK: ldaddalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2023; CHECK-NOT: dmb 2024 2025 ret i8 %old 2026} 2027 2028define dso_local i16 @test_atomic_load_add_i16_acq_rel(i16 %offset) nounwind { 2029; CHECK-LABEL: test_atomic_load_add_i16_acq_rel: 2030; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i16_acq_rel: 2031; OUTLINE-ATOMICS: // %bb.0: 2032; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2033; OUTLINE-ATOMICS-NEXT: adrp x1, var16 2034; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 2035; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel 2036; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2037; OUTLINE-ATOMICS-NEXT: ret 2038 %old = atomicrmw add ptr @var16, i16 %offset acq_rel 2039; CHECK-NOT: dmb 2040; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 2041; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 2042 2043; CHECK: ldaddalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2044; CHECK-NOT: dmb 2045 2046 ret i16 %old 2047} 2048 2049define dso_local i32 @test_atomic_load_add_i32_acq_rel(i32 %offset) nounwind { 2050; CHECK-LABEL: test_atomic_load_add_i32_acq_rel: 2051; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_acq_rel: 2052; OUTLINE-ATOMICS: // %bb.0: 2053; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2054; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2055; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2056; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel 2057; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2058; OUTLINE-ATOMICS-NEXT: ret 2059 %old = atomicrmw add ptr @var32, i32 %offset acq_rel 2060; CHECK-NOT: dmb 2061; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2062; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2063 2064; CHECK: ldaddal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2065; CHECK-NOT: dmb 2066 2067 ret i32 %old 2068} 2069 2070define dso_local i64 @test_atomic_load_add_i64_acq_rel(i64 %offset) nounwind { 2071; CHECK-LABEL: test_atomic_load_add_i64_acq_rel: 2072; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_acq_rel: 2073; OUTLINE-ATOMICS: // %bb.0: 2074; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2075; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2076; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2077; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel 2078; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2079; OUTLINE-ATOMICS-NEXT: ret 2080 %old = atomicrmw add ptr @var64, i64 %offset acq_rel 2081; CHECK-NOT: dmb 2082; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2083; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2084 2085; CHECK: ldaddal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 2086; CHECK-NOT: dmb 2087 2088 ret i64 %old 2089} 2090 2091define dso_local void @test_atomic_load_add_i32_noret_acq_rel(i32 %offset) nounwind { 2092; CHECK-LABEL: test_atomic_load_add_i32_noret_acq_rel: 2093; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_noret_acq_rel: 2094; OUTLINE-ATOMICS: // %bb.0: 2095; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2096; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2097; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2098; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel 2099; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2100; OUTLINE-ATOMICS-NEXT: ret 2101 atomicrmw add ptr @var32, i32 %offset acq_rel 2102; CHECK-NOT: dmb 2103; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2104; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2105 2106; CHECK: ldaddal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 2107; CHECK-NOT: dmb 2108 ret void 2109} 2110 2111define dso_local void @test_atomic_load_add_i64_noret_acq_rel(i64 %offset) nounwind { 2112; CHECK-LABEL: test_atomic_load_add_i64_noret_acq_rel: 2113; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_noret_acq_rel: 2114; OUTLINE-ATOMICS: // %bb.0: 2115; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2116; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2117; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2118; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel 2119; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2120; OUTLINE-ATOMICS-NEXT: ret 2121 atomicrmw add ptr @var64, i64 %offset acq_rel 2122; CHECK-NOT: dmb 2123; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2124; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2125 2126; CHECK: ldaddal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 2127; CHECK-NOT: dmb 2128 ret void 2129} 2130 2131define dso_local i8 @test_atomic_load_add_i8_acquire(i8 %offset) nounwind { 2132; CHECK-LABEL: test_atomic_load_add_i8_acquire: 2133; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i8_acquire: 2134; OUTLINE-ATOMICS: // %bb.0: 2135; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2136; OUTLINE-ATOMICS-NEXT: adrp x1, var8 2137; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 2138; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq 2139; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2140; OUTLINE-ATOMICS-NEXT: ret 2141 %old = atomicrmw add ptr @var8, i8 %offset acquire 2142; CHECK-NOT: dmb 2143; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 2144; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 2145 2146; CHECK: ldaddab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2147; CHECK-NOT: dmb 2148 2149 ret i8 %old 2150} 2151 2152define dso_local i16 @test_atomic_load_add_i16_acquire(i16 %offset) nounwind { 2153; CHECK-LABEL: test_atomic_load_add_i16_acquire: 2154; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i16_acquire: 2155; OUTLINE-ATOMICS: // %bb.0: 2156; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2157; OUTLINE-ATOMICS-NEXT: adrp x1, var16 2158; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 2159; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq 2160; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2161; OUTLINE-ATOMICS-NEXT: ret 2162 %old = atomicrmw add ptr @var16, i16 %offset acquire 2163; CHECK-NOT: dmb 2164; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 2165; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 2166 2167; CHECK: ldaddah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2168; CHECK-NOT: dmb 2169 2170 ret i16 %old 2171} 2172 2173define dso_local i32 @test_atomic_load_add_i32_acquire(i32 %offset) nounwind { 2174; CHECK-LABEL: test_atomic_load_add_i32_acquire: 2175; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_acquire: 2176; OUTLINE-ATOMICS: // %bb.0: 2177; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2178; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2179; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2180; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq 2181; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2182; OUTLINE-ATOMICS-NEXT: ret 2183 %old = atomicrmw add ptr @var32, i32 %offset acquire 2184; CHECK-NOT: dmb 2185; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2186; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2187 2188; CHECK: ldadda w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2189; CHECK-NOT: dmb 2190 2191 ret i32 %old 2192} 2193 2194define dso_local i64 @test_atomic_load_add_i64_acquire(i64 %offset) nounwind { 2195; CHECK-LABEL: test_atomic_load_add_i64_acquire: 2196; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_acquire: 2197; OUTLINE-ATOMICS: // %bb.0: 2198; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2199; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2200; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2201; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq 2202; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2203; OUTLINE-ATOMICS-NEXT: ret 2204 %old = atomicrmw add ptr @var64, i64 %offset acquire 2205; CHECK-NOT: dmb 2206; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2207; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2208 2209; CHECK: ldadda x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 2210; CHECK-NOT: dmb 2211 2212 ret i64 %old 2213} 2214 2215define dso_local void @test_atomic_load_add_i32_noret_acquire(i32 %offset) nounwind { 2216; CHECK-LABEL: test_atomic_load_add_i32_noret_acquire: 2217; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_noret_acquire: 2218; OUTLINE-ATOMICS: // %bb.0: 2219; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2220; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2221; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2222; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq 2223; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2224; OUTLINE-ATOMICS-NEXT: ret 2225 atomicrmw add ptr @var32, i32 %offset acquire 2226; CHECK-NOT: dmb 2227; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2228; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2229 2230; CHECK: ldadda w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 2231; CHECK-NOT: dmb 2232 ret void 2233} 2234 2235define dso_local void @test_atomic_load_add_i64_noret_acquire(i64 %offset) nounwind { 2236; CHECK-LABEL: test_atomic_load_add_i64_noret_acquire: 2237; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_noret_acquire: 2238; OUTLINE-ATOMICS: // %bb.0: 2239; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2240; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2241; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2242; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq 2243; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2244; OUTLINE-ATOMICS-NEXT: ret 2245 atomicrmw add ptr @var64, i64 %offset acquire 2246; CHECK-NOT: dmb 2247; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2248; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2249 2250; CHECK: ldadda x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 2251; CHECK-NOT: dmb 2252 ret void 2253} 2254 2255define dso_local i8 @test_atomic_load_add_i8_monotonic(i8 %offset) nounwind { 2256; CHECK-LABEL: test_atomic_load_add_i8_monotonic: 2257; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i8_monotonic: 2258; OUTLINE-ATOMICS: // %bb.0: 2259; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2260; OUTLINE-ATOMICS-NEXT: adrp x1, var8 2261; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 2262; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_relax 2263; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2264; OUTLINE-ATOMICS-NEXT: ret 2265 %old = atomicrmw add ptr @var8, i8 %offset monotonic 2266; CHECK-NOT: dmb 2267; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 2268; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 2269 2270; CHECK: ldaddb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2271; CHECK-NOT: dmb 2272 2273 ret i8 %old 2274} 2275 2276define dso_local i16 @test_atomic_load_add_i16_monotonic(i16 %offset) nounwind { 2277; CHECK-LABEL: test_atomic_load_add_i16_monotonic: 2278; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i16_monotonic: 2279; OUTLINE-ATOMICS: // %bb.0: 2280; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2281; OUTLINE-ATOMICS-NEXT: adrp x1, var16 2282; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 2283; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_relax 2284; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2285; OUTLINE-ATOMICS-NEXT: ret 2286 %old = atomicrmw add ptr @var16, i16 %offset monotonic 2287; CHECK-NOT: dmb 2288; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 2289; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 2290 2291; CHECK: ldaddh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2292; CHECK-NOT: dmb 2293 2294 ret i16 %old 2295} 2296 2297define dso_local i32 @test_atomic_load_add_i32_monotonic(i32 %offset) nounwind { 2298; CHECK-LABEL: test_atomic_load_add_i32_monotonic: 2299; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_monotonic: 2300; OUTLINE-ATOMICS: // %bb.0: 2301; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2302; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2303; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2304; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_relax 2305; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2306; OUTLINE-ATOMICS-NEXT: ret 2307 %old = atomicrmw add ptr @var32, i32 %offset monotonic 2308; CHECK-NOT: dmb 2309; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2310; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2311 2312; CHECK: ldadd w[[OLD:[0-9]+]], w[[NEW:[0-9,a-z]+]], [x[[ADDR]]] 2313; CHECK-NOT: dmb 2314 2315 ret i32 %old 2316} 2317 2318define dso_local i64 @test_atomic_load_add_i64_monotonic(i64 %offset) nounwind { 2319; CHECK-LABEL: test_atomic_load_add_i64_monotonic: 2320; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_monotonic: 2321; OUTLINE-ATOMICS: // %bb.0: 2322; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2323; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2324; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2325; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_relax 2326; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2327; OUTLINE-ATOMICS-NEXT: ret 2328 %old = atomicrmw add ptr @var64, i64 %offset monotonic 2329; CHECK-NOT: dmb 2330; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2331; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2332 2333; CHECK: ldadd x[[OLD:[0-9]+]], x[[NEW:[0-9,a-z]+]], [x[[ADDR]]] 2334; CHECK-NOT: dmb 2335 2336 ret i64 %old 2337} 2338 2339define dso_local void @test_atomic_load_add_i32_noret_monotonic(i32 %offset) nounwind { 2340; CHECK-LABEL: test_atomic_load_add_i32_noret_monotonic: 2341; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_noret_monotonic: 2342; OUTLINE-ATOMICS: // %bb.0: 2343; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2344; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2345; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2346; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_relax 2347; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2348; OUTLINE-ATOMICS-NEXT: ret 2349 atomicrmw add ptr @var32, i32 %offset monotonic 2350; CHECK-NOT: dmb 2351; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2352; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2353 2354; CHECK: ldadd w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] 2355; CHECK-NOT: dmb 2356 ret void 2357} 2358 2359define dso_local void @test_atomic_load_add_i64_noret_monotonic(i64 %offset) nounwind { 2360; CHECK-LABEL: test_atomic_load_add_i64_noret_monotonic: 2361; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_noret_monotonic: 2362; OUTLINE-ATOMICS: // %bb.0: 2363; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2364; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2365; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2366; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_relax 2367; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2368; OUTLINE-ATOMICS-NEXT: ret 2369 atomicrmw add ptr @var64, i64 %offset monotonic 2370; CHECK-NOT: dmb 2371; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2372; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2373 2374; CHECK: ldadd x{{[0-9]}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] 2375; CHECK-NOT: dmb 2376 ret void 2377} 2378 2379define dso_local i8 @test_atomic_load_add_i8_release(i8 %offset) nounwind { 2380; CHECK-LABEL: test_atomic_load_add_i8_release: 2381; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i8_release: 2382; OUTLINE-ATOMICS: // %bb.0: 2383; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2384; OUTLINE-ATOMICS-NEXT: adrp x1, var8 2385; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 2386; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_rel 2387; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2388; OUTLINE-ATOMICS-NEXT: ret 2389 %old = atomicrmw add ptr @var8, i8 %offset release 2390; CHECK-NOT: dmb 2391; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 2392; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 2393 2394; CHECK: ldaddlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2395; CHECK-NOT: dmb 2396 2397 ret i8 %old 2398} 2399 2400define dso_local i16 @test_atomic_load_add_i16_release(i16 %offset) nounwind { 2401; CHECK-LABEL: test_atomic_load_add_i16_release: 2402; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i16_release: 2403; OUTLINE-ATOMICS: // %bb.0: 2404; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2405; OUTLINE-ATOMICS-NEXT: adrp x1, var16 2406; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 2407; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_rel 2408; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2409; OUTLINE-ATOMICS-NEXT: ret 2410 %old = atomicrmw add ptr @var16, i16 %offset release 2411; CHECK-NOT: dmb 2412; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 2413; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 2414 2415; CHECK: ldaddlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2416; CHECK-NOT: dmb 2417 2418 ret i16 %old 2419} 2420 2421define dso_local i32 @test_atomic_load_add_i32_release(i32 %offset) nounwind { 2422; CHECK-LABEL: test_atomic_load_add_i32_release: 2423; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_release: 2424; OUTLINE-ATOMICS: // %bb.0: 2425; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2426; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2427; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2428; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_rel 2429; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2430; OUTLINE-ATOMICS-NEXT: ret 2431 %old = atomicrmw add ptr @var32, i32 %offset release 2432; CHECK-NOT: dmb 2433; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2434; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2435 2436; CHECK: ldaddl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2437; CHECK-NOT: dmb 2438 2439 ret i32 %old 2440} 2441 2442define dso_local i64 @test_atomic_load_add_i64_release(i64 %offset) nounwind { 2443; CHECK-LABEL: test_atomic_load_add_i64_release: 2444; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_release: 2445; OUTLINE-ATOMICS: // %bb.0: 2446; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2447; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2448; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2449; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_rel 2450; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2451; OUTLINE-ATOMICS-NEXT: ret 2452 %old = atomicrmw add ptr @var64, i64 %offset release 2453; CHECK-NOT: dmb 2454; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2455; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2456 2457; CHECK: ldaddl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 2458; CHECK-NOT: dmb 2459 2460 ret i64 %old 2461} 2462 2463define dso_local void @test_atomic_load_add_i32_noret_release(i32 %offset) nounwind { 2464; CHECK-LABEL: test_atomic_load_add_i32_noret_release: 2465; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_noret_release: 2466; OUTLINE-ATOMICS: // %bb.0: 2467; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2468; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2469; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2470; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_rel 2471; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2472; OUTLINE-ATOMICS-NEXT: ret 2473 atomicrmw add ptr @var32, i32 %offset release 2474; CHECK-NOT: dmb 2475; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2476; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2477 2478; CHECK: ldaddl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] 2479; CHECK-NOT: dmb 2480 ret void 2481} 2482 2483define dso_local void @test_atomic_load_add_i64_noret_release(i64 %offset) nounwind { 2484; CHECK-LABEL: test_atomic_load_add_i64_noret_release: 2485; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_noret_release: 2486; OUTLINE-ATOMICS: // %bb.0: 2487; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2488; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2489; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2490; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_rel 2491; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2492; OUTLINE-ATOMICS-NEXT: ret 2493 atomicrmw add ptr @var64, i64 %offset release 2494; CHECK-NOT: dmb 2495; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2496; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2497 2498; CHECK: ldaddl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] 2499; CHECK-NOT: dmb 2500 ret void 2501} 2502 2503define dso_local i8 @test_atomic_load_add_i8_seq_cst(i8 %offset) nounwind { 2504; CHECK-LABEL: test_atomic_load_add_i8_seq_cst: 2505; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i8_seq_cst: 2506; OUTLINE-ATOMICS: // %bb.0: 2507; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2508; OUTLINE-ATOMICS-NEXT: adrp x1, var8 2509; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 2510; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel 2511; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2512; OUTLINE-ATOMICS-NEXT: ret 2513 %old = atomicrmw add ptr @var8, i8 %offset seq_cst 2514; CHECK-NOT: dmb 2515; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 2516; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 2517 2518; CHECK: ldaddalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2519; CHECK-NOT: dmb 2520 2521 ret i8 %old 2522} 2523 2524define dso_local i16 @test_atomic_load_add_i16_seq_cst(i16 %offset) nounwind { 2525; CHECK-LABEL: test_atomic_load_add_i16_seq_cst: 2526; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i16_seq_cst: 2527; OUTLINE-ATOMICS: // %bb.0: 2528; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2529; OUTLINE-ATOMICS-NEXT: adrp x1, var16 2530; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 2531; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel 2532; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2533; OUTLINE-ATOMICS-NEXT: ret 2534 %old = atomicrmw add ptr @var16, i16 %offset seq_cst 2535; CHECK-NOT: dmb 2536; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 2537; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 2538 2539; CHECK: ldaddalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2540; CHECK-NOT: dmb 2541 2542 ret i16 %old 2543} 2544 2545define dso_local i32 @test_atomic_load_add_i32_seq_cst(i32 %offset) nounwind { 2546; CHECK-LABEL: test_atomic_load_add_i32_seq_cst: 2547; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_seq_cst: 2548; OUTLINE-ATOMICS: // %bb.0: 2549; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2550; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2551; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2552; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel 2553; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2554; OUTLINE-ATOMICS-NEXT: ret 2555 %old = atomicrmw add ptr @var32, i32 %offset seq_cst 2556; CHECK-NOT: dmb 2557; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2558; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2559 2560; CHECK: ldaddal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2561; CHECK-NOT: dmb 2562 2563 ret i32 %old 2564} 2565 2566define dso_local i64 @test_atomic_load_add_i64_seq_cst(i64 %offset) nounwind { 2567; CHECK-LABEL: test_atomic_load_add_i64_seq_cst: 2568; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_seq_cst: 2569; OUTLINE-ATOMICS: // %bb.0: 2570; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2571; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2572; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2573; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel 2574; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2575; OUTLINE-ATOMICS-NEXT: ret 2576 %old = atomicrmw add ptr @var64, i64 %offset seq_cst 2577; CHECK-NOT: dmb 2578; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2579; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2580 2581; CHECK: ldaddal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 2582; CHECK-NOT: dmb 2583 2584 ret i64 %old 2585} 2586 2587define dso_local void @test_atomic_load_add_i32_noret_seq_cst(i32 %offset) nounwind { 2588; CHECK-LABEL: test_atomic_load_add_i32_noret_seq_cst: 2589; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i32_noret_seq_cst: 2590; OUTLINE-ATOMICS: // %bb.0: 2591; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2592; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2593; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2594; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel 2595; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2596; OUTLINE-ATOMICS-NEXT: ret 2597 atomicrmw add ptr @var32, i32 %offset seq_cst 2598; CHECK-NOT: dmb 2599; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2600; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2601 2602; CHECK: ldaddal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 2603; CHECK-NOT: dmb 2604 ret void 2605} 2606 2607define dso_local void @test_atomic_load_add_i64_noret_seq_cst(i64 %offset) nounwind { 2608; CHECK-LABEL: test_atomic_load_add_i64_noret_seq_cst: 2609; OUTLINE-ATOMICS-LABEL: test_atomic_load_add_i64_noret_seq_cst: 2610; OUTLINE-ATOMICS: // %bb.0: 2611; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2612; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2613; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2614; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel 2615; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2616; OUTLINE-ATOMICS-NEXT: ret 2617 atomicrmw add ptr @var64, i64 %offset seq_cst 2618; CHECK-NOT: dmb 2619; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2620; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2621 2622; CHECK: ldaddal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 2623; CHECK-NOT: dmb 2624 ret void 2625} 2626 2627define dso_local i8 @test_atomic_load_and_i8_acq_rel(i8 %offset) nounwind { 2628; CHECK-LABEL: test_atomic_load_and_i8_acq_rel: 2629; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8_acq_rel: 2630; OUTLINE-ATOMICS: // %bb.0: 2631; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2632; OUTLINE-ATOMICS-NEXT: adrp x1, var8 2633; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 2634; OUTLINE-ATOMICS-NEXT: mvn w0, w0 2635; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_acq_rel 2636; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2637; OUTLINE-ATOMICS-NEXT: ret 2638 %old = atomicrmw and ptr @var8, i8 %offset acq_rel 2639; CHECK-NOT: dmb 2640; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 2641; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 2642; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 2643 2644; CHECK: ldclralb w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2645; CHECK-NOT: dmb 2646 ret i8 %old 2647} 2648 2649define dso_local i16 @test_atomic_load_and_i16_acq_rel(i16 %offset) nounwind { 2650; CHECK-LABEL: test_atomic_load_and_i16_acq_rel: 2651; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16_acq_rel: 2652; OUTLINE-ATOMICS: // %bb.0: 2653; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2654; OUTLINE-ATOMICS-NEXT: adrp x1, var16 2655; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 2656; OUTLINE-ATOMICS-NEXT: mvn w0, w0 2657; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_acq_rel 2658; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2659; OUTLINE-ATOMICS-NEXT: ret 2660 %old = atomicrmw and ptr @var16, i16 %offset acq_rel 2661; CHECK-NOT: dmb 2662; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 2663; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 2664; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 2665 2666; CHECK: ldclralh w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2667; CHECK-NOT: dmb 2668 ret i16 %old 2669} 2670 2671define dso_local i32 @test_atomic_load_and_i32_acq_rel(i32 %offset) nounwind { 2672; CHECK-LABEL: test_atomic_load_and_i32_acq_rel: 2673; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_acq_rel: 2674; OUTLINE-ATOMICS: // %bb.0: 2675; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2676; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2677; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2678; OUTLINE-ATOMICS-NEXT: mvn w0, w0 2679; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel 2680; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2681; OUTLINE-ATOMICS-NEXT: ret 2682 %old = atomicrmw and ptr @var32, i32 %offset acq_rel 2683; CHECK-NOT: dmb 2684; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 2685; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2686; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2687 2688; CHECK: ldclral w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2689; CHECK-NOT: dmb 2690 ret i32 %old 2691} 2692 2693define dso_local i64 @test_atomic_load_and_i64_acq_rel(i64 %offset) nounwind { 2694; CHECK-LABEL: test_atomic_load_and_i64_acq_rel: 2695; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_acq_rel: 2696; OUTLINE-ATOMICS: // %bb.0: 2697; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2698; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2699; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2700; OUTLINE-ATOMICS-NEXT: mvn x0, x0 2701; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel 2702; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2703; OUTLINE-ATOMICS-NEXT: ret 2704 %old = atomicrmw and ptr @var64, i64 %offset acq_rel 2705; CHECK-NOT: dmb 2706; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] 2707; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2708; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2709 2710; CHECK: ldclral x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]] 2711; CHECK-NOT: dmb 2712 ret i64 %old 2713} 2714 2715define dso_local void @test_atomic_load_and_i32_noret_acq_rel(i32 %offset) nounwind { 2716; CHECK-LABEL: test_atomic_load_and_i32_noret_acq_rel: 2717; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_noret_acq_rel: 2718; OUTLINE-ATOMICS: // %bb.0: 2719; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2720; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2721; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2722; OUTLINE-ATOMICS-NEXT: mvn w0, w0 2723; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel 2724; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2725; OUTLINE-ATOMICS-NEXT: ret 2726 atomicrmw and ptr @var32, i32 %offset acq_rel 2727; CHECK-NOT: dmb 2728; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 2729; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2730; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2731 2732; CHECK: ldclral w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2733; CHECK-NOT: dmb 2734 ret void 2735} 2736 2737define dso_local void @test_atomic_load_and_i64_noret_acq_rel(i64 %offset) nounwind { 2738; CHECK-LABEL: test_atomic_load_and_i64_noret_acq_rel: 2739; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_noret_acq_rel: 2740; OUTLINE-ATOMICS: // %bb.0: 2741; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2742; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2743; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2744; OUTLINE-ATOMICS-NEXT: mvn x0, x0 2745; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel 2746; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2747; OUTLINE-ATOMICS-NEXT: ret 2748 atomicrmw and ptr @var64, i64 %offset acq_rel 2749; CHECK-NOT: dmb 2750; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] 2751; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2752; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2753 2754; CHECK: ldclral x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]] 2755; CHECK-NOT: dmb 2756 ret void 2757} 2758 2759define dso_local i8 @test_atomic_load_and_i8_acquire(i8 %offset) nounwind { 2760; CHECK-LABEL: test_atomic_load_and_i8_acquire: 2761; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8_acquire: 2762; OUTLINE-ATOMICS: // %bb.0: 2763; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2764; OUTLINE-ATOMICS-NEXT: adrp x1, var8 2765; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 2766; OUTLINE-ATOMICS-NEXT: mvn w0, w0 2767; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_acq 2768; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2769; OUTLINE-ATOMICS-NEXT: ret 2770 %old = atomicrmw and ptr @var8, i8 %offset acquire 2771; CHECK-NOT: dmb 2772; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 2773; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 2774; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 2775 2776; CHECK: ldclrab w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2777; CHECK-NOT: dmb 2778 ret i8 %old 2779} 2780 2781define dso_local i16 @test_atomic_load_and_i16_acquire(i16 %offset) nounwind { 2782; CHECK-LABEL: test_atomic_load_and_i16_acquire: 2783; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16_acquire: 2784; OUTLINE-ATOMICS: // %bb.0: 2785; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2786; OUTLINE-ATOMICS-NEXT: adrp x1, var16 2787; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 2788; OUTLINE-ATOMICS-NEXT: mvn w0, w0 2789; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_acq 2790; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2791; OUTLINE-ATOMICS-NEXT: ret 2792 %old = atomicrmw and ptr @var16, i16 %offset acquire 2793; CHECK-NOT: dmb 2794; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 2795; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 2796; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 2797 2798; CHECK: ldclrah w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2799; CHECK-NOT: dmb 2800 ret i16 %old 2801} 2802 2803define dso_local i32 @test_atomic_load_and_i32_acquire(i32 %offset) nounwind { 2804; CHECK-LABEL: test_atomic_load_and_i32_acquire: 2805; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_acquire: 2806; OUTLINE-ATOMICS: // %bb.0: 2807; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2808; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2809; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2810; OUTLINE-ATOMICS-NEXT: mvn w0, w0 2811; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq 2812; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2813; OUTLINE-ATOMICS-NEXT: ret 2814 %old = atomicrmw and ptr @var32, i32 %offset acquire 2815; CHECK-NOT: dmb 2816; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 2817; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2818; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2819 2820; CHECK: ldclra w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2821; CHECK-NOT: dmb 2822 ret i32 %old 2823} 2824 2825define dso_local i64 @test_atomic_load_and_i64_acquire(i64 %offset) nounwind { 2826; CHECK-LABEL: test_atomic_load_and_i64_acquire: 2827; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_acquire: 2828; OUTLINE-ATOMICS: // %bb.0: 2829; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2830; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2831; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2832; OUTLINE-ATOMICS-NEXT: mvn x0, x0 2833; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq 2834; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2835; OUTLINE-ATOMICS-NEXT: ret 2836 %old = atomicrmw and ptr @var64, i64 %offset acquire 2837; CHECK-NOT: dmb 2838; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] 2839; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2840; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2841 2842; CHECK: ldclra x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]] 2843; CHECK-NOT: dmb 2844 ret i64 %old 2845} 2846 2847define dso_local void @test_atomic_load_and_i32_noret_acquire(i32 %offset) nounwind { 2848; CHECK-LABEL: test_atomic_load_and_i32_noret_acquire: 2849; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_noret_acquire: 2850; OUTLINE-ATOMICS: // %bb.0: 2851; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2852; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2853; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2854; OUTLINE-ATOMICS-NEXT: mvn w0, w0 2855; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq 2856; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2857; OUTLINE-ATOMICS-NEXT: ret 2858 atomicrmw and ptr @var32, i32 %offset acquire 2859; CHECK-NOT: dmb 2860; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 2861; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2862; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2863 2864; CHECK: ldclra w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2865; CHECK-NOT: dmb 2866 ret void 2867} 2868 2869define dso_local void @test_atomic_load_and_i64_noret_acquire(i64 %offset) nounwind { 2870; CHECK-LABEL: test_atomic_load_and_i64_noret_acquire: 2871; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_noret_acquire: 2872; OUTLINE-ATOMICS: // %bb.0: 2873; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2874; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2875; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2876; OUTLINE-ATOMICS-NEXT: mvn x0, x0 2877; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq 2878; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2879; OUTLINE-ATOMICS-NEXT: ret 2880 atomicrmw and ptr @var64, i64 %offset acquire 2881; CHECK-NOT: dmb 2882; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] 2883; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2884; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2885 2886; CHECK: ldclra x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]] 2887; CHECK-NOT: dmb 2888 ret void 2889} 2890 2891define dso_local i8 @test_atomic_load_and_i8_monotonic(i8 %offset) nounwind { 2892; CHECK-LABEL: test_atomic_load_and_i8_monotonic: 2893; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8_monotonic: 2894; OUTLINE-ATOMICS: // %bb.0: 2895; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2896; OUTLINE-ATOMICS-NEXT: adrp x1, var8 2897; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 2898; OUTLINE-ATOMICS-NEXT: mvn w0, w0 2899; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_relax 2900; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2901; OUTLINE-ATOMICS-NEXT: ret 2902 %old = atomicrmw and ptr @var8, i8 %offset monotonic 2903; CHECK-NOT: dmb 2904; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 2905; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 2906; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 2907 2908; CHECK: ldclrb w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2909; CHECK-NOT: dmb 2910 ret i8 %old 2911} 2912 2913define dso_local i16 @test_atomic_load_and_i16_monotonic(i16 %offset) nounwind { 2914; CHECK-LABEL: test_atomic_load_and_i16_monotonic: 2915; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16_monotonic: 2916; OUTLINE-ATOMICS: // %bb.0: 2917; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2918; OUTLINE-ATOMICS-NEXT: adrp x1, var16 2919; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 2920; OUTLINE-ATOMICS-NEXT: mvn w0, w0 2921; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_relax 2922; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2923; OUTLINE-ATOMICS-NEXT: ret 2924 %old = atomicrmw and ptr @var16, i16 %offset monotonic 2925; CHECK-NOT: dmb 2926; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 2927; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 2928; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 2929 2930; CHECK: ldclrh w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2931; CHECK-NOT: dmb 2932 ret i16 %old 2933} 2934 2935define dso_local i32 @test_atomic_load_and_i32_monotonic(i32 %offset) nounwind { 2936; CHECK-LABEL: test_atomic_load_and_i32_monotonic: 2937; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_monotonic: 2938; OUTLINE-ATOMICS: // %bb.0: 2939; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2940; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2941; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2942; OUTLINE-ATOMICS-NEXT: mvn w0, w0 2943; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_relax 2944; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2945; OUTLINE-ATOMICS-NEXT: ret 2946 %old = atomicrmw and ptr @var32, i32 %offset monotonic 2947; CHECK-NOT: dmb 2948; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 2949; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2950; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2951 2952; CHECK: ldclr w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 2953; CHECK-NOT: dmb 2954 ret i32 %old 2955} 2956 2957define dso_local i64 @test_atomic_load_and_i64_monotonic(i64 %offset) nounwind { 2958; CHECK-LABEL: test_atomic_load_and_i64_monotonic: 2959; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_monotonic: 2960; OUTLINE-ATOMICS: // %bb.0: 2961; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2962; OUTLINE-ATOMICS-NEXT: adrp x1, var64 2963; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 2964; OUTLINE-ATOMICS-NEXT: mvn x0, x0 2965; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_relax 2966; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2967; OUTLINE-ATOMICS-NEXT: ret 2968 %old = atomicrmw and ptr @var64, i64 %offset monotonic 2969; CHECK-NOT: dmb 2970; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] 2971; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 2972; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 2973 2974; CHECK: ldclr x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]] 2975; CHECK-NOT: dmb 2976 ret i64 %old 2977} 2978 2979define dso_local void @test_atomic_load_and_i32_noret_monotonic(i32 %offset) nounwind { 2980; CHECK-LABEL: test_atomic_load_and_i32_noret_monotonic: 2981; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_noret_monotonic: 2982; OUTLINE-ATOMICS: // %bb.0: 2983; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 2984; OUTLINE-ATOMICS-NEXT: adrp x1, var32 2985; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 2986; OUTLINE-ATOMICS-NEXT: mvn w0, w0 2987; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_relax 2988; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 2989; OUTLINE-ATOMICS-NEXT: ret 2990 atomicrmw and ptr @var32, i32 %offset monotonic 2991; CHECK-NOT: dmb 2992; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 2993; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 2994; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 2995 2996; CHECK: ldclr w{{[0-9]+}}, w[[NEW:[1-9][0-9]*]], [x[[ADDR]]] 2997; CHECK-NOT: dmb 2998 ret void 2999} 3000 3001define dso_local void @test_atomic_load_and_i64_noret_monotonic(i64 %offset) nounwind { 3002; CHECK-LABEL: test_atomic_load_and_i64_noret_monotonic: 3003; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_noret_monotonic: 3004; OUTLINE-ATOMICS: // %bb.0: 3005; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3006; OUTLINE-ATOMICS-NEXT: adrp x1, var64 3007; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 3008; OUTLINE-ATOMICS-NEXT: mvn x0, x0 3009; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_relax 3010; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3011; OUTLINE-ATOMICS-NEXT: ret 3012 atomicrmw and ptr @var64, i64 %offset monotonic 3013; CHECK-NOT: dmb 3014; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] 3015; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 3016; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 3017 3018; CHECK: ldclr x{{[0-9]+}}, x[[NEW:[1-9][0-9]*]], [x[[ADDR]]] 3019; CHECK-NOT: dmb 3020 ret void 3021} 3022 3023define dso_local i8 @test_atomic_load_and_i8_release(i8 %offset) nounwind { 3024; CHECK-LABEL: test_atomic_load_and_i8_release: 3025; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8_release: 3026; OUTLINE-ATOMICS: // %bb.0: 3027; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3028; OUTLINE-ATOMICS-NEXT: adrp x1, var8 3029; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 3030; OUTLINE-ATOMICS-NEXT: mvn w0, w0 3031; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_rel 3032; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3033; OUTLINE-ATOMICS-NEXT: ret 3034 %old = atomicrmw and ptr @var8, i8 %offset release 3035; CHECK-NOT: dmb 3036; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 3037; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 3038; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 3039 3040; CHECK: ldclrlb w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 3041; CHECK-NOT: dmb 3042 ret i8 %old 3043} 3044 3045define dso_local i16 @test_atomic_load_and_i16_release(i16 %offset) nounwind { 3046; CHECK-LABEL: test_atomic_load_and_i16_release: 3047; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16_release: 3048; OUTLINE-ATOMICS: // %bb.0: 3049; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3050; OUTLINE-ATOMICS-NEXT: adrp x1, var16 3051; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 3052; OUTLINE-ATOMICS-NEXT: mvn w0, w0 3053; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_rel 3054; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3055; OUTLINE-ATOMICS-NEXT: ret 3056 %old = atomicrmw and ptr @var16, i16 %offset release 3057; CHECK-NOT: dmb 3058; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 3059; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 3060; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 3061 3062; CHECK: ldclrlh w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 3063; CHECK-NOT: dmb 3064 ret i16 %old 3065} 3066 3067define dso_local i32 @test_atomic_load_and_i32_release(i32 %offset) nounwind { 3068; CHECK-LABEL: test_atomic_load_and_i32_release: 3069; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_release: 3070; OUTLINE-ATOMICS: // %bb.0: 3071; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3072; OUTLINE-ATOMICS-NEXT: adrp x1, var32 3073; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 3074; OUTLINE-ATOMICS-NEXT: mvn w0, w0 3075; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_rel 3076; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3077; OUTLINE-ATOMICS-NEXT: ret 3078 %old = atomicrmw and ptr @var32, i32 %offset release 3079; CHECK-NOT: dmb 3080; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 3081; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 3082; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 3083 3084; CHECK: ldclrl w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 3085; CHECK-NOT: dmb 3086 ret i32 %old 3087} 3088 3089define dso_local i64 @test_atomic_load_and_i64_release(i64 %offset) nounwind { 3090; CHECK-LABEL: test_atomic_load_and_i64_release: 3091; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_release: 3092; OUTLINE-ATOMICS: // %bb.0: 3093; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3094; OUTLINE-ATOMICS-NEXT: adrp x1, var64 3095; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 3096; OUTLINE-ATOMICS-NEXT: mvn x0, x0 3097; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_rel 3098; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3099; OUTLINE-ATOMICS-NEXT: ret 3100 %old = atomicrmw and ptr @var64, i64 %offset release 3101; CHECK-NOT: dmb 3102; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] 3103; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 3104; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 3105 3106; CHECK: ldclrl x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]] 3107; CHECK-NOT: dmb 3108 ret i64 %old 3109} 3110 3111define dso_local void @test_atomic_load_and_i32_noret_release(i32 %offset) nounwind { 3112; CHECK-LABEL: test_atomic_load_and_i32_noret_release: 3113; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_noret_release: 3114; OUTLINE-ATOMICS: // %bb.0: 3115; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3116; OUTLINE-ATOMICS-NEXT: adrp x1, var32 3117; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 3118; OUTLINE-ATOMICS-NEXT: mvn w0, w0 3119; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_rel 3120; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3121; OUTLINE-ATOMICS-NEXT: ret 3122 atomicrmw and ptr @var32, i32 %offset release 3123; CHECK-NOT: dmb 3124; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 3125; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 3126; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 3127 3128; CHECK: ldclrl w{{[0-9]*}}, w[[NEW:[1-9][0-9]*]], [x[[ADDR]]] 3129; CHECK-NOT: dmb 3130 ret void 3131} 3132 3133define dso_local void @test_atomic_load_and_i64_noret_release(i64 %offset) nounwind { 3134; CHECK-LABEL: test_atomic_load_and_i64_noret_release: 3135; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_noret_release: 3136; OUTLINE-ATOMICS: // %bb.0: 3137; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3138; OUTLINE-ATOMICS-NEXT: adrp x1, var64 3139; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 3140; OUTLINE-ATOMICS-NEXT: mvn x0, x0 3141; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_rel 3142; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3143; OUTLINE-ATOMICS-NEXT: ret 3144 atomicrmw and ptr @var64, i64 %offset release 3145; CHECK-NOT: dmb 3146; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] 3147; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 3148; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 3149 3150; CHECK: ldclrl x{{[0-9]*}}, x[[NEW:[1-9][0-9]*]], [x[[ADDR]]] 3151; CHECK-NOT: dmb 3152 ret void 3153} 3154 3155define dso_local i8 @test_atomic_load_and_i8_seq_cst(i8 %offset) nounwind { 3156; CHECK-LABEL: test_atomic_load_and_i8_seq_cst: 3157; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i8_seq_cst: 3158; OUTLINE-ATOMICS: // %bb.0: 3159; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3160; OUTLINE-ATOMICS-NEXT: adrp x1, var8 3161; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 3162; OUTLINE-ATOMICS-NEXT: mvn w0, w0 3163; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr1_acq_rel 3164; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3165; OUTLINE-ATOMICS-NEXT: ret 3166 %old = atomicrmw and ptr @var8, i8 %offset seq_cst 3167; CHECK-NOT: dmb 3168; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 3169; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 3170; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 3171 3172; CHECK: ldclralb w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 3173; CHECK-NOT: dmb 3174 ret i8 %old 3175} 3176 3177define dso_local i16 @test_atomic_load_and_i16_seq_cst(i16 %offset) nounwind { 3178; CHECK-LABEL: test_atomic_load_and_i16_seq_cst: 3179; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i16_seq_cst: 3180; OUTLINE-ATOMICS: // %bb.0: 3181; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3182; OUTLINE-ATOMICS-NEXT: adrp x1, var16 3183; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 3184; OUTLINE-ATOMICS-NEXT: mvn w0, w0 3185; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr2_acq_rel 3186; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3187; OUTLINE-ATOMICS-NEXT: ret 3188 %old = atomicrmw and ptr @var16, i16 %offset seq_cst 3189; CHECK-NOT: dmb 3190; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 3191; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 3192; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 3193 3194; CHECK: ldclralh w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 3195; CHECK-NOT: dmb 3196 ret i16 %old 3197} 3198 3199define dso_local i32 @test_atomic_load_and_i32_seq_cst(i32 %offset) nounwind { 3200; CHECK-LABEL: test_atomic_load_and_i32_seq_cst: 3201; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_seq_cst: 3202; OUTLINE-ATOMICS: // %bb.0: 3203; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3204; OUTLINE-ATOMICS-NEXT: adrp x1, var32 3205; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 3206; OUTLINE-ATOMICS-NEXT: mvn w0, w0 3207; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel 3208; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3209; OUTLINE-ATOMICS-NEXT: ret 3210 %old = atomicrmw and ptr @var32, i32 %offset seq_cst 3211; CHECK-NOT: dmb 3212; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 3213; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 3214; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 3215 3216; CHECK: ldclral w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 3217; CHECK-NOT: dmb 3218 ret i32 %old 3219} 3220 3221define dso_local i64 @test_atomic_load_and_i64_seq_cst(i64 %offset) nounwind { 3222; CHECK-LABEL: test_atomic_load_and_i64_seq_cst: 3223; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_seq_cst: 3224; OUTLINE-ATOMICS: // %bb.0: 3225; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3226; OUTLINE-ATOMICS-NEXT: adrp x1, var64 3227; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 3228; OUTLINE-ATOMICS-NEXT: mvn x0, x0 3229; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel 3230; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3231; OUTLINE-ATOMICS-NEXT: ret 3232 %old = atomicrmw and ptr @var64, i64 %offset seq_cst 3233; CHECK-NOT: dmb 3234; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] 3235; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 3236; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 3237 3238; CHECK: ldclral x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]] 3239; CHECK-NOT: dmb 3240 ret i64 %old 3241} 3242 3243define dso_local void @test_atomic_load_and_i32_noret_seq_cst(i32 %offset) nounwind { 3244; CHECK-LABEL: test_atomic_load_and_i32_noret_seq_cst: 3245; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i32_noret_seq_cst: 3246; OUTLINE-ATOMICS: // %bb.0: 3247; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3248; OUTLINE-ATOMICS-NEXT: adrp x1, var32 3249; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 3250; OUTLINE-ATOMICS-NEXT: mvn w0, w0 3251; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr4_acq_rel 3252; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3253; OUTLINE-ATOMICS-NEXT: ret 3254 atomicrmw and ptr @var32, i32 %offset seq_cst 3255; CHECK-NOT: dmb 3256; CHECK: mvn w[[NOT:[0-9]+]], w[[OLD:[0-9]+]] 3257; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 3258; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 3259 3260; CHECK: ldclral w[[NOT]], w[[NEW:[0-9]+]], [x[[ADDR]]] 3261; CHECK-NOT: dmb 3262 ret void 3263} 3264 3265define dso_local void @test_atomic_load_and_i64_noret_seq_cst(i64 %offset) nounwind { 3266; CHECK-LABEL: test_atomic_load_and_i64_noret_seq_cst: 3267; OUTLINE-ATOMICS-LABEL: test_atomic_load_and_i64_noret_seq_cst: 3268; OUTLINE-ATOMICS: // %bb.0: 3269; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3270; OUTLINE-ATOMICS-NEXT: adrp x1, var64 3271; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 3272; OUTLINE-ATOMICS-NEXT: mvn x0, x0 3273; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldclr8_acq_rel 3274; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3275; OUTLINE-ATOMICS-NEXT: ret 3276 atomicrmw and ptr @var64, i64 %offset seq_cst 3277; CHECK-NOT: dmb 3278; CHECK: mvn x[[NOT:[0-9]+]], x[[OLD:[0-9]+]] 3279; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 3280; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 3281 3282; CHECK: ldclral x[[NOT]], x[[NEW:[0-9]+]], [x[[ADDR]]] 3283; CHECK-NOT: dmb 3284 ret void 3285} 3286 3287define dso_local i8 @test_atomic_cmpxchg_i8_acquire(i8 %wanted, i8 %new) nounwind { 3288; CHECK-LABEL: test_atomic_cmpxchg_i8_acquire: 3289; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i8_acquire: 3290; OUTLINE-ATOMICS: // %bb.0: 3291; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3292; OUTLINE-ATOMICS-NEXT: adrp x2, var8 3293; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var8 3294; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas1_acq 3295; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3296; OUTLINE-ATOMICS-NEXT: ret 3297 %pair = cmpxchg ptr @var8, i8 %wanted, i8 %new acquire acquire 3298 %old = extractvalue { i8, i1 } %pair, 0 3299 3300; CHECK-NOT: dmb 3301; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 3302; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 3303 3304; CHECK: casab w[[NEW:[0-9]+]], w[[OLD:[0-9]+]], [x[[ADDR]]] 3305; CHECK-NOT: dmb 3306 3307 ret i8 %old 3308} 3309 3310define dso_local i16 @test_atomic_cmpxchg_i16_acquire(i16 %wanted, i16 %new) nounwind { 3311; CHECK-LABEL: test_atomic_cmpxchg_i16_acquire: 3312; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i16_acquire: 3313; OUTLINE-ATOMICS: // %bb.0: 3314; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3315; OUTLINE-ATOMICS-NEXT: adrp x2, var16 3316; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var16 3317; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas2_acq 3318; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3319; OUTLINE-ATOMICS-NEXT: ret 3320 %pair = cmpxchg ptr @var16, i16 %wanted, i16 %new acquire acquire 3321 %old = extractvalue { i16, i1 } %pair, 0 3322 3323; CHECK-NOT: dmb 3324; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 3325; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 3326 3327; CHECK: casah w0, w1, [x[[ADDR]]] 3328; CHECK-NOT: dmb 3329 3330 ret i16 %old 3331} 3332 3333define dso_local i32 @test_atomic_cmpxchg_i32_acquire(i32 %wanted, i32 %new) nounwind { 3334; CHECK-LABEL: test_atomic_cmpxchg_i32_acquire: 3335; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i32_acquire: 3336; OUTLINE-ATOMICS: // %bb.0: 3337; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3338; OUTLINE-ATOMICS-NEXT: adrp x2, var32 3339; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var32 3340; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas4_acq 3341; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3342; OUTLINE-ATOMICS-NEXT: ret 3343 %pair = cmpxchg ptr @var32, i32 %wanted, i32 %new acquire acquire 3344 %old = extractvalue { i32, i1 } %pair, 0 3345 3346; CHECK-NOT: dmb 3347; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 3348; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 3349 3350; CHECK: casa w0, w1, [x[[ADDR]]] 3351; CHECK-NOT: dmb 3352 3353 ret i32 %old 3354} 3355 3356define dso_local i64 @test_atomic_cmpxchg_i64_acquire(i64 %wanted, i64 %new) nounwind { 3357; CHECK-LABEL: test_atomic_cmpxchg_i64_acquire: 3358; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i64_acquire: 3359; OUTLINE-ATOMICS: // %bb.0: 3360; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3361; OUTLINE-ATOMICS-NEXT: adrp x2, var64 3362; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var64 3363; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas8_acq 3364; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3365; OUTLINE-ATOMICS-NEXT: ret 3366 %pair = cmpxchg ptr @var64, i64 %wanted, i64 %new acquire acquire 3367 %old = extractvalue { i64, i1 } %pair, 0 3368 3369; CHECK-NOT: dmb 3370; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 3371; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 3372 3373; CHECK: casa x0, x1, [x[[ADDR]]] 3374; CHECK-NOT: dmb 3375 3376 ret i64 %old 3377} 3378 3379define dso_local i128 @test_atomic_cmpxchg_i128_acquire(i128 %wanted, i128 %new) nounwind { 3380; CHECK-LABEL: test_atomic_cmpxchg_i128_acquire: 3381; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i128_acquire: 3382; OUTLINE-ATOMICS: // %bb.0: 3383; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3384; OUTLINE-ATOMICS-NEXT: adrp x4, var128 3385; OUTLINE-ATOMICS-NEXT: add x4, x4, :lo12:var128 3386; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas16_acq 3387; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3388; OUTLINE-ATOMICS-NEXT: ret 3389 %pair = cmpxchg ptr @var128, i128 %wanted, i128 %new acquire acquire 3390 %old = extractvalue { i128, i1 } %pair, 0 3391 3392; CHECK-NOT: dmb 3393; CHECK: adrp [[TMPADDR:x[0-9]+]], var128 3394; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var128 3395 3396; CHECK: caspa x0, x1, x2, x3, [x[[ADDR]]] 3397; CHECK-NOT: dmb 3398 3399 ret i128 %old 3400} 3401 3402define dso_local i8 @test_atomic_cmpxchg_i8_monotonic(i8 %wanted, i8 %new) nounwind { 3403; CHECK-LABEL: test_atomic_cmpxchg_i8_monotonic: 3404; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i8_monotonic: 3405; OUTLINE-ATOMICS: // %bb.0: 3406; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3407; OUTLINE-ATOMICS-NEXT: adrp x2, var8 3408; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var8 3409; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas1_relax 3410; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3411; OUTLINE-ATOMICS-NEXT: ret 3412 %pair = cmpxchg ptr @var8, i8 %wanted, i8 %new monotonic monotonic 3413 %old = extractvalue { i8, i1 } %pair, 0 3414 3415; CHECK-NOT: dmb 3416; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 3417; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 3418 3419; CHECK: casb w[[NEW:[0-9]+]], w[[OLD:[0-9]+]], [x[[ADDR]]] 3420; CHECK-NOT: dmb 3421 3422 ret i8 %old 3423} 3424 3425define dso_local i16 @test_atomic_cmpxchg_i16_monotonic(i16 %wanted, i16 %new) nounwind { 3426; CHECK-LABEL: test_atomic_cmpxchg_i16_monotonic: 3427; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i16_monotonic: 3428; OUTLINE-ATOMICS: // %bb.0: 3429; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3430; OUTLINE-ATOMICS-NEXT: adrp x2, var16 3431; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var16 3432; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas2_relax 3433; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3434; OUTLINE-ATOMICS-NEXT: ret 3435 %pair = cmpxchg ptr @var16, i16 %wanted, i16 %new monotonic monotonic 3436 %old = extractvalue { i16, i1 } %pair, 0 3437 3438; CHECK-NOT: dmb 3439; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 3440; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 3441 3442; CHECK: cash w0, w1, [x[[ADDR]]] 3443; CHECK-NOT: dmb 3444 3445 ret i16 %old 3446} 3447 3448define dso_local i32 @test_atomic_cmpxchg_i32_monotonic(i32 %wanted, i32 %new) nounwind { 3449; CHECK-LABEL: test_atomic_cmpxchg_i32_monotonic: 3450; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i32_monotonic: 3451; OUTLINE-ATOMICS: // %bb.0: 3452; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3453; OUTLINE-ATOMICS-NEXT: adrp x2, var32 3454; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var32 3455; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas4_relax 3456; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3457; OUTLINE-ATOMICS-NEXT: ret 3458 %pair = cmpxchg ptr @var32, i32 %wanted, i32 %new monotonic monotonic 3459 %old = extractvalue { i32, i1 } %pair, 0 3460 3461; CHECK-NOT: dmb 3462; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 3463; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 3464 3465; CHECK: cas w0, w1, [x[[ADDR]]] 3466; CHECK-NOT: dmb 3467 3468 ret i32 %old 3469} 3470 3471define dso_local i64 @test_atomic_cmpxchg_i64_monotonic(i64 %wanted, i64 %new) nounwind { 3472; CHECK-LABEL: test_atomic_cmpxchg_i64_monotonic: 3473; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i64_monotonic: 3474; OUTLINE-ATOMICS: // %bb.0: 3475; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3476; OUTLINE-ATOMICS-NEXT: adrp x2, var64 3477; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var64 3478; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas8_relax 3479; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3480; OUTLINE-ATOMICS-NEXT: ret 3481 %pair = cmpxchg ptr @var64, i64 %wanted, i64 %new monotonic monotonic 3482 %old = extractvalue { i64, i1 } %pair, 0 3483 3484; CHECK-NOT: dmb 3485; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 3486; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 3487 3488; CHECK: cas x0, x1, [x[[ADDR]]] 3489; CHECK-NOT: dmb 3490 3491 ret i64 %old 3492} 3493 3494define dso_local i128 @test_atomic_cmpxchg_i128_monotonic(i128 %wanted, i128 %new) nounwind { 3495; CHECK-LABEL: test_atomic_cmpxchg_i128_monotonic: 3496; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i128_monotonic: 3497; OUTLINE-ATOMICS: // %bb.0: 3498; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3499; OUTLINE-ATOMICS-NEXT: adrp x4, var128 3500; OUTLINE-ATOMICS-NEXT: add x4, x4, :lo12:var128 3501; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas16_relax 3502; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3503; OUTLINE-ATOMICS-NEXT: ret 3504 %pair = cmpxchg ptr @var128, i128 %wanted, i128 %new monotonic monotonic 3505 %old = extractvalue { i128, i1 } %pair, 0 3506 3507; CHECK-NOT: dmb 3508; CHECK: adrp [[TMPADDR:x[0-9]+]], var128 3509; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var128 3510 3511; CHECK: casp x0, x1, x2, x3, [x[[ADDR]]] 3512; CHECK-NOT: dmb 3513 3514 ret i128 %old 3515} 3516 3517define dso_local i8 @test_atomic_cmpxchg_i8_seq_cst(i8 %wanted, i8 %new) nounwind { 3518; CHECK-LABEL: test_atomic_cmpxchg_i8_seq_cst: 3519; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i8_seq_cst: 3520; OUTLINE-ATOMICS: // %bb.0: 3521; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3522; OUTLINE-ATOMICS-NEXT: adrp x2, var8 3523; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var8 3524; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas1_acq_rel 3525; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3526; OUTLINE-ATOMICS-NEXT: ret 3527 %pair = cmpxchg ptr @var8, i8 %wanted, i8 %new seq_cst seq_cst 3528 %old = extractvalue { i8, i1 } %pair, 0 3529 3530; CHECK-NOT: dmb 3531; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 3532; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 3533 3534; CHECK: casalb w[[NEW:[0-9]+]], w[[OLD:[0-9]+]], [x[[ADDR]]] 3535; CHECK-NOT: dmb 3536 3537 ret i8 %old 3538} 3539 3540define dso_local i16 @test_atomic_cmpxchg_i16_seq_cst(i16 %wanted, i16 %new) nounwind { 3541; CHECK-LABEL: test_atomic_cmpxchg_i16_seq_cst: 3542; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i16_seq_cst: 3543; OUTLINE-ATOMICS: // %bb.0: 3544; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3545; OUTLINE-ATOMICS-NEXT: adrp x2, var16 3546; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var16 3547; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas2_acq_rel 3548; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3549; OUTLINE-ATOMICS-NEXT: ret 3550 %pair = cmpxchg ptr @var16, i16 %wanted, i16 %new seq_cst seq_cst 3551 %old = extractvalue { i16, i1 } %pair, 0 3552 3553; CHECK-NOT: dmb 3554; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 3555; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 3556 3557; CHECK: casalh w0, w1, [x[[ADDR]]] 3558; CHECK-NOT: dmb 3559 3560 ret i16 %old 3561} 3562 3563define dso_local i32 @test_atomic_cmpxchg_i32_seq_cst(i32 %wanted, i32 %new) nounwind { 3564; CHECK-LABEL: test_atomic_cmpxchg_i32_seq_cst: 3565; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i32_seq_cst: 3566; OUTLINE-ATOMICS: // %bb.0: 3567; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3568; OUTLINE-ATOMICS-NEXT: adrp x2, var32 3569; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var32 3570; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas4_acq_rel 3571; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3572; OUTLINE-ATOMICS-NEXT: ret 3573 %pair = cmpxchg ptr @var32, i32 %wanted, i32 %new seq_cst seq_cst 3574 %old = extractvalue { i32, i1 } %pair, 0 3575 3576; CHECK-NOT: dmb 3577; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 3578; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 3579 3580; CHECK: casal w0, w1, [x[[ADDR]]] 3581; CHECK-NOT: dmb 3582 3583 ret i32 %old 3584} 3585 3586define dso_local i32 @test_atomic_cmpxchg_i32_monotonic_seq_cst(i32 %wanted, i32 %new) nounwind { 3587; CHECK-LABEL: test_atomic_cmpxchg_i32_monotonic_seq_cst: 3588; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i32_monotonic_seq_cst: 3589; OUTLINE-ATOMICS: // %bb.0: 3590; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3591; OUTLINE-ATOMICS-NEXT: adrp x2, var32 3592; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var32 3593; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas4_acq_rel 3594; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3595; OUTLINE-ATOMICS-NEXT: ret 3596 %pair = cmpxchg ptr @var32, i32 %wanted, i32 %new monotonic seq_cst 3597 %old = extractvalue { i32, i1 } %pair, 0 3598 3599; CHECK-NOT: dmb 3600; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 3601; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 3602 3603; CHECK: casal w0, w1, [x[[ADDR]]] 3604; CHECK-NOT: dmb 3605 3606 ret i32 %old 3607} 3608 3609define dso_local i32 @test_atomic_cmpxchg_i32_release_acquire(i32 %wanted, i32 %new) nounwind { 3610; CHECK-LABEL: test_atomic_cmpxchg_i32_release_acquire: 3611; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i32_release_acquire: 3612; OUTLINE-ATOMICS: // %bb.0: 3613; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3614; OUTLINE-ATOMICS-NEXT: adrp x2, var32 3615; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var32 3616; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas4_acq_rel 3617; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3618; OUTLINE-ATOMICS-NEXT: ret 3619 %pair = cmpxchg ptr @var32, i32 %wanted, i32 %new release acquire 3620 %old = extractvalue { i32, i1 } %pair, 0 3621 3622; CHECK-NOT: dmb 3623; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 3624; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 3625 3626; CHECK: casal w0, w1, [x[[ADDR]]] 3627; CHECK-NOT: dmb 3628 3629 ret i32 %old 3630} 3631 3632define dso_local i64 @test_atomic_cmpxchg_i64_seq_cst(i64 %wanted, i64 %new) nounwind { 3633; CHECK-LABEL: test_atomic_cmpxchg_i64_seq_cst: 3634; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i64_seq_cst: 3635; OUTLINE-ATOMICS: // %bb.0: 3636; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3637; OUTLINE-ATOMICS-NEXT: adrp x2, var64 3638; OUTLINE-ATOMICS-NEXT: add x2, x2, :lo12:var64 3639; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas8_acq_rel 3640; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3641; OUTLINE-ATOMICS-NEXT: ret 3642 %pair = cmpxchg ptr @var64, i64 %wanted, i64 %new seq_cst seq_cst 3643 %old = extractvalue { i64, i1 } %pair, 0 3644 3645; CHECK-NOT: dmb 3646; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 3647; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 3648 3649; CHECK: casal x0, x1, [x[[ADDR]]] 3650; CHECK-NOT: dmb 3651 3652 ret i64 %old 3653} 3654 3655define dso_local i128 @test_atomic_cmpxchg_i128_seq_cst(i128 %wanted, i128 %new) nounwind { 3656; CHECK-LABEL: test_atomic_cmpxchg_i128_seq_cst: 3657; OUTLINE-ATOMICS-LABEL: test_atomic_cmpxchg_i128_seq_cst: 3658; OUTLINE-ATOMICS: // %bb.0: 3659; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 3660; OUTLINE-ATOMICS-NEXT: adrp x4, var128 3661; OUTLINE-ATOMICS-NEXT: add x4, x4, :lo12:var128 3662; OUTLINE-ATOMICS-NEXT: bl __aarch64_cas16_acq_rel 3663; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 3664; OUTLINE-ATOMICS-NEXT: ret 3665 %pair = cmpxchg ptr @var128, i128 %wanted, i128 %new seq_cst seq_cst 3666 %old = extractvalue { i128, i1 } %pair, 0 3667 3668; CHECK-NOT: dmb 3669; CHECK: adrp [[TMPADDR:x[0-9]+]], var128 3670; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var128 3671 3672; CHECK: caspal x0, x1, x2, x3, [x[[ADDR]]] 3673; CHECK-NOT: dmb 3674 3675 ret i128 %old 3676} 3677 3678define dso_local i8 @test_atomic_load_max_i8_acq_rel(i8 %offset) nounwind { 3679; CHECK-LABEL: test_atomic_load_max_i8_acq_rel: 3680; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_acq_rel: 3681; OUTLINE-ATOMICS: // %bb.0: 3682; OUTLINE-ATOMICS-NEXT: adrp x9, var8 3683; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 3684; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 3685; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 3686; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] 3687; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 3688; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb 3689; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt 3690; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] 3691; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 3692; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 3693; OUTLINE-ATOMICS-NEXT: mov w0, w8 3694; OUTLINE-ATOMICS-NEXT: ret 3695 %old = atomicrmw max ptr @var8, i8 %offset acq_rel 3696; CHECK-NOT: dmb 3697; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 3698; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 3699 3700; CHECK: ldsmaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 3701; CHECK-NOT: dmb 3702 3703 ret i8 %old 3704} 3705 3706define dso_local i16 @test_atomic_load_max_i16_acq_rel(i16 %offset) nounwind { 3707; CHECK-LABEL: test_atomic_load_max_i16_acq_rel: 3708; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_acq_rel: 3709; OUTLINE-ATOMICS: // %bb.0: 3710; OUTLINE-ATOMICS-NEXT: adrp x9, var16 3711; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 3712; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 3713; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 3714; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] 3715; OUTLINE-ATOMICS-NEXT: sxth w8, w10 3716; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth 3717; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt 3718; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] 3719; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 3720; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 3721; OUTLINE-ATOMICS-NEXT: mov w0, w8 3722; OUTLINE-ATOMICS-NEXT: ret 3723 %old = atomicrmw max ptr @var16, i16 %offset acq_rel 3724; CHECK-NOT: dmb 3725; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 3726; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 3727 3728; CHECK: ldsmaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 3729; CHECK-NOT: dmb 3730 3731 ret i16 %old 3732} 3733 3734define dso_local i32 @test_atomic_load_max_i32_acq_rel(i32 %offset) nounwind { 3735; CHECK-LABEL: test_atomic_load_max_i32_acq_rel: 3736; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_acq_rel: 3737; OUTLINE-ATOMICS: // %bb.0: 3738; OUTLINE-ATOMICS-NEXT: adrp x9, var32 3739; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 3740; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 3741; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 3742; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 3743; OUTLINE-ATOMICS-NEXT: cmp w8, w0 3744; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt 3745; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 3746; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 3747; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 3748; OUTLINE-ATOMICS-NEXT: mov w0, w8 3749; OUTLINE-ATOMICS-NEXT: ret 3750 %old = atomicrmw max ptr @var32, i32 %offset acq_rel 3751; CHECK-NOT: dmb 3752; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 3753; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 3754 3755; CHECK: ldsmaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 3756; CHECK-NOT: dmb 3757 3758 ret i32 %old 3759} 3760 3761define dso_local i64 @test_atomic_load_max_i64_acq_rel(i64 %offset) nounwind { 3762; CHECK-LABEL: test_atomic_load_max_i64_acq_rel: 3763; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_acq_rel: 3764; OUTLINE-ATOMICS: // %bb.0: 3765; OUTLINE-ATOMICS-NEXT: adrp x9, var64 3766; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 3767; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 3768; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 3769; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 3770; OUTLINE-ATOMICS-NEXT: cmp x8, x0 3771; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt 3772; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 3773; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 3774; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 3775; OUTLINE-ATOMICS-NEXT: mov x0, x8 3776; OUTLINE-ATOMICS-NEXT: ret 3777 %old = atomicrmw max ptr @var64, i64 %offset acq_rel 3778; CHECK-NOT: dmb 3779; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 3780; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 3781 3782; CHECK: ldsmaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 3783; CHECK-NOT: dmb 3784 3785 ret i64 %old 3786} 3787 3788define dso_local void @test_atomic_load_max_i32_noret_acq_rel(i32 %offset) nounwind { 3789; CHECK-LABEL: test_atomic_load_max_i32_noret_acq_rel: 3790; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_acq_rel: 3791; OUTLINE-ATOMICS: // %bb.0: 3792; OUTLINE-ATOMICS-NEXT: adrp x8, var32 3793; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 3794; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 3795; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 3796; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 3797; OUTLINE-ATOMICS-NEXT: cmp w9, w0 3798; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt 3799; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 3800; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 3801; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 3802; OUTLINE-ATOMICS-NEXT: ret 3803 atomicrmw max ptr @var32, i32 %offset acq_rel 3804; CHECK-NOT: dmb 3805; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 3806; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 3807 3808; CHECK: ldsmaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 3809; CHECK-NOT: dmb 3810 ret void 3811} 3812 3813define dso_local void @test_atomic_load_max_i64_noret_acq_rel(i64 %offset) nounwind { 3814; CHECK-LABEL: test_atomic_load_max_i64_noret_acq_rel: 3815; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_acq_rel: 3816; OUTLINE-ATOMICS: // %bb.0: 3817; OUTLINE-ATOMICS-NEXT: adrp x8, var64 3818; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 3819; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 3820; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 3821; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 3822; OUTLINE-ATOMICS-NEXT: cmp x9, x0 3823; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt 3824; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 3825; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 3826; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 3827; OUTLINE-ATOMICS-NEXT: ret 3828 atomicrmw max ptr @var64, i64 %offset acq_rel 3829; CHECK-NOT: dmb 3830; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 3831; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 3832 3833; CHECK: ldsmaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 3834; CHECK-NOT: dmb 3835 ret void 3836} 3837 3838define dso_local i8 @test_atomic_load_max_i8_acquire(i8 %offset) nounwind { 3839; CHECK-LABEL: test_atomic_load_max_i8_acquire: 3840; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_acquire: 3841; OUTLINE-ATOMICS: // %bb.0: 3842; OUTLINE-ATOMICS-NEXT: adrp x9, var8 3843; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 3844; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 3845; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 3846; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] 3847; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 3848; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb 3849; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt 3850; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] 3851; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 3852; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 3853; OUTLINE-ATOMICS-NEXT: mov w0, w8 3854; OUTLINE-ATOMICS-NEXT: ret 3855 %old = atomicrmw max ptr @var8, i8 %offset acquire 3856; CHECK-NOT: dmb 3857; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 3858; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 3859 3860; CHECK: ldsmaxab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 3861; CHECK-NOT: dmb 3862 3863 ret i8 %old 3864} 3865 3866define dso_local i16 @test_atomic_load_max_i16_acquire(i16 %offset) nounwind { 3867; CHECK-LABEL: test_atomic_load_max_i16_acquire: 3868; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_acquire: 3869; OUTLINE-ATOMICS: // %bb.0: 3870; OUTLINE-ATOMICS-NEXT: adrp x9, var16 3871; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 3872; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 3873; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 3874; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] 3875; OUTLINE-ATOMICS-NEXT: sxth w8, w10 3876; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth 3877; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt 3878; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] 3879; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 3880; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 3881; OUTLINE-ATOMICS-NEXT: mov w0, w8 3882; OUTLINE-ATOMICS-NEXT: ret 3883 %old = atomicrmw max ptr @var16, i16 %offset acquire 3884; CHECK-NOT: dmb 3885; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 3886; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 3887 3888; CHECK: ldsmaxah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 3889; CHECK-NOT: dmb 3890 3891 ret i16 %old 3892} 3893 3894define dso_local i32 @test_atomic_load_max_i32_acquire(i32 %offset) nounwind { 3895; CHECK-LABEL: test_atomic_load_max_i32_acquire: 3896; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_acquire: 3897; OUTLINE-ATOMICS: // %bb.0: 3898; OUTLINE-ATOMICS-NEXT: adrp x9, var32 3899; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 3900; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 3901; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 3902; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 3903; OUTLINE-ATOMICS-NEXT: cmp w8, w0 3904; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt 3905; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] 3906; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 3907; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 3908; OUTLINE-ATOMICS-NEXT: mov w0, w8 3909; OUTLINE-ATOMICS-NEXT: ret 3910 %old = atomicrmw max ptr @var32, i32 %offset acquire 3911; CHECK-NOT: dmb 3912; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 3913; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 3914 3915; CHECK: ldsmaxa w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 3916; CHECK-NOT: dmb 3917 3918 ret i32 %old 3919} 3920 3921define dso_local i64 @test_atomic_load_max_i64_acquire(i64 %offset) nounwind { 3922; CHECK-LABEL: test_atomic_load_max_i64_acquire: 3923; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_acquire: 3924; OUTLINE-ATOMICS: // %bb.0: 3925; OUTLINE-ATOMICS-NEXT: adrp x9, var64 3926; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 3927; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 3928; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 3929; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 3930; OUTLINE-ATOMICS-NEXT: cmp x8, x0 3931; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt 3932; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] 3933; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 3934; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 3935; OUTLINE-ATOMICS-NEXT: mov x0, x8 3936; OUTLINE-ATOMICS-NEXT: ret 3937 %old = atomicrmw max ptr @var64, i64 %offset acquire 3938; CHECK-NOT: dmb 3939; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 3940; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 3941 3942; CHECK: ldsmaxa x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 3943; CHECK-NOT: dmb 3944 3945 ret i64 %old 3946} 3947 3948define dso_local void @test_atomic_load_max_i32_noret_acquire(i32 %offset) nounwind { 3949; CHECK-LABEL: test_atomic_load_max_i32_noret_acquire: 3950; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_acquire: 3951; OUTLINE-ATOMICS: // %bb.0: 3952; OUTLINE-ATOMICS-NEXT: adrp x8, var32 3953; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 3954; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 3955; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 3956; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 3957; OUTLINE-ATOMICS-NEXT: cmp w9, w0 3958; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt 3959; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] 3960; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 3961; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 3962; OUTLINE-ATOMICS-NEXT: ret 3963 atomicrmw max ptr @var32, i32 %offset acquire 3964; CHECK-NOT: dmb 3965; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 3966; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 3967 3968; CHECK: ldsmaxa w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 3969; CHECK-NOT: dmb 3970 ret void 3971} 3972 3973define dso_local void @test_atomic_load_max_i64_noret_acquire(i64 %offset) nounwind { 3974; CHECK-LABEL: test_atomic_load_max_i64_noret_acquire: 3975; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_acquire: 3976; OUTLINE-ATOMICS: // %bb.0: 3977; OUTLINE-ATOMICS-NEXT: adrp x8, var64 3978; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 3979; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 3980; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 3981; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 3982; OUTLINE-ATOMICS-NEXT: cmp x9, x0 3983; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt 3984; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] 3985; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 3986; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 3987; OUTLINE-ATOMICS-NEXT: ret 3988 atomicrmw max ptr @var64, i64 %offset acquire 3989; CHECK-NOT: dmb 3990; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 3991; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 3992 3993; CHECK: ldsmaxa x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 3994; CHECK-NOT: dmb 3995 ret void 3996} 3997 3998define dso_local i8 @test_atomic_load_max_i8_monotonic(i8 %offset) nounwind { 3999; CHECK-LABEL: test_atomic_load_max_i8_monotonic: 4000; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_monotonic: 4001; OUTLINE-ATOMICS: // %bb.0: 4002; OUTLINE-ATOMICS-NEXT: adrp x9, var8 4003; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 4004; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4005; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4006; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9] 4007; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 4008; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb 4009; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt 4010; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] 4011; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4012; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4013; OUTLINE-ATOMICS-NEXT: mov w0, w8 4014; OUTLINE-ATOMICS-NEXT: ret 4015 %old = atomicrmw max ptr @var8, i8 %offset monotonic 4016; CHECK-NOT: dmb 4017; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 4018; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 4019 4020; CHECK: ldsmaxb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4021; CHECK-NOT: dmb 4022 4023 ret i8 %old 4024} 4025 4026define dso_local i16 @test_atomic_load_max_i16_monotonic(i16 %offset) nounwind { 4027; CHECK-LABEL: test_atomic_load_max_i16_monotonic: 4028; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_monotonic: 4029; OUTLINE-ATOMICS: // %bb.0: 4030; OUTLINE-ATOMICS-NEXT: adrp x9, var16 4031; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 4032; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4033; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4034; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9] 4035; OUTLINE-ATOMICS-NEXT: sxth w8, w10 4036; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth 4037; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt 4038; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] 4039; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4040; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4041; OUTLINE-ATOMICS-NEXT: mov w0, w8 4042; OUTLINE-ATOMICS-NEXT: ret 4043 %old = atomicrmw max ptr @var16, i16 %offset monotonic 4044; CHECK-NOT: dmb 4045; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 4046; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 4047 4048; CHECK: ldsmaxh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4049; CHECK-NOT: dmb 4050 4051 ret i16 %old 4052} 4053 4054define dso_local i32 @test_atomic_load_max_i32_monotonic(i32 %offset) nounwind { 4055; CHECK-LABEL: test_atomic_load_max_i32_monotonic: 4056; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_monotonic: 4057; OUTLINE-ATOMICS: // %bb.0: 4058; OUTLINE-ATOMICS-NEXT: adrp x9, var32 4059; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 4060; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4061; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4062; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] 4063; OUTLINE-ATOMICS-NEXT: cmp w8, w0 4064; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt 4065; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] 4066; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4067; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4068; OUTLINE-ATOMICS-NEXT: mov w0, w8 4069; OUTLINE-ATOMICS-NEXT: ret 4070 %old = atomicrmw max ptr @var32, i32 %offset monotonic 4071; CHECK-NOT: dmb 4072; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 4073; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 4074 4075; CHECK: ldsmax w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4076; CHECK-NOT: dmb 4077 4078 ret i32 %old 4079} 4080 4081define dso_local i64 @test_atomic_load_max_i64_monotonic(i64 %offset) nounwind { 4082; CHECK-LABEL: test_atomic_load_max_i64_monotonic: 4083; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_monotonic: 4084; OUTLINE-ATOMICS: // %bb.0: 4085; OUTLINE-ATOMICS-NEXT: adrp x9, var64 4086; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 4087; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4088; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4089; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] 4090; OUTLINE-ATOMICS-NEXT: cmp x8, x0 4091; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt 4092; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] 4093; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4094; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4095; OUTLINE-ATOMICS-NEXT: mov x0, x8 4096; OUTLINE-ATOMICS-NEXT: ret 4097 %old = atomicrmw max ptr @var64, i64 %offset monotonic 4098; CHECK-NOT: dmb 4099; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 4100; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 4101 4102; CHECK: ldsmax x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 4103; CHECK-NOT: dmb 4104 4105 ret i64 %old 4106} 4107 4108define dso_local void @test_atomic_load_max_i32_noret_monotonic(i32 %offset) nounwind { 4109; CHECK-LABEL: test_atomic_load_max_i32_noret_monotonic: 4110; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_monotonic: 4111; OUTLINE-ATOMICS: // %bb.0: 4112; OUTLINE-ATOMICS-NEXT: adrp x8, var32 4113; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 4114; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4115; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4116; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] 4117; OUTLINE-ATOMICS-NEXT: cmp w9, w0 4118; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt 4119; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] 4120; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 4121; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4122; OUTLINE-ATOMICS-NEXT: ret 4123 atomicrmw max ptr @var32, i32 %offset monotonic 4124; CHECK-NOT: dmb 4125; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 4126; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 4127 4128; CHECK: ldsmax w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] 4129; CHECK-NOT: dmb 4130 ret void 4131} 4132 4133define dso_local void @test_atomic_load_max_i64_noret_monotonic(i64 %offset) nounwind { 4134; CHECK-LABEL: test_atomic_load_max_i64_noret_monotonic: 4135; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_monotonic: 4136; OUTLINE-ATOMICS: // %bb.0: 4137; OUTLINE-ATOMICS-NEXT: adrp x8, var64 4138; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 4139; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4140; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4141; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] 4142; OUTLINE-ATOMICS-NEXT: cmp x9, x0 4143; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt 4144; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] 4145; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 4146; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4147; OUTLINE-ATOMICS-NEXT: ret 4148 atomicrmw max ptr @var64, i64 %offset monotonic 4149; CHECK-NOT: dmb 4150; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 4151; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 4152 4153; CHECK: ldsmax x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] 4154; CHECK-NOT: dmb 4155 ret void 4156} 4157 4158define dso_local i8 @test_atomic_load_max_i8_release(i8 %offset) nounwind { 4159; CHECK-LABEL: test_atomic_load_max_i8_release: 4160; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_release: 4161; OUTLINE-ATOMICS: // %bb.0: 4162; OUTLINE-ATOMICS-NEXT: adrp x9, var8 4163; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 4164; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4165; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4166; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9] 4167; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 4168; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb 4169; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt 4170; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] 4171; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4172; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4173; OUTLINE-ATOMICS-NEXT: mov w0, w8 4174; OUTLINE-ATOMICS-NEXT: ret 4175 %old = atomicrmw max ptr @var8, i8 %offset release 4176; CHECK-NOT: dmb 4177; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 4178; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 4179 4180; CHECK: ldsmaxlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4181; CHECK-NOT: dmb 4182 4183 ret i8 %old 4184} 4185 4186define dso_local i16 @test_atomic_load_max_i16_release(i16 %offset) nounwind { 4187; CHECK-LABEL: test_atomic_load_max_i16_release: 4188; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_release: 4189; OUTLINE-ATOMICS: // %bb.0: 4190; OUTLINE-ATOMICS-NEXT: adrp x9, var16 4191; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 4192; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4193; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4194; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9] 4195; OUTLINE-ATOMICS-NEXT: sxth w8, w10 4196; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth 4197; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt 4198; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] 4199; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4200; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4201; OUTLINE-ATOMICS-NEXT: mov w0, w8 4202; OUTLINE-ATOMICS-NEXT: ret 4203 %old = atomicrmw max ptr @var16, i16 %offset release 4204; CHECK-NOT: dmb 4205; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 4206; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 4207 4208; CHECK: ldsmaxlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4209; CHECK-NOT: dmb 4210 4211 ret i16 %old 4212} 4213 4214define dso_local i32 @test_atomic_load_max_i32_release(i32 %offset) nounwind { 4215; CHECK-LABEL: test_atomic_load_max_i32_release: 4216; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_release: 4217; OUTLINE-ATOMICS: // %bb.0: 4218; OUTLINE-ATOMICS-NEXT: adrp x9, var32 4219; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 4220; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4221; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4222; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] 4223; OUTLINE-ATOMICS-NEXT: cmp w8, w0 4224; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt 4225; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 4226; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4227; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4228; OUTLINE-ATOMICS-NEXT: mov w0, w8 4229; OUTLINE-ATOMICS-NEXT: ret 4230 %old = atomicrmw max ptr @var32, i32 %offset release 4231; CHECK-NOT: dmb 4232; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 4233; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 4234 4235; CHECK: ldsmaxl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4236; CHECK-NOT: dmb 4237 4238 ret i32 %old 4239} 4240 4241define dso_local i64 @test_atomic_load_max_i64_release(i64 %offset) nounwind { 4242; CHECK-LABEL: test_atomic_load_max_i64_release: 4243; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_release: 4244; OUTLINE-ATOMICS: // %bb.0: 4245; OUTLINE-ATOMICS-NEXT: adrp x9, var64 4246; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 4247; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4248; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4249; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] 4250; OUTLINE-ATOMICS-NEXT: cmp x8, x0 4251; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt 4252; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 4253; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4254; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4255; OUTLINE-ATOMICS-NEXT: mov x0, x8 4256; OUTLINE-ATOMICS-NEXT: ret 4257 %old = atomicrmw max ptr @var64, i64 %offset release 4258; CHECK-NOT: dmb 4259; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 4260; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 4261 4262; CHECK: ldsmaxl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 4263; CHECK-NOT: dmb 4264 4265 ret i64 %old 4266} 4267 4268define dso_local void @test_atomic_load_max_i32_noret_release(i32 %offset) nounwind { 4269; CHECK-LABEL: test_atomic_load_max_i32_noret_release: 4270; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_release: 4271; OUTLINE-ATOMICS: // %bb.0: 4272; OUTLINE-ATOMICS-NEXT: adrp x8, var32 4273; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 4274; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4275; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4276; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] 4277; OUTLINE-ATOMICS-NEXT: cmp w9, w0 4278; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt 4279; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 4280; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 4281; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4282; OUTLINE-ATOMICS-NEXT: ret 4283 atomicrmw max ptr @var32, i32 %offset release 4284; CHECK-NOT: dmb 4285; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 4286; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 4287 4288; CHECK: ldsmaxl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] 4289; CHECK-NOT: dmb 4290 ret void 4291} 4292 4293define dso_local void @test_atomic_load_max_i64_noret_release(i64 %offset) nounwind { 4294; CHECK-LABEL: test_atomic_load_max_i64_noret_release: 4295; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_release: 4296; OUTLINE-ATOMICS: // %bb.0: 4297; OUTLINE-ATOMICS-NEXT: adrp x8, var64 4298; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 4299; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4300; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4301; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] 4302; OUTLINE-ATOMICS-NEXT: cmp x9, x0 4303; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt 4304; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 4305; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 4306; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4307; OUTLINE-ATOMICS-NEXT: ret 4308 atomicrmw max ptr @var64, i64 %offset release 4309; CHECK-NOT: dmb 4310; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 4311; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 4312 4313; CHECK: ldsmaxl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] 4314; CHECK-NOT: dmb 4315 ret void 4316} 4317 4318define dso_local i8 @test_atomic_load_max_i8_seq_cst(i8 %offset) nounwind { 4319; CHECK-LABEL: test_atomic_load_max_i8_seq_cst: 4320; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i8_seq_cst: 4321; OUTLINE-ATOMICS: // %bb.0: 4322; OUTLINE-ATOMICS-NEXT: adrp x9, var8 4323; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 4324; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4325; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4326; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] 4327; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 4328; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb 4329; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt 4330; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] 4331; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4332; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4333; OUTLINE-ATOMICS-NEXT: mov w0, w8 4334; OUTLINE-ATOMICS-NEXT: ret 4335 %old = atomicrmw max ptr @var8, i8 %offset seq_cst 4336; CHECK-NOT: dmb 4337; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 4338; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 4339 4340; CHECK: ldsmaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4341; CHECK-NOT: dmb 4342 4343 ret i8 %old 4344} 4345 4346define dso_local i16 @test_atomic_load_max_i16_seq_cst(i16 %offset) nounwind { 4347; CHECK-LABEL: test_atomic_load_max_i16_seq_cst: 4348; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i16_seq_cst: 4349; OUTLINE-ATOMICS: // %bb.0: 4350; OUTLINE-ATOMICS-NEXT: adrp x9, var16 4351; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 4352; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4353; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4354; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] 4355; OUTLINE-ATOMICS-NEXT: sxth w8, w10 4356; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth 4357; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, gt 4358; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] 4359; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4360; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4361; OUTLINE-ATOMICS-NEXT: mov w0, w8 4362; OUTLINE-ATOMICS-NEXT: ret 4363 %old = atomicrmw max ptr @var16, i16 %offset seq_cst 4364; CHECK-NOT: dmb 4365; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 4366; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 4367 4368; CHECK: ldsmaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4369; CHECK-NOT: dmb 4370 4371 ret i16 %old 4372} 4373 4374define dso_local i32 @test_atomic_load_max_i32_seq_cst(i32 %offset) nounwind { 4375; CHECK-LABEL: test_atomic_load_max_i32_seq_cst: 4376; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_seq_cst: 4377; OUTLINE-ATOMICS: // %bb.0: 4378; OUTLINE-ATOMICS-NEXT: adrp x9, var32 4379; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 4380; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4381; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4382; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 4383; OUTLINE-ATOMICS-NEXT: cmp w8, w0 4384; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, gt 4385; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 4386; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4387; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4388; OUTLINE-ATOMICS-NEXT: mov w0, w8 4389; OUTLINE-ATOMICS-NEXT: ret 4390 %old = atomicrmw max ptr @var32, i32 %offset seq_cst 4391; CHECK-NOT: dmb 4392; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 4393; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 4394 4395; CHECK: ldsmaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4396; CHECK-NOT: dmb 4397 4398 ret i32 %old 4399} 4400 4401define dso_local i64 @test_atomic_load_max_i64_seq_cst(i64 %offset) nounwind { 4402; CHECK-LABEL: test_atomic_load_max_i64_seq_cst: 4403; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_seq_cst: 4404; OUTLINE-ATOMICS: // %bb.0: 4405; OUTLINE-ATOMICS-NEXT: adrp x9, var64 4406; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 4407; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4408; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4409; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 4410; OUTLINE-ATOMICS-NEXT: cmp x8, x0 4411; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, gt 4412; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 4413; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4414; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4415; OUTLINE-ATOMICS-NEXT: mov x0, x8 4416; OUTLINE-ATOMICS-NEXT: ret 4417 %old = atomicrmw max ptr @var64, i64 %offset seq_cst 4418; CHECK-NOT: dmb 4419; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 4420; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 4421 4422; CHECK: ldsmaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 4423; CHECK-NOT: dmb 4424 4425 ret i64 %old 4426} 4427 4428define dso_local void @test_atomic_load_max_i32_noret_seq_cst(i32 %offset) nounwind { 4429; CHECK-LABEL: test_atomic_load_max_i32_noret_seq_cst: 4430; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i32_noret_seq_cst: 4431; OUTLINE-ATOMICS: // %bb.0: 4432; OUTLINE-ATOMICS-NEXT: adrp x8, var32 4433; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 4434; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4435; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4436; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 4437; OUTLINE-ATOMICS-NEXT: cmp w9, w0 4438; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, gt 4439; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 4440; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 4441; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4442; OUTLINE-ATOMICS-NEXT: ret 4443 atomicrmw max ptr @var32, i32 %offset seq_cst 4444; CHECK-NOT: dmb 4445; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 4446; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 4447 4448; CHECK: ldsmaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 4449; CHECK-NOT: dmb 4450 ret void 4451} 4452 4453define dso_local void @test_atomic_load_max_i64_noret_seq_cst(i64 %offset) nounwind { 4454; CHECK-LABEL: test_atomic_load_max_i64_noret_seq_cst: 4455; OUTLINE-ATOMICS-LABEL: test_atomic_load_max_i64_noret_seq_cst: 4456; OUTLINE-ATOMICS: // %bb.0: 4457; OUTLINE-ATOMICS-NEXT: adrp x8, var64 4458; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 4459; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4460; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4461; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 4462; OUTLINE-ATOMICS-NEXT: cmp x9, x0 4463; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, gt 4464; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 4465; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 4466; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4467; OUTLINE-ATOMICS-NEXT: ret 4468 atomicrmw max ptr @var64, i64 %offset seq_cst 4469; CHECK-NOT: dmb 4470; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 4471; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 4472 4473; CHECK: ldsmaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 4474; CHECK-NOT: dmb 4475 ret void 4476} 4477 4478define dso_local i8 @test_atomic_load_min_i8_acq_rel(i8 %offset) nounwind { 4479; CHECK-LABEL: test_atomic_load_min_i8_acq_rel: 4480; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_acq_rel: 4481; OUTLINE-ATOMICS: // %bb.0: 4482; OUTLINE-ATOMICS-NEXT: adrp x9, var8 4483; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 4484; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4485; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4486; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] 4487; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 4488; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb 4489; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le 4490; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] 4491; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4492; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4493; OUTLINE-ATOMICS-NEXT: mov w0, w8 4494; OUTLINE-ATOMICS-NEXT: ret 4495 %old = atomicrmw min ptr @var8, i8 %offset acq_rel 4496; CHECK-NOT: dmb 4497; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 4498; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 4499 4500; CHECK: ldsminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4501; CHECK-NOT: dmb 4502 4503 ret i8 %old 4504} 4505 4506define dso_local i16 @test_atomic_load_min_i16_acq_rel(i16 %offset) nounwind { 4507; CHECK-LABEL: test_atomic_load_min_i16_acq_rel: 4508; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_acq_rel: 4509; OUTLINE-ATOMICS: // %bb.0: 4510; OUTLINE-ATOMICS-NEXT: adrp x9, var16 4511; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 4512; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4513; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4514; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] 4515; OUTLINE-ATOMICS-NEXT: sxth w8, w10 4516; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth 4517; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le 4518; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] 4519; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4520; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4521; OUTLINE-ATOMICS-NEXT: mov w0, w8 4522; OUTLINE-ATOMICS-NEXT: ret 4523 %old = atomicrmw min ptr @var16, i16 %offset acq_rel 4524; CHECK-NOT: dmb 4525; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 4526; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 4527 4528; CHECK: ldsminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4529; CHECK-NOT: dmb 4530 4531 ret i16 %old 4532} 4533 4534define dso_local i32 @test_atomic_load_min_i32_acq_rel(i32 %offset) nounwind { 4535; CHECK-LABEL: test_atomic_load_min_i32_acq_rel: 4536; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_acq_rel: 4537; OUTLINE-ATOMICS: // %bb.0: 4538; OUTLINE-ATOMICS-NEXT: adrp x9, var32 4539; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 4540; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4541; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4542; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 4543; OUTLINE-ATOMICS-NEXT: cmp w8, w0 4544; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le 4545; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 4546; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4547; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4548; OUTLINE-ATOMICS-NEXT: mov w0, w8 4549; OUTLINE-ATOMICS-NEXT: ret 4550 %old = atomicrmw min ptr @var32, i32 %offset acq_rel 4551; CHECK-NOT: dmb 4552; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 4553; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 4554 4555; CHECK: ldsminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4556; CHECK-NOT: dmb 4557 4558 ret i32 %old 4559} 4560 4561define dso_local i64 @test_atomic_load_min_i64_acq_rel(i64 %offset) nounwind { 4562; CHECK-LABEL: test_atomic_load_min_i64_acq_rel: 4563; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_acq_rel: 4564; OUTLINE-ATOMICS: // %bb.0: 4565; OUTLINE-ATOMICS-NEXT: adrp x9, var64 4566; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 4567; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4568; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4569; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 4570; OUTLINE-ATOMICS-NEXT: cmp x8, x0 4571; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le 4572; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 4573; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4574; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4575; OUTLINE-ATOMICS-NEXT: mov x0, x8 4576; OUTLINE-ATOMICS-NEXT: ret 4577 %old = atomicrmw min ptr @var64, i64 %offset acq_rel 4578; CHECK-NOT: dmb 4579; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 4580; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 4581 4582; CHECK: ldsminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 4583; CHECK-NOT: dmb 4584 4585 ret i64 %old 4586} 4587 4588define dso_local void @test_atomic_load_min_i32_noret_acq_rel(i32 %offset) nounwind { 4589; CHECK-LABEL: test_atomic_load_min_i32_noret_acq_rel: 4590; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_acq_rel: 4591; OUTLINE-ATOMICS: // %bb.0: 4592; OUTLINE-ATOMICS-NEXT: adrp x8, var32 4593; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 4594; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4595; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4596; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 4597; OUTLINE-ATOMICS-NEXT: cmp w9, w0 4598; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le 4599; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 4600; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 4601; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4602; OUTLINE-ATOMICS-NEXT: ret 4603 atomicrmw min ptr @var32, i32 %offset acq_rel 4604; CHECK-NOT: dmb 4605; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 4606; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 4607 4608; CHECK: ldsminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 4609; CHECK-NOT: dmb 4610 ret void 4611} 4612 4613define dso_local void @test_atomic_load_min_i64_noret_acq_rel(i64 %offset) nounwind { 4614; CHECK-LABEL: test_atomic_load_min_i64_noret_acq_rel: 4615; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_acq_rel: 4616; OUTLINE-ATOMICS: // %bb.0: 4617; OUTLINE-ATOMICS-NEXT: adrp x8, var64 4618; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 4619; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4620; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4621; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 4622; OUTLINE-ATOMICS-NEXT: cmp x9, x0 4623; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le 4624; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 4625; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 4626; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4627; OUTLINE-ATOMICS-NEXT: ret 4628 atomicrmw min ptr @var64, i64 %offset acq_rel 4629; CHECK-NOT: dmb 4630; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 4631; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 4632 4633; CHECK: ldsminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 4634; CHECK-NOT: dmb 4635 ret void 4636} 4637 4638define dso_local i8 @test_atomic_load_min_i8_acquire(i8 %offset) nounwind { 4639; CHECK-LABEL: test_atomic_load_min_i8_acquire: 4640; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_acquire: 4641; OUTLINE-ATOMICS: // %bb.0: 4642; OUTLINE-ATOMICS-NEXT: adrp x9, var8 4643; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 4644; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4645; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4646; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] 4647; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 4648; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb 4649; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le 4650; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] 4651; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4652; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4653; OUTLINE-ATOMICS-NEXT: mov w0, w8 4654; OUTLINE-ATOMICS-NEXT: ret 4655 %old = atomicrmw min ptr @var8, i8 %offset acquire 4656; CHECK-NOT: dmb 4657; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 4658; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 4659 4660; CHECK: ldsminab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4661; CHECK-NOT: dmb 4662 4663 ret i8 %old 4664} 4665 4666define dso_local i16 @test_atomic_load_min_i16_acquire(i16 %offset) nounwind { 4667; CHECK-LABEL: test_atomic_load_min_i16_acquire: 4668; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_acquire: 4669; OUTLINE-ATOMICS: // %bb.0: 4670; OUTLINE-ATOMICS-NEXT: adrp x9, var16 4671; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 4672; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4673; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4674; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] 4675; OUTLINE-ATOMICS-NEXT: sxth w8, w10 4676; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth 4677; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le 4678; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] 4679; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4680; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4681; OUTLINE-ATOMICS-NEXT: mov w0, w8 4682; OUTLINE-ATOMICS-NEXT: ret 4683 %old = atomicrmw min ptr @var16, i16 %offset acquire 4684; CHECK-NOT: dmb 4685; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 4686; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 4687 4688; CHECK: ldsminah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4689; CHECK-NOT: dmb 4690 4691 ret i16 %old 4692} 4693 4694define dso_local i32 @test_atomic_load_min_i32_acquire(i32 %offset) nounwind { 4695; CHECK-LABEL: test_atomic_load_min_i32_acquire: 4696; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_acquire: 4697; OUTLINE-ATOMICS: // %bb.0: 4698; OUTLINE-ATOMICS-NEXT: adrp x9, var32 4699; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 4700; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4701; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4702; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 4703; OUTLINE-ATOMICS-NEXT: cmp w8, w0 4704; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le 4705; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] 4706; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4707; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4708; OUTLINE-ATOMICS-NEXT: mov w0, w8 4709; OUTLINE-ATOMICS-NEXT: ret 4710 %old = atomicrmw min ptr @var32, i32 %offset acquire 4711; CHECK-NOT: dmb 4712; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 4713; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 4714 4715; CHECK: ldsmina w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4716; CHECK-NOT: dmb 4717 4718 ret i32 %old 4719} 4720 4721define dso_local i64 @test_atomic_load_min_i64_acquire(i64 %offset) nounwind { 4722; CHECK-LABEL: test_atomic_load_min_i64_acquire: 4723; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_acquire: 4724; OUTLINE-ATOMICS: // %bb.0: 4725; OUTLINE-ATOMICS-NEXT: adrp x9, var64 4726; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 4727; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4728; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4729; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 4730; OUTLINE-ATOMICS-NEXT: cmp x8, x0 4731; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le 4732; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] 4733; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4734; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4735; OUTLINE-ATOMICS-NEXT: mov x0, x8 4736; OUTLINE-ATOMICS-NEXT: ret 4737 %old = atomicrmw min ptr @var64, i64 %offset acquire 4738; CHECK-NOT: dmb 4739; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 4740; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 4741 4742; CHECK: ldsmina x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 4743; CHECK-NOT: dmb 4744 4745 ret i64 %old 4746} 4747 4748define dso_local void @test_atomic_load_min_i32_noret_acquire(i32 %offset) nounwind { 4749; CHECK-LABEL: test_atomic_load_min_i32_noret_acquire: 4750; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_acquire: 4751; OUTLINE-ATOMICS: // %bb.0: 4752; OUTLINE-ATOMICS-NEXT: adrp x8, var32 4753; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 4754; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4755; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4756; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 4757; OUTLINE-ATOMICS-NEXT: cmp w9, w0 4758; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le 4759; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] 4760; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 4761; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4762; OUTLINE-ATOMICS-NEXT: ret 4763 atomicrmw min ptr @var32, i32 %offset acquire 4764; CHECK-NOT: dmb 4765; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 4766; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 4767 4768; CHECK: ldsmina w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 4769; CHECK-NOT: dmb 4770 ret void 4771} 4772 4773define dso_local void @test_atomic_load_min_i64_noret_acquire(i64 %offset) nounwind { 4774; CHECK-LABEL: test_atomic_load_min_i64_noret_acquire: 4775; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_acquire: 4776; OUTLINE-ATOMICS: // %bb.0: 4777; OUTLINE-ATOMICS-NEXT: adrp x8, var64 4778; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 4779; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4780; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4781; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 4782; OUTLINE-ATOMICS-NEXT: cmp x9, x0 4783; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le 4784; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] 4785; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 4786; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4787; OUTLINE-ATOMICS-NEXT: ret 4788 atomicrmw min ptr @var64, i64 %offset acquire 4789; CHECK-NOT: dmb 4790; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 4791; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 4792 4793; CHECK: ldsmina x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 4794; CHECK-NOT: dmb 4795 ret void 4796} 4797 4798define dso_local i8 @test_atomic_load_min_i8_monotonic(i8 %offset) nounwind { 4799; CHECK-LABEL: test_atomic_load_min_i8_monotonic: 4800; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_monotonic: 4801; OUTLINE-ATOMICS: // %bb.0: 4802; OUTLINE-ATOMICS-NEXT: adrp x9, var8 4803; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 4804; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4805; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4806; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9] 4807; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 4808; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb 4809; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le 4810; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x9] 4811; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4812; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4813; OUTLINE-ATOMICS-NEXT: mov w0, w8 4814; OUTLINE-ATOMICS-NEXT: ret 4815 %old = atomicrmw min ptr @var8, i8 %offset monotonic 4816; CHECK-NOT: dmb 4817; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 4818; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 4819 4820; CHECK: ldsminb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4821; CHECK-NOT: dmb 4822 4823 ret i8 %old 4824} 4825 4826define dso_local i16 @test_atomic_load_min_i16_monotonic(i16 %offset) nounwind { 4827; CHECK-LABEL: test_atomic_load_min_i16_monotonic: 4828; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_monotonic: 4829; OUTLINE-ATOMICS: // %bb.0: 4830; OUTLINE-ATOMICS-NEXT: adrp x9, var16 4831; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 4832; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4833; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4834; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9] 4835; OUTLINE-ATOMICS-NEXT: sxth w8, w10 4836; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth 4837; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le 4838; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x9] 4839; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4840; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4841; OUTLINE-ATOMICS-NEXT: mov w0, w8 4842; OUTLINE-ATOMICS-NEXT: ret 4843 %old = atomicrmw min ptr @var16, i16 %offset monotonic 4844; CHECK-NOT: dmb 4845; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 4846; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 4847 4848; CHECK: ldsminh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4849; CHECK-NOT: dmb 4850 4851 ret i16 %old 4852} 4853 4854define dso_local i32 @test_atomic_load_min_i32_monotonic(i32 %offset) nounwind { 4855; CHECK-LABEL: test_atomic_load_min_i32_monotonic: 4856; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_monotonic: 4857; OUTLINE-ATOMICS: // %bb.0: 4858; OUTLINE-ATOMICS-NEXT: adrp x9, var32 4859; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 4860; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4861; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4862; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] 4863; OUTLINE-ATOMICS-NEXT: cmp w8, w0 4864; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le 4865; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] 4866; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4867; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4868; OUTLINE-ATOMICS-NEXT: mov w0, w8 4869; OUTLINE-ATOMICS-NEXT: ret 4870 %old = atomicrmw min ptr @var32, i32 %offset monotonic 4871; CHECK-NOT: dmb 4872; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 4873; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 4874 4875; CHECK: ldsmin w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4876; CHECK-NOT: dmb 4877 4878 ret i32 %old 4879} 4880 4881define dso_local i64 @test_atomic_load_min_i64_monotonic(i64 %offset) nounwind { 4882; CHECK-LABEL: test_atomic_load_min_i64_monotonic: 4883; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_monotonic: 4884; OUTLINE-ATOMICS: // %bb.0: 4885; OUTLINE-ATOMICS-NEXT: adrp x9, var64 4886; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 4887; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4888; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4889; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] 4890; OUTLINE-ATOMICS-NEXT: cmp x8, x0 4891; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le 4892; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] 4893; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4894; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4895; OUTLINE-ATOMICS-NEXT: mov x0, x8 4896; OUTLINE-ATOMICS-NEXT: ret 4897 %old = atomicrmw min ptr @var64, i64 %offset monotonic 4898; CHECK-NOT: dmb 4899; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 4900; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 4901 4902; CHECK: ldsmin x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 4903; CHECK-NOT: dmb 4904 4905 ret i64 %old 4906} 4907 4908define dso_local void @test_atomic_load_min_i32_noret_monotonic(i32 %offset) nounwind { 4909; CHECK-LABEL: test_atomic_load_min_i32_noret_monotonic: 4910; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_monotonic: 4911; OUTLINE-ATOMICS: // %bb.0: 4912; OUTLINE-ATOMICS-NEXT: adrp x8, var32 4913; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 4914; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4915; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4916; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] 4917; OUTLINE-ATOMICS-NEXT: cmp w9, w0 4918; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le 4919; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] 4920; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 4921; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4922; OUTLINE-ATOMICS-NEXT: ret 4923 atomicrmw min ptr @var32, i32 %offset monotonic 4924; CHECK-NOT: dmb 4925; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 4926; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 4927 4928; CHECK: ldsmin w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] 4929; CHECK-NOT: dmb 4930 ret void 4931} 4932 4933define dso_local void @test_atomic_load_min_i64_noret_monotonic(i64 %offset) nounwind { 4934; CHECK-LABEL: test_atomic_load_min_i64_noret_monotonic: 4935; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_monotonic: 4936; OUTLINE-ATOMICS: // %bb.0: 4937; OUTLINE-ATOMICS-NEXT: adrp x8, var64 4938; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 4939; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4940; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4941; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] 4942; OUTLINE-ATOMICS-NEXT: cmp x9, x0 4943; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le 4944; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] 4945; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 4946; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4947; OUTLINE-ATOMICS-NEXT: ret 4948 atomicrmw min ptr @var64, i64 %offset monotonic 4949; CHECK-NOT: dmb 4950; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 4951; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 4952 4953; CHECK: ldsmin x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] 4954; CHECK-NOT: dmb 4955 ret void 4956} 4957 4958define dso_local i8 @test_atomic_load_min_i8_release(i8 %offset) nounwind { 4959; CHECK-LABEL: test_atomic_load_min_i8_release: 4960; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_release: 4961; OUTLINE-ATOMICS: // %bb.0: 4962; OUTLINE-ATOMICS-NEXT: adrp x9, var8 4963; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 4964; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4965; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4966; OUTLINE-ATOMICS-NEXT: ldxrb w10, [x9] 4967; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 4968; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb 4969; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le 4970; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] 4971; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 4972; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 4973; OUTLINE-ATOMICS-NEXT: mov w0, w8 4974; OUTLINE-ATOMICS-NEXT: ret 4975 %old = atomicrmw min ptr @var8, i8 %offset release 4976; CHECK-NOT: dmb 4977; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 4978; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 4979 4980; CHECK: ldsminlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 4981; CHECK-NOT: dmb 4982 4983 ret i8 %old 4984} 4985 4986define dso_local i16 @test_atomic_load_min_i16_release(i16 %offset) nounwind { 4987; CHECK-LABEL: test_atomic_load_min_i16_release: 4988; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_release: 4989; OUTLINE-ATOMICS: // %bb.0: 4990; OUTLINE-ATOMICS-NEXT: adrp x9, var16 4991; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 4992; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 4993; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 4994; OUTLINE-ATOMICS-NEXT: ldxrh w10, [x9] 4995; OUTLINE-ATOMICS-NEXT: sxth w8, w10 4996; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth 4997; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le 4998; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] 4999; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 5000; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 5001; OUTLINE-ATOMICS-NEXT: mov w0, w8 5002; OUTLINE-ATOMICS-NEXT: ret 5003 %old = atomicrmw min ptr @var16, i16 %offset release 5004; CHECK-NOT: dmb 5005; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 5006; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 5007 5008; CHECK: ldsminlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5009; CHECK-NOT: dmb 5010 5011 ret i16 %old 5012} 5013 5014define dso_local i32 @test_atomic_load_min_i32_release(i32 %offset) nounwind { 5015; CHECK-LABEL: test_atomic_load_min_i32_release: 5016; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_release: 5017; OUTLINE-ATOMICS: // %bb.0: 5018; OUTLINE-ATOMICS-NEXT: adrp x9, var32 5019; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 5020; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 5021; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 5022; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] 5023; OUTLINE-ATOMICS-NEXT: cmp w8, w0 5024; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le 5025; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 5026; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 5027; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 5028; OUTLINE-ATOMICS-NEXT: mov w0, w8 5029; OUTLINE-ATOMICS-NEXT: ret 5030 %old = atomicrmw min ptr @var32, i32 %offset release 5031; CHECK-NOT: dmb 5032; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5033; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5034 5035; CHECK: ldsminl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5036; CHECK-NOT: dmb 5037 5038 ret i32 %old 5039} 5040 5041define dso_local i64 @test_atomic_load_min_i64_release(i64 %offset) nounwind { 5042; CHECK-LABEL: test_atomic_load_min_i64_release: 5043; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_release: 5044; OUTLINE-ATOMICS: // %bb.0: 5045; OUTLINE-ATOMICS-NEXT: adrp x9, var64 5046; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 5047; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 5048; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 5049; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] 5050; OUTLINE-ATOMICS-NEXT: cmp x8, x0 5051; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le 5052; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 5053; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 5054; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 5055; OUTLINE-ATOMICS-NEXT: mov x0, x8 5056; OUTLINE-ATOMICS-NEXT: ret 5057 %old = atomicrmw min ptr @var64, i64 %offset release 5058; CHECK-NOT: dmb 5059; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5060; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5061 5062; CHECK: ldsminl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 5063; CHECK-NOT: dmb 5064 5065 ret i64 %old 5066} 5067 5068define dso_local void @test_atomic_load_min_i32_noret_release(i32 %offset) nounwind { 5069; CHECK-LABEL: test_atomic_load_min_i32_noret_release: 5070; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_release: 5071; OUTLINE-ATOMICS: // %bb.0: 5072; OUTLINE-ATOMICS-NEXT: adrp x8, var32 5073; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 5074; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 5075; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 5076; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] 5077; OUTLINE-ATOMICS-NEXT: cmp w9, w0 5078; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le 5079; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 5080; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 5081; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 5082; OUTLINE-ATOMICS-NEXT: ret 5083 atomicrmw min ptr @var32, i32 %offset release 5084; CHECK-NOT: dmb 5085; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5086; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5087 5088; CHECK: ldsminl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] 5089; CHECK-NOT: dmb 5090 ret void 5091} 5092 5093define dso_local void @test_atomic_load_min_i64_noret_release(i64 %offset) nounwind { 5094; CHECK-LABEL: test_atomic_load_min_i64_noret_release: 5095; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_release: 5096; OUTLINE-ATOMICS: // %bb.0: 5097; OUTLINE-ATOMICS-NEXT: adrp x8, var64 5098; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 5099; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 5100; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 5101; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] 5102; OUTLINE-ATOMICS-NEXT: cmp x9, x0 5103; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le 5104; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 5105; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 5106; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 5107; OUTLINE-ATOMICS-NEXT: ret 5108 atomicrmw min ptr @var64, i64 %offset release 5109; CHECK-NOT: dmb 5110; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5111; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5112 5113; CHECK: ldsminl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] 5114; CHECK-NOT: dmb 5115 ret void 5116} 5117 5118define dso_local i8 @test_atomic_load_min_i8_seq_cst(i8 %offset) nounwind { 5119; CHECK-LABEL: test_atomic_load_min_i8_seq_cst: 5120; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i8_seq_cst: 5121; OUTLINE-ATOMICS: // %bb.0: 5122; OUTLINE-ATOMICS-NEXT: adrp x9, var8 5123; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var8 5124; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 5125; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 5126; OUTLINE-ATOMICS-NEXT: ldaxrb w10, [x9] 5127; OUTLINE-ATOMICS-NEXT: sxtb w8, w10 5128; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxtb 5129; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le 5130; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x9] 5131; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 5132; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 5133; OUTLINE-ATOMICS-NEXT: mov w0, w8 5134; OUTLINE-ATOMICS-NEXT: ret 5135 %old = atomicrmw min ptr @var8, i8 %offset seq_cst 5136; CHECK-NOT: dmb 5137; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 5138; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 5139 5140; CHECK: ldsminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5141; CHECK-NOT: dmb 5142 5143 ret i8 %old 5144} 5145 5146define dso_local i16 @test_atomic_load_min_i16_seq_cst(i16 %offset) nounwind { 5147; CHECK-LABEL: test_atomic_load_min_i16_seq_cst: 5148; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i16_seq_cst: 5149; OUTLINE-ATOMICS: // %bb.0: 5150; OUTLINE-ATOMICS-NEXT: adrp x9, var16 5151; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var16 5152; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 5153; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 5154; OUTLINE-ATOMICS-NEXT: ldaxrh w10, [x9] 5155; OUTLINE-ATOMICS-NEXT: sxth w8, w10 5156; OUTLINE-ATOMICS-NEXT: cmp w8, w0, sxth 5157; OUTLINE-ATOMICS-NEXT: csel w10, w10, w0, le 5158; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x9] 5159; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 5160; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 5161; OUTLINE-ATOMICS-NEXT: mov w0, w8 5162; OUTLINE-ATOMICS-NEXT: ret 5163 %old = atomicrmw min ptr @var16, i16 %offset seq_cst 5164; CHECK-NOT: dmb 5165; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 5166; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 5167 5168; CHECK: ldsminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5169; CHECK-NOT: dmb 5170 5171 ret i16 %old 5172} 5173 5174define dso_local i32 @test_atomic_load_min_i32_seq_cst(i32 %offset) nounwind { 5175; CHECK-LABEL: test_atomic_load_min_i32_seq_cst: 5176; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_seq_cst: 5177; OUTLINE-ATOMICS: // %bb.0: 5178; OUTLINE-ATOMICS-NEXT: adrp x9, var32 5179; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 5180; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 5181; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 5182; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 5183; OUTLINE-ATOMICS-NEXT: cmp w8, w0 5184; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, le 5185; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 5186; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 5187; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 5188; OUTLINE-ATOMICS-NEXT: mov w0, w8 5189; OUTLINE-ATOMICS-NEXT: ret 5190 %old = atomicrmw min ptr @var32, i32 %offset seq_cst 5191; CHECK-NOT: dmb 5192; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5193; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5194 5195; CHECK: ldsminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5196; CHECK-NOT: dmb 5197 5198 ret i32 %old 5199} 5200 5201define dso_local i64 @test_atomic_load_min_i64_seq_cst(i64 %offset) nounwind { 5202; CHECK-LABEL: test_atomic_load_min_i64_seq_cst: 5203; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_seq_cst: 5204; OUTLINE-ATOMICS: // %bb.0: 5205; OUTLINE-ATOMICS-NEXT: adrp x9, var64 5206; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 5207; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 5208; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 5209; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 5210; OUTLINE-ATOMICS-NEXT: cmp x8, x0 5211; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, le 5212; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 5213; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 5214; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 5215; OUTLINE-ATOMICS-NEXT: mov x0, x8 5216; OUTLINE-ATOMICS-NEXT: ret 5217 %old = atomicrmw min ptr @var64, i64 %offset seq_cst 5218; CHECK-NOT: dmb 5219; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5220; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5221 5222; CHECK: ldsminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 5223; CHECK-NOT: dmb 5224 5225 ret i64 %old 5226} 5227 5228define dso_local void @test_atomic_load_min_i32_noret_seq_cst(i32 %offset) nounwind { 5229; CHECK-LABEL: test_atomic_load_min_i32_noret_seq_cst: 5230; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i32_noret_seq_cst: 5231; OUTLINE-ATOMICS: // %bb.0: 5232; OUTLINE-ATOMICS-NEXT: adrp x8, var32 5233; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 5234; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 5235; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 5236; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 5237; OUTLINE-ATOMICS-NEXT: cmp w9, w0 5238; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, le 5239; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 5240; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 5241; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 5242; OUTLINE-ATOMICS-NEXT: ret 5243 atomicrmw min ptr @var32, i32 %offset seq_cst 5244; CHECK-NOT: dmb 5245; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5246; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5247 5248; CHECK: ldsminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 5249; CHECK-NOT: dmb 5250 ret void 5251} 5252 5253define dso_local void @test_atomic_load_min_i64_noret_seq_cst(i64 %offset) nounwind { 5254; CHECK-LABEL: test_atomic_load_min_i64_noret_seq_cst: 5255; OUTLINE-ATOMICS-LABEL: test_atomic_load_min_i64_noret_seq_cst: 5256; OUTLINE-ATOMICS: // %bb.0: 5257; OUTLINE-ATOMICS-NEXT: adrp x8, var64 5258; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 5259; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 5260; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 5261; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 5262; OUTLINE-ATOMICS-NEXT: cmp x9, x0 5263; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, le 5264; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 5265; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 5266; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 5267; OUTLINE-ATOMICS-NEXT: ret 5268 atomicrmw min ptr @var64, i64 %offset seq_cst 5269; CHECK-NOT: dmb 5270; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5271; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5272 5273; CHECK: ldsminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 5274; CHECK-NOT: dmb 5275 ret void 5276} 5277 5278define dso_local i8 @test_atomic_load_or_i8_acq_rel(i8 %offset) nounwind { 5279; CHECK-LABEL: test_atomic_load_or_i8_acq_rel: 5280; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i8_acq_rel: 5281; OUTLINE-ATOMICS: // %bb.0: 5282; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5283; OUTLINE-ATOMICS-NEXT: adrp x1, var8 5284; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 5285; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset1_acq_rel 5286; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5287; OUTLINE-ATOMICS-NEXT: ret 5288 %old = atomicrmw or ptr @var8, i8 %offset acq_rel 5289; CHECK-NOT: dmb 5290; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 5291; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 5292 5293; CHECK: ldsetalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5294; CHECK-NOT: dmb 5295 5296 ret i8 %old 5297} 5298 5299define dso_local i16 @test_atomic_load_or_i16_acq_rel(i16 %offset) nounwind { 5300; CHECK-LABEL: test_atomic_load_or_i16_acq_rel: 5301; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i16_acq_rel: 5302; OUTLINE-ATOMICS: // %bb.0: 5303; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5304; OUTLINE-ATOMICS-NEXT: adrp x1, var16 5305; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 5306; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset2_acq_rel 5307; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5308; OUTLINE-ATOMICS-NEXT: ret 5309 %old = atomicrmw or ptr @var16, i16 %offset acq_rel 5310; CHECK-NOT: dmb 5311; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 5312; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 5313 5314; CHECK: ldsetalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5315; CHECK-NOT: dmb 5316 5317 ret i16 %old 5318} 5319 5320define dso_local i32 @test_atomic_load_or_i32_acq_rel(i32 %offset) nounwind { 5321; CHECK-LABEL: test_atomic_load_or_i32_acq_rel: 5322; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_acq_rel: 5323; OUTLINE-ATOMICS: // %bb.0: 5324; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5325; OUTLINE-ATOMICS-NEXT: adrp x1, var32 5326; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 5327; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq_rel 5328; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5329; OUTLINE-ATOMICS-NEXT: ret 5330 %old = atomicrmw or ptr @var32, i32 %offset acq_rel 5331; CHECK-NOT: dmb 5332; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5333; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5334 5335; CHECK: ldsetal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5336; CHECK-NOT: dmb 5337 5338 ret i32 %old 5339} 5340 5341define dso_local i64 @test_atomic_load_or_i64_acq_rel(i64 %offset) nounwind { 5342; CHECK-LABEL: test_atomic_load_or_i64_acq_rel: 5343; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_acq_rel: 5344; OUTLINE-ATOMICS: // %bb.0: 5345; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5346; OUTLINE-ATOMICS-NEXT: adrp x1, var64 5347; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 5348; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq_rel 5349; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5350; OUTLINE-ATOMICS-NEXT: ret 5351 %old = atomicrmw or ptr @var64, i64 %offset acq_rel 5352; CHECK-NOT: dmb 5353; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5354; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5355 5356; CHECK: ldsetal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 5357; CHECK-NOT: dmb 5358 5359 ret i64 %old 5360} 5361 5362define dso_local void @test_atomic_load_or_i32_noret_acq_rel(i32 %offset) nounwind { 5363; CHECK-LABEL: test_atomic_load_or_i32_noret_acq_rel: 5364; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_noret_acq_rel: 5365; OUTLINE-ATOMICS: // %bb.0: 5366; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5367; OUTLINE-ATOMICS-NEXT: adrp x1, var32 5368; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 5369; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq_rel 5370; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5371; OUTLINE-ATOMICS-NEXT: ret 5372 atomicrmw or ptr @var32, i32 %offset acq_rel 5373; CHECK-NOT: dmb 5374; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5375; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5376 5377; CHECK: ldsetal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 5378; CHECK-NOT: dmb 5379 ret void 5380} 5381 5382define dso_local void @test_atomic_load_or_i64_noret_acq_rel(i64 %offset) nounwind { 5383; CHECK-LABEL: test_atomic_load_or_i64_noret_acq_rel: 5384; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_noret_acq_rel: 5385; OUTLINE-ATOMICS: // %bb.0: 5386; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5387; OUTLINE-ATOMICS-NEXT: adrp x1, var64 5388; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 5389; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq_rel 5390; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5391; OUTLINE-ATOMICS-NEXT: ret 5392 atomicrmw or ptr @var64, i64 %offset acq_rel 5393; CHECK-NOT: dmb 5394; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5395; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5396 5397; CHECK: ldsetal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 5398; CHECK-NOT: dmb 5399 ret void 5400} 5401 5402define dso_local i8 @test_atomic_load_or_i8_acquire(i8 %offset) nounwind { 5403; CHECK-LABEL: test_atomic_load_or_i8_acquire: 5404; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i8_acquire: 5405; OUTLINE-ATOMICS: // %bb.0: 5406; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5407; OUTLINE-ATOMICS-NEXT: adrp x1, var8 5408; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 5409; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset1_acq 5410; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5411; OUTLINE-ATOMICS-NEXT: ret 5412 %old = atomicrmw or ptr @var8, i8 %offset acquire 5413; CHECK-NOT: dmb 5414; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 5415; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 5416 5417; CHECK: ldsetab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5418; CHECK-NOT: dmb 5419 5420 ret i8 %old 5421} 5422 5423define dso_local i16 @test_atomic_load_or_i16_acquire(i16 %offset) nounwind { 5424; CHECK-LABEL: test_atomic_load_or_i16_acquire: 5425; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i16_acquire: 5426; OUTLINE-ATOMICS: // %bb.0: 5427; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5428; OUTLINE-ATOMICS-NEXT: adrp x1, var16 5429; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 5430; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset2_acq 5431; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5432; OUTLINE-ATOMICS-NEXT: ret 5433 %old = atomicrmw or ptr @var16, i16 %offset acquire 5434; CHECK-NOT: dmb 5435; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 5436; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 5437 5438; CHECK: ldsetah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5439; CHECK-NOT: dmb 5440 5441 ret i16 %old 5442} 5443 5444define dso_local i32 @test_atomic_load_or_i32_acquire(i32 %offset) nounwind { 5445; CHECK-LABEL: test_atomic_load_or_i32_acquire: 5446; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_acquire: 5447; OUTLINE-ATOMICS: // %bb.0: 5448; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5449; OUTLINE-ATOMICS-NEXT: adrp x1, var32 5450; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 5451; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq 5452; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5453; OUTLINE-ATOMICS-NEXT: ret 5454 %old = atomicrmw or ptr @var32, i32 %offset acquire 5455; CHECK-NOT: dmb 5456; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5457; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5458 5459; CHECK: ldseta w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5460; CHECK-NOT: dmb 5461 5462 ret i32 %old 5463} 5464 5465define dso_local i64 @test_atomic_load_or_i64_acquire(i64 %offset) nounwind { 5466; CHECK-LABEL: test_atomic_load_or_i64_acquire: 5467; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_acquire: 5468; OUTLINE-ATOMICS: // %bb.0: 5469; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5470; OUTLINE-ATOMICS-NEXT: adrp x1, var64 5471; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 5472; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq 5473; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5474; OUTLINE-ATOMICS-NEXT: ret 5475 %old = atomicrmw or ptr @var64, i64 %offset acquire 5476; CHECK-NOT: dmb 5477; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5478; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5479 5480; CHECK: ldseta x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 5481; CHECK-NOT: dmb 5482 5483 ret i64 %old 5484} 5485 5486define dso_local void @test_atomic_load_or_i32_noret_acquire(i32 %offset) nounwind { 5487; CHECK-LABEL: test_atomic_load_or_i32_noret_acquire: 5488; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_noret_acquire: 5489; OUTLINE-ATOMICS: // %bb.0: 5490; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5491; OUTLINE-ATOMICS-NEXT: adrp x1, var32 5492; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 5493; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq 5494; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5495; OUTLINE-ATOMICS-NEXT: ret 5496 atomicrmw or ptr @var32, i32 %offset acquire 5497; CHECK-NOT: dmb 5498; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5499; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5500 5501; CHECK: ldseta w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 5502; CHECK-NOT: dmb 5503 ret void 5504} 5505 5506define dso_local void @test_atomic_load_or_i64_noret_acquire(i64 %offset) nounwind { 5507; CHECK-LABEL: test_atomic_load_or_i64_noret_acquire: 5508; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_noret_acquire: 5509; OUTLINE-ATOMICS: // %bb.0: 5510; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5511; OUTLINE-ATOMICS-NEXT: adrp x1, var64 5512; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 5513; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq 5514; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5515; OUTLINE-ATOMICS-NEXT: ret 5516 atomicrmw or ptr @var64, i64 %offset acquire 5517; CHECK-NOT: dmb 5518; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5519; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5520 5521; CHECK: ldseta x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 5522; CHECK-NOT: dmb 5523 ret void 5524} 5525 5526define dso_local i8 @test_atomic_load_or_i8_monotonic(i8 %offset) nounwind { 5527; CHECK-LABEL: test_atomic_load_or_i8_monotonic: 5528; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i8_monotonic: 5529; OUTLINE-ATOMICS: // %bb.0: 5530; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5531; OUTLINE-ATOMICS-NEXT: adrp x1, var8 5532; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 5533; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset1_relax 5534; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5535; OUTLINE-ATOMICS-NEXT: ret 5536 %old = atomicrmw or ptr @var8, i8 %offset monotonic 5537; CHECK-NOT: dmb 5538; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 5539; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 5540 5541; CHECK: ldsetb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5542; CHECK-NOT: dmb 5543 5544 ret i8 %old 5545} 5546 5547define dso_local i16 @test_atomic_load_or_i16_monotonic(i16 %offset) nounwind { 5548; CHECK-LABEL: test_atomic_load_or_i16_monotonic: 5549; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i16_monotonic: 5550; OUTLINE-ATOMICS: // %bb.0: 5551; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5552; OUTLINE-ATOMICS-NEXT: adrp x1, var16 5553; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 5554; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset2_relax 5555; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5556; OUTLINE-ATOMICS-NEXT: ret 5557 %old = atomicrmw or ptr @var16, i16 %offset monotonic 5558; CHECK-NOT: dmb 5559; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 5560; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 5561 5562; CHECK: ldseth w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5563; CHECK-NOT: dmb 5564 5565 ret i16 %old 5566} 5567 5568define dso_local i32 @test_atomic_load_or_i32_monotonic(i32 %offset) nounwind { 5569; CHECK-LABEL: test_atomic_load_or_i32_monotonic: 5570; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_monotonic: 5571; OUTLINE-ATOMICS: // %bb.0: 5572; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5573; OUTLINE-ATOMICS-NEXT: adrp x1, var32 5574; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 5575; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_relax 5576; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5577; OUTLINE-ATOMICS-NEXT: ret 5578 %old = atomicrmw or ptr @var32, i32 %offset monotonic 5579; CHECK-NOT: dmb 5580; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5581; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5582 5583; CHECK: ldset w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5584; CHECK-NOT: dmb 5585 5586 ret i32 %old 5587} 5588 5589define dso_local i64 @test_atomic_load_or_i64_monotonic(i64 %offset) nounwind { 5590; CHECK-LABEL: test_atomic_load_or_i64_monotonic: 5591; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_monotonic: 5592; OUTLINE-ATOMICS: // %bb.0: 5593; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5594; OUTLINE-ATOMICS-NEXT: adrp x1, var64 5595; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 5596; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_relax 5597; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5598; OUTLINE-ATOMICS-NEXT: ret 5599 %old = atomicrmw or ptr @var64, i64 %offset monotonic 5600; CHECK-NOT: dmb 5601; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5602; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5603 5604; CHECK: ldset x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 5605; CHECK-NOT: dmb 5606 5607 ret i64 %old 5608} 5609 5610define dso_local void @test_atomic_load_or_i32_noret_monotonic(i32 %offset) nounwind { 5611; CHECK-LABEL: test_atomic_load_or_i32_noret_monotonic: 5612; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_noret_monotonic: 5613; OUTLINE-ATOMICS: // %bb.0: 5614; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5615; OUTLINE-ATOMICS-NEXT: adrp x1, var32 5616; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 5617; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_relax 5618; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5619; OUTLINE-ATOMICS-NEXT: ret 5620 atomicrmw or ptr @var32, i32 %offset monotonic 5621; CHECK-NOT: dmb 5622; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5623; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5624 5625; CHECK: ldset w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] 5626; CHECK-NOT: dmb 5627 ret void 5628} 5629 5630define dso_local void @test_atomic_load_or_i64_noret_monotonic(i64 %offset) nounwind { 5631; CHECK-LABEL: test_atomic_load_or_i64_noret_monotonic: 5632; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_noret_monotonic: 5633; OUTLINE-ATOMICS: // %bb.0: 5634; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5635; OUTLINE-ATOMICS-NEXT: adrp x1, var64 5636; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 5637; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_relax 5638; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5639; OUTLINE-ATOMICS-NEXT: ret 5640 atomicrmw or ptr @var64, i64 %offset monotonic 5641; CHECK-NOT: dmb 5642; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5643; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5644 5645; CHECK: ldset x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] 5646; CHECK-NOT: dmb 5647 ret void 5648} 5649 5650define dso_local i8 @test_atomic_load_or_i8_release(i8 %offset) nounwind { 5651; CHECK-LABEL: test_atomic_load_or_i8_release: 5652; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i8_release: 5653; OUTLINE-ATOMICS: // %bb.0: 5654; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5655; OUTLINE-ATOMICS-NEXT: adrp x1, var8 5656; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 5657; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset1_rel 5658; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5659; OUTLINE-ATOMICS-NEXT: ret 5660 %old = atomicrmw or ptr @var8, i8 %offset release 5661; CHECK-NOT: dmb 5662; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 5663; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 5664 5665; CHECK: ldsetlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5666; CHECK-NOT: dmb 5667 5668 ret i8 %old 5669} 5670 5671define dso_local i16 @test_atomic_load_or_i16_release(i16 %offset) nounwind { 5672; CHECK-LABEL: test_atomic_load_or_i16_release: 5673; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i16_release: 5674; OUTLINE-ATOMICS: // %bb.0: 5675; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5676; OUTLINE-ATOMICS-NEXT: adrp x1, var16 5677; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 5678; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset2_rel 5679; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5680; OUTLINE-ATOMICS-NEXT: ret 5681 %old = atomicrmw or ptr @var16, i16 %offset release 5682; CHECK-NOT: dmb 5683; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 5684; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 5685 5686; CHECK: ldsetlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5687; CHECK-NOT: dmb 5688 5689 ret i16 %old 5690} 5691 5692define dso_local i32 @test_atomic_load_or_i32_release(i32 %offset) nounwind { 5693; CHECK-LABEL: test_atomic_load_or_i32_release: 5694; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_release: 5695; OUTLINE-ATOMICS: // %bb.0: 5696; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5697; OUTLINE-ATOMICS-NEXT: adrp x1, var32 5698; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 5699; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_rel 5700; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5701; OUTLINE-ATOMICS-NEXT: ret 5702 %old = atomicrmw or ptr @var32, i32 %offset release 5703; CHECK-NOT: dmb 5704; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5705; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5706 5707; CHECK: ldsetl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5708; CHECK-NOT: dmb 5709 5710 ret i32 %old 5711} 5712 5713define dso_local i64 @test_atomic_load_or_i64_release(i64 %offset) nounwind { 5714; CHECK-LABEL: test_atomic_load_or_i64_release: 5715; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_release: 5716; OUTLINE-ATOMICS: // %bb.0: 5717; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5718; OUTLINE-ATOMICS-NEXT: adrp x1, var64 5719; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 5720; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_rel 5721; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5722; OUTLINE-ATOMICS-NEXT: ret 5723 %old = atomicrmw or ptr @var64, i64 %offset release 5724; CHECK-NOT: dmb 5725; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5726; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5727 5728; CHECK: ldsetl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 5729; CHECK-NOT: dmb 5730 5731 ret i64 %old 5732} 5733 5734define dso_local void @test_atomic_load_or_i32_noret_release(i32 %offset) nounwind { 5735; CHECK-LABEL: test_atomic_load_or_i32_noret_release: 5736; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_noret_release: 5737; OUTLINE-ATOMICS: // %bb.0: 5738; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5739; OUTLINE-ATOMICS-NEXT: adrp x1, var32 5740; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 5741; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_rel 5742; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5743; OUTLINE-ATOMICS-NEXT: ret 5744 atomicrmw or ptr @var32, i32 %offset release 5745; CHECK-NOT: dmb 5746; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5747; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5748 5749; CHECK: ldsetl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] 5750; CHECK-NOT: dmb 5751 ret void 5752} 5753 5754define dso_local void @test_atomic_load_or_i64_noret_release(i64 %offset) nounwind { 5755; CHECK-LABEL: test_atomic_load_or_i64_noret_release: 5756; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_noret_release: 5757; OUTLINE-ATOMICS: // %bb.0: 5758; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5759; OUTLINE-ATOMICS-NEXT: adrp x1, var64 5760; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 5761; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_rel 5762; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5763; OUTLINE-ATOMICS-NEXT: ret 5764 atomicrmw or ptr @var64, i64 %offset release 5765; CHECK-NOT: dmb 5766; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5767; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5768 5769; CHECK: ldsetl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] 5770; CHECK-NOT: dmb 5771 ret void 5772} 5773 5774define dso_local i8 @test_atomic_load_or_i8_seq_cst(i8 %offset) nounwind { 5775; CHECK-LABEL: test_atomic_load_or_i8_seq_cst: 5776; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i8_seq_cst: 5777; OUTLINE-ATOMICS: // %bb.0: 5778; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5779; OUTLINE-ATOMICS-NEXT: adrp x1, var8 5780; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 5781; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset1_acq_rel 5782; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5783; OUTLINE-ATOMICS-NEXT: ret 5784 %old = atomicrmw or ptr @var8, i8 %offset seq_cst 5785; CHECK-NOT: dmb 5786; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 5787; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 5788 5789; CHECK: ldsetalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5790; CHECK-NOT: dmb 5791 5792 ret i8 %old 5793} 5794 5795define dso_local i16 @test_atomic_load_or_i16_seq_cst(i16 %offset) nounwind { 5796; CHECK-LABEL: test_atomic_load_or_i16_seq_cst: 5797; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i16_seq_cst: 5798; OUTLINE-ATOMICS: // %bb.0: 5799; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5800; OUTLINE-ATOMICS-NEXT: adrp x1, var16 5801; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 5802; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset2_acq_rel 5803; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5804; OUTLINE-ATOMICS-NEXT: ret 5805 %old = atomicrmw or ptr @var16, i16 %offset seq_cst 5806; CHECK-NOT: dmb 5807; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 5808; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 5809 5810; CHECK: ldsetalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5811; CHECK-NOT: dmb 5812 5813 ret i16 %old 5814} 5815 5816define dso_local i32 @test_atomic_load_or_i32_seq_cst(i32 %offset) nounwind { 5817; CHECK-LABEL: test_atomic_load_or_i32_seq_cst: 5818; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_seq_cst: 5819; OUTLINE-ATOMICS: // %bb.0: 5820; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5821; OUTLINE-ATOMICS-NEXT: adrp x1, var32 5822; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 5823; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq_rel 5824; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5825; OUTLINE-ATOMICS-NEXT: ret 5826 %old = atomicrmw or ptr @var32, i32 %offset seq_cst 5827; CHECK-NOT: dmb 5828; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5829; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5830 5831; CHECK: ldsetal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5832; CHECK-NOT: dmb 5833 5834 ret i32 %old 5835} 5836 5837define dso_local i64 @test_atomic_load_or_i64_seq_cst(i64 %offset) nounwind { 5838; CHECK-LABEL: test_atomic_load_or_i64_seq_cst: 5839; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_seq_cst: 5840; OUTLINE-ATOMICS: // %bb.0: 5841; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5842; OUTLINE-ATOMICS-NEXT: adrp x1, var64 5843; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 5844; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq_rel 5845; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5846; OUTLINE-ATOMICS-NEXT: ret 5847 %old = atomicrmw or ptr @var64, i64 %offset seq_cst 5848; CHECK-NOT: dmb 5849; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5850; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5851 5852; CHECK: ldsetal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 5853; CHECK-NOT: dmb 5854 5855 ret i64 %old 5856} 5857 5858define dso_local void @test_atomic_load_or_i32_noret_seq_cst(i32 %offset) nounwind { 5859; CHECK-LABEL: test_atomic_load_or_i32_noret_seq_cst: 5860; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i32_noret_seq_cst: 5861; OUTLINE-ATOMICS: // %bb.0: 5862; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5863; OUTLINE-ATOMICS-NEXT: adrp x1, var32 5864; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 5865; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset4_acq_rel 5866; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5867; OUTLINE-ATOMICS-NEXT: ret 5868 atomicrmw or ptr @var32, i32 %offset seq_cst 5869; CHECK-NOT: dmb 5870; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5871; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5872 5873; CHECK: ldsetal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 5874; CHECK-NOT: dmb 5875 ret void 5876} 5877 5878define dso_local void @test_atomic_load_or_i64_noret_seq_cst(i64 %offset) nounwind { 5879; CHECK-LABEL: test_atomic_load_or_i64_noret_seq_cst: 5880; OUTLINE-ATOMICS-LABEL: test_atomic_load_or_i64_noret_seq_cst: 5881; OUTLINE-ATOMICS: // %bb.0: 5882; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5883; OUTLINE-ATOMICS-NEXT: adrp x1, var64 5884; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 5885; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldset8_acq_rel 5886; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5887; OUTLINE-ATOMICS-NEXT: ret 5888 atomicrmw or ptr @var64, i64 %offset seq_cst 5889; CHECK-NOT: dmb 5890; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5891; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5892 5893; CHECK: ldsetal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 5894; CHECK-NOT: dmb 5895 ret void 5896} 5897 5898define dso_local i8 @test_atomic_load_sub_i8_acq_rel(i8 %offset) nounwind { 5899; CHECK-LABEL: test_atomic_load_sub_i8_acq_rel: 5900; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8_acq_rel: 5901; OUTLINE-ATOMICS: // %bb.0: 5902; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5903; OUTLINE-ATOMICS-NEXT: neg w0, w0 5904; OUTLINE-ATOMICS-NEXT: adrp x1, var8 5905; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 5906; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel 5907; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5908; OUTLINE-ATOMICS-NEXT: ret 5909 %old = atomicrmw sub ptr @var8, i8 %offset acq_rel 5910; CHECK-NOT: dmb 5911; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 5912; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 5913; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 5914 5915; CHECK: ldaddalb w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5916; CHECK-NOT: dmb 5917 5918 ret i8 %old 5919} 5920 5921define dso_local i16 @test_atomic_load_sub_i16_acq_rel(i16 %offset) nounwind { 5922; CHECK-LABEL: test_atomic_load_sub_i16_acq_rel: 5923; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16_acq_rel: 5924; OUTLINE-ATOMICS: // %bb.0: 5925; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5926; OUTLINE-ATOMICS-NEXT: neg w0, w0 5927; OUTLINE-ATOMICS-NEXT: adrp x1, var16 5928; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 5929; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel 5930; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5931; OUTLINE-ATOMICS-NEXT: ret 5932 %old = atomicrmw sub ptr @var16, i16 %offset acq_rel 5933; CHECK-NOT: dmb 5934; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 5935; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 5936; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 5937 5938; CHECK: ldaddalh w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5939; CHECK-NOT: dmb 5940 5941 ret i16 %old 5942} 5943 5944define dso_local i32 @test_atomic_load_sub_i32_acq_rel(i32 %offset) nounwind { 5945; CHECK-LABEL: test_atomic_load_sub_i32_acq_rel: 5946; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_acq_rel: 5947; OUTLINE-ATOMICS: // %bb.0: 5948; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5949; OUTLINE-ATOMICS-NEXT: neg w0, w0 5950; OUTLINE-ATOMICS-NEXT: adrp x1, var32 5951; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 5952; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel 5953; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5954; OUTLINE-ATOMICS-NEXT: ret 5955 %old = atomicrmw sub ptr @var32, i32 %offset acq_rel 5956; CHECK-NOT: dmb 5957; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 5958; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 5959; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 5960 5961; CHECK: ldaddal w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 5962; CHECK-NOT: dmb 5963 5964 ret i32 %old 5965} 5966 5967define dso_local i64 @test_atomic_load_sub_i64_acq_rel(i64 %offset) nounwind { 5968; CHECK-LABEL: test_atomic_load_sub_i64_acq_rel: 5969; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_acq_rel: 5970; OUTLINE-ATOMICS: // %bb.0: 5971; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5972; OUTLINE-ATOMICS-NEXT: neg x0, x0 5973; OUTLINE-ATOMICS-NEXT: adrp x1, var64 5974; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 5975; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel 5976; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 5977; OUTLINE-ATOMICS-NEXT: ret 5978 %old = atomicrmw sub ptr @var64, i64 %offset acq_rel 5979; CHECK-NOT: dmb 5980; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] 5981; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 5982; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 5983 5984; CHECK: ldaddal x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]] 5985; CHECK-NOT: dmb 5986 5987 ret i64 %old 5988} 5989 5990define dso_local void @test_atomic_load_sub_i32_noret_acq_rel(i32 %offset) nounwind { 5991; CHECK-LABEL: test_atomic_load_sub_i32_noret_acq_rel: 5992; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_noret_acq_rel: 5993; OUTLINE-ATOMICS: // %bb.0: 5994; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 5995; OUTLINE-ATOMICS-NEXT: neg w0, w0 5996; OUTLINE-ATOMICS-NEXT: adrp x1, var32 5997; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 5998; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel 5999; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6000; OUTLINE-ATOMICS-NEXT: ret 6001 atomicrmw sub ptr @var32, i32 %offset acq_rel 6002; CHECK-NOT: dmb 6003; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6004; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6005; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6006 6007; CHECK: ldaddal w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6008; CHECK-NOT: dmb 6009 6010 ret void 6011} 6012 6013define dso_local void @test_atomic_load_sub_i64_noret_acq_rel(i64 %offset) nounwind { 6014; CHECK-LABEL: test_atomic_load_sub_i64_noret_acq_rel: 6015; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_noret_acq_rel: 6016; OUTLINE-ATOMICS: // %bb.0: 6017; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6018; OUTLINE-ATOMICS-NEXT: neg x0, x0 6019; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6020; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6021; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel 6022; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6023; OUTLINE-ATOMICS-NEXT: ret 6024 atomicrmw sub ptr @var64, i64 %offset acq_rel 6025; CHECK-NOT: dmb 6026; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] 6027; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6028; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6029 6030; CHECK: ldaddal x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]] 6031; CHECK-NOT: dmb 6032 6033 ret void 6034} 6035 6036define dso_local i8 @test_atomic_load_sub_i8_acquire(i8 %offset) nounwind { 6037; CHECK-LABEL: test_atomic_load_sub_i8_acquire: 6038; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8_acquire: 6039; OUTLINE-ATOMICS: // %bb.0: 6040; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6041; OUTLINE-ATOMICS-NEXT: neg w0, w0 6042; OUTLINE-ATOMICS-NEXT: adrp x1, var8 6043; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 6044; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq 6045; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6046; OUTLINE-ATOMICS-NEXT: ret 6047 %old = atomicrmw sub ptr @var8, i8 %offset acquire 6048; CHECK-NOT: dmb 6049; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6050; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 6051; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 6052 6053; CHECK: ldaddab w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6054; CHECK-NOT: dmb 6055 6056 ret i8 %old 6057} 6058 6059define dso_local i16 @test_atomic_load_sub_i16_acquire(i16 %offset) nounwind { 6060; CHECK-LABEL: test_atomic_load_sub_i16_acquire: 6061; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16_acquire: 6062; OUTLINE-ATOMICS: // %bb.0: 6063; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6064; OUTLINE-ATOMICS-NEXT: neg w0, w0 6065; OUTLINE-ATOMICS-NEXT: adrp x1, var16 6066; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 6067; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq 6068; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6069; OUTLINE-ATOMICS-NEXT: ret 6070 %old = atomicrmw sub ptr @var16, i16 %offset acquire 6071; CHECK-NOT: dmb 6072; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6073; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 6074; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 6075 6076; CHECK: ldaddah w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6077; CHECK-NOT: dmb 6078 6079 ret i16 %old 6080} 6081 6082define dso_local i32 @test_atomic_load_sub_i32_acquire(i32 %offset) nounwind { 6083; CHECK-LABEL: test_atomic_load_sub_i32_acquire: 6084; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_acquire: 6085; OUTLINE-ATOMICS: // %bb.0: 6086; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6087; OUTLINE-ATOMICS-NEXT: neg w0, w0 6088; OUTLINE-ATOMICS-NEXT: adrp x1, var32 6089; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 6090; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq 6091; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6092; OUTLINE-ATOMICS-NEXT: ret 6093 %old = atomicrmw sub ptr @var32, i32 %offset acquire 6094; CHECK-NOT: dmb 6095; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6096; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6097; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6098 6099; CHECK: ldadda w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6100; CHECK-NOT: dmb 6101 6102 ret i32 %old 6103} 6104 6105define dso_local i64 @test_atomic_load_sub_i64_acquire(i64 %offset) nounwind { 6106; CHECK-LABEL: test_atomic_load_sub_i64_acquire: 6107; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_acquire: 6108; OUTLINE-ATOMICS: // %bb.0: 6109; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6110; OUTLINE-ATOMICS-NEXT: neg x0, x0 6111; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6112; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6113; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq 6114; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6115; OUTLINE-ATOMICS-NEXT: ret 6116 %old = atomicrmw sub ptr @var64, i64 %offset acquire 6117; CHECK-NOT: dmb 6118; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] 6119; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6120; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6121 6122; CHECK: ldadda x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]] 6123; CHECK-NOT: dmb 6124 6125 ret i64 %old 6126} 6127 6128define dso_local void @test_atomic_load_sub_i32_noret_acquire(i32 %offset) nounwind { 6129; CHECK-LABEL: test_atomic_load_sub_i32_noret_acquire: 6130; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_noret_acquire: 6131; OUTLINE-ATOMICS: // %bb.0: 6132; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6133; OUTLINE-ATOMICS-NEXT: neg w0, w0 6134; OUTLINE-ATOMICS-NEXT: adrp x1, var32 6135; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 6136; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq 6137; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6138; OUTLINE-ATOMICS-NEXT: ret 6139 atomicrmw sub ptr @var32, i32 %offset acquire 6140; CHECK-NOT: dmb 6141; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6142; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6143; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6144 6145; CHECK: ldadda w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6146; CHECK-NOT: dmb 6147 6148 ret void 6149} 6150 6151define dso_local void @test_atomic_load_sub_i64_noret_acquire(i64 %offset) nounwind { 6152; CHECK-LABEL: test_atomic_load_sub_i64_noret_acquire: 6153; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_noret_acquire: 6154; OUTLINE-ATOMICS: // %bb.0: 6155; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6156; OUTLINE-ATOMICS-NEXT: neg x0, x0 6157; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6158; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6159; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq 6160; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6161; OUTLINE-ATOMICS-NEXT: ret 6162 atomicrmw sub ptr @var64, i64 %offset acquire 6163; CHECK-NOT: dmb 6164; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] 6165; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6166; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6167 6168; CHECK: ldadda x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]] 6169; CHECK-NOT: dmb 6170 6171 ret void 6172} 6173 6174define dso_local i8 @test_atomic_load_sub_i8_monotonic(i8 %offset) nounwind { 6175; CHECK-LABEL: test_atomic_load_sub_i8_monotonic: 6176; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8_monotonic: 6177; OUTLINE-ATOMICS: // %bb.0: 6178; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6179; OUTLINE-ATOMICS-NEXT: neg w0, w0 6180; OUTLINE-ATOMICS-NEXT: adrp x1, var8 6181; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 6182; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_relax 6183; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6184; OUTLINE-ATOMICS-NEXT: ret 6185 %old = atomicrmw sub ptr @var8, i8 %offset monotonic 6186; CHECK-NOT: dmb 6187; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6188; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 6189; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 6190 6191; CHECK: ldaddb w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6192; CHECK-NOT: dmb 6193 6194 ret i8 %old 6195} 6196 6197define dso_local i16 @test_atomic_load_sub_i16_monotonic(i16 %offset) nounwind { 6198; CHECK-LABEL: test_atomic_load_sub_i16_monotonic: 6199; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16_monotonic: 6200; OUTLINE-ATOMICS: // %bb.0: 6201; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6202; OUTLINE-ATOMICS-NEXT: neg w0, w0 6203; OUTLINE-ATOMICS-NEXT: adrp x1, var16 6204; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 6205; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_relax 6206; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6207; OUTLINE-ATOMICS-NEXT: ret 6208 %old = atomicrmw sub ptr @var16, i16 %offset monotonic 6209; CHECK-NOT: dmb 6210; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6211; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 6212; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 6213 6214; CHECK: ldaddh w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6215; CHECK-NOT: dmb 6216 6217 ret i16 %old 6218} 6219 6220define dso_local i32 @test_atomic_load_sub_i32_monotonic(i32 %offset) nounwind { 6221; CHECK-LABEL: test_atomic_load_sub_i32_monotonic: 6222; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_monotonic: 6223; OUTLINE-ATOMICS: // %bb.0: 6224; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6225; OUTLINE-ATOMICS-NEXT: neg w0, w0 6226; OUTLINE-ATOMICS-NEXT: adrp x1, var32 6227; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 6228; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_relax 6229; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6230; OUTLINE-ATOMICS-NEXT: ret 6231 %old = atomicrmw sub ptr @var32, i32 %offset monotonic 6232; CHECK-NOT: dmb 6233; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6234; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6235; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6236 6237; CHECK: ldadd w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6238; CHECK-NOT: dmb 6239 6240 ret i32 %old 6241} 6242 6243define dso_local i64 @test_atomic_load_sub_i64_monotonic(i64 %offset) nounwind { 6244; CHECK-LABEL: test_atomic_load_sub_i64_monotonic: 6245; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_monotonic: 6246; OUTLINE-ATOMICS: // %bb.0: 6247; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6248; OUTLINE-ATOMICS-NEXT: neg x0, x0 6249; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6250; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6251; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_relax 6252; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6253; OUTLINE-ATOMICS-NEXT: ret 6254 %old = atomicrmw sub ptr @var64, i64 %offset monotonic 6255; CHECK-NOT: dmb 6256; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] 6257; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6258; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6259 6260; CHECK: ldadd x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]] 6261; CHECK-NOT: dmb 6262 6263 ret i64 %old 6264} 6265 6266define dso_local void @test_atomic_load_sub_i32_noret_monotonic(i32 %offset) nounwind { 6267; CHECK-LABEL: test_atomic_load_sub_i32_noret_monotonic: 6268; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_noret_monotonic: 6269; OUTLINE-ATOMICS: // %bb.0: 6270; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6271; OUTLINE-ATOMICS-NEXT: neg w0, w0 6272; OUTLINE-ATOMICS-NEXT: adrp x1, var32 6273; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 6274; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_relax 6275; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6276; OUTLINE-ATOMICS-NEXT: ret 6277 atomicrmw sub ptr @var32, i32 %offset monotonic 6278; CHECK-NOT: dmb 6279; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6280; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6281; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6282 6283; CHECK: ldadd w{{[0-9]+}}, w[[NEW:[1-9][0-9]*]], [x[[ADDR]]] 6284; CHECK-NOT: dmb 6285 6286 ret void 6287} 6288 6289define dso_local void @test_atomic_load_sub_i64_noret_monotonic(i64 %offset) nounwind { 6290; CHECK-LABEL: test_atomic_load_sub_i64_noret_monotonic: 6291; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_noret_monotonic: 6292; OUTLINE-ATOMICS: // %bb.0: 6293; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6294; OUTLINE-ATOMICS-NEXT: neg x0, x0 6295; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6296; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6297; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_relax 6298; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6299; OUTLINE-ATOMICS-NEXT: ret 6300 atomicrmw sub ptr @var64, i64 %offset monotonic 6301; CHECK-NOT: dmb 6302; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] 6303; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6304; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6305 6306; CHECK: ldadd x{{[0-9]+}}, x[[NEW:[1-9][0-9]*]], [x[[ADDR]]] 6307; CHECK-NOT: dmb 6308 6309 ret void 6310} 6311 6312define dso_local i8 @test_atomic_load_sub_i8_release(i8 %offset) nounwind { 6313; CHECK-LABEL: test_atomic_load_sub_i8_release: 6314; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8_release: 6315; OUTLINE-ATOMICS: // %bb.0: 6316; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6317; OUTLINE-ATOMICS-NEXT: neg w0, w0 6318; OUTLINE-ATOMICS-NEXT: adrp x1, var8 6319; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 6320; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_rel 6321; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6322; OUTLINE-ATOMICS-NEXT: ret 6323 %old = atomicrmw sub ptr @var8, i8 %offset release 6324; CHECK-NOT: dmb 6325; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6326; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 6327; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 6328 6329; CHECK: ldaddlb w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6330; CHECK-NOT: dmb 6331 6332 ret i8 %old 6333} 6334 6335define dso_local i16 @test_atomic_load_sub_i16_release(i16 %offset) nounwind { 6336; CHECK-LABEL: test_atomic_load_sub_i16_release: 6337; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16_release: 6338; OUTLINE-ATOMICS: // %bb.0: 6339; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6340; OUTLINE-ATOMICS-NEXT: neg w0, w0 6341; OUTLINE-ATOMICS-NEXT: adrp x1, var16 6342; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 6343; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_rel 6344; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6345; OUTLINE-ATOMICS-NEXT: ret 6346 %old = atomicrmw sub ptr @var16, i16 %offset release 6347; CHECK-NOT: dmb 6348; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6349; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 6350; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 6351 6352; CHECK: ldaddlh w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6353; CHECK-NOT: dmb 6354 6355 ret i16 %old 6356} 6357 6358define dso_local i32 @test_atomic_load_sub_i32_release(i32 %offset) nounwind { 6359; CHECK-LABEL: test_atomic_load_sub_i32_release: 6360; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_release: 6361; OUTLINE-ATOMICS: // %bb.0: 6362; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6363; OUTLINE-ATOMICS-NEXT: neg w0, w0 6364; OUTLINE-ATOMICS-NEXT: adrp x1, var32 6365; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 6366; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_rel 6367; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6368; OUTLINE-ATOMICS-NEXT: ret 6369 %old = atomicrmw sub ptr @var32, i32 %offset release 6370; CHECK-NOT: dmb 6371; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6372; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6373; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6374 6375; CHECK: ldaddl w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6376; CHECK-NOT: dmb 6377 6378 ret i32 %old 6379} 6380 6381define dso_local i64 @test_atomic_load_sub_i64_release(i64 %offset) nounwind { 6382; CHECK-LABEL: test_atomic_load_sub_i64_release: 6383; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_release: 6384; OUTLINE-ATOMICS: // %bb.0: 6385; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6386; OUTLINE-ATOMICS-NEXT: neg x0, x0 6387; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6388; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6389; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_rel 6390; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6391; OUTLINE-ATOMICS-NEXT: ret 6392 %old = atomicrmw sub ptr @var64, i64 %offset release 6393; CHECK-NOT: dmb 6394; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] 6395; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6396; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6397 6398; CHECK: ldaddl x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]] 6399; CHECK-NOT: dmb 6400 6401 ret i64 %old 6402} 6403 6404define dso_local void @test_atomic_load_sub_i32_noret_release(i32 %offset) nounwind { 6405; CHECK-LABEL: test_atomic_load_sub_i32_noret_release: 6406; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_noret_release: 6407; OUTLINE-ATOMICS: // %bb.0: 6408; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6409; OUTLINE-ATOMICS-NEXT: neg w0, w0 6410; OUTLINE-ATOMICS-NEXT: adrp x1, var32 6411; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 6412; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_rel 6413; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6414; OUTLINE-ATOMICS-NEXT: ret 6415 atomicrmw sub ptr @var32, i32 %offset release 6416; CHECK-NOT: dmb 6417; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6418; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6419; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6420 6421; CHECK: ldaddl w{{[0-9]*}}, w[[NEW:[1-9][0-9]*]], [x[[ADDR]]] 6422; CHECK-NOT: dmb 6423 6424 ret void 6425} 6426 6427define dso_local void @test_atomic_load_sub_i64_noret_release(i64 %offset) nounwind { 6428; CHECK-LABEL: test_atomic_load_sub_i64_noret_release: 6429; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_noret_release: 6430; OUTLINE-ATOMICS: // %bb.0: 6431; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6432; OUTLINE-ATOMICS-NEXT: neg x0, x0 6433; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6434; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6435; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_rel 6436; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6437; OUTLINE-ATOMICS-NEXT: ret 6438 atomicrmw sub ptr @var64, i64 %offset release 6439; CHECK-NOT: dmb 6440; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] 6441; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6442; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6443 6444; CHECK: ldaddl x{{[0-9]*}}, x[[NEW:[1-9][0-9]*]], [x[[ADDR]]] 6445; CHECK-NOT: dmb 6446 6447 ret void 6448} 6449 6450define dso_local i8 @test_atomic_load_sub_i8_seq_cst(i8 %offset) nounwind { 6451; CHECK-LABEL: test_atomic_load_sub_i8_seq_cst: 6452; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i8_seq_cst: 6453; OUTLINE-ATOMICS: // %bb.0: 6454; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6455; OUTLINE-ATOMICS-NEXT: neg w0, w0 6456; OUTLINE-ATOMICS-NEXT: adrp x1, var8 6457; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 6458; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd1_acq_rel 6459; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6460; OUTLINE-ATOMICS-NEXT: ret 6461 %old = atomicrmw sub ptr @var8, i8 %offset seq_cst 6462; CHECK-NOT: dmb 6463; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6464; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 6465; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 6466 6467; CHECK: ldaddalb w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6468; CHECK-NOT: dmb 6469 6470 ret i8 %old 6471} 6472 6473define dso_local i16 @test_atomic_load_sub_i16_seq_cst(i16 %offset) nounwind { 6474; CHECK-LABEL: test_atomic_load_sub_i16_seq_cst: 6475; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i16_seq_cst: 6476; OUTLINE-ATOMICS: // %bb.0: 6477; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6478; OUTLINE-ATOMICS-NEXT: neg w0, w0 6479; OUTLINE-ATOMICS-NEXT: adrp x1, var16 6480; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 6481; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd2_acq_rel 6482; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6483; OUTLINE-ATOMICS-NEXT: ret 6484 %old = atomicrmw sub ptr @var16, i16 %offset seq_cst 6485; CHECK-NOT: dmb 6486; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6487; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 6488; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 6489 6490; CHECK: ldaddalh w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6491; CHECK-NOT: dmb 6492 6493 ret i16 %old 6494} 6495 6496define dso_local i32 @test_atomic_load_sub_i32_seq_cst(i32 %offset) nounwind { 6497; CHECK-LABEL: test_atomic_load_sub_i32_seq_cst: 6498; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_seq_cst: 6499; OUTLINE-ATOMICS: // %bb.0: 6500; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6501; OUTLINE-ATOMICS-NEXT: neg w0, w0 6502; OUTLINE-ATOMICS-NEXT: adrp x1, var32 6503; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 6504; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel 6505; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6506; OUTLINE-ATOMICS-NEXT: ret 6507 %old = atomicrmw sub ptr @var32, i32 %offset seq_cst 6508; CHECK-NOT: dmb 6509; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6510; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6511; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6512 6513; CHECK: ldaddal w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6514; CHECK-NOT: dmb 6515 6516 ret i32 %old 6517} 6518 6519define dso_local i64 @test_atomic_load_sub_i64_seq_cst(i64 %offset) nounwind { 6520; CHECK-LABEL: test_atomic_load_sub_i64_seq_cst: 6521; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_seq_cst: 6522; OUTLINE-ATOMICS: // %bb.0: 6523; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6524; OUTLINE-ATOMICS-NEXT: neg x0, x0 6525; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6526; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6527; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel 6528; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6529; OUTLINE-ATOMICS-NEXT: ret 6530 %old = atomicrmw sub ptr @var64, i64 %offset seq_cst 6531; CHECK-NOT: dmb 6532; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] 6533; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6534; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6535 6536; CHECK: ldaddal x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]] 6537; CHECK-NOT: dmb 6538 6539 ret i64 %old 6540} 6541 6542define dso_local void @test_atomic_load_sub_i32_noret_seq_cst(i32 %offset) nounwind { 6543; CHECK-LABEL: test_atomic_load_sub_i32_noret_seq_cst: 6544; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i32_noret_seq_cst: 6545; OUTLINE-ATOMICS: // %bb.0: 6546; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6547; OUTLINE-ATOMICS-NEXT: neg w0, w0 6548; OUTLINE-ATOMICS-NEXT: adrp x1, var32 6549; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 6550; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd4_acq_rel 6551; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6552; OUTLINE-ATOMICS-NEXT: ret 6553 atomicrmw sub ptr @var32, i32 %offset seq_cst 6554; CHECK-NOT: dmb 6555; CHECK: neg w[[NEG:[0-9]+]], w[[OLD:[0-9]+]] 6556; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6557; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6558 6559; CHECK: ldaddal w[[NEG]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6560; CHECK-NOT: dmb 6561 6562 ret void 6563} 6564 6565define dso_local void @test_atomic_load_sub_i64_noret_seq_cst(i64 %offset) nounwind { 6566; CHECK-LABEL: test_atomic_load_sub_i64_noret_seq_cst: 6567; OUTLINE-ATOMICS-LABEL: test_atomic_load_sub_i64_noret_seq_cst: 6568; OUTLINE-ATOMICS: // %bb.0: 6569; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6570; OUTLINE-ATOMICS-NEXT: neg x0, x0 6571; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6572; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6573; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldadd8_acq_rel 6574; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6575; OUTLINE-ATOMICS-NEXT: ret 6576 atomicrmw sub ptr @var64, i64 %offset seq_cst 6577; CHECK-NOT: dmb 6578; CHECK: neg x[[NEG:[0-9]+]], x[[OLD:[0-9]+]] 6579; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6580; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6581 6582; CHECK: ldaddal x[[NEG]], x[[NEW:[0-9]+]], [x[[ADDR]]] 6583; CHECK-NOT: dmb 6584 6585 ret void 6586} 6587 6588define dso_local i8 @test_atomic_load_xchg_i8_acq_rel(i8 %offset) nounwind { 6589; CHECK-LABEL: test_atomic_load_xchg_i8_acq_rel: 6590; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i8_acq_rel: 6591; OUTLINE-ATOMICS: // %bb.0: 6592; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6593; OUTLINE-ATOMICS-NEXT: adrp x1, var8 6594; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 6595; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp1_acq_rel 6596; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6597; OUTLINE-ATOMICS-NEXT: ret 6598 %old = atomicrmw xchg ptr @var8, i8 %offset acq_rel 6599; CHECK-NOT: dmb 6600; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 6601; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 6602 6603; CHECK: swpalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6604; CHECK-NOT: dmb 6605 6606 ret i8 %old 6607} 6608 6609define dso_local i16 @test_atomic_load_xchg_i16_acq_rel(i16 %offset) nounwind { 6610; CHECK-LABEL: test_atomic_load_xchg_i16_acq_rel: 6611; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i16_acq_rel: 6612; OUTLINE-ATOMICS: // %bb.0: 6613; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6614; OUTLINE-ATOMICS-NEXT: adrp x1, var16 6615; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 6616; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp2_acq_rel 6617; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6618; OUTLINE-ATOMICS-NEXT: ret 6619 %old = atomicrmw xchg ptr @var16, i16 %offset acq_rel 6620; CHECK-NOT: dmb 6621; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 6622; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 6623 6624; CHECK: swpalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6625; CHECK-NOT: dmb 6626 6627 ret i16 %old 6628} 6629 6630define dso_local i32 @test_atomic_load_xchg_i32_acq_rel(i32 %offset) nounwind { 6631; CHECK-LABEL: test_atomic_load_xchg_i32_acq_rel: 6632; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_acq_rel: 6633; OUTLINE-ATOMICS: // %bb.0: 6634; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6635; OUTLINE-ATOMICS-NEXT: adrp x1, var32 6636; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 6637; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq_rel 6638; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6639; OUTLINE-ATOMICS-NEXT: ret 6640 %old = atomicrmw xchg ptr @var32, i32 %offset acq_rel 6641; CHECK-NOT: dmb 6642; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6643; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6644 6645; CHECK: swpal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6646; CHECK-NOT: dmb 6647 6648 ret i32 %old 6649} 6650 6651define dso_local i64 @test_atomic_load_xchg_i64_acq_rel(i64 %offset) nounwind { 6652; CHECK-LABEL: test_atomic_load_xchg_i64_acq_rel: 6653; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_acq_rel: 6654; OUTLINE-ATOMICS: // %bb.0: 6655; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6656; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6657; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6658; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq_rel 6659; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6660; OUTLINE-ATOMICS-NEXT: ret 6661 %old = atomicrmw xchg ptr @var64, i64 %offset acq_rel 6662; CHECK-NOT: dmb 6663; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6664; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6665 6666; CHECK: swpal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 6667; CHECK-NOT: dmb 6668 6669 ret i64 %old 6670} 6671 6672define dso_local void @test_atomic_load_xchg_i32_noret_acq_rel(i32 %offset) nounwind { 6673; CHECK-LABEL: test_atomic_load_xchg_i32_noret_acq_rel: 6674; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_noret_acq_rel: 6675; OUTLINE-ATOMICS: // %bb.0: 6676; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6677; OUTLINE-ATOMICS-NEXT: adrp x1, var32 6678; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 6679; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq_rel 6680; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6681; OUTLINE-ATOMICS-NEXT: ret 6682 atomicrmw xchg ptr @var32, i32 %offset acq_rel 6683; CHECK-NOT: dmb 6684; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6685; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6686 6687; CHECK: swpal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6688; CHECK-NOT: dmb 6689 6690 ret void 6691} 6692 6693define dso_local void @test_atomic_load_xchg_i64_noret_acq_rel(i64 %offset) nounwind { 6694; CHECK-LABEL: test_atomic_load_xchg_i64_noret_acq_rel: 6695; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_noret_acq_rel: 6696; OUTLINE-ATOMICS: // %bb.0: 6697; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6698; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6699; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6700; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq_rel 6701; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6702; OUTLINE-ATOMICS-NEXT: ret 6703 atomicrmw xchg ptr @var64, i64 %offset acq_rel 6704; CHECK-NOT: dmb 6705; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6706; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6707 6708; CHECK: swpal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 6709; CHECK-NOT: dmb 6710 6711 ret void 6712} 6713 6714define dso_local i8 @test_atomic_load_xchg_i8_acquire(i8 %offset) nounwind { 6715; CHECK-LABEL: test_atomic_load_xchg_i8_acquire: 6716; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i8_acquire: 6717; OUTLINE-ATOMICS: // %bb.0: 6718; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6719; OUTLINE-ATOMICS-NEXT: adrp x1, var8 6720; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 6721; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp1_acq 6722; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6723; OUTLINE-ATOMICS-NEXT: ret 6724 %old = atomicrmw xchg ptr @var8, i8 %offset acquire 6725; CHECK-NOT: dmb 6726; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 6727; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 6728 6729; CHECK: swpab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6730; CHECK-NOT: dmb 6731 6732 ret i8 %old 6733} 6734 6735define dso_local i16 @test_atomic_load_xchg_i16_acquire(i16 %offset) nounwind { 6736; CHECK-LABEL: test_atomic_load_xchg_i16_acquire: 6737; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i16_acquire: 6738; OUTLINE-ATOMICS: // %bb.0: 6739; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6740; OUTLINE-ATOMICS-NEXT: adrp x1, var16 6741; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 6742; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp2_acq 6743; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6744; OUTLINE-ATOMICS-NEXT: ret 6745 %old = atomicrmw xchg ptr @var16, i16 %offset acquire 6746; CHECK-NOT: dmb 6747; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 6748; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 6749 6750; CHECK: swpah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6751; CHECK-NOT: dmb 6752 6753 ret i16 %old 6754} 6755 6756define dso_local i32 @test_atomic_load_xchg_i32_acquire(i32 %offset) nounwind { 6757; CHECK-LABEL: test_atomic_load_xchg_i32_acquire: 6758; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_acquire: 6759; OUTLINE-ATOMICS: // %bb.0: 6760; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6761; OUTLINE-ATOMICS-NEXT: adrp x1, var32 6762; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 6763; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq 6764; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6765; OUTLINE-ATOMICS-NEXT: ret 6766 %old = atomicrmw xchg ptr @var32, i32 %offset acquire 6767; CHECK-NOT: dmb 6768; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6769; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6770 6771; CHECK: swpa w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6772; CHECK-NOT: dmb 6773 6774 ret i32 %old 6775} 6776 6777define dso_local i64 @test_atomic_load_xchg_i64_acquire(i64 %offset) nounwind { 6778; CHECK-LABEL: test_atomic_load_xchg_i64_acquire: 6779; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_acquire: 6780; OUTLINE-ATOMICS: // %bb.0: 6781; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6782; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6783; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6784; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq 6785; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6786; OUTLINE-ATOMICS-NEXT: ret 6787 %old = atomicrmw xchg ptr @var64, i64 %offset acquire 6788; CHECK-NOT: dmb 6789; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6790; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6791 6792; CHECK: swpa x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 6793; CHECK-NOT: dmb 6794 6795 ret i64 %old 6796} 6797 6798define dso_local void @test_atomic_load_xchg_i32_noret_acquire(i32 %offset) nounwind { 6799; CHECK-LABEL: test_atomic_load_xchg_i32_noret_acquire: 6800; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_noret_acquire: 6801; OUTLINE-ATOMICS: // %bb.0: 6802; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6803; OUTLINE-ATOMICS-NEXT: adrp x1, var32 6804; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 6805; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq 6806; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6807; OUTLINE-ATOMICS-NEXT: ret 6808 atomicrmw xchg ptr @var32, i32 %offset acquire 6809; CHECK-NOT: dmb 6810; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6811; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6812 6813; CHECK: swpa w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6814; CHECK-NOT: dmb 6815 6816 ret void 6817} 6818 6819define dso_local void @test_atomic_load_xchg_i64_noret_acquire(i64 %offset) nounwind { 6820; CHECK-LABEL: test_atomic_load_xchg_i64_noret_acquire: 6821; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_noret_acquire: 6822; OUTLINE-ATOMICS: // %bb.0: 6823; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6824; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6825; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6826; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq 6827; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6828; OUTLINE-ATOMICS-NEXT: ret 6829 atomicrmw xchg ptr @var64, i64 %offset acquire 6830; CHECK-NOT: dmb 6831; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6832; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6833 6834; CHECK: swpa x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 6835; CHECK-NOT: dmb 6836 6837 ret void 6838} 6839 6840define dso_local i8 @test_atomic_load_xchg_i8_monotonic(i8 %offset) nounwind { 6841; CHECK-LABEL: test_atomic_load_xchg_i8_monotonic: 6842; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i8_monotonic: 6843; OUTLINE-ATOMICS: // %bb.0: 6844; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6845; OUTLINE-ATOMICS-NEXT: adrp x1, var8 6846; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 6847; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp1_relax 6848; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6849; OUTLINE-ATOMICS-NEXT: ret 6850 %old = atomicrmw xchg ptr @var8, i8 %offset monotonic 6851; CHECK-NOT: dmb 6852; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 6853; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 6854 6855; CHECK: swpb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6856; CHECK-NOT: dmb 6857 6858 ret i8 %old 6859} 6860 6861define dso_local i16 @test_atomic_load_xchg_i16_monotonic(i16 %offset) nounwind { 6862; CHECK-LABEL: test_atomic_load_xchg_i16_monotonic: 6863; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i16_monotonic: 6864; OUTLINE-ATOMICS: // %bb.0: 6865; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6866; OUTLINE-ATOMICS-NEXT: adrp x1, var16 6867; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 6868; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp2_relax 6869; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6870; OUTLINE-ATOMICS-NEXT: ret 6871 %old = atomicrmw xchg ptr @var16, i16 %offset monotonic 6872; CHECK-NOT: dmb 6873; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 6874; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 6875 6876; CHECK: swph w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6877; CHECK-NOT: dmb 6878 6879 ret i16 %old 6880} 6881 6882define dso_local i32 @test_atomic_load_xchg_i32_monotonic(i32 %offset) nounwind { 6883; CHECK-LABEL: test_atomic_load_xchg_i32_monotonic: 6884; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_monotonic: 6885; OUTLINE-ATOMICS: // %bb.0: 6886; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6887; OUTLINE-ATOMICS-NEXT: adrp x1, var32 6888; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 6889; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_relax 6890; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6891; OUTLINE-ATOMICS-NEXT: ret 6892 %old = atomicrmw xchg ptr @var32, i32 %offset monotonic 6893; CHECK-NOT: dmb 6894; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6895; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6896 6897; CHECK: swp w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6898; CHECK-NOT: dmb 6899 6900 ret i32 %old 6901} 6902 6903define dso_local i64 @test_atomic_load_xchg_i64_monotonic(i64 %offset) nounwind { 6904; CHECK-LABEL: test_atomic_load_xchg_i64_monotonic: 6905; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_monotonic: 6906; OUTLINE-ATOMICS: // %bb.0: 6907; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6908; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6909; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6910; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_relax 6911; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6912; OUTLINE-ATOMICS-NEXT: ret 6913 %old = atomicrmw xchg ptr @var64, i64 %offset monotonic 6914; CHECK-NOT: dmb 6915; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6916; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6917 6918; CHECK: swp x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 6919; CHECK-NOT: dmb 6920 6921 ret i64 %old 6922} 6923 6924define dso_local void @test_atomic_load_xchg_i32_noret_monotonic(i32 %offset) nounwind { 6925; CHECK-LABEL: test_atomic_load_xchg_i32_noret_monotonic: 6926; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_noret_monotonic: 6927; OUTLINE-ATOMICS: // %bb.0: 6928; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6929; OUTLINE-ATOMICS-NEXT: adrp x1, var32 6930; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 6931; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_relax 6932; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6933; OUTLINE-ATOMICS-NEXT: ret 6934 atomicrmw xchg ptr @var32, i32 %offset monotonic 6935; CHECK-NOT: dmb 6936; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 6937; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 6938 6939; CHECK: swp w[[OLD:[0-9]+]], w[[NEW:[0-9,a-z]+]], [x[[ADDR]]] 6940; CHECK-NOT: dmb 6941 6942 ret void 6943} 6944 6945define dso_local void @test_atomic_load_xchg_i64_noret_monotonic(i64 %offset) nounwind { 6946; CHECK-LABEL: test_atomic_load_xchg_i64_noret_monotonic: 6947; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_noret_monotonic: 6948; OUTLINE-ATOMICS: // %bb.0: 6949; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6950; OUTLINE-ATOMICS-NEXT: adrp x1, var64 6951; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 6952; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_relax 6953; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6954; OUTLINE-ATOMICS-NEXT: ret 6955 atomicrmw xchg ptr @var64, i64 %offset monotonic 6956; CHECK-NOT: dmb 6957; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 6958; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 6959 6960; CHECK: swp x[[OLD:[0-9]+]], x[[NEW:[0-9,a-z]+]], [x[[ADDR]]] 6961; CHECK-NOT: dmb 6962 6963 ret void 6964} 6965 6966define dso_local i8 @test_atomic_load_xchg_i8_release(i8 %offset) nounwind { 6967; CHECK-LABEL: test_atomic_load_xchg_i8_release: 6968; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i8_release: 6969; OUTLINE-ATOMICS: // %bb.0: 6970; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6971; OUTLINE-ATOMICS-NEXT: adrp x1, var8 6972; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 6973; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp1_rel 6974; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6975; OUTLINE-ATOMICS-NEXT: ret 6976 %old = atomicrmw xchg ptr @var8, i8 %offset release 6977; CHECK-NOT: dmb 6978; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 6979; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 6980 6981; CHECK: swplb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 6982; CHECK-NOT: dmb 6983 6984 ret i8 %old 6985} 6986 6987define dso_local i16 @test_atomic_load_xchg_i16_release(i16 %offset) nounwind { 6988; CHECK-LABEL: test_atomic_load_xchg_i16_release: 6989; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i16_release: 6990; OUTLINE-ATOMICS: // %bb.0: 6991; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 6992; OUTLINE-ATOMICS-NEXT: adrp x1, var16 6993; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 6994; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp2_rel 6995; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 6996; OUTLINE-ATOMICS-NEXT: ret 6997 %old = atomicrmw xchg ptr @var16, i16 %offset release 6998; CHECK-NOT: dmb 6999; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 7000; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 7001 7002; CHECK: swplh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7003; CHECK-NOT: dmb 7004 7005 ret i16 %old 7006} 7007 7008define dso_local i32 @test_atomic_load_xchg_i32_release(i32 %offset) nounwind { 7009; CHECK-LABEL: test_atomic_load_xchg_i32_release: 7010; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_release: 7011; OUTLINE-ATOMICS: // %bb.0: 7012; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 7013; OUTLINE-ATOMICS-NEXT: adrp x1, var32 7014; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 7015; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_rel 7016; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 7017; OUTLINE-ATOMICS-NEXT: ret 7018 %old = atomicrmw xchg ptr @var32, i32 %offset release 7019; CHECK-NOT: dmb 7020; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 7021; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 7022 7023; CHECK: swpl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7024; CHECK-NOT: dmb 7025 7026 ret i32 %old 7027} 7028 7029define dso_local i64 @test_atomic_load_xchg_i64_release(i64 %offset) nounwind { 7030; CHECK-LABEL: test_atomic_load_xchg_i64_release: 7031; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_release: 7032; OUTLINE-ATOMICS: // %bb.0: 7033; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 7034; OUTLINE-ATOMICS-NEXT: adrp x1, var64 7035; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 7036; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_rel 7037; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 7038; OUTLINE-ATOMICS-NEXT: ret 7039 %old = atomicrmw xchg ptr @var64, i64 %offset release 7040; CHECK-NOT: dmb 7041; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 7042; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 7043 7044; CHECK: swpl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 7045; CHECK-NOT: dmb 7046 7047 ret i64 %old 7048} 7049 7050define dso_local void @test_atomic_load_xchg_i32_noret_release(i32 %offset) nounwind { 7051; CHECK-LABEL: test_atomic_load_xchg_i32_noret_release: 7052; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_noret_release: 7053; OUTLINE-ATOMICS: // %bb.0: 7054; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 7055; OUTLINE-ATOMICS-NEXT: adrp x1, var32 7056; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 7057; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_rel 7058; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 7059; OUTLINE-ATOMICS-NEXT: ret 7060 atomicrmw xchg ptr @var32, i32 %offset release 7061; CHECK-NOT: dmb 7062; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 7063; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 7064 7065; CHECK: swpl w[[OLD:[0-9]+]], w[[NEW:[0-9,a-z]+]], [x[[ADDR]]] 7066; CHECK-NOT: dmb 7067 7068 ret void 7069} 7070 7071define dso_local void @test_atomic_load_xchg_i64_noret_release(i64 %offset) nounwind { 7072; CHECK-LABEL: test_atomic_load_xchg_i64_noret_release: 7073; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_noret_release: 7074; OUTLINE-ATOMICS: // %bb.0: 7075; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 7076; OUTLINE-ATOMICS-NEXT: adrp x1, var64 7077; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 7078; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_rel 7079; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 7080; OUTLINE-ATOMICS-NEXT: ret 7081 atomicrmw xchg ptr @var64, i64 %offset release 7082; CHECK-NOT: dmb 7083; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 7084; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 7085 7086; CHECK: swpl x[[OLD:[0-9]+]], x[[NEW:[0-9,a-z]+]], [x[[ADDR]]] 7087; CHECK-NOT: dmb 7088 7089 ret void 7090} 7091 7092define dso_local i8 @test_atomic_load_xchg_i8_seq_cst(i8 %offset) nounwind { 7093; CHECK-LABEL: test_atomic_load_xchg_i8_seq_cst: 7094; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i8_seq_cst: 7095; OUTLINE-ATOMICS: // %bb.0: 7096; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 7097; OUTLINE-ATOMICS-NEXT: adrp x1, var8 7098; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 7099; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp1_acq_rel 7100; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 7101; OUTLINE-ATOMICS-NEXT: ret 7102 %old = atomicrmw xchg ptr @var8, i8 %offset seq_cst 7103; CHECK-NOT: dmb 7104; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 7105; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 7106 7107; CHECK: swpalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7108; CHECK-NOT: dmb 7109 7110 ret i8 %old 7111} 7112 7113define dso_local i16 @test_atomic_load_xchg_i16_seq_cst(i16 %offset) nounwind { 7114; CHECK-LABEL: test_atomic_load_xchg_i16_seq_cst: 7115; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i16_seq_cst: 7116; OUTLINE-ATOMICS: // %bb.0: 7117; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 7118; OUTLINE-ATOMICS-NEXT: adrp x1, var16 7119; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 7120; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp2_acq_rel 7121; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 7122; OUTLINE-ATOMICS-NEXT: ret 7123 %old = atomicrmw xchg ptr @var16, i16 %offset seq_cst 7124; CHECK-NOT: dmb 7125; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 7126; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 7127 7128; CHECK: swpalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7129; CHECK-NOT: dmb 7130 7131 ret i16 %old 7132} 7133 7134define dso_local i32 @test_atomic_load_xchg_i32_seq_cst(i32 %offset) nounwind { 7135; CHECK-LABEL: test_atomic_load_xchg_i32_seq_cst: 7136; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_seq_cst: 7137; OUTLINE-ATOMICS: // %bb.0: 7138; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 7139; OUTLINE-ATOMICS-NEXT: adrp x1, var32 7140; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 7141; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq_rel 7142; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 7143; OUTLINE-ATOMICS-NEXT: ret 7144 %old = atomicrmw xchg ptr @var32, i32 %offset seq_cst 7145; CHECK-NOT: dmb 7146; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 7147; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 7148 7149; CHECK: swpal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7150; CHECK-NOT: dmb 7151 7152 ret i32 %old 7153} 7154 7155define dso_local i64 @test_atomic_load_xchg_i64_seq_cst(i64 %offset) nounwind { 7156; CHECK-LABEL: test_atomic_load_xchg_i64_seq_cst: 7157; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_seq_cst: 7158; OUTLINE-ATOMICS: // %bb.0: 7159; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 7160; OUTLINE-ATOMICS-NEXT: adrp x1, var64 7161; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 7162; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq_rel 7163; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 7164; OUTLINE-ATOMICS-NEXT: ret 7165 %old = atomicrmw xchg ptr @var64, i64 %offset seq_cst 7166; CHECK-NOT: dmb 7167; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 7168; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 7169 7170; CHECK: swpal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 7171; CHECK-NOT: dmb 7172 7173 ret i64 %old 7174} 7175 7176define dso_local void @test_atomic_load_xchg_i32_noret_seq_cst(i32 %offset) nounwind { 7177; CHECK-LABEL: test_atomic_load_xchg_i32_noret_seq_cst: 7178; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i32_noret_seq_cst: 7179; OUTLINE-ATOMICS: // %bb.0: 7180; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 7181; OUTLINE-ATOMICS-NEXT: adrp x1, var32 7182; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 7183; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp4_acq_rel 7184; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 7185; OUTLINE-ATOMICS-NEXT: ret 7186 atomicrmw xchg ptr @var32, i32 %offset seq_cst 7187; CHECK-NOT: dmb 7188; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 7189; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 7190 7191; CHECK: swpal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7192; CHECK-NOT: dmb 7193 7194 ret void 7195} 7196 7197define dso_local void @test_atomic_load_xchg_i64_noret_seq_cst(i64 %offset) nounwind { 7198; CHECK-LABEL: test_atomic_load_xchg_i64_noret_seq_cst: 7199; OUTLINE-ATOMICS-LABEL: test_atomic_load_xchg_i64_noret_seq_cst: 7200; OUTLINE-ATOMICS: // %bb.0: 7201; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 7202; OUTLINE-ATOMICS-NEXT: adrp x1, var64 7203; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 7204; OUTLINE-ATOMICS-NEXT: bl __aarch64_swp8_acq_rel 7205; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 7206; OUTLINE-ATOMICS-NEXT: ret 7207 atomicrmw xchg ptr @var64, i64 %offset seq_cst 7208; CHECK-NOT: dmb 7209; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 7210; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 7211 7212; CHECK: swpal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 7213; CHECK-NOT: dmb 7214 7215 ret void 7216} 7217 7218define dso_local i8 @test_atomic_load_umax_i8_acq_rel(i8 %offset) nounwind { 7219; CHECK-LABEL: test_atomic_load_umax_i8_acq_rel: 7220; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_acq_rel: 7221; OUTLINE-ATOMICS: // %bb.0: 7222; OUTLINE-ATOMICS-NEXT: adrp x8, var8 7223; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var8 7224; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xff 7225; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7226; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7227; OUTLINE-ATOMICS-NEXT: ldaxrb w0, [x8] 7228; OUTLINE-ATOMICS-NEXT: cmp w0, w9 7229; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, hi 7230; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x8] 7231; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7232; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7233; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 7234; OUTLINE-ATOMICS-NEXT: ret 7235 %old = atomicrmw umax ptr @var8, i8 %offset acq_rel 7236; CHECK-NOT: dmb 7237; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 7238; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 7239 7240; CHECK: ldumaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7241; CHECK-NOT: dmb 7242 7243 ret i8 %old 7244} 7245 7246define dso_local i16 @test_atomic_load_umax_i16_acq_rel(i16 %offset) nounwind { 7247; CHECK-LABEL: test_atomic_load_umax_i16_acq_rel: 7248; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_acq_rel: 7249; OUTLINE-ATOMICS: // %bb.0: 7250; OUTLINE-ATOMICS-NEXT: adrp x8, var16 7251; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var16 7252; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xffff 7253; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7254; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7255; OUTLINE-ATOMICS-NEXT: ldaxrh w0, [x8] 7256; OUTLINE-ATOMICS-NEXT: cmp w0, w9 7257; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, hi 7258; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x8] 7259; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7260; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7261; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 7262; OUTLINE-ATOMICS-NEXT: ret 7263 %old = atomicrmw umax ptr @var16, i16 %offset acq_rel 7264; CHECK-NOT: dmb 7265; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 7266; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 7267 7268; CHECK: ldumaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7269; CHECK-NOT: dmb 7270 7271 ret i16 %old 7272} 7273 7274define dso_local i32 @test_atomic_load_umax_i32_acq_rel(i32 %offset) nounwind { 7275; CHECK-LABEL: test_atomic_load_umax_i32_acq_rel: 7276; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_acq_rel: 7277; OUTLINE-ATOMICS: // %bb.0: 7278; OUTLINE-ATOMICS-NEXT: adrp x9, var32 7279; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 7280; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7281; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7282; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 7283; OUTLINE-ATOMICS-NEXT: cmp w8, w0 7284; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi 7285; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 7286; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7287; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7288; OUTLINE-ATOMICS-NEXT: mov w0, w8 7289; OUTLINE-ATOMICS-NEXT: ret 7290 %old = atomicrmw umax ptr @var32, i32 %offset acq_rel 7291; CHECK-NOT: dmb 7292; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 7293; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 7294 7295; CHECK: ldumaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7296; CHECK-NOT: dmb 7297 7298 ret i32 %old 7299} 7300 7301define dso_local i64 @test_atomic_load_umax_i64_acq_rel(i64 %offset) nounwind { 7302; CHECK-LABEL: test_atomic_load_umax_i64_acq_rel: 7303; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_acq_rel: 7304; OUTLINE-ATOMICS: // %bb.0: 7305; OUTLINE-ATOMICS-NEXT: adrp x9, var64 7306; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 7307; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7308; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7309; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 7310; OUTLINE-ATOMICS-NEXT: cmp x8, x0 7311; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi 7312; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 7313; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7314; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7315; OUTLINE-ATOMICS-NEXT: mov x0, x8 7316; OUTLINE-ATOMICS-NEXT: ret 7317 %old = atomicrmw umax ptr @var64, i64 %offset acq_rel 7318; CHECK-NOT: dmb 7319; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 7320; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 7321 7322; CHECK: ldumaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 7323; CHECK-NOT: dmb 7324 7325 ret i64 %old 7326} 7327 7328define dso_local void @test_atomic_load_umax_i32_noret_acq_rel(i32 %offset) nounwind { 7329; CHECK-LABEL: test_atomic_load_umax_i32_noret_acq_rel: 7330; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_acq_rel: 7331; OUTLINE-ATOMICS: // %bb.0: 7332; OUTLINE-ATOMICS-NEXT: adrp x8, var32 7333; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 7334; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7335; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7336; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 7337; OUTLINE-ATOMICS-NEXT: cmp w9, w0 7338; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi 7339; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 7340; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 7341; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7342; OUTLINE-ATOMICS-NEXT: ret 7343 atomicrmw umax ptr @var32, i32 %offset acq_rel 7344; CHECK-NOT: dmb 7345; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 7346; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 7347 7348; CHECK: ldumaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 7349; CHECK-NOT: dmb 7350 ret void 7351} 7352 7353define dso_local void @test_atomic_load_umax_i64_noret_acq_rel(i64 %offset) nounwind { 7354; CHECK-LABEL: test_atomic_load_umax_i64_noret_acq_rel: 7355; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_acq_rel: 7356; OUTLINE-ATOMICS: // %bb.0: 7357; OUTLINE-ATOMICS-NEXT: adrp x8, var64 7358; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 7359; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7360; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7361; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 7362; OUTLINE-ATOMICS-NEXT: cmp x9, x0 7363; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi 7364; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 7365; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 7366; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7367; OUTLINE-ATOMICS-NEXT: ret 7368 atomicrmw umax ptr @var64, i64 %offset acq_rel 7369; CHECK-NOT: dmb 7370; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 7371; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 7372 7373; CHECK: ldumaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 7374; CHECK-NOT: dmb 7375 ret void 7376} 7377 7378define dso_local i8 @test_atomic_load_umax_i8_acquire(i8 %offset) nounwind { 7379; CHECK-LABEL: test_atomic_load_umax_i8_acquire: 7380; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_acquire: 7381; OUTLINE-ATOMICS: // %bb.0: 7382; OUTLINE-ATOMICS-NEXT: adrp x8, var8 7383; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var8 7384; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xff 7385; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7386; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7387; OUTLINE-ATOMICS-NEXT: ldaxrb w0, [x8] 7388; OUTLINE-ATOMICS-NEXT: cmp w0, w9 7389; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, hi 7390; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x8] 7391; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7392; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7393; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 7394; OUTLINE-ATOMICS-NEXT: ret 7395 %old = atomicrmw umax ptr @var8, i8 %offset acquire 7396; CHECK-NOT: dmb 7397; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 7398; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 7399 7400; CHECK: ldumaxab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7401; CHECK-NOT: dmb 7402 7403 ret i8 %old 7404} 7405 7406define dso_local i16 @test_atomic_load_umax_i16_acquire(i16 %offset) nounwind { 7407; CHECK-LABEL: test_atomic_load_umax_i16_acquire: 7408; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_acquire: 7409; OUTLINE-ATOMICS: // %bb.0: 7410; OUTLINE-ATOMICS-NEXT: adrp x8, var16 7411; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var16 7412; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xffff 7413; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7414; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7415; OUTLINE-ATOMICS-NEXT: ldaxrh w0, [x8] 7416; OUTLINE-ATOMICS-NEXT: cmp w0, w9 7417; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, hi 7418; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x8] 7419; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7420; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7421; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 7422; OUTLINE-ATOMICS-NEXT: ret 7423 %old = atomicrmw umax ptr @var16, i16 %offset acquire 7424; CHECK-NOT: dmb 7425; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 7426; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 7427 7428; CHECK: ldumaxah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7429; CHECK-NOT: dmb 7430 7431 ret i16 %old 7432} 7433 7434define dso_local i32 @test_atomic_load_umax_i32_acquire(i32 %offset) nounwind { 7435; CHECK-LABEL: test_atomic_load_umax_i32_acquire: 7436; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_acquire: 7437; OUTLINE-ATOMICS: // %bb.0: 7438; OUTLINE-ATOMICS-NEXT: adrp x9, var32 7439; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 7440; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7441; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7442; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 7443; OUTLINE-ATOMICS-NEXT: cmp w8, w0 7444; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi 7445; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] 7446; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7447; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7448; OUTLINE-ATOMICS-NEXT: mov w0, w8 7449; OUTLINE-ATOMICS-NEXT: ret 7450 %old = atomicrmw umax ptr @var32, i32 %offset acquire 7451; CHECK-NOT: dmb 7452; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 7453; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 7454 7455; CHECK: ldumaxa w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7456; CHECK-NOT: dmb 7457 7458 ret i32 %old 7459} 7460 7461define dso_local i64 @test_atomic_load_umax_i64_acquire(i64 %offset) nounwind { 7462; CHECK-LABEL: test_atomic_load_umax_i64_acquire: 7463; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_acquire: 7464; OUTLINE-ATOMICS: // %bb.0: 7465; OUTLINE-ATOMICS-NEXT: adrp x9, var64 7466; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 7467; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7468; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7469; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 7470; OUTLINE-ATOMICS-NEXT: cmp x8, x0 7471; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi 7472; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] 7473; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7474; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7475; OUTLINE-ATOMICS-NEXT: mov x0, x8 7476; OUTLINE-ATOMICS-NEXT: ret 7477 %old = atomicrmw umax ptr @var64, i64 %offset acquire 7478; CHECK-NOT: dmb 7479; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 7480; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 7481 7482; CHECK: ldumaxa x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 7483; CHECK-NOT: dmb 7484 7485 ret i64 %old 7486} 7487 7488define dso_local void @test_atomic_load_umax_i32_noret_acquire(i32 %offset) nounwind { 7489; CHECK-LABEL: test_atomic_load_umax_i32_noret_acquire: 7490; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_acquire: 7491; OUTLINE-ATOMICS: // %bb.0: 7492; OUTLINE-ATOMICS-NEXT: adrp x8, var32 7493; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 7494; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7495; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7496; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 7497; OUTLINE-ATOMICS-NEXT: cmp w9, w0 7498; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi 7499; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] 7500; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 7501; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7502; OUTLINE-ATOMICS-NEXT: ret 7503 atomicrmw umax ptr @var32, i32 %offset acquire 7504; CHECK-NOT: dmb 7505; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 7506; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 7507 7508; CHECK: ldumaxa w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 7509; CHECK-NOT: dmb 7510 ret void 7511} 7512 7513define dso_local void @test_atomic_load_umax_i64_noret_acquire(i64 %offset) nounwind { 7514; CHECK-LABEL: test_atomic_load_umax_i64_noret_acquire: 7515; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_acquire: 7516; OUTLINE-ATOMICS: // %bb.0: 7517; OUTLINE-ATOMICS-NEXT: adrp x8, var64 7518; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 7519; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7520; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7521; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 7522; OUTLINE-ATOMICS-NEXT: cmp x9, x0 7523; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi 7524; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] 7525; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 7526; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7527; OUTLINE-ATOMICS-NEXT: ret 7528 atomicrmw umax ptr @var64, i64 %offset acquire 7529; CHECK-NOT: dmb 7530; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 7531; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 7532 7533; CHECK: ldumaxa x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 7534; CHECK-NOT: dmb 7535 ret void 7536} 7537 7538define dso_local i8 @test_atomic_load_umax_i8_monotonic(i8 %offset) nounwind { 7539; CHECK-LABEL: test_atomic_load_umax_i8_monotonic: 7540; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_monotonic: 7541; OUTLINE-ATOMICS: // %bb.0: 7542; OUTLINE-ATOMICS-NEXT: adrp x8, var8 7543; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var8 7544; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xff 7545; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7546; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7547; OUTLINE-ATOMICS-NEXT: ldxrb w0, [x8] 7548; OUTLINE-ATOMICS-NEXT: cmp w0, w9 7549; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, hi 7550; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x8] 7551; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7552; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7553; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 7554; OUTLINE-ATOMICS-NEXT: ret 7555 %old = atomicrmw umax ptr @var8, i8 %offset monotonic 7556; CHECK-NOT: dmb 7557; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 7558; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 7559 7560; CHECK: ldumaxb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7561; CHECK-NOT: dmb 7562 7563 ret i8 %old 7564} 7565 7566define dso_local i16 @test_atomic_load_umax_i16_monotonic(i16 %offset) nounwind { 7567; CHECK-LABEL: test_atomic_load_umax_i16_monotonic: 7568; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_monotonic: 7569; OUTLINE-ATOMICS: // %bb.0: 7570; OUTLINE-ATOMICS-NEXT: adrp x8, var16 7571; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var16 7572; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xffff 7573; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7574; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7575; OUTLINE-ATOMICS-NEXT: ldxrh w0, [x8] 7576; OUTLINE-ATOMICS-NEXT: cmp w0, w9 7577; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, hi 7578; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x8] 7579; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7580; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7581; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 7582; OUTLINE-ATOMICS-NEXT: ret 7583 %old = atomicrmw umax ptr @var16, i16 %offset monotonic 7584; CHECK-NOT: dmb 7585; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 7586; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 7587 7588; CHECK: ldumaxh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7589; CHECK-NOT: dmb 7590 7591 ret i16 %old 7592} 7593 7594define dso_local i32 @test_atomic_load_umax_i32_monotonic(i32 %offset) nounwind { 7595; CHECK-LABEL: test_atomic_load_umax_i32_monotonic: 7596; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_monotonic: 7597; OUTLINE-ATOMICS: // %bb.0: 7598; OUTLINE-ATOMICS-NEXT: adrp x9, var32 7599; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 7600; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7601; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7602; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] 7603; OUTLINE-ATOMICS-NEXT: cmp w8, w0 7604; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi 7605; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] 7606; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7607; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7608; OUTLINE-ATOMICS-NEXT: mov w0, w8 7609; OUTLINE-ATOMICS-NEXT: ret 7610 %old = atomicrmw umax ptr @var32, i32 %offset monotonic 7611; CHECK-NOT: dmb 7612; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 7613; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 7614 7615; CHECK: ldumax w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7616; CHECK-NOT: dmb 7617 7618 ret i32 %old 7619} 7620 7621define dso_local i64 @test_atomic_load_umax_i64_monotonic(i64 %offset) nounwind { 7622; CHECK-LABEL: test_atomic_load_umax_i64_monotonic: 7623; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_monotonic: 7624; OUTLINE-ATOMICS: // %bb.0: 7625; OUTLINE-ATOMICS-NEXT: adrp x9, var64 7626; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 7627; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7628; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7629; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] 7630; OUTLINE-ATOMICS-NEXT: cmp x8, x0 7631; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi 7632; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] 7633; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7634; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7635; OUTLINE-ATOMICS-NEXT: mov x0, x8 7636; OUTLINE-ATOMICS-NEXT: ret 7637 %old = atomicrmw umax ptr @var64, i64 %offset monotonic 7638; CHECK-NOT: dmb 7639; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 7640; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 7641 7642; CHECK: ldumax x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 7643; CHECK-NOT: dmb 7644 7645 ret i64 %old 7646} 7647 7648define dso_local void @test_atomic_load_umax_i32_noret_monotonic(i32 %offset) nounwind { 7649; CHECK-LABEL: test_atomic_load_umax_i32_noret_monotonic: 7650; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_monotonic: 7651; OUTLINE-ATOMICS: // %bb.0: 7652; OUTLINE-ATOMICS-NEXT: adrp x8, var32 7653; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 7654; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7655; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7656; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] 7657; OUTLINE-ATOMICS-NEXT: cmp w9, w0 7658; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi 7659; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] 7660; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 7661; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7662; OUTLINE-ATOMICS-NEXT: ret 7663 atomicrmw umax ptr @var32, i32 %offset monotonic 7664; CHECK-NOT: dmb 7665; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 7666; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 7667 7668; CHECK: ldumax w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] 7669; CHECK-NOT: dmb 7670 ret void 7671} 7672 7673define dso_local void @test_atomic_load_umax_i64_noret_monotonic(i64 %offset) nounwind { 7674; CHECK-LABEL: test_atomic_load_umax_i64_noret_monotonic: 7675; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_monotonic: 7676; OUTLINE-ATOMICS: // %bb.0: 7677; OUTLINE-ATOMICS-NEXT: adrp x8, var64 7678; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 7679; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7680; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7681; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] 7682; OUTLINE-ATOMICS-NEXT: cmp x9, x0 7683; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi 7684; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] 7685; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 7686; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7687; OUTLINE-ATOMICS-NEXT: ret 7688 atomicrmw umax ptr @var64, i64 %offset monotonic 7689; CHECK-NOT: dmb 7690; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 7691; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 7692 7693; CHECK: ldumax x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] 7694; CHECK-NOT: dmb 7695 ret void 7696} 7697 7698define dso_local i8 @test_atomic_load_umax_i8_release(i8 %offset) nounwind { 7699; CHECK-LABEL: test_atomic_load_umax_i8_release: 7700; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_release: 7701; OUTLINE-ATOMICS: // %bb.0: 7702; OUTLINE-ATOMICS-NEXT: adrp x8, var8 7703; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var8 7704; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xff 7705; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7706; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7707; OUTLINE-ATOMICS-NEXT: ldxrb w0, [x8] 7708; OUTLINE-ATOMICS-NEXT: cmp w0, w9 7709; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, hi 7710; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x8] 7711; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7712; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7713; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 7714; OUTLINE-ATOMICS-NEXT: ret 7715 %old = atomicrmw umax ptr @var8, i8 %offset release 7716; CHECK-NOT: dmb 7717; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 7718; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 7719 7720; CHECK: ldumaxlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7721; CHECK-NOT: dmb 7722 7723 ret i8 %old 7724} 7725 7726define dso_local i16 @test_atomic_load_umax_i16_release(i16 %offset) nounwind { 7727; CHECK-LABEL: test_atomic_load_umax_i16_release: 7728; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_release: 7729; OUTLINE-ATOMICS: // %bb.0: 7730; OUTLINE-ATOMICS-NEXT: adrp x8, var16 7731; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var16 7732; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xffff 7733; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7734; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7735; OUTLINE-ATOMICS-NEXT: ldxrh w0, [x8] 7736; OUTLINE-ATOMICS-NEXT: cmp w0, w9 7737; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, hi 7738; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x8] 7739; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7740; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7741; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 7742; OUTLINE-ATOMICS-NEXT: ret 7743 %old = atomicrmw umax ptr @var16, i16 %offset release 7744; CHECK-NOT: dmb 7745; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 7746; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 7747 7748; CHECK: ldumaxlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7749; CHECK-NOT: dmb 7750 7751 ret i16 %old 7752} 7753 7754define dso_local i32 @test_atomic_load_umax_i32_release(i32 %offset) nounwind { 7755; CHECK-LABEL: test_atomic_load_umax_i32_release: 7756; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_release: 7757; OUTLINE-ATOMICS: // %bb.0: 7758; OUTLINE-ATOMICS-NEXT: adrp x9, var32 7759; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 7760; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7761; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7762; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] 7763; OUTLINE-ATOMICS-NEXT: cmp w8, w0 7764; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi 7765; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 7766; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7767; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7768; OUTLINE-ATOMICS-NEXT: mov w0, w8 7769; OUTLINE-ATOMICS-NEXT: ret 7770 %old = atomicrmw umax ptr @var32, i32 %offset release 7771; CHECK-NOT: dmb 7772; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 7773; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 7774 7775; CHECK: ldumaxl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7776; CHECK-NOT: dmb 7777 7778 ret i32 %old 7779} 7780 7781define dso_local i64 @test_atomic_load_umax_i64_release(i64 %offset) nounwind { 7782; CHECK-LABEL: test_atomic_load_umax_i64_release: 7783; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_release: 7784; OUTLINE-ATOMICS: // %bb.0: 7785; OUTLINE-ATOMICS-NEXT: adrp x9, var64 7786; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 7787; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7788; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7789; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] 7790; OUTLINE-ATOMICS-NEXT: cmp x8, x0 7791; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi 7792; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 7793; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7794; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7795; OUTLINE-ATOMICS-NEXT: mov x0, x8 7796; OUTLINE-ATOMICS-NEXT: ret 7797 %old = atomicrmw umax ptr @var64, i64 %offset release 7798; CHECK-NOT: dmb 7799; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 7800; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 7801 7802; CHECK: ldumaxl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 7803; CHECK-NOT: dmb 7804 7805 ret i64 %old 7806} 7807 7808define dso_local void @test_atomic_load_umax_i32_noret_release(i32 %offset) nounwind { 7809; CHECK-LABEL: test_atomic_load_umax_i32_noret_release: 7810; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_release: 7811; OUTLINE-ATOMICS: // %bb.0: 7812; OUTLINE-ATOMICS-NEXT: adrp x8, var32 7813; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 7814; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7815; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7816; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] 7817; OUTLINE-ATOMICS-NEXT: cmp w9, w0 7818; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi 7819; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 7820; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 7821; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7822; OUTLINE-ATOMICS-NEXT: ret 7823 atomicrmw umax ptr @var32, i32 %offset release 7824; CHECK-NOT: dmb 7825; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 7826; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 7827 7828; CHECK: ldumaxl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] 7829; CHECK-NOT: dmb 7830 ret void 7831} 7832 7833define dso_local void @test_atomic_load_umax_i64_noret_release(i64 %offset) nounwind { 7834; CHECK-LABEL: test_atomic_load_umax_i64_noret_release: 7835; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_release: 7836; OUTLINE-ATOMICS: // %bb.0: 7837; OUTLINE-ATOMICS-NEXT: adrp x8, var64 7838; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 7839; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7840; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7841; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] 7842; OUTLINE-ATOMICS-NEXT: cmp x9, x0 7843; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi 7844; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 7845; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 7846; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7847; OUTLINE-ATOMICS-NEXT: ret 7848 atomicrmw umax ptr @var64, i64 %offset release 7849; CHECK-NOT: dmb 7850; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 7851; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 7852 7853; CHECK: ldumaxl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] 7854; CHECK-NOT: dmb 7855 ret void 7856} 7857 7858define dso_local i8 @test_atomic_load_umax_i8_seq_cst(i8 %offset) nounwind { 7859; CHECK-LABEL: test_atomic_load_umax_i8_seq_cst: 7860; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i8_seq_cst: 7861; OUTLINE-ATOMICS: // %bb.0: 7862; OUTLINE-ATOMICS-NEXT: adrp x8, var8 7863; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var8 7864; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xff 7865; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7866; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7867; OUTLINE-ATOMICS-NEXT: ldaxrb w0, [x8] 7868; OUTLINE-ATOMICS-NEXT: cmp w0, w9 7869; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, hi 7870; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x8] 7871; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7872; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7873; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 7874; OUTLINE-ATOMICS-NEXT: ret 7875 %old = atomicrmw umax ptr @var8, i8 %offset seq_cst 7876; CHECK-NOT: dmb 7877; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 7878; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 7879 7880; CHECK: ldumaxalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7881; CHECK-NOT: dmb 7882 7883 ret i8 %old 7884} 7885 7886define dso_local i16 @test_atomic_load_umax_i16_seq_cst(i16 %offset) nounwind { 7887; CHECK-LABEL: test_atomic_load_umax_i16_seq_cst: 7888; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i16_seq_cst: 7889; OUTLINE-ATOMICS: // %bb.0: 7890; OUTLINE-ATOMICS-NEXT: adrp x8, var16 7891; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var16 7892; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xffff 7893; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7894; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7895; OUTLINE-ATOMICS-NEXT: ldaxrh w0, [x8] 7896; OUTLINE-ATOMICS-NEXT: cmp w0, w9 7897; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, hi 7898; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x8] 7899; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7900; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7901; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 7902; OUTLINE-ATOMICS-NEXT: ret 7903 %old = atomicrmw umax ptr @var16, i16 %offset seq_cst 7904; CHECK-NOT: dmb 7905; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 7906; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 7907 7908; CHECK: ldumaxalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7909; CHECK-NOT: dmb 7910 7911 ret i16 %old 7912} 7913 7914define dso_local i32 @test_atomic_load_umax_i32_seq_cst(i32 %offset) nounwind { 7915; CHECK-LABEL: test_atomic_load_umax_i32_seq_cst: 7916; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_seq_cst: 7917; OUTLINE-ATOMICS: // %bb.0: 7918; OUTLINE-ATOMICS-NEXT: adrp x9, var32 7919; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 7920; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7921; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7922; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 7923; OUTLINE-ATOMICS-NEXT: cmp w8, w0 7924; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, hi 7925; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 7926; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7927; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7928; OUTLINE-ATOMICS-NEXT: mov w0, w8 7929; OUTLINE-ATOMICS-NEXT: ret 7930 %old = atomicrmw umax ptr @var32, i32 %offset seq_cst 7931; CHECK-NOT: dmb 7932; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 7933; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 7934 7935; CHECK: ldumaxal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 7936; CHECK-NOT: dmb 7937 7938 ret i32 %old 7939} 7940 7941define dso_local i64 @test_atomic_load_umax_i64_seq_cst(i64 %offset) nounwind { 7942; CHECK-LABEL: test_atomic_load_umax_i64_seq_cst: 7943; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_seq_cst: 7944; OUTLINE-ATOMICS: // %bb.0: 7945; OUTLINE-ATOMICS-NEXT: adrp x9, var64 7946; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 7947; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7948; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7949; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 7950; OUTLINE-ATOMICS-NEXT: cmp x8, x0 7951; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, hi 7952; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 7953; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 7954; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7955; OUTLINE-ATOMICS-NEXT: mov x0, x8 7956; OUTLINE-ATOMICS-NEXT: ret 7957 %old = atomicrmw umax ptr @var64, i64 %offset seq_cst 7958; CHECK-NOT: dmb 7959; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 7960; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 7961 7962; CHECK: ldumaxal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 7963; CHECK-NOT: dmb 7964 7965 ret i64 %old 7966} 7967 7968define dso_local void @test_atomic_load_umax_i32_noret_seq_cst(i32 %offset) nounwind { 7969; CHECK-LABEL: test_atomic_load_umax_i32_noret_seq_cst: 7970; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i32_noret_seq_cst: 7971; OUTLINE-ATOMICS: // %bb.0: 7972; OUTLINE-ATOMICS-NEXT: adrp x8, var32 7973; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 7974; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 7975; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 7976; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 7977; OUTLINE-ATOMICS-NEXT: cmp w9, w0 7978; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, hi 7979; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 7980; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 7981; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 7982; OUTLINE-ATOMICS-NEXT: ret 7983 atomicrmw umax ptr @var32, i32 %offset seq_cst 7984; CHECK-NOT: dmb 7985; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 7986; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 7987 7988; CHECK: ldumaxal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 7989; CHECK-NOT: dmb 7990 ret void 7991} 7992 7993define dso_local void @test_atomic_load_umax_i64_noret_seq_cst(i64 %offset) nounwind { 7994; CHECK-LABEL: test_atomic_load_umax_i64_noret_seq_cst: 7995; OUTLINE-ATOMICS-LABEL: test_atomic_load_umax_i64_noret_seq_cst: 7996; OUTLINE-ATOMICS: // %bb.0: 7997; OUTLINE-ATOMICS-NEXT: adrp x8, var64 7998; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 7999; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8000; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8001; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 8002; OUTLINE-ATOMICS-NEXT: cmp x9, x0 8003; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, hi 8004; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 8005; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 8006; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8007; OUTLINE-ATOMICS-NEXT: ret 8008 atomicrmw umax ptr @var64, i64 %offset seq_cst 8009; CHECK-NOT: dmb 8010; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 8011; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 8012 8013; CHECK: ldumaxal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 8014; CHECK-NOT: dmb 8015 ret void 8016} 8017 8018define dso_local i8 @test_atomic_load_umin_i8_acq_rel(i8 %offset) nounwind { 8019; CHECK-LABEL: test_atomic_load_umin_i8_acq_rel: 8020; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_acq_rel: 8021; OUTLINE-ATOMICS: // %bb.0: 8022; OUTLINE-ATOMICS-NEXT: adrp x8, var8 8023; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var8 8024; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xff 8025; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8026; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8027; OUTLINE-ATOMICS-NEXT: ldaxrb w0, [x8] 8028; OUTLINE-ATOMICS-NEXT: cmp w0, w9 8029; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, ls 8030; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x8] 8031; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8032; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8033; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 8034; OUTLINE-ATOMICS-NEXT: ret 8035 %old = atomicrmw umin ptr @var8, i8 %offset acq_rel 8036; CHECK-NOT: dmb 8037; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 8038; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 8039 8040; CHECK: lduminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8041; CHECK-NOT: dmb 8042 8043 ret i8 %old 8044} 8045 8046define dso_local i16 @test_atomic_load_umin_i16_acq_rel(i16 %offset) nounwind { 8047; CHECK-LABEL: test_atomic_load_umin_i16_acq_rel: 8048; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_acq_rel: 8049; OUTLINE-ATOMICS: // %bb.0: 8050; OUTLINE-ATOMICS-NEXT: adrp x8, var16 8051; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var16 8052; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xffff 8053; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8054; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8055; OUTLINE-ATOMICS-NEXT: ldaxrh w0, [x8] 8056; OUTLINE-ATOMICS-NEXT: cmp w0, w9 8057; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, ls 8058; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x8] 8059; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8060; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8061; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 8062; OUTLINE-ATOMICS-NEXT: ret 8063 %old = atomicrmw umin ptr @var16, i16 %offset acq_rel 8064; CHECK-NOT: dmb 8065; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 8066; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 8067 8068; CHECK: lduminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8069; CHECK-NOT: dmb 8070 8071 ret i16 %old 8072} 8073 8074define dso_local i32 @test_atomic_load_umin_i32_acq_rel(i32 %offset) nounwind { 8075; CHECK-LABEL: test_atomic_load_umin_i32_acq_rel: 8076; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_acq_rel: 8077; OUTLINE-ATOMICS: // %bb.0: 8078; OUTLINE-ATOMICS-NEXT: adrp x9, var32 8079; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 8080; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8081; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8082; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 8083; OUTLINE-ATOMICS-NEXT: cmp w8, w0 8084; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls 8085; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 8086; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8087; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8088; OUTLINE-ATOMICS-NEXT: mov w0, w8 8089; OUTLINE-ATOMICS-NEXT: ret 8090 %old = atomicrmw umin ptr @var32, i32 %offset acq_rel 8091; CHECK-NOT: dmb 8092; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 8093; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 8094 8095; CHECK: lduminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8096; CHECK-NOT: dmb 8097 8098 ret i32 %old 8099} 8100 8101define dso_local i64 @test_atomic_load_umin_i64_acq_rel(i64 %offset) nounwind { 8102; CHECK-LABEL: test_atomic_load_umin_i64_acq_rel: 8103; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_acq_rel: 8104; OUTLINE-ATOMICS: // %bb.0: 8105; OUTLINE-ATOMICS-NEXT: adrp x9, var64 8106; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 8107; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8108; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8109; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 8110; OUTLINE-ATOMICS-NEXT: cmp x8, x0 8111; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls 8112; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 8113; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8114; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8115; OUTLINE-ATOMICS-NEXT: mov x0, x8 8116; OUTLINE-ATOMICS-NEXT: ret 8117 %old = atomicrmw umin ptr @var64, i64 %offset acq_rel 8118; CHECK-NOT: dmb 8119; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 8120; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 8121 8122; CHECK: lduminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 8123; CHECK-NOT: dmb 8124 8125 ret i64 %old 8126} 8127 8128define dso_local void @test_atomic_load_umin_i32_noret_acq_rel(i32 %offset) nounwind { 8129; CHECK-LABEL: test_atomic_load_umin_i32_noret_acq_rel: 8130; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_acq_rel: 8131; OUTLINE-ATOMICS: // %bb.0: 8132; OUTLINE-ATOMICS-NEXT: adrp x8, var32 8133; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 8134; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8135; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8136; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 8137; OUTLINE-ATOMICS-NEXT: cmp w9, w0 8138; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls 8139; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 8140; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 8141; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8142; OUTLINE-ATOMICS-NEXT: ret 8143 atomicrmw umin ptr @var32, i32 %offset acq_rel 8144; CHECK-NOT: dmb 8145; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 8146; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 8147 8148; CHECK: lduminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 8149; CHECK-NOT: dmb 8150 ret void 8151} 8152 8153define dso_local void @test_atomic_load_umin_i64_noret_acq_rel(i64 %offset) nounwind { 8154; CHECK-LABEL: test_atomic_load_umin_i64_noret_acq_rel: 8155; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_acq_rel: 8156; OUTLINE-ATOMICS: // %bb.0: 8157; OUTLINE-ATOMICS-NEXT: adrp x8, var64 8158; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 8159; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8160; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8161; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 8162; OUTLINE-ATOMICS-NEXT: cmp x9, x0 8163; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls 8164; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 8165; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 8166; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8167; OUTLINE-ATOMICS-NEXT: ret 8168 atomicrmw umin ptr @var64, i64 %offset acq_rel 8169; CHECK-NOT: dmb 8170; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 8171; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 8172 8173; CHECK: lduminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 8174; CHECK-NOT: dmb 8175 ret void 8176} 8177 8178define dso_local i8 @test_atomic_load_umin_i8_acquire(i8 %offset) nounwind { 8179; CHECK-LABEL: test_atomic_load_umin_i8_acquire: 8180; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_acquire: 8181; OUTLINE-ATOMICS: // %bb.0: 8182; OUTLINE-ATOMICS-NEXT: adrp x8, var8 8183; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var8 8184; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xff 8185; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8186; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8187; OUTLINE-ATOMICS-NEXT: ldaxrb w0, [x8] 8188; OUTLINE-ATOMICS-NEXT: cmp w0, w9 8189; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, ls 8190; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x8] 8191; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8192; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8193; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 8194; OUTLINE-ATOMICS-NEXT: ret 8195 %old = atomicrmw umin ptr @var8, i8 %offset acquire 8196; CHECK-NOT: dmb 8197; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 8198; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 8199 8200; CHECK: lduminab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8201; CHECK-NOT: dmb 8202 8203 ret i8 %old 8204} 8205 8206define dso_local i16 @test_atomic_load_umin_i16_acquire(i16 %offset) nounwind { 8207; CHECK-LABEL: test_atomic_load_umin_i16_acquire: 8208; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_acquire: 8209; OUTLINE-ATOMICS: // %bb.0: 8210; OUTLINE-ATOMICS-NEXT: adrp x8, var16 8211; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var16 8212; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xffff 8213; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8214; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8215; OUTLINE-ATOMICS-NEXT: ldaxrh w0, [x8] 8216; OUTLINE-ATOMICS-NEXT: cmp w0, w9 8217; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, ls 8218; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x8] 8219; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8220; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8221; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 8222; OUTLINE-ATOMICS-NEXT: ret 8223 %old = atomicrmw umin ptr @var16, i16 %offset acquire 8224; CHECK-NOT: dmb 8225; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 8226; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 8227 8228; CHECK: lduminah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8229; CHECK-NOT: dmb 8230 8231 ret i16 %old 8232} 8233 8234define dso_local i32 @test_atomic_load_umin_i32_acquire(i32 %offset) nounwind { 8235; CHECK-LABEL: test_atomic_load_umin_i32_acquire: 8236; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_acquire: 8237; OUTLINE-ATOMICS: // %bb.0: 8238; OUTLINE-ATOMICS-NEXT: adrp x9, var32 8239; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 8240; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8241; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8242; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 8243; OUTLINE-ATOMICS-NEXT: cmp w8, w0 8244; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls 8245; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] 8246; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8247; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8248; OUTLINE-ATOMICS-NEXT: mov w0, w8 8249; OUTLINE-ATOMICS-NEXT: ret 8250 %old = atomicrmw umin ptr @var32, i32 %offset acquire 8251; CHECK-NOT: dmb 8252; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 8253; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 8254 8255; CHECK: ldumina w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8256; CHECK-NOT: dmb 8257 8258 ret i32 %old 8259} 8260 8261define dso_local i64 @test_atomic_load_umin_i64_acquire(i64 %offset) nounwind { 8262; CHECK-LABEL: test_atomic_load_umin_i64_acquire: 8263; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_acquire: 8264; OUTLINE-ATOMICS: // %bb.0: 8265; OUTLINE-ATOMICS-NEXT: adrp x9, var64 8266; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 8267; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8268; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8269; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 8270; OUTLINE-ATOMICS-NEXT: cmp x8, x0 8271; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls 8272; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] 8273; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8274; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8275; OUTLINE-ATOMICS-NEXT: mov x0, x8 8276; OUTLINE-ATOMICS-NEXT: ret 8277 %old = atomicrmw umin ptr @var64, i64 %offset acquire 8278; CHECK-NOT: dmb 8279; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 8280; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 8281 8282; CHECK: ldumina x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 8283; CHECK-NOT: dmb 8284 8285 ret i64 %old 8286} 8287 8288define dso_local void @test_atomic_load_umin_i32_noret_acquire(i32 %offset) nounwind { 8289; CHECK-LABEL: test_atomic_load_umin_i32_noret_acquire: 8290; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_acquire: 8291; OUTLINE-ATOMICS: // %bb.0: 8292; OUTLINE-ATOMICS-NEXT: adrp x8, var32 8293; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 8294; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8295; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8296; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 8297; OUTLINE-ATOMICS-NEXT: cmp w9, w0 8298; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls 8299; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] 8300; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 8301; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8302; OUTLINE-ATOMICS-NEXT: ret 8303 atomicrmw umin ptr @var32, i32 %offset acquire 8304; CHECK-NOT: dmb 8305; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 8306; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 8307 8308; CHECK: ldumina w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 8309; CHECK-NOT: dmb 8310 ret void 8311} 8312 8313define dso_local void @test_atomic_load_umin_i64_noret_acquire(i64 %offset) nounwind { 8314; CHECK-LABEL: test_atomic_load_umin_i64_noret_acquire: 8315; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_acquire: 8316; OUTLINE-ATOMICS: // %bb.0: 8317; OUTLINE-ATOMICS-NEXT: adrp x8, var64 8318; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 8319; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8320; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8321; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 8322; OUTLINE-ATOMICS-NEXT: cmp x9, x0 8323; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls 8324; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] 8325; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 8326; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8327; OUTLINE-ATOMICS-NEXT: ret 8328 atomicrmw umin ptr @var64, i64 %offset acquire 8329; CHECK-NOT: dmb 8330; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 8331; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 8332 8333; CHECK: ldumina x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 8334; CHECK-NOT: dmb 8335 ret void 8336} 8337 8338define dso_local i8 @test_atomic_load_umin_i8_monotonic(i8 %offset) nounwind { 8339; CHECK-LABEL: test_atomic_load_umin_i8_monotonic: 8340; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_monotonic: 8341; OUTLINE-ATOMICS: // %bb.0: 8342; OUTLINE-ATOMICS-NEXT: adrp x8, var8 8343; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var8 8344; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xff 8345; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8346; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8347; OUTLINE-ATOMICS-NEXT: ldxrb w0, [x8] 8348; OUTLINE-ATOMICS-NEXT: cmp w0, w9 8349; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, ls 8350; OUTLINE-ATOMICS-NEXT: stxrb w11, w10, [x8] 8351; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8352; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8353; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 8354; OUTLINE-ATOMICS-NEXT: ret 8355 %old = atomicrmw umin ptr @var8, i8 %offset monotonic 8356; CHECK-NOT: dmb 8357; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 8358; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 8359 8360; CHECK: lduminb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8361; CHECK-NOT: dmb 8362 8363 ret i8 %old 8364} 8365 8366define dso_local i16 @test_atomic_load_umin_i16_monotonic(i16 %offset) nounwind { 8367; CHECK-LABEL: test_atomic_load_umin_i16_monotonic: 8368; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_monotonic: 8369; OUTLINE-ATOMICS: // %bb.0: 8370; OUTLINE-ATOMICS-NEXT: adrp x8, var16 8371; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var16 8372; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xffff 8373; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8374; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8375; OUTLINE-ATOMICS-NEXT: ldxrh w0, [x8] 8376; OUTLINE-ATOMICS-NEXT: cmp w0, w9 8377; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, ls 8378; OUTLINE-ATOMICS-NEXT: stxrh w11, w10, [x8] 8379; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8380; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8381; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 8382; OUTLINE-ATOMICS-NEXT: ret 8383 %old = atomicrmw umin ptr @var16, i16 %offset monotonic 8384; CHECK-NOT: dmb 8385; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 8386; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 8387 8388; CHECK: lduminh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8389; CHECK-NOT: dmb 8390 8391 ret i16 %old 8392} 8393 8394define dso_local i32 @test_atomic_load_umin_i32_monotonic(i32 %offset) nounwind { 8395; CHECK-LABEL: test_atomic_load_umin_i32_monotonic: 8396; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_monotonic: 8397; OUTLINE-ATOMICS: // %bb.0: 8398; OUTLINE-ATOMICS-NEXT: adrp x9, var32 8399; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 8400; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8401; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8402; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] 8403; OUTLINE-ATOMICS-NEXT: cmp w8, w0 8404; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls 8405; OUTLINE-ATOMICS-NEXT: stxr w11, w10, [x9] 8406; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8407; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8408; OUTLINE-ATOMICS-NEXT: mov w0, w8 8409; OUTLINE-ATOMICS-NEXT: ret 8410 %old = atomicrmw umin ptr @var32, i32 %offset monotonic 8411; CHECK-NOT: dmb 8412; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 8413; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 8414 8415; CHECK: ldumin w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8416; CHECK-NOT: dmb 8417 8418 ret i32 %old 8419} 8420 8421define dso_local i64 @test_atomic_load_umin_i64_monotonic(i64 %offset) nounwind { 8422; CHECK-LABEL: test_atomic_load_umin_i64_monotonic: 8423; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_monotonic: 8424; OUTLINE-ATOMICS: // %bb.0: 8425; OUTLINE-ATOMICS-NEXT: adrp x9, var64 8426; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 8427; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8428; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8429; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] 8430; OUTLINE-ATOMICS-NEXT: cmp x8, x0 8431; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls 8432; OUTLINE-ATOMICS-NEXT: stxr w11, x10, [x9] 8433; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8434; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8435; OUTLINE-ATOMICS-NEXT: mov x0, x8 8436; OUTLINE-ATOMICS-NEXT: ret 8437 %old = atomicrmw umin ptr @var64, i64 %offset monotonic 8438; CHECK-NOT: dmb 8439; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 8440; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 8441 8442; CHECK: ldumin x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 8443; CHECK-NOT: dmb 8444 8445 ret i64 %old 8446} 8447 8448define dso_local void @test_atomic_load_umin_i32_noret_monotonic(i32 %offset) nounwind { 8449; CHECK-LABEL: test_atomic_load_umin_i32_noret_monotonic: 8450; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_monotonic: 8451; OUTLINE-ATOMICS: // %bb.0: 8452; OUTLINE-ATOMICS-NEXT: adrp x8, var32 8453; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 8454; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8455; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8456; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] 8457; OUTLINE-ATOMICS-NEXT: cmp w9, w0 8458; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls 8459; OUTLINE-ATOMICS-NEXT: stxr w10, w9, [x8] 8460; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 8461; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8462; OUTLINE-ATOMICS-NEXT: ret 8463 atomicrmw umin ptr @var32, i32 %offset monotonic 8464; CHECK-NOT: dmb 8465; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 8466; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 8467 8468; CHECK: ldumin w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] 8469; CHECK-NOT: dmb 8470 ret void 8471} 8472 8473define dso_local void @test_atomic_load_umin_i64_noret_monotonic(i64 %offset) nounwind { 8474; CHECK-LABEL: test_atomic_load_umin_i64_noret_monotonic: 8475; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_monotonic: 8476; OUTLINE-ATOMICS: // %bb.0: 8477; OUTLINE-ATOMICS-NEXT: adrp x8, var64 8478; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 8479; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8480; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8481; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] 8482; OUTLINE-ATOMICS-NEXT: cmp x9, x0 8483; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls 8484; OUTLINE-ATOMICS-NEXT: stxr w10, x9, [x8] 8485; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 8486; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8487; OUTLINE-ATOMICS-NEXT: ret 8488 atomicrmw umin ptr @var64, i64 %offset monotonic 8489; CHECK-NOT: dmb 8490; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 8491; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 8492 8493; CHECK: ldumin x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] 8494; CHECK-NOT: dmb 8495 ret void 8496} 8497 8498define dso_local i8 @test_atomic_load_umin_i8_release(i8 %offset) nounwind { 8499; CHECK-LABEL: test_atomic_load_umin_i8_release: 8500; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_release: 8501; OUTLINE-ATOMICS: // %bb.0: 8502; OUTLINE-ATOMICS-NEXT: adrp x8, var8 8503; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var8 8504; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xff 8505; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8506; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8507; OUTLINE-ATOMICS-NEXT: ldxrb w0, [x8] 8508; OUTLINE-ATOMICS-NEXT: cmp w0, w9 8509; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, ls 8510; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x8] 8511; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8512; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8513; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 8514; OUTLINE-ATOMICS-NEXT: ret 8515 %old = atomicrmw umin ptr @var8, i8 %offset release 8516; CHECK-NOT: dmb 8517; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 8518; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 8519 8520; CHECK: lduminlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8521; CHECK-NOT: dmb 8522 8523 ret i8 %old 8524} 8525 8526define dso_local i16 @test_atomic_load_umin_i16_release(i16 %offset) nounwind { 8527; CHECK-LABEL: test_atomic_load_umin_i16_release: 8528; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_release: 8529; OUTLINE-ATOMICS: // %bb.0: 8530; OUTLINE-ATOMICS-NEXT: adrp x8, var16 8531; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var16 8532; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xffff 8533; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8534; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8535; OUTLINE-ATOMICS-NEXT: ldxrh w0, [x8] 8536; OUTLINE-ATOMICS-NEXT: cmp w0, w9 8537; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, ls 8538; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x8] 8539; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8540; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8541; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 8542; OUTLINE-ATOMICS-NEXT: ret 8543 %old = atomicrmw umin ptr @var16, i16 %offset release 8544; CHECK-NOT: dmb 8545; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 8546; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 8547 8548; CHECK: lduminlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8549; CHECK-NOT: dmb 8550 8551 ret i16 %old 8552} 8553 8554define dso_local i32 @test_atomic_load_umin_i32_release(i32 %offset) nounwind { 8555; CHECK-LABEL: test_atomic_load_umin_i32_release: 8556; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_release: 8557; OUTLINE-ATOMICS: // %bb.0: 8558; OUTLINE-ATOMICS-NEXT: adrp x9, var32 8559; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 8560; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8561; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8562; OUTLINE-ATOMICS-NEXT: ldxr w8, [x9] 8563; OUTLINE-ATOMICS-NEXT: cmp w8, w0 8564; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls 8565; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 8566; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8567; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8568; OUTLINE-ATOMICS-NEXT: mov w0, w8 8569; OUTLINE-ATOMICS-NEXT: ret 8570 %old = atomicrmw umin ptr @var32, i32 %offset release 8571; CHECK-NOT: dmb 8572; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 8573; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 8574 8575; CHECK: lduminl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8576; CHECK-NOT: dmb 8577 8578 ret i32 %old 8579} 8580 8581define dso_local i64 @test_atomic_load_umin_i64_release(i64 %offset) nounwind { 8582; CHECK-LABEL: test_atomic_load_umin_i64_release: 8583; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_release: 8584; OUTLINE-ATOMICS: // %bb.0: 8585; OUTLINE-ATOMICS-NEXT: adrp x9, var64 8586; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 8587; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8588; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8589; OUTLINE-ATOMICS-NEXT: ldxr x8, [x9] 8590; OUTLINE-ATOMICS-NEXT: cmp x8, x0 8591; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls 8592; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 8593; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8594; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8595; OUTLINE-ATOMICS-NEXT: mov x0, x8 8596; OUTLINE-ATOMICS-NEXT: ret 8597 %old = atomicrmw umin ptr @var64, i64 %offset release 8598; CHECK-NOT: dmb 8599; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 8600; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 8601 8602; CHECK: lduminl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 8603; CHECK-NOT: dmb 8604 8605 ret i64 %old 8606} 8607 8608define dso_local void @test_atomic_load_umin_i32_noret_release(i32 %offset) nounwind { 8609; CHECK-LABEL: test_atomic_load_umin_i32_noret_release: 8610; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_release: 8611; OUTLINE-ATOMICS: // %bb.0: 8612; OUTLINE-ATOMICS-NEXT: adrp x8, var32 8613; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 8614; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8615; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8616; OUTLINE-ATOMICS-NEXT: ldxr w9, [x8] 8617; OUTLINE-ATOMICS-NEXT: cmp w9, w0 8618; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls 8619; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 8620; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 8621; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8622; OUTLINE-ATOMICS-NEXT: ret 8623 atomicrmw umin ptr @var32, i32 %offset release 8624; CHECK-NOT: dmb 8625; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 8626; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 8627 8628; CHECK: lduminl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] 8629; CHECK-NOT: dmb 8630 ret void 8631} 8632 8633define dso_local void @test_atomic_load_umin_i64_noret_release(i64 %offset) nounwind { 8634; CHECK-LABEL: test_atomic_load_umin_i64_noret_release: 8635; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_release: 8636; OUTLINE-ATOMICS: // %bb.0: 8637; OUTLINE-ATOMICS-NEXT: adrp x8, var64 8638; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 8639; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8640; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8641; OUTLINE-ATOMICS-NEXT: ldxr x9, [x8] 8642; OUTLINE-ATOMICS-NEXT: cmp x9, x0 8643; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls 8644; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 8645; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 8646; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8647; OUTLINE-ATOMICS-NEXT: ret 8648 atomicrmw umin ptr @var64, i64 %offset release 8649; CHECK-NOT: dmb 8650; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 8651; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 8652 8653; CHECK: lduminl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] 8654; CHECK-NOT: dmb 8655 ret void 8656} 8657 8658define dso_local i8 @test_atomic_load_umin_i8_seq_cst(i8 %offset) nounwind { 8659; CHECK-LABEL: test_atomic_load_umin_i8_seq_cst: 8660; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i8_seq_cst: 8661; OUTLINE-ATOMICS: // %bb.0: 8662; OUTLINE-ATOMICS-NEXT: adrp x8, var8 8663; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var8 8664; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xff 8665; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8666; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8667; OUTLINE-ATOMICS-NEXT: ldaxrb w0, [x8] 8668; OUTLINE-ATOMICS-NEXT: cmp w0, w9 8669; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, ls 8670; OUTLINE-ATOMICS-NEXT: stlxrb w11, w10, [x8] 8671; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8672; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8673; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 8674; OUTLINE-ATOMICS-NEXT: ret 8675 %old = atomicrmw umin ptr @var8, i8 %offset seq_cst 8676; CHECK-NOT: dmb 8677; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 8678; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 8679 8680; CHECK: lduminalb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8681; CHECK-NOT: dmb 8682 8683 ret i8 %old 8684} 8685 8686define dso_local i16 @test_atomic_load_umin_i16_seq_cst(i16 %offset) nounwind { 8687; CHECK-LABEL: test_atomic_load_umin_i16_seq_cst: 8688; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i16_seq_cst: 8689; OUTLINE-ATOMICS: // %bb.0: 8690; OUTLINE-ATOMICS-NEXT: adrp x8, var16 8691; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var16 8692; OUTLINE-ATOMICS-NEXT: and w9, w0, #0xffff 8693; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8694; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8695; OUTLINE-ATOMICS-NEXT: ldaxrh w0, [x8] 8696; OUTLINE-ATOMICS-NEXT: cmp w0, w9 8697; OUTLINE-ATOMICS-NEXT: csel w10, w0, w9, ls 8698; OUTLINE-ATOMICS-NEXT: stlxrh w11, w10, [x8] 8699; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8700; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8701; OUTLINE-ATOMICS-NEXT: // kill: def $w0 killed $w0 killed $x0 8702; OUTLINE-ATOMICS-NEXT: ret 8703 %old = atomicrmw umin ptr @var16, i16 %offset seq_cst 8704; CHECK-NOT: dmb 8705; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 8706; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 8707 8708; CHECK: lduminalh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8709; CHECK-NOT: dmb 8710 8711 ret i16 %old 8712} 8713 8714define dso_local i32 @test_atomic_load_umin_i32_seq_cst(i32 %offset) nounwind { 8715; CHECK-LABEL: test_atomic_load_umin_i32_seq_cst: 8716; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_seq_cst: 8717; OUTLINE-ATOMICS: // %bb.0: 8718; OUTLINE-ATOMICS-NEXT: adrp x9, var32 8719; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var32 8720; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8721; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8722; OUTLINE-ATOMICS-NEXT: ldaxr w8, [x9] 8723; OUTLINE-ATOMICS-NEXT: cmp w8, w0 8724; OUTLINE-ATOMICS-NEXT: csel w10, w8, w0, ls 8725; OUTLINE-ATOMICS-NEXT: stlxr w11, w10, [x9] 8726; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8727; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8728; OUTLINE-ATOMICS-NEXT: mov w0, w8 8729; OUTLINE-ATOMICS-NEXT: ret 8730 %old = atomicrmw umin ptr @var32, i32 %offset seq_cst 8731; CHECK-NOT: dmb 8732; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 8733; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 8734 8735; CHECK: lduminal w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8736; CHECK-NOT: dmb 8737 8738 ret i32 %old 8739} 8740 8741define dso_local i64 @test_atomic_load_umin_i64_seq_cst(i64 %offset) nounwind { 8742; CHECK-LABEL: test_atomic_load_umin_i64_seq_cst: 8743; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_seq_cst: 8744; OUTLINE-ATOMICS: // %bb.0: 8745; OUTLINE-ATOMICS-NEXT: adrp x9, var64 8746; OUTLINE-ATOMICS-NEXT: add x9, x9, :lo12:var64 8747; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8748; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8749; OUTLINE-ATOMICS-NEXT: ldaxr x8, [x9] 8750; OUTLINE-ATOMICS-NEXT: cmp x8, x0 8751; OUTLINE-ATOMICS-NEXT: csel x10, x8, x0, ls 8752; OUTLINE-ATOMICS-NEXT: stlxr w11, x10, [x9] 8753; OUTLINE-ATOMICS-NEXT: cbnz w11, .LBB[[LOOPSTART]] 8754; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8755; OUTLINE-ATOMICS-NEXT: mov x0, x8 8756; OUTLINE-ATOMICS-NEXT: ret 8757 %old = atomicrmw umin ptr @var64, i64 %offset seq_cst 8758; CHECK-NOT: dmb 8759; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 8760; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 8761 8762; CHECK: lduminal x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 8763; CHECK-NOT: dmb 8764 8765 ret i64 %old 8766} 8767 8768define dso_local void @test_atomic_load_umin_i32_noret_seq_cst(i32 %offset) nounwind { 8769; CHECK-LABEL: test_atomic_load_umin_i32_noret_seq_cst: 8770; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i32_noret_seq_cst: 8771; OUTLINE-ATOMICS: // %bb.0: 8772; OUTLINE-ATOMICS-NEXT: adrp x8, var32 8773; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var32 8774; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8775; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8776; OUTLINE-ATOMICS-NEXT: ldaxr w9, [x8] 8777; OUTLINE-ATOMICS-NEXT: cmp w9, w0 8778; OUTLINE-ATOMICS-NEXT: csel w9, w9, w0, ls 8779; OUTLINE-ATOMICS-NEXT: stlxr w10, w9, [x8] 8780; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 8781; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8782; OUTLINE-ATOMICS-NEXT: ret 8783 atomicrmw umin ptr @var32, i32 %offset seq_cst 8784; CHECK-NOT: dmb 8785; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 8786; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 8787 8788; CHECK: lduminal w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 8789; CHECK-NOT: dmb 8790 ret void 8791} 8792 8793define dso_local void @test_atomic_load_umin_i64_noret_seq_cst(i64 %offset) nounwind { 8794; CHECK-LABEL: test_atomic_load_umin_i64_noret_seq_cst: 8795; OUTLINE-ATOMICS-LABEL: test_atomic_load_umin_i64_noret_seq_cst: 8796; OUTLINE-ATOMICS: // %bb.0: 8797; OUTLINE-ATOMICS-NEXT: adrp x8, var64 8798; OUTLINE-ATOMICS-NEXT: add x8, x8, :lo12:var64 8799; OUTLINE-ATOMICS-NEXT: .LBB[[LOOPSTART:.*]]: // %atomicrmw.start 8800; OUTLINE-ATOMICS-NEXT: // =>This Inner Loop Header: Depth=1 8801; OUTLINE-ATOMICS-NEXT: ldaxr x9, [x8] 8802; OUTLINE-ATOMICS-NEXT: cmp x9, x0 8803; OUTLINE-ATOMICS-NEXT: csel x9, x9, x0, ls 8804; OUTLINE-ATOMICS-NEXT: stlxr w10, x9, [x8] 8805; OUTLINE-ATOMICS-NEXT: cbnz w10, .LBB[[LOOPSTART]] 8806; OUTLINE-ATOMICS-NEXT: // %bb.2: // %atomicrmw.end 8807; OUTLINE-ATOMICS-NEXT: ret 8808 atomicrmw umin ptr @var64, i64 %offset seq_cst 8809; CHECK-NOT: dmb 8810; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 8811; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 8812 8813; CHECK: lduminal x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 8814; CHECK-NOT: dmb 8815 ret void 8816} 8817 8818define dso_local i8 @test_atomic_load_xor_i8_acq_rel(i8 %offset) nounwind { 8819; CHECK-LABEL: test_atomic_load_xor_i8_acq_rel: 8820; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i8_acq_rel: 8821; OUTLINE-ATOMICS: // %bb.0: 8822; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 8823; OUTLINE-ATOMICS-NEXT: adrp x1, var8 8824; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 8825; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor1_acq_rel 8826; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 8827; OUTLINE-ATOMICS-NEXT: ret 8828 %old = atomicrmw xor ptr @var8, i8 %offset acq_rel 8829; CHECK-NOT: dmb 8830; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 8831; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 8832 8833; CHECK: ldeoralb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8834; CHECK-NOT: dmb 8835 8836 ret i8 %old 8837} 8838 8839define dso_local i16 @test_atomic_load_xor_i16_acq_rel(i16 %offset) nounwind { 8840; CHECK-LABEL: test_atomic_load_xor_i16_acq_rel: 8841; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i16_acq_rel: 8842; OUTLINE-ATOMICS: // %bb.0: 8843; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 8844; OUTLINE-ATOMICS-NEXT: adrp x1, var16 8845; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 8846; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor2_acq_rel 8847; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 8848; OUTLINE-ATOMICS-NEXT: ret 8849 %old = atomicrmw xor ptr @var16, i16 %offset acq_rel 8850; CHECK-NOT: dmb 8851; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 8852; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 8853 8854; CHECK: ldeoralh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8855; CHECK-NOT: dmb 8856 8857 ret i16 %old 8858} 8859 8860define dso_local i32 @test_atomic_load_xor_i32_acq_rel(i32 %offset) nounwind { 8861; CHECK-LABEL: test_atomic_load_xor_i32_acq_rel: 8862; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_acq_rel: 8863; OUTLINE-ATOMICS: // %bb.0: 8864; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 8865; OUTLINE-ATOMICS-NEXT: adrp x1, var32 8866; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 8867; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel 8868; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 8869; OUTLINE-ATOMICS-NEXT: ret 8870 %old = atomicrmw xor ptr @var32, i32 %offset acq_rel 8871; CHECK-NOT: dmb 8872; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 8873; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 8874 8875; CHECK: ldeoral w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8876; CHECK-NOT: dmb 8877 8878 ret i32 %old 8879} 8880 8881define dso_local i64 @test_atomic_load_xor_i64_acq_rel(i64 %offset) nounwind { 8882; CHECK-LABEL: test_atomic_load_xor_i64_acq_rel: 8883; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_acq_rel: 8884; OUTLINE-ATOMICS: // %bb.0: 8885; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 8886; OUTLINE-ATOMICS-NEXT: adrp x1, var64 8887; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 8888; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq_rel 8889; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 8890; OUTLINE-ATOMICS-NEXT: ret 8891 %old = atomicrmw xor ptr @var64, i64 %offset acq_rel 8892; CHECK-NOT: dmb 8893; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 8894; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 8895 8896; CHECK: ldeoral x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 8897; CHECK-NOT: dmb 8898 8899 ret i64 %old 8900} 8901 8902define dso_local void @test_atomic_load_xor_i32_noret_acq_rel(i32 %offset) nounwind { 8903; CHECK-LABEL: test_atomic_load_xor_i32_noret_acq_rel: 8904; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_noret_acq_rel: 8905; OUTLINE-ATOMICS: // %bb.0: 8906; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 8907; OUTLINE-ATOMICS-NEXT: adrp x1, var32 8908; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 8909; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel 8910; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 8911; OUTLINE-ATOMICS-NEXT: ret 8912 atomicrmw xor ptr @var32, i32 %offset acq_rel 8913; CHECK-NOT: dmb 8914; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 8915; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 8916 8917; CHECK: ldeoral w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 8918; CHECK-NOT: dmb 8919 ret void 8920} 8921 8922define dso_local void @test_atomic_load_xor_i64_noret_acq_rel(i64 %offset) nounwind { 8923; CHECK-LABEL: test_atomic_load_xor_i64_noret_acq_rel: 8924; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_noret_acq_rel: 8925; OUTLINE-ATOMICS: // %bb.0: 8926; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 8927; OUTLINE-ATOMICS-NEXT: adrp x1, var64 8928; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 8929; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq_rel 8930; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 8931; OUTLINE-ATOMICS-NEXT: ret 8932 atomicrmw xor ptr @var64, i64 %offset acq_rel 8933; CHECK-NOT: dmb 8934; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 8935; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 8936 8937; CHECK: ldeoral x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 8938; CHECK-NOT: dmb 8939 ret void 8940} 8941 8942define dso_local i8 @test_atomic_load_xor_i8_acquire(i8 %offset) nounwind { 8943; CHECK-LABEL: test_atomic_load_xor_i8_acquire: 8944; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i8_acquire: 8945; OUTLINE-ATOMICS: // %bb.0: 8946; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 8947; OUTLINE-ATOMICS-NEXT: adrp x1, var8 8948; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 8949; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor1_acq 8950; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 8951; OUTLINE-ATOMICS-NEXT: ret 8952 %old = atomicrmw xor ptr @var8, i8 %offset acquire 8953; CHECK-NOT: dmb 8954; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 8955; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 8956 8957; CHECK: ldeorab w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8958; CHECK-NOT: dmb 8959 8960 ret i8 %old 8961} 8962 8963define dso_local i16 @test_atomic_load_xor_i16_acquire(i16 %offset) nounwind { 8964; CHECK-LABEL: test_atomic_load_xor_i16_acquire: 8965; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i16_acquire: 8966; OUTLINE-ATOMICS: // %bb.0: 8967; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 8968; OUTLINE-ATOMICS-NEXT: adrp x1, var16 8969; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 8970; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor2_acq 8971; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 8972; OUTLINE-ATOMICS-NEXT: ret 8973 %old = atomicrmw xor ptr @var16, i16 %offset acquire 8974; CHECK-NOT: dmb 8975; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 8976; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 8977 8978; CHECK: ldeorah w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 8979; CHECK-NOT: dmb 8980 8981 ret i16 %old 8982} 8983 8984define dso_local i32 @test_atomic_load_xor_i32_acquire(i32 %offset) nounwind { 8985; CHECK-LABEL: test_atomic_load_xor_i32_acquire: 8986; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_acquire: 8987; OUTLINE-ATOMICS: // %bb.0: 8988; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 8989; OUTLINE-ATOMICS-NEXT: adrp x1, var32 8990; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 8991; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq 8992; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 8993; OUTLINE-ATOMICS-NEXT: ret 8994 %old = atomicrmw xor ptr @var32, i32 %offset acquire 8995; CHECK-NOT: dmb 8996; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 8997; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 8998 8999; CHECK: ldeora w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 9000; CHECK-NOT: dmb 9001 9002 ret i32 %old 9003} 9004 9005define dso_local i64 @test_atomic_load_xor_i64_acquire(i64 %offset) nounwind { 9006; CHECK-LABEL: test_atomic_load_xor_i64_acquire: 9007; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_acquire: 9008; OUTLINE-ATOMICS: // %bb.0: 9009; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9010; OUTLINE-ATOMICS-NEXT: adrp x1, var64 9011; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 9012; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq 9013; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9014; OUTLINE-ATOMICS-NEXT: ret 9015 %old = atomicrmw xor ptr @var64, i64 %offset acquire 9016; CHECK-NOT: dmb 9017; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 9018; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 9019 9020; CHECK: ldeora x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 9021; CHECK-NOT: dmb 9022 9023 ret i64 %old 9024} 9025 9026define dso_local void @test_atomic_load_xor_i32_noret_acquire(i32 %offset) nounwind { 9027; CHECK-LABEL: test_atomic_load_xor_i32_noret_acquire: 9028; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_noret_acquire: 9029; OUTLINE-ATOMICS: // %bb.0: 9030; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9031; OUTLINE-ATOMICS-NEXT: adrp x1, var32 9032; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 9033; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq 9034; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9035; OUTLINE-ATOMICS-NEXT: ret 9036 atomicrmw xor ptr @var32, i32 %offset acquire 9037; CHECK-NOT: dmb 9038; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 9039; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 9040 9041; CHECK: ldeora w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 9042; CHECK-NOT: dmb 9043 ret void 9044} 9045 9046define dso_local void @test_atomic_load_xor_i64_noret_acquire(i64 %offset) nounwind { 9047; CHECK-LABEL: test_atomic_load_xor_i64_noret_acquire: 9048; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_noret_acquire: 9049; OUTLINE-ATOMICS: // %bb.0: 9050; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9051; OUTLINE-ATOMICS-NEXT: adrp x1, var64 9052; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 9053; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq 9054; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9055; OUTLINE-ATOMICS-NEXT: ret 9056 atomicrmw xor ptr @var64, i64 %offset acquire 9057; CHECK-NOT: dmb 9058; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 9059; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 9060 9061; CHECK: ldeora x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 9062; CHECK-NOT: dmb 9063 ret void 9064} 9065 9066define dso_local i8 @test_atomic_load_xor_i8_monotonic(i8 %offset) nounwind { 9067; CHECK-LABEL: test_atomic_load_xor_i8_monotonic: 9068; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i8_monotonic: 9069; OUTLINE-ATOMICS: // %bb.0: 9070; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9071; OUTLINE-ATOMICS-NEXT: adrp x1, var8 9072; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 9073; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor1_relax 9074; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9075; OUTLINE-ATOMICS-NEXT: ret 9076 %old = atomicrmw xor ptr @var8, i8 %offset monotonic 9077; CHECK-NOT: dmb 9078; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 9079; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 9080 9081; CHECK: ldeorb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 9082; CHECK-NOT: dmb 9083 9084 ret i8 %old 9085} 9086 9087define dso_local i16 @test_atomic_load_xor_i16_monotonic(i16 %offset) nounwind { 9088; CHECK-LABEL: test_atomic_load_xor_i16_monotonic: 9089; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i16_monotonic: 9090; OUTLINE-ATOMICS: // %bb.0: 9091; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9092; OUTLINE-ATOMICS-NEXT: adrp x1, var16 9093; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 9094; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor2_relax 9095; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9096; OUTLINE-ATOMICS-NEXT: ret 9097 %old = atomicrmw xor ptr @var16, i16 %offset monotonic 9098; CHECK-NOT: dmb 9099; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 9100; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 9101 9102; CHECK: ldeorh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 9103; CHECK-NOT: dmb 9104 9105 ret i16 %old 9106} 9107 9108define dso_local i32 @test_atomic_load_xor_i32_monotonic(i32 %offset) nounwind { 9109; CHECK-LABEL: test_atomic_load_xor_i32_monotonic: 9110; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_monotonic: 9111; OUTLINE-ATOMICS: // %bb.0: 9112; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9113; OUTLINE-ATOMICS-NEXT: adrp x1, var32 9114; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 9115; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_relax 9116; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9117; OUTLINE-ATOMICS-NEXT: ret 9118 %old = atomicrmw xor ptr @var32, i32 %offset monotonic 9119; CHECK-NOT: dmb 9120; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 9121; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 9122 9123; CHECK: ldeor w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 9124; CHECK-NOT: dmb 9125 9126 ret i32 %old 9127} 9128 9129define dso_local i64 @test_atomic_load_xor_i64_monotonic(i64 %offset) nounwind { 9130; CHECK-LABEL: test_atomic_load_xor_i64_monotonic: 9131; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_monotonic: 9132; OUTLINE-ATOMICS: // %bb.0: 9133; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9134; OUTLINE-ATOMICS-NEXT: adrp x1, var64 9135; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 9136; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_relax 9137; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9138; OUTLINE-ATOMICS-NEXT: ret 9139 %old = atomicrmw xor ptr @var64, i64 %offset monotonic 9140; CHECK-NOT: dmb 9141; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 9142; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 9143 9144; CHECK: ldeor x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 9145; CHECK-NOT: dmb 9146 9147 ret i64 %old 9148} 9149 9150define dso_local void @test_atomic_load_xor_i32_noret_monotonic(i32 %offset) nounwind { 9151; CHECK-LABEL: test_atomic_load_xor_i32_noret_monotonic: 9152; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_noret_monotonic: 9153; OUTLINE-ATOMICS: // %bb.0: 9154; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9155; OUTLINE-ATOMICS-NEXT: adrp x1, var32 9156; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 9157; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_relax 9158; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9159; OUTLINE-ATOMICS-NEXT: ret 9160 atomicrmw xor ptr @var32, i32 %offset monotonic 9161; CHECK-NOT: dmb 9162; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 9163; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 9164 9165; CHECK: ldeor w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] 9166; CHECK-NOT: dmb 9167 ret void 9168} 9169 9170define dso_local void @test_atomic_load_xor_i64_noret_monotonic(i64 %offset) nounwind { 9171; CHECK-LABEL: test_atomic_load_xor_i64_noret_monotonic: 9172; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_noret_monotonic: 9173; OUTLINE-ATOMICS: // %bb.0: 9174; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9175; OUTLINE-ATOMICS-NEXT: adrp x1, var64 9176; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 9177; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_relax 9178; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9179; OUTLINE-ATOMICS-NEXT: ret 9180 atomicrmw xor ptr @var64, i64 %offset monotonic 9181; CHECK-NOT: dmb 9182; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 9183; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 9184 9185; CHECK: ldeor x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] 9186; CHECK-NOT: dmb 9187 ret void 9188} 9189 9190define dso_local i8 @test_atomic_load_xor_i8_release(i8 %offset) nounwind { 9191; CHECK-LABEL: test_atomic_load_xor_i8_release: 9192; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i8_release: 9193; OUTLINE-ATOMICS: // %bb.0: 9194; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9195; OUTLINE-ATOMICS-NEXT: adrp x1, var8 9196; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 9197; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor1_rel 9198; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9199; OUTLINE-ATOMICS-NEXT: ret 9200 %old = atomicrmw xor ptr @var8, i8 %offset release 9201; CHECK-NOT: dmb 9202; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 9203; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 9204 9205; CHECK: ldeorlb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 9206; CHECK-NOT: dmb 9207 9208 ret i8 %old 9209} 9210 9211define dso_local i16 @test_atomic_load_xor_i16_release(i16 %offset) nounwind { 9212; CHECK-LABEL: test_atomic_load_xor_i16_release: 9213; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i16_release: 9214; OUTLINE-ATOMICS: // %bb.0: 9215; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9216; OUTLINE-ATOMICS-NEXT: adrp x1, var16 9217; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 9218; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor2_rel 9219; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9220; OUTLINE-ATOMICS-NEXT: ret 9221 %old = atomicrmw xor ptr @var16, i16 %offset release 9222; CHECK-NOT: dmb 9223; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 9224; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 9225 9226; CHECK: ldeorlh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 9227; CHECK-NOT: dmb 9228 9229 ret i16 %old 9230} 9231 9232define dso_local i32 @test_atomic_load_xor_i32_release(i32 %offset) nounwind { 9233; CHECK-LABEL: test_atomic_load_xor_i32_release: 9234; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_release: 9235; OUTLINE-ATOMICS: // %bb.0: 9236; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9237; OUTLINE-ATOMICS-NEXT: adrp x1, var32 9238; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 9239; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_rel 9240; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9241; OUTLINE-ATOMICS-NEXT: ret 9242 %old = atomicrmw xor ptr @var32, i32 %offset release 9243; CHECK-NOT: dmb 9244; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 9245; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 9246 9247; CHECK: ldeorl w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 9248; CHECK-NOT: dmb 9249 9250 ret i32 %old 9251} 9252 9253define dso_local i64 @test_atomic_load_xor_i64_release(i64 %offset) nounwind { 9254; CHECK-LABEL: test_atomic_load_xor_i64_release: 9255; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_release: 9256; OUTLINE-ATOMICS: // %bb.0: 9257; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9258; OUTLINE-ATOMICS-NEXT: adrp x1, var64 9259; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 9260; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_rel 9261; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9262; OUTLINE-ATOMICS-NEXT: ret 9263 %old = atomicrmw xor ptr @var64, i64 %offset release 9264; CHECK-NOT: dmb 9265; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 9266; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 9267 9268; CHECK: ldeorl x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 9269; CHECK-NOT: dmb 9270 9271 ret i64 %old 9272} 9273 9274define dso_local void @test_atomic_load_xor_i32_noret_release(i32 %offset) nounwind { 9275; CHECK-LABEL: test_atomic_load_xor_i32_noret_release: 9276; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_noret_release: 9277; OUTLINE-ATOMICS: // %bb.0: 9278; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9279; OUTLINE-ATOMICS-NEXT: adrp x1, var32 9280; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 9281; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_rel 9282; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9283; OUTLINE-ATOMICS-NEXT: ret 9284 atomicrmw xor ptr @var32, i32 %offset release 9285; CHECK-NOT: dmb 9286; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 9287; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 9288 9289; CHECK: ldeorl w{{[0-9]+}}, w{{[1-9][0-9]*}}, [x[[ADDR]]] 9290; CHECK-NOT: dmb 9291 ret void 9292} 9293 9294define dso_local void @test_atomic_load_xor_i64_noret_release(i64 %offset) nounwind { 9295; CHECK-LABEL: test_atomic_load_xor_i64_noret_release: 9296; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_noret_release: 9297; OUTLINE-ATOMICS: // %bb.0: 9298; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9299; OUTLINE-ATOMICS-NEXT: adrp x1, var64 9300; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 9301; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_rel 9302; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9303; OUTLINE-ATOMICS-NEXT: ret 9304 atomicrmw xor ptr @var64, i64 %offset release 9305; CHECK-NOT: dmb 9306; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 9307; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 9308 9309; CHECK: ldeorl x{{[0-9]+}}, x{{[1-9][0-9]*}}, [x[[ADDR]]] 9310; CHECK-NOT: dmb 9311 ret void 9312} 9313 9314define dso_local i8 @test_atomic_load_xor_i8_seq_cst(i8 %offset) nounwind { 9315; CHECK-LABEL: test_atomic_load_xor_i8_seq_cst: 9316; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i8_seq_cst: 9317; OUTLINE-ATOMICS: // %bb.0: 9318; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9319; OUTLINE-ATOMICS-NEXT: adrp x1, var8 9320; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var8 9321; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor1_acq_rel 9322; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9323; OUTLINE-ATOMICS-NEXT: ret 9324 %old = atomicrmw xor ptr @var8, i8 %offset seq_cst 9325; CHECK-NOT: dmb 9326; CHECK: adrp [[TMPADDR:x[0-9]+]], var8 9327; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var8 9328 9329; CHECK: ldeoralb w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 9330; CHECK-NOT: dmb 9331 9332 ret i8 %old 9333} 9334 9335define dso_local i16 @test_atomic_load_xor_i16_seq_cst(i16 %offset) nounwind { 9336; CHECK-LABEL: test_atomic_load_xor_i16_seq_cst: 9337; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i16_seq_cst: 9338; OUTLINE-ATOMICS: // %bb.0: 9339; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9340; OUTLINE-ATOMICS-NEXT: adrp x1, var16 9341; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var16 9342; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor2_acq_rel 9343; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9344; OUTLINE-ATOMICS-NEXT: ret 9345 %old = atomicrmw xor ptr @var16, i16 %offset seq_cst 9346; CHECK-NOT: dmb 9347; CHECK: adrp [[TMPADDR:x[0-9]+]], var16 9348; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var16 9349 9350; CHECK: ldeoralh w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 9351; CHECK-NOT: dmb 9352 9353 ret i16 %old 9354} 9355 9356define dso_local i32 @test_atomic_load_xor_i32_seq_cst(i32 %offset) nounwind { 9357; CHECK-LABEL: test_atomic_load_xor_i32_seq_cst: 9358; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_seq_cst: 9359; OUTLINE-ATOMICS: // %bb.0: 9360; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9361; OUTLINE-ATOMICS-NEXT: adrp x1, var32 9362; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 9363; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel 9364; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9365; OUTLINE-ATOMICS-NEXT: ret 9366 %old = atomicrmw xor ptr @var32, i32 %offset seq_cst 9367; CHECK-NOT: dmb 9368; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 9369; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 9370 9371; CHECK: ldeoral w[[OLD:[0-9]+]], w[[NEW:[0-9]+]], [x[[ADDR]]] 9372; CHECK-NOT: dmb 9373 9374 ret i32 %old 9375} 9376 9377define dso_local i64 @test_atomic_load_xor_i64_seq_cst(i64 %offset) nounwind { 9378; CHECK-LABEL: test_atomic_load_xor_i64_seq_cst: 9379; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_seq_cst: 9380; OUTLINE-ATOMICS: // %bb.0: 9381; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9382; OUTLINE-ATOMICS-NEXT: adrp x1, var64 9383; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 9384; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq_rel 9385; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9386; OUTLINE-ATOMICS-NEXT: ret 9387 %old = atomicrmw xor ptr @var64, i64 %offset seq_cst 9388; CHECK-NOT: dmb 9389; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 9390; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 9391 9392; CHECK: ldeoral x[[OLD:[0-9]+]], x[[NEW:[0-9]+]], [x[[ADDR]]] 9393; CHECK-NOT: dmb 9394 9395 ret i64 %old 9396} 9397 9398define dso_local void @test_atomic_load_xor_i32_noret_seq_cst(i32 %offset) nounwind { 9399; CHECK-LABEL: test_atomic_load_xor_i32_noret_seq_cst: 9400; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i32_noret_seq_cst: 9401; OUTLINE-ATOMICS: // %bb.0: 9402; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9403; OUTLINE-ATOMICS-NEXT: adrp x1, var32 9404; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var32 9405; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor4_acq_rel 9406; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9407; OUTLINE-ATOMICS-NEXT: ret 9408 atomicrmw xor ptr @var32, i32 %offset seq_cst 9409; CHECK-NOT: dmb 9410; CHECK: adrp [[TMPADDR:x[0-9]+]], var32 9411; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var32 9412 9413; CHECK: ldeoral w0, w[[NEW:[0-9]+]], [x[[ADDR]]] 9414; CHECK-NOT: dmb 9415 ret void 9416} 9417 9418define dso_local void @test_atomic_load_xor_i64_noret_seq_cst(i64 %offset) nounwind { 9419; CHECK-LABEL: test_atomic_load_xor_i64_noret_seq_cst: 9420; OUTLINE-ATOMICS-LABEL: test_atomic_load_xor_i64_noret_seq_cst: 9421; OUTLINE-ATOMICS: // %bb.0: 9422; OUTLINE-ATOMICS-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill 9423; OUTLINE-ATOMICS-NEXT: adrp x1, var64 9424; OUTLINE-ATOMICS-NEXT: add x1, x1, :lo12:var64 9425; OUTLINE-ATOMICS-NEXT: bl __aarch64_ldeor8_acq_rel 9426; OUTLINE-ATOMICS-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload 9427; OUTLINE-ATOMICS-NEXT: ret 9428 atomicrmw xor ptr @var64, i64 %offset seq_cst 9429; CHECK-NOT: dmb 9430; CHECK: adrp [[TMPADDR:x[0-9]+]], var64 9431; CHECK: add x[[ADDR:[0-9]+]], [[TMPADDR]], {{#?}}:lo12:var64 9432 9433; CHECK: ldeoral x0, x[[NEW:[0-9]+]], [x[[ADDR]]] 9434; CHECK-NOT: dmb 9435 ret void 9436} 9437 9438define dso_local i128 @test_atomic_load_i128() nounwind { 9439; CHECK-LABEL: test_atomic_load_i128: 9440; CHECK: casp 9441 9442; OUTLINE-ATOMICS-LABEL: test_atomic_load_i128: 9443; OUTLINE-ATOMICS: ldxp 9444; OUTLINE-ATOMICS: stxp 9445 %pair = load atomic i128, ptr @var128 monotonic, align 16 9446 ret i128 %pair 9447} 9448