1; RUN: llc < %s -mtriple=arm64-eabi -asm-verbose=false -verify-machineinstrs -mcpu=cyclone | FileCheck -enable-var-scope %s 2; RUN: llc < %s -mtriple=arm64-eabi -asm-verbose=false -verify-machineinstrs -mcpu=cyclone -mattr=+outline-atomics | FileCheck -enable-var-scope %s -check-prefix=OUTLINE-ATOMICS 3 4define i32 @val_compare_and_swap(ptr %p, i32 %cmp, i32 %new) #0 { 5; OUTLINE-ATOMICS: bl __aarch64_cas4_acq 6; CHECK-LABEL: val_compare_and_swap: 7; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 8; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: 9; CHECK-NEXT: ldaxr [[RESULT:w[0-9]+]], [x[[ADDR]]] 10; CHECK-NEXT: cmp [[RESULT]], w1 11; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] 12; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], w2, [x[[ADDR]]] 13; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] 14; CHECK-NEXT: ret 15; CHECK-NEXT: [[FAILBB]]: 16; CHECK-NEXT: clrex 17; CHECK-NEXT: ret 18 %pair = cmpxchg ptr %p, i32 %cmp, i32 %new acquire acquire 19 %val = extractvalue { i32, i1 } %pair, 0 20 ret i32 %val 21} 22 23define i32 @val_compare_and_swap_from_load(ptr %p, i32 %cmp, ptr %pnew) #0 { 24; OUTLINE-ATOMICS: bl __aarch64_cas4_acq 25; CHECK-LABEL: val_compare_and_swap_from_load: 26; CHECK-NEXT: ldr [[NEW:w[0-9]+]], [x2] 27; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: 28; CHECK-NEXT: ldaxr w[[RESULT:[0-9]+]], [x0] 29; CHECK-NEXT: cmp w[[RESULT]], w1 30; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] 31; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], [[NEW]], [x0] 32; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] 33; CHECK-NEXT: mov x0, x[[RESULT]] 34; CHECK-NEXT: ret 35; CHECK-NEXT: [[FAILBB]]: 36; CHECK-NEXT: clrex 37; CHECK-NEXT: mov x0, x[[RESULT]] 38; CHECK-NEXT: ret 39 %new = load i32, ptr %pnew 40 %pair = cmpxchg ptr %p, i32 %cmp, i32 %new acquire acquire 41 %val = extractvalue { i32, i1 } %pair, 0 42 ret i32 %val 43} 44 45define i32 @val_compare_and_swap_rel(ptr %p, i32 %cmp, i32 %new) #0 { 46; OUTLINE-ATOMICS: bl __aarch64_cas4_acq_rel 47; CHECK-LABEL: val_compare_and_swap_rel: 48; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 49; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: 50; CHECK-NEXT: ldaxr [[RESULT:w[0-9]+]], [x[[ADDR]]] 51; CHECK-NEXT: cmp [[RESULT]], w1 52; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] 53; CHECK-NEXT: stlxr [[SCRATCH_REG:w[0-9]+]], w2, [x[[ADDR]]] 54; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] 55; CHECK-NEXT: ret 56; CHECK-NEXT: [[FAILBB]]: 57; CHECK-NEXT: clrex 58; CHECK-NEXT: ret 59 %pair = cmpxchg ptr %p, i32 %cmp, i32 %new acq_rel monotonic 60 %val = extractvalue { i32, i1 } %pair, 0 61 ret i32 %val 62} 63 64define i64 @val_compare_and_swap_64(ptr %p, i64 %cmp, i64 %new) #0 { 65; OUTLINE-ATOMICS: bl __aarch64_cas8_relax 66; CHECK-LABEL: val_compare_and_swap_64: 67; CHECK-NEXT: mov x[[ADDR:[0-9]+]], x0 68; CHECK-NEXT: [[TRYBB:.?LBB[0-9_]+]]: 69; CHECK-NEXT: ldxr [[RESULT:x[0-9]+]], [x[[ADDR]]] 70; CHECK-NEXT: cmp [[RESULT]], x1 71; CHECK-NEXT: b.ne [[FAILBB:.?LBB[0-9_]+]] 72; CHECK-NEXT: stxr [[SCRATCH_REG:w[0-9]+]], x2, [x[[ADDR]]] 73; CHECK-NEXT: cbnz [[SCRATCH_REG]], [[TRYBB]] 74; CHECK-NEXT: ret 75; CHECK-NEXT: [[FAILBB]]: 76; CHECK-NEXT: clrex 77; CHECK-NEXT: ret 78 %pair = cmpxchg ptr %p, i64 %cmp, i64 %new monotonic monotonic 79 %val = extractvalue { i64, i1 } %pair, 0 80 ret i64 %val 81} 82 83define i32 @fetch_and_nand(ptr %p) #0 { 84; CHECK-LABEL: fetch_and_nand: 85; CHECK: [[TRYBB:.?LBB[0-9_]+]]: 86; CHECK: ldxr w[[DEST_REG:[0-9]+]], [x0] 87; CHECK: mvn [[TMP_REG:w[0-9]+]], w[[DEST_REG]] 88; CHECK: orr [[SCRATCH2_REG:w[0-9]+]], [[TMP_REG]], #0xfffffff8 89; CHECK-NOT: stlxr [[SCRATCH2_REG]], [[SCRATCH2_REG]] 90; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x0] 91; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] 92; CHECK: mov x0, x[[DEST_REG]] 93 %val = atomicrmw nand ptr %p, i32 7 release 94 ret i32 %val 95} 96 97define i64 @fetch_and_nand_64(ptr %p) #0 { 98; CHECK-LABEL: fetch_and_nand_64: 99; CHECK: mov x[[ADDR:[0-9]+]], x0 100; CHECK: [[TRYBB:.?LBB[0-9_]+]]: 101; CHECK: ldaxr x[[DEST_REG:[0-9]+]], [x[[ADDR]]] 102; CHECK: mvn w[[TMP_REG:[0-9]+]], w[[DEST_REG]] 103; CHECK: orr [[SCRATCH2_REG:x[0-9]+]], x[[TMP_REG]], #0xfffffffffffffff8 104; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x[[ADDR]]] 105; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] 106 107 %val = atomicrmw nand ptr %p, i64 7 acq_rel 108 ret i64 %val 109} 110 111define i32 @fetch_and_or(ptr %p) #0 { 112; OUTLINE-ATOMICS: bl __aarch64_ldset4_acq_rel 113; CHECK-LABEL: fetch_and_or: 114; CHECK: mov [[OLDVAL_REG:w[0-9]+]], #5 115; CHECK: [[TRYBB:.?LBB[0-9_]+]]: 116; CHECK: ldaxr w[[DEST_REG:[0-9]+]], [x0] 117; CHECK: orr [[SCRATCH2_REG:w[0-9]+]], w[[DEST_REG]], [[OLDVAL_REG]] 118; CHECK-NOT: stlxr [[SCRATCH2_REG]], [[SCRATCH2_REG]] 119; CHECK: stlxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x0] 120; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] 121; CHECK: mov x0, x[[DEST_REG]] 122 %val = atomicrmw or ptr %p, i32 5 seq_cst 123 ret i32 %val 124} 125 126define i64 @fetch_and_or_64(ptr %p) #0 { 127; OUTLINE-ATOMICS: bl __aarch64_ldset8_relax 128; CHECK: fetch_and_or_64: 129; CHECK: mov x[[ADDR:[0-9]+]], x0 130; CHECK: [[TRYBB:.?LBB[0-9_]+]]: 131; CHECK: ldxr [[DEST_REG:x[0-9]+]], [x[[ADDR]]] 132; CHECK: orr [[SCRATCH2_REG:x[0-9]+]], [[DEST_REG]], #0x7 133; CHECK: stxr [[SCRATCH_REG:w[0-9]+]], [[SCRATCH2_REG]], [x[[ADDR]]] 134; CHECK: cbnz [[SCRATCH_REG]], [[TRYBB]] 135 %val = atomicrmw or ptr %p, i64 7 monotonic 136 ret i64 %val 137} 138 139define void @acquire_fence() #0 { 140 fence acquire 141 ret void 142 ; CHECK-LABEL: acquire_fence: 143 ; CHECK: dmb ishld 144} 145 146define void @release_fence() #0 { 147 fence release 148 ret void 149 ; CHECK-LABEL: release_fence: 150 ; CHECK: dmb ish{{$}} 151} 152 153define void @seq_cst_fence() #0 { 154 fence seq_cst 155 ret void 156 ; CHECK-LABEL: seq_cst_fence: 157 ; CHECK: dmb ish{{$}} 158} 159 160define i32 @atomic_load(ptr %p) #0 { 161 %r = load atomic i32, ptr %p seq_cst, align 4 162 ret i32 %r 163 ; CHECK-LABEL: atomic_load: 164 ; CHECK: ldar 165} 166 167define i8 @atomic_load_relaxed_8(ptr %p, i32 %off32) #0 { 168; CHECK-LABEL: atomic_load_relaxed_8: 169 %ptr_unsigned = getelementptr i8, ptr %p, i32 4095 170 %val_unsigned = load atomic i8, ptr %ptr_unsigned monotonic, align 1 171; CHECK: ldrb {{w[0-9]+}}, [x0, #4095] 172 173 %ptr_regoff = getelementptr i8, ptr %p, i32 %off32 174 %val_regoff = load atomic i8, ptr %ptr_regoff unordered, align 1 175 %tot1 = add i8 %val_unsigned, %val_regoff 176; CHECK: ldrb {{w[0-9]+}}, [x0, w1, sxtw] 177 178 %ptr_unscaled = getelementptr i8, ptr %p, i32 -256 179 %val_unscaled = load atomic i8, ptr %ptr_unscaled monotonic, align 1 180 %tot2 = add i8 %tot1, %val_unscaled 181; CHECK: ldurb {{w[0-9]+}}, [x0, #-256] 182 183 %ptr_random = getelementptr i8, ptr %p, i32 1191936 ; 0x123000 (i.e. ADD imm) 184 %val_random = load atomic i8, ptr %ptr_random unordered, align 1 185 %tot3 = add i8 %tot2, %val_random 186; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 187; CHECK: ldrb {{w[0-9]+}}, [x[[ADDR]]] 188 189 ret i8 %tot3 190} 191 192define i16 @atomic_load_relaxed_16(ptr %p, i32 %off32) #0 { 193; CHECK-LABEL: atomic_load_relaxed_16: 194 %ptr_unsigned = getelementptr i16, ptr %p, i32 4095 195 %val_unsigned = load atomic i16, ptr %ptr_unsigned monotonic, align 2 196; CHECK: ldrh {{w[0-9]+}}, [x0, #8190] 197 198 %ptr_regoff = getelementptr i16, ptr %p, i32 %off32 199 %val_regoff = load atomic i16, ptr %ptr_regoff unordered, align 2 200 %tot1 = add i16 %val_unsigned, %val_regoff 201; CHECK: ldrh {{w[0-9]+}}, [x0, w1, sxtw #1] 202 203 %ptr_unscaled = getelementptr i16, ptr %p, i32 -128 204 %val_unscaled = load atomic i16, ptr %ptr_unscaled monotonic, align 2 205 %tot2 = add i16 %tot1, %val_unscaled 206; CHECK: ldurh {{w[0-9]+}}, [x0, #-256] 207 208 %ptr_random = getelementptr i16, ptr %p, i32 595968 ; 0x123000/2 (i.e. ADD imm) 209 %val_random = load atomic i16, ptr %ptr_random unordered, align 2 210 %tot3 = add i16 %tot2, %val_random 211; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 212; CHECK: ldrh {{w[0-9]+}}, [x[[ADDR]]] 213 214 ret i16 %tot3 215} 216 217define i32 @atomic_load_relaxed_32(ptr %p, i32 %off32) #0 { 218; CHECK-LABEL: atomic_load_relaxed_32: 219 %ptr_unsigned = getelementptr i32, ptr %p, i32 4095 220 %val_unsigned = load atomic i32, ptr %ptr_unsigned monotonic, align 4 221; CHECK: ldr {{w[0-9]+}}, [x0, #16380] 222 223 %ptr_regoff = getelementptr i32, ptr %p, i32 %off32 224 %val_regoff = load atomic i32, ptr %ptr_regoff unordered, align 4 225 %tot1 = add i32 %val_unsigned, %val_regoff 226; CHECK: ldr {{w[0-9]+}}, [x0, w1, sxtw #2] 227 228 %ptr_unscaled = getelementptr i32, ptr %p, i32 -64 229 %val_unscaled = load atomic i32, ptr %ptr_unscaled monotonic, align 4 230 %tot2 = add i32 %tot1, %val_unscaled 231; CHECK: ldur {{w[0-9]+}}, [x0, #-256] 232 233 %ptr_random = getelementptr i32, ptr %p, i32 297984 ; 0x123000/4 (i.e. ADD imm) 234 %val_random = load atomic i32, ptr %ptr_random unordered, align 4 235 %tot3 = add i32 %tot2, %val_random 236; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 237; CHECK: ldr {{w[0-9]+}}, [x[[ADDR]]] 238 239 ret i32 %tot3 240} 241 242define i64 @atomic_load_relaxed_64(ptr %p, i32 %off32) #0 { 243; CHECK-LABEL: atomic_load_relaxed_64: 244 %ptr_unsigned = getelementptr i64, ptr %p, i32 4095 245 %val_unsigned = load atomic i64, ptr %ptr_unsigned monotonic, align 8 246; CHECK: ldr {{x[0-9]+}}, [x0, #32760] 247 248 %ptr_regoff = getelementptr i64, ptr %p, i32 %off32 249 %val_regoff = load atomic i64, ptr %ptr_regoff unordered, align 8 250 %tot1 = add i64 %val_unsigned, %val_regoff 251; CHECK: ldr {{x[0-9]+}}, [x0, w1, sxtw #3] 252 253 %ptr_unscaled = getelementptr i64, ptr %p, i32 -32 254 %val_unscaled = load atomic i64, ptr %ptr_unscaled monotonic, align 8 255 %tot2 = add i64 %tot1, %val_unscaled 256; CHECK: ldur {{x[0-9]+}}, [x0, #-256] 257 258 %ptr_random = getelementptr i64, ptr %p, i32 148992 ; 0x123000/8 (i.e. ADD imm) 259 %val_random = load atomic i64, ptr %ptr_random unordered, align 8 260 %tot3 = add i64 %tot2, %val_random 261; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 262; CHECK: ldr {{x[0-9]+}}, [x[[ADDR]]] 263 264 ret i64 %tot3 265} 266 267 268define void @atomc_store(ptr %p) #0 { 269 store atomic i32 4, ptr %p seq_cst, align 4 270 ret void 271 ; CHECK-LABEL: atomc_store: 272 ; CHECK: stlr 273} 274 275define void @atomic_store_relaxed_8(ptr %p, i32 %off32, i8 %val) #0 { 276; CHECK-LABEL: atomic_store_relaxed_8: 277 %ptr_unsigned = getelementptr i8, ptr %p, i32 4095 278 store atomic i8 %val, ptr %ptr_unsigned monotonic, align 1 279; CHECK: strb {{w[0-9]+}}, [x0, #4095] 280 281 %ptr_regoff = getelementptr i8, ptr %p, i32 %off32 282 store atomic i8 %val, ptr %ptr_regoff unordered, align 1 283; CHECK: strb {{w[0-9]+}}, [x0, w1, sxtw] 284 285 %ptr_unscaled = getelementptr i8, ptr %p, i32 -256 286 store atomic i8 %val, ptr %ptr_unscaled monotonic, align 1 287; CHECK: sturb {{w[0-9]+}}, [x0, #-256] 288 289 %ptr_random = getelementptr i8, ptr %p, i32 1191936 ; 0x123000 (i.e. ADD imm) 290 store atomic i8 %val, ptr %ptr_random unordered, align 1 291; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 292; CHECK: strb {{w[0-9]+}}, [x[[ADDR]]] 293 294 ret void 295} 296 297define void @atomic_store_relaxed_16(ptr %p, i32 %off32, i16 %val) #0 { 298; CHECK-LABEL: atomic_store_relaxed_16: 299 %ptr_unsigned = getelementptr i16, ptr %p, i32 4095 300 store atomic i16 %val, ptr %ptr_unsigned monotonic, align 2 301; CHECK: strh {{w[0-9]+}}, [x0, #8190] 302 303 %ptr_regoff = getelementptr i16, ptr %p, i32 %off32 304 store atomic i16 %val, ptr %ptr_regoff unordered, align 2 305; CHECK: strh {{w[0-9]+}}, [x0, w1, sxtw #1] 306 307 %ptr_unscaled = getelementptr i16, ptr %p, i32 -128 308 store atomic i16 %val, ptr %ptr_unscaled monotonic, align 2 309; CHECK: sturh {{w[0-9]+}}, [x0, #-256] 310 311 %ptr_random = getelementptr i16, ptr %p, i32 595968 ; 0x123000/2 (i.e. ADD imm) 312 store atomic i16 %val, ptr %ptr_random unordered, align 2 313; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 314; CHECK: strh {{w[0-9]+}}, [x[[ADDR]]] 315 316 ret void 317} 318 319define void @atomic_store_relaxed_32(ptr %p, i32 %off32, i32 %val) #0 { 320; CHECK-LABEL: atomic_store_relaxed_32: 321 %ptr_unsigned = getelementptr i32, ptr %p, i32 4095 322 store atomic i32 %val, ptr %ptr_unsigned monotonic, align 4 323; CHECK: str {{w[0-9]+}}, [x0, #16380] 324 325 %ptr_regoff = getelementptr i32, ptr %p, i32 %off32 326 store atomic i32 %val, ptr %ptr_regoff unordered, align 4 327; CHECK: str {{w[0-9]+}}, [x0, w1, sxtw #2] 328 329 %ptr_unscaled = getelementptr i32, ptr %p, i32 -64 330 store atomic i32 %val, ptr %ptr_unscaled monotonic, align 4 331; CHECK: stur {{w[0-9]+}}, [x0, #-256] 332 333 %ptr_random = getelementptr i32, ptr %p, i32 297984 ; 0x123000/4 (i.e. ADD imm) 334 store atomic i32 %val, ptr %ptr_random unordered, align 4 335; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 336; CHECK: str {{w[0-9]+}}, [x[[ADDR]]] 337 338 ret void 339} 340 341define void @atomic_store_relaxed_64(ptr %p, i32 %off32, i64 %val) #0 { 342; OUTLINE-ATOMICS: bl __aarch64_ldadd4_acq_rel 343; CHECK-LABEL: atomic_store_relaxed_64: 344 %ptr_unsigned = getelementptr i64, ptr %p, i32 4095 345 store atomic i64 %val, ptr %ptr_unsigned monotonic, align 8 346; CHECK: str {{x[0-9]+}}, [x0, #32760] 347 348 %ptr_regoff = getelementptr i64, ptr %p, i32 %off32 349 store atomic i64 %val, ptr %ptr_regoff unordered, align 8 350; CHECK: str {{x[0-9]+}}, [x0, w1, sxtw #3] 351 352 %ptr_unscaled = getelementptr i64, ptr %p, i32 -32 353 store atomic i64 %val, ptr %ptr_unscaled monotonic, align 8 354; CHECK: stur {{x[0-9]+}}, [x0, #-256] 355 356 %ptr_random = getelementptr i64, ptr %p, i32 148992 ; 0x123000/8 (i.e. ADD imm) 357 store atomic i64 %val, ptr %ptr_random unordered, align 8 358; CHECK: add x[[ADDR:[0-9]+]], x0, #291, lsl #12 359; CHECK: str {{x[0-9]+}}, [x[[ADDR]]] 360 361 ret void 362} 363 364; rdar://11531169 365; rdar://11531308 366 367%"class.X::Atomic" = type { %struct.x_atomic_t } 368%struct.x_atomic_t = type { i32 } 369 370@counter = external hidden global %"class.X::Atomic", align 4 371 372define i32 @next_id() nounwind optsize ssp align 2 { 373entry: 374 %0 = atomicrmw add ptr @counter, i32 1 seq_cst 375 %add.i = add i32 %0, 1 376 %tobool = icmp eq i32 %add.i, 0 377 br i1 %tobool, label %if.else, label %return 378 379if.else: ; preds = %entry 380 %1 = atomicrmw add ptr @counter, i32 1 seq_cst 381 %add.i2 = add i32 %1, 1 382 br label %return 383 384return: ; preds = %if.else, %entry 385 %retval.0 = phi i32 [ %add.i2, %if.else ], [ %add.i, %entry ] 386 ret i32 %retval.0 387} 388 389attributes #0 = { nounwind } 390