1; RUN: opt -S -o - -mtriple=armv8-linux-gnueabihf -passes=atomic-expand %s -codegen-opt-level=1 | FileCheck %s 2 3define i8 @test_atomic_xchg_i8(ptr %ptr, i8 %xchgend) { 4; CHECK-LABEL: @test_atomic_xchg_i8 5; CHECK-NOT: fence 6; CHECK: br label %[[LOOP:.*]] 7; CHECK: [[LOOP]]: 8; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldrex.p0(ptr elementtype(i8) %ptr) 9; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i8 10; CHECK: [[NEWVAL32:%.*]] = zext i8 %xchgend to i32 11; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0(i32 [[NEWVAL32]], ptr elementtype(i8) %ptr) 12; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 13; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] 14; CHECK: [[END]]: 15; CHECK-NOT: fence 16; CHECK: ret i8 [[OLDVAL]] 17 %res = atomicrmw xchg ptr %ptr, i8 %xchgend monotonic 18 ret i8 %res 19} 20 21define i16 @test_atomic_add_i16(ptr %ptr, i16 %addend) { 22; CHECK-LABEL: @test_atomic_add_i16 23; CHECK-NOT: fence 24; CHECK: br label %[[LOOP:.*]] 25; CHECK: [[LOOP]]: 26; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0(ptr elementtype(i16) %ptr) 27; CHECK: [[OLDVAL:%.*]] = trunc i32 [[OLDVAL32]] to i16 28; CHECK: [[NEWVAL:%.*]] = add i16 [[OLDVAL]], %addend 29; CHECK: [[NEWVAL32:%.*]] = zext i16 [[NEWVAL]] to i32 30; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.stlex.p0(i32 [[NEWVAL32]], ptr elementtype(i16) %ptr) 31; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 32; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] 33; CHECK: [[END]]: 34; CHECK-NOT: fence 35; CHECK: ret i16 [[OLDVAL]] 36 %res = atomicrmw add ptr %ptr, i16 %addend seq_cst 37 ret i16 %res 38} 39 40define i32 @test_atomic_sub_i32(ptr %ptr, i32 %subend) { 41; CHECK-LABEL: @test_atomic_sub_i32 42; CHECK-NOT: fence 43; CHECK: br label %[[LOOP:.*]] 44; CHECK: [[LOOP]]: 45; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldaex.p0(ptr elementtype(i32) %ptr) 46; CHECK: [[NEWVAL:%.*]] = sub i32 [[OLDVAL]], %subend 47; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0(i32 [[NEWVAL]], ptr elementtype(i32) %ptr) 48; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 49; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] 50; CHECK: [[END]]: 51; CHECK-NOT: fence 52; CHECK: ret i32 [[OLDVAL]] 53 %res = atomicrmw sub ptr %ptr, i32 %subend acquire 54 ret i32 %res 55} 56 57define i64 @test_atomic_or_i64(ptr %ptr, i64 %orend) { 58; CHECK-LABEL: @test_atomic_or_i64 59; CHECK-NOT: fence 60; CHECK: br label %[[LOOP:.*]] 61; CHECK: [[LOOP]]: 62; CHECK: [[LOHI:%.*]] = call { i32, i32 } @llvm.arm.ldaexd(ptr %ptr) 63; CHECK: [[LO:%.*]] = extractvalue { i32, i32 } [[LOHI]], 0 64; CHECK: [[HI:%.*]] = extractvalue { i32, i32 } [[LOHI]], 1 65; CHECK: [[LO64:%.*]] = zext i32 [[LO]] to i64 66; CHECK: [[HI64_TMP:%.*]] = zext i32 [[HI]] to i64 67; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32 68; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]] 69; CHECK: [[NEWVAL:%.*]] = or i64 [[OLDVAL]], %orend 70; CHECK: [[NEWLO:%.*]] = trunc i64 [[NEWVAL]] to i32 71; CHECK: [[NEWHI_TMP:%.*]] = lshr i64 [[NEWVAL]], 32 72; CHECK: [[NEWHI:%.*]] = trunc i64 [[NEWHI_TMP]] to i32 73; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.stlexd(i32 [[NEWLO]], i32 [[NEWHI]], ptr %ptr) 74; CHECK: [[TST:%.*]] = icmp ne i32 [[TRYAGAIN]], 0 75; CHECK: br i1 [[TST]], label %[[LOOP]], label %[[END:.*]] 76; CHECK: [[END]]: 77; CHECK-NOT: fence 78; CHECK: ret i64 [[OLDVAL]] 79 %res = atomicrmw or ptr %ptr, i64 %orend seq_cst 80 ret i64 %res 81} 82 83define i8 @test_cmpxchg_i8_seqcst_seqcst(ptr %ptr, i8 %desired, i8 %newval) { 84; CHECK-LABEL: @test_cmpxchg_i8_seqcst_seqcst 85; CHECK-NOT: fence 86; CHECK: br label %[[LOOP:.*]] 87 88; CHECK: [[LOOP]]: 89; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0(ptr elementtype(i8) %ptr) 90; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i8 91; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i8 [[OLDVAL]], %desired 92; CHECK: br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]] 93 94; CHECK: [[FENCED_STORE]]: 95; CHECK-NEXT: br label %[[TRY_STORE:.*]] 96 97; CHECK: [[TRY_STORE]]: 98; CHECK: [[LOADED_TRYSTORE:%.*]] = phi i8 [ [[OLDVAL]], %[[FENCED_STORE]] ] 99; CHECK: [[NEWVAL32:%.*]] = zext i8 %newval to i32 100; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.stlex.p0(i32 [[NEWVAL32]], ptr elementtype(i8) %ptr) 101; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0 102; CHECK: br i1 [[TST]], label %[[SUCCESS_BB:.*]], label %[[LOOP]] 103 104; CHECK: [[SUCCESS_BB]]: 105; CHECK-NOT: fence_cst 106; CHECK: br label %[[DONE:.*]] 107 108; CHECK: [[NO_STORE_BB]]: 109; CHECK-NEXT: [[LOADED_NOSTORE:%.*]] = phi i8 [ [[OLDVAL]], %[[LOOP]] ] 110; CHECK-NEXT: call void @llvm.arm.clrex() 111; CHECK-NEXT: br label %[[FAILURE_BB:.*]] 112 113; CHECK: [[FAILURE_BB]]: 114; CHECK-NEXT: [[LOADED_FAILURE:%.*]] = phi i8 [ [[LOADED_NOSTORE]], %[[NO_STORE_BB]] ] 115; CHECK-NOT: fence_cst 116; CHECK: br label %[[DONE]] 117 118; CHECK: [[DONE]]: 119; CHECK: [[LOADED_EXIT:%.*]] = phi i8 [ [[LOADED_TRYSTORE]], %[[SUCCESS_BB]] ], [ [[LOADED_FAILURE]], %[[FAILURE_BB]] ] 120; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ] 121; CHECK: ret i8 [[LOADED_EXIT]] 122 123 %pairold = cmpxchg ptr %ptr, i8 %desired, i8 %newval seq_cst seq_cst 124 %old = extractvalue { i8, i1 } %pairold, 0 125 ret i8 %old 126} 127 128define i16 @test_cmpxchg_i16_seqcst_monotonic(ptr %ptr, i16 %desired, i16 %newval) { 129; CHECK-LABEL: @test_cmpxchg_i16_seqcst_monotonic 130; CHECK-NOT: fence 131; CHECK: br label %[[LOOP:.*]] 132 133; CHECK: [[LOOP]]: 134; CHECK: [[OLDVAL32:%.*]] = call i32 @llvm.arm.ldaex.p0(ptr elementtype(i16) %ptr) 135; CHECK: [[OLDVAL:%.*]] = trunc i32 %1 to i16 136; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i16 [[OLDVAL]], %desired 137; CHECK: br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]] 138 139; CHECK: [[FENCED_STORE]]: 140; CHECK-NEXT: br label %[[TRY_STORE:.*]] 141 142; CHECK: [[TRY_STORE]]: 143; CHECK: [[LOADED_TRYSTORE:%.*]] = phi i16 [ [[OLDVAL]], %[[FENCED_STORE]] ] 144; CHECK: [[NEWVAL32:%.*]] = zext i16 %newval to i32 145; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.stlex.p0(i32 [[NEWVAL32]], ptr elementtype(i16) %ptr) 146; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0 147; CHECK: br i1 [[TST]], label %[[SUCCESS_BB:.*]], label %[[LOOP]] 148 149; CHECK: [[SUCCESS_BB]]: 150; CHECK-NOT: fence 151; CHECK: br label %[[DONE:.*]] 152 153; CHECK: [[NO_STORE_BB]]: 154; The PHI is not required. 155; CHECK-NEXT: [[LOADED_NOSTORE:%.*]] = phi i16 [ [[OLDVAL]], %[[LOOP]] ] 156; CHECK-NEXT: call void @llvm.arm.clrex() 157; CHECK-NEXT: br label %[[FAILURE_BB:.*]] 158 159; CHECK: [[FAILURE_BB]]: 160; CHECK-NEXT: [[LOADED_FAILURE:%.*]] = phi i16 [ [[LOADED_NOSTORE]], %[[NO_STORE_BB]] ] 161; CHECK-NOT: fence 162; CHECK: br label %[[DONE]] 163 164; CHECK: [[DONE]]: 165; CHECK: [[LOADED_EXIT:%.*]] = phi i16 [ [[LOADED_TRYSTORE]], %[[SUCCESS_BB]] ], [ [[LOADED_FAILURE]], %[[FAILURE_BB]] ] 166; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ] 167; CHECK: ret i16 [[LOADED_EXIT]] 168 169 %pairold = cmpxchg ptr %ptr, i16 %desired, i16 %newval seq_cst monotonic 170 %old = extractvalue { i16, i1 } %pairold, 0 171 ret i16 %old 172} 173 174define i32 @test_cmpxchg_i32_acquire_acquire(ptr %ptr, i32 %desired, i32 %newval) { 175; CHECK-LABEL: @test_cmpxchg_i32_acquire_acquire 176; CHECK-NOT: fence 177; CHECK: br label %[[LOOP:.*]] 178 179; CHECK: [[LOOP]]: 180; CHECK: [[OLDVAL:%.*]] = call i32 @llvm.arm.ldaex.p0(ptr elementtype(i32) %ptr) 181; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i32 [[OLDVAL]], %desired 182; CHECK: br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]] 183 184; CHECK: [[FENCED_STORE]]: 185; CHECK-NEXT: br label %[[TRY_STORE:.*]] 186 187; CHECK: [[TRY_STORE]]: 188; CHECK: [[LOADED_TRYSTORE:%.*]] = phi i32 [ [[OLDVAL]], %[[FENCED_STORE]] ] 189; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strex.p0(i32 %newval, ptr elementtype(i32) %ptr) 190; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0 191; CHECK: br i1 [[TST]], label %[[SUCCESS_BB:.*]], label %[[LOOP]] 192 193; CHECK: [[SUCCESS_BB]]: 194; CHECK-NOT: fence_cst 195; CHECK: br label %[[DONE:.*]] 196 197; CHECK: [[NO_STORE_BB]]: 198; CHECK-NEXT: [[LOADED_NOSTORE:%.*]] = phi i32 [ [[OLDVAL]], %[[LOOP]] ] 199; CHECK-NEXT: call void @llvm.arm.clrex() 200; CHECK-NEXT: br label %[[FAILURE_BB:.*]] 201 202; CHECK: [[FAILURE_BB]]: 203; CHECK-NEXT: [[LOADED_FAILURE:%.*]] = phi i32 [ [[LOADED_NOSTORE]], %[[NO_STORE_BB]] ] 204; CHECK-NOT: fence_cst 205; CHECK: br label %[[DONE]] 206 207; CHECK: [[DONE]]: 208; CHECK: [[LOADED_EXIT:%.*]] = phi i32 [ [[LOADED_TRYSTORE]], %[[SUCCESS_BB]] ], [ [[LOADED_FAILURE]], %[[FAILURE_BB]] ] 209; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ] 210; CHECK: ret i32 [[LOADED_EXIT]] 211 212 %pairold = cmpxchg ptr %ptr, i32 %desired, i32 %newval acquire acquire 213 %old = extractvalue { i32, i1 } %pairold, 0 214 ret i32 %old 215} 216 217define i64 @test_cmpxchg_i64_monotonic_monotonic(ptr %ptr, i64 %desired, i64 %newval) { 218; CHECK-LABEL: @test_cmpxchg_i64_monotonic_monotonic 219; CHECK-NOT: fence 220; CHECK: br label %[[LOOP:.*]] 221 222; CHECK: [[LOOP]]: 223; CHECK: [[LOHI:%.*]] = call { i32, i32 } @llvm.arm.ldrexd(ptr %ptr) 224; CHECK: [[LO:%.*]] = extractvalue { i32, i32 } [[LOHI]], 0 225; CHECK: [[HI:%.*]] = extractvalue { i32, i32 } [[LOHI]], 1 226; CHECK: [[LO64:%.*]] = zext i32 [[LO]] to i64 227; CHECK: [[HI64_TMP:%.*]] = zext i32 [[HI]] to i64 228; CHECK: [[HI64:%.*]] = shl i64 [[HI64_TMP]], 32 229; CHECK: [[OLDVAL:%.*]] = or i64 [[LO64]], [[HI64]] 230; CHECK: [[SHOULD_STORE:%.*]] = icmp eq i64 [[OLDVAL]], %desired 231; CHECK: br i1 [[SHOULD_STORE]], label %[[FENCED_STORE:.*]], label %[[NO_STORE_BB:.*]] 232 233; CHECK: [[FENCED_STORE]]: 234; CHECK-NEXT: br label %[[TRY_STORE:.*]] 235 236; CHECK: [[TRY_STORE]]: 237; CHECK: [[LOADED_TRYSTORE:%.*]] = phi i64 [ [[OLDVAL]], %[[FENCED_STORE]] ] 238; CHECK: [[NEWLO:%.*]] = trunc i64 %newval to i32 239; CHECK: [[NEWHI_TMP:%.*]] = lshr i64 %newval, 32 240; CHECK: [[NEWHI:%.*]] = trunc i64 [[NEWHI_TMP]] to i32 241; CHECK: [[TRYAGAIN:%.*]] = call i32 @llvm.arm.strexd(i32 [[NEWLO]], i32 [[NEWHI]], ptr %ptr) 242; CHECK: [[TST:%.*]] = icmp eq i32 [[TRYAGAIN]], 0 243; CHECK: br i1 [[TST]], label %[[SUCCESS_BB:.*]], label %[[LOOP]] 244 245; CHECK: [[SUCCESS_BB]]: 246; CHECK-NOT: fence_cst 247; CHECK: br label %[[DONE:.*]] 248 249; CHECK: [[NO_STORE_BB]]: 250; CHECK-NEXT: [[LOADED_NOSTORE:%.*]] = phi i64 [ [[OLDVAL]], %[[LOOP]] ] 251; CHECK-NEXT: call void @llvm.arm.clrex() 252; CHECK-NEXT: br label %[[FAILURE_BB:.*]] 253 254; CHECK: [[FAILURE_BB]]: 255; CHECK-NEXT: [[LOADED_FAILURE:%.*]] = phi i64 [ [[LOADED_NOSTORE]], %[[NO_STORE_BB]] ] 256; CHECK-NOT: fence_cst 257; CHECK: br label %[[DONE]] 258 259; CHECK: [[DONE]]: 260; CHECK: [[LOADED_EXIT:%.*]] = phi i64 [ [[LOADED_TRYSTORE]], %[[SUCCESS_BB]] ], [ [[LOADED_FAILURE]], %[[FAILURE_BB]] ] 261; CHECK: [[SUCCESS:%.*]] = phi i1 [ true, %[[SUCCESS_BB]] ], [ false, %[[FAILURE_BB]] ] 262; CHECK: ret i64 [[LOADED_EXIT]] 263 264 %pairold = cmpxchg ptr %ptr, i64 %desired, i64 %newval monotonic monotonic 265 %old = extractvalue { i64, i1 } %pairold, 0 266 ret i64 %old 267} 268