1 //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass lowers atomic intrinsics to non-atomic form for use in a known 10 // non-preemptible environment. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Transforms/Utils/LowerAtomic.h" 15 #include "llvm/IR/Function.h" 16 #include "llvm/IR/IRBuilder.h" 17 18 using namespace llvm; 19 20 #define DEBUG_TYPE "loweratomic" 21 22 bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) { 23 IRBuilder<> Builder(CXI); 24 Value *Ptr = CXI->getPointerOperand(); 25 Value *Cmp = CXI->getCompareOperand(); 26 Value *Val = CXI->getNewValOperand(); 27 28 auto [Orig, Equal] = 29 buildCmpXchgValue(Builder, Ptr, Cmp, Val, CXI->getAlign()); 30 31 Value *Res = 32 Builder.CreateInsertValue(PoisonValue::get(CXI->getType()), Orig, 0); 33 Res = Builder.CreateInsertValue(Res, Equal, 1); 34 35 CXI->replaceAllUsesWith(Res); 36 CXI->eraseFromParent(); 37 return true; 38 } 39 40 std::pair<Value *, Value *> llvm::buildCmpXchgValue(IRBuilderBase &Builder, 41 Value *Ptr, Value *Cmp, 42 Value *Val, 43 Align Alignment) { 44 LoadInst *Orig = Builder.CreateAlignedLoad(Val->getType(), Ptr, Alignment); 45 Value *Equal = Builder.CreateICmpEQ(Orig, Cmp); 46 Value *Res = Builder.CreateSelect(Equal, Val, Orig); 47 Builder.CreateAlignedStore(Res, Ptr, Alignment); 48 49 return {Orig, Equal}; 50 } 51 52 Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op, 53 IRBuilderBase &Builder, Value *Loaded, 54 Value *Val) { 55 Value *NewVal; 56 switch (Op) { 57 case AtomicRMWInst::Xchg: 58 return Val; 59 case AtomicRMWInst::Add: 60 return Builder.CreateAdd(Loaded, Val, "new"); 61 case AtomicRMWInst::Sub: 62 return Builder.CreateSub(Loaded, Val, "new"); 63 case AtomicRMWInst::And: 64 return Builder.CreateAnd(Loaded, Val, "new"); 65 case AtomicRMWInst::Nand: 66 return Builder.CreateNot(Builder.CreateAnd(Loaded, Val), "new"); 67 case AtomicRMWInst::Or: 68 return Builder.CreateOr(Loaded, Val, "new"); 69 case AtomicRMWInst::Xor: 70 return Builder.CreateXor(Loaded, Val, "new"); 71 case AtomicRMWInst::Max: 72 NewVal = Builder.CreateICmpSGT(Loaded, Val); 73 return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 74 case AtomicRMWInst::Min: 75 NewVal = Builder.CreateICmpSLE(Loaded, Val); 76 return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 77 case AtomicRMWInst::UMax: 78 NewVal = Builder.CreateICmpUGT(Loaded, Val); 79 return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 80 case AtomicRMWInst::UMin: 81 NewVal = Builder.CreateICmpULE(Loaded, Val); 82 return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 83 case AtomicRMWInst::FAdd: 84 return Builder.CreateFAdd(Loaded, Val, "new"); 85 case AtomicRMWInst::FSub: 86 return Builder.CreateFSub(Loaded, Val, "new"); 87 case AtomicRMWInst::FMax: 88 return Builder.CreateMaxNum(Loaded, Val); 89 case AtomicRMWInst::FMin: 90 return Builder.CreateMinNum(Loaded, Val); 91 case AtomicRMWInst::UIncWrap: { 92 Constant *One = ConstantInt::get(Loaded->getType(), 1); 93 Value *Inc = Builder.CreateAdd(Loaded, One); 94 Value *Cmp = Builder.CreateICmpUGE(Loaded, Val); 95 Constant *Zero = ConstantInt::get(Loaded->getType(), 0); 96 return Builder.CreateSelect(Cmp, Zero, Inc, "new"); 97 } 98 case AtomicRMWInst::UDecWrap: { 99 Constant *Zero = ConstantInt::get(Loaded->getType(), 0); 100 Constant *One = ConstantInt::get(Loaded->getType(), 1); 101 102 Value *Dec = Builder.CreateSub(Loaded, One); 103 Value *CmpEq0 = Builder.CreateICmpEQ(Loaded, Zero); 104 Value *CmpOldGtVal = Builder.CreateICmpUGT(Loaded, Val); 105 Value *Or = Builder.CreateOr(CmpEq0, CmpOldGtVal); 106 return Builder.CreateSelect(Or, Val, Dec, "new"); 107 } 108 case AtomicRMWInst::USubCond: { 109 Value *Cmp = Builder.CreateICmpUGE(Loaded, Val); 110 Value *Sub = Builder.CreateSub(Loaded, Val); 111 return Builder.CreateSelect(Cmp, Sub, Loaded, "new"); 112 } 113 case AtomicRMWInst::USubSat: 114 return Builder.CreateIntrinsic(Intrinsic::usub_sat, Loaded->getType(), 115 {Loaded, Val}, nullptr, "new"); 116 default: 117 llvm_unreachable("Unknown atomic op"); 118 } 119 } 120 121 bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) { 122 IRBuilder<> Builder(RMWI); 123 Builder.setIsFPConstrained( 124 RMWI->getFunction()->hasFnAttribute(Attribute::StrictFP)); 125 126 Value *Ptr = RMWI->getPointerOperand(); 127 Value *Val = RMWI->getValOperand(); 128 129 LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr); 130 Value *Res = buildAtomicRMWValue(RMWI->getOperation(), Builder, Orig, Val); 131 Builder.CreateStore(Res, Ptr); 132 RMWI->replaceAllUsesWith(Orig); 133 RMWI->eraseFromParent(); 134 return true; 135 } 136