139f15686SMatt Arsenault //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===// 239f15686SMatt Arsenault // 339f15686SMatt Arsenault // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 439f15686SMatt Arsenault // See https://llvm.org/LICENSE.txt for license information. 539f15686SMatt Arsenault // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 639f15686SMatt Arsenault // 739f15686SMatt Arsenault //===----------------------------------------------------------------------===// 839f15686SMatt Arsenault // 939f15686SMatt Arsenault // This pass lowers atomic intrinsics to non-atomic form for use in a known 1039f15686SMatt Arsenault // non-preemptible environment. 1139f15686SMatt Arsenault // 1239f15686SMatt Arsenault //===----------------------------------------------------------------------===// 1339f15686SMatt Arsenault 1439f15686SMatt Arsenault #include "llvm/Transforms/Utils/LowerAtomic.h" 1539f15686SMatt Arsenault #include "llvm/IR/Function.h" 1639f15686SMatt Arsenault #include "llvm/IR/IRBuilder.h" 17a20f7efbSBjorn Pettersson 1839f15686SMatt Arsenault using namespace llvm; 1939f15686SMatt Arsenault 2039f15686SMatt Arsenault #define DEBUG_TYPE "loweratomic" 2139f15686SMatt Arsenault 2239f15686SMatt Arsenault bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) { 2339f15686SMatt Arsenault IRBuilder<> Builder(CXI); 2439f15686SMatt Arsenault Value *Ptr = CXI->getPointerOperand(); 2539f15686SMatt Arsenault Value *Cmp = CXI->getCompareOperand(); 2639f15686SMatt Arsenault Value *Val = CXI->getNewValOperand(); 2739f15686SMatt Arsenault 28*30dd1297SMatt Arsenault auto [Orig, Equal] = 29*30dd1297SMatt Arsenault buildCmpXchgValue(Builder, Ptr, Cmp, Val, CXI->getAlign()); 3039f15686SMatt Arsenault 31*30dd1297SMatt Arsenault Value *Res = 32*30dd1297SMatt Arsenault Builder.CreateInsertValue(PoisonValue::get(CXI->getType()), Orig, 0); 3339f15686SMatt Arsenault Res = Builder.CreateInsertValue(Res, Equal, 1); 3439f15686SMatt Arsenault 3539f15686SMatt Arsenault CXI->replaceAllUsesWith(Res); 3639f15686SMatt Arsenault CXI->eraseFromParent(); 3739f15686SMatt Arsenault return true; 3839f15686SMatt Arsenault } 3939f15686SMatt Arsenault 40*30dd1297SMatt Arsenault std::pair<Value *, Value *> llvm::buildCmpXchgValue(IRBuilderBase &Builder, 41*30dd1297SMatt Arsenault Value *Ptr, Value *Cmp, 42*30dd1297SMatt Arsenault Value *Val, 43*30dd1297SMatt Arsenault Align Alignment) { 44*30dd1297SMatt Arsenault LoadInst *Orig = Builder.CreateAlignedLoad(Val->getType(), Ptr, Alignment); 45*30dd1297SMatt Arsenault Value *Equal = Builder.CreateICmpEQ(Orig, Cmp); 46*30dd1297SMatt Arsenault Value *Res = Builder.CreateSelect(Equal, Val, Orig); 47*30dd1297SMatt Arsenault Builder.CreateAlignedStore(Res, Ptr, Alignment); 48*30dd1297SMatt Arsenault 49*30dd1297SMatt Arsenault return {Orig, Equal}; 50*30dd1297SMatt Arsenault } 51*30dd1297SMatt Arsenault 529fdd2584SMatt Arsenault Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op, 539fdd2584SMatt Arsenault IRBuilderBase &Builder, Value *Loaded, 54778cf543SMatt Arsenault Value *Val) { 559fdd2584SMatt Arsenault Value *NewVal; 569fdd2584SMatt Arsenault switch (Op) { 579fdd2584SMatt Arsenault case AtomicRMWInst::Xchg: 58778cf543SMatt Arsenault return Val; 599fdd2584SMatt Arsenault case AtomicRMWInst::Add: 60778cf543SMatt Arsenault return Builder.CreateAdd(Loaded, Val, "new"); 619fdd2584SMatt Arsenault case AtomicRMWInst::Sub: 62778cf543SMatt Arsenault return Builder.CreateSub(Loaded, Val, "new"); 639fdd2584SMatt Arsenault case AtomicRMWInst::And: 64778cf543SMatt Arsenault return Builder.CreateAnd(Loaded, Val, "new"); 659fdd2584SMatt Arsenault case AtomicRMWInst::Nand: 66778cf543SMatt Arsenault return Builder.CreateNot(Builder.CreateAnd(Loaded, Val), "new"); 679fdd2584SMatt Arsenault case AtomicRMWInst::Or: 68778cf543SMatt Arsenault return Builder.CreateOr(Loaded, Val, "new"); 699fdd2584SMatt Arsenault case AtomicRMWInst::Xor: 70778cf543SMatt Arsenault return Builder.CreateXor(Loaded, Val, "new"); 719fdd2584SMatt Arsenault case AtomicRMWInst::Max: 72778cf543SMatt Arsenault NewVal = Builder.CreateICmpSGT(Loaded, Val); 73778cf543SMatt Arsenault return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 749fdd2584SMatt Arsenault case AtomicRMWInst::Min: 75778cf543SMatt Arsenault NewVal = Builder.CreateICmpSLE(Loaded, Val); 76778cf543SMatt Arsenault return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 779fdd2584SMatt Arsenault case AtomicRMWInst::UMax: 78778cf543SMatt Arsenault NewVal = Builder.CreateICmpUGT(Loaded, Val); 79778cf543SMatt Arsenault return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 809fdd2584SMatt Arsenault case AtomicRMWInst::UMin: 81778cf543SMatt Arsenault NewVal = Builder.CreateICmpULE(Loaded, Val); 82778cf543SMatt Arsenault return Builder.CreateSelect(NewVal, Loaded, Val, "new"); 839fdd2584SMatt Arsenault case AtomicRMWInst::FAdd: 84778cf543SMatt Arsenault return Builder.CreateFAdd(Loaded, Val, "new"); 859fdd2584SMatt Arsenault case AtomicRMWInst::FSub: 86778cf543SMatt Arsenault return Builder.CreateFSub(Loaded, Val, "new"); 871023ddafSShilei Tian case AtomicRMWInst::FMax: 88778cf543SMatt Arsenault return Builder.CreateMaxNum(Loaded, Val); 891023ddafSShilei Tian case AtomicRMWInst::FMin: 90778cf543SMatt Arsenault return Builder.CreateMinNum(Loaded, Val); 91778cf543SMatt Arsenault case AtomicRMWInst::UIncWrap: { 92778cf543SMatt Arsenault Constant *One = ConstantInt::get(Loaded->getType(), 1); 93778cf543SMatt Arsenault Value *Inc = Builder.CreateAdd(Loaded, One); 94778cf543SMatt Arsenault Value *Cmp = Builder.CreateICmpUGE(Loaded, Val); 95778cf543SMatt Arsenault Constant *Zero = ConstantInt::get(Loaded->getType(), 0); 96778cf543SMatt Arsenault return Builder.CreateSelect(Cmp, Zero, Inc, "new"); 97778cf543SMatt Arsenault } 98778cf543SMatt Arsenault case AtomicRMWInst::UDecWrap: { 99778cf543SMatt Arsenault Constant *Zero = ConstantInt::get(Loaded->getType(), 0); 100778cf543SMatt Arsenault Constant *One = ConstantInt::get(Loaded->getType(), 1); 101778cf543SMatt Arsenault 102778cf543SMatt Arsenault Value *Dec = Builder.CreateSub(Loaded, One); 103778cf543SMatt Arsenault Value *CmpEq0 = Builder.CreateICmpEQ(Loaded, Zero); 104778cf543SMatt Arsenault Value *CmpOldGtVal = Builder.CreateICmpUGT(Loaded, Val); 105778cf543SMatt Arsenault Value *Or = Builder.CreateOr(CmpEq0, CmpOldGtVal); 106778cf543SMatt Arsenault return Builder.CreateSelect(Or, Val, Dec, "new"); 107778cf543SMatt Arsenault } 1084af249feSanjenner case AtomicRMWInst::USubCond: { 1094af249feSanjenner Value *Cmp = Builder.CreateICmpUGE(Loaded, Val); 1104af249feSanjenner Value *Sub = Builder.CreateSub(Loaded, Val); 1114af249feSanjenner return Builder.CreateSelect(Cmp, Sub, Loaded, "new"); 1124af249feSanjenner } 1134af249feSanjenner case AtomicRMWInst::USubSat: 1144af249feSanjenner return Builder.CreateIntrinsic(Intrinsic::usub_sat, Loaded->getType(), 1154af249feSanjenner {Loaded, Val}, nullptr, "new"); 1169fdd2584SMatt Arsenault default: 1179fdd2584SMatt Arsenault llvm_unreachable("Unknown atomic op"); 1189fdd2584SMatt Arsenault } 1199fdd2584SMatt Arsenault } 1209fdd2584SMatt Arsenault 12139f15686SMatt Arsenault bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) { 12239f15686SMatt Arsenault IRBuilder<> Builder(RMWI); 1233701ebe7SMatt Arsenault Builder.setIsFPConstrained( 1243701ebe7SMatt Arsenault RMWI->getFunction()->hasFnAttribute(Attribute::StrictFP)); 1253701ebe7SMatt Arsenault 12639f15686SMatt Arsenault Value *Ptr = RMWI->getPointerOperand(); 12739f15686SMatt Arsenault Value *Val = RMWI->getValOperand(); 12839f15686SMatt Arsenault 12939f15686SMatt Arsenault LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr); 1309fdd2584SMatt Arsenault Value *Res = buildAtomicRMWValue(RMWI->getOperation(), Builder, Orig, Val); 13139f15686SMatt Arsenault Builder.CreateStore(Res, Ptr); 13239f15686SMatt Arsenault RMWI->replaceAllUsesWith(Orig); 13339f15686SMatt Arsenault RMWI->eraseFromParent(); 13439f15686SMatt Arsenault return true; 13539f15686SMatt Arsenault } 136