xref: /llvm-project/llvm/lib/Transforms/Utils/LowerAtomic.cpp (revision 30dd1297fa2cc172ef7a1435775010d7efd673fd)
139f15686SMatt Arsenault //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
239f15686SMatt Arsenault //
339f15686SMatt Arsenault // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
439f15686SMatt Arsenault // See https://llvm.org/LICENSE.txt for license information.
539f15686SMatt Arsenault // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
639f15686SMatt Arsenault //
739f15686SMatt Arsenault //===----------------------------------------------------------------------===//
839f15686SMatt Arsenault //
939f15686SMatt Arsenault // This pass lowers atomic intrinsics to non-atomic form for use in a known
1039f15686SMatt Arsenault // non-preemptible environment.
1139f15686SMatt Arsenault //
1239f15686SMatt Arsenault //===----------------------------------------------------------------------===//
1339f15686SMatt Arsenault 
1439f15686SMatt Arsenault #include "llvm/Transforms/Utils/LowerAtomic.h"
1539f15686SMatt Arsenault #include "llvm/IR/Function.h"
1639f15686SMatt Arsenault #include "llvm/IR/IRBuilder.h"
17a20f7efbSBjorn Pettersson 
1839f15686SMatt Arsenault using namespace llvm;
1939f15686SMatt Arsenault 
2039f15686SMatt Arsenault #define DEBUG_TYPE "loweratomic"
2139f15686SMatt Arsenault 
2239f15686SMatt Arsenault bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
2339f15686SMatt Arsenault   IRBuilder<> Builder(CXI);
2439f15686SMatt Arsenault   Value *Ptr = CXI->getPointerOperand();
2539f15686SMatt Arsenault   Value *Cmp = CXI->getCompareOperand();
2639f15686SMatt Arsenault   Value *Val = CXI->getNewValOperand();
2739f15686SMatt Arsenault 
28*30dd1297SMatt Arsenault   auto [Orig, Equal] =
29*30dd1297SMatt Arsenault       buildCmpXchgValue(Builder, Ptr, Cmp, Val, CXI->getAlign());
3039f15686SMatt Arsenault 
31*30dd1297SMatt Arsenault   Value *Res =
32*30dd1297SMatt Arsenault       Builder.CreateInsertValue(PoisonValue::get(CXI->getType()), Orig, 0);
3339f15686SMatt Arsenault   Res = Builder.CreateInsertValue(Res, Equal, 1);
3439f15686SMatt Arsenault 
3539f15686SMatt Arsenault   CXI->replaceAllUsesWith(Res);
3639f15686SMatt Arsenault   CXI->eraseFromParent();
3739f15686SMatt Arsenault   return true;
3839f15686SMatt Arsenault }
3939f15686SMatt Arsenault 
40*30dd1297SMatt Arsenault std::pair<Value *, Value *> llvm::buildCmpXchgValue(IRBuilderBase &Builder,
41*30dd1297SMatt Arsenault                                                     Value *Ptr, Value *Cmp,
42*30dd1297SMatt Arsenault                                                     Value *Val,
43*30dd1297SMatt Arsenault                                                     Align Alignment) {
44*30dd1297SMatt Arsenault   LoadInst *Orig = Builder.CreateAlignedLoad(Val->getType(), Ptr, Alignment);
45*30dd1297SMatt Arsenault   Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
46*30dd1297SMatt Arsenault   Value *Res = Builder.CreateSelect(Equal, Val, Orig);
47*30dd1297SMatt Arsenault   Builder.CreateAlignedStore(Res, Ptr, Alignment);
48*30dd1297SMatt Arsenault 
49*30dd1297SMatt Arsenault   return {Orig, Equal};
50*30dd1297SMatt Arsenault }
51*30dd1297SMatt Arsenault 
529fdd2584SMatt Arsenault Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op,
539fdd2584SMatt Arsenault                                  IRBuilderBase &Builder, Value *Loaded,
54778cf543SMatt Arsenault                                  Value *Val) {
559fdd2584SMatt Arsenault   Value *NewVal;
569fdd2584SMatt Arsenault   switch (Op) {
579fdd2584SMatt Arsenault   case AtomicRMWInst::Xchg:
58778cf543SMatt Arsenault     return Val;
599fdd2584SMatt Arsenault   case AtomicRMWInst::Add:
60778cf543SMatt Arsenault     return Builder.CreateAdd(Loaded, Val, "new");
619fdd2584SMatt Arsenault   case AtomicRMWInst::Sub:
62778cf543SMatt Arsenault     return Builder.CreateSub(Loaded, Val, "new");
639fdd2584SMatt Arsenault   case AtomicRMWInst::And:
64778cf543SMatt Arsenault     return Builder.CreateAnd(Loaded, Val, "new");
659fdd2584SMatt Arsenault   case AtomicRMWInst::Nand:
66778cf543SMatt Arsenault     return Builder.CreateNot(Builder.CreateAnd(Loaded, Val), "new");
679fdd2584SMatt Arsenault   case AtomicRMWInst::Or:
68778cf543SMatt Arsenault     return Builder.CreateOr(Loaded, Val, "new");
699fdd2584SMatt Arsenault   case AtomicRMWInst::Xor:
70778cf543SMatt Arsenault     return Builder.CreateXor(Loaded, Val, "new");
719fdd2584SMatt Arsenault   case AtomicRMWInst::Max:
72778cf543SMatt Arsenault     NewVal = Builder.CreateICmpSGT(Loaded, Val);
73778cf543SMatt Arsenault     return Builder.CreateSelect(NewVal, Loaded, Val, "new");
749fdd2584SMatt Arsenault   case AtomicRMWInst::Min:
75778cf543SMatt Arsenault     NewVal = Builder.CreateICmpSLE(Loaded, Val);
76778cf543SMatt Arsenault     return Builder.CreateSelect(NewVal, Loaded, Val, "new");
779fdd2584SMatt Arsenault   case AtomicRMWInst::UMax:
78778cf543SMatt Arsenault     NewVal = Builder.CreateICmpUGT(Loaded, Val);
79778cf543SMatt Arsenault     return Builder.CreateSelect(NewVal, Loaded, Val, "new");
809fdd2584SMatt Arsenault   case AtomicRMWInst::UMin:
81778cf543SMatt Arsenault     NewVal = Builder.CreateICmpULE(Loaded, Val);
82778cf543SMatt Arsenault     return Builder.CreateSelect(NewVal, Loaded, Val, "new");
839fdd2584SMatt Arsenault   case AtomicRMWInst::FAdd:
84778cf543SMatt Arsenault     return Builder.CreateFAdd(Loaded, Val, "new");
859fdd2584SMatt Arsenault   case AtomicRMWInst::FSub:
86778cf543SMatt Arsenault     return Builder.CreateFSub(Loaded, Val, "new");
871023ddafSShilei Tian   case AtomicRMWInst::FMax:
88778cf543SMatt Arsenault     return Builder.CreateMaxNum(Loaded, Val);
891023ddafSShilei Tian   case AtomicRMWInst::FMin:
90778cf543SMatt Arsenault     return Builder.CreateMinNum(Loaded, Val);
91778cf543SMatt Arsenault   case AtomicRMWInst::UIncWrap: {
92778cf543SMatt Arsenault     Constant *One = ConstantInt::get(Loaded->getType(), 1);
93778cf543SMatt Arsenault     Value *Inc = Builder.CreateAdd(Loaded, One);
94778cf543SMatt Arsenault     Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);
95778cf543SMatt Arsenault     Constant *Zero = ConstantInt::get(Loaded->getType(), 0);
96778cf543SMatt Arsenault     return Builder.CreateSelect(Cmp, Zero, Inc, "new");
97778cf543SMatt Arsenault   }
98778cf543SMatt Arsenault   case AtomicRMWInst::UDecWrap: {
99778cf543SMatt Arsenault     Constant *Zero = ConstantInt::get(Loaded->getType(), 0);
100778cf543SMatt Arsenault     Constant *One = ConstantInt::get(Loaded->getType(), 1);
101778cf543SMatt Arsenault 
102778cf543SMatt Arsenault     Value *Dec = Builder.CreateSub(Loaded, One);
103778cf543SMatt Arsenault     Value *CmpEq0 = Builder.CreateICmpEQ(Loaded, Zero);
104778cf543SMatt Arsenault     Value *CmpOldGtVal = Builder.CreateICmpUGT(Loaded, Val);
105778cf543SMatt Arsenault     Value *Or = Builder.CreateOr(CmpEq0, CmpOldGtVal);
106778cf543SMatt Arsenault     return Builder.CreateSelect(Or, Val, Dec, "new");
107778cf543SMatt Arsenault   }
1084af249feSanjenner   case AtomicRMWInst::USubCond: {
1094af249feSanjenner     Value *Cmp = Builder.CreateICmpUGE(Loaded, Val);
1104af249feSanjenner     Value *Sub = Builder.CreateSub(Loaded, Val);
1114af249feSanjenner     return Builder.CreateSelect(Cmp, Sub, Loaded, "new");
1124af249feSanjenner   }
1134af249feSanjenner   case AtomicRMWInst::USubSat:
1144af249feSanjenner     return Builder.CreateIntrinsic(Intrinsic::usub_sat, Loaded->getType(),
1154af249feSanjenner                                    {Loaded, Val}, nullptr, "new");
1169fdd2584SMatt Arsenault   default:
1179fdd2584SMatt Arsenault     llvm_unreachable("Unknown atomic op");
1189fdd2584SMatt Arsenault   }
1199fdd2584SMatt Arsenault }
1209fdd2584SMatt Arsenault 
12139f15686SMatt Arsenault bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) {
12239f15686SMatt Arsenault   IRBuilder<> Builder(RMWI);
1233701ebe7SMatt Arsenault   Builder.setIsFPConstrained(
1243701ebe7SMatt Arsenault       RMWI->getFunction()->hasFnAttribute(Attribute::StrictFP));
1253701ebe7SMatt Arsenault 
12639f15686SMatt Arsenault   Value *Ptr = RMWI->getPointerOperand();
12739f15686SMatt Arsenault   Value *Val = RMWI->getValOperand();
12839f15686SMatt Arsenault 
12939f15686SMatt Arsenault   LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
1309fdd2584SMatt Arsenault   Value *Res = buildAtomicRMWValue(RMWI->getOperation(), Builder, Orig, Val);
13139f15686SMatt Arsenault   Builder.CreateStore(Res, Ptr);
13239f15686SMatt Arsenault   RMWI->replaceAllUsesWith(Orig);
13339f15686SMatt Arsenault   RMWI->eraseFromParent();
13439f15686SMatt Arsenault   return true;
13539f15686SMatt Arsenault }
136