181ad6265SDimitry Andric //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===// 281ad6265SDimitry Andric // 381ad6265SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 481ad6265SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 581ad6265SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 681ad6265SDimitry Andric // 781ad6265SDimitry Andric //===----------------------------------------------------------------------===// 881ad6265SDimitry Andric // 981ad6265SDimitry Andric // This pass lowers atomic intrinsics to non-atomic form for use in a known 1081ad6265SDimitry Andric // non-preemptible environment. 1181ad6265SDimitry Andric // 1281ad6265SDimitry Andric //===----------------------------------------------------------------------===// 1381ad6265SDimitry Andric 1481ad6265SDimitry Andric #include "llvm/Transforms/Utils/LowerAtomic.h" 1581ad6265SDimitry Andric #include "llvm/IR/Function.h" 1681ad6265SDimitry Andric #include "llvm/IR/IRBuilder.h" 1781ad6265SDimitry Andric #include "llvm/InitializePasses.h" 1881ad6265SDimitry Andric #include "llvm/Pass.h" 1981ad6265SDimitry Andric using namespace llvm; 2081ad6265SDimitry Andric 2181ad6265SDimitry Andric #define DEBUG_TYPE "loweratomic" 2281ad6265SDimitry Andric 2381ad6265SDimitry Andric bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) { 2481ad6265SDimitry Andric IRBuilder<> Builder(CXI); 2581ad6265SDimitry Andric Value *Ptr = CXI->getPointerOperand(); 2681ad6265SDimitry Andric Value *Cmp = CXI->getCompareOperand(); 2781ad6265SDimitry Andric Value *Val = CXI->getNewValOperand(); 2881ad6265SDimitry Andric 2981ad6265SDimitry Andric LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr); 3081ad6265SDimitry Andric Value *Equal = Builder.CreateICmpEQ(Orig, Cmp); 3181ad6265SDimitry Andric Value *Res = Builder.CreateSelect(Equal, Val, Orig); 3281ad6265SDimitry Andric Builder.CreateStore(Res, Ptr); 3381ad6265SDimitry Andric 3481ad6265SDimitry Andric Res = Builder.CreateInsertValue(UndefValue::get(CXI->getType()), Orig, 0); 3581ad6265SDimitry Andric Res = Builder.CreateInsertValue(Res, Equal, 1); 3681ad6265SDimitry Andric 3781ad6265SDimitry Andric CXI->replaceAllUsesWith(Res); 3881ad6265SDimitry Andric CXI->eraseFromParent(); 3981ad6265SDimitry Andric return true; 4081ad6265SDimitry Andric } 4181ad6265SDimitry Andric 4281ad6265SDimitry Andric Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op, 4381ad6265SDimitry Andric IRBuilderBase &Builder, Value *Loaded, 4481ad6265SDimitry Andric Value *Inc) { 4581ad6265SDimitry Andric Value *NewVal; 4681ad6265SDimitry Andric switch (Op) { 4781ad6265SDimitry Andric case AtomicRMWInst::Xchg: 4881ad6265SDimitry Andric return Inc; 4981ad6265SDimitry Andric case AtomicRMWInst::Add: 5081ad6265SDimitry Andric return Builder.CreateAdd(Loaded, Inc, "new"); 5181ad6265SDimitry Andric case AtomicRMWInst::Sub: 5281ad6265SDimitry Andric return Builder.CreateSub(Loaded, Inc, "new"); 5381ad6265SDimitry Andric case AtomicRMWInst::And: 5481ad6265SDimitry Andric return Builder.CreateAnd(Loaded, Inc, "new"); 5581ad6265SDimitry Andric case AtomicRMWInst::Nand: 5681ad6265SDimitry Andric return Builder.CreateNot(Builder.CreateAnd(Loaded, Inc), "new"); 5781ad6265SDimitry Andric case AtomicRMWInst::Or: 5881ad6265SDimitry Andric return Builder.CreateOr(Loaded, Inc, "new"); 5981ad6265SDimitry Andric case AtomicRMWInst::Xor: 6081ad6265SDimitry Andric return Builder.CreateXor(Loaded, Inc, "new"); 6181ad6265SDimitry Andric case AtomicRMWInst::Max: 6281ad6265SDimitry Andric NewVal = Builder.CreateICmpSGT(Loaded, Inc); 6381ad6265SDimitry Andric return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); 6481ad6265SDimitry Andric case AtomicRMWInst::Min: 6581ad6265SDimitry Andric NewVal = Builder.CreateICmpSLE(Loaded, Inc); 6681ad6265SDimitry Andric return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); 6781ad6265SDimitry Andric case AtomicRMWInst::UMax: 6881ad6265SDimitry Andric NewVal = Builder.CreateICmpUGT(Loaded, Inc); 6981ad6265SDimitry Andric return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); 7081ad6265SDimitry Andric case AtomicRMWInst::UMin: 7181ad6265SDimitry Andric NewVal = Builder.CreateICmpULE(Loaded, Inc); 7281ad6265SDimitry Andric return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); 7381ad6265SDimitry Andric case AtomicRMWInst::FAdd: 7481ad6265SDimitry Andric return Builder.CreateFAdd(Loaded, Inc, "new"); 7581ad6265SDimitry Andric case AtomicRMWInst::FSub: 7681ad6265SDimitry Andric return Builder.CreateFSub(Loaded, Inc, "new"); 77*753f127fSDimitry Andric case AtomicRMWInst::FMax: 78*753f127fSDimitry Andric return Builder.CreateMaxNum(Loaded, Inc); 79*753f127fSDimitry Andric case AtomicRMWInst::FMin: 80*753f127fSDimitry Andric return Builder.CreateMinNum(Loaded, Inc); 8181ad6265SDimitry Andric default: 8281ad6265SDimitry Andric llvm_unreachable("Unknown atomic op"); 8381ad6265SDimitry Andric } 8481ad6265SDimitry Andric } 8581ad6265SDimitry Andric 8681ad6265SDimitry Andric bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) { 8781ad6265SDimitry Andric IRBuilder<> Builder(RMWI); 8881ad6265SDimitry Andric Value *Ptr = RMWI->getPointerOperand(); 8981ad6265SDimitry Andric Value *Val = RMWI->getValOperand(); 9081ad6265SDimitry Andric 9181ad6265SDimitry Andric LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr); 9281ad6265SDimitry Andric Value *Res = buildAtomicRMWValue(RMWI->getOperation(), Builder, Orig, Val); 9381ad6265SDimitry Andric Builder.CreateStore(Res, Ptr); 9481ad6265SDimitry Andric RMWI->replaceAllUsesWith(Orig); 9581ad6265SDimitry Andric RMWI->eraseFromParent(); 9681ad6265SDimitry Andric return true; 9781ad6265SDimitry Andric } 98