Lines Matching defs:Builder
80 insertRMWLLSCLoop(IRBuilderBase &Builder, Type *ResultTy, Value *Addr,
96 IRBuilderBase &Builder, Type *ResultType, Value *Addr, Align AddrAlign,
346 IRBuilder Builder(I);
348 TLI->emitTrailingFence(Builder, I, FenceOrdering)) {
434 ReplacementIRBuilder Builder(I, *DL);
436 auto LeadingFence = TLI->emitLeadingFence(Builder, I, Order);
438 auto TrailingFence = TLI->emitTrailingFence(Builder, I, Order);
463 ReplacementIRBuilder Builder(LI, *DL);
467 auto *NewLI = Builder.CreateLoad(NewTy, Addr);
473 Value *NewVal = Builder.CreateBitCast(NewLI, LI->getType());
487 ReplacementIRBuilder Builder(RMWI, *DL);
492 ? Builder.CreatePtrToInt(Val, NewTy)
493 : Builder.CreateBitCast(Val, NewTy);
495 auto *NewRMWI = Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, Addr, NewVal,
503 ? Builder.CreateIntToPtr(NewRMWI, RMWI->getType())
504 : Builder.CreateBitCast(NewRMWI, RMWI->getType());
518 [](IRBuilderBase &Builder, Value *Loaded) { return Loaded; });
548 ReplacementIRBuilder Builder(LI, *DL);
553 Value *Val = TLI->emitLoadLinked(Builder, LI->getType(),
555 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
564 ReplacementIRBuilder Builder(LI, *DL);
573 Value *Pair = Builder.CreateAtomicCmpXchg(
576 Value *Loaded = Builder.CreateExtractValue(Pair, 0, "loaded");
593 ReplacementIRBuilder Builder(SI, *DL);
597 Value *NewVal = Builder.CreateBitCast(SI->getValueOperand(), NewTy);
601 StoreInst *NewSI = Builder.CreateStore(NewVal, Addr);
617 ReplacementIRBuilder Builder(SI, *DL);
623 AtomicRMWInst *AI = Builder.CreateAtomicRMW(
632 static void createCmpXchgInstFun(IRBuilderBase &Builder, Value *Addr,
643 IntegerType *IntTy = Builder.getIntNTy(OrigTy->getPrimitiveSizeInBits());
644 NewVal = Builder.CreateBitCast(NewVal, IntTy);
645 Loaded = Builder.CreateBitCast(Loaded, IntTy);
648 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
654 Success = Builder.CreateExtractValue(Pair, 1, "success");
655 NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
658 NewLoaded = Builder.CreateBitCast(NewLoaded, OrigTy);
674 auto PerformOp = [&](IRBuilderBase &Builder, Value *Loaded) {
675 return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded,
799 static PartwordMaskValues createMaskInstrs(IRBuilderBase &Builder,
834 PMV.AlignedAddr = Builder.CreateIntrinsic(
839 Value *AddrInt = Builder.CreatePtrToInt(Addr, IntTy);
840 PtrLSB = Builder.CreateAnd(AddrInt, MinWordSize - 1, "PtrLSB");
849 PMV.ShiftAmt = Builder.CreateShl(PtrLSB, 3);
852 PMV.ShiftAmt = Builder.CreateShl(
853 Builder.CreateXor(PtrLSB, MinWordSize - ValueSize), 3);
856 PMV.ShiftAmt = Builder.CreateTrunc(PMV.ShiftAmt, PMV.WordType, "ShiftAmt");
857 PMV.Mask = Builder.CreateShl(
861 PMV.Inv_Mask = Builder.CreateNot(PMV.Mask, "Inv_Mask");
866 static Value *extractMaskedValue(IRBuilderBase &Builder, Value *WideWord,
872 Value *Shift = Builder.CreateLShr(WideWord, PMV.ShiftAmt, "shifted");
873 Value *Trunc = Builder.CreateTrunc(Shift, PMV.IntValueType, "extracted");
874 return Builder.CreateBitCast(Trunc, PMV.ValueType);
877 static Value *insertMaskedValue(IRBuilderBase &Builder, Value *WideWord,
884 Updated = Builder.CreateBitCast(Updated, PMV.IntValueType);
886 Value *ZExt = Builder.CreateZExt(Updated, PMV.WordType, "extended");
888 Builder.CreateShl(ZExt, PMV.ShiftAmt, "shifted", /*HasNUW*/ true);
889 Value *And = Builder.CreateAnd(WideWord, PMV.Inv_Mask, "unmasked");
890 Value *Or = Builder.CreateOr(And, Shift, "inserted");
898 IRBuilderBase &Builder, Value *Loaded,
906 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
907 Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, Shifted_Inc);
918 Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded, Shifted_Inc);
919 Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask);
920 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
921 Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Masked);
939 Value *Loaded_Extract = extractMaskedValue(Builder, Loaded, PMV);
940 Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded_Extract, Inc);
941 Value *FinalVal = insertMaskedValue(Builder, Loaded, NewVal, PMV);
968 ReplacementIRBuilder Builder(AI, *DL);
971 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
977 Value *ValOp = Builder.CreateBitCast(AI->getValOperand(), PMV.IntValueType);
979 Builder.CreateShl(Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,
983 auto PerformPartwordOp = [&](IRBuilderBase &Builder, Value *Loaded) {
984 return performMaskedAtomicOp(Op, Builder, Loaded, ValOperand_Shifted,
991 Builder, PMV.WordType, PMV.AlignedAddr, PMV.AlignedAddrAlignment,
995 OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr,
1000 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV);
1007 ReplacementIRBuilder Builder(AI, *DL);
1015 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
1019 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
1026 Builder.CreateOr(ValOperand_Shifted, PMV.Inv_Mask, "AndOperand");
1030 AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(
1036 Value *FinalOldResult = extractMaskedValue(Builder, NewAI, PMV);
1084 ReplacementIRBuilder Builder(CI, *DL);
1085 LLVMContext &Ctx = Builder.getContext();
1096 Builder.SetInsertPoint(BB);
1099 createMaskInstrs(Builder, CI, CI->getCompareOperand()->getType(), Addr,
1104 Builder.CreateShl(Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt);
1106 Builder.CreateShl(Builder.CreateZExt(Cmp, PMV.WordType), PMV.ShiftAmt);
1110 LoadInst *InitLoaded = Builder.CreateLoad(PMV.WordType, PMV.AlignedAddr);
1112 Value *InitLoaded_MaskOut = Builder.CreateAnd(InitLoaded, PMV.Inv_Mask);
1113 Builder.CreateBr(LoopBB);
1116 Builder.SetInsertPoint(LoopBB);
1117 PHINode *Loaded_MaskOut = Builder.CreatePHI(PMV.WordType, 2);
1121 Value *FullWord_NewVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Shifted);
1122 Value *FullWord_Cmp = Builder.CreateOr(Loaded_MaskOut, Cmp_Shifted);
1123 AtomicCmpXchgInst *NewCI = Builder.CreateAtomicCmpXchg(
1134 Value *OldVal = Builder.CreateExtractValue(NewCI, 0);
1135 Value *Success = Builder.CreateExtractValue(NewCI, 1);
1138 Builder.CreateBr(EndBB);
1140 Builder.CreateCondBr(Success, EndBB, FailureBB);
1143 Builder.SetInsertPoint(FailureBB);
1147 Value *OldVal_MaskOut = Builder.CreateAnd(OldVal, PMV.Inv_Mask);
1148 Value *ShouldContinue = Builder.CreateICmpNE(Loaded_MaskOut, OldVal_MaskOut);
1149 Builder.CreateCondBr(ShouldContinue, LoopBB, EndBB);
1155 Builder.SetInsertPoint(CI);
1157 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV);
1159 Res = Builder.CreateInsertValue(Res, FinalOldVal, 0);
1160 Res = Builder.CreateInsertValue(Res, Success, 1);
1171 ReplacementIRBuilder Builder(I, *DL);
1172 Value *Loaded = insertRMWLLSCLoop(Builder, ResultType, Addr, AddrAlign,
1180 ReplacementIRBuilder Builder(AI, *DL);
1183 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
1194 Value *ValOperand_Shifted = Builder.CreateShl(
1195 Builder.CreateCast(CastOp, AI->getValOperand(), PMV.WordType),
1198 Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt,
1200 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV);
1207 ReplacementIRBuilder Builder(CI, *DL);
1210 Builder, CI, CI->getCompareOperand()->getType(), CI->getPointerOperand(),
1213 Value *CmpVal_Shifted = Builder.CreateShl(
1214 Builder.CreateZExt(CI->getCompareOperand(), PMV.WordType), PMV.ShiftAmt,
1216 Value *NewVal_Shifted = Builder.CreateShl(
1217 Builder.CreateZExt(CI->getNewValOperand(), PMV.WordType), PMV.ShiftAmt,
1220 Builder, CI, PMV.AlignedAddr, CmpVal_Shifted, NewVal_Shifted, PMV.Mask,
1222 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV);
1224 Res = Builder.CreateInsertValue(Res, FinalOldVal, 0);
1225 Value *Success = Builder.CreateICmpEQ(
1226 CmpVal_Shifted, Builder.CreateAnd(OldVal, PMV.Mask), "Success");
1227 Res = Builder.CreateInsertValue(Res, Success, 1);
1234 IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
1237 LLVMContext &Ctx = Builder.getContext();
1238 BasicBlock *BB = Builder.GetInsertBlock();
1258 BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end");
1264 Builder.SetInsertPoint(BB);
1265 Builder.CreateBr(LoopBB);
1268 Builder.SetInsertPoint(LoopBB);
1269 Value *Loaded = TLI->emitLoadLinked(Builder, ResultTy, Addr, MemOpOrder);
1271 Value *NewVal = PerformOp(Builder, Loaded);
1274 TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder);
1275 Value *TryAgain = Builder.CreateICmpNE(
1277 Builder.CreateCondBr(TryAgain, LoopBB, ExitBB);
1279 Builder.SetInsertPoint(ExitBB, ExitBB->begin());
1294 ReplacementIRBuilder Builder(CI, *DL);
1298 Value *NewCmp = Builder.CreatePtrToInt(CI->getCompareOperand(), NewTy);
1299 Value *NewNewVal = Builder.CreatePtrToInt(CI->getNewValOperand(), NewTy);
1301 auto *NewCI = Builder.CreateAtomicCmpXchg(
1308 Value *OldVal = Builder.CreateExtractValue(NewCI, 0);
1309 Value *Succ = Builder.CreateExtractValue(NewCI, 1);
1311 OldVal = Builder.CreateIntToPtr(OldVal, CI->getCompareOperand()->getType());
1314 Res = Builder.CreateInsertValue(Res, OldVal, 0);
1315 Res = Builder.CreateInsertValue(Res, Succ, 1);
1416 ReplacementIRBuilder Builder(CI, *DL);
1422 Builder.SetInsertPoint(BB);
1424 TLI->emitLeadingFence(Builder, CI, SuccessOrder);
1427 createMaskInstrs(Builder, CI, CI->getCompareOperand()->getType(), Addr,
1429 Builder.CreateBr(StartBB);
1432 Builder.SetInsertPoint(StartBB);
1434 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder);
1436 extractMaskedValue(Builder, UnreleasedLoad, PMV);
1437 Value *ShouldStore = Builder.CreateICmpEQ(
1442 Builder.CreateCondBr(ShouldStore, ReleasingStoreBB, NoStoreBB);
1444 Builder.SetInsertPoint(ReleasingStoreBB);
1446 TLI->emitLeadingFence(Builder, CI, SuccessOrder);
1447 Builder.CreateBr(TryStoreBB);
1449 Builder.SetInsertPoint(TryStoreBB);
1451 Builder.CreatePHI(PMV.WordType, 2, "loaded.trystore");
1454 insertMaskedValue(Builder, LoadedTryStore, CI->getNewValOperand(), PMV);
1455 Value *StoreSuccess = TLI->emitStoreConditional(Builder, NewValueInsert,
1457 StoreSuccess = Builder.CreateICmpEQ(
1460 Builder.CreateCondBr(StoreSuccess, SuccessBB,
1463 Builder.SetInsertPoint(ReleasedLoadBB);
1467 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder);
1468 Value *SecondLoadExtract = extractMaskedValue(Builder, SecondLoad, PMV);
1469 ShouldStore = Builder.CreateICmpEQ(SecondLoadExtract,
1474 Builder.CreateCondBr(ShouldStore, TryStoreBB, NoStoreBB);
1478 Builder.CreateUnreachable();
1482 Builder.SetInsertPoint(SuccessBB);
1485 TLI->emitTrailingFence(Builder, CI, SuccessOrder);
1486 Builder.CreateBr(ExitBB);
1488 Builder.SetInsertPoint(NoStoreBB);
1490 Builder.CreatePHI(UnreleasedLoad->getType(), 2, "loaded.nostore");
1498 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
1499 Builder.CreateBr(FailureBB);
1501 Builder.SetInsertPoint(FailureBB);
1503 Builder.CreatePHI(UnreleasedLoad->getType(), 2, "loaded.failure");
1508 TLI->emitTrailingFence(Builder, CI, FailureOrder);
1509 Builder.CreateBr(ExitBB);
1515 Builder.SetInsertPoint(ExitBB, ExitBB->begin());
1517 Builder.CreatePHI(UnreleasedLoad->getType(), 2, "loaded.exit");
1520 PHINode *Success = Builder.CreatePHI(Type::getInt1Ty(Ctx), 2, "success");
1528 Builder.SetInsertPoint(ExitBB, std::next(Success->getIterator()));
1529 Value *Loaded = extractMaskedValue(Builder, LoadedFull, PMV);
1558 Res = Builder.CreateInsertValue(PoisonValue::get(CI->getType()), Loaded, 0);
1559 Res = Builder.CreateInsertValue(Res, Success, 1);
1597 IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
1601 LLVMContext &Ctx = Builder.getContext();
1602 BasicBlock *BB = Builder.GetInsertBlock();
1621 BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end");
1628 Builder.SetInsertPoint(BB);
1629 LoadInst *InitLoaded = Builder.CreateAlignedLoad(ResultTy, Addr, AddrAlign);
1630 Builder.CreateBr(LoopBB);
1633 Builder.SetInsertPoint(LoopBB);
1634 PHINode *Loaded = Builder.CreatePHI(ResultTy, 2, "loaded");
1637 Value *NewVal = PerformOp(Builder, Loaded);
1642 CreateCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign,
1651 Builder.CreateCondBr(Success, ExitBB, LoopBB);
1653 Builder.SetInsertPoint(ExitBB, ExitBB->begin());
1686 ReplacementIRBuilder Builder(AI, AI->getDataLayout());
1687 Builder.setIsFPConstrained(
1693 Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(),
1695 [&](IRBuilderBase &Builder, Value *Loaded) {
1696 return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded,
1849 I, [this](IRBuilderBase &Builder, Value *Addr, Value *Loaded,
1854 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
1860 Success = Builder.CreateExtractValue(Pair, 1, "success");
1861 NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
1884 IRBuilder<> Builder(I);
1984 PtrVal = Builder.CreateAddrSpaceCast(PtrVal, PointerType::getUnqual(Ctx));
1991 Builder.CreateLifetimeStart(AllocaCASExpected, SizeVal64);
1992 Builder.CreateAlignedStore(CASExpected, AllocaCASExpected, AllocaAlignment);
2000 Builder.CreateBitOrPointerCast(ValueOperand, SizedIntTy);
2005 Builder.CreateLifetimeStart(AllocaValue, SizeVal64);
2006 Builder.CreateAlignedStore(ValueOperand, AllocaValue, AllocaAlignment);
2015 Builder.CreateLifetimeStart(AllocaResult, SizeVal64);
2042 CallInst *Call = Builder.CreateCall(LibcallFn, Args);
2048 Builder.CreateLifetimeEnd(AllocaValue, SizeVal64);
2055 Value *ExpectedOut = Builder.CreateAlignedLoad(
2057 Builder.CreateLifetimeEnd(AllocaCASExpected, SizeVal64);
2058 V = Builder.CreateInsertValue(V, ExpectedOut, 0);
2059 V = Builder.CreateInsertValue(V, Result, 1);
2064 V = Builder.CreateBitOrPointerCast(Result, I->getType());
2066 V = Builder.CreateAlignedLoad(I->getType(), AllocaResult,
2068 Builder.CreateLifetimeEnd(AllocaResult, SizeVal64);