Lines Matching full:builder
82 insertRMWLLSCLoop(IRBuilderBase &Builder, Type *ResultTy, Value *Addr,
98 IRBuilderBase &Builder, Type *ResultType, Value *Addr, Align AddrAlign,
143 // Enable StrictFP builder mode when appropriate.
323 IRBuilder Builder(I);
325 TLI->emitTrailingFence(Builder, I, FenceOrdering)) {
379 ReplacementIRBuilder Builder(I, *DL);
381 auto LeadingFence = TLI->emitLeadingFence(Builder, I, Order);
383 auto TrailingFence = TLI->emitTrailingFence(Builder, I, Order);
408 ReplacementIRBuilder Builder(LI, *DL);
412 auto *NewLI = Builder.CreateLoad(NewTy, Addr);
418 Value *NewVal = Builder.CreateBitCast(NewLI, LI->getType());
430 ReplacementIRBuilder Builder(RMWI, *DL);
435 ? Builder.CreatePtrToInt(Val, NewTy)
436 : Builder.CreateBitCast(Val, NewTy);
438 auto *NewRMWI = Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, Addr, NewVal,
445 ? Builder.CreateIntToPtr(NewRMWI, RMWI->getType())
446 : Builder.CreateBitCast(NewRMWI, RMWI->getType());
460 [](IRBuilderBase &Builder, Value *Loaded) { return Loaded; });
490 ReplacementIRBuilder Builder(LI, *DL);
495 Value *Val = TLI->emitLoadLinked(Builder, LI->getType(),
497 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
506 ReplacementIRBuilder Builder(LI, *DL);
515 Value *Pair = Builder.CreateAtomicCmpXchg(
518 Value *Loaded = Builder.CreateExtractValue(Pair, 0, "loaded");
535 ReplacementIRBuilder Builder(SI, *DL);
539 Value *NewVal = Builder.CreateBitCast(SI->getValueOperand(), NewTy);
543 StoreInst *NewSI = Builder.CreateStore(NewVal, Addr);
559 ReplacementIRBuilder Builder(SI, *DL);
565 AtomicRMWInst *AI = Builder.CreateAtomicRMW(
574 static void createCmpXchgInstFun(IRBuilderBase &Builder, Value *Addr,
584 IntegerType *IntTy = Builder.getIntNTy(OrigTy->getPrimitiveSizeInBits());
585 NewVal = Builder.CreateBitCast(NewVal, IntTy);
586 Loaded = Builder.CreateBitCast(Loaded, IntTy);
589 Value *Pair = Builder.CreateAtomicCmpXchg(
592 Success = Builder.CreateExtractValue(Pair, 1, "success");
593 NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
596 NewLoaded = Builder.CreateBitCast(NewLoaded, OrigTy);
612 auto PerformOp = [&](IRBuilderBase &Builder, Value *Loaded) {
613 return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded,
737 static PartwordMaskValues createMaskInstrs(IRBuilderBase &Builder,
772 PMV.AlignedAddr = Builder.CreateIntrinsic(
777 Value *AddrInt = Builder.CreatePtrToInt(Addr, IntTy);
778 PtrLSB = Builder.CreateAnd(AddrInt, MinWordSize - 1, "PtrLSB");
787 PMV.ShiftAmt = Builder.CreateShl(PtrLSB, 3);
790 PMV.ShiftAmt = Builder.CreateShl(
791 Builder.CreateXor(PtrLSB, MinWordSize - ValueSize), 3);
794 PMV.ShiftAmt = Builder.CreateTrunc(PMV.ShiftAmt, PMV.WordType, "ShiftAmt");
795 PMV.Mask = Builder.CreateShl(
799 PMV.Inv_Mask = Builder.CreateNot(PMV.Mask, "Inv_Mask");
804 static Value *extractMaskedValue(IRBuilderBase &Builder, Value *WideWord,
810 Value *Shift = Builder.CreateLShr(WideWord, PMV.ShiftAmt, "shifted");
811 Value *Trunc = Builder.CreateTrunc(Shift, PMV.IntValueType, "extracted");
812 return Builder.CreateBitCast(Trunc, PMV.ValueType);
815 static Value *insertMaskedValue(IRBuilderBase &Builder, Value *WideWord,
822 Updated = Builder.CreateBitCast(Updated, PMV.IntValueType);
824 Value *ZExt = Builder.CreateZExt(Updated, PMV.WordType, "extended");
826 Builder.CreateShl(ZExt, PMV.ShiftAmt, "shifted", /*HasNUW*/ true);
827 Value *And = Builder.CreateAnd(WideWord, PMV.Inv_Mask, "unmasked");
828 Value *Or = Builder.CreateOr(And, Shift, "inserted");
836 IRBuilderBase &Builder, Value *Loaded,
844 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
845 Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, Shifted_Inc);
856 Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded, Shifted_Inc);
857 Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask);
858 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
859 Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Masked);
875 Value *Loaded_Extract = extractMaskedValue(Builder, Loaded, PMV);
876 Value *NewVal = buildAtomicRMWValue(Op, Builder, Loaded_Extract, Inc);
877 Value *FinalVal = insertMaskedValue(Builder, Loaded, NewVal, PMV);
904 ReplacementIRBuilder Builder(AI, *DL);
907 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
913 Value *ValOp = Builder.CreateBitCast(AI->getValOperand(), PMV.IntValueType);
915 Builder.CreateShl(Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,
919 auto PerformPartwordOp = [&](IRBuilderBase &Builder, Value *Loaded) {
920 return performMaskedAtomicOp(Op, Builder, Loaded, ValOperand_Shifted,
926 OldResult = insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr,
931 OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr,
936 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV);
973 ReplacementIRBuilder Builder(AI, *DL);
981 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
985 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
992 Builder.CreateOr(ValOperand_Shifted, PMV.Inv_Mask, "AndOperand");
996 AtomicRMWInst *NewAI = Builder.CreateAtomicRMW(
1002 Value *FinalOldResult = extractMaskedValue(Builder, NewAI, PMV);
1050 ReplacementIRBuilder Builder(CI, *DL);
1051 LLVMContext &Ctx = Builder.getContext();
1062 Builder.SetInsertPoint(BB);
1065 createMaskInstrs(Builder, CI, CI->getCompareOperand()->getType(), Addr,
1070 Builder.CreateShl(Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt);
1072 Builder.CreateShl(Builder.CreateZExt(Cmp, PMV.WordType), PMV.ShiftAmt);
1076 LoadInst *InitLoaded = Builder.CreateLoad(PMV.WordType, PMV.AlignedAddr);
1078 Value *InitLoaded_MaskOut = Builder.CreateAnd(InitLoaded, PMV.Inv_Mask);
1079 Builder.CreateBr(LoopBB);
1082 Builder.SetInsertPoint(LoopBB);
1083 PHINode *Loaded_MaskOut = Builder.CreatePHI(PMV.WordType, 2);
1087 Value *FullWord_NewVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Shifted);
1088 Value *FullWord_Cmp = Builder.CreateOr(Loaded_MaskOut, Cmp_Shifted);
1089 AtomicCmpXchgInst *NewCI = Builder.CreateAtomicCmpXchg(
1100 Value *OldVal = Builder.CreateExtractValue(NewCI, 0);
1101 Value *Success = Builder.CreateExtractValue(NewCI, 1);
1104 Builder.CreateBr(EndBB);
1106 Builder.CreateCondBr(Success, EndBB, FailureBB);
1109 Builder.SetInsertPoint(FailureBB);
1113 Value *OldVal_MaskOut = Builder.CreateAnd(OldVal, PMV.Inv_Mask);
1114 Value *ShouldContinue = Builder.CreateICmpNE(Loaded_MaskOut, OldVal_MaskOut);
1115 Builder.CreateCondBr(ShouldContinue, LoopBB, EndBB);
1121 Builder.SetInsertPoint(CI);
1123 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV);
1125 Res = Builder.CreateInsertValue(Res, FinalOldVal, 0);
1126 Res = Builder.CreateInsertValue(Res, Success, 1);
1137 ReplacementIRBuilder Builder(I, *DL);
1138 Value *Loaded = insertRMWLLSCLoop(Builder, ResultType, Addr, AddrAlign,
1146 ReplacementIRBuilder Builder(AI, *DL);
1149 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
1160 Value *ValOperand_Shifted = Builder.CreateShl(
1161 Builder.CreateCast(CastOp, AI->getValOperand(), PMV.WordType),
1164 Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt,
1166 Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV);
1173 ReplacementIRBuilder Builder(CI, *DL);
1176 Builder, CI, CI->getCompareOperand()->getType(), CI->getPointerOperand(),
1179 Value *CmpVal_Shifted = Builder.CreateShl(
1180 Builder.CreateZExt(CI->getCompareOperand(), PMV.WordType), PMV.ShiftAmt,
1182 Value *NewVal_Shifted = Builder.CreateShl(
1183 Builder.CreateZExt(CI->getNewValOperand(), PMV.WordType), PMV.ShiftAmt,
1186 Builder, CI, PMV.AlignedAddr, CmpVal_Shifted, NewVal_Shifted, PMV.Mask,
1188 Value *FinalOldVal = extractMaskedValue(Builder, OldVal, PMV);
1190 Res = Builder.CreateInsertValue(Res, FinalOldVal, 0);
1191 Value *Success = Builder.CreateICmpEQ(
1192 CmpVal_Shifted, Builder.CreateAnd(OldVal, PMV.Mask), "Success");
1193 Res = Builder.CreateInsertValue(Res, Success, 1);
1200 IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
1203 LLVMContext &Ctx = Builder.getContext();
1204 BasicBlock *BB = Builder.GetInsertBlock();
1224 BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end");
1230 Builder.SetInsertPoint(BB);
1231 Builder.CreateBr(LoopBB);
1234 Builder.SetInsertPoint(LoopBB);
1235 Value *Loaded = TLI->emitLoadLinked(Builder, ResultTy, Addr, MemOpOrder);
1237 Value *NewVal = PerformOp(Builder, Loaded);
1240 TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder);
1241 Value *TryAgain = Builder.CreateICmpNE(
1243 Builder.CreateCondBr(TryAgain, LoopBB, ExitBB);
1245 Builder.SetInsertPoint(ExitBB, ExitBB->begin());
1260 ReplacementIRBuilder Builder(CI, *DL);
1264 Value *NewCmp = Builder.CreatePtrToInt(CI->getCompareOperand(), NewTy);
1265 Value *NewNewVal = Builder.CreatePtrToInt(CI->getNewValOperand(), NewTy);
1267 auto *NewCI = Builder.CreateAtomicCmpXchg(
1274 Value *OldVal = Builder.CreateExtractValue(NewCI, 0);
1275 Value *Succ = Builder.CreateExtractValue(NewCI, 1);
1277 OldVal = Builder.CreateIntToPtr(OldVal, CI->getCompareOperand()->getType());
1280 Res = Builder.CreateInsertValue(Res, OldVal, 0);
1281 Res = Builder.CreateInsertValue(Res, Succ, 1);
1382 ReplacementIRBuilder Builder(CI, *DL);
1388 Builder.SetInsertPoint(BB);
1390 TLI->emitLeadingFence(Builder, CI, SuccessOrder);
1393 createMaskInstrs(Builder, CI, CI->getCompareOperand()->getType(), Addr,
1395 Builder.CreateBr(StartBB);
1398 Builder.SetInsertPoint(StartBB);
1400 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder);
1402 extractMaskedValue(Builder, UnreleasedLoad, PMV);
1403 Value *ShouldStore = Builder.CreateICmpEQ(
1408 Builder.CreateCondBr(ShouldStore, ReleasingStoreBB, NoStoreBB);
1410 Builder.SetInsertPoint(ReleasingStoreBB);
1412 TLI->emitLeadingFence(Builder, CI, SuccessOrder);
1413 Builder.CreateBr(TryStoreBB);
1415 Builder.SetInsertPoint(TryStoreBB);
1417 Builder.CreatePHI(PMV.WordType, 2, "loaded.trystore");
1420 insertMaskedValue(Builder, LoadedTryStore, CI->getNewValOperand(), PMV);
1421 Value *StoreSuccess = TLI->emitStoreConditional(Builder, NewValueInsert,
1423 StoreSuccess = Builder.CreateICmpEQ(
1426 Builder.CreateCondBr(StoreSuccess, SuccessBB,
1429 Builder.SetInsertPoint(ReleasedLoadBB);
1433 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder);
1434 Value *SecondLoadExtract = extractMaskedValue(Builder, SecondLoad, PMV);
1435 ShouldStore = Builder.CreateICmpEQ(SecondLoadExtract,
1440 Builder.CreateCondBr(ShouldStore, TryStoreBB, NoStoreBB);
1444 Builder.CreateUnreachable();
1448 Builder.SetInsertPoint(SuccessBB);
1451 TLI->emitTrailingFence(Builder, CI, SuccessOrder);
1452 Builder.CreateBr(ExitBB);
1454 Builder.SetInsertPoint(NoStoreBB);
1456 Builder.CreatePHI(UnreleasedLoad->getType(), 2, "loaded.nostore");
1464 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
1465 Builder.CreateBr(FailureBB);
1467 Builder.SetInsertPoint(FailureBB);
1469 Builder.CreatePHI(UnreleasedLoad->getType(), 2, "loaded.failure");
1474 TLI->emitTrailingFence(Builder, CI, FailureOrder);
1475 Builder.CreateBr(ExitBB);
1481 Builder.SetInsertPoint(ExitBB, ExitBB->begin());
1483 Builder.CreatePHI(UnreleasedLoad->getType(), 2, "loaded.exit");
1486 PHINode *Success = Builder.CreatePHI(Type::getInt1Ty(Ctx), 2, "success");
1494 Builder.SetInsertPoint(ExitBB, std::next(Success->getIterator()));
1495 Value *Loaded = extractMaskedValue(Builder, LoadedFull, PMV);
1524 Res = Builder.CreateInsertValue(PoisonValue::get(CI->getType()), Loaded, 0);
1525 Res = Builder.CreateInsertValue(Res, Success, 1);
1563 IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign,
1567 LLVMContext &Ctx = Builder.getContext();
1568 BasicBlock *BB = Builder.GetInsertBlock();
1587 BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end");
1594 Builder.SetInsertPoint(BB);
1595 LoadInst *InitLoaded = Builder.CreateAlignedLoad(ResultTy, Addr, AddrAlign);
1596 Builder.CreateBr(LoopBB);
1599 Builder.SetInsertPoint(LoopBB);
1600 PHINode *Loaded = Builder.CreatePHI(ResultTy, 2, "loaded");
1603 Value *NewVal = PerformOp(Builder, Loaded);
1608 CreateCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign,
1617 Builder.CreateCondBr(Success, ExitBB, LoopBB);
1619 Builder.SetInsertPoint(ExitBB, ExitBB->begin());
1648 ReplacementIRBuilder Builder(AI, AI->getDataLayout());
1649 Builder.setIsFPConstrained(
1655 Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(),
1657 [&](IRBuilderBase &Builder, Value *Loaded) {
1658 return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded,
1809 I, [this](IRBuilderBase &Builder, Value *Addr, Value *Loaded,
1813 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
1816 Success = Builder.CreateExtractValue(Pair, 1, "success");
1817 NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
1840 IRBuilder<> Builder(I);
1940 PtrVal = Builder.CreateAddrSpaceCast(PtrVal, PointerType::getUnqual(Ctx));
1947 Builder.CreateLifetimeStart(AllocaCASExpected, SizeVal64);
1948 Builder.CreateAlignedStore(CASExpected, AllocaCASExpected, AllocaAlignment);
1956 Builder.CreateBitOrPointerCast(ValueOperand, SizedIntTy);
1961 Builder.CreateLifetimeStart(AllocaValue, SizeVal64);
1962 Builder.CreateAlignedStore(ValueOperand, AllocaValue, AllocaAlignment);
1971 Builder.CreateLifetimeStart(AllocaResult, SizeVal64);
1998 CallInst *Call = Builder.CreateCall(LibcallFn, Args);
2004 Builder.CreateLifetimeEnd(AllocaValue, SizeVal64);
2011 Value *ExpectedOut = Builder.CreateAlignedLoad(
2013 Builder.CreateLifetimeEnd(AllocaCASExpected, SizeVal64);
2014 V = Builder.CreateInsertValue(V, ExpectedOut, 0);
2015 V = Builder.CreateInsertValue(V, Result, 1);
2020 V = Builder.CreateBitOrPointerCast(Result, I->getType());
2022 V = Builder.CreateAlignedLoad(I->getType(), AllocaResult,
2024 Builder.CreateLifetimeEnd(AllocaResult, SizeVal64);