Lines Matching defs:SI
231 isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG);
735 iterator SI, SJ;
742 Partition(iterator SI) : SI(SI), SJ(SI) {}
765 bool empty() const { return SI == SJ; }
776 iterator begin() const { return SI; }
817 partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE)
818 : P(SI), SE(SE) {
821 if (SI != SE)
829 assert((P.SI != SE || !P.SplitTails.empty()) &&
857 // If P.SI is already at the end, then we've cleared the split tail and
859 if (P.SI == SE) {
866 if (P.SI != P.SJ) {
877 P.SI = P.SJ;
879 // If P.SI is now at the end, we at most have a tail of split slices.
880 if (P.SI == SE) {
889 if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset &&
890 !P.SI->isSplittable()) {
892 P.EndOffset = P.SI->beginOffset();
902 P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset;
903 P.EndOffset = P.SI->endOffset();
908 if (!P.SI->isSplittable()) {
911 assert(P.BeginOffset == P.SI->beginOffset());
929 assert(P.SI->isSplittable() && "Forming a splittable partition!");
952 // The observed positions of partitions is marked by the P.SI iterator and
954 // P.SI == SE, as the end iterator will additionally have an empty split
955 // slices list, but the prior may have the same P.SI and a tail of split
957 if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) {
988 static Value *foldSelectInst(SelectInst &SI) {
992 if (ConstantInt *CI = dyn_cast<ConstantInt>(SI.getCondition()))
993 return SI.getOperand(1 + CI->isZero());
994 if (SI.getOperand(1) == SI.getOperand(2))
995 return SI.getOperand(1);
1123 void visitStoreInst(StoreInst &SI) {
1124 Value *ValOp = SI.getValueOperand();
1126 return PI.setEscapedAndAborted(&SI);
1128 return PI.setAborted(&SI);
1132 return PI.setAborted(&SI);
1148 << " use: " << SI << "\n");
1149 return markAsDead(SI);
1152 assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) &&
1154 handleLoadOrStore(ValOp->getType(), SI, Offset, Size, SI.isVolatile());
1300 if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
1301 Value *Op = SI->getOperand(0);
1303 return SI;
1306 PI.setAborted(SI);
1389 void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(SI); }
1481 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) {
1482 UserTy = SI->getValueOperand()->getType();
1684 isSafeLoadOfSelectToSpeculate(LoadInst &LI, SelectInst &SI, bool PreserveCFG) {
1688 const DataLayout &DL = SI.getDataLayout();
1689 for (Value *Value : {SI.getTrueValue(), SI.getFalseValue()})
1692 Spec.setAsSpeculatable(/*isTrueVal=*/Value == SI.getTrueValue());
1700 SROA::isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG) {
1703 for (User *U : SI.users()) {
1735 isSafeLoadOfSelectToSpeculate(*LI, SI, PreserveCFG);
1746 static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI,
1748 LLVM_DEBUG(dbgs() << " original load: " << SI << "\n");
1750 Value *TV = SI.getTrueValue();
1751 Value *FV = SI.getFalseValue();
1776 Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL,
1784 static void rewriteMemOpOfSelect(SelectInst &SI, T &I,
1793 SplitBlockAndInsertIfThenElse(SI.getCondition(), &I, &ThenTerm, &ElseTerm,
1794 SI.getMetadata(LLVMContext::MD_prof), &DTU);
1796 SplitBlockAndInsertIfThen(SI.getCondition(), &I, /*Unreachable=*/false,
1797 SI.getMetadata(LLVMContext::MD_prof), &DTU,
1825 Value *Ptr = SI.getOperand(1 + SuccIdx);
1845 else if (auto *SI = dyn_cast<StoreInst>(&I))
1846 rewriteMemOpOfSelect(SelInst, *SI, Spec, DTU);
1851 static bool rewriteSelectInstMemOps(SelectInst &SI,
1855 LLVM_DEBUG(dbgs() << " original select: " << SI << "\n");
1868 speculateSelectInstLoads(SI, cast<LoadInst>(*I), IRB);
1871 rewriteMemOpOfSelect(SI, *I, Spec, *DTU);
1877 for (User *U : make_early_inc_range(SI.users()))
1879 SI.eraseFromParent();
2071 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) {
2072 if (SI->isVolatile())
2074 Type *STy = SI->getValueOperand()->getType();
2300 else if (auto *SI = dyn_cast<StoreInst>(S.getUse()->getUser()))
2301 Ty = SI->getValueOperand()->getType();
2386 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) {
2387 Type *ValueTy = SI->getValueOperand()->getType();
2388 if (SI->isVolatile())
2968 bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp,
2992 Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
2997 Pass.DeadInsts.push_back(&SI);
3000 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI,
3006 bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) {
3008 assert(!SI.isVolatile());
3016 V = insertInteger(DL, IRB, Old, SI.getValueOperand(), Offset, "insert");
3020 Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
3026 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI,
3030 Pass.DeadInsts.push_back(&SI);
3035 bool visitStoreInst(StoreInst &SI) {
3036 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
3037 Value *OldOp = SI.getOperand(1);
3040 AAMDNodes AATags = SI.getAAMetadata();
3041 Value *V = SI.getValueOperand();
3050 assert(!SI.isVolatile());
3055 IntegerType *NarrowTy = Type::getIntNTy(SI.getContext(), SliceSize * 8);
3061 return rewriteVectorizedStoreInst(V, SI, OldOp, AATags);
3063 return rewriteIntegerStore(V, SI, AATags);
3071 getPtrToNewAI(SI.getPointerAddressSpace(), SI.isVolatile());
3074 IRB.CreateAlignedStore(V, NewPtr, NewAI.getAlign(), SI.isVolatile());
3076 unsigned AS = SI.getPointerAddressSpace();
3079 IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(), SI.isVolatile());
3081 NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
3086 if (SI.isVolatile())
3087 NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID());
3089 NewSI->setAlignment(SI.getAlign());
3091 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI,
3095 Pass.DeadInsts.push_back(&SI);
3101 !SI.isVolatile();
3582 if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
3583 SI->setAlignment(std::min(SI->getAlign(), getSliceAlign()));
3630 bool visitSelectInst(SelectInst &SI) {
3631 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
3632 assert((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) &&
3639 if (SI.getOperand(1) == OldPtr)
3640 SI.setOperand(1, NewPtr);
3641 if (SI.getOperand(2) == OldPtr)
3642 SI.setOperand(2, NewPtr);
3644 LLVM_DEBUG(dbgs() << " to: " << SI << "\n");
3648 fixLoadStoreAlign(SI);
3653 SelectUsers.insert(&SI);
3906 bool visitStoreInst(StoreInst &SI) {
3907 if (!SI.isSimple() || SI.getPointerOperand() != *U)
3909 Value *V = SI.getValueOperand();
3914 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
3915 StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI,
3916 getAdjustedAlignment(&SI, 0), DL, IRB);
3918 Visited.erase(&SI);
3919 // The stores replacing SI each have markers describing fragments of the
3920 // assignment so delete the assignment markers linked to SI.
3921 at::deleteAssignmentMarkers(&SI);
3922 SI.eraseFromParent();
3945 if (auto *SI = dyn_cast<SelectInst>(Op)) {
3949 Sel = SI;
4035 if (auto *SI = dyn_cast<PHINode>(Op)) {
4039 Phi = SI;
4121 bool visitSelectInst(SelectInst &SI) {
4122 enqueueUsers(SI);
4348 else if (auto *SI = dyn_cast<StoreInst>(I))
4349 if (auto *LI = dyn_cast<LoadInst>(SI->getValueOperand()))
4365 auto *SI = dyn_cast<StoreInst>(LU);
4366 if (!SI || !SI->isSimple())
4377 } else if (auto *SI = dyn_cast<StoreInst>(I)) {
4378 if (S.getUse() != &SI->getOperandUse(SI->getPointerOperandIndex()))
4381 auto *StoredLoad = dyn_cast<LoadInst>(SI->getValueOperand());
4384 assert(!SI->isVolatile() && "Cannot split volatile stores!");
4386 Stores.push_back(SI);
4428 llvm::erase_if(Stores, [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) {
4431 auto *LI = cast<LoadInst>(SI->getValueOperand());
4443 auto &StoreOffsets = SplitOffsetsMap[SI];
4453 << " " << *SI << "\n");
4466 llvm::erase_if(Stores, [&UnsplittableLoads](StoreInst *SI) {
4467 auto *LI = cast<LoadInst>(SI->getValueOperand());
4566 StoreInst *SI = cast<StoreInst>(LU);
4567 if (!Stores.empty() && SplitOffsetsMap.count(SI)) {
4569 LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SI
4574 Value *StoreBasePtr = SI->getPointerOperand();
4575 IRB.SetInsertPoint(SI);
4576 AAMDNodes AATags = SI->getAAMetadata();
4578 LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n");
4583 auto *PartPtrTy = SI->getPointerOperandType();
4585 auto AS = SI->getPointerAddressSpace();
4591 getAdjustedAlignment(SI, PartOffset),
4593 PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access,
4616 DeadInsts.push_back(SI);
4633 for (StoreInst *SI : Stores) {
4634 auto *LI = cast<LoadInst>(SI->getValueOperand());
4640 auto &Offsets = SplitOffsetsMap[SI];
4648 Instruction *StoreBasePtr = cast<Instruction>(SI->getPointerOperand());
4650 LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n");
4668 auto *StorePartPtrTy = SI->getPointerOperandType();
4689 IRB.SetInsertPoint(SI);
4690 auto AS = SI->getPointerAddressSpace();
4696 getAdjustedAlignment(SI, PartOffset),
4698 PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access,
4750 assert(*LI->user_begin() == SI && "Single use isn't this store!");
4753 DeadInsts.push_back(SI);