Lines Matching +full:push +full:- +full:ci +full:- +full:container
1 //===- Local.cpp - Functions to perform local transformations -------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
12 //===----------------------------------------------------------------------===//
99 "phicse-debug-hash",
107 "function is well-behaved w.r.t. its isEqual predicate"));
110 "phicse-num-phi-smallsize", cl::init(32), cl::Hidden,
113 "perform a (faster!) exhaustive search instead of set-driven one."));
116 "max-phi-entries-increase-after-removing-empty-block", cl::init(1000),
125 //===----------------------------------------------------------------------===//
129 /// ConstantFoldTerminator - If a terminator instruction is predicated on a
139 Instruction *T = BB->getTerminator();
142 // Branch - See if we are conditional jumping on constant
144 if (BI->isUnconditional()) return false; // Can't optimize uncond branch
146 BasicBlock *Dest1 = BI->getSuccessor(0);
147 BasicBlock *Dest2 = BI->getSuccessor(1);
155 assert(BI->getParent() && "Terminator not inserted in block!");
156 Dest1->removePredecessor(BI->getParent());
162 NewBI->copyMetadata(*BI, {LLVMContext::MD_loop, LLVMContext::MD_dbg,
165 Value *Cond = BI->getCondition();
166 BI->eraseFromParent();
172 if (auto *Cond = dyn_cast<ConstantInt>(BI->getCondition())) {
175 BasicBlock *Destination = Cond->getZExtValue() ? Dest1 : Dest2;
176 BasicBlock *OldDest = Cond->getZExtValue() ? Dest2 : Dest1;
180 OldDest->removePredecessor(BB);
186 NewBI->copyMetadata(*BI, {LLVMContext::MD_loop, LLVMContext::MD_dbg,
189 BI->eraseFromParent();
191 DTU->applyUpdates({{DominatorTree::Delete, BB, OldDest}});
201 auto *CI = dyn_cast<ConstantInt>(SI->getCondition());
202 BasicBlock *DefaultDest = SI->getDefaultDest();
206 if (isa<UnreachableInst>(DefaultDest->getFirstNonPHIOrDbg()) &&
207 SI->getNumCases() > 0) {
208 TheOnlyDest = SI->case_begin()->getCaseSuccessor();
214 for (auto It = SI->case_begin(), End = SI->case_end(); It != End;) {
216 if (It->getCaseValue() == CI) {
217 TheOnlyDest = It->getCaseSuccessor();
223 if (It->getCaseSuccessor() == DefaultDest) {
225 unsigned NCases = SI->getNumCases();
234 unsigned Idx = It->getCaseIndex();
243 BasicBlock *ParentBB = SI->getParent();
244 DefaultDest->removePredecessor(ParentBB);
245 It = SI->removeCase(It);
246 End = SI->case_end();
249 // case, update CI and restart iteration through the cases.
250 if (auto *NewCI = dyn_cast<ConstantInt>(SI->getCondition())) {
251 CI = NewCI;
252 It = SI->case_begin();
260 // We do this by reseting "TheOnlyDest" to null when we find two non-equal
262 if (It->getCaseSuccessor() != TheOnlyDest)
269 if (CI && !TheOnlyDest) {
272 TheOnlyDest = SI->getDefaultDest();
280 BasicBlock *BB = SI->getParent();
293 Succ->removePredecessor(BB);
298 Value *Cond = SI->getCondition();
299 SI->eraseFromParent();
307 DTU->applyUpdates(Updates);
312 if (SI->getNumCases() == 1) {
314 // instruction if it has only one non-default destination.
315 auto FirstCase = *SI->case_begin();
316 Value *Cond = Builder.CreateICmpEQ(SI->getCondition(),
322 SI->getDefaultDest());
328 NewBr->setMetadata(LLVMContext::MD_prof,
329 MDBuilder(BB->getContext())
333 // Update make.implicit metadata to the newly-created conditional branch.
334 MDNode *MakeImplicitMD = SI->getMetadata(LLVMContext::MD_make_implicit);
336 NewBr->setMetadata(LLVMContext::MD_make_implicit, MakeImplicitMD);
339 SI->eraseFromParent();
346 // indirectbr blockaddress(@F, @BB) -> br label @BB
348 dyn_cast<BlockAddress>(IBI->getAddress()->stripPointerCasts())) {
349 BasicBlock *TheOnlyDest = BA->getBasicBlock();
356 for (unsigned i = 0, e = IBI->getNumDestinations(); i != e; ++i) {
357 BasicBlock *DestBB = IBI->getDestination(i);
360 if (IBI->getDestination(i) == SuccToKeep) {
363 DestBB->removePredecessor(BB);
366 Value *Address = IBI->getAddress();
367 IBI->eraseFromParent();
374 if (BA->use_empty())
375 BA->destroyConstant();
381 BB->getTerminator()->eraseFromParent();
382 new UnreachableInst(BB->getContext(), BB);
390 DTU->applyUpdates(Updates);
399 //===----------------------------------------------------------------------===//
403 /// isInstructionTriviallyDead - Return true if the result produced by the
408 if (!I->use_empty())
418 if (II->getIntrinsicID() == Intrinsic::stacksave ||
419 II->getIntrinsicID() == Intrinsic::launder_invariant_group ||
420 II->isLifetimeStartOrEnd())
427 if (I->isTerminator())
430 // We don't want the landingpad-like instructions removed by anything this
432 if (I->isEHPad())
440 if (DLI->getLabel())
449 if (!I->willReturn()) {
454 switch (II->getIntrinsicID()) {
456 // Guards on true are operationally no-ops. In the future we can
459 auto *Cond = dyn_cast<ConstantInt>(II->getArgOperand(0));
460 return Cond && Cond->isOne();
463 // a well-defined trap.
474 if (!I->mayHaveSideEffects())
481 if (II->getIntrinsicID() == Intrinsic::stacksave ||
482 II->getIntrinsicID() == Intrinsic::launder_invariant_group)
487 if (II->getIntrinsicID() == Intrinsic::allow_runtime_check ||
488 II->getIntrinsicID() == Intrinsic::allow_ubsan_check)
491 if (II->isLifetimeStartOrEnd()) {
492 auto *Arg = II->getArgOperand(1);
493 // Lifetime intrinsics are dead when their right-hand is undef.
496 // If the right-hand is an alloc, global, or argument and the only uses
499 return llvm::all_of(Arg->uses(), [](Use &Use) {
502 return IntrinsicUse->isLifetimeStartOrEnd();
509 if (II->getIntrinsicID() == Intrinsic::assume &&
511 if (ConstantInt *Cond = dyn_cast<ConstantInt>(II->getArgOperand(0)))
512 return !Cond->isZero();
519 FPI->getExceptionBehavior();
527 return C->isNullValue() || isa<UndefValue>(C);
532 // Non-volatile atomic loads from constants can be removed.
535 LI->getPointerOperand()->stripPointerCasts()))
536 if (!LI->isVolatile() && GV->isConstant())
542 /// RecursivelyDeleteTriviallyDeadInstructions - If the specified value is a
592 assert(I->use_empty() && "Instructions with uses are not dead.");
602 for (Use &OpU : I->operands()) {
606 if (!OpV->use_empty())
617 MSSAU->removeMemoryAccess(I);
619 I->eraseFromParent();
628 DII->setKillLocation();
630 DVR->setKillLocation();
634 /// areAllUsesEqual - Check whether the uses of a value are all the same.
639 Value::user_iterator UI = I->user_begin();
640 Value::user_iterator UE = I->user_end();
652 /// RecursivelyDeleteDeadPHINode - If the specified value is an effectively
653 /// dead PHI node, due to being a def-use chain of single-use nodes that
661 for (Instruction *I = PN; areAllUsesEqual(I) && !I->mayHaveSideEffects();
662 I = cast<Instruction>(*I->user_begin())) {
663 if (I->use_empty())
670 I->replaceAllUsesWith(PoisonValue::get(I->getType()));
688 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
689 Value *OpV = I->getOperand(i);
690 I->setOperand(i, nullptr);
692 if (!OpV->use_empty() || I == OpV)
703 I->eraseFromParent();
711 for (User *U : I->users()) {
719 if (!I->use_empty()) {
720 I->replaceAllUsesWith(SimpleV);
724 I->eraseFromParent();
732 /// SimplifyInstructionsInBlock - Scan the specified basic block and try to
740 const DataLayout &DL = BB->getDataLayout();
747 AssertingVH<Instruction> TerminatorVH(&BB->back());
752 // if they actually need to be revisited. This avoids having to pre-init
754 for (BasicBlock::iterator BI = BB->begin(), E = std::prev(BB->end());
756 assert(!BI->isTerminator());
773 //===----------------------------------------------------------------------===//
780 // If BB has single-entry PHI nodes, fold them.
781 while (PHINode *PN = dyn_cast<PHINode>(DestBB->begin())) {
782 Value *NewVal = PN->getIncomingValue(0);
784 if (NewVal == PN) NewVal = PoisonValue::get(PN->getType());
785 PN->replaceAllUsesWith(NewVal);
786 PN->eraseFromParent();
789 BasicBlock *PredBB = DestBB->getSinglePredecessor();
792 bool ReplaceEntryBB = PredBB->isEntryBlock();
816 if (DestBB->hasAddressTaken()) {
819 ConstantInt::get(Type::getInt32Ty(BA->getContext()), 1);
820 BA->replaceAllUsesWith(ConstantExpr::getIntToPtr(Replacement,
821 BA->getType()));
822 BA->destroyConstant();
826 PredBB->replaceAllUsesWith(DestBB);
829 PredBB->getTerminator()->eraseFromParent();
830 DestBB->splice(DestBB->begin(), PredBB);
831 new UnreachableInst(PredBB->getContext(), PredBB);
836 DestBB->moveAfter(PredBB);
839 assert(PredBB->size() == 1 &&
840 isa<UnreachableInst>(PredBB->getTerminator()) &&
843 DTU->applyUpdatesPermissive(Updates);
844 DTU->deleteBB(PredBB);
847 if (ReplaceEntryBB && DTU->hasDomTree()) {
849 // the dominator tree to be notified of this change. In this corner-case
851 DTU->recalculate(*(DestBB->getParent()));
856 PredBB->eraseFromParent(); // Nuke BB if DTU is nullptr.
861 /// other. Note that we will always choose the non-undef value to keep.
866 /// Return true if we can fold BB, an almost-empty BB ending in an unconditional
875 LLVM_DEBUG(dbgs() << "Looking to fold " << BB->getName() << " into "
876 << Succ->getName() << "\n");
879 if (Succ->getSinglePredecessor())
884 for (BasicBlock::iterator I = Succ->begin(); isa<PHINode>(I); ++I) {
890 PHINode *BBPN = dyn_cast<PHINode>(PN->getIncomingValueForBlock(BB));
891 if (BBPN && BBPN->getParent() == BB) {
892 for (unsigned PI = 0, PE = PN->getNumIncomingValues(); PI != PE; ++PI) {
893 BasicBlock *IBB = PN->getIncomingBlock(PI);
895 !CanMergeValues(BBPN->getIncomingValueForBlock(IBB),
896 PN->getIncomingValue(PI))) {
898 << "Can't fold, phi node " << PN->getName() << " in "
899 << Succ->getName() << " is conflicting with "
900 << BBPN->getName() << " with regard to common predecessor "
901 << IBB->getName() << "\n");
906 Value* Val = PN->getIncomingValueForBlock(BB);
907 for (unsigned PI = 0, PE = PN->getNumIncomingValues(); PI != PE; ++PI) {
911 BasicBlock *IBB = PN->getIncomingBlock(PI);
913 !CanMergeValues(Val, PN->getIncomingValue(PI))) {
914 LLVM_DEBUG(dbgs() << "Can't fold, phi node " << PN->getName()
915 << " in " << Succ->getName()
917 << "predecessor " << IBB->getName() << "\n");
938 /// \param IncomingValues A map from block-to-value for other phi inputs
946 IncomingValues.find(BB)->second == OldVal) &&
954 if (It != IncomingValues.end()) return It->second;
962 /// Create a map from block to value for each non-undef value flowing
969 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
970 BasicBlock *BB = PN->getIncomingBlock(i);
971 Value *V = PN->getIncomingValue(i);
979 /// from a block-to-value map.
986 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
987 Value *V = PN->getIncomingValue(i);
991 BasicBlock *BB = PN->getIncomingBlock(i);
1005 PN->setIncomingValue(i, It->second);
1012 return isa<PoisonValue>(PN->getIncomingValue(i));
1016 PN->setIncomingValue(i, UndefValue::get(PN->getType()));
1029 if (BB->phis().empty() || Succ->phis().empty())
1033 if (!BB->hasNPredecessorsOrMore(2))
1037 return isa<IndirectBrInst>(Pred->getTerminator());
1060 if (BB->hasNPredecessors(1))
1064 for (auto &Phi : Succ->phis()) {
1068 if (IncomingPhi->getParent() == BB)
1073 // For every phi that needs to be changed, (NumPreds - 1) new entries will be
1077 return (NumPreds - 1) * NumChangedPhi >
1093 Value *OldVal = PN->removeIncomingValue(BB, false);
1098 // We are merging two blocks - BB, and the block containing PN - and
1105 // consistent with the non-undef values.
1111 if (isa<PHINode>(OldVal) && cast<PHINode>(OldVal)->getParent() == BB) {
1113 for (unsigned i = 0, e = OldValPN->getNumIncomingValues(); i != e; ++i) {
1119 BasicBlock *PredBB = OldValPN->getIncomingBlock(i);
1124 Value *PredVal = OldValPN->getIncomingValue(i);
1130 PN->addIncoming(Selected, PredBB);
1133 PN->addIncoming(OldValPN->getIncomingValueForBlock(CommonPred), BB);
1147 PN->addIncoming(Selected, PredBB);
1150 PN->addIncoming(OldVal, BB);
1158 assert(BB != &BB->getParent()->getEntryBlock() &&
1162 BasicBlock *Succ = cast<BranchInst>(BB->getTerminator())->getSuccessor(0);
1187 // BB dominates Succ, which is non-trivial to calculate in the case where
1189 // constructing the necessary self-referential PHI node doesn't introduce any
1194 // something like a loop pre-header (or rarely, a part of an irreducible CFG);
1196 if (!Succ->getSinglePredecessor()) {
1197 BasicBlock::iterator BBI = BB->begin();
1199 for (Use &U : BBI->uses()) {
1201 if (PN->getIncomingBlock(U) != BB)
1212 LLVM_DEBUG(dbgs() << "Found Common Predecessor between: " << BB->getName()
1213 << " and " << Succ->getName() << " : "
1214 << CommonPred->getName() << "\n");
1216 // 'BB' and 'BB->Pred' are loop latches, bail out to presrve inner loop
1219 // FIXME: This is a stop-gap solution to preserve inner-loop metadata given
1230 // In this case, we expect 'BB' is the latch for outer-loop and 'BB->Pred' is
1231 // the latch for inner-loop (see reason below), so bail out to prerserve
1232 // inner-loop metadata rather than eliminating 'BB' and attaching its metadata
1233 // to this inner-loop.
1234 // - The reason we believe 'BB' and 'BB->Pred' have different inner-most
1235 // loops: assuming 'BB' and 'BB->Pred' are from the same inner-most loop L,
1237 // one self-looping basic block, which is contradictory with the assumption.
1239 // To illustrate how inner-loop metadata is dropped:
1244 // BB->Pred is for.body, attached with loop metadata md1.
1249 // ---> while.cond -------------> while.end
1255 // | for.body <---- (md1)
1267 // dedicated exits for inner-loop (essentially adding `while.cond.exit`
1268 // back), but won't it won't see 'md1' nor restore it for the inner-loop.
1273 // ---> while.cond1 -------------> while.end
1279 // | for.body <---- (md2)
1281 if (Instruction *TI = BB->getTerminator())
1282 if (TI->hasNonDebugLocLoopMetadata())
1284 if (Instruction *PredTI = Pred->getTerminator())
1285 if (PredTI->hasNonDebugLocLoopMetadata())
1321 if (isa<PHINode>(Succ->begin())) {
1328 for (BasicBlock::iterator I = Succ->begin(); isa<PHINode>(I); ++I) {
1334 if (Succ->getSinglePredecessor()) {
1338 BB->getTerminator()->eraseFromParent();
1339 Succ->splice(Succ->getFirstNonPHIIt(), BB);
1341 while (PHINode *PN = dyn_cast<PHINode>(&BB->front())) {
1343 assert(PN->use_empty() && "There shouldn't be any uses here!");
1344 PN->eraseFromParent();
1348 // If the unconditional branch we replaced contains non-debug llvm.loop
1351 if (Instruction *TI = BB->getTerminator())
1352 if (TI->hasNonDebugLocLoopMetadata()) {
1353 MDNode *LoopMD = TI->getMetadata(LLVMContext::MD_loop);
1355 Pred->getTerminator()->setMetadata(LLVMContext::MD_loop, LoopMD);
1360 BB->replaceAllUsesWith(Succ);
1362 if (!Succ->hasName())
1363 Succ->takeName(BB);
1366 if (BB->getTerminator())
1367 BB->back().eraseFromParent();
1369 new UnreachableInst(BB->getContext(), BB);
1374 BB->replaceUsesWithIf(Succ, [BBPreds, CommonPred](Use &U) -> bool {
1376 return UseInst->getParent() != CommonPred &&
1377 BBPreds.contains(UseInst->getParent());
1383 DTU->applyUpdates(Updates);
1403 for (auto I = BB->begin(); PHINode *PN = dyn_cast<PHINode>(I);) {
1411 if (!DuplicatePN->isIdenticalToWhenDefined(PN))
1415 DuplicatePN->replaceAllUsesWith(PN);
1420 I = BB->begin();
1454 hash_combine_range(PN->value_op_begin(), PN->value_op_end()),
1455 hash_combine_range(PN->block_begin(), PN->block_end())));
1460 // If -phicse-debug-hash was specified, return a constant -- this
1473 return LHS->isIdenticalTo(RHS);
1492 for (auto I = BB->begin(); PHINode *PN = dyn_cast<PHINode>(I++);) {
1499 PN->replaceAllUsesWith(*Inserted.first);
1506 I = BB->begin();
1519 hasNItemsOrLess(BB->phis(), PHICSENumPHISmallSize))
1528 PN->eraseFromParent();
1534 V = V->stripPointerCasts();
1542 Align CurrentAlign = AI->getAlign();
1551 AI->setAlignment(PrefAlign);
1557 Align CurrentAlign = GO->getPointerAlignment(DL);
1565 if (!GO->canIncreaseAlignment())
1568 if (GO->isThreadLocal()) {
1569 unsigned MaxTLSAlign = GO->getParent()->getMaxTLSAlignment() / CHAR_BIT;
1574 GO->setAlignment(PrefAlign);
1586 assert(V->getType()->isPointerTy() &&
1597 Align Alignment = Align(1ull << std::min(Known.getBitWidth() - 1, TrailZ));
1606 ///===---------------------------------------------------------------------===//
1621 assert(is_contained(DVI->getValues(), APN));
1622 if ((DVI->getVariable() == DIVar) && (DVI->getExpression() == DIExpr))
1626 assert(is_contained(DVR->location_ops(), APN));
1627 if ((DVR->getVariable() == DIVar) && (DVR->getExpression() == DIExpr))
1640 /// covering an n-bit fragment, if the store size of i1 is at least n bits.
1642 const DataLayout &DL = DII->getDataLayout();
1645 DII->getExpression()->getActiveBits(DII->getVariable()))
1651 if (DII->isAddressOfVariable()) {
1653 assert(DII->getNumVariableLocationOps() == 1 &&
1656 dyn_cast_or_null<AllocaInst>(DII->getVariableLocationOp(0))) {
1658 AI->getAllocationSizeInBits(DL)) {
1669 const DataLayout &DL = DVR->getModule()->getDataLayout();
1672 DVR->getExpression()->getActiveBits(DVR->getVariable()))
1678 if (DVR->isAddressOfVariable()) {
1680 assert(DVR->getNumVariableLocationOps() == 1 &&
1683 dyn_cast_or_null<AllocaInst>(DVR->getVariableLocationOp(0))) {
1684 if (std::optional<TypeSize> FragmentSize = AI->getAllocationSizeInBits(DL)) {
1701 cast<Instruction *>(DbgVal)->insertBefore(Instr);
1703 // RemoveDIs: if we're using the new debug-info format, allocate a
1708 Instr->getParent()->insertDbgRecordBefore(DV, Instr);
1718 cast<Instruction *>(DbgVal)->insertAfter(Instr);
1720 // RemoveDIs: if we're using the new debug-info format, allocate a
1725 Instr->getParent()->insertDbgRecordAfter(DV, &*Instr);
1733 assert(DII->isAddressOfVariable() || isa<DbgAssignIntrinsic>(DII));
1734 auto *DIVar = DII->getVariable();
1736 auto *DIExpr = DII->getExpression();
1737 Value *DV = SI->getValueOperand();
1754 DIExpr->isDeref() || (!DIExpr->startsWithDeref() &&
1755 valueCoversEntireFragment(DV->getType(), DII));
1758 SI->getIterator());
1769 DV = PoisonValue::get(DV->getType());
1771 SI->getIterator());
1775 int NumEltDropped = DIExpr->getElements()[0] == dwarf::DW_OP_LLVM_arg ? 3 : 1;
1776 return DIExpression::get(DIExpr->getContext(),
1777 DIExpr->getElements().drop_front(NumEltDropped));
1782 auto *DIVar = DII->getVariable();
1784 auto *DIExpr = DII->getExpression();
1786 Value *DV = SI->getValueOperand();
1791 SI->getIterator());
1798 auto *DIVar = DII->getVariable();
1799 auto *DIExpr = DII->getExpression();
1802 if (!valueCoversEntireFragment(LI->getType(), DII)) {
1814 // future if multi-location support is added to the IR, it might be
1818 LI->getIterator());
1823 assert(DVR->isAddressOfVariable() || DVR->isDbgAssign());
1824 auto *DIVar = DVR->getVariable();
1826 auto *DIExpr = DVR->getExpression();
1827 Value *DV = SI->getValueOperand();
1844 DIExpr->isDeref() || (!DIExpr->startsWithDeref() &&
1845 valueCoversEntireFragment(DV->getType(), DVR));
1848 SI->getIterator());
1861 DV = PoisonValue::get(DV->getType());
1865 SI->getParent()->insertDbgRecordBefore(NewDVR, SI->getIterator());
1870 auto *DIVar = DVR->getVariable();
1872 auto *DIExpr = DVR->getExpression();
1874 Value *DV = SI->getValueOperand();
1879 SI->getIterator());
1886 auto *DIVar = DII->getVariable();
1887 auto *DIExpr = DII->getExpression();
1893 if (!valueCoversEntireFragment(APN->getType(), DII)) {
1902 BasicBlock *BB = APN->getParent();
1903 auto InsertionPt = BB->getFirstInsertionPt();
1910 if (InsertionPt != BB->end()) {
1918 auto *DIVar = DVR->getVariable();
1919 auto *DIExpr = DVR->getExpression();
1922 if (!valueCoversEntireFragment(LI->getType(), DVR)) {
1934 // future if multi-location support is added to the IR, it might be
1943 LI->getParent()->insertDbgRecordAfter(DV, LI);
1948 return AI->isArrayAllocation() ||
1949 (AI->getAllocatedType() && AI->getAllocatedType()->isArrayTy());
1954 return AI->getAllocatedType() && AI->getAllocatedType()->isStructTy();
1958 auto *DIVar = DVR->getVariable();
1959 auto *DIExpr = DVR->getExpression();
1965 if (!valueCoversEntireFragment(APN->getType(), DVR)) {
1974 BasicBlock *BB = APN->getParent();
1975 auto InsertionPt = BB->getFirstInsertionPt();
1982 if (InsertionPt != BB->end()) {
1988 /// LowerDbgDeclare - Lowers llvm.dbg.declare intrinsics into appropriate set
2011 dyn_cast_or_null<AllocaInst>(DDI->getVariableLocationOp(0));
2016 // the stack slot (and at a lexical-scope granularity). Later
2022 if (llvm::any_of(AI->users(), [](User *U) -> bool {
2024 return LI->isVolatile();
2026 return SI->isVolatile();
2035 for (const auto &AIUse : V->uses()) {
2042 } else if (CallInst *CI = dyn_cast<CallInst>(U)) {
2043 // This is a call by-value or some other instruction that takes a
2046 if (!CI->isLifetimeStartOrEnd()) {
2049 DIExpression::append(DDI->getExpression(), dwarf::DW_OP_deref);
2050 insertDbgValueOrDbgVariableRecord(DIB, AI, DDI->getVariable(),
2052 CI->getIterator());
2055 if (BI->getType()->isPointerTy())
2060 DDI->eraseFromParent();
2074 // RemoveDIs: re-implementation of insertDebugValuesForPHIs, but which pulls the
2075 // debug-info out of the block's DbgVariableRecords rather than dbg.value
2109 BasicBlock *Parent = PHI->getParent();
2110 // Avoid inserting a debug-info record into an EH block.
2111 if (Parent->getFirstNonPHIIt()->isEHPad())
2113 for (auto VI : PHI->operand_values()) {
2116 DbgVariableRecord *DbgII = cast<DbgVariableRecord>(V->second);
2119 DbgVariableRecord *NewDbgII = DbgII->clone();
2122 DbgVariableRecord *NewDbgII = NewDI->second;
2125 if (is_contained(NewDbgII->location_ops(), VI))
2126 NewDbgII->replaceVariableLocationOp(VI, PHI);
2134 auto InsertionPt = Parent->getFirstInsertionPt();
2135 assert(InsertionPt != Parent->end() && "Ill-formed basic block");
2137 Parent->insertDbgRecordBefore(NewDbgII, InsertionPt);
2154 for (Value *V : DbgII->location_ops())
2175 BasicBlock *Parent = PHI->getParent();
2177 if (Parent->getFirstNonPHIIt()->isEHPad())
2179 for (auto *VI : PHI->operand_values()) {
2182 auto *DbgII = cast<DbgVariableIntrinsic>(V->second);
2185 NewDI->second = cast<DbgVariableIntrinsic>(DbgII->clone());
2186 DbgVariableIntrinsic *NewDbgII = NewDI->second;
2189 if (is_contained(NewDbgII->location_ops(), VI))
2190 NewDbgII->replaceVariableLocationOp(VI, PHI);
2198 auto InsertionPt = Parent->getFirstInsertionPt();
2199 assert(InsertionPt != Parent->end() && "Ill-formed basic block");
2200 NewDbgII->insertBefore(InsertionPt);
2211 assert(DII->getVariable() && "Missing variable");
2212 auto *DIExpr = DII->getExpression();
2214 DII->setExpression(DIExpr);
2215 DII->replaceVariableLocationOp(Address, NewAddress);
2232 // This is an alloca-based dbg.value/DbgVariableRecord. The first thing it
2235 if (!DIExpr || DIExpr->getNumElements() < 1 ||
2236 DIExpr->getElement(0) != dwarf::DW_OP_deref)
2244 DVI->setExpression(DIExpr);
2245 DVI->replaceVariableLocationOp(0u, NewAddress);
2248 DVR->setExpression(DIExpr);
2249 DVR->replaceVariableLocationOp(0u, NewAddress);
2261 updateOneDbgValueForAlloca(DVI->getDebugLoc(), DVI->getVariable(),
2262 DVI->getExpression(), NewAllocaAddress, DVI,
2267 updateOneDbgValueForAlloca(DVR->getDebugLoc(), DVR->getVariable(),
2268 DVR->getExpression(), NewAllocaAddress, nullptr,
2282 Instruction *I = dyn_cast<Instruction>(Assign->getAddress());
2287 assert(!Assign->getAddressExpression()->getFragmentInfo().has_value() &&
2288 "address-expression shouldn't have fragment info");
2301 Assign->getAddressExpression(), Ops, 0, /*StackValue=*/false);
2302 assert(!SalvagedExpr->getFragmentInfo().has_value() &&
2303 "address-expression shouldn't have fragment info");
2305 SalvagedExpr = SalvagedExpr->foldConstantMath();
2309 Assign->setAddress(NewV);
2310 Assign->setAddressExpression(SalvagedExpr);
2312 Assign->setKillAddress();
2328 if (DAI->getAddress() == &I) {
2332 if (DAI->getValue() != &I)
2339 auto DIILocation = DII->location_ops();
2349 DIExpression *SalvagedExpr = DII->getExpression();
2354 uint64_t CurrentLocOps = SalvagedExpr->getNumLocationOperands();
2367 SalvagedExpr = SalvagedExpr->foldConstantMath();
2368 DII->replaceVariableLocationOp(&I, Op0);
2369 bool IsValidSalvageExpr = SalvagedExpr->getNumElements() <= MaxExpressionSize;
2371 DII->setExpression(SalvagedExpr);
2373 DII->getNumVariableLocationOps() + AdditionalValues.size() <=
2375 DII->addVariableLocationOps(AdditionalValues, SalvagedExpr);
2380 DII->setKillLocation();
2387 if (DVR->isDbgAssign()) {
2388 if (DVR->getAddress() == &I) {
2392 if (DVR->getValue() != &I)
2400 DVR->getType() != DbgVariableRecord::LocationType::Declare;
2401 auto DVRLocation = DVR->location_ops();
2411 DIExpression *SalvagedExpr = DVR->getExpression();
2416 uint64_t CurrentLocOps = SalvagedExpr->getNumLocationOperands();
2429 SalvagedExpr = SalvagedExpr->foldConstantMath();
2430 DVR->replaceVariableLocationOp(&I, Op0);
2432 SalvagedExpr->getNumElements() <= MaxExpressionSize;
2434 DVR->setExpression(SalvagedExpr);
2435 } else if (DVR->getType() != DbgVariableRecord::LocationType::Declare &&
2437 DVR->getNumVariableLocationOps() + AdditionalValues.size() <=
2439 DVR->addVariableLocationOps(AdditionalValues, SalvagedExpr);
2445 DVR->setKillLocation();
2455 DII->setKillLocation();
2458 DVR->setKillLocation();
2465 unsigned BitWidth = DL.getIndexSizeInBits(GEP->getPointerAddressSpace());
2469 if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset))
2484 return GEP->getOperand(0);
2526 AdditionalValues.push_back(I->getOperand(1));
2533 auto *ConstInt = dyn_cast<ConstantInt>(BI->getOperand(1));
2535 if (ConstInt && ConstInt->getBitWidth() > 64)
2538 Instruction::BinaryOps BinOpcode = BI->getOpcode();
2539 // Push any Constant Int operand onto the expression stack.
2541 uint64_t Val = ConstInt->getSExtValue();
2545 uint64_t Offset = BinOpcode == Instruction::Add ? Val : -int64_t(Val);
2547 return BI->getOperand(0);
2560 return BI->getOperand(0);
2592 auto *ConstInt = dyn_cast<ConstantInt>(Icmp->getOperand(1));
2594 if (ConstInt && ConstInt->getBitWidth() > 64)
2596 // Push any Constant Int operand onto the expression stack.
2598 if (Icmp->isSigned())
2602 uint64_t Val = ConstInt->getSExtValue();
2610 uint64_t DwarfIcmpOp = getDwarfOpForIcmpPred(Icmp->getPredicate());
2614 return Icmp->getOperand(0);
2623 if (auto *CI = dyn_cast<CastInst>(&I)) {
2624 Value *FromValue = CI->getOperand(0);
2625 // No-op casts are irrelevant for debug info.
2626 if (CI->isNoopCast(DL)) {
2630 Type *Type = CI->getType();
2631 if (Type->isPointerTy())
2634 if (Type->isVectorTy() ||
2639 llvm::Type *FromType = FromValue->getType();
2640 if (FromType->isPointerTy())
2643 unsigned FromTypeBitSize = FromType->getScalarSizeInBits();
2644 unsigned ToTypeBitSize = Type->getScalarSizeInBits();
2669 /// possibly moving/undefing users to prevent use-before-def. Returns true if
2682 // Prevent use-before-def of To.
2693 if (DomPointAfterFrom && DII->getNextNonDebugInstruction() == &DomPoint) {
2695 DII->moveAfter(&DomPoint);
2707 Instruction *MarkedInstr = DVR->getMarker()->MarkedInstr;
2711 NextNonDebug = NextNonDebug->getNextNonDebugInstruction();
2715 DVR->removeFromParent();
2717 DomPoint.getParent()->insertDbgRecordAfter(DVR, &DomPoint);
2725 // Update debug users without use-before-def risk.
2734 DII->replaceVariableLocationOp(&From, &To);
2735 DII->setExpression(*DVRepl);
2747 DVR->replaceVariableLocationOp(&From, &To);
2748 DVR->setExpression(*DVRepl);
2767 /// allows semantically unequivalent bitcasts, such as <2 x i64> -> <4 x i32>,
2768 /// and also does not allow lossless pointer <-> integer conversions.
2775 // Handle compatible pointer <-> integer conversions.
2776 if (FromTy->isIntOrPtrTy() && ToTy->isIntOrPtrTy()) {
2798 auto Identity = [&](DbgVariableIntrinsic &DII) -> DbgValReplacement {
2801 auto IdentityDVR = [&](DbgVariableRecord &DVR) -> DbgValReplacement {
2805 // Handle no-op conversions.
2811 // Handle integer-to-integer widening and narrowing.
2813 if (FromTy->isIntegerTy() && ToTy->isIntegerTy()) {
2814 uint64_t FromBits = FromTy->getPrimitiveSizeInBits();
2815 uint64_t ToBits = ToTy->getPrimitiveSizeInBits();
2816 assert(FromBits != ToBits && "Unexpected no-op conversion");
2825 auto SignOrZeroExt = [&](DbgVariableIntrinsic &DII) -> DbgValReplacement {
2829 auto Signedness = Var->getSignedness();
2839 auto SignOrZeroExtDVR = [&](DbgVariableRecord &DVR) -> DbgValReplacement {
2843 auto Signedness = Var->getSignedness();
2855 // TODO: Floating-point conversions, vectors.
2862 // RemoveDIs: erase debug-info on this instruction manually.
2863 I->dropDbgRecords();
2864 for (Use &U : I->operands()) {
2866 if (isa<Instruction>(Op) && !Op->getType()->isTokenTy()) {
2867 U.set(PoisonValue::get(Op->getType()));
2881 // having to update as many def-use and use-def chains.
2882 Instruction *EndInst = BB->getTerminator(); // Last not to be deleted.
2886 while (EndInst != &BB->front()) {
2888 Instruction *Inst = &*--EndInst->getIterator();
2889 if (!Inst->use_empty() && !Inst->getType()->isTokenTy())
2890 Inst->replaceAllUsesWith(PoisonValue::get(Inst->getType()));
2891 if (Inst->isEHPad() || Inst->getType()->isTokenTy()) {
2894 Inst->dropDbgRecords();
2902 // RemoveDIs: erasing debug-info must be done manually.
2903 Inst->dropDbgRecords();
2904 Inst->eraseFromParent();
2912 BasicBlock *BB = I->getParent();
2915 MSSAU->changeToUnreachable(I);
2922 Successor->removePredecessor(BB, PreserveLCSSA);
2926 auto *UI = new UnreachableInst(I->getContext(), I->getIterator());
2927 UI->setDebugLoc(I->getDebugLoc());
2931 BasicBlock::iterator BBI = I->getIterator(), BBE = BB->end();
2933 if (!BBI->use_empty())
2934 BBI->replaceAllUsesWith(PoisonValue::get(BBI->getType()));
2935 BBI++->eraseFromParent();
2943 DTU->applyUpdates(Updates);
2945 BB->flushTerminatorDbgRecords();
2950 SmallVector<Value *, 8> Args(II->args());
2952 II->getOperandBundlesAsDefs(OpBundles);
2953 CallInst *NewCall = CallInst::Create(II->getFunctionType(),
2954 II->getCalledOperand(), Args, OpBundles);
2955 NewCall->setCallingConv(II->getCallingConv());
2956 NewCall->setAttributes(II->getAttributes());
2957 NewCall->setDebugLoc(II->getDebugLoc());
2958 NewCall->copyMetadata(*II);
2962 if (NewCall->extractProfTotalWeight(TotalWeight)) {
2964 MDBuilder MDB(NewCall->getContext());
2968 NewCall->setMetadata(LLVMContext::MD_prof, NewWeights);
2974 // changeToCall - Convert the specified invoke into a normal call.
2977 NewCall->takeName(II);
2978 NewCall->insertBefore(II->getIterator());
2979 II->replaceAllUsesWith(NewCall);
2982 BasicBlock *NormalDestBB = II->getNormalDest();
2983 BranchInst::Create(NormalDestBB, II->getIterator());
2986 BasicBlock *BB = II->getParent();
2987 BasicBlock *UnwindDestBB = II->getUnwindDest();
2988 UnwindDestBB->removePredecessor(BB);
2989 II->eraseFromParent();
2991 DTU->applyUpdates({{DominatorTree::Delete, BB, UnwindDestBB}});
2995 BasicBlock *llvm::changeToInvokeAndSplitBasicBlock(CallInst *CI,
2998 BasicBlock *BB = CI->getParent();
3002 BasicBlock *Split = SplitBlock(BB, CI, DTU, /*LI=*/nullptr, /*MSSAU*/ nullptr,
3003 CI->getName() + ".noexc");
3006 BB->back().eraseFromParent();
3009 SmallVector<Value *, 8> InvokeArgs(CI->args());
3012 CI->getOperandBundlesAsDefs(OpBundles);
3019 InvokeInst::Create(CI->getFunctionType(), CI->getCalledOperand(), Split,
3020 UnwindEdge, InvokeArgs, OpBundles, CI->getName(), BB);
3021 II->setDebugLoc(CI->getDebugLoc());
3022 II->setCallingConv(CI->getCallingConv());
3023 II->setAttributes(CI->getAttributes());
3024 II->setMetadata(LLVMContext::MD_prof, CI->getMetadata(LLVMContext::MD_prof));
3027 DTU->applyUpdates({{DominatorTree::Insert, BB, UnwindEdge}});
3031 CI->replaceAllUsesWith(II);
3034 Split->front().eraseFromParent();
3053 if (auto *CI = dyn_cast<CallInst>(&I)) {
3054 Value *Callee = CI->getCalledOperand();
3057 auto IntrinsicID = F->getIntrinsicID();
3063 if (match(CI->getArgOperand(0), m_CombineOr(m_Zero(), m_Undef()))) {
3065 changeToUnreachable(CI, false, DTU);
3079 if (match(CI->getArgOperand(0), m_Zero()))
3080 if (!isa<UnreachableInst>(CI->getNextNode())) {
3081 changeToUnreachable(CI->getNextNode(), false, DTU);
3087 !NullPointerIsDefined(CI->getFunction(),
3088 cast<PointerType>(Callee->getType())
3089 ->getAddressSpace())) ||
3091 changeToUnreachable(CI, false, DTU);
3095 if (CI->doesNotReturn() && !CI->isMustTailCall()) {
3096 // If we found a call to a no-return function, insert an unreachable
3099 if (!isa<UnreachableInst>(CI->getNextNonDebugInstruction())) {
3101 changeToUnreachable(CI->getNextNonDebugInstruction(), false, DTU);
3112 if (SI->isVolatile()) continue;
3114 Value *Ptr = SI->getOperand(1);
3118 !NullPointerIsDefined(SI->getFunction(),
3119 SI->getPointerAddressSpace()))) {
3127 Instruction *Terminator = BB->getTerminator();
3130 Value *Callee = II->getCalledOperand();
3132 !NullPointerIsDefined(BB->getParent())) ||
3137 if (II->doesNotReturn() &&
3138 !isa<UnreachableInst>(II->getNormalDest()->front())) {
3139 // If we found an invoke of a no-return function,
3144 BasicBlock *OrigNormalDest = II->getNormalDest();
3145 OrigNormalDest->removePredecessor(II->getParent());
3146 LLVMContext &Ctx = II->getContext();
3148 Ctx, OrigNormalDest->getName() + ".unreachable",
3149 II->getFunction(), OrigNormalDest);
3151 II->setNormalDest(UnreachableNormalDest);
3153 DTU->applyUpdates(
3158 if (II->doesNotThrow() && canSimplifyInvokeNoUnwind(&F)) {
3159 if (II->use_empty() && !II->mayHaveSideEffects()) {
3161 BasicBlock *NormalDestBB = II->getNormalDest();
3162 BasicBlock *UnwindDestBB = II->getUnwindDest();
3163 BranchInst::Create(NormalDestBB, II->getIterator());
3164 UnwindDestBB->removePredecessor(II->getParent());
3165 II->eraseFromParent();
3167 DTU->applyUpdates({{DominatorTree::Delete, BB, UnwindDestBB}});
3186 CatchPad->value_op_begin(), CatchPad->value_op_end()));
3193 return LHS->isIdenticalTo(RHS);
3203 for (CatchSwitchInst::handler_iterator I = CatchSwitch->handler_begin(),
3204 E = CatchSwitch->handler_end();
3209 auto *CatchPad = cast<CatchPadInst>(HandlerBB->getFirstNonPHIIt());
3212 --NumPerSuccessorCases[HandlerBB];
3213 CatchSwitch->removeHandler(I);
3214 --I;
3215 --E;
3224 DTU->applyUpdates(Updates);
3237 Instruction *TI = BB->getTerminator();
3246 NewTI = CleanupReturnInst::Create(CRI->getCleanupPad(), nullptr, CRI->getIterator());
3247 UnwindDest = CRI->getUnwindDest();
3250 CatchSwitch->getParentPad(), nullptr, CatchSwitch->getNumHandlers(),
3251 CatchSwitch->getName(), CatchSwitch->getIterator());
3252 for (BasicBlock *PadBB : CatchSwitch->handlers())
3253 NewCatchSwitch->addHandler(PadBB);
3256 UnwindDest = CatchSwitch->getUnwindDest();
3261 NewTI->takeName(TI);
3262 NewTI->setDebugLoc(TI->getDebugLoc());
3263 UnwindDest->removePredecessor(BB);
3264 TI->replaceAllUsesWith(NewTI);
3265 TI->eraseFromParent();
3267 DTU->applyUpdates({{DominatorTree::Delete, BB, UnwindDest}});
3271 /// removeUnreachableBlocks - Remove blocks that are not reachable, even
3291 // Skip already-deleted blocks
3292 if (DTU && DTU->isBBPendingDeletion(&BB))
3304 MSSAU->removeBlocks(BlocksToRemove);
3316 K->getAllMetadataOtherThanDebugLoc(Metadata);
3319 MDNode *JMD = J->getMetadata(Kind);
3325 K->setMetadata(Kind, nullptr); // Remove unknown metadata
3331 K->mergeDIAssignID(J);
3335 K->setMetadata(Kind, MDNode::getMostGenericTBAA(JMD, KMD));
3339 K->setMetadata(Kind, MDNode::getMostGenericAliasScope(JMD, KMD));
3344 K->setMetadata(Kind, MDNode::intersect(JMD, KMD));
3348 K->setMetadata(LLVMContext::MD_access_group,
3352 if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
3353 K->setMetadata(Kind, MDNode::getMostGenericRange(JMD, KMD));
3357 K->setMetadata(Kind, MDNode::getMostGenericFPMath(JMD, KMD));
3363 K->setMetadata(Kind, JMD);
3366 if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
3367 K->setMetadata(Kind, JMD);
3376 if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
3377 K->setMetadata(
3383 K->setMetadata(Kind,
3388 K->setMetadata(Kind, MDNode::getMergedMemProfMetadata(KMD, JMD));
3392 K->setMetadata(Kind, MDNode::getMergedCallsiteMetadata(KMD, JMD));
3400 K->setMetadata(Kind, JMD);
3405 K->setMetadata(Kind, JMD);
3409 K->setMetadata(Kind, MDNode::getMergedProfMetadata(KMD, JMD, K, J));
3413 K->setMetadata(Kind,
3419 // then we will just pick it from J - even when they are different.
3420 // Also make sure that K is load or store - f.e. combining bitcast with load
3424 if (auto *JMD = J->getMetadata(LLVMContext::MD_invariant_group))
3426 K->setMetadata(LLVMContext::MD_invariant_group, JMD);
3431 auto JMMRA = J->getMetadata(LLVMContext::MD_mmra);
3432 auto KMMRA = K->getMetadata(LLVMContext::MD_mmra);
3434 K->setMetadata(LLVMContext::MD_mmra,
3435 MMRAMetadata::combine(K->getContext(), JMMRA, KMMRA));
3489 if (NewType->isPointerTy())
3512 ReplInst->dropPoisonGeneratingFlags();
3518 ReplInst->andIRFlags(I);
3523 bool Success = CB1->tryIntersectAttributes(CB2);
3525 "with non-intersectable attributes");
3532 // same control-flow region (meaning that the execution of one
3537 // In general, GVN unifies expressions over different control-flow
3547 assert(From->getType() == To->getType());
3550 for (Use &U : llvm::make_early_inc_range(From->uses())) {
3552 if (II && II->getIntrinsicID() == Intrinsic::fake_use)
3557 From->printAsOperand(dbgs());
3566 assert(From->getType() == To->getType());
3567 auto *BB = From->getParent();
3570 for (Use &U : llvm::make_early_inc_range(From->uses())) {
3572 if (I->getParent() == BB)
3621 if (Call->hasFnAttr("gc-leaf-function"))
3623 if (const Function *F = Call->getCalledFunction()) {
3624 if (F->hasFnAttribute("gc-leaf-function"))
3627 if (auto IID = F->getIntrinsicID()) {
3637 // marked as 'gc-leaf-function.' All available Libcalls are
3638 // GC-leaf.
3652 if (NewTy->isPointerTy()) {
3659 if (!NewTy->isIntegerTy())
3666 ConstantPointerNull::get(cast<PointerType>(Ptr->getType())), ITy);
3685 if (!NewTy->isPointerTy())
3689 if (BitWidth == OldLI.getType()->getScalarSizeInBits() &&
3701 DII->eraseFromParent();
3703 DVR->eraseFromParent();
3716 // - Remove their debug intrinsic instructions.
3717 // - Set their debug locations to the values from the insertion point.
3730 for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) {
3732 I->dropUBImplyingAttrsAndMetadata();
3733 if (I->isUsedByMetadata())
3735 // RemoveDIs: drop debug-info too as the following code does.
3736 I->dropDbgRecords();
3737 if (I->isDebugOrPseudoInst()) {
3739 II = I->eraseFromParent();
3742 I->setDebugLoc(InsertPt->getDebugLoc());
3745 DomBlock->splice(InsertPt->getIterator(), BB, BB->begin(),
3746 BB->getTerminator()->getIterator());
3752 auto createIntegerExpression = [&DIB](const Constant &CV) -> DIExpression * {
3753 const APInt &API = cast<ConstantInt>(&CV)->getValue();
3765 const APFloat &APF = FP->getValueAPF();
3779 if (CE->getOpcode() == Instruction::IntToPtr) {
3780 const Value *V = CE->getOperand(0);
3781 if (auto CI = dyn_cast_or_null<ConstantInt>(V))
3782 return createIntegerExpression(*CI);
3792 DV->replaceVariableLocationOp(Op, I->second, /*AllowEmpty=*/true);
3796 auto I = Mapping.find(DA->getAddress());
3798 DA->setAddress(I->second);
3801 RemapDebugOperands(DVI, DVI->location_ops());
3804 for (DbgVariableRecord &DVR : filterDbgVars(Inst->getDbgRecordRange())) {
3827 enum { Unset = -1 };
3834 /// piece of a bswap or bitreverse if it can be proved that each non-zero bit in
3843 /// BitPart is returned with Provider set to %X and Provenance[24-31] set to
3844 /// [0-7].
3846 /// For vector types, all analysis is performed at the per-element level. No
3847 /// cross-element analysis is supported (shuffle/insertion/reduction), and all
3852 /// constructed in-place in the \c BPS map. Because of this \c BPS needs to
3857 /// Because we pass around references into \c BPS, we must use a container that
3865 return I->second;
3867 auto &Result = I->second;
3868 auto BitWidth = V->getType()->getScalarSizeInBits();
3889 if (!A || !A->Provider)
3894 if (!B || A->Provider != B->Provider)
3898 Result = BitPart(A->Provider, BitWidth);
3900 if (A->Provenance[BitIdx] != BitPart::Unset &&
3901 B->Provenance[BitIdx] != BitPart::Unset &&
3902 A->Provenance[BitIdx] != B->Provenance[BitIdx])
3905 if (A->Provenance[BitIdx] == BitPart::Unset)
3906 Result->Provenance[BitIdx] = B->Provenance[BitIdx];
3908 Result->Provenance[BitIdx] = A->Provenance[BitIdx];
3922 // For bswap-only, limit shift amounts to whole bytes, for an early exit.
3933 auto &P = Result->Provenance;
3934 if (I->getOpcode() == Instruction::Shl) {
3965 Result->Provenance[BitIdx] = BitPart::Unset;
3976 Result = BitPart(Res->Provider, BitWidth);
3977 auto NarrowBitWidth = X->getType()->getScalarSizeInBits();
3979 Result->Provenance[BitIdx] = Res->Provenance[BitIdx];
3981 Result->Provenance[BitIdx] = BitPart::Unset;
3992 Result = BitPart(Res->Provider, BitWidth);
3994 Result->Provenance[BitIdx] = Res->Provenance[BitIdx];
3998 // BITREVERSE - most likely due to us previous matching a partial
4006 Result = BitPart(Res->Provider, BitWidth);
4008 Result->Provenance[(BitWidth - 1) - BitIdx] = Res->Provenance[BitIdx];
4012 // BSWAP - most likely due to us previous matching a partial bswap.
4020 Result = BitPart(Res->Provider, BitWidth);
4024 Result->Provenance[(BitWidth - 8 - ByteBitOfs) + BitIdx] =
4025 Res->Provenance[ByteBitOfs + BitIdx];
4032 // fshl(X,Y,Z): (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
4033 // fshr(X,Y,Z): (X << (BW - (Z % BW))) | (Y >> (Z % BW))
4037 unsigned ModAmt = C->urem(BitWidth);
4038 if (cast<IntrinsicInst>(I)->getIntrinsicID() == Intrinsic::fshr)
4039 ModAmt = BitWidth - ModAmt;
4041 // For bswap-only, limit shift amounts to whole bytes, for an early exit.
4048 if (!LHS || !LHS->Provider)
4053 if (!RHS || LHS->Provider != RHS->Provider)
4056 unsigned StartBitRHS = BitWidth - ModAmt;
4057 Result = BitPart(LHS->Provider, BitWidth);
4059 Result->Provenance[BitIdx + ModAmt] = LHS->Provenance[BitIdx];
4061 Result->Provenance[BitIdx] = RHS->Provenance[BitIdx + StartBitRHS];
4076 Result->Provenance[BitIdx] = BitIdx;
4088 return From == BitWidth - To - 1;
4093 return From == BitWidth - To - 1;
4106 Type *ITy = I->getType();
4107 if (!ITy->isIntOrIntVectorTy() || ITy->getScalarSizeInBits() > 128)
4117 ArrayRef<int8_t> BitProvenance = Res->Provenance;
4128 return false; // TODO - handle null value?
4129 DemandedTy = Type::getIntNTy(I->getContext(), BitProvenance.size());
4135 unsigned DemandedBW = DemandedTy->getScalarSizeInBits();
4136 if (DemandedBW > ITy->getScalarSizeInBits())
4165 Intrinsic::getOrInsertDeclaration(I->getModule(), Intrin, DemandedTy);
4166 Value *Provider = Res->Provider;
4169 if (DemandedTy != Provider->getType()) {
4171 CastInst::CreateIntegerCast(Provider, DemandedTy, false, "trunc", I->getIterator());
4176 Instruction *Result = CallInst::Create(F, Provider, "rev", I->getIterator());
4181 Result = BinaryOperator::Create(Instruction::And, Result, Mask, "mask", I->getIterator());
4186 if (ITy != Result->getType()) {
4187 auto *ExtInst = CastInst::CreateIntegerCast(Result, ITy, false, "zext", I->getIterator());
4195 // them with target-specific intrinsics. Since that'd skip our interceptors
4200 CallInst *CI, const TargetLibraryInfo *TLI) {
4201 Function *F = CI->getCalledFunction();
4203 if (F && !F->hasLocalLinkage() && F->hasName() &&
4204 TLI->getLibFunc(F->getName(), Func) && TLI->hasOptimizedCodeGen(Func) &&
4205 !F->doesNotAccessMemory())
4206 CI->addFnAttr(Attribute::NoBuiltin);
4211 if (I->getOperand(OpIdx)->getType()->isMetadataTy())
4215 if (!isa<Constant>(I->getOperand(OpIdx)))
4218 switch (I->getOpcode()) {
4229 // Constant bundle operands may need to retain their constant-ness for
4238 OpIdx >= CB.getFunctionType()->getNumParams()) {
4269 // want to make them non-constant.
4270 return !cast<AllocaInst>(I)->isStaticAlloca();
4295 Parent = Inst->getParent();
4297 Parent = &Arg->getParent()->getEntryBlock();
4301 for (User *U : Condition->users())
4303 if (I->getParent() == Parent && match(I, m_Not(m_Specific(Condition))))
4308 BinaryOperator::CreateNot(Condition, Condition->getName() + ".inv");
4310 Inverted->insertAfter(Inst->getIterator());
4312 Inverted->insertBefore(Parent->getFirstInsertionPt());
4341 // can infer by inspecting arguments of argmemonly-ish functions.