Lines Matching +full:reset +full:- +full:bps
1 //===- Local.cpp - Functions to perform local transformations -------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
12 //===----------------------------------------------------------------------===//
99 "phicse-debug-hash",
107 "function is well-behaved w.r.t. its isEqual predicate"));
110 "phicse-num-phi-smallsize", cl::init(32), cl::Hidden,
113 "perform a (faster!) exhaustive search instead of set-driven one."));
119 //===----------------------------------------------------------------------===//
123 /// ConstantFoldTerminator - If a terminator instruction is predicated on a
133 Instruction *T = BB->getTerminator();
136 // Branch - See if we are conditional jumping on constant
138 if (BI->isUnconditional()) return false; // Can't optimize uncond branch
140 BasicBlock *Dest1 = BI->getSuccessor(0);
141 BasicBlock *Dest2 = BI->getSuccessor(1);
149 assert(BI->getParent() && "Terminator not inserted in block!");
150 Dest1->removePredecessor(BI->getParent());
156 NewBI->copyMetadata(*BI, {LLVMContext::MD_loop, LLVMContext::MD_dbg,
159 Value *Cond = BI->getCondition();
160 BI->eraseFromParent();
166 if (auto *Cond = dyn_cast<ConstantInt>(BI->getCondition())) {
169 BasicBlock *Destination = Cond->getZExtValue() ? Dest1 : Dest2;
170 BasicBlock *OldDest = Cond->getZExtValue() ? Dest2 : Dest1;
174 OldDest->removePredecessor(BB);
180 NewBI->copyMetadata(*BI, {LLVMContext::MD_loop, LLVMContext::MD_dbg,
183 BI->eraseFromParent();
185 DTU->applyUpdates({{DominatorTree::Delete, BB, OldDest}});
195 auto *CI = dyn_cast<ConstantInt>(SI->getCondition());
196 BasicBlock *DefaultDest = SI->getDefaultDest();
200 if (isa<UnreachableInst>(DefaultDest->getFirstNonPHIOrDbg()) &&
201 SI->getNumCases() > 0) {
202 TheOnlyDest = SI->case_begin()->getCaseSuccessor();
208 for (auto It = SI->case_begin(), End = SI->case_end(); It != End;) {
210 if (It->getCaseValue() == CI) {
211 TheOnlyDest = It->getCaseSuccessor();
217 if (It->getCaseSuccessor() == DefaultDest) {
219 unsigned NCases = SI->getNumCases();
228 unsigned Idx = It->getCaseIndex();
237 BasicBlock *ParentBB = SI->getParent();
238 DefaultDest->removePredecessor(ParentBB);
239 It = SI->removeCase(It);
240 End = SI->case_end();
244 if (auto *NewCI = dyn_cast<ConstantInt>(SI->getCondition())) {
246 It = SI->case_begin();
254 // We do this by reseting "TheOnlyDest" to null when we find two non-equal
256 if (It->getCaseSuccessor() != TheOnlyDest)
266 TheOnlyDest = SI->getDefaultDest();
274 BasicBlock *BB = SI->getParent();
287 Succ->removePredecessor(BB);
292 Value *Cond = SI->getCondition();
293 SI->eraseFromParent();
301 DTU->applyUpdates(Updates);
306 if (SI->getNumCases() == 1) {
308 // instruction if it has only one non-default destination.
309 auto FirstCase = *SI->case_begin();
310 Value *Cond = Builder.CreateICmpEQ(SI->getCondition(),
316 SI->getDefaultDest());
322 NewBr->setMetadata(LLVMContext::MD_prof,
323 MDBuilder(BB->getContext())
327 // Update make.implicit metadata to the newly-created conditional branch.
328 MDNode *MakeImplicitMD = SI->getMetadata(LLVMContext::MD_make_implicit);
330 NewBr->setMetadata(LLVMContext::MD_make_implicit, MakeImplicitMD);
333 SI->eraseFromParent();
340 // indirectbr blockaddress(@F, @BB) -> br label @BB
342 dyn_cast<BlockAddress>(IBI->getAddress()->stripPointerCasts())) {
343 BasicBlock *TheOnlyDest = BA->getBasicBlock();
350 for (unsigned i = 0, e = IBI->getNumDestinations(); i != e; ++i) {
351 BasicBlock *DestBB = IBI->getDestination(i);
354 if (IBI->getDestination(i) == SuccToKeep) {
357 DestBB->removePredecessor(BB);
360 Value *Address = IBI->getAddress();
361 IBI->eraseFromParent();
368 if (BA->use_empty())
369 BA->destroyConstant();
375 BB->getTerminator()->eraseFromParent();
376 new UnreachableInst(BB->getContext(), BB);
384 DTU->applyUpdates(Updates);
393 //===----------------------------------------------------------------------===//
397 /// isInstructionTriviallyDead - Return true if the result produced by the
402 if (!I->use_empty())
412 if (II->getIntrinsicID() == Intrinsic::stacksave ||
413 II->getIntrinsicID() == Intrinsic::launder_invariant_group ||
414 II->isLifetimeStartOrEnd())
421 if (I->isTerminator())
424 // We don't want the landingpad-like instructions removed by anything this
426 if (I->isEHPad())
434 if (DLI->getLabel())
443 if (!I->willReturn()) {
448 switch (II->getIntrinsicID()) {
450 // Guards on true are operationally no-ops. In the future we can
453 auto *Cond = dyn_cast<ConstantInt>(II->getArgOperand(0));
454 return Cond && Cond->isOne();
457 // a well-defined trap.
468 if (!I->mayHaveSideEffects())
475 if (II->getIntrinsicID() == Intrinsic::stacksave ||
476 II->getIntrinsicID() == Intrinsic::launder_invariant_group)
481 if (II->getIntrinsicID() == Intrinsic::allow_runtime_check ||
482 II->getIntrinsicID() == Intrinsic::allow_ubsan_check)
485 if (II->isLifetimeStartOrEnd()) {
486 auto *Arg = II->getArgOperand(1);
487 // Lifetime intrinsics are dead when their right-hand is undef.
490 // If the right-hand is an alloc, global, or argument and the only uses
493 return llvm::all_of(Arg->uses(), [](Use &Use) {
496 return IntrinsicUse->isLifetimeStartOrEnd();
503 if (II->getIntrinsicID() == Intrinsic::assume &&
505 if (ConstantInt *Cond = dyn_cast<ConstantInt>(II->getArgOperand(0)))
506 return !Cond->isZero();
513 FPI->getExceptionBehavior();
521 return C->isNullValue() || isa<UndefValue>(C);
526 // Non-volatile atomic loads from constants can be removed.
529 LI->getPointerOperand()->stripPointerCasts()))
530 if (!LI->isVolatile() && GV->isConstant())
536 /// RecursivelyDeleteTriviallyDeadInstructions - If the specified value is a
586 assert(I->use_empty() && "Instructions with uses are not dead.");
596 for (Use &OpU : I->operands()) {
600 if (!OpV->use_empty())
611 MSSAU->removeMemoryAccess(I);
613 I->eraseFromParent();
622 DII->setKillLocation();
624 DVR->setKillLocation();
628 /// areAllUsesEqual - Check whether the uses of a value are all the same.
633 Value::user_iterator UI = I->user_begin();
634 Value::user_iterator UE = I->user_end();
646 /// RecursivelyDeleteDeadPHINode - If the specified value is an effectively
647 /// dead PHI node, due to being a def-use chain of single-use nodes that
655 for (Instruction *I = PN; areAllUsesEqual(I) && !I->mayHaveSideEffects();
656 I = cast<Instruction>(*I->user_begin())) {
657 if (I->use_empty())
664 I->replaceAllUsesWith(PoisonValue::get(I->getType()));
682 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
683 Value *OpV = I->getOperand(i);
684 I->setOperand(i, nullptr);
686 if (!OpV->use_empty() || I == OpV)
697 I->eraseFromParent();
705 for (User *U : I->users()) {
713 if (!I->use_empty()) {
714 I->replaceAllUsesWith(SimpleV);
718 I->eraseFromParent();
726 /// SimplifyInstructionsInBlock - Scan the specified basic block and try to
734 const DataLayout &DL = BB->getDataLayout();
741 AssertingVH<Instruction> TerminatorVH(&BB->back());
746 // if they actually need to be revisited. This avoids having to pre-init
748 for (BasicBlock::iterator BI = BB->begin(), E = std::prev(BB->end());
750 assert(!BI->isTerminator());
767 //===----------------------------------------------------------------------===//
774 // If BB has single-entry PHI nodes, fold them.
775 while (PHINode *PN = dyn_cast<PHINode>(DestBB->begin())) {
776 Value *NewVal = PN->getIncomingValue(0);
778 if (NewVal == PN) NewVal = PoisonValue::get(PN->getType());
779 PN->replaceAllUsesWith(NewVal);
780 PN->eraseFromParent();
783 BasicBlock *PredBB = DestBB->getSinglePredecessor();
786 bool ReplaceEntryBB = PredBB->isEntryBlock();
810 if (DestBB->hasAddressTaken()) {
813 ConstantInt::get(Type::getInt32Ty(BA->getContext()), 1);
814 BA->replaceAllUsesWith(ConstantExpr::getIntToPtr(Replacement,
815 BA->getType()));
816 BA->destroyConstant();
820 PredBB->replaceAllUsesWith(DestBB);
823 PredBB->getTerminator()->eraseFromParent();
824 DestBB->splice(DestBB->begin(), PredBB);
825 new UnreachableInst(PredBB->getContext(), PredBB);
830 DestBB->moveAfter(PredBB);
833 assert(PredBB->size() == 1 &&
834 isa<UnreachableInst>(PredBB->getTerminator()) &&
837 DTU->applyUpdatesPermissive(Updates);
838 DTU->deleteBB(PredBB);
841 if (ReplaceEntryBB && DTU->hasDomTree()) {
843 // the dominator tree to be notified of this change. In this corner-case
845 DTU->recalculate(*(DestBB->getParent()));
850 PredBB->eraseFromParent(); // Nuke BB if DTU is nullptr.
855 /// other. Note that we will always choose the non-undef value to keep.
860 /// Return true if we can fold BB, an almost-empty BB ending in an unconditional
869 LLVM_DEBUG(dbgs() << "Looking to fold " << BB->getName() << " into "
870 << Succ->getName() << "\n");
873 if (Succ->getSinglePredecessor())
878 for (BasicBlock::iterator I = Succ->begin(); isa<PHINode>(I); ++I) {
884 PHINode *BBPN = dyn_cast<PHINode>(PN->getIncomingValueForBlock(BB));
885 if (BBPN && BBPN->getParent() == BB) {
886 for (unsigned PI = 0, PE = PN->getNumIncomingValues(); PI != PE; ++PI) {
887 BasicBlock *IBB = PN->getIncomingBlock(PI);
889 !CanMergeValues(BBPN->getIncomingValueForBlock(IBB),
890 PN->getIncomingValue(PI))) {
892 << "Can't fold, phi node " << PN->getName() << " in "
893 << Succ->getName() << " is conflicting with "
894 << BBPN->getName() << " with regard to common predecessor "
895 << IBB->getName() << "\n");
900 Value* Val = PN->getIncomingValueForBlock(BB);
901 for (unsigned PI = 0, PE = PN->getNumIncomingValues(); PI != PE; ++PI) {
905 BasicBlock *IBB = PN->getIncomingBlock(PI);
907 !CanMergeValues(Val, PN->getIncomingValue(PI))) {
908 LLVM_DEBUG(dbgs() << "Can't fold, phi node " << PN->getName()
909 << " in " << Succ->getName()
911 << "predecessor " << IBB->getName() << "\n");
932 /// \param IncomingValues A map from block-to-value for other phi inputs
940 IncomingValues.find(BB)->second == OldVal) &&
948 if (It != IncomingValues.end()) return It->second;
956 /// Create a map from block to value for each non-undef value flowing
963 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
964 BasicBlock *BB = PN->getIncomingBlock(i);
965 Value *V = PN->getIncomingValue(i);
973 /// from a block-to-value map.
980 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
981 Value *V = PN->getIncomingValue(i);
985 BasicBlock *BB = PN->getIncomingBlock(i);
999 PN->setIncomingValue(i, It->second);
1006 return isa<PoisonValue>(PN->getIncomingValue(i));
1010 PN->setIncomingValue(i, UndefValue::get(PN->getType()));
1024 if (BB->phis().empty() || Succ->phis().empty())
1028 if (!BB->hasNPredecessorsOrMore(2))
1032 return isa<IndirectBrInst>(Pred->getTerminator());
1061 Value *OldVal = PN->removeIncomingValue(BB, false);
1066 // We are merging two blocks - BB, and the block containing PN - and
1073 // consistent with the non-undef values.
1079 if (isa<PHINode>(OldVal) && cast<PHINode>(OldVal)->getParent() == BB) {
1081 for (unsigned i = 0, e = OldValPN->getNumIncomingValues(); i != e; ++i) {
1087 BasicBlock *PredBB = OldValPN->getIncomingBlock(i);
1092 Value *PredVal = OldValPN->getIncomingValue(i);
1098 PN->addIncoming(Selected, PredBB);
1101 PN->addIncoming(OldValPN->getIncomingValueForBlock(CommonPred), BB);
1115 PN->addIncoming(Selected, PredBB);
1118 PN->addIncoming(OldVal, BB);
1126 assert(BB != &BB->getParent()->getEntryBlock() &&
1130 BasicBlock *Succ = cast<BranchInst>(BB->getTerminator())->getSuccessor(0);
1157 // BB dominates Succ, which is non-trivial to calculate in the case where
1159 // constructing the necessary self-referential PHI node doesn't introduce any
1164 // something like a loop pre-header (or rarely, a part of an irreducible CFG);
1166 if (!Succ->getSinglePredecessor()) {
1167 BasicBlock::iterator BBI = BB->begin();
1169 for (Use &U : BBI->uses()) {
1171 if (PN->getIncomingBlock(U) != BB)
1182 LLVM_DEBUG(dbgs() << "Found Common Predecessor between: " << BB->getName()
1183 << " and " << Succ->getName() << " : "
1184 << CommonPred->getName() << "\n");
1186 // 'BB' and 'BB->Pred' are loop latches, bail out to presrve inner loop
1189 // FIXME: This is a stop-gap solution to preserve inner-loop metadata given
1200 // In this case, we expect 'BB' is the latch for outer-loop and 'BB->Pred' is
1201 // the latch for inner-loop (see reason below), so bail out to prerserve
1202 // inner-loop metadata rather than eliminating 'BB' and attaching its metadata
1203 // to this inner-loop.
1204 // - The reason we believe 'BB' and 'BB->Pred' have different inner-most
1205 // loops: assuming 'BB' and 'BB->Pred' are from the same inner-most loop L,
1207 // one self-looping basic block, which is contradictory with the assumption.
1209 // To illustrate how inner-loop metadata is dropped:
1214 // BB->Pred is for.body, attached with loop metadata md1.
1219 // ---> while.cond -------------> while.end
1225 // | for.body <---- (md1)
1237 // dedicated exits for inner-loop (essentially adding `while.cond.exit`
1238 // back), but won't it won't see 'md1' nor restore it for the inner-loop.
1243 // ---> while.cond1 -------------> while.end
1249 // | for.body <---- (md2)
1251 if (Instruction *TI = BB->getTerminator())
1252 if (TI->hasMetadata(LLVMContext::MD_loop))
1254 if (Instruction *PredTI = Pred->getTerminator())
1255 if (PredTI->hasMetadata(LLVMContext::MD_loop))
1291 if (isa<PHINode>(Succ->begin())) {
1298 for (BasicBlock::iterator I = Succ->begin(); isa<PHINode>(I); ++I) {
1304 if (Succ->getSinglePredecessor()) {
1308 BB->getTerminator()->eraseFromParent();
1309 Succ->splice(Succ->getFirstNonPHIIt(), BB);
1311 while (PHINode *PN = dyn_cast<PHINode>(&BB->front())) {
1313 assert(PN->use_empty() && "There shouldn't be any uses here!");
1314 PN->eraseFromParent();
1320 if (Instruction *TI = BB->getTerminator())
1321 if (MDNode *LoopMD = TI->getMetadata(LLVMContext::MD_loop))
1323 Pred->getTerminator()->setMetadata(LLVMContext::MD_loop, LoopMD);
1327 BB->replaceAllUsesWith(Succ);
1329 if (!Succ->hasName())
1330 Succ->takeName(BB);
1333 if (BB->getTerminator())
1334 BB->back().eraseFromParent();
1336 new UnreachableInst(BB->getContext(), BB);
1341 BB->replaceUsesWithIf(Succ, [BBPreds, CommonPred](Use &U) -> bool {
1343 return UseInst->getParent() != CommonPred &&
1344 BBPreds.contains(UseInst->getParent());
1350 DTU->applyUpdates(Updates);
1370 for (auto I = BB->begin(); PHINode *PN = dyn_cast<PHINode>(I);) {
1378 if (!DuplicatePN->isIdenticalToWhenDefined(PN))
1382 DuplicatePN->replaceAllUsesWith(PN);
1387 I = BB->begin();
1421 hash_combine_range(PN->value_op_begin(), PN->value_op_end()),
1422 hash_combine_range(PN->block_begin(), PN->block_end())));
1427 // If -phicse-debug-hash was specified, return a constant -- this
1440 return LHS->isIdenticalTo(RHS);
1459 for (auto I = BB->begin(); PHINode *PN = dyn_cast<PHINode>(I++);) {
1466 PN->replaceAllUsesWith(*Inserted.first);
1473 I = BB->begin();
1486 hasNItemsOrLess(BB->phis(), PHICSENumPHISmallSize))
1495 PN->eraseFromParent();
1501 V = V->stripPointerCasts();
1509 Align CurrentAlign = AI->getAlign();
1517 AI->setAlignment(PrefAlign);
1523 Align CurrentAlign = GO->getPointerAlignment(DL);
1531 if (!GO->canIncreaseAlignment())
1534 if (GO->isThreadLocal()) {
1535 unsigned MaxTLSAlign = GO->getParent()->getMaxTLSAlignment() / CHAR_BIT;
1540 GO->setAlignment(PrefAlign);
1552 assert(V->getType()->isPointerTy() &&
1563 Align Alignment = Align(1ull << std::min(Known.getBitWidth() - 1, TrailZ));
1572 ///===---------------------------------------------------------------------===//
1587 assert(is_contained(DVI->getValues(), APN));
1588 if ((DVI->getVariable() == DIVar) && (DVI->getExpression() == DIExpr))
1592 assert(is_contained(DVR->location_ops(), APN));
1593 if ((DVR->getVariable() == DIVar) && (DVR->getExpression() == DIExpr))
1606 /// covering an n-bit fragment, if the store size of i1 is at least n bits.
1608 const DataLayout &DL = DII->getDataLayout();
1611 DII->getExpression()->getActiveBits(DII->getVariable()))
1617 if (DII->isAddressOfVariable()) {
1619 assert(DII->getNumVariableLocationOps() == 1 &&
1622 dyn_cast_or_null<AllocaInst>(DII->getVariableLocationOp(0))) {
1624 AI->getAllocationSizeInBits(DL)) {
1635 const DataLayout &DL = DVR->getModule()->getDataLayout();
1638 DVR->getExpression()->getActiveBits(DVR->getVariable()))
1644 if (DVR->isAddressOfVariable()) {
1646 assert(DVR->getNumVariableLocationOps() == 1 &&
1649 dyn_cast_or_null<AllocaInst>(DVR->getVariableLocationOp(0))) {
1650 if (std::optional<TypeSize> FragmentSize = AI->getAllocationSizeInBits(DL)) {
1667 DbgVal.get<Instruction *>()->insertBefore(Instr);
1669 // RemoveDIs: if we're using the new debug-info format, allocate a
1674 Instr->getParent()->insertDbgRecordBefore(DV, Instr);
1684 DbgVal.get<Instruction *>()->insertAfter(&*Instr);
1686 // RemoveDIs: if we're using the new debug-info format, allocate a
1691 Instr->getParent()->insertDbgRecordAfter(DV, &*Instr);
1699 assert(DII->isAddressOfVariable() || isa<DbgAssignIntrinsic>(DII));
1700 auto *DIVar = DII->getVariable();
1702 auto *DIExpr = DII->getExpression();
1703 Value *DV = SI->getValueOperand();
1720 DIExpr->isDeref() || (!DIExpr->startsWithDeref() &&
1721 valueCoversEntireFragment(DV->getType(), DII));
1724 SI->getIterator());
1735 DV = UndefValue::get(DV->getType());
1737 SI->getIterator());
1744 auto *DIVar = DII->getVariable();
1745 auto *DIExpr = DII->getExpression();
1748 if (!valueCoversEntireFragment(LI->getType(), DII)) {
1760 // future if multi-location support is added to the IR, it might be
1764 LI->getIterator());
1769 assert(DVR->isAddressOfVariable() || DVR->isDbgAssign());
1770 auto *DIVar = DVR->getVariable();
1772 auto *DIExpr = DVR->getExpression();
1773 Value *DV = SI->getValueOperand();
1790 DIExpr->isDeref() || (!DIExpr->startsWithDeref() &&
1791 valueCoversEntireFragment(DV->getType(), DVR));
1794 SI->getIterator());
1807 DV = UndefValue::get(DV->getType());
1811 SI->getParent()->insertDbgRecordBefore(NewDVR, SI->getIterator());
1818 auto *DIVar = DII->getVariable();
1819 auto *DIExpr = DII->getExpression();
1825 if (!valueCoversEntireFragment(APN->getType(), DII)) {
1834 BasicBlock *BB = APN->getParent();
1835 auto InsertionPt = BB->getFirstInsertionPt();
1842 if (InsertionPt != BB->end()) {
1850 auto *DIVar = DVR->getVariable();
1851 auto *DIExpr = DVR->getExpression();
1854 if (!valueCoversEntireFragment(LI->getType(), DVR)) {
1866 // future if multi-location support is added to the IR, it might be
1875 LI->getParent()->insertDbgRecordAfter(DV, LI);
1880 return AI->isArrayAllocation() ||
1881 (AI->getAllocatedType() && AI->getAllocatedType()->isArrayTy());
1886 return AI->getAllocatedType() && AI->getAllocatedType()->isStructTy();
1890 auto *DIVar = DVR->getVariable();
1891 auto *DIExpr = DVR->getExpression();
1897 if (!valueCoversEntireFragment(APN->getType(), DVR)) {
1906 BasicBlock *BB = APN->getParent();
1907 auto InsertionPt = BB->getFirstInsertionPt();
1914 if (InsertionPt != BB->end()) {
1920 /// LowerDbgDeclare - Lowers llvm.dbg.declare intrinsics into appropriate set
1943 dyn_cast_or_null<AllocaInst>(DDI->getVariableLocationOp(0));
1948 // the stack slot (and at a lexical-scope granularity). Later
1954 if (llvm::any_of(AI->users(), [](User *U) -> bool {
1956 return LI->isVolatile();
1958 return SI->isVolatile();
1967 for (const auto &AIUse : V->uses()) {
1975 // This is a call by-value or some other instruction that takes a
1978 if (!CI->isLifetimeStartOrEnd()) {
1981 DIExpression::append(DDI->getExpression(), dwarf::DW_OP_deref);
1982 insertDbgValueOrDbgVariableRecord(DIB, AI, DDI->getVariable(),
1984 CI->getIterator());
1987 if (BI->getType()->isPointerTy())
1992 DDI->eraseFromParent();
2006 // RemoveDIs: re-implementation of insertDebugValuesForPHIs, but which pulls the
2007 // debug-info out of the block's DbgVariableRecords rather than dbg.value
2041 BasicBlock *Parent = PHI->getParent();
2042 // Avoid inserting a debug-info record into an EH block.
2043 if (Parent->getFirstNonPHI()->isEHPad())
2045 for (auto VI : PHI->operand_values()) {
2048 DbgVariableRecord *DbgII = cast<DbgVariableRecord>(V->second);
2051 DbgVariableRecord *NewDbgII = DbgII->clone();
2054 DbgVariableRecord *NewDbgII = NewDI->second;
2057 if (is_contained(NewDbgII->location_ops(), VI))
2058 NewDbgII->replaceVariableLocationOp(VI, PHI);
2066 auto InsertionPt = Parent->getFirstInsertionPt();
2067 assert(InsertionPt != Parent->end() && "Ill-formed basic block");
2069 Parent->insertDbgRecordBefore(NewDbgII, InsertionPt);
2086 for (Value *V : DbgII->location_ops())
2107 BasicBlock *Parent = PHI->getParent();
2109 if (Parent->getFirstNonPHI()->isEHPad())
2111 for (auto *VI : PHI->operand_values()) {
2114 auto *DbgII = cast<DbgVariableIntrinsic>(V->second);
2117 auto *NewDbgII = cast<DbgVariableIntrinsic>(DbgII->clone());
2120 DbgVariableIntrinsic *NewDbgII = NewDI->second;
2123 if (is_contained(NewDbgII->location_ops(), VI))
2124 NewDbgII->replaceVariableLocationOp(VI, PHI);
2132 auto InsertionPt = Parent->getFirstInsertionPt();
2133 assert(InsertionPt != Parent->end() && "Ill-formed basic block");
2134 NewDbgII->insertBefore(&*InsertionPt);
2145 assert(DII->getVariable() && "Missing variable");
2146 auto *DIExpr = DII->getExpression();
2148 DII->setExpression(DIExpr);
2149 DII->replaceVariableLocationOp(Address, NewAddress);
2166 // This is an alloca-based dbg.value/DbgVariableRecord. The first thing it
2169 if (!DIExpr || DIExpr->getNumElements() < 1 ||
2170 DIExpr->getElement(0) != dwarf::DW_OP_deref)
2178 DVI->setExpression(DIExpr);
2179 DVI->replaceVariableLocationOp(0u, NewAddress);
2182 DVR->setExpression(DIExpr);
2183 DVR->replaceVariableLocationOp(0u, NewAddress);
2195 updateOneDbgValueForAlloca(DVI->getDebugLoc(), DVI->getVariable(),
2196 DVI->getExpression(), NewAllocaAddress, DVI,
2201 updateOneDbgValueForAlloca(DVR->getDebugLoc(), DVR->getVariable(),
2202 DVR->getExpression(), NewAllocaAddress, nullptr,
2216 Instruction *I = dyn_cast<Instruction>(Assign->getAddress());
2221 assert(!Assign->getAddressExpression()->getFragmentInfo().has_value() &&
2222 "address-expression shouldn't have fragment info");
2235 Assign->getAddressExpression(), Ops, 0, /*StackValue=*/false);
2236 assert(!SalvagedExpr->getFragmentInfo().has_value() &&
2237 "address-expression shouldn't have fragment info");
2239 SalvagedExpr = SalvagedExpr->foldConstantMath();
2243 Assign->setAddress(NewV);
2244 Assign->setAddressExpression(SalvagedExpr);
2246 Assign->setKillAddress();
2262 if (DAI->getAddress() == &I) {
2266 if (DAI->getValue() != &I)
2273 auto DIILocation = DII->location_ops();
2283 DIExpression *SalvagedExpr = DII->getExpression();
2288 uint64_t CurrentLocOps = SalvagedExpr->getNumLocationOperands();
2301 SalvagedExpr = SalvagedExpr->foldConstantMath();
2302 DII->replaceVariableLocationOp(&I, Op0);
2303 bool IsValidSalvageExpr = SalvagedExpr->getNumElements() <= MaxExpressionSize;
2305 DII->setExpression(SalvagedExpr);
2307 DII->getNumVariableLocationOps() + AdditionalValues.size() <=
2309 DII->addVariableLocationOps(AdditionalValues, SalvagedExpr);
2314 DII->setKillLocation();
2321 if (DVR->isDbgAssign()) {
2322 if (DVR->getAddress() == &I) {
2326 if (DVR->getValue() != &I)
2334 DVR->getType() != DbgVariableRecord::LocationType::Declare;
2335 auto DVRLocation = DVR->location_ops();
2345 DIExpression *SalvagedExpr = DVR->getExpression();
2350 uint64_t CurrentLocOps = SalvagedExpr->getNumLocationOperands();
2363 SalvagedExpr = SalvagedExpr->foldConstantMath();
2364 DVR->replaceVariableLocationOp(&I, Op0);
2366 SalvagedExpr->getNumElements() <= MaxExpressionSize;
2368 DVR->setExpression(SalvagedExpr);
2369 } else if (DVR->getType() != DbgVariableRecord::LocationType::Declare &&
2371 DVR->getNumVariableLocationOps() + AdditionalValues.size() <=
2373 DVR->addVariableLocationOps(AdditionalValues, SalvagedExpr);
2379 DVR->setKillLocation();
2389 DII->setKillLocation();
2392 DVR->setKillLocation();
2399 unsigned BitWidth = DL.getIndexSizeInBits(GEP->getPointerAddressSpace());
2403 if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset))
2418 return GEP->getOperand(0);
2460 AdditionalValues.push_back(I->getOperand(1));
2467 auto *ConstInt = dyn_cast<ConstantInt>(BI->getOperand(1));
2469 if (ConstInt && ConstInt->getBitWidth() > 64)
2472 Instruction::BinaryOps BinOpcode = BI->getOpcode();
2475 uint64_t Val = ConstInt->getSExtValue();
2479 uint64_t Offset = BinOpcode == Instruction::Add ? Val : -int64_t(Val);
2481 return BI->getOperand(0);
2494 return BI->getOperand(0);
2526 auto *ConstInt = dyn_cast<ConstantInt>(Icmp->getOperand(1));
2528 if (ConstInt && ConstInt->getBitWidth() > 64)
2532 if (Icmp->isSigned())
2536 uint64_t Val = ConstInt->getSExtValue();
2544 uint64_t DwarfIcmpOp = getDwarfOpForIcmpPred(Icmp->getPredicate());
2548 return Icmp->getOperand(0);
2558 Value *FromValue = CI->getOperand(0);
2559 // No-op casts are irrelevant for debug info.
2560 if (CI->isNoopCast(DL)) {
2564 Type *Type = CI->getType();
2565 if (Type->isPointerTy())
2568 if (Type->isVectorTy() ||
2573 llvm::Type *FromType = FromValue->getType();
2574 if (FromType->isPointerTy())
2577 unsigned FromTypeBitSize = FromType->getScalarSizeInBits();
2578 unsigned ToTypeBitSize = Type->getScalarSizeInBits();
2603 /// possibly moving/undefing users to prevent use-before-def. Returns true if
2616 // Prevent use-before-def of To.
2627 if (DomPointAfterFrom && DII->getNextNonDebugInstruction() == &DomPoint) {
2629 DII->moveAfter(&DomPoint);
2641 Instruction *MarkedInstr = DVR->getMarker()->MarkedInstr;
2645 NextNonDebug = NextNonDebug->getNextNonDebugInstruction();
2649 DVR->removeFromParent();
2651 DomPoint.getParent()->insertDbgRecordAfter(DVR, &DomPoint);
2659 // Update debug users without use-before-def risk.
2668 DII->replaceVariableLocationOp(&From, &To);
2669 DII->setExpression(*DVRepl);
2681 DVR->replaceVariableLocationOp(&From, &To);
2682 DVR->setExpression(*DVRepl);
2701 /// allows semantically unequivalent bitcasts, such as <2 x i64> -> <4 x i32>,
2702 /// and also does not allow lossless pointer <-> integer conversions.
2709 // Handle compatible pointer <-> integer conversions.
2710 if (FromTy->isIntOrPtrTy() && ToTy->isIntOrPtrTy()) {
2732 auto Identity = [&](DbgVariableIntrinsic &DII) -> DbgValReplacement {
2735 auto IdentityDVR = [&](DbgVariableRecord &DVR) -> DbgValReplacement {
2739 // Handle no-op conversions.
2745 // Handle integer-to-integer widening and narrowing.
2747 if (FromTy->isIntegerTy() && ToTy->isIntegerTy()) {
2748 uint64_t FromBits = FromTy->getPrimitiveSizeInBits();
2749 uint64_t ToBits = ToTy->getPrimitiveSizeInBits();
2750 assert(FromBits != ToBits && "Unexpected no-op conversion");
2759 auto SignOrZeroExt = [&](DbgVariableIntrinsic &DII) -> DbgValReplacement {
2763 auto Signedness = Var->getSignedness();
2773 auto SignOrZeroExtDVR = [&](DbgVariableRecord &DVR) -> DbgValReplacement {
2777 auto Signedness = Var->getSignedness();
2789 // TODO: Floating-point conversions, vectors.
2796 // RemoveDIs: erase debug-info on this instruction manually.
2797 I->dropDbgRecords();
2798 for (Use &U : I->operands()) {
2800 if (isa<Instruction>(Op) && !Op->getType()->isTokenTy()) {
2801 U.set(PoisonValue::get(Op->getType()));
2815 // having to update as many def-use and use-def chains.
2816 Instruction *EndInst = BB->getTerminator(); // Last not to be deleted.
2820 while (EndInst != &BB->front()) {
2822 Instruction *Inst = &*--EndInst->getIterator();
2823 if (!Inst->use_empty() && !Inst->getType()->isTokenTy())
2824 Inst->replaceAllUsesWith(PoisonValue::get(Inst->getType()));
2825 if (Inst->isEHPad() || Inst->getType()->isTokenTy()) {
2828 Inst->dropDbgRecords();
2836 // RemoveDIs: erasing debug-info must be done manually.
2837 Inst->dropDbgRecords();
2838 Inst->eraseFromParent();
2846 BasicBlock *BB = I->getParent();
2849 MSSAU->changeToUnreachable(I);
2856 Successor->removePredecessor(BB, PreserveLCSSA);
2860 auto *UI = new UnreachableInst(I->getContext(), I->getIterator());
2861 UI->setDebugLoc(I->getDebugLoc());
2865 BasicBlock::iterator BBI = I->getIterator(), BBE = BB->end();
2867 if (!BBI->use_empty())
2868 BBI->replaceAllUsesWith(PoisonValue::get(BBI->getType()));
2869 BBI++->eraseFromParent();
2877 DTU->applyUpdates(Updates);
2879 BB->flushTerminatorDbgRecords();
2884 SmallVector<Value *, 8> Args(II->args());
2886 II->getOperandBundlesAsDefs(OpBundles);
2887 CallInst *NewCall = CallInst::Create(II->getFunctionType(),
2888 II->getCalledOperand(), Args, OpBundles);
2889 NewCall->setCallingConv(II->getCallingConv());
2890 NewCall->setAttributes(II->getAttributes());
2891 NewCall->setDebugLoc(II->getDebugLoc());
2892 NewCall->copyMetadata(*II);
2896 if (NewCall->extractProfTotalWeight(TotalWeight)) {
2897 // Set the total weight if it fits into i32, otherwise reset.
2898 MDBuilder MDB(NewCall->getContext());
2902 NewCall->setMetadata(LLVMContext::MD_prof, NewWeights);
2908 // changeToCall - Convert the specified invoke into a normal call.
2911 NewCall->takeName(II);
2912 NewCall->insertBefore(II);
2913 II->replaceAllUsesWith(NewCall);
2916 BasicBlock *NormalDestBB = II->getNormalDest();
2917 BranchInst::Create(NormalDestBB, II->getIterator());
2920 BasicBlock *BB = II->getParent();
2921 BasicBlock *UnwindDestBB = II->getUnwindDest();
2922 UnwindDestBB->removePredecessor(BB);
2923 II->eraseFromParent();
2925 DTU->applyUpdates({{DominatorTree::Delete, BB, UnwindDestBB}});
2932 BasicBlock *BB = CI->getParent();
2937 CI->getName() + ".noexc");
2940 BB->back().eraseFromParent();
2943 SmallVector<Value *, 8> InvokeArgs(CI->args());
2946 CI->getOperandBundlesAsDefs(OpBundles);
2953 InvokeInst::Create(CI->getFunctionType(), CI->getCalledOperand(), Split,
2954 UnwindEdge, InvokeArgs, OpBundles, CI->getName(), BB);
2955 II->setDebugLoc(CI->getDebugLoc());
2956 II->setCallingConv(CI->getCallingConv());
2957 II->setAttributes(CI->getAttributes());
2958 II->setMetadata(LLVMContext::MD_prof, CI->getMetadata(LLVMContext::MD_prof));
2961 DTU->applyUpdates({{DominatorTree::Insert, BB, UnwindEdge}});
2965 CI->replaceAllUsesWith(II);
2968 Split->front().eraseFromParent();
2988 Value *Callee = CI->getCalledOperand();
2991 auto IntrinsicID = F->getIntrinsicID();
2997 if (match(CI->getArgOperand(0), m_CombineOr(m_Zero(), m_Undef()))) {
3013 if (match(CI->getArgOperand(0), m_Zero()))
3014 if (!isa<UnreachableInst>(CI->getNextNode())) {
3015 changeToUnreachable(CI->getNextNode(), false, DTU);
3021 !NullPointerIsDefined(CI->getFunction(),
3022 cast<PointerType>(Callee->getType())
3023 ->getAddressSpace())) ||
3029 if (CI->doesNotReturn() && !CI->isMustTailCall()) {
3030 // If we found a call to a no-return function, insert an unreachable
3033 if (!isa<UnreachableInst>(CI->getNextNonDebugInstruction())) {
3035 changeToUnreachable(CI->getNextNonDebugInstruction(), false, DTU);
3046 if (SI->isVolatile()) continue;
3048 Value *Ptr = SI->getOperand(1);
3052 !NullPointerIsDefined(SI->getFunction(),
3053 SI->getPointerAddressSpace()))) {
3061 Instruction *Terminator = BB->getTerminator();
3064 Value *Callee = II->getCalledOperand();
3066 !NullPointerIsDefined(BB->getParent())) ||
3071 if (II->doesNotReturn() &&
3072 !isa<UnreachableInst>(II->getNormalDest()->front())) {
3073 // If we found an invoke of a no-return function,
3078 BasicBlock *OrigNormalDest = II->getNormalDest();
3079 OrigNormalDest->removePredecessor(II->getParent());
3080 LLVMContext &Ctx = II->getContext();
3082 Ctx, OrigNormalDest->getName() + ".unreachable",
3083 II->getFunction(), OrigNormalDest);
3085 II->setNormalDest(UnreachableNormalDest);
3087 DTU->applyUpdates(
3092 if (II->doesNotThrow() && canSimplifyInvokeNoUnwind(&F)) {
3093 if (II->use_empty() && !II->mayHaveSideEffects()) {
3095 BasicBlock *NormalDestBB = II->getNormalDest();
3096 BasicBlock *UnwindDestBB = II->getUnwindDest();
3097 BranchInst::Create(NormalDestBB, II->getIterator());
3098 UnwindDestBB->removePredecessor(II->getParent());
3099 II->eraseFromParent();
3101 DTU->applyUpdates({{DominatorTree::Delete, BB, UnwindDestBB}});
3120 CatchPad->value_op_begin(), CatchPad->value_op_end()));
3127 return LHS->isIdenticalTo(RHS);
3137 for (CatchSwitchInst::handler_iterator I = CatchSwitch->handler_begin(),
3138 E = CatchSwitch->handler_end();
3143 auto *CatchPad = cast<CatchPadInst>(HandlerBB->getFirstNonPHI());
3146 --NumPerSuccessorCases[HandlerBB];
3147 CatchSwitch->removeHandler(I);
3148 --I;
3149 --E;
3158 DTU->applyUpdates(Updates);
3171 Instruction *TI = BB->getTerminator();
3180 NewTI = CleanupReturnInst::Create(CRI->getCleanupPad(), nullptr, CRI->getIterator());
3181 UnwindDest = CRI->getUnwindDest();
3184 CatchSwitch->getParentPad(), nullptr, CatchSwitch->getNumHandlers(),
3185 CatchSwitch->getName(), CatchSwitch->getIterator());
3186 for (BasicBlock *PadBB : CatchSwitch->handlers())
3187 NewCatchSwitch->addHandler(PadBB);
3190 UnwindDest = CatchSwitch->getUnwindDest();
3195 NewTI->takeName(TI);
3196 NewTI->setDebugLoc(TI->getDebugLoc());
3197 UnwindDest->removePredecessor(BB);
3198 TI->replaceAllUsesWith(NewTI);
3199 TI->eraseFromParent();
3201 DTU->applyUpdates({{DominatorTree::Delete, BB, UnwindDest}});
3205 /// removeUnreachableBlocks - Remove blocks that are not reachable, even
3225 // Skip already-deleted blocks
3226 if (DTU && DTU->isBBPendingDeletion(&BB))
3238 MSSAU->removeBlocks(BlocksToRemove);
3248 K->dropUnknownNonDebugMetadata(KnownIDs);
3249 K->getAllMetadataOtherThanDebugLoc(Metadata);
3252 MDNode *JMD = J->getMetadata(Kind);
3257 K->setMetadata(Kind, nullptr); // Remove unknown metadata
3262 K->mergeDIAssignID(J);
3265 K->setMetadata(Kind, MDNode::getMostGenericTBAA(JMD, KMD));
3268 K->setMetadata(Kind, MDNode::getMostGenericAliasScope(JMD, KMD));
3272 K->setMetadata(Kind, MDNode::intersect(JMD, KMD));
3275 K->setMetadata(LLVMContext::MD_access_group,
3279 if (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef))
3280 K->setMetadata(Kind, MDNode::getMostGenericRange(JMD, KMD));
3283 K->setMetadata(Kind, MDNode::getMostGenericFPMath(JMD, KMD));
3289 K->setMetadata(Kind, JMD);
3292 if (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef))
3293 K->setMetadata(Kind, JMD);
3302 if (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef))
3303 K->setMetadata(
3309 K->setMetadata(Kind,
3318 K->setMetadata(Kind, JMD);
3322 K->setMetadata(Kind, JMD);
3326 K->setMetadata(Kind, MDNode::getMergedProfMetadata(KMD, JMD, K, J));
3331 // then we will just pick it from J - even when they are different.
3332 // Also make sure that K is load or store - f.e. combining bitcast with load
3336 if (auto *JMD = J->getMetadata(LLVMContext::MD_invariant_group))
3338 K->setMetadata(LLVMContext::MD_invariant_group, JMD);
3343 auto JMMRA = J->getMetadata(LLVMContext::MD_mmra);
3344 auto KMMRA = K->getMetadata(LLVMContext::MD_mmra);
3346 K->setMetadata(LLVMContext::MD_mmra,
3347 MMRAMetadata::combine(K->getContext(), JMMRA, KMMRA));
3414 if (NewType->isPointerTy())
3437 ReplInst->dropPoisonGeneratingFlags();
3443 ReplInst->andIRFlags(I);
3446 // same control-flow region (meaning that the execution of one
3451 // In general, GVN unifies expressions over different control-flow
3461 assert(From->getType() == To->getType());
3464 for (Use &U : llvm::make_early_inc_range(From->uses())) {
3468 From->printAsOperand(dbgs());
3477 assert(From->getType() == To->getType());
3478 auto *BB = From->getParent();
3481 for (Use &U : llvm::make_early_inc_range(From->uses())) {
3483 if (I->getParent() == BB)
3532 if (Call->hasFnAttr("gc-leaf-function"))
3534 if (const Function *F = Call->getCalledFunction()) {
3535 if (F->hasFnAttribute("gc-leaf-function"))
3538 if (auto IID = F->getIntrinsicID()) {
3548 // marked as 'gc-leaf-function.' All available Libcalls are
3549 // GC-leaf.
3563 if (NewTy->isPointerTy()) {
3570 if (!NewTy->isIntegerTy())
3577 ConstantPointerNull::get(cast<PointerType>(Ptr->getType())), ITy);
3596 if (!NewTy->isPointerTy())
3600 if (BitWidth == OldLI.getType()->getScalarSizeInBits() &&
3612 DII->eraseFromParent();
3614 DVR->eraseFromParent();
3627 // - Remove their debug intrinsic instructions.
3628 // - Set their debug locations to the values from the insertion point.
3641 for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) {
3643 I->dropUBImplyingAttrsAndMetadata();
3644 if (I->isUsedByMetadata())
3646 // RemoveDIs: drop debug-info too as the following code does.
3647 I->dropDbgRecords();
3648 if (I->isDebugOrPseudoInst()) {
3650 II = I->eraseFromParent();
3653 I->setDebugLoc(InsertPt->getDebugLoc());
3656 DomBlock->splice(InsertPt->getIterator(), BB, BB->begin(),
3657 BB->getTerminator()->getIterator());
3663 auto createIntegerExpression = [&DIB](const Constant &CV) -> DIExpression * {
3664 const APInt &API = cast<ConstantInt>(&CV)->getValue();
3676 const APFloat &APF = FP->getValueAPF();
3690 if (CE->getOpcode() == Instruction::IntToPtr) {
3691 const Value *V = CE->getOperand(0);
3703 DV->replaceVariableLocationOp(Op, I->second, /*AllowEmpty=*/true);
3707 auto I = Mapping.find(DA->getAddress());
3709 DA->setAddress(I->second);
3712 RemapDebugOperands(DVI, DVI->location_ops());
3715 for (DbgVariableRecord &DVR : filterDbgVars(Inst->getDbgRecordRange())) {
3738 enum { Unset = -1 };
3745 /// piece of a bswap or bitreverse if it can be proved that each non-zero bit in
3754 /// BitPart is returned with Provider set to %X and Provenance[24-31] set to
3755 /// [0-7].
3757 /// For vector types, all analysis is performed at the per-element level. No
3758 /// cross-element analysis is supported (shuffle/insertion/reduction), and all
3763 /// constructed in-place in the \c BPS map. Because of this \c BPS needs to
3768 /// Because we pass around references into \c BPS, we must use a container that
3772 std::map<Value *, std::optional<BitPart>> &BPS, int Depth,
3774 auto I = BPS.find(V);
3775 if (I != BPS.end())
3776 return I->second;
3778 auto &Result = BPS[V] = std::nullopt;
3779 auto BitWidth = V->getType()->getScalarSizeInBits();
3798 const auto &A = collectBitParts(X, MatchBSwaps, MatchBitReversals, BPS,
3800 if (!A || !A->Provider)
3803 const auto &B = collectBitParts(Y, MatchBSwaps, MatchBitReversals, BPS,
3805 if (!B || A->Provider != B->Provider)
3809 Result = BitPart(A->Provider, BitWidth);
3811 if (A->Provenance[BitIdx] != BitPart::Unset &&
3812 B->Provenance[BitIdx] != BitPart::Unset &&
3813 A->Provenance[BitIdx] != B->Provenance[BitIdx])
3816 if (A->Provenance[BitIdx] == BitPart::Unset)
3817 Result->Provenance[BitIdx] = B->Provenance[BitIdx];
3819 Result->Provenance[BitIdx] = A->Provenance[BitIdx];
3833 // For bswap-only, limit shift amounts to whole bytes, for an early exit.
3837 const auto &Res = collectBitParts(X, MatchBSwaps, MatchBitReversals, BPS,
3844 auto &P = Result->Provenance;
3845 if (I->getOpcode() == Instruction::Shl) {
3867 const auto &Res = collectBitParts(X, MatchBSwaps, MatchBitReversals, BPS,
3876 Result->Provenance[BitIdx] = BitPart::Unset;
3882 const auto &Res = collectBitParts(X, MatchBSwaps, MatchBitReversals, BPS,
3887 Result = BitPart(Res->Provider, BitWidth);
3888 auto NarrowBitWidth = X->getType()->getScalarSizeInBits();
3890 Result->Provenance[BitIdx] = Res->Provenance[BitIdx];
3892 Result->Provenance[BitIdx] = BitPart::Unset;
3898 const auto &Res = collectBitParts(X, MatchBSwaps, MatchBitReversals, BPS,
3903 Result = BitPart(Res->Provider, BitWidth);
3905 Result->Provenance[BitIdx] = Res->Provenance[BitIdx];
3909 // BITREVERSE - most likely due to us previous matching a partial
3912 const auto &Res = collectBitParts(X, MatchBSwaps, MatchBitReversals, BPS,
3917 Result = BitPart(Res->Provider, BitWidth);
3919 Result->Provenance[(BitWidth - 1) - BitIdx] = Res->Provenance[BitIdx];
3923 // BSWAP - most likely due to us previous matching a partial bswap.
3925 const auto &Res = collectBitParts(X, MatchBSwaps, MatchBitReversals, BPS,
3931 Result = BitPart(Res->Provider, BitWidth);
3935 Result->Provenance[(BitWidth - 8 - ByteBitOfs) + BitIdx] =
3936 Res->Provenance[ByteBitOfs + BitIdx];
3943 // fshl(X,Y,Z): (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3944 // fshr(X,Y,Z): (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3948 unsigned ModAmt = C->urem(BitWidth);
3949 if (cast<IntrinsicInst>(I)->getIntrinsicID() == Intrinsic::fshr)
3950 ModAmt = BitWidth - ModAmt;
3952 // For bswap-only, limit shift amounts to whole bytes, for an early exit.
3957 const auto &LHS = collectBitParts(X, MatchBSwaps, MatchBitReversals, BPS,
3959 if (!LHS || !LHS->Provider)
3962 const auto &RHS = collectBitParts(Y, MatchBSwaps, MatchBitReversals, BPS,
3964 if (!RHS || LHS->Provider != RHS->Provider)
3967 unsigned StartBitRHS = BitWidth - ModAmt;
3968 Result = BitPart(LHS->Provider, BitWidth);
3970 Result->Provenance[BitIdx + ModAmt] = LHS->Provenance[BitIdx];
3972 Result->Provenance[BitIdx] = RHS->Provenance[BitIdx + StartBitRHS];
3987 Result->Provenance[BitIdx] = BitIdx;
3999 return From == BitWidth - To - 1;
4004 return From == BitWidth - To - 1;
4017 Type *ITy = I->getType();
4018 if (!ITy->isIntOrIntVectorTy() || ITy->getScalarSizeInBits() > 128)
4023 std::map<Value *, std::optional<BitPart>> BPS;
4025 collectBitParts(I, MatchBSwaps, MatchBitReversals, BPS, 0, FoundRoot);
4028 ArrayRef<int8_t> BitProvenance = Res->Provenance;
4039 return false; // TODO - handle null value?
4040 DemandedTy = Type::getIntNTy(I->getContext(), BitProvenance.size());
4046 unsigned DemandedBW = DemandedTy->getScalarSizeInBits();
4047 if (DemandedBW > ITy->getScalarSizeInBits())
4075 Function *F = Intrinsic::getDeclaration(I->getModule(), Intrin, DemandedTy);
4076 Value *Provider = Res->Provider;
4079 if (DemandedTy != Provider->getType()) {
4081 CastInst::CreateIntegerCast(Provider, DemandedTy, false, "trunc", I->getIterator());
4086 Instruction *Result = CallInst::Create(F, Provider, "rev", I->getIterator());
4091 Result = BinaryOperator::Create(Instruction::And, Result, Mask, "mask", I->getIterator());
4096 if (ITy != Result->getType()) {
4097 auto *ExtInst = CastInst::CreateIntegerCast(Result, ITy, false, "zext", I->getIterator());
4105 // them with target-specific intrinsics. Since that'd skip our interceptors
4111 Function *F = CI->getCalledFunction();
4113 if (F && !F->hasLocalLinkage() && F->hasName() &&
4114 TLI->getLibFunc(F->getName(), Func) && TLI->hasOptimizedCodeGen(Func) &&
4115 !F->doesNotAccessMemory())
4116 CI->addFnAttr(Attribute::NoBuiltin);
4121 if (I->getOperand(OpIdx)->getType()->isMetadataTy())
4125 if (!isa<Constant>(I->getOperand(OpIdx)))
4128 switch (I->getOpcode()) {
4139 // Constant bundle operands may need to retain their constant-ness for
4148 OpIdx >= CB.getFunctionType()->getNumParams()) {
4179 // want to make them non-constant.
4180 return !cast<AllocaInst>(I)->isStaticAlloca();
4205 Parent = Inst->getParent();
4207 Parent = &Arg->getParent()->getEntryBlock();
4211 for (User *U : Condition->users())
4213 if (I->getParent() == Parent && match(I, m_Not(m_Specific(Condition))))
4218 BinaryOperator::CreateNot(Condition, Condition->getName() + ".inv");
4220 Inverted->insertAfter(Inst);
4222 Inverted->insertBefore(&*Parent->getFirstInsertionPt());
4251 // can infer by inspecting arguments of argmemonly-ish functions.