15ffd83dbSDimitry Andric //===- AssumeBundleBuilder.cpp - tools to preserve informations -*- C++ -*-===// 25ffd83dbSDimitry Andric // 35ffd83dbSDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 45ffd83dbSDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 55ffd83dbSDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 65ffd83dbSDimitry Andric // 75ffd83dbSDimitry Andric //===----------------------------------------------------------------------===// 85ffd83dbSDimitry Andric 95ffd83dbSDimitry Andric #include "llvm/Transforms/Utils/AssumeBundleBuilder.h" 105ffd83dbSDimitry Andric #include "llvm/ADT/DepthFirstIterator.h" 115ffd83dbSDimitry Andric #include "llvm/ADT/MapVector.h" 125ffd83dbSDimitry Andric #include "llvm/ADT/Statistic.h" 135ffd83dbSDimitry Andric #include "llvm/Analysis/AssumeBundleQueries.h" 145ffd83dbSDimitry Andric #include "llvm/Analysis/AssumptionCache.h" 155ffd83dbSDimitry Andric #include "llvm/Analysis/ValueTracking.h" 165ffd83dbSDimitry Andric #include "llvm/IR/Dominators.h" 175ffd83dbSDimitry Andric #include "llvm/IR/Function.h" 185ffd83dbSDimitry Andric #include "llvm/IR/InstIterator.h" 195ffd83dbSDimitry Andric #include "llvm/IR/IntrinsicInst.h" 205ffd83dbSDimitry Andric #include "llvm/IR/Module.h" 2181ad6265SDimitry Andric #include "llvm/IR/Operator.h" 225ffd83dbSDimitry Andric #include "llvm/InitializePasses.h" 235ffd83dbSDimitry Andric #include "llvm/Support/CommandLine.h" 245ffd83dbSDimitry Andric #include "llvm/Support/DebugCounter.h" 255ffd83dbSDimitry Andric #include "llvm/Transforms/Utils/Local.h" 265ffd83dbSDimitry Andric 275ffd83dbSDimitry Andric using namespace llvm; 285ffd83dbSDimitry Andric 29fe6060f1SDimitry Andric namespace llvm { 305ffd83dbSDimitry Andric cl::opt<bool> ShouldPreserveAllAttributes( 315ffd83dbSDimitry Andric "assume-preserve-all", cl::init(false), cl::Hidden, 325ffd83dbSDimitry Andric cl::desc("enable preservation of all attrbitues. even those that are " 335ffd83dbSDimitry Andric "unlikely to be usefull")); 345ffd83dbSDimitry Andric 355ffd83dbSDimitry Andric cl::opt<bool> EnableKnowledgeRetention( 365ffd83dbSDimitry Andric "enable-knowledge-retention", cl::init(false), cl::Hidden, 375ffd83dbSDimitry Andric cl::desc( 385ffd83dbSDimitry Andric "enable preservation of attributes throughout code transformation")); 39fe6060f1SDimitry Andric } // namespace llvm 40fe6060f1SDimitry Andric 41fe6060f1SDimitry Andric #define DEBUG_TYPE "assume-builder" 425ffd83dbSDimitry Andric 435ffd83dbSDimitry Andric STATISTIC(NumAssumeBuilt, "Number of assume built by the assume builder"); 445ffd83dbSDimitry Andric STATISTIC(NumBundlesInAssumes, "Total number of Bundles in the assume built"); 455ffd83dbSDimitry Andric STATISTIC(NumAssumesMerged, 465ffd83dbSDimitry Andric "Number of assume merged by the assume simplify pass"); 475ffd83dbSDimitry Andric STATISTIC(NumAssumesRemoved, 485ffd83dbSDimitry Andric "Number of assume removed by the assume simplify pass"); 495ffd83dbSDimitry Andric 505ffd83dbSDimitry Andric DEBUG_COUNTER(BuildAssumeCounter, "assume-builder-counter", 515ffd83dbSDimitry Andric "Controls which assumes gets created"); 525ffd83dbSDimitry Andric 535ffd83dbSDimitry Andric namespace { 545ffd83dbSDimitry Andric 555ffd83dbSDimitry Andric bool isUsefullToPreserve(Attribute::AttrKind Kind) { 565ffd83dbSDimitry Andric switch (Kind) { 575ffd83dbSDimitry Andric case Attribute::NonNull: 58e8d8bef9SDimitry Andric case Attribute::NoUndef: 595ffd83dbSDimitry Andric case Attribute::Alignment: 605ffd83dbSDimitry Andric case Attribute::Dereferenceable: 615ffd83dbSDimitry Andric case Attribute::DereferenceableOrNull: 625ffd83dbSDimitry Andric case Attribute::Cold: 635ffd83dbSDimitry Andric return true; 645ffd83dbSDimitry Andric default: 655ffd83dbSDimitry Andric return false; 665ffd83dbSDimitry Andric } 675ffd83dbSDimitry Andric } 685ffd83dbSDimitry Andric 695ffd83dbSDimitry Andric /// This function will try to transform the given knowledge into a more 705ffd83dbSDimitry Andric /// canonical one. the canonical knowledge maybe the given one. 71349cc55cSDimitry Andric RetainedKnowledge canonicalizedKnowledge(RetainedKnowledge RK, 72349cc55cSDimitry Andric const DataLayout &DL) { 735ffd83dbSDimitry Andric switch (RK.AttrKind) { 745ffd83dbSDimitry Andric default: 755ffd83dbSDimitry Andric return RK; 765ffd83dbSDimitry Andric case Attribute::NonNull: 77e8d8bef9SDimitry Andric RK.WasOn = getUnderlyingObject(RK.WasOn); 785ffd83dbSDimitry Andric return RK; 795ffd83dbSDimitry Andric case Attribute::Alignment: { 805ffd83dbSDimitry Andric Value *V = RK.WasOn->stripInBoundsOffsets([&](const Value *Strip) { 815ffd83dbSDimitry Andric if (auto *GEP = dyn_cast<GEPOperator>(Strip)) 825ffd83dbSDimitry Andric RK.ArgValue = 83fe6060f1SDimitry Andric MinAlign(RK.ArgValue, GEP->getMaxPreservedAlignment(DL).value()); 845ffd83dbSDimitry Andric }); 855ffd83dbSDimitry Andric RK.WasOn = V; 865ffd83dbSDimitry Andric return RK; 875ffd83dbSDimitry Andric } 885ffd83dbSDimitry Andric case Attribute::Dereferenceable: 895ffd83dbSDimitry Andric case Attribute::DereferenceableOrNull: { 905ffd83dbSDimitry Andric int64_t Offset = 0; 91fe6060f1SDimitry Andric Value *V = GetPointerBaseWithConstantOffset(RK.WasOn, Offset, DL, 92fe6060f1SDimitry Andric /*AllowNonInBounds*/ false); 935ffd83dbSDimitry Andric if (Offset < 0) 945ffd83dbSDimitry Andric return RK; 955ffd83dbSDimitry Andric RK.ArgValue = RK.ArgValue + Offset; 965ffd83dbSDimitry Andric RK.WasOn = V; 975ffd83dbSDimitry Andric } 985ffd83dbSDimitry Andric } 995ffd83dbSDimitry Andric return RK; 1005ffd83dbSDimitry Andric } 1015ffd83dbSDimitry Andric 1025ffd83dbSDimitry Andric /// This class contain all knowledge that have been gather while building an 1035ffd83dbSDimitry Andric /// llvm.assume and the function to manipulate it. 1045ffd83dbSDimitry Andric struct AssumeBuilderState { 1055ffd83dbSDimitry Andric Module *M; 1065ffd83dbSDimitry Andric 1075ffd83dbSDimitry Andric using MapKey = std::pair<Value *, Attribute::AttrKind>; 108349cc55cSDimitry Andric SmallMapVector<MapKey, uint64_t, 8> AssumedKnowledgeMap; 109fe6060f1SDimitry Andric Instruction *InstBeingModified = nullptr; 1105ffd83dbSDimitry Andric AssumptionCache* AC = nullptr; 1115ffd83dbSDimitry Andric DominatorTree* DT = nullptr; 1125ffd83dbSDimitry Andric 1135ffd83dbSDimitry Andric AssumeBuilderState(Module *M, Instruction *I = nullptr, 1145ffd83dbSDimitry Andric AssumptionCache *AC = nullptr, DominatorTree *DT = nullptr) 115fe6060f1SDimitry Andric : M(M), InstBeingModified(I), AC(AC), DT(DT) {} 1165ffd83dbSDimitry Andric 1175ffd83dbSDimitry Andric bool tryToPreserveWithoutAddingAssume(RetainedKnowledge RK) { 118fe6060f1SDimitry Andric if (!InstBeingModified || !RK.WasOn) 1195ffd83dbSDimitry Andric return false; 1205ffd83dbSDimitry Andric bool HasBeenPreserved = false; 1215ffd83dbSDimitry Andric Use* ToUpdate = nullptr; 1225ffd83dbSDimitry Andric getKnowledgeForValue( 1235ffd83dbSDimitry Andric RK.WasOn, {RK.AttrKind}, AC, 1245ffd83dbSDimitry Andric [&](RetainedKnowledge RKOther, Instruction *Assume, 1255ffd83dbSDimitry Andric const CallInst::BundleOpInfo *Bundle) { 126fe6060f1SDimitry Andric if (!isValidAssumeForContext(Assume, InstBeingModified, DT)) 1275ffd83dbSDimitry Andric return false; 1285ffd83dbSDimitry Andric if (RKOther.ArgValue >= RK.ArgValue) { 1295ffd83dbSDimitry Andric HasBeenPreserved = true; 1305ffd83dbSDimitry Andric return true; 131fe6060f1SDimitry Andric } else if (isValidAssumeForContext(InstBeingModified, Assume, DT)) { 1325ffd83dbSDimitry Andric HasBeenPreserved = true; 1335ffd83dbSDimitry Andric IntrinsicInst *Intr = cast<IntrinsicInst>(Assume); 1345ffd83dbSDimitry Andric ToUpdate = &Intr->op_begin()[Bundle->Begin + ABA_Argument]; 1355ffd83dbSDimitry Andric return true; 1365ffd83dbSDimitry Andric } 1375ffd83dbSDimitry Andric return false; 1385ffd83dbSDimitry Andric }); 1395ffd83dbSDimitry Andric if (ToUpdate) 1405ffd83dbSDimitry Andric ToUpdate->set( 1415ffd83dbSDimitry Andric ConstantInt::get(Type::getInt64Ty(M->getContext()), RK.ArgValue)); 1425ffd83dbSDimitry Andric return HasBeenPreserved; 1435ffd83dbSDimitry Andric } 1445ffd83dbSDimitry Andric 1455ffd83dbSDimitry Andric bool isKnowledgeWorthPreserving(RetainedKnowledge RK) { 1465ffd83dbSDimitry Andric if (!RK) 1475ffd83dbSDimitry Andric return false; 1485ffd83dbSDimitry Andric if (!RK.WasOn) 1495ffd83dbSDimitry Andric return true; 1505ffd83dbSDimitry Andric if (RK.WasOn->getType()->isPointerTy()) { 151e8d8bef9SDimitry Andric Value *UnderlyingPtr = getUnderlyingObject(RK.WasOn); 1525ffd83dbSDimitry Andric if (isa<AllocaInst>(UnderlyingPtr) || isa<GlobalValue>(UnderlyingPtr)) 1535ffd83dbSDimitry Andric return false; 1545ffd83dbSDimitry Andric } 1555ffd83dbSDimitry Andric if (auto *Arg = dyn_cast<Argument>(RK.WasOn)) { 1565ffd83dbSDimitry Andric if (Arg->hasAttribute(RK.AttrKind) && 157fe6060f1SDimitry Andric (!Attribute::isIntAttrKind(RK.AttrKind) || 1585ffd83dbSDimitry Andric Arg->getAttribute(RK.AttrKind).getValueAsInt() >= RK.ArgValue)) 1595ffd83dbSDimitry Andric return false; 1605ffd83dbSDimitry Andric return true; 1615ffd83dbSDimitry Andric } 1625ffd83dbSDimitry Andric if (auto *Inst = dyn_cast<Instruction>(RK.WasOn)) 1635ffd83dbSDimitry Andric if (wouldInstructionBeTriviallyDead(Inst)) { 1645ffd83dbSDimitry Andric if (RK.WasOn->use_empty()) 1655ffd83dbSDimitry Andric return false; 1665ffd83dbSDimitry Andric Use *SingleUse = RK.WasOn->getSingleUndroppableUse(); 167fe6060f1SDimitry Andric if (SingleUse && SingleUse->getUser() == InstBeingModified) 1685ffd83dbSDimitry Andric return false; 1695ffd83dbSDimitry Andric } 1705ffd83dbSDimitry Andric return true; 1715ffd83dbSDimitry Andric } 1725ffd83dbSDimitry Andric 1735ffd83dbSDimitry Andric void addKnowledge(RetainedKnowledge RK) { 174fe6060f1SDimitry Andric RK = canonicalizedKnowledge(RK, M->getDataLayout()); 1755ffd83dbSDimitry Andric 1765ffd83dbSDimitry Andric if (!isKnowledgeWorthPreserving(RK)) 1775ffd83dbSDimitry Andric return; 1785ffd83dbSDimitry Andric 1795ffd83dbSDimitry Andric if (tryToPreserveWithoutAddingAssume(RK)) 1805ffd83dbSDimitry Andric return; 1815ffd83dbSDimitry Andric MapKey Key{RK.WasOn, RK.AttrKind}; 1825ffd83dbSDimitry Andric auto Lookup = AssumedKnowledgeMap.find(Key); 1835ffd83dbSDimitry Andric if (Lookup == AssumedKnowledgeMap.end()) { 1845ffd83dbSDimitry Andric AssumedKnowledgeMap[Key] = RK.ArgValue; 1855ffd83dbSDimitry Andric return; 1865ffd83dbSDimitry Andric } 1875ffd83dbSDimitry Andric assert(((Lookup->second == 0 && RK.ArgValue == 0) || 1885ffd83dbSDimitry Andric (Lookup->second != 0 && RK.ArgValue != 0)) && 1895ffd83dbSDimitry Andric "inconsistent argument value"); 1905ffd83dbSDimitry Andric 1915ffd83dbSDimitry Andric /// This is only desirable because for all attributes taking an argument 1925ffd83dbSDimitry Andric /// higher is better. 1935ffd83dbSDimitry Andric Lookup->second = std::max(Lookup->second, RK.ArgValue); 1945ffd83dbSDimitry Andric } 1955ffd83dbSDimitry Andric 1965ffd83dbSDimitry Andric void addAttribute(Attribute Attr, Value *WasOn) { 1975ffd83dbSDimitry Andric if (Attr.isTypeAttribute() || Attr.isStringAttribute() || 1985ffd83dbSDimitry Andric (!ShouldPreserveAllAttributes && 1995ffd83dbSDimitry Andric !isUsefullToPreserve(Attr.getKindAsEnum()))) 2005ffd83dbSDimitry Andric return; 201349cc55cSDimitry Andric uint64_t AttrArg = 0; 2025ffd83dbSDimitry Andric if (Attr.isIntAttribute()) 2035ffd83dbSDimitry Andric AttrArg = Attr.getValueAsInt(); 2045ffd83dbSDimitry Andric addKnowledge({Attr.getKindAsEnum(), AttrArg, WasOn}); 2055ffd83dbSDimitry Andric } 2065ffd83dbSDimitry Andric 2075ffd83dbSDimitry Andric void addCall(const CallBase *Call) { 208349cc55cSDimitry Andric auto addAttrList = [&](AttributeList AttrList, unsigned NumArgs) { 209349cc55cSDimitry Andric for (unsigned Idx = 0; Idx < NumArgs; Idx++) 210349cc55cSDimitry Andric for (Attribute Attr : AttrList.getParamAttrs(Idx)) { 211fe6060f1SDimitry Andric bool IsPoisonAttr = Attr.hasAttribute(Attribute::NonNull) || 212fe6060f1SDimitry Andric Attr.hasAttribute(Attribute::Alignment); 213349cc55cSDimitry Andric if (!IsPoisonAttr || Call->isPassingUndefUB(Idx)) 214349cc55cSDimitry Andric addAttribute(Attr, Call->getArgOperand(Idx)); 215fe6060f1SDimitry Andric } 216349cc55cSDimitry Andric for (Attribute Attr : AttrList.getFnAttrs()) 2175ffd83dbSDimitry Andric addAttribute(Attr, nullptr); 2185ffd83dbSDimitry Andric }; 219349cc55cSDimitry Andric addAttrList(Call->getAttributes(), Call->arg_size()); 2205ffd83dbSDimitry Andric if (Function *Fn = Call->getCalledFunction()) 221349cc55cSDimitry Andric addAttrList(Fn->getAttributes(), Fn->arg_size()); 2225ffd83dbSDimitry Andric } 2235ffd83dbSDimitry Andric 224fe6060f1SDimitry Andric AssumeInst *build() { 2255ffd83dbSDimitry Andric if (AssumedKnowledgeMap.empty()) 2265ffd83dbSDimitry Andric return nullptr; 2275ffd83dbSDimitry Andric if (!DebugCounter::shouldExecute(BuildAssumeCounter)) 2285ffd83dbSDimitry Andric return nullptr; 2295ffd83dbSDimitry Andric Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume); 2305ffd83dbSDimitry Andric LLVMContext &C = M->getContext(); 2315ffd83dbSDimitry Andric SmallVector<OperandBundleDef, 8> OpBundle; 2325ffd83dbSDimitry Andric for (auto &MapElem : AssumedKnowledgeMap) { 2335ffd83dbSDimitry Andric SmallVector<Value *, 2> Args; 2345ffd83dbSDimitry Andric if (MapElem.first.first) 2355ffd83dbSDimitry Andric Args.push_back(MapElem.first.first); 2365ffd83dbSDimitry Andric 2375ffd83dbSDimitry Andric /// This is only valid because for all attribute that currently exist a 2385ffd83dbSDimitry Andric /// value of 0 is useless. and should not be preserved. 2395ffd83dbSDimitry Andric if (MapElem.second) 2405ffd83dbSDimitry Andric Args.push_back(ConstantInt::get(Type::getInt64Ty(M->getContext()), 2415ffd83dbSDimitry Andric MapElem.second)); 2425ffd83dbSDimitry Andric OpBundle.push_back(OperandBundleDefT<Value *>( 2435ffd83dbSDimitry Andric std::string(Attribute::getNameFromAttrKind(MapElem.first.second)), 2445ffd83dbSDimitry Andric Args)); 2455ffd83dbSDimitry Andric NumBundlesInAssumes++; 2465ffd83dbSDimitry Andric } 2475ffd83dbSDimitry Andric NumAssumeBuilt++; 248fe6060f1SDimitry Andric return cast<AssumeInst>(CallInst::Create( 2495ffd83dbSDimitry Andric FnAssume, ArrayRef<Value *>({ConstantInt::getTrue(C)}), OpBundle)); 2505ffd83dbSDimitry Andric } 2515ffd83dbSDimitry Andric 2525ffd83dbSDimitry Andric void addAccessedPtr(Instruction *MemInst, Value *Pointer, Type *AccType, 2535ffd83dbSDimitry Andric MaybeAlign MA) { 2545ffd83dbSDimitry Andric unsigned DerefSize = MemInst->getModule() 2555ffd83dbSDimitry Andric ->getDataLayout() 2565ffd83dbSDimitry Andric .getTypeStoreSize(AccType) 257bdd1243dSDimitry Andric .getKnownMinValue(); 2585ffd83dbSDimitry Andric if (DerefSize != 0) { 2595ffd83dbSDimitry Andric addKnowledge({Attribute::Dereferenceable, DerefSize, Pointer}); 2605ffd83dbSDimitry Andric if (!NullPointerIsDefined(MemInst->getFunction(), 2615ffd83dbSDimitry Andric Pointer->getType()->getPointerAddressSpace())) 2625ffd83dbSDimitry Andric addKnowledge({Attribute::NonNull, 0u, Pointer}); 2635ffd83dbSDimitry Andric } 2645ffd83dbSDimitry Andric if (MA.valueOrOne() > 1) 265349cc55cSDimitry Andric addKnowledge({Attribute::Alignment, MA.valueOrOne().value(), Pointer}); 2665ffd83dbSDimitry Andric } 2675ffd83dbSDimitry Andric 2685ffd83dbSDimitry Andric void addInstruction(Instruction *I) { 2695ffd83dbSDimitry Andric if (auto *Call = dyn_cast<CallBase>(I)) 2705ffd83dbSDimitry Andric return addCall(Call); 2715ffd83dbSDimitry Andric if (auto *Load = dyn_cast<LoadInst>(I)) 2725ffd83dbSDimitry Andric return addAccessedPtr(I, Load->getPointerOperand(), Load->getType(), 2735ffd83dbSDimitry Andric Load->getAlign()); 2745ffd83dbSDimitry Andric if (auto *Store = dyn_cast<StoreInst>(I)) 2755ffd83dbSDimitry Andric return addAccessedPtr(I, Store->getPointerOperand(), 2765ffd83dbSDimitry Andric Store->getValueOperand()->getType(), 2775ffd83dbSDimitry Andric Store->getAlign()); 2785ffd83dbSDimitry Andric // TODO: Add support for the other Instructions. 2795ffd83dbSDimitry Andric // TODO: Maybe we should look around and merge with other llvm.assume. 2805ffd83dbSDimitry Andric } 2815ffd83dbSDimitry Andric }; 2825ffd83dbSDimitry Andric 2835ffd83dbSDimitry Andric } // namespace 2845ffd83dbSDimitry Andric 285fe6060f1SDimitry Andric AssumeInst *llvm::buildAssumeFromInst(Instruction *I) { 2865ffd83dbSDimitry Andric if (!EnableKnowledgeRetention) 2875ffd83dbSDimitry Andric return nullptr; 2885ffd83dbSDimitry Andric AssumeBuilderState Builder(I->getModule()); 2895ffd83dbSDimitry Andric Builder.addInstruction(I); 2905ffd83dbSDimitry Andric return Builder.build(); 2915ffd83dbSDimitry Andric } 2925ffd83dbSDimitry Andric 293*06c3fb27SDimitry Andric bool llvm::salvageKnowledge(Instruction *I, AssumptionCache *AC, 2945ffd83dbSDimitry Andric DominatorTree *DT) { 2955ffd83dbSDimitry Andric if (!EnableKnowledgeRetention || I->isTerminator()) 296*06c3fb27SDimitry Andric return false; 297*06c3fb27SDimitry Andric bool Changed = false; 2985ffd83dbSDimitry Andric AssumeBuilderState Builder(I->getModule(), I, AC, DT); 2995ffd83dbSDimitry Andric Builder.addInstruction(I); 300fe6060f1SDimitry Andric if (auto *Intr = Builder.build()) { 3015ffd83dbSDimitry Andric Intr->insertBefore(I); 302*06c3fb27SDimitry Andric Changed = true; 3035ffd83dbSDimitry Andric if (AC) 3045ffd83dbSDimitry Andric AC->registerAssumption(Intr); 3055ffd83dbSDimitry Andric } 306*06c3fb27SDimitry Andric return Changed; 3075ffd83dbSDimitry Andric } 3085ffd83dbSDimitry Andric 309fe6060f1SDimitry Andric AssumeInst * 310fe6060f1SDimitry Andric llvm::buildAssumeFromKnowledge(ArrayRef<RetainedKnowledge> Knowledge, 311fe6060f1SDimitry Andric Instruction *CtxI, AssumptionCache *AC, 312fe6060f1SDimitry Andric DominatorTree *DT) { 313fe6060f1SDimitry Andric AssumeBuilderState Builder(CtxI->getModule(), CtxI, AC, DT); 314fe6060f1SDimitry Andric for (const RetainedKnowledge &RK : Knowledge) 315fe6060f1SDimitry Andric Builder.addKnowledge(RK); 316fe6060f1SDimitry Andric return Builder.build(); 317fe6060f1SDimitry Andric } 318fe6060f1SDimitry Andric 319fe6060f1SDimitry Andric RetainedKnowledge llvm::simplifyRetainedKnowledge(AssumeInst *Assume, 320fe6060f1SDimitry Andric RetainedKnowledge RK, 321fe6060f1SDimitry Andric AssumptionCache *AC, 322fe6060f1SDimitry Andric DominatorTree *DT) { 323fe6060f1SDimitry Andric AssumeBuilderState Builder(Assume->getModule(), Assume, AC, DT); 324fe6060f1SDimitry Andric RK = canonicalizedKnowledge(RK, Assume->getModule()->getDataLayout()); 325fe6060f1SDimitry Andric 326fe6060f1SDimitry Andric if (!Builder.isKnowledgeWorthPreserving(RK)) 327fe6060f1SDimitry Andric return RetainedKnowledge::none(); 328fe6060f1SDimitry Andric 329fe6060f1SDimitry Andric if (Builder.tryToPreserveWithoutAddingAssume(RK)) 330fe6060f1SDimitry Andric return RetainedKnowledge::none(); 331fe6060f1SDimitry Andric return RK; 332fe6060f1SDimitry Andric } 333fe6060f1SDimitry Andric 3345ffd83dbSDimitry Andric namespace { 3355ffd83dbSDimitry Andric 3365ffd83dbSDimitry Andric struct AssumeSimplify { 3375ffd83dbSDimitry Andric Function &F; 3385ffd83dbSDimitry Andric AssumptionCache &AC; 3395ffd83dbSDimitry Andric DominatorTree *DT; 3405ffd83dbSDimitry Andric LLVMContext &C; 3415ffd83dbSDimitry Andric SmallDenseSet<IntrinsicInst *> CleanupToDo; 3425ffd83dbSDimitry Andric StringMapEntry<uint32_t> *IgnoreTag; 3435ffd83dbSDimitry Andric SmallDenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 4>, 8> BBToAssume; 3445ffd83dbSDimitry Andric bool MadeChange = false; 3455ffd83dbSDimitry Andric 3465ffd83dbSDimitry Andric AssumeSimplify(Function &F, AssumptionCache &AC, DominatorTree *DT, 3475ffd83dbSDimitry Andric LLVMContext &C) 3485ffd83dbSDimitry Andric : F(F), AC(AC), DT(DT), C(C), 3495ffd83dbSDimitry Andric IgnoreTag(C.getOrInsertBundleTag(IgnoreBundleTag)) {} 3505ffd83dbSDimitry Andric 3515ffd83dbSDimitry Andric void buildMapping(bool FilterBooleanArgument) { 3525ffd83dbSDimitry Andric BBToAssume.clear(); 3535ffd83dbSDimitry Andric for (Value *V : AC.assumptions()) { 3545ffd83dbSDimitry Andric if (!V) 3555ffd83dbSDimitry Andric continue; 3565ffd83dbSDimitry Andric IntrinsicInst *Assume = cast<IntrinsicInst>(V); 3575ffd83dbSDimitry Andric if (FilterBooleanArgument) { 3585ffd83dbSDimitry Andric auto *Arg = dyn_cast<ConstantInt>(Assume->getOperand(0)); 3595ffd83dbSDimitry Andric if (!Arg || Arg->isZero()) 3605ffd83dbSDimitry Andric continue; 3615ffd83dbSDimitry Andric } 3625ffd83dbSDimitry Andric BBToAssume[Assume->getParent()].push_back(Assume); 3635ffd83dbSDimitry Andric } 3645ffd83dbSDimitry Andric 3655ffd83dbSDimitry Andric for (auto &Elem : BBToAssume) { 3665ffd83dbSDimitry Andric llvm::sort(Elem.second, 3675ffd83dbSDimitry Andric [](const IntrinsicInst *LHS, const IntrinsicInst *RHS) { 3685ffd83dbSDimitry Andric return LHS->comesBefore(RHS); 3695ffd83dbSDimitry Andric }); 3705ffd83dbSDimitry Andric } 3715ffd83dbSDimitry Andric } 3725ffd83dbSDimitry Andric 3735ffd83dbSDimitry Andric /// Remove all asumes in CleanupToDo if there boolean argument is true and 3745ffd83dbSDimitry Andric /// ForceCleanup is set or the assume doesn't hold valuable knowledge. 3755ffd83dbSDimitry Andric void RunCleanup(bool ForceCleanup) { 3765ffd83dbSDimitry Andric for (IntrinsicInst *Assume : CleanupToDo) { 3775ffd83dbSDimitry Andric auto *Arg = dyn_cast<ConstantInt>(Assume->getOperand(0)); 3785ffd83dbSDimitry Andric if (!Arg || Arg->isZero() || 379fe6060f1SDimitry Andric (!ForceCleanup && 380fe6060f1SDimitry Andric !isAssumeWithEmptyBundle(cast<AssumeInst>(*Assume)))) 3815ffd83dbSDimitry Andric continue; 3825ffd83dbSDimitry Andric MadeChange = true; 3835ffd83dbSDimitry Andric if (ForceCleanup) 3845ffd83dbSDimitry Andric NumAssumesMerged++; 3855ffd83dbSDimitry Andric else 3865ffd83dbSDimitry Andric NumAssumesRemoved++; 3875ffd83dbSDimitry Andric Assume->eraseFromParent(); 3885ffd83dbSDimitry Andric } 3895ffd83dbSDimitry Andric CleanupToDo.clear(); 3905ffd83dbSDimitry Andric } 3915ffd83dbSDimitry Andric 3925ffd83dbSDimitry Andric /// Remove knowledge stored in assume when it is already know by an attribute 3935ffd83dbSDimitry Andric /// or an other assume. This can when valid update an existing knowledge in an 3945ffd83dbSDimitry Andric /// attribute or an other assume. 3955ffd83dbSDimitry Andric void dropRedundantKnowledge() { 3965ffd83dbSDimitry Andric struct MapValue { 3975ffd83dbSDimitry Andric IntrinsicInst *Assume; 398349cc55cSDimitry Andric uint64_t ArgValue; 3995ffd83dbSDimitry Andric CallInst::BundleOpInfo *BOI; 4005ffd83dbSDimitry Andric }; 4015ffd83dbSDimitry Andric buildMapping(false); 4025ffd83dbSDimitry Andric SmallDenseMap<std::pair<Value *, Attribute::AttrKind>, 4035ffd83dbSDimitry Andric SmallVector<MapValue, 2>, 16> 4045ffd83dbSDimitry Andric Knowledge; 4055ffd83dbSDimitry Andric for (BasicBlock *BB : depth_first(&F)) 4065ffd83dbSDimitry Andric for (Value *V : BBToAssume[BB]) { 4075ffd83dbSDimitry Andric if (!V) 4085ffd83dbSDimitry Andric continue; 4095ffd83dbSDimitry Andric IntrinsicInst *Assume = cast<IntrinsicInst>(V); 4105ffd83dbSDimitry Andric for (CallInst::BundleOpInfo &BOI : Assume->bundle_op_infos()) { 4115ffd83dbSDimitry Andric auto RemoveFromAssume = [&]() { 4125ffd83dbSDimitry Andric CleanupToDo.insert(Assume); 4135ffd83dbSDimitry Andric if (BOI.Begin != BOI.End) { 4145ffd83dbSDimitry Andric Use *U = &Assume->op_begin()[BOI.Begin + ABA_WasOn]; 4155ffd83dbSDimitry Andric U->set(UndefValue::get(U->get()->getType())); 4165ffd83dbSDimitry Andric } 4175ffd83dbSDimitry Andric BOI.Tag = IgnoreTag; 4185ffd83dbSDimitry Andric }; 4195ffd83dbSDimitry Andric if (BOI.Tag == IgnoreTag) { 4205ffd83dbSDimitry Andric CleanupToDo.insert(Assume); 4215ffd83dbSDimitry Andric continue; 4225ffd83dbSDimitry Andric } 423fe6060f1SDimitry Andric RetainedKnowledge RK = 424fe6060f1SDimitry Andric getKnowledgeFromBundle(cast<AssumeInst>(*Assume), BOI); 4255ffd83dbSDimitry Andric if (auto *Arg = dyn_cast_or_null<Argument>(RK.WasOn)) { 4265ffd83dbSDimitry Andric bool HasSameKindAttr = Arg->hasAttribute(RK.AttrKind); 4275ffd83dbSDimitry Andric if (HasSameKindAttr) 428fe6060f1SDimitry Andric if (!Attribute::isIntAttrKind(RK.AttrKind) || 4295ffd83dbSDimitry Andric Arg->getAttribute(RK.AttrKind).getValueAsInt() >= 4305ffd83dbSDimitry Andric RK.ArgValue) { 4315ffd83dbSDimitry Andric RemoveFromAssume(); 4325ffd83dbSDimitry Andric continue; 4335ffd83dbSDimitry Andric } 4345ffd83dbSDimitry Andric if (isValidAssumeForContext( 4355ffd83dbSDimitry Andric Assume, &*F.getEntryBlock().getFirstInsertionPt()) || 4365ffd83dbSDimitry Andric Assume == &*F.getEntryBlock().getFirstInsertionPt()) { 4375ffd83dbSDimitry Andric if (HasSameKindAttr) 4385ffd83dbSDimitry Andric Arg->removeAttr(RK.AttrKind); 4395ffd83dbSDimitry Andric Arg->addAttr(Attribute::get(C, RK.AttrKind, RK.ArgValue)); 4405ffd83dbSDimitry Andric MadeChange = true; 4415ffd83dbSDimitry Andric RemoveFromAssume(); 4425ffd83dbSDimitry Andric continue; 4435ffd83dbSDimitry Andric } 4445ffd83dbSDimitry Andric } 4455ffd83dbSDimitry Andric auto &Lookup = Knowledge[{RK.WasOn, RK.AttrKind}]; 4465ffd83dbSDimitry Andric for (MapValue &Elem : Lookup) { 4475ffd83dbSDimitry Andric if (!isValidAssumeForContext(Elem.Assume, Assume, DT)) 4485ffd83dbSDimitry Andric continue; 4495ffd83dbSDimitry Andric if (Elem.ArgValue >= RK.ArgValue) { 4505ffd83dbSDimitry Andric RemoveFromAssume(); 4515ffd83dbSDimitry Andric continue; 4525ffd83dbSDimitry Andric } else if (isValidAssumeForContext(Assume, Elem.Assume, DT)) { 4535ffd83dbSDimitry Andric Elem.Assume->op_begin()[Elem.BOI->Begin + ABA_Argument].set( 4545ffd83dbSDimitry Andric ConstantInt::get(Type::getInt64Ty(C), RK.ArgValue)); 4555ffd83dbSDimitry Andric MadeChange = true; 4565ffd83dbSDimitry Andric RemoveFromAssume(); 4575ffd83dbSDimitry Andric continue; 4585ffd83dbSDimitry Andric } 4595ffd83dbSDimitry Andric } 4605ffd83dbSDimitry Andric Lookup.push_back({Assume, RK.ArgValue, &BOI}); 4615ffd83dbSDimitry Andric } 4625ffd83dbSDimitry Andric } 4635ffd83dbSDimitry Andric } 4645ffd83dbSDimitry Andric 4655ffd83dbSDimitry Andric using MergeIterator = SmallVectorImpl<IntrinsicInst *>::iterator; 4665ffd83dbSDimitry Andric 4675ffd83dbSDimitry Andric /// Merge all Assumes from Begin to End in and insert the resulting assume as 4685ffd83dbSDimitry Andric /// high as possible in the basicblock. 4695ffd83dbSDimitry Andric void mergeRange(BasicBlock *BB, MergeIterator Begin, MergeIterator End) { 4705ffd83dbSDimitry Andric if (Begin == End || std::next(Begin) == End) 4715ffd83dbSDimitry Andric return; 4725ffd83dbSDimitry Andric /// Provide no additional information so that AssumeBuilderState doesn't 4735ffd83dbSDimitry Andric /// try to do any punning since it already has been done better. 4745ffd83dbSDimitry Andric AssumeBuilderState Builder(F.getParent()); 4755ffd83dbSDimitry Andric 4765ffd83dbSDimitry Andric /// For now it is initialized to the best value it could have 4775ffd83dbSDimitry Andric Instruction *InsertPt = BB->getFirstNonPHI(); 4785ffd83dbSDimitry Andric if (isa<LandingPadInst>(InsertPt)) 4795ffd83dbSDimitry Andric InsertPt = InsertPt->getNextNode(); 4805ffd83dbSDimitry Andric for (IntrinsicInst *I : make_range(Begin, End)) { 4815ffd83dbSDimitry Andric CleanupToDo.insert(I); 4825ffd83dbSDimitry Andric for (CallInst::BundleOpInfo &BOI : I->bundle_op_infos()) { 483fe6060f1SDimitry Andric RetainedKnowledge RK = 484fe6060f1SDimitry Andric getKnowledgeFromBundle(cast<AssumeInst>(*I), BOI); 4855ffd83dbSDimitry Andric if (!RK) 4865ffd83dbSDimitry Andric continue; 4875ffd83dbSDimitry Andric Builder.addKnowledge(RK); 4885ffd83dbSDimitry Andric if (auto *I = dyn_cast_or_null<Instruction>(RK.WasOn)) 4895ffd83dbSDimitry Andric if (I->getParent() == InsertPt->getParent() && 4905ffd83dbSDimitry Andric (InsertPt->comesBefore(I) || InsertPt == I)) 4915ffd83dbSDimitry Andric InsertPt = I->getNextNode(); 4925ffd83dbSDimitry Andric } 4935ffd83dbSDimitry Andric } 4945ffd83dbSDimitry Andric 4955ffd83dbSDimitry Andric /// Adjust InsertPt if it is before Begin, since mergeAssumes only 4965ffd83dbSDimitry Andric /// guarantees we can place the resulting assume between Begin and End. 4975ffd83dbSDimitry Andric if (InsertPt->comesBefore(*Begin)) 4985ffd83dbSDimitry Andric for (auto It = (*Begin)->getIterator(), E = InsertPt->getIterator(); 4995ffd83dbSDimitry Andric It != E; --It) 5005ffd83dbSDimitry Andric if (!isGuaranteedToTransferExecutionToSuccessor(&*It)) { 5015ffd83dbSDimitry Andric InsertPt = It->getNextNode(); 5025ffd83dbSDimitry Andric break; 5035ffd83dbSDimitry Andric } 504fe6060f1SDimitry Andric auto *MergedAssume = Builder.build(); 5055ffd83dbSDimitry Andric if (!MergedAssume) 5065ffd83dbSDimitry Andric return; 5075ffd83dbSDimitry Andric MadeChange = true; 5085ffd83dbSDimitry Andric MergedAssume->insertBefore(InsertPt); 5095ffd83dbSDimitry Andric AC.registerAssumption(MergedAssume); 5105ffd83dbSDimitry Andric } 5115ffd83dbSDimitry Andric 5125ffd83dbSDimitry Andric /// Merge assume when they are in the same BasicBlock and for all instruction 5135ffd83dbSDimitry Andric /// between them isGuaranteedToTransferExecutionToSuccessor returns true. 5145ffd83dbSDimitry Andric void mergeAssumes() { 5155ffd83dbSDimitry Andric buildMapping(true); 5165ffd83dbSDimitry Andric 5175ffd83dbSDimitry Andric SmallVector<MergeIterator, 4> SplitPoints; 5185ffd83dbSDimitry Andric for (auto &Elem : BBToAssume) { 5195ffd83dbSDimitry Andric SmallVectorImpl<IntrinsicInst *> &AssumesInBB = Elem.second; 5205ffd83dbSDimitry Andric if (AssumesInBB.size() < 2) 5215ffd83dbSDimitry Andric continue; 5225ffd83dbSDimitry Andric /// AssumesInBB is already sorted by order in the block. 5235ffd83dbSDimitry Andric 5245ffd83dbSDimitry Andric BasicBlock::iterator It = AssumesInBB.front()->getIterator(); 5255ffd83dbSDimitry Andric BasicBlock::iterator E = AssumesInBB.back()->getIterator(); 5265ffd83dbSDimitry Andric SplitPoints.push_back(AssumesInBB.begin()); 5275ffd83dbSDimitry Andric MergeIterator LastSplit = AssumesInBB.begin(); 5285ffd83dbSDimitry Andric for (; It != E; ++It) 5295ffd83dbSDimitry Andric if (!isGuaranteedToTransferExecutionToSuccessor(&*It)) { 5305ffd83dbSDimitry Andric for (; (*LastSplit)->comesBefore(&*It); ++LastSplit) 5315ffd83dbSDimitry Andric ; 5325ffd83dbSDimitry Andric if (SplitPoints.back() != LastSplit) 5335ffd83dbSDimitry Andric SplitPoints.push_back(LastSplit); 5345ffd83dbSDimitry Andric } 5355ffd83dbSDimitry Andric SplitPoints.push_back(AssumesInBB.end()); 5365ffd83dbSDimitry Andric for (auto SplitIt = SplitPoints.begin(); 5375ffd83dbSDimitry Andric SplitIt != std::prev(SplitPoints.end()); SplitIt++) { 5385ffd83dbSDimitry Andric mergeRange(Elem.first, *SplitIt, *(SplitIt + 1)); 5395ffd83dbSDimitry Andric } 5405ffd83dbSDimitry Andric SplitPoints.clear(); 5415ffd83dbSDimitry Andric } 5425ffd83dbSDimitry Andric } 5435ffd83dbSDimitry Andric }; 5445ffd83dbSDimitry Andric 5455ffd83dbSDimitry Andric bool simplifyAssumes(Function &F, AssumptionCache *AC, DominatorTree *DT) { 5465ffd83dbSDimitry Andric AssumeSimplify AS(F, *AC, DT, F.getContext()); 5475ffd83dbSDimitry Andric 5485ffd83dbSDimitry Andric /// Remove knowledge that is already known by a dominating other assume or an 5495ffd83dbSDimitry Andric /// attribute. 5505ffd83dbSDimitry Andric AS.dropRedundantKnowledge(); 5515ffd83dbSDimitry Andric 5525ffd83dbSDimitry Andric /// Remove assume that are empty. 5535ffd83dbSDimitry Andric AS.RunCleanup(false); 5545ffd83dbSDimitry Andric 5555ffd83dbSDimitry Andric /// Merge assume in the same basicblock when possible. 5565ffd83dbSDimitry Andric AS.mergeAssumes(); 5575ffd83dbSDimitry Andric 5585ffd83dbSDimitry Andric /// Remove assume that were merged. 5595ffd83dbSDimitry Andric AS.RunCleanup(true); 5605ffd83dbSDimitry Andric return AS.MadeChange; 5615ffd83dbSDimitry Andric } 5625ffd83dbSDimitry Andric 5635ffd83dbSDimitry Andric } // namespace 5645ffd83dbSDimitry Andric 5655ffd83dbSDimitry Andric PreservedAnalyses AssumeSimplifyPass::run(Function &F, 5665ffd83dbSDimitry Andric FunctionAnalysisManager &AM) { 5675ffd83dbSDimitry Andric if (!EnableKnowledgeRetention) 5685ffd83dbSDimitry Andric return PreservedAnalyses::all(); 569*06c3fb27SDimitry Andric if (!simplifyAssumes(F, &AM.getResult<AssumptionAnalysis>(F), 570*06c3fb27SDimitry Andric AM.getCachedResult<DominatorTreeAnalysis>(F))) 5715ffd83dbSDimitry Andric return PreservedAnalyses::all(); 572*06c3fb27SDimitry Andric PreservedAnalyses PA; 573*06c3fb27SDimitry Andric PA.preserveSet<CFGAnalyses>(); 574*06c3fb27SDimitry Andric return PA; 5755ffd83dbSDimitry Andric } 5765ffd83dbSDimitry Andric 5775ffd83dbSDimitry Andric PreservedAnalyses AssumeBuilderPass::run(Function &F, 5785ffd83dbSDimitry Andric FunctionAnalysisManager &AM) { 5795ffd83dbSDimitry Andric AssumptionCache *AC = &AM.getResult<AssumptionAnalysis>(F); 5805ffd83dbSDimitry Andric DominatorTree* DT = AM.getCachedResult<DominatorTreeAnalysis>(F); 581*06c3fb27SDimitry Andric bool Changed = false; 5825ffd83dbSDimitry Andric for (Instruction &I : instructions(F)) 583*06c3fb27SDimitry Andric Changed |= salvageKnowledge(&I, AC, DT); 584*06c3fb27SDimitry Andric if (!Changed) 585*06c3fb27SDimitry Andric PreservedAnalyses::all(); 586*06c3fb27SDimitry Andric PreservedAnalyses PA; 587*06c3fb27SDimitry Andric PA.preserveSet<CFGAnalyses>(); 588*06c3fb27SDimitry Andric return PA; 5895ffd83dbSDimitry Andric } 5905ffd83dbSDimitry Andric 5915ffd83dbSDimitry Andric namespace { 5925ffd83dbSDimitry Andric class AssumeBuilderPassLegacyPass : public FunctionPass { 5935ffd83dbSDimitry Andric public: 5945ffd83dbSDimitry Andric static char ID; 5955ffd83dbSDimitry Andric 5965ffd83dbSDimitry Andric AssumeBuilderPassLegacyPass() : FunctionPass(ID) { 5975ffd83dbSDimitry Andric initializeAssumeBuilderPassLegacyPassPass(*PassRegistry::getPassRegistry()); 5985ffd83dbSDimitry Andric } 5995ffd83dbSDimitry Andric bool runOnFunction(Function &F) override { 6005ffd83dbSDimitry Andric AssumptionCache &AC = 6015ffd83dbSDimitry Andric getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F); 6025ffd83dbSDimitry Andric DominatorTreeWrapperPass *DTWP = 6035ffd83dbSDimitry Andric getAnalysisIfAvailable<DominatorTreeWrapperPass>(); 6045ffd83dbSDimitry Andric for (Instruction &I : instructions(F)) 6055ffd83dbSDimitry Andric salvageKnowledge(&I, &AC, DTWP ? &DTWP->getDomTree() : nullptr); 6065ffd83dbSDimitry Andric return true; 6075ffd83dbSDimitry Andric } 6085ffd83dbSDimitry Andric 6095ffd83dbSDimitry Andric void getAnalysisUsage(AnalysisUsage &AU) const override { 6105ffd83dbSDimitry Andric AU.addRequired<AssumptionCacheTracker>(); 6115ffd83dbSDimitry Andric 6125ffd83dbSDimitry Andric AU.setPreservesAll(); 6135ffd83dbSDimitry Andric } 6145ffd83dbSDimitry Andric }; 6155ffd83dbSDimitry Andric } // namespace 6165ffd83dbSDimitry Andric 6175ffd83dbSDimitry Andric char AssumeBuilderPassLegacyPass::ID = 0; 6185ffd83dbSDimitry Andric 6195ffd83dbSDimitry Andric INITIALIZE_PASS_BEGIN(AssumeBuilderPassLegacyPass, "assume-builder", 6205ffd83dbSDimitry Andric "Assume Builder", false, false) 6215ffd83dbSDimitry Andric INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) 6225ffd83dbSDimitry Andric INITIALIZE_PASS_END(AssumeBuilderPassLegacyPass, "assume-builder", 6235ffd83dbSDimitry Andric "Assume Builder", false, false) 624