xref: /freebsd-src/contrib/llvm-project/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp (revision 06c3fb2749bda94cb5201f81ffdb8fa6c3161b2e)
15ffd83dbSDimitry Andric //===- AssumeBundleBuilder.cpp - tools to preserve informations -*- C++ -*-===//
25ffd83dbSDimitry Andric //
35ffd83dbSDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
45ffd83dbSDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
55ffd83dbSDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
65ffd83dbSDimitry Andric //
75ffd83dbSDimitry Andric //===----------------------------------------------------------------------===//
85ffd83dbSDimitry Andric 
95ffd83dbSDimitry Andric #include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
105ffd83dbSDimitry Andric #include "llvm/ADT/DepthFirstIterator.h"
115ffd83dbSDimitry Andric #include "llvm/ADT/MapVector.h"
125ffd83dbSDimitry Andric #include "llvm/ADT/Statistic.h"
135ffd83dbSDimitry Andric #include "llvm/Analysis/AssumeBundleQueries.h"
145ffd83dbSDimitry Andric #include "llvm/Analysis/AssumptionCache.h"
155ffd83dbSDimitry Andric #include "llvm/Analysis/ValueTracking.h"
165ffd83dbSDimitry Andric #include "llvm/IR/Dominators.h"
175ffd83dbSDimitry Andric #include "llvm/IR/Function.h"
185ffd83dbSDimitry Andric #include "llvm/IR/InstIterator.h"
195ffd83dbSDimitry Andric #include "llvm/IR/IntrinsicInst.h"
205ffd83dbSDimitry Andric #include "llvm/IR/Module.h"
2181ad6265SDimitry Andric #include "llvm/IR/Operator.h"
225ffd83dbSDimitry Andric #include "llvm/InitializePasses.h"
235ffd83dbSDimitry Andric #include "llvm/Support/CommandLine.h"
245ffd83dbSDimitry Andric #include "llvm/Support/DebugCounter.h"
255ffd83dbSDimitry Andric #include "llvm/Transforms/Utils/Local.h"
265ffd83dbSDimitry Andric 
275ffd83dbSDimitry Andric using namespace llvm;
285ffd83dbSDimitry Andric 
29fe6060f1SDimitry Andric namespace llvm {
305ffd83dbSDimitry Andric cl::opt<bool> ShouldPreserveAllAttributes(
315ffd83dbSDimitry Andric     "assume-preserve-all", cl::init(false), cl::Hidden,
325ffd83dbSDimitry Andric     cl::desc("enable preservation of all attrbitues. even those that are "
335ffd83dbSDimitry Andric              "unlikely to be usefull"));
345ffd83dbSDimitry Andric 
355ffd83dbSDimitry Andric cl::opt<bool> EnableKnowledgeRetention(
365ffd83dbSDimitry Andric     "enable-knowledge-retention", cl::init(false), cl::Hidden,
375ffd83dbSDimitry Andric     cl::desc(
385ffd83dbSDimitry Andric         "enable preservation of attributes throughout code transformation"));
39fe6060f1SDimitry Andric } // namespace llvm
40fe6060f1SDimitry Andric 
41fe6060f1SDimitry Andric #define DEBUG_TYPE "assume-builder"
425ffd83dbSDimitry Andric 
435ffd83dbSDimitry Andric STATISTIC(NumAssumeBuilt, "Number of assume built by the assume builder");
445ffd83dbSDimitry Andric STATISTIC(NumBundlesInAssumes, "Total number of Bundles in the assume built");
455ffd83dbSDimitry Andric STATISTIC(NumAssumesMerged,
465ffd83dbSDimitry Andric           "Number of assume merged by the assume simplify pass");
475ffd83dbSDimitry Andric STATISTIC(NumAssumesRemoved,
485ffd83dbSDimitry Andric           "Number of assume removed by the assume simplify pass");
495ffd83dbSDimitry Andric 
505ffd83dbSDimitry Andric DEBUG_COUNTER(BuildAssumeCounter, "assume-builder-counter",
515ffd83dbSDimitry Andric               "Controls which assumes gets created");
525ffd83dbSDimitry Andric 
535ffd83dbSDimitry Andric namespace {
545ffd83dbSDimitry Andric 
555ffd83dbSDimitry Andric bool isUsefullToPreserve(Attribute::AttrKind Kind) {
565ffd83dbSDimitry Andric   switch (Kind) {
575ffd83dbSDimitry Andric     case Attribute::NonNull:
58e8d8bef9SDimitry Andric     case Attribute::NoUndef:
595ffd83dbSDimitry Andric     case Attribute::Alignment:
605ffd83dbSDimitry Andric     case Attribute::Dereferenceable:
615ffd83dbSDimitry Andric     case Attribute::DereferenceableOrNull:
625ffd83dbSDimitry Andric     case Attribute::Cold:
635ffd83dbSDimitry Andric       return true;
645ffd83dbSDimitry Andric     default:
655ffd83dbSDimitry Andric       return false;
665ffd83dbSDimitry Andric   }
675ffd83dbSDimitry Andric }
685ffd83dbSDimitry Andric 
695ffd83dbSDimitry Andric /// This function will try to transform the given knowledge into a more
705ffd83dbSDimitry Andric /// canonical one. the canonical knowledge maybe the given one.
71349cc55cSDimitry Andric RetainedKnowledge canonicalizedKnowledge(RetainedKnowledge RK,
72349cc55cSDimitry Andric                                          const DataLayout &DL) {
735ffd83dbSDimitry Andric   switch (RK.AttrKind) {
745ffd83dbSDimitry Andric   default:
755ffd83dbSDimitry Andric     return RK;
765ffd83dbSDimitry Andric   case Attribute::NonNull:
77e8d8bef9SDimitry Andric     RK.WasOn = getUnderlyingObject(RK.WasOn);
785ffd83dbSDimitry Andric     return RK;
795ffd83dbSDimitry Andric   case Attribute::Alignment: {
805ffd83dbSDimitry Andric     Value *V = RK.WasOn->stripInBoundsOffsets([&](const Value *Strip) {
815ffd83dbSDimitry Andric       if (auto *GEP = dyn_cast<GEPOperator>(Strip))
825ffd83dbSDimitry Andric         RK.ArgValue =
83fe6060f1SDimitry Andric             MinAlign(RK.ArgValue, GEP->getMaxPreservedAlignment(DL).value());
845ffd83dbSDimitry Andric     });
855ffd83dbSDimitry Andric     RK.WasOn = V;
865ffd83dbSDimitry Andric     return RK;
875ffd83dbSDimitry Andric   }
885ffd83dbSDimitry Andric   case Attribute::Dereferenceable:
895ffd83dbSDimitry Andric   case Attribute::DereferenceableOrNull: {
905ffd83dbSDimitry Andric     int64_t Offset = 0;
91fe6060f1SDimitry Andric     Value *V = GetPointerBaseWithConstantOffset(RK.WasOn, Offset, DL,
92fe6060f1SDimitry Andric                                                 /*AllowNonInBounds*/ false);
935ffd83dbSDimitry Andric     if (Offset < 0)
945ffd83dbSDimitry Andric       return RK;
955ffd83dbSDimitry Andric     RK.ArgValue = RK.ArgValue + Offset;
965ffd83dbSDimitry Andric     RK.WasOn = V;
975ffd83dbSDimitry Andric   }
985ffd83dbSDimitry Andric   }
995ffd83dbSDimitry Andric   return RK;
1005ffd83dbSDimitry Andric }
1015ffd83dbSDimitry Andric 
1025ffd83dbSDimitry Andric /// This class contain all knowledge that have been gather while building an
1035ffd83dbSDimitry Andric /// llvm.assume and the function to manipulate it.
1045ffd83dbSDimitry Andric struct AssumeBuilderState {
1055ffd83dbSDimitry Andric   Module *M;
1065ffd83dbSDimitry Andric 
1075ffd83dbSDimitry Andric   using MapKey = std::pair<Value *, Attribute::AttrKind>;
108349cc55cSDimitry Andric   SmallMapVector<MapKey, uint64_t, 8> AssumedKnowledgeMap;
109fe6060f1SDimitry Andric   Instruction *InstBeingModified = nullptr;
1105ffd83dbSDimitry Andric   AssumptionCache* AC = nullptr;
1115ffd83dbSDimitry Andric   DominatorTree* DT = nullptr;
1125ffd83dbSDimitry Andric 
1135ffd83dbSDimitry Andric   AssumeBuilderState(Module *M, Instruction *I = nullptr,
1145ffd83dbSDimitry Andric                      AssumptionCache *AC = nullptr, DominatorTree *DT = nullptr)
115fe6060f1SDimitry Andric       : M(M), InstBeingModified(I), AC(AC), DT(DT) {}
1165ffd83dbSDimitry Andric 
1175ffd83dbSDimitry Andric   bool tryToPreserveWithoutAddingAssume(RetainedKnowledge RK) {
118fe6060f1SDimitry Andric     if (!InstBeingModified || !RK.WasOn)
1195ffd83dbSDimitry Andric       return false;
1205ffd83dbSDimitry Andric     bool HasBeenPreserved = false;
1215ffd83dbSDimitry Andric     Use* ToUpdate = nullptr;
1225ffd83dbSDimitry Andric     getKnowledgeForValue(
1235ffd83dbSDimitry Andric         RK.WasOn, {RK.AttrKind}, AC,
1245ffd83dbSDimitry Andric         [&](RetainedKnowledge RKOther, Instruction *Assume,
1255ffd83dbSDimitry Andric             const CallInst::BundleOpInfo *Bundle) {
126fe6060f1SDimitry Andric           if (!isValidAssumeForContext(Assume, InstBeingModified, DT))
1275ffd83dbSDimitry Andric             return false;
1285ffd83dbSDimitry Andric           if (RKOther.ArgValue >= RK.ArgValue) {
1295ffd83dbSDimitry Andric             HasBeenPreserved = true;
1305ffd83dbSDimitry Andric             return true;
131fe6060f1SDimitry Andric           } else if (isValidAssumeForContext(InstBeingModified, Assume, DT)) {
1325ffd83dbSDimitry Andric             HasBeenPreserved = true;
1335ffd83dbSDimitry Andric             IntrinsicInst *Intr = cast<IntrinsicInst>(Assume);
1345ffd83dbSDimitry Andric             ToUpdate = &Intr->op_begin()[Bundle->Begin + ABA_Argument];
1355ffd83dbSDimitry Andric             return true;
1365ffd83dbSDimitry Andric           }
1375ffd83dbSDimitry Andric           return false;
1385ffd83dbSDimitry Andric         });
1395ffd83dbSDimitry Andric     if (ToUpdate)
1405ffd83dbSDimitry Andric       ToUpdate->set(
1415ffd83dbSDimitry Andric           ConstantInt::get(Type::getInt64Ty(M->getContext()), RK.ArgValue));
1425ffd83dbSDimitry Andric     return HasBeenPreserved;
1435ffd83dbSDimitry Andric   }
1445ffd83dbSDimitry Andric 
1455ffd83dbSDimitry Andric   bool isKnowledgeWorthPreserving(RetainedKnowledge RK) {
1465ffd83dbSDimitry Andric     if (!RK)
1475ffd83dbSDimitry Andric       return false;
1485ffd83dbSDimitry Andric     if (!RK.WasOn)
1495ffd83dbSDimitry Andric       return true;
1505ffd83dbSDimitry Andric     if (RK.WasOn->getType()->isPointerTy()) {
151e8d8bef9SDimitry Andric       Value *UnderlyingPtr = getUnderlyingObject(RK.WasOn);
1525ffd83dbSDimitry Andric       if (isa<AllocaInst>(UnderlyingPtr) || isa<GlobalValue>(UnderlyingPtr))
1535ffd83dbSDimitry Andric         return false;
1545ffd83dbSDimitry Andric     }
1555ffd83dbSDimitry Andric     if (auto *Arg = dyn_cast<Argument>(RK.WasOn)) {
1565ffd83dbSDimitry Andric       if (Arg->hasAttribute(RK.AttrKind) &&
157fe6060f1SDimitry Andric           (!Attribute::isIntAttrKind(RK.AttrKind) ||
1585ffd83dbSDimitry Andric            Arg->getAttribute(RK.AttrKind).getValueAsInt() >= RK.ArgValue))
1595ffd83dbSDimitry Andric         return false;
1605ffd83dbSDimitry Andric       return true;
1615ffd83dbSDimitry Andric     }
1625ffd83dbSDimitry Andric     if (auto *Inst = dyn_cast<Instruction>(RK.WasOn))
1635ffd83dbSDimitry Andric       if (wouldInstructionBeTriviallyDead(Inst)) {
1645ffd83dbSDimitry Andric         if (RK.WasOn->use_empty())
1655ffd83dbSDimitry Andric           return false;
1665ffd83dbSDimitry Andric         Use *SingleUse = RK.WasOn->getSingleUndroppableUse();
167fe6060f1SDimitry Andric         if (SingleUse && SingleUse->getUser() == InstBeingModified)
1685ffd83dbSDimitry Andric           return false;
1695ffd83dbSDimitry Andric       }
1705ffd83dbSDimitry Andric     return true;
1715ffd83dbSDimitry Andric   }
1725ffd83dbSDimitry Andric 
1735ffd83dbSDimitry Andric   void addKnowledge(RetainedKnowledge RK) {
174fe6060f1SDimitry Andric     RK = canonicalizedKnowledge(RK, M->getDataLayout());
1755ffd83dbSDimitry Andric 
1765ffd83dbSDimitry Andric     if (!isKnowledgeWorthPreserving(RK))
1775ffd83dbSDimitry Andric       return;
1785ffd83dbSDimitry Andric 
1795ffd83dbSDimitry Andric     if (tryToPreserveWithoutAddingAssume(RK))
1805ffd83dbSDimitry Andric       return;
1815ffd83dbSDimitry Andric     MapKey Key{RK.WasOn, RK.AttrKind};
1825ffd83dbSDimitry Andric     auto Lookup = AssumedKnowledgeMap.find(Key);
1835ffd83dbSDimitry Andric     if (Lookup == AssumedKnowledgeMap.end()) {
1845ffd83dbSDimitry Andric       AssumedKnowledgeMap[Key] = RK.ArgValue;
1855ffd83dbSDimitry Andric       return;
1865ffd83dbSDimitry Andric     }
1875ffd83dbSDimitry Andric     assert(((Lookup->second == 0 && RK.ArgValue == 0) ||
1885ffd83dbSDimitry Andric             (Lookup->second != 0 && RK.ArgValue != 0)) &&
1895ffd83dbSDimitry Andric            "inconsistent argument value");
1905ffd83dbSDimitry Andric 
1915ffd83dbSDimitry Andric     /// This is only desirable because for all attributes taking an argument
1925ffd83dbSDimitry Andric     /// higher is better.
1935ffd83dbSDimitry Andric     Lookup->second = std::max(Lookup->second, RK.ArgValue);
1945ffd83dbSDimitry Andric   }
1955ffd83dbSDimitry Andric 
1965ffd83dbSDimitry Andric   void addAttribute(Attribute Attr, Value *WasOn) {
1975ffd83dbSDimitry Andric     if (Attr.isTypeAttribute() || Attr.isStringAttribute() ||
1985ffd83dbSDimitry Andric         (!ShouldPreserveAllAttributes &&
1995ffd83dbSDimitry Andric          !isUsefullToPreserve(Attr.getKindAsEnum())))
2005ffd83dbSDimitry Andric       return;
201349cc55cSDimitry Andric     uint64_t AttrArg = 0;
2025ffd83dbSDimitry Andric     if (Attr.isIntAttribute())
2035ffd83dbSDimitry Andric       AttrArg = Attr.getValueAsInt();
2045ffd83dbSDimitry Andric     addKnowledge({Attr.getKindAsEnum(), AttrArg, WasOn});
2055ffd83dbSDimitry Andric   }
2065ffd83dbSDimitry Andric 
2075ffd83dbSDimitry Andric   void addCall(const CallBase *Call) {
208349cc55cSDimitry Andric     auto addAttrList = [&](AttributeList AttrList, unsigned NumArgs) {
209349cc55cSDimitry Andric       for (unsigned Idx = 0; Idx < NumArgs; Idx++)
210349cc55cSDimitry Andric         for (Attribute Attr : AttrList.getParamAttrs(Idx)) {
211fe6060f1SDimitry Andric           bool IsPoisonAttr = Attr.hasAttribute(Attribute::NonNull) ||
212fe6060f1SDimitry Andric                               Attr.hasAttribute(Attribute::Alignment);
213349cc55cSDimitry Andric           if (!IsPoisonAttr || Call->isPassingUndefUB(Idx))
214349cc55cSDimitry Andric             addAttribute(Attr, Call->getArgOperand(Idx));
215fe6060f1SDimitry Andric         }
216349cc55cSDimitry Andric       for (Attribute Attr : AttrList.getFnAttrs())
2175ffd83dbSDimitry Andric         addAttribute(Attr, nullptr);
2185ffd83dbSDimitry Andric     };
219349cc55cSDimitry Andric     addAttrList(Call->getAttributes(), Call->arg_size());
2205ffd83dbSDimitry Andric     if (Function *Fn = Call->getCalledFunction())
221349cc55cSDimitry Andric       addAttrList(Fn->getAttributes(), Fn->arg_size());
2225ffd83dbSDimitry Andric   }
2235ffd83dbSDimitry Andric 
224fe6060f1SDimitry Andric   AssumeInst *build() {
2255ffd83dbSDimitry Andric     if (AssumedKnowledgeMap.empty())
2265ffd83dbSDimitry Andric       return nullptr;
2275ffd83dbSDimitry Andric     if (!DebugCounter::shouldExecute(BuildAssumeCounter))
2285ffd83dbSDimitry Andric       return nullptr;
2295ffd83dbSDimitry Andric     Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
2305ffd83dbSDimitry Andric     LLVMContext &C = M->getContext();
2315ffd83dbSDimitry Andric     SmallVector<OperandBundleDef, 8> OpBundle;
2325ffd83dbSDimitry Andric     for (auto &MapElem : AssumedKnowledgeMap) {
2335ffd83dbSDimitry Andric       SmallVector<Value *, 2> Args;
2345ffd83dbSDimitry Andric       if (MapElem.first.first)
2355ffd83dbSDimitry Andric         Args.push_back(MapElem.first.first);
2365ffd83dbSDimitry Andric 
2375ffd83dbSDimitry Andric       /// This is only valid because for all attribute that currently exist a
2385ffd83dbSDimitry Andric       /// value of 0 is useless. and should not be preserved.
2395ffd83dbSDimitry Andric       if (MapElem.second)
2405ffd83dbSDimitry Andric         Args.push_back(ConstantInt::get(Type::getInt64Ty(M->getContext()),
2415ffd83dbSDimitry Andric                                         MapElem.second));
2425ffd83dbSDimitry Andric       OpBundle.push_back(OperandBundleDefT<Value *>(
2435ffd83dbSDimitry Andric           std::string(Attribute::getNameFromAttrKind(MapElem.first.second)),
2445ffd83dbSDimitry Andric           Args));
2455ffd83dbSDimitry Andric       NumBundlesInAssumes++;
2465ffd83dbSDimitry Andric     }
2475ffd83dbSDimitry Andric     NumAssumeBuilt++;
248fe6060f1SDimitry Andric     return cast<AssumeInst>(CallInst::Create(
2495ffd83dbSDimitry Andric         FnAssume, ArrayRef<Value *>({ConstantInt::getTrue(C)}), OpBundle));
2505ffd83dbSDimitry Andric   }
2515ffd83dbSDimitry Andric 
2525ffd83dbSDimitry Andric   void addAccessedPtr(Instruction *MemInst, Value *Pointer, Type *AccType,
2535ffd83dbSDimitry Andric                       MaybeAlign MA) {
2545ffd83dbSDimitry Andric     unsigned DerefSize = MemInst->getModule()
2555ffd83dbSDimitry Andric                              ->getDataLayout()
2565ffd83dbSDimitry Andric                              .getTypeStoreSize(AccType)
257bdd1243dSDimitry Andric                              .getKnownMinValue();
2585ffd83dbSDimitry Andric     if (DerefSize != 0) {
2595ffd83dbSDimitry Andric       addKnowledge({Attribute::Dereferenceable, DerefSize, Pointer});
2605ffd83dbSDimitry Andric       if (!NullPointerIsDefined(MemInst->getFunction(),
2615ffd83dbSDimitry Andric                                 Pointer->getType()->getPointerAddressSpace()))
2625ffd83dbSDimitry Andric         addKnowledge({Attribute::NonNull, 0u, Pointer});
2635ffd83dbSDimitry Andric     }
2645ffd83dbSDimitry Andric     if (MA.valueOrOne() > 1)
265349cc55cSDimitry Andric       addKnowledge({Attribute::Alignment, MA.valueOrOne().value(), Pointer});
2665ffd83dbSDimitry Andric   }
2675ffd83dbSDimitry Andric 
2685ffd83dbSDimitry Andric   void addInstruction(Instruction *I) {
2695ffd83dbSDimitry Andric     if (auto *Call = dyn_cast<CallBase>(I))
2705ffd83dbSDimitry Andric       return addCall(Call);
2715ffd83dbSDimitry Andric     if (auto *Load = dyn_cast<LoadInst>(I))
2725ffd83dbSDimitry Andric       return addAccessedPtr(I, Load->getPointerOperand(), Load->getType(),
2735ffd83dbSDimitry Andric                             Load->getAlign());
2745ffd83dbSDimitry Andric     if (auto *Store = dyn_cast<StoreInst>(I))
2755ffd83dbSDimitry Andric       return addAccessedPtr(I, Store->getPointerOperand(),
2765ffd83dbSDimitry Andric                             Store->getValueOperand()->getType(),
2775ffd83dbSDimitry Andric                             Store->getAlign());
2785ffd83dbSDimitry Andric     // TODO: Add support for the other Instructions.
2795ffd83dbSDimitry Andric     // TODO: Maybe we should look around and merge with other llvm.assume.
2805ffd83dbSDimitry Andric   }
2815ffd83dbSDimitry Andric };
2825ffd83dbSDimitry Andric 
2835ffd83dbSDimitry Andric } // namespace
2845ffd83dbSDimitry Andric 
285fe6060f1SDimitry Andric AssumeInst *llvm::buildAssumeFromInst(Instruction *I) {
2865ffd83dbSDimitry Andric   if (!EnableKnowledgeRetention)
2875ffd83dbSDimitry Andric     return nullptr;
2885ffd83dbSDimitry Andric   AssumeBuilderState Builder(I->getModule());
2895ffd83dbSDimitry Andric   Builder.addInstruction(I);
2905ffd83dbSDimitry Andric   return Builder.build();
2915ffd83dbSDimitry Andric }
2925ffd83dbSDimitry Andric 
293*06c3fb27SDimitry Andric bool llvm::salvageKnowledge(Instruction *I, AssumptionCache *AC,
2945ffd83dbSDimitry Andric                             DominatorTree *DT) {
2955ffd83dbSDimitry Andric   if (!EnableKnowledgeRetention || I->isTerminator())
296*06c3fb27SDimitry Andric     return false;
297*06c3fb27SDimitry Andric   bool Changed = false;
2985ffd83dbSDimitry Andric   AssumeBuilderState Builder(I->getModule(), I, AC, DT);
2995ffd83dbSDimitry Andric   Builder.addInstruction(I);
300fe6060f1SDimitry Andric   if (auto *Intr = Builder.build()) {
3015ffd83dbSDimitry Andric     Intr->insertBefore(I);
302*06c3fb27SDimitry Andric     Changed = true;
3035ffd83dbSDimitry Andric     if (AC)
3045ffd83dbSDimitry Andric       AC->registerAssumption(Intr);
3055ffd83dbSDimitry Andric   }
306*06c3fb27SDimitry Andric   return Changed;
3075ffd83dbSDimitry Andric }
3085ffd83dbSDimitry Andric 
309fe6060f1SDimitry Andric AssumeInst *
310fe6060f1SDimitry Andric llvm::buildAssumeFromKnowledge(ArrayRef<RetainedKnowledge> Knowledge,
311fe6060f1SDimitry Andric                                Instruction *CtxI, AssumptionCache *AC,
312fe6060f1SDimitry Andric                                DominatorTree *DT) {
313fe6060f1SDimitry Andric   AssumeBuilderState Builder(CtxI->getModule(), CtxI, AC, DT);
314fe6060f1SDimitry Andric   for (const RetainedKnowledge &RK : Knowledge)
315fe6060f1SDimitry Andric     Builder.addKnowledge(RK);
316fe6060f1SDimitry Andric   return Builder.build();
317fe6060f1SDimitry Andric }
318fe6060f1SDimitry Andric 
319fe6060f1SDimitry Andric RetainedKnowledge llvm::simplifyRetainedKnowledge(AssumeInst *Assume,
320fe6060f1SDimitry Andric                                                   RetainedKnowledge RK,
321fe6060f1SDimitry Andric                                                   AssumptionCache *AC,
322fe6060f1SDimitry Andric                                                   DominatorTree *DT) {
323fe6060f1SDimitry Andric   AssumeBuilderState Builder(Assume->getModule(), Assume, AC, DT);
324fe6060f1SDimitry Andric   RK = canonicalizedKnowledge(RK, Assume->getModule()->getDataLayout());
325fe6060f1SDimitry Andric 
326fe6060f1SDimitry Andric   if (!Builder.isKnowledgeWorthPreserving(RK))
327fe6060f1SDimitry Andric     return RetainedKnowledge::none();
328fe6060f1SDimitry Andric 
329fe6060f1SDimitry Andric   if (Builder.tryToPreserveWithoutAddingAssume(RK))
330fe6060f1SDimitry Andric     return RetainedKnowledge::none();
331fe6060f1SDimitry Andric   return RK;
332fe6060f1SDimitry Andric }
333fe6060f1SDimitry Andric 
3345ffd83dbSDimitry Andric namespace {
3355ffd83dbSDimitry Andric 
3365ffd83dbSDimitry Andric struct AssumeSimplify {
3375ffd83dbSDimitry Andric   Function &F;
3385ffd83dbSDimitry Andric   AssumptionCache &AC;
3395ffd83dbSDimitry Andric   DominatorTree *DT;
3405ffd83dbSDimitry Andric   LLVMContext &C;
3415ffd83dbSDimitry Andric   SmallDenseSet<IntrinsicInst *> CleanupToDo;
3425ffd83dbSDimitry Andric   StringMapEntry<uint32_t> *IgnoreTag;
3435ffd83dbSDimitry Andric   SmallDenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 4>, 8> BBToAssume;
3445ffd83dbSDimitry Andric   bool MadeChange = false;
3455ffd83dbSDimitry Andric 
3465ffd83dbSDimitry Andric   AssumeSimplify(Function &F, AssumptionCache &AC, DominatorTree *DT,
3475ffd83dbSDimitry Andric                  LLVMContext &C)
3485ffd83dbSDimitry Andric       : F(F), AC(AC), DT(DT), C(C),
3495ffd83dbSDimitry Andric         IgnoreTag(C.getOrInsertBundleTag(IgnoreBundleTag)) {}
3505ffd83dbSDimitry Andric 
3515ffd83dbSDimitry Andric   void buildMapping(bool FilterBooleanArgument) {
3525ffd83dbSDimitry Andric     BBToAssume.clear();
3535ffd83dbSDimitry Andric     for (Value *V : AC.assumptions()) {
3545ffd83dbSDimitry Andric       if (!V)
3555ffd83dbSDimitry Andric         continue;
3565ffd83dbSDimitry Andric       IntrinsicInst *Assume = cast<IntrinsicInst>(V);
3575ffd83dbSDimitry Andric       if (FilterBooleanArgument) {
3585ffd83dbSDimitry Andric         auto *Arg = dyn_cast<ConstantInt>(Assume->getOperand(0));
3595ffd83dbSDimitry Andric         if (!Arg || Arg->isZero())
3605ffd83dbSDimitry Andric           continue;
3615ffd83dbSDimitry Andric       }
3625ffd83dbSDimitry Andric       BBToAssume[Assume->getParent()].push_back(Assume);
3635ffd83dbSDimitry Andric     }
3645ffd83dbSDimitry Andric 
3655ffd83dbSDimitry Andric     for (auto &Elem : BBToAssume) {
3665ffd83dbSDimitry Andric       llvm::sort(Elem.second,
3675ffd83dbSDimitry Andric                  [](const IntrinsicInst *LHS, const IntrinsicInst *RHS) {
3685ffd83dbSDimitry Andric                    return LHS->comesBefore(RHS);
3695ffd83dbSDimitry Andric                  });
3705ffd83dbSDimitry Andric     }
3715ffd83dbSDimitry Andric   }
3725ffd83dbSDimitry Andric 
3735ffd83dbSDimitry Andric   /// Remove all asumes in CleanupToDo if there boolean argument is true and
3745ffd83dbSDimitry Andric   /// ForceCleanup is set or the assume doesn't hold valuable knowledge.
3755ffd83dbSDimitry Andric   void RunCleanup(bool ForceCleanup) {
3765ffd83dbSDimitry Andric     for (IntrinsicInst *Assume : CleanupToDo) {
3775ffd83dbSDimitry Andric       auto *Arg = dyn_cast<ConstantInt>(Assume->getOperand(0));
3785ffd83dbSDimitry Andric       if (!Arg || Arg->isZero() ||
379fe6060f1SDimitry Andric           (!ForceCleanup &&
380fe6060f1SDimitry Andric            !isAssumeWithEmptyBundle(cast<AssumeInst>(*Assume))))
3815ffd83dbSDimitry Andric         continue;
3825ffd83dbSDimitry Andric       MadeChange = true;
3835ffd83dbSDimitry Andric       if (ForceCleanup)
3845ffd83dbSDimitry Andric         NumAssumesMerged++;
3855ffd83dbSDimitry Andric       else
3865ffd83dbSDimitry Andric         NumAssumesRemoved++;
3875ffd83dbSDimitry Andric       Assume->eraseFromParent();
3885ffd83dbSDimitry Andric     }
3895ffd83dbSDimitry Andric     CleanupToDo.clear();
3905ffd83dbSDimitry Andric   }
3915ffd83dbSDimitry Andric 
3925ffd83dbSDimitry Andric   /// Remove knowledge stored in assume when it is already know by an attribute
3935ffd83dbSDimitry Andric   /// or an other assume. This can when valid update an existing knowledge in an
3945ffd83dbSDimitry Andric   /// attribute or an other assume.
3955ffd83dbSDimitry Andric   void dropRedundantKnowledge() {
3965ffd83dbSDimitry Andric     struct MapValue {
3975ffd83dbSDimitry Andric       IntrinsicInst *Assume;
398349cc55cSDimitry Andric       uint64_t ArgValue;
3995ffd83dbSDimitry Andric       CallInst::BundleOpInfo *BOI;
4005ffd83dbSDimitry Andric     };
4015ffd83dbSDimitry Andric     buildMapping(false);
4025ffd83dbSDimitry Andric     SmallDenseMap<std::pair<Value *, Attribute::AttrKind>,
4035ffd83dbSDimitry Andric                   SmallVector<MapValue, 2>, 16>
4045ffd83dbSDimitry Andric         Knowledge;
4055ffd83dbSDimitry Andric     for (BasicBlock *BB : depth_first(&F))
4065ffd83dbSDimitry Andric       for (Value *V : BBToAssume[BB]) {
4075ffd83dbSDimitry Andric         if (!V)
4085ffd83dbSDimitry Andric           continue;
4095ffd83dbSDimitry Andric         IntrinsicInst *Assume = cast<IntrinsicInst>(V);
4105ffd83dbSDimitry Andric         for (CallInst::BundleOpInfo &BOI : Assume->bundle_op_infos()) {
4115ffd83dbSDimitry Andric           auto RemoveFromAssume = [&]() {
4125ffd83dbSDimitry Andric             CleanupToDo.insert(Assume);
4135ffd83dbSDimitry Andric             if (BOI.Begin != BOI.End) {
4145ffd83dbSDimitry Andric               Use *U = &Assume->op_begin()[BOI.Begin + ABA_WasOn];
4155ffd83dbSDimitry Andric               U->set(UndefValue::get(U->get()->getType()));
4165ffd83dbSDimitry Andric             }
4175ffd83dbSDimitry Andric             BOI.Tag = IgnoreTag;
4185ffd83dbSDimitry Andric           };
4195ffd83dbSDimitry Andric           if (BOI.Tag == IgnoreTag) {
4205ffd83dbSDimitry Andric             CleanupToDo.insert(Assume);
4215ffd83dbSDimitry Andric             continue;
4225ffd83dbSDimitry Andric           }
423fe6060f1SDimitry Andric           RetainedKnowledge RK =
424fe6060f1SDimitry Andric             getKnowledgeFromBundle(cast<AssumeInst>(*Assume), BOI);
4255ffd83dbSDimitry Andric           if (auto *Arg = dyn_cast_or_null<Argument>(RK.WasOn)) {
4265ffd83dbSDimitry Andric             bool HasSameKindAttr = Arg->hasAttribute(RK.AttrKind);
4275ffd83dbSDimitry Andric             if (HasSameKindAttr)
428fe6060f1SDimitry Andric               if (!Attribute::isIntAttrKind(RK.AttrKind) ||
4295ffd83dbSDimitry Andric                   Arg->getAttribute(RK.AttrKind).getValueAsInt() >=
4305ffd83dbSDimitry Andric                       RK.ArgValue) {
4315ffd83dbSDimitry Andric                 RemoveFromAssume();
4325ffd83dbSDimitry Andric                 continue;
4335ffd83dbSDimitry Andric               }
4345ffd83dbSDimitry Andric             if (isValidAssumeForContext(
4355ffd83dbSDimitry Andric                     Assume, &*F.getEntryBlock().getFirstInsertionPt()) ||
4365ffd83dbSDimitry Andric                 Assume == &*F.getEntryBlock().getFirstInsertionPt()) {
4375ffd83dbSDimitry Andric               if (HasSameKindAttr)
4385ffd83dbSDimitry Andric                 Arg->removeAttr(RK.AttrKind);
4395ffd83dbSDimitry Andric               Arg->addAttr(Attribute::get(C, RK.AttrKind, RK.ArgValue));
4405ffd83dbSDimitry Andric               MadeChange = true;
4415ffd83dbSDimitry Andric               RemoveFromAssume();
4425ffd83dbSDimitry Andric               continue;
4435ffd83dbSDimitry Andric             }
4445ffd83dbSDimitry Andric           }
4455ffd83dbSDimitry Andric           auto &Lookup = Knowledge[{RK.WasOn, RK.AttrKind}];
4465ffd83dbSDimitry Andric           for (MapValue &Elem : Lookup) {
4475ffd83dbSDimitry Andric             if (!isValidAssumeForContext(Elem.Assume, Assume, DT))
4485ffd83dbSDimitry Andric               continue;
4495ffd83dbSDimitry Andric             if (Elem.ArgValue >= RK.ArgValue) {
4505ffd83dbSDimitry Andric               RemoveFromAssume();
4515ffd83dbSDimitry Andric               continue;
4525ffd83dbSDimitry Andric             } else if (isValidAssumeForContext(Assume, Elem.Assume, DT)) {
4535ffd83dbSDimitry Andric               Elem.Assume->op_begin()[Elem.BOI->Begin + ABA_Argument].set(
4545ffd83dbSDimitry Andric                   ConstantInt::get(Type::getInt64Ty(C), RK.ArgValue));
4555ffd83dbSDimitry Andric               MadeChange = true;
4565ffd83dbSDimitry Andric               RemoveFromAssume();
4575ffd83dbSDimitry Andric               continue;
4585ffd83dbSDimitry Andric             }
4595ffd83dbSDimitry Andric           }
4605ffd83dbSDimitry Andric           Lookup.push_back({Assume, RK.ArgValue, &BOI});
4615ffd83dbSDimitry Andric         }
4625ffd83dbSDimitry Andric       }
4635ffd83dbSDimitry Andric   }
4645ffd83dbSDimitry Andric 
4655ffd83dbSDimitry Andric   using MergeIterator = SmallVectorImpl<IntrinsicInst *>::iterator;
4665ffd83dbSDimitry Andric 
4675ffd83dbSDimitry Andric   /// Merge all Assumes from Begin to End in and insert the resulting assume as
4685ffd83dbSDimitry Andric   /// high as possible in the basicblock.
4695ffd83dbSDimitry Andric   void mergeRange(BasicBlock *BB, MergeIterator Begin, MergeIterator End) {
4705ffd83dbSDimitry Andric     if (Begin == End || std::next(Begin) == End)
4715ffd83dbSDimitry Andric       return;
4725ffd83dbSDimitry Andric     /// Provide no additional information so that AssumeBuilderState doesn't
4735ffd83dbSDimitry Andric     /// try to do any punning since it already has been done better.
4745ffd83dbSDimitry Andric     AssumeBuilderState Builder(F.getParent());
4755ffd83dbSDimitry Andric 
4765ffd83dbSDimitry Andric     /// For now it is initialized to the best value it could have
4775ffd83dbSDimitry Andric     Instruction *InsertPt = BB->getFirstNonPHI();
4785ffd83dbSDimitry Andric     if (isa<LandingPadInst>(InsertPt))
4795ffd83dbSDimitry Andric       InsertPt = InsertPt->getNextNode();
4805ffd83dbSDimitry Andric     for (IntrinsicInst *I : make_range(Begin, End)) {
4815ffd83dbSDimitry Andric       CleanupToDo.insert(I);
4825ffd83dbSDimitry Andric       for (CallInst::BundleOpInfo &BOI : I->bundle_op_infos()) {
483fe6060f1SDimitry Andric         RetainedKnowledge RK =
484fe6060f1SDimitry Andric           getKnowledgeFromBundle(cast<AssumeInst>(*I), BOI);
4855ffd83dbSDimitry Andric         if (!RK)
4865ffd83dbSDimitry Andric           continue;
4875ffd83dbSDimitry Andric         Builder.addKnowledge(RK);
4885ffd83dbSDimitry Andric         if (auto *I = dyn_cast_or_null<Instruction>(RK.WasOn))
4895ffd83dbSDimitry Andric           if (I->getParent() == InsertPt->getParent() &&
4905ffd83dbSDimitry Andric               (InsertPt->comesBefore(I) || InsertPt == I))
4915ffd83dbSDimitry Andric             InsertPt = I->getNextNode();
4925ffd83dbSDimitry Andric       }
4935ffd83dbSDimitry Andric     }
4945ffd83dbSDimitry Andric 
4955ffd83dbSDimitry Andric     /// Adjust InsertPt if it is before Begin, since mergeAssumes only
4965ffd83dbSDimitry Andric     /// guarantees we can place the resulting assume between Begin and End.
4975ffd83dbSDimitry Andric     if (InsertPt->comesBefore(*Begin))
4985ffd83dbSDimitry Andric       for (auto It = (*Begin)->getIterator(), E = InsertPt->getIterator();
4995ffd83dbSDimitry Andric            It != E; --It)
5005ffd83dbSDimitry Andric         if (!isGuaranteedToTransferExecutionToSuccessor(&*It)) {
5015ffd83dbSDimitry Andric           InsertPt = It->getNextNode();
5025ffd83dbSDimitry Andric           break;
5035ffd83dbSDimitry Andric         }
504fe6060f1SDimitry Andric     auto *MergedAssume = Builder.build();
5055ffd83dbSDimitry Andric     if (!MergedAssume)
5065ffd83dbSDimitry Andric       return;
5075ffd83dbSDimitry Andric     MadeChange = true;
5085ffd83dbSDimitry Andric     MergedAssume->insertBefore(InsertPt);
5095ffd83dbSDimitry Andric     AC.registerAssumption(MergedAssume);
5105ffd83dbSDimitry Andric   }
5115ffd83dbSDimitry Andric 
5125ffd83dbSDimitry Andric   /// Merge assume when they are in the same BasicBlock and for all instruction
5135ffd83dbSDimitry Andric   /// between them isGuaranteedToTransferExecutionToSuccessor returns true.
5145ffd83dbSDimitry Andric   void mergeAssumes() {
5155ffd83dbSDimitry Andric     buildMapping(true);
5165ffd83dbSDimitry Andric 
5175ffd83dbSDimitry Andric     SmallVector<MergeIterator, 4> SplitPoints;
5185ffd83dbSDimitry Andric     for (auto &Elem : BBToAssume) {
5195ffd83dbSDimitry Andric       SmallVectorImpl<IntrinsicInst *> &AssumesInBB = Elem.second;
5205ffd83dbSDimitry Andric       if (AssumesInBB.size() < 2)
5215ffd83dbSDimitry Andric         continue;
5225ffd83dbSDimitry Andric       /// AssumesInBB is already sorted by order in the block.
5235ffd83dbSDimitry Andric 
5245ffd83dbSDimitry Andric       BasicBlock::iterator It = AssumesInBB.front()->getIterator();
5255ffd83dbSDimitry Andric       BasicBlock::iterator E = AssumesInBB.back()->getIterator();
5265ffd83dbSDimitry Andric       SplitPoints.push_back(AssumesInBB.begin());
5275ffd83dbSDimitry Andric       MergeIterator LastSplit = AssumesInBB.begin();
5285ffd83dbSDimitry Andric       for (; It != E; ++It)
5295ffd83dbSDimitry Andric         if (!isGuaranteedToTransferExecutionToSuccessor(&*It)) {
5305ffd83dbSDimitry Andric           for (; (*LastSplit)->comesBefore(&*It); ++LastSplit)
5315ffd83dbSDimitry Andric             ;
5325ffd83dbSDimitry Andric           if (SplitPoints.back() != LastSplit)
5335ffd83dbSDimitry Andric             SplitPoints.push_back(LastSplit);
5345ffd83dbSDimitry Andric         }
5355ffd83dbSDimitry Andric       SplitPoints.push_back(AssumesInBB.end());
5365ffd83dbSDimitry Andric       for (auto SplitIt = SplitPoints.begin();
5375ffd83dbSDimitry Andric            SplitIt != std::prev(SplitPoints.end()); SplitIt++) {
5385ffd83dbSDimitry Andric         mergeRange(Elem.first, *SplitIt, *(SplitIt + 1));
5395ffd83dbSDimitry Andric       }
5405ffd83dbSDimitry Andric       SplitPoints.clear();
5415ffd83dbSDimitry Andric     }
5425ffd83dbSDimitry Andric   }
5435ffd83dbSDimitry Andric };
5445ffd83dbSDimitry Andric 
5455ffd83dbSDimitry Andric bool simplifyAssumes(Function &F, AssumptionCache *AC, DominatorTree *DT) {
5465ffd83dbSDimitry Andric   AssumeSimplify AS(F, *AC, DT, F.getContext());
5475ffd83dbSDimitry Andric 
5485ffd83dbSDimitry Andric   /// Remove knowledge that is already known by a dominating other assume or an
5495ffd83dbSDimitry Andric   /// attribute.
5505ffd83dbSDimitry Andric   AS.dropRedundantKnowledge();
5515ffd83dbSDimitry Andric 
5525ffd83dbSDimitry Andric   /// Remove assume that are empty.
5535ffd83dbSDimitry Andric   AS.RunCleanup(false);
5545ffd83dbSDimitry Andric 
5555ffd83dbSDimitry Andric   /// Merge assume in the same basicblock when possible.
5565ffd83dbSDimitry Andric   AS.mergeAssumes();
5575ffd83dbSDimitry Andric 
5585ffd83dbSDimitry Andric   /// Remove assume that were merged.
5595ffd83dbSDimitry Andric   AS.RunCleanup(true);
5605ffd83dbSDimitry Andric   return AS.MadeChange;
5615ffd83dbSDimitry Andric }
5625ffd83dbSDimitry Andric 
5635ffd83dbSDimitry Andric } // namespace
5645ffd83dbSDimitry Andric 
5655ffd83dbSDimitry Andric PreservedAnalyses AssumeSimplifyPass::run(Function &F,
5665ffd83dbSDimitry Andric                                           FunctionAnalysisManager &AM) {
5675ffd83dbSDimitry Andric   if (!EnableKnowledgeRetention)
5685ffd83dbSDimitry Andric     return PreservedAnalyses::all();
569*06c3fb27SDimitry Andric   if (!simplifyAssumes(F, &AM.getResult<AssumptionAnalysis>(F),
570*06c3fb27SDimitry Andric                        AM.getCachedResult<DominatorTreeAnalysis>(F)))
5715ffd83dbSDimitry Andric     return PreservedAnalyses::all();
572*06c3fb27SDimitry Andric   PreservedAnalyses PA;
573*06c3fb27SDimitry Andric   PA.preserveSet<CFGAnalyses>();
574*06c3fb27SDimitry Andric   return PA;
5755ffd83dbSDimitry Andric }
5765ffd83dbSDimitry Andric 
5775ffd83dbSDimitry Andric PreservedAnalyses AssumeBuilderPass::run(Function &F,
5785ffd83dbSDimitry Andric                                          FunctionAnalysisManager &AM) {
5795ffd83dbSDimitry Andric   AssumptionCache *AC = &AM.getResult<AssumptionAnalysis>(F);
5805ffd83dbSDimitry Andric   DominatorTree* DT = AM.getCachedResult<DominatorTreeAnalysis>(F);
581*06c3fb27SDimitry Andric   bool Changed = false;
5825ffd83dbSDimitry Andric   for (Instruction &I : instructions(F))
583*06c3fb27SDimitry Andric     Changed |= salvageKnowledge(&I, AC, DT);
584*06c3fb27SDimitry Andric   if (!Changed)
585*06c3fb27SDimitry Andric     PreservedAnalyses::all();
586*06c3fb27SDimitry Andric   PreservedAnalyses PA;
587*06c3fb27SDimitry Andric   PA.preserveSet<CFGAnalyses>();
588*06c3fb27SDimitry Andric   return PA;
5895ffd83dbSDimitry Andric }
5905ffd83dbSDimitry Andric 
5915ffd83dbSDimitry Andric namespace {
5925ffd83dbSDimitry Andric class AssumeBuilderPassLegacyPass : public FunctionPass {
5935ffd83dbSDimitry Andric public:
5945ffd83dbSDimitry Andric   static char ID;
5955ffd83dbSDimitry Andric 
5965ffd83dbSDimitry Andric   AssumeBuilderPassLegacyPass() : FunctionPass(ID) {
5975ffd83dbSDimitry Andric     initializeAssumeBuilderPassLegacyPassPass(*PassRegistry::getPassRegistry());
5985ffd83dbSDimitry Andric   }
5995ffd83dbSDimitry Andric   bool runOnFunction(Function &F) override {
6005ffd83dbSDimitry Andric     AssumptionCache &AC =
6015ffd83dbSDimitry Andric         getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
6025ffd83dbSDimitry Andric     DominatorTreeWrapperPass *DTWP =
6035ffd83dbSDimitry Andric         getAnalysisIfAvailable<DominatorTreeWrapperPass>();
6045ffd83dbSDimitry Andric     for (Instruction &I : instructions(F))
6055ffd83dbSDimitry Andric       salvageKnowledge(&I, &AC, DTWP ? &DTWP->getDomTree() : nullptr);
6065ffd83dbSDimitry Andric     return true;
6075ffd83dbSDimitry Andric   }
6085ffd83dbSDimitry Andric 
6095ffd83dbSDimitry Andric   void getAnalysisUsage(AnalysisUsage &AU) const override {
6105ffd83dbSDimitry Andric     AU.addRequired<AssumptionCacheTracker>();
6115ffd83dbSDimitry Andric 
6125ffd83dbSDimitry Andric     AU.setPreservesAll();
6135ffd83dbSDimitry Andric   }
6145ffd83dbSDimitry Andric };
6155ffd83dbSDimitry Andric } // namespace
6165ffd83dbSDimitry Andric 
6175ffd83dbSDimitry Andric char AssumeBuilderPassLegacyPass::ID = 0;
6185ffd83dbSDimitry Andric 
6195ffd83dbSDimitry Andric INITIALIZE_PASS_BEGIN(AssumeBuilderPassLegacyPass, "assume-builder",
6205ffd83dbSDimitry Andric                       "Assume Builder", false, false)
6215ffd83dbSDimitry Andric INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
6225ffd83dbSDimitry Andric INITIALIZE_PASS_END(AssumeBuilderPassLegacyPass, "assume-builder",
6235ffd83dbSDimitry Andric                     "Assume Builder", false, false)
624