xref: /openbsd-src/gnu/llvm/llvm/lib/Transforms/Utils/AssumeBundleBuilder.cpp (revision d415bd752c734aee168c4ee86ff32e8cc249eb16)
1097a140dSpatrick //===- AssumeBundleBuilder.cpp - tools to preserve informations -*- C++ -*-===//
2097a140dSpatrick //
3097a140dSpatrick // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4097a140dSpatrick // See https://llvm.org/LICENSE.txt for license information.
5097a140dSpatrick // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6097a140dSpatrick //
7097a140dSpatrick //===----------------------------------------------------------------------===//
8097a140dSpatrick 
9097a140dSpatrick #include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
10097a140dSpatrick #include "llvm/ADT/DepthFirstIterator.h"
11097a140dSpatrick #include "llvm/ADT/MapVector.h"
12097a140dSpatrick #include "llvm/ADT/Statistic.h"
13097a140dSpatrick #include "llvm/Analysis/AssumeBundleQueries.h"
14097a140dSpatrick #include "llvm/Analysis/AssumptionCache.h"
15097a140dSpatrick #include "llvm/Analysis/ValueTracking.h"
16097a140dSpatrick #include "llvm/IR/Dominators.h"
17097a140dSpatrick #include "llvm/IR/Function.h"
18097a140dSpatrick #include "llvm/IR/InstIterator.h"
19097a140dSpatrick #include "llvm/IR/IntrinsicInst.h"
20097a140dSpatrick #include "llvm/IR/Module.h"
21*d415bd75Srobert #include "llvm/IR/Operator.h"
22097a140dSpatrick #include "llvm/InitializePasses.h"
23097a140dSpatrick #include "llvm/Support/CommandLine.h"
24097a140dSpatrick #include "llvm/Support/DebugCounter.h"
25097a140dSpatrick #include "llvm/Transforms/Utils/Local.h"
26097a140dSpatrick 
27097a140dSpatrick using namespace llvm;
28097a140dSpatrick 
2973471bf0Spatrick namespace llvm {
30097a140dSpatrick cl::opt<bool> ShouldPreserveAllAttributes(
31097a140dSpatrick     "assume-preserve-all", cl::init(false), cl::Hidden,
32097a140dSpatrick     cl::desc("enable preservation of all attrbitues. even those that are "
33097a140dSpatrick              "unlikely to be usefull"));
34097a140dSpatrick 
35097a140dSpatrick cl::opt<bool> EnableKnowledgeRetention(
36097a140dSpatrick     "enable-knowledge-retention", cl::init(false), cl::Hidden,
37097a140dSpatrick     cl::desc(
38097a140dSpatrick         "enable preservation of attributes throughout code transformation"));
3973471bf0Spatrick } // namespace llvm
4073471bf0Spatrick 
4173471bf0Spatrick #define DEBUG_TYPE "assume-builder"
42097a140dSpatrick 
43097a140dSpatrick STATISTIC(NumAssumeBuilt, "Number of assume built by the assume builder");
44097a140dSpatrick STATISTIC(NumBundlesInAssumes, "Total number of Bundles in the assume built");
45097a140dSpatrick STATISTIC(NumAssumesMerged,
46097a140dSpatrick           "Number of assume merged by the assume simplify pass");
47097a140dSpatrick STATISTIC(NumAssumesRemoved,
48097a140dSpatrick           "Number of assume removed by the assume simplify pass");
49097a140dSpatrick 
50097a140dSpatrick DEBUG_COUNTER(BuildAssumeCounter, "assume-builder-counter",
51097a140dSpatrick               "Controls which assumes gets created");
52097a140dSpatrick 
53097a140dSpatrick namespace {
54097a140dSpatrick 
isUsefullToPreserve(Attribute::AttrKind Kind)55097a140dSpatrick bool isUsefullToPreserve(Attribute::AttrKind Kind) {
56097a140dSpatrick   switch (Kind) {
57097a140dSpatrick     case Attribute::NonNull:
5873471bf0Spatrick     case Attribute::NoUndef:
59097a140dSpatrick     case Attribute::Alignment:
60097a140dSpatrick     case Attribute::Dereferenceable:
61097a140dSpatrick     case Attribute::DereferenceableOrNull:
62097a140dSpatrick     case Attribute::Cold:
63097a140dSpatrick       return true;
64097a140dSpatrick     default:
65097a140dSpatrick       return false;
66097a140dSpatrick   }
67097a140dSpatrick }
68097a140dSpatrick 
69097a140dSpatrick /// This function will try to transform the given knowledge into a more
70097a140dSpatrick /// canonical one. the canonical knowledge maybe the given one.
canonicalizedKnowledge(RetainedKnowledge RK,const DataLayout & DL)71*d415bd75Srobert RetainedKnowledge canonicalizedKnowledge(RetainedKnowledge RK,
72*d415bd75Srobert                                          const DataLayout &DL) {
73097a140dSpatrick   switch (RK.AttrKind) {
74097a140dSpatrick   default:
75097a140dSpatrick     return RK;
76097a140dSpatrick   case Attribute::NonNull:
7773471bf0Spatrick     RK.WasOn = getUnderlyingObject(RK.WasOn);
78097a140dSpatrick     return RK;
79097a140dSpatrick   case Attribute::Alignment: {
80097a140dSpatrick     Value *V = RK.WasOn->stripInBoundsOffsets([&](const Value *Strip) {
81097a140dSpatrick       if (auto *GEP = dyn_cast<GEPOperator>(Strip))
82097a140dSpatrick         RK.ArgValue =
8373471bf0Spatrick             MinAlign(RK.ArgValue, GEP->getMaxPreservedAlignment(DL).value());
84097a140dSpatrick     });
85097a140dSpatrick     RK.WasOn = V;
86097a140dSpatrick     return RK;
87097a140dSpatrick   }
88097a140dSpatrick   case Attribute::Dereferenceable:
89097a140dSpatrick   case Attribute::DereferenceableOrNull: {
90097a140dSpatrick     int64_t Offset = 0;
9173471bf0Spatrick     Value *V = GetPointerBaseWithConstantOffset(RK.WasOn, Offset, DL,
9273471bf0Spatrick                                                 /*AllowNonInBounds*/ false);
93097a140dSpatrick     if (Offset < 0)
94097a140dSpatrick       return RK;
95097a140dSpatrick     RK.ArgValue = RK.ArgValue + Offset;
96097a140dSpatrick     RK.WasOn = V;
97097a140dSpatrick   }
98097a140dSpatrick   }
99097a140dSpatrick   return RK;
100097a140dSpatrick }
101097a140dSpatrick 
102097a140dSpatrick /// This class contain all knowledge that have been gather while building an
103097a140dSpatrick /// llvm.assume and the function to manipulate it.
104097a140dSpatrick struct AssumeBuilderState {
105097a140dSpatrick   Module *M;
106097a140dSpatrick 
107097a140dSpatrick   using MapKey = std::pair<Value *, Attribute::AttrKind>;
108*d415bd75Srobert   SmallMapVector<MapKey, uint64_t, 8> AssumedKnowledgeMap;
10973471bf0Spatrick   Instruction *InstBeingModified = nullptr;
110097a140dSpatrick   AssumptionCache* AC = nullptr;
111097a140dSpatrick   DominatorTree* DT = nullptr;
112097a140dSpatrick 
AssumeBuilderState__anon0b0cb1140111::AssumeBuilderState113097a140dSpatrick   AssumeBuilderState(Module *M, Instruction *I = nullptr,
114097a140dSpatrick                      AssumptionCache *AC = nullptr, DominatorTree *DT = nullptr)
11573471bf0Spatrick       : M(M), InstBeingModified(I), AC(AC), DT(DT) {}
116097a140dSpatrick 
tryToPreserveWithoutAddingAssume__anon0b0cb1140111::AssumeBuilderState117097a140dSpatrick   bool tryToPreserveWithoutAddingAssume(RetainedKnowledge RK) {
11873471bf0Spatrick     if (!InstBeingModified || !RK.WasOn)
119097a140dSpatrick       return false;
120097a140dSpatrick     bool HasBeenPreserved = false;
121097a140dSpatrick     Use* ToUpdate = nullptr;
122097a140dSpatrick     getKnowledgeForValue(
123097a140dSpatrick         RK.WasOn, {RK.AttrKind}, AC,
124097a140dSpatrick         [&](RetainedKnowledge RKOther, Instruction *Assume,
125097a140dSpatrick             const CallInst::BundleOpInfo *Bundle) {
12673471bf0Spatrick           if (!isValidAssumeForContext(Assume, InstBeingModified, DT))
127097a140dSpatrick             return false;
128097a140dSpatrick           if (RKOther.ArgValue >= RK.ArgValue) {
129097a140dSpatrick             HasBeenPreserved = true;
130097a140dSpatrick             return true;
13173471bf0Spatrick           } else if (isValidAssumeForContext(InstBeingModified, Assume, DT)) {
132097a140dSpatrick             HasBeenPreserved = true;
133097a140dSpatrick             IntrinsicInst *Intr = cast<IntrinsicInst>(Assume);
134097a140dSpatrick             ToUpdate = &Intr->op_begin()[Bundle->Begin + ABA_Argument];
135097a140dSpatrick             return true;
136097a140dSpatrick           }
137097a140dSpatrick           return false;
138097a140dSpatrick         });
139097a140dSpatrick     if (ToUpdate)
140097a140dSpatrick       ToUpdate->set(
141097a140dSpatrick           ConstantInt::get(Type::getInt64Ty(M->getContext()), RK.ArgValue));
142097a140dSpatrick     return HasBeenPreserved;
143097a140dSpatrick   }
144097a140dSpatrick 
isKnowledgeWorthPreserving__anon0b0cb1140111::AssumeBuilderState145097a140dSpatrick   bool isKnowledgeWorthPreserving(RetainedKnowledge RK) {
146097a140dSpatrick     if (!RK)
147097a140dSpatrick       return false;
148097a140dSpatrick     if (!RK.WasOn)
149097a140dSpatrick       return true;
150097a140dSpatrick     if (RK.WasOn->getType()->isPointerTy()) {
15173471bf0Spatrick       Value *UnderlyingPtr = getUnderlyingObject(RK.WasOn);
152097a140dSpatrick       if (isa<AllocaInst>(UnderlyingPtr) || isa<GlobalValue>(UnderlyingPtr))
153097a140dSpatrick         return false;
154097a140dSpatrick     }
155097a140dSpatrick     if (auto *Arg = dyn_cast<Argument>(RK.WasOn)) {
156097a140dSpatrick       if (Arg->hasAttribute(RK.AttrKind) &&
15773471bf0Spatrick           (!Attribute::isIntAttrKind(RK.AttrKind) ||
158097a140dSpatrick            Arg->getAttribute(RK.AttrKind).getValueAsInt() >= RK.ArgValue))
159097a140dSpatrick         return false;
160097a140dSpatrick       return true;
161097a140dSpatrick     }
162097a140dSpatrick     if (auto *Inst = dyn_cast<Instruction>(RK.WasOn))
163097a140dSpatrick       if (wouldInstructionBeTriviallyDead(Inst)) {
164097a140dSpatrick         if (RK.WasOn->use_empty())
165097a140dSpatrick           return false;
166097a140dSpatrick         Use *SingleUse = RK.WasOn->getSingleUndroppableUse();
16773471bf0Spatrick         if (SingleUse && SingleUse->getUser() == InstBeingModified)
168097a140dSpatrick           return false;
169097a140dSpatrick       }
170097a140dSpatrick     return true;
171097a140dSpatrick   }
172097a140dSpatrick 
addKnowledge__anon0b0cb1140111::AssumeBuilderState173097a140dSpatrick   void addKnowledge(RetainedKnowledge RK) {
17473471bf0Spatrick     RK = canonicalizedKnowledge(RK, M->getDataLayout());
175097a140dSpatrick 
176097a140dSpatrick     if (!isKnowledgeWorthPreserving(RK))
177097a140dSpatrick       return;
178097a140dSpatrick 
179097a140dSpatrick     if (tryToPreserveWithoutAddingAssume(RK))
180097a140dSpatrick       return;
181097a140dSpatrick     MapKey Key{RK.WasOn, RK.AttrKind};
182097a140dSpatrick     auto Lookup = AssumedKnowledgeMap.find(Key);
183097a140dSpatrick     if (Lookup == AssumedKnowledgeMap.end()) {
184097a140dSpatrick       AssumedKnowledgeMap[Key] = RK.ArgValue;
185097a140dSpatrick       return;
186097a140dSpatrick     }
187097a140dSpatrick     assert(((Lookup->second == 0 && RK.ArgValue == 0) ||
188097a140dSpatrick             (Lookup->second != 0 && RK.ArgValue != 0)) &&
189097a140dSpatrick            "inconsistent argument value");
190097a140dSpatrick 
191097a140dSpatrick     /// This is only desirable because for all attributes taking an argument
192097a140dSpatrick     /// higher is better.
193097a140dSpatrick     Lookup->second = std::max(Lookup->second, RK.ArgValue);
194097a140dSpatrick   }
195097a140dSpatrick 
addAttribute__anon0b0cb1140111::AssumeBuilderState196097a140dSpatrick   void addAttribute(Attribute Attr, Value *WasOn) {
197097a140dSpatrick     if (Attr.isTypeAttribute() || Attr.isStringAttribute() ||
198097a140dSpatrick         (!ShouldPreserveAllAttributes &&
199097a140dSpatrick          !isUsefullToPreserve(Attr.getKindAsEnum())))
200097a140dSpatrick       return;
201*d415bd75Srobert     uint64_t AttrArg = 0;
202097a140dSpatrick     if (Attr.isIntAttribute())
203097a140dSpatrick       AttrArg = Attr.getValueAsInt();
204097a140dSpatrick     addKnowledge({Attr.getKindAsEnum(), AttrArg, WasOn});
205097a140dSpatrick   }
206097a140dSpatrick 
addCall__anon0b0cb1140111::AssumeBuilderState207097a140dSpatrick   void addCall(const CallBase *Call) {
208*d415bd75Srobert     auto addAttrList = [&](AttributeList AttrList, unsigned NumArgs) {
209*d415bd75Srobert       for (unsigned Idx = 0; Idx < NumArgs; Idx++)
210*d415bd75Srobert         for (Attribute Attr : AttrList.getParamAttrs(Idx)) {
21173471bf0Spatrick           bool IsPoisonAttr = Attr.hasAttribute(Attribute::NonNull) ||
21273471bf0Spatrick                               Attr.hasAttribute(Attribute::Alignment);
213*d415bd75Srobert           if (!IsPoisonAttr || Call->isPassingUndefUB(Idx))
214*d415bd75Srobert             addAttribute(Attr, Call->getArgOperand(Idx));
21573471bf0Spatrick         }
216*d415bd75Srobert       for (Attribute Attr : AttrList.getFnAttrs())
217097a140dSpatrick         addAttribute(Attr, nullptr);
218097a140dSpatrick     };
219*d415bd75Srobert     addAttrList(Call->getAttributes(), Call->arg_size());
220097a140dSpatrick     if (Function *Fn = Call->getCalledFunction())
221*d415bd75Srobert       addAttrList(Fn->getAttributes(), Fn->arg_size());
222097a140dSpatrick   }
223097a140dSpatrick 
build__anon0b0cb1140111::AssumeBuilderState22473471bf0Spatrick   AssumeInst *build() {
225097a140dSpatrick     if (AssumedKnowledgeMap.empty())
226097a140dSpatrick       return nullptr;
227097a140dSpatrick     if (!DebugCounter::shouldExecute(BuildAssumeCounter))
228097a140dSpatrick       return nullptr;
229097a140dSpatrick     Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
230097a140dSpatrick     LLVMContext &C = M->getContext();
231097a140dSpatrick     SmallVector<OperandBundleDef, 8> OpBundle;
232097a140dSpatrick     for (auto &MapElem : AssumedKnowledgeMap) {
233097a140dSpatrick       SmallVector<Value *, 2> Args;
234097a140dSpatrick       if (MapElem.first.first)
235097a140dSpatrick         Args.push_back(MapElem.first.first);
236097a140dSpatrick 
237097a140dSpatrick       /// This is only valid because for all attribute that currently exist a
238097a140dSpatrick       /// value of 0 is useless. and should not be preserved.
239097a140dSpatrick       if (MapElem.second)
240097a140dSpatrick         Args.push_back(ConstantInt::get(Type::getInt64Ty(M->getContext()),
241097a140dSpatrick                                         MapElem.second));
242097a140dSpatrick       OpBundle.push_back(OperandBundleDefT<Value *>(
243097a140dSpatrick           std::string(Attribute::getNameFromAttrKind(MapElem.first.second)),
244097a140dSpatrick           Args));
245097a140dSpatrick       NumBundlesInAssumes++;
246097a140dSpatrick     }
247097a140dSpatrick     NumAssumeBuilt++;
24873471bf0Spatrick     return cast<AssumeInst>(CallInst::Create(
249097a140dSpatrick         FnAssume, ArrayRef<Value *>({ConstantInt::getTrue(C)}), OpBundle));
250097a140dSpatrick   }
251097a140dSpatrick 
addAccessedPtr__anon0b0cb1140111::AssumeBuilderState252097a140dSpatrick   void addAccessedPtr(Instruction *MemInst, Value *Pointer, Type *AccType,
253097a140dSpatrick                       MaybeAlign MA) {
254097a140dSpatrick     unsigned DerefSize = MemInst->getModule()
255097a140dSpatrick                              ->getDataLayout()
256097a140dSpatrick                              .getTypeStoreSize(AccType)
257*d415bd75Srobert                              .getKnownMinValue();
258097a140dSpatrick     if (DerefSize != 0) {
259097a140dSpatrick       addKnowledge({Attribute::Dereferenceable, DerefSize, Pointer});
260097a140dSpatrick       if (!NullPointerIsDefined(MemInst->getFunction(),
261097a140dSpatrick                                 Pointer->getType()->getPointerAddressSpace()))
262097a140dSpatrick         addKnowledge({Attribute::NonNull, 0u, Pointer});
263097a140dSpatrick     }
264097a140dSpatrick     if (MA.valueOrOne() > 1)
265*d415bd75Srobert       addKnowledge({Attribute::Alignment, MA.valueOrOne().value(), Pointer});
266097a140dSpatrick   }
267097a140dSpatrick 
addInstruction__anon0b0cb1140111::AssumeBuilderState268097a140dSpatrick   void addInstruction(Instruction *I) {
269097a140dSpatrick     if (auto *Call = dyn_cast<CallBase>(I))
270097a140dSpatrick       return addCall(Call);
271097a140dSpatrick     if (auto *Load = dyn_cast<LoadInst>(I))
272097a140dSpatrick       return addAccessedPtr(I, Load->getPointerOperand(), Load->getType(),
273097a140dSpatrick                             Load->getAlign());
274097a140dSpatrick     if (auto *Store = dyn_cast<StoreInst>(I))
275097a140dSpatrick       return addAccessedPtr(I, Store->getPointerOperand(),
276097a140dSpatrick                             Store->getValueOperand()->getType(),
277097a140dSpatrick                             Store->getAlign());
278097a140dSpatrick     // TODO: Add support for the other Instructions.
279097a140dSpatrick     // TODO: Maybe we should look around and merge with other llvm.assume.
280097a140dSpatrick   }
281097a140dSpatrick };
282097a140dSpatrick 
283097a140dSpatrick } // namespace
284097a140dSpatrick 
buildAssumeFromInst(Instruction * I)28573471bf0Spatrick AssumeInst *llvm::buildAssumeFromInst(Instruction *I) {
286097a140dSpatrick   if (!EnableKnowledgeRetention)
287097a140dSpatrick     return nullptr;
288097a140dSpatrick   AssumeBuilderState Builder(I->getModule());
289097a140dSpatrick   Builder.addInstruction(I);
290097a140dSpatrick   return Builder.build();
291097a140dSpatrick }
292097a140dSpatrick 
salvageKnowledge(Instruction * I,AssumptionCache * AC,DominatorTree * DT)293097a140dSpatrick void llvm::salvageKnowledge(Instruction *I, AssumptionCache *AC,
294097a140dSpatrick                             DominatorTree *DT) {
295097a140dSpatrick   if (!EnableKnowledgeRetention || I->isTerminator())
296097a140dSpatrick     return;
297097a140dSpatrick   AssumeBuilderState Builder(I->getModule(), I, AC, DT);
298097a140dSpatrick   Builder.addInstruction(I);
29973471bf0Spatrick   if (auto *Intr = Builder.build()) {
300097a140dSpatrick     Intr->insertBefore(I);
301097a140dSpatrick     if (AC)
302097a140dSpatrick       AC->registerAssumption(Intr);
303097a140dSpatrick   }
304097a140dSpatrick }
305097a140dSpatrick 
30673471bf0Spatrick AssumeInst *
buildAssumeFromKnowledge(ArrayRef<RetainedKnowledge> Knowledge,Instruction * CtxI,AssumptionCache * AC,DominatorTree * DT)30773471bf0Spatrick llvm::buildAssumeFromKnowledge(ArrayRef<RetainedKnowledge> Knowledge,
30873471bf0Spatrick                                Instruction *CtxI, AssumptionCache *AC,
30973471bf0Spatrick                                DominatorTree *DT) {
31073471bf0Spatrick   AssumeBuilderState Builder(CtxI->getModule(), CtxI, AC, DT);
31173471bf0Spatrick   for (const RetainedKnowledge &RK : Knowledge)
31273471bf0Spatrick     Builder.addKnowledge(RK);
31373471bf0Spatrick   return Builder.build();
31473471bf0Spatrick }
31573471bf0Spatrick 
simplifyRetainedKnowledge(AssumeInst * Assume,RetainedKnowledge RK,AssumptionCache * AC,DominatorTree * DT)31673471bf0Spatrick RetainedKnowledge llvm::simplifyRetainedKnowledge(AssumeInst *Assume,
31773471bf0Spatrick                                                   RetainedKnowledge RK,
31873471bf0Spatrick                                                   AssumptionCache *AC,
31973471bf0Spatrick                                                   DominatorTree *DT) {
32073471bf0Spatrick   AssumeBuilderState Builder(Assume->getModule(), Assume, AC, DT);
32173471bf0Spatrick   RK = canonicalizedKnowledge(RK, Assume->getModule()->getDataLayout());
32273471bf0Spatrick 
32373471bf0Spatrick   if (!Builder.isKnowledgeWorthPreserving(RK))
32473471bf0Spatrick     return RetainedKnowledge::none();
32573471bf0Spatrick 
32673471bf0Spatrick   if (Builder.tryToPreserveWithoutAddingAssume(RK))
32773471bf0Spatrick     return RetainedKnowledge::none();
32873471bf0Spatrick   return RK;
32973471bf0Spatrick }
33073471bf0Spatrick 
331097a140dSpatrick namespace {
332097a140dSpatrick 
333097a140dSpatrick struct AssumeSimplify {
334097a140dSpatrick   Function &F;
335097a140dSpatrick   AssumptionCache &AC;
336097a140dSpatrick   DominatorTree *DT;
337097a140dSpatrick   LLVMContext &C;
338097a140dSpatrick   SmallDenseSet<IntrinsicInst *> CleanupToDo;
339097a140dSpatrick   StringMapEntry<uint32_t> *IgnoreTag;
340097a140dSpatrick   SmallDenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 4>, 8> BBToAssume;
341097a140dSpatrick   bool MadeChange = false;
342097a140dSpatrick 
AssumeSimplify__anon0b0cb1140511::AssumeSimplify343097a140dSpatrick   AssumeSimplify(Function &F, AssumptionCache &AC, DominatorTree *DT,
344097a140dSpatrick                  LLVMContext &C)
345097a140dSpatrick       : F(F), AC(AC), DT(DT), C(C),
346097a140dSpatrick         IgnoreTag(C.getOrInsertBundleTag(IgnoreBundleTag)) {}
347097a140dSpatrick 
buildMapping__anon0b0cb1140511::AssumeSimplify348097a140dSpatrick   void buildMapping(bool FilterBooleanArgument) {
349097a140dSpatrick     BBToAssume.clear();
350097a140dSpatrick     for (Value *V : AC.assumptions()) {
351097a140dSpatrick       if (!V)
352097a140dSpatrick         continue;
353097a140dSpatrick       IntrinsicInst *Assume = cast<IntrinsicInst>(V);
354097a140dSpatrick       if (FilterBooleanArgument) {
355097a140dSpatrick         auto *Arg = dyn_cast<ConstantInt>(Assume->getOperand(0));
356097a140dSpatrick         if (!Arg || Arg->isZero())
357097a140dSpatrick           continue;
358097a140dSpatrick       }
359097a140dSpatrick       BBToAssume[Assume->getParent()].push_back(Assume);
360097a140dSpatrick     }
361097a140dSpatrick 
362097a140dSpatrick     for (auto &Elem : BBToAssume) {
363097a140dSpatrick       llvm::sort(Elem.second,
364097a140dSpatrick                  [](const IntrinsicInst *LHS, const IntrinsicInst *RHS) {
365097a140dSpatrick                    return LHS->comesBefore(RHS);
366097a140dSpatrick                  });
367097a140dSpatrick     }
368097a140dSpatrick   }
369097a140dSpatrick 
370097a140dSpatrick   /// Remove all asumes in CleanupToDo if there boolean argument is true and
371097a140dSpatrick   /// ForceCleanup is set or the assume doesn't hold valuable knowledge.
RunCleanup__anon0b0cb1140511::AssumeSimplify372097a140dSpatrick   void RunCleanup(bool ForceCleanup) {
373097a140dSpatrick     for (IntrinsicInst *Assume : CleanupToDo) {
374097a140dSpatrick       auto *Arg = dyn_cast<ConstantInt>(Assume->getOperand(0));
375097a140dSpatrick       if (!Arg || Arg->isZero() ||
37673471bf0Spatrick           (!ForceCleanup &&
37773471bf0Spatrick            !isAssumeWithEmptyBundle(cast<AssumeInst>(*Assume))))
378097a140dSpatrick         continue;
379097a140dSpatrick       MadeChange = true;
380097a140dSpatrick       if (ForceCleanup)
381097a140dSpatrick         NumAssumesMerged++;
382097a140dSpatrick       else
383097a140dSpatrick         NumAssumesRemoved++;
384097a140dSpatrick       Assume->eraseFromParent();
385097a140dSpatrick     }
386097a140dSpatrick     CleanupToDo.clear();
387097a140dSpatrick   }
388097a140dSpatrick 
389097a140dSpatrick   /// Remove knowledge stored in assume when it is already know by an attribute
390097a140dSpatrick   /// or an other assume. This can when valid update an existing knowledge in an
391097a140dSpatrick   /// attribute or an other assume.
dropRedundantKnowledge__anon0b0cb1140511::AssumeSimplify392097a140dSpatrick   void dropRedundantKnowledge() {
393097a140dSpatrick     struct MapValue {
394097a140dSpatrick       IntrinsicInst *Assume;
395*d415bd75Srobert       uint64_t ArgValue;
396097a140dSpatrick       CallInst::BundleOpInfo *BOI;
397097a140dSpatrick     };
398097a140dSpatrick     buildMapping(false);
399097a140dSpatrick     SmallDenseMap<std::pair<Value *, Attribute::AttrKind>,
400097a140dSpatrick                   SmallVector<MapValue, 2>, 16>
401097a140dSpatrick         Knowledge;
402097a140dSpatrick     for (BasicBlock *BB : depth_first(&F))
403097a140dSpatrick       for (Value *V : BBToAssume[BB]) {
404097a140dSpatrick         if (!V)
405097a140dSpatrick           continue;
406097a140dSpatrick         IntrinsicInst *Assume = cast<IntrinsicInst>(V);
407097a140dSpatrick         for (CallInst::BundleOpInfo &BOI : Assume->bundle_op_infos()) {
408097a140dSpatrick           auto RemoveFromAssume = [&]() {
409097a140dSpatrick             CleanupToDo.insert(Assume);
410097a140dSpatrick             if (BOI.Begin != BOI.End) {
411097a140dSpatrick               Use *U = &Assume->op_begin()[BOI.Begin + ABA_WasOn];
412097a140dSpatrick               U->set(UndefValue::get(U->get()->getType()));
413097a140dSpatrick             }
414097a140dSpatrick             BOI.Tag = IgnoreTag;
415097a140dSpatrick           };
416097a140dSpatrick           if (BOI.Tag == IgnoreTag) {
417097a140dSpatrick             CleanupToDo.insert(Assume);
418097a140dSpatrick             continue;
419097a140dSpatrick           }
42073471bf0Spatrick           RetainedKnowledge RK =
42173471bf0Spatrick             getKnowledgeFromBundle(cast<AssumeInst>(*Assume), BOI);
422097a140dSpatrick           if (auto *Arg = dyn_cast_or_null<Argument>(RK.WasOn)) {
423097a140dSpatrick             bool HasSameKindAttr = Arg->hasAttribute(RK.AttrKind);
424097a140dSpatrick             if (HasSameKindAttr)
42573471bf0Spatrick               if (!Attribute::isIntAttrKind(RK.AttrKind) ||
426097a140dSpatrick                   Arg->getAttribute(RK.AttrKind).getValueAsInt() >=
427097a140dSpatrick                       RK.ArgValue) {
428097a140dSpatrick                 RemoveFromAssume();
429097a140dSpatrick                 continue;
430097a140dSpatrick               }
431097a140dSpatrick             if (isValidAssumeForContext(
432097a140dSpatrick                     Assume, &*F.getEntryBlock().getFirstInsertionPt()) ||
433097a140dSpatrick                 Assume == &*F.getEntryBlock().getFirstInsertionPt()) {
434097a140dSpatrick               if (HasSameKindAttr)
435097a140dSpatrick                 Arg->removeAttr(RK.AttrKind);
436097a140dSpatrick               Arg->addAttr(Attribute::get(C, RK.AttrKind, RK.ArgValue));
437097a140dSpatrick               MadeChange = true;
438097a140dSpatrick               RemoveFromAssume();
439097a140dSpatrick               continue;
440097a140dSpatrick             }
441097a140dSpatrick           }
442097a140dSpatrick           auto &Lookup = Knowledge[{RK.WasOn, RK.AttrKind}];
443097a140dSpatrick           for (MapValue &Elem : Lookup) {
444097a140dSpatrick             if (!isValidAssumeForContext(Elem.Assume, Assume, DT))
445097a140dSpatrick               continue;
446097a140dSpatrick             if (Elem.ArgValue >= RK.ArgValue) {
447097a140dSpatrick               RemoveFromAssume();
448097a140dSpatrick               continue;
449097a140dSpatrick             } else if (isValidAssumeForContext(Assume, Elem.Assume, DT)) {
450097a140dSpatrick               Elem.Assume->op_begin()[Elem.BOI->Begin + ABA_Argument].set(
451097a140dSpatrick                   ConstantInt::get(Type::getInt64Ty(C), RK.ArgValue));
452097a140dSpatrick               MadeChange = true;
453097a140dSpatrick               RemoveFromAssume();
454097a140dSpatrick               continue;
455097a140dSpatrick             }
456097a140dSpatrick           }
457097a140dSpatrick           Lookup.push_back({Assume, RK.ArgValue, &BOI});
458097a140dSpatrick         }
459097a140dSpatrick       }
460097a140dSpatrick   }
461097a140dSpatrick 
462097a140dSpatrick   using MergeIterator = SmallVectorImpl<IntrinsicInst *>::iterator;
463097a140dSpatrick 
464097a140dSpatrick   /// Merge all Assumes from Begin to End in and insert the resulting assume as
465097a140dSpatrick   /// high as possible in the basicblock.
mergeRange__anon0b0cb1140511::AssumeSimplify466097a140dSpatrick   void mergeRange(BasicBlock *BB, MergeIterator Begin, MergeIterator End) {
467097a140dSpatrick     if (Begin == End || std::next(Begin) == End)
468097a140dSpatrick       return;
469097a140dSpatrick     /// Provide no additional information so that AssumeBuilderState doesn't
470097a140dSpatrick     /// try to do any punning since it already has been done better.
471097a140dSpatrick     AssumeBuilderState Builder(F.getParent());
472097a140dSpatrick 
473097a140dSpatrick     /// For now it is initialized to the best value it could have
474097a140dSpatrick     Instruction *InsertPt = BB->getFirstNonPHI();
475097a140dSpatrick     if (isa<LandingPadInst>(InsertPt))
476097a140dSpatrick       InsertPt = InsertPt->getNextNode();
477097a140dSpatrick     for (IntrinsicInst *I : make_range(Begin, End)) {
478097a140dSpatrick       CleanupToDo.insert(I);
479097a140dSpatrick       for (CallInst::BundleOpInfo &BOI : I->bundle_op_infos()) {
48073471bf0Spatrick         RetainedKnowledge RK =
48173471bf0Spatrick           getKnowledgeFromBundle(cast<AssumeInst>(*I), BOI);
482097a140dSpatrick         if (!RK)
483097a140dSpatrick           continue;
484097a140dSpatrick         Builder.addKnowledge(RK);
485097a140dSpatrick         if (auto *I = dyn_cast_or_null<Instruction>(RK.WasOn))
486097a140dSpatrick           if (I->getParent() == InsertPt->getParent() &&
487097a140dSpatrick               (InsertPt->comesBefore(I) || InsertPt == I))
488097a140dSpatrick             InsertPt = I->getNextNode();
489097a140dSpatrick       }
490097a140dSpatrick     }
491097a140dSpatrick 
492097a140dSpatrick     /// Adjust InsertPt if it is before Begin, since mergeAssumes only
493097a140dSpatrick     /// guarantees we can place the resulting assume between Begin and End.
494097a140dSpatrick     if (InsertPt->comesBefore(*Begin))
495097a140dSpatrick       for (auto It = (*Begin)->getIterator(), E = InsertPt->getIterator();
496097a140dSpatrick            It != E; --It)
497097a140dSpatrick         if (!isGuaranteedToTransferExecutionToSuccessor(&*It)) {
498097a140dSpatrick           InsertPt = It->getNextNode();
499097a140dSpatrick           break;
500097a140dSpatrick         }
50173471bf0Spatrick     auto *MergedAssume = Builder.build();
502097a140dSpatrick     if (!MergedAssume)
503097a140dSpatrick       return;
504097a140dSpatrick     MadeChange = true;
505097a140dSpatrick     MergedAssume->insertBefore(InsertPt);
506097a140dSpatrick     AC.registerAssumption(MergedAssume);
507097a140dSpatrick   }
508097a140dSpatrick 
509097a140dSpatrick   /// Merge assume when they are in the same BasicBlock and for all instruction
510097a140dSpatrick   /// between them isGuaranteedToTransferExecutionToSuccessor returns true.
mergeAssumes__anon0b0cb1140511::AssumeSimplify511097a140dSpatrick   void mergeAssumes() {
512097a140dSpatrick     buildMapping(true);
513097a140dSpatrick 
514097a140dSpatrick     SmallVector<MergeIterator, 4> SplitPoints;
515097a140dSpatrick     for (auto &Elem : BBToAssume) {
516097a140dSpatrick       SmallVectorImpl<IntrinsicInst *> &AssumesInBB = Elem.second;
517097a140dSpatrick       if (AssumesInBB.size() < 2)
518097a140dSpatrick         continue;
519097a140dSpatrick       /// AssumesInBB is already sorted by order in the block.
520097a140dSpatrick 
521097a140dSpatrick       BasicBlock::iterator It = AssumesInBB.front()->getIterator();
522097a140dSpatrick       BasicBlock::iterator E = AssumesInBB.back()->getIterator();
523097a140dSpatrick       SplitPoints.push_back(AssumesInBB.begin());
524097a140dSpatrick       MergeIterator LastSplit = AssumesInBB.begin();
525097a140dSpatrick       for (; It != E; ++It)
526097a140dSpatrick         if (!isGuaranteedToTransferExecutionToSuccessor(&*It)) {
527097a140dSpatrick           for (; (*LastSplit)->comesBefore(&*It); ++LastSplit)
528097a140dSpatrick             ;
529097a140dSpatrick           if (SplitPoints.back() != LastSplit)
530097a140dSpatrick             SplitPoints.push_back(LastSplit);
531097a140dSpatrick         }
532097a140dSpatrick       SplitPoints.push_back(AssumesInBB.end());
533097a140dSpatrick       for (auto SplitIt = SplitPoints.begin();
534097a140dSpatrick            SplitIt != std::prev(SplitPoints.end()); SplitIt++) {
535097a140dSpatrick         mergeRange(Elem.first, *SplitIt, *(SplitIt + 1));
536097a140dSpatrick       }
537097a140dSpatrick       SplitPoints.clear();
538097a140dSpatrick     }
539097a140dSpatrick   }
540097a140dSpatrick };
541097a140dSpatrick 
simplifyAssumes(Function & F,AssumptionCache * AC,DominatorTree * DT)542097a140dSpatrick bool simplifyAssumes(Function &F, AssumptionCache *AC, DominatorTree *DT) {
543097a140dSpatrick   AssumeSimplify AS(F, *AC, DT, F.getContext());
544097a140dSpatrick 
545097a140dSpatrick   /// Remove knowledge that is already known by a dominating other assume or an
546097a140dSpatrick   /// attribute.
547097a140dSpatrick   AS.dropRedundantKnowledge();
548097a140dSpatrick 
549097a140dSpatrick   /// Remove assume that are empty.
550097a140dSpatrick   AS.RunCleanup(false);
551097a140dSpatrick 
552097a140dSpatrick   /// Merge assume in the same basicblock when possible.
553097a140dSpatrick   AS.mergeAssumes();
554097a140dSpatrick 
555097a140dSpatrick   /// Remove assume that were merged.
556097a140dSpatrick   AS.RunCleanup(true);
557097a140dSpatrick   return AS.MadeChange;
558097a140dSpatrick }
559097a140dSpatrick 
560097a140dSpatrick } // namespace
561097a140dSpatrick 
run(Function & F,FunctionAnalysisManager & AM)562097a140dSpatrick PreservedAnalyses AssumeSimplifyPass::run(Function &F,
563097a140dSpatrick                                           FunctionAnalysisManager &AM) {
564097a140dSpatrick   if (!EnableKnowledgeRetention)
565097a140dSpatrick     return PreservedAnalyses::all();
566097a140dSpatrick   simplifyAssumes(F, &AM.getResult<AssumptionAnalysis>(F),
567097a140dSpatrick                   AM.getCachedResult<DominatorTreeAnalysis>(F));
568097a140dSpatrick   return PreservedAnalyses::all();
569097a140dSpatrick }
570097a140dSpatrick 
571097a140dSpatrick namespace {
572097a140dSpatrick class AssumeSimplifyPassLegacyPass : public FunctionPass {
573097a140dSpatrick public:
574097a140dSpatrick   static char ID;
575097a140dSpatrick 
AssumeSimplifyPassLegacyPass()576097a140dSpatrick   AssumeSimplifyPassLegacyPass() : FunctionPass(ID) {
577097a140dSpatrick     initializeAssumeSimplifyPassLegacyPassPass(
578097a140dSpatrick         *PassRegistry::getPassRegistry());
579097a140dSpatrick   }
runOnFunction(Function & F)580097a140dSpatrick   bool runOnFunction(Function &F) override {
581097a140dSpatrick     if (skipFunction(F) || !EnableKnowledgeRetention)
582097a140dSpatrick       return false;
583097a140dSpatrick     AssumptionCache &AC =
584097a140dSpatrick         getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
585097a140dSpatrick     DominatorTreeWrapperPass *DTWP =
586097a140dSpatrick         getAnalysisIfAvailable<DominatorTreeWrapperPass>();
587097a140dSpatrick     return simplifyAssumes(F, &AC, DTWP ? &DTWP->getDomTree() : nullptr);
588097a140dSpatrick   }
589097a140dSpatrick 
getAnalysisUsage(AnalysisUsage & AU) const590097a140dSpatrick   void getAnalysisUsage(AnalysisUsage &AU) const override {
591097a140dSpatrick     AU.addRequired<AssumptionCacheTracker>();
592097a140dSpatrick 
593097a140dSpatrick     AU.setPreservesAll();
594097a140dSpatrick   }
595097a140dSpatrick };
596097a140dSpatrick } // namespace
597097a140dSpatrick 
598097a140dSpatrick char AssumeSimplifyPassLegacyPass::ID = 0;
599097a140dSpatrick 
600097a140dSpatrick INITIALIZE_PASS_BEGIN(AssumeSimplifyPassLegacyPass, "assume-simplify",
601097a140dSpatrick                       "Assume Simplify", false, false)
INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)602097a140dSpatrick INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
603097a140dSpatrick INITIALIZE_PASS_END(AssumeSimplifyPassLegacyPass, "assume-simplify",
604097a140dSpatrick                     "Assume Simplify", false, false)
605097a140dSpatrick 
606097a140dSpatrick FunctionPass *llvm::createAssumeSimplifyPass() {
607097a140dSpatrick   return new AssumeSimplifyPassLegacyPass();
608097a140dSpatrick }
609097a140dSpatrick 
run(Function & F,FunctionAnalysisManager & AM)610097a140dSpatrick PreservedAnalyses AssumeBuilderPass::run(Function &F,
611097a140dSpatrick                                          FunctionAnalysisManager &AM) {
612097a140dSpatrick   AssumptionCache *AC = &AM.getResult<AssumptionAnalysis>(F);
613097a140dSpatrick   DominatorTree* DT = AM.getCachedResult<DominatorTreeAnalysis>(F);
614097a140dSpatrick   for (Instruction &I : instructions(F))
615097a140dSpatrick     salvageKnowledge(&I, AC, DT);
616097a140dSpatrick   return PreservedAnalyses::all();
617097a140dSpatrick }
618097a140dSpatrick 
619097a140dSpatrick namespace {
620097a140dSpatrick class AssumeBuilderPassLegacyPass : public FunctionPass {
621097a140dSpatrick public:
622097a140dSpatrick   static char ID;
623097a140dSpatrick 
AssumeBuilderPassLegacyPass()624097a140dSpatrick   AssumeBuilderPassLegacyPass() : FunctionPass(ID) {
625097a140dSpatrick     initializeAssumeBuilderPassLegacyPassPass(*PassRegistry::getPassRegistry());
626097a140dSpatrick   }
runOnFunction(Function & F)627097a140dSpatrick   bool runOnFunction(Function &F) override {
628097a140dSpatrick     AssumptionCache &AC =
629097a140dSpatrick         getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
630097a140dSpatrick     DominatorTreeWrapperPass *DTWP =
631097a140dSpatrick         getAnalysisIfAvailable<DominatorTreeWrapperPass>();
632097a140dSpatrick     for (Instruction &I : instructions(F))
633097a140dSpatrick       salvageKnowledge(&I, &AC, DTWP ? &DTWP->getDomTree() : nullptr);
634097a140dSpatrick     return true;
635097a140dSpatrick   }
636097a140dSpatrick 
getAnalysisUsage(AnalysisUsage & AU) const637097a140dSpatrick   void getAnalysisUsage(AnalysisUsage &AU) const override {
638097a140dSpatrick     AU.addRequired<AssumptionCacheTracker>();
639097a140dSpatrick 
640097a140dSpatrick     AU.setPreservesAll();
641097a140dSpatrick   }
642097a140dSpatrick };
643097a140dSpatrick } // namespace
644097a140dSpatrick 
645097a140dSpatrick char AssumeBuilderPassLegacyPass::ID = 0;
646097a140dSpatrick 
647097a140dSpatrick INITIALIZE_PASS_BEGIN(AssumeBuilderPassLegacyPass, "assume-builder",
648097a140dSpatrick                       "Assume Builder", false, false)
649097a140dSpatrick INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
650097a140dSpatrick INITIALIZE_PASS_END(AssumeBuilderPassLegacyPass, "assume-builder",
651097a140dSpatrick                     "Assume Builder", false, false)
652