xref: /netbsd-src/external/apache2/llvm/dist/llvm/lib/Transforms/Utils/SimplifyCFG.cpp (revision 82d56013d7b633d116a93943de88e08335357a7c)
1 //===- SimplifyCFG.cpp - Code to perform CFG simplification ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Peephole optimize the CFG.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ADT/APInt.h"
14 #include "llvm/ADT/ArrayRef.h"
15 #include "llvm/ADT/DenseMap.h"
16 #include "llvm/ADT/MapVector.h"
17 #include "llvm/ADT/Optional.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/ADT/ScopeExit.h"
20 #include "llvm/ADT/Sequence.h"
21 #include "llvm/ADT/SetOperations.h"
22 #include "llvm/ADT/SetVector.h"
23 #include "llvm/ADT/SmallPtrSet.h"
24 #include "llvm/ADT/SmallVector.h"
25 #include "llvm/ADT/Statistic.h"
26 #include "llvm/ADT/StringRef.h"
27 #include "llvm/Analysis/AssumptionCache.h"
28 #include "llvm/Analysis/ConstantFolding.h"
29 #include "llvm/Analysis/EHPersonalities.h"
30 #include "llvm/Analysis/GuardUtils.h"
31 #include "llvm/Analysis/InstructionSimplify.h"
32 #include "llvm/Analysis/MemorySSA.h"
33 #include "llvm/Analysis/MemorySSAUpdater.h"
34 #include "llvm/Analysis/TargetTransformInfo.h"
35 #include "llvm/Analysis/ValueTracking.h"
36 #include "llvm/IR/Attributes.h"
37 #include "llvm/IR/BasicBlock.h"
38 #include "llvm/IR/CFG.h"
39 #include "llvm/IR/Constant.h"
40 #include "llvm/IR/ConstantRange.h"
41 #include "llvm/IR/Constants.h"
42 #include "llvm/IR/DataLayout.h"
43 #include "llvm/IR/DerivedTypes.h"
44 #include "llvm/IR/Function.h"
45 #include "llvm/IR/GlobalValue.h"
46 #include "llvm/IR/GlobalVariable.h"
47 #include "llvm/IR/IRBuilder.h"
48 #include "llvm/IR/InstrTypes.h"
49 #include "llvm/IR/Instruction.h"
50 #include "llvm/IR/Instructions.h"
51 #include "llvm/IR/IntrinsicInst.h"
52 #include "llvm/IR/Intrinsics.h"
53 #include "llvm/IR/LLVMContext.h"
54 #include "llvm/IR/MDBuilder.h"
55 #include "llvm/IR/Metadata.h"
56 #include "llvm/IR/Module.h"
57 #include "llvm/IR/NoFolder.h"
58 #include "llvm/IR/Operator.h"
59 #include "llvm/IR/PatternMatch.h"
60 #include "llvm/IR/PseudoProbe.h"
61 #include "llvm/IR/Type.h"
62 #include "llvm/IR/Use.h"
63 #include "llvm/IR/User.h"
64 #include "llvm/IR/Value.h"
65 #include "llvm/IR/ValueHandle.h"
66 #include "llvm/Support/BranchProbability.h"
67 #include "llvm/Support/Casting.h"
68 #include "llvm/Support/CommandLine.h"
69 #include "llvm/Support/Debug.h"
70 #include "llvm/Support/ErrorHandling.h"
71 #include "llvm/Support/KnownBits.h"
72 #include "llvm/Support/MathExtras.h"
73 #include "llvm/Support/raw_ostream.h"
74 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
75 #include "llvm/Transforms/Utils/Local.h"
76 #include "llvm/Transforms/Utils/SSAUpdater.h"
77 #include "llvm/Transforms/Utils/ValueMapper.h"
78 #include <algorithm>
79 #include <cassert>
80 #include <climits>
81 #include <cstddef>
82 #include <cstdint>
83 #include <iterator>
84 #include <map>
85 #include <set>
86 #include <tuple>
87 #include <utility>
88 #include <vector>
89 
90 using namespace llvm;
91 using namespace PatternMatch;
92 
93 #define DEBUG_TYPE "simplifycfg"
94 
95 cl::opt<bool> llvm::RequireAndPreserveDomTree(
96     "simplifycfg-require-and-preserve-domtree", cl::Hidden, cl::ZeroOrMore,
97     cl::init(false),
98     cl::desc("Temorary development switch used to gradually uplift SimplifyCFG "
99              "into preserving DomTree,"));
100 
101 // Chosen as 2 so as to be cheap, but still to have enough power to fold
102 // a select, so the "clamp" idiom (of a min followed by a max) will be caught.
103 // To catch this, we need to fold a compare and a select, hence '2' being the
104 // minimum reasonable default.
105 static cl::opt<unsigned> PHINodeFoldingThreshold(
106     "phi-node-folding-threshold", cl::Hidden, cl::init(2),
107     cl::desc(
108         "Control the amount of phi node folding to perform (default = 2)"));
109 
110 static cl::opt<unsigned> TwoEntryPHINodeFoldingThreshold(
111     "two-entry-phi-node-folding-threshold", cl::Hidden, cl::init(4),
112     cl::desc("Control the maximal total instruction cost that we are willing "
113              "to speculatively execute to fold a 2-entry PHI node into a "
114              "select (default = 4)"));
115 
116 static cl::opt<bool> DupRet(
117     "simplifycfg-dup-ret", cl::Hidden, cl::init(false),
118     cl::desc("Duplicate return instructions into unconditional branches"));
119 
120 static cl::opt<bool>
121     HoistCommon("simplifycfg-hoist-common", cl::Hidden, cl::init(true),
122                 cl::desc("Hoist common instructions up to the parent block"));
123 
124 static cl::opt<bool>
125     SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true),
126                cl::desc("Sink common instructions down to the end block"));
127 
128 static cl::opt<bool> HoistCondStores(
129     "simplifycfg-hoist-cond-stores", cl::Hidden, cl::init(true),
130     cl::desc("Hoist conditional stores if an unconditional store precedes"));
131 
132 static cl::opt<bool> MergeCondStores(
133     "simplifycfg-merge-cond-stores", cl::Hidden, cl::init(true),
134     cl::desc("Hoist conditional stores even if an unconditional store does not "
135              "precede - hoist multiple conditional stores into a single "
136              "predicated store"));
137 
138 static cl::opt<bool> MergeCondStoresAggressively(
139     "simplifycfg-merge-cond-stores-aggressively", cl::Hidden, cl::init(false),
140     cl::desc("When merging conditional stores, do so even if the resultant "
141              "basic blocks are unlikely to be if-converted as a result"));
142 
143 static cl::opt<bool> SpeculateOneExpensiveInst(
144     "speculate-one-expensive-inst", cl::Hidden, cl::init(true),
145     cl::desc("Allow exactly one expensive instruction to be speculatively "
146              "executed"));
147 
148 static cl::opt<unsigned> MaxSpeculationDepth(
149     "max-speculation-depth", cl::Hidden, cl::init(10),
150     cl::desc("Limit maximum recursion depth when calculating costs of "
151              "speculatively executed instructions"));
152 
153 static cl::opt<int>
154     MaxSmallBlockSize("simplifycfg-max-small-block-size", cl::Hidden,
155                       cl::init(10),
156                       cl::desc("Max size of a block which is still considered "
157                                "small enough to thread through"));
158 
159 // Two is chosen to allow one negation and a logical combine.
160 static cl::opt<unsigned>
161     BranchFoldThreshold("simplifycfg-branch-fold-threshold", cl::Hidden,
162                         cl::init(2),
163                         cl::desc("Maximum cost of combining conditions when "
164                                  "folding branches"));
165 
166 STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps");
167 STATISTIC(NumLinearMaps,
168           "Number of switch instructions turned into linear mapping");
169 STATISTIC(NumLookupTables,
170           "Number of switch instructions turned into lookup tables");
171 STATISTIC(
172     NumLookupTablesHoles,
173     "Number of switch instructions turned into lookup tables (holes checked)");
174 STATISTIC(NumTableCmpReuses, "Number of reused switch table lookup compares");
175 STATISTIC(NumFoldValueComparisonIntoPredecessors,
176           "Number of value comparisons folded into predecessor basic blocks");
177 STATISTIC(NumFoldBranchToCommonDest,
178           "Number of branches folded into predecessor basic block");
179 STATISTIC(
180     NumHoistCommonCode,
181     "Number of common instruction 'blocks' hoisted up to the begin block");
182 STATISTIC(NumHoistCommonInstrs,
183           "Number of common instructions hoisted up to the begin block");
184 STATISTIC(NumSinkCommonCode,
185           "Number of common instruction 'blocks' sunk down to the end block");
186 STATISTIC(NumSinkCommonInstrs,
187           "Number of common instructions sunk down to the end block");
188 STATISTIC(NumSpeculations, "Number of speculative executed instructions");
189 STATISTIC(NumInvokes,
190           "Number of invokes with empty resume blocks simplified into calls");
191 
192 namespace {
193 
194 // The first field contains the value that the switch produces when a certain
195 // case group is selected, and the second field is a vector containing the
196 // cases composing the case group.
197 using SwitchCaseResultVectorTy =
198     SmallVector<std::pair<Constant *, SmallVector<ConstantInt *, 4>>, 2>;
199 
200 // The first field contains the phi node that generates a result of the switch
201 // and the second field contains the value generated for a certain case in the
202 // switch for that PHI.
203 using SwitchCaseResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>;
204 
205 /// ValueEqualityComparisonCase - Represents a case of a switch.
206 struct ValueEqualityComparisonCase {
207   ConstantInt *Value;
208   BasicBlock *Dest;
209 
ValueEqualityComparisonCase__anon9ac0a6d60111::ValueEqualityComparisonCase210   ValueEqualityComparisonCase(ConstantInt *Value, BasicBlock *Dest)
211       : Value(Value), Dest(Dest) {}
212 
operator <__anon9ac0a6d60111::ValueEqualityComparisonCase213   bool operator<(ValueEqualityComparisonCase RHS) const {
214     // Comparing pointers is ok as we only rely on the order for uniquing.
215     return Value < RHS.Value;
216   }
217 
operator ==__anon9ac0a6d60111::ValueEqualityComparisonCase218   bool operator==(BasicBlock *RHSDest) const { return Dest == RHSDest; }
219 };
220 
221 class SimplifyCFGOpt {
222   const TargetTransformInfo &TTI;
223   DomTreeUpdater *DTU;
224   const DataLayout &DL;
225   ArrayRef<WeakVH> LoopHeaders;
226   const SimplifyCFGOptions &Options;
227   bool Resimplify;
228 
229   Value *isValueEqualityComparison(Instruction *TI);
230   BasicBlock *GetValueEqualityComparisonCases(
231       Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases);
232   bool SimplifyEqualityComparisonWithOnlyPredecessor(Instruction *TI,
233                                                      BasicBlock *Pred,
234                                                      IRBuilder<> &Builder);
235   bool PerformValueComparisonIntoPredecessorFolding(Instruction *TI, Value *&CV,
236                                                     Instruction *PTI,
237                                                     IRBuilder<> &Builder);
238   bool FoldValueComparisonIntoPredecessors(Instruction *TI,
239                                            IRBuilder<> &Builder);
240 
241   bool simplifyReturn(ReturnInst *RI, IRBuilder<> &Builder);
242   bool simplifyResume(ResumeInst *RI, IRBuilder<> &Builder);
243   bool simplifySingleResume(ResumeInst *RI);
244   bool simplifyCommonResume(ResumeInst *RI);
245   bool simplifyCleanupReturn(CleanupReturnInst *RI);
246   bool simplifyUnreachable(UnreachableInst *UI);
247   bool simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder);
248   bool simplifyIndirectBr(IndirectBrInst *IBI);
249   bool simplifyBranch(BranchInst *Branch, IRBuilder<> &Builder);
250   bool simplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder);
251   bool simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder);
252   bool SimplifyCondBranchToTwoReturns(BranchInst *BI, IRBuilder<> &Builder);
253 
254   bool tryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
255                                              IRBuilder<> &Builder);
256 
257   bool HoistThenElseCodeToIf(BranchInst *BI, const TargetTransformInfo &TTI,
258                              bool EqTermsOnly);
259   bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB,
260                               const TargetTransformInfo &TTI);
261   bool SimplifyTerminatorOnSelect(Instruction *OldTerm, Value *Cond,
262                                   BasicBlock *TrueBB, BasicBlock *FalseBB,
263                                   uint32_t TrueWeight, uint32_t FalseWeight);
264   bool SimplifyBranchOnICmpChain(BranchInst *BI, IRBuilder<> &Builder,
265                                  const DataLayout &DL);
266   bool SimplifySwitchOnSelect(SwitchInst *SI, SelectInst *Select);
267   bool SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI);
268   bool TurnSwitchRangeIntoICmp(SwitchInst *SI, IRBuilder<> &Builder);
269 
270 public:
SimplifyCFGOpt(const TargetTransformInfo & TTI,DomTreeUpdater * DTU,const DataLayout & DL,ArrayRef<WeakVH> LoopHeaders,const SimplifyCFGOptions & Opts)271   SimplifyCFGOpt(const TargetTransformInfo &TTI, DomTreeUpdater *DTU,
272                  const DataLayout &DL, ArrayRef<WeakVH> LoopHeaders,
273                  const SimplifyCFGOptions &Opts)
274       : TTI(TTI), DTU(DTU), DL(DL), LoopHeaders(LoopHeaders), Options(Opts) {
275     assert((!DTU || !DTU->hasPostDomTree()) &&
276            "SimplifyCFG is not yet capable of maintaining validity of a "
277            "PostDomTree, so don't ask for it.");
278   }
279 
280   bool simplifyOnce(BasicBlock *BB);
281   bool simplifyOnceImpl(BasicBlock *BB);
282   bool run(BasicBlock *BB);
283 
284   // Helper to set Resimplify and return change indication.
requestResimplify()285   bool requestResimplify() {
286     Resimplify = true;
287     return true;
288   }
289 };
290 
291 } // end anonymous namespace
292 
293 /// Return true if it is safe to merge these two
294 /// terminator instructions together.
295 static bool
SafeToMergeTerminators(Instruction * SI1,Instruction * SI2,SmallSetVector<BasicBlock *,4> * FailBlocks=nullptr)296 SafeToMergeTerminators(Instruction *SI1, Instruction *SI2,
297                        SmallSetVector<BasicBlock *, 4> *FailBlocks = nullptr) {
298   if (SI1 == SI2)
299     return false; // Can't merge with self!
300 
301   // It is not safe to merge these two switch instructions if they have a common
302   // successor, and if that successor has a PHI node, and if *that* PHI node has
303   // conflicting incoming values from the two switch blocks.
304   BasicBlock *SI1BB = SI1->getParent();
305   BasicBlock *SI2BB = SI2->getParent();
306 
307   SmallPtrSet<BasicBlock *, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB));
308   bool Fail = false;
309   for (BasicBlock *Succ : successors(SI2BB))
310     if (SI1Succs.count(Succ))
311       for (BasicBlock::iterator BBI = Succ->begin(); isa<PHINode>(BBI); ++BBI) {
312         PHINode *PN = cast<PHINode>(BBI);
313         if (PN->getIncomingValueForBlock(SI1BB) !=
314             PN->getIncomingValueForBlock(SI2BB)) {
315           if (FailBlocks)
316             FailBlocks->insert(Succ);
317           Fail = true;
318         }
319       }
320 
321   return !Fail;
322 }
323 
324 /// Update PHI nodes in Succ to indicate that there will now be entries in it
325 /// from the 'NewPred' block. The values that will be flowing into the PHI nodes
326 /// will be the same as those coming in from ExistPred, an existing predecessor
327 /// of Succ.
AddPredecessorToBlock(BasicBlock * Succ,BasicBlock * NewPred,BasicBlock * ExistPred,MemorySSAUpdater * MSSAU=nullptr)328 static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred,
329                                   BasicBlock *ExistPred,
330                                   MemorySSAUpdater *MSSAU = nullptr) {
331   for (PHINode &PN : Succ->phis())
332     PN.addIncoming(PN.getIncomingValueForBlock(ExistPred), NewPred);
333   if (MSSAU)
334     if (auto *MPhi = MSSAU->getMemorySSA()->getMemoryAccess(Succ))
335       MPhi->addIncoming(MPhi->getIncomingValueForBlock(ExistPred), NewPred);
336 }
337 
338 /// Compute an abstract "cost" of speculating the given instruction,
339 /// which is assumed to be safe to speculate. TCC_Free means cheap,
340 /// TCC_Basic means less cheap, and TCC_Expensive means prohibitively
341 /// expensive.
computeSpeculationCost(const User * I,const TargetTransformInfo & TTI)342 static InstructionCost computeSpeculationCost(const User *I,
343                                               const TargetTransformInfo &TTI) {
344   assert(isSafeToSpeculativelyExecute(I) &&
345          "Instruction is not safe to speculatively execute!");
346   return TTI.getUserCost(I, TargetTransformInfo::TCK_SizeAndLatency);
347 }
348 
349 /// If we have a merge point of an "if condition" as accepted above,
350 /// return true if the specified value dominates the block.  We
351 /// don't handle the true generality of domination here, just a special case
352 /// which works well enough for us.
353 ///
354 /// If AggressiveInsts is non-null, and if V does not dominate BB, we check to
355 /// see if V (which must be an instruction) and its recursive operands
356 /// that do not dominate BB have a combined cost lower than Budget and
357 /// are non-trapping.  If both are true, the instruction is inserted into the
358 /// set and true is returned.
359 ///
360 /// The cost for most non-trapping instructions is defined as 1 except for
361 /// Select whose cost is 2.
362 ///
363 /// After this function returns, Cost is increased by the cost of
364 /// V plus its non-dominating operands.  If that cost is greater than
365 /// Budget, false is returned and Cost is undefined.
dominatesMergePoint(Value * V,BasicBlock * BB,SmallPtrSetImpl<Instruction * > & AggressiveInsts,InstructionCost & Cost,InstructionCost Budget,const TargetTransformInfo & TTI,unsigned Depth=0)366 static bool dominatesMergePoint(Value *V, BasicBlock *BB,
367                                 SmallPtrSetImpl<Instruction *> &AggressiveInsts,
368                                 InstructionCost &Cost,
369                                 InstructionCost Budget,
370                                 const TargetTransformInfo &TTI,
371                                 unsigned Depth = 0) {
372   // It is possible to hit a zero-cost cycle (phi/gep instructions for example),
373   // so limit the recursion depth.
374   // TODO: While this recursion limit does prevent pathological behavior, it
375   // would be better to track visited instructions to avoid cycles.
376   if (Depth == MaxSpeculationDepth)
377     return false;
378 
379   Instruction *I = dyn_cast<Instruction>(V);
380   if (!I) {
381     // Non-instructions all dominate instructions, but not all constantexprs
382     // can be executed unconditionally.
383     if (ConstantExpr *C = dyn_cast<ConstantExpr>(V))
384       if (C->canTrap())
385         return false;
386     return true;
387   }
388   BasicBlock *PBB = I->getParent();
389 
390   // We don't want to allow weird loops that might have the "if condition" in
391   // the bottom of this block.
392   if (PBB == BB)
393     return false;
394 
395   // If this instruction is defined in a block that contains an unconditional
396   // branch to BB, then it must be in the 'conditional' part of the "if
397   // statement".  If not, it definitely dominates the region.
398   BranchInst *BI = dyn_cast<BranchInst>(PBB->getTerminator());
399   if (!BI || BI->isConditional() || BI->getSuccessor(0) != BB)
400     return true;
401 
402   // If we have seen this instruction before, don't count it again.
403   if (AggressiveInsts.count(I))
404     return true;
405 
406   // Okay, it looks like the instruction IS in the "condition".  Check to
407   // see if it's a cheap instruction to unconditionally compute, and if it
408   // only uses stuff defined outside of the condition.  If so, hoist it out.
409   if (!isSafeToSpeculativelyExecute(I))
410     return false;
411 
412   Cost += computeSpeculationCost(I, TTI);
413 
414   // Allow exactly one instruction to be speculated regardless of its cost
415   // (as long as it is safe to do so).
416   // This is intended to flatten the CFG even if the instruction is a division
417   // or other expensive operation. The speculation of an expensive instruction
418   // is expected to be undone in CodeGenPrepare if the speculation has not
419   // enabled further IR optimizations.
420   if (Cost > Budget &&
421       (!SpeculateOneExpensiveInst || !AggressiveInsts.empty() || Depth > 0 ||
422        !Cost.isValid()))
423     return false;
424 
425   // Okay, we can only really hoist these out if their operands do
426   // not take us over the cost threshold.
427   for (Use &Op : I->operands())
428     if (!dominatesMergePoint(Op, BB, AggressiveInsts, Cost, Budget, TTI,
429                              Depth + 1))
430       return false;
431   // Okay, it's safe to do this!  Remember this instruction.
432   AggressiveInsts.insert(I);
433   return true;
434 }
435 
436 /// Extract ConstantInt from value, looking through IntToPtr
437 /// and PointerNullValue. Return NULL if value is not a constant int.
GetConstantInt(Value * V,const DataLayout & DL)438 static ConstantInt *GetConstantInt(Value *V, const DataLayout &DL) {
439   // Normal constant int.
440   ConstantInt *CI = dyn_cast<ConstantInt>(V);
441   if (CI || !isa<Constant>(V) || !V->getType()->isPointerTy())
442     return CI;
443 
444   // This is some kind of pointer constant. Turn it into a pointer-sized
445   // ConstantInt if possible.
446   IntegerType *PtrTy = cast<IntegerType>(DL.getIntPtrType(V->getType()));
447 
448   // Null pointer means 0, see SelectionDAGBuilder::getValue(const Value*).
449   if (isa<ConstantPointerNull>(V))
450     return ConstantInt::get(PtrTy, 0);
451 
452   // IntToPtr const int.
453   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
454     if (CE->getOpcode() == Instruction::IntToPtr)
455       if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(0))) {
456         // The constant is very likely to have the right type already.
457         if (CI->getType() == PtrTy)
458           return CI;
459         else
460           return cast<ConstantInt>(
461               ConstantExpr::getIntegerCast(CI, PtrTy, /*isSigned=*/false));
462       }
463   return nullptr;
464 }
465 
466 namespace {
467 
468 /// Given a chain of or (||) or and (&&) comparison of a value against a
469 /// constant, this will try to recover the information required for a switch
470 /// structure.
471 /// It will depth-first traverse the chain of comparison, seeking for patterns
472 /// like %a == 12 or %a < 4 and combine them to produce a set of integer
473 /// representing the different cases for the switch.
474 /// Note that if the chain is composed of '||' it will build the set of elements
475 /// that matches the comparisons (i.e. any of this value validate the chain)
476 /// while for a chain of '&&' it will build the set elements that make the test
477 /// fail.
478 struct ConstantComparesGatherer {
479   const DataLayout &DL;
480 
481   /// Value found for the switch comparison
482   Value *CompValue = nullptr;
483 
484   /// Extra clause to be checked before the switch
485   Value *Extra = nullptr;
486 
487   /// Set of integers to match in switch
488   SmallVector<ConstantInt *, 8> Vals;
489 
490   /// Number of comparisons matched in the and/or chain
491   unsigned UsedICmps = 0;
492 
493   /// Construct and compute the result for the comparison instruction Cond
ConstantComparesGatherer__anon9ac0a6d60211::ConstantComparesGatherer494   ConstantComparesGatherer(Instruction *Cond, const DataLayout &DL) : DL(DL) {
495     gather(Cond);
496   }
497 
498   ConstantComparesGatherer(const ConstantComparesGatherer &) = delete;
499   ConstantComparesGatherer &
500   operator=(const ConstantComparesGatherer &) = delete;
501 
502 private:
503   /// Try to set the current value used for the comparison, it succeeds only if
504   /// it wasn't set before or if the new value is the same as the old one
setValueOnce__anon9ac0a6d60211::ConstantComparesGatherer505   bool setValueOnce(Value *NewVal) {
506     if (CompValue && CompValue != NewVal)
507       return false;
508     CompValue = NewVal;
509     return (CompValue != nullptr);
510   }
511 
512   /// Try to match Instruction "I" as a comparison against a constant and
513   /// populates the array Vals with the set of values that match (or do not
514   /// match depending on isEQ).
515   /// Return false on failure. On success, the Value the comparison matched
516   /// against is placed in CompValue.
517   /// If CompValue is already set, the function is expected to fail if a match
518   /// is found but the value compared to is different.
matchInstruction__anon9ac0a6d60211::ConstantComparesGatherer519   bool matchInstruction(Instruction *I, bool isEQ) {
520     // If this is an icmp against a constant, handle this as one of the cases.
521     ICmpInst *ICI;
522     ConstantInt *C;
523     if (!((ICI = dyn_cast<ICmpInst>(I)) &&
524           (C = GetConstantInt(I->getOperand(1), DL)))) {
525       return false;
526     }
527 
528     Value *RHSVal;
529     const APInt *RHSC;
530 
531     // Pattern match a special case
532     // (x & ~2^z) == y --> x == y || x == y|2^z
533     // This undoes a transformation done by instcombine to fuse 2 compares.
534     if (ICI->getPredicate() == (isEQ ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE)) {
535       // It's a little bit hard to see why the following transformations are
536       // correct. Here is a CVC3 program to verify them for 64-bit values:
537 
538       /*
539          ONE  : BITVECTOR(64) = BVZEROEXTEND(0bin1, 63);
540          x    : BITVECTOR(64);
541          y    : BITVECTOR(64);
542          z    : BITVECTOR(64);
543          mask : BITVECTOR(64) = BVSHL(ONE, z);
544          QUERY( (y & ~mask = y) =>
545                 ((x & ~mask = y) <=> (x = y OR x = (y |  mask)))
546          );
547          QUERY( (y |  mask = y) =>
548                 ((x |  mask = y) <=> (x = y OR x = (y & ~mask)))
549          );
550       */
551 
552       // Please note that each pattern must be a dual implication (<--> or
553       // iff). One directional implication can create spurious matches. If the
554       // implication is only one-way, an unsatisfiable condition on the left
555       // side can imply a satisfiable condition on the right side. Dual
556       // implication ensures that satisfiable conditions are transformed to
557       // other satisfiable conditions and unsatisfiable conditions are
558       // transformed to other unsatisfiable conditions.
559 
560       // Here is a concrete example of a unsatisfiable condition on the left
561       // implying a satisfiable condition on the right:
562       //
563       // mask = (1 << z)
564       // (x & ~mask) == y  --> (x == y || x == (y | mask))
565       //
566       // Substituting y = 3, z = 0 yields:
567       // (x & -2) == 3 --> (x == 3 || x == 2)
568 
569       // Pattern match a special case:
570       /*
571         QUERY( (y & ~mask = y) =>
572                ((x & ~mask = y) <=> (x = y OR x = (y |  mask)))
573         );
574       */
575       if (match(ICI->getOperand(0),
576                 m_And(m_Value(RHSVal), m_APInt(RHSC)))) {
577         APInt Mask = ~*RHSC;
578         if (Mask.isPowerOf2() && (C->getValue() & ~Mask) == C->getValue()) {
579           // If we already have a value for the switch, it has to match!
580           if (!setValueOnce(RHSVal))
581             return false;
582 
583           Vals.push_back(C);
584           Vals.push_back(
585               ConstantInt::get(C->getContext(),
586                                C->getValue() | Mask));
587           UsedICmps++;
588           return true;
589         }
590       }
591 
592       // Pattern match a special case:
593       /*
594         QUERY( (y |  mask = y) =>
595                ((x |  mask = y) <=> (x = y OR x = (y & ~mask)))
596         );
597       */
598       if (match(ICI->getOperand(0),
599                 m_Or(m_Value(RHSVal), m_APInt(RHSC)))) {
600         APInt Mask = *RHSC;
601         if (Mask.isPowerOf2() && (C->getValue() | Mask) == C->getValue()) {
602           // If we already have a value for the switch, it has to match!
603           if (!setValueOnce(RHSVal))
604             return false;
605 
606           Vals.push_back(C);
607           Vals.push_back(ConstantInt::get(C->getContext(),
608                                           C->getValue() & ~Mask));
609           UsedICmps++;
610           return true;
611         }
612       }
613 
614       // If we already have a value for the switch, it has to match!
615       if (!setValueOnce(ICI->getOperand(0)))
616         return false;
617 
618       UsedICmps++;
619       Vals.push_back(C);
620       return ICI->getOperand(0);
621     }
622 
623     // If we have "x ult 3", for example, then we can add 0,1,2 to the set.
624     ConstantRange Span = ConstantRange::makeAllowedICmpRegion(
625         ICI->getPredicate(), C->getValue());
626 
627     // Shift the range if the compare is fed by an add. This is the range
628     // compare idiom as emitted by instcombine.
629     Value *CandidateVal = I->getOperand(0);
630     if (match(I->getOperand(0), m_Add(m_Value(RHSVal), m_APInt(RHSC)))) {
631       Span = Span.subtract(*RHSC);
632       CandidateVal = RHSVal;
633     }
634 
635     // If this is an and/!= check, then we are looking to build the set of
636     // value that *don't* pass the and chain. I.e. to turn "x ugt 2" into
637     // x != 0 && x != 1.
638     if (!isEQ)
639       Span = Span.inverse();
640 
641     // If there are a ton of values, we don't want to make a ginormous switch.
642     if (Span.isSizeLargerThan(8) || Span.isEmptySet()) {
643       return false;
644     }
645 
646     // If we already have a value for the switch, it has to match!
647     if (!setValueOnce(CandidateVal))
648       return false;
649 
650     // Add all values from the range to the set
651     for (APInt Tmp = Span.getLower(); Tmp != Span.getUpper(); ++Tmp)
652       Vals.push_back(ConstantInt::get(I->getContext(), Tmp));
653 
654     UsedICmps++;
655     return true;
656   }
657 
658   /// Given a potentially 'or'd or 'and'd together collection of icmp
659   /// eq/ne/lt/gt instructions that compare a value against a constant, extract
660   /// the value being compared, and stick the list constants into the Vals
661   /// vector.
662   /// One "Extra" case is allowed to differ from the other.
gather__anon9ac0a6d60211::ConstantComparesGatherer663   void gather(Value *V) {
664     bool isEQ = match(V, m_LogicalOr(m_Value(), m_Value()));
665 
666     // Keep a stack (SmallVector for efficiency) for depth-first traversal
667     SmallVector<Value *, 8> DFT;
668     SmallPtrSet<Value *, 8> Visited;
669 
670     // Initialize
671     Visited.insert(V);
672     DFT.push_back(V);
673 
674     while (!DFT.empty()) {
675       V = DFT.pop_back_val();
676 
677       if (Instruction *I = dyn_cast<Instruction>(V)) {
678         // If it is a || (or && depending on isEQ), process the operands.
679         Value *Op0, *Op1;
680         if (isEQ ? match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1)))
681                  : match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) {
682           if (Visited.insert(Op1).second)
683             DFT.push_back(Op1);
684           if (Visited.insert(Op0).second)
685             DFT.push_back(Op0);
686 
687           continue;
688         }
689 
690         // Try to match the current instruction
691         if (matchInstruction(I, isEQ))
692           // Match succeed, continue the loop
693           continue;
694       }
695 
696       // One element of the sequence of || (or &&) could not be match as a
697       // comparison against the same value as the others.
698       // We allow only one "Extra" case to be checked before the switch
699       if (!Extra) {
700         Extra = V;
701         continue;
702       }
703       // Failed to parse a proper sequence, abort now
704       CompValue = nullptr;
705       break;
706     }
707   }
708 };
709 
710 } // end anonymous namespace
711 
EraseTerminatorAndDCECond(Instruction * TI,MemorySSAUpdater * MSSAU=nullptr)712 static void EraseTerminatorAndDCECond(Instruction *TI,
713                                       MemorySSAUpdater *MSSAU = nullptr) {
714   Instruction *Cond = nullptr;
715   if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
716     Cond = dyn_cast<Instruction>(SI->getCondition());
717   } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
718     if (BI->isConditional())
719       Cond = dyn_cast<Instruction>(BI->getCondition());
720   } else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(TI)) {
721     Cond = dyn_cast<Instruction>(IBI->getAddress());
722   }
723 
724   TI->eraseFromParent();
725   if (Cond)
726     RecursivelyDeleteTriviallyDeadInstructions(Cond, nullptr, MSSAU);
727 }
728 
729 /// Return true if the specified terminator checks
730 /// to see if a value is equal to constant integer value.
isValueEqualityComparison(Instruction * TI)731 Value *SimplifyCFGOpt::isValueEqualityComparison(Instruction *TI) {
732   Value *CV = nullptr;
733   if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
734     // Do not permit merging of large switch instructions into their
735     // predecessors unless there is only one predecessor.
736     if (!SI->getParent()->hasNPredecessorsOrMore(128 / SI->getNumSuccessors()))
737       CV = SI->getCondition();
738   } else if (BranchInst *BI = dyn_cast<BranchInst>(TI))
739     if (BI->isConditional() && BI->getCondition()->hasOneUse())
740       if (ICmpInst *ICI = dyn_cast<ICmpInst>(BI->getCondition())) {
741         if (ICI->isEquality() && GetConstantInt(ICI->getOperand(1), DL))
742           CV = ICI->getOperand(0);
743       }
744 
745   // Unwrap any lossless ptrtoint cast.
746   if (CV) {
747     if (PtrToIntInst *PTII = dyn_cast<PtrToIntInst>(CV)) {
748       Value *Ptr = PTII->getPointerOperand();
749       if (PTII->getType() == DL.getIntPtrType(Ptr->getType()))
750         CV = Ptr;
751     }
752   }
753   return CV;
754 }
755 
756 /// Given a value comparison instruction,
757 /// decode all of the 'cases' that it represents and return the 'default' block.
GetValueEqualityComparisonCases(Instruction * TI,std::vector<ValueEqualityComparisonCase> & Cases)758 BasicBlock *SimplifyCFGOpt::GetValueEqualityComparisonCases(
759     Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases) {
760   if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
761     Cases.reserve(SI->getNumCases());
762     for (auto Case : SI->cases())
763       Cases.push_back(ValueEqualityComparisonCase(Case.getCaseValue(),
764                                                   Case.getCaseSuccessor()));
765     return SI->getDefaultDest();
766   }
767 
768   BranchInst *BI = cast<BranchInst>(TI);
769   ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
770   BasicBlock *Succ = BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_NE);
771   Cases.push_back(ValueEqualityComparisonCase(
772       GetConstantInt(ICI->getOperand(1), DL), Succ));
773   return BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_EQ);
774 }
775 
776 /// Given a vector of bb/value pairs, remove any entries
777 /// in the list that match the specified block.
778 static void
EliminateBlockCases(BasicBlock * BB,std::vector<ValueEqualityComparisonCase> & Cases)779 EliminateBlockCases(BasicBlock *BB,
780                     std::vector<ValueEqualityComparisonCase> &Cases) {
781   llvm::erase_value(Cases, BB);
782 }
783 
784 /// Return true if there are any keys in C1 that exist in C2 as well.
ValuesOverlap(std::vector<ValueEqualityComparisonCase> & C1,std::vector<ValueEqualityComparisonCase> & C2)785 static bool ValuesOverlap(std::vector<ValueEqualityComparisonCase> &C1,
786                           std::vector<ValueEqualityComparisonCase> &C2) {
787   std::vector<ValueEqualityComparisonCase> *V1 = &C1, *V2 = &C2;
788 
789   // Make V1 be smaller than V2.
790   if (V1->size() > V2->size())
791     std::swap(V1, V2);
792 
793   if (V1->empty())
794     return false;
795   if (V1->size() == 1) {
796     // Just scan V2.
797     ConstantInt *TheVal = (*V1)[0].Value;
798     for (unsigned i = 0, e = V2->size(); i != e; ++i)
799       if (TheVal == (*V2)[i].Value)
800         return true;
801   }
802 
803   // Otherwise, just sort both lists and compare element by element.
804   array_pod_sort(V1->begin(), V1->end());
805   array_pod_sort(V2->begin(), V2->end());
806   unsigned i1 = 0, i2 = 0, e1 = V1->size(), e2 = V2->size();
807   while (i1 != e1 && i2 != e2) {
808     if ((*V1)[i1].Value == (*V2)[i2].Value)
809       return true;
810     if ((*V1)[i1].Value < (*V2)[i2].Value)
811       ++i1;
812     else
813       ++i2;
814   }
815   return false;
816 }
817 
818 // Set branch weights on SwitchInst. This sets the metadata if there is at
819 // least one non-zero weight.
setBranchWeights(SwitchInst * SI,ArrayRef<uint32_t> Weights)820 static void setBranchWeights(SwitchInst *SI, ArrayRef<uint32_t> Weights) {
821   // Check that there is at least one non-zero weight. Otherwise, pass
822   // nullptr to setMetadata which will erase the existing metadata.
823   MDNode *N = nullptr;
824   if (llvm::any_of(Weights, [](uint32_t W) { return W != 0; }))
825     N = MDBuilder(SI->getParent()->getContext()).createBranchWeights(Weights);
826   SI->setMetadata(LLVMContext::MD_prof, N);
827 }
828 
829 // Similar to the above, but for branch and select instructions that take
830 // exactly 2 weights.
setBranchWeights(Instruction * I,uint32_t TrueWeight,uint32_t FalseWeight)831 static void setBranchWeights(Instruction *I, uint32_t TrueWeight,
832                              uint32_t FalseWeight) {
833   assert(isa<BranchInst>(I) || isa<SelectInst>(I));
834   // Check that there is at least one non-zero weight. Otherwise, pass
835   // nullptr to setMetadata which will erase the existing metadata.
836   MDNode *N = nullptr;
837   if (TrueWeight || FalseWeight)
838     N = MDBuilder(I->getParent()->getContext())
839             .createBranchWeights(TrueWeight, FalseWeight);
840   I->setMetadata(LLVMContext::MD_prof, N);
841 }
842 
843 /// If TI is known to be a terminator instruction and its block is known to
844 /// only have a single predecessor block, check to see if that predecessor is
845 /// also a value comparison with the same value, and if that comparison
846 /// determines the outcome of this comparison. If so, simplify TI. This does a
847 /// very limited form of jump threading.
SimplifyEqualityComparisonWithOnlyPredecessor(Instruction * TI,BasicBlock * Pred,IRBuilder<> & Builder)848 bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor(
849     Instruction *TI, BasicBlock *Pred, IRBuilder<> &Builder) {
850   Value *PredVal = isValueEqualityComparison(Pred->getTerminator());
851   if (!PredVal)
852     return false; // Not a value comparison in predecessor.
853 
854   Value *ThisVal = isValueEqualityComparison(TI);
855   assert(ThisVal && "This isn't a value comparison!!");
856   if (ThisVal != PredVal)
857     return false; // Different predicates.
858 
859   // TODO: Preserve branch weight metadata, similarly to how
860   // FoldValueComparisonIntoPredecessors preserves it.
861 
862   // Find out information about when control will move from Pred to TI's block.
863   std::vector<ValueEqualityComparisonCase> PredCases;
864   BasicBlock *PredDef =
865       GetValueEqualityComparisonCases(Pred->getTerminator(), PredCases);
866   EliminateBlockCases(PredDef, PredCases); // Remove default from cases.
867 
868   // Find information about how control leaves this block.
869   std::vector<ValueEqualityComparisonCase> ThisCases;
870   BasicBlock *ThisDef = GetValueEqualityComparisonCases(TI, ThisCases);
871   EliminateBlockCases(ThisDef, ThisCases); // Remove default from cases.
872 
873   // If TI's block is the default block from Pred's comparison, potentially
874   // simplify TI based on this knowledge.
875   if (PredDef == TI->getParent()) {
876     // If we are here, we know that the value is none of those cases listed in
877     // PredCases.  If there are any cases in ThisCases that are in PredCases, we
878     // can simplify TI.
879     if (!ValuesOverlap(PredCases, ThisCases))
880       return false;
881 
882     if (isa<BranchInst>(TI)) {
883       // Okay, one of the successors of this condbr is dead.  Convert it to a
884       // uncond br.
885       assert(ThisCases.size() == 1 && "Branch can only have one case!");
886       // Insert the new branch.
887       Instruction *NI = Builder.CreateBr(ThisDef);
888       (void)NI;
889 
890       // Remove PHI node entries for the dead edge.
891       ThisCases[0].Dest->removePredecessor(PredDef);
892 
893       LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
894                         << "Through successor TI: " << *TI << "Leaving: " << *NI
895                         << "\n");
896 
897       EraseTerminatorAndDCECond(TI);
898 
899       if (DTU)
900         DTU->applyUpdates(
901             {{DominatorTree::Delete, PredDef, ThisCases[0].Dest}});
902 
903       return true;
904     }
905 
906     SwitchInstProfUpdateWrapper SI = *cast<SwitchInst>(TI);
907     // Okay, TI has cases that are statically dead, prune them away.
908     SmallPtrSet<Constant *, 16> DeadCases;
909     for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
910       DeadCases.insert(PredCases[i].Value);
911 
912     LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
913                       << "Through successor TI: " << *TI);
914 
915     SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases;
916     for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) {
917       --i;
918       auto *Successor = i->getCaseSuccessor();
919       if (DTU)
920         ++NumPerSuccessorCases[Successor];
921       if (DeadCases.count(i->getCaseValue())) {
922         Successor->removePredecessor(PredDef);
923         SI.removeCase(i);
924         if (DTU)
925           --NumPerSuccessorCases[Successor];
926       }
927     }
928 
929     if (DTU) {
930       std::vector<DominatorTree::UpdateType> Updates;
931       for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases)
932         if (I.second == 0)
933           Updates.push_back({DominatorTree::Delete, PredDef, I.first});
934       DTU->applyUpdates(Updates);
935     }
936 
937     LLVM_DEBUG(dbgs() << "Leaving: " << *TI << "\n");
938     return true;
939   }
940 
941   // Otherwise, TI's block must correspond to some matched value.  Find out
942   // which value (or set of values) this is.
943   ConstantInt *TIV = nullptr;
944   BasicBlock *TIBB = TI->getParent();
945   for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
946     if (PredCases[i].Dest == TIBB) {
947       if (TIV)
948         return false; // Cannot handle multiple values coming to this block.
949       TIV = PredCases[i].Value;
950     }
951   assert(TIV && "No edge from pred to succ?");
952 
953   // Okay, we found the one constant that our value can be if we get into TI's
954   // BB.  Find out which successor will unconditionally be branched to.
955   BasicBlock *TheRealDest = nullptr;
956   for (unsigned i = 0, e = ThisCases.size(); i != e; ++i)
957     if (ThisCases[i].Value == TIV) {
958       TheRealDest = ThisCases[i].Dest;
959       break;
960     }
961 
962   // If not handled by any explicit cases, it is handled by the default case.
963   if (!TheRealDest)
964     TheRealDest = ThisDef;
965 
966   SmallPtrSet<BasicBlock *, 2> RemovedSuccs;
967 
968   // Remove PHI node entries for dead edges.
969   BasicBlock *CheckEdge = TheRealDest;
970   for (BasicBlock *Succ : successors(TIBB))
971     if (Succ != CheckEdge) {
972       if (Succ != TheRealDest)
973         RemovedSuccs.insert(Succ);
974       Succ->removePredecessor(TIBB);
975     } else
976       CheckEdge = nullptr;
977 
978   // Insert the new branch.
979   Instruction *NI = Builder.CreateBr(TheRealDest);
980   (void)NI;
981 
982   LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
983                     << "Through successor TI: " << *TI << "Leaving: " << *NI
984                     << "\n");
985 
986   EraseTerminatorAndDCECond(TI);
987   if (DTU) {
988     SmallVector<DominatorTree::UpdateType, 2> Updates;
989     Updates.reserve(RemovedSuccs.size());
990     for (auto *RemovedSucc : RemovedSuccs)
991       Updates.push_back({DominatorTree::Delete, TIBB, RemovedSucc});
992     DTU->applyUpdates(Updates);
993   }
994   return true;
995 }
996 
997 namespace {
998 
999 /// This class implements a stable ordering of constant
1000 /// integers that does not depend on their address.  This is important for
1001 /// applications that sort ConstantInt's to ensure uniqueness.
1002 struct ConstantIntOrdering {
operator ()__anon9ac0a6d60411::ConstantIntOrdering1003   bool operator()(const ConstantInt *LHS, const ConstantInt *RHS) const {
1004     return LHS->getValue().ult(RHS->getValue());
1005   }
1006 };
1007 
1008 } // end anonymous namespace
1009 
ConstantIntSortPredicate(ConstantInt * const * P1,ConstantInt * const * P2)1010 static int ConstantIntSortPredicate(ConstantInt *const *P1,
1011                                     ConstantInt *const *P2) {
1012   const ConstantInt *LHS = *P1;
1013   const ConstantInt *RHS = *P2;
1014   if (LHS == RHS)
1015     return 0;
1016   return LHS->getValue().ult(RHS->getValue()) ? 1 : -1;
1017 }
1018 
HasBranchWeights(const Instruction * I)1019 static inline bool HasBranchWeights(const Instruction *I) {
1020   MDNode *ProfMD = I->getMetadata(LLVMContext::MD_prof);
1021   if (ProfMD && ProfMD->getOperand(0))
1022     if (MDString *MDS = dyn_cast<MDString>(ProfMD->getOperand(0)))
1023       return MDS->getString().equals("branch_weights");
1024 
1025   return false;
1026 }
1027 
1028 /// Get Weights of a given terminator, the default weight is at the front
1029 /// of the vector. If TI is a conditional eq, we need to swap the branch-weight
1030 /// metadata.
GetBranchWeights(Instruction * TI,SmallVectorImpl<uint64_t> & Weights)1031 static void GetBranchWeights(Instruction *TI,
1032                              SmallVectorImpl<uint64_t> &Weights) {
1033   MDNode *MD = TI->getMetadata(LLVMContext::MD_prof);
1034   assert(MD);
1035   for (unsigned i = 1, e = MD->getNumOperands(); i < e; ++i) {
1036     ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(i));
1037     Weights.push_back(CI->getValue().getZExtValue());
1038   }
1039 
1040   // If TI is a conditional eq, the default case is the false case,
1041   // and the corresponding branch-weight data is at index 2. We swap the
1042   // default weight to be the first entry.
1043   if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
1044     assert(Weights.size() == 2);
1045     ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
1046     if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
1047       std::swap(Weights.front(), Weights.back());
1048   }
1049 }
1050 
1051 /// Keep halving the weights until all can fit in uint32_t.
FitWeights(MutableArrayRef<uint64_t> Weights)1052 static void FitWeights(MutableArrayRef<uint64_t> Weights) {
1053   uint64_t Max = *std::max_element(Weights.begin(), Weights.end());
1054   if (Max > UINT_MAX) {
1055     unsigned Offset = 32 - countLeadingZeros(Max);
1056     for (uint64_t &I : Weights)
1057       I >>= Offset;
1058   }
1059 }
1060 
CloneInstructionsIntoPredecessorBlockAndUpdateSSAUses(BasicBlock * BB,BasicBlock * PredBlock,ValueToValueMapTy & VMap)1061 static void CloneInstructionsIntoPredecessorBlockAndUpdateSSAUses(
1062     BasicBlock *BB, BasicBlock *PredBlock, ValueToValueMapTy &VMap) {
1063   Instruction *PTI = PredBlock->getTerminator();
1064 
1065   // If we have bonus instructions, clone them into the predecessor block.
1066   // Note that there may be multiple predecessor blocks, so we cannot move
1067   // bonus instructions to a predecessor block.
1068   for (Instruction &BonusInst : *BB) {
1069     if (isa<DbgInfoIntrinsic>(BonusInst) || BonusInst.isTerminator())
1070       continue;
1071 
1072     Instruction *NewBonusInst = BonusInst.clone();
1073 
1074     if (PTI->getDebugLoc() != NewBonusInst->getDebugLoc()) {
1075       // Unless the instruction has the same !dbg location as the original
1076       // branch, drop it. When we fold the bonus instructions we want to make
1077       // sure we reset their debug locations in order to avoid stepping on
1078       // dead code caused by folding dead branches.
1079       NewBonusInst->setDebugLoc(DebugLoc());
1080     }
1081 
1082     RemapInstruction(NewBonusInst, VMap,
1083                      RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
1084     VMap[&BonusInst] = NewBonusInst;
1085 
1086     // If we moved a load, we cannot any longer claim any knowledge about
1087     // its potential value. The previous information might have been valid
1088     // only given the branch precondition.
1089     // For an analogous reason, we must also drop all the metadata whose
1090     // semantics we don't understand. We *can* preserve !annotation, because
1091     // it is tied to the instruction itself, not the value or position.
1092     NewBonusInst->dropUnknownNonDebugMetadata(LLVMContext::MD_annotation);
1093 
1094     PredBlock->getInstList().insert(PTI->getIterator(), NewBonusInst);
1095     NewBonusInst->takeName(&BonusInst);
1096     BonusInst.setName(NewBonusInst->getName() + ".old");
1097 
1098     // Update (liveout) uses of bonus instructions,
1099     // now that the bonus instruction has been cloned into predecessor.
1100     SSAUpdater SSAUpdate;
1101     SSAUpdate.Initialize(BonusInst.getType(),
1102                          (NewBonusInst->getName() + ".merge").str());
1103     SSAUpdate.AddAvailableValue(BB, &BonusInst);
1104     SSAUpdate.AddAvailableValue(PredBlock, NewBonusInst);
1105     for (Use &U : make_early_inc_range(BonusInst.uses())) {
1106       auto *UI = cast<Instruction>(U.getUser());
1107       if (UI->getParent() != PredBlock)
1108         SSAUpdate.RewriteUseAfterInsertions(U);
1109       else // Use is in the same block as, and comes before, NewBonusInst.
1110         SSAUpdate.RewriteUse(U);
1111     }
1112   }
1113 }
1114 
PerformValueComparisonIntoPredecessorFolding(Instruction * TI,Value * & CV,Instruction * PTI,IRBuilder<> & Builder)1115 bool SimplifyCFGOpt::PerformValueComparisonIntoPredecessorFolding(
1116     Instruction *TI, Value *&CV, Instruction *PTI, IRBuilder<> &Builder) {
1117   BasicBlock *BB = TI->getParent();
1118   BasicBlock *Pred = PTI->getParent();
1119 
1120   SmallVector<DominatorTree::UpdateType, 32> Updates;
1121 
1122   // Figure out which 'cases' to copy from SI to PSI.
1123   std::vector<ValueEqualityComparisonCase> BBCases;
1124   BasicBlock *BBDefault = GetValueEqualityComparisonCases(TI, BBCases);
1125 
1126   std::vector<ValueEqualityComparisonCase> PredCases;
1127   BasicBlock *PredDefault = GetValueEqualityComparisonCases(PTI, PredCases);
1128 
1129   // Based on whether the default edge from PTI goes to BB or not, fill in
1130   // PredCases and PredDefault with the new switch cases we would like to
1131   // build.
1132   SmallMapVector<BasicBlock *, int, 8> NewSuccessors;
1133 
1134   // Update the branch weight metadata along the way
1135   SmallVector<uint64_t, 8> Weights;
1136   bool PredHasWeights = HasBranchWeights(PTI);
1137   bool SuccHasWeights = HasBranchWeights(TI);
1138 
1139   if (PredHasWeights) {
1140     GetBranchWeights(PTI, Weights);
1141     // branch-weight metadata is inconsistent here.
1142     if (Weights.size() != 1 + PredCases.size())
1143       PredHasWeights = SuccHasWeights = false;
1144   } else if (SuccHasWeights)
1145     // If there are no predecessor weights but there are successor weights,
1146     // populate Weights with 1, which will later be scaled to the sum of
1147     // successor's weights
1148     Weights.assign(1 + PredCases.size(), 1);
1149 
1150   SmallVector<uint64_t, 8> SuccWeights;
1151   if (SuccHasWeights) {
1152     GetBranchWeights(TI, SuccWeights);
1153     // branch-weight metadata is inconsistent here.
1154     if (SuccWeights.size() != 1 + BBCases.size())
1155       PredHasWeights = SuccHasWeights = false;
1156   } else if (PredHasWeights)
1157     SuccWeights.assign(1 + BBCases.size(), 1);
1158 
1159   if (PredDefault == BB) {
1160     // If this is the default destination from PTI, only the edges in TI
1161     // that don't occur in PTI, or that branch to BB will be activated.
1162     std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1163     for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1164       if (PredCases[i].Dest != BB)
1165         PTIHandled.insert(PredCases[i].Value);
1166       else {
1167         // The default destination is BB, we don't need explicit targets.
1168         std::swap(PredCases[i], PredCases.back());
1169 
1170         if (PredHasWeights || SuccHasWeights) {
1171           // Increase weight for the default case.
1172           Weights[0] += Weights[i + 1];
1173           std::swap(Weights[i + 1], Weights.back());
1174           Weights.pop_back();
1175         }
1176 
1177         PredCases.pop_back();
1178         --i;
1179         --e;
1180       }
1181 
1182     // Reconstruct the new switch statement we will be building.
1183     if (PredDefault != BBDefault) {
1184       PredDefault->removePredecessor(Pred);
1185       if (DTU && PredDefault != BB)
1186         Updates.push_back({DominatorTree::Delete, Pred, PredDefault});
1187       PredDefault = BBDefault;
1188       ++NewSuccessors[BBDefault];
1189     }
1190 
1191     unsigned CasesFromPred = Weights.size();
1192     uint64_t ValidTotalSuccWeight = 0;
1193     for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1194       if (!PTIHandled.count(BBCases[i].Value) && BBCases[i].Dest != BBDefault) {
1195         PredCases.push_back(BBCases[i]);
1196         ++NewSuccessors[BBCases[i].Dest];
1197         if (SuccHasWeights || PredHasWeights) {
1198           // The default weight is at index 0, so weight for the ith case
1199           // should be at index i+1. Scale the cases from successor by
1200           // PredDefaultWeight (Weights[0]).
1201           Weights.push_back(Weights[0] * SuccWeights[i + 1]);
1202           ValidTotalSuccWeight += SuccWeights[i + 1];
1203         }
1204       }
1205 
1206     if (SuccHasWeights || PredHasWeights) {
1207       ValidTotalSuccWeight += SuccWeights[0];
1208       // Scale the cases from predecessor by ValidTotalSuccWeight.
1209       for (unsigned i = 1; i < CasesFromPred; ++i)
1210         Weights[i] *= ValidTotalSuccWeight;
1211       // Scale the default weight by SuccDefaultWeight (SuccWeights[0]).
1212       Weights[0] *= SuccWeights[0];
1213     }
1214   } else {
1215     // If this is not the default destination from PSI, only the edges
1216     // in SI that occur in PSI with a destination of BB will be
1217     // activated.
1218     std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1219     std::map<ConstantInt *, uint64_t> WeightsForHandled;
1220     for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1221       if (PredCases[i].Dest == BB) {
1222         PTIHandled.insert(PredCases[i].Value);
1223 
1224         if (PredHasWeights || SuccHasWeights) {
1225           WeightsForHandled[PredCases[i].Value] = Weights[i + 1];
1226           std::swap(Weights[i + 1], Weights.back());
1227           Weights.pop_back();
1228         }
1229 
1230         std::swap(PredCases[i], PredCases.back());
1231         PredCases.pop_back();
1232         --i;
1233         --e;
1234       }
1235 
1236     // Okay, now we know which constants were sent to BB from the
1237     // predecessor.  Figure out where they will all go now.
1238     for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1239       if (PTIHandled.count(BBCases[i].Value)) {
1240         // If this is one we are capable of getting...
1241         if (PredHasWeights || SuccHasWeights)
1242           Weights.push_back(WeightsForHandled[BBCases[i].Value]);
1243         PredCases.push_back(BBCases[i]);
1244         ++NewSuccessors[BBCases[i].Dest];
1245         PTIHandled.erase(BBCases[i].Value); // This constant is taken care of
1246       }
1247 
1248     // If there are any constants vectored to BB that TI doesn't handle,
1249     // they must go to the default destination of TI.
1250     for (ConstantInt *I : PTIHandled) {
1251       if (PredHasWeights || SuccHasWeights)
1252         Weights.push_back(WeightsForHandled[I]);
1253       PredCases.push_back(ValueEqualityComparisonCase(I, BBDefault));
1254       ++NewSuccessors[BBDefault];
1255     }
1256   }
1257 
1258   // Okay, at this point, we know which new successor Pred will get.  Make
1259   // sure we update the number of entries in the PHI nodes for these
1260   // successors.
1261   SmallPtrSet<BasicBlock *, 2> SuccsOfPred;
1262   if (DTU) {
1263     SuccsOfPred = {succ_begin(Pred), succ_end(Pred)};
1264     Updates.reserve(Updates.size() + NewSuccessors.size());
1265   }
1266   for (const std::pair<BasicBlock *, int /*Num*/> &NewSuccessor :
1267        NewSuccessors) {
1268     for (auto I : seq(0, NewSuccessor.second)) {
1269       (void)I;
1270       AddPredecessorToBlock(NewSuccessor.first, Pred, BB);
1271     }
1272     if (DTU && !SuccsOfPred.contains(NewSuccessor.first))
1273       Updates.push_back({DominatorTree::Insert, Pred, NewSuccessor.first});
1274   }
1275 
1276   Builder.SetInsertPoint(PTI);
1277   // Convert pointer to int before we switch.
1278   if (CV->getType()->isPointerTy()) {
1279     CV =
1280         Builder.CreatePtrToInt(CV, DL.getIntPtrType(CV->getType()), "magicptr");
1281   }
1282 
1283   // Now that the successors are updated, create the new Switch instruction.
1284   SwitchInst *NewSI = Builder.CreateSwitch(CV, PredDefault, PredCases.size());
1285   NewSI->setDebugLoc(PTI->getDebugLoc());
1286   for (ValueEqualityComparisonCase &V : PredCases)
1287     NewSI->addCase(V.Value, V.Dest);
1288 
1289   if (PredHasWeights || SuccHasWeights) {
1290     // Halve the weights if any of them cannot fit in an uint32_t
1291     FitWeights(Weights);
1292 
1293     SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
1294 
1295     setBranchWeights(NewSI, MDWeights);
1296   }
1297 
1298   EraseTerminatorAndDCECond(PTI);
1299 
1300   // Okay, last check.  If BB is still a successor of PSI, then we must
1301   // have an infinite loop case.  If so, add an infinitely looping block
1302   // to handle the case to preserve the behavior of the code.
1303   BasicBlock *InfLoopBlock = nullptr;
1304   for (unsigned i = 0, e = NewSI->getNumSuccessors(); i != e; ++i)
1305     if (NewSI->getSuccessor(i) == BB) {
1306       if (!InfLoopBlock) {
1307         // Insert it at the end of the function, because it's either code,
1308         // or it won't matter if it's hot. :)
1309         InfLoopBlock =
1310             BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
1311         BranchInst::Create(InfLoopBlock, InfLoopBlock);
1312         if (DTU)
1313           Updates.push_back(
1314               {DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
1315       }
1316       NewSI->setSuccessor(i, InfLoopBlock);
1317     }
1318 
1319   if (DTU) {
1320     if (InfLoopBlock)
1321       Updates.push_back({DominatorTree::Insert, Pred, InfLoopBlock});
1322 
1323     Updates.push_back({DominatorTree::Delete, Pred, BB});
1324 
1325     DTU->applyUpdates(Updates);
1326   }
1327 
1328   // Here the BB is not a dead block but folded into its predecessors, so move
1329   // the probe and mark it as dangling.
1330   moveAndDanglePseudoProbes(BB, NewSI);
1331 
1332   ++NumFoldValueComparisonIntoPredecessors;
1333   return true;
1334 }
1335 
1336 /// The specified terminator is a value equality comparison instruction
1337 /// (either a switch or a branch on "X == c").
1338 /// See if any of the predecessors of the terminator block are value comparisons
1339 /// on the same value.  If so, and if safe to do so, fold them together.
FoldValueComparisonIntoPredecessors(Instruction * TI,IRBuilder<> & Builder)1340 bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(Instruction *TI,
1341                                                          IRBuilder<> &Builder) {
1342   BasicBlock *BB = TI->getParent();
1343   Value *CV = isValueEqualityComparison(TI); // CondVal
1344   assert(CV && "Not a comparison?");
1345 
1346   bool Changed = false;
1347 
1348   SmallSetVector<BasicBlock *, 16> Preds(pred_begin(BB), pred_end(BB));
1349   while (!Preds.empty()) {
1350     BasicBlock *Pred = Preds.pop_back_val();
1351     Instruction *PTI = Pred->getTerminator();
1352 
1353     // Don't try to fold into itself.
1354     if (Pred == BB)
1355       continue;
1356 
1357     // See if the predecessor is a comparison with the same value.
1358     Value *PCV = isValueEqualityComparison(PTI); // PredCondVal
1359     if (PCV != CV)
1360       continue;
1361 
1362     SmallSetVector<BasicBlock *, 4> FailBlocks;
1363     if (!SafeToMergeTerminators(TI, PTI, &FailBlocks)) {
1364       for (auto *Succ : FailBlocks) {
1365         if (!SplitBlockPredecessors(Succ, TI->getParent(), ".fold.split", DTU))
1366           return false;
1367       }
1368     }
1369 
1370     PerformValueComparisonIntoPredecessorFolding(TI, CV, PTI, Builder);
1371     Changed = true;
1372   }
1373   return Changed;
1374 }
1375 
1376 // If we would need to insert a select that uses the value of this invoke
1377 // (comments in HoistThenElseCodeToIf explain why we would need to do this), we
1378 // can't hoist the invoke, as there is nowhere to put the select in this case.
isSafeToHoistInvoke(BasicBlock * BB1,BasicBlock * BB2,Instruction * I1,Instruction * I2)1379 static bool isSafeToHoistInvoke(BasicBlock *BB1, BasicBlock *BB2,
1380                                 Instruction *I1, Instruction *I2) {
1381   for (BasicBlock *Succ : successors(BB1)) {
1382     for (const PHINode &PN : Succ->phis()) {
1383       Value *BB1V = PN.getIncomingValueForBlock(BB1);
1384       Value *BB2V = PN.getIncomingValueForBlock(BB2);
1385       if (BB1V != BB2V && (BB1V == I1 || BB2V == I2)) {
1386         return false;
1387       }
1388     }
1389   }
1390   return true;
1391 }
1392 
1393 static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified = false);
1394 
1395 /// Given a conditional branch that goes to BB1 and BB2, hoist any common code
1396 /// in the two blocks up into the branch block. The caller of this function
1397 /// guarantees that BI's block dominates BB1 and BB2. If EqTermsOnly is given,
1398 /// only perform hoisting in case both blocks only contain a terminator. In that
1399 /// case, only the original BI will be replaced and selects for PHIs are added.
HoistThenElseCodeToIf(BranchInst * BI,const TargetTransformInfo & TTI,bool EqTermsOnly)1400 bool SimplifyCFGOpt::HoistThenElseCodeToIf(BranchInst *BI,
1401                                            const TargetTransformInfo &TTI,
1402                                            bool EqTermsOnly) {
1403   // This does very trivial matching, with limited scanning, to find identical
1404   // instructions in the two blocks.  In particular, we don't want to get into
1405   // O(M*N) situations here where M and N are the sizes of BB1 and BB2.  As
1406   // such, we currently just scan for obviously identical instructions in an
1407   // identical order.
1408   BasicBlock *BB1 = BI->getSuccessor(0); // The true destination.
1409   BasicBlock *BB2 = BI->getSuccessor(1); // The false destination
1410 
1411   BasicBlock::iterator BB1_Itr = BB1->begin();
1412   BasicBlock::iterator BB2_Itr = BB2->begin();
1413 
1414   Instruction *I1 = &*BB1_Itr++, *I2 = &*BB2_Itr++;
1415   // Skip debug info if it is not identical.
1416   DbgInfoIntrinsic *DBI1 = dyn_cast<DbgInfoIntrinsic>(I1);
1417   DbgInfoIntrinsic *DBI2 = dyn_cast<DbgInfoIntrinsic>(I2);
1418   if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) {
1419     while (isa<DbgInfoIntrinsic>(I1))
1420       I1 = &*BB1_Itr++;
1421     while (isa<DbgInfoIntrinsic>(I2))
1422       I2 = &*BB2_Itr++;
1423   }
1424   // FIXME: Can we define a safety predicate for CallBr?
1425   if (isa<PHINode>(I1) || !I1->isIdenticalToWhenDefined(I2) ||
1426       (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) ||
1427       isa<CallBrInst>(I1))
1428     return false;
1429 
1430   BasicBlock *BIParent = BI->getParent();
1431 
1432   bool Changed = false;
1433 
1434   auto _ = make_scope_exit([&]() {
1435     if (Changed)
1436       ++NumHoistCommonCode;
1437   });
1438 
1439   // Check if only hoisting terminators is allowed. This does not add new
1440   // instructions to the hoist location.
1441   if (EqTermsOnly) {
1442     // Skip any debug intrinsics, as they are free to hoist.
1443     auto *I1NonDbg = &*skipDebugIntrinsics(I1->getIterator());
1444     auto *I2NonDbg = &*skipDebugIntrinsics(I2->getIterator());
1445     if (!I1NonDbg->isIdenticalToWhenDefined(I2NonDbg))
1446       return false;
1447     if (!I1NonDbg->isTerminator())
1448       return false;
1449     // Now we know that we only need to hoist debug instrinsics and the
1450     // terminator. Let the loop below handle those 2 cases.
1451   }
1452 
1453   do {
1454     // If we are hoisting the terminator instruction, don't move one (making a
1455     // broken BB), instead clone it, and remove BI.
1456     if (I1->isTerminator())
1457       goto HoistTerminator;
1458 
1459     // If we're going to hoist a call, make sure that the two instructions we're
1460     // commoning/hoisting are both marked with musttail, or neither of them is
1461     // marked as such. Otherwise, we might end up in a situation where we hoist
1462     // from a block where the terminator is a `ret` to a block where the terminator
1463     // is a `br`, and `musttail` calls expect to be followed by a return.
1464     auto *C1 = dyn_cast<CallInst>(I1);
1465     auto *C2 = dyn_cast<CallInst>(I2);
1466     if (C1 && C2)
1467       if (C1->isMustTailCall() != C2->isMustTailCall())
1468         return Changed;
1469 
1470     if (!TTI.isProfitableToHoist(I1) || !TTI.isProfitableToHoist(I2))
1471       return Changed;
1472 
1473     // If any of the two call sites has nomerge attribute, stop hoisting.
1474     if (const auto *CB1 = dyn_cast<CallBase>(I1))
1475       if (CB1->cannotMerge())
1476         return Changed;
1477     if (const auto *CB2 = dyn_cast<CallBase>(I2))
1478       if (CB2->cannotMerge())
1479         return Changed;
1480 
1481     if (isa<DbgInfoIntrinsic>(I1) || isa<DbgInfoIntrinsic>(I2)) {
1482       assert (isa<DbgInfoIntrinsic>(I1) && isa<DbgInfoIntrinsic>(I2));
1483       // The debug location is an integral part of a debug info intrinsic
1484       // and can't be separated from it or replaced.  Instead of attempting
1485       // to merge locations, simply hoist both copies of the intrinsic.
1486       BIParent->getInstList().splice(BI->getIterator(),
1487                                      BB1->getInstList(), I1);
1488       BIParent->getInstList().splice(BI->getIterator(),
1489                                      BB2->getInstList(), I2);
1490       Changed = true;
1491     } else {
1492       // For a normal instruction, we just move one to right before the branch,
1493       // then replace all uses of the other with the first.  Finally, we remove
1494       // the now redundant second instruction.
1495       BIParent->getInstList().splice(BI->getIterator(),
1496                                      BB1->getInstList(), I1);
1497       if (!I2->use_empty())
1498         I2->replaceAllUsesWith(I1);
1499       I1->andIRFlags(I2);
1500       unsigned KnownIDs[] = {LLVMContext::MD_tbaa,
1501                              LLVMContext::MD_range,
1502                              LLVMContext::MD_fpmath,
1503                              LLVMContext::MD_invariant_load,
1504                              LLVMContext::MD_nonnull,
1505                              LLVMContext::MD_invariant_group,
1506                              LLVMContext::MD_align,
1507                              LLVMContext::MD_dereferenceable,
1508                              LLVMContext::MD_dereferenceable_or_null,
1509                              LLVMContext::MD_mem_parallel_loop_access,
1510                              LLVMContext::MD_access_group,
1511                              LLVMContext::MD_preserve_access_index};
1512       combineMetadata(I1, I2, KnownIDs, true);
1513 
1514       // I1 and I2 are being combined into a single instruction.  Its debug
1515       // location is the merged locations of the original instructions.
1516       I1->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc());
1517 
1518       I2->eraseFromParent();
1519       Changed = true;
1520     }
1521     ++NumHoistCommonInstrs;
1522 
1523     I1 = &*BB1_Itr++;
1524     I2 = &*BB2_Itr++;
1525     // Skip debug info if it is not identical.
1526     DbgInfoIntrinsic *DBI1 = dyn_cast<DbgInfoIntrinsic>(I1);
1527     DbgInfoIntrinsic *DBI2 = dyn_cast<DbgInfoIntrinsic>(I2);
1528     if (!DBI1 || !DBI2 || !DBI1->isIdenticalToWhenDefined(DBI2)) {
1529       while (isa<DbgInfoIntrinsic>(I1))
1530         I1 = &*BB1_Itr++;
1531       while (isa<DbgInfoIntrinsic>(I2))
1532         I2 = &*BB2_Itr++;
1533     }
1534   } while (I1->isIdenticalToWhenDefined(I2));
1535 
1536   return true;
1537 
1538 HoistTerminator:
1539   // It may not be possible to hoist an invoke.
1540   // FIXME: Can we define a safety predicate for CallBr?
1541   if (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2))
1542     return Changed;
1543 
1544   // TODO: callbr hoisting currently disabled pending further study.
1545   if (isa<CallBrInst>(I1))
1546     return Changed;
1547 
1548   for (BasicBlock *Succ : successors(BB1)) {
1549     for (PHINode &PN : Succ->phis()) {
1550       Value *BB1V = PN.getIncomingValueForBlock(BB1);
1551       Value *BB2V = PN.getIncomingValueForBlock(BB2);
1552       if (BB1V == BB2V)
1553         continue;
1554 
1555       // Check for passingValueIsAlwaysUndefined here because we would rather
1556       // eliminate undefined control flow then converting it to a select.
1557       if (passingValueIsAlwaysUndefined(BB1V, &PN) ||
1558           passingValueIsAlwaysUndefined(BB2V, &PN))
1559         return Changed;
1560 
1561       if (isa<ConstantExpr>(BB1V) && !isSafeToSpeculativelyExecute(BB1V))
1562         return Changed;
1563       if (isa<ConstantExpr>(BB2V) && !isSafeToSpeculativelyExecute(BB2V))
1564         return Changed;
1565     }
1566   }
1567 
1568   // Okay, it is safe to hoist the terminator.
1569   Instruction *NT = I1->clone();
1570   BIParent->getInstList().insert(BI->getIterator(), NT);
1571   if (!NT->getType()->isVoidTy()) {
1572     I1->replaceAllUsesWith(NT);
1573     I2->replaceAllUsesWith(NT);
1574     NT->takeName(I1);
1575   }
1576   Changed = true;
1577   ++NumHoistCommonInstrs;
1578 
1579   // Ensure terminator gets a debug location, even an unknown one, in case
1580   // it involves inlinable calls.
1581   NT->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc());
1582 
1583   // PHIs created below will adopt NT's merged DebugLoc.
1584   IRBuilder<NoFolder> Builder(NT);
1585 
1586   // Hoisting one of the terminators from our successor is a great thing.
1587   // Unfortunately, the successors of the if/else blocks may have PHI nodes in
1588   // them.  If they do, all PHI entries for BB1/BB2 must agree for all PHI
1589   // nodes, so we insert select instruction to compute the final result.
1590   std::map<std::pair<Value *, Value *>, SelectInst *> InsertedSelects;
1591   for (BasicBlock *Succ : successors(BB1)) {
1592     for (PHINode &PN : Succ->phis()) {
1593       Value *BB1V = PN.getIncomingValueForBlock(BB1);
1594       Value *BB2V = PN.getIncomingValueForBlock(BB2);
1595       if (BB1V == BB2V)
1596         continue;
1597 
1598       // These values do not agree.  Insert a select instruction before NT
1599       // that determines the right value.
1600       SelectInst *&SI = InsertedSelects[std::make_pair(BB1V, BB2V)];
1601       if (!SI) {
1602         // Propagate fast-math-flags from phi node to its replacement select.
1603         IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
1604         if (isa<FPMathOperator>(PN))
1605           Builder.setFastMathFlags(PN.getFastMathFlags());
1606 
1607         SI = cast<SelectInst>(
1608             Builder.CreateSelect(BI->getCondition(), BB1V, BB2V,
1609                                  BB1V->getName() + "." + BB2V->getName(), BI));
1610       }
1611 
1612       // Make the PHI node use the select for all incoming values for BB1/BB2
1613       for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
1614         if (PN.getIncomingBlock(i) == BB1 || PN.getIncomingBlock(i) == BB2)
1615           PN.setIncomingValue(i, SI);
1616     }
1617   }
1618 
1619   SmallVector<DominatorTree::UpdateType, 4> Updates;
1620 
1621   // Update any PHI nodes in our new successors.
1622   for (BasicBlock *Succ : successors(BB1)) {
1623     AddPredecessorToBlock(Succ, BIParent, BB1);
1624     if (DTU)
1625       Updates.push_back({DominatorTree::Insert, BIParent, Succ});
1626   }
1627 
1628   if (DTU)
1629     for (BasicBlock *Succ : successors(BI))
1630       Updates.push_back({DominatorTree::Delete, BIParent, Succ});
1631 
1632   EraseTerminatorAndDCECond(BI);
1633   if (DTU)
1634     DTU->applyUpdates(Updates);
1635   return Changed;
1636 }
1637 
1638 // Check lifetime markers.
isLifeTimeMarker(const Instruction * I)1639 static bool isLifeTimeMarker(const Instruction *I) {
1640   if (auto II = dyn_cast<IntrinsicInst>(I)) {
1641     switch (II->getIntrinsicID()) {
1642     default:
1643       break;
1644     case Intrinsic::lifetime_start:
1645     case Intrinsic::lifetime_end:
1646       return true;
1647     }
1648   }
1649   return false;
1650 }
1651 
1652 // TODO: Refine this. This should avoid cases like turning constant memcpy sizes
1653 // into variables.
replacingOperandWithVariableIsCheap(const Instruction * I,int OpIdx)1654 static bool replacingOperandWithVariableIsCheap(const Instruction *I,
1655                                                 int OpIdx) {
1656   return !isa<IntrinsicInst>(I);
1657 }
1658 
1659 // All instructions in Insts belong to different blocks that all unconditionally
1660 // branch to a common successor. Analyze each instruction and return true if it
1661 // would be possible to sink them into their successor, creating one common
1662 // instruction instead. For every value that would be required to be provided by
1663 // PHI node (because an operand varies in each input block), add to PHIOperands.
canSinkInstructions(ArrayRef<Instruction * > Insts,DenseMap<Instruction *,SmallVector<Value *,4>> & PHIOperands)1664 static bool canSinkInstructions(
1665     ArrayRef<Instruction *> Insts,
1666     DenseMap<Instruction *, SmallVector<Value *, 4>> &PHIOperands) {
1667   // Prune out obviously bad instructions to move. Each instruction must have
1668   // exactly zero or one use, and we check later that use is by a single, common
1669   // PHI instruction in the successor.
1670   bool HasUse = !Insts.front()->user_empty();
1671   for (auto *I : Insts) {
1672     // These instructions may change or break semantics if moved.
1673     if (isa<PHINode>(I) || I->isEHPad() || isa<AllocaInst>(I) ||
1674         I->getType()->isTokenTy())
1675       return false;
1676 
1677     // Do not try to sink an instruction in an infinite loop - it can cause
1678     // this algorithm to infinite loop.
1679     if (I->getParent()->getSingleSuccessor() == I->getParent())
1680       return false;
1681 
1682     // Conservatively return false if I is an inline-asm instruction. Sinking
1683     // and merging inline-asm instructions can potentially create arguments
1684     // that cannot satisfy the inline-asm constraints.
1685     // If the instruction has nomerge attribute, return false.
1686     if (const auto *C = dyn_cast<CallBase>(I))
1687       if (C->isInlineAsm() || C->cannotMerge())
1688         return false;
1689 
1690     // Each instruction must have zero or one use.
1691     if (HasUse && !I->hasOneUse())
1692       return false;
1693     if (!HasUse && !I->user_empty())
1694       return false;
1695   }
1696 
1697   const Instruction *I0 = Insts.front();
1698   for (auto *I : Insts)
1699     if (!I->isSameOperationAs(I0))
1700       return false;
1701 
1702   // All instructions in Insts are known to be the same opcode. If they have a
1703   // use, check that the only user is a PHI or in the same block as the
1704   // instruction, because if a user is in the same block as an instruction we're
1705   // contemplating sinking, it must already be determined to be sinkable.
1706   if (HasUse) {
1707     auto *PNUse = dyn_cast<PHINode>(*I0->user_begin());
1708     auto *Succ = I0->getParent()->getTerminator()->getSuccessor(0);
1709     if (!all_of(Insts, [&PNUse,&Succ](const Instruction *I) -> bool {
1710           auto *U = cast<Instruction>(*I->user_begin());
1711           return (PNUse &&
1712                   PNUse->getParent() == Succ &&
1713                   PNUse->getIncomingValueForBlock(I->getParent()) == I) ||
1714                  U->getParent() == I->getParent();
1715         }))
1716       return false;
1717   }
1718 
1719   // Because SROA can't handle speculating stores of selects, try not to sink
1720   // loads, stores or lifetime markers of allocas when we'd have to create a
1721   // PHI for the address operand. Also, because it is likely that loads or
1722   // stores of allocas will disappear when Mem2Reg/SROA is run, don't sink
1723   // them.
1724   // This can cause code churn which can have unintended consequences down
1725   // the line - see https://llvm.org/bugs/show_bug.cgi?id=30244.
1726   // FIXME: This is a workaround for a deficiency in SROA - see
1727   // https://llvm.org/bugs/show_bug.cgi?id=30188
1728   if (isa<StoreInst>(I0) && any_of(Insts, [](const Instruction *I) {
1729         return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
1730       }))
1731     return false;
1732   if (isa<LoadInst>(I0) && any_of(Insts, [](const Instruction *I) {
1733         return isa<AllocaInst>(I->getOperand(0)->stripPointerCasts());
1734       }))
1735     return false;
1736   if (isLifeTimeMarker(I0) && any_of(Insts, [](const Instruction *I) {
1737         return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
1738       }))
1739     return false;
1740 
1741   // For calls to be sinkable, they must all be indirect, or have same callee.
1742   // I.e. if we have two direct calls to different callees, we don't want to
1743   // turn that into an indirect call. Likewise, if we have an indirect call,
1744   // and a direct call, we don't actually want to have a single indirect call.
1745   if (isa<CallBase>(I0)) {
1746     auto IsIndirectCall = [](const Instruction *I) {
1747       return cast<CallBase>(I)->isIndirectCall();
1748     };
1749     bool HaveIndirectCalls = any_of(Insts, IsIndirectCall);
1750     bool AllCallsAreIndirect = all_of(Insts, IsIndirectCall);
1751     if (HaveIndirectCalls) {
1752       if (!AllCallsAreIndirect)
1753         return false;
1754     } else {
1755       // All callees must be identical.
1756       Value *Callee = nullptr;
1757       for (const Instruction *I : Insts) {
1758         Value *CurrCallee = cast<CallBase>(I)->getCalledOperand();
1759         if (!Callee)
1760           Callee = CurrCallee;
1761         else if (Callee != CurrCallee)
1762           return false;
1763       }
1764     }
1765   }
1766 
1767   for (unsigned OI = 0, OE = I0->getNumOperands(); OI != OE; ++OI) {
1768     Value *Op = I0->getOperand(OI);
1769     if (Op->getType()->isTokenTy())
1770       // Don't touch any operand of token type.
1771       return false;
1772 
1773     auto SameAsI0 = [&I0, OI](const Instruction *I) {
1774       assert(I->getNumOperands() == I0->getNumOperands());
1775       return I->getOperand(OI) == I0->getOperand(OI);
1776     };
1777     if (!all_of(Insts, SameAsI0)) {
1778       if ((isa<Constant>(Op) && !replacingOperandWithVariableIsCheap(I0, OI)) ||
1779           !canReplaceOperandWithVariable(I0, OI))
1780         // We can't create a PHI from this GEP.
1781         return false;
1782       for (auto *I : Insts)
1783         PHIOperands[I].push_back(I->getOperand(OI));
1784     }
1785   }
1786   return true;
1787 }
1788 
1789 // Assuming canSinkInstructions(Blocks) has returned true, sink the last
1790 // instruction of every block in Blocks to their common successor, commoning
1791 // into one instruction.
sinkLastInstruction(ArrayRef<BasicBlock * > Blocks)1792 static bool sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) {
1793   auto *BBEnd = Blocks[0]->getTerminator()->getSuccessor(0);
1794 
1795   // canSinkInstructions returning true guarantees that every block has at
1796   // least one non-terminator instruction.
1797   SmallVector<Instruction*,4> Insts;
1798   for (auto *BB : Blocks) {
1799     Instruction *I = BB->getTerminator();
1800     do {
1801       I = I->getPrevNode();
1802     } while (isa<DbgInfoIntrinsic>(I) && I != &BB->front());
1803     if (!isa<DbgInfoIntrinsic>(I))
1804       Insts.push_back(I);
1805   }
1806 
1807   // The only checking we need to do now is that all users of all instructions
1808   // are the same PHI node. canSinkInstructions should have checked this but
1809   // it is slightly over-aggressive - it gets confused by commutative
1810   // instructions so double-check it here.
1811   Instruction *I0 = Insts.front();
1812   if (!I0->user_empty()) {
1813     auto *PNUse = dyn_cast<PHINode>(*I0->user_begin());
1814     if (!all_of(Insts, [&PNUse](const Instruction *I) -> bool {
1815           auto *U = cast<Instruction>(*I->user_begin());
1816           return U == PNUse;
1817         }))
1818       return false;
1819   }
1820 
1821   // We don't need to do any more checking here; canSinkInstructions should
1822   // have done it all for us.
1823   SmallVector<Value*, 4> NewOperands;
1824   for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) {
1825     // This check is different to that in canSinkInstructions. There, we
1826     // cared about the global view once simplifycfg (and instcombine) have
1827     // completed - it takes into account PHIs that become trivially
1828     // simplifiable.  However here we need a more local view; if an operand
1829     // differs we create a PHI and rely on instcombine to clean up the very
1830     // small mess we may make.
1831     bool NeedPHI = any_of(Insts, [&I0, O](const Instruction *I) {
1832       return I->getOperand(O) != I0->getOperand(O);
1833     });
1834     if (!NeedPHI) {
1835       NewOperands.push_back(I0->getOperand(O));
1836       continue;
1837     }
1838 
1839     // Create a new PHI in the successor block and populate it.
1840     auto *Op = I0->getOperand(O);
1841     assert(!Op->getType()->isTokenTy() && "Can't PHI tokens!");
1842     auto *PN = PHINode::Create(Op->getType(), Insts.size(),
1843                                Op->getName() + ".sink", &BBEnd->front());
1844     for (auto *I : Insts)
1845       PN->addIncoming(I->getOperand(O), I->getParent());
1846     NewOperands.push_back(PN);
1847   }
1848 
1849   // Arbitrarily use I0 as the new "common" instruction; remap its operands
1850   // and move it to the start of the successor block.
1851   for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O)
1852     I0->getOperandUse(O).set(NewOperands[O]);
1853   I0->moveBefore(&*BBEnd->getFirstInsertionPt());
1854 
1855   // Update metadata and IR flags, and merge debug locations.
1856   for (auto *I : Insts)
1857     if (I != I0) {
1858       // The debug location for the "common" instruction is the merged locations
1859       // of all the commoned instructions.  We start with the original location
1860       // of the "common" instruction and iteratively merge each location in the
1861       // loop below.
1862       // This is an N-way merge, which will be inefficient if I0 is a CallInst.
1863       // However, as N-way merge for CallInst is rare, so we use simplified API
1864       // instead of using complex API for N-way merge.
1865       I0->applyMergedLocation(I0->getDebugLoc(), I->getDebugLoc());
1866       combineMetadataForCSE(I0, I, true);
1867       I0->andIRFlags(I);
1868     }
1869 
1870   if (!I0->user_empty()) {
1871     // canSinkLastInstruction checked that all instructions were used by
1872     // one and only one PHI node. Find that now, RAUW it to our common
1873     // instruction and nuke it.
1874     auto *PN = cast<PHINode>(*I0->user_begin());
1875     PN->replaceAllUsesWith(I0);
1876     PN->eraseFromParent();
1877   }
1878 
1879   // Finally nuke all instructions apart from the common instruction.
1880   for (auto *I : Insts)
1881     if (I != I0)
1882       I->eraseFromParent();
1883 
1884   return true;
1885 }
1886 
1887 namespace {
1888 
1889   // LockstepReverseIterator - Iterates through instructions
1890   // in a set of blocks in reverse order from the first non-terminator.
1891   // For example (assume all blocks have size n):
1892   //   LockstepReverseIterator I([B1, B2, B3]);
1893   //   *I-- = [B1[n], B2[n], B3[n]];
1894   //   *I-- = [B1[n-1], B2[n-1], B3[n-1]];
1895   //   *I-- = [B1[n-2], B2[n-2], B3[n-2]];
1896   //   ...
1897   class LockstepReverseIterator {
1898     ArrayRef<BasicBlock*> Blocks;
1899     SmallVector<Instruction*,4> Insts;
1900     bool Fail;
1901 
1902   public:
LockstepReverseIterator(ArrayRef<BasicBlock * > Blocks)1903     LockstepReverseIterator(ArrayRef<BasicBlock*> Blocks) : Blocks(Blocks) {
1904       reset();
1905     }
1906 
reset()1907     void reset() {
1908       Fail = false;
1909       Insts.clear();
1910       for (auto *BB : Blocks) {
1911         Instruction *Inst = BB->getTerminator();
1912         for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
1913           Inst = Inst->getPrevNode();
1914         if (!Inst) {
1915           // Block wasn't big enough.
1916           Fail = true;
1917           return;
1918         }
1919         Insts.push_back(Inst);
1920       }
1921     }
1922 
isValid() const1923     bool isValid() const {
1924       return !Fail;
1925     }
1926 
operator --()1927     void operator--() {
1928       if (Fail)
1929         return;
1930       for (auto *&Inst : Insts) {
1931         for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
1932           Inst = Inst->getPrevNode();
1933         // Already at beginning of block.
1934         if (!Inst) {
1935           Fail = true;
1936           return;
1937         }
1938       }
1939     }
1940 
operator ++()1941     void operator++() {
1942       if (Fail)
1943         return;
1944       for (auto *&Inst : Insts) {
1945         for (Inst = Inst->getNextNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
1946           Inst = Inst->getNextNode();
1947         // Already at end of block.
1948         if (!Inst) {
1949           Fail = true;
1950           return;
1951         }
1952       }
1953     }
1954 
operator *() const1955     ArrayRef<Instruction*> operator * () const {
1956       return Insts;
1957     }
1958   };
1959 
1960 } // end anonymous namespace
1961 
1962 /// Check whether BB's predecessors end with unconditional branches. If it is
1963 /// true, sink any common code from the predecessors to BB.
SinkCommonCodeFromPredecessors(BasicBlock * BB,DomTreeUpdater * DTU)1964 static bool SinkCommonCodeFromPredecessors(BasicBlock *BB,
1965                                            DomTreeUpdater *DTU) {
1966   // We support two situations:
1967   //   (1) all incoming arcs are unconditional
1968   //   (2) there are non-unconditional incoming arcs
1969   //
1970   // (2) is very common in switch defaults and
1971   // else-if patterns;
1972   //
1973   //   if (a) f(1);
1974   //   else if (b) f(2);
1975   //
1976   // produces:
1977   //
1978   //       [if]
1979   //      /    \
1980   //    [f(1)] [if]
1981   //      |     | \
1982   //      |     |  |
1983   //      |  [f(2)]|
1984   //       \    | /
1985   //        [ end ]
1986   //
1987   // [end] has two unconditional predecessor arcs and one conditional. The
1988   // conditional refers to the implicit empty 'else' arc. This conditional
1989   // arc can also be caused by an empty default block in a switch.
1990   //
1991   // In this case, we attempt to sink code from all *unconditional* arcs.
1992   // If we can sink instructions from these arcs (determined during the scan
1993   // phase below) we insert a common successor for all unconditional arcs and
1994   // connect that to [end], to enable sinking:
1995   //
1996   //       [if]
1997   //      /    \
1998   //    [x(1)] [if]
1999   //      |     | \
2000   //      |     |  \
2001   //      |  [x(2)] |
2002   //       \   /    |
2003   //   [sink.split] |
2004   //         \     /
2005   //         [ end ]
2006   //
2007   SmallVector<BasicBlock*,4> UnconditionalPreds;
2008   bool HaveNonUnconditionalPredecessors = false;
2009   for (auto *PredBB : predecessors(BB)) {
2010     auto *PredBr = dyn_cast<BranchInst>(PredBB->getTerminator());
2011     if (PredBr && PredBr->isUnconditional())
2012       UnconditionalPreds.push_back(PredBB);
2013     else
2014       HaveNonUnconditionalPredecessors = true;
2015   }
2016   if (UnconditionalPreds.size() < 2)
2017     return false;
2018 
2019   // We take a two-step approach to tail sinking. First we scan from the end of
2020   // each block upwards in lockstep. If the n'th instruction from the end of each
2021   // block can be sunk, those instructions are added to ValuesToSink and we
2022   // carry on. If we can sink an instruction but need to PHI-merge some operands
2023   // (because they're not identical in each instruction) we add these to
2024   // PHIOperands.
2025   int ScanIdx = 0;
2026   SmallPtrSet<Value*,4> InstructionsToSink;
2027   DenseMap<Instruction*, SmallVector<Value*,4>> PHIOperands;
2028   LockstepReverseIterator LRI(UnconditionalPreds);
2029   while (LRI.isValid() &&
2030          canSinkInstructions(*LRI, PHIOperands)) {
2031     LLVM_DEBUG(dbgs() << "SINK: instruction can be sunk: " << *(*LRI)[0]
2032                       << "\n");
2033     InstructionsToSink.insert((*LRI).begin(), (*LRI).end());
2034     ++ScanIdx;
2035     --LRI;
2036   }
2037 
2038   // If no instructions can be sunk, early-return.
2039   if (ScanIdx == 0)
2040     return false;
2041 
2042   // Okay, we *could* sink last ScanIdx instructions. But how many can we
2043   // actually sink before encountering instruction that is unprofitable to sink?
2044   auto ProfitableToSinkInstruction = [&](LockstepReverseIterator &LRI) {
2045     unsigned NumPHIdValues = 0;
2046     for (auto *I : *LRI)
2047       for (auto *V : PHIOperands[I]) {
2048         if (InstructionsToSink.count(V) == 0)
2049           ++NumPHIdValues;
2050         // FIXME: this check is overly optimistic. We may end up not sinking
2051         // said instruction, due to the very same profitability check.
2052         // See @creating_too_many_phis in sink-common-code.ll.
2053       }
2054     LLVM_DEBUG(dbgs() << "SINK: #phid values: " << NumPHIdValues << "\n");
2055     unsigned NumPHIInsts = NumPHIdValues / UnconditionalPreds.size();
2056     if ((NumPHIdValues % UnconditionalPreds.size()) != 0)
2057         NumPHIInsts++;
2058 
2059     return NumPHIInsts <= 1;
2060   };
2061 
2062   // If no instructions can be sunk, early-return.
2063   if (ScanIdx == 0)
2064     return false;
2065 
2066   // We've determined that we are going to sink last ScanIdx instructions,
2067   // and recorded them in InstructionsToSink. Now, some instructions may be
2068   // unprofitable to sink. But that determination depends on the instructions
2069   // that we are going to sink.
2070 
2071   // First, forward scan: find the first instruction unprofitable to sink,
2072   // recording all the ones that are profitable to sink.
2073   // FIXME: would it be better, after we detect that not all are profitable.
2074   // to either record the profitable ones, or erase the unprofitable ones?
2075   // Maybe we need to choose (at runtime) the one that will touch least instrs?
2076   LRI.reset();
2077   int Idx = 0;
2078   SmallPtrSet<Value *, 4> InstructionsProfitableToSink;
2079   while (Idx < ScanIdx) {
2080     if (!ProfitableToSinkInstruction(LRI)) {
2081       // Too many PHIs would be created.
2082       LLVM_DEBUG(
2083           dbgs() << "SINK: stopping here, too many PHIs would be created!\n");
2084       break;
2085     }
2086     InstructionsProfitableToSink.insert((*LRI).begin(), (*LRI).end());
2087     --LRI;
2088     ++Idx;
2089   }
2090 
2091   // If no instructions can be sunk, early-return.
2092   if (Idx == 0)
2093     return false;
2094 
2095   // Did we determine that (only) some instructions are unprofitable to sink?
2096   if (Idx < ScanIdx) {
2097     // Okay, some instructions are unprofitable.
2098     ScanIdx = Idx;
2099     InstructionsToSink = InstructionsProfitableToSink;
2100 
2101     // But, that may make other instructions unprofitable, too.
2102     // So, do a backward scan, do any earlier instructions become unprofitable?
2103     assert(!ProfitableToSinkInstruction(LRI) &&
2104            "We already know that the last instruction is unprofitable to sink");
2105     ++LRI;
2106     --Idx;
2107     while (Idx >= 0) {
2108       // If we detect that an instruction becomes unprofitable to sink,
2109       // all earlier instructions won't be sunk either,
2110       // so preemptively keep InstructionsProfitableToSink in sync.
2111       // FIXME: is this the most performant approach?
2112       for (auto *I : *LRI)
2113         InstructionsProfitableToSink.erase(I);
2114       if (!ProfitableToSinkInstruction(LRI)) {
2115         // Everything starting with this instruction won't be sunk.
2116         ScanIdx = Idx;
2117         InstructionsToSink = InstructionsProfitableToSink;
2118       }
2119       ++LRI;
2120       --Idx;
2121     }
2122   }
2123 
2124   // If no instructions can be sunk, early-return.
2125   if (ScanIdx == 0)
2126     return false;
2127 
2128   bool Changed = false;
2129 
2130   if (HaveNonUnconditionalPredecessors) {
2131     // It is always legal to sink common instructions from unconditional
2132     // predecessors. However, if not all predecessors are unconditional,
2133     // this transformation might be pessimizing. So as a rule of thumb,
2134     // don't do it unless we'd sink at least one non-speculatable instruction.
2135     // See https://bugs.llvm.org/show_bug.cgi?id=30244
2136     LRI.reset();
2137     int Idx = 0;
2138     bool Profitable = false;
2139     while (Idx < ScanIdx) {
2140       if (!isSafeToSpeculativelyExecute((*LRI)[0])) {
2141         Profitable = true;
2142         break;
2143       }
2144       --LRI;
2145       ++Idx;
2146     }
2147     if (!Profitable)
2148       return false;
2149 
2150     LLVM_DEBUG(dbgs() << "SINK: Splitting edge\n");
2151     // We have a conditional edge and we're going to sink some instructions.
2152     // Insert a new block postdominating all blocks we're going to sink from.
2153     if (!SplitBlockPredecessors(BB, UnconditionalPreds, ".sink.split", DTU))
2154       // Edges couldn't be split.
2155       return false;
2156     Changed = true;
2157   }
2158 
2159   // Now that we've analyzed all potential sinking candidates, perform the
2160   // actual sink. We iteratively sink the last non-terminator of the source
2161   // blocks into their common successor unless doing so would require too
2162   // many PHI instructions to be generated (currently only one PHI is allowed
2163   // per sunk instruction).
2164   //
2165   // We can use InstructionsToSink to discount values needing PHI-merging that will
2166   // actually be sunk in a later iteration. This allows us to be more
2167   // aggressive in what we sink. This does allow a false positive where we
2168   // sink presuming a later value will also be sunk, but stop half way through
2169   // and never actually sink it which means we produce more PHIs than intended.
2170   // This is unlikely in practice though.
2171   int SinkIdx = 0;
2172   for (; SinkIdx != ScanIdx; ++SinkIdx) {
2173     LLVM_DEBUG(dbgs() << "SINK: Sink: "
2174                       << *UnconditionalPreds[0]->getTerminator()->getPrevNode()
2175                       << "\n");
2176 
2177     // Because we've sunk every instruction in turn, the current instruction to
2178     // sink is always at index 0.
2179     LRI.reset();
2180 
2181     if (!sinkLastInstruction(UnconditionalPreds)) {
2182       LLVM_DEBUG(
2183           dbgs()
2184           << "SINK: stopping here, failed to actually sink instruction!\n");
2185       break;
2186     }
2187 
2188     NumSinkCommonInstrs++;
2189     Changed = true;
2190   }
2191   if (SinkIdx != 0)
2192     ++NumSinkCommonCode;
2193   return Changed;
2194 }
2195 
2196 /// Determine if we can hoist sink a sole store instruction out of a
2197 /// conditional block.
2198 ///
2199 /// We are looking for code like the following:
2200 ///   BrBB:
2201 ///     store i32 %add, i32* %arrayidx2
2202 ///     ... // No other stores or function calls (we could be calling a memory
2203 ///     ... // function).
2204 ///     %cmp = icmp ult %x, %y
2205 ///     br i1 %cmp, label %EndBB, label %ThenBB
2206 ///   ThenBB:
2207 ///     store i32 %add5, i32* %arrayidx2
2208 ///     br label EndBB
2209 ///   EndBB:
2210 ///     ...
2211 ///   We are going to transform this into:
2212 ///   BrBB:
2213 ///     store i32 %add, i32* %arrayidx2
2214 ///     ... //
2215 ///     %cmp = icmp ult %x, %y
2216 ///     %add.add5 = select i1 %cmp, i32 %add, %add5
2217 ///     store i32 %add.add5, i32* %arrayidx2
2218 ///     ...
2219 ///
2220 /// \return The pointer to the value of the previous store if the store can be
2221 ///         hoisted into the predecessor block. 0 otherwise.
isSafeToSpeculateStore(Instruction * I,BasicBlock * BrBB,BasicBlock * StoreBB,BasicBlock * EndBB)2222 static Value *isSafeToSpeculateStore(Instruction *I, BasicBlock *BrBB,
2223                                      BasicBlock *StoreBB, BasicBlock *EndBB) {
2224   StoreInst *StoreToHoist = dyn_cast<StoreInst>(I);
2225   if (!StoreToHoist)
2226     return nullptr;
2227 
2228   // Volatile or atomic.
2229   if (!StoreToHoist->isSimple())
2230     return nullptr;
2231 
2232   Value *StorePtr = StoreToHoist->getPointerOperand();
2233 
2234   // Look for a store to the same pointer in BrBB.
2235   unsigned MaxNumInstToLookAt = 9;
2236   // Skip pseudo probe intrinsic calls which are not really killing any memory
2237   // accesses.
2238   for (Instruction &CurI : reverse(BrBB->instructionsWithoutDebug(true))) {
2239     if (!MaxNumInstToLookAt)
2240       break;
2241     --MaxNumInstToLookAt;
2242 
2243     // Could be calling an instruction that affects memory like free().
2244     if (CurI.mayHaveSideEffects() && !isa<StoreInst>(CurI))
2245       return nullptr;
2246 
2247     if (auto *SI = dyn_cast<StoreInst>(&CurI)) {
2248       // Found the previous store make sure it stores to the same location.
2249       if (SI->getPointerOperand() == StorePtr)
2250         // Found the previous store, return its value operand.
2251         return SI->getValueOperand();
2252       return nullptr; // Unknown store.
2253     }
2254   }
2255 
2256   return nullptr;
2257 }
2258 
2259 /// Estimate the cost of the insertion(s) and check that the PHI nodes can be
2260 /// converted to selects.
validateAndCostRequiredSelects(BasicBlock * BB,BasicBlock * ThenBB,BasicBlock * EndBB,unsigned & SpeculatedInstructions,InstructionCost & Cost,const TargetTransformInfo & TTI)2261 static bool validateAndCostRequiredSelects(BasicBlock *BB, BasicBlock *ThenBB,
2262                                            BasicBlock *EndBB,
2263                                            unsigned &SpeculatedInstructions,
2264                                            InstructionCost &Cost,
2265                                            const TargetTransformInfo &TTI) {
2266   TargetTransformInfo::TargetCostKind CostKind =
2267     BB->getParent()->hasMinSize()
2268     ? TargetTransformInfo::TCK_CodeSize
2269     : TargetTransformInfo::TCK_SizeAndLatency;
2270 
2271   bool HaveRewritablePHIs = false;
2272   for (PHINode &PN : EndBB->phis()) {
2273     Value *OrigV = PN.getIncomingValueForBlock(BB);
2274     Value *ThenV = PN.getIncomingValueForBlock(ThenBB);
2275 
2276     // FIXME: Try to remove some of the duplication with HoistThenElseCodeToIf.
2277     // Skip PHIs which are trivial.
2278     if (ThenV == OrigV)
2279       continue;
2280 
2281     Cost += TTI.getCmpSelInstrCost(Instruction::Select, PN.getType(), nullptr,
2282                                    CmpInst::BAD_ICMP_PREDICATE, CostKind);
2283 
2284     // Don't convert to selects if we could remove undefined behavior instead.
2285     if (passingValueIsAlwaysUndefined(OrigV, &PN) ||
2286         passingValueIsAlwaysUndefined(ThenV, &PN))
2287       return false;
2288 
2289     HaveRewritablePHIs = true;
2290     ConstantExpr *OrigCE = dyn_cast<ConstantExpr>(OrigV);
2291     ConstantExpr *ThenCE = dyn_cast<ConstantExpr>(ThenV);
2292     if (!OrigCE && !ThenCE)
2293       continue; // Known safe and cheap.
2294 
2295     if ((ThenCE && !isSafeToSpeculativelyExecute(ThenCE)) ||
2296         (OrigCE && !isSafeToSpeculativelyExecute(OrigCE)))
2297       return false;
2298     InstructionCost OrigCost = OrigCE ? computeSpeculationCost(OrigCE, TTI) : 0;
2299     InstructionCost ThenCost = ThenCE ? computeSpeculationCost(ThenCE, TTI) : 0;
2300     InstructionCost MaxCost =
2301         2 * PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
2302     if (OrigCost + ThenCost > MaxCost)
2303       return false;
2304 
2305     // Account for the cost of an unfolded ConstantExpr which could end up
2306     // getting expanded into Instructions.
2307     // FIXME: This doesn't account for how many operations are combined in the
2308     // constant expression.
2309     ++SpeculatedInstructions;
2310     if (SpeculatedInstructions > 1)
2311       return false;
2312   }
2313 
2314   return HaveRewritablePHIs;
2315 }
2316 
2317 /// Speculate a conditional basic block flattening the CFG.
2318 ///
2319 /// Note that this is a very risky transform currently. Speculating
2320 /// instructions like this is most often not desirable. Instead, there is an MI
2321 /// pass which can do it with full awareness of the resource constraints.
2322 /// However, some cases are "obvious" and we should do directly. An example of
2323 /// this is speculating a single, reasonably cheap instruction.
2324 ///
2325 /// There is only one distinct advantage to flattening the CFG at the IR level:
2326 /// it makes very common but simplistic optimizations such as are common in
2327 /// instcombine and the DAG combiner more powerful by removing CFG edges and
2328 /// modeling their effects with easier to reason about SSA value graphs.
2329 ///
2330 ///
2331 /// An illustration of this transform is turning this IR:
2332 /// \code
2333 ///   BB:
2334 ///     %cmp = icmp ult %x, %y
2335 ///     br i1 %cmp, label %EndBB, label %ThenBB
2336 ///   ThenBB:
2337 ///     %sub = sub %x, %y
2338 ///     br label BB2
2339 ///   EndBB:
2340 ///     %phi = phi [ %sub, %ThenBB ], [ 0, %EndBB ]
2341 ///     ...
2342 /// \endcode
2343 ///
2344 /// Into this IR:
2345 /// \code
2346 ///   BB:
2347 ///     %cmp = icmp ult %x, %y
2348 ///     %sub = sub %x, %y
2349 ///     %cond = select i1 %cmp, 0, %sub
2350 ///     ...
2351 /// \endcode
2352 ///
2353 /// \returns true if the conditional block is removed.
SpeculativelyExecuteBB(BranchInst * BI,BasicBlock * ThenBB,const TargetTransformInfo & TTI)2354 bool SimplifyCFGOpt::SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB,
2355                                             const TargetTransformInfo &TTI) {
2356   // Be conservative for now. FP select instruction can often be expensive.
2357   Value *BrCond = BI->getCondition();
2358   if (isa<FCmpInst>(BrCond))
2359     return false;
2360 
2361   BasicBlock *BB = BI->getParent();
2362   BasicBlock *EndBB = ThenBB->getTerminator()->getSuccessor(0);
2363   InstructionCost Budget =
2364       PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
2365 
2366   // If ThenBB is actually on the false edge of the conditional branch, remember
2367   // to swap the select operands later.
2368   bool Invert = false;
2369   if (ThenBB != BI->getSuccessor(0)) {
2370     assert(ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?");
2371     Invert = true;
2372   }
2373   assert(EndBB == BI->getSuccessor(!Invert) && "No edge from to end block");
2374 
2375   // Keep a count of how many times instructions are used within ThenBB when
2376   // they are candidates for sinking into ThenBB. Specifically:
2377   // - They are defined in BB, and
2378   // - They have no side effects, and
2379   // - All of their uses are in ThenBB.
2380   SmallDenseMap<Instruction *, unsigned, 4> SinkCandidateUseCounts;
2381 
2382   SmallVector<Instruction *, 4> SpeculatedDbgIntrinsics;
2383 
2384   unsigned SpeculatedInstructions = 0;
2385   Value *SpeculatedStoreValue = nullptr;
2386   StoreInst *SpeculatedStore = nullptr;
2387   for (BasicBlock::iterator BBI = ThenBB->begin(),
2388                             BBE = std::prev(ThenBB->end());
2389        BBI != BBE; ++BBI) {
2390     Instruction *I = &*BBI;
2391     // Skip debug info.
2392     if (isa<DbgInfoIntrinsic>(I)) {
2393       SpeculatedDbgIntrinsics.push_back(I);
2394       continue;
2395     }
2396 
2397     // Skip pseudo probes. The consequence is we lose track of the branch
2398     // probability for ThenBB, which is fine since the optimization here takes
2399     // place regardless of the branch probability.
2400     if (isa<PseudoProbeInst>(I)) {
2401       continue;
2402     }
2403 
2404     // Only speculatively execute a single instruction (not counting the
2405     // terminator) for now.
2406     ++SpeculatedInstructions;
2407     if (SpeculatedInstructions > 1)
2408       return false;
2409 
2410     // Don't hoist the instruction if it's unsafe or expensive.
2411     if (!isSafeToSpeculativelyExecute(I) &&
2412         !(HoistCondStores && (SpeculatedStoreValue = isSafeToSpeculateStore(
2413                                   I, BB, ThenBB, EndBB))))
2414       return false;
2415     if (!SpeculatedStoreValue &&
2416         computeSpeculationCost(I, TTI) >
2417             PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic)
2418       return false;
2419 
2420     // Store the store speculation candidate.
2421     if (SpeculatedStoreValue)
2422       SpeculatedStore = cast<StoreInst>(I);
2423 
2424     // Do not hoist the instruction if any of its operands are defined but not
2425     // used in BB. The transformation will prevent the operand from
2426     // being sunk into the use block.
2427     for (Use &Op : I->operands()) {
2428       Instruction *OpI = dyn_cast<Instruction>(Op);
2429       if (!OpI || OpI->getParent() != BB || OpI->mayHaveSideEffects())
2430         continue; // Not a candidate for sinking.
2431 
2432       ++SinkCandidateUseCounts[OpI];
2433     }
2434   }
2435 
2436   // Consider any sink candidates which are only used in ThenBB as costs for
2437   // speculation. Note, while we iterate over a DenseMap here, we are summing
2438   // and so iteration order isn't significant.
2439   for (SmallDenseMap<Instruction *, unsigned, 4>::iterator
2440            I = SinkCandidateUseCounts.begin(),
2441            E = SinkCandidateUseCounts.end();
2442        I != E; ++I)
2443     if (I->first->hasNUses(I->second)) {
2444       ++SpeculatedInstructions;
2445       if (SpeculatedInstructions > 1)
2446         return false;
2447     }
2448 
2449   // Check that we can insert the selects and that it's not too expensive to do
2450   // so.
2451   bool Convert = SpeculatedStore != nullptr;
2452   InstructionCost Cost = 0;
2453   Convert |= validateAndCostRequiredSelects(BB, ThenBB, EndBB,
2454                                             SpeculatedInstructions,
2455                                             Cost, TTI);
2456   if (!Convert || Cost > Budget)
2457     return false;
2458 
2459   // If we get here, we can hoist the instruction and if-convert.
2460   LLVM_DEBUG(dbgs() << "SPECULATIVELY EXECUTING BB" << *ThenBB << "\n";);
2461 
2462   // Insert a select of the value of the speculated store.
2463   if (SpeculatedStoreValue) {
2464     IRBuilder<NoFolder> Builder(BI);
2465     Value *TrueV = SpeculatedStore->getValueOperand();
2466     Value *FalseV = SpeculatedStoreValue;
2467     if (Invert)
2468       std::swap(TrueV, FalseV);
2469     Value *S = Builder.CreateSelect(
2470         BrCond, TrueV, FalseV, "spec.store.select", BI);
2471     SpeculatedStore->setOperand(0, S);
2472     SpeculatedStore->applyMergedLocation(BI->getDebugLoc(),
2473                                          SpeculatedStore->getDebugLoc());
2474   }
2475 
2476   // A hoisted conditional probe should be treated as dangling so that it will
2477   // not be over-counted when the samples collected on the non-conditional path
2478   // are counted towards the conditional path. We leave it for the counts
2479   // inference algorithm to figure out a proper count for a danglng probe.
2480   moveAndDanglePseudoProbes(ThenBB, BI);
2481 
2482   // Metadata can be dependent on the condition we are hoisting above.
2483   // Conservatively strip all metadata on the instruction. Drop the debug loc
2484   // to avoid making it appear as if the condition is a constant, which would
2485   // be misleading while debugging.
2486   for (auto &I : *ThenBB) {
2487     assert(!isa<PseudoProbeInst>(I) &&
2488            "Should not drop debug info from any pseudo probes.");
2489     if (!SpeculatedStoreValue || &I != SpeculatedStore)
2490       I.setDebugLoc(DebugLoc());
2491     I.dropUnknownNonDebugMetadata();
2492   }
2493 
2494   // Hoist the instructions.
2495   BB->getInstList().splice(BI->getIterator(), ThenBB->getInstList(),
2496                            ThenBB->begin(), std::prev(ThenBB->end()));
2497 
2498   // Insert selects and rewrite the PHI operands.
2499   IRBuilder<NoFolder> Builder(BI);
2500   for (PHINode &PN : EndBB->phis()) {
2501     unsigned OrigI = PN.getBasicBlockIndex(BB);
2502     unsigned ThenI = PN.getBasicBlockIndex(ThenBB);
2503     Value *OrigV = PN.getIncomingValue(OrigI);
2504     Value *ThenV = PN.getIncomingValue(ThenI);
2505 
2506     // Skip PHIs which are trivial.
2507     if (OrigV == ThenV)
2508       continue;
2509 
2510     // Create a select whose true value is the speculatively executed value and
2511     // false value is the pre-existing value. Swap them if the branch
2512     // destinations were inverted.
2513     Value *TrueV = ThenV, *FalseV = OrigV;
2514     if (Invert)
2515       std::swap(TrueV, FalseV);
2516     Value *V = Builder.CreateSelect(BrCond, TrueV, FalseV, "spec.select", BI);
2517     PN.setIncomingValue(OrigI, V);
2518     PN.setIncomingValue(ThenI, V);
2519   }
2520 
2521   // Remove speculated dbg intrinsics.
2522   // FIXME: Is it possible to do this in a more elegant way? Moving/merging the
2523   // dbg value for the different flows and inserting it after the select.
2524   for (Instruction *I : SpeculatedDbgIntrinsics)
2525     I->eraseFromParent();
2526 
2527   ++NumSpeculations;
2528   return true;
2529 }
2530 
2531 /// Return true if we can thread a branch across this block.
BlockIsSimpleEnoughToThreadThrough(BasicBlock * BB)2532 static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB) {
2533   int Size = 0;
2534 
2535   SmallPtrSet<const Value *, 32> EphValues;
2536   auto IsEphemeral = [&](const Value *V) {
2537     if (isa<AssumeInst>(V))
2538       return true;
2539     return isSafeToSpeculativelyExecute(V) &&
2540            all_of(V->users(),
2541                   [&](const User *U) { return EphValues.count(U); });
2542   };
2543 
2544   // Walk the loop in reverse so that we can identify ephemeral values properly
2545   // (values only feeding assumes).
2546   for (Instruction &I : reverse(BB->instructionsWithoutDebug())) {
2547     // Can't fold blocks that contain noduplicate or convergent calls.
2548     if (CallInst *CI = dyn_cast<CallInst>(&I))
2549       if (CI->cannotDuplicate() || CI->isConvergent())
2550         return false;
2551 
2552     // Ignore ephemeral values which are deleted during codegen.
2553     if (IsEphemeral(&I))
2554       EphValues.insert(&I);
2555     // We will delete Phis while threading, so Phis should not be accounted in
2556     // block's size.
2557     else if (!isa<PHINode>(I)) {
2558       if (Size++ > MaxSmallBlockSize)
2559         return false; // Don't clone large BB's.
2560     }
2561 
2562     // We can only support instructions that do not define values that are
2563     // live outside of the current basic block.
2564     for (User *U : I.users()) {
2565       Instruction *UI = cast<Instruction>(U);
2566       if (UI->getParent() != BB || isa<PHINode>(UI))
2567         return false;
2568     }
2569 
2570     // Looks ok, continue checking.
2571   }
2572 
2573   return true;
2574 }
2575 
2576 /// If we have a conditional branch on a PHI node value that is defined in the
2577 /// same block as the branch and if any PHI entries are constants, thread edges
2578 /// corresponding to that entry to be branches to their ultimate destination.
FoldCondBranchOnPHI(BranchInst * BI,DomTreeUpdater * DTU,const DataLayout & DL,AssumptionCache * AC)2579 static bool FoldCondBranchOnPHI(BranchInst *BI, DomTreeUpdater *DTU,
2580                                 const DataLayout &DL, AssumptionCache *AC) {
2581   BasicBlock *BB = BI->getParent();
2582   PHINode *PN = dyn_cast<PHINode>(BI->getCondition());
2583   // NOTE: we currently cannot transform this case if the PHI node is used
2584   // outside of the block.
2585   if (!PN || PN->getParent() != BB || !PN->hasOneUse())
2586     return false;
2587 
2588   // Degenerate case of a single entry PHI.
2589   if (PN->getNumIncomingValues() == 1) {
2590     FoldSingleEntryPHINodes(PN->getParent());
2591     return true;
2592   }
2593 
2594   // Now we know that this block has multiple preds and two succs.
2595   if (!BlockIsSimpleEnoughToThreadThrough(BB))
2596     return false;
2597 
2598   // Okay, this is a simple enough basic block.  See if any phi values are
2599   // constants.
2600   for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
2601     ConstantInt *CB = dyn_cast<ConstantInt>(PN->getIncomingValue(i));
2602     if (!CB || !CB->getType()->isIntegerTy(1))
2603       continue;
2604 
2605     // Okay, we now know that all edges from PredBB should be revectored to
2606     // branch to RealDest.
2607     BasicBlock *PredBB = PN->getIncomingBlock(i);
2608     BasicBlock *RealDest = BI->getSuccessor(!CB->getZExtValue());
2609 
2610     if (RealDest == BB)
2611       continue; // Skip self loops.
2612     // Skip if the predecessor's terminator is an indirect branch.
2613     if (isa<IndirectBrInst>(PredBB->getTerminator()))
2614       continue;
2615 
2616     SmallVector<DominatorTree::UpdateType, 3> Updates;
2617 
2618     // The dest block might have PHI nodes, other predecessors and other
2619     // difficult cases.  Instead of being smart about this, just insert a new
2620     // block that jumps to the destination block, effectively splitting
2621     // the edge we are about to create.
2622     BasicBlock *EdgeBB =
2623         BasicBlock::Create(BB->getContext(), RealDest->getName() + ".critedge",
2624                            RealDest->getParent(), RealDest);
2625     BranchInst *CritEdgeBranch = BranchInst::Create(RealDest, EdgeBB);
2626     if (DTU)
2627       Updates.push_back({DominatorTree::Insert, EdgeBB, RealDest});
2628     CritEdgeBranch->setDebugLoc(BI->getDebugLoc());
2629 
2630     // Update PHI nodes.
2631     AddPredecessorToBlock(RealDest, EdgeBB, BB);
2632 
2633     // BB may have instructions that are being threaded over.  Clone these
2634     // instructions into EdgeBB.  We know that there will be no uses of the
2635     // cloned instructions outside of EdgeBB.
2636     BasicBlock::iterator InsertPt = EdgeBB->begin();
2637     DenseMap<Value *, Value *> TranslateMap; // Track translated values.
2638     for (BasicBlock::iterator BBI = BB->begin(); &*BBI != BI; ++BBI) {
2639       if (PHINode *PN = dyn_cast<PHINode>(BBI)) {
2640         TranslateMap[PN] = PN->getIncomingValueForBlock(PredBB);
2641         continue;
2642       }
2643       // Clone the instruction.
2644       Instruction *N = BBI->clone();
2645       if (BBI->hasName())
2646         N->setName(BBI->getName() + ".c");
2647 
2648       // Update operands due to translation.
2649       for (Use &Op : N->operands()) {
2650         DenseMap<Value *, Value *>::iterator PI = TranslateMap.find(Op);
2651         if (PI != TranslateMap.end())
2652           Op = PI->second;
2653       }
2654 
2655       // Check for trivial simplification.
2656       if (Value *V = SimplifyInstruction(N, {DL, nullptr, nullptr, AC})) {
2657         if (!BBI->use_empty())
2658           TranslateMap[&*BBI] = V;
2659         if (!N->mayHaveSideEffects()) {
2660           N->deleteValue(); // Instruction folded away, don't need actual inst
2661           N = nullptr;
2662         }
2663       } else {
2664         if (!BBI->use_empty())
2665           TranslateMap[&*BBI] = N;
2666       }
2667       if (N) {
2668         // Insert the new instruction into its new home.
2669         EdgeBB->getInstList().insert(InsertPt, N);
2670 
2671         // Register the new instruction with the assumption cache if necessary.
2672         if (auto *Assume = dyn_cast<AssumeInst>(N))
2673           if (AC)
2674             AC->registerAssumption(Assume);
2675       }
2676     }
2677 
2678     // Loop over all of the edges from PredBB to BB, changing them to branch
2679     // to EdgeBB instead.
2680     Instruction *PredBBTI = PredBB->getTerminator();
2681     for (unsigned i = 0, e = PredBBTI->getNumSuccessors(); i != e; ++i)
2682       if (PredBBTI->getSuccessor(i) == BB) {
2683         BB->removePredecessor(PredBB);
2684         PredBBTI->setSuccessor(i, EdgeBB);
2685       }
2686 
2687     if (DTU) {
2688       Updates.push_back({DominatorTree::Insert, PredBB, EdgeBB});
2689       Updates.push_back({DominatorTree::Delete, PredBB, BB});
2690 
2691       DTU->applyUpdates(Updates);
2692     }
2693 
2694     // Recurse, simplifying any other constants.
2695     return FoldCondBranchOnPHI(BI, DTU, DL, AC) || true;
2696   }
2697 
2698   return false;
2699 }
2700 
2701 /// Given a BB that starts with the specified two-entry PHI node,
2702 /// see if we can eliminate it.
FoldTwoEntryPHINode(PHINode * PN,const TargetTransformInfo & TTI,DomTreeUpdater * DTU,const DataLayout & DL)2703 static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI,
2704                                 DomTreeUpdater *DTU, const DataLayout &DL) {
2705   // Ok, this is a two entry PHI node.  Check to see if this is a simple "if
2706   // statement", which has a very simple dominance structure.  Basically, we
2707   // are trying to find the condition that is being branched on, which
2708   // subsequently causes this merge to happen.  We really want control
2709   // dependence information for this check, but simplifycfg can't keep it up
2710   // to date, and this catches most of the cases we care about anyway.
2711   BasicBlock *BB = PN->getParent();
2712 
2713   BasicBlock *IfTrue, *IfFalse;
2714   Value *IfCond = GetIfCondition(BB, IfTrue, IfFalse);
2715   if (!IfCond ||
2716       // Don't bother if the branch will be constant folded trivially.
2717       isa<ConstantInt>(IfCond))
2718     return false;
2719 
2720   // Okay, we found that we can merge this two-entry phi node into a select.
2721   // Doing so would require us to fold *all* two entry phi nodes in this block.
2722   // At some point this becomes non-profitable (particularly if the target
2723   // doesn't support cmov's).  Only do this transformation if there are two or
2724   // fewer PHI nodes in this block.
2725   unsigned NumPhis = 0;
2726   for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++NumPhis, ++I)
2727     if (NumPhis > 2)
2728       return false;
2729 
2730   // Loop over the PHI's seeing if we can promote them all to select
2731   // instructions.  While we are at it, keep track of the instructions
2732   // that need to be moved to the dominating block.
2733   SmallPtrSet<Instruction *, 4> AggressiveInsts;
2734   InstructionCost Cost = 0;
2735   InstructionCost Budget =
2736       TwoEntryPHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
2737 
2738   bool Changed = false;
2739   for (BasicBlock::iterator II = BB->begin(); isa<PHINode>(II);) {
2740     PHINode *PN = cast<PHINode>(II++);
2741     if (Value *V = SimplifyInstruction(PN, {DL, PN})) {
2742       PN->replaceAllUsesWith(V);
2743       PN->eraseFromParent();
2744       Changed = true;
2745       continue;
2746     }
2747 
2748     if (!dominatesMergePoint(PN->getIncomingValue(0), BB, AggressiveInsts,
2749                              Cost, Budget, TTI) ||
2750         !dominatesMergePoint(PN->getIncomingValue(1), BB, AggressiveInsts,
2751                              Cost, Budget, TTI))
2752       return Changed;
2753   }
2754 
2755   // If we folded the first phi, PN dangles at this point.  Refresh it.  If
2756   // we ran out of PHIs then we simplified them all.
2757   PN = dyn_cast<PHINode>(BB->begin());
2758   if (!PN)
2759     return true;
2760 
2761   // Return true if at least one of these is a 'not', and another is either
2762   // a 'not' too, or a constant.
2763   auto CanHoistNotFromBothValues = [](Value *V0, Value *V1) {
2764     if (!match(V0, m_Not(m_Value())))
2765       std::swap(V0, V1);
2766     auto Invertible = m_CombineOr(m_Not(m_Value()), m_AnyIntegralConstant());
2767     return match(V0, m_Not(m_Value())) && match(V1, Invertible);
2768   };
2769 
2770   // Don't fold i1 branches on PHIs which contain binary operators or
2771   // select form of or/ands, unless one of the incoming values is an 'not' and
2772   // another one is freely invertible.
2773   // These can often be turned into switches and other things.
2774   auto IsBinOpOrAnd = [](Value *V) {
2775     return match(
2776         V, m_CombineOr(m_BinOp(), m_CombineOr(m_LogicalAnd(), m_LogicalOr())));
2777   };
2778   if (PN->getType()->isIntegerTy(1) &&
2779       (IsBinOpOrAnd(PN->getIncomingValue(0)) ||
2780        IsBinOpOrAnd(PN->getIncomingValue(1)) || IsBinOpOrAnd(IfCond)) &&
2781       !CanHoistNotFromBothValues(PN->getIncomingValue(0),
2782                                  PN->getIncomingValue(1)))
2783     return Changed;
2784 
2785   // If all PHI nodes are promotable, check to make sure that all instructions
2786   // in the predecessor blocks can be promoted as well. If not, we won't be able
2787   // to get rid of the control flow, so it's not worth promoting to select
2788   // instructions.
2789   BasicBlock *DomBlock = nullptr;
2790   BasicBlock *IfBlock1 = PN->getIncomingBlock(0);
2791   BasicBlock *IfBlock2 = PN->getIncomingBlock(1);
2792   if (cast<BranchInst>(IfBlock1->getTerminator())->isConditional()) {
2793     IfBlock1 = nullptr;
2794   } else {
2795     DomBlock = *pred_begin(IfBlock1);
2796     for (BasicBlock::iterator I = IfBlock1->begin(); !I->isTerminator(); ++I)
2797       if (!AggressiveInsts.count(&*I) && !isa<DbgInfoIntrinsic>(I) &&
2798           !isa<PseudoProbeInst>(I)) {
2799         // This is not an aggressive instruction that we can promote.
2800         // Because of this, we won't be able to get rid of the control flow, so
2801         // the xform is not worth it.
2802         return Changed;
2803       }
2804   }
2805 
2806   if (cast<BranchInst>(IfBlock2->getTerminator())->isConditional()) {
2807     IfBlock2 = nullptr;
2808   } else {
2809     DomBlock = *pred_begin(IfBlock2);
2810     for (BasicBlock::iterator I = IfBlock2->begin(); !I->isTerminator(); ++I)
2811       if (!AggressiveInsts.count(&*I) && !isa<DbgInfoIntrinsic>(I) &&
2812           !isa<PseudoProbeInst>(I)) {
2813         // This is not an aggressive instruction that we can promote.
2814         // Because of this, we won't be able to get rid of the control flow, so
2815         // the xform is not worth it.
2816         return Changed;
2817       }
2818   }
2819   assert(DomBlock && "Failed to find root DomBlock");
2820 
2821   LLVM_DEBUG(dbgs() << "FOUND IF CONDITION!  " << *IfCond
2822                     << "  T: " << IfTrue->getName()
2823                     << "  F: " << IfFalse->getName() << "\n");
2824 
2825   // If we can still promote the PHI nodes after this gauntlet of tests,
2826   // do all of the PHI's now.
2827   Instruction *InsertPt = DomBlock->getTerminator();
2828   IRBuilder<NoFolder> Builder(InsertPt);
2829 
2830   // Move all 'aggressive' instructions, which are defined in the
2831   // conditional parts of the if's up to the dominating block.
2832   if (IfBlock1)
2833     hoistAllInstructionsInto(DomBlock, InsertPt, IfBlock1);
2834   if (IfBlock2)
2835     hoistAllInstructionsInto(DomBlock, InsertPt, IfBlock2);
2836 
2837   // Propagate fast-math-flags from phi nodes to replacement selects.
2838   IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
2839   while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) {
2840     if (isa<FPMathOperator>(PN))
2841       Builder.setFastMathFlags(PN->getFastMathFlags());
2842 
2843     // Change the PHI node into a select instruction.
2844     Value *TrueVal = PN->getIncomingValue(PN->getIncomingBlock(0) == IfFalse);
2845     Value *FalseVal = PN->getIncomingValue(PN->getIncomingBlock(0) == IfTrue);
2846 
2847     Value *Sel = Builder.CreateSelect(IfCond, TrueVal, FalseVal, "", InsertPt);
2848     PN->replaceAllUsesWith(Sel);
2849     Sel->takeName(PN);
2850     PN->eraseFromParent();
2851   }
2852 
2853   // At this point, IfBlock1 and IfBlock2 are both empty, so our if statement
2854   // has been flattened.  Change DomBlock to jump directly to our new block to
2855   // avoid other simplifycfg's kicking in on the diamond.
2856   Instruction *OldTI = DomBlock->getTerminator();
2857   Builder.SetInsertPoint(OldTI);
2858   Builder.CreateBr(BB);
2859 
2860   SmallVector<DominatorTree::UpdateType, 3> Updates;
2861   if (DTU) {
2862     Updates.push_back({DominatorTree::Insert, DomBlock, BB});
2863     for (auto *Successor : successors(DomBlock))
2864       Updates.push_back({DominatorTree::Delete, DomBlock, Successor});
2865   }
2866 
2867   OldTI->eraseFromParent();
2868   if (DTU)
2869     DTU->applyUpdates(Updates);
2870 
2871   return true;
2872 }
2873 
2874 /// If we found a conditional branch that goes to two returning blocks,
2875 /// try to merge them together into one return,
2876 /// introducing a select if the return values disagree.
SimplifyCondBranchToTwoReturns(BranchInst * BI,IRBuilder<> & Builder)2877 bool SimplifyCFGOpt::SimplifyCondBranchToTwoReturns(BranchInst *BI,
2878                                                     IRBuilder<> &Builder) {
2879   auto *BB = BI->getParent();
2880   assert(BI->isConditional() && "Must be a conditional branch");
2881   BasicBlock *TrueSucc = BI->getSuccessor(0);
2882   BasicBlock *FalseSucc = BI->getSuccessor(1);
2883   // NOTE: destinations may match, this could be degenerate uncond branch.
2884   ReturnInst *TrueRet = cast<ReturnInst>(TrueSucc->getTerminator());
2885   ReturnInst *FalseRet = cast<ReturnInst>(FalseSucc->getTerminator());
2886 
2887   // Check to ensure both blocks are empty (just a return) or optionally empty
2888   // with PHI nodes.  If there are other instructions, merging would cause extra
2889   // computation on one path or the other.
2890   if (!TrueSucc->getFirstNonPHIOrDbg()->isTerminator())
2891     return false;
2892   if (!FalseSucc->getFirstNonPHIOrDbg()->isTerminator())
2893     return false;
2894 
2895   Builder.SetInsertPoint(BI);
2896   // Okay, we found a branch that is going to two return nodes.  If
2897   // there is no return value for this function, just change the
2898   // branch into a return.
2899   if (FalseRet->getNumOperands() == 0) {
2900     TrueSucc->removePredecessor(BB);
2901     FalseSucc->removePredecessor(BB);
2902     Builder.CreateRetVoid();
2903     EraseTerminatorAndDCECond(BI);
2904     if (DTU) {
2905       SmallVector<DominatorTree::UpdateType, 2> Updates;
2906       Updates.push_back({DominatorTree::Delete, BB, TrueSucc});
2907       if (TrueSucc != FalseSucc)
2908         Updates.push_back({DominatorTree::Delete, BB, FalseSucc});
2909       DTU->applyUpdates(Updates);
2910     }
2911     return true;
2912   }
2913 
2914   // Otherwise, figure out what the true and false return values are
2915   // so we can insert a new select instruction.
2916   Value *TrueValue = TrueRet->getReturnValue();
2917   Value *FalseValue = FalseRet->getReturnValue();
2918 
2919   // Unwrap any PHI nodes in the return blocks.
2920   if (PHINode *TVPN = dyn_cast_or_null<PHINode>(TrueValue))
2921     if (TVPN->getParent() == TrueSucc)
2922       TrueValue = TVPN->getIncomingValueForBlock(BB);
2923   if (PHINode *FVPN = dyn_cast_or_null<PHINode>(FalseValue))
2924     if (FVPN->getParent() == FalseSucc)
2925       FalseValue = FVPN->getIncomingValueForBlock(BB);
2926 
2927   // In order for this transformation to be safe, we must be able to
2928   // unconditionally execute both operands to the return.  This is
2929   // normally the case, but we could have a potentially-trapping
2930   // constant expression that prevents this transformation from being
2931   // safe.
2932   if (ConstantExpr *TCV = dyn_cast_or_null<ConstantExpr>(TrueValue))
2933     if (TCV->canTrap())
2934       return false;
2935   if (ConstantExpr *FCV = dyn_cast_or_null<ConstantExpr>(FalseValue))
2936     if (FCV->canTrap())
2937       return false;
2938 
2939   // Okay, we collected all the mapped values and checked them for sanity, and
2940   // defined to really do this transformation.  First, update the CFG.
2941   TrueSucc->removePredecessor(BB);
2942   FalseSucc->removePredecessor(BB);
2943 
2944   // Insert select instructions where needed.
2945   Value *BrCond = BI->getCondition();
2946   if (TrueValue) {
2947     // Insert a select if the results differ.
2948     if (TrueValue == FalseValue || isa<UndefValue>(FalseValue)) {
2949     } else if (isa<UndefValue>(TrueValue)) {
2950       TrueValue = FalseValue;
2951     } else {
2952       TrueValue =
2953           Builder.CreateSelect(BrCond, TrueValue, FalseValue, "retval", BI);
2954     }
2955   }
2956 
2957   Value *RI =
2958       !TrueValue ? Builder.CreateRetVoid() : Builder.CreateRet(TrueValue);
2959 
2960   (void)RI;
2961 
2962   LLVM_DEBUG(dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"
2963                     << "\n  " << *BI << "\nNewRet = " << *RI << "\nTRUEBLOCK: "
2964                     << *TrueSucc << "\nFALSEBLOCK: " << *FalseSucc);
2965 
2966   EraseTerminatorAndDCECond(BI);
2967   if (DTU) {
2968     SmallVector<DominatorTree::UpdateType, 2> Updates;
2969     Updates.push_back({DominatorTree::Delete, BB, TrueSucc});
2970     if (TrueSucc != FalseSucc)
2971       Updates.push_back({DominatorTree::Delete, BB, FalseSucc});
2972     DTU->applyUpdates(Updates);
2973   }
2974 
2975   return true;
2976 }
2977 
createLogicalOp(IRBuilderBase & Builder,Instruction::BinaryOps Opc,Value * LHS,Value * RHS,const Twine & Name="")2978 static Value *createLogicalOp(IRBuilderBase &Builder,
2979                               Instruction::BinaryOps Opc, Value *LHS,
2980                               Value *RHS, const Twine &Name = "") {
2981   // Try to relax logical op to binary op.
2982   if (impliesPoison(RHS, LHS))
2983     return Builder.CreateBinOp(Opc, LHS, RHS, Name);
2984   if (Opc == Instruction::And)
2985     return Builder.CreateLogicalAnd(LHS, RHS, Name);
2986   if (Opc == Instruction::Or)
2987     return Builder.CreateLogicalOr(LHS, RHS, Name);
2988   llvm_unreachable("Invalid logical opcode");
2989 }
2990 
2991 /// Return true if either PBI or BI has branch weight available, and store
2992 /// the weights in {Pred|Succ}{True|False}Weight. If one of PBI and BI does
2993 /// not have branch weight, use 1:1 as its weight.
extractPredSuccWeights(BranchInst * PBI,BranchInst * BI,uint64_t & PredTrueWeight,uint64_t & PredFalseWeight,uint64_t & SuccTrueWeight,uint64_t & SuccFalseWeight)2994 static bool extractPredSuccWeights(BranchInst *PBI, BranchInst *BI,
2995                                    uint64_t &PredTrueWeight,
2996                                    uint64_t &PredFalseWeight,
2997                                    uint64_t &SuccTrueWeight,
2998                                    uint64_t &SuccFalseWeight) {
2999   bool PredHasWeights =
3000       PBI->extractProfMetadata(PredTrueWeight, PredFalseWeight);
3001   bool SuccHasWeights =
3002       BI->extractProfMetadata(SuccTrueWeight, SuccFalseWeight);
3003   if (PredHasWeights || SuccHasWeights) {
3004     if (!PredHasWeights)
3005       PredTrueWeight = PredFalseWeight = 1;
3006     if (!SuccHasWeights)
3007       SuccTrueWeight = SuccFalseWeight = 1;
3008     return true;
3009   } else {
3010     return false;
3011   }
3012 }
3013 
3014 /// Determine if the two branches share a common destination and deduce a glue
3015 /// that joins the branches' conditions to arrive at the common destination if
3016 /// that would be profitable.
3017 static Optional<std::pair<Instruction::BinaryOps, bool>>
shouldFoldCondBranchesToCommonDestination(BranchInst * BI,BranchInst * PBI,const TargetTransformInfo * TTI)3018 shouldFoldCondBranchesToCommonDestination(BranchInst *BI, BranchInst *PBI,
3019                                           const TargetTransformInfo *TTI) {
3020   assert(BI && PBI && BI->isConditional() && PBI->isConditional() &&
3021          "Both blocks must end with a conditional branches.");
3022   assert(is_contained(predecessors(BI->getParent()), PBI->getParent()) &&
3023          "PredBB must be a predecessor of BB.");
3024 
3025   // We have the potential to fold the conditions together, but if the
3026   // predecessor branch is predictable, we may not want to merge them.
3027   uint64_t PTWeight, PFWeight;
3028   BranchProbability PBITrueProb, Likely;
3029   if (TTI && PBI->extractProfMetadata(PTWeight, PFWeight) &&
3030       (PTWeight + PFWeight) != 0) {
3031     PBITrueProb =
3032         BranchProbability::getBranchProbability(PTWeight, PTWeight + PFWeight);
3033     Likely = TTI->getPredictableBranchThreshold();
3034   }
3035 
3036   if (PBI->getSuccessor(0) == BI->getSuccessor(0)) {
3037     // Speculate the 2nd condition unless the 1st is probably true.
3038     if (PBITrueProb.isUnknown() || PBITrueProb < Likely)
3039       return {{Instruction::Or, false}};
3040   } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) {
3041     // Speculate the 2nd condition unless the 1st is probably false.
3042     if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely)
3043       return {{Instruction::And, false}};
3044   } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) {
3045     // Speculate the 2nd condition unless the 1st is probably true.
3046     if (PBITrueProb.isUnknown() || PBITrueProb < Likely)
3047       return {{Instruction::And, true}};
3048   } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) {
3049     // Speculate the 2nd condition unless the 1st is probably false.
3050     if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely)
3051       return {{Instruction::Or, true}};
3052   }
3053   return None;
3054 }
3055 
performBranchToCommonDestFolding(BranchInst * BI,BranchInst * PBI,DomTreeUpdater * DTU,MemorySSAUpdater * MSSAU,const TargetTransformInfo * TTI)3056 static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
3057                                              DomTreeUpdater *DTU,
3058                                              MemorySSAUpdater *MSSAU,
3059                                              const TargetTransformInfo *TTI) {
3060   BasicBlock *BB = BI->getParent();
3061   BasicBlock *PredBlock = PBI->getParent();
3062 
3063   // Determine if the two branches share a common destination.
3064   Instruction::BinaryOps Opc;
3065   bool InvertPredCond;
3066   std::tie(Opc, InvertPredCond) =
3067       *shouldFoldCondBranchesToCommonDestination(BI, PBI, TTI);
3068 
3069   LLVM_DEBUG(dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB);
3070 
3071   IRBuilder<> Builder(PBI);
3072   // The builder is used to create instructions to eliminate the branch in BB.
3073   // If BB's terminator has !annotation metadata, add it to the new
3074   // instructions.
3075   Builder.CollectMetadataToCopy(BB->getTerminator(),
3076                                 {LLVMContext::MD_annotation});
3077 
3078   // If we need to invert the condition in the pred block to match, do so now.
3079   if (InvertPredCond) {
3080     Value *NewCond = PBI->getCondition();
3081     if (NewCond->hasOneUse() && isa<CmpInst>(NewCond)) {
3082       CmpInst *CI = cast<CmpInst>(NewCond);
3083       CI->setPredicate(CI->getInversePredicate());
3084     } else {
3085       NewCond =
3086           Builder.CreateNot(NewCond, PBI->getCondition()->getName() + ".not");
3087     }
3088 
3089     PBI->setCondition(NewCond);
3090     PBI->swapSuccessors();
3091   }
3092 
3093   BasicBlock *UniqueSucc =
3094       PBI->getSuccessor(0) == BB ? BI->getSuccessor(0) : BI->getSuccessor(1);
3095 
3096   // Before cloning instructions, notify the successor basic block that it
3097   // is about to have a new predecessor. This will update PHI nodes,
3098   // which will allow us to update live-out uses of bonus instructions.
3099   AddPredecessorToBlock(UniqueSucc, PredBlock, BB, MSSAU);
3100 
3101   // Try to update branch weights.
3102   uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
3103   if (extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
3104                              SuccTrueWeight, SuccFalseWeight)) {
3105     SmallVector<uint64_t, 8> NewWeights;
3106 
3107     if (PBI->getSuccessor(0) == BB) {
3108       // PBI: br i1 %x, BB, FalseDest
3109       // BI:  br i1 %y, UniqueSucc, FalseDest
3110       // TrueWeight is TrueWeight for PBI * TrueWeight for BI.
3111       NewWeights.push_back(PredTrueWeight * SuccTrueWeight);
3112       // FalseWeight is FalseWeight for PBI * TotalWeight for BI +
3113       //               TrueWeight for PBI * FalseWeight for BI.
3114       // We assume that total weights of a BranchInst can fit into 32 bits.
3115       // Therefore, we will not have overflow using 64-bit arithmetic.
3116       NewWeights.push_back(PredFalseWeight *
3117                                (SuccFalseWeight + SuccTrueWeight) +
3118                            PredTrueWeight * SuccFalseWeight);
3119     } else {
3120       // PBI: br i1 %x, TrueDest, BB
3121       // BI:  br i1 %y, TrueDest, UniqueSucc
3122       // TrueWeight is TrueWeight for PBI * TotalWeight for BI +
3123       //              FalseWeight for PBI * TrueWeight for BI.
3124       NewWeights.push_back(PredTrueWeight * (SuccFalseWeight + SuccTrueWeight) +
3125                            PredFalseWeight * SuccTrueWeight);
3126       // FalseWeight is FalseWeight for PBI * FalseWeight for BI.
3127       NewWeights.push_back(PredFalseWeight * SuccFalseWeight);
3128     }
3129 
3130     // Halve the weights if any of them cannot fit in an uint32_t
3131     FitWeights(NewWeights);
3132 
3133     SmallVector<uint32_t, 8> MDWeights(NewWeights.begin(), NewWeights.end());
3134     setBranchWeights(PBI, MDWeights[0], MDWeights[1]);
3135 
3136     // TODO: If BB is reachable from all paths through PredBlock, then we
3137     // could replace PBI's branch probabilities with BI's.
3138   } else
3139     PBI->setMetadata(LLVMContext::MD_prof, nullptr);
3140 
3141   // Now, update the CFG.
3142   PBI->setSuccessor(PBI->getSuccessor(0) != BB, UniqueSucc);
3143 
3144   if (DTU)
3145     DTU->applyUpdates({{DominatorTree::Insert, PredBlock, UniqueSucc},
3146                        {DominatorTree::Delete, PredBlock, BB}});
3147 
3148   // If BI was a loop latch, it may have had associated loop metadata.
3149   // We need to copy it to the new latch, that is, PBI.
3150   if (MDNode *LoopMD = BI->getMetadata(LLVMContext::MD_loop))
3151     PBI->setMetadata(LLVMContext::MD_loop, LoopMD);
3152 
3153   ValueToValueMapTy VMap; // maps original values to cloned values
3154   CloneInstructionsIntoPredecessorBlockAndUpdateSSAUses(BB, PredBlock, VMap);
3155 
3156   // Now that the Cond was cloned into the predecessor basic block,
3157   // or/and the two conditions together.
3158   Value *BICond = VMap[BI->getCondition()];
3159   PBI->setCondition(
3160       createLogicalOp(Builder, Opc, PBI->getCondition(), BICond, "or.cond"));
3161 
3162   // Copy any debug value intrinsics into the end of PredBlock.
3163   for (Instruction &I : *BB) {
3164     if (isa<DbgInfoIntrinsic>(I)) {
3165       Instruction *NewI = I.clone();
3166       RemapInstruction(NewI, VMap,
3167                        RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
3168       NewI->insertBefore(PBI);
3169     }
3170   }
3171 
3172   ++NumFoldBranchToCommonDest;
3173   return true;
3174 }
3175 
3176 /// If this basic block is simple enough, and if a predecessor branches to us
3177 /// and one of our successors, fold the block into the predecessor and use
3178 /// logical operations to pick the right destination.
FoldBranchToCommonDest(BranchInst * BI,DomTreeUpdater * DTU,MemorySSAUpdater * MSSAU,const TargetTransformInfo * TTI,unsigned BonusInstThreshold)3179 bool llvm::FoldBranchToCommonDest(BranchInst *BI, DomTreeUpdater *DTU,
3180                                   MemorySSAUpdater *MSSAU,
3181                                   const TargetTransformInfo *TTI,
3182                                   unsigned BonusInstThreshold) {
3183   // If this block ends with an unconditional branch,
3184   // let SpeculativelyExecuteBB() deal with it.
3185   if (!BI->isConditional())
3186     return false;
3187 
3188   BasicBlock *BB = BI->getParent();
3189 
3190   bool Changed = false;
3191 
3192   TargetTransformInfo::TargetCostKind CostKind =
3193     BB->getParent()->hasMinSize() ? TargetTransformInfo::TCK_CodeSize
3194                                   : TargetTransformInfo::TCK_SizeAndLatency;
3195 
3196   Instruction *Cond = dyn_cast<Instruction>(BI->getCondition());
3197 
3198   if (!Cond || (!isa<CmpInst>(Cond) && !isa<BinaryOperator>(Cond)) ||
3199       Cond->getParent() != BB || !Cond->hasOneUse())
3200     return Changed;
3201 
3202   // Cond is known to be a compare or binary operator.  Check to make sure that
3203   // neither operand is a potentially-trapping constant expression.
3204   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(0)))
3205     if (CE->canTrap())
3206       return Changed;
3207   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(1)))
3208     if (CE->canTrap())
3209       return Changed;
3210 
3211   // Finally, don't infinitely unroll conditional loops.
3212   if (is_contained(successors(BB), BB))
3213     return Changed;
3214 
3215   // With which predecessors will we want to deal with?
3216   SmallVector<BasicBlock *, 8> Preds;
3217   for (BasicBlock *PredBlock : predecessors(BB)) {
3218     BranchInst *PBI = dyn_cast<BranchInst>(PredBlock->getTerminator());
3219 
3220     // Check that we have two conditional branches.  If there is a PHI node in
3221     // the common successor, verify that the same value flows in from both
3222     // blocks.
3223     if (!PBI || PBI->isUnconditional() || !SafeToMergeTerminators(BI, PBI))
3224       continue;
3225 
3226     // Determine if the two branches share a common destination.
3227     Instruction::BinaryOps Opc;
3228     bool InvertPredCond;
3229     if (auto Recipe = shouldFoldCondBranchesToCommonDestination(BI, PBI, TTI))
3230       std::tie(Opc, InvertPredCond) = *Recipe;
3231     else
3232       continue;
3233 
3234     // Check the cost of inserting the necessary logic before performing the
3235     // transformation.
3236     if (TTI) {
3237       Type *Ty = BI->getCondition()->getType();
3238       InstructionCost Cost = TTI->getArithmeticInstrCost(Opc, Ty, CostKind);
3239       if (InvertPredCond && (!PBI->getCondition()->hasOneUse() ||
3240           !isa<CmpInst>(PBI->getCondition())))
3241         Cost += TTI->getArithmeticInstrCost(Instruction::Xor, Ty, CostKind);
3242 
3243       if (Cost > BranchFoldThreshold)
3244         continue;
3245     }
3246 
3247     // Ok, we do want to deal with this predecessor. Record it.
3248     Preds.emplace_back(PredBlock);
3249   }
3250 
3251   // If there aren't any predecessors into which we can fold,
3252   // don't bother checking the cost.
3253   if (Preds.empty())
3254     return Changed;
3255 
3256   // Only allow this transformation if computing the condition doesn't involve
3257   // too many instructions and these involved instructions can be executed
3258   // unconditionally. We denote all involved instructions except the condition
3259   // as "bonus instructions", and only allow this transformation when the
3260   // number of the bonus instructions we'll need to create when cloning into
3261   // each predecessor does not exceed a certain threshold.
3262   unsigned NumBonusInsts = 0;
3263   const unsigned PredCount = Preds.size();
3264   for (Instruction &I : *BB) {
3265     // Don't check the branch condition comparison itself.
3266     if (&I == Cond)
3267       continue;
3268     // Ignore dbg intrinsics, and the terminator.
3269     if (isa<DbgInfoIntrinsic>(I) || isa<BranchInst>(I))
3270       continue;
3271     // I must be safe to execute unconditionally.
3272     if (!isSafeToSpeculativelyExecute(&I))
3273       return Changed;
3274 
3275     // Account for the cost of duplicating this instruction into each
3276     // predecessor.
3277     NumBonusInsts += PredCount;
3278     // Early exits once we reach the limit.
3279     if (NumBonusInsts > BonusInstThreshold)
3280       return Changed;
3281   }
3282 
3283   // Ok, we have the budget. Perform the transformation.
3284   for (BasicBlock *PredBlock : Preds) {
3285     auto *PBI = cast<BranchInst>(PredBlock->getTerminator());
3286     return performBranchToCommonDestFolding(BI, PBI, DTU, MSSAU, TTI);
3287   }
3288   return Changed;
3289 }
3290 
3291 // If there is only one store in BB1 and BB2, return it, otherwise return
3292 // nullptr.
findUniqueStoreInBlocks(BasicBlock * BB1,BasicBlock * BB2)3293 static StoreInst *findUniqueStoreInBlocks(BasicBlock *BB1, BasicBlock *BB2) {
3294   StoreInst *S = nullptr;
3295   for (auto *BB : {BB1, BB2}) {
3296     if (!BB)
3297       continue;
3298     for (auto &I : *BB)
3299       if (auto *SI = dyn_cast<StoreInst>(&I)) {
3300         if (S)
3301           // Multiple stores seen.
3302           return nullptr;
3303         else
3304           S = SI;
3305       }
3306   }
3307   return S;
3308 }
3309 
ensureValueAvailableInSuccessor(Value * V,BasicBlock * BB,Value * AlternativeV=nullptr)3310 static Value *ensureValueAvailableInSuccessor(Value *V, BasicBlock *BB,
3311                                               Value *AlternativeV = nullptr) {
3312   // PHI is going to be a PHI node that allows the value V that is defined in
3313   // BB to be referenced in BB's only successor.
3314   //
3315   // If AlternativeV is nullptr, the only value we care about in PHI is V. It
3316   // doesn't matter to us what the other operand is (it'll never get used). We
3317   // could just create a new PHI with an undef incoming value, but that could
3318   // increase register pressure if EarlyCSE/InstCombine can't fold it with some
3319   // other PHI. So here we directly look for some PHI in BB's successor with V
3320   // as an incoming operand. If we find one, we use it, else we create a new
3321   // one.
3322   //
3323   // If AlternativeV is not nullptr, we care about both incoming values in PHI.
3324   // PHI must be exactly: phi <ty> [ %BB, %V ], [ %OtherBB, %AlternativeV]
3325   // where OtherBB is the single other predecessor of BB's only successor.
3326   PHINode *PHI = nullptr;
3327   BasicBlock *Succ = BB->getSingleSuccessor();
3328 
3329   for (auto I = Succ->begin(); isa<PHINode>(I); ++I)
3330     if (cast<PHINode>(I)->getIncomingValueForBlock(BB) == V) {
3331       PHI = cast<PHINode>(I);
3332       if (!AlternativeV)
3333         break;
3334 
3335       assert(Succ->hasNPredecessors(2));
3336       auto PredI = pred_begin(Succ);
3337       BasicBlock *OtherPredBB = *PredI == BB ? *++PredI : *PredI;
3338       if (PHI->getIncomingValueForBlock(OtherPredBB) == AlternativeV)
3339         break;
3340       PHI = nullptr;
3341     }
3342   if (PHI)
3343     return PHI;
3344 
3345   // If V is not an instruction defined in BB, just return it.
3346   if (!AlternativeV &&
3347       (!isa<Instruction>(V) || cast<Instruction>(V)->getParent() != BB))
3348     return V;
3349 
3350   PHI = PHINode::Create(V->getType(), 2, "simplifycfg.merge", &Succ->front());
3351   PHI->addIncoming(V, BB);
3352   for (BasicBlock *PredBB : predecessors(Succ))
3353     if (PredBB != BB)
3354       PHI->addIncoming(
3355           AlternativeV ? AlternativeV : UndefValue::get(V->getType()), PredBB);
3356   return PHI;
3357 }
3358 
mergeConditionalStoreToAddress(BasicBlock * PTB,BasicBlock * PFB,BasicBlock * QTB,BasicBlock * QFB,BasicBlock * PostBB,Value * Address,bool InvertPCond,bool InvertQCond,DomTreeUpdater * DTU,const DataLayout & DL,const TargetTransformInfo & TTI)3359 static bool mergeConditionalStoreToAddress(
3360     BasicBlock *PTB, BasicBlock *PFB, BasicBlock *QTB, BasicBlock *QFB,
3361     BasicBlock *PostBB, Value *Address, bool InvertPCond, bool InvertQCond,
3362     DomTreeUpdater *DTU, const DataLayout &DL, const TargetTransformInfo &TTI) {
3363   // For every pointer, there must be exactly two stores, one coming from
3364   // PTB or PFB, and the other from QTB or QFB. We don't support more than one
3365   // store (to any address) in PTB,PFB or QTB,QFB.
3366   // FIXME: We could relax this restriction with a bit more work and performance
3367   // testing.
3368   StoreInst *PStore = findUniqueStoreInBlocks(PTB, PFB);
3369   StoreInst *QStore = findUniqueStoreInBlocks(QTB, QFB);
3370   if (!PStore || !QStore)
3371     return false;
3372 
3373   // Now check the stores are compatible.
3374   if (!QStore->isUnordered() || !PStore->isUnordered())
3375     return false;
3376 
3377   // Check that sinking the store won't cause program behavior changes. Sinking
3378   // the store out of the Q blocks won't change any behavior as we're sinking
3379   // from a block to its unconditional successor. But we're moving a store from
3380   // the P blocks down through the middle block (QBI) and past both QFB and QTB.
3381   // So we need to check that there are no aliasing loads or stores in
3382   // QBI, QTB and QFB. We also need to check there are no conflicting memory
3383   // operations between PStore and the end of its parent block.
3384   //
3385   // The ideal way to do this is to query AliasAnalysis, but we don't
3386   // preserve AA currently so that is dangerous. Be super safe and just
3387   // check there are no other memory operations at all.
3388   for (auto &I : *QFB->getSinglePredecessor())
3389     if (I.mayReadOrWriteMemory())
3390       return false;
3391   for (auto &I : *QFB)
3392     if (&I != QStore && I.mayReadOrWriteMemory())
3393       return false;
3394   if (QTB)
3395     for (auto &I : *QTB)
3396       if (&I != QStore && I.mayReadOrWriteMemory())
3397         return false;
3398   for (auto I = BasicBlock::iterator(PStore), E = PStore->getParent()->end();
3399        I != E; ++I)
3400     if (&*I != PStore && I->mayReadOrWriteMemory())
3401       return false;
3402 
3403   // If we're not in aggressive mode, we only optimize if we have some
3404   // confidence that by optimizing we'll allow P and/or Q to be if-converted.
3405   auto IsWorthwhile = [&](BasicBlock *BB, ArrayRef<StoreInst *> FreeStores) {
3406     if (!BB)
3407       return true;
3408     // Heuristic: if the block can be if-converted/phi-folded and the
3409     // instructions inside are all cheap (arithmetic/GEPs), it's worthwhile to
3410     // thread this store.
3411     InstructionCost Cost = 0;
3412     InstructionCost Budget =
3413         PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
3414     for (auto &I : BB->instructionsWithoutDebug()) {
3415       // Consider terminator instruction to be free.
3416       if (I.isTerminator())
3417         continue;
3418       // If this is one the stores that we want to speculate out of this BB,
3419       // then don't count it's cost, consider it to be free.
3420       if (auto *S = dyn_cast<StoreInst>(&I))
3421         if (llvm::find(FreeStores, S))
3422           continue;
3423       // Else, we have a white-list of instructions that we are ak speculating.
3424       if (!isa<BinaryOperator>(I) && !isa<GetElementPtrInst>(I))
3425         return false; // Not in white-list - not worthwhile folding.
3426       // And finally, if this is a non-free instruction that we are okay
3427       // speculating, ensure that we consider the speculation budget.
3428       Cost += TTI.getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency);
3429       if (Cost > Budget)
3430         return false; // Eagerly refuse to fold as soon as we're out of budget.
3431     }
3432     assert(Cost <= Budget &&
3433            "When we run out of budget we will eagerly return from within the "
3434            "per-instruction loop.");
3435     return true;
3436   };
3437 
3438   const std::array<StoreInst *, 2> FreeStores = {PStore, QStore};
3439   if (!MergeCondStoresAggressively &&
3440       (!IsWorthwhile(PTB, FreeStores) || !IsWorthwhile(PFB, FreeStores) ||
3441        !IsWorthwhile(QTB, FreeStores) || !IsWorthwhile(QFB, FreeStores)))
3442     return false;
3443 
3444   // If PostBB has more than two predecessors, we need to split it so we can
3445   // sink the store.
3446   if (std::next(pred_begin(PostBB), 2) != pred_end(PostBB)) {
3447     // We know that QFB's only successor is PostBB. And QFB has a single
3448     // predecessor. If QTB exists, then its only successor is also PostBB.
3449     // If QTB does not exist, then QFB's only predecessor has a conditional
3450     // branch to QFB and PostBB.
3451     BasicBlock *TruePred = QTB ? QTB : QFB->getSinglePredecessor();
3452     BasicBlock *NewBB =
3453         SplitBlockPredecessors(PostBB, {QFB, TruePred}, "condstore.split", DTU);
3454     if (!NewBB)
3455       return false;
3456     PostBB = NewBB;
3457   }
3458 
3459   // OK, we're going to sink the stores to PostBB. The store has to be
3460   // conditional though, so first create the predicate.
3461   Value *PCond = cast<BranchInst>(PFB->getSinglePredecessor()->getTerminator())
3462                      ->getCondition();
3463   Value *QCond = cast<BranchInst>(QFB->getSinglePredecessor()->getTerminator())
3464                      ->getCondition();
3465 
3466   Value *PPHI = ensureValueAvailableInSuccessor(PStore->getValueOperand(),
3467                                                 PStore->getParent());
3468   Value *QPHI = ensureValueAvailableInSuccessor(QStore->getValueOperand(),
3469                                                 QStore->getParent(), PPHI);
3470 
3471   IRBuilder<> QB(&*PostBB->getFirstInsertionPt());
3472 
3473   Value *PPred = PStore->getParent() == PTB ? PCond : QB.CreateNot(PCond);
3474   Value *QPred = QStore->getParent() == QTB ? QCond : QB.CreateNot(QCond);
3475 
3476   if (InvertPCond)
3477     PPred = QB.CreateNot(PPred);
3478   if (InvertQCond)
3479     QPred = QB.CreateNot(QPred);
3480   Value *CombinedPred = QB.CreateOr(PPred, QPred);
3481 
3482   auto *T = SplitBlockAndInsertIfThen(CombinedPred, &*QB.GetInsertPoint(),
3483                                       /*Unreachable=*/false,
3484                                       /*BranchWeights=*/nullptr, DTU);
3485   QB.SetInsertPoint(T);
3486   StoreInst *SI = cast<StoreInst>(QB.CreateStore(QPHI, Address));
3487   AAMDNodes AAMD;
3488   PStore->getAAMetadata(AAMD, /*Merge=*/false);
3489   PStore->getAAMetadata(AAMD, /*Merge=*/true);
3490   SI->setAAMetadata(AAMD);
3491   // Choose the minimum alignment. If we could prove both stores execute, we
3492   // could use biggest one.  In this case, though, we only know that one of the
3493   // stores executes.  And we don't know it's safe to take the alignment from a
3494   // store that doesn't execute.
3495   SI->setAlignment(std::min(PStore->getAlign(), QStore->getAlign()));
3496 
3497   QStore->eraseFromParent();
3498   PStore->eraseFromParent();
3499 
3500   return true;
3501 }
3502 
mergeConditionalStores(BranchInst * PBI,BranchInst * QBI,DomTreeUpdater * DTU,const DataLayout & DL,const TargetTransformInfo & TTI)3503 static bool mergeConditionalStores(BranchInst *PBI, BranchInst *QBI,
3504                                    DomTreeUpdater *DTU, const DataLayout &DL,
3505                                    const TargetTransformInfo &TTI) {
3506   // The intention here is to find diamonds or triangles (see below) where each
3507   // conditional block contains a store to the same address. Both of these
3508   // stores are conditional, so they can't be unconditionally sunk. But it may
3509   // be profitable to speculatively sink the stores into one merged store at the
3510   // end, and predicate the merged store on the union of the two conditions of
3511   // PBI and QBI.
3512   //
3513   // This can reduce the number of stores executed if both of the conditions are
3514   // true, and can allow the blocks to become small enough to be if-converted.
3515   // This optimization will also chain, so that ladders of test-and-set
3516   // sequences can be if-converted away.
3517   //
3518   // We only deal with simple diamonds or triangles:
3519   //
3520   //     PBI       or      PBI        or a combination of the two
3521   //    /   \               | \
3522   //   PTB  PFB             |  PFB
3523   //    \   /               | /
3524   //     QBI                QBI
3525   //    /  \                | \
3526   //   QTB  QFB             |  QFB
3527   //    \  /                | /
3528   //    PostBB            PostBB
3529   //
3530   // We model triangles as a type of diamond with a nullptr "true" block.
3531   // Triangles are canonicalized so that the fallthrough edge is represented by
3532   // a true condition, as in the diagram above.
3533   BasicBlock *PTB = PBI->getSuccessor(0);
3534   BasicBlock *PFB = PBI->getSuccessor(1);
3535   BasicBlock *QTB = QBI->getSuccessor(0);
3536   BasicBlock *QFB = QBI->getSuccessor(1);
3537   BasicBlock *PostBB = QFB->getSingleSuccessor();
3538 
3539   // Make sure we have a good guess for PostBB. If QTB's only successor is
3540   // QFB, then QFB is a better PostBB.
3541   if (QTB->getSingleSuccessor() == QFB)
3542     PostBB = QFB;
3543 
3544   // If we couldn't find a good PostBB, stop.
3545   if (!PostBB)
3546     return false;
3547 
3548   bool InvertPCond = false, InvertQCond = false;
3549   // Canonicalize fallthroughs to the true branches.
3550   if (PFB == QBI->getParent()) {
3551     std::swap(PFB, PTB);
3552     InvertPCond = true;
3553   }
3554   if (QFB == PostBB) {
3555     std::swap(QFB, QTB);
3556     InvertQCond = true;
3557   }
3558 
3559   // From this point on we can assume PTB or QTB may be fallthroughs but PFB
3560   // and QFB may not. Model fallthroughs as a nullptr block.
3561   if (PTB == QBI->getParent())
3562     PTB = nullptr;
3563   if (QTB == PostBB)
3564     QTB = nullptr;
3565 
3566   // Legality bailouts. We must have at least the non-fallthrough blocks and
3567   // the post-dominating block, and the non-fallthroughs must only have one
3568   // predecessor.
3569   auto HasOnePredAndOneSucc = [](BasicBlock *BB, BasicBlock *P, BasicBlock *S) {
3570     return BB->getSinglePredecessor() == P && BB->getSingleSuccessor() == S;
3571   };
3572   if (!HasOnePredAndOneSucc(PFB, PBI->getParent(), QBI->getParent()) ||
3573       !HasOnePredAndOneSucc(QFB, QBI->getParent(), PostBB))
3574     return false;
3575   if ((PTB && !HasOnePredAndOneSucc(PTB, PBI->getParent(), QBI->getParent())) ||
3576       (QTB && !HasOnePredAndOneSucc(QTB, QBI->getParent(), PostBB)))
3577     return false;
3578   if (!QBI->getParent()->hasNUses(2))
3579     return false;
3580 
3581   // OK, this is a sequence of two diamonds or triangles.
3582   // Check if there are stores in PTB or PFB that are repeated in QTB or QFB.
3583   SmallPtrSet<Value *, 4> PStoreAddresses, QStoreAddresses;
3584   for (auto *BB : {PTB, PFB}) {
3585     if (!BB)
3586       continue;
3587     for (auto &I : *BB)
3588       if (StoreInst *SI = dyn_cast<StoreInst>(&I))
3589         PStoreAddresses.insert(SI->getPointerOperand());
3590   }
3591   for (auto *BB : {QTB, QFB}) {
3592     if (!BB)
3593       continue;
3594     for (auto &I : *BB)
3595       if (StoreInst *SI = dyn_cast<StoreInst>(&I))
3596         QStoreAddresses.insert(SI->getPointerOperand());
3597   }
3598 
3599   set_intersect(PStoreAddresses, QStoreAddresses);
3600   // set_intersect mutates PStoreAddresses in place. Rename it here to make it
3601   // clear what it contains.
3602   auto &CommonAddresses = PStoreAddresses;
3603 
3604   bool Changed = false;
3605   for (auto *Address : CommonAddresses)
3606     Changed |=
3607         mergeConditionalStoreToAddress(PTB, PFB, QTB, QFB, PostBB, Address,
3608                                        InvertPCond, InvertQCond, DTU, DL, TTI);
3609   return Changed;
3610 }
3611 
3612 /// If the previous block ended with a widenable branch, determine if reusing
3613 /// the target block is profitable and legal.  This will have the effect of
3614 /// "widening" PBI, but doesn't require us to reason about hosting safety.
tryWidenCondBranchToCondBranch(BranchInst * PBI,BranchInst * BI,DomTreeUpdater * DTU)3615 static bool tryWidenCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
3616                                            DomTreeUpdater *DTU) {
3617   // TODO: This can be generalized in two important ways:
3618   // 1) We can allow phi nodes in IfFalseBB and simply reuse all the input
3619   //    values from the PBI edge.
3620   // 2) We can sink side effecting instructions into BI's fallthrough
3621   //    successor provided they doesn't contribute to computation of
3622   //    BI's condition.
3623   Value *CondWB, *WC;
3624   BasicBlock *IfTrueBB, *IfFalseBB;
3625   if (!parseWidenableBranch(PBI, CondWB, WC, IfTrueBB, IfFalseBB) ||
3626       IfTrueBB != BI->getParent() || !BI->getParent()->getSinglePredecessor())
3627     return false;
3628   if (!IfFalseBB->phis().empty())
3629     return false; // TODO
3630   // Use lambda to lazily compute expensive condition after cheap ones.
3631   auto NoSideEffects = [](BasicBlock &BB) {
3632     return !llvm::any_of(BB, [](const Instruction &I) {
3633         return I.mayWriteToMemory() || I.mayHaveSideEffects();
3634       });
3635   };
3636   if (BI->getSuccessor(1) != IfFalseBB && // no inf looping
3637       BI->getSuccessor(1)->getTerminatingDeoptimizeCall() && // profitability
3638       NoSideEffects(*BI->getParent())) {
3639     auto *OldSuccessor = BI->getSuccessor(1);
3640     OldSuccessor->removePredecessor(BI->getParent());
3641     BI->setSuccessor(1, IfFalseBB);
3642     if (DTU)
3643       DTU->applyUpdates(
3644           {{DominatorTree::Insert, BI->getParent(), IfFalseBB},
3645            {DominatorTree::Delete, BI->getParent(), OldSuccessor}});
3646     return true;
3647   }
3648   if (BI->getSuccessor(0) != IfFalseBB && // no inf looping
3649       BI->getSuccessor(0)->getTerminatingDeoptimizeCall() && // profitability
3650       NoSideEffects(*BI->getParent())) {
3651     auto *OldSuccessor = BI->getSuccessor(0);
3652     OldSuccessor->removePredecessor(BI->getParent());
3653     BI->setSuccessor(0, IfFalseBB);
3654     if (DTU)
3655       DTU->applyUpdates(
3656           {{DominatorTree::Insert, BI->getParent(), IfFalseBB},
3657            {DominatorTree::Delete, BI->getParent(), OldSuccessor}});
3658     return true;
3659   }
3660   return false;
3661 }
3662 
3663 /// If we have a conditional branch as a predecessor of another block,
3664 /// this function tries to simplify it.  We know
3665 /// that PBI and BI are both conditional branches, and BI is in one of the
3666 /// successor blocks of PBI - PBI branches to BI.
SimplifyCondBranchToCondBranch(BranchInst * PBI,BranchInst * BI,DomTreeUpdater * DTU,const DataLayout & DL,const TargetTransformInfo & TTI)3667 static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
3668                                            DomTreeUpdater *DTU,
3669                                            const DataLayout &DL,
3670                                            const TargetTransformInfo &TTI) {
3671   assert(PBI->isConditional() && BI->isConditional());
3672   BasicBlock *BB = BI->getParent();
3673 
3674   // If this block ends with a branch instruction, and if there is a
3675   // predecessor that ends on a branch of the same condition, make
3676   // this conditional branch redundant.
3677   if (PBI->getCondition() == BI->getCondition() &&
3678       PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
3679     // Okay, the outcome of this conditional branch is statically
3680     // knowable.  If this block had a single pred, handle specially.
3681     if (BB->getSinglePredecessor()) {
3682       // Turn this into a branch on constant.
3683       bool CondIsTrue = PBI->getSuccessor(0) == BB;
3684       BI->setCondition(
3685           ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue));
3686       return true; // Nuke the branch on constant.
3687     }
3688 
3689     // Otherwise, if there are multiple predecessors, insert a PHI that merges
3690     // in the constant and simplify the block result.  Subsequent passes of
3691     // simplifycfg will thread the block.
3692     if (BlockIsSimpleEnoughToThreadThrough(BB)) {
3693       pred_iterator PB = pred_begin(BB), PE = pred_end(BB);
3694       PHINode *NewPN = PHINode::Create(
3695           Type::getInt1Ty(BB->getContext()), std::distance(PB, PE),
3696           BI->getCondition()->getName() + ".pr", &BB->front());
3697       // Okay, we're going to insert the PHI node.  Since PBI is not the only
3698       // predecessor, compute the PHI'd conditional value for all of the preds.
3699       // Any predecessor where the condition is not computable we keep symbolic.
3700       for (pred_iterator PI = PB; PI != PE; ++PI) {
3701         BasicBlock *P = *PI;
3702         if ((PBI = dyn_cast<BranchInst>(P->getTerminator())) && PBI != BI &&
3703             PBI->isConditional() && PBI->getCondition() == BI->getCondition() &&
3704             PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
3705           bool CondIsTrue = PBI->getSuccessor(0) == BB;
3706           NewPN->addIncoming(
3707               ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue),
3708               P);
3709         } else {
3710           NewPN->addIncoming(BI->getCondition(), P);
3711         }
3712       }
3713 
3714       BI->setCondition(NewPN);
3715       return true;
3716     }
3717   }
3718 
3719   // If the previous block ended with a widenable branch, determine if reusing
3720   // the target block is profitable and legal.  This will have the effect of
3721   // "widening" PBI, but doesn't require us to reason about hosting safety.
3722   if (tryWidenCondBranchToCondBranch(PBI, BI, DTU))
3723     return true;
3724 
3725   if (auto *CE = dyn_cast<ConstantExpr>(BI->getCondition()))
3726     if (CE->canTrap())
3727       return false;
3728 
3729   // If both branches are conditional and both contain stores to the same
3730   // address, remove the stores from the conditionals and create a conditional
3731   // merged store at the end.
3732   if (MergeCondStores && mergeConditionalStores(PBI, BI, DTU, DL, TTI))
3733     return true;
3734 
3735   // If this is a conditional branch in an empty block, and if any
3736   // predecessors are a conditional branch to one of our destinations,
3737   // fold the conditions into logical ops and one cond br.
3738 
3739   // Ignore dbg intrinsics.
3740   if (&*BB->instructionsWithoutDebug().begin() != BI)
3741     return false;
3742 
3743   int PBIOp, BIOp;
3744   if (PBI->getSuccessor(0) == BI->getSuccessor(0)) {
3745     PBIOp = 0;
3746     BIOp = 0;
3747   } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) {
3748     PBIOp = 0;
3749     BIOp = 1;
3750   } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) {
3751     PBIOp = 1;
3752     BIOp = 0;
3753   } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) {
3754     PBIOp = 1;
3755     BIOp = 1;
3756   } else {
3757     return false;
3758   }
3759 
3760   // Check to make sure that the other destination of this branch
3761   // isn't BB itself.  If so, this is an infinite loop that will
3762   // keep getting unwound.
3763   if (PBI->getSuccessor(PBIOp) == BB)
3764     return false;
3765 
3766   // Do not perform this transformation if it would require
3767   // insertion of a large number of select instructions. For targets
3768   // without predication/cmovs, this is a big pessimization.
3769 
3770   // Also do not perform this transformation if any phi node in the common
3771   // destination block can trap when reached by BB or PBB (PR17073). In that
3772   // case, it would be unsafe to hoist the operation into a select instruction.
3773 
3774   BasicBlock *CommonDest = PBI->getSuccessor(PBIOp);
3775   BasicBlock *RemovedDest = PBI->getSuccessor(PBIOp ^ 1);
3776   unsigned NumPhis = 0;
3777   for (BasicBlock::iterator II = CommonDest->begin(); isa<PHINode>(II);
3778        ++II, ++NumPhis) {
3779     if (NumPhis > 2) // Disable this xform.
3780       return false;
3781 
3782     PHINode *PN = cast<PHINode>(II);
3783     Value *BIV = PN->getIncomingValueForBlock(BB);
3784     if (ConstantExpr *CE = dyn_cast<ConstantExpr>(BIV))
3785       if (CE->canTrap())
3786         return false;
3787 
3788     unsigned PBBIdx = PN->getBasicBlockIndex(PBI->getParent());
3789     Value *PBIV = PN->getIncomingValue(PBBIdx);
3790     if (ConstantExpr *CE = dyn_cast<ConstantExpr>(PBIV))
3791       if (CE->canTrap())
3792         return false;
3793   }
3794 
3795   // Finally, if everything is ok, fold the branches to logical ops.
3796   BasicBlock *OtherDest = BI->getSuccessor(BIOp ^ 1);
3797 
3798   LLVM_DEBUG(dbgs() << "FOLDING BRs:" << *PBI->getParent()
3799                     << "AND: " << *BI->getParent());
3800 
3801   SmallVector<DominatorTree::UpdateType, 5> Updates;
3802 
3803   // If OtherDest *is* BB, then BB is a basic block with a single conditional
3804   // branch in it, where one edge (OtherDest) goes back to itself but the other
3805   // exits.  We don't *know* that the program avoids the infinite loop
3806   // (even though that seems likely).  If we do this xform naively, we'll end up
3807   // recursively unpeeling the loop.  Since we know that (after the xform is
3808   // done) that the block *is* infinite if reached, we just make it an obviously
3809   // infinite loop with no cond branch.
3810   if (OtherDest == BB) {
3811     // Insert it at the end of the function, because it's either code,
3812     // or it won't matter if it's hot. :)
3813     BasicBlock *InfLoopBlock =
3814         BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
3815     BranchInst::Create(InfLoopBlock, InfLoopBlock);
3816     if (DTU)
3817       Updates.push_back({DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
3818     OtherDest = InfLoopBlock;
3819   }
3820 
3821   LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent());
3822 
3823   // BI may have other predecessors.  Because of this, we leave
3824   // it alone, but modify PBI.
3825 
3826   // Make sure we get to CommonDest on True&True directions.
3827   Value *PBICond = PBI->getCondition();
3828   IRBuilder<NoFolder> Builder(PBI);
3829   if (PBIOp)
3830     PBICond = Builder.CreateNot(PBICond, PBICond->getName() + ".not");
3831 
3832   Value *BICond = BI->getCondition();
3833   if (BIOp)
3834     BICond = Builder.CreateNot(BICond, BICond->getName() + ".not");
3835 
3836   // Merge the conditions.
3837   Value *Cond =
3838       createLogicalOp(Builder, Instruction::Or, PBICond, BICond, "brmerge");
3839 
3840   // Modify PBI to branch on the new condition to the new dests.
3841   PBI->setCondition(Cond);
3842   PBI->setSuccessor(0, CommonDest);
3843   PBI->setSuccessor(1, OtherDest);
3844 
3845   if (DTU) {
3846     Updates.push_back({DominatorTree::Insert, PBI->getParent(), OtherDest});
3847     Updates.push_back({DominatorTree::Delete, PBI->getParent(), RemovedDest});
3848 
3849     DTU->applyUpdates(Updates);
3850   }
3851 
3852   // Update branch weight for PBI.
3853   uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
3854   uint64_t PredCommon, PredOther, SuccCommon, SuccOther;
3855   bool HasWeights =
3856       extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
3857                              SuccTrueWeight, SuccFalseWeight);
3858   if (HasWeights) {
3859     PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
3860     PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
3861     SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
3862     SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
3863     // The weight to CommonDest should be PredCommon * SuccTotal +
3864     //                                    PredOther * SuccCommon.
3865     // The weight to OtherDest should be PredOther * SuccOther.
3866     uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther) +
3867                                   PredOther * SuccCommon,
3868                               PredOther * SuccOther};
3869     // Halve the weights if any of them cannot fit in an uint32_t
3870     FitWeights(NewWeights);
3871 
3872     setBranchWeights(PBI, NewWeights[0], NewWeights[1]);
3873   }
3874 
3875   // OtherDest may have phi nodes.  If so, add an entry from PBI's
3876   // block that are identical to the entries for BI's block.
3877   AddPredecessorToBlock(OtherDest, PBI->getParent(), BB);
3878 
3879   // We know that the CommonDest already had an edge from PBI to
3880   // it.  If it has PHIs though, the PHIs may have different
3881   // entries for BB and PBI's BB.  If so, insert a select to make
3882   // them agree.
3883   for (PHINode &PN : CommonDest->phis()) {
3884     Value *BIV = PN.getIncomingValueForBlock(BB);
3885     unsigned PBBIdx = PN.getBasicBlockIndex(PBI->getParent());
3886     Value *PBIV = PN.getIncomingValue(PBBIdx);
3887     if (BIV != PBIV) {
3888       // Insert a select in PBI to pick the right value.
3889       SelectInst *NV = cast<SelectInst>(
3890           Builder.CreateSelect(PBICond, PBIV, BIV, PBIV->getName() + ".mux"));
3891       PN.setIncomingValue(PBBIdx, NV);
3892       // Although the select has the same condition as PBI, the original branch
3893       // weights for PBI do not apply to the new select because the select's
3894       // 'logical' edges are incoming edges of the phi that is eliminated, not
3895       // the outgoing edges of PBI.
3896       if (HasWeights) {
3897         uint64_t PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
3898         uint64_t PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
3899         uint64_t SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
3900         uint64_t SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
3901         // The weight to PredCommonDest should be PredCommon * SuccTotal.
3902         // The weight to PredOtherDest should be PredOther * SuccCommon.
3903         uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther),
3904                                   PredOther * SuccCommon};
3905 
3906         FitWeights(NewWeights);
3907 
3908         setBranchWeights(NV, NewWeights[0], NewWeights[1]);
3909       }
3910     }
3911   }
3912 
3913   LLVM_DEBUG(dbgs() << "INTO: " << *PBI->getParent());
3914   LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent());
3915 
3916   // This basic block is probably dead.  We know it has at least
3917   // one fewer predecessor.
3918   return true;
3919 }
3920 
3921 // Simplifies a terminator by replacing it with a branch to TrueBB if Cond is
3922 // true or to FalseBB if Cond is false.
3923 // Takes care of updating the successors and removing the old terminator.
3924 // Also makes sure not to introduce new successors by assuming that edges to
3925 // non-successor TrueBBs and FalseBBs aren't reachable.
SimplifyTerminatorOnSelect(Instruction * OldTerm,Value * Cond,BasicBlock * TrueBB,BasicBlock * FalseBB,uint32_t TrueWeight,uint32_t FalseWeight)3926 bool SimplifyCFGOpt::SimplifyTerminatorOnSelect(Instruction *OldTerm,
3927                                                 Value *Cond, BasicBlock *TrueBB,
3928                                                 BasicBlock *FalseBB,
3929                                                 uint32_t TrueWeight,
3930                                                 uint32_t FalseWeight) {
3931   auto *BB = OldTerm->getParent();
3932   // Remove any superfluous successor edges from the CFG.
3933   // First, figure out which successors to preserve.
3934   // If TrueBB and FalseBB are equal, only try to preserve one copy of that
3935   // successor.
3936   BasicBlock *KeepEdge1 = TrueBB;
3937   BasicBlock *KeepEdge2 = TrueBB != FalseBB ? FalseBB : nullptr;
3938 
3939   SmallPtrSet<BasicBlock *, 2> RemovedSuccessors;
3940 
3941   // Then remove the rest.
3942   for (BasicBlock *Succ : successors(OldTerm)) {
3943     // Make sure only to keep exactly one copy of each edge.
3944     if (Succ == KeepEdge1)
3945       KeepEdge1 = nullptr;
3946     else if (Succ == KeepEdge2)
3947       KeepEdge2 = nullptr;
3948     else {
3949       Succ->removePredecessor(BB,
3950                               /*KeepOneInputPHIs=*/true);
3951 
3952       if (Succ != TrueBB && Succ != FalseBB)
3953         RemovedSuccessors.insert(Succ);
3954     }
3955   }
3956 
3957   IRBuilder<> Builder(OldTerm);
3958   Builder.SetCurrentDebugLocation(OldTerm->getDebugLoc());
3959 
3960   // Insert an appropriate new terminator.
3961   if (!KeepEdge1 && !KeepEdge2) {
3962     if (TrueBB == FalseBB) {
3963       // We were only looking for one successor, and it was present.
3964       // Create an unconditional branch to it.
3965       Builder.CreateBr(TrueBB);
3966     } else {
3967       // We found both of the successors we were looking for.
3968       // Create a conditional branch sharing the condition of the select.
3969       BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB);
3970       if (TrueWeight != FalseWeight)
3971         setBranchWeights(NewBI, TrueWeight, FalseWeight);
3972     }
3973   } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) {
3974     // Neither of the selected blocks were successors, so this
3975     // terminator must be unreachable.
3976     new UnreachableInst(OldTerm->getContext(), OldTerm);
3977   } else {
3978     // One of the selected values was a successor, but the other wasn't.
3979     // Insert an unconditional branch to the one that was found;
3980     // the edge to the one that wasn't must be unreachable.
3981     if (!KeepEdge1) {
3982       // Only TrueBB was found.
3983       Builder.CreateBr(TrueBB);
3984     } else {
3985       // Only FalseBB was found.
3986       Builder.CreateBr(FalseBB);
3987     }
3988   }
3989 
3990   EraseTerminatorAndDCECond(OldTerm);
3991 
3992   if (DTU) {
3993     SmallVector<DominatorTree::UpdateType, 2> Updates;
3994     Updates.reserve(RemovedSuccessors.size());
3995     for (auto *RemovedSuccessor : RemovedSuccessors)
3996       Updates.push_back({DominatorTree::Delete, BB, RemovedSuccessor});
3997     DTU->applyUpdates(Updates);
3998   }
3999 
4000   return true;
4001 }
4002 
4003 // Replaces
4004 //   (switch (select cond, X, Y)) on constant X, Y
4005 // with a branch - conditional if X and Y lead to distinct BBs,
4006 // unconditional otherwise.
SimplifySwitchOnSelect(SwitchInst * SI,SelectInst * Select)4007 bool SimplifyCFGOpt::SimplifySwitchOnSelect(SwitchInst *SI,
4008                                             SelectInst *Select) {
4009   // Check for constant integer values in the select.
4010   ConstantInt *TrueVal = dyn_cast<ConstantInt>(Select->getTrueValue());
4011   ConstantInt *FalseVal = dyn_cast<ConstantInt>(Select->getFalseValue());
4012   if (!TrueVal || !FalseVal)
4013     return false;
4014 
4015   // Find the relevant condition and destinations.
4016   Value *Condition = Select->getCondition();
4017   BasicBlock *TrueBB = SI->findCaseValue(TrueVal)->getCaseSuccessor();
4018   BasicBlock *FalseBB = SI->findCaseValue(FalseVal)->getCaseSuccessor();
4019 
4020   // Get weight for TrueBB and FalseBB.
4021   uint32_t TrueWeight = 0, FalseWeight = 0;
4022   SmallVector<uint64_t, 8> Weights;
4023   bool HasWeights = HasBranchWeights(SI);
4024   if (HasWeights) {
4025     GetBranchWeights(SI, Weights);
4026     if (Weights.size() == 1 + SI->getNumCases()) {
4027       TrueWeight =
4028           (uint32_t)Weights[SI->findCaseValue(TrueVal)->getSuccessorIndex()];
4029       FalseWeight =
4030           (uint32_t)Weights[SI->findCaseValue(FalseVal)->getSuccessorIndex()];
4031     }
4032   }
4033 
4034   // Perform the actual simplification.
4035   return SimplifyTerminatorOnSelect(SI, Condition, TrueBB, FalseBB, TrueWeight,
4036                                     FalseWeight);
4037 }
4038 
4039 // Replaces
4040 //   (indirectbr (select cond, blockaddress(@fn, BlockA),
4041 //                             blockaddress(@fn, BlockB)))
4042 // with
4043 //   (br cond, BlockA, BlockB).
SimplifyIndirectBrOnSelect(IndirectBrInst * IBI,SelectInst * SI)4044 bool SimplifyCFGOpt::SimplifyIndirectBrOnSelect(IndirectBrInst *IBI,
4045                                                 SelectInst *SI) {
4046   // Check that both operands of the select are block addresses.
4047   BlockAddress *TBA = dyn_cast<BlockAddress>(SI->getTrueValue());
4048   BlockAddress *FBA = dyn_cast<BlockAddress>(SI->getFalseValue());
4049   if (!TBA || !FBA)
4050     return false;
4051 
4052   // Extract the actual blocks.
4053   BasicBlock *TrueBB = TBA->getBasicBlock();
4054   BasicBlock *FalseBB = FBA->getBasicBlock();
4055 
4056   // Perform the actual simplification.
4057   return SimplifyTerminatorOnSelect(IBI, SI->getCondition(), TrueBB, FalseBB, 0,
4058                                     0);
4059 }
4060 
4061 /// This is called when we find an icmp instruction
4062 /// (a seteq/setne with a constant) as the only instruction in a
4063 /// block that ends with an uncond branch.  We are looking for a very specific
4064 /// pattern that occurs when "A == 1 || A == 2 || A == 3" gets simplified.  In
4065 /// this case, we merge the first two "or's of icmp" into a switch, but then the
4066 /// default value goes to an uncond block with a seteq in it, we get something
4067 /// like:
4068 ///
4069 ///   switch i8 %A, label %DEFAULT [ i8 1, label %end    i8 2, label %end ]
4070 /// DEFAULT:
4071 ///   %tmp = icmp eq i8 %A, 92
4072 ///   br label %end
4073 /// end:
4074 ///   ... = phi i1 [ true, %entry ], [ %tmp, %DEFAULT ], [ true, %entry ]
4075 ///
4076 /// We prefer to split the edge to 'end' so that there is a true/false entry to
4077 /// the PHI, merging the third icmp into the switch.
tryToSimplifyUncondBranchWithICmpInIt(ICmpInst * ICI,IRBuilder<> & Builder)4078 bool SimplifyCFGOpt::tryToSimplifyUncondBranchWithICmpInIt(
4079     ICmpInst *ICI, IRBuilder<> &Builder) {
4080   BasicBlock *BB = ICI->getParent();
4081 
4082   // If the block has any PHIs in it or the icmp has multiple uses, it is too
4083   // complex.
4084   if (isa<PHINode>(BB->begin()) || !ICI->hasOneUse())
4085     return false;
4086 
4087   Value *V = ICI->getOperand(0);
4088   ConstantInt *Cst = cast<ConstantInt>(ICI->getOperand(1));
4089 
4090   // The pattern we're looking for is where our only predecessor is a switch on
4091   // 'V' and this block is the default case for the switch.  In this case we can
4092   // fold the compared value into the switch to simplify things.
4093   BasicBlock *Pred = BB->getSinglePredecessor();
4094   if (!Pred || !isa<SwitchInst>(Pred->getTerminator()))
4095     return false;
4096 
4097   SwitchInst *SI = cast<SwitchInst>(Pred->getTerminator());
4098   if (SI->getCondition() != V)
4099     return false;
4100 
4101   // If BB is reachable on a non-default case, then we simply know the value of
4102   // V in this block.  Substitute it and constant fold the icmp instruction
4103   // away.
4104   if (SI->getDefaultDest() != BB) {
4105     ConstantInt *VVal = SI->findCaseDest(BB);
4106     assert(VVal && "Should have a unique destination value");
4107     ICI->setOperand(0, VVal);
4108 
4109     if (Value *V = SimplifyInstruction(ICI, {DL, ICI})) {
4110       ICI->replaceAllUsesWith(V);
4111       ICI->eraseFromParent();
4112     }
4113     // BB is now empty, so it is likely to simplify away.
4114     return requestResimplify();
4115   }
4116 
4117   // Ok, the block is reachable from the default dest.  If the constant we're
4118   // comparing exists in one of the other edges, then we can constant fold ICI
4119   // and zap it.
4120   if (SI->findCaseValue(Cst) != SI->case_default()) {
4121     Value *V;
4122     if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
4123       V = ConstantInt::getFalse(BB->getContext());
4124     else
4125       V = ConstantInt::getTrue(BB->getContext());
4126 
4127     ICI->replaceAllUsesWith(V);
4128     ICI->eraseFromParent();
4129     // BB is now empty, so it is likely to simplify away.
4130     return requestResimplify();
4131   }
4132 
4133   // The use of the icmp has to be in the 'end' block, by the only PHI node in
4134   // the block.
4135   BasicBlock *SuccBlock = BB->getTerminator()->getSuccessor(0);
4136   PHINode *PHIUse = dyn_cast<PHINode>(ICI->user_back());
4137   if (PHIUse == nullptr || PHIUse != &SuccBlock->front() ||
4138       isa<PHINode>(++BasicBlock::iterator(PHIUse)))
4139     return false;
4140 
4141   // If the icmp is a SETEQ, then the default dest gets false, the new edge gets
4142   // true in the PHI.
4143   Constant *DefaultCst = ConstantInt::getTrue(BB->getContext());
4144   Constant *NewCst = ConstantInt::getFalse(BB->getContext());
4145 
4146   if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
4147     std::swap(DefaultCst, NewCst);
4148 
4149   // Replace ICI (which is used by the PHI for the default value) with true or
4150   // false depending on if it is EQ or NE.
4151   ICI->replaceAllUsesWith(DefaultCst);
4152   ICI->eraseFromParent();
4153 
4154   SmallVector<DominatorTree::UpdateType, 2> Updates;
4155 
4156   // Okay, the switch goes to this block on a default value.  Add an edge from
4157   // the switch to the merge point on the compared value.
4158   BasicBlock *NewBB =
4159       BasicBlock::Create(BB->getContext(), "switch.edge", BB->getParent(), BB);
4160   {
4161     SwitchInstProfUpdateWrapper SIW(*SI);
4162     auto W0 = SIW.getSuccessorWeight(0);
4163     SwitchInstProfUpdateWrapper::CaseWeightOpt NewW;
4164     if (W0) {
4165       NewW = ((uint64_t(*W0) + 1) >> 1);
4166       SIW.setSuccessorWeight(0, *NewW);
4167     }
4168     SIW.addCase(Cst, NewBB, NewW);
4169     if (DTU)
4170       Updates.push_back({DominatorTree::Insert, Pred, NewBB});
4171   }
4172 
4173   // NewBB branches to the phi block, add the uncond branch and the phi entry.
4174   Builder.SetInsertPoint(NewBB);
4175   Builder.SetCurrentDebugLocation(SI->getDebugLoc());
4176   Builder.CreateBr(SuccBlock);
4177   PHIUse->addIncoming(NewCst, NewBB);
4178   if (DTU) {
4179     Updates.push_back({DominatorTree::Insert, NewBB, SuccBlock});
4180     DTU->applyUpdates(Updates);
4181   }
4182   return true;
4183 }
4184 
4185 /// The specified branch is a conditional branch.
4186 /// Check to see if it is branching on an or/and chain of icmp instructions, and
4187 /// fold it into a switch instruction if so.
SimplifyBranchOnICmpChain(BranchInst * BI,IRBuilder<> & Builder,const DataLayout & DL)4188 bool SimplifyCFGOpt::SimplifyBranchOnICmpChain(BranchInst *BI,
4189                                                IRBuilder<> &Builder,
4190                                                const DataLayout &DL) {
4191   Instruction *Cond = dyn_cast<Instruction>(BI->getCondition());
4192   if (!Cond)
4193     return false;
4194 
4195   // Change br (X == 0 | X == 1), T, F into a switch instruction.
4196   // If this is a bunch of seteq's or'd together, or if it's a bunch of
4197   // 'setne's and'ed together, collect them.
4198 
4199   // Try to gather values from a chain of and/or to be turned into a switch
4200   ConstantComparesGatherer ConstantCompare(Cond, DL);
4201   // Unpack the result
4202   SmallVectorImpl<ConstantInt *> &Values = ConstantCompare.Vals;
4203   Value *CompVal = ConstantCompare.CompValue;
4204   unsigned UsedICmps = ConstantCompare.UsedICmps;
4205   Value *ExtraCase = ConstantCompare.Extra;
4206 
4207   // If we didn't have a multiply compared value, fail.
4208   if (!CompVal)
4209     return false;
4210 
4211   // Avoid turning single icmps into a switch.
4212   if (UsedICmps <= 1)
4213     return false;
4214 
4215   bool TrueWhenEqual = match(Cond, m_LogicalOr(m_Value(), m_Value()));
4216 
4217   // There might be duplicate constants in the list, which the switch
4218   // instruction can't handle, remove them now.
4219   array_pod_sort(Values.begin(), Values.end(), ConstantIntSortPredicate);
4220   Values.erase(std::unique(Values.begin(), Values.end()), Values.end());
4221 
4222   // If Extra was used, we require at least two switch values to do the
4223   // transformation.  A switch with one value is just a conditional branch.
4224   if (ExtraCase && Values.size() < 2)
4225     return false;
4226 
4227   // TODO: Preserve branch weight metadata, similarly to how
4228   // FoldValueComparisonIntoPredecessors preserves it.
4229 
4230   // Figure out which block is which destination.
4231   BasicBlock *DefaultBB = BI->getSuccessor(1);
4232   BasicBlock *EdgeBB = BI->getSuccessor(0);
4233   if (!TrueWhenEqual)
4234     std::swap(DefaultBB, EdgeBB);
4235 
4236   BasicBlock *BB = BI->getParent();
4237 
4238   // MSAN does not like undefs as branch condition which can be introduced
4239   // with "explicit branch".
4240   if (ExtraCase && BB->getParent()->hasFnAttribute(Attribute::SanitizeMemory))
4241     return false;
4242 
4243   LLVM_DEBUG(dbgs() << "Converting 'icmp' chain with " << Values.size()
4244                     << " cases into SWITCH.  BB is:\n"
4245                     << *BB);
4246 
4247   SmallVector<DominatorTree::UpdateType, 2> Updates;
4248 
4249   // If there are any extra values that couldn't be folded into the switch
4250   // then we evaluate them with an explicit branch first. Split the block
4251   // right before the condbr to handle it.
4252   if (ExtraCase) {
4253     BasicBlock *NewBB = SplitBlock(BB, BI, DTU, /*LI=*/nullptr,
4254                                    /*MSSAU=*/nullptr, "switch.early.test");
4255 
4256     // Remove the uncond branch added to the old block.
4257     Instruction *OldTI = BB->getTerminator();
4258     Builder.SetInsertPoint(OldTI);
4259 
4260     if (TrueWhenEqual)
4261       Builder.CreateCondBr(ExtraCase, EdgeBB, NewBB);
4262     else
4263       Builder.CreateCondBr(ExtraCase, NewBB, EdgeBB);
4264 
4265     OldTI->eraseFromParent();
4266 
4267     if (DTU)
4268       Updates.push_back({DominatorTree::Insert, BB, EdgeBB});
4269 
4270     // If there are PHI nodes in EdgeBB, then we need to add a new entry to them
4271     // for the edge we just added.
4272     AddPredecessorToBlock(EdgeBB, BB, NewBB);
4273 
4274     LLVM_DEBUG(dbgs() << "  ** 'icmp' chain unhandled condition: " << *ExtraCase
4275                       << "\nEXTRABB = " << *BB);
4276     BB = NewBB;
4277   }
4278 
4279   Builder.SetInsertPoint(BI);
4280   // Convert pointer to int before we switch.
4281   if (CompVal->getType()->isPointerTy()) {
4282     CompVal = Builder.CreatePtrToInt(
4283         CompVal, DL.getIntPtrType(CompVal->getType()), "magicptr");
4284   }
4285 
4286   // Create the new switch instruction now.
4287   SwitchInst *New = Builder.CreateSwitch(CompVal, DefaultBB, Values.size());
4288 
4289   // Add all of the 'cases' to the switch instruction.
4290   for (unsigned i = 0, e = Values.size(); i != e; ++i)
4291     New->addCase(Values[i], EdgeBB);
4292 
4293   // We added edges from PI to the EdgeBB.  As such, if there were any
4294   // PHI nodes in EdgeBB, they need entries to be added corresponding to
4295   // the number of edges added.
4296   for (BasicBlock::iterator BBI = EdgeBB->begin(); isa<PHINode>(BBI); ++BBI) {
4297     PHINode *PN = cast<PHINode>(BBI);
4298     Value *InVal = PN->getIncomingValueForBlock(BB);
4299     for (unsigned i = 0, e = Values.size() - 1; i != e; ++i)
4300       PN->addIncoming(InVal, BB);
4301   }
4302 
4303   // Erase the old branch instruction.
4304   EraseTerminatorAndDCECond(BI);
4305   if (DTU)
4306     DTU->applyUpdates(Updates);
4307 
4308   LLVM_DEBUG(dbgs() << "  ** 'icmp' chain result is:\n" << *BB << '\n');
4309   return true;
4310 }
4311 
simplifyResume(ResumeInst * RI,IRBuilder<> & Builder)4312 bool SimplifyCFGOpt::simplifyResume(ResumeInst *RI, IRBuilder<> &Builder) {
4313   if (isa<PHINode>(RI->getValue()))
4314     return simplifyCommonResume(RI);
4315   else if (isa<LandingPadInst>(RI->getParent()->getFirstNonPHI()) &&
4316            RI->getValue() == RI->getParent()->getFirstNonPHI())
4317     // The resume must unwind the exception that caused control to branch here.
4318     return simplifySingleResume(RI);
4319 
4320   return false;
4321 }
4322 
4323 // Check if cleanup block is empty
isCleanupBlockEmpty(iterator_range<BasicBlock::iterator> R)4324 static bool isCleanupBlockEmpty(iterator_range<BasicBlock::iterator> R) {
4325   for (Instruction &I : R) {
4326     auto *II = dyn_cast<IntrinsicInst>(&I);
4327     if (!II)
4328       return false;
4329 
4330     Intrinsic::ID IntrinsicID = II->getIntrinsicID();
4331     switch (IntrinsicID) {
4332     case Intrinsic::dbg_declare:
4333     case Intrinsic::dbg_value:
4334     case Intrinsic::dbg_label:
4335     case Intrinsic::lifetime_end:
4336       break;
4337     default:
4338       return false;
4339     }
4340   }
4341   return true;
4342 }
4343 
4344 // Simplify resume that is shared by several landing pads (phi of landing pad).
simplifyCommonResume(ResumeInst * RI)4345 bool SimplifyCFGOpt::simplifyCommonResume(ResumeInst *RI) {
4346   BasicBlock *BB = RI->getParent();
4347 
4348   // Check that there are no other instructions except for debug and lifetime
4349   // intrinsics between the phi's and resume instruction.
4350   if (!isCleanupBlockEmpty(
4351           make_range(RI->getParent()->getFirstNonPHI(), BB->getTerminator())))
4352     return false;
4353 
4354   SmallSetVector<BasicBlock *, 4> TrivialUnwindBlocks;
4355   auto *PhiLPInst = cast<PHINode>(RI->getValue());
4356 
4357   // Check incoming blocks to see if any of them are trivial.
4358   for (unsigned Idx = 0, End = PhiLPInst->getNumIncomingValues(); Idx != End;
4359        Idx++) {
4360     auto *IncomingBB = PhiLPInst->getIncomingBlock(Idx);
4361     auto *IncomingValue = PhiLPInst->getIncomingValue(Idx);
4362 
4363     // If the block has other successors, we can not delete it because
4364     // it has other dependents.
4365     if (IncomingBB->getUniqueSuccessor() != BB)
4366       continue;
4367 
4368     auto *LandingPad = dyn_cast<LandingPadInst>(IncomingBB->getFirstNonPHI());
4369     // Not the landing pad that caused the control to branch here.
4370     if (IncomingValue != LandingPad)
4371       continue;
4372 
4373     if (isCleanupBlockEmpty(
4374             make_range(LandingPad->getNextNode(), IncomingBB->getTerminator())))
4375       TrivialUnwindBlocks.insert(IncomingBB);
4376   }
4377 
4378   // If no trivial unwind blocks, don't do any simplifications.
4379   if (TrivialUnwindBlocks.empty())
4380     return false;
4381 
4382   // Turn all invokes that unwind here into calls.
4383   for (auto *TrivialBB : TrivialUnwindBlocks) {
4384     // Blocks that will be simplified should be removed from the phi node.
4385     // Note there could be multiple edges to the resume block, and we need
4386     // to remove them all.
4387     while (PhiLPInst->getBasicBlockIndex(TrivialBB) != -1)
4388       BB->removePredecessor(TrivialBB, true);
4389 
4390     for (BasicBlock *Pred :
4391          llvm::make_early_inc_range(predecessors(TrivialBB))) {
4392       removeUnwindEdge(Pred, DTU);
4393       ++NumInvokes;
4394     }
4395 
4396     // In each SimplifyCFG run, only the current processed block can be erased.
4397     // Otherwise, it will break the iteration of SimplifyCFG pass. So instead
4398     // of erasing TrivialBB, we only remove the branch to the common resume
4399     // block so that we can later erase the resume block since it has no
4400     // predecessors.
4401     TrivialBB->getTerminator()->eraseFromParent();
4402     new UnreachableInst(RI->getContext(), TrivialBB);
4403     if (DTU)
4404       DTU->applyUpdates({{DominatorTree::Delete, TrivialBB, BB}});
4405   }
4406 
4407   // Delete the resume block if all its predecessors have been removed.
4408   if (pred_empty(BB))
4409     DeleteDeadBlock(BB, DTU);
4410 
4411   return !TrivialUnwindBlocks.empty();
4412 }
4413 
4414 // Simplify resume that is only used by a single (non-phi) landing pad.
simplifySingleResume(ResumeInst * RI)4415 bool SimplifyCFGOpt::simplifySingleResume(ResumeInst *RI) {
4416   BasicBlock *BB = RI->getParent();
4417   auto *LPInst = cast<LandingPadInst>(BB->getFirstNonPHI());
4418   assert(RI->getValue() == LPInst &&
4419          "Resume must unwind the exception that caused control to here");
4420 
4421   // Check that there are no other instructions except for debug intrinsics.
4422   if (!isCleanupBlockEmpty(
4423           make_range<Instruction *>(LPInst->getNextNode(), RI)))
4424     return false;
4425 
4426   // Turn all invokes that unwind here into calls and delete the basic block.
4427   for (BasicBlock *Pred : llvm::make_early_inc_range(predecessors(BB))) {
4428     removeUnwindEdge(Pred, DTU);
4429     ++NumInvokes;
4430   }
4431 
4432   // The landingpad is now unreachable.  Zap it.
4433   DeleteDeadBlock(BB, DTU);
4434   return true;
4435 }
4436 
removeEmptyCleanup(CleanupReturnInst * RI,DomTreeUpdater * DTU)4437 static bool removeEmptyCleanup(CleanupReturnInst *RI, DomTreeUpdater *DTU) {
4438   // If this is a trivial cleanup pad that executes no instructions, it can be
4439   // eliminated.  If the cleanup pad continues to the caller, any predecessor
4440   // that is an EH pad will be updated to continue to the caller and any
4441   // predecessor that terminates with an invoke instruction will have its invoke
4442   // instruction converted to a call instruction.  If the cleanup pad being
4443   // simplified does not continue to the caller, each predecessor will be
4444   // updated to continue to the unwind destination of the cleanup pad being
4445   // simplified.
4446   BasicBlock *BB = RI->getParent();
4447   CleanupPadInst *CPInst = RI->getCleanupPad();
4448   if (CPInst->getParent() != BB)
4449     // This isn't an empty cleanup.
4450     return false;
4451 
4452   // We cannot kill the pad if it has multiple uses.  This typically arises
4453   // from unreachable basic blocks.
4454   if (!CPInst->hasOneUse())
4455     return false;
4456 
4457   // Check that there are no other instructions except for benign intrinsics.
4458   if (!isCleanupBlockEmpty(
4459           make_range<Instruction *>(CPInst->getNextNode(), RI)))
4460     return false;
4461 
4462   // If the cleanup return we are simplifying unwinds to the caller, this will
4463   // set UnwindDest to nullptr.
4464   BasicBlock *UnwindDest = RI->getUnwindDest();
4465   Instruction *DestEHPad = UnwindDest ? UnwindDest->getFirstNonPHI() : nullptr;
4466 
4467   // We're about to remove BB from the control flow.  Before we do, sink any
4468   // PHINodes into the unwind destination.  Doing this before changing the
4469   // control flow avoids some potentially slow checks, since we can currently
4470   // be certain that UnwindDest and BB have no common predecessors (since they
4471   // are both EH pads).
4472   if (UnwindDest) {
4473     // First, go through the PHI nodes in UnwindDest and update any nodes that
4474     // reference the block we are removing
4475     for (PHINode &DestPN : UnwindDest->phis()) {
4476       int Idx = DestPN.getBasicBlockIndex(BB);
4477       // Since BB unwinds to UnwindDest, it has to be in the PHI node.
4478       assert(Idx != -1);
4479       // This PHI node has an incoming value that corresponds to a control
4480       // path through the cleanup pad we are removing.  If the incoming
4481       // value is in the cleanup pad, it must be a PHINode (because we
4482       // verified above that the block is otherwise empty).  Otherwise, the
4483       // value is either a constant or a value that dominates the cleanup
4484       // pad being removed.
4485       //
4486       // Because BB and UnwindDest are both EH pads, all of their
4487       // predecessors must unwind to these blocks, and since no instruction
4488       // can have multiple unwind destinations, there will be no overlap in
4489       // incoming blocks between SrcPN and DestPN.
4490       Value *SrcVal = DestPN.getIncomingValue(Idx);
4491       PHINode *SrcPN = dyn_cast<PHINode>(SrcVal);
4492 
4493       bool NeedPHITranslation = SrcPN && SrcPN->getParent() == BB;
4494       for (auto *Pred : predecessors(BB)) {
4495         Value *Incoming =
4496             NeedPHITranslation ? SrcPN->getIncomingValueForBlock(Pred) : SrcVal;
4497         DestPN.addIncoming(Incoming, Pred);
4498       }
4499     }
4500 
4501     // Sink any remaining PHI nodes directly into UnwindDest.
4502     Instruction *InsertPt = DestEHPad;
4503     for (PHINode &PN : BB->phis()) {
4504       // The iterator must be incremented here because the instructions are
4505       // being moved to another block.
4506       if (PN.use_empty() || !PN.isUsedOutsideOfBlock(BB))
4507         // If the PHI node has no uses or all of its uses are in this basic
4508         // block (meaning they are debug or lifetime intrinsics), just leave
4509         // it.  It will be erased when we erase BB below.
4510         continue;
4511 
4512       // Otherwise, sink this PHI node into UnwindDest.
4513       // Any predecessors to UnwindDest which are not already represented
4514       // must be back edges which inherit the value from the path through
4515       // BB.  In this case, the PHI value must reference itself.
4516       for (auto *pred : predecessors(UnwindDest))
4517         if (pred != BB)
4518           PN.addIncoming(&PN, pred);
4519       PN.moveBefore(InsertPt);
4520       // Also, add a dummy incoming value for the original BB itself,
4521       // so that the PHI is well-formed until we drop said predecessor.
4522       PN.addIncoming(UndefValue::get(PN.getType()), BB);
4523     }
4524   }
4525 
4526   std::vector<DominatorTree::UpdateType> Updates;
4527 
4528   // We use make_early_inc_range here because we will remove all predecessors.
4529   for (BasicBlock *PredBB : llvm::make_early_inc_range(predecessors(BB))) {
4530     if (UnwindDest == nullptr) {
4531       if (DTU) {
4532         DTU->applyUpdates(Updates);
4533         Updates.clear();
4534       }
4535       removeUnwindEdge(PredBB, DTU);
4536       ++NumInvokes;
4537     } else {
4538       BB->removePredecessor(PredBB);
4539       Instruction *TI = PredBB->getTerminator();
4540       TI->replaceUsesOfWith(BB, UnwindDest);
4541       if (DTU) {
4542         Updates.push_back({DominatorTree::Insert, PredBB, UnwindDest});
4543         Updates.push_back({DominatorTree::Delete, PredBB, BB});
4544       }
4545     }
4546   }
4547 
4548   if (DTU)
4549     DTU->applyUpdates(Updates);
4550 
4551   DeleteDeadBlock(BB, DTU);
4552 
4553   return true;
4554 }
4555 
4556 // Try to merge two cleanuppads together.
mergeCleanupPad(CleanupReturnInst * RI)4557 static bool mergeCleanupPad(CleanupReturnInst *RI) {
4558   // Skip any cleanuprets which unwind to caller, there is nothing to merge
4559   // with.
4560   BasicBlock *UnwindDest = RI->getUnwindDest();
4561   if (!UnwindDest)
4562     return false;
4563 
4564   // This cleanupret isn't the only predecessor of this cleanuppad, it wouldn't
4565   // be safe to merge without code duplication.
4566   if (UnwindDest->getSinglePredecessor() != RI->getParent())
4567     return false;
4568 
4569   // Verify that our cleanuppad's unwind destination is another cleanuppad.
4570   auto *SuccessorCleanupPad = dyn_cast<CleanupPadInst>(&UnwindDest->front());
4571   if (!SuccessorCleanupPad)
4572     return false;
4573 
4574   CleanupPadInst *PredecessorCleanupPad = RI->getCleanupPad();
4575   // Replace any uses of the successor cleanupad with the predecessor pad
4576   // The only cleanuppad uses should be this cleanupret, it's cleanupret and
4577   // funclet bundle operands.
4578   SuccessorCleanupPad->replaceAllUsesWith(PredecessorCleanupPad);
4579   // Remove the old cleanuppad.
4580   SuccessorCleanupPad->eraseFromParent();
4581   // Now, we simply replace the cleanupret with a branch to the unwind
4582   // destination.
4583   BranchInst::Create(UnwindDest, RI->getParent());
4584   RI->eraseFromParent();
4585 
4586   return true;
4587 }
4588 
simplifyCleanupReturn(CleanupReturnInst * RI)4589 bool SimplifyCFGOpt::simplifyCleanupReturn(CleanupReturnInst *RI) {
4590   // It is possible to transiantly have an undef cleanuppad operand because we
4591   // have deleted some, but not all, dead blocks.
4592   // Eventually, this block will be deleted.
4593   if (isa<UndefValue>(RI->getOperand(0)))
4594     return false;
4595 
4596   if (mergeCleanupPad(RI))
4597     return true;
4598 
4599   if (removeEmptyCleanup(RI, DTU))
4600     return true;
4601 
4602   return false;
4603 }
4604 
simplifyReturn(ReturnInst * RI,IRBuilder<> & Builder)4605 bool SimplifyCFGOpt::simplifyReturn(ReturnInst *RI, IRBuilder<> &Builder) {
4606   BasicBlock *BB = RI->getParent();
4607   if (!BB->getFirstNonPHIOrDbg()->isTerminator())
4608     return false;
4609 
4610   // Find predecessors that end with branches.
4611   SmallVector<BasicBlock *, 8> UncondBranchPreds;
4612   SmallVector<BranchInst *, 8> CondBranchPreds;
4613   for (BasicBlock *P : predecessors(BB)) {
4614     Instruction *PTI = P->getTerminator();
4615     if (BranchInst *BI = dyn_cast<BranchInst>(PTI)) {
4616       if (BI->isUnconditional())
4617         UncondBranchPreds.push_back(P);
4618       else
4619         CondBranchPreds.push_back(BI);
4620     }
4621   }
4622 
4623   // If we found some, do the transformation!
4624   if (!UncondBranchPreds.empty() && DupRet) {
4625     while (!UncondBranchPreds.empty()) {
4626       BasicBlock *Pred = UncondBranchPreds.pop_back_val();
4627       LLVM_DEBUG(dbgs() << "FOLDING: " << *BB
4628                         << "INTO UNCOND BRANCH PRED: " << *Pred);
4629       (void)FoldReturnIntoUncondBranch(RI, BB, Pred, DTU);
4630     }
4631 
4632     // If we eliminated all predecessors of the block, delete the block now.
4633     if (pred_empty(BB))
4634       DeleteDeadBlock(BB, DTU);
4635 
4636     return true;
4637   }
4638 
4639   // Check out all of the conditional branches going to this return
4640   // instruction.  If any of them just select between returns, change the
4641   // branch itself into a select/return pair.
4642   while (!CondBranchPreds.empty()) {
4643     BranchInst *BI = CondBranchPreds.pop_back_val();
4644 
4645     // Check to see if the non-BB successor is also a return block.
4646     if (isa<ReturnInst>(BI->getSuccessor(0)->getTerminator()) &&
4647         isa<ReturnInst>(BI->getSuccessor(1)->getTerminator()) &&
4648         SimplifyCondBranchToTwoReturns(BI, Builder))
4649       return true;
4650   }
4651   return false;
4652 }
4653 
simplifyUnreachable(UnreachableInst * UI)4654 bool SimplifyCFGOpt::simplifyUnreachable(UnreachableInst *UI) {
4655   BasicBlock *BB = UI->getParent();
4656 
4657   bool Changed = false;
4658 
4659   // If there are any instructions immediately before the unreachable that can
4660   // be removed, do so.
4661   while (UI->getIterator() != BB->begin()) {
4662     BasicBlock::iterator BBI = UI->getIterator();
4663     --BBI;
4664     // Do not delete instructions that can have side effects which might cause
4665     // the unreachable to not be reachable; specifically, calls and volatile
4666     // operations may have this effect.
4667     if (isa<CallInst>(BBI) && !isa<DbgInfoIntrinsic>(BBI))
4668       break;
4669 
4670     if (BBI->mayHaveSideEffects()) {
4671       if (auto *SI = dyn_cast<StoreInst>(BBI)) {
4672         if (SI->isVolatile())
4673           break;
4674       } else if (auto *LI = dyn_cast<LoadInst>(BBI)) {
4675         if (LI->isVolatile())
4676           break;
4677       } else if (auto *RMWI = dyn_cast<AtomicRMWInst>(BBI)) {
4678         if (RMWI->isVolatile())
4679           break;
4680       } else if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(BBI)) {
4681         if (CXI->isVolatile())
4682           break;
4683       } else if (isa<CatchPadInst>(BBI)) {
4684         // A catchpad may invoke exception object constructors and such, which
4685         // in some languages can be arbitrary code, so be conservative by
4686         // default.
4687         // For CoreCLR, it just involves a type test, so can be removed.
4688         if (classifyEHPersonality(BB->getParent()->getPersonalityFn()) !=
4689             EHPersonality::CoreCLR)
4690           break;
4691       } else if (!isa<FenceInst>(BBI) && !isa<VAArgInst>(BBI) &&
4692                  !isa<LandingPadInst>(BBI)) {
4693         break;
4694       }
4695       // Note that deleting LandingPad's here is in fact okay, although it
4696       // involves a bit of subtle reasoning. If this inst is a LandingPad,
4697       // all the predecessors of this block will be the unwind edges of Invokes,
4698       // and we can therefore guarantee this block will be erased.
4699     }
4700 
4701     // Delete this instruction (any uses are guaranteed to be dead)
4702     if (!BBI->use_empty())
4703       BBI->replaceAllUsesWith(UndefValue::get(BBI->getType()));
4704     BBI->eraseFromParent();
4705     Changed = true;
4706   }
4707 
4708   // If the unreachable instruction is the first in the block, take a gander
4709   // at all of the predecessors of this instruction, and simplify them.
4710   if (&BB->front() != UI)
4711     return Changed;
4712 
4713   std::vector<DominatorTree::UpdateType> Updates;
4714 
4715   SmallSetVector<BasicBlock *, 8> Preds(pred_begin(BB), pred_end(BB));
4716   for (unsigned i = 0, e = Preds.size(); i != e; ++i) {
4717     auto *Predecessor = Preds[i];
4718     Instruction *TI = Predecessor->getTerminator();
4719     IRBuilder<> Builder(TI);
4720     if (auto *BI = dyn_cast<BranchInst>(TI)) {
4721       // We could either have a proper unconditional branch,
4722       // or a degenerate conditional branch with matching destinations.
4723       if (all_of(BI->successors(),
4724                  [BB](auto *Successor) { return Successor == BB; })) {
4725         new UnreachableInst(TI->getContext(), TI);
4726         TI->eraseFromParent();
4727         Changed = true;
4728       } else {
4729         assert(BI->isConditional() && "Can't get here with an uncond branch.");
4730         Value* Cond = BI->getCondition();
4731         assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
4732                "The destinations are guaranteed to be different here.");
4733         if (BI->getSuccessor(0) == BB) {
4734           Builder.CreateAssumption(Builder.CreateNot(Cond));
4735           Builder.CreateBr(BI->getSuccessor(1));
4736         } else {
4737           assert(BI->getSuccessor(1) == BB && "Incorrect CFG");
4738           Builder.CreateAssumption(Cond);
4739           Builder.CreateBr(BI->getSuccessor(0));
4740         }
4741         EraseTerminatorAndDCECond(BI);
4742         Changed = true;
4743       }
4744       if (DTU)
4745         Updates.push_back({DominatorTree::Delete, Predecessor, BB});
4746     } else if (auto *SI = dyn_cast<SwitchInst>(TI)) {
4747       SwitchInstProfUpdateWrapper SU(*SI);
4748       for (auto i = SU->case_begin(), e = SU->case_end(); i != e;) {
4749         if (i->getCaseSuccessor() != BB) {
4750           ++i;
4751           continue;
4752         }
4753         BB->removePredecessor(SU->getParent());
4754         i = SU.removeCase(i);
4755         e = SU->case_end();
4756         Changed = true;
4757       }
4758       // Note that the default destination can't be removed!
4759       if (DTU && SI->getDefaultDest() != BB)
4760         Updates.push_back({DominatorTree::Delete, Predecessor, BB});
4761     } else if (auto *II = dyn_cast<InvokeInst>(TI)) {
4762       if (II->getUnwindDest() == BB) {
4763         if (DTU) {
4764           DTU->applyUpdates(Updates);
4765           Updates.clear();
4766         }
4767         removeUnwindEdge(TI->getParent(), DTU);
4768         Changed = true;
4769       }
4770     } else if (auto *CSI = dyn_cast<CatchSwitchInst>(TI)) {
4771       if (CSI->getUnwindDest() == BB) {
4772         if (DTU) {
4773           DTU->applyUpdates(Updates);
4774           Updates.clear();
4775         }
4776         removeUnwindEdge(TI->getParent(), DTU);
4777         Changed = true;
4778         continue;
4779       }
4780 
4781       for (CatchSwitchInst::handler_iterator I = CSI->handler_begin(),
4782                                              E = CSI->handler_end();
4783            I != E; ++I) {
4784         if (*I == BB) {
4785           CSI->removeHandler(I);
4786           --I;
4787           --E;
4788           Changed = true;
4789         }
4790       }
4791       if (DTU)
4792         Updates.push_back({DominatorTree::Delete, Predecessor, BB});
4793       if (CSI->getNumHandlers() == 0) {
4794         if (CSI->hasUnwindDest()) {
4795           // Redirect all predecessors of the block containing CatchSwitchInst
4796           // to instead branch to the CatchSwitchInst's unwind destination.
4797           if (DTU) {
4798             for (auto *PredecessorOfPredecessor : predecessors(Predecessor)) {
4799               Updates.push_back({DominatorTree::Insert,
4800                                  PredecessorOfPredecessor,
4801                                  CSI->getUnwindDest()});
4802               Updates.push_back({DominatorTree::Delete,
4803                                  PredecessorOfPredecessor, Predecessor});
4804             }
4805           }
4806           Predecessor->replaceAllUsesWith(CSI->getUnwindDest());
4807         } else {
4808           // Rewrite all preds to unwind to caller (or from invoke to call).
4809           if (DTU) {
4810             DTU->applyUpdates(Updates);
4811             Updates.clear();
4812           }
4813           SmallVector<BasicBlock *, 8> EHPreds(predecessors(Predecessor));
4814           for (BasicBlock *EHPred : EHPreds)
4815             removeUnwindEdge(EHPred, DTU);
4816         }
4817         // The catchswitch is no longer reachable.
4818         new UnreachableInst(CSI->getContext(), CSI);
4819         CSI->eraseFromParent();
4820         Changed = true;
4821       }
4822     } else if (auto *CRI = dyn_cast<CleanupReturnInst>(TI)) {
4823       (void)CRI;
4824       assert(CRI->hasUnwindDest() && CRI->getUnwindDest() == BB &&
4825              "Expected to always have an unwind to BB.");
4826       if (DTU)
4827         Updates.push_back({DominatorTree::Delete, Predecessor, BB});
4828       new UnreachableInst(TI->getContext(), TI);
4829       TI->eraseFromParent();
4830       Changed = true;
4831     }
4832   }
4833 
4834   if (DTU)
4835     DTU->applyUpdates(Updates);
4836 
4837   // If this block is now dead, remove it.
4838   if (pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) {
4839     DeleteDeadBlock(BB, DTU);
4840     return true;
4841   }
4842 
4843   return Changed;
4844 }
4845 
CasesAreContiguous(SmallVectorImpl<ConstantInt * > & Cases)4846 static bool CasesAreContiguous(SmallVectorImpl<ConstantInt *> &Cases) {
4847   assert(Cases.size() >= 1);
4848 
4849   array_pod_sort(Cases.begin(), Cases.end(), ConstantIntSortPredicate);
4850   for (size_t I = 1, E = Cases.size(); I != E; ++I) {
4851     if (Cases[I - 1]->getValue() != Cases[I]->getValue() + 1)
4852       return false;
4853   }
4854   return true;
4855 }
4856 
createUnreachableSwitchDefault(SwitchInst * Switch,DomTreeUpdater * DTU)4857 static void createUnreachableSwitchDefault(SwitchInst *Switch,
4858                                            DomTreeUpdater *DTU) {
4859   LLVM_DEBUG(dbgs() << "SimplifyCFG: switch default is dead.\n");
4860   auto *BB = Switch->getParent();
4861   BasicBlock *NewDefaultBlock = SplitBlockPredecessors(
4862       Switch->getDefaultDest(), Switch->getParent(), "", DTU);
4863   auto *OrigDefaultBlock = Switch->getDefaultDest();
4864   Switch->setDefaultDest(&*NewDefaultBlock);
4865   if (DTU)
4866     DTU->applyUpdates({{DominatorTree::Insert, BB, &*NewDefaultBlock},
4867                        {DominatorTree::Delete, BB, OrigDefaultBlock}});
4868   SplitBlock(&*NewDefaultBlock, &NewDefaultBlock->front(), DTU);
4869   SmallVector<DominatorTree::UpdateType, 2> Updates;
4870   if (DTU)
4871     for (auto *Successor : successors(NewDefaultBlock))
4872       Updates.push_back({DominatorTree::Delete, NewDefaultBlock, Successor});
4873   auto *NewTerminator = NewDefaultBlock->getTerminator();
4874   new UnreachableInst(Switch->getContext(), NewTerminator);
4875   EraseTerminatorAndDCECond(NewTerminator);
4876   if (DTU)
4877     DTU->applyUpdates(Updates);
4878 }
4879 
4880 /// Turn a switch with two reachable destinations into an integer range
4881 /// comparison and branch.
TurnSwitchRangeIntoICmp(SwitchInst * SI,IRBuilder<> & Builder)4882 bool SimplifyCFGOpt::TurnSwitchRangeIntoICmp(SwitchInst *SI,
4883                                              IRBuilder<> &Builder) {
4884   assert(SI->getNumCases() > 1 && "Degenerate switch?");
4885 
4886   bool HasDefault =
4887       !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg());
4888 
4889   auto *BB = SI->getParent();
4890 
4891   // Partition the cases into two sets with different destinations.
4892   BasicBlock *DestA = HasDefault ? SI->getDefaultDest() : nullptr;
4893   BasicBlock *DestB = nullptr;
4894   SmallVector<ConstantInt *, 16> CasesA;
4895   SmallVector<ConstantInt *, 16> CasesB;
4896 
4897   for (auto Case : SI->cases()) {
4898     BasicBlock *Dest = Case.getCaseSuccessor();
4899     if (!DestA)
4900       DestA = Dest;
4901     if (Dest == DestA) {
4902       CasesA.push_back(Case.getCaseValue());
4903       continue;
4904     }
4905     if (!DestB)
4906       DestB = Dest;
4907     if (Dest == DestB) {
4908       CasesB.push_back(Case.getCaseValue());
4909       continue;
4910     }
4911     return false; // More than two destinations.
4912   }
4913 
4914   assert(DestA && DestB &&
4915          "Single-destination switch should have been folded.");
4916   assert(DestA != DestB);
4917   assert(DestB != SI->getDefaultDest());
4918   assert(!CasesB.empty() && "There must be non-default cases.");
4919   assert(!CasesA.empty() || HasDefault);
4920 
4921   // Figure out if one of the sets of cases form a contiguous range.
4922   SmallVectorImpl<ConstantInt *> *ContiguousCases = nullptr;
4923   BasicBlock *ContiguousDest = nullptr;
4924   BasicBlock *OtherDest = nullptr;
4925   if (!CasesA.empty() && CasesAreContiguous(CasesA)) {
4926     ContiguousCases = &CasesA;
4927     ContiguousDest = DestA;
4928     OtherDest = DestB;
4929   } else if (CasesAreContiguous(CasesB)) {
4930     ContiguousCases = &CasesB;
4931     ContiguousDest = DestB;
4932     OtherDest = DestA;
4933   } else
4934     return false;
4935 
4936   // Start building the compare and branch.
4937 
4938   Constant *Offset = ConstantExpr::getNeg(ContiguousCases->back());
4939   Constant *NumCases =
4940       ConstantInt::get(Offset->getType(), ContiguousCases->size());
4941 
4942   Value *Sub = SI->getCondition();
4943   if (!Offset->isNullValue())
4944     Sub = Builder.CreateAdd(Sub, Offset, Sub->getName() + ".off");
4945 
4946   Value *Cmp;
4947   // If NumCases overflowed, then all possible values jump to the successor.
4948   if (NumCases->isNullValue() && !ContiguousCases->empty())
4949     Cmp = ConstantInt::getTrue(SI->getContext());
4950   else
4951     Cmp = Builder.CreateICmpULT(Sub, NumCases, "switch");
4952   BranchInst *NewBI = Builder.CreateCondBr(Cmp, ContiguousDest, OtherDest);
4953 
4954   // Update weight for the newly-created conditional branch.
4955   if (HasBranchWeights(SI)) {
4956     SmallVector<uint64_t, 8> Weights;
4957     GetBranchWeights(SI, Weights);
4958     if (Weights.size() == 1 + SI->getNumCases()) {
4959       uint64_t TrueWeight = 0;
4960       uint64_t FalseWeight = 0;
4961       for (size_t I = 0, E = Weights.size(); I != E; ++I) {
4962         if (SI->getSuccessor(I) == ContiguousDest)
4963           TrueWeight += Weights[I];
4964         else
4965           FalseWeight += Weights[I];
4966       }
4967       while (TrueWeight > UINT32_MAX || FalseWeight > UINT32_MAX) {
4968         TrueWeight /= 2;
4969         FalseWeight /= 2;
4970       }
4971       setBranchWeights(NewBI, TrueWeight, FalseWeight);
4972     }
4973   }
4974 
4975   // Prune obsolete incoming values off the successors' PHI nodes.
4976   for (auto BBI = ContiguousDest->begin(); isa<PHINode>(BBI); ++BBI) {
4977     unsigned PreviousEdges = ContiguousCases->size();
4978     if (ContiguousDest == SI->getDefaultDest())
4979       ++PreviousEdges;
4980     for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I)
4981       cast<PHINode>(BBI)->removeIncomingValue(SI->getParent());
4982   }
4983   for (auto BBI = OtherDest->begin(); isa<PHINode>(BBI); ++BBI) {
4984     unsigned PreviousEdges = SI->getNumCases() - ContiguousCases->size();
4985     if (OtherDest == SI->getDefaultDest())
4986       ++PreviousEdges;
4987     for (unsigned I = 0, E = PreviousEdges - 1; I != E; ++I)
4988       cast<PHINode>(BBI)->removeIncomingValue(SI->getParent());
4989   }
4990 
4991   // Clean up the default block - it may have phis or other instructions before
4992   // the unreachable terminator.
4993   if (!HasDefault)
4994     createUnreachableSwitchDefault(SI, DTU);
4995 
4996   auto *UnreachableDefault = SI->getDefaultDest();
4997 
4998   // Drop the switch.
4999   SI->eraseFromParent();
5000 
5001   if (!HasDefault && DTU)
5002     DTU->applyUpdates({{DominatorTree::Delete, BB, UnreachableDefault}});
5003 
5004   return true;
5005 }
5006 
5007 /// Compute masked bits for the condition of a switch
5008 /// and use it to remove dead cases.
eliminateDeadSwitchCases(SwitchInst * SI,DomTreeUpdater * DTU,AssumptionCache * AC,const DataLayout & DL)5009 static bool eliminateDeadSwitchCases(SwitchInst *SI, DomTreeUpdater *DTU,
5010                                      AssumptionCache *AC,
5011                                      const DataLayout &DL) {
5012   Value *Cond = SI->getCondition();
5013   unsigned Bits = Cond->getType()->getIntegerBitWidth();
5014   KnownBits Known = computeKnownBits(Cond, DL, 0, AC, SI);
5015 
5016   // We can also eliminate cases by determining that their values are outside of
5017   // the limited range of the condition based on how many significant (non-sign)
5018   // bits are in the condition value.
5019   unsigned ExtraSignBits = ComputeNumSignBits(Cond, DL, 0, AC, SI) - 1;
5020   unsigned MaxSignificantBitsInCond = Bits - ExtraSignBits;
5021 
5022   // Gather dead cases.
5023   SmallVector<ConstantInt *, 8> DeadCases;
5024   SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases;
5025   for (auto &Case : SI->cases()) {
5026     auto *Successor = Case.getCaseSuccessor();
5027     if (DTU)
5028       ++NumPerSuccessorCases[Successor];
5029     const APInt &CaseVal = Case.getCaseValue()->getValue();
5030     if (Known.Zero.intersects(CaseVal) || !Known.One.isSubsetOf(CaseVal) ||
5031         (CaseVal.getMinSignedBits() > MaxSignificantBitsInCond)) {
5032       DeadCases.push_back(Case.getCaseValue());
5033       if (DTU)
5034         --NumPerSuccessorCases[Successor];
5035       LLVM_DEBUG(dbgs() << "SimplifyCFG: switch case " << CaseVal
5036                         << " is dead.\n");
5037     }
5038   }
5039 
5040   // If we can prove that the cases must cover all possible values, the
5041   // default destination becomes dead and we can remove it.  If we know some
5042   // of the bits in the value, we can use that to more precisely compute the
5043   // number of possible unique case values.
5044   bool HasDefault =
5045       !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg());
5046   const unsigned NumUnknownBits =
5047       Bits - (Known.Zero | Known.One).countPopulation();
5048   assert(NumUnknownBits <= Bits);
5049   if (HasDefault && DeadCases.empty() &&
5050       NumUnknownBits < 64 /* avoid overflow */ &&
5051       SI->getNumCases() == (1ULL << NumUnknownBits)) {
5052     createUnreachableSwitchDefault(SI, DTU);
5053     return true;
5054   }
5055 
5056   if (DeadCases.empty())
5057     return false;
5058 
5059   SwitchInstProfUpdateWrapper SIW(*SI);
5060   for (ConstantInt *DeadCase : DeadCases) {
5061     SwitchInst::CaseIt CaseI = SI->findCaseValue(DeadCase);
5062     assert(CaseI != SI->case_default() &&
5063            "Case was not found. Probably mistake in DeadCases forming.");
5064     // Prune unused values from PHI nodes.
5065     CaseI->getCaseSuccessor()->removePredecessor(SI->getParent());
5066     SIW.removeCase(CaseI);
5067   }
5068 
5069   if (DTU) {
5070     std::vector<DominatorTree::UpdateType> Updates;
5071     for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases)
5072       if (I.second == 0)
5073         Updates.push_back({DominatorTree::Delete, SI->getParent(), I.first});
5074     DTU->applyUpdates(Updates);
5075   }
5076 
5077   return true;
5078 }
5079 
5080 /// If BB would be eligible for simplification by
5081 /// TryToSimplifyUncondBranchFromEmptyBlock (i.e. it is empty and terminated
5082 /// by an unconditional branch), look at the phi node for BB in the successor
5083 /// block and see if the incoming value is equal to CaseValue. If so, return
5084 /// the phi node, and set PhiIndex to BB's index in the phi node.
FindPHIForConditionForwarding(ConstantInt * CaseValue,BasicBlock * BB,int * PhiIndex)5085 static PHINode *FindPHIForConditionForwarding(ConstantInt *CaseValue,
5086                                               BasicBlock *BB, int *PhiIndex) {
5087   if (BB->getFirstNonPHIOrDbg() != BB->getTerminator())
5088     return nullptr; // BB must be empty to be a candidate for simplification.
5089   if (!BB->getSinglePredecessor())
5090     return nullptr; // BB must be dominated by the switch.
5091 
5092   BranchInst *Branch = dyn_cast<BranchInst>(BB->getTerminator());
5093   if (!Branch || !Branch->isUnconditional())
5094     return nullptr; // Terminator must be unconditional branch.
5095 
5096   BasicBlock *Succ = Branch->getSuccessor(0);
5097 
5098   for (PHINode &PHI : Succ->phis()) {
5099     int Idx = PHI.getBasicBlockIndex(BB);
5100     assert(Idx >= 0 && "PHI has no entry for predecessor?");
5101 
5102     Value *InValue = PHI.getIncomingValue(Idx);
5103     if (InValue != CaseValue)
5104       continue;
5105 
5106     *PhiIndex = Idx;
5107     return &PHI;
5108   }
5109 
5110   return nullptr;
5111 }
5112 
5113 /// Try to forward the condition of a switch instruction to a phi node
5114 /// dominated by the switch, if that would mean that some of the destination
5115 /// blocks of the switch can be folded away. Return true if a change is made.
ForwardSwitchConditionToPHI(SwitchInst * SI)5116 static bool ForwardSwitchConditionToPHI(SwitchInst *SI) {
5117   using ForwardingNodesMap = DenseMap<PHINode *, SmallVector<int, 4>>;
5118 
5119   ForwardingNodesMap ForwardingNodes;
5120   BasicBlock *SwitchBlock = SI->getParent();
5121   bool Changed = false;
5122   for (auto &Case : SI->cases()) {
5123     ConstantInt *CaseValue = Case.getCaseValue();
5124     BasicBlock *CaseDest = Case.getCaseSuccessor();
5125 
5126     // Replace phi operands in successor blocks that are using the constant case
5127     // value rather than the switch condition variable:
5128     //   switchbb:
5129     //   switch i32 %x, label %default [
5130     //     i32 17, label %succ
5131     //   ...
5132     //   succ:
5133     //     %r = phi i32 ... [ 17, %switchbb ] ...
5134     // -->
5135     //     %r = phi i32 ... [ %x, %switchbb ] ...
5136 
5137     for (PHINode &Phi : CaseDest->phis()) {
5138       // This only works if there is exactly 1 incoming edge from the switch to
5139       // a phi. If there is >1, that means multiple cases of the switch map to 1
5140       // value in the phi, and that phi value is not the switch condition. Thus,
5141       // this transform would not make sense (the phi would be invalid because
5142       // a phi can't have different incoming values from the same block).
5143       int SwitchBBIdx = Phi.getBasicBlockIndex(SwitchBlock);
5144       if (Phi.getIncomingValue(SwitchBBIdx) == CaseValue &&
5145           count(Phi.blocks(), SwitchBlock) == 1) {
5146         Phi.setIncomingValue(SwitchBBIdx, SI->getCondition());
5147         Changed = true;
5148       }
5149     }
5150 
5151     // Collect phi nodes that are indirectly using this switch's case constants.
5152     int PhiIdx;
5153     if (auto *Phi = FindPHIForConditionForwarding(CaseValue, CaseDest, &PhiIdx))
5154       ForwardingNodes[Phi].push_back(PhiIdx);
5155   }
5156 
5157   for (auto &ForwardingNode : ForwardingNodes) {
5158     PHINode *Phi = ForwardingNode.first;
5159     SmallVectorImpl<int> &Indexes = ForwardingNode.second;
5160     if (Indexes.size() < 2)
5161       continue;
5162 
5163     for (int Index : Indexes)
5164       Phi->setIncomingValue(Index, SI->getCondition());
5165     Changed = true;
5166   }
5167 
5168   return Changed;
5169 }
5170 
5171 /// Return true if the backend will be able to handle
5172 /// initializing an array of constants like C.
ValidLookupTableConstant(Constant * C,const TargetTransformInfo & TTI)5173 static bool ValidLookupTableConstant(Constant *C, const TargetTransformInfo &TTI) {
5174   if (C->isThreadDependent())
5175     return false;
5176   if (C->isDLLImportDependent())
5177     return false;
5178 
5179   if (!isa<ConstantFP>(C) && !isa<ConstantInt>(C) &&
5180       !isa<ConstantPointerNull>(C) && !isa<GlobalValue>(C) &&
5181       !isa<UndefValue>(C) && !isa<ConstantExpr>(C))
5182     return false;
5183 
5184   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
5185     if (!CE->isGEPWithNoNotionalOverIndexing())
5186       return false;
5187     if (!ValidLookupTableConstant(CE->getOperand(0), TTI))
5188       return false;
5189   }
5190 
5191   if (!TTI.shouldBuildLookupTablesForConstant(C))
5192     return false;
5193 
5194   return true;
5195 }
5196 
5197 /// If V is a Constant, return it. Otherwise, try to look up
5198 /// its constant value in ConstantPool, returning 0 if it's not there.
5199 static Constant *
LookupConstant(Value * V,const SmallDenseMap<Value *,Constant * > & ConstantPool)5200 LookupConstant(Value *V,
5201                const SmallDenseMap<Value *, Constant *> &ConstantPool) {
5202   if (Constant *C = dyn_cast<Constant>(V))
5203     return C;
5204   return ConstantPool.lookup(V);
5205 }
5206 
5207 /// Try to fold instruction I into a constant. This works for
5208 /// simple instructions such as binary operations where both operands are
5209 /// constant or can be replaced by constants from the ConstantPool. Returns the
5210 /// resulting constant on success, 0 otherwise.
5211 static Constant *
ConstantFold(Instruction * I,const DataLayout & DL,const SmallDenseMap<Value *,Constant * > & ConstantPool)5212 ConstantFold(Instruction *I, const DataLayout &DL,
5213              const SmallDenseMap<Value *, Constant *> &ConstantPool) {
5214   if (SelectInst *Select = dyn_cast<SelectInst>(I)) {
5215     Constant *A = LookupConstant(Select->getCondition(), ConstantPool);
5216     if (!A)
5217       return nullptr;
5218     if (A->isAllOnesValue())
5219       return LookupConstant(Select->getTrueValue(), ConstantPool);
5220     if (A->isNullValue())
5221       return LookupConstant(Select->getFalseValue(), ConstantPool);
5222     return nullptr;
5223   }
5224 
5225   SmallVector<Constant *, 4> COps;
5226   for (unsigned N = 0, E = I->getNumOperands(); N != E; ++N) {
5227     if (Constant *A = LookupConstant(I->getOperand(N), ConstantPool))
5228       COps.push_back(A);
5229     else
5230       return nullptr;
5231   }
5232 
5233   if (CmpInst *Cmp = dyn_cast<CmpInst>(I)) {
5234     return ConstantFoldCompareInstOperands(Cmp->getPredicate(), COps[0],
5235                                            COps[1], DL);
5236   }
5237 
5238   return ConstantFoldInstOperands(I, COps, DL);
5239 }
5240 
5241 /// Try to determine the resulting constant values in phi nodes
5242 /// at the common destination basic block, *CommonDest, for one of the case
5243 /// destionations CaseDest corresponding to value CaseVal (0 for the default
5244 /// case), of a switch instruction SI.
5245 static bool
GetCaseResults(SwitchInst * SI,ConstantInt * CaseVal,BasicBlock * CaseDest,BasicBlock ** CommonDest,SmallVectorImpl<std::pair<PHINode *,Constant * >> & Res,const DataLayout & DL,const TargetTransformInfo & TTI)5246 GetCaseResults(SwitchInst *SI, ConstantInt *CaseVal, BasicBlock *CaseDest,
5247                BasicBlock **CommonDest,
5248                SmallVectorImpl<std::pair<PHINode *, Constant *>> &Res,
5249                const DataLayout &DL, const TargetTransformInfo &TTI) {
5250   // The block from which we enter the common destination.
5251   BasicBlock *Pred = SI->getParent();
5252 
5253   // If CaseDest is empty except for some side-effect free instructions through
5254   // which we can constant-propagate the CaseVal, continue to its successor.
5255   SmallDenseMap<Value *, Constant *> ConstantPool;
5256   ConstantPool.insert(std::make_pair(SI->getCondition(), CaseVal));
5257   for (Instruction &I :CaseDest->instructionsWithoutDebug()) {
5258     if (I.isTerminator()) {
5259       // If the terminator is a simple branch, continue to the next block.
5260       if (I.getNumSuccessors() != 1 || I.isExceptionalTerminator())
5261         return false;
5262       Pred = CaseDest;
5263       CaseDest = I.getSuccessor(0);
5264     } else if (Constant *C = ConstantFold(&I, DL, ConstantPool)) {
5265       // Instruction is side-effect free and constant.
5266 
5267       // If the instruction has uses outside this block or a phi node slot for
5268       // the block, it is not safe to bypass the instruction since it would then
5269       // no longer dominate all its uses.
5270       for (auto &Use : I.uses()) {
5271         User *User = Use.getUser();
5272         if (Instruction *I = dyn_cast<Instruction>(User))
5273           if (I->getParent() == CaseDest)
5274             continue;
5275         if (PHINode *Phi = dyn_cast<PHINode>(User))
5276           if (Phi->getIncomingBlock(Use) == CaseDest)
5277             continue;
5278         return false;
5279       }
5280 
5281       ConstantPool.insert(std::make_pair(&I, C));
5282     } else {
5283       break;
5284     }
5285   }
5286 
5287   // If we did not have a CommonDest before, use the current one.
5288   if (!*CommonDest)
5289     *CommonDest = CaseDest;
5290   // If the destination isn't the common one, abort.
5291   if (CaseDest != *CommonDest)
5292     return false;
5293 
5294   // Get the values for this case from phi nodes in the destination block.
5295   for (PHINode &PHI : (*CommonDest)->phis()) {
5296     int Idx = PHI.getBasicBlockIndex(Pred);
5297     if (Idx == -1)
5298       continue;
5299 
5300     Constant *ConstVal =
5301         LookupConstant(PHI.getIncomingValue(Idx), ConstantPool);
5302     if (!ConstVal)
5303       return false;
5304 
5305     // Be conservative about which kinds of constants we support.
5306     if (!ValidLookupTableConstant(ConstVal, TTI))
5307       return false;
5308 
5309     Res.push_back(std::make_pair(&PHI, ConstVal));
5310   }
5311 
5312   return Res.size() > 0;
5313 }
5314 
5315 // Helper function used to add CaseVal to the list of cases that generate
5316 // Result. Returns the updated number of cases that generate this result.
MapCaseToResult(ConstantInt * CaseVal,SwitchCaseResultVectorTy & UniqueResults,Constant * Result)5317 static uintptr_t MapCaseToResult(ConstantInt *CaseVal,
5318                                  SwitchCaseResultVectorTy &UniqueResults,
5319                                  Constant *Result) {
5320   for (auto &I : UniqueResults) {
5321     if (I.first == Result) {
5322       I.second.push_back(CaseVal);
5323       return I.second.size();
5324     }
5325   }
5326   UniqueResults.push_back(
5327       std::make_pair(Result, SmallVector<ConstantInt *, 4>(1, CaseVal)));
5328   return 1;
5329 }
5330 
5331 // Helper function that initializes a map containing
5332 // results for the PHI node of the common destination block for a switch
5333 // instruction. Returns false if multiple PHI nodes have been found or if
5334 // there is not a common destination block for the switch.
5335 static bool
InitializeUniqueCases(SwitchInst * SI,PHINode * & PHI,BasicBlock * & CommonDest,SwitchCaseResultVectorTy & UniqueResults,Constant * & DefaultResult,const DataLayout & DL,const TargetTransformInfo & TTI,uintptr_t MaxUniqueResults,uintptr_t MaxCasesPerResult)5336 InitializeUniqueCases(SwitchInst *SI, PHINode *&PHI, BasicBlock *&CommonDest,
5337                       SwitchCaseResultVectorTy &UniqueResults,
5338                       Constant *&DefaultResult, const DataLayout &DL,
5339                       const TargetTransformInfo &TTI,
5340                       uintptr_t MaxUniqueResults, uintptr_t MaxCasesPerResult) {
5341   for (auto &I : SI->cases()) {
5342     ConstantInt *CaseVal = I.getCaseValue();
5343 
5344     // Resulting value at phi nodes for this case value.
5345     SwitchCaseResultsTy Results;
5346     if (!GetCaseResults(SI, CaseVal, I.getCaseSuccessor(), &CommonDest, Results,
5347                         DL, TTI))
5348       return false;
5349 
5350     // Only one value per case is permitted.
5351     if (Results.size() > 1)
5352       return false;
5353 
5354     // Add the case->result mapping to UniqueResults.
5355     const uintptr_t NumCasesForResult =
5356         MapCaseToResult(CaseVal, UniqueResults, Results.begin()->second);
5357 
5358     // Early out if there are too many cases for this result.
5359     if (NumCasesForResult > MaxCasesPerResult)
5360       return false;
5361 
5362     // Early out if there are too many unique results.
5363     if (UniqueResults.size() > MaxUniqueResults)
5364       return false;
5365 
5366     // Check the PHI consistency.
5367     if (!PHI)
5368       PHI = Results[0].first;
5369     else if (PHI != Results[0].first)
5370       return false;
5371   }
5372   // Find the default result value.
5373   SmallVector<std::pair<PHINode *, Constant *>, 1> DefaultResults;
5374   BasicBlock *DefaultDest = SI->getDefaultDest();
5375   GetCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest, DefaultResults,
5376                  DL, TTI);
5377   // If the default value is not found abort unless the default destination
5378   // is unreachable.
5379   DefaultResult =
5380       DefaultResults.size() == 1 ? DefaultResults.begin()->second : nullptr;
5381   if ((!DefaultResult &&
5382        !isa<UnreachableInst>(DefaultDest->getFirstNonPHIOrDbg())))
5383     return false;
5384 
5385   return true;
5386 }
5387 
5388 // Helper function that checks if it is possible to transform a switch with only
5389 // two cases (or two cases + default) that produces a result into a select.
5390 // Example:
5391 // switch (a) {
5392 //   case 10:                %0 = icmp eq i32 %a, 10
5393 //     return 10;            %1 = select i1 %0, i32 10, i32 4
5394 //   case 20:        ---->   %2 = icmp eq i32 %a, 20
5395 //     return 2;             %3 = select i1 %2, i32 2, i32 %1
5396 //   default:
5397 //     return 4;
5398 // }
ConvertTwoCaseSwitch(const SwitchCaseResultVectorTy & ResultVector,Constant * DefaultResult,Value * Condition,IRBuilder<> & Builder)5399 static Value *ConvertTwoCaseSwitch(const SwitchCaseResultVectorTy &ResultVector,
5400                                    Constant *DefaultResult, Value *Condition,
5401                                    IRBuilder<> &Builder) {
5402   // If we are selecting between only two cases transform into a simple
5403   // select or a two-way select if default is possible.
5404   if (ResultVector.size() == 2 && ResultVector[0].second.size() == 1 &&
5405       ResultVector[1].second.size() == 1) {
5406     ConstantInt *const FirstCase = ResultVector[0].second[0];
5407     ConstantInt *const SecondCase = ResultVector[1].second[0];
5408 
5409     bool DefaultCanTrigger = DefaultResult;
5410     Value *SelectValue = ResultVector[1].first;
5411     if (DefaultCanTrigger) {
5412       Value *const ValueCompare =
5413           Builder.CreateICmpEQ(Condition, SecondCase, "switch.selectcmp");
5414       SelectValue = Builder.CreateSelect(ValueCompare, ResultVector[1].first,
5415                                          DefaultResult, "switch.select");
5416     }
5417     Value *const ValueCompare =
5418         Builder.CreateICmpEQ(Condition, FirstCase, "switch.selectcmp");
5419     return Builder.CreateSelect(ValueCompare, ResultVector[0].first,
5420                                 SelectValue, "switch.select");
5421   }
5422 
5423   // Handle the degenerate case where two cases have the same value.
5424   if (ResultVector.size() == 1 && ResultVector[0].second.size() == 2 &&
5425       DefaultResult) {
5426     Value *Cmp1 = Builder.CreateICmpEQ(
5427         Condition, ResultVector[0].second[0], "switch.selectcmp.case1");
5428     Value *Cmp2 = Builder.CreateICmpEQ(
5429         Condition, ResultVector[0].second[1], "switch.selectcmp.case2");
5430     Value *Cmp = Builder.CreateOr(Cmp1, Cmp2, "switch.selectcmp");
5431     return Builder.CreateSelect(Cmp, ResultVector[0].first, DefaultResult);
5432   }
5433 
5434   return nullptr;
5435 }
5436 
5437 // Helper function to cleanup a switch instruction that has been converted into
5438 // a select, fixing up PHI nodes and basic blocks.
RemoveSwitchAfterSelectConversion(SwitchInst * SI,PHINode * PHI,Value * SelectValue,IRBuilder<> & Builder,DomTreeUpdater * DTU)5439 static void RemoveSwitchAfterSelectConversion(SwitchInst *SI, PHINode *PHI,
5440                                               Value *SelectValue,
5441                                               IRBuilder<> &Builder,
5442                                               DomTreeUpdater *DTU) {
5443   std::vector<DominatorTree::UpdateType> Updates;
5444 
5445   BasicBlock *SelectBB = SI->getParent();
5446   BasicBlock *DestBB = PHI->getParent();
5447 
5448   if (DTU && !is_contained(predecessors(DestBB), SelectBB))
5449     Updates.push_back({DominatorTree::Insert, SelectBB, DestBB});
5450   Builder.CreateBr(DestBB);
5451 
5452   // Remove the switch.
5453 
5454   while (PHI->getBasicBlockIndex(SelectBB) >= 0)
5455     PHI->removeIncomingValue(SelectBB);
5456   PHI->addIncoming(SelectValue, SelectBB);
5457 
5458   SmallPtrSet<BasicBlock *, 4> RemovedSuccessors;
5459   for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) {
5460     BasicBlock *Succ = SI->getSuccessor(i);
5461 
5462     if (Succ == DestBB)
5463       continue;
5464     Succ->removePredecessor(SelectBB);
5465     if (DTU && RemovedSuccessors.insert(Succ).second)
5466       Updates.push_back({DominatorTree::Delete, SelectBB, Succ});
5467   }
5468   SI->eraseFromParent();
5469   if (DTU)
5470     DTU->applyUpdates(Updates);
5471 }
5472 
5473 /// If the switch is only used to initialize one or more
5474 /// phi nodes in a common successor block with only two different
5475 /// constant values, replace the switch with select.
switchToSelect(SwitchInst * SI,IRBuilder<> & Builder,DomTreeUpdater * DTU,const DataLayout & DL,const TargetTransformInfo & TTI)5476 static bool switchToSelect(SwitchInst *SI, IRBuilder<> &Builder,
5477                            DomTreeUpdater *DTU, const DataLayout &DL,
5478                            const TargetTransformInfo &TTI) {
5479   Value *const Cond = SI->getCondition();
5480   PHINode *PHI = nullptr;
5481   BasicBlock *CommonDest = nullptr;
5482   Constant *DefaultResult;
5483   SwitchCaseResultVectorTy UniqueResults;
5484   // Collect all the cases that will deliver the same value from the switch.
5485   if (!InitializeUniqueCases(SI, PHI, CommonDest, UniqueResults, DefaultResult,
5486                              DL, TTI, /*MaxUniqueResults*/2,
5487                              /*MaxCasesPerResult*/2))
5488     return false;
5489   assert(PHI != nullptr && "PHI for value select not found");
5490 
5491   Builder.SetInsertPoint(SI);
5492   Value *SelectValue =
5493       ConvertTwoCaseSwitch(UniqueResults, DefaultResult, Cond, Builder);
5494   if (SelectValue) {
5495     RemoveSwitchAfterSelectConversion(SI, PHI, SelectValue, Builder, DTU);
5496     return true;
5497   }
5498   // The switch couldn't be converted into a select.
5499   return false;
5500 }
5501 
5502 namespace {
5503 
5504 /// This class represents a lookup table that can be used to replace a switch.
5505 class SwitchLookupTable {
5506 public:
5507   /// Create a lookup table to use as a switch replacement with the contents
5508   /// of Values, using DefaultValue to fill any holes in the table.
5509   SwitchLookupTable(
5510       Module &M, uint64_t TableSize, ConstantInt *Offset,
5511       const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values,
5512       Constant *DefaultValue, const DataLayout &DL, const StringRef &FuncName);
5513 
5514   /// Build instructions with Builder to retrieve the value at
5515   /// the position given by Index in the lookup table.
5516   Value *BuildLookup(Value *Index, IRBuilder<> &Builder);
5517 
5518   /// Return true if a table with TableSize elements of
5519   /// type ElementType would fit in a target-legal register.
5520   static bool WouldFitInRegister(const DataLayout &DL, uint64_t TableSize,
5521                                  Type *ElementType);
5522 
5523 private:
5524   // Depending on the contents of the table, it can be represented in
5525   // different ways.
5526   enum {
5527     // For tables where each element contains the same value, we just have to
5528     // store that single value and return it for each lookup.
5529     SingleValueKind,
5530 
5531     // For tables where there is a linear relationship between table index
5532     // and values. We calculate the result with a simple multiplication
5533     // and addition instead of a table lookup.
5534     LinearMapKind,
5535 
5536     // For small tables with integer elements, we can pack them into a bitmap
5537     // that fits into a target-legal register. Values are retrieved by
5538     // shift and mask operations.
5539     BitMapKind,
5540 
5541     // The table is stored as an array of values. Values are retrieved by load
5542     // instructions from the table.
5543     ArrayKind
5544   } Kind;
5545 
5546   // For SingleValueKind, this is the single value.
5547   Constant *SingleValue = nullptr;
5548 
5549   // For BitMapKind, this is the bitmap.
5550   ConstantInt *BitMap = nullptr;
5551   IntegerType *BitMapElementTy = nullptr;
5552 
5553   // For LinearMapKind, these are the constants used to derive the value.
5554   ConstantInt *LinearOffset = nullptr;
5555   ConstantInt *LinearMultiplier = nullptr;
5556 
5557   // For ArrayKind, this is the array.
5558   GlobalVariable *Array = nullptr;
5559 };
5560 
5561 } // end anonymous namespace
5562 
SwitchLookupTable(Module & M,uint64_t TableSize,ConstantInt * Offset,const SmallVectorImpl<std::pair<ConstantInt *,Constant * >> & Values,Constant * DefaultValue,const DataLayout & DL,const StringRef & FuncName)5563 SwitchLookupTable::SwitchLookupTable(
5564     Module &M, uint64_t TableSize, ConstantInt *Offset,
5565     const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values,
5566     Constant *DefaultValue, const DataLayout &DL, const StringRef &FuncName) {
5567   assert(Values.size() && "Can't build lookup table without values!");
5568   assert(TableSize >= Values.size() && "Can't fit values in table!");
5569 
5570   // If all values in the table are equal, this is that value.
5571   SingleValue = Values.begin()->second;
5572 
5573   Type *ValueType = Values.begin()->second->getType();
5574 
5575   // Build up the table contents.
5576   SmallVector<Constant *, 64> TableContents(TableSize);
5577   for (size_t I = 0, E = Values.size(); I != E; ++I) {
5578     ConstantInt *CaseVal = Values[I].first;
5579     Constant *CaseRes = Values[I].second;
5580     assert(CaseRes->getType() == ValueType);
5581 
5582     uint64_t Idx = (CaseVal->getValue() - Offset->getValue()).getLimitedValue();
5583     TableContents[Idx] = CaseRes;
5584 
5585     if (CaseRes != SingleValue)
5586       SingleValue = nullptr;
5587   }
5588 
5589   // Fill in any holes in the table with the default result.
5590   if (Values.size() < TableSize) {
5591     assert(DefaultValue &&
5592            "Need a default value to fill the lookup table holes.");
5593     assert(DefaultValue->getType() == ValueType);
5594     for (uint64_t I = 0; I < TableSize; ++I) {
5595       if (!TableContents[I])
5596         TableContents[I] = DefaultValue;
5597     }
5598 
5599     if (DefaultValue != SingleValue)
5600       SingleValue = nullptr;
5601   }
5602 
5603   // If each element in the table contains the same value, we only need to store
5604   // that single value.
5605   if (SingleValue) {
5606     Kind = SingleValueKind;
5607     return;
5608   }
5609 
5610   // Check if we can derive the value with a linear transformation from the
5611   // table index.
5612   if (isa<IntegerType>(ValueType)) {
5613     bool LinearMappingPossible = true;
5614     APInt PrevVal;
5615     APInt DistToPrev;
5616     assert(TableSize >= 2 && "Should be a SingleValue table.");
5617     // Check if there is the same distance between two consecutive values.
5618     for (uint64_t I = 0; I < TableSize; ++I) {
5619       ConstantInt *ConstVal = dyn_cast<ConstantInt>(TableContents[I]);
5620       if (!ConstVal) {
5621         // This is an undef. We could deal with it, but undefs in lookup tables
5622         // are very seldom. It's probably not worth the additional complexity.
5623         LinearMappingPossible = false;
5624         break;
5625       }
5626       const APInt &Val = ConstVal->getValue();
5627       if (I != 0) {
5628         APInt Dist = Val - PrevVal;
5629         if (I == 1) {
5630           DistToPrev = Dist;
5631         } else if (Dist != DistToPrev) {
5632           LinearMappingPossible = false;
5633           break;
5634         }
5635       }
5636       PrevVal = Val;
5637     }
5638     if (LinearMappingPossible) {
5639       LinearOffset = cast<ConstantInt>(TableContents[0]);
5640       LinearMultiplier = ConstantInt::get(M.getContext(), DistToPrev);
5641       Kind = LinearMapKind;
5642       ++NumLinearMaps;
5643       return;
5644     }
5645   }
5646 
5647   // If the type is integer and the table fits in a register, build a bitmap.
5648   if (WouldFitInRegister(DL, TableSize, ValueType)) {
5649     IntegerType *IT = cast<IntegerType>(ValueType);
5650     APInt TableInt(TableSize * IT->getBitWidth(), 0);
5651     for (uint64_t I = TableSize; I > 0; --I) {
5652       TableInt <<= IT->getBitWidth();
5653       // Insert values into the bitmap. Undef values are set to zero.
5654       if (!isa<UndefValue>(TableContents[I - 1])) {
5655         ConstantInt *Val = cast<ConstantInt>(TableContents[I - 1]);
5656         TableInt |= Val->getValue().zext(TableInt.getBitWidth());
5657       }
5658     }
5659     BitMap = ConstantInt::get(M.getContext(), TableInt);
5660     BitMapElementTy = IT;
5661     Kind = BitMapKind;
5662     ++NumBitMaps;
5663     return;
5664   }
5665 
5666   // Store the table in an array.
5667   ArrayType *ArrayTy = ArrayType::get(ValueType, TableSize);
5668   Constant *Initializer = ConstantArray::get(ArrayTy, TableContents);
5669 
5670   Array = new GlobalVariable(M, ArrayTy, /*isConstant=*/true,
5671                              GlobalVariable::PrivateLinkage, Initializer,
5672                              "switch.table." + FuncName);
5673   Array->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
5674   // Set the alignment to that of an array items. We will be only loading one
5675   // value out of it.
5676   Array->setAlignment(Align(DL.getPrefTypeAlignment(ValueType)));
5677   Kind = ArrayKind;
5678 }
5679 
BuildLookup(Value * Index,IRBuilder<> & Builder)5680 Value *SwitchLookupTable::BuildLookup(Value *Index, IRBuilder<> &Builder) {
5681   switch (Kind) {
5682   case SingleValueKind:
5683     return SingleValue;
5684   case LinearMapKind: {
5685     // Derive the result value from the input value.
5686     Value *Result = Builder.CreateIntCast(Index, LinearMultiplier->getType(),
5687                                           false, "switch.idx.cast");
5688     if (!LinearMultiplier->isOne())
5689       Result = Builder.CreateMul(Result, LinearMultiplier, "switch.idx.mult");
5690     if (!LinearOffset->isZero())
5691       Result = Builder.CreateAdd(Result, LinearOffset, "switch.offset");
5692     return Result;
5693   }
5694   case BitMapKind: {
5695     // Type of the bitmap (e.g. i59).
5696     IntegerType *MapTy = BitMap->getType();
5697 
5698     // Cast Index to the same type as the bitmap.
5699     // Note: The Index is <= the number of elements in the table, so
5700     // truncating it to the width of the bitmask is safe.
5701     Value *ShiftAmt = Builder.CreateZExtOrTrunc(Index, MapTy, "switch.cast");
5702 
5703     // Multiply the shift amount by the element width.
5704     ShiftAmt = Builder.CreateMul(
5705         ShiftAmt, ConstantInt::get(MapTy, BitMapElementTy->getBitWidth()),
5706         "switch.shiftamt");
5707 
5708     // Shift down.
5709     Value *DownShifted =
5710         Builder.CreateLShr(BitMap, ShiftAmt, "switch.downshift");
5711     // Mask off.
5712     return Builder.CreateTrunc(DownShifted, BitMapElementTy, "switch.masked");
5713   }
5714   case ArrayKind: {
5715     // Make sure the table index will not overflow when treated as signed.
5716     IntegerType *IT = cast<IntegerType>(Index->getType());
5717     uint64_t TableSize =
5718         Array->getInitializer()->getType()->getArrayNumElements();
5719     if (TableSize > (1ULL << (IT->getBitWidth() - 1)))
5720       Index = Builder.CreateZExt(
5721           Index, IntegerType::get(IT->getContext(), IT->getBitWidth() + 1),
5722           "switch.tableidx.zext");
5723 
5724     Value *GEPIndices[] = {Builder.getInt32(0), Index};
5725     Value *GEP = Builder.CreateInBoundsGEP(Array->getValueType(), Array,
5726                                            GEPIndices, "switch.gep");
5727     return Builder.CreateLoad(
5728         cast<ArrayType>(Array->getValueType())->getElementType(), GEP,
5729         "switch.load");
5730   }
5731   }
5732   llvm_unreachable("Unknown lookup table kind!");
5733 }
5734 
WouldFitInRegister(const DataLayout & DL,uint64_t TableSize,Type * ElementType)5735 bool SwitchLookupTable::WouldFitInRegister(const DataLayout &DL,
5736                                            uint64_t TableSize,
5737                                            Type *ElementType) {
5738   auto *IT = dyn_cast<IntegerType>(ElementType);
5739   if (!IT)
5740     return false;
5741   // FIXME: If the type is wider than it needs to be, e.g. i8 but all values
5742   // are <= 15, we could try to narrow the type.
5743 
5744   // Avoid overflow, fitsInLegalInteger uses unsigned int for the width.
5745   if (TableSize >= UINT_MAX / IT->getBitWidth())
5746     return false;
5747   return DL.fitsInLegalInteger(TableSize * IT->getBitWidth());
5748 }
5749 
5750 /// Determine whether a lookup table should be built for this switch, based on
5751 /// the number of cases, size of the table, and the types of the results.
5752 static bool
ShouldBuildLookupTable(SwitchInst * SI,uint64_t TableSize,const TargetTransformInfo & TTI,const DataLayout & DL,const SmallDenseMap<PHINode *,Type * > & ResultTypes)5753 ShouldBuildLookupTable(SwitchInst *SI, uint64_t TableSize,
5754                        const TargetTransformInfo &TTI, const DataLayout &DL,
5755                        const SmallDenseMap<PHINode *, Type *> &ResultTypes) {
5756   if (SI->getNumCases() > TableSize || TableSize >= UINT64_MAX / 10)
5757     return false; // TableSize overflowed, or mul below might overflow.
5758 
5759   bool AllTablesFitInRegister = true;
5760   bool HasIllegalType = false;
5761   for (const auto &I : ResultTypes) {
5762     Type *Ty = I.second;
5763 
5764     // Saturate this flag to true.
5765     HasIllegalType = HasIllegalType || !TTI.isTypeLegal(Ty);
5766 
5767     // Saturate this flag to false.
5768     AllTablesFitInRegister =
5769         AllTablesFitInRegister &&
5770         SwitchLookupTable::WouldFitInRegister(DL, TableSize, Ty);
5771 
5772     // If both flags saturate, we're done. NOTE: This *only* works with
5773     // saturating flags, and all flags have to saturate first due to the
5774     // non-deterministic behavior of iterating over a dense map.
5775     if (HasIllegalType && !AllTablesFitInRegister)
5776       break;
5777   }
5778 
5779   // If each table would fit in a register, we should build it anyway.
5780   if (AllTablesFitInRegister)
5781     return true;
5782 
5783   // Don't build a table that doesn't fit in-register if it has illegal types.
5784   if (HasIllegalType)
5785     return false;
5786 
5787   // The table density should be at least 40%. This is the same criterion as for
5788   // jump tables, see SelectionDAGBuilder::handleJTSwitchCase.
5789   // FIXME: Find the best cut-off.
5790   return SI->getNumCases() * 10 >= TableSize * 4;
5791 }
5792 
5793 /// Try to reuse the switch table index compare. Following pattern:
5794 /// \code
5795 ///     if (idx < tablesize)
5796 ///        r = table[idx]; // table does not contain default_value
5797 ///     else
5798 ///        r = default_value;
5799 ///     if (r != default_value)
5800 ///        ...
5801 /// \endcode
5802 /// Is optimized to:
5803 /// \code
5804 ///     cond = idx < tablesize;
5805 ///     if (cond)
5806 ///        r = table[idx];
5807 ///     else
5808 ///        r = default_value;
5809 ///     if (cond)
5810 ///        ...
5811 /// \endcode
5812 /// Jump threading will then eliminate the second if(cond).
reuseTableCompare(User * PhiUser,BasicBlock * PhiBlock,BranchInst * RangeCheckBranch,Constant * DefaultValue,const SmallVectorImpl<std::pair<ConstantInt *,Constant * >> & Values)5813 static void reuseTableCompare(
5814     User *PhiUser, BasicBlock *PhiBlock, BranchInst *RangeCheckBranch,
5815     Constant *DefaultValue,
5816     const SmallVectorImpl<std::pair<ConstantInt *, Constant *>> &Values) {
5817   ICmpInst *CmpInst = dyn_cast<ICmpInst>(PhiUser);
5818   if (!CmpInst)
5819     return;
5820 
5821   // We require that the compare is in the same block as the phi so that jump
5822   // threading can do its work afterwards.
5823   if (CmpInst->getParent() != PhiBlock)
5824     return;
5825 
5826   Constant *CmpOp1 = dyn_cast<Constant>(CmpInst->getOperand(1));
5827   if (!CmpOp1)
5828     return;
5829 
5830   Value *RangeCmp = RangeCheckBranch->getCondition();
5831   Constant *TrueConst = ConstantInt::getTrue(RangeCmp->getType());
5832   Constant *FalseConst = ConstantInt::getFalse(RangeCmp->getType());
5833 
5834   // Check if the compare with the default value is constant true or false.
5835   Constant *DefaultConst = ConstantExpr::getICmp(CmpInst->getPredicate(),
5836                                                  DefaultValue, CmpOp1, true);
5837   if (DefaultConst != TrueConst && DefaultConst != FalseConst)
5838     return;
5839 
5840   // Check if the compare with the case values is distinct from the default
5841   // compare result.
5842   for (auto ValuePair : Values) {
5843     Constant *CaseConst = ConstantExpr::getICmp(CmpInst->getPredicate(),
5844                                                 ValuePair.second, CmpOp1, true);
5845     if (!CaseConst || CaseConst == DefaultConst || isa<UndefValue>(CaseConst))
5846       return;
5847     assert((CaseConst == TrueConst || CaseConst == FalseConst) &&
5848            "Expect true or false as compare result.");
5849   }
5850 
5851   // Check if the branch instruction dominates the phi node. It's a simple
5852   // dominance check, but sufficient for our needs.
5853   // Although this check is invariant in the calling loops, it's better to do it
5854   // at this late stage. Practically we do it at most once for a switch.
5855   BasicBlock *BranchBlock = RangeCheckBranch->getParent();
5856   for (BasicBlock *Pred : predecessors(PhiBlock)) {
5857     if (Pred != BranchBlock && Pred->getUniquePredecessor() != BranchBlock)
5858       return;
5859   }
5860 
5861   if (DefaultConst == FalseConst) {
5862     // The compare yields the same result. We can replace it.
5863     CmpInst->replaceAllUsesWith(RangeCmp);
5864     ++NumTableCmpReuses;
5865   } else {
5866     // The compare yields the same result, just inverted. We can replace it.
5867     Value *InvertedTableCmp = BinaryOperator::CreateXor(
5868         RangeCmp, ConstantInt::get(RangeCmp->getType(), 1), "inverted.cmp",
5869         RangeCheckBranch);
5870     CmpInst->replaceAllUsesWith(InvertedTableCmp);
5871     ++NumTableCmpReuses;
5872   }
5873 }
5874 
5875 /// If the switch is only used to initialize one or more phi nodes in a common
5876 /// successor block with different constant values, replace the switch with
5877 /// lookup tables.
SwitchToLookupTable(SwitchInst * SI,IRBuilder<> & Builder,DomTreeUpdater * DTU,const DataLayout & DL,const TargetTransformInfo & TTI)5878 static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
5879                                 DomTreeUpdater *DTU, const DataLayout &DL,
5880                                 const TargetTransformInfo &TTI) {
5881   assert(SI->getNumCases() > 1 && "Degenerate switch?");
5882 
5883   BasicBlock *BB = SI->getParent();
5884   Function *Fn = BB->getParent();
5885   // Only build lookup table when we have a target that supports it or the
5886   // attribute is not set.
5887   if (!TTI.shouldBuildLookupTables() ||
5888       (Fn->getFnAttribute("no-jump-tables").getValueAsBool()))
5889     return false;
5890 
5891   // FIXME: If the switch is too sparse for a lookup table, perhaps we could
5892   // split off a dense part and build a lookup table for that.
5893 
5894   // FIXME: This creates arrays of GEPs to constant strings, which means each
5895   // GEP needs a runtime relocation in PIC code. We should just build one big
5896   // string and lookup indices into that.
5897 
5898   // Ignore switches with less than three cases. Lookup tables will not make
5899   // them faster, so we don't analyze them.
5900   if (SI->getNumCases() < 3)
5901     return false;
5902 
5903   // Figure out the corresponding result for each case value and phi node in the
5904   // common destination, as well as the min and max case values.
5905   assert(!SI->cases().empty());
5906   SwitchInst::CaseIt CI = SI->case_begin();
5907   ConstantInt *MinCaseVal = CI->getCaseValue();
5908   ConstantInt *MaxCaseVal = CI->getCaseValue();
5909 
5910   BasicBlock *CommonDest = nullptr;
5911 
5912   using ResultListTy = SmallVector<std::pair<ConstantInt *, Constant *>, 4>;
5913   SmallDenseMap<PHINode *, ResultListTy> ResultLists;
5914 
5915   SmallDenseMap<PHINode *, Constant *> DefaultResults;
5916   SmallDenseMap<PHINode *, Type *> ResultTypes;
5917   SmallVector<PHINode *, 4> PHIs;
5918 
5919   for (SwitchInst::CaseIt E = SI->case_end(); CI != E; ++CI) {
5920     ConstantInt *CaseVal = CI->getCaseValue();
5921     if (CaseVal->getValue().slt(MinCaseVal->getValue()))
5922       MinCaseVal = CaseVal;
5923     if (CaseVal->getValue().sgt(MaxCaseVal->getValue()))
5924       MaxCaseVal = CaseVal;
5925 
5926     // Resulting value at phi nodes for this case value.
5927     using ResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>;
5928     ResultsTy Results;
5929     if (!GetCaseResults(SI, CaseVal, CI->getCaseSuccessor(), &CommonDest,
5930                         Results, DL, TTI))
5931       return false;
5932 
5933     // Append the result from this case to the list for each phi.
5934     for (const auto &I : Results) {
5935       PHINode *PHI = I.first;
5936       Constant *Value = I.second;
5937       if (!ResultLists.count(PHI))
5938         PHIs.push_back(PHI);
5939       ResultLists[PHI].push_back(std::make_pair(CaseVal, Value));
5940     }
5941   }
5942 
5943   // Keep track of the result types.
5944   for (PHINode *PHI : PHIs) {
5945     ResultTypes[PHI] = ResultLists[PHI][0].second->getType();
5946   }
5947 
5948   uint64_t NumResults = ResultLists[PHIs[0]].size();
5949   APInt RangeSpread = MaxCaseVal->getValue() - MinCaseVal->getValue();
5950   uint64_t TableSize = RangeSpread.getLimitedValue() + 1;
5951   bool TableHasHoles = (NumResults < TableSize);
5952 
5953   // If the table has holes, we need a constant result for the default case
5954   // or a bitmask that fits in a register.
5955   SmallVector<std::pair<PHINode *, Constant *>, 4> DefaultResultsList;
5956   bool HasDefaultResults =
5957       GetCaseResults(SI, nullptr, SI->getDefaultDest(), &CommonDest,
5958                      DefaultResultsList, DL, TTI);
5959 
5960   bool NeedMask = (TableHasHoles && !HasDefaultResults);
5961   if (NeedMask) {
5962     // As an extra penalty for the validity test we require more cases.
5963     if (SI->getNumCases() < 4) // FIXME: Find best threshold value (benchmark).
5964       return false;
5965     if (!DL.fitsInLegalInteger(TableSize))
5966       return false;
5967   }
5968 
5969   for (const auto &I : DefaultResultsList) {
5970     PHINode *PHI = I.first;
5971     Constant *Result = I.second;
5972     DefaultResults[PHI] = Result;
5973   }
5974 
5975   if (!ShouldBuildLookupTable(SI, TableSize, TTI, DL, ResultTypes))
5976     return false;
5977 
5978   std::vector<DominatorTree::UpdateType> Updates;
5979 
5980   // Create the BB that does the lookups.
5981   Module &Mod = *CommonDest->getParent()->getParent();
5982   BasicBlock *LookupBB = BasicBlock::Create(
5983       Mod.getContext(), "switch.lookup", CommonDest->getParent(), CommonDest);
5984 
5985   // Compute the table index value.
5986   Builder.SetInsertPoint(SI);
5987   Value *TableIndex;
5988   if (MinCaseVal->isNullValue())
5989     TableIndex = SI->getCondition();
5990   else
5991     TableIndex = Builder.CreateSub(SI->getCondition(), MinCaseVal,
5992                                    "switch.tableidx");
5993 
5994   // Compute the maximum table size representable by the integer type we are
5995   // switching upon.
5996   unsigned CaseSize = MinCaseVal->getType()->getPrimitiveSizeInBits();
5997   uint64_t MaxTableSize = CaseSize > 63 ? UINT64_MAX : 1ULL << CaseSize;
5998   assert(MaxTableSize >= TableSize &&
5999          "It is impossible for a switch to have more entries than the max "
6000          "representable value of its input integer type's size.");
6001 
6002   // If the default destination is unreachable, or if the lookup table covers
6003   // all values of the conditional variable, branch directly to the lookup table
6004   // BB. Otherwise, check that the condition is within the case range.
6005   const bool DefaultIsReachable =
6006       !isa<UnreachableInst>(SI->getDefaultDest()->getFirstNonPHIOrDbg());
6007   const bool GeneratingCoveredLookupTable = (MaxTableSize == TableSize);
6008   BranchInst *RangeCheckBranch = nullptr;
6009 
6010   if (!DefaultIsReachable || GeneratingCoveredLookupTable) {
6011     Builder.CreateBr(LookupBB);
6012     if (DTU)
6013       Updates.push_back({DominatorTree::Insert, BB, LookupBB});
6014     // Note: We call removeProdecessor later since we need to be able to get the
6015     // PHI value for the default case in case we're using a bit mask.
6016   } else {
6017     Value *Cmp = Builder.CreateICmpULT(
6018         TableIndex, ConstantInt::get(MinCaseVal->getType(), TableSize));
6019     RangeCheckBranch =
6020         Builder.CreateCondBr(Cmp, LookupBB, SI->getDefaultDest());
6021     if (DTU)
6022       Updates.push_back({DominatorTree::Insert, BB, LookupBB});
6023   }
6024 
6025   // Populate the BB that does the lookups.
6026   Builder.SetInsertPoint(LookupBB);
6027 
6028   if (NeedMask) {
6029     // Before doing the lookup, we do the hole check. The LookupBB is therefore
6030     // re-purposed to do the hole check, and we create a new LookupBB.
6031     BasicBlock *MaskBB = LookupBB;
6032     MaskBB->setName("switch.hole_check");
6033     LookupBB = BasicBlock::Create(Mod.getContext(), "switch.lookup",
6034                                   CommonDest->getParent(), CommonDest);
6035 
6036     // Make the mask's bitwidth at least 8-bit and a power-of-2 to avoid
6037     // unnecessary illegal types.
6038     uint64_t TableSizePowOf2 = NextPowerOf2(std::max(7ULL, TableSize - 1ULL));
6039     APInt MaskInt(TableSizePowOf2, 0);
6040     APInt One(TableSizePowOf2, 1);
6041     // Build bitmask; fill in a 1 bit for every case.
6042     const ResultListTy &ResultList = ResultLists[PHIs[0]];
6043     for (size_t I = 0, E = ResultList.size(); I != E; ++I) {
6044       uint64_t Idx = (ResultList[I].first->getValue() - MinCaseVal->getValue())
6045                          .getLimitedValue();
6046       MaskInt |= One << Idx;
6047     }
6048     ConstantInt *TableMask = ConstantInt::get(Mod.getContext(), MaskInt);
6049 
6050     // Get the TableIndex'th bit of the bitmask.
6051     // If this bit is 0 (meaning hole) jump to the default destination,
6052     // else continue with table lookup.
6053     IntegerType *MapTy = TableMask->getType();
6054     Value *MaskIndex =
6055         Builder.CreateZExtOrTrunc(TableIndex, MapTy, "switch.maskindex");
6056     Value *Shifted = Builder.CreateLShr(TableMask, MaskIndex, "switch.shifted");
6057     Value *LoBit = Builder.CreateTrunc(
6058         Shifted, Type::getInt1Ty(Mod.getContext()), "switch.lobit");
6059     Builder.CreateCondBr(LoBit, LookupBB, SI->getDefaultDest());
6060     if (DTU) {
6061       Updates.push_back({DominatorTree::Insert, MaskBB, LookupBB});
6062       Updates.push_back({DominatorTree::Insert, MaskBB, SI->getDefaultDest()});
6063     }
6064     Builder.SetInsertPoint(LookupBB);
6065     AddPredecessorToBlock(SI->getDefaultDest(), MaskBB, BB);
6066   }
6067 
6068   if (!DefaultIsReachable || GeneratingCoveredLookupTable) {
6069     // We cached PHINodes in PHIs. To avoid accessing deleted PHINodes later,
6070     // do not delete PHINodes here.
6071     SI->getDefaultDest()->removePredecessor(BB,
6072                                             /*KeepOneInputPHIs=*/true);
6073     if (DTU)
6074       Updates.push_back({DominatorTree::Delete, BB, SI->getDefaultDest()});
6075   }
6076 
6077   bool ReturnedEarly = false;
6078   for (PHINode *PHI : PHIs) {
6079     const ResultListTy &ResultList = ResultLists[PHI];
6080 
6081     // If using a bitmask, use any value to fill the lookup table holes.
6082     Constant *DV = NeedMask ? ResultLists[PHI][0].second : DefaultResults[PHI];
6083     StringRef FuncName = Fn->getName();
6084     SwitchLookupTable Table(Mod, TableSize, MinCaseVal, ResultList, DV, DL,
6085                             FuncName);
6086 
6087     Value *Result = Table.BuildLookup(TableIndex, Builder);
6088 
6089     // If the result is used to return immediately from the function, we want to
6090     // do that right here.
6091     if (PHI->hasOneUse() && isa<ReturnInst>(*PHI->user_begin()) &&
6092         PHI->user_back() == CommonDest->getFirstNonPHIOrDbg()) {
6093       Builder.CreateRet(Result);
6094       ReturnedEarly = true;
6095       break;
6096     }
6097 
6098     // Do a small peephole optimization: re-use the switch table compare if
6099     // possible.
6100     if (!TableHasHoles && HasDefaultResults && RangeCheckBranch) {
6101       BasicBlock *PhiBlock = PHI->getParent();
6102       // Search for compare instructions which use the phi.
6103       for (auto *User : PHI->users()) {
6104         reuseTableCompare(User, PhiBlock, RangeCheckBranch, DV, ResultList);
6105       }
6106     }
6107 
6108     PHI->addIncoming(Result, LookupBB);
6109   }
6110 
6111   if (!ReturnedEarly) {
6112     Builder.CreateBr(CommonDest);
6113     if (DTU)
6114       Updates.push_back({DominatorTree::Insert, LookupBB, CommonDest});
6115   }
6116 
6117   // Remove the switch.
6118   SmallPtrSet<BasicBlock *, 8> RemovedSuccessors;
6119   for (unsigned i = 0, e = SI->getNumSuccessors(); i < e; ++i) {
6120     BasicBlock *Succ = SI->getSuccessor(i);
6121 
6122     if (Succ == SI->getDefaultDest())
6123       continue;
6124     Succ->removePredecessor(BB);
6125     RemovedSuccessors.insert(Succ);
6126   }
6127   SI->eraseFromParent();
6128 
6129   if (DTU) {
6130     for (BasicBlock *RemovedSuccessor : RemovedSuccessors)
6131       Updates.push_back({DominatorTree::Delete, BB, RemovedSuccessor});
6132     DTU->applyUpdates(Updates);
6133   }
6134 
6135   ++NumLookupTables;
6136   if (NeedMask)
6137     ++NumLookupTablesHoles;
6138   return true;
6139 }
6140 
isSwitchDense(ArrayRef<int64_t> Values)6141 static bool isSwitchDense(ArrayRef<int64_t> Values) {
6142   // See also SelectionDAGBuilder::isDense(), which this function was based on.
6143   uint64_t Diff = (uint64_t)Values.back() - (uint64_t)Values.front();
6144   uint64_t Range = Diff + 1;
6145   uint64_t NumCases = Values.size();
6146   // 40% is the default density for building a jump table in optsize/minsize mode.
6147   uint64_t MinDensity = 40;
6148 
6149   return NumCases * 100 >= Range * MinDensity;
6150 }
6151 
6152 /// Try to transform a switch that has "holes" in it to a contiguous sequence
6153 /// of cases.
6154 ///
6155 /// A switch such as: switch(i) {case 5: case 9: case 13: case 17:} can be
6156 /// range-reduced to: switch ((i-5) / 4) {case 0: case 1: case 2: case 3:}.
6157 ///
6158 /// This converts a sparse switch into a dense switch which allows better
6159 /// lowering and could also allow transforming into a lookup table.
ReduceSwitchRange(SwitchInst * SI,IRBuilder<> & Builder,const DataLayout & DL,const TargetTransformInfo & TTI)6160 static bool ReduceSwitchRange(SwitchInst *SI, IRBuilder<> &Builder,
6161                               const DataLayout &DL,
6162                               const TargetTransformInfo &TTI) {
6163   auto *CondTy = cast<IntegerType>(SI->getCondition()->getType());
6164   if (CondTy->getIntegerBitWidth() > 64 ||
6165       !DL.fitsInLegalInteger(CondTy->getIntegerBitWidth()))
6166     return false;
6167   // Only bother with this optimization if there are more than 3 switch cases;
6168   // SDAG will only bother creating jump tables for 4 or more cases.
6169   if (SI->getNumCases() < 4)
6170     return false;
6171 
6172   // This transform is agnostic to the signedness of the input or case values. We
6173   // can treat the case values as signed or unsigned. We can optimize more common
6174   // cases such as a sequence crossing zero {-4,0,4,8} if we interpret case values
6175   // as signed.
6176   SmallVector<int64_t,4> Values;
6177   for (auto &C : SI->cases())
6178     Values.push_back(C.getCaseValue()->getValue().getSExtValue());
6179   llvm::sort(Values);
6180 
6181   // If the switch is already dense, there's nothing useful to do here.
6182   if (isSwitchDense(Values))
6183     return false;
6184 
6185   // First, transform the values such that they start at zero and ascend.
6186   int64_t Base = Values[0];
6187   for (auto &V : Values)
6188     V -= (uint64_t)(Base);
6189 
6190   // Now we have signed numbers that have been shifted so that, given enough
6191   // precision, there are no negative values. Since the rest of the transform
6192   // is bitwise only, we switch now to an unsigned representation.
6193 
6194   // This transform can be done speculatively because it is so cheap - it
6195   // results in a single rotate operation being inserted.
6196   // FIXME: It's possible that optimizing a switch on powers of two might also
6197   // be beneficial - flag values are often powers of two and we could use a CLZ
6198   // as the key function.
6199 
6200   // countTrailingZeros(0) returns 64. As Values is guaranteed to have more than
6201   // one element and LLVM disallows duplicate cases, Shift is guaranteed to be
6202   // less than 64.
6203   unsigned Shift = 64;
6204   for (auto &V : Values)
6205     Shift = std::min(Shift, countTrailingZeros((uint64_t)V));
6206   assert(Shift < 64);
6207   if (Shift > 0)
6208     for (auto &V : Values)
6209       V = (int64_t)((uint64_t)V >> Shift);
6210 
6211   if (!isSwitchDense(Values))
6212     // Transform didn't create a dense switch.
6213     return false;
6214 
6215   // The obvious transform is to shift the switch condition right and emit a
6216   // check that the condition actually cleanly divided by GCD, i.e.
6217   //   C & (1 << Shift - 1) == 0
6218   // inserting a new CFG edge to handle the case where it didn't divide cleanly.
6219   //
6220   // A cheaper way of doing this is a simple ROTR(C, Shift). This performs the
6221   // shift and puts the shifted-off bits in the uppermost bits. If any of these
6222   // are nonzero then the switch condition will be very large and will hit the
6223   // default case.
6224 
6225   auto *Ty = cast<IntegerType>(SI->getCondition()->getType());
6226   Builder.SetInsertPoint(SI);
6227   auto *ShiftC = ConstantInt::get(Ty, Shift);
6228   auto *Sub = Builder.CreateSub(SI->getCondition(), ConstantInt::get(Ty, Base));
6229   auto *LShr = Builder.CreateLShr(Sub, ShiftC);
6230   auto *Shl = Builder.CreateShl(Sub, Ty->getBitWidth() - Shift);
6231   auto *Rot = Builder.CreateOr(LShr, Shl);
6232   SI->replaceUsesOfWith(SI->getCondition(), Rot);
6233 
6234   for (auto Case : SI->cases()) {
6235     auto *Orig = Case.getCaseValue();
6236     auto Sub = Orig->getValue() - APInt(Ty->getBitWidth(), Base);
6237     Case.setValue(
6238         cast<ConstantInt>(ConstantInt::get(Ty, Sub.lshr(ShiftC->getValue()))));
6239   }
6240   return true;
6241 }
6242 
simplifySwitch(SwitchInst * SI,IRBuilder<> & Builder)6243 bool SimplifyCFGOpt::simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) {
6244   BasicBlock *BB = SI->getParent();
6245 
6246   if (isValueEqualityComparison(SI)) {
6247     // If we only have one predecessor, and if it is a branch on this value,
6248     // see if that predecessor totally determines the outcome of this switch.
6249     if (BasicBlock *OnlyPred = BB->getSinglePredecessor())
6250       if (SimplifyEqualityComparisonWithOnlyPredecessor(SI, OnlyPred, Builder))
6251         return requestResimplify();
6252 
6253     Value *Cond = SI->getCondition();
6254     if (SelectInst *Select = dyn_cast<SelectInst>(Cond))
6255       if (SimplifySwitchOnSelect(SI, Select))
6256         return requestResimplify();
6257 
6258     // If the block only contains the switch, see if we can fold the block
6259     // away into any preds.
6260     if (SI == &*BB->instructionsWithoutDebug().begin())
6261       if (FoldValueComparisonIntoPredecessors(SI, Builder))
6262         return requestResimplify();
6263   }
6264 
6265   // Try to transform the switch into an icmp and a branch.
6266   if (TurnSwitchRangeIntoICmp(SI, Builder))
6267     return requestResimplify();
6268 
6269   // Remove unreachable cases.
6270   if (eliminateDeadSwitchCases(SI, DTU, Options.AC, DL))
6271     return requestResimplify();
6272 
6273   if (switchToSelect(SI, Builder, DTU, DL, TTI))
6274     return requestResimplify();
6275 
6276   if (Options.ForwardSwitchCondToPhi && ForwardSwitchConditionToPHI(SI))
6277     return requestResimplify();
6278 
6279   // The conversion from switch to lookup tables results in difficult-to-analyze
6280   // code and makes pruning branches much harder. This is a problem if the
6281   // switch expression itself can still be restricted as a result of inlining or
6282   // CVP. Therefore, only apply this transformation during late stages of the
6283   // optimisation pipeline.
6284   if (Options.ConvertSwitchToLookupTable &&
6285       SwitchToLookupTable(SI, Builder, DTU, DL, TTI))
6286     return requestResimplify();
6287 
6288   if (ReduceSwitchRange(SI, Builder, DL, TTI))
6289     return requestResimplify();
6290 
6291   return false;
6292 }
6293 
simplifyIndirectBr(IndirectBrInst * IBI)6294 bool SimplifyCFGOpt::simplifyIndirectBr(IndirectBrInst *IBI) {
6295   BasicBlock *BB = IBI->getParent();
6296   bool Changed = false;
6297 
6298   // Eliminate redundant destinations.
6299   SmallPtrSet<Value *, 8> Succs;
6300   SmallPtrSet<BasicBlock *, 8> RemovedSuccs;
6301   for (unsigned i = 0, e = IBI->getNumDestinations(); i != e; ++i) {
6302     BasicBlock *Dest = IBI->getDestination(i);
6303     if (!Dest->hasAddressTaken() || !Succs.insert(Dest).second) {
6304       if (!Dest->hasAddressTaken())
6305         RemovedSuccs.insert(Dest);
6306       Dest->removePredecessor(BB);
6307       IBI->removeDestination(i);
6308       --i;
6309       --e;
6310       Changed = true;
6311     }
6312   }
6313 
6314   if (DTU) {
6315     std::vector<DominatorTree::UpdateType> Updates;
6316     Updates.reserve(RemovedSuccs.size());
6317     for (auto *RemovedSucc : RemovedSuccs)
6318       Updates.push_back({DominatorTree::Delete, BB, RemovedSucc});
6319     DTU->applyUpdates(Updates);
6320   }
6321 
6322   if (IBI->getNumDestinations() == 0) {
6323     // If the indirectbr has no successors, change it to unreachable.
6324     new UnreachableInst(IBI->getContext(), IBI);
6325     EraseTerminatorAndDCECond(IBI);
6326     return true;
6327   }
6328 
6329   if (IBI->getNumDestinations() == 1) {
6330     // If the indirectbr has one successor, change it to a direct branch.
6331     BranchInst::Create(IBI->getDestination(0), IBI);
6332     EraseTerminatorAndDCECond(IBI);
6333     return true;
6334   }
6335 
6336   if (SelectInst *SI = dyn_cast<SelectInst>(IBI->getAddress())) {
6337     if (SimplifyIndirectBrOnSelect(IBI, SI))
6338       return requestResimplify();
6339   }
6340   return Changed;
6341 }
6342 
6343 /// Given an block with only a single landing pad and a unconditional branch
6344 /// try to find another basic block which this one can be merged with.  This
6345 /// handles cases where we have multiple invokes with unique landing pads, but
6346 /// a shared handler.
6347 ///
6348 /// We specifically choose to not worry about merging non-empty blocks
6349 /// here.  That is a PRE/scheduling problem and is best solved elsewhere.  In
6350 /// practice, the optimizer produces empty landing pad blocks quite frequently
6351 /// when dealing with exception dense code.  (see: instcombine, gvn, if-else
6352 /// sinking in this file)
6353 ///
6354 /// This is primarily a code size optimization.  We need to avoid performing
6355 /// any transform which might inhibit optimization (such as our ability to
6356 /// specialize a particular handler via tail commoning).  We do this by not
6357 /// merging any blocks which require us to introduce a phi.  Since the same
6358 /// values are flowing through both blocks, we don't lose any ability to
6359 /// specialize.  If anything, we make such specialization more likely.
6360 ///
6361 /// TODO - This transformation could remove entries from a phi in the target
6362 /// block when the inputs in the phi are the same for the two blocks being
6363 /// merged.  In some cases, this could result in removal of the PHI entirely.
TryToMergeLandingPad(LandingPadInst * LPad,BranchInst * BI,BasicBlock * BB,DomTreeUpdater * DTU)6364 static bool TryToMergeLandingPad(LandingPadInst *LPad, BranchInst *BI,
6365                                  BasicBlock *BB, DomTreeUpdater *DTU) {
6366   auto Succ = BB->getUniqueSuccessor();
6367   assert(Succ);
6368   // If there's a phi in the successor block, we'd likely have to introduce
6369   // a phi into the merged landing pad block.
6370   if (isa<PHINode>(*Succ->begin()))
6371     return false;
6372 
6373   for (BasicBlock *OtherPred : predecessors(Succ)) {
6374     if (BB == OtherPred)
6375       continue;
6376     BasicBlock::iterator I = OtherPred->begin();
6377     LandingPadInst *LPad2 = dyn_cast<LandingPadInst>(I);
6378     if (!LPad2 || !LPad2->isIdenticalTo(LPad))
6379       continue;
6380     for (++I; isa<DbgInfoIntrinsic>(I); ++I)
6381       ;
6382     BranchInst *BI2 = dyn_cast<BranchInst>(I);
6383     if (!BI2 || !BI2->isIdenticalTo(BI))
6384       continue;
6385 
6386     std::vector<DominatorTree::UpdateType> Updates;
6387 
6388     // We've found an identical block.  Update our predecessors to take that
6389     // path instead and make ourselves dead.
6390     SmallPtrSet<BasicBlock *, 16> Preds(pred_begin(BB), pred_end(BB));
6391     for (BasicBlock *Pred : Preds) {
6392       InvokeInst *II = cast<InvokeInst>(Pred->getTerminator());
6393       assert(II->getNormalDest() != BB && II->getUnwindDest() == BB &&
6394              "unexpected successor");
6395       II->setUnwindDest(OtherPred);
6396       if (DTU) {
6397         Updates.push_back({DominatorTree::Insert, Pred, OtherPred});
6398         Updates.push_back({DominatorTree::Delete, Pred, BB});
6399       }
6400     }
6401 
6402     // The debug info in OtherPred doesn't cover the merged control flow that
6403     // used to go through BB.  We need to delete it or update it.
6404     for (auto I = OtherPred->begin(), E = OtherPred->end(); I != E;) {
6405       Instruction &Inst = *I;
6406       I++;
6407       if (isa<DbgInfoIntrinsic>(Inst))
6408         Inst.eraseFromParent();
6409     }
6410 
6411     SmallPtrSet<BasicBlock *, 16> Succs(succ_begin(BB), succ_end(BB));
6412     for (BasicBlock *Succ : Succs) {
6413       Succ->removePredecessor(BB);
6414       if (DTU)
6415         Updates.push_back({DominatorTree::Delete, BB, Succ});
6416     }
6417 
6418     IRBuilder<> Builder(BI);
6419     Builder.CreateUnreachable();
6420     BI->eraseFromParent();
6421     if (DTU)
6422       DTU->applyUpdates(Updates);
6423     return true;
6424   }
6425   return false;
6426 }
6427 
simplifyBranch(BranchInst * Branch,IRBuilder<> & Builder)6428 bool SimplifyCFGOpt::simplifyBranch(BranchInst *Branch, IRBuilder<> &Builder) {
6429   return Branch->isUnconditional() ? simplifyUncondBranch(Branch, Builder)
6430                                    : simplifyCondBranch(Branch, Builder);
6431 }
6432 
simplifyUncondBranch(BranchInst * BI,IRBuilder<> & Builder)6433 bool SimplifyCFGOpt::simplifyUncondBranch(BranchInst *BI,
6434                                           IRBuilder<> &Builder) {
6435   BasicBlock *BB = BI->getParent();
6436   BasicBlock *Succ = BI->getSuccessor(0);
6437 
6438   // If the Terminator is the only non-phi instruction, simplify the block.
6439   // If LoopHeader is provided, check if the block or its successor is a loop
6440   // header. (This is for early invocations before loop simplify and
6441   // vectorization to keep canonical loop forms for nested loops. These blocks
6442   // can be eliminated when the pass is invoked later in the back-end.)
6443   // Note that if BB has only one predecessor then we do not introduce new
6444   // backedge, so we can eliminate BB.
6445   bool NeedCanonicalLoop =
6446       Options.NeedCanonicalLoop &&
6447       (!LoopHeaders.empty() && BB->hasNPredecessorsOrMore(2) &&
6448        (is_contained(LoopHeaders, BB) || is_contained(LoopHeaders, Succ)));
6449   BasicBlock::iterator I = BB->getFirstNonPHIOrDbg(true)->getIterator();
6450   if (I->isTerminator() && BB != &BB->getParent()->getEntryBlock() &&
6451       !NeedCanonicalLoop && TryToSimplifyUncondBranchFromEmptyBlock(BB, DTU))
6452     return true;
6453 
6454   // If the only instruction in the block is a seteq/setne comparison against a
6455   // constant, try to simplify the block.
6456   if (ICmpInst *ICI = dyn_cast<ICmpInst>(I))
6457     if (ICI->isEquality() && isa<ConstantInt>(ICI->getOperand(1))) {
6458       for (++I; isa<DbgInfoIntrinsic>(I); ++I)
6459         ;
6460       if (I->isTerminator() &&
6461           tryToSimplifyUncondBranchWithICmpInIt(ICI, Builder))
6462         return true;
6463     }
6464 
6465   // See if we can merge an empty landing pad block with another which is
6466   // equivalent.
6467   if (LandingPadInst *LPad = dyn_cast<LandingPadInst>(I)) {
6468     for (++I; isa<DbgInfoIntrinsic>(I); ++I)
6469       ;
6470     if (I->isTerminator() && TryToMergeLandingPad(LPad, BI, BB, DTU))
6471       return true;
6472   }
6473 
6474   // If this basic block is ONLY a compare and a branch, and if a predecessor
6475   // branches to us and our successor, fold the comparison into the
6476   // predecessor and use logical operations to update the incoming value
6477   // for PHI nodes in common successor.
6478   if (FoldBranchToCommonDest(BI, DTU, /*MSSAU=*/nullptr, &TTI,
6479                              Options.BonusInstThreshold))
6480     return requestResimplify();
6481   return false;
6482 }
6483 
allPredecessorsComeFromSameSource(BasicBlock * BB)6484 static BasicBlock *allPredecessorsComeFromSameSource(BasicBlock *BB) {
6485   BasicBlock *PredPred = nullptr;
6486   for (auto *P : predecessors(BB)) {
6487     BasicBlock *PPred = P->getSinglePredecessor();
6488     if (!PPred || (PredPred && PredPred != PPred))
6489       return nullptr;
6490     PredPred = PPred;
6491   }
6492   return PredPred;
6493 }
6494 
simplifyCondBranch(BranchInst * BI,IRBuilder<> & Builder)6495 bool SimplifyCFGOpt::simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
6496   BasicBlock *BB = BI->getParent();
6497   if (!Options.SimplifyCondBranch)
6498     return false;
6499 
6500   // Conditional branch
6501   if (isValueEqualityComparison(BI)) {
6502     // If we only have one predecessor, and if it is a branch on this value,
6503     // see if that predecessor totally determines the outcome of this
6504     // switch.
6505     if (BasicBlock *OnlyPred = BB->getSinglePredecessor())
6506       if (SimplifyEqualityComparisonWithOnlyPredecessor(BI, OnlyPred, Builder))
6507         return requestResimplify();
6508 
6509     // This block must be empty, except for the setcond inst, if it exists.
6510     // Ignore dbg and pseudo intrinsics.
6511     auto I = BB->instructionsWithoutDebug(true).begin();
6512     if (&*I == BI) {
6513       if (FoldValueComparisonIntoPredecessors(BI, Builder))
6514         return requestResimplify();
6515     } else if (&*I == cast<Instruction>(BI->getCondition())) {
6516       ++I;
6517       if (&*I == BI && FoldValueComparisonIntoPredecessors(BI, Builder))
6518         return requestResimplify();
6519     }
6520   }
6521 
6522   // Try to turn "br (X == 0 | X == 1), T, F" into a switch instruction.
6523   if (SimplifyBranchOnICmpChain(BI, Builder, DL))
6524     return true;
6525 
6526   // If this basic block has dominating predecessor blocks and the dominating
6527   // blocks' conditions imply BI's condition, we know the direction of BI.
6528   Optional<bool> Imp = isImpliedByDomCondition(BI->getCondition(), BI, DL);
6529   if (Imp) {
6530     // Turn this into a branch on constant.
6531     auto *OldCond = BI->getCondition();
6532     ConstantInt *TorF = *Imp ? ConstantInt::getTrue(BB->getContext())
6533                              : ConstantInt::getFalse(BB->getContext());
6534     BI->setCondition(TorF);
6535     RecursivelyDeleteTriviallyDeadInstructions(OldCond);
6536     return requestResimplify();
6537   }
6538 
6539   // If this basic block is ONLY a compare and a branch, and if a predecessor
6540   // branches to us and one of our successors, fold the comparison into the
6541   // predecessor and use logical operations to pick the right destination.
6542   if (FoldBranchToCommonDest(BI, DTU, /*MSSAU=*/nullptr, &TTI,
6543                              Options.BonusInstThreshold))
6544     return requestResimplify();
6545 
6546   // We have a conditional branch to two blocks that are only reachable
6547   // from BI.  We know that the condbr dominates the two blocks, so see if
6548   // there is any identical code in the "then" and "else" blocks.  If so, we
6549   // can hoist it up to the branching block.
6550   if (BI->getSuccessor(0)->getSinglePredecessor()) {
6551     if (BI->getSuccessor(1)->getSinglePredecessor()) {
6552       if (HoistCommon &&
6553           HoistThenElseCodeToIf(BI, TTI, !Options.HoistCommonInsts))
6554         return requestResimplify();
6555     } else {
6556       // If Successor #1 has multiple preds, we may be able to conditionally
6557       // execute Successor #0 if it branches to Successor #1.
6558       Instruction *Succ0TI = BI->getSuccessor(0)->getTerminator();
6559       if (Succ0TI->getNumSuccessors() == 1 &&
6560           Succ0TI->getSuccessor(0) == BI->getSuccessor(1))
6561         if (SpeculativelyExecuteBB(BI, BI->getSuccessor(0), TTI))
6562           return requestResimplify();
6563     }
6564   } else if (BI->getSuccessor(1)->getSinglePredecessor()) {
6565     // If Successor #0 has multiple preds, we may be able to conditionally
6566     // execute Successor #1 if it branches to Successor #0.
6567     Instruction *Succ1TI = BI->getSuccessor(1)->getTerminator();
6568     if (Succ1TI->getNumSuccessors() == 1 &&
6569         Succ1TI->getSuccessor(0) == BI->getSuccessor(0))
6570       if (SpeculativelyExecuteBB(BI, BI->getSuccessor(1), TTI))
6571         return requestResimplify();
6572   }
6573 
6574   // If this is a branch on a phi node in the current block, thread control
6575   // through this block if any PHI node entries are constants.
6576   if (PHINode *PN = dyn_cast<PHINode>(BI->getCondition()))
6577     if (PN->getParent() == BI->getParent())
6578       if (FoldCondBranchOnPHI(BI, DTU, DL, Options.AC))
6579         return requestResimplify();
6580 
6581   // Scan predecessor blocks for conditional branches.
6582   for (BasicBlock *Pred : predecessors(BB))
6583     if (BranchInst *PBI = dyn_cast<BranchInst>(Pred->getTerminator()))
6584       if (PBI != BI && PBI->isConditional())
6585         if (SimplifyCondBranchToCondBranch(PBI, BI, DTU, DL, TTI))
6586           return requestResimplify();
6587 
6588   // Look for diamond patterns.
6589   if (MergeCondStores)
6590     if (BasicBlock *PrevBB = allPredecessorsComeFromSameSource(BB))
6591       if (BranchInst *PBI = dyn_cast<BranchInst>(PrevBB->getTerminator()))
6592         if (PBI != BI && PBI->isConditional())
6593           if (mergeConditionalStores(PBI, BI, DTU, DL, TTI))
6594             return requestResimplify();
6595 
6596   return false;
6597 }
6598 
6599 /// Check if passing a value to an instruction will cause undefined behavior.
passingValueIsAlwaysUndefined(Value * V,Instruction * I,bool PtrValueMayBeModified)6600 static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified) {
6601   Constant *C = dyn_cast<Constant>(V);
6602   if (!C)
6603     return false;
6604 
6605   if (I->use_empty())
6606     return false;
6607 
6608   if (C->isNullValue() || isa<UndefValue>(C)) {
6609     // Only look at the first use, avoid hurting compile time with long uselists
6610     User *Use = *I->user_begin();
6611 
6612     // Now make sure that there are no instructions in between that can alter
6613     // control flow (eg. calls)
6614     for (BasicBlock::iterator
6615              i = ++BasicBlock::iterator(I),
6616              UI = BasicBlock::iterator(dyn_cast<Instruction>(Use));
6617          i != UI; ++i) {
6618       if (i == I->getParent()->end())
6619         return false;
6620       if (!isGuaranteedToTransferExecutionToSuccessor(&*i))
6621         return false;
6622     }
6623 
6624     // Look through GEPs. A load from a GEP derived from NULL is still undefined
6625     if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Use))
6626       if (GEP->getPointerOperand() == I) {
6627         if (!GEP->isInBounds() || !GEP->hasAllZeroIndices())
6628           PtrValueMayBeModified = true;
6629         return passingValueIsAlwaysUndefined(V, GEP, PtrValueMayBeModified);
6630       }
6631 
6632     // Look through bitcasts.
6633     if (BitCastInst *BC = dyn_cast<BitCastInst>(Use))
6634       return passingValueIsAlwaysUndefined(V, BC, PtrValueMayBeModified);
6635 
6636     // Load from null is undefined.
6637     if (LoadInst *LI = dyn_cast<LoadInst>(Use))
6638       if (!LI->isVolatile())
6639         return !NullPointerIsDefined(LI->getFunction(),
6640                                      LI->getPointerAddressSpace());
6641 
6642     // Store to null is undefined.
6643     if (StoreInst *SI = dyn_cast<StoreInst>(Use))
6644       if (!SI->isVolatile())
6645         return (!NullPointerIsDefined(SI->getFunction(),
6646                                       SI->getPointerAddressSpace())) &&
6647                SI->getPointerOperand() == I;
6648 
6649     if (auto *CB = dyn_cast<CallBase>(Use)) {
6650       if (C->isNullValue() && NullPointerIsDefined(CB->getFunction()))
6651         return false;
6652       // A call to null is undefined.
6653       if (CB->getCalledOperand() == I)
6654         return true;
6655 
6656       if (C->isNullValue()) {
6657         for (const llvm::Use &Arg : CB->args())
6658           if (Arg == I) {
6659             unsigned ArgIdx = CB->getArgOperandNo(&Arg);
6660             if (CB->isPassingUndefUB(ArgIdx) &&
6661                 CB->paramHasAttr(ArgIdx, Attribute::NonNull)) {
6662               // Passing null to a nonnnull+noundef argument is undefined.
6663               return !PtrValueMayBeModified;
6664             }
6665           }
6666       } else if (isa<UndefValue>(C)) {
6667         // Passing undef to a noundef argument is undefined.
6668         for (const llvm::Use &Arg : CB->args())
6669           if (Arg == I) {
6670             unsigned ArgIdx = CB->getArgOperandNo(&Arg);
6671             if (CB->isPassingUndefUB(ArgIdx)) {
6672               // Passing undef to a noundef argument is undefined.
6673               return true;
6674             }
6675           }
6676       }
6677     }
6678   }
6679   return false;
6680 }
6681 
6682 /// If BB has an incoming value that will always trigger undefined behavior
6683 /// (eg. null pointer dereference), remove the branch leading here.
removeUndefIntroducingPredecessor(BasicBlock * BB,DomTreeUpdater * DTU)6684 static bool removeUndefIntroducingPredecessor(BasicBlock *BB,
6685                                               DomTreeUpdater *DTU) {
6686   for (PHINode &PHI : BB->phis())
6687     for (unsigned i = 0, e = PHI.getNumIncomingValues(); i != e; ++i)
6688       if (passingValueIsAlwaysUndefined(PHI.getIncomingValue(i), &PHI)) {
6689         BasicBlock *Predecessor = PHI.getIncomingBlock(i);
6690         Instruction *T = Predecessor->getTerminator();
6691         IRBuilder<> Builder(T);
6692         if (BranchInst *BI = dyn_cast<BranchInst>(T)) {
6693           BB->removePredecessor(Predecessor);
6694           // Turn uncoditional branches into unreachables and remove the dead
6695           // destination from conditional branches.
6696           if (BI->isUnconditional())
6697             Builder.CreateUnreachable();
6698           else
6699             Builder.CreateBr(BI->getSuccessor(0) == BB ? BI->getSuccessor(1)
6700                                                        : BI->getSuccessor(0));
6701           BI->eraseFromParent();
6702           if (DTU)
6703             DTU->applyUpdates({{DominatorTree::Delete, Predecessor, BB}});
6704           return true;
6705         }
6706         // TODO: SwitchInst.
6707       }
6708 
6709   return false;
6710 }
6711 
simplifyOnceImpl(BasicBlock * BB)6712 bool SimplifyCFGOpt::simplifyOnceImpl(BasicBlock *BB) {
6713   bool Changed = false;
6714 
6715   assert(BB && BB->getParent() && "Block not embedded in function!");
6716   assert(BB->getTerminator() && "Degenerate basic block encountered!");
6717 
6718   // Remove basic blocks that have no predecessors (except the entry block)...
6719   // or that just have themself as a predecessor.  These are unreachable.
6720   if ((pred_empty(BB) && BB != &BB->getParent()->getEntryBlock()) ||
6721       BB->getSinglePredecessor() == BB) {
6722     LLVM_DEBUG(dbgs() << "Removing BB: \n" << *BB);
6723     DeleteDeadBlock(BB, DTU);
6724     return true;
6725   }
6726 
6727   // Check to see if we can constant propagate this terminator instruction
6728   // away...
6729   Changed |= ConstantFoldTerminator(BB, /*DeleteDeadConditions=*/true,
6730                                     /*TLI=*/nullptr, DTU);
6731 
6732   // Check for and eliminate duplicate PHI nodes in this block.
6733   Changed |= EliminateDuplicatePHINodes(BB);
6734 
6735   // Check for and remove branches that will always cause undefined behavior.
6736   Changed |= removeUndefIntroducingPredecessor(BB, DTU);
6737 
6738   // Merge basic blocks into their predecessor if there is only one distinct
6739   // pred, and if there is only one distinct successor of the predecessor, and
6740   // if there are no PHI nodes.
6741   if (MergeBlockIntoPredecessor(BB, DTU))
6742     return true;
6743 
6744   if (SinkCommon && Options.SinkCommonInsts)
6745     Changed |= SinkCommonCodeFromPredecessors(BB, DTU);
6746 
6747   IRBuilder<> Builder(BB);
6748 
6749   if (Options.FoldTwoEntryPHINode) {
6750     // If there is a trivial two-entry PHI node in this basic block, and we can
6751     // eliminate it, do so now.
6752     if (auto *PN = dyn_cast<PHINode>(BB->begin()))
6753       if (PN->getNumIncomingValues() == 2)
6754         Changed |= FoldTwoEntryPHINode(PN, TTI, DTU, DL);
6755   }
6756 
6757   Instruction *Terminator = BB->getTerminator();
6758   Builder.SetInsertPoint(Terminator);
6759   switch (Terminator->getOpcode()) {
6760   case Instruction::Br:
6761     Changed |= simplifyBranch(cast<BranchInst>(Terminator), Builder);
6762     break;
6763   case Instruction::Ret:
6764     Changed |= simplifyReturn(cast<ReturnInst>(Terminator), Builder);
6765     break;
6766   case Instruction::Resume:
6767     Changed |= simplifyResume(cast<ResumeInst>(Terminator), Builder);
6768     break;
6769   case Instruction::CleanupRet:
6770     Changed |= simplifyCleanupReturn(cast<CleanupReturnInst>(Terminator));
6771     break;
6772   case Instruction::Switch:
6773     Changed |= simplifySwitch(cast<SwitchInst>(Terminator), Builder);
6774     break;
6775   case Instruction::Unreachable:
6776     Changed |= simplifyUnreachable(cast<UnreachableInst>(Terminator));
6777     break;
6778   case Instruction::IndirectBr:
6779     Changed |= simplifyIndirectBr(cast<IndirectBrInst>(Terminator));
6780     break;
6781   }
6782 
6783   return Changed;
6784 }
6785 
simplifyOnce(BasicBlock * BB)6786 bool SimplifyCFGOpt::simplifyOnce(BasicBlock *BB) {
6787   bool Changed = simplifyOnceImpl(BB);
6788 
6789   return Changed;
6790 }
6791 
run(BasicBlock * BB)6792 bool SimplifyCFGOpt::run(BasicBlock *BB) {
6793   bool Changed = false;
6794 
6795   // Repeated simplify BB as long as resimplification is requested.
6796   do {
6797     Resimplify = false;
6798 
6799     // Perform one round of simplifcation. Resimplify flag will be set if
6800     // another iteration is requested.
6801     Changed |= simplifyOnce(BB);
6802   } while (Resimplify);
6803 
6804   return Changed;
6805 }
6806 
simplifyCFG(BasicBlock * BB,const TargetTransformInfo & TTI,DomTreeUpdater * DTU,const SimplifyCFGOptions & Options,ArrayRef<WeakVH> LoopHeaders)6807 bool llvm::simplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI,
6808                        DomTreeUpdater *DTU, const SimplifyCFGOptions &Options,
6809                        ArrayRef<WeakVH> LoopHeaders) {
6810   return SimplifyCFGOpt(TTI, DTU, BB->getModule()->getDataLayout(), LoopHeaders,
6811                         Options)
6812       .run(BB);
6813 }
6814