10b57cec5SDimitry Andric //===- InstCombineSelect.cpp ----------------------------------------------===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric // 90b57cec5SDimitry Andric // This file implements the visitSelect function. 100b57cec5SDimitry Andric // 110b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 120b57cec5SDimitry Andric 130b57cec5SDimitry Andric #include "InstCombineInternal.h" 140b57cec5SDimitry Andric #include "llvm/ADT/APInt.h" 150b57cec5SDimitry Andric #include "llvm/ADT/STLExtras.h" 160b57cec5SDimitry Andric #include "llvm/ADT/SmallVector.h" 170b57cec5SDimitry Andric #include "llvm/Analysis/AssumptionCache.h" 180b57cec5SDimitry Andric #include "llvm/Analysis/CmpInstAnalysis.h" 190b57cec5SDimitry Andric #include "llvm/Analysis/InstructionSimplify.h" 20fe6060f1SDimitry Andric #include "llvm/Analysis/OverflowInstAnalysis.h" 210b57cec5SDimitry Andric #include "llvm/Analysis/ValueTracking.h" 22bdd1243dSDimitry Andric #include "llvm/Analysis/VectorUtils.h" 230b57cec5SDimitry Andric #include "llvm/IR/BasicBlock.h" 240b57cec5SDimitry Andric #include "llvm/IR/Constant.h" 2581ad6265SDimitry Andric #include "llvm/IR/ConstantRange.h" 260b57cec5SDimitry Andric #include "llvm/IR/Constants.h" 270b57cec5SDimitry Andric #include "llvm/IR/DerivedTypes.h" 280b57cec5SDimitry Andric #include "llvm/IR/IRBuilder.h" 290b57cec5SDimitry Andric #include "llvm/IR/InstrTypes.h" 300b57cec5SDimitry Andric #include "llvm/IR/Instruction.h" 310b57cec5SDimitry Andric #include "llvm/IR/Instructions.h" 320b57cec5SDimitry Andric #include "llvm/IR/IntrinsicInst.h" 330b57cec5SDimitry Andric #include "llvm/IR/Intrinsics.h" 340b57cec5SDimitry Andric #include "llvm/IR/Operator.h" 350b57cec5SDimitry Andric #include "llvm/IR/PatternMatch.h" 360b57cec5SDimitry Andric #include "llvm/IR/Type.h" 370b57cec5SDimitry Andric #include "llvm/IR/User.h" 380b57cec5SDimitry Andric #include "llvm/IR/Value.h" 390b57cec5SDimitry Andric #include "llvm/Support/Casting.h" 400b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h" 410b57cec5SDimitry Andric #include "llvm/Support/KnownBits.h" 42e8d8bef9SDimitry Andric #include "llvm/Transforms/InstCombine/InstCombiner.h" 430b57cec5SDimitry Andric #include <cassert> 440b57cec5SDimitry Andric #include <utility> 450b57cec5SDimitry Andric 46349cc55cSDimitry Andric #define DEBUG_TYPE "instcombine" 47349cc55cSDimitry Andric #include "llvm/Transforms/Utils/InstructionWorklist.h" 48349cc55cSDimitry Andric 490b57cec5SDimitry Andric using namespace llvm; 500b57cec5SDimitry Andric using namespace PatternMatch; 510b57cec5SDimitry Andric 520b57cec5SDimitry Andric 530b57cec5SDimitry Andric /// Replace a select operand based on an equality comparison with the identity 540b57cec5SDimitry Andric /// constant of a binop. 550b57cec5SDimitry Andric static Instruction *foldSelectBinOpIdentity(SelectInst &Sel, 565ffd83dbSDimitry Andric const TargetLibraryInfo &TLI, 57e8d8bef9SDimitry Andric InstCombinerImpl &IC) { 580b57cec5SDimitry Andric // The select condition must be an equality compare with a constant operand. 590b57cec5SDimitry Andric Value *X; 600b57cec5SDimitry Andric Constant *C; 610b57cec5SDimitry Andric CmpInst::Predicate Pred; 620b57cec5SDimitry Andric if (!match(Sel.getCondition(), m_Cmp(Pred, m_Value(X), m_Constant(C)))) 630b57cec5SDimitry Andric return nullptr; 640b57cec5SDimitry Andric 650b57cec5SDimitry Andric bool IsEq; 660b57cec5SDimitry Andric if (ICmpInst::isEquality(Pred)) 670b57cec5SDimitry Andric IsEq = Pred == ICmpInst::ICMP_EQ; 680b57cec5SDimitry Andric else if (Pred == FCmpInst::FCMP_OEQ) 690b57cec5SDimitry Andric IsEq = true; 700b57cec5SDimitry Andric else if (Pred == FCmpInst::FCMP_UNE) 710b57cec5SDimitry Andric IsEq = false; 720b57cec5SDimitry Andric else 730b57cec5SDimitry Andric return nullptr; 740b57cec5SDimitry Andric 750b57cec5SDimitry Andric // A select operand must be a binop. 760b57cec5SDimitry Andric BinaryOperator *BO; 770b57cec5SDimitry Andric if (!match(Sel.getOperand(IsEq ? 1 : 2), m_BinOp(BO))) 780b57cec5SDimitry Andric return nullptr; 790b57cec5SDimitry Andric 800b57cec5SDimitry Andric // The compare constant must be the identity constant for that binop. 810b57cec5SDimitry Andric // If this a floating-point compare with 0.0, any zero constant will do. 820b57cec5SDimitry Andric Type *Ty = BO->getType(); 830b57cec5SDimitry Andric Constant *IdC = ConstantExpr::getBinOpIdentity(BO->getOpcode(), Ty, true); 840b57cec5SDimitry Andric if (IdC != C) { 850b57cec5SDimitry Andric if (!IdC || !CmpInst::isFPPredicate(Pred)) 860b57cec5SDimitry Andric return nullptr; 870b57cec5SDimitry Andric if (!match(IdC, m_AnyZeroFP()) || !match(C, m_AnyZeroFP())) 880b57cec5SDimitry Andric return nullptr; 890b57cec5SDimitry Andric } 900b57cec5SDimitry Andric 910b57cec5SDimitry Andric // Last, match the compare variable operand with a binop operand. 920b57cec5SDimitry Andric Value *Y; 930b57cec5SDimitry Andric if (!BO->isCommutative() && !match(BO, m_BinOp(m_Value(Y), m_Specific(X)))) 940b57cec5SDimitry Andric return nullptr; 950b57cec5SDimitry Andric if (!match(BO, m_c_BinOp(m_Value(Y), m_Specific(X)))) 960b57cec5SDimitry Andric return nullptr; 970b57cec5SDimitry Andric 980b57cec5SDimitry Andric // +0.0 compares equal to -0.0, and so it does not behave as required for this 990b57cec5SDimitry Andric // transform. Bail out if we can not exclude that possibility. 1000b57cec5SDimitry Andric if (isa<FPMathOperator>(BO)) 10106c3fb27SDimitry Andric if (!BO->hasNoSignedZeros() && 102*0fca6ea1SDimitry Andric !cannotBeNegativeZero(Y, 0, 103*0fca6ea1SDimitry Andric IC.getSimplifyQuery().getWithInstruction(&Sel))) 1040b57cec5SDimitry Andric return nullptr; 1050b57cec5SDimitry Andric 1060b57cec5SDimitry Andric // BO = binop Y, X 1070b57cec5SDimitry Andric // S = { select (cmp eq X, C), BO, ? } or { select (cmp ne X, C), ?, BO } 1080b57cec5SDimitry Andric // => 1090b57cec5SDimitry Andric // S = { select (cmp eq X, C), Y, ? } or { select (cmp ne X, C), ?, Y } 1105ffd83dbSDimitry Andric return IC.replaceOperand(Sel, IsEq ? 1 : 2, Y); 1110b57cec5SDimitry Andric } 1120b57cec5SDimitry Andric 1130b57cec5SDimitry Andric /// This folds: 1140b57cec5SDimitry Andric /// select (icmp eq (and X, C1)), TC, FC 1150b57cec5SDimitry Andric /// iff C1 is a power 2 and the difference between TC and FC is a power-of-2. 1160b57cec5SDimitry Andric /// To something like: 1170b57cec5SDimitry Andric /// (shr (and (X, C1)), (log2(C1) - log2(TC-FC))) + FC 1180b57cec5SDimitry Andric /// Or: 1190b57cec5SDimitry Andric /// (shl (and (X, C1)), (log2(TC-FC) - log2(C1))) + FC 1200b57cec5SDimitry Andric /// With some variations depending if FC is larger than TC, or the shift 1210b57cec5SDimitry Andric /// isn't needed, or the bit widths don't match. 1220b57cec5SDimitry Andric static Value *foldSelectICmpAnd(SelectInst &Sel, ICmpInst *Cmp, 1230b57cec5SDimitry Andric InstCombiner::BuilderTy &Builder) { 1240b57cec5SDimitry Andric const APInt *SelTC, *SelFC; 1250b57cec5SDimitry Andric if (!match(Sel.getTrueValue(), m_APInt(SelTC)) || 1260b57cec5SDimitry Andric !match(Sel.getFalseValue(), m_APInt(SelFC))) 1270b57cec5SDimitry Andric return nullptr; 1280b57cec5SDimitry Andric 1290b57cec5SDimitry Andric // If this is a vector select, we need a vector compare. 1300b57cec5SDimitry Andric Type *SelType = Sel.getType(); 1310b57cec5SDimitry Andric if (SelType->isVectorTy() != Cmp->getType()->isVectorTy()) 1320b57cec5SDimitry Andric return nullptr; 1330b57cec5SDimitry Andric 1340b57cec5SDimitry Andric Value *V; 1350b57cec5SDimitry Andric APInt AndMask; 1360b57cec5SDimitry Andric bool CreateAnd = false; 1370b57cec5SDimitry Andric ICmpInst::Predicate Pred = Cmp->getPredicate(); 1380b57cec5SDimitry Andric if (ICmpInst::isEquality(Pred)) { 1390b57cec5SDimitry Andric if (!match(Cmp->getOperand(1), m_Zero())) 1400b57cec5SDimitry Andric return nullptr; 1410b57cec5SDimitry Andric 1420b57cec5SDimitry Andric V = Cmp->getOperand(0); 1430b57cec5SDimitry Andric const APInt *AndRHS; 1440b57cec5SDimitry Andric if (!match(V, m_And(m_Value(), m_Power2(AndRHS)))) 1450b57cec5SDimitry Andric return nullptr; 1460b57cec5SDimitry Andric 1470b57cec5SDimitry Andric AndMask = *AndRHS; 1480b57cec5SDimitry Andric } else if (decomposeBitTestICmp(Cmp->getOperand(0), Cmp->getOperand(1), 1490b57cec5SDimitry Andric Pred, V, AndMask)) { 1500b57cec5SDimitry Andric assert(ICmpInst::isEquality(Pred) && "Not equality test?"); 1510b57cec5SDimitry Andric if (!AndMask.isPowerOf2()) 1520b57cec5SDimitry Andric return nullptr; 1530b57cec5SDimitry Andric 1540b57cec5SDimitry Andric CreateAnd = true; 1550b57cec5SDimitry Andric } else { 1560b57cec5SDimitry Andric return nullptr; 1570b57cec5SDimitry Andric } 1580b57cec5SDimitry Andric 1590b57cec5SDimitry Andric // In general, when both constants are non-zero, we would need an offset to 1600b57cec5SDimitry Andric // replace the select. This would require more instructions than we started 1610b57cec5SDimitry Andric // with. But there's one special-case that we handle here because it can 1620b57cec5SDimitry Andric // simplify/reduce the instructions. 1630b57cec5SDimitry Andric APInt TC = *SelTC; 1640b57cec5SDimitry Andric APInt FC = *SelFC; 165349cc55cSDimitry Andric if (!TC.isZero() && !FC.isZero()) { 1660b57cec5SDimitry Andric // If the select constants differ by exactly one bit and that's the same 1670b57cec5SDimitry Andric // bit that is masked and checked by the select condition, the select can 1680b57cec5SDimitry Andric // be replaced by bitwise logic to set/clear one bit of the constant result. 1690b57cec5SDimitry Andric if (TC.getBitWidth() != AndMask.getBitWidth() || (TC ^ FC) != AndMask) 1700b57cec5SDimitry Andric return nullptr; 1710b57cec5SDimitry Andric if (CreateAnd) { 1720b57cec5SDimitry Andric // If we have to create an 'and', then we must kill the cmp to not 1730b57cec5SDimitry Andric // increase the instruction count. 1740b57cec5SDimitry Andric if (!Cmp->hasOneUse()) 1750b57cec5SDimitry Andric return nullptr; 1760b57cec5SDimitry Andric V = Builder.CreateAnd(V, ConstantInt::get(SelType, AndMask)); 1770b57cec5SDimitry Andric } 1780b57cec5SDimitry Andric bool ExtraBitInTC = TC.ugt(FC); 1790b57cec5SDimitry Andric if (Pred == ICmpInst::ICMP_EQ) { 1800b57cec5SDimitry Andric // If the masked bit in V is clear, clear or set the bit in the result: 1810b57cec5SDimitry Andric // (V & AndMaskC) == 0 ? TC : FC --> (V & AndMaskC) ^ TC 1820b57cec5SDimitry Andric // (V & AndMaskC) == 0 ? TC : FC --> (V & AndMaskC) | TC 1830b57cec5SDimitry Andric Constant *C = ConstantInt::get(SelType, TC); 1840b57cec5SDimitry Andric return ExtraBitInTC ? Builder.CreateXor(V, C) : Builder.CreateOr(V, C); 1850b57cec5SDimitry Andric } 1860b57cec5SDimitry Andric if (Pred == ICmpInst::ICMP_NE) { 1870b57cec5SDimitry Andric // If the masked bit in V is set, set or clear the bit in the result: 1880b57cec5SDimitry Andric // (V & AndMaskC) != 0 ? TC : FC --> (V & AndMaskC) | FC 1890b57cec5SDimitry Andric // (V & AndMaskC) != 0 ? TC : FC --> (V & AndMaskC) ^ FC 1900b57cec5SDimitry Andric Constant *C = ConstantInt::get(SelType, FC); 1910b57cec5SDimitry Andric return ExtraBitInTC ? Builder.CreateOr(V, C) : Builder.CreateXor(V, C); 1920b57cec5SDimitry Andric } 1930b57cec5SDimitry Andric llvm_unreachable("Only expecting equality predicates"); 1940b57cec5SDimitry Andric } 1950b57cec5SDimitry Andric 1960b57cec5SDimitry Andric // Make sure one of the select arms is a power-of-2. 1970b57cec5SDimitry Andric if (!TC.isPowerOf2() && !FC.isPowerOf2()) 1980b57cec5SDimitry Andric return nullptr; 1990b57cec5SDimitry Andric 2000b57cec5SDimitry Andric // Determine which shift is needed to transform result of the 'and' into the 2010b57cec5SDimitry Andric // desired result. 202349cc55cSDimitry Andric const APInt &ValC = !TC.isZero() ? TC : FC; 2030b57cec5SDimitry Andric unsigned ValZeros = ValC.logBase2(); 2040b57cec5SDimitry Andric unsigned AndZeros = AndMask.logBase2(); 205*0fca6ea1SDimitry Andric bool ShouldNotVal = !TC.isZero(); 206*0fca6ea1SDimitry Andric ShouldNotVal ^= Pred == ICmpInst::ICMP_NE; 207*0fca6ea1SDimitry Andric 208*0fca6ea1SDimitry Andric // If we would need to create an 'and' + 'shift' + 'xor' to replace a 'select' 209*0fca6ea1SDimitry Andric // + 'icmp', then this transformation would result in more instructions and 210*0fca6ea1SDimitry Andric // potentially interfere with other folding. 211*0fca6ea1SDimitry Andric if (CreateAnd && ShouldNotVal && ValZeros != AndZeros) 212*0fca6ea1SDimitry Andric return nullptr; 2130b57cec5SDimitry Andric 2140b57cec5SDimitry Andric // Insert the 'and' instruction on the input to the truncate. 2150b57cec5SDimitry Andric if (CreateAnd) 2160b57cec5SDimitry Andric V = Builder.CreateAnd(V, ConstantInt::get(V->getType(), AndMask)); 2170b57cec5SDimitry Andric 2180b57cec5SDimitry Andric // If types don't match, we can still convert the select by introducing a zext 2190b57cec5SDimitry Andric // or a trunc of the 'and'. 2200b57cec5SDimitry Andric if (ValZeros > AndZeros) { 2210b57cec5SDimitry Andric V = Builder.CreateZExtOrTrunc(V, SelType); 2220b57cec5SDimitry Andric V = Builder.CreateShl(V, ValZeros - AndZeros); 2230b57cec5SDimitry Andric } else if (ValZeros < AndZeros) { 2240b57cec5SDimitry Andric V = Builder.CreateLShr(V, AndZeros - ValZeros); 2250b57cec5SDimitry Andric V = Builder.CreateZExtOrTrunc(V, SelType); 2260b57cec5SDimitry Andric } else { 2270b57cec5SDimitry Andric V = Builder.CreateZExtOrTrunc(V, SelType); 2280b57cec5SDimitry Andric } 2290b57cec5SDimitry Andric 2300b57cec5SDimitry Andric // Okay, now we know that everything is set up, we just don't know whether we 2310b57cec5SDimitry Andric // have a icmp_ne or icmp_eq and whether the true or false val is the zero. 2320b57cec5SDimitry Andric if (ShouldNotVal) 2330b57cec5SDimitry Andric V = Builder.CreateXor(V, ValC); 2340b57cec5SDimitry Andric 2350b57cec5SDimitry Andric return V; 2360b57cec5SDimitry Andric } 2370b57cec5SDimitry Andric 2380b57cec5SDimitry Andric /// We want to turn code that looks like this: 2390b57cec5SDimitry Andric /// %C = or %A, %B 2400b57cec5SDimitry Andric /// %D = select %cond, %C, %A 2410b57cec5SDimitry Andric /// into: 2420b57cec5SDimitry Andric /// %C = select %cond, %B, 0 2430b57cec5SDimitry Andric /// %D = or %A, %C 2440b57cec5SDimitry Andric /// 2450b57cec5SDimitry Andric /// Assuming that the specified instruction is an operand to the select, return 2460b57cec5SDimitry Andric /// a bitmask indicating which operands of this instruction are foldable if they 2470b57cec5SDimitry Andric /// equal the other incoming value of the select. 2480b57cec5SDimitry Andric static unsigned getSelectFoldableOperands(BinaryOperator *I) { 2490b57cec5SDimitry Andric switch (I->getOpcode()) { 2500b57cec5SDimitry Andric case Instruction::Add: 2514824e7fdSDimitry Andric case Instruction::FAdd: 2520b57cec5SDimitry Andric case Instruction::Mul: 2534824e7fdSDimitry Andric case Instruction::FMul: 2540b57cec5SDimitry Andric case Instruction::And: 2550b57cec5SDimitry Andric case Instruction::Or: 2560b57cec5SDimitry Andric case Instruction::Xor: 2570b57cec5SDimitry Andric return 3; // Can fold through either operand. 2580b57cec5SDimitry Andric case Instruction::Sub: // Can only fold on the amount subtracted. 2594824e7fdSDimitry Andric case Instruction::FSub: 2604824e7fdSDimitry Andric case Instruction::FDiv: // Can only fold on the divisor amount. 2610b57cec5SDimitry Andric case Instruction::Shl: // Can only fold on the shift amount. 2620b57cec5SDimitry Andric case Instruction::LShr: 2630b57cec5SDimitry Andric case Instruction::AShr: 2640b57cec5SDimitry Andric return 1; 2650b57cec5SDimitry Andric default: 2660b57cec5SDimitry Andric return 0; // Cannot fold 2670b57cec5SDimitry Andric } 2680b57cec5SDimitry Andric } 2690b57cec5SDimitry Andric 2700b57cec5SDimitry Andric /// We have (select c, TI, FI), and we know that TI and FI have the same opcode. 271e8d8bef9SDimitry Andric Instruction *InstCombinerImpl::foldSelectOpOp(SelectInst &SI, Instruction *TI, 2720b57cec5SDimitry Andric Instruction *FI) { 2730b57cec5SDimitry Andric // Don't break up min/max patterns. The hasOneUse checks below prevent that 2740b57cec5SDimitry Andric // for most cases, but vector min/max with bitcasts can be transformed. If the 2750b57cec5SDimitry Andric // one-use restrictions are eased for other patterns, we still don't want to 2760b57cec5SDimitry Andric // obfuscate min/max. 2770b57cec5SDimitry Andric if ((match(&SI, m_SMin(m_Value(), m_Value())) || 2780b57cec5SDimitry Andric match(&SI, m_SMax(m_Value(), m_Value())) || 2790b57cec5SDimitry Andric match(&SI, m_UMin(m_Value(), m_Value())) || 2800b57cec5SDimitry Andric match(&SI, m_UMax(m_Value(), m_Value())))) 2810b57cec5SDimitry Andric return nullptr; 2820b57cec5SDimitry Andric 2830b57cec5SDimitry Andric // If this is a cast from the same type, merge. 2840b57cec5SDimitry Andric Value *Cond = SI.getCondition(); 2850b57cec5SDimitry Andric Type *CondTy = Cond->getType(); 2860b57cec5SDimitry Andric if (TI->getNumOperands() == 1 && TI->isCast()) { 2870b57cec5SDimitry Andric Type *FIOpndTy = FI->getOperand(0)->getType(); 2880b57cec5SDimitry Andric if (TI->getOperand(0)->getType() != FIOpndTy) 2890b57cec5SDimitry Andric return nullptr; 2900b57cec5SDimitry Andric 2910b57cec5SDimitry Andric // The select condition may be a vector. We may only change the operand 2920b57cec5SDimitry Andric // type if the vector width remains the same (and matches the condition). 2935ffd83dbSDimitry Andric if (auto *CondVTy = dyn_cast<VectorType>(CondTy)) { 294e8d8bef9SDimitry Andric if (!FIOpndTy->isVectorTy() || 295e8d8bef9SDimitry Andric CondVTy->getElementCount() != 296e8d8bef9SDimitry Andric cast<VectorType>(FIOpndTy)->getElementCount()) 2970b57cec5SDimitry Andric return nullptr; 2980b57cec5SDimitry Andric 2990b57cec5SDimitry Andric // TODO: If the backend knew how to deal with casts better, we could 3000b57cec5SDimitry Andric // remove this limitation. For now, there's too much potential to create 3010b57cec5SDimitry Andric // worse codegen by promoting the select ahead of size-altering casts 3020b57cec5SDimitry Andric // (PR28160). 3030b57cec5SDimitry Andric // 3040b57cec5SDimitry Andric // Note that ValueTracking's matchSelectPattern() looks through casts 3050b57cec5SDimitry Andric // without checking 'hasOneUse' when it matches min/max patterns, so this 3060b57cec5SDimitry Andric // transform may end up happening anyway. 3070b57cec5SDimitry Andric if (TI->getOpcode() != Instruction::BitCast && 3080b57cec5SDimitry Andric (!TI->hasOneUse() || !FI->hasOneUse())) 3090b57cec5SDimitry Andric return nullptr; 3100b57cec5SDimitry Andric } else if (!TI->hasOneUse() || !FI->hasOneUse()) { 3110b57cec5SDimitry Andric // TODO: The one-use restrictions for a scalar select could be eased if 3120b57cec5SDimitry Andric // the fold of a select in visitLoadInst() was enhanced to match a pattern 3130b57cec5SDimitry Andric // that includes a cast. 3140b57cec5SDimitry Andric return nullptr; 3150b57cec5SDimitry Andric } 3160b57cec5SDimitry Andric 3170b57cec5SDimitry Andric // Fold this by inserting a select from the input values. 3180b57cec5SDimitry Andric Value *NewSI = 3190b57cec5SDimitry Andric Builder.CreateSelect(Cond, TI->getOperand(0), FI->getOperand(0), 3200b57cec5SDimitry Andric SI.getName() + ".v", &SI); 3210b57cec5SDimitry Andric return CastInst::Create(Instruction::CastOps(TI->getOpcode()), NewSI, 3220b57cec5SDimitry Andric TI->getType()); 3230b57cec5SDimitry Andric } 3240b57cec5SDimitry Andric 325bdd1243dSDimitry Andric Value *OtherOpT, *OtherOpF; 326bdd1243dSDimitry Andric bool MatchIsOpZero; 327bdd1243dSDimitry Andric auto getCommonOp = [&](Instruction *TI, Instruction *FI, bool Commute, 328bdd1243dSDimitry Andric bool Swapped = false) -> Value * { 329bdd1243dSDimitry Andric assert(!(Commute && Swapped) && 330bdd1243dSDimitry Andric "Commute and Swapped can't set at the same time"); 331bdd1243dSDimitry Andric if (!Swapped) { 332bdd1243dSDimitry Andric if (TI->getOperand(0) == FI->getOperand(0)) { 333bdd1243dSDimitry Andric OtherOpT = TI->getOperand(1); 334bdd1243dSDimitry Andric OtherOpF = FI->getOperand(1); 335bdd1243dSDimitry Andric MatchIsOpZero = true; 336bdd1243dSDimitry Andric return TI->getOperand(0); 337bdd1243dSDimitry Andric } else if (TI->getOperand(1) == FI->getOperand(1)) { 338bdd1243dSDimitry Andric OtherOpT = TI->getOperand(0); 339bdd1243dSDimitry Andric OtherOpF = FI->getOperand(0); 340bdd1243dSDimitry Andric MatchIsOpZero = false; 341bdd1243dSDimitry Andric return TI->getOperand(1); 342bdd1243dSDimitry Andric } 343bdd1243dSDimitry Andric } 344bdd1243dSDimitry Andric 345bdd1243dSDimitry Andric if (!Commute && !Swapped) 346bdd1243dSDimitry Andric return nullptr; 347bdd1243dSDimitry Andric 348bdd1243dSDimitry Andric // If we are allowing commute or swap of operands, then 349bdd1243dSDimitry Andric // allow a cross-operand match. In that case, MatchIsOpZero 350bdd1243dSDimitry Andric // means that TI's operand 0 (FI's operand 1) is the common op. 351bdd1243dSDimitry Andric if (TI->getOperand(0) == FI->getOperand(1)) { 352bdd1243dSDimitry Andric OtherOpT = TI->getOperand(1); 353bdd1243dSDimitry Andric OtherOpF = FI->getOperand(0); 354bdd1243dSDimitry Andric MatchIsOpZero = true; 355bdd1243dSDimitry Andric return TI->getOperand(0); 356bdd1243dSDimitry Andric } else if (TI->getOperand(1) == FI->getOperand(0)) { 357bdd1243dSDimitry Andric OtherOpT = TI->getOperand(0); 358bdd1243dSDimitry Andric OtherOpF = FI->getOperand(1); 359bdd1243dSDimitry Andric MatchIsOpZero = false; 360bdd1243dSDimitry Andric return TI->getOperand(1); 361bdd1243dSDimitry Andric } 362bdd1243dSDimitry Andric return nullptr; 363bdd1243dSDimitry Andric }; 364bdd1243dSDimitry Andric 365bdd1243dSDimitry Andric if (TI->hasOneUse() || FI->hasOneUse()) { 3660b57cec5SDimitry Andric // Cond ? -X : -Y --> -(Cond ? X : Y) 3670b57cec5SDimitry Andric Value *X, *Y; 368bdd1243dSDimitry Andric if (match(TI, m_FNeg(m_Value(X))) && match(FI, m_FNeg(m_Value(Y)))) { 369bdd1243dSDimitry Andric // Intersect FMF from the fneg instructions and union those with the 370bdd1243dSDimitry Andric // select. 371349cc55cSDimitry Andric FastMathFlags FMF = TI->getFastMathFlags(); 372349cc55cSDimitry Andric FMF &= FI->getFastMathFlags(); 373349cc55cSDimitry Andric FMF |= SI.getFastMathFlags(); 374bdd1243dSDimitry Andric Value *NewSel = 375bdd1243dSDimitry Andric Builder.CreateSelect(Cond, X, Y, SI.getName() + ".v", &SI); 376349cc55cSDimitry Andric if (auto *NewSelI = dyn_cast<Instruction>(NewSel)) 377349cc55cSDimitry Andric NewSelI->setFastMathFlags(FMF); 378349cc55cSDimitry Andric Instruction *NewFNeg = UnaryOperator::CreateFNeg(NewSel); 379349cc55cSDimitry Andric NewFNeg->setFastMathFlags(FMF); 380349cc55cSDimitry Andric return NewFNeg; 3810b57cec5SDimitry Andric } 3820b57cec5SDimitry Andric 383bdd1243dSDimitry Andric // Min/max intrinsic with a common operand can have the common operand 384bdd1243dSDimitry Andric // pulled after the select. This is the same transform as below for binops, 385bdd1243dSDimitry Andric // but specialized for intrinsic matching and without the restrictive uses 386bdd1243dSDimitry Andric // clause. 387fe6060f1SDimitry Andric auto *TII = dyn_cast<IntrinsicInst>(TI); 388fe6060f1SDimitry Andric auto *FII = dyn_cast<IntrinsicInst>(FI); 389bdd1243dSDimitry Andric if (TII && FII && TII->getIntrinsicID() == FII->getIntrinsicID()) { 390bdd1243dSDimitry Andric if (match(TII, m_MaxOrMin(m_Value(), m_Value()))) { 391bdd1243dSDimitry Andric if (Value *MatchOp = getCommonOp(TI, FI, true)) { 392bdd1243dSDimitry Andric Value *NewSel = 393bdd1243dSDimitry Andric Builder.CreateSelect(Cond, OtherOpT, OtherOpF, "minmaxop", &SI); 394bdd1243dSDimitry Andric return CallInst::Create(TII->getCalledFunction(), {NewSel, MatchOp}); 395fe6060f1SDimitry Andric } 396fe6060f1SDimitry Andric } 39706c3fb27SDimitry Andric 39806c3fb27SDimitry Andric // select c, (ldexp v, e0), (ldexp v, e1) -> ldexp v, (select c, e0, e1) 39906c3fb27SDimitry Andric // select c, (ldexp v0, e), (ldexp v1, e) -> ldexp (select c, v0, v1), e 40006c3fb27SDimitry Andric // 40106c3fb27SDimitry Andric // select c, (ldexp v0, e0), (ldexp v1, e1) -> 40206c3fb27SDimitry Andric // ldexp (select c, v0, v1), (select c, e0, e1) 40306c3fb27SDimitry Andric if (TII->getIntrinsicID() == Intrinsic::ldexp) { 40406c3fb27SDimitry Andric Value *LdexpVal0 = TII->getArgOperand(0); 40506c3fb27SDimitry Andric Value *LdexpExp0 = TII->getArgOperand(1); 40606c3fb27SDimitry Andric Value *LdexpVal1 = FII->getArgOperand(0); 40706c3fb27SDimitry Andric Value *LdexpExp1 = FII->getArgOperand(1); 40806c3fb27SDimitry Andric if (LdexpExp0->getType() == LdexpExp1->getType()) { 40906c3fb27SDimitry Andric FPMathOperator *SelectFPOp = cast<FPMathOperator>(&SI); 41006c3fb27SDimitry Andric FastMathFlags FMF = cast<FPMathOperator>(TII)->getFastMathFlags(); 41106c3fb27SDimitry Andric FMF &= cast<FPMathOperator>(FII)->getFastMathFlags(); 41206c3fb27SDimitry Andric FMF |= SelectFPOp->getFastMathFlags(); 41306c3fb27SDimitry Andric 41406c3fb27SDimitry Andric Value *SelectVal = Builder.CreateSelect(Cond, LdexpVal0, LdexpVal1); 41506c3fb27SDimitry Andric Value *SelectExp = Builder.CreateSelect(Cond, LdexpExp0, LdexpExp1); 41606c3fb27SDimitry Andric 41706c3fb27SDimitry Andric CallInst *NewLdexp = Builder.CreateIntrinsic( 41806c3fb27SDimitry Andric TII->getType(), Intrinsic::ldexp, {SelectVal, SelectExp}); 41906c3fb27SDimitry Andric NewLdexp->setFastMathFlags(FMF); 42006c3fb27SDimitry Andric return replaceInstUsesWith(SI, NewLdexp); 42106c3fb27SDimitry Andric } 42206c3fb27SDimitry Andric } 423fe6060f1SDimitry Andric } 424bdd1243dSDimitry Andric 425bdd1243dSDimitry Andric // icmp with a common operand also can have the common operand 426bdd1243dSDimitry Andric // pulled after the select. 427bdd1243dSDimitry Andric ICmpInst::Predicate TPred, FPred; 428bdd1243dSDimitry Andric if (match(TI, m_ICmp(TPred, m_Value(), m_Value())) && 429bdd1243dSDimitry Andric match(FI, m_ICmp(FPred, m_Value(), m_Value()))) { 430bdd1243dSDimitry Andric if (TPred == FPred || TPred == CmpInst::getSwappedPredicate(FPred)) { 431bdd1243dSDimitry Andric bool Swapped = TPred != FPred; 432bdd1243dSDimitry Andric if (Value *MatchOp = 433bdd1243dSDimitry Andric getCommonOp(TI, FI, ICmpInst::isEquality(TPred), Swapped)) { 434bdd1243dSDimitry Andric Value *NewSel = Builder.CreateSelect(Cond, OtherOpT, OtherOpF, 435bdd1243dSDimitry Andric SI.getName() + ".v", &SI); 436bdd1243dSDimitry Andric return new ICmpInst( 437bdd1243dSDimitry Andric MatchIsOpZero ? TPred : CmpInst::getSwappedPredicate(TPred), 438bdd1243dSDimitry Andric MatchOp, NewSel); 439bdd1243dSDimitry Andric } 440fe6060f1SDimitry Andric } 441fe6060f1SDimitry Andric } 442fe6060f1SDimitry Andric } 443fe6060f1SDimitry Andric 4440b57cec5SDimitry Andric // Only handle binary operators (including two-operand getelementptr) with 4450b57cec5SDimitry Andric // one-use here. As with the cast case above, it may be possible to relax the 4460b57cec5SDimitry Andric // one-use constraint, but that needs be examined carefully since it may not 4470b57cec5SDimitry Andric // reduce the total number of instructions. 4480b57cec5SDimitry Andric if (TI->getNumOperands() != 2 || FI->getNumOperands() != 2 || 44981ad6265SDimitry Andric !TI->isSameOperationAs(FI) || 4500b57cec5SDimitry Andric (!isa<BinaryOperator>(TI) && !isa<GetElementPtrInst>(TI)) || 4510b57cec5SDimitry Andric !TI->hasOneUse() || !FI->hasOneUse()) 4520b57cec5SDimitry Andric return nullptr; 4530b57cec5SDimitry Andric 4540b57cec5SDimitry Andric // Figure out if the operations have any operands in common. 455bdd1243dSDimitry Andric Value *MatchOp = getCommonOp(TI, FI, TI->isCommutative()); 456bdd1243dSDimitry Andric if (!MatchOp) 4570b57cec5SDimitry Andric return nullptr; 4580b57cec5SDimitry Andric 4590b57cec5SDimitry Andric // If the select condition is a vector, the operands of the original select's 4600b57cec5SDimitry Andric // operands also must be vectors. This may not be the case for getelementptr 4610b57cec5SDimitry Andric // for example. 4620b57cec5SDimitry Andric if (CondTy->isVectorTy() && (!OtherOpT->getType()->isVectorTy() || 4630b57cec5SDimitry Andric !OtherOpF->getType()->isVectorTy())) 4640b57cec5SDimitry Andric return nullptr; 4650b57cec5SDimitry Andric 46606c3fb27SDimitry Andric // If we are sinking div/rem after a select, we may need to freeze the 46706c3fb27SDimitry Andric // condition because div/rem may induce immediate UB with a poison operand. 46806c3fb27SDimitry Andric // For example, the following transform is not safe if Cond can ever be poison 46906c3fb27SDimitry Andric // because we can replace poison with zero and then we have div-by-zero that 47006c3fb27SDimitry Andric // didn't exist in the original code: 47106c3fb27SDimitry Andric // Cond ? x/y : x/z --> x / (Cond ? y : z) 47206c3fb27SDimitry Andric auto *BO = dyn_cast<BinaryOperator>(TI); 47306c3fb27SDimitry Andric if (BO && BO->isIntDivRem() && !isGuaranteedNotToBePoison(Cond)) { 47406c3fb27SDimitry Andric // A udiv/urem with a common divisor is safe because UB can only occur with 47506c3fb27SDimitry Andric // div-by-zero, and that would be present in the original code. 47606c3fb27SDimitry Andric if (BO->getOpcode() == Instruction::SDiv || 47706c3fb27SDimitry Andric BO->getOpcode() == Instruction::SRem || MatchIsOpZero) 47806c3fb27SDimitry Andric Cond = Builder.CreateFreeze(Cond); 47906c3fb27SDimitry Andric } 48006c3fb27SDimitry Andric 4810b57cec5SDimitry Andric // If we reach here, they do have operations in common. 4820b57cec5SDimitry Andric Value *NewSI = Builder.CreateSelect(Cond, OtherOpT, OtherOpF, 4830b57cec5SDimitry Andric SI.getName() + ".v", &SI); 4840b57cec5SDimitry Andric Value *Op0 = MatchIsOpZero ? MatchOp : NewSI; 4850b57cec5SDimitry Andric Value *Op1 = MatchIsOpZero ? NewSI : MatchOp; 4860b57cec5SDimitry Andric if (auto *BO = dyn_cast<BinaryOperator>(TI)) { 4870b57cec5SDimitry Andric BinaryOperator *NewBO = BinaryOperator::Create(BO->getOpcode(), Op0, Op1); 4880b57cec5SDimitry Andric NewBO->copyIRFlags(TI); 4890b57cec5SDimitry Andric NewBO->andIRFlags(FI); 4900b57cec5SDimitry Andric return NewBO; 4910b57cec5SDimitry Andric } 4920b57cec5SDimitry Andric if (auto *TGEP = dyn_cast<GetElementPtrInst>(TI)) { 4930b57cec5SDimitry Andric auto *FGEP = cast<GetElementPtrInst>(FI); 494*0fca6ea1SDimitry Andric Type *ElementType = TGEP->getSourceElementType(); 495*0fca6ea1SDimitry Andric return GetElementPtrInst::Create( 496*0fca6ea1SDimitry Andric ElementType, Op0, Op1, TGEP->getNoWrapFlags() & FGEP->getNoWrapFlags()); 4970b57cec5SDimitry Andric } 4980b57cec5SDimitry Andric llvm_unreachable("Expected BinaryOperator or GEP"); 4990b57cec5SDimitry Andric return nullptr; 5000b57cec5SDimitry Andric } 5010b57cec5SDimitry Andric 5020b57cec5SDimitry Andric static bool isSelect01(const APInt &C1I, const APInt &C2I) { 503349cc55cSDimitry Andric if (!C1I.isZero() && !C2I.isZero()) // One side must be zero. 5040b57cec5SDimitry Andric return false; 505349cc55cSDimitry Andric return C1I.isOne() || C1I.isAllOnes() || C2I.isOne() || C2I.isAllOnes(); 5060b57cec5SDimitry Andric } 5070b57cec5SDimitry Andric 5080b57cec5SDimitry Andric /// Try to fold the select into one of the operands to allow further 5090b57cec5SDimitry Andric /// optimization. 510e8d8bef9SDimitry Andric Instruction *InstCombinerImpl::foldSelectIntoOp(SelectInst &SI, Value *TrueVal, 5110b57cec5SDimitry Andric Value *FalseVal) { 51206c3fb27SDimitry Andric // See the comment above getSelectFoldableOperands for a description of the 5130b57cec5SDimitry Andric // transformation we are doing here. 51481ad6265SDimitry Andric auto TryFoldSelectIntoOp = [&](SelectInst &SI, Value *TrueVal, 51581ad6265SDimitry Andric Value *FalseVal, 51681ad6265SDimitry Andric bool Swapped) -> Instruction * { 517bdd1243dSDimitry Andric auto *TVI = dyn_cast<BinaryOperator>(TrueVal); 518bdd1243dSDimitry Andric if (!TVI || !TVI->hasOneUse() || isa<Constant>(FalseVal)) 519bdd1243dSDimitry Andric return nullptr; 520bdd1243dSDimitry Andric 521bdd1243dSDimitry Andric unsigned SFO = getSelectFoldableOperands(TVI); 5220b57cec5SDimitry Andric unsigned OpToFold = 0; 52381ad6265SDimitry Andric if ((SFO & 1) && FalseVal == TVI->getOperand(0)) 5240b57cec5SDimitry Andric OpToFold = 1; 52581ad6265SDimitry Andric else if ((SFO & 2) && FalseVal == TVI->getOperand(1)) 5260b57cec5SDimitry Andric OpToFold = 2; 5270b57cec5SDimitry Andric 528bdd1243dSDimitry Andric if (!OpToFold) 529bdd1243dSDimitry Andric return nullptr; 530bdd1243dSDimitry Andric 53181ad6265SDimitry Andric // TODO: We probably ought to revisit cases where the select and FP 53281ad6265SDimitry Andric // instructions have different flags and add tests to ensure the 53381ad6265SDimitry Andric // behaviour is correct. 534bdd1243dSDimitry Andric FastMathFlags FMF; 53581ad6265SDimitry Andric if (isa<FPMathOperator>(&SI)) 53681ad6265SDimitry Andric FMF = SI.getFastMathFlags(); 53781ad6265SDimitry Andric Constant *C = ConstantExpr::getBinOpIdentity( 53881ad6265SDimitry Andric TVI->getOpcode(), TVI->getType(), true, FMF.noSignedZeros()); 5390b57cec5SDimitry Andric Value *OOp = TVI->getOperand(2 - OpToFold); 5400b57cec5SDimitry Andric // Avoid creating select between 2 constants unless it's selecting 5410b57cec5SDimitry Andric // between 0, 1 and -1. 5420b57cec5SDimitry Andric const APInt *OOpC; 5430b57cec5SDimitry Andric bool OOpIsAPInt = match(OOp, m_APInt(OOpC)); 544*0fca6ea1SDimitry Andric if (isa<Constant>(OOp) && 545*0fca6ea1SDimitry Andric (!OOpIsAPInt || !isSelect01(C->getUniqueInteger(), *OOpC))) 546*0fca6ea1SDimitry Andric return nullptr; 547*0fca6ea1SDimitry Andric 548*0fca6ea1SDimitry Andric // If the false value is a NaN then we have that the floating point math 549*0fca6ea1SDimitry Andric // operation in the transformed code may not preserve the exact NaN 550*0fca6ea1SDimitry Andric // bit-pattern -- e.g. `fadd sNaN, 0.0 -> qNaN`. 551*0fca6ea1SDimitry Andric // This makes the transformation incorrect since the original program would 552*0fca6ea1SDimitry Andric // have preserved the exact NaN bit-pattern. 553*0fca6ea1SDimitry Andric // Avoid the folding if the false value might be a NaN. 554*0fca6ea1SDimitry Andric if (isa<FPMathOperator>(&SI) && 555*0fca6ea1SDimitry Andric !computeKnownFPClass(FalseVal, FMF, fcNan, &SI).isKnownNeverNaN()) 556*0fca6ea1SDimitry Andric return nullptr; 557*0fca6ea1SDimitry Andric 558bdd1243dSDimitry Andric Value *NewSel = Builder.CreateSelect(SI.getCondition(), Swapped ? C : OOp, 55906c3fb27SDimitry Andric Swapped ? OOp : C, "", &SI); 56081ad6265SDimitry Andric if (isa<FPMathOperator>(&SI)) 56181ad6265SDimitry Andric cast<Instruction>(NewSel)->setFastMathFlags(FMF); 5620b57cec5SDimitry Andric NewSel->takeName(TVI); 56381ad6265SDimitry Andric BinaryOperator *BO = 56481ad6265SDimitry Andric BinaryOperator::Create(TVI->getOpcode(), FalseVal, NewSel); 5650b57cec5SDimitry Andric BO->copyIRFlags(TVI); 5660b57cec5SDimitry Andric return BO; 56781ad6265SDimitry Andric }; 5680b57cec5SDimitry Andric 56981ad6265SDimitry Andric if (Instruction *R = TryFoldSelectIntoOp(SI, TrueVal, FalseVal, false)) 57081ad6265SDimitry Andric return R; 5710b57cec5SDimitry Andric 57281ad6265SDimitry Andric if (Instruction *R = TryFoldSelectIntoOp(SI, FalseVal, TrueVal, true)) 57381ad6265SDimitry Andric return R; 5740b57cec5SDimitry Andric 5750b57cec5SDimitry Andric return nullptr; 5760b57cec5SDimitry Andric } 5770b57cec5SDimitry Andric 5780b57cec5SDimitry Andric /// We want to turn: 5790b57cec5SDimitry Andric /// (select (icmp eq (and X, Y), 0), (and (lshr X, Z), 1), 1) 5800b57cec5SDimitry Andric /// into: 5810b57cec5SDimitry Andric /// zext (icmp ne i32 (and X, (or Y, (shl 1, Z))), 0) 5820b57cec5SDimitry Andric /// Note: 5830b57cec5SDimitry Andric /// Z may be 0 if lshr is missing. 5840b57cec5SDimitry Andric /// Worst-case scenario is that we will replace 5 instructions with 5 different 5850b57cec5SDimitry Andric /// instructions, but we got rid of select. 5860b57cec5SDimitry Andric static Instruction *foldSelectICmpAndAnd(Type *SelType, const ICmpInst *Cmp, 5870b57cec5SDimitry Andric Value *TVal, Value *FVal, 5880b57cec5SDimitry Andric InstCombiner::BuilderTy &Builder) { 5890b57cec5SDimitry Andric if (!(Cmp->hasOneUse() && Cmp->getOperand(0)->hasOneUse() && 5900b57cec5SDimitry Andric Cmp->getPredicate() == ICmpInst::ICMP_EQ && 5910b57cec5SDimitry Andric match(Cmp->getOperand(1), m_Zero()) && match(FVal, m_One()))) 5920b57cec5SDimitry Andric return nullptr; 5930b57cec5SDimitry Andric 5940b57cec5SDimitry Andric // The TrueVal has general form of: and %B, 1 5950b57cec5SDimitry Andric Value *B; 5960b57cec5SDimitry Andric if (!match(TVal, m_OneUse(m_And(m_Value(B), m_One())))) 5970b57cec5SDimitry Andric return nullptr; 5980b57cec5SDimitry Andric 5990b57cec5SDimitry Andric // Where %B may be optionally shifted: lshr %X, %Z. 6000b57cec5SDimitry Andric Value *X, *Z; 6010b57cec5SDimitry Andric const bool HasShift = match(B, m_OneUse(m_LShr(m_Value(X), m_Value(Z)))); 60281ad6265SDimitry Andric 60381ad6265SDimitry Andric // The shift must be valid. 60481ad6265SDimitry Andric // TODO: This restricts the fold to constant shift amounts. Is there a way to 60581ad6265SDimitry Andric // handle variable shifts safely? PR47012 60681ad6265SDimitry Andric if (HasShift && 60781ad6265SDimitry Andric !match(Z, m_SpecificInt_ICMP(CmpInst::ICMP_ULT, 60881ad6265SDimitry Andric APInt(SelType->getScalarSizeInBits(), 60981ad6265SDimitry Andric SelType->getScalarSizeInBits())))) 61081ad6265SDimitry Andric return nullptr; 61181ad6265SDimitry Andric 6120b57cec5SDimitry Andric if (!HasShift) 6130b57cec5SDimitry Andric X = B; 6140b57cec5SDimitry Andric 6150b57cec5SDimitry Andric Value *Y; 6160b57cec5SDimitry Andric if (!match(Cmp->getOperand(0), m_c_And(m_Specific(X), m_Value(Y)))) 6170b57cec5SDimitry Andric return nullptr; 6180b57cec5SDimitry Andric 6190b57cec5SDimitry Andric // ((X & Y) == 0) ? ((X >> Z) & 1) : 1 --> (X & (Y | (1 << Z))) != 0 6200b57cec5SDimitry Andric // ((X & Y) == 0) ? (X & 1) : 1 --> (X & (Y | 1)) != 0 6210b57cec5SDimitry Andric Constant *One = ConstantInt::get(SelType, 1); 6220b57cec5SDimitry Andric Value *MaskB = HasShift ? Builder.CreateShl(One, Z) : One; 6230b57cec5SDimitry Andric Value *FullMask = Builder.CreateOr(Y, MaskB); 6240b57cec5SDimitry Andric Value *MaskedX = Builder.CreateAnd(X, FullMask); 6250b57cec5SDimitry Andric Value *ICmpNeZero = Builder.CreateIsNotNull(MaskedX); 6260b57cec5SDimitry Andric return new ZExtInst(ICmpNeZero, SelType); 6270b57cec5SDimitry Andric } 6280b57cec5SDimitry Andric 6290b57cec5SDimitry Andric /// We want to turn: 63006c3fb27SDimitry Andric /// (select (icmp eq (and X, C1), 0), 0, (shl [nsw/nuw] X, C2)); 63106c3fb27SDimitry Andric /// iff C1 is a mask and the number of its leading zeros is equal to C2 63206c3fb27SDimitry Andric /// into: 63306c3fb27SDimitry Andric /// shl X, C2 63406c3fb27SDimitry Andric static Value *foldSelectICmpAndZeroShl(const ICmpInst *Cmp, Value *TVal, 63506c3fb27SDimitry Andric Value *FVal, 63606c3fb27SDimitry Andric InstCombiner::BuilderTy &Builder) { 63706c3fb27SDimitry Andric ICmpInst::Predicate Pred; 63806c3fb27SDimitry Andric Value *AndVal; 63906c3fb27SDimitry Andric if (!match(Cmp, m_ICmp(Pred, m_Value(AndVal), m_Zero()))) 64006c3fb27SDimitry Andric return nullptr; 64106c3fb27SDimitry Andric 64206c3fb27SDimitry Andric if (Pred == ICmpInst::ICMP_NE) { 64306c3fb27SDimitry Andric Pred = ICmpInst::ICMP_EQ; 64406c3fb27SDimitry Andric std::swap(TVal, FVal); 64506c3fb27SDimitry Andric } 64606c3fb27SDimitry Andric 64706c3fb27SDimitry Andric Value *X; 64806c3fb27SDimitry Andric const APInt *C2, *C1; 64906c3fb27SDimitry Andric if (Pred != ICmpInst::ICMP_EQ || 65006c3fb27SDimitry Andric !match(AndVal, m_And(m_Value(X), m_APInt(C1))) || 65106c3fb27SDimitry Andric !match(TVal, m_Zero()) || !match(FVal, m_Shl(m_Specific(X), m_APInt(C2)))) 65206c3fb27SDimitry Andric return nullptr; 65306c3fb27SDimitry Andric 65406c3fb27SDimitry Andric if (!C1->isMask() || 65506c3fb27SDimitry Andric C1->countLeadingZeros() != static_cast<unsigned>(C2->getZExtValue())) 65606c3fb27SDimitry Andric return nullptr; 65706c3fb27SDimitry Andric 65806c3fb27SDimitry Andric auto *FI = dyn_cast<Instruction>(FVal); 65906c3fb27SDimitry Andric if (!FI) 66006c3fb27SDimitry Andric return nullptr; 66106c3fb27SDimitry Andric 66206c3fb27SDimitry Andric FI->setHasNoSignedWrap(false); 66306c3fb27SDimitry Andric FI->setHasNoUnsignedWrap(false); 66406c3fb27SDimitry Andric return FVal; 66506c3fb27SDimitry Andric } 66606c3fb27SDimitry Andric 66706c3fb27SDimitry Andric /// We want to turn: 6680b57cec5SDimitry Andric /// (select (icmp sgt x, C), lshr (X, Y), ashr (X, Y)); iff C s>= -1 6690b57cec5SDimitry Andric /// (select (icmp slt x, C), ashr (X, Y), lshr (X, Y)); iff C s>= 0 6700b57cec5SDimitry Andric /// into: 6710b57cec5SDimitry Andric /// ashr (X, Y) 6720b57cec5SDimitry Andric static Value *foldSelectICmpLshrAshr(const ICmpInst *IC, Value *TrueVal, 6730b57cec5SDimitry Andric Value *FalseVal, 6740b57cec5SDimitry Andric InstCombiner::BuilderTy &Builder) { 6750b57cec5SDimitry Andric ICmpInst::Predicate Pred = IC->getPredicate(); 6760b57cec5SDimitry Andric Value *CmpLHS = IC->getOperand(0); 6770b57cec5SDimitry Andric Value *CmpRHS = IC->getOperand(1); 6780b57cec5SDimitry Andric if (!CmpRHS->getType()->isIntOrIntVectorTy()) 6790b57cec5SDimitry Andric return nullptr; 6800b57cec5SDimitry Andric 6810b57cec5SDimitry Andric Value *X, *Y; 6820b57cec5SDimitry Andric unsigned Bitwidth = CmpRHS->getType()->getScalarSizeInBits(); 6830b57cec5SDimitry Andric if ((Pred != ICmpInst::ICMP_SGT || 6840b57cec5SDimitry Andric !match(CmpRHS, 6850b57cec5SDimitry Andric m_SpecificInt_ICMP(ICmpInst::ICMP_SGE, APInt(Bitwidth, -1)))) && 6860b57cec5SDimitry Andric (Pred != ICmpInst::ICMP_SLT || 6870b57cec5SDimitry Andric !match(CmpRHS, 6880b57cec5SDimitry Andric m_SpecificInt_ICMP(ICmpInst::ICMP_SGE, APInt(Bitwidth, 0))))) 6890b57cec5SDimitry Andric return nullptr; 6900b57cec5SDimitry Andric 6910b57cec5SDimitry Andric // Canonicalize so that ashr is in FalseVal. 6920b57cec5SDimitry Andric if (Pred == ICmpInst::ICMP_SLT) 6930b57cec5SDimitry Andric std::swap(TrueVal, FalseVal); 6940b57cec5SDimitry Andric 6950b57cec5SDimitry Andric if (match(TrueVal, m_LShr(m_Value(X), m_Value(Y))) && 6960b57cec5SDimitry Andric match(FalseVal, m_AShr(m_Specific(X), m_Specific(Y))) && 6970b57cec5SDimitry Andric match(CmpLHS, m_Specific(X))) { 6980b57cec5SDimitry Andric const auto *Ashr = cast<Instruction>(FalseVal); 6990b57cec5SDimitry Andric // if lshr is not exact and ashr is, this new ashr must not be exact. 7000b57cec5SDimitry Andric bool IsExact = Ashr->isExact() && cast<Instruction>(TrueVal)->isExact(); 7010b57cec5SDimitry Andric return Builder.CreateAShr(X, Y, IC->getName(), IsExact); 7020b57cec5SDimitry Andric } 7030b57cec5SDimitry Andric 7040b57cec5SDimitry Andric return nullptr; 7050b57cec5SDimitry Andric } 7060b57cec5SDimitry Andric 7070b57cec5SDimitry Andric /// We want to turn: 7085f757f3fSDimitry Andric /// (select (icmp eq (and X, C1), 0), Y, (BinOp Y, C2)) 7090b57cec5SDimitry Andric /// into: 7105f757f3fSDimitry Andric /// IF C2 u>= C1 7115f757f3fSDimitry Andric /// (BinOp Y, (shl (and X, C1), C3)) 7125f757f3fSDimitry Andric /// ELSE 7135f757f3fSDimitry Andric /// (BinOp Y, (lshr (and X, C1), C3)) 7140b57cec5SDimitry Andric /// iff: 7155f757f3fSDimitry Andric /// 0 on the RHS is the identity value (i.e add, xor, shl, etc...) 7160b57cec5SDimitry Andric /// C1 and C2 are both powers of 2 7170b57cec5SDimitry Andric /// where: 7185f757f3fSDimitry Andric /// IF C2 u>= C1 7190b57cec5SDimitry Andric /// C3 = Log(C2) - Log(C1) 7205f757f3fSDimitry Andric /// ELSE 7215f757f3fSDimitry Andric /// C3 = Log(C1) - Log(C2) 7220b57cec5SDimitry Andric /// 7230b57cec5SDimitry Andric /// This transform handles cases where: 7240b57cec5SDimitry Andric /// 1. The icmp predicate is inverted 7250b57cec5SDimitry Andric /// 2. The select operands are reversed 7260b57cec5SDimitry Andric /// 3. The magnitude of C2 and C1 are flipped 7275f757f3fSDimitry Andric static Value *foldSelectICmpAndBinOp(const ICmpInst *IC, Value *TrueVal, 7280b57cec5SDimitry Andric Value *FalseVal, 7290b57cec5SDimitry Andric InstCombiner::BuilderTy &Builder) { 7300b57cec5SDimitry Andric // Only handle integer compares. Also, if this is a vector select, we need a 7310b57cec5SDimitry Andric // vector compare. 7320b57cec5SDimitry Andric if (!TrueVal->getType()->isIntOrIntVectorTy() || 7330b57cec5SDimitry Andric TrueVal->getType()->isVectorTy() != IC->getType()->isVectorTy()) 7340b57cec5SDimitry Andric return nullptr; 7350b57cec5SDimitry Andric 7360b57cec5SDimitry Andric Value *CmpLHS = IC->getOperand(0); 7370b57cec5SDimitry Andric Value *CmpRHS = IC->getOperand(1); 7380b57cec5SDimitry Andric 7390b57cec5SDimitry Andric unsigned C1Log; 7400b57cec5SDimitry Andric bool NeedAnd = false; 7415f757f3fSDimitry Andric CmpInst::Predicate Pred = IC->getPredicate(); 7420b57cec5SDimitry Andric if (IC->isEquality()) { 7430b57cec5SDimitry Andric if (!match(CmpRHS, m_Zero())) 7440b57cec5SDimitry Andric return nullptr; 7450b57cec5SDimitry Andric 7460b57cec5SDimitry Andric const APInt *C1; 7470b57cec5SDimitry Andric if (!match(CmpLHS, m_And(m_Value(), m_Power2(C1)))) 7480b57cec5SDimitry Andric return nullptr; 7490b57cec5SDimitry Andric 7500b57cec5SDimitry Andric C1Log = C1->logBase2(); 7515f757f3fSDimitry Andric } else { 7525f757f3fSDimitry Andric APInt C1; 7535f757f3fSDimitry Andric if (!decomposeBitTestICmp(CmpLHS, CmpRHS, Pred, CmpLHS, C1) || 7545f757f3fSDimitry Andric !C1.isPowerOf2()) 7550b57cec5SDimitry Andric return nullptr; 7560b57cec5SDimitry Andric 7575f757f3fSDimitry Andric C1Log = C1.logBase2(); 7580b57cec5SDimitry Andric NeedAnd = true; 7595f757f3fSDimitry Andric } 7605f757f3fSDimitry Andric 7615f757f3fSDimitry Andric Value *Y, *V = CmpLHS; 7625f757f3fSDimitry Andric BinaryOperator *BinOp; 7635f757f3fSDimitry Andric const APInt *C2; 7645f757f3fSDimitry Andric bool NeedXor; 7655f757f3fSDimitry Andric if (match(FalseVal, m_BinOp(m_Specific(TrueVal), m_Power2(C2)))) { 7665f757f3fSDimitry Andric Y = TrueVal; 7675f757f3fSDimitry Andric BinOp = cast<BinaryOperator>(FalseVal); 7685f757f3fSDimitry Andric NeedXor = Pred == ICmpInst::ICMP_NE; 7695f757f3fSDimitry Andric } else if (match(TrueVal, m_BinOp(m_Specific(FalseVal), m_Power2(C2)))) { 7705f757f3fSDimitry Andric Y = FalseVal; 7715f757f3fSDimitry Andric BinOp = cast<BinaryOperator>(TrueVal); 7725f757f3fSDimitry Andric NeedXor = Pred == ICmpInst::ICMP_EQ; 7730b57cec5SDimitry Andric } else { 7740b57cec5SDimitry Andric return nullptr; 7750b57cec5SDimitry Andric } 7760b57cec5SDimitry Andric 7775f757f3fSDimitry Andric // Check that 0 on RHS is identity value for this binop. 7785f757f3fSDimitry Andric auto *IdentityC = 7795f757f3fSDimitry Andric ConstantExpr::getBinOpIdentity(BinOp->getOpcode(), BinOp->getType(), 7805f757f3fSDimitry Andric /*AllowRHSConstant*/ true); 7815f757f3fSDimitry Andric if (IdentityC == nullptr || !IdentityC->isNullValue()) 7820b57cec5SDimitry Andric return nullptr; 7830b57cec5SDimitry Andric 7840b57cec5SDimitry Andric unsigned C2Log = C2->logBase2(); 7850b57cec5SDimitry Andric 7860b57cec5SDimitry Andric bool NeedShift = C1Log != C2Log; 7870b57cec5SDimitry Andric bool NeedZExtTrunc = Y->getType()->getScalarSizeInBits() != 7880b57cec5SDimitry Andric V->getType()->getScalarSizeInBits(); 7890b57cec5SDimitry Andric 7900b57cec5SDimitry Andric // Make sure we don't create more instructions than we save. 7915f757f3fSDimitry Andric if ((NeedShift + NeedXor + NeedZExtTrunc + NeedAnd) > 7925f757f3fSDimitry Andric (IC->hasOneUse() + BinOp->hasOneUse())) 7930b57cec5SDimitry Andric return nullptr; 7940b57cec5SDimitry Andric 7950b57cec5SDimitry Andric if (NeedAnd) { 7960b57cec5SDimitry Andric // Insert the AND instruction on the input to the truncate. 7970b57cec5SDimitry Andric APInt C1 = APInt::getOneBitSet(V->getType()->getScalarSizeInBits(), C1Log); 7980b57cec5SDimitry Andric V = Builder.CreateAnd(V, ConstantInt::get(V->getType(), C1)); 7990b57cec5SDimitry Andric } 8000b57cec5SDimitry Andric 8010b57cec5SDimitry Andric if (C2Log > C1Log) { 8020b57cec5SDimitry Andric V = Builder.CreateZExtOrTrunc(V, Y->getType()); 8030b57cec5SDimitry Andric V = Builder.CreateShl(V, C2Log - C1Log); 8040b57cec5SDimitry Andric } else if (C1Log > C2Log) { 8050b57cec5SDimitry Andric V = Builder.CreateLShr(V, C1Log - C2Log); 8060b57cec5SDimitry Andric V = Builder.CreateZExtOrTrunc(V, Y->getType()); 8070b57cec5SDimitry Andric } else 8080b57cec5SDimitry Andric V = Builder.CreateZExtOrTrunc(V, Y->getType()); 8090b57cec5SDimitry Andric 8100b57cec5SDimitry Andric if (NeedXor) 8110b57cec5SDimitry Andric V = Builder.CreateXor(V, *C2); 8120b57cec5SDimitry Andric 8135f757f3fSDimitry Andric return Builder.CreateBinOp(BinOp->getOpcode(), Y, V); 8140b57cec5SDimitry Andric } 8150b57cec5SDimitry Andric 8165ffd83dbSDimitry Andric /// Canonicalize a set or clear of a masked set of constant bits to 8175ffd83dbSDimitry Andric /// select-of-constants form. 8185ffd83dbSDimitry Andric static Instruction *foldSetClearBits(SelectInst &Sel, 8195ffd83dbSDimitry Andric InstCombiner::BuilderTy &Builder) { 8205ffd83dbSDimitry Andric Value *Cond = Sel.getCondition(); 8215ffd83dbSDimitry Andric Value *T = Sel.getTrueValue(); 8225ffd83dbSDimitry Andric Value *F = Sel.getFalseValue(); 8235ffd83dbSDimitry Andric Type *Ty = Sel.getType(); 8245ffd83dbSDimitry Andric Value *X; 8255ffd83dbSDimitry Andric const APInt *NotC, *C; 8265ffd83dbSDimitry Andric 8275ffd83dbSDimitry Andric // Cond ? (X & ~C) : (X | C) --> (X & ~C) | (Cond ? 0 : C) 8285ffd83dbSDimitry Andric if (match(T, m_And(m_Value(X), m_APInt(NotC))) && 8295ffd83dbSDimitry Andric match(F, m_OneUse(m_Or(m_Specific(X), m_APInt(C)))) && *NotC == ~(*C)) { 8305ffd83dbSDimitry Andric Constant *Zero = ConstantInt::getNullValue(Ty); 8315ffd83dbSDimitry Andric Constant *OrC = ConstantInt::get(Ty, *C); 8325ffd83dbSDimitry Andric Value *NewSel = Builder.CreateSelect(Cond, Zero, OrC, "masksel", &Sel); 8335ffd83dbSDimitry Andric return BinaryOperator::CreateOr(T, NewSel); 8345ffd83dbSDimitry Andric } 8355ffd83dbSDimitry Andric 8365ffd83dbSDimitry Andric // Cond ? (X | C) : (X & ~C) --> (X & ~C) | (Cond ? C : 0) 8375ffd83dbSDimitry Andric if (match(F, m_And(m_Value(X), m_APInt(NotC))) && 8385ffd83dbSDimitry Andric match(T, m_OneUse(m_Or(m_Specific(X), m_APInt(C)))) && *NotC == ~(*C)) { 8395ffd83dbSDimitry Andric Constant *Zero = ConstantInt::getNullValue(Ty); 8405ffd83dbSDimitry Andric Constant *OrC = ConstantInt::get(Ty, *C); 8415ffd83dbSDimitry Andric Value *NewSel = Builder.CreateSelect(Cond, OrC, Zero, "masksel", &Sel); 8425ffd83dbSDimitry Andric return BinaryOperator::CreateOr(F, NewSel); 8435ffd83dbSDimitry Andric } 8445ffd83dbSDimitry Andric 8455ffd83dbSDimitry Andric return nullptr; 8465ffd83dbSDimitry Andric } 8475ffd83dbSDimitry Andric 848349cc55cSDimitry Andric // select (x == 0), 0, x * y --> freeze(y) * x 849349cc55cSDimitry Andric // select (y == 0), 0, x * y --> freeze(x) * y 850349cc55cSDimitry Andric // select (x == 0), undef, x * y --> freeze(y) * x 851349cc55cSDimitry Andric // select (x == undef), 0, x * y --> freeze(y) * x 852349cc55cSDimitry Andric // Usage of mul instead of 0 will make the result more poisonous, 853349cc55cSDimitry Andric // so the operand that was not checked in the condition should be frozen. 854349cc55cSDimitry Andric // The latter folding is applied only when a constant compared with x is 855349cc55cSDimitry Andric // is a vector consisting of 0 and undefs. If a constant compared with x 856349cc55cSDimitry Andric // is a scalar undefined value or undefined vector then an expression 857349cc55cSDimitry Andric // should be already folded into a constant. 858349cc55cSDimitry Andric static Instruction *foldSelectZeroOrMul(SelectInst &SI, InstCombinerImpl &IC) { 859349cc55cSDimitry Andric auto *CondVal = SI.getCondition(); 860349cc55cSDimitry Andric auto *TrueVal = SI.getTrueValue(); 861349cc55cSDimitry Andric auto *FalseVal = SI.getFalseValue(); 862349cc55cSDimitry Andric Value *X, *Y; 863349cc55cSDimitry Andric ICmpInst::Predicate Predicate; 864349cc55cSDimitry Andric 865349cc55cSDimitry Andric // Assuming that constant compared with zero is not undef (but it may be 866349cc55cSDimitry Andric // a vector with some undef elements). Otherwise (when a constant is undef) 867349cc55cSDimitry Andric // the select expression should be already simplified. 868349cc55cSDimitry Andric if (!match(CondVal, m_ICmp(Predicate, m_Value(X), m_Zero())) || 869349cc55cSDimitry Andric !ICmpInst::isEquality(Predicate)) 870349cc55cSDimitry Andric return nullptr; 871349cc55cSDimitry Andric 872349cc55cSDimitry Andric if (Predicate == ICmpInst::ICMP_NE) 873349cc55cSDimitry Andric std::swap(TrueVal, FalseVal); 874349cc55cSDimitry Andric 875349cc55cSDimitry Andric // Check that TrueVal is a constant instead of matching it with m_Zero() 876349cc55cSDimitry Andric // to handle the case when it is a scalar undef value or a vector containing 877349cc55cSDimitry Andric // non-zero elements that are masked by undef elements in the compare 878349cc55cSDimitry Andric // constant. 879349cc55cSDimitry Andric auto *TrueValC = dyn_cast<Constant>(TrueVal); 880349cc55cSDimitry Andric if (TrueValC == nullptr || 881349cc55cSDimitry Andric !match(FalseVal, m_c_Mul(m_Specific(X), m_Value(Y))) || 882349cc55cSDimitry Andric !isa<Instruction>(FalseVal)) 883349cc55cSDimitry Andric return nullptr; 884349cc55cSDimitry Andric 885349cc55cSDimitry Andric auto *ZeroC = cast<Constant>(cast<Instruction>(CondVal)->getOperand(1)); 886349cc55cSDimitry Andric auto *MergedC = Constant::mergeUndefsWith(TrueValC, ZeroC); 887349cc55cSDimitry Andric // If X is compared with 0 then TrueVal could be either zero or undef. 888349cc55cSDimitry Andric // m_Zero match vectors containing some undef elements, but for scalars 889349cc55cSDimitry Andric // m_Undef should be used explicitly. 890349cc55cSDimitry Andric if (!match(MergedC, m_Zero()) && !match(MergedC, m_Undef())) 891349cc55cSDimitry Andric return nullptr; 892349cc55cSDimitry Andric 893349cc55cSDimitry Andric auto *FalseValI = cast<Instruction>(FalseVal); 894349cc55cSDimitry Andric auto *FrY = IC.InsertNewInstBefore(new FreezeInst(Y, Y->getName() + ".fr"), 8955f757f3fSDimitry Andric FalseValI->getIterator()); 896349cc55cSDimitry Andric IC.replaceOperand(*FalseValI, FalseValI->getOperand(0) == Y ? 0 : 1, FrY); 897349cc55cSDimitry Andric return IC.replaceInstUsesWith(SI, FalseValI); 898349cc55cSDimitry Andric } 899349cc55cSDimitry Andric 9000b57cec5SDimitry Andric /// Transform patterns such as (a > b) ? a - b : 0 into usub.sat(a, b). 9010b57cec5SDimitry Andric /// There are 8 commuted/swapped variants of this pattern. 9020b57cec5SDimitry Andric /// TODO: Also support a - UMIN(a,b) patterns. 9030b57cec5SDimitry Andric static Value *canonicalizeSaturatedSubtract(const ICmpInst *ICI, 9040b57cec5SDimitry Andric const Value *TrueVal, 9050b57cec5SDimitry Andric const Value *FalseVal, 9060b57cec5SDimitry Andric InstCombiner::BuilderTy &Builder) { 9070b57cec5SDimitry Andric ICmpInst::Predicate Pred = ICI->getPredicate(); 908bdd1243dSDimitry Andric Value *A = ICI->getOperand(0); 909bdd1243dSDimitry Andric Value *B = ICI->getOperand(1); 9100b57cec5SDimitry Andric 9110b57cec5SDimitry Andric // (b > a) ? 0 : a - b -> (b <= a) ? a - b : 0 912bdd1243dSDimitry Andric // (a == 0) ? 0 : a - 1 -> (a != 0) ? a - 1 : 0 9130b57cec5SDimitry Andric if (match(TrueVal, m_Zero())) { 9140b57cec5SDimitry Andric Pred = ICmpInst::getInversePredicate(Pred); 9150b57cec5SDimitry Andric std::swap(TrueVal, FalseVal); 9160b57cec5SDimitry Andric } 917bdd1243dSDimitry Andric 9180b57cec5SDimitry Andric if (!match(FalseVal, m_Zero())) 9190b57cec5SDimitry Andric return nullptr; 9200b57cec5SDimitry Andric 921bdd1243dSDimitry Andric // ugt 0 is canonicalized to ne 0 and requires special handling 922bdd1243dSDimitry Andric // (a != 0) ? a + -1 : 0 -> usub.sat(a, 1) 923bdd1243dSDimitry Andric if (Pred == ICmpInst::ICMP_NE) { 924bdd1243dSDimitry Andric if (match(B, m_Zero()) && match(TrueVal, m_Add(m_Specific(A), m_AllOnes()))) 925bdd1243dSDimitry Andric return Builder.CreateBinaryIntrinsic(Intrinsic::usub_sat, A, 926bdd1243dSDimitry Andric ConstantInt::get(A->getType(), 1)); 927bdd1243dSDimitry Andric return nullptr; 928bdd1243dSDimitry Andric } 929bdd1243dSDimitry Andric 930bdd1243dSDimitry Andric if (!ICmpInst::isUnsigned(Pred)) 931bdd1243dSDimitry Andric return nullptr; 932bdd1243dSDimitry Andric 9330b57cec5SDimitry Andric if (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_ULT) { 9340b57cec5SDimitry Andric // (b < a) ? a - b : 0 -> (a > b) ? a - b : 0 9350b57cec5SDimitry Andric std::swap(A, B); 9360b57cec5SDimitry Andric Pred = ICmpInst::getSwappedPredicate(Pred); 9370b57cec5SDimitry Andric } 9380b57cec5SDimitry Andric 9390b57cec5SDimitry Andric assert((Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_UGT) && 9400b57cec5SDimitry Andric "Unexpected isUnsigned predicate!"); 9410b57cec5SDimitry Andric 942480093f4SDimitry Andric // Ensure the sub is of the form: 943480093f4SDimitry Andric // (a > b) ? a - b : 0 -> usub.sat(a, b) 944480093f4SDimitry Andric // (a > b) ? b - a : 0 -> -usub.sat(a, b) 945480093f4SDimitry Andric // Checking for both a-b and a+(-b) as a constant. 9460b57cec5SDimitry Andric bool IsNegative = false; 947480093f4SDimitry Andric const APInt *C; 948480093f4SDimitry Andric if (match(TrueVal, m_Sub(m_Specific(B), m_Specific(A))) || 949480093f4SDimitry Andric (match(A, m_APInt(C)) && 950480093f4SDimitry Andric match(TrueVal, m_Add(m_Specific(B), m_SpecificInt(-*C))))) 9510b57cec5SDimitry Andric IsNegative = true; 952480093f4SDimitry Andric else if (!match(TrueVal, m_Sub(m_Specific(A), m_Specific(B))) && 953480093f4SDimitry Andric !(match(B, m_APInt(C)) && 954480093f4SDimitry Andric match(TrueVal, m_Add(m_Specific(A), m_SpecificInt(-*C))))) 9550b57cec5SDimitry Andric return nullptr; 9560b57cec5SDimitry Andric 957480093f4SDimitry Andric // If we are adding a negate and the sub and icmp are used anywhere else, we 958480093f4SDimitry Andric // would end up with more instructions. 959480093f4SDimitry Andric if (IsNegative && !TrueVal->hasOneUse() && !ICI->hasOneUse()) 9600b57cec5SDimitry Andric return nullptr; 9610b57cec5SDimitry Andric 9620b57cec5SDimitry Andric // (a > b) ? a - b : 0 -> usub.sat(a, b) 9630b57cec5SDimitry Andric // (a > b) ? b - a : 0 -> -usub.sat(a, b) 9640b57cec5SDimitry Andric Value *Result = Builder.CreateBinaryIntrinsic(Intrinsic::usub_sat, A, B); 9650b57cec5SDimitry Andric if (IsNegative) 9660b57cec5SDimitry Andric Result = Builder.CreateNeg(Result); 9670b57cec5SDimitry Andric return Result; 9680b57cec5SDimitry Andric } 9690b57cec5SDimitry Andric 9700b57cec5SDimitry Andric static Value *canonicalizeSaturatedAdd(ICmpInst *Cmp, Value *TVal, Value *FVal, 9710b57cec5SDimitry Andric InstCombiner::BuilderTy &Builder) { 9720b57cec5SDimitry Andric if (!Cmp->hasOneUse()) 9730b57cec5SDimitry Andric return nullptr; 9740b57cec5SDimitry Andric 9750b57cec5SDimitry Andric // Match unsigned saturated add with constant. 9760b57cec5SDimitry Andric Value *Cmp0 = Cmp->getOperand(0); 9770b57cec5SDimitry Andric Value *Cmp1 = Cmp->getOperand(1); 9780b57cec5SDimitry Andric ICmpInst::Predicate Pred = Cmp->getPredicate(); 9790b57cec5SDimitry Andric Value *X; 9800b57cec5SDimitry Andric const APInt *C, *CmpC; 9810b57cec5SDimitry Andric if (Pred == ICmpInst::ICMP_ULT && 9820b57cec5SDimitry Andric match(TVal, m_Add(m_Value(X), m_APInt(C))) && X == Cmp0 && 9830b57cec5SDimitry Andric match(FVal, m_AllOnes()) && match(Cmp1, m_APInt(CmpC)) && *CmpC == ~*C) { 9840b57cec5SDimitry Andric // (X u< ~C) ? (X + C) : -1 --> uadd.sat(X, C) 9850b57cec5SDimitry Andric return Builder.CreateBinaryIntrinsic( 9860b57cec5SDimitry Andric Intrinsic::uadd_sat, X, ConstantInt::get(X->getType(), *C)); 9870b57cec5SDimitry Andric } 9880b57cec5SDimitry Andric 9890b57cec5SDimitry Andric // Match unsigned saturated add of 2 variables with an unnecessary 'not'. 9900b57cec5SDimitry Andric // There are 8 commuted variants. 991eaeb601bSDimitry Andric // Canonicalize -1 (saturated result) to true value of the select. 9920b57cec5SDimitry Andric if (match(FVal, m_AllOnes())) { 9930b57cec5SDimitry Andric std::swap(TVal, FVal); 994eaeb601bSDimitry Andric Pred = CmpInst::getInversePredicate(Pred); 9950b57cec5SDimitry Andric } 9960b57cec5SDimitry Andric if (!match(TVal, m_AllOnes())) 9970b57cec5SDimitry Andric return nullptr; 9980b57cec5SDimitry Andric 999eaeb601bSDimitry Andric // Canonicalize predicate to less-than or less-or-equal-than. 1000eaeb601bSDimitry Andric if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) { 10010b57cec5SDimitry Andric std::swap(Cmp0, Cmp1); 1002eaeb601bSDimitry Andric Pred = CmpInst::getSwappedPredicate(Pred); 10030b57cec5SDimitry Andric } 1004eaeb601bSDimitry Andric if (Pred != ICmpInst::ICMP_ULT && Pred != ICmpInst::ICMP_ULE) 10050b57cec5SDimitry Andric return nullptr; 10060b57cec5SDimitry Andric 10070b57cec5SDimitry Andric // Match unsigned saturated add of 2 variables with an unnecessary 'not'. 1008eaeb601bSDimitry Andric // Strictness of the comparison is irrelevant. 10090b57cec5SDimitry Andric Value *Y; 10100b57cec5SDimitry Andric if (match(Cmp0, m_Not(m_Value(X))) && 10110b57cec5SDimitry Andric match(FVal, m_c_Add(m_Specific(X), m_Value(Y))) && Y == Cmp1) { 10120b57cec5SDimitry Andric // (~X u< Y) ? -1 : (X + Y) --> uadd.sat(X, Y) 10130b57cec5SDimitry Andric // (~X u< Y) ? -1 : (Y + X) --> uadd.sat(X, Y) 10140b57cec5SDimitry Andric return Builder.CreateBinaryIntrinsic(Intrinsic::uadd_sat, X, Y); 10150b57cec5SDimitry Andric } 10160b57cec5SDimitry Andric // The 'not' op may be included in the sum but not the compare. 1017eaeb601bSDimitry Andric // Strictness of the comparison is irrelevant. 10180b57cec5SDimitry Andric X = Cmp0; 10190b57cec5SDimitry Andric Y = Cmp1; 10200b57cec5SDimitry Andric if (match(FVal, m_c_Add(m_Not(m_Specific(X)), m_Specific(Y)))) { 10210b57cec5SDimitry Andric // (X u< Y) ? -1 : (~X + Y) --> uadd.sat(~X, Y) 10220b57cec5SDimitry Andric // (X u< Y) ? -1 : (Y + ~X) --> uadd.sat(Y, ~X) 10230b57cec5SDimitry Andric BinaryOperator *BO = cast<BinaryOperator>(FVal); 10240b57cec5SDimitry Andric return Builder.CreateBinaryIntrinsic( 10250b57cec5SDimitry Andric Intrinsic::uadd_sat, BO->getOperand(0), BO->getOperand(1)); 10260b57cec5SDimitry Andric } 1027480093f4SDimitry Andric // The overflow may be detected via the add wrapping round. 1028eaeb601bSDimitry Andric // This is only valid for strict comparison! 1029eaeb601bSDimitry Andric if (Pred == ICmpInst::ICMP_ULT && 1030eaeb601bSDimitry Andric match(Cmp0, m_c_Add(m_Specific(Cmp1), m_Value(Y))) && 1031480093f4SDimitry Andric match(FVal, m_c_Add(m_Specific(Cmp1), m_Specific(Y)))) { 1032480093f4SDimitry Andric // ((X + Y) u< X) ? -1 : (X + Y) --> uadd.sat(X, Y) 1033480093f4SDimitry Andric // ((X + Y) u< Y) ? -1 : (X + Y) --> uadd.sat(X, Y) 1034480093f4SDimitry Andric return Builder.CreateBinaryIntrinsic(Intrinsic::uadd_sat, Cmp1, Y); 1035480093f4SDimitry Andric } 10360b57cec5SDimitry Andric 10370b57cec5SDimitry Andric return nullptr; 10380b57cec5SDimitry Andric } 10390b57cec5SDimitry Andric 104006c3fb27SDimitry Andric /// Try to match patterns with select and subtract as absolute difference. 104106c3fb27SDimitry Andric static Value *foldAbsDiff(ICmpInst *Cmp, Value *TVal, Value *FVal, 104206c3fb27SDimitry Andric InstCombiner::BuilderTy &Builder) { 104306c3fb27SDimitry Andric auto *TI = dyn_cast<Instruction>(TVal); 104406c3fb27SDimitry Andric auto *FI = dyn_cast<Instruction>(FVal); 104506c3fb27SDimitry Andric if (!TI || !FI) 104606c3fb27SDimitry Andric return nullptr; 104706c3fb27SDimitry Andric 104806c3fb27SDimitry Andric // Normalize predicate to gt/lt rather than ge/le. 104906c3fb27SDimitry Andric ICmpInst::Predicate Pred = Cmp->getStrictPredicate(); 105006c3fb27SDimitry Andric Value *A = Cmp->getOperand(0); 105106c3fb27SDimitry Andric Value *B = Cmp->getOperand(1); 105206c3fb27SDimitry Andric 105306c3fb27SDimitry Andric // Normalize "A - B" as the true value of the select. 105406c3fb27SDimitry Andric if (match(FI, m_Sub(m_Specific(A), m_Specific(B)))) { 105506c3fb27SDimitry Andric std::swap(FI, TI); 105606c3fb27SDimitry Andric Pred = ICmpInst::getSwappedPredicate(Pred); 105706c3fb27SDimitry Andric } 105806c3fb27SDimitry Andric 105906c3fb27SDimitry Andric // With any pair of no-wrap subtracts: 106006c3fb27SDimitry Andric // (A > B) ? (A - B) : (B - A) --> abs(A - B) 106106c3fb27SDimitry Andric if (Pred == CmpInst::ICMP_SGT && 106206c3fb27SDimitry Andric match(TI, m_Sub(m_Specific(A), m_Specific(B))) && 106306c3fb27SDimitry Andric match(FI, m_Sub(m_Specific(B), m_Specific(A))) && 106406c3fb27SDimitry Andric (TI->hasNoSignedWrap() || TI->hasNoUnsignedWrap()) && 106506c3fb27SDimitry Andric (FI->hasNoSignedWrap() || FI->hasNoUnsignedWrap())) { 106606c3fb27SDimitry Andric // The remaining subtract is not "nuw" any more. 106706c3fb27SDimitry Andric // If there's one use of the subtract (no other use than the use we are 106806c3fb27SDimitry Andric // about to replace), then we know that the sub is "nsw" in this context 106906c3fb27SDimitry Andric // even if it was only "nuw" before. If there's another use, then we can't 107006c3fb27SDimitry Andric // add "nsw" to the existing instruction because it may not be safe in the 107106c3fb27SDimitry Andric // other user's context. 107206c3fb27SDimitry Andric TI->setHasNoUnsignedWrap(false); 107306c3fb27SDimitry Andric if (!TI->hasNoSignedWrap()) 107406c3fb27SDimitry Andric TI->setHasNoSignedWrap(TI->hasOneUse()); 107506c3fb27SDimitry Andric return Builder.CreateBinaryIntrinsic(Intrinsic::abs, TI, Builder.getTrue()); 107606c3fb27SDimitry Andric } 107706c3fb27SDimitry Andric 107806c3fb27SDimitry Andric return nullptr; 107906c3fb27SDimitry Andric } 108006c3fb27SDimitry Andric 10818bcb0991SDimitry Andric /// Fold the following code sequence: 10828bcb0991SDimitry Andric /// \code 10838bcb0991SDimitry Andric /// int a = ctlz(x & -x); 10848bcb0991SDimitry Andric // x ? 31 - a : a; 108506c3fb27SDimitry Andric // // or 108606c3fb27SDimitry Andric // x ? 31 - a : 32; 10878bcb0991SDimitry Andric /// \code 10888bcb0991SDimitry Andric /// 10898bcb0991SDimitry Andric /// into: 10908bcb0991SDimitry Andric /// cttz(x) 10918bcb0991SDimitry Andric static Instruction *foldSelectCtlzToCttz(ICmpInst *ICI, Value *TrueVal, 10928bcb0991SDimitry Andric Value *FalseVal, 10938bcb0991SDimitry Andric InstCombiner::BuilderTy &Builder) { 10948bcb0991SDimitry Andric unsigned BitWidth = TrueVal->getType()->getScalarSizeInBits(); 10958bcb0991SDimitry Andric if (!ICI->isEquality() || !match(ICI->getOperand(1), m_Zero())) 10968bcb0991SDimitry Andric return nullptr; 10978bcb0991SDimitry Andric 10988bcb0991SDimitry Andric if (ICI->getPredicate() == ICmpInst::ICMP_NE) 10998bcb0991SDimitry Andric std::swap(TrueVal, FalseVal); 11008bcb0991SDimitry Andric 110106c3fb27SDimitry Andric Value *Ctlz; 11028bcb0991SDimitry Andric if (!match(FalseVal, 110306c3fb27SDimitry Andric m_Xor(m_Value(Ctlz), m_SpecificInt(BitWidth - 1)))) 11048bcb0991SDimitry Andric return nullptr; 11058bcb0991SDimitry Andric 110606c3fb27SDimitry Andric if (!match(Ctlz, m_Intrinsic<Intrinsic::ctlz>())) 110706c3fb27SDimitry Andric return nullptr; 110806c3fb27SDimitry Andric 110906c3fb27SDimitry Andric if (TrueVal != Ctlz && !match(TrueVal, m_SpecificInt(BitWidth))) 11108bcb0991SDimitry Andric return nullptr; 11118bcb0991SDimitry Andric 11128bcb0991SDimitry Andric Value *X = ICI->getOperand(0); 111306c3fb27SDimitry Andric auto *II = cast<IntrinsicInst>(Ctlz); 11148bcb0991SDimitry Andric if (!match(II->getOperand(0), m_c_And(m_Specific(X), m_Neg(m_Specific(X))))) 11158bcb0991SDimitry Andric return nullptr; 11168bcb0991SDimitry Andric 11178bcb0991SDimitry Andric Function *F = Intrinsic::getDeclaration(II->getModule(), Intrinsic::cttz, 11188bcb0991SDimitry Andric II->getType()); 11198bcb0991SDimitry Andric return CallInst::Create(F, {X, II->getArgOperand(1)}); 11208bcb0991SDimitry Andric } 11218bcb0991SDimitry Andric 11220b57cec5SDimitry Andric /// Attempt to fold a cttz/ctlz followed by a icmp plus select into a single 112304eeddc0SDimitry Andric /// call to cttz/ctlz with flag 'is_zero_poison' cleared. 11240b57cec5SDimitry Andric /// 11250b57cec5SDimitry Andric /// For example, we can fold the following code sequence: 11260b57cec5SDimitry Andric /// \code 11270b57cec5SDimitry Andric /// %0 = tail call i32 @llvm.cttz.i32(i32 %x, i1 true) 11280b57cec5SDimitry Andric /// %1 = icmp ne i32 %x, 0 11290b57cec5SDimitry Andric /// %2 = select i1 %1, i32 %0, i32 32 11300b57cec5SDimitry Andric /// \code 11310b57cec5SDimitry Andric /// 11320b57cec5SDimitry Andric /// into: 11330b57cec5SDimitry Andric /// %0 = tail call i32 @llvm.cttz.i32(i32 %x, i1 false) 11340b57cec5SDimitry Andric static Value *foldSelectCttzCtlz(ICmpInst *ICI, Value *TrueVal, Value *FalseVal, 1135*0fca6ea1SDimitry Andric InstCombinerImpl &IC) { 11360b57cec5SDimitry Andric ICmpInst::Predicate Pred = ICI->getPredicate(); 11370b57cec5SDimitry Andric Value *CmpLHS = ICI->getOperand(0); 11380b57cec5SDimitry Andric Value *CmpRHS = ICI->getOperand(1); 11390b57cec5SDimitry Andric 1140bdd1243dSDimitry Andric // Check if the select condition compares a value for equality. 1141bdd1243dSDimitry Andric if (!ICI->isEquality()) 11420b57cec5SDimitry Andric return nullptr; 11430b57cec5SDimitry Andric 11445ffd83dbSDimitry Andric Value *SelectArg = FalseVal; 11450b57cec5SDimitry Andric Value *ValueOnZero = TrueVal; 11460b57cec5SDimitry Andric if (Pred == ICmpInst::ICMP_NE) 11475ffd83dbSDimitry Andric std::swap(SelectArg, ValueOnZero); 11480b57cec5SDimitry Andric 11490b57cec5SDimitry Andric // Skip zero extend/truncate. 11505ffd83dbSDimitry Andric Value *Count = nullptr; 11515ffd83dbSDimitry Andric if (!match(SelectArg, m_ZExt(m_Value(Count))) && 11525ffd83dbSDimitry Andric !match(SelectArg, m_Trunc(m_Value(Count)))) 11535ffd83dbSDimitry Andric Count = SelectArg; 11540b57cec5SDimitry Andric 11550b57cec5SDimitry Andric // Check that 'Count' is a call to intrinsic cttz/ctlz. Also check that the 11560b57cec5SDimitry Andric // input to the cttz/ctlz is used as LHS for the compare instruction. 1157bdd1243dSDimitry Andric Value *X; 1158bdd1243dSDimitry Andric if (!match(Count, m_Intrinsic<Intrinsic::cttz>(m_Value(X))) && 1159bdd1243dSDimitry Andric !match(Count, m_Intrinsic<Intrinsic::ctlz>(m_Value(X)))) 1160bdd1243dSDimitry Andric return nullptr; 1161bdd1243dSDimitry Andric 1162bdd1243dSDimitry Andric // (X == 0) ? BitWidth : ctz(X) 1163bdd1243dSDimitry Andric // (X == -1) ? BitWidth : ctz(~X) 1164bdd1243dSDimitry Andric if ((X != CmpLHS || !match(CmpRHS, m_Zero())) && 1165bdd1243dSDimitry Andric (!match(X, m_Not(m_Specific(CmpLHS))) || !match(CmpRHS, m_AllOnes()))) 11660b57cec5SDimitry Andric return nullptr; 11670b57cec5SDimitry Andric 11680b57cec5SDimitry Andric IntrinsicInst *II = cast<IntrinsicInst>(Count); 11690b57cec5SDimitry Andric 11700b57cec5SDimitry Andric // Check if the value propagated on zero is a constant number equal to the 11710b57cec5SDimitry Andric // sizeof in bits of 'Count'. 11720b57cec5SDimitry Andric unsigned SizeOfInBits = Count->getType()->getScalarSizeInBits(); 11730b57cec5SDimitry Andric if (match(ValueOnZero, m_SpecificInt(SizeOfInBits))) { 117404eeddc0SDimitry Andric // Explicitly clear the 'is_zero_poison' flag. It's always valid to go from 11755ffd83dbSDimitry Andric // true to false on this flag, so we can replace it for all users. 11765ffd83dbSDimitry Andric II->setArgOperand(1, ConstantInt::getFalse(II->getContext())); 1177*0fca6ea1SDimitry Andric // A range annotation on the intrinsic may no longer be valid. 1178*0fca6ea1SDimitry Andric II->dropPoisonGeneratingAnnotations(); 1179*0fca6ea1SDimitry Andric IC.addToWorklist(II); 11805ffd83dbSDimitry Andric return SelectArg; 11810b57cec5SDimitry Andric } 11820b57cec5SDimitry Andric 11835ffd83dbSDimitry Andric // The ValueOnZero is not the bitwidth. But if the cttz/ctlz (and optional 11845ffd83dbSDimitry Andric // zext/trunc) have one use (ending at the select), the cttz/ctlz result will 118504eeddc0SDimitry Andric // not be used if the input is zero. Relax to 'zero is poison' for that case. 11865ffd83dbSDimitry Andric if (II->hasOneUse() && SelectArg->hasOneUse() && 11875ffd83dbSDimitry Andric !match(II->getArgOperand(1), m_One())) 11880b57cec5SDimitry Andric II->setArgOperand(1, ConstantInt::getTrue(II->getContext())); 11890b57cec5SDimitry Andric 11900b57cec5SDimitry Andric return nullptr; 11910b57cec5SDimitry Andric } 11920b57cec5SDimitry Andric 1193647cbc5dSDimitry Andric static Value *canonicalizeSPF(ICmpInst &Cmp, Value *TrueVal, Value *FalseVal, 1194e8d8bef9SDimitry Andric InstCombinerImpl &IC) { 11950b57cec5SDimitry Andric Value *LHS, *RHS; 119681ad6265SDimitry Andric // TODO: What to do with pointer min/max patterns? 1197647cbc5dSDimitry Andric if (!TrueVal->getType()->isIntOrIntVectorTy()) 11980b57cec5SDimitry Andric return nullptr; 11990b57cec5SDimitry Andric 1200647cbc5dSDimitry Andric SelectPatternFlavor SPF = 1201647cbc5dSDimitry Andric matchDecomposedSelectPattern(&Cmp, TrueVal, FalseVal, LHS, RHS).Flavor; 120281ad6265SDimitry Andric if (SPF == SelectPatternFlavor::SPF_ABS || 120381ad6265SDimitry Andric SPF == SelectPatternFlavor::SPF_NABS) { 120481ad6265SDimitry Andric if (!Cmp.hasOneUse() && !RHS->hasOneUse()) 120581ad6265SDimitry Andric return nullptr; // TODO: Relax this restriction. 12060b57cec5SDimitry Andric 1207e8d8bef9SDimitry Andric // Note that NSW flag can only be propagated for normal, non-negated abs! 1208e8d8bef9SDimitry Andric bool IntMinIsPoison = SPF == SelectPatternFlavor::SPF_ABS && 1209e8d8bef9SDimitry Andric match(RHS, m_NSWNeg(m_Specific(LHS))); 1210e8d8bef9SDimitry Andric Constant *IntMinIsPoisonC = 1211647cbc5dSDimitry Andric ConstantInt::get(Type::getInt1Ty(Cmp.getContext()), IntMinIsPoison); 1212*0fca6ea1SDimitry Andric Value *Abs = 1213e8d8bef9SDimitry Andric IC.Builder.CreateBinaryIntrinsic(Intrinsic::abs, LHS, IntMinIsPoisonC); 12140b57cec5SDimitry Andric 1215e8d8bef9SDimitry Andric if (SPF == SelectPatternFlavor::SPF_NABS) 1216647cbc5dSDimitry Andric return IC.Builder.CreateNeg(Abs); // Always without NSW flag! 1217647cbc5dSDimitry Andric return Abs; 12180b57cec5SDimitry Andric } 12190b57cec5SDimitry Andric 122081ad6265SDimitry Andric if (SelectPatternResult::isMinOrMax(SPF)) { 122181ad6265SDimitry Andric Intrinsic::ID IntrinsicID; 122281ad6265SDimitry Andric switch (SPF) { 122381ad6265SDimitry Andric case SelectPatternFlavor::SPF_UMIN: 122481ad6265SDimitry Andric IntrinsicID = Intrinsic::umin; 122581ad6265SDimitry Andric break; 122681ad6265SDimitry Andric case SelectPatternFlavor::SPF_UMAX: 122781ad6265SDimitry Andric IntrinsicID = Intrinsic::umax; 122881ad6265SDimitry Andric break; 122981ad6265SDimitry Andric case SelectPatternFlavor::SPF_SMIN: 123081ad6265SDimitry Andric IntrinsicID = Intrinsic::smin; 123181ad6265SDimitry Andric break; 123281ad6265SDimitry Andric case SelectPatternFlavor::SPF_SMAX: 123381ad6265SDimitry Andric IntrinsicID = Intrinsic::smax; 123481ad6265SDimitry Andric break; 123581ad6265SDimitry Andric default: 123681ad6265SDimitry Andric llvm_unreachable("Unexpected SPF"); 123781ad6265SDimitry Andric } 1238647cbc5dSDimitry Andric return IC.Builder.CreateBinaryIntrinsic(IntrinsicID, LHS, RHS); 123981ad6265SDimitry Andric } 124081ad6265SDimitry Andric 124181ad6265SDimitry Andric return nullptr; 124281ad6265SDimitry Andric } 124381ad6265SDimitry Andric 124406c3fb27SDimitry Andric bool InstCombinerImpl::replaceInInstruction(Value *V, Value *Old, Value *New, 124506c3fb27SDimitry Andric unsigned Depth) { 1246bdd1243dSDimitry Andric // Conservatively limit replacement to two instructions upwards. 1247bdd1243dSDimitry Andric if (Depth == 2) 1248bdd1243dSDimitry Andric return false; 1249bdd1243dSDimitry Andric 1250*0fca6ea1SDimitry Andric assert(!isa<Constant>(Old) && "Only replace non-constant values"); 1251*0fca6ea1SDimitry Andric 1252bdd1243dSDimitry Andric auto *I = dyn_cast<Instruction>(V); 1253*0fca6ea1SDimitry Andric if (!I || !I->hasOneUse() || 1254*0fca6ea1SDimitry Andric !isSafeToSpeculativelyExecuteWithVariableReplaced(I)) 1255bdd1243dSDimitry Andric return false; 1256bdd1243dSDimitry Andric 1257bdd1243dSDimitry Andric bool Changed = false; 1258bdd1243dSDimitry Andric for (Use &U : I->operands()) { 1259bdd1243dSDimitry Andric if (U == Old) { 126006c3fb27SDimitry Andric replaceUse(U, New); 126106c3fb27SDimitry Andric Worklist.add(I); 1262bdd1243dSDimitry Andric Changed = true; 1263bdd1243dSDimitry Andric } else { 126406c3fb27SDimitry Andric Changed |= replaceInInstruction(U, Old, New, Depth + 1); 1265bdd1243dSDimitry Andric } 1266bdd1243dSDimitry Andric } 1267bdd1243dSDimitry Andric return Changed; 1268bdd1243dSDimitry Andric } 1269bdd1243dSDimitry Andric 12708bcb0991SDimitry Andric /// If we have a select with an equality comparison, then we know the value in 12718bcb0991SDimitry Andric /// one of the arms of the select. See if substituting this value into an arm 12728bcb0991SDimitry Andric /// and simplifying the result yields the same value as the other arm. 12738bcb0991SDimitry Andric /// 12748bcb0991SDimitry Andric /// To make this transform safe, we must drop poison-generating flags 12758bcb0991SDimitry Andric /// (nsw, etc) if we simplified to a binop because the select may be guarding 12768bcb0991SDimitry Andric /// that poison from propagating. If the existing binop already had no 12778bcb0991SDimitry Andric /// poison-generating flags, then this transform can be done by instsimplify. 12788bcb0991SDimitry Andric /// 12798bcb0991SDimitry Andric /// Consider: 12808bcb0991SDimitry Andric /// %cmp = icmp eq i32 %x, 2147483647 12818bcb0991SDimitry Andric /// %add = add nsw i32 %x, 1 12828bcb0991SDimitry Andric /// %sel = select i1 %cmp, i32 -2147483648, i32 %add 12838bcb0991SDimitry Andric /// 12848bcb0991SDimitry Andric /// We can't replace %sel with %add unless we strip away the flags. 12858bcb0991SDimitry Andric /// TODO: Wrapping flags could be preserved in some cases with better analysis. 1286e8d8bef9SDimitry Andric Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel, 1287e8d8bef9SDimitry Andric ICmpInst &Cmp) { 1288bdd1243dSDimitry Andric if (!Cmp.isEquality()) 12898bcb0991SDimitry Andric return nullptr; 12908bcb0991SDimitry Andric 12918bcb0991SDimitry Andric // Canonicalize the pattern to ICMP_EQ by swapping the select operands. 12928bcb0991SDimitry Andric Value *TrueVal = Sel.getTrueValue(), *FalseVal = Sel.getFalseValue(); 1293e8d8bef9SDimitry Andric bool Swapped = false; 1294e8d8bef9SDimitry Andric if (Cmp.getPredicate() == ICmpInst::ICMP_NE) { 12958bcb0991SDimitry Andric std::swap(TrueVal, FalseVal); 1296e8d8bef9SDimitry Andric Swapped = true; 1297e8d8bef9SDimitry Andric } 1298e8d8bef9SDimitry Andric 1299e8d8bef9SDimitry Andric Value *CmpLHS = Cmp.getOperand(0), *CmpRHS = Cmp.getOperand(1); 1300*0fca6ea1SDimitry Andric auto ReplaceOldOpWithNewOp = [&](Value *OldOp, 1301*0fca6ea1SDimitry Andric Value *NewOp) -> Instruction * { 1302*0fca6ea1SDimitry Andric // In X == Y ? f(X) : Z, try to evaluate f(Y) and replace the operand. 1303*0fca6ea1SDimitry Andric // Take care to avoid replacing X == Y ? X : Z with X == Y ? Y : Z, as that 1304*0fca6ea1SDimitry Andric // would lead to an infinite replacement cycle. 1305*0fca6ea1SDimitry Andric // If we will be able to evaluate f(Y) to a constant, we can allow undef, 1306*0fca6ea1SDimitry Andric // otherwise Y cannot be undef as we might pick different values for undef 1307*0fca6ea1SDimitry Andric // in the icmp and in f(Y). 1308*0fca6ea1SDimitry Andric if (TrueVal == OldOp) 1309*0fca6ea1SDimitry Andric return nullptr; 1310*0fca6ea1SDimitry Andric 1311*0fca6ea1SDimitry Andric if (Value *V = simplifyWithOpReplaced(TrueVal, OldOp, NewOp, SQ, 1312*0fca6ea1SDimitry Andric /* AllowRefinement=*/true)) { 1313*0fca6ea1SDimitry Andric // Need some guarantees about the new simplified op to ensure we don't inf 1314*0fca6ea1SDimitry Andric // loop. 1315*0fca6ea1SDimitry Andric // If we simplify to a constant, replace if we aren't creating new undef. 1316*0fca6ea1SDimitry Andric if (match(V, m_ImmConstant()) && 1317*0fca6ea1SDimitry Andric isGuaranteedNotToBeUndef(V, SQ.AC, &Sel, &DT)) 1318e8d8bef9SDimitry Andric return replaceOperand(Sel, Swapped ? 2 : 1, V); 1319e8d8bef9SDimitry Andric 1320*0fca6ea1SDimitry Andric // If NewOp is a constant and OldOp is not replace iff NewOp doesn't 1321*0fca6ea1SDimitry Andric // contain and undef elements. 1322*0fca6ea1SDimitry Andric if (match(NewOp, m_ImmConstant()) || NewOp == V) { 1323*0fca6ea1SDimitry Andric if (isGuaranteedNotToBeUndef(NewOp, SQ.AC, &Sel, &DT)) 1324*0fca6ea1SDimitry Andric return replaceOperand(Sel, Swapped ? 2 : 1, V); 1325*0fca6ea1SDimitry Andric return nullptr; 1326*0fca6ea1SDimitry Andric } 1327*0fca6ea1SDimitry Andric } 1328*0fca6ea1SDimitry Andric 1329e8d8bef9SDimitry Andric // Even if TrueVal does not simplify, we can directly replace a use of 1330e8d8bef9SDimitry Andric // CmpLHS with CmpRHS, as long as the instruction is not used anywhere 1331e8d8bef9SDimitry Andric // else and is safe to speculatively execute (we may end up executing it 1332e8d8bef9SDimitry Andric // with different operands, which should not cause side-effects or trigger 1333e8d8bef9SDimitry Andric // undefined behavior). Only do this if CmpRHS is a constant, as 1334e8d8bef9SDimitry Andric // profitability is not clear for other cases. 1335bdd1243dSDimitry Andric // FIXME: Support vectors. 1336*0fca6ea1SDimitry Andric if (OldOp == CmpLHS && match(NewOp, m_ImmConstant()) && 1337*0fca6ea1SDimitry Andric !match(OldOp, m_Constant()) && !Cmp.getType()->isVectorTy() && 1338*0fca6ea1SDimitry Andric isGuaranteedNotToBeUndef(NewOp, SQ.AC, &Sel, &DT)) 1339*0fca6ea1SDimitry Andric if (replaceInInstruction(TrueVal, OldOp, NewOp)) 1340e8d8bef9SDimitry Andric return &Sel; 1341*0fca6ea1SDimitry Andric return nullptr; 1342*0fca6ea1SDimitry Andric }; 1343*0fca6ea1SDimitry Andric 1344*0fca6ea1SDimitry Andric if (Instruction *R = ReplaceOldOpWithNewOp(CmpLHS, CmpRHS)) 1345*0fca6ea1SDimitry Andric return R; 1346*0fca6ea1SDimitry Andric if (Instruction *R = ReplaceOldOpWithNewOp(CmpRHS, CmpLHS)) 1347*0fca6ea1SDimitry Andric return R; 13488bcb0991SDimitry Andric 134916d6b3b3SDimitry Andric auto *FalseInst = dyn_cast<Instruction>(FalseVal); 135016d6b3b3SDimitry Andric if (!FalseInst) 135116d6b3b3SDimitry Andric return nullptr; 135216d6b3b3SDimitry Andric 135316d6b3b3SDimitry Andric // InstSimplify already performed this fold if it was possible subject to 13545f757f3fSDimitry Andric // current poison-generating flags. Check whether dropping poison-generating 13555f757f3fSDimitry Andric // flags enables the transform. 135616d6b3b3SDimitry Andric 13578bcb0991SDimitry Andric // Try each equivalence substitution possibility. 13588bcb0991SDimitry Andric // We have an 'EQ' comparison, so the select's false value will propagate. 13598bcb0991SDimitry Andric // Example: 13608bcb0991SDimitry Andric // (X == 42) ? 43 : (X + 1) --> (X == 42) ? (X + 1) : (X + 1) --> X + 1 13615f757f3fSDimitry Andric SmallVector<Instruction *> DropFlags; 1362fe6060f1SDimitry Andric if (simplifyWithOpReplaced(FalseVal, CmpLHS, CmpRHS, SQ, 13635f757f3fSDimitry Andric /* AllowRefinement */ false, 13645f757f3fSDimitry Andric &DropFlags) == TrueVal || 1365fe6060f1SDimitry Andric simplifyWithOpReplaced(FalseVal, CmpRHS, CmpLHS, SQ, 13665f757f3fSDimitry Andric /* AllowRefinement */ false, 13675f757f3fSDimitry Andric &DropFlags) == TrueVal) { 13685f757f3fSDimitry Andric for (Instruction *I : DropFlags) { 1369*0fca6ea1SDimitry Andric I->dropPoisonGeneratingAnnotations(); 13705f757f3fSDimitry Andric Worklist.add(I); 13718bcb0991SDimitry Andric } 137216d6b3b3SDimitry Andric 13735f757f3fSDimitry Andric return replaceInstUsesWith(Sel, FalseVal); 13745f757f3fSDimitry Andric } 137516d6b3b3SDimitry Andric 13768bcb0991SDimitry Andric return nullptr; 13778bcb0991SDimitry Andric } 13788bcb0991SDimitry Andric 13798bcb0991SDimitry Andric // See if this is a pattern like: 13808bcb0991SDimitry Andric // %old_cmp1 = icmp slt i32 %x, C2 13818bcb0991SDimitry Andric // %old_replacement = select i1 %old_cmp1, i32 %target_low, i32 %target_high 13828bcb0991SDimitry Andric // %old_x_offseted = add i32 %x, C1 13838bcb0991SDimitry Andric // %old_cmp0 = icmp ult i32 %old_x_offseted, C0 13848bcb0991SDimitry Andric // %r = select i1 %old_cmp0, i32 %x, i32 %old_replacement 13858bcb0991SDimitry Andric // This can be rewritten as more canonical pattern: 13868bcb0991SDimitry Andric // %new_cmp1 = icmp slt i32 %x, -C1 13878bcb0991SDimitry Andric // %new_cmp2 = icmp sge i32 %x, C0-C1 13888bcb0991SDimitry Andric // %new_clamped_low = select i1 %new_cmp1, i32 %target_low, i32 %x 13898bcb0991SDimitry Andric // %r = select i1 %new_cmp2, i32 %target_high, i32 %new_clamped_low 13908bcb0991SDimitry Andric // Iff -C1 s<= C2 s<= C0-C1 13918bcb0991SDimitry Andric // Also ULT predicate can also be UGT iff C0 != -1 (+invert result) 13928bcb0991SDimitry Andric // SLT predicate can also be SGT iff C2 != INT_MAX (+invert res.) 1393349cc55cSDimitry Andric static Value *canonicalizeClampLike(SelectInst &Sel0, ICmpInst &Cmp0, 1394*0fca6ea1SDimitry Andric InstCombiner::BuilderTy &Builder, 1395*0fca6ea1SDimitry Andric InstCombiner &IC) { 13968bcb0991SDimitry Andric Value *X = Sel0.getTrueValue(); 13978bcb0991SDimitry Andric Value *Sel1 = Sel0.getFalseValue(); 13988bcb0991SDimitry Andric 13998bcb0991SDimitry Andric // First match the condition of the outermost select. 14008bcb0991SDimitry Andric // Said condition must be one-use. 14018bcb0991SDimitry Andric if (!Cmp0.hasOneUse()) 14028bcb0991SDimitry Andric return nullptr; 1403349cc55cSDimitry Andric ICmpInst::Predicate Pred0 = Cmp0.getPredicate(); 14048bcb0991SDimitry Andric Value *Cmp00 = Cmp0.getOperand(0); 14058bcb0991SDimitry Andric Constant *C0; 14068bcb0991SDimitry Andric if (!match(Cmp0.getOperand(1), 14078bcb0991SDimitry Andric m_CombineAnd(m_AnyIntegralConstant(), m_Constant(C0)))) 14088bcb0991SDimitry Andric return nullptr; 1409349cc55cSDimitry Andric 1410349cc55cSDimitry Andric if (!isa<SelectInst>(Sel1)) { 1411349cc55cSDimitry Andric Pred0 = ICmpInst::getInversePredicate(Pred0); 1412349cc55cSDimitry Andric std::swap(X, Sel1); 1413349cc55cSDimitry Andric } 1414349cc55cSDimitry Andric 1415349cc55cSDimitry Andric // Canonicalize Cmp0 into ult or uge. 14168bcb0991SDimitry Andric // FIXME: we shouldn't care about lanes that are 'undef' in the end? 1417349cc55cSDimitry Andric switch (Pred0) { 14188bcb0991SDimitry Andric case ICmpInst::Predicate::ICMP_ULT: 1419349cc55cSDimitry Andric case ICmpInst::Predicate::ICMP_UGE: 1420349cc55cSDimitry Andric // Although icmp ult %x, 0 is an unusual thing to try and should generally 1421349cc55cSDimitry Andric // have been simplified, it does not verify with undef inputs so ensure we 1422349cc55cSDimitry Andric // are not in a strange state. 1423349cc55cSDimitry Andric if (!match(C0, m_SpecificInt_ICMP( 1424349cc55cSDimitry Andric ICmpInst::Predicate::ICMP_NE, 1425349cc55cSDimitry Andric APInt::getZero(C0->getType()->getScalarSizeInBits())))) 1426349cc55cSDimitry Andric return nullptr; 14278bcb0991SDimitry Andric break; // Great! 14288bcb0991SDimitry Andric case ICmpInst::Predicate::ICMP_ULE: 14298bcb0991SDimitry Andric case ICmpInst::Predicate::ICMP_UGT: 1430349cc55cSDimitry Andric // We want to canonicalize it to 'ult' or 'uge', so we'll need to increment 1431349cc55cSDimitry Andric // C0, which again means it must not have any all-ones elements. 14328bcb0991SDimitry Andric if (!match(C0, 1433349cc55cSDimitry Andric m_SpecificInt_ICMP( 1434349cc55cSDimitry Andric ICmpInst::Predicate::ICMP_NE, 1435349cc55cSDimitry Andric APInt::getAllOnes(C0->getType()->getScalarSizeInBits())))) 14368bcb0991SDimitry Andric return nullptr; // Can't do, have all-ones element[s]. 143781ad6265SDimitry Andric Pred0 = ICmpInst::getFlippedStrictnessPredicate(Pred0); 1438e8d8bef9SDimitry Andric C0 = InstCombiner::AddOne(C0); 14398bcb0991SDimitry Andric break; 14408bcb0991SDimitry Andric default: 14418bcb0991SDimitry Andric return nullptr; // Unknown predicate. 14428bcb0991SDimitry Andric } 14438bcb0991SDimitry Andric 14448bcb0991SDimitry Andric // Now that we've canonicalized the ICmp, we know the X we expect; 14458bcb0991SDimitry Andric // the select in other hand should be one-use. 14468bcb0991SDimitry Andric if (!Sel1->hasOneUse()) 14478bcb0991SDimitry Andric return nullptr; 14488bcb0991SDimitry Andric 1449349cc55cSDimitry Andric // If the types do not match, look through any truncs to the underlying 1450349cc55cSDimitry Andric // instruction. 1451349cc55cSDimitry Andric if (Cmp00->getType() != X->getType() && X->hasOneUse()) 1452349cc55cSDimitry Andric match(X, m_TruncOrSelf(m_Value(X))); 1453349cc55cSDimitry Andric 14548bcb0991SDimitry Andric // We now can finish matching the condition of the outermost select: 14558bcb0991SDimitry Andric // it should either be the X itself, or an addition of some constant to X. 14568bcb0991SDimitry Andric Constant *C1; 14578bcb0991SDimitry Andric if (Cmp00 == X) 1458349cc55cSDimitry Andric C1 = ConstantInt::getNullValue(X->getType()); 14598bcb0991SDimitry Andric else if (!match(Cmp00, 14608bcb0991SDimitry Andric m_Add(m_Specific(X), 14618bcb0991SDimitry Andric m_CombineAnd(m_AnyIntegralConstant(), m_Constant(C1))))) 14628bcb0991SDimitry Andric return nullptr; 14638bcb0991SDimitry Andric 14648bcb0991SDimitry Andric Value *Cmp1; 14658bcb0991SDimitry Andric ICmpInst::Predicate Pred1; 14668bcb0991SDimitry Andric Constant *C2; 14678bcb0991SDimitry Andric Value *ReplacementLow, *ReplacementHigh; 14688bcb0991SDimitry Andric if (!match(Sel1, m_Select(m_Value(Cmp1), m_Value(ReplacementLow), 14698bcb0991SDimitry Andric m_Value(ReplacementHigh))) || 14708bcb0991SDimitry Andric !match(Cmp1, 14718bcb0991SDimitry Andric m_ICmp(Pred1, m_Specific(X), 14728bcb0991SDimitry Andric m_CombineAnd(m_AnyIntegralConstant(), m_Constant(C2))))) 14738bcb0991SDimitry Andric return nullptr; 14748bcb0991SDimitry Andric 14758bcb0991SDimitry Andric if (!Cmp1->hasOneUse() && (Cmp00 == X || !Cmp00->hasOneUse())) 14768bcb0991SDimitry Andric return nullptr; // Not enough one-use instructions for the fold. 14778bcb0991SDimitry Andric // FIXME: this restriction could be relaxed if Cmp1 can be reused as one of 14788bcb0991SDimitry Andric // two comparisons we'll need to build. 14798bcb0991SDimitry Andric 14808bcb0991SDimitry Andric // Canonicalize Cmp1 into the form we expect. 14818bcb0991SDimitry Andric // FIXME: we shouldn't care about lanes that are 'undef' in the end? 14828bcb0991SDimitry Andric switch (Pred1) { 14838bcb0991SDimitry Andric case ICmpInst::Predicate::ICMP_SLT: 14848bcb0991SDimitry Andric break; 14858bcb0991SDimitry Andric case ICmpInst::Predicate::ICMP_SLE: 14868bcb0991SDimitry Andric // We'd have to increment C2 by one, and for that it must not have signed 14878bcb0991SDimitry Andric // max element, but then it would have been canonicalized to 'slt' before 14888bcb0991SDimitry Andric // we get here. So we can't do anything useful with 'sle'. 14898bcb0991SDimitry Andric return nullptr; 14908bcb0991SDimitry Andric case ICmpInst::Predicate::ICMP_SGT: 14918bcb0991SDimitry Andric // We want to canonicalize it to 'slt', so we'll need to increment C2, 14928bcb0991SDimitry Andric // which again means it must not have any signed max elements. 14938bcb0991SDimitry Andric if (!match(C2, 14948bcb0991SDimitry Andric m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_NE, 14958bcb0991SDimitry Andric APInt::getSignedMaxValue( 14968bcb0991SDimitry Andric C2->getType()->getScalarSizeInBits())))) 14978bcb0991SDimitry Andric return nullptr; // Can't do, have signed max element[s]. 1498e8d8bef9SDimitry Andric C2 = InstCombiner::AddOne(C2); 1499bdd1243dSDimitry Andric [[fallthrough]]; 15008bcb0991SDimitry Andric case ICmpInst::Predicate::ICMP_SGE: 15018bcb0991SDimitry Andric // Also non-canonical, but here we don't need to change C2, 15028bcb0991SDimitry Andric // so we don't have any restrictions on C2, so we can just handle it. 150381ad6265SDimitry Andric Pred1 = ICmpInst::Predicate::ICMP_SLT; 15048bcb0991SDimitry Andric std::swap(ReplacementLow, ReplacementHigh); 15058bcb0991SDimitry Andric break; 15068bcb0991SDimitry Andric default: 15078bcb0991SDimitry Andric return nullptr; // Unknown predicate. 15088bcb0991SDimitry Andric } 150981ad6265SDimitry Andric assert(Pred1 == ICmpInst::Predicate::ICMP_SLT && 151081ad6265SDimitry Andric "Unexpected predicate type."); 15118bcb0991SDimitry Andric 15128bcb0991SDimitry Andric // The thresholds of this clamp-like pattern. 15138bcb0991SDimitry Andric auto *ThresholdLowIncl = ConstantExpr::getNeg(C1); 15148bcb0991SDimitry Andric auto *ThresholdHighExcl = ConstantExpr::getSub(C0, C1); 151581ad6265SDimitry Andric 151681ad6265SDimitry Andric assert((Pred0 == ICmpInst::Predicate::ICMP_ULT || 151781ad6265SDimitry Andric Pred0 == ICmpInst::Predicate::ICMP_UGE) && 151881ad6265SDimitry Andric "Unexpected predicate type."); 1519349cc55cSDimitry Andric if (Pred0 == ICmpInst::Predicate::ICMP_UGE) 1520349cc55cSDimitry Andric std::swap(ThresholdLowIncl, ThresholdHighExcl); 15218bcb0991SDimitry Andric 15228bcb0991SDimitry Andric // The fold has a precondition 1: C2 s>= ThresholdLow 1523*0fca6ea1SDimitry Andric auto *Precond1 = ConstantFoldCompareInstOperands( 1524*0fca6ea1SDimitry Andric ICmpInst::Predicate::ICMP_SGE, C2, ThresholdLowIncl, IC.getDataLayout()); 1525*0fca6ea1SDimitry Andric if (!Precond1 || !match(Precond1, m_One())) 15268bcb0991SDimitry Andric return nullptr; 15278bcb0991SDimitry Andric // The fold has a precondition 2: C2 s<= ThresholdHigh 1528*0fca6ea1SDimitry Andric auto *Precond2 = ConstantFoldCompareInstOperands( 1529*0fca6ea1SDimitry Andric ICmpInst::Predicate::ICMP_SLE, C2, ThresholdHighExcl, IC.getDataLayout()); 1530*0fca6ea1SDimitry Andric if (!Precond2 || !match(Precond2, m_One())) 15318bcb0991SDimitry Andric return nullptr; 15328bcb0991SDimitry Andric 1533349cc55cSDimitry Andric // If we are matching from a truncated input, we need to sext the 1534349cc55cSDimitry Andric // ReplacementLow and ReplacementHigh values. Only do the transform if they 1535349cc55cSDimitry Andric // are free to extend due to being constants. 1536349cc55cSDimitry Andric if (X->getType() != Sel0.getType()) { 1537349cc55cSDimitry Andric Constant *LowC, *HighC; 1538349cc55cSDimitry Andric if (!match(ReplacementLow, m_ImmConstant(LowC)) || 1539349cc55cSDimitry Andric !match(ReplacementHigh, m_ImmConstant(HighC))) 1540349cc55cSDimitry Andric return nullptr; 1541*0fca6ea1SDimitry Andric const DataLayout &DL = Sel0.getDataLayout(); 15425f757f3fSDimitry Andric ReplacementLow = 15435f757f3fSDimitry Andric ConstantFoldCastOperand(Instruction::SExt, LowC, X->getType(), DL); 15445f757f3fSDimitry Andric ReplacementHigh = 15455f757f3fSDimitry Andric ConstantFoldCastOperand(Instruction::SExt, HighC, X->getType(), DL); 15465f757f3fSDimitry Andric assert(ReplacementLow && ReplacementHigh && 15475f757f3fSDimitry Andric "Constant folding of ImmConstant cannot fail"); 1548349cc55cSDimitry Andric } 1549349cc55cSDimitry Andric 15508bcb0991SDimitry Andric // All good, finally emit the new pattern. 15518bcb0991SDimitry Andric Value *ShouldReplaceLow = Builder.CreateICmpSLT(X, ThresholdLowIncl); 15528bcb0991SDimitry Andric Value *ShouldReplaceHigh = Builder.CreateICmpSGE(X, ThresholdHighExcl); 15538bcb0991SDimitry Andric Value *MaybeReplacedLow = 15548bcb0991SDimitry Andric Builder.CreateSelect(ShouldReplaceLow, ReplacementLow, X); 15558bcb0991SDimitry Andric 1556349cc55cSDimitry Andric // Create the final select. If we looked through a truncate above, we will 1557349cc55cSDimitry Andric // need to retruncate the result. 1558349cc55cSDimitry Andric Value *MaybeReplacedHigh = Builder.CreateSelect( 1559349cc55cSDimitry Andric ShouldReplaceHigh, ReplacementHigh, MaybeReplacedLow); 1560349cc55cSDimitry Andric return Builder.CreateTrunc(MaybeReplacedHigh, Sel0.getType()); 15618bcb0991SDimitry Andric } 15628bcb0991SDimitry Andric 15638bcb0991SDimitry Andric // If we have 15648bcb0991SDimitry Andric // %cmp = icmp [canonical predicate] i32 %x, C0 15658bcb0991SDimitry Andric // %r = select i1 %cmp, i32 %y, i32 C1 15668bcb0991SDimitry Andric // Where C0 != C1 and %x may be different from %y, see if the constant that we 15678bcb0991SDimitry Andric // will have if we flip the strictness of the predicate (i.e. without changing 15688bcb0991SDimitry Andric // the result) is identical to the C1 in select. If it matches we can change 15698bcb0991SDimitry Andric // original comparison to one with swapped predicate, reuse the constant, 15708bcb0991SDimitry Andric // and swap the hands of select. 15718bcb0991SDimitry Andric static Instruction * 15728bcb0991SDimitry Andric tryToReuseConstantFromSelectInComparison(SelectInst &Sel, ICmpInst &Cmp, 1573e8d8bef9SDimitry Andric InstCombinerImpl &IC) { 15748bcb0991SDimitry Andric ICmpInst::Predicate Pred; 15758bcb0991SDimitry Andric Value *X; 15768bcb0991SDimitry Andric Constant *C0; 15778bcb0991SDimitry Andric if (!match(&Cmp, m_OneUse(m_ICmp( 15788bcb0991SDimitry Andric Pred, m_Value(X), 15798bcb0991SDimitry Andric m_CombineAnd(m_AnyIntegralConstant(), m_Constant(C0)))))) 15808bcb0991SDimitry Andric return nullptr; 15818bcb0991SDimitry Andric 15828bcb0991SDimitry Andric // If comparison predicate is non-relational, we won't be able to do anything. 15838bcb0991SDimitry Andric if (ICmpInst::isEquality(Pred)) 15848bcb0991SDimitry Andric return nullptr; 15858bcb0991SDimitry Andric 15868bcb0991SDimitry Andric // If comparison predicate is non-canonical, then we certainly won't be able 15878bcb0991SDimitry Andric // to make it canonical; canonicalizeCmpWithConstant() already tried. 1588e8d8bef9SDimitry Andric if (!InstCombiner::isCanonicalPredicate(Pred)) 15898bcb0991SDimitry Andric return nullptr; 15908bcb0991SDimitry Andric 15918bcb0991SDimitry Andric // If the [input] type of comparison and select type are different, lets abort 15928bcb0991SDimitry Andric // for now. We could try to compare constants with trunc/[zs]ext though. 15938bcb0991SDimitry Andric if (C0->getType() != Sel.getType()) 15948bcb0991SDimitry Andric return nullptr; 15958bcb0991SDimitry Andric 15960eae32dcSDimitry Andric // ULT with 'add' of a constant is canonical. See foldICmpAddConstant(). 15970eae32dcSDimitry Andric // FIXME: Are there more magic icmp predicate+constant pairs we must avoid? 15980eae32dcSDimitry Andric // Or should we just abandon this transform entirely? 15990eae32dcSDimitry Andric if (Pred == CmpInst::ICMP_ULT && match(X, m_Add(m_Value(), m_Constant()))) 16000eae32dcSDimitry Andric return nullptr; 16010eae32dcSDimitry Andric 16028bcb0991SDimitry Andric 16038bcb0991SDimitry Andric Value *SelVal0, *SelVal1; // We do not care which one is from where. 16048bcb0991SDimitry Andric match(&Sel, m_Select(m_Value(), m_Value(SelVal0), m_Value(SelVal1))); 16058bcb0991SDimitry Andric // At least one of these values we are selecting between must be a constant 16068bcb0991SDimitry Andric // else we'll never succeed. 16078bcb0991SDimitry Andric if (!match(SelVal0, m_AnyIntegralConstant()) && 16088bcb0991SDimitry Andric !match(SelVal1, m_AnyIntegralConstant())) 16098bcb0991SDimitry Andric return nullptr; 16108bcb0991SDimitry Andric 16118bcb0991SDimitry Andric // Does this constant C match any of the `select` values? 16128bcb0991SDimitry Andric auto MatchesSelectValue = [SelVal0, SelVal1](Constant *C) { 16138bcb0991SDimitry Andric return C->isElementWiseEqual(SelVal0) || C->isElementWiseEqual(SelVal1); 16148bcb0991SDimitry Andric }; 16158bcb0991SDimitry Andric 16168bcb0991SDimitry Andric // If C0 *already* matches true/false value of select, we are done. 16178bcb0991SDimitry Andric if (MatchesSelectValue(C0)) 16188bcb0991SDimitry Andric return nullptr; 16198bcb0991SDimitry Andric 16208bcb0991SDimitry Andric // Check the constant we'd have with flipped-strictness predicate. 1621e8d8bef9SDimitry Andric auto FlippedStrictness = 1622e8d8bef9SDimitry Andric InstCombiner::getFlippedStrictnessPredicateAndConstant(Pred, C0); 16238bcb0991SDimitry Andric if (!FlippedStrictness) 16248bcb0991SDimitry Andric return nullptr; 16258bcb0991SDimitry Andric 16268bcb0991SDimitry Andric // If said constant doesn't match either, then there is no hope, 16278bcb0991SDimitry Andric if (!MatchesSelectValue(FlippedStrictness->second)) 16288bcb0991SDimitry Andric return nullptr; 16298bcb0991SDimitry Andric 16308bcb0991SDimitry Andric // It matched! Lets insert the new comparison just before select. 16315ffd83dbSDimitry Andric InstCombiner::BuilderTy::InsertPointGuard Guard(IC.Builder); 16325ffd83dbSDimitry Andric IC.Builder.SetInsertPoint(&Sel); 16338bcb0991SDimitry Andric 16348bcb0991SDimitry Andric Pred = ICmpInst::getSwappedPredicate(Pred); // Yes, swapped. 16355ffd83dbSDimitry Andric Value *NewCmp = IC.Builder.CreateICmp(Pred, X, FlippedStrictness->second, 16368bcb0991SDimitry Andric Cmp.getName() + ".inv"); 16375ffd83dbSDimitry Andric IC.replaceOperand(Sel, 0, NewCmp); 16388bcb0991SDimitry Andric Sel.swapValues(); 16398bcb0991SDimitry Andric Sel.swapProfMetadata(); 16408bcb0991SDimitry Andric 16418bcb0991SDimitry Andric return &Sel; 16428bcb0991SDimitry Andric } 16438bcb0991SDimitry Andric 164481ad6265SDimitry Andric static Instruction *foldSelectZeroOrOnes(ICmpInst *Cmp, Value *TVal, 164581ad6265SDimitry Andric Value *FVal, 164681ad6265SDimitry Andric InstCombiner::BuilderTy &Builder) { 164781ad6265SDimitry Andric if (!Cmp->hasOneUse()) 164881ad6265SDimitry Andric return nullptr; 164981ad6265SDimitry Andric 165081ad6265SDimitry Andric const APInt *CmpC; 1651*0fca6ea1SDimitry Andric if (!match(Cmp->getOperand(1), m_APIntAllowPoison(CmpC))) 165281ad6265SDimitry Andric return nullptr; 165381ad6265SDimitry Andric 165481ad6265SDimitry Andric // (X u< 2) ? -X : -1 --> sext (X != 0) 165581ad6265SDimitry Andric Value *X = Cmp->getOperand(0); 165681ad6265SDimitry Andric if (Cmp->getPredicate() == ICmpInst::ICMP_ULT && *CmpC == 2 && 165781ad6265SDimitry Andric match(TVal, m_Neg(m_Specific(X))) && match(FVal, m_AllOnes())) 165881ad6265SDimitry Andric return new SExtInst(Builder.CreateIsNotNull(X), TVal->getType()); 165981ad6265SDimitry Andric 166081ad6265SDimitry Andric // (X u> 1) ? -1 : -X --> sext (X != 0) 166181ad6265SDimitry Andric if (Cmp->getPredicate() == ICmpInst::ICMP_UGT && *CmpC == 1 && 166281ad6265SDimitry Andric match(FVal, m_Neg(m_Specific(X))) && match(TVal, m_AllOnes())) 166381ad6265SDimitry Andric return new SExtInst(Builder.CreateIsNotNull(X), TVal->getType()); 166481ad6265SDimitry Andric 166581ad6265SDimitry Andric return nullptr; 166681ad6265SDimitry Andric } 166781ad6265SDimitry Andric 166806c3fb27SDimitry Andric static Value *foldSelectInstWithICmpConst(SelectInst &SI, ICmpInst *ICI, 166906c3fb27SDimitry Andric InstCombiner::BuilderTy &Builder) { 167081ad6265SDimitry Andric const APInt *CmpC; 167181ad6265SDimitry Andric Value *V; 167281ad6265SDimitry Andric CmpInst::Predicate Pred; 167381ad6265SDimitry Andric if (!match(ICI, m_ICmp(Pred, m_Value(V), m_APInt(CmpC)))) 167481ad6265SDimitry Andric return nullptr; 167581ad6265SDimitry Andric 167606c3fb27SDimitry Andric // Match clamp away from min/max value as a max/min operation. 167706c3fb27SDimitry Andric Value *TVal = SI.getTrueValue(); 167806c3fb27SDimitry Andric Value *FVal = SI.getFalseValue(); 167906c3fb27SDimitry Andric if (Pred == ICmpInst::ICMP_EQ && V == FVal) { 168006c3fb27SDimitry Andric // (V == UMIN) ? UMIN+1 : V --> umax(V, UMIN+1) 168106c3fb27SDimitry Andric if (CmpC->isMinValue() && match(TVal, m_SpecificInt(*CmpC + 1))) 168206c3fb27SDimitry Andric return Builder.CreateBinaryIntrinsic(Intrinsic::umax, V, TVal); 168306c3fb27SDimitry Andric // (V == UMAX) ? UMAX-1 : V --> umin(V, UMAX-1) 168406c3fb27SDimitry Andric if (CmpC->isMaxValue() && match(TVal, m_SpecificInt(*CmpC - 1))) 168506c3fb27SDimitry Andric return Builder.CreateBinaryIntrinsic(Intrinsic::umin, V, TVal); 168606c3fb27SDimitry Andric // (V == SMIN) ? SMIN+1 : V --> smax(V, SMIN+1) 168706c3fb27SDimitry Andric if (CmpC->isMinSignedValue() && match(TVal, m_SpecificInt(*CmpC + 1))) 168806c3fb27SDimitry Andric return Builder.CreateBinaryIntrinsic(Intrinsic::smax, V, TVal); 168906c3fb27SDimitry Andric // (V == SMAX) ? SMAX-1 : V --> smin(V, SMAX-1) 169006c3fb27SDimitry Andric if (CmpC->isMaxSignedValue() && match(TVal, m_SpecificInt(*CmpC - 1))) 169106c3fb27SDimitry Andric return Builder.CreateBinaryIntrinsic(Intrinsic::smin, V, TVal); 169206c3fb27SDimitry Andric } 169306c3fb27SDimitry Andric 169481ad6265SDimitry Andric BinaryOperator *BO; 169581ad6265SDimitry Andric const APInt *C; 169681ad6265SDimitry Andric CmpInst::Predicate CPred; 169781ad6265SDimitry Andric if (match(&SI, m_Select(m_Specific(ICI), m_APInt(C), m_BinOp(BO)))) 169881ad6265SDimitry Andric CPred = ICI->getPredicate(); 169981ad6265SDimitry Andric else if (match(&SI, m_Select(m_Specific(ICI), m_BinOp(BO), m_APInt(C)))) 170081ad6265SDimitry Andric CPred = ICI->getInversePredicate(); 170181ad6265SDimitry Andric else 170281ad6265SDimitry Andric return nullptr; 170381ad6265SDimitry Andric 170481ad6265SDimitry Andric const APInt *BinOpC; 170581ad6265SDimitry Andric if (!match(BO, m_BinOp(m_Specific(V), m_APInt(BinOpC)))) 170681ad6265SDimitry Andric return nullptr; 170781ad6265SDimitry Andric 170881ad6265SDimitry Andric ConstantRange R = ConstantRange::makeExactICmpRegion(CPred, *CmpC) 170981ad6265SDimitry Andric .binaryOp(BO->getOpcode(), *BinOpC); 171081ad6265SDimitry Andric if (R == *C) { 171181ad6265SDimitry Andric BO->dropPoisonGeneratingFlags(); 171281ad6265SDimitry Andric return BO; 171381ad6265SDimitry Andric } 171481ad6265SDimitry Andric return nullptr; 171581ad6265SDimitry Andric } 171681ad6265SDimitry Andric 1717*0fca6ea1SDimitry Andric static Instruction *foldSelectICmpEq(SelectInst &SI, ICmpInst *ICI, 1718*0fca6ea1SDimitry Andric InstCombinerImpl &IC) { 1719*0fca6ea1SDimitry Andric ICmpInst::Predicate Pred = ICI->getPredicate(); 1720*0fca6ea1SDimitry Andric if (!ICmpInst::isEquality(Pred)) 1721*0fca6ea1SDimitry Andric return nullptr; 1722*0fca6ea1SDimitry Andric 1723*0fca6ea1SDimitry Andric Value *TrueVal = SI.getTrueValue(); 1724*0fca6ea1SDimitry Andric Value *FalseVal = SI.getFalseValue(); 1725*0fca6ea1SDimitry Andric Value *CmpLHS = ICI->getOperand(0); 1726*0fca6ea1SDimitry Andric Value *CmpRHS = ICI->getOperand(1); 1727*0fca6ea1SDimitry Andric 1728*0fca6ea1SDimitry Andric if (Pred == ICmpInst::ICMP_NE) 1729*0fca6ea1SDimitry Andric std::swap(TrueVal, FalseVal); 1730*0fca6ea1SDimitry Andric 1731*0fca6ea1SDimitry Andric // Transform (X == C) ? X : Y -> (X == C) ? C : Y 1732*0fca6ea1SDimitry Andric // specific handling for Bitwise operation. 1733*0fca6ea1SDimitry Andric // x&y -> (x|y) ^ (x^y) or (x|y) & ~(x^y) 1734*0fca6ea1SDimitry Andric // x|y -> (x&y) | (x^y) or (x&y) ^ (x^y) 1735*0fca6ea1SDimitry Andric // x^y -> (x|y) ^ (x&y) or (x|y) & ~(x&y) 1736*0fca6ea1SDimitry Andric Value *X, *Y; 1737*0fca6ea1SDimitry Andric if (!match(CmpLHS, m_BitwiseLogic(m_Value(X), m_Value(Y))) || 1738*0fca6ea1SDimitry Andric !match(TrueVal, m_c_BitwiseLogic(m_Specific(X), m_Specific(Y)))) 1739*0fca6ea1SDimitry Andric return nullptr; 1740*0fca6ea1SDimitry Andric 1741*0fca6ea1SDimitry Andric const unsigned AndOps = Instruction::And, OrOps = Instruction::Or, 1742*0fca6ea1SDimitry Andric XorOps = Instruction::Xor, NoOps = 0; 1743*0fca6ea1SDimitry Andric enum NotMask { None = 0, NotInner, NotRHS }; 1744*0fca6ea1SDimitry Andric 1745*0fca6ea1SDimitry Andric auto matchFalseVal = [&](unsigned OuterOpc, unsigned InnerOpc, 1746*0fca6ea1SDimitry Andric unsigned NotMask) { 1747*0fca6ea1SDimitry Andric auto matchInner = m_c_BinOp(InnerOpc, m_Specific(X), m_Specific(Y)); 1748*0fca6ea1SDimitry Andric if (OuterOpc == NoOps) 1749*0fca6ea1SDimitry Andric return match(CmpRHS, m_Zero()) && match(FalseVal, matchInner); 1750*0fca6ea1SDimitry Andric 1751*0fca6ea1SDimitry Andric if (NotMask == NotInner) { 1752*0fca6ea1SDimitry Andric return match(FalseVal, m_c_BinOp(OuterOpc, m_NotForbidPoison(matchInner), 1753*0fca6ea1SDimitry Andric m_Specific(CmpRHS))); 1754*0fca6ea1SDimitry Andric } else if (NotMask == NotRHS) { 1755*0fca6ea1SDimitry Andric return match(FalseVal, m_c_BinOp(OuterOpc, matchInner, 1756*0fca6ea1SDimitry Andric m_NotForbidPoison(m_Specific(CmpRHS)))); 1757*0fca6ea1SDimitry Andric } else { 1758*0fca6ea1SDimitry Andric return match(FalseVal, 1759*0fca6ea1SDimitry Andric m_c_BinOp(OuterOpc, matchInner, m_Specific(CmpRHS))); 1760*0fca6ea1SDimitry Andric } 1761*0fca6ea1SDimitry Andric }; 1762*0fca6ea1SDimitry Andric 1763*0fca6ea1SDimitry Andric // (X&Y)==C ? X|Y : X^Y -> (X^Y)|C : X^Y or (X^Y)^ C : X^Y 1764*0fca6ea1SDimitry Andric // (X&Y)==C ? X^Y : X|Y -> (X|Y)^C : X|Y or (X|Y)&~C : X|Y 1765*0fca6ea1SDimitry Andric if (match(CmpLHS, m_And(m_Value(X), m_Value(Y)))) { 1766*0fca6ea1SDimitry Andric if (match(TrueVal, m_c_Or(m_Specific(X), m_Specific(Y)))) { 1767*0fca6ea1SDimitry Andric // (X&Y)==C ? X|Y : (X^Y)|C -> (X^Y)|C : (X^Y)|C -> (X^Y)|C 1768*0fca6ea1SDimitry Andric // (X&Y)==C ? X|Y : (X^Y)^C -> (X^Y)^C : (X^Y)^C -> (X^Y)^C 1769*0fca6ea1SDimitry Andric if (matchFalseVal(OrOps, XorOps, None) || 1770*0fca6ea1SDimitry Andric matchFalseVal(XorOps, XorOps, None)) 1771*0fca6ea1SDimitry Andric return IC.replaceInstUsesWith(SI, FalseVal); 1772*0fca6ea1SDimitry Andric } else if (match(TrueVal, m_c_Xor(m_Specific(X), m_Specific(Y)))) { 1773*0fca6ea1SDimitry Andric // (X&Y)==C ? X^Y : (X|Y)^ C -> (X|Y)^ C : (X|Y)^ C -> (X|Y)^ C 1774*0fca6ea1SDimitry Andric // (X&Y)==C ? X^Y : (X|Y)&~C -> (X|Y)&~C : (X|Y)&~C -> (X|Y)&~C 1775*0fca6ea1SDimitry Andric if (matchFalseVal(XorOps, OrOps, None) || 1776*0fca6ea1SDimitry Andric matchFalseVal(AndOps, OrOps, NotRHS)) 1777*0fca6ea1SDimitry Andric return IC.replaceInstUsesWith(SI, FalseVal); 1778*0fca6ea1SDimitry Andric } 1779*0fca6ea1SDimitry Andric } 1780*0fca6ea1SDimitry Andric 1781*0fca6ea1SDimitry Andric // (X|Y)==C ? X&Y : X^Y -> (X^Y)^C : X^Y or ~(X^Y)&C : X^Y 1782*0fca6ea1SDimitry Andric // (X|Y)==C ? X^Y : X&Y -> (X&Y)^C : X&Y or ~(X&Y)&C : X&Y 1783*0fca6ea1SDimitry Andric if (match(CmpLHS, m_Or(m_Value(X), m_Value(Y)))) { 1784*0fca6ea1SDimitry Andric if (match(TrueVal, m_c_And(m_Specific(X), m_Specific(Y)))) { 1785*0fca6ea1SDimitry Andric // (X|Y)==C ? X&Y: (X^Y)^C -> (X^Y)^C: (X^Y)^C -> (X^Y)^C 1786*0fca6ea1SDimitry Andric // (X|Y)==C ? X&Y:~(X^Y)&C ->~(X^Y)&C:~(X^Y)&C -> ~(X^Y)&C 1787*0fca6ea1SDimitry Andric if (matchFalseVal(XorOps, XorOps, None) || 1788*0fca6ea1SDimitry Andric matchFalseVal(AndOps, XorOps, NotInner)) 1789*0fca6ea1SDimitry Andric return IC.replaceInstUsesWith(SI, FalseVal); 1790*0fca6ea1SDimitry Andric } else if (match(TrueVal, m_c_Xor(m_Specific(X), m_Specific(Y)))) { 1791*0fca6ea1SDimitry Andric // (X|Y)==C ? X^Y : (X&Y)^C -> (X&Y)^C : (X&Y)^C -> (X&Y)^C 1792*0fca6ea1SDimitry Andric // (X|Y)==C ? X^Y :~(X&Y)&C -> ~(X&Y)&C :~(X&Y)&C -> ~(X&Y)&C 1793*0fca6ea1SDimitry Andric if (matchFalseVal(XorOps, AndOps, None) || 1794*0fca6ea1SDimitry Andric matchFalseVal(AndOps, AndOps, NotInner)) 1795*0fca6ea1SDimitry Andric return IC.replaceInstUsesWith(SI, FalseVal); 1796*0fca6ea1SDimitry Andric } 1797*0fca6ea1SDimitry Andric } 1798*0fca6ea1SDimitry Andric 1799*0fca6ea1SDimitry Andric // (X^Y)==C ? X&Y : X|Y -> (X|Y)^C : X|Y or (X|Y)&~C : X|Y 1800*0fca6ea1SDimitry Andric // (X^Y)==C ? X|Y : X&Y -> (X&Y)|C : X&Y or (X&Y)^ C : X&Y 1801*0fca6ea1SDimitry Andric if (match(CmpLHS, m_Xor(m_Value(X), m_Value(Y)))) { 1802*0fca6ea1SDimitry Andric if ((match(TrueVal, m_c_And(m_Specific(X), m_Specific(Y))))) { 1803*0fca6ea1SDimitry Andric // (X^Y)==C ? X&Y : (X|Y)^C -> (X|Y)^C 1804*0fca6ea1SDimitry Andric // (X^Y)==C ? X&Y : (X|Y)&~C -> (X|Y)&~C 1805*0fca6ea1SDimitry Andric if (matchFalseVal(XorOps, OrOps, None) || 1806*0fca6ea1SDimitry Andric matchFalseVal(AndOps, OrOps, NotRHS)) 1807*0fca6ea1SDimitry Andric return IC.replaceInstUsesWith(SI, FalseVal); 1808*0fca6ea1SDimitry Andric } else if (match(TrueVal, m_c_Or(m_Specific(X), m_Specific(Y)))) { 1809*0fca6ea1SDimitry Andric // (X^Y)==C ? (X|Y) : (X&Y)|C -> (X&Y)|C 1810*0fca6ea1SDimitry Andric // (X^Y)==C ? (X|Y) : (X&Y)^C -> (X&Y)^C 1811*0fca6ea1SDimitry Andric if (matchFalseVal(OrOps, AndOps, None) || 1812*0fca6ea1SDimitry Andric matchFalseVal(XorOps, AndOps, None)) 1813*0fca6ea1SDimitry Andric return IC.replaceInstUsesWith(SI, FalseVal); 1814*0fca6ea1SDimitry Andric } 1815*0fca6ea1SDimitry Andric } 1816*0fca6ea1SDimitry Andric 1817*0fca6ea1SDimitry Andric return nullptr; 1818*0fca6ea1SDimitry Andric } 1819*0fca6ea1SDimitry Andric 18200b57cec5SDimitry Andric /// Visit a SelectInst that has an ICmpInst as its first operand. 1821e8d8bef9SDimitry Andric Instruction *InstCombinerImpl::foldSelectInstWithICmp(SelectInst &SI, 18220b57cec5SDimitry Andric ICmpInst *ICI) { 1823e8d8bef9SDimitry Andric if (Instruction *NewSel = foldSelectValueEquivalence(SI, *ICI)) 1824e8d8bef9SDimitry Andric return NewSel; 18250b57cec5SDimitry Andric 1826647cbc5dSDimitry Andric if (Value *V = 1827647cbc5dSDimitry Andric canonicalizeSPF(*ICI, SI.getTrueValue(), SI.getFalseValue(), *this)) 1828647cbc5dSDimitry Andric return replaceInstUsesWith(SI, V); 18290b57cec5SDimitry Andric 183006c3fb27SDimitry Andric if (Value *V = foldSelectInstWithICmpConst(SI, ICI, Builder)) 183181ad6265SDimitry Andric return replaceInstUsesWith(SI, V); 18320b57cec5SDimitry Andric 1833*0fca6ea1SDimitry Andric if (Value *V = canonicalizeClampLike(SI, *ICI, Builder, *this)) 1834349cc55cSDimitry Andric return replaceInstUsesWith(SI, V); 18358bcb0991SDimitry Andric 18368bcb0991SDimitry Andric if (Instruction *NewSel = 18375ffd83dbSDimitry Andric tryToReuseConstantFromSelectInComparison(SI, *ICI, *this)) 18388bcb0991SDimitry Andric return NewSel; 18398bcb0991SDimitry Andric 18400b57cec5SDimitry Andric if (Value *V = foldSelectICmpAnd(SI, ICI, Builder)) 18410b57cec5SDimitry Andric return replaceInstUsesWith(SI, V); 18420b57cec5SDimitry Andric 18430b57cec5SDimitry Andric // NOTE: if we wanted to, this is where to detect integer MIN/MAX 184406c3fb27SDimitry Andric bool Changed = false; 18458bcb0991SDimitry Andric Value *TrueVal = SI.getTrueValue(); 18468bcb0991SDimitry Andric Value *FalseVal = SI.getFalseValue(); 18470b57cec5SDimitry Andric ICmpInst::Predicate Pred = ICI->getPredicate(); 18480b57cec5SDimitry Andric Value *CmpLHS = ICI->getOperand(0); 18490b57cec5SDimitry Andric Value *CmpRHS = ICI->getOperand(1); 185006c3fb27SDimitry Andric if (CmpRHS != CmpLHS && isa<Constant>(CmpRHS) && !isa<Constant>(CmpLHS)) { 18510b57cec5SDimitry Andric if (CmpLHS == TrueVal && Pred == ICmpInst::ICMP_EQ) { 18520b57cec5SDimitry Andric // Transform (X == C) ? X : Y -> (X == C) ? C : Y 1853297eecfbSDimitry Andric replaceOperand(SI, 1, CmpRHS); 18540b57cec5SDimitry Andric Changed = true; 18550b57cec5SDimitry Andric } else if (CmpLHS == FalseVal && Pred == ICmpInst::ICMP_NE) { 18560b57cec5SDimitry Andric // Transform (X != C) ? Y : X -> (X != C) ? Y : C 1857297eecfbSDimitry Andric replaceOperand(SI, 2, CmpRHS); 18580b57cec5SDimitry Andric Changed = true; 18590b57cec5SDimitry Andric } 18600b57cec5SDimitry Andric } 18610b57cec5SDimitry Andric 1862*0fca6ea1SDimitry Andric if (Instruction *NewSel = foldSelectICmpEq(SI, ICI, *this)) 1863*0fca6ea1SDimitry Andric return NewSel; 1864*0fca6ea1SDimitry Andric 18653a9a9c0cSDimitry Andric // Canonicalize a signbit condition to use zero constant by swapping: 18663a9a9c0cSDimitry Andric // (CmpLHS > -1) ? TV : FV --> (CmpLHS < 0) ? FV : TV 18673a9a9c0cSDimitry Andric // To avoid conflicts (infinite loops) with other canonicalizations, this is 18683a9a9c0cSDimitry Andric // not applied with any constant select arm. 18693a9a9c0cSDimitry Andric if (Pred == ICmpInst::ICMP_SGT && match(CmpRHS, m_AllOnes()) && 18703a9a9c0cSDimitry Andric !match(TrueVal, m_Constant()) && !match(FalseVal, m_Constant()) && 18713a9a9c0cSDimitry Andric ICI->hasOneUse()) { 18723a9a9c0cSDimitry Andric InstCombiner::BuilderTy::InsertPointGuard Guard(Builder); 18733a9a9c0cSDimitry Andric Builder.SetInsertPoint(&SI); 187481ad6265SDimitry Andric Value *IsNeg = Builder.CreateIsNeg(CmpLHS, ICI->getName()); 18753a9a9c0cSDimitry Andric replaceOperand(SI, 0, IsNeg); 18763a9a9c0cSDimitry Andric SI.swapValues(); 18773a9a9c0cSDimitry Andric SI.swapProfMetadata(); 18783a9a9c0cSDimitry Andric return &SI; 18793a9a9c0cSDimitry Andric } 18803a9a9c0cSDimitry Andric 18810b57cec5SDimitry Andric // FIXME: This code is nearly duplicated in InstSimplify. Using/refactoring 18820b57cec5SDimitry Andric // decomposeBitTestICmp() might help. 188306c3fb27SDimitry Andric if (TrueVal->getType()->isIntOrIntVectorTy()) { 18840b57cec5SDimitry Andric unsigned BitWidth = 18850b57cec5SDimitry Andric DL.getTypeSizeInBits(TrueVal->getType()->getScalarType()); 18860b57cec5SDimitry Andric APInt MinSignedValue = APInt::getSignedMinValue(BitWidth); 18870b57cec5SDimitry Andric Value *X; 18880b57cec5SDimitry Andric const APInt *Y, *C; 18890b57cec5SDimitry Andric bool TrueWhenUnset; 18900b57cec5SDimitry Andric bool IsBitTest = false; 18910b57cec5SDimitry Andric if (ICmpInst::isEquality(Pred) && 18920b57cec5SDimitry Andric match(CmpLHS, m_And(m_Value(X), m_Power2(Y))) && 18930b57cec5SDimitry Andric match(CmpRHS, m_Zero())) { 18940b57cec5SDimitry Andric IsBitTest = true; 18950b57cec5SDimitry Andric TrueWhenUnset = Pred == ICmpInst::ICMP_EQ; 18960b57cec5SDimitry Andric } else if (Pred == ICmpInst::ICMP_SLT && match(CmpRHS, m_Zero())) { 18970b57cec5SDimitry Andric X = CmpLHS; 18980b57cec5SDimitry Andric Y = &MinSignedValue; 18990b57cec5SDimitry Andric IsBitTest = true; 19000b57cec5SDimitry Andric TrueWhenUnset = false; 19010b57cec5SDimitry Andric } else if (Pred == ICmpInst::ICMP_SGT && match(CmpRHS, m_AllOnes())) { 19020b57cec5SDimitry Andric X = CmpLHS; 19030b57cec5SDimitry Andric Y = &MinSignedValue; 19040b57cec5SDimitry Andric IsBitTest = true; 19050b57cec5SDimitry Andric TrueWhenUnset = true; 19060b57cec5SDimitry Andric } 19070b57cec5SDimitry Andric if (IsBitTest) { 19080b57cec5SDimitry Andric Value *V = nullptr; 19090b57cec5SDimitry Andric // (X & Y) == 0 ? X : X ^ Y --> X & ~Y 19100b57cec5SDimitry Andric if (TrueWhenUnset && TrueVal == X && 19110b57cec5SDimitry Andric match(FalseVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C) 19120b57cec5SDimitry Andric V = Builder.CreateAnd(X, ~(*Y)); 19130b57cec5SDimitry Andric // (X & Y) != 0 ? X ^ Y : X --> X & ~Y 19140b57cec5SDimitry Andric else if (!TrueWhenUnset && FalseVal == X && 19150b57cec5SDimitry Andric match(TrueVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C) 19160b57cec5SDimitry Andric V = Builder.CreateAnd(X, ~(*Y)); 19170b57cec5SDimitry Andric // (X & Y) == 0 ? X ^ Y : X --> X | Y 19180b57cec5SDimitry Andric else if (TrueWhenUnset && FalseVal == X && 19190b57cec5SDimitry Andric match(TrueVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C) 19200b57cec5SDimitry Andric V = Builder.CreateOr(X, *Y); 19210b57cec5SDimitry Andric // (X & Y) != 0 ? X : X ^ Y --> X | Y 19220b57cec5SDimitry Andric else if (!TrueWhenUnset && TrueVal == X && 19230b57cec5SDimitry Andric match(FalseVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C) 19240b57cec5SDimitry Andric V = Builder.CreateOr(X, *Y); 19250b57cec5SDimitry Andric 19260b57cec5SDimitry Andric if (V) 19270b57cec5SDimitry Andric return replaceInstUsesWith(SI, V); 19280b57cec5SDimitry Andric } 19290b57cec5SDimitry Andric } 19300b57cec5SDimitry Andric 19310b57cec5SDimitry Andric if (Instruction *V = 19320b57cec5SDimitry Andric foldSelectICmpAndAnd(SI.getType(), ICI, TrueVal, FalseVal, Builder)) 19330b57cec5SDimitry Andric return V; 19340b57cec5SDimitry Andric 193506c3fb27SDimitry Andric if (Value *V = foldSelectICmpAndZeroShl(ICI, TrueVal, FalseVal, Builder)) 193606c3fb27SDimitry Andric return replaceInstUsesWith(SI, V); 193706c3fb27SDimitry Andric 19388bcb0991SDimitry Andric if (Instruction *V = foldSelectCtlzToCttz(ICI, TrueVal, FalseVal, Builder)) 19398bcb0991SDimitry Andric return V; 19408bcb0991SDimitry Andric 194181ad6265SDimitry Andric if (Instruction *V = foldSelectZeroOrOnes(ICI, TrueVal, FalseVal, Builder)) 194281ad6265SDimitry Andric return V; 194381ad6265SDimitry Andric 19445f757f3fSDimitry Andric if (Value *V = foldSelectICmpAndBinOp(ICI, TrueVal, FalseVal, Builder)) 19450b57cec5SDimitry Andric return replaceInstUsesWith(SI, V); 19460b57cec5SDimitry Andric 19470b57cec5SDimitry Andric if (Value *V = foldSelectICmpLshrAshr(ICI, TrueVal, FalseVal, Builder)) 19480b57cec5SDimitry Andric return replaceInstUsesWith(SI, V); 19490b57cec5SDimitry Andric 1950*0fca6ea1SDimitry Andric if (Value *V = foldSelectCttzCtlz(ICI, TrueVal, FalseVal, *this)) 19510b57cec5SDimitry Andric return replaceInstUsesWith(SI, V); 19520b57cec5SDimitry Andric 19530b57cec5SDimitry Andric if (Value *V = canonicalizeSaturatedSubtract(ICI, TrueVal, FalseVal, Builder)) 19540b57cec5SDimitry Andric return replaceInstUsesWith(SI, V); 19550b57cec5SDimitry Andric 19560b57cec5SDimitry Andric if (Value *V = canonicalizeSaturatedAdd(ICI, TrueVal, FalseVal, Builder)) 19570b57cec5SDimitry Andric return replaceInstUsesWith(SI, V); 19580b57cec5SDimitry Andric 195906c3fb27SDimitry Andric if (Value *V = foldAbsDiff(ICI, TrueVal, FalseVal, Builder)) 196006c3fb27SDimitry Andric return replaceInstUsesWith(SI, V); 196106c3fb27SDimitry Andric 19620b57cec5SDimitry Andric return Changed ? &SI : nullptr; 19630b57cec5SDimitry Andric } 19640b57cec5SDimitry Andric 19650b57cec5SDimitry Andric /// SI is a select whose condition is a PHI node (but the two may be in 19660b57cec5SDimitry Andric /// different blocks). See if the true/false values (V) are live in all of the 19670b57cec5SDimitry Andric /// predecessor blocks of the PHI. For example, cases like this can't be mapped: 19680b57cec5SDimitry Andric /// 19690b57cec5SDimitry Andric /// X = phi [ C1, BB1], [C2, BB2] 19700b57cec5SDimitry Andric /// Y = add 19710b57cec5SDimitry Andric /// Z = select X, Y, 0 19720b57cec5SDimitry Andric /// 19730b57cec5SDimitry Andric /// because Y is not live in BB1/BB2. 19740b57cec5SDimitry Andric static bool canSelectOperandBeMappingIntoPredBlock(const Value *V, 19750b57cec5SDimitry Andric const SelectInst &SI) { 19760b57cec5SDimitry Andric // If the value is a non-instruction value like a constant or argument, it 19770b57cec5SDimitry Andric // can always be mapped. 19780b57cec5SDimitry Andric const Instruction *I = dyn_cast<Instruction>(V); 19790b57cec5SDimitry Andric if (!I) return true; 19800b57cec5SDimitry Andric 19810b57cec5SDimitry Andric // If V is a PHI node defined in the same block as the condition PHI, we can 19820b57cec5SDimitry Andric // map the arguments. 19830b57cec5SDimitry Andric const PHINode *CondPHI = cast<PHINode>(SI.getCondition()); 19840b57cec5SDimitry Andric 19850b57cec5SDimitry Andric if (const PHINode *VP = dyn_cast<PHINode>(I)) 19860b57cec5SDimitry Andric if (VP->getParent() == CondPHI->getParent()) 19870b57cec5SDimitry Andric return true; 19880b57cec5SDimitry Andric 19890b57cec5SDimitry Andric // Otherwise, if the PHI and select are defined in the same block and if V is 19900b57cec5SDimitry Andric // defined in a different block, then we can transform it. 19910b57cec5SDimitry Andric if (SI.getParent() == CondPHI->getParent() && 19920b57cec5SDimitry Andric I->getParent() != CondPHI->getParent()) 19930b57cec5SDimitry Andric return true; 19940b57cec5SDimitry Andric 19950b57cec5SDimitry Andric // Otherwise we have a 'hard' case and we can't tell without doing more 19960b57cec5SDimitry Andric // detailed dominator based analysis, punt. 19970b57cec5SDimitry Andric return false; 19980b57cec5SDimitry Andric } 19990b57cec5SDimitry Andric 20000b57cec5SDimitry Andric /// We have an SPF (e.g. a min or max) of an SPF of the form: 20010b57cec5SDimitry Andric /// SPF2(SPF1(A, B), C) 2002e8d8bef9SDimitry Andric Instruction *InstCombinerImpl::foldSPFofSPF(Instruction *Inner, 2003e8d8bef9SDimitry Andric SelectPatternFlavor SPF1, Value *A, 2004e8d8bef9SDimitry Andric Value *B, Instruction &Outer, 2005e8d8bef9SDimitry Andric SelectPatternFlavor SPF2, 2006e8d8bef9SDimitry Andric Value *C) { 20070b57cec5SDimitry Andric if (Outer.getType() != Inner->getType()) 20080b57cec5SDimitry Andric return nullptr; 20090b57cec5SDimitry Andric 20100b57cec5SDimitry Andric if (C == A || C == B) { 20110b57cec5SDimitry Andric // MAX(MAX(A, B), B) -> MAX(A, B) 20120b57cec5SDimitry Andric // MIN(MIN(a, b), a) -> MIN(a, b) 20130b57cec5SDimitry Andric // TODO: This could be done in instsimplify. 20140b57cec5SDimitry Andric if (SPF1 == SPF2 && SelectPatternResult::isMinOrMax(SPF1)) 20150b57cec5SDimitry Andric return replaceInstUsesWith(Outer, Inner); 20160b57cec5SDimitry Andric } 20170b57cec5SDimitry Andric 20180b57cec5SDimitry Andric return nullptr; 20190b57cec5SDimitry Andric } 20200b57cec5SDimitry Andric 20210b57cec5SDimitry Andric /// Turn select C, (X + Y), (X - Y) --> (X + (select C, Y, (-Y))). 20220b57cec5SDimitry Andric /// This is even legal for FP. 20230b57cec5SDimitry Andric static Instruction *foldAddSubSelect(SelectInst &SI, 20240b57cec5SDimitry Andric InstCombiner::BuilderTy &Builder) { 20250b57cec5SDimitry Andric Value *CondVal = SI.getCondition(); 20260b57cec5SDimitry Andric Value *TrueVal = SI.getTrueValue(); 20270b57cec5SDimitry Andric Value *FalseVal = SI.getFalseValue(); 20280b57cec5SDimitry Andric auto *TI = dyn_cast<Instruction>(TrueVal); 20290b57cec5SDimitry Andric auto *FI = dyn_cast<Instruction>(FalseVal); 20300b57cec5SDimitry Andric if (!TI || !FI || !TI->hasOneUse() || !FI->hasOneUse()) 20310b57cec5SDimitry Andric return nullptr; 20320b57cec5SDimitry Andric 20330b57cec5SDimitry Andric Instruction *AddOp = nullptr, *SubOp = nullptr; 20340b57cec5SDimitry Andric if ((TI->getOpcode() == Instruction::Sub && 20350b57cec5SDimitry Andric FI->getOpcode() == Instruction::Add) || 20360b57cec5SDimitry Andric (TI->getOpcode() == Instruction::FSub && 20370b57cec5SDimitry Andric FI->getOpcode() == Instruction::FAdd)) { 20380b57cec5SDimitry Andric AddOp = FI; 20390b57cec5SDimitry Andric SubOp = TI; 20400b57cec5SDimitry Andric } else if ((FI->getOpcode() == Instruction::Sub && 20410b57cec5SDimitry Andric TI->getOpcode() == Instruction::Add) || 20420b57cec5SDimitry Andric (FI->getOpcode() == Instruction::FSub && 20430b57cec5SDimitry Andric TI->getOpcode() == Instruction::FAdd)) { 20440b57cec5SDimitry Andric AddOp = TI; 20450b57cec5SDimitry Andric SubOp = FI; 20460b57cec5SDimitry Andric } 20470b57cec5SDimitry Andric 20480b57cec5SDimitry Andric if (AddOp) { 20490b57cec5SDimitry Andric Value *OtherAddOp = nullptr; 20500b57cec5SDimitry Andric if (SubOp->getOperand(0) == AddOp->getOperand(0)) { 20510b57cec5SDimitry Andric OtherAddOp = AddOp->getOperand(1); 20520b57cec5SDimitry Andric } else if (SubOp->getOperand(0) == AddOp->getOperand(1)) { 20530b57cec5SDimitry Andric OtherAddOp = AddOp->getOperand(0); 20540b57cec5SDimitry Andric } 20550b57cec5SDimitry Andric 20560b57cec5SDimitry Andric if (OtherAddOp) { 20570b57cec5SDimitry Andric // So at this point we know we have (Y -> OtherAddOp): 20580b57cec5SDimitry Andric // select C, (add X, Y), (sub X, Z) 20590b57cec5SDimitry Andric Value *NegVal; // Compute -Z 20600b57cec5SDimitry Andric if (SI.getType()->isFPOrFPVectorTy()) { 20610b57cec5SDimitry Andric NegVal = Builder.CreateFNeg(SubOp->getOperand(1)); 20620b57cec5SDimitry Andric if (Instruction *NegInst = dyn_cast<Instruction>(NegVal)) { 20630b57cec5SDimitry Andric FastMathFlags Flags = AddOp->getFastMathFlags(); 20640b57cec5SDimitry Andric Flags &= SubOp->getFastMathFlags(); 20650b57cec5SDimitry Andric NegInst->setFastMathFlags(Flags); 20660b57cec5SDimitry Andric } 20670b57cec5SDimitry Andric } else { 20680b57cec5SDimitry Andric NegVal = Builder.CreateNeg(SubOp->getOperand(1)); 20690b57cec5SDimitry Andric } 20700b57cec5SDimitry Andric 20710b57cec5SDimitry Andric Value *NewTrueOp = OtherAddOp; 20720b57cec5SDimitry Andric Value *NewFalseOp = NegVal; 20730b57cec5SDimitry Andric if (AddOp != TI) 20740b57cec5SDimitry Andric std::swap(NewTrueOp, NewFalseOp); 20750b57cec5SDimitry Andric Value *NewSel = Builder.CreateSelect(CondVal, NewTrueOp, NewFalseOp, 20760b57cec5SDimitry Andric SI.getName() + ".p", &SI); 20770b57cec5SDimitry Andric 20780b57cec5SDimitry Andric if (SI.getType()->isFPOrFPVectorTy()) { 20790b57cec5SDimitry Andric Instruction *RI = 20800b57cec5SDimitry Andric BinaryOperator::CreateFAdd(SubOp->getOperand(0), NewSel); 20810b57cec5SDimitry Andric 20820b57cec5SDimitry Andric FastMathFlags Flags = AddOp->getFastMathFlags(); 20830b57cec5SDimitry Andric Flags &= SubOp->getFastMathFlags(); 20840b57cec5SDimitry Andric RI->setFastMathFlags(Flags); 20850b57cec5SDimitry Andric return RI; 20860b57cec5SDimitry Andric } else 20870b57cec5SDimitry Andric return BinaryOperator::CreateAdd(SubOp->getOperand(0), NewSel); 20880b57cec5SDimitry Andric } 20890b57cec5SDimitry Andric } 20900b57cec5SDimitry Andric return nullptr; 20910b57cec5SDimitry Andric } 20920b57cec5SDimitry Andric 2093480093f4SDimitry Andric /// Turn X + Y overflows ? -1 : X + Y -> uadd_sat X, Y 2094480093f4SDimitry Andric /// And X - Y overflows ? 0 : X - Y -> usub_sat X, Y 2095480093f4SDimitry Andric /// Along with a number of patterns similar to: 2096480093f4SDimitry Andric /// X + Y overflows ? (X < 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y 2097480093f4SDimitry Andric /// X - Y overflows ? (X > 0 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y 2098480093f4SDimitry Andric static Instruction * 2099480093f4SDimitry Andric foldOverflowingAddSubSelect(SelectInst &SI, InstCombiner::BuilderTy &Builder) { 2100480093f4SDimitry Andric Value *CondVal = SI.getCondition(); 2101480093f4SDimitry Andric Value *TrueVal = SI.getTrueValue(); 2102480093f4SDimitry Andric Value *FalseVal = SI.getFalseValue(); 2103480093f4SDimitry Andric 2104480093f4SDimitry Andric WithOverflowInst *II; 2105480093f4SDimitry Andric if (!match(CondVal, m_ExtractValue<1>(m_WithOverflowInst(II))) || 2106480093f4SDimitry Andric !match(FalseVal, m_ExtractValue<0>(m_Specific(II)))) 2107480093f4SDimitry Andric return nullptr; 2108480093f4SDimitry Andric 2109480093f4SDimitry Andric Value *X = II->getLHS(); 2110480093f4SDimitry Andric Value *Y = II->getRHS(); 2111480093f4SDimitry Andric 2112480093f4SDimitry Andric auto IsSignedSaturateLimit = [&](Value *Limit, bool IsAdd) { 2113480093f4SDimitry Andric Type *Ty = Limit->getType(); 2114480093f4SDimitry Andric 2115480093f4SDimitry Andric ICmpInst::Predicate Pred; 2116480093f4SDimitry Andric Value *TrueVal, *FalseVal, *Op; 2117480093f4SDimitry Andric const APInt *C; 2118480093f4SDimitry Andric if (!match(Limit, m_Select(m_ICmp(Pred, m_Value(Op), m_APInt(C)), 2119480093f4SDimitry Andric m_Value(TrueVal), m_Value(FalseVal)))) 2120480093f4SDimitry Andric return false; 2121480093f4SDimitry Andric 2122349cc55cSDimitry Andric auto IsZeroOrOne = [](const APInt &C) { return C.isZero() || C.isOne(); }; 2123480093f4SDimitry Andric auto IsMinMax = [&](Value *Min, Value *Max) { 2124480093f4SDimitry Andric APInt MinVal = APInt::getSignedMinValue(Ty->getScalarSizeInBits()); 2125480093f4SDimitry Andric APInt MaxVal = APInt::getSignedMaxValue(Ty->getScalarSizeInBits()); 2126480093f4SDimitry Andric return match(Min, m_SpecificInt(MinVal)) && 2127480093f4SDimitry Andric match(Max, m_SpecificInt(MaxVal)); 2128480093f4SDimitry Andric }; 2129480093f4SDimitry Andric 2130480093f4SDimitry Andric if (Op != X && Op != Y) 2131480093f4SDimitry Andric return false; 2132480093f4SDimitry Andric 2133480093f4SDimitry Andric if (IsAdd) { 2134480093f4SDimitry Andric // X + Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y 2135480093f4SDimitry Andric // X + Y overflows ? (X <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y 2136480093f4SDimitry Andric // X + Y overflows ? (Y <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y 2137480093f4SDimitry Andric // X + Y overflows ? (Y <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y 2138480093f4SDimitry Andric if (Pred == ICmpInst::ICMP_SLT && IsZeroOrOne(*C) && 2139480093f4SDimitry Andric IsMinMax(TrueVal, FalseVal)) 2140480093f4SDimitry Andric return true; 2141480093f4SDimitry Andric // X + Y overflows ? (X >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y 2142480093f4SDimitry Andric // X + Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y 2143480093f4SDimitry Andric // X + Y overflows ? (Y >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y 2144480093f4SDimitry Andric // X + Y overflows ? (Y >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y 2145480093f4SDimitry Andric if (Pred == ICmpInst::ICMP_SGT && IsZeroOrOne(*C + 1) && 2146480093f4SDimitry Andric IsMinMax(FalseVal, TrueVal)) 2147480093f4SDimitry Andric return true; 2148480093f4SDimitry Andric } else { 2149480093f4SDimitry Andric // X - Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y 2150480093f4SDimitry Andric // X - Y overflows ? (X <s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y 2151480093f4SDimitry Andric if (Op == X && Pred == ICmpInst::ICMP_SLT && IsZeroOrOne(*C + 1) && 2152480093f4SDimitry Andric IsMinMax(TrueVal, FalseVal)) 2153480093f4SDimitry Andric return true; 2154480093f4SDimitry Andric // X - Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y 2155480093f4SDimitry Andric // X - Y overflows ? (X >s -2 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y 2156480093f4SDimitry Andric if (Op == X && Pred == ICmpInst::ICMP_SGT && IsZeroOrOne(*C + 2) && 2157480093f4SDimitry Andric IsMinMax(FalseVal, TrueVal)) 2158480093f4SDimitry Andric return true; 2159480093f4SDimitry Andric // X - Y overflows ? (Y <s 0 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y 2160480093f4SDimitry Andric // X - Y overflows ? (Y <s 1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y 2161480093f4SDimitry Andric if (Op == Y && Pred == ICmpInst::ICMP_SLT && IsZeroOrOne(*C) && 2162480093f4SDimitry Andric IsMinMax(FalseVal, TrueVal)) 2163480093f4SDimitry Andric return true; 2164480093f4SDimitry Andric // X - Y overflows ? (Y >s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y 2165480093f4SDimitry Andric // X - Y overflows ? (Y >s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y 2166480093f4SDimitry Andric if (Op == Y && Pred == ICmpInst::ICMP_SGT && IsZeroOrOne(*C + 1) && 2167480093f4SDimitry Andric IsMinMax(TrueVal, FalseVal)) 2168480093f4SDimitry Andric return true; 2169480093f4SDimitry Andric } 2170480093f4SDimitry Andric 2171480093f4SDimitry Andric return false; 2172480093f4SDimitry Andric }; 2173480093f4SDimitry Andric 2174480093f4SDimitry Andric Intrinsic::ID NewIntrinsicID; 2175480093f4SDimitry Andric if (II->getIntrinsicID() == Intrinsic::uadd_with_overflow && 2176480093f4SDimitry Andric match(TrueVal, m_AllOnes())) 2177480093f4SDimitry Andric // X + Y overflows ? -1 : X + Y -> uadd_sat X, Y 2178480093f4SDimitry Andric NewIntrinsicID = Intrinsic::uadd_sat; 2179480093f4SDimitry Andric else if (II->getIntrinsicID() == Intrinsic::usub_with_overflow && 2180480093f4SDimitry Andric match(TrueVal, m_Zero())) 2181480093f4SDimitry Andric // X - Y overflows ? 0 : X - Y -> usub_sat X, Y 2182480093f4SDimitry Andric NewIntrinsicID = Intrinsic::usub_sat; 2183480093f4SDimitry Andric else if (II->getIntrinsicID() == Intrinsic::sadd_with_overflow && 2184480093f4SDimitry Andric IsSignedSaturateLimit(TrueVal, /*IsAdd=*/true)) 2185480093f4SDimitry Andric // X + Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y 2186480093f4SDimitry Andric // X + Y overflows ? (X <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y 2187480093f4SDimitry Andric // X + Y overflows ? (X >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y 2188480093f4SDimitry Andric // X + Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y 2189480093f4SDimitry Andric // X + Y overflows ? (Y <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y 2190480093f4SDimitry Andric // X + Y overflows ? (Y <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y 2191480093f4SDimitry Andric // X + Y overflows ? (Y >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y 2192480093f4SDimitry Andric // X + Y overflows ? (Y >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y 2193480093f4SDimitry Andric NewIntrinsicID = Intrinsic::sadd_sat; 2194480093f4SDimitry Andric else if (II->getIntrinsicID() == Intrinsic::ssub_with_overflow && 2195480093f4SDimitry Andric IsSignedSaturateLimit(TrueVal, /*IsAdd=*/false)) 2196480093f4SDimitry Andric // X - Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y 2197480093f4SDimitry Andric // X - Y overflows ? (X <s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y 2198480093f4SDimitry Andric // X - Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y 2199480093f4SDimitry Andric // X - Y overflows ? (X >s -2 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y 2200480093f4SDimitry Andric // X - Y overflows ? (Y <s 0 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y 2201480093f4SDimitry Andric // X - Y overflows ? (Y <s 1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y 2202480093f4SDimitry Andric // X - Y overflows ? (Y >s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y 2203480093f4SDimitry Andric // X - Y overflows ? (Y >s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y 2204480093f4SDimitry Andric NewIntrinsicID = Intrinsic::ssub_sat; 2205480093f4SDimitry Andric else 2206480093f4SDimitry Andric return nullptr; 2207480093f4SDimitry Andric 2208480093f4SDimitry Andric Function *F = 2209480093f4SDimitry Andric Intrinsic::getDeclaration(SI.getModule(), NewIntrinsicID, SI.getType()); 2210480093f4SDimitry Andric return CallInst::Create(F, {X, Y}); 2211480093f4SDimitry Andric } 2212480093f4SDimitry Andric 2213e8d8bef9SDimitry Andric Instruction *InstCombinerImpl::foldSelectExtConst(SelectInst &Sel) { 22140b57cec5SDimitry Andric Constant *C; 22150b57cec5SDimitry Andric if (!match(Sel.getTrueValue(), m_Constant(C)) && 22160b57cec5SDimitry Andric !match(Sel.getFalseValue(), m_Constant(C))) 22170b57cec5SDimitry Andric return nullptr; 22180b57cec5SDimitry Andric 22190b57cec5SDimitry Andric Instruction *ExtInst; 22200b57cec5SDimitry Andric if (!match(Sel.getTrueValue(), m_Instruction(ExtInst)) && 22210b57cec5SDimitry Andric !match(Sel.getFalseValue(), m_Instruction(ExtInst))) 22220b57cec5SDimitry Andric return nullptr; 22230b57cec5SDimitry Andric 22240b57cec5SDimitry Andric auto ExtOpcode = ExtInst->getOpcode(); 22250b57cec5SDimitry Andric if (ExtOpcode != Instruction::ZExt && ExtOpcode != Instruction::SExt) 22260b57cec5SDimitry Andric return nullptr; 22270b57cec5SDimitry Andric 22280b57cec5SDimitry Andric // If we are extending from a boolean type or if we can create a select that 22290b57cec5SDimitry Andric // has the same size operands as its condition, try to narrow the select. 22300b57cec5SDimitry Andric Value *X = ExtInst->getOperand(0); 22310b57cec5SDimitry Andric Type *SmallType = X->getType(); 22320b57cec5SDimitry Andric Value *Cond = Sel.getCondition(); 22330b57cec5SDimitry Andric auto *Cmp = dyn_cast<CmpInst>(Cond); 22340b57cec5SDimitry Andric if (!SmallType->isIntOrIntVectorTy(1) && 22350b57cec5SDimitry Andric (!Cmp || Cmp->getOperand(0)->getType() != SmallType)) 22360b57cec5SDimitry Andric return nullptr; 22370b57cec5SDimitry Andric 22380b57cec5SDimitry Andric // If the constant is the same after truncation to the smaller type and 22390b57cec5SDimitry Andric // extension to the original type, we can narrow the select. 22400b57cec5SDimitry Andric Type *SelType = Sel.getType(); 22415f757f3fSDimitry Andric Constant *TruncC = getLosslessTrunc(C, SmallType, ExtOpcode); 22425f757f3fSDimitry Andric if (TruncC && ExtInst->hasOneUse()) { 22430b57cec5SDimitry Andric Value *TruncCVal = cast<Value>(TruncC); 22440b57cec5SDimitry Andric if (ExtInst == Sel.getFalseValue()) 22450b57cec5SDimitry Andric std::swap(X, TruncCVal); 22460b57cec5SDimitry Andric 22470b57cec5SDimitry Andric // select Cond, (ext X), C --> ext(select Cond, X, C') 22480b57cec5SDimitry Andric // select Cond, C, (ext X) --> ext(select Cond, C', X) 22490b57cec5SDimitry Andric Value *NewSel = Builder.CreateSelect(Cond, X, TruncCVal, "narrow", &Sel); 22500b57cec5SDimitry Andric return CastInst::Create(Instruction::CastOps(ExtOpcode), NewSel, SelType); 22510b57cec5SDimitry Andric } 22520b57cec5SDimitry Andric 22530b57cec5SDimitry Andric return nullptr; 22540b57cec5SDimitry Andric } 22550b57cec5SDimitry Andric 22560b57cec5SDimitry Andric /// Try to transform a vector select with a constant condition vector into a 22570b57cec5SDimitry Andric /// shuffle for easier combining with other shuffles and insert/extract. 22580b57cec5SDimitry Andric static Instruction *canonicalizeSelectToShuffle(SelectInst &SI) { 22590b57cec5SDimitry Andric Value *CondVal = SI.getCondition(); 22600b57cec5SDimitry Andric Constant *CondC; 2261e8d8bef9SDimitry Andric auto *CondValTy = dyn_cast<FixedVectorType>(CondVal->getType()); 2262e8d8bef9SDimitry Andric if (!CondValTy || !match(CondVal, m_Constant(CondC))) 22630b57cec5SDimitry Andric return nullptr; 22640b57cec5SDimitry Andric 2265e8d8bef9SDimitry Andric unsigned NumElts = CondValTy->getNumElements(); 22665ffd83dbSDimitry Andric SmallVector<int, 16> Mask; 22670b57cec5SDimitry Andric Mask.reserve(NumElts); 22680b57cec5SDimitry Andric for (unsigned i = 0; i != NumElts; ++i) { 22690b57cec5SDimitry Andric Constant *Elt = CondC->getAggregateElement(i); 22700b57cec5SDimitry Andric if (!Elt) 22710b57cec5SDimitry Andric return nullptr; 22720b57cec5SDimitry Andric 22730b57cec5SDimitry Andric if (Elt->isOneValue()) { 22740b57cec5SDimitry Andric // If the select condition element is true, choose from the 1st vector. 22755ffd83dbSDimitry Andric Mask.push_back(i); 22760b57cec5SDimitry Andric } else if (Elt->isNullValue()) { 22770b57cec5SDimitry Andric // If the select condition element is false, choose from the 2nd vector. 22785ffd83dbSDimitry Andric Mask.push_back(i + NumElts); 22790b57cec5SDimitry Andric } else if (isa<UndefValue>(Elt)) { 22800b57cec5SDimitry Andric // Undef in a select condition (choose one of the operands) does not mean 22810b57cec5SDimitry Andric // the same thing as undef in a shuffle mask (any value is acceptable), so 22820b57cec5SDimitry Andric // give up. 22830b57cec5SDimitry Andric return nullptr; 22840b57cec5SDimitry Andric } else { 22850b57cec5SDimitry Andric // Bail out on a constant expression. 22860b57cec5SDimitry Andric return nullptr; 22870b57cec5SDimitry Andric } 22880b57cec5SDimitry Andric } 22890b57cec5SDimitry Andric 22905ffd83dbSDimitry Andric return new ShuffleVectorInst(SI.getTrueValue(), SI.getFalseValue(), Mask); 22910b57cec5SDimitry Andric } 22920b57cec5SDimitry Andric 22938bcb0991SDimitry Andric /// If we have a select of vectors with a scalar condition, try to convert that 22948bcb0991SDimitry Andric /// to a vector select by splatting the condition. A splat may get folded with 22958bcb0991SDimitry Andric /// other operations in IR and having all operands of a select be vector types 22968bcb0991SDimitry Andric /// is likely better for vector codegen. 2297e8d8bef9SDimitry Andric static Instruction *canonicalizeScalarSelectOfVecs(SelectInst &Sel, 2298e8d8bef9SDimitry Andric InstCombinerImpl &IC) { 22995ffd83dbSDimitry Andric auto *Ty = dyn_cast<VectorType>(Sel.getType()); 23005ffd83dbSDimitry Andric if (!Ty) 23018bcb0991SDimitry Andric return nullptr; 23028bcb0991SDimitry Andric 23038bcb0991SDimitry Andric // We can replace a single-use extract with constant index. 23048bcb0991SDimitry Andric Value *Cond = Sel.getCondition(); 23055ffd83dbSDimitry Andric if (!match(Cond, m_OneUse(m_ExtractElt(m_Value(), m_ConstantInt())))) 23068bcb0991SDimitry Andric return nullptr; 23078bcb0991SDimitry Andric 23088bcb0991SDimitry Andric // select (extelt V, Index), T, F --> select (splat V, Index), T, F 23098bcb0991SDimitry Andric // Splatting the extracted condition reduces code (we could directly create a 23108bcb0991SDimitry Andric // splat shuffle of the source vector to eliminate the intermediate step). 2311e8d8bef9SDimitry Andric return IC.replaceOperand( 2312e8d8bef9SDimitry Andric Sel, 0, IC.Builder.CreateVectorSplat(Ty->getElementCount(), Cond)); 23138bcb0991SDimitry Andric } 23148bcb0991SDimitry Andric 23150b57cec5SDimitry Andric /// Reuse bitcasted operands between a compare and select: 23160b57cec5SDimitry Andric /// select (cmp (bitcast C), (bitcast D)), (bitcast' C), (bitcast' D) --> 23170b57cec5SDimitry Andric /// bitcast (select (cmp (bitcast C), (bitcast D)), (bitcast C), (bitcast D)) 23180b57cec5SDimitry Andric static Instruction *foldSelectCmpBitcasts(SelectInst &Sel, 23190b57cec5SDimitry Andric InstCombiner::BuilderTy &Builder) { 23200b57cec5SDimitry Andric Value *Cond = Sel.getCondition(); 23210b57cec5SDimitry Andric Value *TVal = Sel.getTrueValue(); 23220b57cec5SDimitry Andric Value *FVal = Sel.getFalseValue(); 23230b57cec5SDimitry Andric 23240b57cec5SDimitry Andric CmpInst::Predicate Pred; 23250b57cec5SDimitry Andric Value *A, *B; 23260b57cec5SDimitry Andric if (!match(Cond, m_Cmp(Pred, m_Value(A), m_Value(B)))) 23270b57cec5SDimitry Andric return nullptr; 23280b57cec5SDimitry Andric 23290b57cec5SDimitry Andric // The select condition is a compare instruction. If the select's true/false 23300b57cec5SDimitry Andric // values are already the same as the compare operands, there's nothing to do. 23310b57cec5SDimitry Andric if (TVal == A || TVal == B || FVal == A || FVal == B) 23320b57cec5SDimitry Andric return nullptr; 23330b57cec5SDimitry Andric 23340b57cec5SDimitry Andric Value *C, *D; 23350b57cec5SDimitry Andric if (!match(A, m_BitCast(m_Value(C))) || !match(B, m_BitCast(m_Value(D)))) 23360b57cec5SDimitry Andric return nullptr; 23370b57cec5SDimitry Andric 23380b57cec5SDimitry Andric // select (cmp (bitcast C), (bitcast D)), (bitcast TSrc), (bitcast FSrc) 23390b57cec5SDimitry Andric Value *TSrc, *FSrc; 23400b57cec5SDimitry Andric if (!match(TVal, m_BitCast(m_Value(TSrc))) || 23410b57cec5SDimitry Andric !match(FVal, m_BitCast(m_Value(FSrc)))) 23420b57cec5SDimitry Andric return nullptr; 23430b57cec5SDimitry Andric 23440b57cec5SDimitry Andric // If the select true/false values are *different bitcasts* of the same source 23450b57cec5SDimitry Andric // operands, make the select operands the same as the compare operands and 23460b57cec5SDimitry Andric // cast the result. This is the canonical select form for min/max. 23470b57cec5SDimitry Andric Value *NewSel; 23480b57cec5SDimitry Andric if (TSrc == C && FSrc == D) { 23490b57cec5SDimitry Andric // select (cmp (bitcast C), (bitcast D)), (bitcast' C), (bitcast' D) --> 23500b57cec5SDimitry Andric // bitcast (select (cmp A, B), A, B) 23510b57cec5SDimitry Andric NewSel = Builder.CreateSelect(Cond, A, B, "", &Sel); 23520b57cec5SDimitry Andric } else if (TSrc == D && FSrc == C) { 23530b57cec5SDimitry Andric // select (cmp (bitcast C), (bitcast D)), (bitcast' D), (bitcast' C) --> 23540b57cec5SDimitry Andric // bitcast (select (cmp A, B), B, A) 23550b57cec5SDimitry Andric NewSel = Builder.CreateSelect(Cond, B, A, "", &Sel); 23560b57cec5SDimitry Andric } else { 23570b57cec5SDimitry Andric return nullptr; 23580b57cec5SDimitry Andric } 23590b57cec5SDimitry Andric return CastInst::CreateBitOrPointerCast(NewSel, Sel.getType()); 23600b57cec5SDimitry Andric } 23610b57cec5SDimitry Andric 23620b57cec5SDimitry Andric /// Try to eliminate select instructions that test the returned flag of cmpxchg 23630b57cec5SDimitry Andric /// instructions. 23640b57cec5SDimitry Andric /// 23650b57cec5SDimitry Andric /// If a select instruction tests the returned flag of a cmpxchg instruction and 23660b57cec5SDimitry Andric /// selects between the returned value of the cmpxchg instruction its compare 23670b57cec5SDimitry Andric /// operand, the result of the select will always be equal to its false value. 23680b57cec5SDimitry Andric /// For example: 23690b57cec5SDimitry Andric /// 2370*0fca6ea1SDimitry Andric /// %cmpxchg = cmpxchg ptr %ptr, i64 %compare, i64 %new_value seq_cst seq_cst 2371*0fca6ea1SDimitry Andric /// %val = extractvalue { i64, i1 } %cmpxchg, 0 2372*0fca6ea1SDimitry Andric /// %success = extractvalue { i64, i1 } %cmpxchg, 1 2373*0fca6ea1SDimitry Andric /// %sel = select i1 %success, i64 %compare, i64 %val 2374*0fca6ea1SDimitry Andric /// ret i64 %sel 23750b57cec5SDimitry Andric /// 2376*0fca6ea1SDimitry Andric /// The returned value of the cmpxchg instruction (%val) is the original value 2377*0fca6ea1SDimitry Andric /// located at %ptr prior to any update. If the cmpxchg operation succeeds, %val 23780b57cec5SDimitry Andric /// must have been equal to %compare. Thus, the result of the select is always 2379*0fca6ea1SDimitry Andric /// equal to %val, and the code can be simplified to: 23800b57cec5SDimitry Andric /// 2381*0fca6ea1SDimitry Andric /// %cmpxchg = cmpxchg ptr %ptr, i64 %compare, i64 %new_value seq_cst seq_cst 2382*0fca6ea1SDimitry Andric /// %val = extractvalue { i64, i1 } %cmpxchg, 0 2383*0fca6ea1SDimitry Andric /// ret i64 %val 23840b57cec5SDimitry Andric /// 23855ffd83dbSDimitry Andric static Value *foldSelectCmpXchg(SelectInst &SI) { 23860b57cec5SDimitry Andric // A helper that determines if V is an extractvalue instruction whose 23870b57cec5SDimitry Andric // aggregate operand is a cmpxchg instruction and whose single index is equal 23880b57cec5SDimitry Andric // to I. If such conditions are true, the helper returns the cmpxchg 23890b57cec5SDimitry Andric // instruction; otherwise, a nullptr is returned. 23900b57cec5SDimitry Andric auto isExtractFromCmpXchg = [](Value *V, unsigned I) -> AtomicCmpXchgInst * { 23910b57cec5SDimitry Andric auto *Extract = dyn_cast<ExtractValueInst>(V); 23920b57cec5SDimitry Andric if (!Extract) 23930b57cec5SDimitry Andric return nullptr; 23940b57cec5SDimitry Andric if (Extract->getIndices()[0] != I) 23950b57cec5SDimitry Andric return nullptr; 23960b57cec5SDimitry Andric return dyn_cast<AtomicCmpXchgInst>(Extract->getAggregateOperand()); 23970b57cec5SDimitry Andric }; 23980b57cec5SDimitry Andric 23990b57cec5SDimitry Andric // If the select has a single user, and this user is a select instruction that 24000b57cec5SDimitry Andric // we can simplify, skip the cmpxchg simplification for now. 24010b57cec5SDimitry Andric if (SI.hasOneUse()) 24020b57cec5SDimitry Andric if (auto *Select = dyn_cast<SelectInst>(SI.user_back())) 24030b57cec5SDimitry Andric if (Select->getCondition() == SI.getCondition()) 24040b57cec5SDimitry Andric if (Select->getFalseValue() == SI.getTrueValue() || 24050b57cec5SDimitry Andric Select->getTrueValue() == SI.getFalseValue()) 24060b57cec5SDimitry Andric return nullptr; 24070b57cec5SDimitry Andric 24080b57cec5SDimitry Andric // Ensure the select condition is the returned flag of a cmpxchg instruction. 24090b57cec5SDimitry Andric auto *CmpXchg = isExtractFromCmpXchg(SI.getCondition(), 1); 24100b57cec5SDimitry Andric if (!CmpXchg) 24110b57cec5SDimitry Andric return nullptr; 24120b57cec5SDimitry Andric 24130b57cec5SDimitry Andric // Check the true value case: The true value of the select is the returned 24140b57cec5SDimitry Andric // value of the same cmpxchg used by the condition, and the false value is the 24150b57cec5SDimitry Andric // cmpxchg instruction's compare operand. 24160b57cec5SDimitry Andric if (auto *X = isExtractFromCmpXchg(SI.getTrueValue(), 0)) 24175ffd83dbSDimitry Andric if (X == CmpXchg && X->getCompareOperand() == SI.getFalseValue()) 24185ffd83dbSDimitry Andric return SI.getFalseValue(); 24190b57cec5SDimitry Andric 24200b57cec5SDimitry Andric // Check the false value case: The false value of the select is the returned 24210b57cec5SDimitry Andric // value of the same cmpxchg used by the condition, and the true value is the 24220b57cec5SDimitry Andric // cmpxchg instruction's compare operand. 24230b57cec5SDimitry Andric if (auto *X = isExtractFromCmpXchg(SI.getFalseValue(), 0)) 24245ffd83dbSDimitry Andric if (X == CmpXchg && X->getCompareOperand() == SI.getTrueValue()) 24255ffd83dbSDimitry Andric return SI.getFalseValue(); 24260b57cec5SDimitry Andric 24270b57cec5SDimitry Andric return nullptr; 24280b57cec5SDimitry Andric } 24290b57cec5SDimitry Andric 2430e8d8bef9SDimitry Andric /// Try to reduce a funnel/rotate pattern that includes a compare and select 2431e8d8bef9SDimitry Andric /// into a funnel shift intrinsic. Example: 24320b57cec5SDimitry Andric /// rotl32(a, b) --> (b == 0 ? a : ((a >> (32 - b)) | (a << b))) 24330b57cec5SDimitry Andric /// --> call llvm.fshl.i32(a, a, b) 2434e8d8bef9SDimitry Andric /// fshl32(a, b, c) --> (c == 0 ? a : ((b >> (32 - c)) | (a << c))) 2435e8d8bef9SDimitry Andric /// --> call llvm.fshl.i32(a, b, c) 2436e8d8bef9SDimitry Andric /// fshr32(a, b, c) --> (c == 0 ? b : ((a >> (32 - c)) | (b << c))) 2437e8d8bef9SDimitry Andric /// --> call llvm.fshr.i32(a, b, c) 2438e8d8bef9SDimitry Andric static Instruction *foldSelectFunnelShift(SelectInst &Sel, 2439e8d8bef9SDimitry Andric InstCombiner::BuilderTy &Builder) { 2440e8d8bef9SDimitry Andric // This must be a power-of-2 type for a bitmasking transform to be valid. 24410b57cec5SDimitry Andric unsigned Width = Sel.getType()->getScalarSizeInBits(); 24420b57cec5SDimitry Andric if (!isPowerOf2_32(Width)) 24430b57cec5SDimitry Andric return nullptr; 24440b57cec5SDimitry Andric 2445e8d8bef9SDimitry Andric BinaryOperator *Or0, *Or1; 2446e8d8bef9SDimitry Andric if (!match(Sel.getFalseValue(), m_OneUse(m_Or(m_BinOp(Or0), m_BinOp(Or1))))) 2447e8d8bef9SDimitry Andric return nullptr; 2448e8d8bef9SDimitry Andric 2449e8d8bef9SDimitry Andric Value *SV0, *SV1, *SA0, *SA1; 2450e8d8bef9SDimitry Andric if (!match(Or0, m_OneUse(m_LogicalShift(m_Value(SV0), 2451e8d8bef9SDimitry Andric m_ZExtOrSelf(m_Value(SA0))))) || 2452e8d8bef9SDimitry Andric !match(Or1, m_OneUse(m_LogicalShift(m_Value(SV1), 2453e8d8bef9SDimitry Andric m_ZExtOrSelf(m_Value(SA1))))) || 2454e8d8bef9SDimitry Andric Or0->getOpcode() == Or1->getOpcode()) 2455e8d8bef9SDimitry Andric return nullptr; 2456e8d8bef9SDimitry Andric 2457e8d8bef9SDimitry Andric // Canonicalize to or(shl(SV0, SA0), lshr(SV1, SA1)). 2458e8d8bef9SDimitry Andric if (Or0->getOpcode() == BinaryOperator::LShr) { 2459e8d8bef9SDimitry Andric std::swap(Or0, Or1); 2460e8d8bef9SDimitry Andric std::swap(SV0, SV1); 2461e8d8bef9SDimitry Andric std::swap(SA0, SA1); 2462e8d8bef9SDimitry Andric } 2463e8d8bef9SDimitry Andric assert(Or0->getOpcode() == BinaryOperator::Shl && 2464e8d8bef9SDimitry Andric Or1->getOpcode() == BinaryOperator::LShr && 2465e8d8bef9SDimitry Andric "Illegal or(shift,shift) pair"); 2466e8d8bef9SDimitry Andric 24670b57cec5SDimitry Andric // Check the shift amounts to see if they are an opposite pair. 24680b57cec5SDimitry Andric Value *ShAmt; 24690b57cec5SDimitry Andric if (match(SA1, m_OneUse(m_Sub(m_SpecificInt(Width), m_Specific(SA0))))) 24700b57cec5SDimitry Andric ShAmt = SA0; 24710b57cec5SDimitry Andric else if (match(SA0, m_OneUse(m_Sub(m_SpecificInt(Width), m_Specific(SA1))))) 24720b57cec5SDimitry Andric ShAmt = SA1; 24730b57cec5SDimitry Andric else 24740b57cec5SDimitry Andric return nullptr; 24750b57cec5SDimitry Andric 2476e8d8bef9SDimitry Andric // We should now have this pattern: 2477e8d8bef9SDimitry Andric // select ?, TVal, (or (shl SV0, SA0), (lshr SV1, SA1)) 2478e8d8bef9SDimitry Andric // The false value of the select must be a funnel-shift of the true value: 2479e8d8bef9SDimitry Andric // IsFShl -> TVal must be SV0 else TVal must be SV1. 2480e8d8bef9SDimitry Andric bool IsFshl = (ShAmt == SA0); 2481e8d8bef9SDimitry Andric Value *TVal = Sel.getTrueValue(); 2482e8d8bef9SDimitry Andric if ((IsFshl && TVal != SV0) || (!IsFshl && TVal != SV1)) 2483e8d8bef9SDimitry Andric return nullptr; 2484e8d8bef9SDimitry Andric 24850b57cec5SDimitry Andric // Finally, see if the select is filtering out a shift-by-zero. 24860b57cec5SDimitry Andric Value *Cond = Sel.getCondition(); 24870b57cec5SDimitry Andric ICmpInst::Predicate Pred; 24880b57cec5SDimitry Andric if (!match(Cond, m_OneUse(m_ICmp(Pred, m_Specific(ShAmt), m_ZeroInt()))) || 24890b57cec5SDimitry Andric Pred != ICmpInst::ICMP_EQ) 24900b57cec5SDimitry Andric return nullptr; 24910b57cec5SDimitry Andric 2492e8d8bef9SDimitry Andric // If this is not a rotate then the select was blocking poison from the 2493e8d8bef9SDimitry Andric // 'shift-by-zero' non-TVal, but a funnel shift won't - so freeze it. 2494e8d8bef9SDimitry Andric if (SV0 != SV1) { 2495e8d8bef9SDimitry Andric if (IsFshl && !llvm::isGuaranteedNotToBePoison(SV1)) 2496e8d8bef9SDimitry Andric SV1 = Builder.CreateFreeze(SV1); 2497e8d8bef9SDimitry Andric else if (!IsFshl && !llvm::isGuaranteedNotToBePoison(SV0)) 2498e8d8bef9SDimitry Andric SV0 = Builder.CreateFreeze(SV0); 2499e8d8bef9SDimitry Andric } 2500e8d8bef9SDimitry Andric 2501e8d8bef9SDimitry Andric // This is a funnel/rotate that avoids shift-by-bitwidth UB in a suboptimal way. 25020b57cec5SDimitry Andric // Convert to funnel shift intrinsic. 25030b57cec5SDimitry Andric Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr; 25040b57cec5SDimitry Andric Function *F = Intrinsic::getDeclaration(Sel.getModule(), IID, Sel.getType()); 2505e8d8bef9SDimitry Andric ShAmt = Builder.CreateZExt(ShAmt, Sel.getType()); 2506fe6060f1SDimitry Andric return CallInst::Create(F, { SV0, SV1, ShAmt }); 25070b57cec5SDimitry Andric } 25080b57cec5SDimitry Andric 25095ffd83dbSDimitry Andric static Instruction *foldSelectToCopysign(SelectInst &Sel, 25105ffd83dbSDimitry Andric InstCombiner::BuilderTy &Builder) { 25115ffd83dbSDimitry Andric Value *Cond = Sel.getCondition(); 25125ffd83dbSDimitry Andric Value *TVal = Sel.getTrueValue(); 25135ffd83dbSDimitry Andric Value *FVal = Sel.getFalseValue(); 25145ffd83dbSDimitry Andric Type *SelType = Sel.getType(); 25155ffd83dbSDimitry Andric 25165ffd83dbSDimitry Andric // Match select ?, TC, FC where the constants are equal but negated. 25175ffd83dbSDimitry Andric // TODO: Generalize to handle a negated variable operand? 25185ffd83dbSDimitry Andric const APFloat *TC, *FC; 2519*0fca6ea1SDimitry Andric if (!match(TVal, m_APFloatAllowPoison(TC)) || 2520*0fca6ea1SDimitry Andric !match(FVal, m_APFloatAllowPoison(FC)) || 25215ffd83dbSDimitry Andric !abs(*TC).bitwiseIsEqual(abs(*FC))) 25225ffd83dbSDimitry Andric return nullptr; 25235ffd83dbSDimitry Andric 25245ffd83dbSDimitry Andric assert(TC != FC && "Expected equal select arms to simplify"); 25255ffd83dbSDimitry Andric 25265ffd83dbSDimitry Andric Value *X; 25275ffd83dbSDimitry Andric const APInt *C; 25285ffd83dbSDimitry Andric bool IsTrueIfSignSet; 25295ffd83dbSDimitry Andric ICmpInst::Predicate Pred; 2530*0fca6ea1SDimitry Andric if (!match(Cond, m_OneUse(m_ICmp(Pred, m_ElementWiseBitCast(m_Value(X)), 2531*0fca6ea1SDimitry Andric m_APInt(C)))) || 2532*0fca6ea1SDimitry Andric !isSignBitCheck(Pred, *C, IsTrueIfSignSet) || X->getType() != SelType) 25335ffd83dbSDimitry Andric return nullptr; 25345ffd83dbSDimitry Andric 25355ffd83dbSDimitry Andric // If needed, negate the value that will be the sign argument of the copysign: 25365ffd83dbSDimitry Andric // (bitcast X) < 0 ? -TC : TC --> copysign(TC, X) 25375ffd83dbSDimitry Andric // (bitcast X) < 0 ? TC : -TC --> copysign(TC, -X) 25385ffd83dbSDimitry Andric // (bitcast X) >= 0 ? -TC : TC --> copysign(TC, -X) 25395ffd83dbSDimitry Andric // (bitcast X) >= 0 ? TC : -TC --> copysign(TC, X) 254081ad6265SDimitry Andric // Note: FMF from the select can not be propagated to the new instructions. 25415ffd83dbSDimitry Andric if (IsTrueIfSignSet ^ TC->isNegative()) 254281ad6265SDimitry Andric X = Builder.CreateFNeg(X); 25435ffd83dbSDimitry Andric 25445ffd83dbSDimitry Andric // Canonicalize the magnitude argument as the positive constant since we do 25455ffd83dbSDimitry Andric // not care about its sign. 254681ad6265SDimitry Andric Value *MagArg = ConstantFP::get(SelType, abs(*TC)); 25475ffd83dbSDimitry Andric Function *F = Intrinsic::getDeclaration(Sel.getModule(), Intrinsic::copysign, 25485ffd83dbSDimitry Andric Sel.getType()); 254981ad6265SDimitry Andric return CallInst::Create(F, { MagArg, X }); 25505ffd83dbSDimitry Andric } 25515ffd83dbSDimitry Andric 2552e8d8bef9SDimitry Andric Instruction *InstCombinerImpl::foldVectorSelect(SelectInst &Sel) { 2553bdd1243dSDimitry Andric if (!isa<VectorType>(Sel.getType())) 2554bdd1243dSDimitry Andric return nullptr; 2555bdd1243dSDimitry Andric 2556bdd1243dSDimitry Andric Value *Cond = Sel.getCondition(); 2557bdd1243dSDimitry Andric Value *TVal = Sel.getTrueValue(); 2558bdd1243dSDimitry Andric Value *FVal = Sel.getFalseValue(); 2559bdd1243dSDimitry Andric Value *C, *X, *Y; 2560bdd1243dSDimitry Andric 2561bdd1243dSDimitry Andric if (match(Cond, m_VecReverse(m_Value(C)))) { 2562bdd1243dSDimitry Andric auto createSelReverse = [&](Value *C, Value *X, Value *Y) { 2563bdd1243dSDimitry Andric Value *V = Builder.CreateSelect(C, X, Y, Sel.getName(), &Sel); 2564bdd1243dSDimitry Andric if (auto *I = dyn_cast<Instruction>(V)) 2565bdd1243dSDimitry Andric I->copyIRFlags(&Sel); 2566bdd1243dSDimitry Andric Module *M = Sel.getModule(); 2567*0fca6ea1SDimitry Andric Function *F = 2568*0fca6ea1SDimitry Andric Intrinsic::getDeclaration(M, Intrinsic::vector_reverse, V->getType()); 2569bdd1243dSDimitry Andric return CallInst::Create(F, V); 2570bdd1243dSDimitry Andric }; 2571bdd1243dSDimitry Andric 2572bdd1243dSDimitry Andric if (match(TVal, m_VecReverse(m_Value(X)))) { 2573bdd1243dSDimitry Andric // select rev(C), rev(X), rev(Y) --> rev(select C, X, Y) 2574bdd1243dSDimitry Andric if (match(FVal, m_VecReverse(m_Value(Y))) && 2575bdd1243dSDimitry Andric (Cond->hasOneUse() || TVal->hasOneUse() || FVal->hasOneUse())) 2576bdd1243dSDimitry Andric return createSelReverse(C, X, Y); 2577bdd1243dSDimitry Andric 2578bdd1243dSDimitry Andric // select rev(C), rev(X), FValSplat --> rev(select C, X, FValSplat) 2579bdd1243dSDimitry Andric if ((Cond->hasOneUse() || TVal->hasOneUse()) && isSplatValue(FVal)) 2580bdd1243dSDimitry Andric return createSelReverse(C, X, FVal); 2581bdd1243dSDimitry Andric } 2582bdd1243dSDimitry Andric // select rev(C), TValSplat, rev(Y) --> rev(select C, TValSplat, Y) 2583bdd1243dSDimitry Andric else if (isSplatValue(TVal) && match(FVal, m_VecReverse(m_Value(Y))) && 2584bdd1243dSDimitry Andric (Cond->hasOneUse() || FVal->hasOneUse())) 2585bdd1243dSDimitry Andric return createSelReverse(C, TVal, Y); 2586bdd1243dSDimitry Andric } 2587bdd1243dSDimitry Andric 25885ffd83dbSDimitry Andric auto *VecTy = dyn_cast<FixedVectorType>(Sel.getType()); 25895ffd83dbSDimitry Andric if (!VecTy) 25905ffd83dbSDimitry Andric return nullptr; 25915ffd83dbSDimitry Andric 25925ffd83dbSDimitry Andric unsigned NumElts = VecTy->getNumElements(); 2593cb14a3feSDimitry Andric APInt PoisonElts(NumElts, 0); 2594349cc55cSDimitry Andric APInt AllOnesEltMask(APInt::getAllOnes(NumElts)); 2595cb14a3feSDimitry Andric if (Value *V = SimplifyDemandedVectorElts(&Sel, AllOnesEltMask, PoisonElts)) { 25965ffd83dbSDimitry Andric if (V != &Sel) 25975ffd83dbSDimitry Andric return replaceInstUsesWith(Sel, V); 25985ffd83dbSDimitry Andric return &Sel; 25995ffd83dbSDimitry Andric } 26005ffd83dbSDimitry Andric 26015ffd83dbSDimitry Andric // A select of a "select shuffle" with a common operand can be rearranged 26025ffd83dbSDimitry Andric // to select followed by "select shuffle". Because of poison, this only works 26035ffd83dbSDimitry Andric // in the case of a shuffle with no undefined mask elements. 26045ffd83dbSDimitry Andric ArrayRef<int> Mask; 26055ffd83dbSDimitry Andric if (match(TVal, m_OneUse(m_Shuffle(m_Value(X), m_Value(Y), m_Mask(Mask)))) && 260606c3fb27SDimitry Andric !is_contained(Mask, PoisonMaskElem) && 26075ffd83dbSDimitry Andric cast<ShuffleVectorInst>(TVal)->isSelect()) { 26085ffd83dbSDimitry Andric if (X == FVal) { 26095ffd83dbSDimitry Andric // select Cond, (shuf_sel X, Y), X --> shuf_sel X, (select Cond, Y, X) 26105ffd83dbSDimitry Andric Value *NewSel = Builder.CreateSelect(Cond, Y, X, "sel", &Sel); 26115ffd83dbSDimitry Andric return new ShuffleVectorInst(X, NewSel, Mask); 26125ffd83dbSDimitry Andric } 26135ffd83dbSDimitry Andric if (Y == FVal) { 26145ffd83dbSDimitry Andric // select Cond, (shuf_sel X, Y), Y --> shuf_sel (select Cond, X, Y), Y 26155ffd83dbSDimitry Andric Value *NewSel = Builder.CreateSelect(Cond, X, Y, "sel", &Sel); 26165ffd83dbSDimitry Andric return new ShuffleVectorInst(NewSel, Y, Mask); 26175ffd83dbSDimitry Andric } 26185ffd83dbSDimitry Andric } 26195ffd83dbSDimitry Andric if (match(FVal, m_OneUse(m_Shuffle(m_Value(X), m_Value(Y), m_Mask(Mask)))) && 262006c3fb27SDimitry Andric !is_contained(Mask, PoisonMaskElem) && 26215ffd83dbSDimitry Andric cast<ShuffleVectorInst>(FVal)->isSelect()) { 26225ffd83dbSDimitry Andric if (X == TVal) { 26235ffd83dbSDimitry Andric // select Cond, X, (shuf_sel X, Y) --> shuf_sel X, (select Cond, X, Y) 26245ffd83dbSDimitry Andric Value *NewSel = Builder.CreateSelect(Cond, X, Y, "sel", &Sel); 26255ffd83dbSDimitry Andric return new ShuffleVectorInst(X, NewSel, Mask); 26265ffd83dbSDimitry Andric } 26275ffd83dbSDimitry Andric if (Y == TVal) { 26285ffd83dbSDimitry Andric // select Cond, Y, (shuf_sel X, Y) --> shuf_sel (select Cond, Y, X), Y 26295ffd83dbSDimitry Andric Value *NewSel = Builder.CreateSelect(Cond, Y, X, "sel", &Sel); 26305ffd83dbSDimitry Andric return new ShuffleVectorInst(NewSel, Y, Mask); 26315ffd83dbSDimitry Andric } 26325ffd83dbSDimitry Andric } 26335ffd83dbSDimitry Andric 26345ffd83dbSDimitry Andric return nullptr; 26355ffd83dbSDimitry Andric } 26365ffd83dbSDimitry Andric 26375ffd83dbSDimitry Andric static Instruction *foldSelectToPhiImpl(SelectInst &Sel, BasicBlock *BB, 26385ffd83dbSDimitry Andric const DominatorTree &DT, 26395ffd83dbSDimitry Andric InstCombiner::BuilderTy &Builder) { 26405ffd83dbSDimitry Andric // Find the block's immediate dominator that ends with a conditional branch 26415ffd83dbSDimitry Andric // that matches select's condition (maybe inverted). 26425ffd83dbSDimitry Andric auto *IDomNode = DT[BB]->getIDom(); 26435ffd83dbSDimitry Andric if (!IDomNode) 26445ffd83dbSDimitry Andric return nullptr; 26455ffd83dbSDimitry Andric BasicBlock *IDom = IDomNode->getBlock(); 26465ffd83dbSDimitry Andric 26475ffd83dbSDimitry Andric Value *Cond = Sel.getCondition(); 26485ffd83dbSDimitry Andric Value *IfTrue, *IfFalse; 26495ffd83dbSDimitry Andric BasicBlock *TrueSucc, *FalseSucc; 26505ffd83dbSDimitry Andric if (match(IDom->getTerminator(), 26515ffd83dbSDimitry Andric m_Br(m_Specific(Cond), m_BasicBlock(TrueSucc), 26525ffd83dbSDimitry Andric m_BasicBlock(FalseSucc)))) { 26535ffd83dbSDimitry Andric IfTrue = Sel.getTrueValue(); 26545ffd83dbSDimitry Andric IfFalse = Sel.getFalseValue(); 26555ffd83dbSDimitry Andric } else if (match(IDom->getTerminator(), 26565ffd83dbSDimitry Andric m_Br(m_Not(m_Specific(Cond)), m_BasicBlock(TrueSucc), 26575ffd83dbSDimitry Andric m_BasicBlock(FalseSucc)))) { 26585ffd83dbSDimitry Andric IfTrue = Sel.getFalseValue(); 26595ffd83dbSDimitry Andric IfFalse = Sel.getTrueValue(); 26605ffd83dbSDimitry Andric } else 26615ffd83dbSDimitry Andric return nullptr; 26625ffd83dbSDimitry Andric 2663590d96feSDimitry Andric // Make sure the branches are actually different. 2664590d96feSDimitry Andric if (TrueSucc == FalseSucc) 2665590d96feSDimitry Andric return nullptr; 2666590d96feSDimitry Andric 26675ffd83dbSDimitry Andric // We want to replace select %cond, %a, %b with a phi that takes value %a 26685ffd83dbSDimitry Andric // for all incoming edges that are dominated by condition `%cond == true`, 26695ffd83dbSDimitry Andric // and value %b for edges dominated by condition `%cond == false`. If %a 26705ffd83dbSDimitry Andric // or %b are also phis from the same basic block, we can go further and take 26715ffd83dbSDimitry Andric // their incoming values from the corresponding blocks. 26725ffd83dbSDimitry Andric BasicBlockEdge TrueEdge(IDom, TrueSucc); 26735ffd83dbSDimitry Andric BasicBlockEdge FalseEdge(IDom, FalseSucc); 26745ffd83dbSDimitry Andric DenseMap<BasicBlock *, Value *> Inputs; 26755ffd83dbSDimitry Andric for (auto *Pred : predecessors(BB)) { 26765ffd83dbSDimitry Andric // Check implication. 26775ffd83dbSDimitry Andric BasicBlockEdge Incoming(Pred, BB); 26785ffd83dbSDimitry Andric if (DT.dominates(TrueEdge, Incoming)) 26795ffd83dbSDimitry Andric Inputs[Pred] = IfTrue->DoPHITranslation(BB, Pred); 26805ffd83dbSDimitry Andric else if (DT.dominates(FalseEdge, Incoming)) 26815ffd83dbSDimitry Andric Inputs[Pred] = IfFalse->DoPHITranslation(BB, Pred); 26825ffd83dbSDimitry Andric else 26835ffd83dbSDimitry Andric return nullptr; 26845ffd83dbSDimitry Andric // Check availability. 26855ffd83dbSDimitry Andric if (auto *Insn = dyn_cast<Instruction>(Inputs[Pred])) 26865ffd83dbSDimitry Andric if (!DT.dominates(Insn, Pred->getTerminator())) 26875ffd83dbSDimitry Andric return nullptr; 26885ffd83dbSDimitry Andric } 26895ffd83dbSDimitry Andric 26905f757f3fSDimitry Andric Builder.SetInsertPoint(BB, BB->begin()); 26915ffd83dbSDimitry Andric auto *PN = Builder.CreatePHI(Sel.getType(), Inputs.size()); 26925ffd83dbSDimitry Andric for (auto *Pred : predecessors(BB)) 26935ffd83dbSDimitry Andric PN->addIncoming(Inputs[Pred], Pred); 26945ffd83dbSDimitry Andric PN->takeName(&Sel); 26955ffd83dbSDimitry Andric return PN; 26965ffd83dbSDimitry Andric } 26975ffd83dbSDimitry Andric 26985ffd83dbSDimitry Andric static Instruction *foldSelectToPhi(SelectInst &Sel, const DominatorTree &DT, 26995ffd83dbSDimitry Andric InstCombiner::BuilderTy &Builder) { 27005ffd83dbSDimitry Andric // Try to replace this select with Phi in one of these blocks. 27015ffd83dbSDimitry Andric SmallSetVector<BasicBlock *, 4> CandidateBlocks; 27025ffd83dbSDimitry Andric CandidateBlocks.insert(Sel.getParent()); 27035ffd83dbSDimitry Andric for (Value *V : Sel.operands()) 27045ffd83dbSDimitry Andric if (auto *I = dyn_cast<Instruction>(V)) 27055ffd83dbSDimitry Andric CandidateBlocks.insert(I->getParent()); 27065ffd83dbSDimitry Andric 27075ffd83dbSDimitry Andric for (BasicBlock *BB : CandidateBlocks) 27085ffd83dbSDimitry Andric if (auto *PN = foldSelectToPhiImpl(Sel, BB, DT, Builder)) 27095ffd83dbSDimitry Andric return PN; 27105ffd83dbSDimitry Andric return nullptr; 27115ffd83dbSDimitry Andric } 27125ffd83dbSDimitry Andric 27135f757f3fSDimitry Andric /// Tries to reduce a pattern that arises when calculating the remainder of the 27145f757f3fSDimitry Andric /// Euclidean division. When the divisor is a power of two and is guaranteed not 27155f757f3fSDimitry Andric /// to be negative, a signed remainder can be folded with a bitwise and. 27165f757f3fSDimitry Andric /// 27175f757f3fSDimitry Andric /// (x % n) < 0 ? (x % n) + n : (x % n) 27185f757f3fSDimitry Andric /// -> x & (n - 1) 27195f757f3fSDimitry Andric static Instruction *foldSelectWithSRem(SelectInst &SI, InstCombinerImpl &IC, 27205f757f3fSDimitry Andric IRBuilderBase &Builder) { 27215f757f3fSDimitry Andric Value *CondVal = SI.getCondition(); 27225f757f3fSDimitry Andric Value *TrueVal = SI.getTrueValue(); 27235f757f3fSDimitry Andric Value *FalseVal = SI.getFalseValue(); 27245f757f3fSDimitry Andric 27255f757f3fSDimitry Andric ICmpInst::Predicate Pred; 27265f757f3fSDimitry Andric Value *Op, *RemRes, *Remainder; 27275f757f3fSDimitry Andric const APInt *C; 27285f757f3fSDimitry Andric bool TrueIfSigned = false; 27295f757f3fSDimitry Andric 27305f757f3fSDimitry Andric if (!(match(CondVal, m_ICmp(Pred, m_Value(RemRes), m_APInt(C))) && 2731*0fca6ea1SDimitry Andric isSignBitCheck(Pred, *C, TrueIfSigned))) 27325f757f3fSDimitry Andric return nullptr; 27335f757f3fSDimitry Andric 27345f757f3fSDimitry Andric // If the sign bit is not set, we have a SGE/SGT comparison, and the operands 27355f757f3fSDimitry Andric // of the select are inverted. 27365f757f3fSDimitry Andric if (!TrueIfSigned) 27375f757f3fSDimitry Andric std::swap(TrueVal, FalseVal); 27385f757f3fSDimitry Andric 27395f757f3fSDimitry Andric auto FoldToBitwiseAnd = [&](Value *Remainder) -> Instruction * { 27405f757f3fSDimitry Andric Value *Add = Builder.CreateAdd( 27415f757f3fSDimitry Andric Remainder, Constant::getAllOnesValue(RemRes->getType())); 27425f757f3fSDimitry Andric return BinaryOperator::CreateAnd(Op, Add); 27435f757f3fSDimitry Andric }; 27445f757f3fSDimitry Andric 27455f757f3fSDimitry Andric // Match the general case: 27465f757f3fSDimitry Andric // %rem = srem i32 %x, %n 27475f757f3fSDimitry Andric // %cnd = icmp slt i32 %rem, 0 27485f757f3fSDimitry Andric // %add = add i32 %rem, %n 27495f757f3fSDimitry Andric // %sel = select i1 %cnd, i32 %add, i32 %rem 27505678d1d9SDimitry Andric if (match(TrueVal, m_Add(m_Specific(RemRes), m_Value(Remainder))) && 27515f757f3fSDimitry Andric match(RemRes, m_SRem(m_Value(Op), m_Specific(Remainder))) && 27525f757f3fSDimitry Andric IC.isKnownToBeAPowerOfTwo(Remainder, /*OrZero*/ true) && 27535f757f3fSDimitry Andric FalseVal == RemRes) 27545f757f3fSDimitry Andric return FoldToBitwiseAnd(Remainder); 27555f757f3fSDimitry Andric 27565f757f3fSDimitry Andric // Match the case where the one arm has been replaced by constant 1: 27575f757f3fSDimitry Andric // %rem = srem i32 %n, 2 27585f757f3fSDimitry Andric // %cnd = icmp slt i32 %rem, 0 27595f757f3fSDimitry Andric // %sel = select i1 %cnd, i32 1, i32 %rem 27605f757f3fSDimitry Andric if (match(TrueVal, m_One()) && 27615f757f3fSDimitry Andric match(RemRes, m_SRem(m_Value(Op), m_SpecificInt(2))) && 27625f757f3fSDimitry Andric FalseVal == RemRes) 27635f757f3fSDimitry Andric return FoldToBitwiseAnd(ConstantInt::get(RemRes->getType(), 2)); 27645f757f3fSDimitry Andric 27655f757f3fSDimitry Andric return nullptr; 27665f757f3fSDimitry Andric } 27675f757f3fSDimitry Andric 2768e8d8bef9SDimitry Andric static Value *foldSelectWithFrozenICmp(SelectInst &Sel, InstCombiner::BuilderTy &Builder) { 2769e8d8bef9SDimitry Andric FreezeInst *FI = dyn_cast<FreezeInst>(Sel.getCondition()); 2770e8d8bef9SDimitry Andric if (!FI) 2771e8d8bef9SDimitry Andric return nullptr; 2772e8d8bef9SDimitry Andric 2773e8d8bef9SDimitry Andric Value *Cond = FI->getOperand(0); 2774e8d8bef9SDimitry Andric Value *TrueVal = Sel.getTrueValue(), *FalseVal = Sel.getFalseValue(); 2775e8d8bef9SDimitry Andric 2776e8d8bef9SDimitry Andric // select (freeze(x == y)), x, y --> y 2777e8d8bef9SDimitry Andric // select (freeze(x != y)), x, y --> x 2778e8d8bef9SDimitry Andric // The freeze should be only used by this select. Otherwise, remaining uses of 2779e8d8bef9SDimitry Andric // the freeze can observe a contradictory value. 2780e8d8bef9SDimitry Andric // c = freeze(x == y) ; Let's assume that y = poison & x = 42; c is 0 or 1 2781e8d8bef9SDimitry Andric // a = select c, x, y ; 2782e8d8bef9SDimitry Andric // f(a, c) ; f(poison, 1) cannot happen, but if a is folded 2783e8d8bef9SDimitry Andric // ; to y, this can happen. 2784e8d8bef9SDimitry Andric CmpInst::Predicate Pred; 2785e8d8bef9SDimitry Andric if (FI->hasOneUse() && 2786e8d8bef9SDimitry Andric match(Cond, m_c_ICmp(Pred, m_Specific(TrueVal), m_Specific(FalseVal))) && 2787e8d8bef9SDimitry Andric (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE)) { 2788e8d8bef9SDimitry Andric return Pred == ICmpInst::ICMP_EQ ? FalseVal : TrueVal; 2789e8d8bef9SDimitry Andric } 2790e8d8bef9SDimitry Andric 2791e8d8bef9SDimitry Andric return nullptr; 2792e8d8bef9SDimitry Andric } 2793e8d8bef9SDimitry Andric 2794*0fca6ea1SDimitry Andric /// Given that \p CondVal is known to be \p CondIsTrue, try to simplify \p SI. 2795*0fca6ea1SDimitry Andric static Value *simplifyNestedSelectsUsingImpliedCond(SelectInst &SI, 2796*0fca6ea1SDimitry Andric Value *CondVal, 2797*0fca6ea1SDimitry Andric bool CondIsTrue, 2798*0fca6ea1SDimitry Andric const DataLayout &DL) { 2799*0fca6ea1SDimitry Andric Value *InnerCondVal = SI.getCondition(); 2800*0fca6ea1SDimitry Andric Value *InnerTrueVal = SI.getTrueValue(); 2801*0fca6ea1SDimitry Andric Value *InnerFalseVal = SI.getFalseValue(); 2802*0fca6ea1SDimitry Andric assert(CondVal->getType() == InnerCondVal->getType() && 2803*0fca6ea1SDimitry Andric "The type of inner condition must match with the outer."); 2804*0fca6ea1SDimitry Andric if (auto Implied = isImpliedCondition(CondVal, InnerCondVal, DL, CondIsTrue)) 2805*0fca6ea1SDimitry Andric return *Implied ? InnerTrueVal : InnerFalseVal; 2806*0fca6ea1SDimitry Andric return nullptr; 2807*0fca6ea1SDimitry Andric } 2808*0fca6ea1SDimitry Andric 2809fe6060f1SDimitry Andric Instruction *InstCombinerImpl::foldAndOrOfSelectUsingImpliedCond(Value *Op, 2810fe6060f1SDimitry Andric SelectInst &SI, 2811fe6060f1SDimitry Andric bool IsAnd) { 2812fe6060f1SDimitry Andric assert(Op->getType()->isIntOrIntVectorTy(1) && 2813fe6060f1SDimitry Andric "Op must be either i1 or vector of i1."); 2814*0fca6ea1SDimitry Andric if (SI.getCondition()->getType() != Op->getType()) 2815fe6060f1SDimitry Andric return nullptr; 2816*0fca6ea1SDimitry Andric if (Value *V = simplifyNestedSelectsUsingImpliedCond(SI, Op, IsAnd, DL)) 2817*0fca6ea1SDimitry Andric return SelectInst::Create(Op, 2818*0fca6ea1SDimitry Andric IsAnd ? V : ConstantInt::getTrue(Op->getType()), 2819*0fca6ea1SDimitry Andric IsAnd ? ConstantInt::getFalse(Op->getType()) : V); 2820*0fca6ea1SDimitry Andric return nullptr; 2821fe6060f1SDimitry Andric } 2822fe6060f1SDimitry Andric 282381ad6265SDimitry Andric // Canonicalize select with fcmp to fabs(). -0.0 makes this tricky. We need 282481ad6265SDimitry Andric // fast-math-flags (nsz) or fsub with +0.0 (not fneg) for this to work. 282581ad6265SDimitry Andric static Instruction *foldSelectWithFCmpToFabs(SelectInst &SI, 282681ad6265SDimitry Andric InstCombinerImpl &IC) { 282781ad6265SDimitry Andric Value *CondVal = SI.getCondition(); 282881ad6265SDimitry Andric 2829bdd1243dSDimitry Andric bool ChangedFMF = false; 283081ad6265SDimitry Andric for (bool Swap : {false, true}) { 283181ad6265SDimitry Andric Value *TrueVal = SI.getTrueValue(); 283281ad6265SDimitry Andric Value *X = SI.getFalseValue(); 283381ad6265SDimitry Andric CmpInst::Predicate Pred; 283481ad6265SDimitry Andric 283581ad6265SDimitry Andric if (Swap) 283681ad6265SDimitry Andric std::swap(TrueVal, X); 283781ad6265SDimitry Andric 283881ad6265SDimitry Andric if (!match(CondVal, m_FCmp(Pred, m_Specific(X), m_AnyZeroFP()))) 283981ad6265SDimitry Andric continue; 284081ad6265SDimitry Andric 284181ad6265SDimitry Andric // fold (X <= +/-0.0) ? (0.0 - X) : X to fabs(X), when 'Swap' is false 284281ad6265SDimitry Andric // fold (X > +/-0.0) ? X : (0.0 - X) to fabs(X), when 'Swap' is true 284381ad6265SDimitry Andric if (match(TrueVal, m_FSub(m_PosZeroFP(), m_Specific(X)))) { 284481ad6265SDimitry Andric if (!Swap && (Pred == FCmpInst::FCMP_OLE || Pred == FCmpInst::FCMP_ULE)) { 284581ad6265SDimitry Andric Value *Fabs = IC.Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, &SI); 284681ad6265SDimitry Andric return IC.replaceInstUsesWith(SI, Fabs); 284781ad6265SDimitry Andric } 284881ad6265SDimitry Andric if (Swap && (Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_UGT)) { 284981ad6265SDimitry Andric Value *Fabs = IC.Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, &SI); 285081ad6265SDimitry Andric return IC.replaceInstUsesWith(SI, Fabs); 285181ad6265SDimitry Andric } 285281ad6265SDimitry Andric } 285381ad6265SDimitry Andric 2854bdd1243dSDimitry Andric if (!match(TrueVal, m_FNeg(m_Specific(X)))) 2855bdd1243dSDimitry Andric return nullptr; 2856bdd1243dSDimitry Andric 2857bdd1243dSDimitry Andric // Forward-propagate nnan and ninf from the fneg to the select. 2858bdd1243dSDimitry Andric // If all inputs are not those values, then the select is not either. 2859bdd1243dSDimitry Andric // Note: nsz is defined differently, so it may not be correct to propagate. 2860bdd1243dSDimitry Andric FastMathFlags FMF = cast<FPMathOperator>(TrueVal)->getFastMathFlags(); 2861bdd1243dSDimitry Andric if (FMF.noNaNs() && !SI.hasNoNaNs()) { 2862bdd1243dSDimitry Andric SI.setHasNoNaNs(true); 2863bdd1243dSDimitry Andric ChangedFMF = true; 2864bdd1243dSDimitry Andric } 2865bdd1243dSDimitry Andric if (FMF.noInfs() && !SI.hasNoInfs()) { 2866bdd1243dSDimitry Andric SI.setHasNoInfs(true); 2867bdd1243dSDimitry Andric ChangedFMF = true; 2868bdd1243dSDimitry Andric } 2869bdd1243dSDimitry Andric 287081ad6265SDimitry Andric // With nsz, when 'Swap' is false: 287181ad6265SDimitry Andric // fold (X < +/-0.0) ? -X : X or (X <= +/-0.0) ? -X : X to fabs(X) 287281ad6265SDimitry Andric // fold (X > +/-0.0) ? -X : X or (X >= +/-0.0) ? -X : X to -fabs(x) 287381ad6265SDimitry Andric // when 'Swap' is true: 287481ad6265SDimitry Andric // fold (X > +/-0.0) ? X : -X or (X >= +/-0.0) ? X : -X to fabs(X) 287581ad6265SDimitry Andric // fold (X < +/-0.0) ? X : -X or (X <= +/-0.0) ? X : -X to -fabs(X) 2876bdd1243dSDimitry Andric // 2877bdd1243dSDimitry Andric // Note: We require "nnan" for this fold because fcmp ignores the signbit 2878bdd1243dSDimitry Andric // of NAN, but IEEE-754 specifies the signbit of NAN values with 2879bdd1243dSDimitry Andric // fneg/fabs operations. 2880bdd1243dSDimitry Andric if (!SI.hasNoSignedZeros() || !SI.hasNoNaNs()) 288181ad6265SDimitry Andric return nullptr; 288281ad6265SDimitry Andric 288381ad6265SDimitry Andric if (Swap) 288481ad6265SDimitry Andric Pred = FCmpInst::getSwappedPredicate(Pred); 288581ad6265SDimitry Andric 288681ad6265SDimitry Andric bool IsLTOrLE = Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_OLE || 288781ad6265SDimitry Andric Pred == FCmpInst::FCMP_ULT || Pred == FCmpInst::FCMP_ULE; 288881ad6265SDimitry Andric bool IsGTOrGE = Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_OGE || 288981ad6265SDimitry Andric Pred == FCmpInst::FCMP_UGT || Pred == FCmpInst::FCMP_UGE; 289081ad6265SDimitry Andric 289181ad6265SDimitry Andric if (IsLTOrLE) { 289281ad6265SDimitry Andric Value *Fabs = IC.Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, &SI); 289381ad6265SDimitry Andric return IC.replaceInstUsesWith(SI, Fabs); 289481ad6265SDimitry Andric } 289581ad6265SDimitry Andric if (IsGTOrGE) { 289681ad6265SDimitry Andric Value *Fabs = IC.Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, &SI); 289781ad6265SDimitry Andric Instruction *NewFNeg = UnaryOperator::CreateFNeg(Fabs); 289881ad6265SDimitry Andric NewFNeg->setFastMathFlags(SI.getFastMathFlags()); 289981ad6265SDimitry Andric return NewFNeg; 290081ad6265SDimitry Andric } 290181ad6265SDimitry Andric } 290281ad6265SDimitry Andric 2903*0fca6ea1SDimitry Andric // Match select with (icmp slt (bitcast X to int), 0) 2904*0fca6ea1SDimitry Andric // or (icmp sgt (bitcast X to int), -1) 2905*0fca6ea1SDimitry Andric 2906*0fca6ea1SDimitry Andric for (bool Swap : {false, true}) { 2907*0fca6ea1SDimitry Andric Value *TrueVal = SI.getTrueValue(); 2908*0fca6ea1SDimitry Andric Value *X = SI.getFalseValue(); 2909*0fca6ea1SDimitry Andric 2910*0fca6ea1SDimitry Andric if (Swap) 2911*0fca6ea1SDimitry Andric std::swap(TrueVal, X); 2912*0fca6ea1SDimitry Andric 2913*0fca6ea1SDimitry Andric CmpInst::Predicate Pred; 2914*0fca6ea1SDimitry Andric const APInt *C; 2915*0fca6ea1SDimitry Andric bool TrueIfSigned; 2916*0fca6ea1SDimitry Andric if (!match(CondVal, 2917*0fca6ea1SDimitry Andric m_ICmp(Pred, m_ElementWiseBitCast(m_Specific(X)), m_APInt(C))) || 2918*0fca6ea1SDimitry Andric !isSignBitCheck(Pred, *C, TrueIfSigned)) 2919*0fca6ea1SDimitry Andric continue; 2920*0fca6ea1SDimitry Andric if (!match(TrueVal, m_FNeg(m_Specific(X)))) 2921*0fca6ea1SDimitry Andric return nullptr; 2922*0fca6ea1SDimitry Andric if (Swap == TrueIfSigned && !CondVal->hasOneUse() && !TrueVal->hasOneUse()) 2923*0fca6ea1SDimitry Andric return nullptr; 2924*0fca6ea1SDimitry Andric 2925*0fca6ea1SDimitry Andric // Fold (IsNeg ? -X : X) or (!IsNeg ? X : -X) to fabs(X) 2926*0fca6ea1SDimitry Andric // Fold (IsNeg ? X : -X) or (!IsNeg ? -X : X) to -fabs(X) 2927*0fca6ea1SDimitry Andric Value *Fabs = IC.Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, &SI); 2928*0fca6ea1SDimitry Andric if (Swap != TrueIfSigned) 2929*0fca6ea1SDimitry Andric return IC.replaceInstUsesWith(SI, Fabs); 2930*0fca6ea1SDimitry Andric return UnaryOperator::CreateFNegFMF(Fabs, &SI); 2931*0fca6ea1SDimitry Andric } 2932*0fca6ea1SDimitry Andric 2933bdd1243dSDimitry Andric return ChangedFMF ? &SI : nullptr; 293481ad6265SDimitry Andric } 293581ad6265SDimitry Andric 293681ad6265SDimitry Andric // Match the following IR pattern: 293781ad6265SDimitry Andric // %x.lowbits = and i8 %x, %lowbitmask 293881ad6265SDimitry Andric // %x.lowbits.are.zero = icmp eq i8 %x.lowbits, 0 293981ad6265SDimitry Andric // %x.biased = add i8 %x, %bias 294081ad6265SDimitry Andric // %x.biased.highbits = and i8 %x.biased, %highbitmask 294181ad6265SDimitry Andric // %x.roundedup = select i1 %x.lowbits.are.zero, i8 %x, i8 %x.biased.highbits 294281ad6265SDimitry Andric // Define: 294381ad6265SDimitry Andric // %alignment = add i8 %lowbitmask, 1 294481ad6265SDimitry Andric // Iff 1. an %alignment is a power-of-two (aka, %lowbitmask is a low bit mask) 294581ad6265SDimitry Andric // and 2. %bias is equal to either %lowbitmask or %alignment, 294681ad6265SDimitry Andric // and 3. %highbitmask is equal to ~%lowbitmask (aka, to -%alignment) 294781ad6265SDimitry Andric // then this pattern can be transformed into: 294881ad6265SDimitry Andric // %x.offset = add i8 %x, %lowbitmask 294981ad6265SDimitry Andric // %x.roundedup = and i8 %x.offset, %highbitmask 295081ad6265SDimitry Andric static Value * 295181ad6265SDimitry Andric foldRoundUpIntegerWithPow2Alignment(SelectInst &SI, 295281ad6265SDimitry Andric InstCombiner::BuilderTy &Builder) { 295381ad6265SDimitry Andric Value *Cond = SI.getCondition(); 295481ad6265SDimitry Andric Value *X = SI.getTrueValue(); 295581ad6265SDimitry Andric Value *XBiasedHighBits = SI.getFalseValue(); 295681ad6265SDimitry Andric 295781ad6265SDimitry Andric ICmpInst::Predicate Pred; 295881ad6265SDimitry Andric Value *XLowBits; 295981ad6265SDimitry Andric if (!match(Cond, m_ICmp(Pred, m_Value(XLowBits), m_ZeroInt())) || 296081ad6265SDimitry Andric !ICmpInst::isEquality(Pred)) 296181ad6265SDimitry Andric return nullptr; 296281ad6265SDimitry Andric 296381ad6265SDimitry Andric if (Pred == ICmpInst::Predicate::ICMP_NE) 296481ad6265SDimitry Andric std::swap(X, XBiasedHighBits); 296581ad6265SDimitry Andric 296681ad6265SDimitry Andric // FIXME: we could support non non-splats here. 296781ad6265SDimitry Andric 296881ad6265SDimitry Andric const APInt *LowBitMaskCst; 2969*0fca6ea1SDimitry Andric if (!match(XLowBits, m_And(m_Specific(X), m_APIntAllowPoison(LowBitMaskCst)))) 297081ad6265SDimitry Andric return nullptr; 297181ad6265SDimitry Andric 2972bdd1243dSDimitry Andric // Match even if the AND and ADD are swapped. 297381ad6265SDimitry Andric const APInt *BiasCst, *HighBitMaskCst; 297481ad6265SDimitry Andric if (!match(XBiasedHighBits, 2975*0fca6ea1SDimitry Andric m_And(m_Add(m_Specific(X), m_APIntAllowPoison(BiasCst)), 2976*0fca6ea1SDimitry Andric m_APIntAllowPoison(HighBitMaskCst))) && 2977bdd1243dSDimitry Andric !match(XBiasedHighBits, 2978*0fca6ea1SDimitry Andric m_Add(m_And(m_Specific(X), m_APIntAllowPoison(HighBitMaskCst)), 2979*0fca6ea1SDimitry Andric m_APIntAllowPoison(BiasCst)))) 298081ad6265SDimitry Andric return nullptr; 298181ad6265SDimitry Andric 298281ad6265SDimitry Andric if (!LowBitMaskCst->isMask()) 298381ad6265SDimitry Andric return nullptr; 298481ad6265SDimitry Andric 298581ad6265SDimitry Andric APInt InvertedLowBitMaskCst = ~*LowBitMaskCst; 298681ad6265SDimitry Andric if (InvertedLowBitMaskCst != *HighBitMaskCst) 298781ad6265SDimitry Andric return nullptr; 298881ad6265SDimitry Andric 298981ad6265SDimitry Andric APInt AlignmentCst = *LowBitMaskCst + 1; 299081ad6265SDimitry Andric 299181ad6265SDimitry Andric if (*BiasCst != AlignmentCst && *BiasCst != *LowBitMaskCst) 299281ad6265SDimitry Andric return nullptr; 299381ad6265SDimitry Andric 299481ad6265SDimitry Andric if (!XBiasedHighBits->hasOneUse()) { 2995*0fca6ea1SDimitry Andric // We can't directly return XBiasedHighBits if it is more poisonous. 2996*0fca6ea1SDimitry Andric if (*BiasCst == *LowBitMaskCst && impliesPoison(XBiasedHighBits, X)) 299781ad6265SDimitry Andric return XBiasedHighBits; 299881ad6265SDimitry Andric return nullptr; 299981ad6265SDimitry Andric } 300081ad6265SDimitry Andric 300181ad6265SDimitry Andric // FIXME: could we preserve undef's here? 300281ad6265SDimitry Andric Type *Ty = X->getType(); 300381ad6265SDimitry Andric Value *XOffset = Builder.CreateAdd(X, ConstantInt::get(Ty, *LowBitMaskCst), 300481ad6265SDimitry Andric X->getName() + ".biased"); 300581ad6265SDimitry Andric Value *R = Builder.CreateAnd(XOffset, ConstantInt::get(Ty, *HighBitMaskCst)); 300681ad6265SDimitry Andric R->takeName(&SI); 300781ad6265SDimitry Andric return R; 300881ad6265SDimitry Andric } 300981ad6265SDimitry Andric 3010bdd1243dSDimitry Andric namespace { 3011bdd1243dSDimitry Andric struct DecomposedSelect { 3012bdd1243dSDimitry Andric Value *Cond = nullptr; 3013bdd1243dSDimitry Andric Value *TrueVal = nullptr; 3014bdd1243dSDimitry Andric Value *FalseVal = nullptr; 3015bdd1243dSDimitry Andric }; 3016bdd1243dSDimitry Andric } // namespace 3017bdd1243dSDimitry Andric 3018*0fca6ea1SDimitry Andric /// Folds patterns like: 3019*0fca6ea1SDimitry Andric /// select c2 (select c1 a b) (select c1 b a) 3020*0fca6ea1SDimitry Andric /// into: 3021*0fca6ea1SDimitry Andric /// select (xor c1 c2) b a 3022*0fca6ea1SDimitry Andric static Instruction * 3023*0fca6ea1SDimitry Andric foldSelectOfSymmetricSelect(SelectInst &OuterSelVal, 3024*0fca6ea1SDimitry Andric InstCombiner::BuilderTy &Builder) { 3025*0fca6ea1SDimitry Andric 3026*0fca6ea1SDimitry Andric Value *OuterCond, *InnerCond, *InnerTrueVal, *InnerFalseVal; 3027*0fca6ea1SDimitry Andric if (!match( 3028*0fca6ea1SDimitry Andric &OuterSelVal, 3029*0fca6ea1SDimitry Andric m_Select(m_Value(OuterCond), 3030*0fca6ea1SDimitry Andric m_OneUse(m_Select(m_Value(InnerCond), m_Value(InnerTrueVal), 3031*0fca6ea1SDimitry Andric m_Value(InnerFalseVal))), 3032*0fca6ea1SDimitry Andric m_OneUse(m_Select(m_Deferred(InnerCond), 3033*0fca6ea1SDimitry Andric m_Deferred(InnerFalseVal), 3034*0fca6ea1SDimitry Andric m_Deferred(InnerTrueVal)))))) 3035*0fca6ea1SDimitry Andric return nullptr; 3036*0fca6ea1SDimitry Andric 3037*0fca6ea1SDimitry Andric if (OuterCond->getType() != InnerCond->getType()) 3038*0fca6ea1SDimitry Andric return nullptr; 3039*0fca6ea1SDimitry Andric 3040*0fca6ea1SDimitry Andric Value *Xor = Builder.CreateXor(InnerCond, OuterCond); 3041*0fca6ea1SDimitry Andric return SelectInst::Create(Xor, InnerFalseVal, InnerTrueVal); 3042*0fca6ea1SDimitry Andric } 3043*0fca6ea1SDimitry Andric 3044bdd1243dSDimitry Andric /// Look for patterns like 3045bdd1243dSDimitry Andric /// %outer.cond = select i1 %inner.cond, i1 %alt.cond, i1 false 3046bdd1243dSDimitry Andric /// %inner.sel = select i1 %inner.cond, i8 %inner.sel.t, i8 %inner.sel.f 3047bdd1243dSDimitry Andric /// %outer.sel = select i1 %outer.cond, i8 %outer.sel.t, i8 %inner.sel 3048bdd1243dSDimitry Andric /// and rewrite it as 3049bdd1243dSDimitry Andric /// %inner.sel = select i1 %cond.alternative, i8 %sel.outer.t, i8 %sel.inner.t 3050bdd1243dSDimitry Andric /// %sel.outer = select i1 %cond.inner, i8 %inner.sel, i8 %sel.inner.f 3051bdd1243dSDimitry Andric static Instruction *foldNestedSelects(SelectInst &OuterSelVal, 3052bdd1243dSDimitry Andric InstCombiner::BuilderTy &Builder) { 3053bdd1243dSDimitry Andric // We must start with a `select`. 3054bdd1243dSDimitry Andric DecomposedSelect OuterSel; 3055bdd1243dSDimitry Andric match(&OuterSelVal, 3056bdd1243dSDimitry Andric m_Select(m_Value(OuterSel.Cond), m_Value(OuterSel.TrueVal), 3057bdd1243dSDimitry Andric m_Value(OuterSel.FalseVal))); 3058bdd1243dSDimitry Andric 3059bdd1243dSDimitry Andric // Canonicalize inversion of the outermost `select`'s condition. 3060bdd1243dSDimitry Andric if (match(OuterSel.Cond, m_Not(m_Value(OuterSel.Cond)))) 3061bdd1243dSDimitry Andric std::swap(OuterSel.TrueVal, OuterSel.FalseVal); 3062bdd1243dSDimitry Andric 3063bdd1243dSDimitry Andric // The condition of the outermost select must be an `and`/`or`. 3064bdd1243dSDimitry Andric if (!match(OuterSel.Cond, m_c_LogicalOp(m_Value(), m_Value()))) 3065bdd1243dSDimitry Andric return nullptr; 3066bdd1243dSDimitry Andric 3067bdd1243dSDimitry Andric // Depending on the logical op, inner select might be in different hand. 3068bdd1243dSDimitry Andric bool IsAndVariant = match(OuterSel.Cond, m_LogicalAnd()); 3069bdd1243dSDimitry Andric Value *InnerSelVal = IsAndVariant ? OuterSel.FalseVal : OuterSel.TrueVal; 3070bdd1243dSDimitry Andric 3071bdd1243dSDimitry Andric // Profitability check - avoid increasing instruction count. 3072bdd1243dSDimitry Andric if (none_of(ArrayRef<Value *>({OuterSelVal.getCondition(), InnerSelVal}), 3073bdd1243dSDimitry Andric [](Value *V) { return V->hasOneUse(); })) 3074bdd1243dSDimitry Andric return nullptr; 3075bdd1243dSDimitry Andric 3076bdd1243dSDimitry Andric // The appropriate hand of the outermost `select` must be a select itself. 3077bdd1243dSDimitry Andric DecomposedSelect InnerSel; 3078bdd1243dSDimitry Andric if (!match(InnerSelVal, 3079bdd1243dSDimitry Andric m_Select(m_Value(InnerSel.Cond), m_Value(InnerSel.TrueVal), 3080bdd1243dSDimitry Andric m_Value(InnerSel.FalseVal)))) 3081bdd1243dSDimitry Andric return nullptr; 3082bdd1243dSDimitry Andric 3083bdd1243dSDimitry Andric // Canonicalize inversion of the innermost `select`'s condition. 3084bdd1243dSDimitry Andric if (match(InnerSel.Cond, m_Not(m_Value(InnerSel.Cond)))) 3085bdd1243dSDimitry Andric std::swap(InnerSel.TrueVal, InnerSel.FalseVal); 3086bdd1243dSDimitry Andric 3087bdd1243dSDimitry Andric Value *AltCond = nullptr; 30885f757f3fSDimitry Andric auto matchOuterCond = [OuterSel, IsAndVariant, &AltCond](auto m_InnerCond) { 30895f757f3fSDimitry Andric // An unsimplified select condition can match both LogicalAnd and LogicalOr 30905f757f3fSDimitry Andric // (select true, true, false). Since below we assume that LogicalAnd implies 30915f757f3fSDimitry Andric // InnerSel match the FVal and vice versa for LogicalOr, we can't match the 30925f757f3fSDimitry Andric // alternative pattern here. 30935f757f3fSDimitry Andric return IsAndVariant ? match(OuterSel.Cond, 30945f757f3fSDimitry Andric m_c_LogicalAnd(m_InnerCond, m_Value(AltCond))) 30955f757f3fSDimitry Andric : match(OuterSel.Cond, 30965f757f3fSDimitry Andric m_c_LogicalOr(m_InnerCond, m_Value(AltCond))); 3097bdd1243dSDimitry Andric }; 3098bdd1243dSDimitry Andric 3099bdd1243dSDimitry Andric // Finally, match the condition that was driving the outermost `select`, 3100bdd1243dSDimitry Andric // it should be a logical operation between the condition that was driving 3101bdd1243dSDimitry Andric // the innermost `select` (after accounting for the possible inversions 3102bdd1243dSDimitry Andric // of the condition), and some other condition. 3103bdd1243dSDimitry Andric if (matchOuterCond(m_Specific(InnerSel.Cond))) { 3104bdd1243dSDimitry Andric // Done! 3105bdd1243dSDimitry Andric } else if (Value * NotInnerCond; matchOuterCond(m_CombineAnd( 3106bdd1243dSDimitry Andric m_Not(m_Specific(InnerSel.Cond)), m_Value(NotInnerCond)))) { 3107bdd1243dSDimitry Andric // Done! 3108bdd1243dSDimitry Andric std::swap(InnerSel.TrueVal, InnerSel.FalseVal); 3109bdd1243dSDimitry Andric InnerSel.Cond = NotInnerCond; 3110bdd1243dSDimitry Andric } else // Not the pattern we were looking for. 3111bdd1243dSDimitry Andric return nullptr; 3112bdd1243dSDimitry Andric 3113bdd1243dSDimitry Andric Value *SelInner = Builder.CreateSelect( 3114bdd1243dSDimitry Andric AltCond, IsAndVariant ? OuterSel.TrueVal : InnerSel.FalseVal, 3115bdd1243dSDimitry Andric IsAndVariant ? InnerSel.TrueVal : OuterSel.FalseVal); 3116bdd1243dSDimitry Andric SelInner->takeName(InnerSelVal); 3117bdd1243dSDimitry Andric return SelectInst::Create(InnerSel.Cond, 3118bdd1243dSDimitry Andric IsAndVariant ? SelInner : InnerSel.TrueVal, 3119bdd1243dSDimitry Andric !IsAndVariant ? SelInner : InnerSel.FalseVal); 3120bdd1243dSDimitry Andric } 3121bdd1243dSDimitry Andric 3122bdd1243dSDimitry Andric Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) { 31230b57cec5SDimitry Andric Value *CondVal = SI.getCondition(); 31240b57cec5SDimitry Andric Value *TrueVal = SI.getTrueValue(); 31250b57cec5SDimitry Andric Value *FalseVal = SI.getFalseValue(); 31260b57cec5SDimitry Andric Type *SelType = SI.getType(); 31270b57cec5SDimitry Andric 3128fe6060f1SDimitry Andric // Avoid potential infinite loops by checking for non-constant condition. 3129fe6060f1SDimitry Andric // TODO: Can we assert instead by improving canonicalizeSelectToShuffle()? 3130fe6060f1SDimitry Andric // Scalar select must have simplified? 3131bdd1243dSDimitry Andric if (!SelType->isIntOrIntVectorTy(1) || isa<Constant>(CondVal) || 3132bdd1243dSDimitry Andric TrueVal->getType() != CondVal->getType()) 3133bdd1243dSDimitry Andric return nullptr; 3134bdd1243dSDimitry Andric 3135bdd1243dSDimitry Andric auto *One = ConstantInt::getTrue(SelType); 3136bdd1243dSDimitry Andric auto *Zero = ConstantInt::getFalse(SelType); 3137bdd1243dSDimitry Andric Value *A, *B, *C, *D; 3138bdd1243dSDimitry Andric 3139fe6060f1SDimitry Andric // Folding select to and/or i1 isn't poison safe in general. impliesPoison 3140fe6060f1SDimitry Andric // checks whether folding it does not convert a well-defined value into 3141fe6060f1SDimitry Andric // poison. 314281ad6265SDimitry Andric if (match(TrueVal, m_One())) { 314381ad6265SDimitry Andric if (impliesPoison(FalseVal, CondVal)) { 31440b57cec5SDimitry Andric // Change: A = select B, true, C --> A = or B, C 31450b57cec5SDimitry Andric return BinaryOperator::CreateOr(CondVal, FalseVal); 31460b57cec5SDimitry Andric } 314781ad6265SDimitry Andric 3148*0fca6ea1SDimitry Andric if (match(CondVal, m_OneUse(m_Select(m_Value(A), m_One(), m_Value(B)))) && 3149*0fca6ea1SDimitry Andric impliesPoison(FalseVal, B)) { 3150*0fca6ea1SDimitry Andric // (A || B) || C --> A || (B | C) 3151*0fca6ea1SDimitry Andric return replaceInstUsesWith( 3152*0fca6ea1SDimitry Andric SI, Builder.CreateLogicalOr(A, Builder.CreateOr(B, FalseVal))); 3153*0fca6ea1SDimitry Andric } 3154*0fca6ea1SDimitry Andric 315581ad6265SDimitry Andric if (auto *LHS = dyn_cast<FCmpInst>(CondVal)) 315681ad6265SDimitry Andric if (auto *RHS = dyn_cast<FCmpInst>(FalseVal)) 315781ad6265SDimitry Andric if (Value *V = foldLogicOfFCmps(LHS, RHS, /*IsAnd*/ false, 315881ad6265SDimitry Andric /*IsSelectLogical*/ true)) 315981ad6265SDimitry Andric return replaceInstUsesWith(SI, V); 3160bdd1243dSDimitry Andric 3161bdd1243dSDimitry Andric // (A && B) || (C && B) --> (A || C) && B 3162bdd1243dSDimitry Andric if (match(CondVal, m_LogicalAnd(m_Value(A), m_Value(B))) && 3163bdd1243dSDimitry Andric match(FalseVal, m_LogicalAnd(m_Value(C), m_Value(D))) && 3164bdd1243dSDimitry Andric (CondVal->hasOneUse() || FalseVal->hasOneUse())) { 3165bdd1243dSDimitry Andric bool CondLogicAnd = isa<SelectInst>(CondVal); 3166bdd1243dSDimitry Andric bool FalseLogicAnd = isa<SelectInst>(FalseVal); 3167bdd1243dSDimitry Andric auto AndFactorization = [&](Value *Common, Value *InnerCond, 3168bdd1243dSDimitry Andric Value *InnerVal, 3169bdd1243dSDimitry Andric bool SelFirst = false) -> Instruction * { 3170bdd1243dSDimitry Andric Value *InnerSel = Builder.CreateSelect(InnerCond, One, InnerVal); 3171bdd1243dSDimitry Andric if (SelFirst) 3172bdd1243dSDimitry Andric std::swap(Common, InnerSel); 3173bdd1243dSDimitry Andric if (FalseLogicAnd || (CondLogicAnd && Common == A)) 3174bdd1243dSDimitry Andric return SelectInst::Create(Common, InnerSel, Zero); 3175bdd1243dSDimitry Andric else 3176bdd1243dSDimitry Andric return BinaryOperator::CreateAnd(Common, InnerSel); 3177bdd1243dSDimitry Andric }; 3178bdd1243dSDimitry Andric 3179bdd1243dSDimitry Andric if (A == C) 3180bdd1243dSDimitry Andric return AndFactorization(A, B, D); 3181bdd1243dSDimitry Andric if (A == D) 3182bdd1243dSDimitry Andric return AndFactorization(A, B, C); 3183bdd1243dSDimitry Andric if (B == C) 3184bdd1243dSDimitry Andric return AndFactorization(B, A, D); 3185bdd1243dSDimitry Andric if (B == D) 3186bdd1243dSDimitry Andric return AndFactorization(B, A, C, CondLogicAnd && FalseLogicAnd); 318781ad6265SDimitry Andric } 3188bdd1243dSDimitry Andric } 3189bdd1243dSDimitry Andric 319081ad6265SDimitry Andric if (match(FalseVal, m_Zero())) { 319181ad6265SDimitry Andric if (impliesPoison(TrueVal, CondVal)) { 31920b57cec5SDimitry Andric // Change: A = select B, C, false --> A = and B, C 31930b57cec5SDimitry Andric return BinaryOperator::CreateAnd(CondVal, TrueVal); 31940b57cec5SDimitry Andric } 3195e8d8bef9SDimitry Andric 3196*0fca6ea1SDimitry Andric if (match(CondVal, m_OneUse(m_Select(m_Value(A), m_Value(B), m_Zero()))) && 3197*0fca6ea1SDimitry Andric impliesPoison(TrueVal, B)) { 3198*0fca6ea1SDimitry Andric // (A && B) && C --> A && (B & C) 3199*0fca6ea1SDimitry Andric return replaceInstUsesWith( 3200*0fca6ea1SDimitry Andric SI, Builder.CreateLogicalAnd(A, Builder.CreateAnd(B, TrueVal))); 3201*0fca6ea1SDimitry Andric } 3202*0fca6ea1SDimitry Andric 320381ad6265SDimitry Andric if (auto *LHS = dyn_cast<FCmpInst>(CondVal)) 320481ad6265SDimitry Andric if (auto *RHS = dyn_cast<FCmpInst>(TrueVal)) 320581ad6265SDimitry Andric if (Value *V = foldLogicOfFCmps(LHS, RHS, /*IsAnd*/ true, 320681ad6265SDimitry Andric /*IsSelectLogical*/ true)) 320781ad6265SDimitry Andric return replaceInstUsesWith(SI, V); 320881ad6265SDimitry Andric 3209bdd1243dSDimitry Andric // (A || B) && (C || B) --> (A && C) || B 3210bdd1243dSDimitry Andric if (match(CondVal, m_LogicalOr(m_Value(A), m_Value(B))) && 3211bdd1243dSDimitry Andric match(TrueVal, m_LogicalOr(m_Value(C), m_Value(D))) && 3212bdd1243dSDimitry Andric (CondVal->hasOneUse() || TrueVal->hasOneUse())) { 3213bdd1243dSDimitry Andric bool CondLogicOr = isa<SelectInst>(CondVal); 3214bdd1243dSDimitry Andric bool TrueLogicOr = isa<SelectInst>(TrueVal); 3215bdd1243dSDimitry Andric auto OrFactorization = [&](Value *Common, Value *InnerCond, 3216bdd1243dSDimitry Andric Value *InnerVal, 3217bdd1243dSDimitry Andric bool SelFirst = false) -> Instruction * { 3218bdd1243dSDimitry Andric Value *InnerSel = Builder.CreateSelect(InnerCond, InnerVal, Zero); 3219bdd1243dSDimitry Andric if (SelFirst) 3220bdd1243dSDimitry Andric std::swap(Common, InnerSel); 3221bdd1243dSDimitry Andric if (TrueLogicOr || (CondLogicOr && Common == A)) 3222bdd1243dSDimitry Andric return SelectInst::Create(Common, One, InnerSel); 3223bdd1243dSDimitry Andric else 3224bdd1243dSDimitry Andric return BinaryOperator::CreateOr(Common, InnerSel); 3225bdd1243dSDimitry Andric }; 3226bdd1243dSDimitry Andric 3227bdd1243dSDimitry Andric if (A == C) 3228bdd1243dSDimitry Andric return OrFactorization(A, B, D); 3229bdd1243dSDimitry Andric if (A == D) 3230bdd1243dSDimitry Andric return OrFactorization(A, B, C); 3231bdd1243dSDimitry Andric if (B == C) 3232bdd1243dSDimitry Andric return OrFactorization(B, A, D); 3233bdd1243dSDimitry Andric if (B == D) 3234bdd1243dSDimitry Andric return OrFactorization(B, A, C, CondLogicOr && TrueLogicOr); 3235bdd1243dSDimitry Andric } 3236bdd1243dSDimitry Andric } 3237fe6060f1SDimitry Andric 3238fe6060f1SDimitry Andric // We match the "full" 0 or 1 constant here to avoid a potential infinite 3239fe6060f1SDimitry Andric // loop with vectors that may have undefined/poison elements. 3240e8d8bef9SDimitry Andric // select a, false, b -> select !a, b, false 3241fe6060f1SDimitry Andric if (match(TrueVal, m_Specific(Zero))) { 32420b57cec5SDimitry Andric Value *NotCond = Builder.CreateNot(CondVal, "not." + CondVal->getName()); 3243fe6060f1SDimitry Andric return SelectInst::Create(NotCond, FalseVal, Zero); 3244e8d8bef9SDimitry Andric } 3245e8d8bef9SDimitry Andric // select a, b, true -> select !a, true, b 3246fe6060f1SDimitry Andric if (match(FalseVal, m_Specific(One))) { 3247e8d8bef9SDimitry Andric Value *NotCond = Builder.CreateNot(CondVal, "not." + CondVal->getName()); 3248fe6060f1SDimitry Andric return SelectInst::Create(NotCond, One, TrueVal); 32490b57cec5SDimitry Andric } 32500b57cec5SDimitry Andric 3251fe6060f1SDimitry Andric // DeMorgan in select form: !a && !b --> !(a || b) 3252fe6060f1SDimitry Andric // select !a, !b, false --> not (select a, true, b) 3253fe6060f1SDimitry Andric if (match(&SI, m_LogicalAnd(m_Not(m_Value(A)), m_Not(m_Value(B)))) && 3254fe6060f1SDimitry Andric (CondVal->hasOneUse() || TrueVal->hasOneUse()) && 3255fe6060f1SDimitry Andric !match(A, m_ConstantExpr()) && !match(B, m_ConstantExpr())) 3256fe6060f1SDimitry Andric return BinaryOperator::CreateNot(Builder.CreateSelect(A, One, B)); 3257fe6060f1SDimitry Andric 3258fe6060f1SDimitry Andric // DeMorgan in select form: !a || !b --> !(a && b) 3259fe6060f1SDimitry Andric // select !a, true, !b --> not (select a, b, false) 3260fe6060f1SDimitry Andric if (match(&SI, m_LogicalOr(m_Not(m_Value(A)), m_Not(m_Value(B)))) && 3261fe6060f1SDimitry Andric (CondVal->hasOneUse() || FalseVal->hasOneUse()) && 3262fe6060f1SDimitry Andric !match(A, m_ConstantExpr()) && !match(B, m_ConstantExpr())) 3263fe6060f1SDimitry Andric return BinaryOperator::CreateNot(Builder.CreateSelect(A, B, Zero)); 3264fe6060f1SDimitry Andric 3265fe6060f1SDimitry Andric // select (select a, true, b), true, b -> select a, true, b 3266fe6060f1SDimitry Andric if (match(CondVal, m_Select(m_Value(A), m_One(), m_Value(B))) && 3267fe6060f1SDimitry Andric match(TrueVal, m_One()) && match(FalseVal, m_Specific(B))) 3268fe6060f1SDimitry Andric return replaceOperand(SI, 0, A); 3269fe6060f1SDimitry Andric // select (select a, b, false), b, false -> select a, b, false 3270fe6060f1SDimitry Andric if (match(CondVal, m_Select(m_Value(A), m_Value(B), m_Zero())) && 3271fe6060f1SDimitry Andric match(TrueVal, m_Specific(B)) && match(FalseVal, m_Zero())) 3272fe6060f1SDimitry Andric return replaceOperand(SI, 0, A); 3273fe6060f1SDimitry Andric 3274bdd1243dSDimitry Andric // ~(A & B) & (A | B) --> A ^ B 3275bdd1243dSDimitry Andric if (match(&SI, m_c_LogicalAnd(m_Not(m_LogicalAnd(m_Value(A), m_Value(B))), 3276bdd1243dSDimitry Andric m_c_LogicalOr(m_Deferred(A), m_Deferred(B))))) 3277bdd1243dSDimitry Andric return BinaryOperator::CreateXor(A, B); 3278bdd1243dSDimitry Andric 32795f757f3fSDimitry Andric // select (~a | c), a, b -> select a, (select c, true, b), false 32805f757f3fSDimitry Andric if (match(CondVal, 32815f757f3fSDimitry Andric m_OneUse(m_c_Or(m_Not(m_Specific(TrueVal)), m_Value(C))))) { 32825f757f3fSDimitry Andric Value *OrV = Builder.CreateSelect(C, One, FalseVal); 32835f757f3fSDimitry Andric return SelectInst::Create(TrueVal, OrV, Zero); 328481ad6265SDimitry Andric } 32855f757f3fSDimitry Andric // select (c & b), a, b -> select b, (select ~c, true, a), false 32865f757f3fSDimitry Andric if (match(CondVal, m_OneUse(m_c_And(m_Value(C), m_Specific(FalseVal))))) { 32875f757f3fSDimitry Andric if (Value *NotC = getFreelyInverted(C, C->hasOneUse(), &Builder)) { 32885f757f3fSDimitry Andric Value *OrV = Builder.CreateSelect(NotC, One, TrueVal); 32895f757f3fSDimitry Andric return SelectInst::Create(FalseVal, OrV, Zero); 32905f757f3fSDimitry Andric } 32915f757f3fSDimitry Andric } 32925f757f3fSDimitry Andric // select (a | c), a, b -> select a, true, (select ~c, b, false) 32935f757f3fSDimitry Andric if (match(CondVal, m_OneUse(m_c_Or(m_Specific(TrueVal), m_Value(C))))) { 32945f757f3fSDimitry Andric if (Value *NotC = getFreelyInverted(C, C->hasOneUse(), &Builder)) { 32955f757f3fSDimitry Andric Value *AndV = Builder.CreateSelect(NotC, FalseVal, Zero); 32965f757f3fSDimitry Andric return SelectInst::Create(TrueVal, One, AndV); 32975f757f3fSDimitry Andric } 32985f757f3fSDimitry Andric } 32995f757f3fSDimitry Andric // select (c & ~b), a, b -> select b, true, (select c, a, false) 33005f757f3fSDimitry Andric if (match(CondVal, 33015f757f3fSDimitry Andric m_OneUse(m_c_And(m_Value(C), m_Not(m_Specific(FalseVal)))))) { 33025f757f3fSDimitry Andric Value *AndV = Builder.CreateSelect(C, TrueVal, Zero); 33035f757f3fSDimitry Andric return SelectInst::Create(FalseVal, One, AndV); 330481ad6265SDimitry Andric } 330581ad6265SDimitry Andric 3306fe6060f1SDimitry Andric if (match(FalseVal, m_Zero()) || match(TrueVal, m_One())) { 3307fe6060f1SDimitry Andric Use *Y = nullptr; 3308fe6060f1SDimitry Andric bool IsAnd = match(FalseVal, m_Zero()) ? true : false; 3309fe6060f1SDimitry Andric Value *Op1 = IsAnd ? TrueVal : FalseVal; 3310fe6060f1SDimitry Andric if (isCheckForZeroAndMulWithOverflow(CondVal, Op1, IsAnd, Y)) { 3311fe6060f1SDimitry Andric auto *FI = new FreezeInst(*Y, (*Y)->getName() + ".fr"); 33125f757f3fSDimitry Andric InsertNewInstBefore(FI, cast<Instruction>(Y->getUser())->getIterator()); 3313fe6060f1SDimitry Andric replaceUse(*Y, FI); 3314fe6060f1SDimitry Andric return replaceInstUsesWith(SI, Op1); 3315fe6060f1SDimitry Andric } 3316fe6060f1SDimitry Andric 331781ad6265SDimitry Andric if (auto *ICmp0 = dyn_cast<ICmpInst>(CondVal)) 331881ad6265SDimitry Andric if (auto *ICmp1 = dyn_cast<ICmpInst>(Op1)) 331981ad6265SDimitry Andric if (auto *V = foldAndOrOfICmps(ICmp0, ICmp1, SI, IsAnd, 3320fe6060f1SDimitry Andric /* IsLogical */ true)) 3321fe6060f1SDimitry Andric return replaceInstUsesWith(SI, V); 3322fe6060f1SDimitry Andric } 3323fe6060f1SDimitry Andric 3324bdd1243dSDimitry Andric // select (a || b), c, false -> select a, c, false 3325bdd1243dSDimitry Andric // select c, (a || b), false -> select c, a, false 3326fe6060f1SDimitry Andric // if c implies that b is false. 3327bdd1243dSDimitry Andric if (match(CondVal, m_LogicalOr(m_Value(A), m_Value(B))) && 3328fe6060f1SDimitry Andric match(FalseVal, m_Zero())) { 3329bdd1243dSDimitry Andric std::optional<bool> Res = isImpliedCondition(TrueVal, B, DL); 3330fe6060f1SDimitry Andric if (Res && *Res == false) 3331fe6060f1SDimitry Andric return replaceOperand(SI, 0, A); 3332fe6060f1SDimitry Andric } 3333bdd1243dSDimitry Andric if (match(TrueVal, m_LogicalOr(m_Value(A), m_Value(B))) && 3334fe6060f1SDimitry Andric match(FalseVal, m_Zero())) { 3335bdd1243dSDimitry Andric std::optional<bool> Res = isImpliedCondition(CondVal, B, DL); 3336fe6060f1SDimitry Andric if (Res && *Res == false) 3337fe6060f1SDimitry Andric return replaceOperand(SI, 1, A); 3338fe6060f1SDimitry Andric } 3339bdd1243dSDimitry Andric // select c, true, (a && b) -> select c, true, a 3340bdd1243dSDimitry Andric // select (a && b), true, c -> select a, true, c 3341fe6060f1SDimitry Andric // if c = false implies that b = true 3342fe6060f1SDimitry Andric if (match(TrueVal, m_One()) && 3343bdd1243dSDimitry Andric match(FalseVal, m_LogicalAnd(m_Value(A), m_Value(B)))) { 3344bdd1243dSDimitry Andric std::optional<bool> Res = isImpliedCondition(CondVal, B, DL, false); 3345fe6060f1SDimitry Andric if (Res && *Res == true) 3346fe6060f1SDimitry Andric return replaceOperand(SI, 2, A); 3347fe6060f1SDimitry Andric } 3348bdd1243dSDimitry Andric if (match(CondVal, m_LogicalAnd(m_Value(A), m_Value(B))) && 3349fe6060f1SDimitry Andric match(TrueVal, m_One())) { 3350bdd1243dSDimitry Andric std::optional<bool> Res = isImpliedCondition(FalseVal, B, DL, false); 3351fe6060f1SDimitry Andric if (Res && *Res == true) 3352fe6060f1SDimitry Andric return replaceOperand(SI, 0, A); 3353fe6060f1SDimitry Andric } 3354fe6060f1SDimitry Andric 3355bdd1243dSDimitry Andric if (match(TrueVal, m_One())) { 3356bdd1243dSDimitry Andric Value *C; 3357bdd1243dSDimitry Andric 3358bdd1243dSDimitry Andric // (C && A) || (!C && B) --> sel C, A, B 3359bdd1243dSDimitry Andric // (A && C) || (!C && B) --> sel C, A, B 3360bdd1243dSDimitry Andric // (C && A) || (B && !C) --> sel C, A, B 3361bdd1243dSDimitry Andric // (A && C) || (B && !C) --> sel C, A, B (may require freeze) 3362bdd1243dSDimitry Andric if (match(FalseVal, m_c_LogicalAnd(m_Not(m_Value(C)), m_Value(B))) && 3363bdd1243dSDimitry Andric match(CondVal, m_c_LogicalAnd(m_Specific(C), m_Value(A)))) { 3364bdd1243dSDimitry Andric auto *SelCond = dyn_cast<SelectInst>(CondVal); 3365bdd1243dSDimitry Andric auto *SelFVal = dyn_cast<SelectInst>(FalseVal); 3366bdd1243dSDimitry Andric bool MayNeedFreeze = SelCond && SelFVal && 3367bdd1243dSDimitry Andric match(SelFVal->getTrueValue(), 3368bdd1243dSDimitry Andric m_Not(m_Specific(SelCond->getTrueValue()))); 3369bdd1243dSDimitry Andric if (MayNeedFreeze) 3370bdd1243dSDimitry Andric C = Builder.CreateFreeze(C); 3371bdd1243dSDimitry Andric return SelectInst::Create(C, A, B); 3372bdd1243dSDimitry Andric } 3373bdd1243dSDimitry Andric 3374bdd1243dSDimitry Andric // (!C && A) || (C && B) --> sel C, B, A 3375bdd1243dSDimitry Andric // (A && !C) || (C && B) --> sel C, B, A 3376bdd1243dSDimitry Andric // (!C && A) || (B && C) --> sel C, B, A 3377bdd1243dSDimitry Andric // (A && !C) || (B && C) --> sel C, B, A (may require freeze) 3378bdd1243dSDimitry Andric if (match(CondVal, m_c_LogicalAnd(m_Not(m_Value(C)), m_Value(A))) && 3379bdd1243dSDimitry Andric match(FalseVal, m_c_LogicalAnd(m_Specific(C), m_Value(B)))) { 3380bdd1243dSDimitry Andric auto *SelCond = dyn_cast<SelectInst>(CondVal); 3381bdd1243dSDimitry Andric auto *SelFVal = dyn_cast<SelectInst>(FalseVal); 3382bdd1243dSDimitry Andric bool MayNeedFreeze = SelCond && SelFVal && 3383bdd1243dSDimitry Andric match(SelCond->getTrueValue(), 3384bdd1243dSDimitry Andric m_Not(m_Specific(SelFVal->getTrueValue()))); 3385bdd1243dSDimitry Andric if (MayNeedFreeze) 3386bdd1243dSDimitry Andric C = Builder.CreateFreeze(C); 3387bdd1243dSDimitry Andric return SelectInst::Create(C, B, A); 3388fe6060f1SDimitry Andric } 33890b57cec5SDimitry Andric } 33900b57cec5SDimitry Andric 3391bdd1243dSDimitry Andric return nullptr; 3392bdd1243dSDimitry Andric } 3393bdd1243dSDimitry Andric 339406c3fb27SDimitry Andric // Return true if we can safely remove the select instruction for std::bit_ceil 339506c3fb27SDimitry Andric // pattern. 339606c3fb27SDimitry Andric static bool isSafeToRemoveBitCeilSelect(ICmpInst::Predicate Pred, Value *Cond0, 339706c3fb27SDimitry Andric const APInt *Cond1, Value *CtlzOp, 33983a079333SDimitry Andric unsigned BitWidth, 33993a079333SDimitry Andric bool &ShouldDropNUW) { 340006c3fb27SDimitry Andric // The challenge in recognizing std::bit_ceil(X) is that the operand is used 340106c3fb27SDimitry Andric // for the CTLZ proper and select condition, each possibly with some 340206c3fb27SDimitry Andric // operation like add and sub. 340306c3fb27SDimitry Andric // 340406c3fb27SDimitry Andric // Our aim is to make sure that -ctlz & (BitWidth - 1) == 0 even when the 340506c3fb27SDimitry Andric // select instruction would select 1, which allows us to get rid of the select 340606c3fb27SDimitry Andric // instruction. 340706c3fb27SDimitry Andric // 340806c3fb27SDimitry Andric // To see if we can do so, we do some symbolic execution with ConstantRange. 340906c3fb27SDimitry Andric // Specifically, we compute the range of values that Cond0 could take when 341006c3fb27SDimitry Andric // Cond == false. Then we successively transform the range until we obtain 341106c3fb27SDimitry Andric // the range of values that CtlzOp could take. 341206c3fb27SDimitry Andric // 341306c3fb27SDimitry Andric // Conceptually, we follow the def-use chain backward from Cond0 while 341406c3fb27SDimitry Andric // transforming the range for Cond0 until we meet the common ancestor of Cond0 341506c3fb27SDimitry Andric // and CtlzOp. Then we follow the def-use chain forward until we obtain the 341606c3fb27SDimitry Andric // range for CtlzOp. That said, we only follow at most one ancestor from 341706c3fb27SDimitry Andric // Cond0. Likewise, we only follow at most one ancestor from CtrlOp. 341806c3fb27SDimitry Andric 341906c3fb27SDimitry Andric ConstantRange CR = ConstantRange::makeExactICmpRegion( 342006c3fb27SDimitry Andric CmpInst::getInversePredicate(Pred), *Cond1); 342106c3fb27SDimitry Andric 34223a079333SDimitry Andric ShouldDropNUW = false; 34233a079333SDimitry Andric 342406c3fb27SDimitry Andric // Match the operation that's used to compute CtlzOp from CommonAncestor. If 342506c3fb27SDimitry Andric // CtlzOp == CommonAncestor, return true as no operation is needed. If a 342606c3fb27SDimitry Andric // match is found, execute the operation on CR, update CR, and return true. 342706c3fb27SDimitry Andric // Otherwise, return false. 342806c3fb27SDimitry Andric auto MatchForward = [&](Value *CommonAncestor) { 342906c3fb27SDimitry Andric const APInt *C = nullptr; 343006c3fb27SDimitry Andric if (CtlzOp == CommonAncestor) 343106c3fb27SDimitry Andric return true; 343206c3fb27SDimitry Andric if (match(CtlzOp, m_Add(m_Specific(CommonAncestor), m_APInt(C)))) { 343306c3fb27SDimitry Andric CR = CR.add(*C); 343406c3fb27SDimitry Andric return true; 343506c3fb27SDimitry Andric } 343606c3fb27SDimitry Andric if (match(CtlzOp, m_Sub(m_APInt(C), m_Specific(CommonAncestor)))) { 34373a079333SDimitry Andric ShouldDropNUW = true; 343806c3fb27SDimitry Andric CR = ConstantRange(*C).sub(CR); 343906c3fb27SDimitry Andric return true; 344006c3fb27SDimitry Andric } 344106c3fb27SDimitry Andric if (match(CtlzOp, m_Not(m_Specific(CommonAncestor)))) { 344206c3fb27SDimitry Andric CR = CR.binaryNot(); 344306c3fb27SDimitry Andric return true; 344406c3fb27SDimitry Andric } 344506c3fb27SDimitry Andric return false; 344606c3fb27SDimitry Andric }; 344706c3fb27SDimitry Andric 344806c3fb27SDimitry Andric const APInt *C = nullptr; 344906c3fb27SDimitry Andric Value *CommonAncestor; 345006c3fb27SDimitry Andric if (MatchForward(Cond0)) { 345106c3fb27SDimitry Andric // Cond0 is either CtlzOp or CtlzOp's parent. CR has been updated. 345206c3fb27SDimitry Andric } else if (match(Cond0, m_Add(m_Value(CommonAncestor), m_APInt(C)))) { 345306c3fb27SDimitry Andric CR = CR.sub(*C); 345406c3fb27SDimitry Andric if (!MatchForward(CommonAncestor)) 345506c3fb27SDimitry Andric return false; 345606c3fb27SDimitry Andric // Cond0's parent is either CtlzOp or CtlzOp's parent. CR has been updated. 345706c3fb27SDimitry Andric } else { 345806c3fb27SDimitry Andric return false; 345906c3fb27SDimitry Andric } 346006c3fb27SDimitry Andric 346106c3fb27SDimitry Andric // Return true if all the values in the range are either 0 or negative (if 346206c3fb27SDimitry Andric // treated as signed). We do so by evaluating: 346306c3fb27SDimitry Andric // 346406c3fb27SDimitry Andric // CR - 1 u>= (1 << BitWidth) - 1. 346506c3fb27SDimitry Andric APInt IntMax = APInt::getSignMask(BitWidth) - 1; 346606c3fb27SDimitry Andric CR = CR.sub(APInt(BitWidth, 1)); 346706c3fb27SDimitry Andric return CR.icmp(ICmpInst::ICMP_UGE, IntMax); 346806c3fb27SDimitry Andric } 346906c3fb27SDimitry Andric 347006c3fb27SDimitry Andric // Transform the std::bit_ceil(X) pattern like: 347106c3fb27SDimitry Andric // 347206c3fb27SDimitry Andric // %dec = add i32 %x, -1 347306c3fb27SDimitry Andric // %ctlz = tail call i32 @llvm.ctlz.i32(i32 %dec, i1 false) 347406c3fb27SDimitry Andric // %sub = sub i32 32, %ctlz 347506c3fb27SDimitry Andric // %shl = shl i32 1, %sub 347606c3fb27SDimitry Andric // %ugt = icmp ugt i32 %x, 1 347706c3fb27SDimitry Andric // %sel = select i1 %ugt, i32 %shl, i32 1 347806c3fb27SDimitry Andric // 347906c3fb27SDimitry Andric // into: 348006c3fb27SDimitry Andric // 348106c3fb27SDimitry Andric // %dec = add i32 %x, -1 348206c3fb27SDimitry Andric // %ctlz = tail call i32 @llvm.ctlz.i32(i32 %dec, i1 false) 348306c3fb27SDimitry Andric // %neg = sub i32 0, %ctlz 348406c3fb27SDimitry Andric // %masked = and i32 %ctlz, 31 348506c3fb27SDimitry Andric // %shl = shl i32 1, %sub 348606c3fb27SDimitry Andric // 348706c3fb27SDimitry Andric // Note that the select is optimized away while the shift count is masked with 348806c3fb27SDimitry Andric // 31. We handle some variations of the input operand like std::bit_ceil(X + 348906c3fb27SDimitry Andric // 1). 349006c3fb27SDimitry Andric static Instruction *foldBitCeil(SelectInst &SI, IRBuilderBase &Builder) { 349106c3fb27SDimitry Andric Type *SelType = SI.getType(); 349206c3fb27SDimitry Andric unsigned BitWidth = SelType->getScalarSizeInBits(); 349306c3fb27SDimitry Andric 349406c3fb27SDimitry Andric Value *FalseVal = SI.getFalseValue(); 349506c3fb27SDimitry Andric Value *TrueVal = SI.getTrueValue(); 349606c3fb27SDimitry Andric ICmpInst::Predicate Pred; 349706c3fb27SDimitry Andric const APInt *Cond1; 349806c3fb27SDimitry Andric Value *Cond0, *Ctlz, *CtlzOp; 349906c3fb27SDimitry Andric if (!match(SI.getCondition(), m_ICmp(Pred, m_Value(Cond0), m_APInt(Cond1)))) 350006c3fb27SDimitry Andric return nullptr; 350106c3fb27SDimitry Andric 350206c3fb27SDimitry Andric if (match(TrueVal, m_One())) { 350306c3fb27SDimitry Andric std::swap(FalseVal, TrueVal); 350406c3fb27SDimitry Andric Pred = CmpInst::getInversePredicate(Pred); 350506c3fb27SDimitry Andric } 350606c3fb27SDimitry Andric 35073a079333SDimitry Andric bool ShouldDropNUW; 35083a079333SDimitry Andric 350906c3fb27SDimitry Andric if (!match(FalseVal, m_One()) || 351006c3fb27SDimitry Andric !match(TrueVal, 351106c3fb27SDimitry Andric m_OneUse(m_Shl(m_One(), m_OneUse(m_Sub(m_SpecificInt(BitWidth), 351206c3fb27SDimitry Andric m_Value(Ctlz)))))) || 351306c3fb27SDimitry Andric !match(Ctlz, m_Intrinsic<Intrinsic::ctlz>(m_Value(CtlzOp), m_Zero())) || 35143a079333SDimitry Andric !isSafeToRemoveBitCeilSelect(Pred, Cond0, Cond1, CtlzOp, BitWidth, 35153a079333SDimitry Andric ShouldDropNUW)) 351606c3fb27SDimitry Andric return nullptr; 351706c3fb27SDimitry Andric 35183a079333SDimitry Andric if (ShouldDropNUW) 35193a079333SDimitry Andric cast<Instruction>(CtlzOp)->setHasNoUnsignedWrap(false); 35203a079333SDimitry Andric 352106c3fb27SDimitry Andric // Build 1 << (-CTLZ & (BitWidth-1)). The negation likely corresponds to a 352206c3fb27SDimitry Andric // single hardware instruction as opposed to BitWidth - CTLZ, where BitWidth 352306c3fb27SDimitry Andric // is an integer constant. Masking with BitWidth-1 comes free on some 352406c3fb27SDimitry Andric // hardware as part of the shift instruction. 352506c3fb27SDimitry Andric Value *Neg = Builder.CreateNeg(Ctlz); 352606c3fb27SDimitry Andric Value *Masked = 352706c3fb27SDimitry Andric Builder.CreateAnd(Neg, ConstantInt::get(SelType, BitWidth - 1)); 352806c3fb27SDimitry Andric return BinaryOperator::Create(Instruction::Shl, ConstantInt::get(SelType, 1), 352906c3fb27SDimitry Andric Masked); 353006c3fb27SDimitry Andric } 353106c3fb27SDimitry Andric 35325f757f3fSDimitry Andric bool InstCombinerImpl::fmulByZeroIsZero(Value *MulVal, FastMathFlags FMF, 35335f757f3fSDimitry Andric const Instruction *CtxI) const { 35345f757f3fSDimitry Andric KnownFPClass Known = computeKnownFPClass(MulVal, FMF, fcNegative, CtxI); 35355f757f3fSDimitry Andric 35365f757f3fSDimitry Andric return Known.isKnownNeverNaN() && Known.isKnownNeverInfinity() && 35375f757f3fSDimitry Andric (FMF.noSignedZeros() || Known.signBitIsZeroOrNaN()); 35385f757f3fSDimitry Andric } 35395f757f3fSDimitry Andric 35405f757f3fSDimitry Andric static bool matchFMulByZeroIfResultEqZero(InstCombinerImpl &IC, Value *Cmp0, 35415f757f3fSDimitry Andric Value *Cmp1, Value *TrueVal, 35425f757f3fSDimitry Andric Value *FalseVal, Instruction &CtxI, 35435f757f3fSDimitry Andric bool SelectIsNSZ) { 35445f757f3fSDimitry Andric Value *MulRHS; 35455f757f3fSDimitry Andric if (match(Cmp1, m_PosZeroFP()) && 35465f757f3fSDimitry Andric match(TrueVal, m_c_FMul(m_Specific(Cmp0), m_Value(MulRHS)))) { 35475f757f3fSDimitry Andric FastMathFlags FMF = cast<FPMathOperator>(TrueVal)->getFastMathFlags(); 35485f757f3fSDimitry Andric // nsz must be on the select, it must be ignored on the multiply. We 35495f757f3fSDimitry Andric // need nnan and ninf on the multiply for the other value. 35505f757f3fSDimitry Andric FMF.setNoSignedZeros(SelectIsNSZ); 35515f757f3fSDimitry Andric return IC.fmulByZeroIsZero(MulRHS, FMF, &CtxI); 35525f757f3fSDimitry Andric } 35535f757f3fSDimitry Andric 35545f757f3fSDimitry Andric return false; 35555f757f3fSDimitry Andric } 35565f757f3fSDimitry Andric 3557*0fca6ea1SDimitry Andric /// Check whether the KnownBits of a select arm may be affected by the 3558*0fca6ea1SDimitry Andric /// select condition. 3559*0fca6ea1SDimitry Andric static bool hasAffectedValue(Value *V, SmallPtrSetImpl<Value *> &Affected, 3560*0fca6ea1SDimitry Andric unsigned Depth) { 3561*0fca6ea1SDimitry Andric if (Depth == MaxAnalysisRecursionDepth) 3562*0fca6ea1SDimitry Andric return false; 3563*0fca6ea1SDimitry Andric 3564*0fca6ea1SDimitry Andric // Ignore the case where the select arm itself is affected. These cases 3565*0fca6ea1SDimitry Andric // are handled more efficiently by other optimizations. 3566*0fca6ea1SDimitry Andric if (Depth != 0 && Affected.contains(V)) 3567*0fca6ea1SDimitry Andric return true; 3568*0fca6ea1SDimitry Andric 3569*0fca6ea1SDimitry Andric if (auto *I = dyn_cast<Instruction>(V)) { 3570*0fca6ea1SDimitry Andric if (isa<PHINode>(I)) { 3571*0fca6ea1SDimitry Andric if (Depth == MaxAnalysisRecursionDepth - 1) 3572*0fca6ea1SDimitry Andric return false; 3573*0fca6ea1SDimitry Andric Depth = MaxAnalysisRecursionDepth - 2; 3574*0fca6ea1SDimitry Andric } 3575*0fca6ea1SDimitry Andric return any_of(I->operands(), [&](Value *Op) { 3576*0fca6ea1SDimitry Andric return Op->getType()->isIntOrIntVectorTy() && 3577*0fca6ea1SDimitry Andric hasAffectedValue(Op, Affected, Depth + 1); 3578*0fca6ea1SDimitry Andric }); 3579*0fca6ea1SDimitry Andric } 3580*0fca6ea1SDimitry Andric 3581*0fca6ea1SDimitry Andric return false; 3582*0fca6ea1SDimitry Andric } 3583*0fca6ea1SDimitry Andric 3584bdd1243dSDimitry Andric Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) { 3585bdd1243dSDimitry Andric Value *CondVal = SI.getCondition(); 3586bdd1243dSDimitry Andric Value *TrueVal = SI.getTrueValue(); 3587bdd1243dSDimitry Andric Value *FalseVal = SI.getFalseValue(); 3588bdd1243dSDimitry Andric Type *SelType = SI.getType(); 3589bdd1243dSDimitry Andric 3590bdd1243dSDimitry Andric if (Value *V = simplifySelectInst(CondVal, TrueVal, FalseVal, 3591bdd1243dSDimitry Andric SQ.getWithInstruction(&SI))) 3592bdd1243dSDimitry Andric return replaceInstUsesWith(SI, V); 3593bdd1243dSDimitry Andric 3594bdd1243dSDimitry Andric if (Instruction *I = canonicalizeSelectToShuffle(SI)) 3595bdd1243dSDimitry Andric return I; 3596bdd1243dSDimitry Andric 3597bdd1243dSDimitry Andric if (Instruction *I = canonicalizeScalarSelectOfVecs(SI, *this)) 3598bdd1243dSDimitry Andric return I; 3599bdd1243dSDimitry Andric 3600bdd1243dSDimitry Andric // If the type of select is not an integer type or if the condition and 3601bdd1243dSDimitry Andric // the selection type are not both scalar nor both vector types, there is no 3602bdd1243dSDimitry Andric // point in attempting to match these patterns. 3603bdd1243dSDimitry Andric Type *CondType = CondVal->getType(); 3604bdd1243dSDimitry Andric if (!isa<Constant>(CondVal) && SelType->isIntOrIntVectorTy() && 3605bdd1243dSDimitry Andric CondType->isVectorTy() == SelType->isVectorTy()) { 3606bdd1243dSDimitry Andric if (Value *S = simplifyWithOpReplaced(TrueVal, CondVal, 3607bdd1243dSDimitry Andric ConstantInt::getTrue(CondType), SQ, 3608bdd1243dSDimitry Andric /* AllowRefinement */ true)) 3609bdd1243dSDimitry Andric return replaceOperand(SI, 1, S); 3610bdd1243dSDimitry Andric 3611bdd1243dSDimitry Andric if (Value *S = simplifyWithOpReplaced(FalseVal, CondVal, 3612bdd1243dSDimitry Andric ConstantInt::getFalse(CondType), SQ, 3613bdd1243dSDimitry Andric /* AllowRefinement */ true)) 3614bdd1243dSDimitry Andric return replaceOperand(SI, 2, S); 3615bdd1243dSDimitry Andric } 3616bdd1243dSDimitry Andric 3617bdd1243dSDimitry Andric if (Instruction *R = foldSelectOfBools(SI)) 3618bdd1243dSDimitry Andric return R; 3619bdd1243dSDimitry Andric 36200b57cec5SDimitry Andric // Selecting between two integer or vector splat integer constants? 36210b57cec5SDimitry Andric // 36220b57cec5SDimitry Andric // Note that we don't handle a scalar select of vectors: 36230b57cec5SDimitry Andric // select i1 %c, <2 x i8> <1, 1>, <2 x i8> <0, 0> 36240b57cec5SDimitry Andric // because that may need 3 instructions to splat the condition value: 36250b57cec5SDimitry Andric // extend, insertelement, shufflevector. 3626e8d8bef9SDimitry Andric // 3627e8d8bef9SDimitry Andric // Do not handle i1 TrueVal and FalseVal otherwise would result in 3628e8d8bef9SDimitry Andric // zext/sext i1 to i1. 3629e8d8bef9SDimitry Andric if (SelType->isIntOrIntVectorTy() && !SelType->isIntOrIntVectorTy(1) && 36300b57cec5SDimitry Andric CondVal->getType()->isVectorTy() == SelType->isVectorTy()) { 36310b57cec5SDimitry Andric // select C, 1, 0 -> zext C to int 36320b57cec5SDimitry Andric if (match(TrueVal, m_One()) && match(FalseVal, m_Zero())) 36330b57cec5SDimitry Andric return new ZExtInst(CondVal, SelType); 36340b57cec5SDimitry Andric 36350b57cec5SDimitry Andric // select C, -1, 0 -> sext C to int 36360b57cec5SDimitry Andric if (match(TrueVal, m_AllOnes()) && match(FalseVal, m_Zero())) 36370b57cec5SDimitry Andric return new SExtInst(CondVal, SelType); 36380b57cec5SDimitry Andric 36390b57cec5SDimitry Andric // select C, 0, 1 -> zext !C to int 36400b57cec5SDimitry Andric if (match(TrueVal, m_Zero()) && match(FalseVal, m_One())) { 36410b57cec5SDimitry Andric Value *NotCond = Builder.CreateNot(CondVal, "not." + CondVal->getName()); 36420b57cec5SDimitry Andric return new ZExtInst(NotCond, SelType); 36430b57cec5SDimitry Andric } 36440b57cec5SDimitry Andric 36450b57cec5SDimitry Andric // select C, 0, -1 -> sext !C to int 36460b57cec5SDimitry Andric if (match(TrueVal, m_Zero()) && match(FalseVal, m_AllOnes())) { 36470b57cec5SDimitry Andric Value *NotCond = Builder.CreateNot(CondVal, "not." + CondVal->getName()); 36480b57cec5SDimitry Andric return new SExtInst(NotCond, SelType); 36490b57cec5SDimitry Andric } 36500b57cec5SDimitry Andric } 36510b57cec5SDimitry Andric 36525f757f3fSDimitry Andric auto *SIFPOp = dyn_cast<FPMathOperator>(&SI); 36535f757f3fSDimitry Andric 3654fe6060f1SDimitry Andric if (auto *FCmp = dyn_cast<FCmpInst>(CondVal)) { 36555f757f3fSDimitry Andric FCmpInst::Predicate Pred = FCmp->getPredicate(); 3656fe6060f1SDimitry Andric Value *Cmp0 = FCmp->getOperand(0), *Cmp1 = FCmp->getOperand(1); 3657fe6060f1SDimitry Andric // Are we selecting a value based on a comparison of the two values? 3658480093f4SDimitry Andric if ((Cmp0 == TrueVal && Cmp1 == FalseVal) || 3659480093f4SDimitry Andric (Cmp0 == FalseVal && Cmp1 == TrueVal)) { 36600b57cec5SDimitry Andric // Canonicalize to use ordered comparisons by swapping the select 36610b57cec5SDimitry Andric // operands. 36620b57cec5SDimitry Andric // 36630b57cec5SDimitry Andric // e.g. 36640b57cec5SDimitry Andric // (X ugt Y) ? X : Y -> (X ole Y) ? Y : X 36655f757f3fSDimitry Andric if (FCmp->hasOneUse() && FCmpInst::isUnordered(Pred)) { 3666fe6060f1SDimitry Andric FCmpInst::Predicate InvPred = FCmp->getInversePredicate(); 36670b57cec5SDimitry Andric IRBuilder<>::FastMathFlagGuard FMFG(Builder); 3668480093f4SDimitry Andric // FIXME: The FMF should propagate from the select, not the fcmp. 3669fe6060f1SDimitry Andric Builder.setFastMathFlags(FCmp->getFastMathFlags()); 3670480093f4SDimitry Andric Value *NewCond = Builder.CreateFCmp(InvPred, Cmp0, Cmp1, 3671fe6060f1SDimitry Andric FCmp->getName() + ".inv"); 3672480093f4SDimitry Andric Value *NewSel = Builder.CreateSelect(NewCond, FalseVal, TrueVal); 3673480093f4SDimitry Andric return replaceInstUsesWith(SI, NewSel); 36740b57cec5SDimitry Andric } 3675bdd1243dSDimitry Andric } 36765f757f3fSDimitry Andric 36775f757f3fSDimitry Andric if (SIFPOp) { 36785f757f3fSDimitry Andric // Fold out scale-if-equals-zero pattern. 36795f757f3fSDimitry Andric // 36805f757f3fSDimitry Andric // This pattern appears in code with denormal range checks after it's 36815f757f3fSDimitry Andric // assumed denormals are treated as zero. This drops a canonicalization. 36825f757f3fSDimitry Andric 36835f757f3fSDimitry Andric // TODO: Could relax the signed zero logic. We just need to know the sign 36845f757f3fSDimitry Andric // of the result matches (fmul x, y has the same sign as x). 36855f757f3fSDimitry Andric // 36865f757f3fSDimitry Andric // TODO: Handle always-canonicalizing variant that selects some value or 1 36875f757f3fSDimitry Andric // scaling factor in the fmul visitor. 36885f757f3fSDimitry Andric 36895f757f3fSDimitry Andric // TODO: Handle ldexp too 36905f757f3fSDimitry Andric 36915f757f3fSDimitry Andric Value *MatchCmp0 = nullptr; 36925f757f3fSDimitry Andric Value *MatchCmp1 = nullptr; 36935f757f3fSDimitry Andric 36945f757f3fSDimitry Andric // (select (fcmp [ou]eq x, 0.0), (fmul x, K), x => x 36955f757f3fSDimitry Andric // (select (fcmp [ou]ne x, 0.0), x, (fmul x, K) => x 36965f757f3fSDimitry Andric if (Pred == CmpInst::FCMP_OEQ || Pred == CmpInst::FCMP_UEQ) { 36975f757f3fSDimitry Andric MatchCmp0 = FalseVal; 36985f757f3fSDimitry Andric MatchCmp1 = TrueVal; 36995f757f3fSDimitry Andric } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UNE) { 37005f757f3fSDimitry Andric MatchCmp0 = TrueVal; 37015f757f3fSDimitry Andric MatchCmp1 = FalseVal; 3702bdd1243dSDimitry Andric } 37030b57cec5SDimitry Andric 37045f757f3fSDimitry Andric if (Cmp0 == MatchCmp0 && 37055f757f3fSDimitry Andric matchFMulByZeroIfResultEqZero(*this, Cmp0, Cmp1, MatchCmp1, MatchCmp0, 37065f757f3fSDimitry Andric SI, SIFPOp->hasNoSignedZeros())) 37075f757f3fSDimitry Andric return replaceInstUsesWith(SI, Cmp0); 37085f757f3fSDimitry Andric } 37095f757f3fSDimitry Andric } 37105f757f3fSDimitry Andric 37115f757f3fSDimitry Andric if (SIFPOp) { 3712bdd1243dSDimitry Andric // TODO: Try to forward-propagate FMF from select arms to the select. 3713bdd1243dSDimitry Andric 3714bdd1243dSDimitry Andric // Canonicalize select of FP values where NaN and -0.0 are not valid as 3715bdd1243dSDimitry Andric // minnum/maxnum intrinsics. 37165f757f3fSDimitry Andric if (SIFPOp->hasNoNaNs() && SIFPOp->hasNoSignedZeros()) { 3717bdd1243dSDimitry Andric Value *X, *Y; 3718bdd1243dSDimitry Andric if (match(&SI, m_OrdFMax(m_Value(X), m_Value(Y)))) 3719bdd1243dSDimitry Andric return replaceInstUsesWith( 3720bdd1243dSDimitry Andric SI, Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, X, Y, &SI)); 3721bdd1243dSDimitry Andric 3722bdd1243dSDimitry Andric if (match(&SI, m_OrdFMin(m_Value(X), m_Value(Y)))) 3723bdd1243dSDimitry Andric return replaceInstUsesWith( 3724bdd1243dSDimitry Andric SI, Builder.CreateBinaryIntrinsic(Intrinsic::minnum, X, Y, &SI)); 37250b57cec5SDimitry Andric } 37260b57cec5SDimitry Andric } 37270b57cec5SDimitry Andric 372881ad6265SDimitry Andric // Fold selecting to fabs. 372981ad6265SDimitry Andric if (Instruction *Fabs = foldSelectWithFCmpToFabs(SI, *this)) 373081ad6265SDimitry Andric return Fabs; 37310b57cec5SDimitry Andric 37320b57cec5SDimitry Andric // See if we are selecting two values based on a comparison of the two values. 37330b57cec5SDimitry Andric if (ICmpInst *ICI = dyn_cast<ICmpInst>(CondVal)) 37340b57cec5SDimitry Andric if (Instruction *Result = foldSelectInstWithICmp(SI, ICI)) 37350b57cec5SDimitry Andric return Result; 37360b57cec5SDimitry Andric 37370b57cec5SDimitry Andric if (Instruction *Add = foldAddSubSelect(SI, Builder)) 37380b57cec5SDimitry Andric return Add; 3739480093f4SDimitry Andric if (Instruction *Add = foldOverflowingAddSubSelect(SI, Builder)) 3740480093f4SDimitry Andric return Add; 37415ffd83dbSDimitry Andric if (Instruction *Or = foldSetClearBits(SI, Builder)) 37425ffd83dbSDimitry Andric return Or; 3743349cc55cSDimitry Andric if (Instruction *Mul = foldSelectZeroOrMul(SI, *this)) 3744349cc55cSDimitry Andric return Mul; 37450b57cec5SDimitry Andric 37460b57cec5SDimitry Andric // Turn (select C, (op X, Y), (op X, Z)) -> (op X, (select C, Y, Z)) 37470b57cec5SDimitry Andric auto *TI = dyn_cast<Instruction>(TrueVal); 37480b57cec5SDimitry Andric auto *FI = dyn_cast<Instruction>(FalseVal); 37490b57cec5SDimitry Andric if (TI && FI && TI->getOpcode() == FI->getOpcode()) 37500b57cec5SDimitry Andric if (Instruction *IV = foldSelectOpOp(SI, TI, FI)) 37510b57cec5SDimitry Andric return IV; 37520b57cec5SDimitry Andric 37530b57cec5SDimitry Andric if (Instruction *I = foldSelectExtConst(SI)) 37540b57cec5SDimitry Andric return I; 37550b57cec5SDimitry Andric 37565f757f3fSDimitry Andric if (Instruction *I = foldSelectWithSRem(SI, *this, Builder)) 37575f757f3fSDimitry Andric return I; 37585f757f3fSDimitry Andric 3759fe6060f1SDimitry Andric // Fold (select C, (gep Ptr, Idx), Ptr) -> (gep Ptr, (select C, Idx, 0)) 3760fe6060f1SDimitry Andric // Fold (select C, Ptr, (gep Ptr, Idx)) -> (gep Ptr, (select C, 0, Idx)) 3761fe6060f1SDimitry Andric auto SelectGepWithBase = [&](GetElementPtrInst *Gep, Value *Base, 3762fe6060f1SDimitry Andric bool Swap) -> GetElementPtrInst * { 3763fe6060f1SDimitry Andric Value *Ptr = Gep->getPointerOperand(); 3764fe6060f1SDimitry Andric if (Gep->getNumOperands() != 2 || Gep->getPointerOperand() != Base || 3765fe6060f1SDimitry Andric !Gep->hasOneUse()) 3766fe6060f1SDimitry Andric return nullptr; 3767fe6060f1SDimitry Andric Value *Idx = Gep->getOperand(1); 3768349cc55cSDimitry Andric if (isa<VectorType>(CondVal->getType()) && !isa<VectorType>(Idx->getType())) 3769349cc55cSDimitry Andric return nullptr; 3770*0fca6ea1SDimitry Andric Type *ElementType = Gep->getSourceElementType(); 3771fe6060f1SDimitry Andric Value *NewT = Idx; 3772fe6060f1SDimitry Andric Value *NewF = Constant::getNullValue(Idx->getType()); 3773fe6060f1SDimitry Andric if (Swap) 3774fe6060f1SDimitry Andric std::swap(NewT, NewF); 3775fe6060f1SDimitry Andric Value *NewSI = 3776fe6060f1SDimitry Andric Builder.CreateSelect(CondVal, NewT, NewF, SI.getName() + ".idx", &SI); 3777*0fca6ea1SDimitry Andric return GetElementPtrInst::Create(ElementType, Ptr, NewSI, 3778*0fca6ea1SDimitry Andric Gep->getNoWrapFlags()); 3779fe6060f1SDimitry Andric }; 3780fe6060f1SDimitry Andric if (auto *TrueGep = dyn_cast<GetElementPtrInst>(TrueVal)) 3781fe6060f1SDimitry Andric if (auto *NewGep = SelectGepWithBase(TrueGep, FalseVal, false)) 3782fe6060f1SDimitry Andric return NewGep; 3783fe6060f1SDimitry Andric if (auto *FalseGep = dyn_cast<GetElementPtrInst>(FalseVal)) 3784fe6060f1SDimitry Andric if (auto *NewGep = SelectGepWithBase(FalseGep, TrueVal, true)) 3785fe6060f1SDimitry Andric return NewGep; 3786fe6060f1SDimitry Andric 37870b57cec5SDimitry Andric // See if we can fold the select into one of our operands. 37880b57cec5SDimitry Andric if (SelType->isIntOrIntVectorTy() || SelType->isFPOrFPVectorTy()) { 37890b57cec5SDimitry Andric if (Instruction *FoldI = foldSelectIntoOp(SI, TrueVal, FalseVal)) 37900b57cec5SDimitry Andric return FoldI; 37910b57cec5SDimitry Andric 37920b57cec5SDimitry Andric Value *LHS, *RHS; 37930b57cec5SDimitry Andric Instruction::CastOps CastOp; 37940b57cec5SDimitry Andric SelectPatternResult SPR = matchSelectPattern(&SI, LHS, RHS, &CastOp); 37950b57cec5SDimitry Andric auto SPF = SPR.Flavor; 37960b57cec5SDimitry Andric if (SPF) { 37970b57cec5SDimitry Andric Value *LHS2, *RHS2; 37980b57cec5SDimitry Andric if (SelectPatternFlavor SPF2 = matchSelectPattern(LHS, LHS2, RHS2).Flavor) 37990b57cec5SDimitry Andric if (Instruction *R = foldSPFofSPF(cast<Instruction>(LHS), SPF2, LHS2, 38000b57cec5SDimitry Andric RHS2, SI, SPF, RHS)) 38010b57cec5SDimitry Andric return R; 38020b57cec5SDimitry Andric if (SelectPatternFlavor SPF2 = matchSelectPattern(RHS, LHS2, RHS2).Flavor) 38030b57cec5SDimitry Andric if (Instruction *R = foldSPFofSPF(cast<Instruction>(RHS), SPF2, LHS2, 38040b57cec5SDimitry Andric RHS2, SI, SPF, LHS)) 38050b57cec5SDimitry Andric return R; 38060b57cec5SDimitry Andric } 38070b57cec5SDimitry Andric 38080b57cec5SDimitry Andric if (SelectPatternResult::isMinOrMax(SPF)) { 38090b57cec5SDimitry Andric // Canonicalize so that 38100b57cec5SDimitry Andric // - type casts are outside select patterns. 38110b57cec5SDimitry Andric // - float clamp is transformed to min/max pattern 38120b57cec5SDimitry Andric 38130b57cec5SDimitry Andric bool IsCastNeeded = LHS->getType() != SelType; 38140b57cec5SDimitry Andric Value *CmpLHS = cast<CmpInst>(CondVal)->getOperand(0); 38150b57cec5SDimitry Andric Value *CmpRHS = cast<CmpInst>(CondVal)->getOperand(1); 38160b57cec5SDimitry Andric if (IsCastNeeded || 38170b57cec5SDimitry Andric (LHS->getType()->isFPOrFPVectorTy() && 38180b57cec5SDimitry Andric ((CmpLHS != LHS && CmpLHS != RHS) || 38190b57cec5SDimitry Andric (CmpRHS != LHS && CmpRHS != RHS)))) { 38208bcb0991SDimitry Andric CmpInst::Predicate MinMaxPred = getMinMaxPred(SPF, SPR.Ordered); 38210b57cec5SDimitry Andric 38220b57cec5SDimitry Andric Value *Cmp; 38238bcb0991SDimitry Andric if (CmpInst::isIntPredicate(MinMaxPred)) { 38248bcb0991SDimitry Andric Cmp = Builder.CreateICmp(MinMaxPred, LHS, RHS); 38250b57cec5SDimitry Andric } else { 38260b57cec5SDimitry Andric IRBuilder<>::FastMathFlagGuard FMFG(Builder); 38278bcb0991SDimitry Andric auto FMF = 38288bcb0991SDimitry Andric cast<FPMathOperator>(SI.getCondition())->getFastMathFlags(); 38290b57cec5SDimitry Andric Builder.setFastMathFlags(FMF); 38308bcb0991SDimitry Andric Cmp = Builder.CreateFCmp(MinMaxPred, LHS, RHS); 38310b57cec5SDimitry Andric } 38320b57cec5SDimitry Andric 38330b57cec5SDimitry Andric Value *NewSI = Builder.CreateSelect(Cmp, LHS, RHS, SI.getName(), &SI); 38340b57cec5SDimitry Andric if (!IsCastNeeded) 38350b57cec5SDimitry Andric return replaceInstUsesWith(SI, NewSI); 38360b57cec5SDimitry Andric 38370b57cec5SDimitry Andric Value *NewCast = Builder.CreateCast(CastOp, NewSI, SelType); 38380b57cec5SDimitry Andric return replaceInstUsesWith(SI, NewCast); 38390b57cec5SDimitry Andric } 38400b57cec5SDimitry Andric } 38410b57cec5SDimitry Andric } 38420b57cec5SDimitry Andric 38430b57cec5SDimitry Andric // See if we can fold the select into a phi node if the condition is a select. 38440b57cec5SDimitry Andric if (auto *PN = dyn_cast<PHINode>(SI.getCondition())) 38450b57cec5SDimitry Andric // The true/false values have to be live in the PHI predecessor's blocks. 38460b57cec5SDimitry Andric if (canSelectOperandBeMappingIntoPredBlock(TrueVal, SI) && 38470b57cec5SDimitry Andric canSelectOperandBeMappingIntoPredBlock(FalseVal, SI)) 38480b57cec5SDimitry Andric if (Instruction *NV = foldOpIntoPhi(SI, PN)) 38490b57cec5SDimitry Andric return NV; 38500b57cec5SDimitry Andric 38510b57cec5SDimitry Andric if (SelectInst *TrueSI = dyn_cast<SelectInst>(TrueVal)) { 38520b57cec5SDimitry Andric if (TrueSI->getCondition()->getType() == CondVal->getType()) { 3853*0fca6ea1SDimitry Andric // Fold nested selects if the inner condition can be implied by the outer 3854*0fca6ea1SDimitry Andric // condition. 3855*0fca6ea1SDimitry Andric if (Value *V = simplifyNestedSelectsUsingImpliedCond( 3856*0fca6ea1SDimitry Andric *TrueSI, CondVal, /*CondIsTrue=*/true, DL)) 3857*0fca6ea1SDimitry Andric return replaceOperand(SI, 1, V); 3858*0fca6ea1SDimitry Andric 38590b57cec5SDimitry Andric // select(C0, select(C1, a, b), b) -> select(C0&C1, a, b) 3860e8d8bef9SDimitry Andric // We choose this as normal form to enable folding on the And and 3861e8d8bef9SDimitry Andric // shortening paths for the values (this helps getUnderlyingObjects() for 3862e8d8bef9SDimitry Andric // example). 38630b57cec5SDimitry Andric if (TrueSI->getFalseValue() == FalseVal && TrueSI->hasOneUse()) { 3864fe6060f1SDimitry Andric Value *And = Builder.CreateLogicalAnd(CondVal, TrueSI->getCondition()); 38655ffd83dbSDimitry Andric replaceOperand(SI, 0, And); 38665ffd83dbSDimitry Andric replaceOperand(SI, 1, TrueSI->getTrueValue()); 38670b57cec5SDimitry Andric return &SI; 38680b57cec5SDimitry Andric } 38690b57cec5SDimitry Andric } 38700b57cec5SDimitry Andric } 38710b57cec5SDimitry Andric if (SelectInst *FalseSI = dyn_cast<SelectInst>(FalseVal)) { 38720b57cec5SDimitry Andric if (FalseSI->getCondition()->getType() == CondVal->getType()) { 3873*0fca6ea1SDimitry Andric // Fold nested selects if the inner condition can be implied by the outer 3874*0fca6ea1SDimitry Andric // condition. 3875*0fca6ea1SDimitry Andric if (Value *V = simplifyNestedSelectsUsingImpliedCond( 3876*0fca6ea1SDimitry Andric *FalseSI, CondVal, /*CondIsTrue=*/false, DL)) 3877*0fca6ea1SDimitry Andric return replaceOperand(SI, 2, V); 3878*0fca6ea1SDimitry Andric 38790b57cec5SDimitry Andric // select(C0, a, select(C1, a, b)) -> select(C0|C1, a, b) 38800b57cec5SDimitry Andric if (FalseSI->getTrueValue() == TrueVal && FalseSI->hasOneUse()) { 3881fe6060f1SDimitry Andric Value *Or = Builder.CreateLogicalOr(CondVal, FalseSI->getCondition()); 38825ffd83dbSDimitry Andric replaceOperand(SI, 0, Or); 38835ffd83dbSDimitry Andric replaceOperand(SI, 2, FalseSI->getFalseValue()); 38840b57cec5SDimitry Andric return &SI; 38850b57cec5SDimitry Andric } 38860b57cec5SDimitry Andric } 38870b57cec5SDimitry Andric } 38880b57cec5SDimitry Andric 38890b57cec5SDimitry Andric // Try to simplify a binop sandwiched between 2 selects with the same 389006c3fb27SDimitry Andric // condition. This is not valid for div/rem because the select might be 389106c3fb27SDimitry Andric // preventing a division-by-zero. 389206c3fb27SDimitry Andric // TODO: A div/rem restriction is conservative; use something like 389306c3fb27SDimitry Andric // isSafeToSpeculativelyExecute(). 38940b57cec5SDimitry Andric // select(C, binop(select(C, X, Y), W), Z) -> select(C, binop(X, W), Z) 38950b57cec5SDimitry Andric BinaryOperator *TrueBO; 389606c3fb27SDimitry Andric if (match(TrueVal, m_OneUse(m_BinOp(TrueBO))) && !TrueBO->isIntDivRem()) { 38970b57cec5SDimitry Andric if (auto *TrueBOSI = dyn_cast<SelectInst>(TrueBO->getOperand(0))) { 38980b57cec5SDimitry Andric if (TrueBOSI->getCondition() == CondVal) { 38995ffd83dbSDimitry Andric replaceOperand(*TrueBO, 0, TrueBOSI->getTrueValue()); 39005ffd83dbSDimitry Andric Worklist.push(TrueBO); 39010b57cec5SDimitry Andric return &SI; 39020b57cec5SDimitry Andric } 39030b57cec5SDimitry Andric } 39040b57cec5SDimitry Andric if (auto *TrueBOSI = dyn_cast<SelectInst>(TrueBO->getOperand(1))) { 39050b57cec5SDimitry Andric if (TrueBOSI->getCondition() == CondVal) { 39065ffd83dbSDimitry Andric replaceOperand(*TrueBO, 1, TrueBOSI->getTrueValue()); 39075ffd83dbSDimitry Andric Worklist.push(TrueBO); 39080b57cec5SDimitry Andric return &SI; 39090b57cec5SDimitry Andric } 39100b57cec5SDimitry Andric } 39110b57cec5SDimitry Andric } 39120b57cec5SDimitry Andric 39130b57cec5SDimitry Andric // select(C, Z, binop(select(C, X, Y), W)) -> select(C, Z, binop(Y, W)) 39140b57cec5SDimitry Andric BinaryOperator *FalseBO; 391506c3fb27SDimitry Andric if (match(FalseVal, m_OneUse(m_BinOp(FalseBO))) && !FalseBO->isIntDivRem()) { 39160b57cec5SDimitry Andric if (auto *FalseBOSI = dyn_cast<SelectInst>(FalseBO->getOperand(0))) { 39170b57cec5SDimitry Andric if (FalseBOSI->getCondition() == CondVal) { 39185ffd83dbSDimitry Andric replaceOperand(*FalseBO, 0, FalseBOSI->getFalseValue()); 39195ffd83dbSDimitry Andric Worklist.push(FalseBO); 39200b57cec5SDimitry Andric return &SI; 39210b57cec5SDimitry Andric } 39220b57cec5SDimitry Andric } 39230b57cec5SDimitry Andric if (auto *FalseBOSI = dyn_cast<SelectInst>(FalseBO->getOperand(1))) { 39240b57cec5SDimitry Andric if (FalseBOSI->getCondition() == CondVal) { 39255ffd83dbSDimitry Andric replaceOperand(*FalseBO, 1, FalseBOSI->getFalseValue()); 39265ffd83dbSDimitry Andric Worklist.push(FalseBO); 39270b57cec5SDimitry Andric return &SI; 39280b57cec5SDimitry Andric } 39290b57cec5SDimitry Andric } 39300b57cec5SDimitry Andric } 39310b57cec5SDimitry Andric 39320b57cec5SDimitry Andric Value *NotCond; 3933e8d8bef9SDimitry Andric if (match(CondVal, m_Not(m_Value(NotCond))) && 3934e8d8bef9SDimitry Andric !InstCombiner::shouldAvoidAbsorbingNotIntoSelect(SI)) { 39355ffd83dbSDimitry Andric replaceOperand(SI, 0, NotCond); 39365ffd83dbSDimitry Andric SI.swapValues(); 39370b57cec5SDimitry Andric SI.swapProfMetadata(); 39380b57cec5SDimitry Andric return &SI; 39390b57cec5SDimitry Andric } 39400b57cec5SDimitry Andric 39415ffd83dbSDimitry Andric if (Instruction *I = foldVectorSelect(SI)) 39425ffd83dbSDimitry Andric return I; 39430b57cec5SDimitry Andric 39440b57cec5SDimitry Andric // If we can compute the condition, there's no need for a select. 39450b57cec5SDimitry Andric // Like the above fold, we are attempting to reduce compile-time cost by 39460b57cec5SDimitry Andric // putting this fold here with limitations rather than in InstSimplify. 39470b57cec5SDimitry Andric // The motivation for this call into value tracking is to take advantage of 39480b57cec5SDimitry Andric // the assumption cache, so make sure that is populated. 39490b57cec5SDimitry Andric if (!CondVal->getType()->isVectorTy() && !AC.assumptions().empty()) { 39500b57cec5SDimitry Andric KnownBits Known(1); 39510b57cec5SDimitry Andric computeKnownBits(CondVal, Known, 0, &SI); 3952349cc55cSDimitry Andric if (Known.One.isOne()) 39530b57cec5SDimitry Andric return replaceInstUsesWith(SI, TrueVal); 3954349cc55cSDimitry Andric if (Known.Zero.isOne()) 39550b57cec5SDimitry Andric return replaceInstUsesWith(SI, FalseVal); 39560b57cec5SDimitry Andric } 39570b57cec5SDimitry Andric 39580b57cec5SDimitry Andric if (Instruction *BitCastSel = foldSelectCmpBitcasts(SI, Builder)) 39590b57cec5SDimitry Andric return BitCastSel; 39600b57cec5SDimitry Andric 39610b57cec5SDimitry Andric // Simplify selects that test the returned flag of cmpxchg instructions. 39625ffd83dbSDimitry Andric if (Value *V = foldSelectCmpXchg(SI)) 39635ffd83dbSDimitry Andric return replaceInstUsesWith(SI, V); 39640b57cec5SDimitry Andric 39655ffd83dbSDimitry Andric if (Instruction *Select = foldSelectBinOpIdentity(SI, TLI, *this)) 39660b57cec5SDimitry Andric return Select; 39670b57cec5SDimitry Andric 3968e8d8bef9SDimitry Andric if (Instruction *Funnel = foldSelectFunnelShift(SI, Builder)) 3969e8d8bef9SDimitry Andric return Funnel; 39700b57cec5SDimitry Andric 39715ffd83dbSDimitry Andric if (Instruction *Copysign = foldSelectToCopysign(SI, Builder)) 39725ffd83dbSDimitry Andric return Copysign; 39735ffd83dbSDimitry Andric 39745ffd83dbSDimitry Andric if (Instruction *PN = foldSelectToPhi(SI, DT, Builder)) 39755ffd83dbSDimitry Andric return replaceInstUsesWith(SI, PN); 39765ffd83dbSDimitry Andric 3977e8d8bef9SDimitry Andric if (Value *Fr = foldSelectWithFrozenICmp(SI, Builder)) 3978e8d8bef9SDimitry Andric return replaceInstUsesWith(SI, Fr); 3979e8d8bef9SDimitry Andric 398081ad6265SDimitry Andric if (Value *V = foldRoundUpIntegerWithPow2Alignment(SI, Builder)) 398181ad6265SDimitry Andric return replaceInstUsesWith(SI, V); 398281ad6265SDimitry Andric 3983fe6060f1SDimitry Andric // select(mask, mload(,,mask,0), 0) -> mload(,,mask,0) 3984fe6060f1SDimitry Andric // Load inst is intentionally not checked for hasOneUse() 3985fe6060f1SDimitry Andric if (match(FalseVal, m_Zero()) && 398681ad6265SDimitry Andric (match(TrueVal, m_MaskedLoad(m_Value(), m_Value(), m_Specific(CondVal), 398781ad6265SDimitry Andric m_CombineOr(m_Undef(), m_Zero()))) || 398881ad6265SDimitry Andric match(TrueVal, m_MaskedGather(m_Value(), m_Value(), m_Specific(CondVal), 398981ad6265SDimitry Andric m_CombineOr(m_Undef(), m_Zero()))))) { 399081ad6265SDimitry Andric auto *MaskedInst = cast<IntrinsicInst>(TrueVal); 399181ad6265SDimitry Andric if (isa<UndefValue>(MaskedInst->getArgOperand(3))) 399281ad6265SDimitry Andric MaskedInst->setArgOperand(3, FalseVal /* Zero */); 399381ad6265SDimitry Andric return replaceInstUsesWith(SI, MaskedInst); 3994fe6060f1SDimitry Andric } 3995fe6060f1SDimitry Andric 3996fe6060f1SDimitry Andric Value *Mask; 3997fe6060f1SDimitry Andric if (match(TrueVal, m_Zero()) && 399881ad6265SDimitry Andric (match(FalseVal, m_MaskedLoad(m_Value(), m_Value(), m_Value(Mask), 399981ad6265SDimitry Andric m_CombineOr(m_Undef(), m_Zero()))) || 400081ad6265SDimitry Andric match(FalseVal, m_MaskedGather(m_Value(), m_Value(), m_Value(Mask), 400181ad6265SDimitry Andric m_CombineOr(m_Undef(), m_Zero())))) && 40026e75b2fbSDimitry Andric (CondVal->getType() == Mask->getType())) { 4003fe6060f1SDimitry Andric // We can remove the select by ensuring the load zeros all lanes the 4004fe6060f1SDimitry Andric // select would have. We determine this by proving there is no overlap 4005fe6060f1SDimitry Andric // between the load and select masks. 4006fe6060f1SDimitry Andric // (i.e (load_mask & select_mask) == 0 == no overlap) 4007fe6060f1SDimitry Andric bool CanMergeSelectIntoLoad = false; 400881ad6265SDimitry Andric if (Value *V = simplifyAndInst(CondVal, Mask, SQ.getWithInstruction(&SI))) 4009fe6060f1SDimitry Andric CanMergeSelectIntoLoad = match(V, m_Zero()); 4010fe6060f1SDimitry Andric 4011fe6060f1SDimitry Andric if (CanMergeSelectIntoLoad) { 401281ad6265SDimitry Andric auto *MaskedInst = cast<IntrinsicInst>(FalseVal); 401381ad6265SDimitry Andric if (isa<UndefValue>(MaskedInst->getArgOperand(3))) 401481ad6265SDimitry Andric MaskedInst->setArgOperand(3, TrueVal /* Zero */); 401581ad6265SDimitry Andric return replaceInstUsesWith(SI, MaskedInst); 4016fe6060f1SDimitry Andric } 4017fe6060f1SDimitry Andric } 4018fe6060f1SDimitry Andric 4019*0fca6ea1SDimitry Andric if (Instruction *I = foldSelectOfSymmetricSelect(SI, Builder)) 4020*0fca6ea1SDimitry Andric return I; 4021*0fca6ea1SDimitry Andric 4022bdd1243dSDimitry Andric if (Instruction *I = foldNestedSelects(SI, Builder)) 4023bdd1243dSDimitry Andric return I; 4024bdd1243dSDimitry Andric 4025bdd1243dSDimitry Andric // Match logical variants of the pattern, 4026bdd1243dSDimitry Andric // and transform them iff that gets rid of inversions. 4027bdd1243dSDimitry Andric // (~x) | y --> ~(x & (~y)) 4028bdd1243dSDimitry Andric // (~x) & y --> ~(x | (~y)) 4029bdd1243dSDimitry Andric if (sinkNotIntoOtherHandOfLogicalOp(SI)) 4030bdd1243dSDimitry Andric return &SI; 4031bdd1243dSDimitry Andric 403206c3fb27SDimitry Andric if (Instruction *I = foldBitCeil(SI, Builder)) 403306c3fb27SDimitry Andric return I; 403406c3fb27SDimitry Andric 4035647cbc5dSDimitry Andric // Fold: 4036647cbc5dSDimitry Andric // (select A && B, T, F) -> (select A, (select B, T, F), F) 4037647cbc5dSDimitry Andric // (select A || B, T, F) -> (select A, T, (select B, T, F)) 4038647cbc5dSDimitry Andric // if (select B, T, F) is foldable. 4039647cbc5dSDimitry Andric // TODO: preserve FMF flags 4040647cbc5dSDimitry Andric auto FoldSelectWithAndOrCond = [&](bool IsAnd, Value *A, 4041647cbc5dSDimitry Andric Value *B) -> Instruction * { 4042647cbc5dSDimitry Andric if (Value *V = simplifySelectInst(B, TrueVal, FalseVal, 4043647cbc5dSDimitry Andric SQ.getWithInstruction(&SI))) 4044647cbc5dSDimitry Andric return SelectInst::Create(A, IsAnd ? V : TrueVal, IsAnd ? FalseVal : V); 4045647cbc5dSDimitry Andric 4046647cbc5dSDimitry Andric // Is (select B, T, F) a SPF? 4047647cbc5dSDimitry Andric if (CondVal->hasOneUse() && SelType->isIntOrIntVectorTy()) { 4048647cbc5dSDimitry Andric if (ICmpInst *Cmp = dyn_cast<ICmpInst>(B)) 4049647cbc5dSDimitry Andric if (Value *V = canonicalizeSPF(*Cmp, TrueVal, FalseVal, *this)) 4050647cbc5dSDimitry Andric return SelectInst::Create(A, IsAnd ? V : TrueVal, 4051647cbc5dSDimitry Andric IsAnd ? FalseVal : V); 4052647cbc5dSDimitry Andric } 4053647cbc5dSDimitry Andric 4054647cbc5dSDimitry Andric return nullptr; 4055647cbc5dSDimitry Andric }; 4056647cbc5dSDimitry Andric 4057647cbc5dSDimitry Andric Value *LHS, *RHS; 4058647cbc5dSDimitry Andric if (match(CondVal, m_And(m_Value(LHS), m_Value(RHS)))) { 4059647cbc5dSDimitry Andric if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ true, LHS, RHS)) 4060647cbc5dSDimitry Andric return I; 4061647cbc5dSDimitry Andric if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ true, RHS, LHS)) 4062647cbc5dSDimitry Andric return I; 4063647cbc5dSDimitry Andric } else if (match(CondVal, m_Or(m_Value(LHS), m_Value(RHS)))) { 4064647cbc5dSDimitry Andric if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ false, LHS, RHS)) 4065647cbc5dSDimitry Andric return I; 4066647cbc5dSDimitry Andric if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ false, RHS, LHS)) 4067647cbc5dSDimitry Andric return I; 4068647cbc5dSDimitry Andric } else { 4069647cbc5dSDimitry Andric // We cannot swap the operands of logical and/or. 4070647cbc5dSDimitry Andric // TODO: Can we swap the operands by inserting a freeze? 4071647cbc5dSDimitry Andric if (match(CondVal, m_LogicalAnd(m_Value(LHS), m_Value(RHS)))) { 4072647cbc5dSDimitry Andric if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ true, LHS, RHS)) 4073647cbc5dSDimitry Andric return I; 4074647cbc5dSDimitry Andric } else if (match(CondVal, m_LogicalOr(m_Value(LHS), m_Value(RHS)))) { 4075647cbc5dSDimitry Andric if (Instruction *I = FoldSelectWithAndOrCond(/*IsAnd*/ false, LHS, RHS)) 4076647cbc5dSDimitry Andric return I; 4077647cbc5dSDimitry Andric } 4078647cbc5dSDimitry Andric } 4079647cbc5dSDimitry Andric 4080*0fca6ea1SDimitry Andric // select Cond, !X, X -> xor Cond, X 4081*0fca6ea1SDimitry Andric if (CondVal->getType() == SI.getType() && isKnownInversion(FalseVal, TrueVal)) 4082*0fca6ea1SDimitry Andric return BinaryOperator::CreateXor(CondVal, FalseVal); 4083*0fca6ea1SDimitry Andric 4084*0fca6ea1SDimitry Andric // For vectors, this transform is only safe if the simplification does not 4085*0fca6ea1SDimitry Andric // look through any lane-crossing operations. For now, limit to scalars only. 4086*0fca6ea1SDimitry Andric if (SelType->isIntegerTy() && 4087*0fca6ea1SDimitry Andric (!isa<Constant>(TrueVal) || !isa<Constant>(FalseVal))) { 4088*0fca6ea1SDimitry Andric // Try to simplify select arms based on KnownBits implied by the condition. 4089*0fca6ea1SDimitry Andric CondContext CC(CondVal); 4090*0fca6ea1SDimitry Andric findValuesAffectedByCondition(CondVal, /*IsAssume=*/false, [&](Value *V) { 4091*0fca6ea1SDimitry Andric CC.AffectedValues.insert(V); 4092*0fca6ea1SDimitry Andric }); 4093*0fca6ea1SDimitry Andric SimplifyQuery Q = SQ.getWithInstruction(&SI).getWithCondContext(CC); 4094*0fca6ea1SDimitry Andric if (!CC.AffectedValues.empty()) { 4095*0fca6ea1SDimitry Andric if (!isa<Constant>(TrueVal) && 4096*0fca6ea1SDimitry Andric hasAffectedValue(TrueVal, CC.AffectedValues, /*Depth=*/0)) { 4097*0fca6ea1SDimitry Andric KnownBits Known = llvm::computeKnownBits(TrueVal, /*Depth=*/0, Q); 4098*0fca6ea1SDimitry Andric if (Known.isConstant()) 4099*0fca6ea1SDimitry Andric return replaceOperand(SI, 1, 4100*0fca6ea1SDimitry Andric ConstantInt::get(SelType, Known.getConstant())); 4101*0fca6ea1SDimitry Andric } 4102*0fca6ea1SDimitry Andric 4103*0fca6ea1SDimitry Andric CC.Invert = true; 4104*0fca6ea1SDimitry Andric if (!isa<Constant>(FalseVal) && 4105*0fca6ea1SDimitry Andric hasAffectedValue(FalseVal, CC.AffectedValues, /*Depth=*/0)) { 4106*0fca6ea1SDimitry Andric KnownBits Known = llvm::computeKnownBits(FalseVal, /*Depth=*/0, Q); 4107*0fca6ea1SDimitry Andric if (Known.isConstant()) 4108*0fca6ea1SDimitry Andric return replaceOperand(SI, 2, 4109*0fca6ea1SDimitry Andric ConstantInt::get(SelType, Known.getConstant())); 4110*0fca6ea1SDimitry Andric } 4111*0fca6ea1SDimitry Andric } 4112*0fca6ea1SDimitry Andric } 4113*0fca6ea1SDimitry Andric 41140b57cec5SDimitry Andric return nullptr; 41150b57cec5SDimitry Andric } 4116