1 //===----- CodeGen/ExpandVectorPredication.cpp - Expand VP intrinsics -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass implements IR expansion for vector predication intrinsics, allowing 10 // targets to enable vector predication until just before codegen. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/CodeGen/ExpandVectorPredication.h" 15 #include "llvm/ADT/Statistic.h" 16 #include "llvm/Analysis/TargetTransformInfo.h" 17 #include "llvm/Analysis/ValueTracking.h" 18 #include "llvm/CodeGen/Passes.h" 19 #include "llvm/IR/Constants.h" 20 #include "llvm/IR/Function.h" 21 #include "llvm/IR/IRBuilder.h" 22 #include "llvm/IR/InstIterator.h" 23 #include "llvm/IR/Instructions.h" 24 #include "llvm/IR/IntrinsicInst.h" 25 #include "llvm/IR/Intrinsics.h" 26 #include "llvm/InitializePasses.h" 27 #include "llvm/Pass.h" 28 #include "llvm/Support/CommandLine.h" 29 #include "llvm/Support/Compiler.h" 30 #include "llvm/Support/Debug.h" 31 32 using namespace llvm; 33 34 using VPLegalization = TargetTransformInfo::VPLegalization; 35 using VPTransform = TargetTransformInfo::VPLegalization::VPTransform; 36 37 // Keep this in sync with TargetTransformInfo::VPLegalization. 38 #define VPINTERNAL_VPLEGAL_CASES \ 39 VPINTERNAL_CASE(Legal) \ 40 VPINTERNAL_CASE(Discard) \ 41 VPINTERNAL_CASE(Convert) 42 43 #define VPINTERNAL_CASE(X) "|" #X 44 45 // Override options. 46 static cl::opt<std::string> EVLTransformOverride( 47 "expandvp-override-evl-transform", cl::init(""), cl::Hidden, 48 cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES 49 ". If non-empty, ignore " 50 "TargetTransformInfo and " 51 "always use this transformation for the %evl parameter (Used in " 52 "testing).")); 53 54 static cl::opt<std::string> MaskTransformOverride( 55 "expandvp-override-mask-transform", cl::init(""), cl::Hidden, 56 cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES 57 ". If non-empty, Ignore " 58 "TargetTransformInfo and " 59 "always use this transformation for the %mask parameter (Used in " 60 "testing).")); 61 62 #undef VPINTERNAL_CASE 63 #define VPINTERNAL_CASE(X) .Case(#X, VPLegalization::X) 64 65 static VPTransform parseOverrideOption(const std::string &TextOpt) { 66 return StringSwitch<VPTransform>(TextOpt) VPINTERNAL_VPLEGAL_CASES; 67 } 68 69 #undef VPINTERNAL_VPLEGAL_CASES 70 71 // Whether any override options are set. 72 static bool anyExpandVPOverridesSet() { 73 return !EVLTransformOverride.empty() || !MaskTransformOverride.empty(); 74 } 75 76 #define DEBUG_TYPE "expandvp" 77 78 STATISTIC(NumFoldedVL, "Number of folded vector length params"); 79 STATISTIC(NumLoweredVPOps, "Number of folded vector predication operations"); 80 81 ///// Helpers { 82 83 /// \returns Whether the vector mask \p MaskVal has all lane bits set. 84 static bool isAllTrueMask(Value *MaskVal) { 85 auto *ConstVec = dyn_cast<ConstantVector>(MaskVal); 86 return ConstVec && ConstVec->isAllOnesValue(); 87 } 88 89 /// \returns A non-excepting divisor constant for this type. 90 static Constant *getSafeDivisor(Type *DivTy) { 91 assert(DivTy->isIntOrIntVectorTy() && "Unsupported divisor type"); 92 return ConstantInt::get(DivTy, 1u, false); 93 } 94 95 /// Transfer operation properties from \p OldVPI to \p NewVal. 96 static void transferDecorations(Value &NewVal, VPIntrinsic &VPI) { 97 auto *NewInst = dyn_cast<Instruction>(&NewVal); 98 if (!NewInst || !isa<FPMathOperator>(NewVal)) 99 return; 100 101 auto *OldFMOp = dyn_cast<FPMathOperator>(&VPI); 102 if (!OldFMOp) 103 return; 104 105 NewInst->setFastMathFlags(OldFMOp->getFastMathFlags()); 106 } 107 108 /// Transfer all properties from \p OldOp to \p NewOp and replace all uses. 109 /// OldVP gets erased. 110 static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp) { 111 transferDecorations(NewOp, OldOp); 112 OldOp.replaceAllUsesWith(&NewOp); 113 OldOp.eraseFromParent(); 114 } 115 116 static bool maySpeculateLanes(VPIntrinsic &VPI) { 117 // The result of VP reductions depends on the mask and evl. 118 if (isa<VPReductionIntrinsic>(VPI)) 119 return false; 120 // Fallback to whether the intrinsic is speculatable. 121 Optional<unsigned> OpcOpt = VPI.getFunctionalOpcode(); 122 unsigned FunctionalOpc = OpcOpt.value_or((unsigned)Instruction::Call); 123 return isSafeToSpeculativelyExecuteWithOpcode(FunctionalOpc, 124 cast<Operator>(&VPI)); 125 } 126 127 //// } Helpers 128 129 namespace { 130 131 // Expansion pass state at function scope. 132 struct CachingVPExpander { 133 Function &F; 134 const TargetTransformInfo &TTI; 135 136 /// \returns A (fixed length) vector with ascending integer indices 137 /// (<0, 1, ..., NumElems-1>). 138 /// \p Builder 139 /// Used for instruction creation. 140 /// \p LaneTy 141 /// Integer element type of the result vector. 142 /// \p NumElems 143 /// Number of vector elements. 144 Value *createStepVector(IRBuilder<> &Builder, Type *LaneTy, 145 unsigned NumElems); 146 147 /// \returns A bitmask that is true where the lane position is less-than \p 148 /// EVLParam 149 /// 150 /// \p Builder 151 /// Used for instruction creation. 152 /// \p VLParam 153 /// The explicit vector length parameter to test against the lane 154 /// positions. 155 /// \p ElemCount 156 /// Static (potentially scalable) number of vector elements. 157 Value *convertEVLToMask(IRBuilder<> &Builder, Value *EVLParam, 158 ElementCount ElemCount); 159 160 Value *foldEVLIntoMask(VPIntrinsic &VPI); 161 162 /// "Remove" the %evl parameter of \p PI by setting it to the static vector 163 /// length of the operation. 164 void discardEVLParameter(VPIntrinsic &PI); 165 166 /// \brief Lower this VP binary operator to a unpredicated binary operator. 167 Value *expandPredicationInBinaryOperator(IRBuilder<> &Builder, 168 VPIntrinsic &PI); 169 170 /// \brief Lower this VP reduction to a call to an unpredicated reduction 171 /// intrinsic. 172 Value *expandPredicationInReduction(IRBuilder<> &Builder, 173 VPReductionIntrinsic &PI); 174 175 /// \brief Query TTI and expand the vector predication in \p P accordingly. 176 Value *expandPredication(VPIntrinsic &PI); 177 178 /// \brief Determine how and whether the VPIntrinsic \p VPI shall be 179 /// expanded. This overrides TTI with the cl::opts listed at the top of this 180 /// file. 181 VPLegalization getVPLegalizationStrategy(const VPIntrinsic &VPI) const; 182 bool UsingTTIOverrides; 183 184 public: 185 CachingVPExpander(Function &F, const TargetTransformInfo &TTI) 186 : F(F), TTI(TTI), UsingTTIOverrides(anyExpandVPOverridesSet()) {} 187 188 bool expandVectorPredication(); 189 }; 190 191 //// CachingVPExpander { 192 193 Value *CachingVPExpander::createStepVector(IRBuilder<> &Builder, Type *LaneTy, 194 unsigned NumElems) { 195 // TODO add caching 196 SmallVector<Constant *, 16> ConstElems; 197 198 for (unsigned Idx = 0; Idx < NumElems; ++Idx) 199 ConstElems.push_back(ConstantInt::get(LaneTy, Idx, false)); 200 201 return ConstantVector::get(ConstElems); 202 } 203 204 Value *CachingVPExpander::convertEVLToMask(IRBuilder<> &Builder, 205 Value *EVLParam, 206 ElementCount ElemCount) { 207 // TODO add caching 208 // Scalable vector %evl conversion. 209 if (ElemCount.isScalable()) { 210 auto *M = Builder.GetInsertBlock()->getModule(); 211 Type *BoolVecTy = VectorType::get(Builder.getInt1Ty(), ElemCount); 212 Function *ActiveMaskFunc = Intrinsic::getDeclaration( 213 M, Intrinsic::get_active_lane_mask, {BoolVecTy, EVLParam->getType()}); 214 // `get_active_lane_mask` performs an implicit less-than comparison. 215 Value *ConstZero = Builder.getInt32(0); 216 return Builder.CreateCall(ActiveMaskFunc, {ConstZero, EVLParam}); 217 } 218 219 // Fixed vector %evl conversion. 220 Type *LaneTy = EVLParam->getType(); 221 unsigned NumElems = ElemCount.getFixedValue(); 222 Value *VLSplat = Builder.CreateVectorSplat(NumElems, EVLParam); 223 Value *IdxVec = createStepVector(Builder, LaneTy, NumElems); 224 return Builder.CreateICmp(CmpInst::ICMP_ULT, IdxVec, VLSplat); 225 } 226 227 Value * 228 CachingVPExpander::expandPredicationInBinaryOperator(IRBuilder<> &Builder, 229 VPIntrinsic &VPI) { 230 assert((maySpeculateLanes(VPI) || VPI.canIgnoreVectorLengthParam()) && 231 "Implicitly dropping %evl in non-speculatable operator!"); 232 233 auto OC = static_cast<Instruction::BinaryOps>(*VPI.getFunctionalOpcode()); 234 assert(Instruction::isBinaryOp(OC)); 235 236 Value *Op0 = VPI.getOperand(0); 237 Value *Op1 = VPI.getOperand(1); 238 Value *Mask = VPI.getMaskParam(); 239 240 // Blend in safe operands. 241 if (Mask && !isAllTrueMask(Mask)) { 242 switch (OC) { 243 default: 244 // Can safely ignore the predicate. 245 break; 246 247 // Division operators need a safe divisor on masked-off lanes (1). 248 case Instruction::UDiv: 249 case Instruction::SDiv: 250 case Instruction::URem: 251 case Instruction::SRem: 252 // 2nd operand must not be zero. 253 Value *SafeDivisor = getSafeDivisor(VPI.getType()); 254 Op1 = Builder.CreateSelect(Mask, Op1, SafeDivisor); 255 } 256 } 257 258 Value *NewBinOp = Builder.CreateBinOp(OC, Op0, Op1, VPI.getName()); 259 260 replaceOperation(*NewBinOp, VPI); 261 return NewBinOp; 262 } 263 264 static Value *getNeutralReductionElement(const VPReductionIntrinsic &VPI, 265 Type *EltTy) { 266 bool Negative = false; 267 unsigned EltBits = EltTy->getScalarSizeInBits(); 268 switch (VPI.getIntrinsicID()) { 269 default: 270 llvm_unreachable("Expecting a VP reduction intrinsic"); 271 case Intrinsic::vp_reduce_add: 272 case Intrinsic::vp_reduce_or: 273 case Intrinsic::vp_reduce_xor: 274 case Intrinsic::vp_reduce_umax: 275 return Constant::getNullValue(EltTy); 276 case Intrinsic::vp_reduce_mul: 277 return ConstantInt::get(EltTy, 1, /*IsSigned*/ false); 278 case Intrinsic::vp_reduce_and: 279 case Intrinsic::vp_reduce_umin: 280 return ConstantInt::getAllOnesValue(EltTy); 281 case Intrinsic::vp_reduce_smin: 282 return ConstantInt::get(EltTy->getContext(), 283 APInt::getSignedMaxValue(EltBits)); 284 case Intrinsic::vp_reduce_smax: 285 return ConstantInt::get(EltTy->getContext(), 286 APInt::getSignedMinValue(EltBits)); 287 case Intrinsic::vp_reduce_fmax: 288 Negative = true; 289 LLVM_FALLTHROUGH; 290 case Intrinsic::vp_reduce_fmin: { 291 FastMathFlags Flags = VPI.getFastMathFlags(); 292 const fltSemantics &Semantics = EltTy->getFltSemantics(); 293 return !Flags.noNaNs() ? ConstantFP::getQNaN(EltTy, Negative) 294 : !Flags.noInfs() 295 ? ConstantFP::getInfinity(EltTy, Negative) 296 : ConstantFP::get(EltTy, 297 APFloat::getLargest(Semantics, Negative)); 298 } 299 case Intrinsic::vp_reduce_fadd: 300 return ConstantFP::getNegativeZero(EltTy); 301 case Intrinsic::vp_reduce_fmul: 302 return ConstantFP::get(EltTy, 1.0); 303 } 304 } 305 306 Value * 307 CachingVPExpander::expandPredicationInReduction(IRBuilder<> &Builder, 308 VPReductionIntrinsic &VPI) { 309 assert((maySpeculateLanes(VPI) || VPI.canIgnoreVectorLengthParam()) && 310 "Implicitly dropping %evl in non-speculatable operator!"); 311 312 Value *Mask = VPI.getMaskParam(); 313 Value *RedOp = VPI.getOperand(VPI.getVectorParamPos()); 314 315 // Insert neutral element in masked-out positions 316 if (Mask && !isAllTrueMask(Mask)) { 317 auto *NeutralElt = getNeutralReductionElement(VPI, VPI.getType()); 318 auto *NeutralVector = Builder.CreateVectorSplat( 319 cast<VectorType>(RedOp->getType())->getElementCount(), NeutralElt); 320 RedOp = Builder.CreateSelect(Mask, RedOp, NeutralVector); 321 } 322 323 Value *Reduction; 324 Value *Start = VPI.getOperand(VPI.getStartParamPos()); 325 326 switch (VPI.getIntrinsicID()) { 327 default: 328 llvm_unreachable("Impossible reduction kind"); 329 case Intrinsic::vp_reduce_add: 330 Reduction = Builder.CreateAddReduce(RedOp); 331 Reduction = Builder.CreateAdd(Reduction, Start); 332 break; 333 case Intrinsic::vp_reduce_mul: 334 Reduction = Builder.CreateMulReduce(RedOp); 335 Reduction = Builder.CreateMul(Reduction, Start); 336 break; 337 case Intrinsic::vp_reduce_and: 338 Reduction = Builder.CreateAndReduce(RedOp); 339 Reduction = Builder.CreateAnd(Reduction, Start); 340 break; 341 case Intrinsic::vp_reduce_or: 342 Reduction = Builder.CreateOrReduce(RedOp); 343 Reduction = Builder.CreateOr(Reduction, Start); 344 break; 345 case Intrinsic::vp_reduce_xor: 346 Reduction = Builder.CreateXorReduce(RedOp); 347 Reduction = Builder.CreateXor(Reduction, Start); 348 break; 349 case Intrinsic::vp_reduce_smax: 350 Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ true); 351 Reduction = 352 Builder.CreateBinaryIntrinsic(Intrinsic::smax, Reduction, Start); 353 break; 354 case Intrinsic::vp_reduce_smin: 355 Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ true); 356 Reduction = 357 Builder.CreateBinaryIntrinsic(Intrinsic::smin, Reduction, Start); 358 break; 359 case Intrinsic::vp_reduce_umax: 360 Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ false); 361 Reduction = 362 Builder.CreateBinaryIntrinsic(Intrinsic::umax, Reduction, Start); 363 break; 364 case Intrinsic::vp_reduce_umin: 365 Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ false); 366 Reduction = 367 Builder.CreateBinaryIntrinsic(Intrinsic::umin, Reduction, Start); 368 break; 369 case Intrinsic::vp_reduce_fmax: 370 Reduction = Builder.CreateFPMaxReduce(RedOp); 371 transferDecorations(*Reduction, VPI); 372 Reduction = 373 Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, Reduction, Start); 374 break; 375 case Intrinsic::vp_reduce_fmin: 376 Reduction = Builder.CreateFPMinReduce(RedOp); 377 transferDecorations(*Reduction, VPI); 378 Reduction = 379 Builder.CreateBinaryIntrinsic(Intrinsic::minnum, Reduction, Start); 380 break; 381 case Intrinsic::vp_reduce_fadd: 382 Reduction = Builder.CreateFAddReduce(Start, RedOp); 383 break; 384 case Intrinsic::vp_reduce_fmul: 385 Reduction = Builder.CreateFMulReduce(Start, RedOp); 386 break; 387 } 388 389 replaceOperation(*Reduction, VPI); 390 return Reduction; 391 } 392 393 void CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) { 394 LLVM_DEBUG(dbgs() << "Discard EVL parameter in " << VPI << "\n"); 395 396 if (VPI.canIgnoreVectorLengthParam()) 397 return; 398 399 Value *EVLParam = VPI.getVectorLengthParam(); 400 if (!EVLParam) 401 return; 402 403 ElementCount StaticElemCount = VPI.getStaticVectorLength(); 404 Value *MaxEVL = nullptr; 405 Type *Int32Ty = Type::getInt32Ty(VPI.getContext()); 406 if (StaticElemCount.isScalable()) { 407 // TODO add caching 408 auto *M = VPI.getModule(); 409 Function *VScaleFunc = 410 Intrinsic::getDeclaration(M, Intrinsic::vscale, Int32Ty); 411 IRBuilder<> Builder(VPI.getParent(), VPI.getIterator()); 412 Value *FactorConst = Builder.getInt32(StaticElemCount.getKnownMinValue()); 413 Value *VScale = Builder.CreateCall(VScaleFunc, {}, "vscale"); 414 MaxEVL = Builder.CreateMul(VScale, FactorConst, "scalable_size", 415 /*NUW*/ true, /*NSW*/ false); 416 } else { 417 MaxEVL = ConstantInt::get(Int32Ty, StaticElemCount.getFixedValue(), false); 418 } 419 VPI.setVectorLengthParam(MaxEVL); 420 } 421 422 Value *CachingVPExpander::foldEVLIntoMask(VPIntrinsic &VPI) { 423 LLVM_DEBUG(dbgs() << "Folding vlen for " << VPI << '\n'); 424 425 IRBuilder<> Builder(&VPI); 426 427 // Ineffective %evl parameter and so nothing to do here. 428 if (VPI.canIgnoreVectorLengthParam()) 429 return &VPI; 430 431 // Only VP intrinsics can have an %evl parameter. 432 Value *OldMaskParam = VPI.getMaskParam(); 433 Value *OldEVLParam = VPI.getVectorLengthParam(); 434 assert(OldMaskParam && "no mask param to fold the vl param into"); 435 assert(OldEVLParam && "no EVL param to fold away"); 436 437 LLVM_DEBUG(dbgs() << "OLD evl: " << *OldEVLParam << '\n'); 438 LLVM_DEBUG(dbgs() << "OLD mask: " << *OldMaskParam << '\n'); 439 440 // Convert the %evl predication into vector mask predication. 441 ElementCount ElemCount = VPI.getStaticVectorLength(); 442 Value *VLMask = convertEVLToMask(Builder, OldEVLParam, ElemCount); 443 Value *NewMaskParam = Builder.CreateAnd(VLMask, OldMaskParam); 444 VPI.setMaskParam(NewMaskParam); 445 446 // Drop the %evl parameter. 447 discardEVLParameter(VPI); 448 assert(VPI.canIgnoreVectorLengthParam() && 449 "transformation did not render the evl param ineffective!"); 450 451 // Reassess the modified instruction. 452 return &VPI; 453 } 454 455 Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) { 456 LLVM_DEBUG(dbgs() << "Lowering to unpredicated op: " << VPI << '\n'); 457 458 IRBuilder<> Builder(&VPI); 459 460 // Try lowering to a LLVM instruction first. 461 auto OC = VPI.getFunctionalOpcode(); 462 463 if (OC && Instruction::isBinaryOp(*OC)) 464 return expandPredicationInBinaryOperator(Builder, VPI); 465 466 if (auto *VPRI = dyn_cast<VPReductionIntrinsic>(&VPI)) 467 return expandPredicationInReduction(Builder, *VPRI); 468 469 return &VPI; 470 } 471 472 //// } CachingVPExpander 473 474 struct TransformJob { 475 VPIntrinsic *PI; 476 TargetTransformInfo::VPLegalization Strategy; 477 TransformJob(VPIntrinsic *PI, TargetTransformInfo::VPLegalization InitStrat) 478 : PI(PI), Strategy(InitStrat) {} 479 480 bool isDone() const { return Strategy.shouldDoNothing(); } 481 }; 482 483 void sanitizeStrategy(VPIntrinsic &VPI, VPLegalization &LegalizeStrat) { 484 // Operations with speculatable lanes do not strictly need predication. 485 if (maySpeculateLanes(VPI)) { 486 // Converting a speculatable VP intrinsic means dropping %mask and %evl. 487 // No need to expand %evl into the %mask only to ignore that code. 488 if (LegalizeStrat.OpStrategy == VPLegalization::Convert) 489 LegalizeStrat.EVLParamStrategy = VPLegalization::Discard; 490 return; 491 } 492 493 // We have to preserve the predicating effect of %evl for this 494 // non-speculatable VP intrinsic. 495 // 1) Never discard %evl. 496 // 2) If this VP intrinsic will be expanded to non-VP code, make sure that 497 // %evl gets folded into %mask. 498 if ((LegalizeStrat.EVLParamStrategy == VPLegalization::Discard) || 499 (LegalizeStrat.OpStrategy == VPLegalization::Convert)) { 500 LegalizeStrat.EVLParamStrategy = VPLegalization::Convert; 501 } 502 } 503 504 VPLegalization 505 CachingVPExpander::getVPLegalizationStrategy(const VPIntrinsic &VPI) const { 506 auto VPStrat = TTI.getVPLegalizationStrategy(VPI); 507 if (LLVM_LIKELY(!UsingTTIOverrides)) { 508 // No overrides - we are in production. 509 return VPStrat; 510 } 511 512 // Overrides set - we are in testing, the following does not need to be 513 // efficient. 514 VPStrat.EVLParamStrategy = parseOverrideOption(EVLTransformOverride); 515 VPStrat.OpStrategy = parseOverrideOption(MaskTransformOverride); 516 return VPStrat; 517 } 518 519 /// \brief Expand llvm.vp.* intrinsics as requested by \p TTI. 520 bool CachingVPExpander::expandVectorPredication() { 521 SmallVector<TransformJob, 16> Worklist; 522 523 // Collect all VPIntrinsics that need expansion and determine their expansion 524 // strategy. 525 for (auto &I : instructions(F)) { 526 auto *VPI = dyn_cast<VPIntrinsic>(&I); 527 if (!VPI) 528 continue; 529 auto VPStrat = getVPLegalizationStrategy(*VPI); 530 sanitizeStrategy(*VPI, VPStrat); 531 if (!VPStrat.shouldDoNothing()) 532 Worklist.emplace_back(VPI, VPStrat); 533 } 534 if (Worklist.empty()) 535 return false; 536 537 // Transform all VPIntrinsics on the worklist. 538 LLVM_DEBUG(dbgs() << "\n:::: Transforming " << Worklist.size() 539 << " instructions ::::\n"); 540 for (TransformJob Job : Worklist) { 541 // Transform the EVL parameter. 542 switch (Job.Strategy.EVLParamStrategy) { 543 case VPLegalization::Legal: 544 break; 545 case VPLegalization::Discard: 546 discardEVLParameter(*Job.PI); 547 break; 548 case VPLegalization::Convert: 549 if (foldEVLIntoMask(*Job.PI)) 550 ++NumFoldedVL; 551 break; 552 } 553 Job.Strategy.EVLParamStrategy = VPLegalization::Legal; 554 555 // Replace with a non-predicated operation. 556 switch (Job.Strategy.OpStrategy) { 557 case VPLegalization::Legal: 558 break; 559 case VPLegalization::Discard: 560 llvm_unreachable("Invalid strategy for operators."); 561 case VPLegalization::Convert: 562 expandPredication(*Job.PI); 563 ++NumLoweredVPOps; 564 break; 565 } 566 Job.Strategy.OpStrategy = VPLegalization::Legal; 567 568 assert(Job.isDone() && "incomplete transformation"); 569 } 570 571 return true; 572 } 573 class ExpandVectorPredication : public FunctionPass { 574 public: 575 static char ID; 576 ExpandVectorPredication() : FunctionPass(ID) { 577 initializeExpandVectorPredicationPass(*PassRegistry::getPassRegistry()); 578 } 579 580 bool runOnFunction(Function &F) override { 581 const auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); 582 CachingVPExpander VPExpander(F, *TTI); 583 return VPExpander.expandVectorPredication(); 584 } 585 586 void getAnalysisUsage(AnalysisUsage &AU) const override { 587 AU.addRequired<TargetTransformInfoWrapperPass>(); 588 AU.setPreservesCFG(); 589 } 590 }; 591 } // namespace 592 593 char ExpandVectorPredication::ID; 594 INITIALIZE_PASS_BEGIN(ExpandVectorPredication, "expandvp", 595 "Expand vector predication intrinsics", false, false) 596 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) 597 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) 598 INITIALIZE_PASS_END(ExpandVectorPredication, "expandvp", 599 "Expand vector predication intrinsics", false, false) 600 601 FunctionPass *llvm::createExpandVectorPredicationPass() { 602 return new ExpandVectorPredication(); 603 } 604 605 PreservedAnalyses 606 ExpandVectorPredicationPass::run(Function &F, FunctionAnalysisManager &AM) { 607 const auto &TTI = AM.getResult<TargetIRAnalysis>(F); 608 CachingVPExpander VPExpander(F, TTI); 609 if (!VPExpander.expandVectorPredication()) 610 return PreservedAnalyses::all(); 611 PreservedAnalyses PA; 612 PA.preserveSet<CFGAnalyses>(); 613 return PA; 614 } 615