1 //===--- ExpandReductions.cpp - Expand experimental reduction intrinsics --===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This pass implements IR expansion for reduction intrinsics, allowing targets 11 // to enable the experimental intrinsics until just before codegen. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "llvm/CodeGen/ExpandReductions.h" 16 #include "llvm/Analysis/TargetTransformInfo.h" 17 #include "llvm/CodeGen/Passes.h" 18 #include "llvm/IR/Function.h" 19 #include "llvm/IR/IRBuilder.h" 20 #include "llvm/IR/InstIterator.h" 21 #include "llvm/IR/IntrinsicInst.h" 22 #include "llvm/IR/Intrinsics.h" 23 #include "llvm/IR/Module.h" 24 #include "llvm/Pass.h" 25 #include "llvm/Transforms/Utils/LoopUtils.h" 26 27 using namespace llvm; 28 29 namespace { 30 31 unsigned getOpcode(Intrinsic::ID ID) { 32 switch (ID) { 33 case Intrinsic::experimental_vector_reduce_fadd: 34 return Instruction::FAdd; 35 case Intrinsic::experimental_vector_reduce_fmul: 36 return Instruction::FMul; 37 case Intrinsic::experimental_vector_reduce_add: 38 return Instruction::Add; 39 case Intrinsic::experimental_vector_reduce_mul: 40 return Instruction::Mul; 41 case Intrinsic::experimental_vector_reduce_and: 42 return Instruction::And; 43 case Intrinsic::experimental_vector_reduce_or: 44 return Instruction::Or; 45 case Intrinsic::experimental_vector_reduce_xor: 46 return Instruction::Xor; 47 case Intrinsic::experimental_vector_reduce_smax: 48 case Intrinsic::experimental_vector_reduce_smin: 49 case Intrinsic::experimental_vector_reduce_umax: 50 case Intrinsic::experimental_vector_reduce_umin: 51 return Instruction::ICmp; 52 case Intrinsic::experimental_vector_reduce_fmax: 53 case Intrinsic::experimental_vector_reduce_fmin: 54 return Instruction::FCmp; 55 default: 56 llvm_unreachable("Unexpected ID"); 57 } 58 } 59 60 RecurrenceDescriptor::MinMaxRecurrenceKind getMRK(Intrinsic::ID ID) { 61 switch (ID) { 62 case Intrinsic::experimental_vector_reduce_smax: 63 return RecurrenceDescriptor::MRK_SIntMax; 64 case Intrinsic::experimental_vector_reduce_smin: 65 return RecurrenceDescriptor::MRK_SIntMin; 66 case Intrinsic::experimental_vector_reduce_umax: 67 return RecurrenceDescriptor::MRK_UIntMax; 68 case Intrinsic::experimental_vector_reduce_umin: 69 return RecurrenceDescriptor::MRK_UIntMin; 70 case Intrinsic::experimental_vector_reduce_fmax: 71 return RecurrenceDescriptor::MRK_FloatMax; 72 case Intrinsic::experimental_vector_reduce_fmin: 73 return RecurrenceDescriptor::MRK_FloatMin; 74 default: 75 return RecurrenceDescriptor::MRK_Invalid; 76 } 77 } 78 79 bool expandReductions(Function &F, const TargetTransformInfo *TTI) { 80 bool Changed = false; 81 SmallVector<IntrinsicInst*, 4> Worklist; 82 for (inst_iterator I = inst_begin(F), E = inst_end(F); I != E; ++I) 83 if (auto II = dyn_cast<IntrinsicInst>(&*I)) 84 Worklist.push_back(II); 85 86 for (auto *II : Worklist) { 87 IRBuilder<> Builder(II); 88 Value *Vec = nullptr; 89 auto ID = II->getIntrinsicID(); 90 auto MRK = RecurrenceDescriptor::MRK_Invalid; 91 switch (ID) { 92 case Intrinsic::experimental_vector_reduce_fadd: 93 case Intrinsic::experimental_vector_reduce_fmul: 94 // FMFs must be attached to the call, otherwise it's an ordered reduction 95 // and it can't be handled by generating this shuffle sequence. 96 // TODO: Implement scalarization of ordered reductions here for targets 97 // without native support. 98 if (!II->getFastMathFlags().unsafeAlgebra()) 99 continue; 100 Vec = II->getArgOperand(1); 101 break; 102 case Intrinsic::experimental_vector_reduce_add: 103 case Intrinsic::experimental_vector_reduce_mul: 104 case Intrinsic::experimental_vector_reduce_and: 105 case Intrinsic::experimental_vector_reduce_or: 106 case Intrinsic::experimental_vector_reduce_xor: 107 case Intrinsic::experimental_vector_reduce_smax: 108 case Intrinsic::experimental_vector_reduce_smin: 109 case Intrinsic::experimental_vector_reduce_umax: 110 case Intrinsic::experimental_vector_reduce_umin: 111 case Intrinsic::experimental_vector_reduce_fmax: 112 case Intrinsic::experimental_vector_reduce_fmin: 113 Vec = II->getArgOperand(0); 114 MRK = getMRK(ID); 115 break; 116 default: 117 continue; 118 } 119 if (!TTI->shouldExpandReduction(II)) 120 continue; 121 auto Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK); 122 II->replaceAllUsesWith(Rdx); 123 II->eraseFromParent(); 124 Changed = true; 125 } 126 return Changed; 127 } 128 129 class ExpandReductions : public FunctionPass { 130 public: 131 static char ID; 132 ExpandReductions() : FunctionPass(ID) { 133 initializeExpandReductionsPass(*PassRegistry::getPassRegistry()); 134 } 135 136 bool runOnFunction(Function &F) override { 137 const auto *TTI =&getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); 138 return expandReductions(F, TTI); 139 } 140 141 void getAnalysisUsage(AnalysisUsage &AU) const override { 142 AU.addRequired<TargetTransformInfoWrapperPass>(); 143 AU.setPreservesCFG(); 144 } 145 }; 146 } 147 148 char ExpandReductions::ID; 149 INITIALIZE_PASS_BEGIN(ExpandReductions, "expand-reductions", 150 "Expand reduction intrinsics", false, false) 151 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) 152 INITIALIZE_PASS_END(ExpandReductions, "expand-reductions", 153 "Expand reduction intrinsics", false, false) 154 155 FunctionPass *llvm::createExpandReductionsPass() { 156 return new ExpandReductions(); 157 } 158 159 PreservedAnalyses ExpandReductionsPass::run(Function &F, 160 FunctionAnalysisManager &AM) { 161 const auto &TTI = AM.getResult<TargetIRAnalysis>(F); 162 if (!expandReductions(F, &TTI)) 163 return PreservedAnalyses::all(); 164 PreservedAnalyses PA; 165 PA.preserveSet<CFGAnalyses>(); 166 return PA; 167 } 168