1 //===- NVVMReflect.cpp - NVVM Emulate conditional compilation -------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass replaces occurrences of __nvvm_reflect("foo") and llvm.nvvm.reflect 10 // with an integer. 11 // 12 // We choose the value we use by looking at metadata in the module itself. Note 13 // that we intentionally only have one way to choose these values, because other 14 // parts of LLVM (particularly, InstCombineCall) rely on being able to predict 15 // the values chosen by this pass. 16 // 17 // If we see an unknown string, we replace its call with 0. 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "NVPTX.h" 22 #include "llvm/ADT/SmallVector.h" 23 #include "llvm/Analysis/ConstantFolding.h" 24 #include "llvm/CodeGen/CommandFlags.h" 25 #include "llvm/IR/Constants.h" 26 #include "llvm/IR/DerivedTypes.h" 27 #include "llvm/IR/Function.h" 28 #include "llvm/IR/InstIterator.h" 29 #include "llvm/IR/Instructions.h" 30 #include "llvm/IR/Intrinsics.h" 31 #include "llvm/IR/IntrinsicsNVPTX.h" 32 #include "llvm/IR/Module.h" 33 #include "llvm/IR/PassManager.h" 34 #include "llvm/IR/Type.h" 35 #include "llvm/Pass.h" 36 #include "llvm/Support/CommandLine.h" 37 #include "llvm/Support/Debug.h" 38 #include "llvm/Support/raw_ostream.h" 39 #include "llvm/Transforms/Scalar.h" 40 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 41 #include "llvm/Transforms/Utils/Local.h" 42 #include <algorithm> 43 #define NVVM_REFLECT_FUNCTION "__nvvm_reflect" 44 #define NVVM_REFLECT_OCL_FUNCTION "__nvvm_reflect_ocl" 45 46 using namespace llvm; 47 48 #define DEBUG_TYPE "nvptx-reflect" 49 50 namespace llvm { void initializeNVVMReflectPass(PassRegistry &); } 51 52 namespace { 53 class NVVMReflect : public FunctionPass { 54 public: 55 static char ID; 56 unsigned int SmVersion; 57 NVVMReflect() : NVVMReflect(0) {} 58 explicit NVVMReflect(unsigned int Sm) : FunctionPass(ID), SmVersion(Sm) { 59 initializeNVVMReflectPass(*PassRegistry::getPassRegistry()); 60 } 61 62 bool runOnFunction(Function &) override; 63 }; 64 } 65 66 FunctionPass *llvm::createNVVMReflectPass(unsigned int SmVersion) { 67 return new NVVMReflect(SmVersion); 68 } 69 70 static cl::opt<bool> 71 NVVMReflectEnabled("nvvm-reflect-enable", cl::init(true), cl::Hidden, 72 cl::desc("NVVM reflection, enabled by default")); 73 74 char NVVMReflect::ID = 0; 75 INITIALIZE_PASS(NVVMReflect, "nvvm-reflect", 76 "Replace occurrences of __nvvm_reflect() calls with 0/1", false, 77 false) 78 79 static bool runNVVMReflect(Function &F, unsigned SmVersion) { 80 if (!NVVMReflectEnabled) 81 return false; 82 83 if (F.getName() == NVVM_REFLECT_FUNCTION || 84 F.getName() == NVVM_REFLECT_OCL_FUNCTION) { 85 assert(F.isDeclaration() && "_reflect function should not have a body"); 86 assert(F.getReturnType()->isIntegerTy() && 87 "_reflect's return type should be integer"); 88 return false; 89 } 90 91 SmallVector<Instruction *, 4> ToRemove; 92 SmallVector<Instruction *, 4> ToSimplify; 93 94 // Go through the calls in this function. Each call to __nvvm_reflect or 95 // llvm.nvvm.reflect should be a CallInst with a ConstantArray argument. 96 // First validate that. If the c-string corresponding to the ConstantArray can 97 // be found successfully, see if it can be found in VarMap. If so, replace the 98 // uses of CallInst with the value found in VarMap. If not, replace the use 99 // with value 0. 100 101 // The IR for __nvvm_reflect calls differs between CUDA versions. 102 // 103 // CUDA 6.5 and earlier uses this sequence: 104 // %ptr = tail call i8* @llvm.nvvm.ptr.constant.to.gen.p0i8.p4i8 105 // (i8 addrspace(4)* getelementptr inbounds 106 // ([8 x i8], [8 x i8] addrspace(4)* @str, i32 0, i32 0)) 107 // %reflect = tail call i32 @__nvvm_reflect(i8* %ptr) 108 // 109 // The value returned by Sym->getOperand(0) is a Constant with a 110 // ConstantDataSequential operand which can be converted to string and used 111 // for lookup. 112 // 113 // CUDA 7.0 does it slightly differently: 114 // %reflect = call i32 @__nvvm_reflect(i8* addrspacecast 115 // (i8 addrspace(1)* getelementptr inbounds 116 // ([8 x i8], [8 x i8] addrspace(1)* @str, i32 0, i32 0) to i8*)) 117 // 118 // In this case, we get a Constant with a GlobalVariable operand and we need 119 // to dig deeper to find its initializer with the string we'll use for lookup. 120 for (Instruction &I : instructions(F)) { 121 CallInst *Call = dyn_cast<CallInst>(&I); 122 if (!Call) 123 continue; 124 Function *Callee = Call->getCalledFunction(); 125 if (!Callee || (Callee->getName() != NVVM_REFLECT_FUNCTION && 126 Callee->getName() != NVVM_REFLECT_OCL_FUNCTION && 127 Callee->getIntrinsicID() != Intrinsic::nvvm_reflect)) 128 continue; 129 130 // FIXME: Improve error handling here and elsewhere in this pass. 131 assert(Call->getNumOperands() == 2 && 132 "Wrong number of operands to __nvvm_reflect function"); 133 134 // In cuda 6.5 and earlier, we will have an extra constant-to-generic 135 // conversion of the string. 136 const Value *Str = Call->getArgOperand(0); 137 if (const CallInst *ConvCall = dyn_cast<CallInst>(Str)) { 138 // FIXME: Add assertions about ConvCall. 139 Str = ConvCall->getArgOperand(0); 140 } 141 // Pre opaque pointers we have a constant expression wrapping the constant 142 // string. 143 Str = Str->stripPointerCasts(); 144 assert(isa<Constant>(Str) && 145 "Format of __nvvm_reflect function not recognized"); 146 147 const Value *Operand = cast<Constant>(Str)->getOperand(0); 148 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Operand)) { 149 // For CUDA-7.0 style __nvvm_reflect calls, we need to find the operand's 150 // initializer. 151 assert(GV->hasInitializer() && 152 "Format of _reflect function not recognized"); 153 const Constant *Initializer = GV->getInitializer(); 154 Operand = Initializer; 155 } 156 157 assert(isa<ConstantDataSequential>(Operand) && 158 "Format of _reflect function not recognized"); 159 assert(cast<ConstantDataSequential>(Operand)->isCString() && 160 "Format of _reflect function not recognized"); 161 162 StringRef ReflectArg = cast<ConstantDataSequential>(Operand)->getAsString(); 163 ReflectArg = ReflectArg.substr(0, ReflectArg.size() - 1); 164 LLVM_DEBUG(dbgs() << "Arg of _reflect : " << ReflectArg << "\n"); 165 166 int ReflectVal = 0; // The default value is 0 167 if (ReflectArg == "__CUDA_FTZ") { 168 // Try to pull __CUDA_FTZ from the nvvm-reflect-ftz module flag. Our 169 // choice here must be kept in sync with AutoUpgrade, which uses the same 170 // technique to detect whether ftz is enabled. 171 if (auto *Flag = mdconst::extract_or_null<ConstantInt>( 172 F.getParent()->getModuleFlag("nvvm-reflect-ftz"))) 173 ReflectVal = Flag->getSExtValue(); 174 } else if (ReflectArg == "__CUDA_ARCH") { 175 ReflectVal = SmVersion * 10; 176 } 177 178 // If the immediate user is a simple comparison we want to simplify it. 179 for (User *U : Call->users()) 180 if (Instruction *I = dyn_cast<Instruction>(U)) 181 ToSimplify.push_back(I); 182 183 Call->replaceAllUsesWith(ConstantInt::get(Call->getType(), ReflectVal)); 184 ToRemove.push_back(Call); 185 } 186 187 // The code guarded by __nvvm_reflect may be invalid for the target machine. 188 // Traverse the use-def chain, continually simplifying constant expressions 189 // until we find a terminator that we can then remove. 190 while (!ToSimplify.empty()) { 191 Instruction *I = ToSimplify.pop_back_val(); 192 if (Constant *C = 193 ConstantFoldInstruction(I, F.getDataLayout())) { 194 for (User *U : I->users()) 195 if (Instruction *I = dyn_cast<Instruction>(U)) 196 ToSimplify.push_back(I); 197 198 I->replaceAllUsesWith(C); 199 if (isInstructionTriviallyDead(I)) { 200 ToRemove.push_back(I); 201 } 202 } else if (I->isTerminator()) { 203 ConstantFoldTerminator(I->getParent()); 204 } 205 } 206 207 // Removing via isInstructionTriviallyDead may add duplicates to the ToRemove 208 // array. Filter out the duplicates before starting to erase from parent. 209 std::sort(ToRemove.begin(), ToRemove.end()); 210 auto NewLastIter = llvm::unique(ToRemove); 211 ToRemove.erase(NewLastIter, ToRemove.end()); 212 213 for (Instruction *I : ToRemove) 214 I->eraseFromParent(); 215 216 return ToRemove.size() > 0; 217 } 218 219 bool NVVMReflect::runOnFunction(Function &F) { 220 return runNVVMReflect(F, SmVersion); 221 } 222 223 NVVMReflectPass::NVVMReflectPass() { 224 // Get the CPU string from the command line if not provided. 225 std::string MCPU = codegen::getMCPU(); 226 StringRef SM = MCPU; 227 if (!SM.consume_front("sm_") || SM.consumeInteger(10, SmVersion)) 228 SmVersion = 0; 229 } 230 231 PreservedAnalyses NVVMReflectPass::run(Function &F, 232 FunctionAnalysisManager &AM) { 233 return runNVVMReflect(F, SmVersion) ? PreservedAnalyses::none() 234 : PreservedAnalyses::all(); 235 } 236