1 //===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass implements IR lowering for the llvm.memcpy, llvm.memmove, 10 // llvm.memset, llvm.load.relative and llvm.objc.* intrinsics. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/CodeGen/PreISelIntrinsicLowering.h" 15 #include "llvm/Analysis/ObjCARCInstKind.h" 16 #include "llvm/Analysis/ObjCARCUtil.h" 17 #include "llvm/Analysis/TargetTransformInfo.h" 18 #include "llvm/CodeGen/Passes.h" 19 #include "llvm/CodeGen/TargetLowering.h" 20 #include "llvm/CodeGen/TargetPassConfig.h" 21 #include "llvm/IR/Function.h" 22 #include "llvm/IR/IRBuilder.h" 23 #include "llvm/IR/Instructions.h" 24 #include "llvm/IR/IntrinsicInst.h" 25 #include "llvm/IR/Module.h" 26 #include "llvm/IR/Type.h" 27 #include "llvm/InitializePasses.h" 28 #include "llvm/Pass.h" 29 #include "llvm/Support/Casting.h" 30 #include "llvm/Target/TargetMachine.h" 31 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h" 32 33 using namespace llvm; 34 35 /// Threshold to leave statically sized memory intrinsic calls. Calls of known 36 /// size larger than this will be expanded by the pass. Calls of unknown or 37 /// lower size will be left for expansion in codegen. 38 static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt( 39 "mem-intrinsic-expand-size", 40 cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1), 41 cl::Hidden); 42 43 namespace { 44 45 struct PreISelIntrinsicLowering { 46 const TargetMachine &TM; 47 const function_ref<TargetTransformInfo &(Function &)> LookupTTI; 48 49 /// If this is true, assume it's preferably to leave memory intrinsic calls 50 /// for replacement with a library call later. Otherwise this depends on 51 /// TargetLoweringInfo availability of the corresponding function. 52 const bool UseMemIntrinsicLibFunc; 53 54 explicit PreISelIntrinsicLowering( 55 const TargetMachine &TM_, 56 function_ref<TargetTransformInfo &(Function &)> LookupTTI_, 57 bool UseMemIntrinsicLibFunc_ = true) 58 : TM(TM_), LookupTTI(LookupTTI_), 59 UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {} 60 61 static bool shouldExpandMemIntrinsicWithSize(Value *Size, 62 const TargetTransformInfo &TTI); 63 bool expandMemIntrinsicUses(Function &F) const; 64 bool lowerIntrinsics(Module &M) const; 65 }; 66 67 } // namespace 68 69 static bool lowerLoadRelative(Function &F) { 70 if (F.use_empty()) 71 return false; 72 73 bool Changed = false; 74 Type *Int32Ty = Type::getInt32Ty(F.getContext()); 75 Type *Int8Ty = Type::getInt8Ty(F.getContext()); 76 77 for (Use &U : llvm::make_early_inc_range(F.uses())) { 78 auto CI = dyn_cast<CallInst>(U.getUser()); 79 if (!CI || CI->getCalledOperand() != &F) 80 continue; 81 82 IRBuilder<> B(CI); 83 Value *OffsetPtr = 84 B.CreateGEP(Int8Ty, CI->getArgOperand(0), CI->getArgOperand(1)); 85 Value *OffsetI32 = B.CreateAlignedLoad(Int32Ty, OffsetPtr, Align(4)); 86 87 Value *ResultPtr = B.CreateGEP(Int8Ty, CI->getArgOperand(0), OffsetI32); 88 89 CI->replaceAllUsesWith(ResultPtr); 90 CI->eraseFromParent(); 91 Changed = true; 92 } 93 94 return Changed; 95 } 96 97 // ObjCARC has knowledge about whether an obj-c runtime function needs to be 98 // always tail-called or never tail-called. 99 static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) { 100 objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(&F); 101 if (objcarc::IsAlwaysTail(Kind)) 102 return CallInst::TCK_Tail; 103 else if (objcarc::IsNeverTail(Kind)) 104 return CallInst::TCK_NoTail; 105 return CallInst::TCK_None; 106 } 107 108 static bool lowerObjCCall(Function &F, const char *NewFn, 109 bool setNonLazyBind = false) { 110 assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) && 111 "Pre-ISel intrinsics do lower into regular function calls"); 112 if (F.use_empty()) 113 return false; 114 115 // If we haven't already looked up this function, check to see if the 116 // program already contains a function with this name. 117 Module *M = F.getParent(); 118 FunctionCallee FCache = M->getOrInsertFunction(NewFn, F.getFunctionType()); 119 120 if (Function *Fn = dyn_cast<Function>(FCache.getCallee())) { 121 Fn->setLinkage(F.getLinkage()); 122 if (setNonLazyBind && !Fn->isWeakForLinker()) { 123 // If we have Native ARC, set nonlazybind attribute for these APIs for 124 // performance. 125 Fn->addFnAttr(Attribute::NonLazyBind); 126 } 127 } 128 129 CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F); 130 131 for (Use &U : llvm::make_early_inc_range(F.uses())) { 132 auto *CB = cast<CallBase>(U.getUser()); 133 134 if (CB->getCalledFunction() != &F) { 135 objcarc::ARCInstKind Kind = objcarc::getAttachedARCFunctionKind(CB); 136 (void)Kind; 137 assert((Kind == objcarc::ARCInstKind::RetainRV || 138 Kind == objcarc::ARCInstKind::UnsafeClaimRV) && 139 "use expected to be the argument of operand bundle " 140 "\"clang.arc.attachedcall\""); 141 U.set(FCache.getCallee()); 142 continue; 143 } 144 145 auto *CI = cast<CallInst>(CB); 146 assert(CI->getCalledFunction() && "Cannot lower an indirect call!"); 147 148 IRBuilder<> Builder(CI->getParent(), CI->getIterator()); 149 SmallVector<Value *, 8> Args(CI->args()); 150 SmallVector<llvm::OperandBundleDef, 1> BundleList; 151 CI->getOperandBundlesAsDefs(BundleList); 152 CallInst *NewCI = Builder.CreateCall(FCache, Args, BundleList); 153 NewCI->setName(CI->getName()); 154 155 // Try to set the most appropriate TailCallKind based on both the current 156 // attributes and the ones that we could get from ObjCARC's special 157 // knowledge of the runtime functions. 158 // 159 // std::max respects both requirements of notail and tail here: 160 // * notail on either the call or from ObjCARC becomes notail 161 // * tail on either side is stronger than none, but not notail 162 CallInst::TailCallKind TCK = CI->getTailCallKind(); 163 NewCI->setTailCallKind(std::max(TCK, OverridingTCK)); 164 165 // Transfer the 'returned' attribute from the intrinsic to the call site. 166 // By applying this only to intrinsic call sites, we avoid applying it to 167 // non-ARC explicit calls to things like objc_retain which have not been 168 // auto-upgraded to use the intrinsics. 169 unsigned Index; 170 if (F.getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) && 171 Index) 172 NewCI->addParamAttr(Index - AttributeList::FirstArgIndex, 173 Attribute::Returned); 174 175 if (!CI->use_empty()) 176 CI->replaceAllUsesWith(NewCI); 177 CI->eraseFromParent(); 178 } 179 180 return true; 181 } 182 183 // TODO: Should refine based on estimated number of accesses (e.g. does it 184 // require splitting based on alignment) 185 bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize( 186 Value *Size, const TargetTransformInfo &TTI) { 187 ConstantInt *CI = dyn_cast<ConstantInt>(Size); 188 if (!CI) 189 return true; 190 uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences() 191 ? MemIntrinsicExpandSizeThresholdOpt 192 : TTI.getMaxMemIntrinsicInlineSizeThreshold(); 193 uint64_t SizeVal = CI->getZExtValue(); 194 195 // Treat a threshold of 0 as a special case to force expansion of all 196 // intrinsics, including size 0. 197 return SizeVal > Threshold || Threshold == 0; 198 } 199 200 static bool canEmitLibcall(const TargetMachine &TM, Function *F, 201 RTLIB::Libcall LC) { 202 // TODO: Should this consider the address space of the memcpy? 203 const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering(); 204 return TLI->getLibcallName(LC) != nullptr; 205 } 206 207 // TODO: Handle atomic memcpy and memcpy.inline 208 // TODO: Pass ScalarEvolution 209 bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const { 210 Intrinsic::ID ID = F.getIntrinsicID(); 211 bool Changed = false; 212 213 for (User *U : llvm::make_early_inc_range(F.users())) { 214 Instruction *Inst = cast<Instruction>(U); 215 216 switch (ID) { 217 case Intrinsic::memcpy: { 218 auto *Memcpy = cast<MemCpyInst>(Inst); 219 Function *ParentFunc = Memcpy->getFunction(); 220 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); 221 if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) { 222 if (UseMemIntrinsicLibFunc && 223 canEmitLibcall(TM, ParentFunc, RTLIB::MEMCPY)) 224 break; 225 226 // TODO: For optsize, emit the loop into a separate function 227 expandMemCpyAsLoop(Memcpy, TTI); 228 Changed = true; 229 Memcpy->eraseFromParent(); 230 } 231 232 break; 233 } 234 case Intrinsic::memmove: { 235 auto *Memmove = cast<MemMoveInst>(Inst); 236 Function *ParentFunc = Memmove->getFunction(); 237 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); 238 if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) { 239 if (UseMemIntrinsicLibFunc && 240 canEmitLibcall(TM, ParentFunc, RTLIB::MEMMOVE)) 241 break; 242 243 if (expandMemMoveAsLoop(Memmove, TTI)) { 244 Changed = true; 245 Memmove->eraseFromParent(); 246 } 247 } 248 249 break; 250 } 251 case Intrinsic::memset: { 252 auto *Memset = cast<MemSetInst>(Inst); 253 Function *ParentFunc = Memset->getFunction(); 254 const TargetTransformInfo &TTI = LookupTTI(*ParentFunc); 255 if (shouldExpandMemIntrinsicWithSize(Memset->getLength(), TTI)) { 256 if (UseMemIntrinsicLibFunc && 257 canEmitLibcall(TM, ParentFunc, RTLIB::MEMSET)) 258 break; 259 260 expandMemSetAsLoop(Memset); 261 Changed = true; 262 Memset->eraseFromParent(); 263 } 264 265 break; 266 } 267 default: 268 llvm_unreachable("unhandled intrinsic"); 269 } 270 } 271 272 return Changed; 273 } 274 275 bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const { 276 bool Changed = false; 277 for (Function &F : M) { 278 switch (F.getIntrinsicID()) { 279 default: 280 break; 281 case Intrinsic::memcpy: 282 case Intrinsic::memmove: 283 case Intrinsic::memset: 284 Changed |= expandMemIntrinsicUses(F); 285 break; 286 case Intrinsic::load_relative: 287 Changed |= lowerLoadRelative(F); 288 break; 289 case Intrinsic::objc_autorelease: 290 Changed |= lowerObjCCall(F, "objc_autorelease"); 291 break; 292 case Intrinsic::objc_autoreleasePoolPop: 293 Changed |= lowerObjCCall(F, "objc_autoreleasePoolPop"); 294 break; 295 case Intrinsic::objc_autoreleasePoolPush: 296 Changed |= lowerObjCCall(F, "objc_autoreleasePoolPush"); 297 break; 298 case Intrinsic::objc_autoreleaseReturnValue: 299 Changed |= lowerObjCCall(F, "objc_autoreleaseReturnValue"); 300 break; 301 case Intrinsic::objc_copyWeak: 302 Changed |= lowerObjCCall(F, "objc_copyWeak"); 303 break; 304 case Intrinsic::objc_destroyWeak: 305 Changed |= lowerObjCCall(F, "objc_destroyWeak"); 306 break; 307 case Intrinsic::objc_initWeak: 308 Changed |= lowerObjCCall(F, "objc_initWeak"); 309 break; 310 case Intrinsic::objc_loadWeak: 311 Changed |= lowerObjCCall(F, "objc_loadWeak"); 312 break; 313 case Intrinsic::objc_loadWeakRetained: 314 Changed |= lowerObjCCall(F, "objc_loadWeakRetained"); 315 break; 316 case Intrinsic::objc_moveWeak: 317 Changed |= lowerObjCCall(F, "objc_moveWeak"); 318 break; 319 case Intrinsic::objc_release: 320 Changed |= lowerObjCCall(F, "objc_release", true); 321 break; 322 case Intrinsic::objc_retain: 323 Changed |= lowerObjCCall(F, "objc_retain", true); 324 break; 325 case Intrinsic::objc_retainAutorelease: 326 Changed |= lowerObjCCall(F, "objc_retainAutorelease"); 327 break; 328 case Intrinsic::objc_retainAutoreleaseReturnValue: 329 Changed |= lowerObjCCall(F, "objc_retainAutoreleaseReturnValue"); 330 break; 331 case Intrinsic::objc_retainAutoreleasedReturnValue: 332 Changed |= lowerObjCCall(F, "objc_retainAutoreleasedReturnValue"); 333 break; 334 case Intrinsic::objc_retainBlock: 335 Changed |= lowerObjCCall(F, "objc_retainBlock"); 336 break; 337 case Intrinsic::objc_storeStrong: 338 Changed |= lowerObjCCall(F, "objc_storeStrong"); 339 break; 340 case Intrinsic::objc_storeWeak: 341 Changed |= lowerObjCCall(F, "objc_storeWeak"); 342 break; 343 case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue: 344 Changed |= lowerObjCCall(F, "objc_unsafeClaimAutoreleasedReturnValue"); 345 break; 346 case Intrinsic::objc_retainedObject: 347 Changed |= lowerObjCCall(F, "objc_retainedObject"); 348 break; 349 case Intrinsic::objc_unretainedObject: 350 Changed |= lowerObjCCall(F, "objc_unretainedObject"); 351 break; 352 case Intrinsic::objc_unretainedPointer: 353 Changed |= lowerObjCCall(F, "objc_unretainedPointer"); 354 break; 355 case Intrinsic::objc_retain_autorelease: 356 Changed |= lowerObjCCall(F, "objc_retain_autorelease"); 357 break; 358 case Intrinsic::objc_sync_enter: 359 Changed |= lowerObjCCall(F, "objc_sync_enter"); 360 break; 361 case Intrinsic::objc_sync_exit: 362 Changed |= lowerObjCCall(F, "objc_sync_exit"); 363 break; 364 } 365 } 366 return Changed; 367 } 368 369 namespace { 370 371 class PreISelIntrinsicLoweringLegacyPass : public ModulePass { 372 public: 373 static char ID; 374 375 PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {} 376 377 void getAnalysisUsage(AnalysisUsage &AU) const override { 378 AU.addRequired<TargetTransformInfoWrapperPass>(); 379 AU.addRequired<TargetPassConfig>(); 380 } 381 382 bool runOnModule(Module &M) override { 383 auto LookupTTI = [this](Function &F) -> TargetTransformInfo & { 384 return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); 385 }; 386 387 const auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>(); 388 PreISelIntrinsicLowering Lowering(TM, LookupTTI); 389 return Lowering.lowerIntrinsics(M); 390 } 391 }; 392 393 } // end anonymous namespace 394 395 char PreISelIntrinsicLoweringLegacyPass::ID; 396 397 INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass, 398 "pre-isel-intrinsic-lowering", 399 "Pre-ISel Intrinsic Lowering", false, false) 400 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) 401 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) 402 INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass, 403 "pre-isel-intrinsic-lowering", 404 "Pre-ISel Intrinsic Lowering", false, false) 405 406 ModulePass *llvm::createPreISelIntrinsicLoweringPass() { 407 return new PreISelIntrinsicLoweringLegacyPass(); 408 } 409 410 PreservedAnalyses PreISelIntrinsicLoweringPass::run(Module &M, 411 ModuleAnalysisManager &AM) { 412 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 413 414 auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & { 415 return FAM.getResult<TargetIRAnalysis>(F); 416 }; 417 418 PreISelIntrinsicLowering Lowering(TM, LookupTTI); 419 if (!Lowering.lowerIntrinsics(M)) 420 return PreservedAnalyses::all(); 421 else 422 return PreservedAnalyses::none(); 423 } 424