1 //===- AMDGPULibCalls.cpp -------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file does AMD library function optimizations. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPU.h" 15 #include "AMDGPULibFunc.h" 16 #include "GCNSubtarget.h" 17 #include "llvm/Analysis/AliasAnalysis.h" 18 #include "llvm/Analysis/Loads.h" 19 #include "llvm/IR/IRBuilder.h" 20 #include "llvm/IR/IntrinsicInst.h" 21 #include "llvm/IR/IntrinsicsAMDGPU.h" 22 #include "llvm/InitializePasses.h" 23 #include "llvm/Target/TargetMachine.h" 24 #include <cmath> 25 26 #define DEBUG_TYPE "amdgpu-simplifylib" 27 28 using namespace llvm; 29 30 static cl::opt<bool> EnablePreLink("amdgpu-prelink", 31 cl::desc("Enable pre-link mode optimizations"), 32 cl::init(false), 33 cl::Hidden); 34 35 static cl::list<std::string> UseNative("amdgpu-use-native", 36 cl::desc("Comma separated list of functions to replace with native, or all"), 37 cl::CommaSeparated, cl::ValueOptional, 38 cl::Hidden); 39 40 #define MATH_PI numbers::pi 41 #define MATH_E numbers::e 42 #define MATH_SQRT2 numbers::sqrt2 43 #define MATH_SQRT1_2 numbers::inv_sqrt2 44 45 namespace llvm { 46 47 class AMDGPULibCalls { 48 private: 49 50 typedef llvm::AMDGPULibFunc FuncInfo; 51 52 const TargetMachine *TM; 53 54 // -fuse-native. 55 bool AllNative = false; 56 57 bool useNativeFunc(const StringRef F) const; 58 59 // Return a pointer (pointer expr) to the function if function definition with 60 // "FuncName" exists. It may create a new function prototype in pre-link mode. 61 FunctionCallee getFunction(Module *M, const FuncInfo &fInfo); 62 63 bool parseFunctionName(const StringRef &FMangledName, FuncInfo &FInfo); 64 65 bool TDOFold(CallInst *CI, const FuncInfo &FInfo); 66 67 /* Specialized optimizations */ 68 69 // recip (half or native) 70 bool fold_recip(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 71 72 // divide (half or native) 73 bool fold_divide(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 74 75 // pow/powr/pown 76 bool fold_pow(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 77 78 // rootn 79 bool fold_rootn(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 80 81 // fma/mad 82 bool fold_fma_mad(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 83 84 // -fuse-native for sincos 85 bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo); 86 87 // evaluate calls if calls' arguments are constants. 88 bool evaluateScalarMathFunc(const FuncInfo &FInfo, double& Res0, 89 double& Res1, Constant *copr0, Constant *copr1, Constant *copr2); 90 bool evaluateCall(CallInst *aCI, const FuncInfo &FInfo); 91 92 // sqrt 93 bool fold_sqrt(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 94 95 // sin/cos 96 bool fold_sincos(CallInst * CI, IRBuilder<> &B, AliasAnalysis * AA); 97 98 // __read_pipe/__write_pipe 99 bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, 100 const FuncInfo &FInfo); 101 102 // llvm.amdgcn.wavefrontsize 103 bool fold_wavefrontsize(CallInst *CI, IRBuilder<> &B); 104 105 // Get insertion point at entry. 106 BasicBlock::iterator getEntryIns(CallInst * UI); 107 // Insert an Alloc instruction. 108 AllocaInst* insertAlloca(CallInst * UI, IRBuilder<> &B, const char *prefix); 109 // Get a scalar native builtin single argument FP function 110 FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo); 111 112 protected: 113 CallInst *CI; 114 115 bool isUnsafeMath(const CallInst *CI) const; 116 117 void replaceCall(Value *With) { 118 CI->replaceAllUsesWith(With); 119 CI->eraseFromParent(); 120 } 121 122 public: 123 AMDGPULibCalls(const TargetMachine *TM_ = nullptr) : TM(TM_) {} 124 125 bool fold(CallInst *CI, AliasAnalysis *AA = nullptr); 126 127 void initNativeFuncs(); 128 129 // Replace a normal math function call with that native version 130 bool useNative(CallInst *CI); 131 }; 132 133 } // end llvm namespace 134 135 namespace { 136 137 class AMDGPUSimplifyLibCalls : public FunctionPass { 138 139 AMDGPULibCalls Simplifier; 140 141 public: 142 static char ID; // Pass identification 143 144 AMDGPUSimplifyLibCalls(const TargetMachine *TM = nullptr) 145 : FunctionPass(ID), Simplifier(TM) { 146 initializeAMDGPUSimplifyLibCallsPass(*PassRegistry::getPassRegistry()); 147 } 148 149 void getAnalysisUsage(AnalysisUsage &AU) const override { 150 AU.addRequired<AAResultsWrapperPass>(); 151 } 152 153 bool runOnFunction(Function &M) override; 154 }; 155 156 class AMDGPUUseNativeCalls : public FunctionPass { 157 158 AMDGPULibCalls Simplifier; 159 160 public: 161 static char ID; // Pass identification 162 163 AMDGPUUseNativeCalls() : FunctionPass(ID) { 164 initializeAMDGPUUseNativeCallsPass(*PassRegistry::getPassRegistry()); 165 Simplifier.initNativeFuncs(); 166 } 167 168 bool runOnFunction(Function &F) override; 169 }; 170 171 } // end anonymous namespace. 172 173 char AMDGPUSimplifyLibCalls::ID = 0; 174 char AMDGPUUseNativeCalls::ID = 0; 175 176 INITIALIZE_PASS_BEGIN(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", 177 "Simplify well-known AMD library calls", false, false) 178 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 179 INITIALIZE_PASS_END(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", 180 "Simplify well-known AMD library calls", false, false) 181 182 INITIALIZE_PASS(AMDGPUUseNativeCalls, "amdgpu-usenative", 183 "Replace builtin math calls with that native versions.", 184 false, false) 185 186 template <typename IRB> 187 static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg, 188 const Twine &Name = "") { 189 CallInst *R = B.CreateCall(Callee, Arg, Name); 190 if (Function *F = dyn_cast<Function>(Callee.getCallee())) 191 R->setCallingConv(F->getCallingConv()); 192 return R; 193 } 194 195 template <typename IRB> 196 static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1, 197 Value *Arg2, const Twine &Name = "") { 198 CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name); 199 if (Function *F = dyn_cast<Function>(Callee.getCallee())) 200 R->setCallingConv(F->getCallingConv()); 201 return R; 202 } 203 204 // Data structures for table-driven optimizations. 205 // FuncTbl works for both f32 and f64 functions with 1 input argument 206 207 struct TableEntry { 208 double result; 209 double input; 210 }; 211 212 /* a list of {result, input} */ 213 static const TableEntry tbl_acos[] = { 214 {MATH_PI / 2.0, 0.0}, 215 {MATH_PI / 2.0, -0.0}, 216 {0.0, 1.0}, 217 {MATH_PI, -1.0} 218 }; 219 static const TableEntry tbl_acosh[] = { 220 {0.0, 1.0} 221 }; 222 static const TableEntry tbl_acospi[] = { 223 {0.5, 0.0}, 224 {0.5, -0.0}, 225 {0.0, 1.0}, 226 {1.0, -1.0} 227 }; 228 static const TableEntry tbl_asin[] = { 229 {0.0, 0.0}, 230 {-0.0, -0.0}, 231 {MATH_PI / 2.0, 1.0}, 232 {-MATH_PI / 2.0, -1.0} 233 }; 234 static const TableEntry tbl_asinh[] = { 235 {0.0, 0.0}, 236 {-0.0, -0.0} 237 }; 238 static const TableEntry tbl_asinpi[] = { 239 {0.0, 0.0}, 240 {-0.0, -0.0}, 241 {0.5, 1.0}, 242 {-0.5, -1.0} 243 }; 244 static const TableEntry tbl_atan[] = { 245 {0.0, 0.0}, 246 {-0.0, -0.0}, 247 {MATH_PI / 4.0, 1.0}, 248 {-MATH_PI / 4.0, -1.0} 249 }; 250 static const TableEntry tbl_atanh[] = { 251 {0.0, 0.0}, 252 {-0.0, -0.0} 253 }; 254 static const TableEntry tbl_atanpi[] = { 255 {0.0, 0.0}, 256 {-0.0, -0.0}, 257 {0.25, 1.0}, 258 {-0.25, -1.0} 259 }; 260 static const TableEntry tbl_cbrt[] = { 261 {0.0, 0.0}, 262 {-0.0, -0.0}, 263 {1.0, 1.0}, 264 {-1.0, -1.0}, 265 }; 266 static const TableEntry tbl_cos[] = { 267 {1.0, 0.0}, 268 {1.0, -0.0} 269 }; 270 static const TableEntry tbl_cosh[] = { 271 {1.0, 0.0}, 272 {1.0, -0.0} 273 }; 274 static const TableEntry tbl_cospi[] = { 275 {1.0, 0.0}, 276 {1.0, -0.0} 277 }; 278 static const TableEntry tbl_erfc[] = { 279 {1.0, 0.0}, 280 {1.0, -0.0} 281 }; 282 static const TableEntry tbl_erf[] = { 283 {0.0, 0.0}, 284 {-0.0, -0.0} 285 }; 286 static const TableEntry tbl_exp[] = { 287 {1.0, 0.0}, 288 {1.0, -0.0}, 289 {MATH_E, 1.0} 290 }; 291 static const TableEntry tbl_exp2[] = { 292 {1.0, 0.0}, 293 {1.0, -0.0}, 294 {2.0, 1.0} 295 }; 296 static const TableEntry tbl_exp10[] = { 297 {1.0, 0.0}, 298 {1.0, -0.0}, 299 {10.0, 1.0} 300 }; 301 static const TableEntry tbl_expm1[] = { 302 {0.0, 0.0}, 303 {-0.0, -0.0} 304 }; 305 static const TableEntry tbl_log[] = { 306 {0.0, 1.0}, 307 {1.0, MATH_E} 308 }; 309 static const TableEntry tbl_log2[] = { 310 {0.0, 1.0}, 311 {1.0, 2.0} 312 }; 313 static const TableEntry tbl_log10[] = { 314 {0.0, 1.0}, 315 {1.0, 10.0} 316 }; 317 static const TableEntry tbl_rsqrt[] = { 318 {1.0, 1.0}, 319 {MATH_SQRT1_2, 2.0} 320 }; 321 static const TableEntry tbl_sin[] = { 322 {0.0, 0.0}, 323 {-0.0, -0.0} 324 }; 325 static const TableEntry tbl_sinh[] = { 326 {0.0, 0.0}, 327 {-0.0, -0.0} 328 }; 329 static const TableEntry tbl_sinpi[] = { 330 {0.0, 0.0}, 331 {-0.0, -0.0} 332 }; 333 static const TableEntry tbl_sqrt[] = { 334 {0.0, 0.0}, 335 {1.0, 1.0}, 336 {MATH_SQRT2, 2.0} 337 }; 338 static const TableEntry tbl_tan[] = { 339 {0.0, 0.0}, 340 {-0.0, -0.0} 341 }; 342 static const TableEntry tbl_tanh[] = { 343 {0.0, 0.0}, 344 {-0.0, -0.0} 345 }; 346 static const TableEntry tbl_tanpi[] = { 347 {0.0, 0.0}, 348 {-0.0, -0.0} 349 }; 350 static const TableEntry tbl_tgamma[] = { 351 {1.0, 1.0}, 352 {1.0, 2.0}, 353 {2.0, 3.0}, 354 {6.0, 4.0} 355 }; 356 357 static bool HasNative(AMDGPULibFunc::EFuncId id) { 358 switch(id) { 359 case AMDGPULibFunc::EI_DIVIDE: 360 case AMDGPULibFunc::EI_COS: 361 case AMDGPULibFunc::EI_EXP: 362 case AMDGPULibFunc::EI_EXP2: 363 case AMDGPULibFunc::EI_EXP10: 364 case AMDGPULibFunc::EI_LOG: 365 case AMDGPULibFunc::EI_LOG2: 366 case AMDGPULibFunc::EI_LOG10: 367 case AMDGPULibFunc::EI_POWR: 368 case AMDGPULibFunc::EI_RECIP: 369 case AMDGPULibFunc::EI_RSQRT: 370 case AMDGPULibFunc::EI_SIN: 371 case AMDGPULibFunc::EI_SINCOS: 372 case AMDGPULibFunc::EI_SQRT: 373 case AMDGPULibFunc::EI_TAN: 374 return true; 375 default:; 376 } 377 return false; 378 } 379 380 using TableRef = ArrayRef<TableEntry>; 381 382 static TableRef getOptTable(AMDGPULibFunc::EFuncId id) { 383 switch(id) { 384 case AMDGPULibFunc::EI_ACOS: return TableRef(tbl_acos); 385 case AMDGPULibFunc::EI_ACOSH: return TableRef(tbl_acosh); 386 case AMDGPULibFunc::EI_ACOSPI: return TableRef(tbl_acospi); 387 case AMDGPULibFunc::EI_ASIN: return TableRef(tbl_asin); 388 case AMDGPULibFunc::EI_ASINH: return TableRef(tbl_asinh); 389 case AMDGPULibFunc::EI_ASINPI: return TableRef(tbl_asinpi); 390 case AMDGPULibFunc::EI_ATAN: return TableRef(tbl_atan); 391 case AMDGPULibFunc::EI_ATANH: return TableRef(tbl_atanh); 392 case AMDGPULibFunc::EI_ATANPI: return TableRef(tbl_atanpi); 393 case AMDGPULibFunc::EI_CBRT: return TableRef(tbl_cbrt); 394 case AMDGPULibFunc::EI_NCOS: 395 case AMDGPULibFunc::EI_COS: return TableRef(tbl_cos); 396 case AMDGPULibFunc::EI_COSH: return TableRef(tbl_cosh); 397 case AMDGPULibFunc::EI_COSPI: return TableRef(tbl_cospi); 398 case AMDGPULibFunc::EI_ERFC: return TableRef(tbl_erfc); 399 case AMDGPULibFunc::EI_ERF: return TableRef(tbl_erf); 400 case AMDGPULibFunc::EI_EXP: return TableRef(tbl_exp); 401 case AMDGPULibFunc::EI_NEXP2: 402 case AMDGPULibFunc::EI_EXP2: return TableRef(tbl_exp2); 403 case AMDGPULibFunc::EI_EXP10: return TableRef(tbl_exp10); 404 case AMDGPULibFunc::EI_EXPM1: return TableRef(tbl_expm1); 405 case AMDGPULibFunc::EI_LOG: return TableRef(tbl_log); 406 case AMDGPULibFunc::EI_NLOG2: 407 case AMDGPULibFunc::EI_LOG2: return TableRef(tbl_log2); 408 case AMDGPULibFunc::EI_LOG10: return TableRef(tbl_log10); 409 case AMDGPULibFunc::EI_NRSQRT: 410 case AMDGPULibFunc::EI_RSQRT: return TableRef(tbl_rsqrt); 411 case AMDGPULibFunc::EI_NSIN: 412 case AMDGPULibFunc::EI_SIN: return TableRef(tbl_sin); 413 case AMDGPULibFunc::EI_SINH: return TableRef(tbl_sinh); 414 case AMDGPULibFunc::EI_SINPI: return TableRef(tbl_sinpi); 415 case AMDGPULibFunc::EI_NSQRT: 416 case AMDGPULibFunc::EI_SQRT: return TableRef(tbl_sqrt); 417 case AMDGPULibFunc::EI_TAN: return TableRef(tbl_tan); 418 case AMDGPULibFunc::EI_TANH: return TableRef(tbl_tanh); 419 case AMDGPULibFunc::EI_TANPI: return TableRef(tbl_tanpi); 420 case AMDGPULibFunc::EI_TGAMMA: return TableRef(tbl_tgamma); 421 default:; 422 } 423 return TableRef(); 424 } 425 426 static inline int getVecSize(const AMDGPULibFunc& FInfo) { 427 return FInfo.getLeads()[0].VectorSize; 428 } 429 430 static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) { 431 return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType; 432 } 433 434 FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) { 435 // If we are doing PreLinkOpt, the function is external. So it is safe to 436 // use getOrInsertFunction() at this stage. 437 438 return EnablePreLink ? AMDGPULibFunc::getOrInsertFunction(M, fInfo) 439 : AMDGPULibFunc::getFunction(M, fInfo); 440 } 441 442 bool AMDGPULibCalls::parseFunctionName(const StringRef &FMangledName, 443 FuncInfo &FInfo) { 444 return AMDGPULibFunc::parse(FMangledName, FInfo); 445 } 446 447 bool AMDGPULibCalls::isUnsafeMath(const CallInst *CI) const { 448 if (auto Op = dyn_cast<FPMathOperator>(CI)) 449 if (Op->isFast()) 450 return true; 451 const Function *F = CI->getParent()->getParent(); 452 Attribute Attr = F->getFnAttribute("unsafe-fp-math"); 453 return Attr.getValueAsBool(); 454 } 455 456 bool AMDGPULibCalls::useNativeFunc(const StringRef F) const { 457 return AllNative || llvm::is_contained(UseNative, F); 458 } 459 460 void AMDGPULibCalls::initNativeFuncs() { 461 AllNative = useNativeFunc("all") || 462 (UseNative.getNumOccurrences() && UseNative.size() == 1 && 463 UseNative.begin()->empty()); 464 } 465 466 bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) { 467 bool native_sin = useNativeFunc("sin"); 468 bool native_cos = useNativeFunc("cos"); 469 470 if (native_sin && native_cos) { 471 Module *M = aCI->getModule(); 472 Value *opr0 = aCI->getArgOperand(0); 473 474 AMDGPULibFunc nf; 475 nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType; 476 nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize; 477 478 nf.setPrefix(AMDGPULibFunc::NATIVE); 479 nf.setId(AMDGPULibFunc::EI_SIN); 480 FunctionCallee sinExpr = getFunction(M, nf); 481 482 nf.setPrefix(AMDGPULibFunc::NATIVE); 483 nf.setId(AMDGPULibFunc::EI_COS); 484 FunctionCallee cosExpr = getFunction(M, nf); 485 if (sinExpr && cosExpr) { 486 Value *sinval = CallInst::Create(sinExpr, opr0, "splitsin", aCI); 487 Value *cosval = CallInst::Create(cosExpr, opr0, "splitcos", aCI); 488 new StoreInst(cosval, aCI->getArgOperand(1), aCI); 489 490 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI 491 << " with native version of sin/cos"); 492 493 replaceCall(sinval); 494 return true; 495 } 496 } 497 return false; 498 } 499 500 bool AMDGPULibCalls::useNative(CallInst *aCI) { 501 CI = aCI; 502 Function *Callee = aCI->getCalledFunction(); 503 504 FuncInfo FInfo; 505 if (!parseFunctionName(Callee->getName(), FInfo) || !FInfo.isMangled() || 506 FInfo.getPrefix() != AMDGPULibFunc::NOPFX || 507 getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()) || 508 !(AllNative || useNativeFunc(FInfo.getName()))) { 509 return false; 510 } 511 512 if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS) 513 return sincosUseNative(aCI, FInfo); 514 515 FInfo.setPrefix(AMDGPULibFunc::NATIVE); 516 FunctionCallee F = getFunction(aCI->getModule(), FInfo); 517 if (!F) 518 return false; 519 520 aCI->setCalledFunction(F); 521 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI 522 << " with native version"); 523 return true; 524 } 525 526 // Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe 527 // builtin, with appended type size and alignment arguments, where 2 or 4 528 // indicates the original number of arguments. The library has optimized version 529 // of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same 530 // power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N 531 // for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ..., 532 // 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4. 533 bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, 534 const FuncInfo &FInfo) { 535 auto *Callee = CI->getCalledFunction(); 536 if (!Callee->isDeclaration()) 537 return false; 538 539 assert(Callee->hasName() && "Invalid read_pipe/write_pipe function"); 540 auto *M = Callee->getParent(); 541 std::string Name = std::string(Callee->getName()); 542 auto NumArg = CI->arg_size(); 543 if (NumArg != 4 && NumArg != 6) 544 return false; 545 auto *PacketSize = CI->getArgOperand(NumArg - 2); 546 auto *PacketAlign = CI->getArgOperand(NumArg - 1); 547 if (!isa<ConstantInt>(PacketSize) || !isa<ConstantInt>(PacketAlign)) 548 return false; 549 unsigned Size = cast<ConstantInt>(PacketSize)->getZExtValue(); 550 Align Alignment = cast<ConstantInt>(PacketAlign)->getAlignValue(); 551 if (Alignment != Size) 552 return false; 553 554 unsigned PtrArgLoc = CI->arg_size() - 3; 555 Value *PtrArg = CI->getArgOperand(PtrArgLoc); 556 Type *PtrTy = PtrArg->getType(); 557 558 SmallVector<llvm::Type *, 6> ArgTys; 559 for (unsigned I = 0; I != PtrArgLoc; ++I) 560 ArgTys.push_back(CI->getArgOperand(I)->getType()); 561 ArgTys.push_back(PtrTy); 562 563 Name = Name + "_" + std::to_string(Size); 564 auto *FTy = FunctionType::get(Callee->getReturnType(), 565 ArrayRef<Type *>(ArgTys), false); 566 AMDGPULibFunc NewLibFunc(Name, FTy); 567 FunctionCallee F = AMDGPULibFunc::getOrInsertFunction(M, NewLibFunc); 568 if (!F) 569 return false; 570 571 auto *BCast = B.CreatePointerCast(PtrArg, PtrTy); 572 SmallVector<Value *, 6> Args; 573 for (unsigned I = 0; I != PtrArgLoc; ++I) 574 Args.push_back(CI->getArgOperand(I)); 575 Args.push_back(BCast); 576 577 auto *NCI = B.CreateCall(F, Args); 578 NCI->setAttributes(CI->getAttributes()); 579 CI->replaceAllUsesWith(NCI); 580 CI->dropAllReferences(); 581 CI->eraseFromParent(); 582 583 return true; 584 } 585 586 // This function returns false if no change; return true otherwise. 587 bool AMDGPULibCalls::fold(CallInst *CI, AliasAnalysis *AA) { 588 this->CI = CI; 589 Function *Callee = CI->getCalledFunction(); 590 591 // Ignore indirect calls. 592 if (Callee == nullptr) 593 return false; 594 595 BasicBlock *BB = CI->getParent(); 596 LLVMContext &Context = CI->getParent()->getContext(); 597 IRBuilder<> B(Context); 598 599 // Set the builder to the instruction after the call. 600 B.SetInsertPoint(BB, CI->getIterator()); 601 602 // Copy fast flags from the original call. 603 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(CI)) 604 B.setFastMathFlags(FPOp->getFastMathFlags()); 605 606 switch (Callee->getIntrinsicID()) { 607 default: 608 break; 609 case Intrinsic::amdgcn_wavefrontsize: 610 return !EnablePreLink && fold_wavefrontsize(CI, B); 611 } 612 613 FuncInfo FInfo; 614 if (!parseFunctionName(Callee->getName(), FInfo)) 615 return false; 616 617 // Further check the number of arguments to see if they match. 618 if (CI->arg_size() != FInfo.getNumArgs()) 619 return false; 620 621 if (TDOFold(CI, FInfo)) 622 return true; 623 624 // Under unsafe-math, evaluate calls if possible. 625 // According to Brian Sumner, we can do this for all f32 function calls 626 // using host's double function calls. 627 if (isUnsafeMath(CI) && evaluateCall(CI, FInfo)) 628 return true; 629 630 // Specialized optimizations for each function call 631 switch (FInfo.getId()) { 632 case AMDGPULibFunc::EI_RECIP: 633 // skip vector function 634 assert ((FInfo.getPrefix() == AMDGPULibFunc::NATIVE || 635 FInfo.getPrefix() == AMDGPULibFunc::HALF) && 636 "recip must be an either native or half function"); 637 return (getVecSize(FInfo) != 1) ? false : fold_recip(CI, B, FInfo); 638 639 case AMDGPULibFunc::EI_DIVIDE: 640 // skip vector function 641 assert ((FInfo.getPrefix() == AMDGPULibFunc::NATIVE || 642 FInfo.getPrefix() == AMDGPULibFunc::HALF) && 643 "divide must be an either native or half function"); 644 return (getVecSize(FInfo) != 1) ? false : fold_divide(CI, B, FInfo); 645 646 case AMDGPULibFunc::EI_POW: 647 case AMDGPULibFunc::EI_POWR: 648 case AMDGPULibFunc::EI_POWN: 649 return fold_pow(CI, B, FInfo); 650 651 case AMDGPULibFunc::EI_ROOTN: 652 // skip vector function 653 return (getVecSize(FInfo) != 1) ? false : fold_rootn(CI, B, FInfo); 654 655 case AMDGPULibFunc::EI_FMA: 656 case AMDGPULibFunc::EI_MAD: 657 case AMDGPULibFunc::EI_NFMA: 658 // skip vector function 659 return (getVecSize(FInfo) != 1) ? false : fold_fma_mad(CI, B, FInfo); 660 661 case AMDGPULibFunc::EI_SQRT: 662 return isUnsafeMath(CI) && fold_sqrt(CI, B, FInfo); 663 case AMDGPULibFunc::EI_COS: 664 case AMDGPULibFunc::EI_SIN: 665 if ((getArgType(FInfo) == AMDGPULibFunc::F32 || 666 getArgType(FInfo) == AMDGPULibFunc::F64) 667 && (FInfo.getPrefix() == AMDGPULibFunc::NOPFX)) 668 return fold_sincos(CI, B, AA); 669 670 break; 671 case AMDGPULibFunc::EI_READ_PIPE_2: 672 case AMDGPULibFunc::EI_READ_PIPE_4: 673 case AMDGPULibFunc::EI_WRITE_PIPE_2: 674 case AMDGPULibFunc::EI_WRITE_PIPE_4: 675 return fold_read_write_pipe(CI, B, FInfo); 676 677 default: 678 break; 679 } 680 681 return false; 682 } 683 684 bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) { 685 // Table-Driven optimization 686 const TableRef tr = getOptTable(FInfo.getId()); 687 if (tr.empty()) 688 return false; 689 690 int const sz = (int)tr.size(); 691 Value *opr0 = CI->getArgOperand(0); 692 693 if (getVecSize(FInfo) > 1) { 694 if (ConstantDataVector *CV = dyn_cast<ConstantDataVector>(opr0)) { 695 SmallVector<double, 0> DVal; 696 for (int eltNo = 0; eltNo < getVecSize(FInfo); ++eltNo) { 697 ConstantFP *eltval = dyn_cast<ConstantFP>( 698 CV->getElementAsConstant((unsigned)eltNo)); 699 assert(eltval && "Non-FP arguments in math function!"); 700 bool found = false; 701 for (int i=0; i < sz; ++i) { 702 if (eltval->isExactlyValue(tr[i].input)) { 703 DVal.push_back(tr[i].result); 704 found = true; 705 break; 706 } 707 } 708 if (!found) { 709 // This vector constants not handled yet. 710 return false; 711 } 712 } 713 LLVMContext &context = CI->getParent()->getParent()->getContext(); 714 Constant *nval; 715 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 716 SmallVector<float, 0> FVal; 717 for (unsigned i = 0; i < DVal.size(); ++i) { 718 FVal.push_back((float)DVal[i]); 719 } 720 ArrayRef<float> tmp(FVal); 721 nval = ConstantDataVector::get(context, tmp); 722 } else { // F64 723 ArrayRef<double> tmp(DVal); 724 nval = ConstantDataVector::get(context, tmp); 725 } 726 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 727 replaceCall(nval); 728 return true; 729 } 730 } else { 731 // Scalar version 732 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) { 733 for (int i = 0; i < sz; ++i) { 734 if (CF->isExactlyValue(tr[i].input)) { 735 Value *nval = ConstantFP::get(CF->getType(), tr[i].result); 736 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 737 replaceCall(nval); 738 return true; 739 } 740 } 741 } 742 } 743 744 return false; 745 } 746 747 // [native_]half_recip(c) ==> 1.0/c 748 bool AMDGPULibCalls::fold_recip(CallInst *CI, IRBuilder<> &B, 749 const FuncInfo &FInfo) { 750 Value *opr0 = CI->getArgOperand(0); 751 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) { 752 // Just create a normal div. Later, InstCombine will be able 753 // to compute the divide into a constant (avoid check float infinity 754 // or subnormal at this point). 755 Value *nval = B.CreateFDiv(ConstantFP::get(CF->getType(), 1.0), 756 opr0, 757 "recip2div"); 758 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 759 replaceCall(nval); 760 return true; 761 } 762 return false; 763 } 764 765 // [native_]half_divide(x, c) ==> x/c 766 bool AMDGPULibCalls::fold_divide(CallInst *CI, IRBuilder<> &B, 767 const FuncInfo &FInfo) { 768 Value *opr0 = CI->getArgOperand(0); 769 Value *opr1 = CI->getArgOperand(1); 770 ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0); 771 ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1); 772 773 if ((CF0 && CF1) || // both are constants 774 (CF1 && (getArgType(FInfo) == AMDGPULibFunc::F32))) 775 // CF1 is constant && f32 divide 776 { 777 Value *nval1 = B.CreateFDiv(ConstantFP::get(opr1->getType(), 1.0), 778 opr1, "__div2recip"); 779 Value *nval = B.CreateFMul(opr0, nval1, "__div2mul"); 780 replaceCall(nval); 781 return true; 782 } 783 return false; 784 } 785 786 namespace llvm { 787 static double log2(double V) { 788 #if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L 789 return ::log2(V); 790 #else 791 return log(V) / numbers::ln2; 792 #endif 793 } 794 } 795 796 bool AMDGPULibCalls::fold_pow(CallInst *CI, IRBuilder<> &B, 797 const FuncInfo &FInfo) { 798 assert((FInfo.getId() == AMDGPULibFunc::EI_POW || 799 FInfo.getId() == AMDGPULibFunc::EI_POWR || 800 FInfo.getId() == AMDGPULibFunc::EI_POWN) && 801 "fold_pow: encounter a wrong function call"); 802 803 Value *opr0, *opr1; 804 ConstantFP *CF; 805 ConstantInt *CINT; 806 ConstantAggregateZero *CZero; 807 Type *eltType; 808 809 opr0 = CI->getArgOperand(0); 810 opr1 = CI->getArgOperand(1); 811 CZero = dyn_cast<ConstantAggregateZero>(opr1); 812 if (getVecSize(FInfo) == 1) { 813 eltType = opr0->getType(); 814 CF = dyn_cast<ConstantFP>(opr1); 815 CINT = dyn_cast<ConstantInt>(opr1); 816 } else { 817 VectorType *VTy = dyn_cast<VectorType>(opr0->getType()); 818 assert(VTy && "Oprand of vector function should be of vectortype"); 819 eltType = VTy->getElementType(); 820 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1); 821 822 // Now, only Handle vector const whose elements have the same value. 823 CF = CDV ? dyn_cast_or_null<ConstantFP>(CDV->getSplatValue()) : nullptr; 824 CINT = CDV ? dyn_cast_or_null<ConstantInt>(CDV->getSplatValue()) : nullptr; 825 } 826 827 // No unsafe math , no constant argument, do nothing 828 if (!isUnsafeMath(CI) && !CF && !CINT && !CZero) 829 return false; 830 831 // 0x1111111 means that we don't do anything for this call. 832 int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111); 833 834 if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0) || CZero) { 835 // pow/powr/pown(x, 0) == 1 836 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1\n"); 837 Constant *cnval = ConstantFP::get(eltType, 1.0); 838 if (getVecSize(FInfo) > 1) { 839 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 840 } 841 replaceCall(cnval); 842 return true; 843 } 844 if ((CF && CF->isExactlyValue(1.0)) || (CINT && ci_opr1 == 1)) { 845 // pow/powr/pown(x, 1.0) = x 846 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n"); 847 replaceCall(opr0); 848 return true; 849 } 850 if ((CF && CF->isExactlyValue(2.0)) || (CINT && ci_opr1 == 2)) { 851 // pow/powr/pown(x, 2.0) = x*x 852 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * " << *opr0 853 << "\n"); 854 Value *nval = B.CreateFMul(opr0, opr0, "__pow2"); 855 replaceCall(nval); 856 return true; 857 } 858 if ((CF && CF->isExactlyValue(-1.0)) || (CINT && ci_opr1 == -1)) { 859 // pow/powr/pown(x, -1.0) = 1.0/x 860 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1 / " << *opr0 << "\n"); 861 Constant *cnval = ConstantFP::get(eltType, 1.0); 862 if (getVecSize(FInfo) > 1) { 863 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 864 } 865 Value *nval = B.CreateFDiv(cnval, opr0, "__powrecip"); 866 replaceCall(nval); 867 return true; 868 } 869 870 Module *M = CI->getModule(); 871 if (CF && (CF->isExactlyValue(0.5) || CF->isExactlyValue(-0.5))) { 872 // pow[r](x, [-]0.5) = sqrt(x) 873 bool issqrt = CF->isExactlyValue(0.5); 874 if (FunctionCallee FPExpr = 875 getFunction(M, AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT 876 : AMDGPULibFunc::EI_RSQRT, 877 FInfo))) { 878 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " 879 << FInfo.getName().c_str() << "(" << *opr0 << ")\n"); 880 Value *nval = CreateCallEx(B,FPExpr, opr0, issqrt ? "__pow2sqrt" 881 : "__pow2rsqrt"); 882 replaceCall(nval); 883 return true; 884 } 885 } 886 887 if (!isUnsafeMath(CI)) 888 return false; 889 890 // Unsafe Math optimization 891 892 // Remember that ci_opr1 is set if opr1 is integral 893 if (CF) { 894 double dval = (getArgType(FInfo) == AMDGPULibFunc::F32) 895 ? (double)CF->getValueAPF().convertToFloat() 896 : CF->getValueAPF().convertToDouble(); 897 int ival = (int)dval; 898 if ((double)ival == dval) { 899 ci_opr1 = ival; 900 } else 901 ci_opr1 = 0x11111111; 902 } 903 904 // pow/powr/pown(x, c) = [1/](x*x*..x); where 905 // trunc(c) == c && the number of x == c && |c| <= 12 906 unsigned abs_opr1 = (ci_opr1 < 0) ? -ci_opr1 : ci_opr1; 907 if (abs_opr1 <= 12) { 908 Constant *cnval; 909 Value *nval; 910 if (abs_opr1 == 0) { 911 cnval = ConstantFP::get(eltType, 1.0); 912 if (getVecSize(FInfo) > 1) { 913 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 914 } 915 nval = cnval; 916 } else { 917 Value *valx2 = nullptr; 918 nval = nullptr; 919 while (abs_opr1 > 0) { 920 valx2 = valx2 ? B.CreateFMul(valx2, valx2, "__powx2") : opr0; 921 if (abs_opr1 & 1) { 922 nval = nval ? B.CreateFMul(nval, valx2, "__powprod") : valx2; 923 } 924 abs_opr1 >>= 1; 925 } 926 } 927 928 if (ci_opr1 < 0) { 929 cnval = ConstantFP::get(eltType, 1.0); 930 if (getVecSize(FInfo) > 1) { 931 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 932 } 933 nval = B.CreateFDiv(cnval, nval, "__1powprod"); 934 } 935 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " 936 << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr0 937 << ")\n"); 938 replaceCall(nval); 939 return true; 940 } 941 942 // powr ---> exp2(y * log2(x)) 943 // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31)) 944 FunctionCallee ExpExpr = 945 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo)); 946 if (!ExpExpr) 947 return false; 948 949 bool needlog = false; 950 bool needabs = false; 951 bool needcopysign = false; 952 Constant *cnval = nullptr; 953 if (getVecSize(FInfo) == 1) { 954 CF = dyn_cast<ConstantFP>(opr0); 955 956 if (CF) { 957 double V = (getArgType(FInfo) == AMDGPULibFunc::F32) 958 ? (double)CF->getValueAPF().convertToFloat() 959 : CF->getValueAPF().convertToDouble(); 960 961 V = log2(std::abs(V)); 962 cnval = ConstantFP::get(eltType, V); 963 needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR) && 964 CF->isNegative(); 965 } else { 966 needlog = true; 967 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR && 968 (!CF || CF->isNegative()); 969 } 970 } else { 971 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr0); 972 973 if (!CDV) { 974 needlog = true; 975 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR; 976 } else { 977 assert ((int)CDV->getNumElements() == getVecSize(FInfo) && 978 "Wrong vector size detected"); 979 980 SmallVector<double, 0> DVal; 981 for (int i=0; i < getVecSize(FInfo); ++i) { 982 double V = (getArgType(FInfo) == AMDGPULibFunc::F32) 983 ? (double)CDV->getElementAsFloat(i) 984 : CDV->getElementAsDouble(i); 985 if (V < 0.0) needcopysign = true; 986 V = log2(std::abs(V)); 987 DVal.push_back(V); 988 } 989 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 990 SmallVector<float, 0> FVal; 991 for (unsigned i=0; i < DVal.size(); ++i) { 992 FVal.push_back((float)DVal[i]); 993 } 994 ArrayRef<float> tmp(FVal); 995 cnval = ConstantDataVector::get(M->getContext(), tmp); 996 } else { 997 ArrayRef<double> tmp(DVal); 998 cnval = ConstantDataVector::get(M->getContext(), tmp); 999 } 1000 } 1001 } 1002 1003 if (needcopysign && (FInfo.getId() == AMDGPULibFunc::EI_POW)) { 1004 // We cannot handle corner cases for a general pow() function, give up 1005 // unless y is a constant integral value. Then proceed as if it were pown. 1006 if (getVecSize(FInfo) == 1) { 1007 if (const ConstantFP *CF = dyn_cast<ConstantFP>(opr1)) { 1008 double y = (getArgType(FInfo) == AMDGPULibFunc::F32) 1009 ? (double)CF->getValueAPF().convertToFloat() 1010 : CF->getValueAPF().convertToDouble(); 1011 if (y != (double)(int64_t)y) 1012 return false; 1013 } else 1014 return false; 1015 } else { 1016 if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1)) { 1017 for (int i=0; i < getVecSize(FInfo); ++i) { 1018 double y = (getArgType(FInfo) == AMDGPULibFunc::F32) 1019 ? (double)CDV->getElementAsFloat(i) 1020 : CDV->getElementAsDouble(i); 1021 if (y != (double)(int64_t)y) 1022 return false; 1023 } 1024 } else 1025 return false; 1026 } 1027 } 1028 1029 Value *nval; 1030 if (needabs) { 1031 FunctionCallee AbsExpr = 1032 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_FABS, FInfo)); 1033 if (!AbsExpr) 1034 return false; 1035 nval = CreateCallEx(B, AbsExpr, opr0, "__fabs"); 1036 } else { 1037 nval = cnval ? cnval : opr0; 1038 } 1039 if (needlog) { 1040 FunctionCallee LogExpr = 1041 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo)); 1042 if (!LogExpr) 1043 return false; 1044 nval = CreateCallEx(B,LogExpr, nval, "__log2"); 1045 } 1046 1047 if (FInfo.getId() == AMDGPULibFunc::EI_POWN) { 1048 // convert int(32) to fp(f32 or f64) 1049 opr1 = B.CreateSIToFP(opr1, nval->getType(), "pownI2F"); 1050 } 1051 nval = B.CreateFMul(opr1, nval, "__ylogx"); 1052 nval = CreateCallEx(B,ExpExpr, nval, "__exp2"); 1053 1054 if (needcopysign) { 1055 Value *opr_n; 1056 Type* rTy = opr0->getType(); 1057 Type* nTyS = eltType->isDoubleTy() ? B.getInt64Ty() : B.getInt32Ty(); 1058 Type *nTy = nTyS; 1059 if (const auto *vTy = dyn_cast<FixedVectorType>(rTy)) 1060 nTy = FixedVectorType::get(nTyS, vTy); 1061 unsigned size = nTy->getScalarSizeInBits(); 1062 opr_n = CI->getArgOperand(1); 1063 if (opr_n->getType()->isIntegerTy()) 1064 opr_n = B.CreateZExtOrBitCast(opr_n, nTy, "__ytou"); 1065 else 1066 opr_n = B.CreateFPToSI(opr1, nTy, "__ytou"); 1067 1068 Value *sign = B.CreateShl(opr_n, size-1, "__yeven"); 1069 sign = B.CreateAnd(B.CreateBitCast(opr0, nTy), sign, "__pow_sign"); 1070 nval = B.CreateOr(B.CreateBitCast(nval, nTy), sign); 1071 nval = B.CreateBitCast(nval, opr0->getType()); 1072 } 1073 1074 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " 1075 << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n"); 1076 replaceCall(nval); 1077 1078 return true; 1079 } 1080 1081 bool AMDGPULibCalls::fold_rootn(CallInst *CI, IRBuilder<> &B, 1082 const FuncInfo &FInfo) { 1083 Value *opr0 = CI->getArgOperand(0); 1084 Value *opr1 = CI->getArgOperand(1); 1085 1086 ConstantInt *CINT = dyn_cast<ConstantInt>(opr1); 1087 if (!CINT) { 1088 return false; 1089 } 1090 int ci_opr1 = (int)CINT->getSExtValue(); 1091 if (ci_opr1 == 1) { // rootn(x, 1) = x 1092 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n"); 1093 replaceCall(opr0); 1094 return true; 1095 } 1096 if (ci_opr1 == 2) { // rootn(x, 2) = sqrt(x) 1097 Module *M = CI->getModule(); 1098 if (FunctionCallee FPExpr = 1099 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) { 1100 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> sqrt(" << *opr0 << ")\n"); 1101 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2sqrt"); 1102 replaceCall(nval); 1103 return true; 1104 } 1105 } else if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x) 1106 Module *M = CI->getModule(); 1107 if (FunctionCallee FPExpr = 1108 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) { 1109 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> cbrt(" << *opr0 << ")\n"); 1110 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2cbrt"); 1111 replaceCall(nval); 1112 return true; 1113 } 1114 } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x 1115 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1.0 / " << *opr0 << "\n"); 1116 Value *nval = B.CreateFDiv(ConstantFP::get(opr0->getType(), 1.0), 1117 opr0, 1118 "__rootn2div"); 1119 replaceCall(nval); 1120 return true; 1121 } else if (ci_opr1 == -2) { // rootn(x, -2) = rsqrt(x) 1122 Module *M = CI->getModule(); 1123 if (FunctionCallee FPExpr = 1124 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_RSQRT, FInfo))) { 1125 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> rsqrt(" << *opr0 1126 << ")\n"); 1127 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2rsqrt"); 1128 replaceCall(nval); 1129 return true; 1130 } 1131 } 1132 return false; 1133 } 1134 1135 bool AMDGPULibCalls::fold_fma_mad(CallInst *CI, IRBuilder<> &B, 1136 const FuncInfo &FInfo) { 1137 Value *opr0 = CI->getArgOperand(0); 1138 Value *opr1 = CI->getArgOperand(1); 1139 Value *opr2 = CI->getArgOperand(2); 1140 1141 ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0); 1142 ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1); 1143 if ((CF0 && CF0->isZero()) || (CF1 && CF1->isZero())) { 1144 // fma/mad(a, b, c) = c if a=0 || b=0 1145 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr2 << "\n"); 1146 replaceCall(opr2); 1147 return true; 1148 } 1149 if (CF0 && CF0->isExactlyValue(1.0f)) { 1150 // fma/mad(a, b, c) = b+c if a=1 1151 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr1 << " + " << *opr2 1152 << "\n"); 1153 Value *nval = B.CreateFAdd(opr1, opr2, "fmaadd"); 1154 replaceCall(nval); 1155 return true; 1156 } 1157 if (CF1 && CF1->isExactlyValue(1.0f)) { 1158 // fma/mad(a, b, c) = a+c if b=1 1159 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " + " << *opr2 1160 << "\n"); 1161 Value *nval = B.CreateFAdd(opr0, opr2, "fmaadd"); 1162 replaceCall(nval); 1163 return true; 1164 } 1165 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr2)) { 1166 if (CF->isZero()) { 1167 // fma/mad(a, b, c) = a*b if c=0 1168 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * " 1169 << *opr1 << "\n"); 1170 Value *nval = B.CreateFMul(opr0, opr1, "fmamul"); 1171 replaceCall(nval); 1172 return true; 1173 } 1174 } 1175 1176 return false; 1177 } 1178 1179 // Get a scalar native builtin single argument FP function 1180 FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M, 1181 const FuncInfo &FInfo) { 1182 if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId())) 1183 return nullptr; 1184 FuncInfo nf = FInfo; 1185 nf.setPrefix(AMDGPULibFunc::NATIVE); 1186 return getFunction(M, nf); 1187 } 1188 1189 // fold sqrt -> native_sqrt (x) 1190 bool AMDGPULibCalls::fold_sqrt(CallInst *CI, IRBuilder<> &B, 1191 const FuncInfo &FInfo) { 1192 if (getArgType(FInfo) == AMDGPULibFunc::F32 && (getVecSize(FInfo) == 1) && 1193 (FInfo.getPrefix() != AMDGPULibFunc::NATIVE)) { 1194 if (FunctionCallee FPExpr = getNativeFunction( 1195 CI->getModule(), AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) { 1196 Value *opr0 = CI->getArgOperand(0); 1197 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " 1198 << "sqrt(" << *opr0 << ")\n"); 1199 Value *nval = CreateCallEx(B,FPExpr, opr0, "__sqrt"); 1200 replaceCall(nval); 1201 return true; 1202 } 1203 } 1204 return false; 1205 } 1206 1207 // fold sin, cos -> sincos. 1208 bool AMDGPULibCalls::fold_sincos(CallInst *CI, IRBuilder<> &B, 1209 AliasAnalysis *AA) { 1210 AMDGPULibFunc fInfo; 1211 if (!AMDGPULibFunc::parse(CI->getCalledFunction()->getName(), fInfo)) 1212 return false; 1213 1214 assert(fInfo.getId() == AMDGPULibFunc::EI_SIN || 1215 fInfo.getId() == AMDGPULibFunc::EI_COS); 1216 bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN; 1217 1218 Value *CArgVal = CI->getArgOperand(0); 1219 BasicBlock * const CBB = CI->getParent(); 1220 1221 int const MaxScan = 30; 1222 bool Changed = false; 1223 1224 { // fold in load value. 1225 LoadInst *LI = dyn_cast<LoadInst>(CArgVal); 1226 if (LI && LI->getParent() == CBB) { 1227 BasicBlock::iterator BBI = LI->getIterator(); 1228 Value *AvailableVal = FindAvailableLoadedValue(LI, CBB, BBI, MaxScan, AA); 1229 if (AvailableVal) { 1230 Changed = true; 1231 CArgVal->replaceAllUsesWith(AvailableVal); 1232 if (CArgVal->getNumUses() == 0) 1233 LI->eraseFromParent(); 1234 CArgVal = CI->getArgOperand(0); 1235 } 1236 } 1237 } 1238 1239 Module *M = CI->getModule(); 1240 fInfo.setId(isSin ? AMDGPULibFunc::EI_COS : AMDGPULibFunc::EI_SIN); 1241 std::string const PairName = fInfo.mangle(); 1242 1243 CallInst *UI = nullptr; 1244 for (User* U : CArgVal->users()) { 1245 CallInst *XI = dyn_cast_or_null<CallInst>(U); 1246 if (!XI || XI == CI || XI->getParent() != CBB) 1247 continue; 1248 1249 Function *UCallee = XI->getCalledFunction(); 1250 if (!UCallee || !UCallee->getName().equals(PairName)) 1251 continue; 1252 1253 BasicBlock::iterator BBI = CI->getIterator(); 1254 if (BBI == CI->getParent()->begin()) 1255 break; 1256 --BBI; 1257 for (int I = MaxScan; I > 0 && BBI != CBB->begin(); --BBI, --I) { 1258 if (cast<Instruction>(BBI) == XI) { 1259 UI = XI; 1260 break; 1261 } 1262 } 1263 if (UI) break; 1264 } 1265 1266 if (!UI) 1267 return Changed; 1268 1269 // Merge the sin and cos. 1270 1271 // for OpenCL 2.0 we have only generic implementation of sincos 1272 // function. 1273 AMDGPULibFunc nf(AMDGPULibFunc::EI_SINCOS, fInfo); 1274 nf.getLeads()[0].PtrKind = AMDGPULibFunc::getEPtrKindFromAddrSpace(AMDGPUAS::FLAT_ADDRESS); 1275 FunctionCallee Fsincos = getFunction(M, nf); 1276 if (!Fsincos) 1277 return Changed; 1278 1279 BasicBlock::iterator ItOld = B.GetInsertPoint(); 1280 AllocaInst *Alloc = insertAlloca(UI, B, "__sincos_"); 1281 B.SetInsertPoint(UI); 1282 1283 Value *P = Alloc; 1284 Type *PTy = Fsincos.getFunctionType()->getParamType(1); 1285 // The allocaInst allocates the memory in private address space. This need 1286 // to be bitcasted to point to the address space of cos pointer type. 1287 // In OpenCL 2.0 this is generic, while in 1.2 that is private. 1288 if (PTy->getPointerAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) 1289 P = B.CreateAddrSpaceCast(Alloc, PTy); 1290 CallInst *Call = CreateCallEx2(B, Fsincos, UI->getArgOperand(0), P); 1291 1292 LLVM_DEBUG(errs() << "AMDIC: fold_sincos (" << *CI << ", " << *UI << ") with " 1293 << *Call << "\n"); 1294 1295 if (!isSin) { // CI->cos, UI->sin 1296 B.SetInsertPoint(&*ItOld); 1297 UI->replaceAllUsesWith(&*Call); 1298 Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc); 1299 CI->replaceAllUsesWith(Reload); 1300 UI->eraseFromParent(); 1301 CI->eraseFromParent(); 1302 } else { // CI->sin, UI->cos 1303 Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc); 1304 UI->replaceAllUsesWith(Reload); 1305 CI->replaceAllUsesWith(Call); 1306 UI->eraseFromParent(); 1307 CI->eraseFromParent(); 1308 } 1309 return true; 1310 } 1311 1312 bool AMDGPULibCalls::fold_wavefrontsize(CallInst *CI, IRBuilder<> &B) { 1313 if (!TM) 1314 return false; 1315 1316 StringRef CPU = TM->getTargetCPU(); 1317 StringRef Features = TM->getTargetFeatureString(); 1318 if ((CPU.empty() || CPU.equals_insensitive("generic")) && 1319 (Features.empty() || !Features.contains_insensitive("wavefrontsize"))) 1320 return false; 1321 1322 Function *F = CI->getParent()->getParent(); 1323 const GCNSubtarget &ST = TM->getSubtarget<GCNSubtarget>(*F); 1324 unsigned N = ST.getWavefrontSize(); 1325 1326 LLVM_DEBUG(errs() << "AMDIC: fold_wavefrontsize (" << *CI << ") with " 1327 << N << "\n"); 1328 1329 CI->replaceAllUsesWith(ConstantInt::get(B.getInt32Ty(), N)); 1330 CI->eraseFromParent(); 1331 return true; 1332 } 1333 1334 // Get insertion point at entry. 1335 BasicBlock::iterator AMDGPULibCalls::getEntryIns(CallInst * UI) { 1336 Function * Func = UI->getParent()->getParent(); 1337 BasicBlock * BB = &Func->getEntryBlock(); 1338 assert(BB && "Entry block not found!"); 1339 BasicBlock::iterator ItNew = BB->begin(); 1340 return ItNew; 1341 } 1342 1343 // Insert a AllocsInst at the beginning of function entry block. 1344 AllocaInst* AMDGPULibCalls::insertAlloca(CallInst *UI, IRBuilder<> &B, 1345 const char *prefix) { 1346 BasicBlock::iterator ItNew = getEntryIns(UI); 1347 Function *UCallee = UI->getCalledFunction(); 1348 Type *RetType = UCallee->getReturnType(); 1349 B.SetInsertPoint(&*ItNew); 1350 AllocaInst *Alloc = 1351 B.CreateAlloca(RetType, nullptr, std::string(prefix) + UI->getName()); 1352 Alloc->setAlignment( 1353 Align(UCallee->getParent()->getDataLayout().getTypeAllocSize(RetType))); 1354 return Alloc; 1355 } 1356 1357 bool AMDGPULibCalls::evaluateScalarMathFunc(const FuncInfo &FInfo, 1358 double& Res0, double& Res1, 1359 Constant *copr0, Constant *copr1, 1360 Constant *copr2) { 1361 // By default, opr0/opr1/opr3 holds values of float/double type. 1362 // If they are not float/double, each function has to its 1363 // operand separately. 1364 double opr0=0.0, opr1=0.0, opr2=0.0; 1365 ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(copr0); 1366 ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(copr1); 1367 ConstantFP *fpopr2 = dyn_cast_or_null<ConstantFP>(copr2); 1368 if (fpopr0) { 1369 opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1370 ? fpopr0->getValueAPF().convertToDouble() 1371 : (double)fpopr0->getValueAPF().convertToFloat(); 1372 } 1373 1374 if (fpopr1) { 1375 opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1376 ? fpopr1->getValueAPF().convertToDouble() 1377 : (double)fpopr1->getValueAPF().convertToFloat(); 1378 } 1379 1380 if (fpopr2) { 1381 opr2 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1382 ? fpopr2->getValueAPF().convertToDouble() 1383 : (double)fpopr2->getValueAPF().convertToFloat(); 1384 } 1385 1386 switch (FInfo.getId()) { 1387 default : return false; 1388 1389 case AMDGPULibFunc::EI_ACOS: 1390 Res0 = acos(opr0); 1391 return true; 1392 1393 case AMDGPULibFunc::EI_ACOSH: 1394 // acosh(x) == log(x + sqrt(x*x - 1)) 1395 Res0 = log(opr0 + sqrt(opr0*opr0 - 1.0)); 1396 return true; 1397 1398 case AMDGPULibFunc::EI_ACOSPI: 1399 Res0 = acos(opr0) / MATH_PI; 1400 return true; 1401 1402 case AMDGPULibFunc::EI_ASIN: 1403 Res0 = asin(opr0); 1404 return true; 1405 1406 case AMDGPULibFunc::EI_ASINH: 1407 // asinh(x) == log(x + sqrt(x*x + 1)) 1408 Res0 = log(opr0 + sqrt(opr0*opr0 + 1.0)); 1409 return true; 1410 1411 case AMDGPULibFunc::EI_ASINPI: 1412 Res0 = asin(opr0) / MATH_PI; 1413 return true; 1414 1415 case AMDGPULibFunc::EI_ATAN: 1416 Res0 = atan(opr0); 1417 return true; 1418 1419 case AMDGPULibFunc::EI_ATANH: 1420 // atanh(x) == (log(x+1) - log(x-1))/2; 1421 Res0 = (log(opr0 + 1.0) - log(opr0 - 1.0))/2.0; 1422 return true; 1423 1424 case AMDGPULibFunc::EI_ATANPI: 1425 Res0 = atan(opr0) / MATH_PI; 1426 return true; 1427 1428 case AMDGPULibFunc::EI_CBRT: 1429 Res0 = (opr0 < 0.0) ? -pow(-opr0, 1.0/3.0) : pow(opr0, 1.0/3.0); 1430 return true; 1431 1432 case AMDGPULibFunc::EI_COS: 1433 Res0 = cos(opr0); 1434 return true; 1435 1436 case AMDGPULibFunc::EI_COSH: 1437 Res0 = cosh(opr0); 1438 return true; 1439 1440 case AMDGPULibFunc::EI_COSPI: 1441 Res0 = cos(MATH_PI * opr0); 1442 return true; 1443 1444 case AMDGPULibFunc::EI_EXP: 1445 Res0 = exp(opr0); 1446 return true; 1447 1448 case AMDGPULibFunc::EI_EXP2: 1449 Res0 = pow(2.0, opr0); 1450 return true; 1451 1452 case AMDGPULibFunc::EI_EXP10: 1453 Res0 = pow(10.0, opr0); 1454 return true; 1455 1456 case AMDGPULibFunc::EI_EXPM1: 1457 Res0 = exp(opr0) - 1.0; 1458 return true; 1459 1460 case AMDGPULibFunc::EI_LOG: 1461 Res0 = log(opr0); 1462 return true; 1463 1464 case AMDGPULibFunc::EI_LOG2: 1465 Res0 = log(opr0) / log(2.0); 1466 return true; 1467 1468 case AMDGPULibFunc::EI_LOG10: 1469 Res0 = log(opr0) / log(10.0); 1470 return true; 1471 1472 case AMDGPULibFunc::EI_RSQRT: 1473 Res0 = 1.0 / sqrt(opr0); 1474 return true; 1475 1476 case AMDGPULibFunc::EI_SIN: 1477 Res0 = sin(opr0); 1478 return true; 1479 1480 case AMDGPULibFunc::EI_SINH: 1481 Res0 = sinh(opr0); 1482 return true; 1483 1484 case AMDGPULibFunc::EI_SINPI: 1485 Res0 = sin(MATH_PI * opr0); 1486 return true; 1487 1488 case AMDGPULibFunc::EI_SQRT: 1489 Res0 = sqrt(opr0); 1490 return true; 1491 1492 case AMDGPULibFunc::EI_TAN: 1493 Res0 = tan(opr0); 1494 return true; 1495 1496 case AMDGPULibFunc::EI_TANH: 1497 Res0 = tanh(opr0); 1498 return true; 1499 1500 case AMDGPULibFunc::EI_TANPI: 1501 Res0 = tan(MATH_PI * opr0); 1502 return true; 1503 1504 case AMDGPULibFunc::EI_RECIP: 1505 Res0 = 1.0 / opr0; 1506 return true; 1507 1508 // two-arg functions 1509 case AMDGPULibFunc::EI_DIVIDE: 1510 Res0 = opr0 / opr1; 1511 return true; 1512 1513 case AMDGPULibFunc::EI_POW: 1514 case AMDGPULibFunc::EI_POWR: 1515 Res0 = pow(opr0, opr1); 1516 return true; 1517 1518 case AMDGPULibFunc::EI_POWN: { 1519 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) { 1520 double val = (double)iopr1->getSExtValue(); 1521 Res0 = pow(opr0, val); 1522 return true; 1523 } 1524 return false; 1525 } 1526 1527 case AMDGPULibFunc::EI_ROOTN: { 1528 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) { 1529 double val = (double)iopr1->getSExtValue(); 1530 Res0 = pow(opr0, 1.0 / val); 1531 return true; 1532 } 1533 return false; 1534 } 1535 1536 // with ptr arg 1537 case AMDGPULibFunc::EI_SINCOS: 1538 Res0 = sin(opr0); 1539 Res1 = cos(opr0); 1540 return true; 1541 1542 // three-arg functions 1543 case AMDGPULibFunc::EI_FMA: 1544 case AMDGPULibFunc::EI_MAD: 1545 Res0 = opr0 * opr1 + opr2; 1546 return true; 1547 } 1548 1549 return false; 1550 } 1551 1552 bool AMDGPULibCalls::evaluateCall(CallInst *aCI, const FuncInfo &FInfo) { 1553 int numArgs = (int)aCI->arg_size(); 1554 if (numArgs > 3) 1555 return false; 1556 1557 Constant *copr0 = nullptr; 1558 Constant *copr1 = nullptr; 1559 Constant *copr2 = nullptr; 1560 if (numArgs > 0) { 1561 if ((copr0 = dyn_cast<Constant>(aCI->getArgOperand(0))) == nullptr) 1562 return false; 1563 } 1564 1565 if (numArgs > 1) { 1566 if ((copr1 = dyn_cast<Constant>(aCI->getArgOperand(1))) == nullptr) { 1567 if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS) 1568 return false; 1569 } 1570 } 1571 1572 if (numArgs > 2) { 1573 if ((copr2 = dyn_cast<Constant>(aCI->getArgOperand(2))) == nullptr) 1574 return false; 1575 } 1576 1577 // At this point, all arguments to aCI are constants. 1578 1579 // max vector size is 16, and sincos will generate two results. 1580 double DVal0[16], DVal1[16]; 1581 int FuncVecSize = getVecSize(FInfo); 1582 bool hasTwoResults = (FInfo.getId() == AMDGPULibFunc::EI_SINCOS); 1583 if (FuncVecSize == 1) { 1584 if (!evaluateScalarMathFunc(FInfo, DVal0[0], 1585 DVal1[0], copr0, copr1, copr2)) { 1586 return false; 1587 } 1588 } else { 1589 ConstantDataVector *CDV0 = dyn_cast_or_null<ConstantDataVector>(copr0); 1590 ConstantDataVector *CDV1 = dyn_cast_or_null<ConstantDataVector>(copr1); 1591 ConstantDataVector *CDV2 = dyn_cast_or_null<ConstantDataVector>(copr2); 1592 for (int i = 0; i < FuncVecSize; ++i) { 1593 Constant *celt0 = CDV0 ? CDV0->getElementAsConstant(i) : nullptr; 1594 Constant *celt1 = CDV1 ? CDV1->getElementAsConstant(i) : nullptr; 1595 Constant *celt2 = CDV2 ? CDV2->getElementAsConstant(i) : nullptr; 1596 if (!evaluateScalarMathFunc(FInfo, DVal0[i], 1597 DVal1[i], celt0, celt1, celt2)) { 1598 return false; 1599 } 1600 } 1601 } 1602 1603 LLVMContext &context = CI->getParent()->getParent()->getContext(); 1604 Constant *nval0, *nval1; 1605 if (FuncVecSize == 1) { 1606 nval0 = ConstantFP::get(CI->getType(), DVal0[0]); 1607 if (hasTwoResults) 1608 nval1 = ConstantFP::get(CI->getType(), DVal1[0]); 1609 } else { 1610 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 1611 SmallVector <float, 0> FVal0, FVal1; 1612 for (int i = 0; i < FuncVecSize; ++i) 1613 FVal0.push_back((float)DVal0[i]); 1614 ArrayRef<float> tmp0(FVal0); 1615 nval0 = ConstantDataVector::get(context, tmp0); 1616 if (hasTwoResults) { 1617 for (int i = 0; i < FuncVecSize; ++i) 1618 FVal1.push_back((float)DVal1[i]); 1619 ArrayRef<float> tmp1(FVal1); 1620 nval1 = ConstantDataVector::get(context, tmp1); 1621 } 1622 } else { 1623 ArrayRef<double> tmp0(DVal0); 1624 nval0 = ConstantDataVector::get(context, tmp0); 1625 if (hasTwoResults) { 1626 ArrayRef<double> tmp1(DVal1); 1627 nval1 = ConstantDataVector::get(context, tmp1); 1628 } 1629 } 1630 } 1631 1632 if (hasTwoResults) { 1633 // sincos 1634 assert(FInfo.getId() == AMDGPULibFunc::EI_SINCOS && 1635 "math function with ptr arg not supported yet"); 1636 new StoreInst(nval1, aCI->getArgOperand(1), aCI); 1637 } 1638 1639 replaceCall(nval0); 1640 return true; 1641 } 1642 1643 // Public interface to the Simplify LibCalls pass. 1644 FunctionPass *llvm::createAMDGPUSimplifyLibCallsPass(const TargetMachine *TM) { 1645 return new AMDGPUSimplifyLibCalls(TM); 1646 } 1647 1648 FunctionPass *llvm::createAMDGPUUseNativeCallsPass() { 1649 return new AMDGPUUseNativeCalls(); 1650 } 1651 1652 bool AMDGPUSimplifyLibCalls::runOnFunction(Function &F) { 1653 if (skipFunction(F)) 1654 return false; 1655 1656 bool Changed = false; 1657 auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); 1658 1659 LLVM_DEBUG(dbgs() << "AMDIC: process function "; 1660 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';); 1661 1662 for (auto &BB : F) { 1663 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) { 1664 // Ignore non-calls. 1665 CallInst *CI = dyn_cast<CallInst>(I); 1666 ++I; 1667 // Ignore intrinsics that do not become real instructions. 1668 if (!CI || isa<DbgInfoIntrinsic>(CI) || CI->isLifetimeStartOrEnd()) 1669 continue; 1670 1671 // Ignore indirect calls. 1672 Function *Callee = CI->getCalledFunction(); 1673 if (Callee == nullptr) 1674 continue; 1675 1676 LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << "\n"; 1677 dbgs().flush()); 1678 if(Simplifier.fold(CI, AA)) 1679 Changed = true; 1680 } 1681 } 1682 return Changed; 1683 } 1684 1685 PreservedAnalyses AMDGPUSimplifyLibCallsPass::run(Function &F, 1686 FunctionAnalysisManager &AM) { 1687 AMDGPULibCalls Simplifier(&TM); 1688 Simplifier.initNativeFuncs(); 1689 1690 bool Changed = false; 1691 auto AA = &AM.getResult<AAManager>(F); 1692 1693 LLVM_DEBUG(dbgs() << "AMDIC: process function "; 1694 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';); 1695 1696 for (auto &BB : F) { 1697 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) { 1698 // Ignore non-calls. 1699 CallInst *CI = dyn_cast<CallInst>(I); 1700 ++I; 1701 // Ignore intrinsics that do not become real instructions. 1702 if (!CI || isa<DbgInfoIntrinsic>(CI) || CI->isLifetimeStartOrEnd()) 1703 continue; 1704 1705 // Ignore indirect calls. 1706 Function *Callee = CI->getCalledFunction(); 1707 if (Callee == nullptr) 1708 continue; 1709 1710 LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << "\n"; 1711 dbgs().flush()); 1712 if (Simplifier.fold(CI, AA)) 1713 Changed = true; 1714 } 1715 } 1716 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 1717 } 1718 1719 bool AMDGPUUseNativeCalls::runOnFunction(Function &F) { 1720 if (skipFunction(F) || UseNative.empty()) 1721 return false; 1722 1723 bool Changed = false; 1724 for (auto &BB : F) { 1725 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) { 1726 // Ignore non-calls. 1727 CallInst *CI = dyn_cast<CallInst>(I); 1728 ++I; 1729 if (!CI) continue; 1730 1731 // Ignore indirect calls. 1732 Function *Callee = CI->getCalledFunction(); 1733 if (Callee == nullptr) 1734 continue; 1735 1736 if (Simplifier.useNative(CI)) 1737 Changed = true; 1738 } 1739 } 1740 return Changed; 1741 } 1742 1743 PreservedAnalyses AMDGPUUseNativeCallsPass::run(Function &F, 1744 FunctionAnalysisManager &AM) { 1745 if (UseNative.empty()) 1746 return PreservedAnalyses::all(); 1747 1748 AMDGPULibCalls Simplifier; 1749 Simplifier.initNativeFuncs(); 1750 1751 bool Changed = false; 1752 for (auto &BB : F) { 1753 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) { 1754 // Ignore non-calls. 1755 CallInst *CI = dyn_cast<CallInst>(I); 1756 ++I; 1757 if (!CI) 1758 continue; 1759 1760 // Ignore indirect calls. 1761 Function *Callee = CI->getCalledFunction(); 1762 if (Callee == nullptr) 1763 continue; 1764 1765 if (Simplifier.useNative(CI)) 1766 Changed = true; 1767 } 1768 } 1769 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 1770 } 1771