1 //===- AMDGPULibCalls.cpp -------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file does AMD library function optimizations. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPU.h" 15 #include "AMDGPULibFunc.h" 16 #include "GCNSubtarget.h" 17 #include "llvm/Analysis/AliasAnalysis.h" 18 #include "llvm/Analysis/Loads.h" 19 #include "llvm/IR/IRBuilder.h" 20 #include "llvm/IR/IntrinsicInst.h" 21 #include "llvm/IR/IntrinsicsAMDGPU.h" 22 #include "llvm/InitializePasses.h" 23 #include "llvm/Target/TargetMachine.h" 24 #include <cmath> 25 26 #define DEBUG_TYPE "amdgpu-simplifylib" 27 28 using namespace llvm; 29 30 static cl::opt<bool> EnablePreLink("amdgpu-prelink", 31 cl::desc("Enable pre-link mode optimizations"), 32 cl::init(false), 33 cl::Hidden); 34 35 static cl::list<std::string> UseNative("amdgpu-use-native", 36 cl::desc("Comma separated list of functions to replace with native, or all"), 37 cl::CommaSeparated, cl::ValueOptional, 38 cl::Hidden); 39 40 #define MATH_PI numbers::pi 41 #define MATH_E numbers::e 42 #define MATH_SQRT2 numbers::sqrt2 43 #define MATH_SQRT1_2 numbers::inv_sqrt2 44 45 namespace llvm { 46 47 class AMDGPULibCalls { 48 private: 49 50 typedef llvm::AMDGPULibFunc FuncInfo; 51 52 const TargetMachine *TM; 53 54 // -fuse-native. 55 bool AllNative = false; 56 57 bool useNativeFunc(const StringRef F) const; 58 59 // Return a pointer (pointer expr) to the function if function definition with 60 // "FuncName" exists. It may create a new function prototype in pre-link mode. 61 FunctionCallee getFunction(Module *M, const FuncInfo &fInfo); 62 63 bool parseFunctionName(const StringRef &FMangledName, FuncInfo &FInfo); 64 65 bool TDOFold(CallInst *CI, const FuncInfo &FInfo); 66 67 /* Specialized optimizations */ 68 69 // recip (half or native) 70 bool fold_recip(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 71 72 // divide (half or native) 73 bool fold_divide(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 74 75 // pow/powr/pown 76 bool fold_pow(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 77 78 // rootn 79 bool fold_rootn(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 80 81 // fma/mad 82 bool fold_fma_mad(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 83 84 // -fuse-native for sincos 85 bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo); 86 87 // evaluate calls if calls' arguments are constants. 88 bool evaluateScalarMathFunc(const FuncInfo &FInfo, double& Res0, 89 double& Res1, Constant *copr0, Constant *copr1, Constant *copr2); 90 bool evaluateCall(CallInst *aCI, const FuncInfo &FInfo); 91 92 // sqrt 93 bool fold_sqrt(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 94 95 // sin/cos 96 bool fold_sincos(CallInst * CI, IRBuilder<> &B, AliasAnalysis * AA); 97 98 // __read_pipe/__write_pipe 99 bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, 100 const FuncInfo &FInfo); 101 102 // llvm.amdgcn.wavefrontsize 103 bool fold_wavefrontsize(CallInst *CI, IRBuilder<> &B); 104 105 // Get insertion point at entry. 106 BasicBlock::iterator getEntryIns(CallInst * UI); 107 // Insert an Alloc instruction. 108 AllocaInst* insertAlloca(CallInst * UI, IRBuilder<> &B, const char *prefix); 109 // Get a scalar native builtin single argument FP function 110 FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo); 111 112 protected: 113 CallInst *CI; 114 115 bool isUnsafeMath(const CallInst *CI) const; 116 117 void replaceCall(Value *With) { 118 CI->replaceAllUsesWith(With); 119 CI->eraseFromParent(); 120 } 121 122 public: 123 AMDGPULibCalls(const TargetMachine *TM_ = nullptr) : TM(TM_) {} 124 125 bool fold(CallInst *CI, AliasAnalysis *AA = nullptr); 126 127 void initNativeFuncs(); 128 129 // Replace a normal math function call with that native version 130 bool useNative(CallInst *CI); 131 }; 132 133 } // end llvm namespace 134 135 namespace { 136 137 class AMDGPUSimplifyLibCalls : public FunctionPass { 138 139 AMDGPULibCalls Simplifier; 140 141 public: 142 static char ID; // Pass identification 143 144 AMDGPUSimplifyLibCalls(const TargetMachine *TM = nullptr) 145 : FunctionPass(ID), Simplifier(TM) { 146 initializeAMDGPUSimplifyLibCallsPass(*PassRegistry::getPassRegistry()); 147 } 148 149 void getAnalysisUsage(AnalysisUsage &AU) const override { 150 AU.addRequired<AAResultsWrapperPass>(); 151 } 152 153 bool runOnFunction(Function &M) override; 154 }; 155 156 class AMDGPUUseNativeCalls : public FunctionPass { 157 158 AMDGPULibCalls Simplifier; 159 160 public: 161 static char ID; // Pass identification 162 163 AMDGPUUseNativeCalls() : FunctionPass(ID) { 164 initializeAMDGPUUseNativeCallsPass(*PassRegistry::getPassRegistry()); 165 Simplifier.initNativeFuncs(); 166 } 167 168 bool runOnFunction(Function &F) override; 169 }; 170 171 } // end anonymous namespace. 172 173 char AMDGPUSimplifyLibCalls::ID = 0; 174 char AMDGPUUseNativeCalls::ID = 0; 175 176 INITIALIZE_PASS_BEGIN(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", 177 "Simplify well-known AMD library calls", false, false) 178 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 179 INITIALIZE_PASS_END(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", 180 "Simplify well-known AMD library calls", false, false) 181 182 INITIALIZE_PASS(AMDGPUUseNativeCalls, "amdgpu-usenative", 183 "Replace builtin math calls with that native versions.", 184 false, false) 185 186 template <typename IRB> 187 static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg, 188 const Twine &Name = "") { 189 CallInst *R = B.CreateCall(Callee, Arg, Name); 190 if (Function *F = dyn_cast<Function>(Callee.getCallee())) 191 R->setCallingConv(F->getCallingConv()); 192 return R; 193 } 194 195 template <typename IRB> 196 static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1, 197 Value *Arg2, const Twine &Name = "") { 198 CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name); 199 if (Function *F = dyn_cast<Function>(Callee.getCallee())) 200 R->setCallingConv(F->getCallingConv()); 201 return R; 202 } 203 204 // Data structures for table-driven optimizations. 205 // FuncTbl works for both f32 and f64 functions with 1 input argument 206 207 struct TableEntry { 208 double result; 209 double input; 210 }; 211 212 /* a list of {result, input} */ 213 static const TableEntry tbl_acos[] = { 214 {MATH_PI / 2.0, 0.0}, 215 {MATH_PI / 2.0, -0.0}, 216 {0.0, 1.0}, 217 {MATH_PI, -1.0} 218 }; 219 static const TableEntry tbl_acosh[] = { 220 {0.0, 1.0} 221 }; 222 static const TableEntry tbl_acospi[] = { 223 {0.5, 0.0}, 224 {0.5, -0.0}, 225 {0.0, 1.0}, 226 {1.0, -1.0} 227 }; 228 static const TableEntry tbl_asin[] = { 229 {0.0, 0.0}, 230 {-0.0, -0.0}, 231 {MATH_PI / 2.0, 1.0}, 232 {-MATH_PI / 2.0, -1.0} 233 }; 234 static const TableEntry tbl_asinh[] = { 235 {0.0, 0.0}, 236 {-0.0, -0.0} 237 }; 238 static const TableEntry tbl_asinpi[] = { 239 {0.0, 0.0}, 240 {-0.0, -0.0}, 241 {0.5, 1.0}, 242 {-0.5, -1.0} 243 }; 244 static const TableEntry tbl_atan[] = { 245 {0.0, 0.0}, 246 {-0.0, -0.0}, 247 {MATH_PI / 4.0, 1.0}, 248 {-MATH_PI / 4.0, -1.0} 249 }; 250 static const TableEntry tbl_atanh[] = { 251 {0.0, 0.0}, 252 {-0.0, -0.0} 253 }; 254 static const TableEntry tbl_atanpi[] = { 255 {0.0, 0.0}, 256 {-0.0, -0.0}, 257 {0.25, 1.0}, 258 {-0.25, -1.0} 259 }; 260 static const TableEntry tbl_cbrt[] = { 261 {0.0, 0.0}, 262 {-0.0, -0.0}, 263 {1.0, 1.0}, 264 {-1.0, -1.0}, 265 }; 266 static const TableEntry tbl_cos[] = { 267 {1.0, 0.0}, 268 {1.0, -0.0} 269 }; 270 static const TableEntry tbl_cosh[] = { 271 {1.0, 0.0}, 272 {1.0, -0.0} 273 }; 274 static const TableEntry tbl_cospi[] = { 275 {1.0, 0.0}, 276 {1.0, -0.0} 277 }; 278 static const TableEntry tbl_erfc[] = { 279 {1.0, 0.0}, 280 {1.0, -0.0} 281 }; 282 static const TableEntry tbl_erf[] = { 283 {0.0, 0.0}, 284 {-0.0, -0.0} 285 }; 286 static const TableEntry tbl_exp[] = { 287 {1.0, 0.0}, 288 {1.0, -0.0}, 289 {MATH_E, 1.0} 290 }; 291 static const TableEntry tbl_exp2[] = { 292 {1.0, 0.0}, 293 {1.0, -0.0}, 294 {2.0, 1.0} 295 }; 296 static const TableEntry tbl_exp10[] = { 297 {1.0, 0.0}, 298 {1.0, -0.0}, 299 {10.0, 1.0} 300 }; 301 static const TableEntry tbl_expm1[] = { 302 {0.0, 0.0}, 303 {-0.0, -0.0} 304 }; 305 static const TableEntry tbl_log[] = { 306 {0.0, 1.0}, 307 {1.0, MATH_E} 308 }; 309 static const TableEntry tbl_log2[] = { 310 {0.0, 1.0}, 311 {1.0, 2.0} 312 }; 313 static const TableEntry tbl_log10[] = { 314 {0.0, 1.0}, 315 {1.0, 10.0} 316 }; 317 static const TableEntry tbl_rsqrt[] = { 318 {1.0, 1.0}, 319 {MATH_SQRT1_2, 2.0} 320 }; 321 static const TableEntry tbl_sin[] = { 322 {0.0, 0.0}, 323 {-0.0, -0.0} 324 }; 325 static const TableEntry tbl_sinh[] = { 326 {0.0, 0.0}, 327 {-0.0, -0.0} 328 }; 329 static const TableEntry tbl_sinpi[] = { 330 {0.0, 0.0}, 331 {-0.0, -0.0} 332 }; 333 static const TableEntry tbl_sqrt[] = { 334 {0.0, 0.0}, 335 {1.0, 1.0}, 336 {MATH_SQRT2, 2.0} 337 }; 338 static const TableEntry tbl_tan[] = { 339 {0.0, 0.0}, 340 {-0.0, -0.0} 341 }; 342 static const TableEntry tbl_tanh[] = { 343 {0.0, 0.0}, 344 {-0.0, -0.0} 345 }; 346 static const TableEntry tbl_tanpi[] = { 347 {0.0, 0.0}, 348 {-0.0, -0.0} 349 }; 350 static const TableEntry tbl_tgamma[] = { 351 {1.0, 1.0}, 352 {1.0, 2.0}, 353 {2.0, 3.0}, 354 {6.0, 4.0} 355 }; 356 357 static bool HasNative(AMDGPULibFunc::EFuncId id) { 358 switch(id) { 359 case AMDGPULibFunc::EI_DIVIDE: 360 case AMDGPULibFunc::EI_COS: 361 case AMDGPULibFunc::EI_EXP: 362 case AMDGPULibFunc::EI_EXP2: 363 case AMDGPULibFunc::EI_EXP10: 364 case AMDGPULibFunc::EI_LOG: 365 case AMDGPULibFunc::EI_LOG2: 366 case AMDGPULibFunc::EI_LOG10: 367 case AMDGPULibFunc::EI_POWR: 368 case AMDGPULibFunc::EI_RECIP: 369 case AMDGPULibFunc::EI_RSQRT: 370 case AMDGPULibFunc::EI_SIN: 371 case AMDGPULibFunc::EI_SINCOS: 372 case AMDGPULibFunc::EI_SQRT: 373 case AMDGPULibFunc::EI_TAN: 374 return true; 375 default:; 376 } 377 return false; 378 } 379 380 using TableRef = ArrayRef<TableEntry>; 381 382 static TableRef getOptTable(AMDGPULibFunc::EFuncId id) { 383 switch(id) { 384 case AMDGPULibFunc::EI_ACOS: return TableRef(tbl_acos); 385 case AMDGPULibFunc::EI_ACOSH: return TableRef(tbl_acosh); 386 case AMDGPULibFunc::EI_ACOSPI: return TableRef(tbl_acospi); 387 case AMDGPULibFunc::EI_ASIN: return TableRef(tbl_asin); 388 case AMDGPULibFunc::EI_ASINH: return TableRef(tbl_asinh); 389 case AMDGPULibFunc::EI_ASINPI: return TableRef(tbl_asinpi); 390 case AMDGPULibFunc::EI_ATAN: return TableRef(tbl_atan); 391 case AMDGPULibFunc::EI_ATANH: return TableRef(tbl_atanh); 392 case AMDGPULibFunc::EI_ATANPI: return TableRef(tbl_atanpi); 393 case AMDGPULibFunc::EI_CBRT: return TableRef(tbl_cbrt); 394 case AMDGPULibFunc::EI_NCOS: 395 case AMDGPULibFunc::EI_COS: return TableRef(tbl_cos); 396 case AMDGPULibFunc::EI_COSH: return TableRef(tbl_cosh); 397 case AMDGPULibFunc::EI_COSPI: return TableRef(tbl_cospi); 398 case AMDGPULibFunc::EI_ERFC: return TableRef(tbl_erfc); 399 case AMDGPULibFunc::EI_ERF: return TableRef(tbl_erf); 400 case AMDGPULibFunc::EI_EXP: return TableRef(tbl_exp); 401 case AMDGPULibFunc::EI_NEXP2: 402 case AMDGPULibFunc::EI_EXP2: return TableRef(tbl_exp2); 403 case AMDGPULibFunc::EI_EXP10: return TableRef(tbl_exp10); 404 case AMDGPULibFunc::EI_EXPM1: return TableRef(tbl_expm1); 405 case AMDGPULibFunc::EI_LOG: return TableRef(tbl_log); 406 case AMDGPULibFunc::EI_NLOG2: 407 case AMDGPULibFunc::EI_LOG2: return TableRef(tbl_log2); 408 case AMDGPULibFunc::EI_LOG10: return TableRef(tbl_log10); 409 case AMDGPULibFunc::EI_NRSQRT: 410 case AMDGPULibFunc::EI_RSQRT: return TableRef(tbl_rsqrt); 411 case AMDGPULibFunc::EI_NSIN: 412 case AMDGPULibFunc::EI_SIN: return TableRef(tbl_sin); 413 case AMDGPULibFunc::EI_SINH: return TableRef(tbl_sinh); 414 case AMDGPULibFunc::EI_SINPI: return TableRef(tbl_sinpi); 415 case AMDGPULibFunc::EI_NSQRT: 416 case AMDGPULibFunc::EI_SQRT: return TableRef(tbl_sqrt); 417 case AMDGPULibFunc::EI_TAN: return TableRef(tbl_tan); 418 case AMDGPULibFunc::EI_TANH: return TableRef(tbl_tanh); 419 case AMDGPULibFunc::EI_TANPI: return TableRef(tbl_tanpi); 420 case AMDGPULibFunc::EI_TGAMMA: return TableRef(tbl_tgamma); 421 default:; 422 } 423 return TableRef(); 424 } 425 426 static inline int getVecSize(const AMDGPULibFunc& FInfo) { 427 return FInfo.getLeads()[0].VectorSize; 428 } 429 430 static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) { 431 return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType; 432 } 433 434 FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) { 435 // If we are doing PreLinkOpt, the function is external. So it is safe to 436 // use getOrInsertFunction() at this stage. 437 438 return EnablePreLink ? AMDGPULibFunc::getOrInsertFunction(M, fInfo) 439 : AMDGPULibFunc::getFunction(M, fInfo); 440 } 441 442 bool AMDGPULibCalls::parseFunctionName(const StringRef &FMangledName, 443 FuncInfo &FInfo) { 444 return AMDGPULibFunc::parse(FMangledName, FInfo); 445 } 446 447 bool AMDGPULibCalls::isUnsafeMath(const CallInst *CI) const { 448 if (auto Op = dyn_cast<FPMathOperator>(CI)) 449 if (Op->isFast()) 450 return true; 451 const Function *F = CI->getParent()->getParent(); 452 Attribute Attr = F->getFnAttribute("unsafe-fp-math"); 453 return Attr.getValueAsBool(); 454 } 455 456 bool AMDGPULibCalls::useNativeFunc(const StringRef F) const { 457 return AllNative || llvm::is_contained(UseNative, F); 458 } 459 460 void AMDGPULibCalls::initNativeFuncs() { 461 AllNative = useNativeFunc("all") || 462 (UseNative.getNumOccurrences() && UseNative.size() == 1 && 463 UseNative.begin()->empty()); 464 } 465 466 bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) { 467 bool native_sin = useNativeFunc("sin"); 468 bool native_cos = useNativeFunc("cos"); 469 470 if (native_sin && native_cos) { 471 Module *M = aCI->getModule(); 472 Value *opr0 = aCI->getArgOperand(0); 473 474 AMDGPULibFunc nf; 475 nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType; 476 nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize; 477 478 nf.setPrefix(AMDGPULibFunc::NATIVE); 479 nf.setId(AMDGPULibFunc::EI_SIN); 480 FunctionCallee sinExpr = getFunction(M, nf); 481 482 nf.setPrefix(AMDGPULibFunc::NATIVE); 483 nf.setId(AMDGPULibFunc::EI_COS); 484 FunctionCallee cosExpr = getFunction(M, nf); 485 if (sinExpr && cosExpr) { 486 Value *sinval = CallInst::Create(sinExpr, opr0, "splitsin", aCI); 487 Value *cosval = CallInst::Create(cosExpr, opr0, "splitcos", aCI); 488 new StoreInst(cosval, aCI->getArgOperand(1), aCI); 489 490 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI 491 << " with native version of sin/cos"); 492 493 replaceCall(sinval); 494 return true; 495 } 496 } 497 return false; 498 } 499 500 bool AMDGPULibCalls::useNative(CallInst *aCI) { 501 CI = aCI; 502 Function *Callee = aCI->getCalledFunction(); 503 if (!Callee) 504 return false; 505 506 FuncInfo FInfo; 507 if (!parseFunctionName(Callee->getName(), FInfo) || !FInfo.isMangled() || 508 FInfo.getPrefix() != AMDGPULibFunc::NOPFX || 509 getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()) || 510 !(AllNative || useNativeFunc(FInfo.getName()))) { 511 return false; 512 } 513 514 if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS) 515 return sincosUseNative(aCI, FInfo); 516 517 FInfo.setPrefix(AMDGPULibFunc::NATIVE); 518 FunctionCallee F = getFunction(aCI->getModule(), FInfo); 519 if (!F) 520 return false; 521 522 aCI->setCalledFunction(F); 523 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI 524 << " with native version"); 525 return true; 526 } 527 528 // Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe 529 // builtin, with appended type size and alignment arguments, where 2 or 4 530 // indicates the original number of arguments. The library has optimized version 531 // of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same 532 // power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N 533 // for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ..., 534 // 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4. 535 bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, 536 const FuncInfo &FInfo) { 537 auto *Callee = CI->getCalledFunction(); 538 if (!Callee->isDeclaration()) 539 return false; 540 541 assert(Callee->hasName() && "Invalid read_pipe/write_pipe function"); 542 auto *M = Callee->getParent(); 543 std::string Name = std::string(Callee->getName()); 544 auto NumArg = CI->arg_size(); 545 if (NumArg != 4 && NumArg != 6) 546 return false; 547 auto *PacketSize = CI->getArgOperand(NumArg - 2); 548 auto *PacketAlign = CI->getArgOperand(NumArg - 1); 549 if (!isa<ConstantInt>(PacketSize) || !isa<ConstantInt>(PacketAlign)) 550 return false; 551 unsigned Size = cast<ConstantInt>(PacketSize)->getZExtValue(); 552 Align Alignment = cast<ConstantInt>(PacketAlign)->getAlignValue(); 553 if (Alignment != Size) 554 return false; 555 556 unsigned PtrArgLoc = CI->arg_size() - 3; 557 Value *PtrArg = CI->getArgOperand(PtrArgLoc); 558 Type *PtrTy = PtrArg->getType(); 559 560 SmallVector<llvm::Type *, 6> ArgTys; 561 for (unsigned I = 0; I != PtrArgLoc; ++I) 562 ArgTys.push_back(CI->getArgOperand(I)->getType()); 563 ArgTys.push_back(PtrTy); 564 565 Name = Name + "_" + std::to_string(Size); 566 auto *FTy = FunctionType::get(Callee->getReturnType(), 567 ArrayRef<Type *>(ArgTys), false); 568 AMDGPULibFunc NewLibFunc(Name, FTy); 569 FunctionCallee F = AMDGPULibFunc::getOrInsertFunction(M, NewLibFunc); 570 if (!F) 571 return false; 572 573 auto *BCast = B.CreatePointerCast(PtrArg, PtrTy); 574 SmallVector<Value *, 6> Args; 575 for (unsigned I = 0; I != PtrArgLoc; ++I) 576 Args.push_back(CI->getArgOperand(I)); 577 Args.push_back(BCast); 578 579 auto *NCI = B.CreateCall(F, Args); 580 NCI->setAttributes(CI->getAttributes()); 581 CI->replaceAllUsesWith(NCI); 582 CI->dropAllReferences(); 583 CI->eraseFromParent(); 584 585 return true; 586 } 587 588 // This function returns false if no change; return true otherwise. 589 bool AMDGPULibCalls::fold(CallInst *CI, AliasAnalysis *AA) { 590 this->CI = CI; 591 Function *Callee = CI->getCalledFunction(); 592 // Ignore indirect calls. 593 if (!Callee) 594 return false; 595 596 IRBuilder<> B(CI); 597 switch (Callee->getIntrinsicID()) { 598 case Intrinsic::not_intrinsic: 599 break; 600 case Intrinsic::amdgcn_wavefrontsize: 601 return !EnablePreLink && fold_wavefrontsize(CI, B); 602 default: 603 return false; 604 } 605 606 FuncInfo FInfo; 607 if (!parseFunctionName(Callee->getName(), FInfo)) 608 return false; 609 610 // Further check the number of arguments to see if they match. 611 if (CI->arg_size() != FInfo.getNumArgs()) 612 return false; 613 614 LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << '\n'); 615 616 if (TDOFold(CI, FInfo)) 617 return true; 618 619 // Under unsafe-math, evaluate calls if possible. 620 // According to Brian Sumner, we can do this for all f32 function calls 621 // using host's double function calls. 622 if (isUnsafeMath(CI) && evaluateCall(CI, FInfo)) 623 return true; 624 625 // Copy fast flags from the original call. 626 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(CI)) 627 B.setFastMathFlags(FPOp->getFastMathFlags()); 628 629 // Specialized optimizations for each function call 630 switch (FInfo.getId()) { 631 case AMDGPULibFunc::EI_RECIP: 632 // skip vector function 633 assert ((FInfo.getPrefix() == AMDGPULibFunc::NATIVE || 634 FInfo.getPrefix() == AMDGPULibFunc::HALF) && 635 "recip must be an either native or half function"); 636 return (getVecSize(FInfo) != 1) ? false : fold_recip(CI, B, FInfo); 637 638 case AMDGPULibFunc::EI_DIVIDE: 639 // skip vector function 640 assert ((FInfo.getPrefix() == AMDGPULibFunc::NATIVE || 641 FInfo.getPrefix() == AMDGPULibFunc::HALF) && 642 "divide must be an either native or half function"); 643 return (getVecSize(FInfo) != 1) ? false : fold_divide(CI, B, FInfo); 644 645 case AMDGPULibFunc::EI_POW: 646 case AMDGPULibFunc::EI_POWR: 647 case AMDGPULibFunc::EI_POWN: 648 return fold_pow(CI, B, FInfo); 649 650 case AMDGPULibFunc::EI_ROOTN: 651 // skip vector function 652 return (getVecSize(FInfo) != 1) ? false : fold_rootn(CI, B, FInfo); 653 654 case AMDGPULibFunc::EI_FMA: 655 case AMDGPULibFunc::EI_MAD: 656 case AMDGPULibFunc::EI_NFMA: 657 // skip vector function 658 return (getVecSize(FInfo) != 1) ? false : fold_fma_mad(CI, B, FInfo); 659 660 case AMDGPULibFunc::EI_SQRT: 661 return isUnsafeMath(CI) && fold_sqrt(CI, B, FInfo); 662 case AMDGPULibFunc::EI_COS: 663 case AMDGPULibFunc::EI_SIN: 664 if ((getArgType(FInfo) == AMDGPULibFunc::F32 || 665 getArgType(FInfo) == AMDGPULibFunc::F64) 666 && (FInfo.getPrefix() == AMDGPULibFunc::NOPFX)) 667 return fold_sincos(CI, B, AA); 668 669 break; 670 case AMDGPULibFunc::EI_READ_PIPE_2: 671 case AMDGPULibFunc::EI_READ_PIPE_4: 672 case AMDGPULibFunc::EI_WRITE_PIPE_2: 673 case AMDGPULibFunc::EI_WRITE_PIPE_4: 674 return fold_read_write_pipe(CI, B, FInfo); 675 676 default: 677 break; 678 } 679 680 return false; 681 } 682 683 bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) { 684 // Table-Driven optimization 685 const TableRef tr = getOptTable(FInfo.getId()); 686 if (tr.empty()) 687 return false; 688 689 int const sz = (int)tr.size(); 690 Value *opr0 = CI->getArgOperand(0); 691 692 if (getVecSize(FInfo) > 1) { 693 if (ConstantDataVector *CV = dyn_cast<ConstantDataVector>(opr0)) { 694 SmallVector<double, 0> DVal; 695 for (int eltNo = 0; eltNo < getVecSize(FInfo); ++eltNo) { 696 ConstantFP *eltval = dyn_cast<ConstantFP>( 697 CV->getElementAsConstant((unsigned)eltNo)); 698 assert(eltval && "Non-FP arguments in math function!"); 699 bool found = false; 700 for (int i=0; i < sz; ++i) { 701 if (eltval->isExactlyValue(tr[i].input)) { 702 DVal.push_back(tr[i].result); 703 found = true; 704 break; 705 } 706 } 707 if (!found) { 708 // This vector constants not handled yet. 709 return false; 710 } 711 } 712 LLVMContext &context = CI->getParent()->getParent()->getContext(); 713 Constant *nval; 714 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 715 SmallVector<float, 0> FVal; 716 for (unsigned i = 0; i < DVal.size(); ++i) { 717 FVal.push_back((float)DVal[i]); 718 } 719 ArrayRef<float> tmp(FVal); 720 nval = ConstantDataVector::get(context, tmp); 721 } else { // F64 722 ArrayRef<double> tmp(DVal); 723 nval = ConstantDataVector::get(context, tmp); 724 } 725 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 726 replaceCall(nval); 727 return true; 728 } 729 } else { 730 // Scalar version 731 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) { 732 for (int i = 0; i < sz; ++i) { 733 if (CF->isExactlyValue(tr[i].input)) { 734 Value *nval = ConstantFP::get(CF->getType(), tr[i].result); 735 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 736 replaceCall(nval); 737 return true; 738 } 739 } 740 } 741 } 742 743 return false; 744 } 745 746 // [native_]half_recip(c) ==> 1.0/c 747 bool AMDGPULibCalls::fold_recip(CallInst *CI, IRBuilder<> &B, 748 const FuncInfo &FInfo) { 749 Value *opr0 = CI->getArgOperand(0); 750 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) { 751 // Just create a normal div. Later, InstCombine will be able 752 // to compute the divide into a constant (avoid check float infinity 753 // or subnormal at this point). 754 Value *nval = B.CreateFDiv(ConstantFP::get(CF->getType(), 1.0), 755 opr0, 756 "recip2div"); 757 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 758 replaceCall(nval); 759 return true; 760 } 761 return false; 762 } 763 764 // [native_]half_divide(x, c) ==> x/c 765 bool AMDGPULibCalls::fold_divide(CallInst *CI, IRBuilder<> &B, 766 const FuncInfo &FInfo) { 767 Value *opr0 = CI->getArgOperand(0); 768 Value *opr1 = CI->getArgOperand(1); 769 ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0); 770 ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1); 771 772 if ((CF0 && CF1) || // both are constants 773 (CF1 && (getArgType(FInfo) == AMDGPULibFunc::F32))) 774 // CF1 is constant && f32 divide 775 { 776 Value *nval1 = B.CreateFDiv(ConstantFP::get(opr1->getType(), 1.0), 777 opr1, "__div2recip"); 778 Value *nval = B.CreateFMul(opr0, nval1, "__div2mul"); 779 replaceCall(nval); 780 return true; 781 } 782 return false; 783 } 784 785 namespace llvm { 786 static double log2(double V) { 787 #if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L 788 return ::log2(V); 789 #else 790 return log(V) / numbers::ln2; 791 #endif 792 } 793 } 794 795 bool AMDGPULibCalls::fold_pow(CallInst *CI, IRBuilder<> &B, 796 const FuncInfo &FInfo) { 797 assert((FInfo.getId() == AMDGPULibFunc::EI_POW || 798 FInfo.getId() == AMDGPULibFunc::EI_POWR || 799 FInfo.getId() == AMDGPULibFunc::EI_POWN) && 800 "fold_pow: encounter a wrong function call"); 801 802 Value *opr0, *opr1; 803 ConstantFP *CF; 804 ConstantInt *CINT; 805 ConstantAggregateZero *CZero; 806 Type *eltType; 807 808 opr0 = CI->getArgOperand(0); 809 opr1 = CI->getArgOperand(1); 810 CZero = dyn_cast<ConstantAggregateZero>(opr1); 811 if (getVecSize(FInfo) == 1) { 812 eltType = opr0->getType(); 813 CF = dyn_cast<ConstantFP>(opr1); 814 CINT = dyn_cast<ConstantInt>(opr1); 815 } else { 816 VectorType *VTy = dyn_cast<VectorType>(opr0->getType()); 817 assert(VTy && "Oprand of vector function should be of vectortype"); 818 eltType = VTy->getElementType(); 819 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1); 820 821 // Now, only Handle vector const whose elements have the same value. 822 CF = CDV ? dyn_cast_or_null<ConstantFP>(CDV->getSplatValue()) : nullptr; 823 CINT = CDV ? dyn_cast_or_null<ConstantInt>(CDV->getSplatValue()) : nullptr; 824 } 825 826 // No unsafe math , no constant argument, do nothing 827 if (!isUnsafeMath(CI) && !CF && !CINT && !CZero) 828 return false; 829 830 // 0x1111111 means that we don't do anything for this call. 831 int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111); 832 833 if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0) || CZero) { 834 // pow/powr/pown(x, 0) == 1 835 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1\n"); 836 Constant *cnval = ConstantFP::get(eltType, 1.0); 837 if (getVecSize(FInfo) > 1) { 838 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 839 } 840 replaceCall(cnval); 841 return true; 842 } 843 if ((CF && CF->isExactlyValue(1.0)) || (CINT && ci_opr1 == 1)) { 844 // pow/powr/pown(x, 1.0) = x 845 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n"); 846 replaceCall(opr0); 847 return true; 848 } 849 if ((CF && CF->isExactlyValue(2.0)) || (CINT && ci_opr1 == 2)) { 850 // pow/powr/pown(x, 2.0) = x*x 851 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * " << *opr0 852 << "\n"); 853 Value *nval = B.CreateFMul(opr0, opr0, "__pow2"); 854 replaceCall(nval); 855 return true; 856 } 857 if ((CF && CF->isExactlyValue(-1.0)) || (CINT && ci_opr1 == -1)) { 858 // pow/powr/pown(x, -1.0) = 1.0/x 859 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1 / " << *opr0 << "\n"); 860 Constant *cnval = ConstantFP::get(eltType, 1.0); 861 if (getVecSize(FInfo) > 1) { 862 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 863 } 864 Value *nval = B.CreateFDiv(cnval, opr0, "__powrecip"); 865 replaceCall(nval); 866 return true; 867 } 868 869 Module *M = CI->getModule(); 870 if (CF && (CF->isExactlyValue(0.5) || CF->isExactlyValue(-0.5))) { 871 // pow[r](x, [-]0.5) = sqrt(x) 872 bool issqrt = CF->isExactlyValue(0.5); 873 if (FunctionCallee FPExpr = 874 getFunction(M, AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT 875 : AMDGPULibFunc::EI_RSQRT, 876 FInfo))) { 877 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " 878 << FInfo.getName().c_str() << "(" << *opr0 << ")\n"); 879 Value *nval = CreateCallEx(B,FPExpr, opr0, issqrt ? "__pow2sqrt" 880 : "__pow2rsqrt"); 881 replaceCall(nval); 882 return true; 883 } 884 } 885 886 if (!isUnsafeMath(CI)) 887 return false; 888 889 // Unsafe Math optimization 890 891 // Remember that ci_opr1 is set if opr1 is integral 892 if (CF) { 893 double dval = (getArgType(FInfo) == AMDGPULibFunc::F32) 894 ? (double)CF->getValueAPF().convertToFloat() 895 : CF->getValueAPF().convertToDouble(); 896 int ival = (int)dval; 897 if ((double)ival == dval) { 898 ci_opr1 = ival; 899 } else 900 ci_opr1 = 0x11111111; 901 } 902 903 // pow/powr/pown(x, c) = [1/](x*x*..x); where 904 // trunc(c) == c && the number of x == c && |c| <= 12 905 unsigned abs_opr1 = (ci_opr1 < 0) ? -ci_opr1 : ci_opr1; 906 if (abs_opr1 <= 12) { 907 Constant *cnval; 908 Value *nval; 909 if (abs_opr1 == 0) { 910 cnval = ConstantFP::get(eltType, 1.0); 911 if (getVecSize(FInfo) > 1) { 912 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 913 } 914 nval = cnval; 915 } else { 916 Value *valx2 = nullptr; 917 nval = nullptr; 918 while (abs_opr1 > 0) { 919 valx2 = valx2 ? B.CreateFMul(valx2, valx2, "__powx2") : opr0; 920 if (abs_opr1 & 1) { 921 nval = nval ? B.CreateFMul(nval, valx2, "__powprod") : valx2; 922 } 923 abs_opr1 >>= 1; 924 } 925 } 926 927 if (ci_opr1 < 0) { 928 cnval = ConstantFP::get(eltType, 1.0); 929 if (getVecSize(FInfo) > 1) { 930 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 931 } 932 nval = B.CreateFDiv(cnval, nval, "__1powprod"); 933 } 934 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " 935 << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr0 936 << ")\n"); 937 replaceCall(nval); 938 return true; 939 } 940 941 // powr ---> exp2(y * log2(x)) 942 // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31)) 943 FunctionCallee ExpExpr = 944 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo)); 945 if (!ExpExpr) 946 return false; 947 948 bool needlog = false; 949 bool needabs = false; 950 bool needcopysign = false; 951 Constant *cnval = nullptr; 952 if (getVecSize(FInfo) == 1) { 953 CF = dyn_cast<ConstantFP>(opr0); 954 955 if (CF) { 956 double V = (getArgType(FInfo) == AMDGPULibFunc::F32) 957 ? (double)CF->getValueAPF().convertToFloat() 958 : CF->getValueAPF().convertToDouble(); 959 960 V = log2(std::abs(V)); 961 cnval = ConstantFP::get(eltType, V); 962 needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR) && 963 CF->isNegative(); 964 } else { 965 needlog = true; 966 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR && 967 (!CF || CF->isNegative()); 968 } 969 } else { 970 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr0); 971 972 if (!CDV) { 973 needlog = true; 974 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR; 975 } else { 976 assert ((int)CDV->getNumElements() == getVecSize(FInfo) && 977 "Wrong vector size detected"); 978 979 SmallVector<double, 0> DVal; 980 for (int i=0; i < getVecSize(FInfo); ++i) { 981 double V = (getArgType(FInfo) == AMDGPULibFunc::F32) 982 ? (double)CDV->getElementAsFloat(i) 983 : CDV->getElementAsDouble(i); 984 if (V < 0.0) needcopysign = true; 985 V = log2(std::abs(V)); 986 DVal.push_back(V); 987 } 988 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 989 SmallVector<float, 0> FVal; 990 for (unsigned i=0; i < DVal.size(); ++i) { 991 FVal.push_back((float)DVal[i]); 992 } 993 ArrayRef<float> tmp(FVal); 994 cnval = ConstantDataVector::get(M->getContext(), tmp); 995 } else { 996 ArrayRef<double> tmp(DVal); 997 cnval = ConstantDataVector::get(M->getContext(), tmp); 998 } 999 } 1000 } 1001 1002 if (needcopysign && (FInfo.getId() == AMDGPULibFunc::EI_POW)) { 1003 // We cannot handle corner cases for a general pow() function, give up 1004 // unless y is a constant integral value. Then proceed as if it were pown. 1005 if (getVecSize(FInfo) == 1) { 1006 if (const ConstantFP *CF = dyn_cast<ConstantFP>(opr1)) { 1007 double y = (getArgType(FInfo) == AMDGPULibFunc::F32) 1008 ? (double)CF->getValueAPF().convertToFloat() 1009 : CF->getValueAPF().convertToDouble(); 1010 if (y != (double)(int64_t)y) 1011 return false; 1012 } else 1013 return false; 1014 } else { 1015 if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1)) { 1016 for (int i=0; i < getVecSize(FInfo); ++i) { 1017 double y = (getArgType(FInfo) == AMDGPULibFunc::F32) 1018 ? (double)CDV->getElementAsFloat(i) 1019 : CDV->getElementAsDouble(i); 1020 if (y != (double)(int64_t)y) 1021 return false; 1022 } 1023 } else 1024 return false; 1025 } 1026 } 1027 1028 Value *nval; 1029 if (needabs) { 1030 FunctionCallee AbsExpr = 1031 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_FABS, FInfo)); 1032 if (!AbsExpr) 1033 return false; 1034 nval = CreateCallEx(B, AbsExpr, opr0, "__fabs"); 1035 } else { 1036 nval = cnval ? cnval : opr0; 1037 } 1038 if (needlog) { 1039 FunctionCallee LogExpr = 1040 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo)); 1041 if (!LogExpr) 1042 return false; 1043 nval = CreateCallEx(B,LogExpr, nval, "__log2"); 1044 } 1045 1046 if (FInfo.getId() == AMDGPULibFunc::EI_POWN) { 1047 // convert int(32) to fp(f32 or f64) 1048 opr1 = B.CreateSIToFP(opr1, nval->getType(), "pownI2F"); 1049 } 1050 nval = B.CreateFMul(opr1, nval, "__ylogx"); 1051 nval = CreateCallEx(B,ExpExpr, nval, "__exp2"); 1052 1053 if (needcopysign) { 1054 Value *opr_n; 1055 Type* rTy = opr0->getType(); 1056 Type* nTyS = eltType->isDoubleTy() ? B.getInt64Ty() : B.getInt32Ty(); 1057 Type *nTy = nTyS; 1058 if (const auto *vTy = dyn_cast<FixedVectorType>(rTy)) 1059 nTy = FixedVectorType::get(nTyS, vTy); 1060 unsigned size = nTy->getScalarSizeInBits(); 1061 opr_n = CI->getArgOperand(1); 1062 if (opr_n->getType()->isIntegerTy()) 1063 opr_n = B.CreateZExtOrBitCast(opr_n, nTy, "__ytou"); 1064 else 1065 opr_n = B.CreateFPToSI(opr1, nTy, "__ytou"); 1066 1067 Value *sign = B.CreateShl(opr_n, size-1, "__yeven"); 1068 sign = B.CreateAnd(B.CreateBitCast(opr0, nTy), sign, "__pow_sign"); 1069 nval = B.CreateOr(B.CreateBitCast(nval, nTy), sign); 1070 nval = B.CreateBitCast(nval, opr0->getType()); 1071 } 1072 1073 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " 1074 << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n"); 1075 replaceCall(nval); 1076 1077 return true; 1078 } 1079 1080 bool AMDGPULibCalls::fold_rootn(CallInst *CI, IRBuilder<> &B, 1081 const FuncInfo &FInfo) { 1082 Value *opr0 = CI->getArgOperand(0); 1083 Value *opr1 = CI->getArgOperand(1); 1084 1085 ConstantInt *CINT = dyn_cast<ConstantInt>(opr1); 1086 if (!CINT) { 1087 return false; 1088 } 1089 int ci_opr1 = (int)CINT->getSExtValue(); 1090 if (ci_opr1 == 1) { // rootn(x, 1) = x 1091 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n"); 1092 replaceCall(opr0); 1093 return true; 1094 } 1095 if (ci_opr1 == 2) { // rootn(x, 2) = sqrt(x) 1096 Module *M = CI->getModule(); 1097 if (FunctionCallee FPExpr = 1098 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) { 1099 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> sqrt(" << *opr0 << ")\n"); 1100 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2sqrt"); 1101 replaceCall(nval); 1102 return true; 1103 } 1104 } else if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x) 1105 Module *M = CI->getModule(); 1106 if (FunctionCallee FPExpr = 1107 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) { 1108 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> cbrt(" << *opr0 << ")\n"); 1109 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2cbrt"); 1110 replaceCall(nval); 1111 return true; 1112 } 1113 } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x 1114 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1.0 / " << *opr0 << "\n"); 1115 Value *nval = B.CreateFDiv(ConstantFP::get(opr0->getType(), 1.0), 1116 opr0, 1117 "__rootn2div"); 1118 replaceCall(nval); 1119 return true; 1120 } else if (ci_opr1 == -2) { // rootn(x, -2) = rsqrt(x) 1121 Module *M = CI->getModule(); 1122 if (FunctionCallee FPExpr = 1123 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_RSQRT, FInfo))) { 1124 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> rsqrt(" << *opr0 1125 << ")\n"); 1126 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2rsqrt"); 1127 replaceCall(nval); 1128 return true; 1129 } 1130 } 1131 return false; 1132 } 1133 1134 bool AMDGPULibCalls::fold_fma_mad(CallInst *CI, IRBuilder<> &B, 1135 const FuncInfo &FInfo) { 1136 Value *opr0 = CI->getArgOperand(0); 1137 Value *opr1 = CI->getArgOperand(1); 1138 Value *opr2 = CI->getArgOperand(2); 1139 1140 ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0); 1141 ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1); 1142 if ((CF0 && CF0->isZero()) || (CF1 && CF1->isZero())) { 1143 // fma/mad(a, b, c) = c if a=0 || b=0 1144 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr2 << "\n"); 1145 replaceCall(opr2); 1146 return true; 1147 } 1148 if (CF0 && CF0->isExactlyValue(1.0f)) { 1149 // fma/mad(a, b, c) = b+c if a=1 1150 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr1 << " + " << *opr2 1151 << "\n"); 1152 Value *nval = B.CreateFAdd(opr1, opr2, "fmaadd"); 1153 replaceCall(nval); 1154 return true; 1155 } 1156 if (CF1 && CF1->isExactlyValue(1.0f)) { 1157 // fma/mad(a, b, c) = a+c if b=1 1158 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " + " << *opr2 1159 << "\n"); 1160 Value *nval = B.CreateFAdd(opr0, opr2, "fmaadd"); 1161 replaceCall(nval); 1162 return true; 1163 } 1164 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr2)) { 1165 if (CF->isZero()) { 1166 // fma/mad(a, b, c) = a*b if c=0 1167 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * " 1168 << *opr1 << "\n"); 1169 Value *nval = B.CreateFMul(opr0, opr1, "fmamul"); 1170 replaceCall(nval); 1171 return true; 1172 } 1173 } 1174 1175 return false; 1176 } 1177 1178 // Get a scalar native builtin single argument FP function 1179 FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M, 1180 const FuncInfo &FInfo) { 1181 if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId())) 1182 return nullptr; 1183 FuncInfo nf = FInfo; 1184 nf.setPrefix(AMDGPULibFunc::NATIVE); 1185 return getFunction(M, nf); 1186 } 1187 1188 // fold sqrt -> native_sqrt (x) 1189 bool AMDGPULibCalls::fold_sqrt(CallInst *CI, IRBuilder<> &B, 1190 const FuncInfo &FInfo) { 1191 if (getArgType(FInfo) == AMDGPULibFunc::F32 && (getVecSize(FInfo) == 1) && 1192 (FInfo.getPrefix() != AMDGPULibFunc::NATIVE)) { 1193 if (FunctionCallee FPExpr = getNativeFunction( 1194 CI->getModule(), AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) { 1195 Value *opr0 = CI->getArgOperand(0); 1196 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " 1197 << "sqrt(" << *opr0 << ")\n"); 1198 Value *nval = CreateCallEx(B,FPExpr, opr0, "__sqrt"); 1199 replaceCall(nval); 1200 return true; 1201 } 1202 } 1203 return false; 1204 } 1205 1206 // fold sin, cos -> sincos. 1207 bool AMDGPULibCalls::fold_sincos(CallInst *CI, IRBuilder<> &B, 1208 AliasAnalysis *AA) { 1209 AMDGPULibFunc fInfo; 1210 if (!AMDGPULibFunc::parse(CI->getCalledFunction()->getName(), fInfo)) 1211 return false; 1212 1213 assert(fInfo.getId() == AMDGPULibFunc::EI_SIN || 1214 fInfo.getId() == AMDGPULibFunc::EI_COS); 1215 bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN; 1216 1217 Value *CArgVal = CI->getArgOperand(0); 1218 BasicBlock * const CBB = CI->getParent(); 1219 1220 int const MaxScan = 30; 1221 bool Changed = false; 1222 1223 { // fold in load value. 1224 LoadInst *LI = dyn_cast<LoadInst>(CArgVal); 1225 if (LI && LI->getParent() == CBB) { 1226 BasicBlock::iterator BBI = LI->getIterator(); 1227 Value *AvailableVal = FindAvailableLoadedValue(LI, CBB, BBI, MaxScan, AA); 1228 if (AvailableVal) { 1229 Changed = true; 1230 CArgVal->replaceAllUsesWith(AvailableVal); 1231 if (CArgVal->getNumUses() == 0) 1232 LI->eraseFromParent(); 1233 CArgVal = CI->getArgOperand(0); 1234 } 1235 } 1236 } 1237 1238 Module *M = CI->getModule(); 1239 fInfo.setId(isSin ? AMDGPULibFunc::EI_COS : AMDGPULibFunc::EI_SIN); 1240 std::string const PairName = fInfo.mangle(); 1241 1242 CallInst *UI = nullptr; 1243 for (User* U : CArgVal->users()) { 1244 CallInst *XI = dyn_cast_or_null<CallInst>(U); 1245 if (!XI || XI == CI || XI->getParent() != CBB) 1246 continue; 1247 1248 Function *UCallee = XI->getCalledFunction(); 1249 if (!UCallee || !UCallee->getName().equals(PairName)) 1250 continue; 1251 1252 BasicBlock::iterator BBI = CI->getIterator(); 1253 if (BBI == CI->getParent()->begin()) 1254 break; 1255 --BBI; 1256 for (int I = MaxScan; I > 0 && BBI != CBB->begin(); --BBI, --I) { 1257 if (cast<Instruction>(BBI) == XI) { 1258 UI = XI; 1259 break; 1260 } 1261 } 1262 if (UI) break; 1263 } 1264 1265 if (!UI) 1266 return Changed; 1267 1268 // Merge the sin and cos. 1269 1270 // for OpenCL 2.0 we have only generic implementation of sincos 1271 // function. 1272 AMDGPULibFunc nf(AMDGPULibFunc::EI_SINCOS, fInfo); 1273 nf.getLeads()[0].PtrKind = AMDGPULibFunc::getEPtrKindFromAddrSpace(AMDGPUAS::FLAT_ADDRESS); 1274 FunctionCallee Fsincos = getFunction(M, nf); 1275 if (!Fsincos) 1276 return Changed; 1277 1278 BasicBlock::iterator ItOld = B.GetInsertPoint(); 1279 AllocaInst *Alloc = insertAlloca(UI, B, "__sincos_"); 1280 B.SetInsertPoint(UI); 1281 1282 Value *P = Alloc; 1283 Type *PTy = Fsincos.getFunctionType()->getParamType(1); 1284 // The allocaInst allocates the memory in private address space. This need 1285 // to be bitcasted to point to the address space of cos pointer type. 1286 // In OpenCL 2.0 this is generic, while in 1.2 that is private. 1287 if (PTy->getPointerAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) 1288 P = B.CreateAddrSpaceCast(Alloc, PTy); 1289 CallInst *Call = CreateCallEx2(B, Fsincos, UI->getArgOperand(0), P); 1290 1291 LLVM_DEBUG(errs() << "AMDIC: fold_sincos (" << *CI << ", " << *UI << ") with " 1292 << *Call << "\n"); 1293 1294 if (!isSin) { // CI->cos, UI->sin 1295 B.SetInsertPoint(&*ItOld); 1296 UI->replaceAllUsesWith(&*Call); 1297 Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc); 1298 CI->replaceAllUsesWith(Reload); 1299 UI->eraseFromParent(); 1300 CI->eraseFromParent(); 1301 } else { // CI->sin, UI->cos 1302 Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc); 1303 UI->replaceAllUsesWith(Reload); 1304 CI->replaceAllUsesWith(Call); 1305 UI->eraseFromParent(); 1306 CI->eraseFromParent(); 1307 } 1308 return true; 1309 } 1310 1311 bool AMDGPULibCalls::fold_wavefrontsize(CallInst *CI, IRBuilder<> &B) { 1312 if (!TM) 1313 return false; 1314 1315 StringRef CPU = TM->getTargetCPU(); 1316 StringRef Features = TM->getTargetFeatureString(); 1317 if ((CPU.empty() || CPU.equals_insensitive("generic")) && 1318 (Features.empty() || !Features.contains_insensitive("wavefrontsize"))) 1319 return false; 1320 1321 Function *F = CI->getParent()->getParent(); 1322 const GCNSubtarget &ST = TM->getSubtarget<GCNSubtarget>(*F); 1323 unsigned N = ST.getWavefrontSize(); 1324 1325 LLVM_DEBUG(errs() << "AMDIC: fold_wavefrontsize (" << *CI << ") with " 1326 << N << "\n"); 1327 1328 CI->replaceAllUsesWith(ConstantInt::get(B.getInt32Ty(), N)); 1329 CI->eraseFromParent(); 1330 return true; 1331 } 1332 1333 // Get insertion point at entry. 1334 BasicBlock::iterator AMDGPULibCalls::getEntryIns(CallInst * UI) { 1335 Function * Func = UI->getParent()->getParent(); 1336 BasicBlock * BB = &Func->getEntryBlock(); 1337 assert(BB && "Entry block not found!"); 1338 BasicBlock::iterator ItNew = BB->begin(); 1339 return ItNew; 1340 } 1341 1342 // Insert a AllocsInst at the beginning of function entry block. 1343 AllocaInst* AMDGPULibCalls::insertAlloca(CallInst *UI, IRBuilder<> &B, 1344 const char *prefix) { 1345 BasicBlock::iterator ItNew = getEntryIns(UI); 1346 Function *UCallee = UI->getCalledFunction(); 1347 Type *RetType = UCallee->getReturnType(); 1348 B.SetInsertPoint(&*ItNew); 1349 AllocaInst *Alloc = 1350 B.CreateAlloca(RetType, nullptr, std::string(prefix) + UI->getName()); 1351 Alloc->setAlignment( 1352 Align(UCallee->getParent()->getDataLayout().getTypeAllocSize(RetType))); 1353 return Alloc; 1354 } 1355 1356 bool AMDGPULibCalls::evaluateScalarMathFunc(const FuncInfo &FInfo, 1357 double& Res0, double& Res1, 1358 Constant *copr0, Constant *copr1, 1359 Constant *copr2) { 1360 // By default, opr0/opr1/opr3 holds values of float/double type. 1361 // If they are not float/double, each function has to its 1362 // operand separately. 1363 double opr0=0.0, opr1=0.0, opr2=0.0; 1364 ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(copr0); 1365 ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(copr1); 1366 ConstantFP *fpopr2 = dyn_cast_or_null<ConstantFP>(copr2); 1367 if (fpopr0) { 1368 opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1369 ? fpopr0->getValueAPF().convertToDouble() 1370 : (double)fpopr0->getValueAPF().convertToFloat(); 1371 } 1372 1373 if (fpopr1) { 1374 opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1375 ? fpopr1->getValueAPF().convertToDouble() 1376 : (double)fpopr1->getValueAPF().convertToFloat(); 1377 } 1378 1379 if (fpopr2) { 1380 opr2 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1381 ? fpopr2->getValueAPF().convertToDouble() 1382 : (double)fpopr2->getValueAPF().convertToFloat(); 1383 } 1384 1385 switch (FInfo.getId()) { 1386 default : return false; 1387 1388 case AMDGPULibFunc::EI_ACOS: 1389 Res0 = acos(opr0); 1390 return true; 1391 1392 case AMDGPULibFunc::EI_ACOSH: 1393 // acosh(x) == log(x + sqrt(x*x - 1)) 1394 Res0 = log(opr0 + sqrt(opr0*opr0 - 1.0)); 1395 return true; 1396 1397 case AMDGPULibFunc::EI_ACOSPI: 1398 Res0 = acos(opr0) / MATH_PI; 1399 return true; 1400 1401 case AMDGPULibFunc::EI_ASIN: 1402 Res0 = asin(opr0); 1403 return true; 1404 1405 case AMDGPULibFunc::EI_ASINH: 1406 // asinh(x) == log(x + sqrt(x*x + 1)) 1407 Res0 = log(opr0 + sqrt(opr0*opr0 + 1.0)); 1408 return true; 1409 1410 case AMDGPULibFunc::EI_ASINPI: 1411 Res0 = asin(opr0) / MATH_PI; 1412 return true; 1413 1414 case AMDGPULibFunc::EI_ATAN: 1415 Res0 = atan(opr0); 1416 return true; 1417 1418 case AMDGPULibFunc::EI_ATANH: 1419 // atanh(x) == (log(x+1) - log(x-1))/2; 1420 Res0 = (log(opr0 + 1.0) - log(opr0 - 1.0))/2.0; 1421 return true; 1422 1423 case AMDGPULibFunc::EI_ATANPI: 1424 Res0 = atan(opr0) / MATH_PI; 1425 return true; 1426 1427 case AMDGPULibFunc::EI_CBRT: 1428 Res0 = (opr0 < 0.0) ? -pow(-opr0, 1.0/3.0) : pow(opr0, 1.0/3.0); 1429 return true; 1430 1431 case AMDGPULibFunc::EI_COS: 1432 Res0 = cos(opr0); 1433 return true; 1434 1435 case AMDGPULibFunc::EI_COSH: 1436 Res0 = cosh(opr0); 1437 return true; 1438 1439 case AMDGPULibFunc::EI_COSPI: 1440 Res0 = cos(MATH_PI * opr0); 1441 return true; 1442 1443 case AMDGPULibFunc::EI_EXP: 1444 Res0 = exp(opr0); 1445 return true; 1446 1447 case AMDGPULibFunc::EI_EXP2: 1448 Res0 = pow(2.0, opr0); 1449 return true; 1450 1451 case AMDGPULibFunc::EI_EXP10: 1452 Res0 = pow(10.0, opr0); 1453 return true; 1454 1455 case AMDGPULibFunc::EI_EXPM1: 1456 Res0 = exp(opr0) - 1.0; 1457 return true; 1458 1459 case AMDGPULibFunc::EI_LOG: 1460 Res0 = log(opr0); 1461 return true; 1462 1463 case AMDGPULibFunc::EI_LOG2: 1464 Res0 = log(opr0) / log(2.0); 1465 return true; 1466 1467 case AMDGPULibFunc::EI_LOG10: 1468 Res0 = log(opr0) / log(10.0); 1469 return true; 1470 1471 case AMDGPULibFunc::EI_RSQRT: 1472 Res0 = 1.0 / sqrt(opr0); 1473 return true; 1474 1475 case AMDGPULibFunc::EI_SIN: 1476 Res0 = sin(opr0); 1477 return true; 1478 1479 case AMDGPULibFunc::EI_SINH: 1480 Res0 = sinh(opr0); 1481 return true; 1482 1483 case AMDGPULibFunc::EI_SINPI: 1484 Res0 = sin(MATH_PI * opr0); 1485 return true; 1486 1487 case AMDGPULibFunc::EI_SQRT: 1488 Res0 = sqrt(opr0); 1489 return true; 1490 1491 case AMDGPULibFunc::EI_TAN: 1492 Res0 = tan(opr0); 1493 return true; 1494 1495 case AMDGPULibFunc::EI_TANH: 1496 Res0 = tanh(opr0); 1497 return true; 1498 1499 case AMDGPULibFunc::EI_TANPI: 1500 Res0 = tan(MATH_PI * opr0); 1501 return true; 1502 1503 case AMDGPULibFunc::EI_RECIP: 1504 Res0 = 1.0 / opr0; 1505 return true; 1506 1507 // two-arg functions 1508 case AMDGPULibFunc::EI_DIVIDE: 1509 Res0 = opr0 / opr1; 1510 return true; 1511 1512 case AMDGPULibFunc::EI_POW: 1513 case AMDGPULibFunc::EI_POWR: 1514 Res0 = pow(opr0, opr1); 1515 return true; 1516 1517 case AMDGPULibFunc::EI_POWN: { 1518 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) { 1519 double val = (double)iopr1->getSExtValue(); 1520 Res0 = pow(opr0, val); 1521 return true; 1522 } 1523 return false; 1524 } 1525 1526 case AMDGPULibFunc::EI_ROOTN: { 1527 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) { 1528 double val = (double)iopr1->getSExtValue(); 1529 Res0 = pow(opr0, 1.0 / val); 1530 return true; 1531 } 1532 return false; 1533 } 1534 1535 // with ptr arg 1536 case AMDGPULibFunc::EI_SINCOS: 1537 Res0 = sin(opr0); 1538 Res1 = cos(opr0); 1539 return true; 1540 1541 // three-arg functions 1542 case AMDGPULibFunc::EI_FMA: 1543 case AMDGPULibFunc::EI_MAD: 1544 Res0 = opr0 * opr1 + opr2; 1545 return true; 1546 } 1547 1548 return false; 1549 } 1550 1551 bool AMDGPULibCalls::evaluateCall(CallInst *aCI, const FuncInfo &FInfo) { 1552 int numArgs = (int)aCI->arg_size(); 1553 if (numArgs > 3) 1554 return false; 1555 1556 Constant *copr0 = nullptr; 1557 Constant *copr1 = nullptr; 1558 Constant *copr2 = nullptr; 1559 if (numArgs > 0) { 1560 if ((copr0 = dyn_cast<Constant>(aCI->getArgOperand(0))) == nullptr) 1561 return false; 1562 } 1563 1564 if (numArgs > 1) { 1565 if ((copr1 = dyn_cast<Constant>(aCI->getArgOperand(1))) == nullptr) { 1566 if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS) 1567 return false; 1568 } 1569 } 1570 1571 if (numArgs > 2) { 1572 if ((copr2 = dyn_cast<Constant>(aCI->getArgOperand(2))) == nullptr) 1573 return false; 1574 } 1575 1576 // At this point, all arguments to aCI are constants. 1577 1578 // max vector size is 16, and sincos will generate two results. 1579 double DVal0[16], DVal1[16]; 1580 int FuncVecSize = getVecSize(FInfo); 1581 bool hasTwoResults = (FInfo.getId() == AMDGPULibFunc::EI_SINCOS); 1582 if (FuncVecSize == 1) { 1583 if (!evaluateScalarMathFunc(FInfo, DVal0[0], 1584 DVal1[0], copr0, copr1, copr2)) { 1585 return false; 1586 } 1587 } else { 1588 ConstantDataVector *CDV0 = dyn_cast_or_null<ConstantDataVector>(copr0); 1589 ConstantDataVector *CDV1 = dyn_cast_or_null<ConstantDataVector>(copr1); 1590 ConstantDataVector *CDV2 = dyn_cast_or_null<ConstantDataVector>(copr2); 1591 for (int i = 0; i < FuncVecSize; ++i) { 1592 Constant *celt0 = CDV0 ? CDV0->getElementAsConstant(i) : nullptr; 1593 Constant *celt1 = CDV1 ? CDV1->getElementAsConstant(i) : nullptr; 1594 Constant *celt2 = CDV2 ? CDV2->getElementAsConstant(i) : nullptr; 1595 if (!evaluateScalarMathFunc(FInfo, DVal0[i], 1596 DVal1[i], celt0, celt1, celt2)) { 1597 return false; 1598 } 1599 } 1600 } 1601 1602 LLVMContext &context = CI->getParent()->getParent()->getContext(); 1603 Constant *nval0, *nval1; 1604 if (FuncVecSize == 1) { 1605 nval0 = ConstantFP::get(CI->getType(), DVal0[0]); 1606 if (hasTwoResults) 1607 nval1 = ConstantFP::get(CI->getType(), DVal1[0]); 1608 } else { 1609 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 1610 SmallVector <float, 0> FVal0, FVal1; 1611 for (int i = 0; i < FuncVecSize; ++i) 1612 FVal0.push_back((float)DVal0[i]); 1613 ArrayRef<float> tmp0(FVal0); 1614 nval0 = ConstantDataVector::get(context, tmp0); 1615 if (hasTwoResults) { 1616 for (int i = 0; i < FuncVecSize; ++i) 1617 FVal1.push_back((float)DVal1[i]); 1618 ArrayRef<float> tmp1(FVal1); 1619 nval1 = ConstantDataVector::get(context, tmp1); 1620 } 1621 } else { 1622 ArrayRef<double> tmp0(DVal0); 1623 nval0 = ConstantDataVector::get(context, tmp0); 1624 if (hasTwoResults) { 1625 ArrayRef<double> tmp1(DVal1); 1626 nval1 = ConstantDataVector::get(context, tmp1); 1627 } 1628 } 1629 } 1630 1631 if (hasTwoResults) { 1632 // sincos 1633 assert(FInfo.getId() == AMDGPULibFunc::EI_SINCOS && 1634 "math function with ptr arg not supported yet"); 1635 new StoreInst(nval1, aCI->getArgOperand(1), aCI); 1636 } 1637 1638 replaceCall(nval0); 1639 return true; 1640 } 1641 1642 // Public interface to the Simplify LibCalls pass. 1643 FunctionPass *llvm::createAMDGPUSimplifyLibCallsPass(const TargetMachine *TM) { 1644 return new AMDGPUSimplifyLibCalls(TM); 1645 } 1646 1647 FunctionPass *llvm::createAMDGPUUseNativeCallsPass() { 1648 return new AMDGPUUseNativeCalls(); 1649 } 1650 1651 bool AMDGPUSimplifyLibCalls::runOnFunction(Function &F) { 1652 if (skipFunction(F)) 1653 return false; 1654 1655 bool Changed = false; 1656 auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); 1657 1658 LLVM_DEBUG(dbgs() << "AMDIC: process function "; 1659 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';); 1660 1661 for (auto &BB : F) { 1662 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) { 1663 // Ignore non-calls. 1664 CallInst *CI = dyn_cast<CallInst>(I); 1665 ++I; 1666 if (CI) { 1667 if (Simplifier.fold(CI, AA)) 1668 Changed = true; 1669 } 1670 } 1671 } 1672 return Changed; 1673 } 1674 1675 PreservedAnalyses AMDGPUSimplifyLibCallsPass::run(Function &F, 1676 FunctionAnalysisManager &AM) { 1677 AMDGPULibCalls Simplifier(&TM); 1678 Simplifier.initNativeFuncs(); 1679 1680 bool Changed = false; 1681 auto AA = &AM.getResult<AAManager>(F); 1682 1683 LLVM_DEBUG(dbgs() << "AMDIC: process function "; 1684 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';); 1685 1686 for (auto &BB : F) { 1687 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) { 1688 // Ignore non-calls. 1689 CallInst *CI = dyn_cast<CallInst>(I); 1690 ++I; 1691 1692 if (CI) { 1693 if (Simplifier.fold(CI, AA)) 1694 Changed = true; 1695 } 1696 } 1697 } 1698 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 1699 } 1700 1701 bool AMDGPUUseNativeCalls::runOnFunction(Function &F) { 1702 if (skipFunction(F) || UseNative.empty()) 1703 return false; 1704 1705 bool Changed = false; 1706 for (auto &BB : F) { 1707 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) { 1708 // Ignore non-calls. 1709 CallInst *CI = dyn_cast<CallInst>(I); 1710 ++I; 1711 if (CI && Simplifier.useNative(CI)) 1712 Changed = true; 1713 } 1714 } 1715 return Changed; 1716 } 1717 1718 PreservedAnalyses AMDGPUUseNativeCallsPass::run(Function &F, 1719 FunctionAnalysisManager &AM) { 1720 if (UseNative.empty()) 1721 return PreservedAnalyses::all(); 1722 1723 AMDGPULibCalls Simplifier; 1724 Simplifier.initNativeFuncs(); 1725 1726 bool Changed = false; 1727 for (auto &BB : F) { 1728 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) { 1729 // Ignore non-calls. 1730 CallInst *CI = dyn_cast<CallInst>(I); 1731 ++I; 1732 if (CI && Simplifier.useNative(CI)) 1733 Changed = true; 1734 } 1735 } 1736 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 1737 } 1738