1 //===- AMDGPULibCalls.cpp -------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file does AMD library function optimizations. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPU.h" 15 #include "AMDGPULibFunc.h" 16 #include "GCNSubtarget.h" 17 #include "llvm/Analysis/AliasAnalysis.h" 18 #include "llvm/Analysis/Loads.h" 19 #include "llvm/IR/IRBuilder.h" 20 #include "llvm/IR/IntrinsicInst.h" 21 #include "llvm/IR/IntrinsicsAMDGPU.h" 22 #include "llvm/InitializePasses.h" 23 #include "llvm/Target/TargetMachine.h" 24 #include <cmath> 25 26 #define DEBUG_TYPE "amdgpu-simplifylib" 27 28 using namespace llvm; 29 30 static cl::opt<bool> EnablePreLink("amdgpu-prelink", 31 cl::desc("Enable pre-link mode optimizations"), 32 cl::init(false), 33 cl::Hidden); 34 35 static cl::list<std::string> UseNative("amdgpu-use-native", 36 cl::desc("Comma separated list of functions to replace with native, or all"), 37 cl::CommaSeparated, cl::ValueOptional, 38 cl::Hidden); 39 40 #define MATH_PI numbers::pi 41 #define MATH_E numbers::e 42 #define MATH_SQRT2 numbers::sqrt2 43 #define MATH_SQRT1_2 numbers::inv_sqrt2 44 45 namespace llvm { 46 47 class AMDGPULibCalls { 48 private: 49 50 typedef llvm::AMDGPULibFunc FuncInfo; 51 52 const TargetMachine *TM; 53 54 bool UnsafeFPMath = false; 55 56 // -fuse-native. 57 bool AllNative = false; 58 59 bool useNativeFunc(const StringRef F) const; 60 61 // Return a pointer (pointer expr) to the function if function definition with 62 // "FuncName" exists. It may create a new function prototype in pre-link mode. 63 FunctionCallee getFunction(Module *M, const FuncInfo &fInfo); 64 65 bool parseFunctionName(const StringRef &FMangledName, FuncInfo &FInfo); 66 67 bool TDOFold(CallInst *CI, const FuncInfo &FInfo); 68 69 /* Specialized optimizations */ 70 71 // recip (half or native) 72 bool fold_recip(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 73 74 // divide (half or native) 75 bool fold_divide(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 76 77 // pow/powr/pown 78 bool fold_pow(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo); 79 80 // rootn 81 bool fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo); 82 83 // fma/mad 84 bool fold_fma_mad(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 85 86 // -fuse-native for sincos 87 bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo); 88 89 // evaluate calls if calls' arguments are constants. 90 bool evaluateScalarMathFunc(const FuncInfo &FInfo, double& Res0, 91 double& Res1, Constant *copr0, Constant *copr1, Constant *copr2); 92 bool evaluateCall(CallInst *aCI, const FuncInfo &FInfo); 93 94 // sqrt 95 bool fold_sqrt(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo); 96 97 // sin/cos 98 bool fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo, 99 AliasAnalysis *AA); 100 101 // __read_pipe/__write_pipe 102 bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, 103 const FuncInfo &FInfo); 104 105 // llvm.amdgcn.wavefrontsize 106 bool fold_wavefrontsize(CallInst *CI, IRBuilder<> &B); 107 108 // Get insertion point at entry. 109 BasicBlock::iterator getEntryIns(CallInst * UI); 110 // Insert an Alloc instruction. 111 AllocaInst* insertAlloca(CallInst * UI, IRBuilder<> &B, const char *prefix); 112 // Get a scalar native builtin single argument FP function 113 FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo); 114 115 protected: 116 bool isUnsafeMath(const FPMathOperator *FPOp) const; 117 118 bool canIncreasePrecisionOfConstantFold(const FPMathOperator *FPOp) const; 119 120 static void replaceCall(Instruction *I, Value *With) { 121 I->replaceAllUsesWith(With); 122 I->eraseFromParent(); 123 } 124 125 static void replaceCall(FPMathOperator *I, Value *With) { 126 replaceCall(cast<Instruction>(I), With); 127 } 128 129 public: 130 AMDGPULibCalls(const TargetMachine *TM_ = nullptr) : TM(TM_) {} 131 132 bool fold(CallInst *CI, AliasAnalysis *AA = nullptr); 133 134 void initFunction(const Function &F); 135 void initNativeFuncs(); 136 137 // Replace a normal math function call with that native version 138 bool useNative(CallInst *CI); 139 }; 140 141 } // end llvm namespace 142 143 namespace { 144 145 class AMDGPUSimplifyLibCalls : public FunctionPass { 146 147 AMDGPULibCalls Simplifier; 148 149 public: 150 static char ID; // Pass identification 151 152 AMDGPUSimplifyLibCalls(const TargetMachine *TM = nullptr) 153 : FunctionPass(ID), Simplifier(TM) { 154 initializeAMDGPUSimplifyLibCallsPass(*PassRegistry::getPassRegistry()); 155 } 156 157 void getAnalysisUsage(AnalysisUsage &AU) const override { 158 AU.addRequired<AAResultsWrapperPass>(); 159 } 160 161 bool runOnFunction(Function &M) override; 162 }; 163 164 class AMDGPUUseNativeCalls : public FunctionPass { 165 166 AMDGPULibCalls Simplifier; 167 168 public: 169 static char ID; // Pass identification 170 171 AMDGPUUseNativeCalls() : FunctionPass(ID) { 172 initializeAMDGPUUseNativeCallsPass(*PassRegistry::getPassRegistry()); 173 Simplifier.initNativeFuncs(); 174 } 175 176 bool runOnFunction(Function &F) override; 177 }; 178 179 } // end anonymous namespace. 180 181 char AMDGPUSimplifyLibCalls::ID = 0; 182 char AMDGPUUseNativeCalls::ID = 0; 183 184 INITIALIZE_PASS_BEGIN(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", 185 "Simplify well-known AMD library calls", false, false) 186 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 187 INITIALIZE_PASS_END(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", 188 "Simplify well-known AMD library calls", false, false) 189 190 INITIALIZE_PASS(AMDGPUUseNativeCalls, "amdgpu-usenative", 191 "Replace builtin math calls with that native versions.", 192 false, false) 193 194 template <typename IRB> 195 static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg, 196 const Twine &Name = "") { 197 CallInst *R = B.CreateCall(Callee, Arg, Name); 198 if (Function *F = dyn_cast<Function>(Callee.getCallee())) 199 R->setCallingConv(F->getCallingConv()); 200 return R; 201 } 202 203 template <typename IRB> 204 static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1, 205 Value *Arg2, const Twine &Name = "") { 206 CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name); 207 if (Function *F = dyn_cast<Function>(Callee.getCallee())) 208 R->setCallingConv(F->getCallingConv()); 209 return R; 210 } 211 212 // Data structures for table-driven optimizations. 213 // FuncTbl works for both f32 and f64 functions with 1 input argument 214 215 struct TableEntry { 216 double result; 217 double input; 218 }; 219 220 /* a list of {result, input} */ 221 static const TableEntry tbl_acos[] = { 222 {MATH_PI / 2.0, 0.0}, 223 {MATH_PI / 2.0, -0.0}, 224 {0.0, 1.0}, 225 {MATH_PI, -1.0} 226 }; 227 static const TableEntry tbl_acosh[] = { 228 {0.0, 1.0} 229 }; 230 static const TableEntry tbl_acospi[] = { 231 {0.5, 0.0}, 232 {0.5, -0.0}, 233 {0.0, 1.0}, 234 {1.0, -1.0} 235 }; 236 static const TableEntry tbl_asin[] = { 237 {0.0, 0.0}, 238 {-0.0, -0.0}, 239 {MATH_PI / 2.0, 1.0}, 240 {-MATH_PI / 2.0, -1.0} 241 }; 242 static const TableEntry tbl_asinh[] = { 243 {0.0, 0.0}, 244 {-0.0, -0.0} 245 }; 246 static const TableEntry tbl_asinpi[] = { 247 {0.0, 0.0}, 248 {-0.0, -0.0}, 249 {0.5, 1.0}, 250 {-0.5, -1.0} 251 }; 252 static const TableEntry tbl_atan[] = { 253 {0.0, 0.0}, 254 {-0.0, -0.0}, 255 {MATH_PI / 4.0, 1.0}, 256 {-MATH_PI / 4.0, -1.0} 257 }; 258 static const TableEntry tbl_atanh[] = { 259 {0.0, 0.0}, 260 {-0.0, -0.0} 261 }; 262 static const TableEntry tbl_atanpi[] = { 263 {0.0, 0.0}, 264 {-0.0, -0.0}, 265 {0.25, 1.0}, 266 {-0.25, -1.0} 267 }; 268 static const TableEntry tbl_cbrt[] = { 269 {0.0, 0.0}, 270 {-0.0, -0.0}, 271 {1.0, 1.0}, 272 {-1.0, -1.0}, 273 }; 274 static const TableEntry tbl_cos[] = { 275 {1.0, 0.0}, 276 {1.0, -0.0} 277 }; 278 static const TableEntry tbl_cosh[] = { 279 {1.0, 0.0}, 280 {1.0, -0.0} 281 }; 282 static const TableEntry tbl_cospi[] = { 283 {1.0, 0.0}, 284 {1.0, -0.0} 285 }; 286 static const TableEntry tbl_erfc[] = { 287 {1.0, 0.0}, 288 {1.0, -0.0} 289 }; 290 static const TableEntry tbl_erf[] = { 291 {0.0, 0.0}, 292 {-0.0, -0.0} 293 }; 294 static const TableEntry tbl_exp[] = { 295 {1.0, 0.0}, 296 {1.0, -0.0}, 297 {MATH_E, 1.0} 298 }; 299 static const TableEntry tbl_exp2[] = { 300 {1.0, 0.0}, 301 {1.0, -0.0}, 302 {2.0, 1.0} 303 }; 304 static const TableEntry tbl_exp10[] = { 305 {1.0, 0.0}, 306 {1.0, -0.0}, 307 {10.0, 1.0} 308 }; 309 static const TableEntry tbl_expm1[] = { 310 {0.0, 0.0}, 311 {-0.0, -0.0} 312 }; 313 static const TableEntry tbl_log[] = { 314 {0.0, 1.0}, 315 {1.0, MATH_E} 316 }; 317 static const TableEntry tbl_log2[] = { 318 {0.0, 1.0}, 319 {1.0, 2.0} 320 }; 321 static const TableEntry tbl_log10[] = { 322 {0.0, 1.0}, 323 {1.0, 10.0} 324 }; 325 static const TableEntry tbl_rsqrt[] = { 326 {1.0, 1.0}, 327 {MATH_SQRT1_2, 2.0} 328 }; 329 static const TableEntry tbl_sin[] = { 330 {0.0, 0.0}, 331 {-0.0, -0.0} 332 }; 333 static const TableEntry tbl_sinh[] = { 334 {0.0, 0.0}, 335 {-0.0, -0.0} 336 }; 337 static const TableEntry tbl_sinpi[] = { 338 {0.0, 0.0}, 339 {-0.0, -0.0} 340 }; 341 static const TableEntry tbl_sqrt[] = { 342 {0.0, 0.0}, 343 {1.0, 1.0}, 344 {MATH_SQRT2, 2.0} 345 }; 346 static const TableEntry tbl_tan[] = { 347 {0.0, 0.0}, 348 {-0.0, -0.0} 349 }; 350 static const TableEntry tbl_tanh[] = { 351 {0.0, 0.0}, 352 {-0.0, -0.0} 353 }; 354 static const TableEntry tbl_tanpi[] = { 355 {0.0, 0.0}, 356 {-0.0, -0.0} 357 }; 358 static const TableEntry tbl_tgamma[] = { 359 {1.0, 1.0}, 360 {1.0, 2.0}, 361 {2.0, 3.0}, 362 {6.0, 4.0} 363 }; 364 365 static bool HasNative(AMDGPULibFunc::EFuncId id) { 366 switch(id) { 367 case AMDGPULibFunc::EI_DIVIDE: 368 case AMDGPULibFunc::EI_COS: 369 case AMDGPULibFunc::EI_EXP: 370 case AMDGPULibFunc::EI_EXP2: 371 case AMDGPULibFunc::EI_EXP10: 372 case AMDGPULibFunc::EI_LOG: 373 case AMDGPULibFunc::EI_LOG2: 374 case AMDGPULibFunc::EI_LOG10: 375 case AMDGPULibFunc::EI_POWR: 376 case AMDGPULibFunc::EI_RECIP: 377 case AMDGPULibFunc::EI_RSQRT: 378 case AMDGPULibFunc::EI_SIN: 379 case AMDGPULibFunc::EI_SINCOS: 380 case AMDGPULibFunc::EI_SQRT: 381 case AMDGPULibFunc::EI_TAN: 382 return true; 383 default:; 384 } 385 return false; 386 } 387 388 using TableRef = ArrayRef<TableEntry>; 389 390 static TableRef getOptTable(AMDGPULibFunc::EFuncId id) { 391 switch(id) { 392 case AMDGPULibFunc::EI_ACOS: return TableRef(tbl_acos); 393 case AMDGPULibFunc::EI_ACOSH: return TableRef(tbl_acosh); 394 case AMDGPULibFunc::EI_ACOSPI: return TableRef(tbl_acospi); 395 case AMDGPULibFunc::EI_ASIN: return TableRef(tbl_asin); 396 case AMDGPULibFunc::EI_ASINH: return TableRef(tbl_asinh); 397 case AMDGPULibFunc::EI_ASINPI: return TableRef(tbl_asinpi); 398 case AMDGPULibFunc::EI_ATAN: return TableRef(tbl_atan); 399 case AMDGPULibFunc::EI_ATANH: return TableRef(tbl_atanh); 400 case AMDGPULibFunc::EI_ATANPI: return TableRef(tbl_atanpi); 401 case AMDGPULibFunc::EI_CBRT: return TableRef(tbl_cbrt); 402 case AMDGPULibFunc::EI_NCOS: 403 case AMDGPULibFunc::EI_COS: return TableRef(tbl_cos); 404 case AMDGPULibFunc::EI_COSH: return TableRef(tbl_cosh); 405 case AMDGPULibFunc::EI_COSPI: return TableRef(tbl_cospi); 406 case AMDGPULibFunc::EI_ERFC: return TableRef(tbl_erfc); 407 case AMDGPULibFunc::EI_ERF: return TableRef(tbl_erf); 408 case AMDGPULibFunc::EI_EXP: return TableRef(tbl_exp); 409 case AMDGPULibFunc::EI_NEXP2: 410 case AMDGPULibFunc::EI_EXP2: return TableRef(tbl_exp2); 411 case AMDGPULibFunc::EI_EXP10: return TableRef(tbl_exp10); 412 case AMDGPULibFunc::EI_EXPM1: return TableRef(tbl_expm1); 413 case AMDGPULibFunc::EI_LOG: return TableRef(tbl_log); 414 case AMDGPULibFunc::EI_NLOG2: 415 case AMDGPULibFunc::EI_LOG2: return TableRef(tbl_log2); 416 case AMDGPULibFunc::EI_LOG10: return TableRef(tbl_log10); 417 case AMDGPULibFunc::EI_NRSQRT: 418 case AMDGPULibFunc::EI_RSQRT: return TableRef(tbl_rsqrt); 419 case AMDGPULibFunc::EI_NSIN: 420 case AMDGPULibFunc::EI_SIN: return TableRef(tbl_sin); 421 case AMDGPULibFunc::EI_SINH: return TableRef(tbl_sinh); 422 case AMDGPULibFunc::EI_SINPI: return TableRef(tbl_sinpi); 423 case AMDGPULibFunc::EI_NSQRT: 424 case AMDGPULibFunc::EI_SQRT: return TableRef(tbl_sqrt); 425 case AMDGPULibFunc::EI_TAN: return TableRef(tbl_tan); 426 case AMDGPULibFunc::EI_TANH: return TableRef(tbl_tanh); 427 case AMDGPULibFunc::EI_TANPI: return TableRef(tbl_tanpi); 428 case AMDGPULibFunc::EI_TGAMMA: return TableRef(tbl_tgamma); 429 default:; 430 } 431 return TableRef(); 432 } 433 434 static inline int getVecSize(const AMDGPULibFunc& FInfo) { 435 return FInfo.getLeads()[0].VectorSize; 436 } 437 438 static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) { 439 return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType; 440 } 441 442 FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) { 443 // If we are doing PreLinkOpt, the function is external. So it is safe to 444 // use getOrInsertFunction() at this stage. 445 446 return EnablePreLink ? AMDGPULibFunc::getOrInsertFunction(M, fInfo) 447 : AMDGPULibFunc::getFunction(M, fInfo); 448 } 449 450 bool AMDGPULibCalls::parseFunctionName(const StringRef &FMangledName, 451 FuncInfo &FInfo) { 452 return AMDGPULibFunc::parse(FMangledName, FInfo); 453 } 454 455 bool AMDGPULibCalls::isUnsafeMath(const FPMathOperator *FPOp) const { 456 return UnsafeFPMath || FPOp->isFast(); 457 } 458 459 bool AMDGPULibCalls::canIncreasePrecisionOfConstantFold( 460 const FPMathOperator *FPOp) const { 461 // TODO: Refine to approxFunc or contract 462 return isUnsafeMath(FPOp); 463 } 464 465 void AMDGPULibCalls::initFunction(const Function &F) { 466 UnsafeFPMath = F.getFnAttribute("unsafe-fp-math").getValueAsBool(); 467 } 468 469 bool AMDGPULibCalls::useNativeFunc(const StringRef F) const { 470 return AllNative || llvm::is_contained(UseNative, F); 471 } 472 473 void AMDGPULibCalls::initNativeFuncs() { 474 AllNative = useNativeFunc("all") || 475 (UseNative.getNumOccurrences() && UseNative.size() == 1 && 476 UseNative.begin()->empty()); 477 } 478 479 bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) { 480 bool native_sin = useNativeFunc("sin"); 481 bool native_cos = useNativeFunc("cos"); 482 483 if (native_sin && native_cos) { 484 Module *M = aCI->getModule(); 485 Value *opr0 = aCI->getArgOperand(0); 486 487 AMDGPULibFunc nf; 488 nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType; 489 nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize; 490 491 nf.setPrefix(AMDGPULibFunc::NATIVE); 492 nf.setId(AMDGPULibFunc::EI_SIN); 493 FunctionCallee sinExpr = getFunction(M, nf); 494 495 nf.setPrefix(AMDGPULibFunc::NATIVE); 496 nf.setId(AMDGPULibFunc::EI_COS); 497 FunctionCallee cosExpr = getFunction(M, nf); 498 if (sinExpr && cosExpr) { 499 Value *sinval = CallInst::Create(sinExpr, opr0, "splitsin", aCI); 500 Value *cosval = CallInst::Create(cosExpr, opr0, "splitcos", aCI); 501 new StoreInst(cosval, aCI->getArgOperand(1), aCI); 502 503 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI 504 << " with native version of sin/cos"); 505 506 replaceCall(aCI, sinval); 507 return true; 508 } 509 } 510 return false; 511 } 512 513 bool AMDGPULibCalls::useNative(CallInst *aCI) { 514 Function *Callee = aCI->getCalledFunction(); 515 if (!Callee || aCI->isNoBuiltin()) 516 return false; 517 518 FuncInfo FInfo; 519 if (!parseFunctionName(Callee->getName(), FInfo) || !FInfo.isMangled() || 520 FInfo.getPrefix() != AMDGPULibFunc::NOPFX || 521 getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()) || 522 !(AllNative || useNativeFunc(FInfo.getName()))) { 523 return false; 524 } 525 526 if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS) 527 return sincosUseNative(aCI, FInfo); 528 529 FInfo.setPrefix(AMDGPULibFunc::NATIVE); 530 FunctionCallee F = getFunction(aCI->getModule(), FInfo); 531 if (!F) 532 return false; 533 534 aCI->setCalledFunction(F); 535 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI 536 << " with native version"); 537 return true; 538 } 539 540 // Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe 541 // builtin, with appended type size and alignment arguments, where 2 or 4 542 // indicates the original number of arguments. The library has optimized version 543 // of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same 544 // power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N 545 // for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ..., 546 // 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4. 547 bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, 548 const FuncInfo &FInfo) { 549 auto *Callee = CI->getCalledFunction(); 550 if (!Callee->isDeclaration()) 551 return false; 552 553 assert(Callee->hasName() && "Invalid read_pipe/write_pipe function"); 554 auto *M = Callee->getParent(); 555 std::string Name = std::string(Callee->getName()); 556 auto NumArg = CI->arg_size(); 557 if (NumArg != 4 && NumArg != 6) 558 return false; 559 ConstantInt *PacketSize = 560 dyn_cast<ConstantInt>(CI->getArgOperand(NumArg - 2)); 561 ConstantInt *PacketAlign = 562 dyn_cast<ConstantInt>(CI->getArgOperand(NumArg - 1)); 563 if (!PacketSize || !PacketAlign) 564 return false; 565 566 unsigned Size = PacketSize->getZExtValue(); 567 Align Alignment = PacketAlign->getAlignValue(); 568 if (Alignment != Size) 569 return false; 570 571 unsigned PtrArgLoc = CI->arg_size() - 3; 572 Value *PtrArg = CI->getArgOperand(PtrArgLoc); 573 Type *PtrTy = PtrArg->getType(); 574 575 SmallVector<llvm::Type *, 6> ArgTys; 576 for (unsigned I = 0; I != PtrArgLoc; ++I) 577 ArgTys.push_back(CI->getArgOperand(I)->getType()); 578 ArgTys.push_back(PtrTy); 579 580 Name = Name + "_" + std::to_string(Size); 581 auto *FTy = FunctionType::get(Callee->getReturnType(), 582 ArrayRef<Type *>(ArgTys), false); 583 AMDGPULibFunc NewLibFunc(Name, FTy); 584 FunctionCallee F = AMDGPULibFunc::getOrInsertFunction(M, NewLibFunc); 585 if (!F) 586 return false; 587 588 auto *BCast = B.CreatePointerCast(PtrArg, PtrTy); 589 SmallVector<Value *, 6> Args; 590 for (unsigned I = 0; I != PtrArgLoc; ++I) 591 Args.push_back(CI->getArgOperand(I)); 592 Args.push_back(BCast); 593 594 auto *NCI = B.CreateCall(F, Args); 595 NCI->setAttributes(CI->getAttributes()); 596 CI->replaceAllUsesWith(NCI); 597 CI->dropAllReferences(); 598 CI->eraseFromParent(); 599 600 return true; 601 } 602 603 // This function returns false if no change; return true otherwise. 604 bool AMDGPULibCalls::fold(CallInst *CI, AliasAnalysis *AA) { 605 Function *Callee = CI->getCalledFunction(); 606 // Ignore indirect calls. 607 if (!Callee || CI->isNoBuiltin()) 608 return false; 609 610 IRBuilder<> B(CI); 611 switch (Callee->getIntrinsicID()) { 612 case Intrinsic::not_intrinsic: 613 break; 614 case Intrinsic::amdgcn_wavefrontsize: 615 return !EnablePreLink && fold_wavefrontsize(CI, B); 616 default: 617 return false; 618 } 619 620 FuncInfo FInfo; 621 if (!parseFunctionName(Callee->getName(), FInfo)) 622 return false; 623 624 // Further check the number of arguments to see if they match. 625 // TODO: Check calling convention matches too 626 if (!FInfo.isCompatibleSignature(CI->getFunctionType())) 627 return false; 628 629 LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << '\n'); 630 631 if (TDOFold(CI, FInfo)) 632 return true; 633 634 if (FPMathOperator *FPOp = dyn_cast<FPMathOperator>(CI)) { 635 // Under unsafe-math, evaluate calls if possible. 636 // According to Brian Sumner, we can do this for all f32 function calls 637 // using host's double function calls. 638 if (canIncreasePrecisionOfConstantFold(FPOp) && evaluateCall(CI, FInfo)) 639 return true; 640 641 // Copy fast flags from the original call. 642 B.setFastMathFlags(FPOp->getFastMathFlags()); 643 644 // Specialized optimizations for each function call 645 switch (FInfo.getId()) { 646 case AMDGPULibFunc::EI_POW: 647 case AMDGPULibFunc::EI_POWR: 648 case AMDGPULibFunc::EI_POWN: 649 return fold_pow(FPOp, B, FInfo); 650 case AMDGPULibFunc::EI_ROOTN: 651 return fold_rootn(FPOp, B, FInfo); 652 case AMDGPULibFunc::EI_SQRT: 653 return fold_sqrt(FPOp, B, FInfo); 654 case AMDGPULibFunc::EI_COS: 655 case AMDGPULibFunc::EI_SIN: 656 return fold_sincos(FPOp, B, FInfo, AA); 657 case AMDGPULibFunc::EI_RECIP: 658 // skip vector function 659 assert((FInfo.getPrefix() == AMDGPULibFunc::NATIVE || 660 FInfo.getPrefix() == AMDGPULibFunc::HALF) && 661 "recip must be an either native or half function"); 662 return (getVecSize(FInfo) != 1) ? false : fold_recip(CI, B, FInfo); 663 664 case AMDGPULibFunc::EI_DIVIDE: 665 // skip vector function 666 assert((FInfo.getPrefix() == AMDGPULibFunc::NATIVE || 667 FInfo.getPrefix() == AMDGPULibFunc::HALF) && 668 "divide must be an either native or half function"); 669 return (getVecSize(FInfo) != 1) ? false : fold_divide(CI, B, FInfo); 670 case AMDGPULibFunc::EI_FMA: 671 case AMDGPULibFunc::EI_MAD: 672 case AMDGPULibFunc::EI_NFMA: 673 // skip vector function 674 return (getVecSize(FInfo) != 1) ? false : fold_fma_mad(CI, B, FInfo); 675 default: 676 break; 677 } 678 } else { 679 // Specialized optimizations for each function call 680 switch (FInfo.getId()) { 681 case AMDGPULibFunc::EI_READ_PIPE_2: 682 case AMDGPULibFunc::EI_READ_PIPE_4: 683 case AMDGPULibFunc::EI_WRITE_PIPE_2: 684 case AMDGPULibFunc::EI_WRITE_PIPE_4: 685 return fold_read_write_pipe(CI, B, FInfo); 686 default: 687 break; 688 } 689 } 690 691 return false; 692 } 693 694 bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) { 695 // Table-Driven optimization 696 const TableRef tr = getOptTable(FInfo.getId()); 697 if (tr.empty()) 698 return false; 699 700 int const sz = (int)tr.size(); 701 Value *opr0 = CI->getArgOperand(0); 702 703 if (getVecSize(FInfo) > 1) { 704 if (ConstantDataVector *CV = dyn_cast<ConstantDataVector>(opr0)) { 705 SmallVector<double, 0> DVal; 706 for (int eltNo = 0; eltNo < getVecSize(FInfo); ++eltNo) { 707 ConstantFP *eltval = dyn_cast<ConstantFP>( 708 CV->getElementAsConstant((unsigned)eltNo)); 709 assert(eltval && "Non-FP arguments in math function!"); 710 bool found = false; 711 for (int i=0; i < sz; ++i) { 712 if (eltval->isExactlyValue(tr[i].input)) { 713 DVal.push_back(tr[i].result); 714 found = true; 715 break; 716 } 717 } 718 if (!found) { 719 // This vector constants not handled yet. 720 return false; 721 } 722 } 723 LLVMContext &context = CI->getParent()->getParent()->getContext(); 724 Constant *nval; 725 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 726 SmallVector<float, 0> FVal; 727 for (unsigned i = 0; i < DVal.size(); ++i) { 728 FVal.push_back((float)DVal[i]); 729 } 730 ArrayRef<float> tmp(FVal); 731 nval = ConstantDataVector::get(context, tmp); 732 } else { // F64 733 ArrayRef<double> tmp(DVal); 734 nval = ConstantDataVector::get(context, tmp); 735 } 736 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 737 replaceCall(CI, nval); 738 return true; 739 } 740 } else { 741 // Scalar version 742 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) { 743 for (int i = 0; i < sz; ++i) { 744 if (CF->isExactlyValue(tr[i].input)) { 745 Value *nval = ConstantFP::get(CF->getType(), tr[i].result); 746 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 747 replaceCall(CI, nval); 748 return true; 749 } 750 } 751 } 752 } 753 754 return false; 755 } 756 757 // [native_]half_recip(c) ==> 1.0/c 758 bool AMDGPULibCalls::fold_recip(CallInst *CI, IRBuilder<> &B, 759 const FuncInfo &FInfo) { 760 Value *opr0 = CI->getArgOperand(0); 761 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) { 762 // Just create a normal div. Later, InstCombine will be able 763 // to compute the divide into a constant (avoid check float infinity 764 // or subnormal at this point). 765 Value *nval = B.CreateFDiv(ConstantFP::get(CF->getType(), 1.0), 766 opr0, 767 "recip2div"); 768 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 769 replaceCall(CI, nval); 770 return true; 771 } 772 return false; 773 } 774 775 // [native_]half_divide(x, c) ==> x/c 776 bool AMDGPULibCalls::fold_divide(CallInst *CI, IRBuilder<> &B, 777 const FuncInfo &FInfo) { 778 Value *opr0 = CI->getArgOperand(0); 779 Value *opr1 = CI->getArgOperand(1); 780 ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0); 781 ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1); 782 783 if ((CF0 && CF1) || // both are constants 784 (CF1 && (getArgType(FInfo) == AMDGPULibFunc::F32))) 785 // CF1 is constant && f32 divide 786 { 787 Value *nval1 = B.CreateFDiv(ConstantFP::get(opr1->getType(), 1.0), 788 opr1, "__div2recip"); 789 Value *nval = B.CreateFMul(opr0, nval1, "__div2mul"); 790 replaceCall(CI, nval); 791 return true; 792 } 793 return false; 794 } 795 796 namespace llvm { 797 static double log2(double V) { 798 #if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L 799 return ::log2(V); 800 #else 801 return log(V) / numbers::ln2; 802 #endif 803 } 804 } 805 806 bool AMDGPULibCalls::fold_pow(FPMathOperator *FPOp, IRBuilder<> &B, 807 const FuncInfo &FInfo) { 808 assert((FInfo.getId() == AMDGPULibFunc::EI_POW || 809 FInfo.getId() == AMDGPULibFunc::EI_POWR || 810 FInfo.getId() == AMDGPULibFunc::EI_POWN) && 811 "fold_pow: encounter a wrong function call"); 812 813 Module *M = B.GetInsertBlock()->getModule(); 814 ConstantFP *CF; 815 ConstantInt *CINT; 816 Type *eltType; 817 Value *opr0 = FPOp->getOperand(0); 818 Value *opr1 = FPOp->getOperand(1); 819 ConstantAggregateZero *CZero = dyn_cast<ConstantAggregateZero>(opr1); 820 821 if (getVecSize(FInfo) == 1) { 822 eltType = opr0->getType(); 823 CF = dyn_cast<ConstantFP>(opr1); 824 CINT = dyn_cast<ConstantInt>(opr1); 825 } else { 826 VectorType *VTy = dyn_cast<VectorType>(opr0->getType()); 827 assert(VTy && "Oprand of vector function should be of vectortype"); 828 eltType = VTy->getElementType(); 829 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1); 830 831 // Now, only Handle vector const whose elements have the same value. 832 CF = CDV ? dyn_cast_or_null<ConstantFP>(CDV->getSplatValue()) : nullptr; 833 CINT = CDV ? dyn_cast_or_null<ConstantInt>(CDV->getSplatValue()) : nullptr; 834 } 835 836 // No unsafe math , no constant argument, do nothing 837 if (!isUnsafeMath(FPOp) && !CF && !CINT && !CZero) 838 return false; 839 840 // 0x1111111 means that we don't do anything for this call. 841 int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111); 842 843 if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0) || CZero) { 844 // pow/powr/pown(x, 0) == 1 845 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1\n"); 846 Constant *cnval = ConstantFP::get(eltType, 1.0); 847 if (getVecSize(FInfo) > 1) { 848 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 849 } 850 replaceCall(FPOp, cnval); 851 return true; 852 } 853 if ((CF && CF->isExactlyValue(1.0)) || (CINT && ci_opr1 == 1)) { 854 // pow/powr/pown(x, 1.0) = x 855 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << "\n"); 856 replaceCall(FPOp, opr0); 857 return true; 858 } 859 if ((CF && CF->isExactlyValue(2.0)) || (CINT && ci_opr1 == 2)) { 860 // pow/powr/pown(x, 2.0) = x*x 861 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << " * " 862 << *opr0 << "\n"); 863 Value *nval = B.CreateFMul(opr0, opr0, "__pow2"); 864 replaceCall(FPOp, nval); 865 return true; 866 } 867 if ((CF && CF->isExactlyValue(-1.0)) || (CINT && ci_opr1 == -1)) { 868 // pow/powr/pown(x, -1.0) = 1.0/x 869 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1 / " << *opr0 << "\n"); 870 Constant *cnval = ConstantFP::get(eltType, 1.0); 871 if (getVecSize(FInfo) > 1) { 872 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 873 } 874 Value *nval = B.CreateFDiv(cnval, opr0, "__powrecip"); 875 replaceCall(FPOp, nval); 876 return true; 877 } 878 879 if (CF && (CF->isExactlyValue(0.5) || CF->isExactlyValue(-0.5))) { 880 // pow[r](x, [-]0.5) = sqrt(x) 881 bool issqrt = CF->isExactlyValue(0.5); 882 if (FunctionCallee FPExpr = 883 getFunction(M, AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT 884 : AMDGPULibFunc::EI_RSQRT, 885 FInfo))) { 886 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << FInfo.getName() 887 << '(' << *opr0 << ")\n"); 888 Value *nval = CreateCallEx(B,FPExpr, opr0, issqrt ? "__pow2sqrt" 889 : "__pow2rsqrt"); 890 replaceCall(FPOp, nval); 891 return true; 892 } 893 } 894 895 if (!isUnsafeMath(FPOp)) 896 return false; 897 898 // Unsafe Math optimization 899 900 // Remember that ci_opr1 is set if opr1 is integral 901 if (CF) { 902 double dval = (getArgType(FInfo) == AMDGPULibFunc::F32) 903 ? (double)CF->getValueAPF().convertToFloat() 904 : CF->getValueAPF().convertToDouble(); 905 int ival = (int)dval; 906 if ((double)ival == dval) { 907 ci_opr1 = ival; 908 } else 909 ci_opr1 = 0x11111111; 910 } 911 912 // pow/powr/pown(x, c) = [1/](x*x*..x); where 913 // trunc(c) == c && the number of x == c && |c| <= 12 914 unsigned abs_opr1 = (ci_opr1 < 0) ? -ci_opr1 : ci_opr1; 915 if (abs_opr1 <= 12) { 916 Constant *cnval; 917 Value *nval; 918 if (abs_opr1 == 0) { 919 cnval = ConstantFP::get(eltType, 1.0); 920 if (getVecSize(FInfo) > 1) { 921 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 922 } 923 nval = cnval; 924 } else { 925 Value *valx2 = nullptr; 926 nval = nullptr; 927 while (abs_opr1 > 0) { 928 valx2 = valx2 ? B.CreateFMul(valx2, valx2, "__powx2") : opr0; 929 if (abs_opr1 & 1) { 930 nval = nval ? B.CreateFMul(nval, valx2, "__powprod") : valx2; 931 } 932 abs_opr1 >>= 1; 933 } 934 } 935 936 if (ci_opr1 < 0) { 937 cnval = ConstantFP::get(eltType, 1.0); 938 if (getVecSize(FInfo) > 1) { 939 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 940 } 941 nval = B.CreateFDiv(cnval, nval, "__1powprod"); 942 } 943 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " 944 << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr0 945 << ")\n"); 946 replaceCall(FPOp, nval); 947 return true; 948 } 949 950 // powr ---> exp2(y * log2(x)) 951 // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31)) 952 FunctionCallee ExpExpr = 953 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo)); 954 if (!ExpExpr) 955 return false; 956 957 bool needlog = false; 958 bool needabs = false; 959 bool needcopysign = false; 960 Constant *cnval = nullptr; 961 if (getVecSize(FInfo) == 1) { 962 CF = dyn_cast<ConstantFP>(opr0); 963 964 if (CF) { 965 double V = (getArgType(FInfo) == AMDGPULibFunc::F32) 966 ? (double)CF->getValueAPF().convertToFloat() 967 : CF->getValueAPF().convertToDouble(); 968 969 V = log2(std::abs(V)); 970 cnval = ConstantFP::get(eltType, V); 971 needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR) && 972 CF->isNegative(); 973 } else { 974 needlog = true; 975 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR && 976 (!CF || CF->isNegative()); 977 } 978 } else { 979 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr0); 980 981 if (!CDV) { 982 needlog = true; 983 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR; 984 } else { 985 assert ((int)CDV->getNumElements() == getVecSize(FInfo) && 986 "Wrong vector size detected"); 987 988 SmallVector<double, 0> DVal; 989 for (int i=0; i < getVecSize(FInfo); ++i) { 990 double V = (getArgType(FInfo) == AMDGPULibFunc::F32) 991 ? (double)CDV->getElementAsFloat(i) 992 : CDV->getElementAsDouble(i); 993 if (V < 0.0) needcopysign = true; 994 V = log2(std::abs(V)); 995 DVal.push_back(V); 996 } 997 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 998 SmallVector<float, 0> FVal; 999 for (unsigned i=0; i < DVal.size(); ++i) { 1000 FVal.push_back((float)DVal[i]); 1001 } 1002 ArrayRef<float> tmp(FVal); 1003 cnval = ConstantDataVector::get(M->getContext(), tmp); 1004 } else { 1005 ArrayRef<double> tmp(DVal); 1006 cnval = ConstantDataVector::get(M->getContext(), tmp); 1007 } 1008 } 1009 } 1010 1011 if (needcopysign && (FInfo.getId() == AMDGPULibFunc::EI_POW)) { 1012 // We cannot handle corner cases for a general pow() function, give up 1013 // unless y is a constant integral value. Then proceed as if it were pown. 1014 if (getVecSize(FInfo) == 1) { 1015 if (const ConstantFP *CF = dyn_cast<ConstantFP>(opr1)) { 1016 double y = (getArgType(FInfo) == AMDGPULibFunc::F32) 1017 ? (double)CF->getValueAPF().convertToFloat() 1018 : CF->getValueAPF().convertToDouble(); 1019 if (y != (double)(int64_t)y) 1020 return false; 1021 } else 1022 return false; 1023 } else { 1024 if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1)) { 1025 for (int i=0; i < getVecSize(FInfo); ++i) { 1026 double y = (getArgType(FInfo) == AMDGPULibFunc::F32) 1027 ? (double)CDV->getElementAsFloat(i) 1028 : CDV->getElementAsDouble(i); 1029 if (y != (double)(int64_t)y) 1030 return false; 1031 } 1032 } else 1033 return false; 1034 } 1035 } 1036 1037 Value *nval; 1038 if (needabs) { 1039 FunctionCallee AbsExpr = 1040 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_FABS, FInfo)); 1041 if (!AbsExpr) 1042 return false; 1043 nval = CreateCallEx(B, AbsExpr, opr0, "__fabs"); 1044 } else { 1045 nval = cnval ? cnval : opr0; 1046 } 1047 if (needlog) { 1048 FunctionCallee LogExpr = 1049 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo)); 1050 if (!LogExpr) 1051 return false; 1052 nval = CreateCallEx(B,LogExpr, nval, "__log2"); 1053 } 1054 1055 if (FInfo.getId() == AMDGPULibFunc::EI_POWN) { 1056 // convert int(32) to fp(f32 or f64) 1057 opr1 = B.CreateSIToFP(opr1, nval->getType(), "pownI2F"); 1058 } 1059 nval = B.CreateFMul(opr1, nval, "__ylogx"); 1060 nval = CreateCallEx(B,ExpExpr, nval, "__exp2"); 1061 1062 if (needcopysign) { 1063 Value *opr_n; 1064 Type* rTy = opr0->getType(); 1065 Type* nTyS = eltType->isDoubleTy() ? B.getInt64Ty() : B.getInt32Ty(); 1066 Type *nTy = nTyS; 1067 if (const auto *vTy = dyn_cast<FixedVectorType>(rTy)) 1068 nTy = FixedVectorType::get(nTyS, vTy); 1069 unsigned size = nTy->getScalarSizeInBits(); 1070 opr_n = FPOp->getOperand(1); 1071 if (opr_n->getType()->isIntegerTy()) 1072 opr_n = B.CreateZExtOrBitCast(opr_n, nTy, "__ytou"); 1073 else 1074 opr_n = B.CreateFPToSI(opr1, nTy, "__ytou"); 1075 1076 Value *sign = B.CreateShl(opr_n, size-1, "__yeven"); 1077 sign = B.CreateAnd(B.CreateBitCast(opr0, nTy), sign, "__pow_sign"); 1078 nval = B.CreateOr(B.CreateBitCast(nval, nTy), sign); 1079 nval = B.CreateBitCast(nval, opr0->getType()); 1080 } 1081 1082 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " 1083 << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n"); 1084 replaceCall(FPOp, nval); 1085 1086 return true; 1087 } 1088 1089 bool AMDGPULibCalls::fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B, 1090 const FuncInfo &FInfo) { 1091 // skip vector function 1092 if (getVecSize(FInfo) != 1) 1093 return false; 1094 1095 Value *opr0 = FPOp->getOperand(0); 1096 Value *opr1 = FPOp->getOperand(1); 1097 1098 ConstantInt *CINT = dyn_cast<ConstantInt>(opr1); 1099 if (!CINT) { 1100 return false; 1101 } 1102 int ci_opr1 = (int)CINT->getSExtValue(); 1103 if (ci_opr1 == 1) { // rootn(x, 1) = x 1104 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << "\n"); 1105 replaceCall(FPOp, opr0); 1106 return true; 1107 } 1108 1109 Module *M = B.GetInsertBlock()->getModule(); 1110 if (ci_opr1 == 2) { // rootn(x, 2) = sqrt(x) 1111 if (FunctionCallee FPExpr = 1112 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) { 1113 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> sqrt(" << *opr0 1114 << ")\n"); 1115 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2sqrt"); 1116 replaceCall(FPOp, nval); 1117 return true; 1118 } 1119 } else if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x) 1120 if (FunctionCallee FPExpr = 1121 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) { 1122 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> cbrt(" << *opr0 1123 << ")\n"); 1124 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2cbrt"); 1125 replaceCall(FPOp, nval); 1126 return true; 1127 } 1128 } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x 1129 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1.0 / " << *opr0 << "\n"); 1130 Value *nval = B.CreateFDiv(ConstantFP::get(opr0->getType(), 1.0), 1131 opr0, 1132 "__rootn2div"); 1133 replaceCall(FPOp, nval); 1134 return true; 1135 } else if (ci_opr1 == -2) { // rootn(x, -2) = rsqrt(x) 1136 if (FunctionCallee FPExpr = 1137 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_RSQRT, FInfo))) { 1138 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> rsqrt(" << *opr0 1139 << ")\n"); 1140 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2rsqrt"); 1141 replaceCall(FPOp, nval); 1142 return true; 1143 } 1144 } 1145 return false; 1146 } 1147 1148 bool AMDGPULibCalls::fold_fma_mad(CallInst *CI, IRBuilder<> &B, 1149 const FuncInfo &FInfo) { 1150 Value *opr0 = CI->getArgOperand(0); 1151 Value *opr1 = CI->getArgOperand(1); 1152 Value *opr2 = CI->getArgOperand(2); 1153 1154 ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0); 1155 ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1); 1156 if ((CF0 && CF0->isZero()) || (CF1 && CF1->isZero())) { 1157 // fma/mad(a, b, c) = c if a=0 || b=0 1158 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr2 << "\n"); 1159 replaceCall(CI, opr2); 1160 return true; 1161 } 1162 if (CF0 && CF0->isExactlyValue(1.0f)) { 1163 // fma/mad(a, b, c) = b+c if a=1 1164 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr1 << " + " << *opr2 1165 << "\n"); 1166 Value *nval = B.CreateFAdd(opr1, opr2, "fmaadd"); 1167 replaceCall(CI, nval); 1168 return true; 1169 } 1170 if (CF1 && CF1->isExactlyValue(1.0f)) { 1171 // fma/mad(a, b, c) = a+c if b=1 1172 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " + " << *opr2 1173 << "\n"); 1174 Value *nval = B.CreateFAdd(opr0, opr2, "fmaadd"); 1175 replaceCall(CI, nval); 1176 return true; 1177 } 1178 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr2)) { 1179 if (CF->isZero()) { 1180 // fma/mad(a, b, c) = a*b if c=0 1181 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * " 1182 << *opr1 << "\n"); 1183 Value *nval = B.CreateFMul(opr0, opr1, "fmamul"); 1184 replaceCall(CI, nval); 1185 return true; 1186 } 1187 } 1188 1189 return false; 1190 } 1191 1192 // Get a scalar native builtin single argument FP function 1193 FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M, 1194 const FuncInfo &FInfo) { 1195 if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId())) 1196 return nullptr; 1197 FuncInfo nf = FInfo; 1198 nf.setPrefix(AMDGPULibFunc::NATIVE); 1199 return getFunction(M, nf); 1200 } 1201 1202 // fold sqrt -> native_sqrt (x) 1203 bool AMDGPULibCalls::fold_sqrt(FPMathOperator *FPOp, IRBuilder<> &B, 1204 const FuncInfo &FInfo) { 1205 if (!isUnsafeMath(FPOp)) 1206 return false; 1207 1208 if (getArgType(FInfo) == AMDGPULibFunc::F32 && (getVecSize(FInfo) == 1) && 1209 (FInfo.getPrefix() != AMDGPULibFunc::NATIVE)) { 1210 Module *M = B.GetInsertBlock()->getModule(); 1211 1212 if (FunctionCallee FPExpr = getNativeFunction( 1213 M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) { 1214 Value *opr0 = FPOp->getOperand(0); 1215 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " 1216 << "sqrt(" << *opr0 << ")\n"); 1217 Value *nval = CreateCallEx(B,FPExpr, opr0, "__sqrt"); 1218 replaceCall(FPOp, nval); 1219 return true; 1220 } 1221 } 1222 return false; 1223 } 1224 1225 // fold sin, cos -> sincos. 1226 bool AMDGPULibCalls::fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B, 1227 const FuncInfo &fInfo, AliasAnalysis *AA) { 1228 assert(fInfo.getId() == AMDGPULibFunc::EI_SIN || 1229 fInfo.getId() == AMDGPULibFunc::EI_COS); 1230 1231 if ((getArgType(fInfo) != AMDGPULibFunc::F32 && 1232 getArgType(fInfo) != AMDGPULibFunc::F64) || 1233 fInfo.getPrefix() != AMDGPULibFunc::NOPFX) 1234 return false; 1235 1236 bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN; 1237 1238 Value *CArgVal = FPOp->getOperand(0); 1239 CallInst *CI = cast<CallInst>(FPOp); 1240 BasicBlock * const CBB = CI->getParent(); 1241 1242 int const MaxScan = 30; 1243 bool Changed = false; 1244 1245 { // fold in load value. 1246 LoadInst *LI = dyn_cast<LoadInst>(CArgVal); 1247 if (LI && LI->getParent() == CBB) { 1248 BasicBlock::iterator BBI = LI->getIterator(); 1249 Value *AvailableVal = FindAvailableLoadedValue(LI, CBB, BBI, MaxScan, AA); 1250 if (AvailableVal) { 1251 Changed = true; 1252 CArgVal->replaceAllUsesWith(AvailableVal); 1253 if (CArgVal->getNumUses() == 0) 1254 LI->eraseFromParent(); 1255 CArgVal = FPOp->getOperand(0); 1256 } 1257 } 1258 } 1259 1260 Module *M = CI->getModule(); 1261 FuncInfo PartnerInfo(isSin ? AMDGPULibFunc::EI_COS : AMDGPULibFunc::EI_SIN, 1262 fInfo); 1263 const std::string PairName = PartnerInfo.mangle(); 1264 1265 CallInst *UI = nullptr; 1266 for (User* U : CArgVal->users()) { 1267 CallInst *XI = dyn_cast_or_null<CallInst>(U); 1268 if (!XI || XI == CI || XI->getParent() != CBB) 1269 continue; 1270 1271 Function *UCallee = XI->getCalledFunction(); 1272 if (!UCallee || !UCallee->getName().equals(PairName)) 1273 continue; 1274 1275 BasicBlock::iterator BBI = CI->getIterator(); 1276 if (BBI == CI->getParent()->begin()) 1277 break; 1278 --BBI; 1279 for (int I = MaxScan; I > 0 && BBI != CBB->begin(); --BBI, --I) { 1280 if (cast<Instruction>(BBI) == XI) { 1281 UI = XI; 1282 break; 1283 } 1284 } 1285 if (UI) break; 1286 } 1287 1288 if (!UI) 1289 return Changed; 1290 1291 // Merge the sin and cos. 1292 1293 // for OpenCL 2.0 we have only generic implementation of sincos 1294 // function. 1295 AMDGPULibFunc nf(AMDGPULibFunc::EI_SINCOS, fInfo); 1296 nf.getLeads()[0].PtrKind = AMDGPULibFunc::getEPtrKindFromAddrSpace(AMDGPUAS::FLAT_ADDRESS); 1297 FunctionCallee Fsincos = getFunction(M, nf); 1298 if (!Fsincos) 1299 return Changed; 1300 1301 BasicBlock::iterator ItOld = B.GetInsertPoint(); 1302 AllocaInst *Alloc = insertAlloca(UI, B, "__sincos_"); 1303 B.SetInsertPoint(UI); 1304 1305 Value *P = Alloc; 1306 Type *PTy = Fsincos.getFunctionType()->getParamType(1); 1307 // The allocaInst allocates the memory in private address space. This need 1308 // to be bitcasted to point to the address space of cos pointer type. 1309 // In OpenCL 2.0 this is generic, while in 1.2 that is private. 1310 if (PTy->getPointerAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) 1311 P = B.CreateAddrSpaceCast(Alloc, PTy); 1312 CallInst *Call = CreateCallEx2(B, Fsincos, UI->getArgOperand(0), P); 1313 1314 LLVM_DEBUG(errs() << "AMDIC: fold_sincos (" << *CI << ", " << *UI << ") with " 1315 << *Call << "\n"); 1316 1317 if (!isSin) { // CI->cos, UI->sin 1318 B.SetInsertPoint(&*ItOld); 1319 UI->replaceAllUsesWith(&*Call); 1320 Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc); 1321 CI->replaceAllUsesWith(Reload); 1322 UI->eraseFromParent(); 1323 CI->eraseFromParent(); 1324 } else { // CI->sin, UI->cos 1325 Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc); 1326 UI->replaceAllUsesWith(Reload); 1327 CI->replaceAllUsesWith(Call); 1328 UI->eraseFromParent(); 1329 CI->eraseFromParent(); 1330 } 1331 return true; 1332 } 1333 1334 bool AMDGPULibCalls::fold_wavefrontsize(CallInst *CI, IRBuilder<> &B) { 1335 if (!TM) 1336 return false; 1337 1338 StringRef CPU = TM->getTargetCPU(); 1339 StringRef Features = TM->getTargetFeatureString(); 1340 if ((CPU.empty() || CPU.equals_insensitive("generic")) && 1341 (Features.empty() || !Features.contains_insensitive("wavefrontsize"))) 1342 return false; 1343 1344 Function *F = CI->getParent()->getParent(); 1345 const GCNSubtarget &ST = TM->getSubtarget<GCNSubtarget>(*F); 1346 unsigned N = ST.getWavefrontSize(); 1347 1348 LLVM_DEBUG(errs() << "AMDIC: fold_wavefrontsize (" << *CI << ") with " 1349 << N << "\n"); 1350 1351 CI->replaceAllUsesWith(ConstantInt::get(B.getInt32Ty(), N)); 1352 CI->eraseFromParent(); 1353 return true; 1354 } 1355 1356 // Get insertion point at entry. 1357 BasicBlock::iterator AMDGPULibCalls::getEntryIns(CallInst * UI) { 1358 Function * Func = UI->getParent()->getParent(); 1359 BasicBlock * BB = &Func->getEntryBlock(); 1360 assert(BB && "Entry block not found!"); 1361 BasicBlock::iterator ItNew = BB->begin(); 1362 return ItNew; 1363 } 1364 1365 // Insert a AllocsInst at the beginning of function entry block. 1366 AllocaInst* AMDGPULibCalls::insertAlloca(CallInst *UI, IRBuilder<> &B, 1367 const char *prefix) { 1368 BasicBlock::iterator ItNew = getEntryIns(UI); 1369 Function *UCallee = UI->getCalledFunction(); 1370 Type *RetType = UCallee->getReturnType(); 1371 B.SetInsertPoint(&*ItNew); 1372 AllocaInst *Alloc = 1373 B.CreateAlloca(RetType, nullptr, std::string(prefix) + UI->getName()); 1374 Alloc->setAlignment( 1375 Align(UCallee->getParent()->getDataLayout().getTypeAllocSize(RetType))); 1376 return Alloc; 1377 } 1378 1379 bool AMDGPULibCalls::evaluateScalarMathFunc(const FuncInfo &FInfo, 1380 double& Res0, double& Res1, 1381 Constant *copr0, Constant *copr1, 1382 Constant *copr2) { 1383 // By default, opr0/opr1/opr3 holds values of float/double type. 1384 // If they are not float/double, each function has to its 1385 // operand separately. 1386 double opr0=0.0, opr1=0.0, opr2=0.0; 1387 ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(copr0); 1388 ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(copr1); 1389 ConstantFP *fpopr2 = dyn_cast_or_null<ConstantFP>(copr2); 1390 if (fpopr0) { 1391 opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1392 ? fpopr0->getValueAPF().convertToDouble() 1393 : (double)fpopr0->getValueAPF().convertToFloat(); 1394 } 1395 1396 if (fpopr1) { 1397 opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1398 ? fpopr1->getValueAPF().convertToDouble() 1399 : (double)fpopr1->getValueAPF().convertToFloat(); 1400 } 1401 1402 if (fpopr2) { 1403 opr2 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1404 ? fpopr2->getValueAPF().convertToDouble() 1405 : (double)fpopr2->getValueAPF().convertToFloat(); 1406 } 1407 1408 switch (FInfo.getId()) { 1409 default : return false; 1410 1411 case AMDGPULibFunc::EI_ACOS: 1412 Res0 = acos(opr0); 1413 return true; 1414 1415 case AMDGPULibFunc::EI_ACOSH: 1416 // acosh(x) == log(x + sqrt(x*x - 1)) 1417 Res0 = log(opr0 + sqrt(opr0*opr0 - 1.0)); 1418 return true; 1419 1420 case AMDGPULibFunc::EI_ACOSPI: 1421 Res0 = acos(opr0) / MATH_PI; 1422 return true; 1423 1424 case AMDGPULibFunc::EI_ASIN: 1425 Res0 = asin(opr0); 1426 return true; 1427 1428 case AMDGPULibFunc::EI_ASINH: 1429 // asinh(x) == log(x + sqrt(x*x + 1)) 1430 Res0 = log(opr0 + sqrt(opr0*opr0 + 1.0)); 1431 return true; 1432 1433 case AMDGPULibFunc::EI_ASINPI: 1434 Res0 = asin(opr0) / MATH_PI; 1435 return true; 1436 1437 case AMDGPULibFunc::EI_ATAN: 1438 Res0 = atan(opr0); 1439 return true; 1440 1441 case AMDGPULibFunc::EI_ATANH: 1442 // atanh(x) == (log(x+1) - log(x-1))/2; 1443 Res0 = (log(opr0 + 1.0) - log(opr0 - 1.0))/2.0; 1444 return true; 1445 1446 case AMDGPULibFunc::EI_ATANPI: 1447 Res0 = atan(opr0) / MATH_PI; 1448 return true; 1449 1450 case AMDGPULibFunc::EI_CBRT: 1451 Res0 = (opr0 < 0.0) ? -pow(-opr0, 1.0/3.0) : pow(opr0, 1.0/3.0); 1452 return true; 1453 1454 case AMDGPULibFunc::EI_COS: 1455 Res0 = cos(opr0); 1456 return true; 1457 1458 case AMDGPULibFunc::EI_COSH: 1459 Res0 = cosh(opr0); 1460 return true; 1461 1462 case AMDGPULibFunc::EI_COSPI: 1463 Res0 = cos(MATH_PI * opr0); 1464 return true; 1465 1466 case AMDGPULibFunc::EI_EXP: 1467 Res0 = exp(opr0); 1468 return true; 1469 1470 case AMDGPULibFunc::EI_EXP2: 1471 Res0 = pow(2.0, opr0); 1472 return true; 1473 1474 case AMDGPULibFunc::EI_EXP10: 1475 Res0 = pow(10.0, opr0); 1476 return true; 1477 1478 case AMDGPULibFunc::EI_EXPM1: 1479 Res0 = exp(opr0) - 1.0; 1480 return true; 1481 1482 case AMDGPULibFunc::EI_LOG: 1483 Res0 = log(opr0); 1484 return true; 1485 1486 case AMDGPULibFunc::EI_LOG2: 1487 Res0 = log(opr0) / log(2.0); 1488 return true; 1489 1490 case AMDGPULibFunc::EI_LOG10: 1491 Res0 = log(opr0) / log(10.0); 1492 return true; 1493 1494 case AMDGPULibFunc::EI_RSQRT: 1495 Res0 = 1.0 / sqrt(opr0); 1496 return true; 1497 1498 case AMDGPULibFunc::EI_SIN: 1499 Res0 = sin(opr0); 1500 return true; 1501 1502 case AMDGPULibFunc::EI_SINH: 1503 Res0 = sinh(opr0); 1504 return true; 1505 1506 case AMDGPULibFunc::EI_SINPI: 1507 Res0 = sin(MATH_PI * opr0); 1508 return true; 1509 1510 case AMDGPULibFunc::EI_SQRT: 1511 Res0 = sqrt(opr0); 1512 return true; 1513 1514 case AMDGPULibFunc::EI_TAN: 1515 Res0 = tan(opr0); 1516 return true; 1517 1518 case AMDGPULibFunc::EI_TANH: 1519 Res0 = tanh(opr0); 1520 return true; 1521 1522 case AMDGPULibFunc::EI_TANPI: 1523 Res0 = tan(MATH_PI * opr0); 1524 return true; 1525 1526 case AMDGPULibFunc::EI_RECIP: 1527 Res0 = 1.0 / opr0; 1528 return true; 1529 1530 // two-arg functions 1531 case AMDGPULibFunc::EI_DIVIDE: 1532 Res0 = opr0 / opr1; 1533 return true; 1534 1535 case AMDGPULibFunc::EI_POW: 1536 case AMDGPULibFunc::EI_POWR: 1537 Res0 = pow(opr0, opr1); 1538 return true; 1539 1540 case AMDGPULibFunc::EI_POWN: { 1541 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) { 1542 double val = (double)iopr1->getSExtValue(); 1543 Res0 = pow(opr0, val); 1544 return true; 1545 } 1546 return false; 1547 } 1548 1549 case AMDGPULibFunc::EI_ROOTN: { 1550 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) { 1551 double val = (double)iopr1->getSExtValue(); 1552 Res0 = pow(opr0, 1.0 / val); 1553 return true; 1554 } 1555 return false; 1556 } 1557 1558 // with ptr arg 1559 case AMDGPULibFunc::EI_SINCOS: 1560 Res0 = sin(opr0); 1561 Res1 = cos(opr0); 1562 return true; 1563 1564 // three-arg functions 1565 case AMDGPULibFunc::EI_FMA: 1566 case AMDGPULibFunc::EI_MAD: 1567 Res0 = opr0 * opr1 + opr2; 1568 return true; 1569 } 1570 1571 return false; 1572 } 1573 1574 bool AMDGPULibCalls::evaluateCall(CallInst *aCI, const FuncInfo &FInfo) { 1575 int numArgs = (int)aCI->arg_size(); 1576 if (numArgs > 3) 1577 return false; 1578 1579 Constant *copr0 = nullptr; 1580 Constant *copr1 = nullptr; 1581 Constant *copr2 = nullptr; 1582 if (numArgs > 0) { 1583 if ((copr0 = dyn_cast<Constant>(aCI->getArgOperand(0))) == nullptr) 1584 return false; 1585 } 1586 1587 if (numArgs > 1) { 1588 if ((copr1 = dyn_cast<Constant>(aCI->getArgOperand(1))) == nullptr) { 1589 if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS) 1590 return false; 1591 } 1592 } 1593 1594 if (numArgs > 2) { 1595 if ((copr2 = dyn_cast<Constant>(aCI->getArgOperand(2))) == nullptr) 1596 return false; 1597 } 1598 1599 // At this point, all arguments to aCI are constants. 1600 1601 // max vector size is 16, and sincos will generate two results. 1602 double DVal0[16], DVal1[16]; 1603 int FuncVecSize = getVecSize(FInfo); 1604 bool hasTwoResults = (FInfo.getId() == AMDGPULibFunc::EI_SINCOS); 1605 if (FuncVecSize == 1) { 1606 if (!evaluateScalarMathFunc(FInfo, DVal0[0], 1607 DVal1[0], copr0, copr1, copr2)) { 1608 return false; 1609 } 1610 } else { 1611 ConstantDataVector *CDV0 = dyn_cast_or_null<ConstantDataVector>(copr0); 1612 ConstantDataVector *CDV1 = dyn_cast_or_null<ConstantDataVector>(copr1); 1613 ConstantDataVector *CDV2 = dyn_cast_or_null<ConstantDataVector>(copr2); 1614 for (int i = 0; i < FuncVecSize; ++i) { 1615 Constant *celt0 = CDV0 ? CDV0->getElementAsConstant(i) : nullptr; 1616 Constant *celt1 = CDV1 ? CDV1->getElementAsConstant(i) : nullptr; 1617 Constant *celt2 = CDV2 ? CDV2->getElementAsConstant(i) : nullptr; 1618 if (!evaluateScalarMathFunc(FInfo, DVal0[i], 1619 DVal1[i], celt0, celt1, celt2)) { 1620 return false; 1621 } 1622 } 1623 } 1624 1625 LLVMContext &context = aCI->getContext(); 1626 Constant *nval0, *nval1; 1627 if (FuncVecSize == 1) { 1628 nval0 = ConstantFP::get(aCI->getType(), DVal0[0]); 1629 if (hasTwoResults) 1630 nval1 = ConstantFP::get(aCI->getType(), DVal1[0]); 1631 } else { 1632 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 1633 SmallVector <float, 0> FVal0, FVal1; 1634 for (int i = 0; i < FuncVecSize; ++i) 1635 FVal0.push_back((float)DVal0[i]); 1636 ArrayRef<float> tmp0(FVal0); 1637 nval0 = ConstantDataVector::get(context, tmp0); 1638 if (hasTwoResults) { 1639 for (int i = 0; i < FuncVecSize; ++i) 1640 FVal1.push_back((float)DVal1[i]); 1641 ArrayRef<float> tmp1(FVal1); 1642 nval1 = ConstantDataVector::get(context, tmp1); 1643 } 1644 } else { 1645 ArrayRef<double> tmp0(DVal0); 1646 nval0 = ConstantDataVector::get(context, tmp0); 1647 if (hasTwoResults) { 1648 ArrayRef<double> tmp1(DVal1); 1649 nval1 = ConstantDataVector::get(context, tmp1); 1650 } 1651 } 1652 } 1653 1654 if (hasTwoResults) { 1655 // sincos 1656 assert(FInfo.getId() == AMDGPULibFunc::EI_SINCOS && 1657 "math function with ptr arg not supported yet"); 1658 new StoreInst(nval1, aCI->getArgOperand(1), aCI); 1659 } 1660 1661 replaceCall(aCI, nval0); 1662 return true; 1663 } 1664 1665 // Public interface to the Simplify LibCalls pass. 1666 FunctionPass *llvm::createAMDGPUSimplifyLibCallsPass(const TargetMachine *TM) { 1667 return new AMDGPUSimplifyLibCalls(TM); 1668 } 1669 1670 FunctionPass *llvm::createAMDGPUUseNativeCallsPass() { 1671 return new AMDGPUUseNativeCalls(); 1672 } 1673 1674 bool AMDGPUSimplifyLibCalls::runOnFunction(Function &F) { 1675 if (skipFunction(F)) 1676 return false; 1677 1678 Simplifier.initFunction(F); 1679 1680 bool Changed = false; 1681 auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); 1682 1683 LLVM_DEBUG(dbgs() << "AMDIC: process function "; 1684 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';); 1685 1686 for (auto &BB : F) { 1687 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) { 1688 // Ignore non-calls. 1689 CallInst *CI = dyn_cast<CallInst>(I); 1690 ++I; 1691 if (CI) { 1692 if (Simplifier.fold(CI, AA)) 1693 Changed = true; 1694 } 1695 } 1696 } 1697 return Changed; 1698 } 1699 1700 PreservedAnalyses AMDGPUSimplifyLibCallsPass::run(Function &F, 1701 FunctionAnalysisManager &AM) { 1702 AMDGPULibCalls Simplifier(&TM); 1703 Simplifier.initNativeFuncs(); 1704 Simplifier.initFunction(F); 1705 1706 bool Changed = false; 1707 auto AA = &AM.getResult<AAManager>(F); 1708 1709 LLVM_DEBUG(dbgs() << "AMDIC: process function "; 1710 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';); 1711 1712 for (auto &BB : F) { 1713 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) { 1714 // Ignore non-calls. 1715 CallInst *CI = dyn_cast<CallInst>(I); 1716 ++I; 1717 1718 if (CI) { 1719 if (Simplifier.fold(CI, AA)) 1720 Changed = true; 1721 } 1722 } 1723 } 1724 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 1725 } 1726 1727 bool AMDGPUUseNativeCalls::runOnFunction(Function &F) { 1728 if (skipFunction(F) || UseNative.empty()) 1729 return false; 1730 1731 Simplifier.initFunction(F); 1732 1733 bool Changed = false; 1734 for (auto &BB : F) { 1735 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) { 1736 // Ignore non-calls. 1737 CallInst *CI = dyn_cast<CallInst>(I); 1738 ++I; 1739 if (CI && Simplifier.useNative(CI)) 1740 Changed = true; 1741 } 1742 } 1743 return Changed; 1744 } 1745 1746 PreservedAnalyses AMDGPUUseNativeCallsPass::run(Function &F, 1747 FunctionAnalysisManager &AM) { 1748 if (UseNative.empty()) 1749 return PreservedAnalyses::all(); 1750 1751 AMDGPULibCalls Simplifier; 1752 Simplifier.initNativeFuncs(); 1753 Simplifier.initFunction(F); 1754 1755 bool Changed = false; 1756 for (auto &BB : F) { 1757 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) { 1758 // Ignore non-calls. 1759 CallInst *CI = dyn_cast<CallInst>(I); 1760 ++I; 1761 if (CI && Simplifier.useNative(CI)) 1762 Changed = true; 1763 } 1764 } 1765 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 1766 } 1767