1 //===- AMDGPULibCalls.cpp -------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file does AMD library function optimizations. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPU.h" 15 #include "AMDGPULibFunc.h" 16 #include "GCNSubtarget.h" 17 #include "llvm/Analysis/AliasAnalysis.h" 18 #include "llvm/Analysis/Loads.h" 19 #include "llvm/IR/IRBuilder.h" 20 #include "llvm/IR/IntrinsicInst.h" 21 #include "llvm/IR/IntrinsicsAMDGPU.h" 22 #include "llvm/InitializePasses.h" 23 #include "llvm/Target/TargetMachine.h" 24 #include <cmath> 25 26 #define DEBUG_TYPE "amdgpu-simplifylib" 27 28 using namespace llvm; 29 30 static cl::opt<bool> EnablePreLink("amdgpu-prelink", 31 cl::desc("Enable pre-link mode optimizations"), 32 cl::init(false), 33 cl::Hidden); 34 35 static cl::list<std::string> UseNative("amdgpu-use-native", 36 cl::desc("Comma separated list of functions to replace with native, or all"), 37 cl::CommaSeparated, cl::ValueOptional, 38 cl::Hidden); 39 40 #define MATH_PI numbers::pi 41 #define MATH_E numbers::e 42 #define MATH_SQRT2 numbers::sqrt2 43 #define MATH_SQRT1_2 numbers::inv_sqrt2 44 45 namespace llvm { 46 47 class AMDGPULibCalls { 48 private: 49 50 typedef llvm::AMDGPULibFunc FuncInfo; 51 52 const TargetMachine *TM; 53 54 bool UnsafeFPMath = false; 55 56 // -fuse-native. 57 bool AllNative = false; 58 59 bool useNativeFunc(const StringRef F) const; 60 61 // Return a pointer (pointer expr) to the function if function definition with 62 // "FuncName" exists. It may create a new function prototype in pre-link mode. 63 FunctionCallee getFunction(Module *M, const FuncInfo &fInfo); 64 65 bool parseFunctionName(const StringRef &FMangledName, FuncInfo &FInfo); 66 67 bool TDOFold(CallInst *CI, const FuncInfo &FInfo); 68 69 /* Specialized optimizations */ 70 71 // recip (half or native) 72 bool fold_recip(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 73 74 // divide (half or native) 75 bool fold_divide(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 76 77 // pow/powr/pown 78 bool fold_pow(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo); 79 80 // rootn 81 bool fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo); 82 83 // fma/mad 84 bool fold_fma_mad(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 85 86 // -fuse-native for sincos 87 bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo); 88 89 // evaluate calls if calls' arguments are constants. 90 bool evaluateScalarMathFunc(const FuncInfo &FInfo, double& Res0, 91 double& Res1, Constant *copr0, Constant *copr1, Constant *copr2); 92 bool evaluateCall(CallInst *aCI, const FuncInfo &FInfo); 93 94 // sqrt 95 bool fold_sqrt(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo); 96 97 // sin/cos 98 bool fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo, 99 AliasAnalysis *AA); 100 101 // __read_pipe/__write_pipe 102 bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, 103 const FuncInfo &FInfo); 104 105 // llvm.amdgcn.wavefrontsize 106 bool fold_wavefrontsize(CallInst *CI, IRBuilder<> &B); 107 108 // Get insertion point at entry. 109 BasicBlock::iterator getEntryIns(CallInst * UI); 110 // Insert an Alloc instruction. 111 AllocaInst* insertAlloca(CallInst * UI, IRBuilder<> &B, const char *prefix); 112 // Get a scalar native builtin single argument FP function 113 FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo); 114 115 protected: 116 bool isUnsafeMath(const FPMathOperator *FPOp) const; 117 118 bool canIncreasePrecisionOfConstantFold(const FPMathOperator *FPOp) const; 119 120 static void replaceCall(Instruction *I, Value *With) { 121 I->replaceAllUsesWith(With); 122 I->eraseFromParent(); 123 } 124 125 static void replaceCall(FPMathOperator *I, Value *With) { 126 replaceCall(cast<Instruction>(I), With); 127 } 128 129 public: 130 AMDGPULibCalls(const TargetMachine *TM_ = nullptr) : TM(TM_) {} 131 132 bool fold(CallInst *CI, AliasAnalysis *AA = nullptr); 133 134 void initFunction(const Function &F); 135 void initNativeFuncs(); 136 137 // Replace a normal math function call with that native version 138 bool useNative(CallInst *CI); 139 }; 140 141 } // end llvm namespace 142 143 namespace { 144 145 class AMDGPUSimplifyLibCalls : public FunctionPass { 146 147 AMDGPULibCalls Simplifier; 148 149 public: 150 static char ID; // Pass identification 151 152 AMDGPUSimplifyLibCalls(const TargetMachine *TM = nullptr) 153 : FunctionPass(ID), Simplifier(TM) { 154 initializeAMDGPUSimplifyLibCallsPass(*PassRegistry::getPassRegistry()); 155 } 156 157 void getAnalysisUsage(AnalysisUsage &AU) const override { 158 AU.addRequired<AAResultsWrapperPass>(); 159 } 160 161 bool runOnFunction(Function &M) override; 162 }; 163 164 class AMDGPUUseNativeCalls : public FunctionPass { 165 166 AMDGPULibCalls Simplifier; 167 168 public: 169 static char ID; // Pass identification 170 171 AMDGPUUseNativeCalls() : FunctionPass(ID) { 172 initializeAMDGPUUseNativeCallsPass(*PassRegistry::getPassRegistry()); 173 Simplifier.initNativeFuncs(); 174 } 175 176 bool runOnFunction(Function &F) override; 177 }; 178 179 } // end anonymous namespace. 180 181 char AMDGPUSimplifyLibCalls::ID = 0; 182 char AMDGPUUseNativeCalls::ID = 0; 183 184 INITIALIZE_PASS_BEGIN(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", 185 "Simplify well-known AMD library calls", false, false) 186 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 187 INITIALIZE_PASS_END(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", 188 "Simplify well-known AMD library calls", false, false) 189 190 INITIALIZE_PASS(AMDGPUUseNativeCalls, "amdgpu-usenative", 191 "Replace builtin math calls with that native versions.", 192 false, false) 193 194 template <typename IRB> 195 static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg, 196 const Twine &Name = "") { 197 CallInst *R = B.CreateCall(Callee, Arg, Name); 198 if (Function *F = dyn_cast<Function>(Callee.getCallee())) 199 R->setCallingConv(F->getCallingConv()); 200 return R; 201 } 202 203 template <typename IRB> 204 static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1, 205 Value *Arg2, const Twine &Name = "") { 206 CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name); 207 if (Function *F = dyn_cast<Function>(Callee.getCallee())) 208 R->setCallingConv(F->getCallingConv()); 209 return R; 210 } 211 212 // Data structures for table-driven optimizations. 213 // FuncTbl works for both f32 and f64 functions with 1 input argument 214 215 struct TableEntry { 216 double result; 217 double input; 218 }; 219 220 /* a list of {result, input} */ 221 static const TableEntry tbl_acos[] = { 222 {MATH_PI / 2.0, 0.0}, 223 {MATH_PI / 2.0, -0.0}, 224 {0.0, 1.0}, 225 {MATH_PI, -1.0} 226 }; 227 static const TableEntry tbl_acosh[] = { 228 {0.0, 1.0} 229 }; 230 static const TableEntry tbl_acospi[] = { 231 {0.5, 0.0}, 232 {0.5, -0.0}, 233 {0.0, 1.0}, 234 {1.0, -1.0} 235 }; 236 static const TableEntry tbl_asin[] = { 237 {0.0, 0.0}, 238 {-0.0, -0.0}, 239 {MATH_PI / 2.0, 1.0}, 240 {-MATH_PI / 2.0, -1.0} 241 }; 242 static const TableEntry tbl_asinh[] = { 243 {0.0, 0.0}, 244 {-0.0, -0.0} 245 }; 246 static const TableEntry tbl_asinpi[] = { 247 {0.0, 0.0}, 248 {-0.0, -0.0}, 249 {0.5, 1.0}, 250 {-0.5, -1.0} 251 }; 252 static const TableEntry tbl_atan[] = { 253 {0.0, 0.0}, 254 {-0.0, -0.0}, 255 {MATH_PI / 4.0, 1.0}, 256 {-MATH_PI / 4.0, -1.0} 257 }; 258 static const TableEntry tbl_atanh[] = { 259 {0.0, 0.0}, 260 {-0.0, -0.0} 261 }; 262 static const TableEntry tbl_atanpi[] = { 263 {0.0, 0.0}, 264 {-0.0, -0.0}, 265 {0.25, 1.0}, 266 {-0.25, -1.0} 267 }; 268 static const TableEntry tbl_cbrt[] = { 269 {0.0, 0.0}, 270 {-0.0, -0.0}, 271 {1.0, 1.0}, 272 {-1.0, -1.0}, 273 }; 274 static const TableEntry tbl_cos[] = { 275 {1.0, 0.0}, 276 {1.0, -0.0} 277 }; 278 static const TableEntry tbl_cosh[] = { 279 {1.0, 0.0}, 280 {1.0, -0.0} 281 }; 282 static const TableEntry tbl_cospi[] = { 283 {1.0, 0.0}, 284 {1.0, -0.0} 285 }; 286 static const TableEntry tbl_erfc[] = { 287 {1.0, 0.0}, 288 {1.0, -0.0} 289 }; 290 static const TableEntry tbl_erf[] = { 291 {0.0, 0.0}, 292 {-0.0, -0.0} 293 }; 294 static const TableEntry tbl_exp[] = { 295 {1.0, 0.0}, 296 {1.0, -0.0}, 297 {MATH_E, 1.0} 298 }; 299 static const TableEntry tbl_exp2[] = { 300 {1.0, 0.0}, 301 {1.0, -0.0}, 302 {2.0, 1.0} 303 }; 304 static const TableEntry tbl_exp10[] = { 305 {1.0, 0.0}, 306 {1.0, -0.0}, 307 {10.0, 1.0} 308 }; 309 static const TableEntry tbl_expm1[] = { 310 {0.0, 0.0}, 311 {-0.0, -0.0} 312 }; 313 static const TableEntry tbl_log[] = { 314 {0.0, 1.0}, 315 {1.0, MATH_E} 316 }; 317 static const TableEntry tbl_log2[] = { 318 {0.0, 1.0}, 319 {1.0, 2.0} 320 }; 321 static const TableEntry tbl_log10[] = { 322 {0.0, 1.0}, 323 {1.0, 10.0} 324 }; 325 static const TableEntry tbl_rsqrt[] = { 326 {1.0, 1.0}, 327 {MATH_SQRT1_2, 2.0} 328 }; 329 static const TableEntry tbl_sin[] = { 330 {0.0, 0.0}, 331 {-0.0, -0.0} 332 }; 333 static const TableEntry tbl_sinh[] = { 334 {0.0, 0.0}, 335 {-0.0, -0.0} 336 }; 337 static const TableEntry tbl_sinpi[] = { 338 {0.0, 0.0}, 339 {-0.0, -0.0} 340 }; 341 static const TableEntry tbl_sqrt[] = { 342 {0.0, 0.0}, 343 {1.0, 1.0}, 344 {MATH_SQRT2, 2.0} 345 }; 346 static const TableEntry tbl_tan[] = { 347 {0.0, 0.0}, 348 {-0.0, -0.0} 349 }; 350 static const TableEntry tbl_tanh[] = { 351 {0.0, 0.0}, 352 {-0.0, -0.0} 353 }; 354 static const TableEntry tbl_tanpi[] = { 355 {0.0, 0.0}, 356 {-0.0, -0.0} 357 }; 358 static const TableEntry tbl_tgamma[] = { 359 {1.0, 1.0}, 360 {1.0, 2.0}, 361 {2.0, 3.0}, 362 {6.0, 4.0} 363 }; 364 365 static bool HasNative(AMDGPULibFunc::EFuncId id) { 366 switch(id) { 367 case AMDGPULibFunc::EI_DIVIDE: 368 case AMDGPULibFunc::EI_COS: 369 case AMDGPULibFunc::EI_EXP: 370 case AMDGPULibFunc::EI_EXP2: 371 case AMDGPULibFunc::EI_EXP10: 372 case AMDGPULibFunc::EI_LOG: 373 case AMDGPULibFunc::EI_LOG2: 374 case AMDGPULibFunc::EI_LOG10: 375 case AMDGPULibFunc::EI_POWR: 376 case AMDGPULibFunc::EI_RECIP: 377 case AMDGPULibFunc::EI_RSQRT: 378 case AMDGPULibFunc::EI_SIN: 379 case AMDGPULibFunc::EI_SINCOS: 380 case AMDGPULibFunc::EI_SQRT: 381 case AMDGPULibFunc::EI_TAN: 382 return true; 383 default:; 384 } 385 return false; 386 } 387 388 using TableRef = ArrayRef<TableEntry>; 389 390 static TableRef getOptTable(AMDGPULibFunc::EFuncId id) { 391 switch(id) { 392 case AMDGPULibFunc::EI_ACOS: return TableRef(tbl_acos); 393 case AMDGPULibFunc::EI_ACOSH: return TableRef(tbl_acosh); 394 case AMDGPULibFunc::EI_ACOSPI: return TableRef(tbl_acospi); 395 case AMDGPULibFunc::EI_ASIN: return TableRef(tbl_asin); 396 case AMDGPULibFunc::EI_ASINH: return TableRef(tbl_asinh); 397 case AMDGPULibFunc::EI_ASINPI: return TableRef(tbl_asinpi); 398 case AMDGPULibFunc::EI_ATAN: return TableRef(tbl_atan); 399 case AMDGPULibFunc::EI_ATANH: return TableRef(tbl_atanh); 400 case AMDGPULibFunc::EI_ATANPI: return TableRef(tbl_atanpi); 401 case AMDGPULibFunc::EI_CBRT: return TableRef(tbl_cbrt); 402 case AMDGPULibFunc::EI_NCOS: 403 case AMDGPULibFunc::EI_COS: return TableRef(tbl_cos); 404 case AMDGPULibFunc::EI_COSH: return TableRef(tbl_cosh); 405 case AMDGPULibFunc::EI_COSPI: return TableRef(tbl_cospi); 406 case AMDGPULibFunc::EI_ERFC: return TableRef(tbl_erfc); 407 case AMDGPULibFunc::EI_ERF: return TableRef(tbl_erf); 408 case AMDGPULibFunc::EI_EXP: return TableRef(tbl_exp); 409 case AMDGPULibFunc::EI_NEXP2: 410 case AMDGPULibFunc::EI_EXP2: return TableRef(tbl_exp2); 411 case AMDGPULibFunc::EI_EXP10: return TableRef(tbl_exp10); 412 case AMDGPULibFunc::EI_EXPM1: return TableRef(tbl_expm1); 413 case AMDGPULibFunc::EI_LOG: return TableRef(tbl_log); 414 case AMDGPULibFunc::EI_NLOG2: 415 case AMDGPULibFunc::EI_LOG2: return TableRef(tbl_log2); 416 case AMDGPULibFunc::EI_LOG10: return TableRef(tbl_log10); 417 case AMDGPULibFunc::EI_NRSQRT: 418 case AMDGPULibFunc::EI_RSQRT: return TableRef(tbl_rsqrt); 419 case AMDGPULibFunc::EI_NSIN: 420 case AMDGPULibFunc::EI_SIN: return TableRef(tbl_sin); 421 case AMDGPULibFunc::EI_SINH: return TableRef(tbl_sinh); 422 case AMDGPULibFunc::EI_SINPI: return TableRef(tbl_sinpi); 423 case AMDGPULibFunc::EI_NSQRT: 424 case AMDGPULibFunc::EI_SQRT: return TableRef(tbl_sqrt); 425 case AMDGPULibFunc::EI_TAN: return TableRef(tbl_tan); 426 case AMDGPULibFunc::EI_TANH: return TableRef(tbl_tanh); 427 case AMDGPULibFunc::EI_TANPI: return TableRef(tbl_tanpi); 428 case AMDGPULibFunc::EI_TGAMMA: return TableRef(tbl_tgamma); 429 default:; 430 } 431 return TableRef(); 432 } 433 434 static inline int getVecSize(const AMDGPULibFunc& FInfo) { 435 return FInfo.getLeads()[0].VectorSize; 436 } 437 438 static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) { 439 return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType; 440 } 441 442 FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) { 443 // If we are doing PreLinkOpt, the function is external. So it is safe to 444 // use getOrInsertFunction() at this stage. 445 446 return EnablePreLink ? AMDGPULibFunc::getOrInsertFunction(M, fInfo) 447 : AMDGPULibFunc::getFunction(M, fInfo); 448 } 449 450 bool AMDGPULibCalls::parseFunctionName(const StringRef &FMangledName, 451 FuncInfo &FInfo) { 452 return AMDGPULibFunc::parse(FMangledName, FInfo); 453 } 454 455 bool AMDGPULibCalls::isUnsafeMath(const FPMathOperator *FPOp) const { 456 return UnsafeFPMath || FPOp->isFast(); 457 } 458 459 bool AMDGPULibCalls::canIncreasePrecisionOfConstantFold( 460 const FPMathOperator *FPOp) const { 461 // TODO: Refine to approxFunc or contract 462 return isUnsafeMath(FPOp); 463 } 464 465 void AMDGPULibCalls::initFunction(const Function &F) { 466 UnsafeFPMath = F.getFnAttribute("unsafe-fp-math").getValueAsBool(); 467 } 468 469 bool AMDGPULibCalls::useNativeFunc(const StringRef F) const { 470 return AllNative || llvm::is_contained(UseNative, F); 471 } 472 473 void AMDGPULibCalls::initNativeFuncs() { 474 AllNative = useNativeFunc("all") || 475 (UseNative.getNumOccurrences() && UseNative.size() == 1 && 476 UseNative.begin()->empty()); 477 } 478 479 bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) { 480 bool native_sin = useNativeFunc("sin"); 481 bool native_cos = useNativeFunc("cos"); 482 483 if (native_sin && native_cos) { 484 Module *M = aCI->getModule(); 485 Value *opr0 = aCI->getArgOperand(0); 486 487 AMDGPULibFunc nf; 488 nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType; 489 nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize; 490 491 nf.setPrefix(AMDGPULibFunc::NATIVE); 492 nf.setId(AMDGPULibFunc::EI_SIN); 493 FunctionCallee sinExpr = getFunction(M, nf); 494 495 nf.setPrefix(AMDGPULibFunc::NATIVE); 496 nf.setId(AMDGPULibFunc::EI_COS); 497 FunctionCallee cosExpr = getFunction(M, nf); 498 if (sinExpr && cosExpr) { 499 Value *sinval = CallInst::Create(sinExpr, opr0, "splitsin", aCI); 500 Value *cosval = CallInst::Create(cosExpr, opr0, "splitcos", aCI); 501 new StoreInst(cosval, aCI->getArgOperand(1), aCI); 502 503 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI 504 << " with native version of sin/cos"); 505 506 replaceCall(aCI, sinval); 507 return true; 508 } 509 } 510 return false; 511 } 512 513 bool AMDGPULibCalls::useNative(CallInst *aCI) { 514 Function *Callee = aCI->getCalledFunction(); 515 if (!Callee || aCI->isNoBuiltin()) 516 return false; 517 518 FuncInfo FInfo; 519 if (!parseFunctionName(Callee->getName(), FInfo) || !FInfo.isMangled() || 520 FInfo.getPrefix() != AMDGPULibFunc::NOPFX || 521 getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()) || 522 !(AllNative || useNativeFunc(FInfo.getName()))) { 523 return false; 524 } 525 526 if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS) 527 return sincosUseNative(aCI, FInfo); 528 529 FInfo.setPrefix(AMDGPULibFunc::NATIVE); 530 FunctionCallee F = getFunction(aCI->getModule(), FInfo); 531 if (!F) 532 return false; 533 534 aCI->setCalledFunction(F); 535 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI 536 << " with native version"); 537 return true; 538 } 539 540 // Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe 541 // builtin, with appended type size and alignment arguments, where 2 or 4 542 // indicates the original number of arguments. The library has optimized version 543 // of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same 544 // power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N 545 // for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ..., 546 // 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4. 547 bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, 548 const FuncInfo &FInfo) { 549 auto *Callee = CI->getCalledFunction(); 550 if (!Callee->isDeclaration()) 551 return false; 552 553 assert(Callee->hasName() && "Invalid read_pipe/write_pipe function"); 554 auto *M = Callee->getParent(); 555 std::string Name = std::string(Callee->getName()); 556 auto NumArg = CI->arg_size(); 557 if (NumArg != 4 && NumArg != 6) 558 return false; 559 ConstantInt *PacketSize = 560 dyn_cast<ConstantInt>(CI->getArgOperand(NumArg - 2)); 561 ConstantInt *PacketAlign = 562 dyn_cast<ConstantInt>(CI->getArgOperand(NumArg - 1)); 563 if (!PacketSize || !PacketAlign) 564 return false; 565 566 unsigned Size = PacketSize->getZExtValue(); 567 Align Alignment = PacketAlign->getAlignValue(); 568 if (Alignment != Size) 569 return false; 570 571 unsigned PtrArgLoc = CI->arg_size() - 3; 572 Value *PtrArg = CI->getArgOperand(PtrArgLoc); 573 Type *PtrTy = PtrArg->getType(); 574 575 SmallVector<llvm::Type *, 6> ArgTys; 576 for (unsigned I = 0; I != PtrArgLoc; ++I) 577 ArgTys.push_back(CI->getArgOperand(I)->getType()); 578 ArgTys.push_back(PtrTy); 579 580 Name = Name + "_" + std::to_string(Size); 581 auto *FTy = FunctionType::get(Callee->getReturnType(), 582 ArrayRef<Type *>(ArgTys), false); 583 AMDGPULibFunc NewLibFunc(Name, FTy); 584 FunctionCallee F = AMDGPULibFunc::getOrInsertFunction(M, NewLibFunc); 585 if (!F) 586 return false; 587 588 auto *BCast = B.CreatePointerCast(PtrArg, PtrTy); 589 SmallVector<Value *, 6> Args; 590 for (unsigned I = 0; I != PtrArgLoc; ++I) 591 Args.push_back(CI->getArgOperand(I)); 592 Args.push_back(BCast); 593 594 auto *NCI = B.CreateCall(F, Args); 595 NCI->setAttributes(CI->getAttributes()); 596 CI->replaceAllUsesWith(NCI); 597 CI->dropAllReferences(); 598 CI->eraseFromParent(); 599 600 return true; 601 } 602 603 // This function returns false if no change; return true otherwise. 604 bool AMDGPULibCalls::fold(CallInst *CI, AliasAnalysis *AA) { 605 Function *Callee = CI->getCalledFunction(); 606 // Ignore indirect calls. 607 if (!Callee || CI->isNoBuiltin()) 608 return false; 609 610 IRBuilder<> B(CI); 611 switch (Callee->getIntrinsicID()) { 612 case Intrinsic::not_intrinsic: 613 break; 614 case Intrinsic::amdgcn_wavefrontsize: 615 return !EnablePreLink && fold_wavefrontsize(CI, B); 616 default: 617 return false; 618 } 619 620 FuncInfo FInfo; 621 if (!parseFunctionName(Callee->getName(), FInfo)) 622 return false; 623 624 // Further check the number of arguments to see if they match. 625 if (CI->arg_size() != FInfo.getNumArgs()) 626 return false; 627 628 LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << '\n'); 629 630 if (TDOFold(CI, FInfo)) 631 return true; 632 633 if (FPMathOperator *FPOp = dyn_cast<FPMathOperator>(CI)) { 634 // Under unsafe-math, evaluate calls if possible. 635 // According to Brian Sumner, we can do this for all f32 function calls 636 // using host's double function calls. 637 if (canIncreasePrecisionOfConstantFold(FPOp) && evaluateCall(CI, FInfo)) 638 return true; 639 640 // Copy fast flags from the original call. 641 B.setFastMathFlags(FPOp->getFastMathFlags()); 642 643 // Specialized optimizations for each function call 644 switch (FInfo.getId()) { 645 case AMDGPULibFunc::EI_POW: 646 case AMDGPULibFunc::EI_POWR: 647 case AMDGPULibFunc::EI_POWN: 648 return fold_pow(FPOp, B, FInfo); 649 case AMDGPULibFunc::EI_ROOTN: 650 return fold_rootn(FPOp, B, FInfo); 651 case AMDGPULibFunc::EI_SQRT: 652 return fold_sqrt(FPOp, B, FInfo); 653 case AMDGPULibFunc::EI_COS: 654 case AMDGPULibFunc::EI_SIN: 655 return fold_sincos(FPOp, B, FInfo, AA); 656 case AMDGPULibFunc::EI_RECIP: 657 // skip vector function 658 assert((FInfo.getPrefix() == AMDGPULibFunc::NATIVE || 659 FInfo.getPrefix() == AMDGPULibFunc::HALF) && 660 "recip must be an either native or half function"); 661 return (getVecSize(FInfo) != 1) ? false : fold_recip(CI, B, FInfo); 662 663 case AMDGPULibFunc::EI_DIVIDE: 664 // skip vector function 665 assert((FInfo.getPrefix() == AMDGPULibFunc::NATIVE || 666 FInfo.getPrefix() == AMDGPULibFunc::HALF) && 667 "divide must be an either native or half function"); 668 return (getVecSize(FInfo) != 1) ? false : fold_divide(CI, B, FInfo); 669 case AMDGPULibFunc::EI_FMA: 670 case AMDGPULibFunc::EI_MAD: 671 case AMDGPULibFunc::EI_NFMA: 672 // skip vector function 673 return (getVecSize(FInfo) != 1) ? false : fold_fma_mad(CI, B, FInfo); 674 default: 675 break; 676 } 677 } else { 678 // Specialized optimizations for each function call 679 switch (FInfo.getId()) { 680 case AMDGPULibFunc::EI_READ_PIPE_2: 681 case AMDGPULibFunc::EI_READ_PIPE_4: 682 case AMDGPULibFunc::EI_WRITE_PIPE_2: 683 case AMDGPULibFunc::EI_WRITE_PIPE_4: 684 return fold_read_write_pipe(CI, B, FInfo); 685 default: 686 break; 687 } 688 } 689 690 return false; 691 } 692 693 bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) { 694 // Table-Driven optimization 695 const TableRef tr = getOptTable(FInfo.getId()); 696 if (tr.empty()) 697 return false; 698 699 int const sz = (int)tr.size(); 700 Value *opr0 = CI->getArgOperand(0); 701 702 if (getVecSize(FInfo) > 1) { 703 if (ConstantDataVector *CV = dyn_cast<ConstantDataVector>(opr0)) { 704 SmallVector<double, 0> DVal; 705 for (int eltNo = 0; eltNo < getVecSize(FInfo); ++eltNo) { 706 ConstantFP *eltval = dyn_cast<ConstantFP>( 707 CV->getElementAsConstant((unsigned)eltNo)); 708 assert(eltval && "Non-FP arguments in math function!"); 709 bool found = false; 710 for (int i=0; i < sz; ++i) { 711 if (eltval->isExactlyValue(tr[i].input)) { 712 DVal.push_back(tr[i].result); 713 found = true; 714 break; 715 } 716 } 717 if (!found) { 718 // This vector constants not handled yet. 719 return false; 720 } 721 } 722 LLVMContext &context = CI->getParent()->getParent()->getContext(); 723 Constant *nval; 724 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 725 SmallVector<float, 0> FVal; 726 for (unsigned i = 0; i < DVal.size(); ++i) { 727 FVal.push_back((float)DVal[i]); 728 } 729 ArrayRef<float> tmp(FVal); 730 nval = ConstantDataVector::get(context, tmp); 731 } else { // F64 732 ArrayRef<double> tmp(DVal); 733 nval = ConstantDataVector::get(context, tmp); 734 } 735 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 736 replaceCall(CI, nval); 737 return true; 738 } 739 } else { 740 // Scalar version 741 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) { 742 for (int i = 0; i < sz; ++i) { 743 if (CF->isExactlyValue(tr[i].input)) { 744 Value *nval = ConstantFP::get(CF->getType(), tr[i].result); 745 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 746 replaceCall(CI, nval); 747 return true; 748 } 749 } 750 } 751 } 752 753 return false; 754 } 755 756 // [native_]half_recip(c) ==> 1.0/c 757 bool AMDGPULibCalls::fold_recip(CallInst *CI, IRBuilder<> &B, 758 const FuncInfo &FInfo) { 759 Value *opr0 = CI->getArgOperand(0); 760 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) { 761 // Just create a normal div. Later, InstCombine will be able 762 // to compute the divide into a constant (avoid check float infinity 763 // or subnormal at this point). 764 Value *nval = B.CreateFDiv(ConstantFP::get(CF->getType(), 1.0), 765 opr0, 766 "recip2div"); 767 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 768 replaceCall(CI, nval); 769 return true; 770 } 771 return false; 772 } 773 774 // [native_]half_divide(x, c) ==> x/c 775 bool AMDGPULibCalls::fold_divide(CallInst *CI, IRBuilder<> &B, 776 const FuncInfo &FInfo) { 777 Value *opr0 = CI->getArgOperand(0); 778 Value *opr1 = CI->getArgOperand(1); 779 ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0); 780 ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1); 781 782 if ((CF0 && CF1) || // both are constants 783 (CF1 && (getArgType(FInfo) == AMDGPULibFunc::F32))) 784 // CF1 is constant && f32 divide 785 { 786 Value *nval1 = B.CreateFDiv(ConstantFP::get(opr1->getType(), 1.0), 787 opr1, "__div2recip"); 788 Value *nval = B.CreateFMul(opr0, nval1, "__div2mul"); 789 replaceCall(CI, nval); 790 return true; 791 } 792 return false; 793 } 794 795 namespace llvm { 796 static double log2(double V) { 797 #if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L 798 return ::log2(V); 799 #else 800 return log(V) / numbers::ln2; 801 #endif 802 } 803 } 804 805 bool AMDGPULibCalls::fold_pow(FPMathOperator *FPOp, IRBuilder<> &B, 806 const FuncInfo &FInfo) { 807 assert((FInfo.getId() == AMDGPULibFunc::EI_POW || 808 FInfo.getId() == AMDGPULibFunc::EI_POWR || 809 FInfo.getId() == AMDGPULibFunc::EI_POWN) && 810 "fold_pow: encounter a wrong function call"); 811 812 Module *M = B.GetInsertBlock()->getModule(); 813 ConstantFP *CF; 814 ConstantInt *CINT; 815 Type *eltType; 816 Value *opr0 = FPOp->getOperand(0); 817 Value *opr1 = FPOp->getOperand(1); 818 ConstantAggregateZero *CZero = dyn_cast<ConstantAggregateZero>(opr1); 819 820 if (getVecSize(FInfo) == 1) { 821 eltType = opr0->getType(); 822 CF = dyn_cast<ConstantFP>(opr1); 823 CINT = dyn_cast<ConstantInt>(opr1); 824 } else { 825 VectorType *VTy = dyn_cast<VectorType>(opr0->getType()); 826 assert(VTy && "Oprand of vector function should be of vectortype"); 827 eltType = VTy->getElementType(); 828 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1); 829 830 // Now, only Handle vector const whose elements have the same value. 831 CF = CDV ? dyn_cast_or_null<ConstantFP>(CDV->getSplatValue()) : nullptr; 832 CINT = CDV ? dyn_cast_or_null<ConstantInt>(CDV->getSplatValue()) : nullptr; 833 } 834 835 // No unsafe math , no constant argument, do nothing 836 if (!isUnsafeMath(FPOp) && !CF && !CINT && !CZero) 837 return false; 838 839 // 0x1111111 means that we don't do anything for this call. 840 int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111); 841 842 if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0) || CZero) { 843 // pow/powr/pown(x, 0) == 1 844 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1\n"); 845 Constant *cnval = ConstantFP::get(eltType, 1.0); 846 if (getVecSize(FInfo) > 1) { 847 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 848 } 849 replaceCall(FPOp, cnval); 850 return true; 851 } 852 if ((CF && CF->isExactlyValue(1.0)) || (CINT && ci_opr1 == 1)) { 853 // pow/powr/pown(x, 1.0) = x 854 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << "\n"); 855 replaceCall(FPOp, opr0); 856 return true; 857 } 858 if ((CF && CF->isExactlyValue(2.0)) || (CINT && ci_opr1 == 2)) { 859 // pow/powr/pown(x, 2.0) = x*x 860 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << " * " 861 << *opr0 << "\n"); 862 Value *nval = B.CreateFMul(opr0, opr0, "__pow2"); 863 replaceCall(FPOp, nval); 864 return true; 865 } 866 if ((CF && CF->isExactlyValue(-1.0)) || (CINT && ci_opr1 == -1)) { 867 // pow/powr/pown(x, -1.0) = 1.0/x 868 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1 / " << *opr0 << "\n"); 869 Constant *cnval = ConstantFP::get(eltType, 1.0); 870 if (getVecSize(FInfo) > 1) { 871 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 872 } 873 Value *nval = B.CreateFDiv(cnval, opr0, "__powrecip"); 874 replaceCall(FPOp, nval); 875 return true; 876 } 877 878 if (CF && (CF->isExactlyValue(0.5) || CF->isExactlyValue(-0.5))) { 879 // pow[r](x, [-]0.5) = sqrt(x) 880 bool issqrt = CF->isExactlyValue(0.5); 881 if (FunctionCallee FPExpr = 882 getFunction(M, AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT 883 : AMDGPULibFunc::EI_RSQRT, 884 FInfo))) { 885 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << FInfo.getName() 886 << '(' << *opr0 << ")\n"); 887 Value *nval = CreateCallEx(B,FPExpr, opr0, issqrt ? "__pow2sqrt" 888 : "__pow2rsqrt"); 889 replaceCall(FPOp, nval); 890 return true; 891 } 892 } 893 894 if (!isUnsafeMath(FPOp)) 895 return false; 896 897 // Unsafe Math optimization 898 899 // Remember that ci_opr1 is set if opr1 is integral 900 if (CF) { 901 double dval = (getArgType(FInfo) == AMDGPULibFunc::F32) 902 ? (double)CF->getValueAPF().convertToFloat() 903 : CF->getValueAPF().convertToDouble(); 904 int ival = (int)dval; 905 if ((double)ival == dval) { 906 ci_opr1 = ival; 907 } else 908 ci_opr1 = 0x11111111; 909 } 910 911 // pow/powr/pown(x, c) = [1/](x*x*..x); where 912 // trunc(c) == c && the number of x == c && |c| <= 12 913 unsigned abs_opr1 = (ci_opr1 < 0) ? -ci_opr1 : ci_opr1; 914 if (abs_opr1 <= 12) { 915 Constant *cnval; 916 Value *nval; 917 if (abs_opr1 == 0) { 918 cnval = ConstantFP::get(eltType, 1.0); 919 if (getVecSize(FInfo) > 1) { 920 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 921 } 922 nval = cnval; 923 } else { 924 Value *valx2 = nullptr; 925 nval = nullptr; 926 while (abs_opr1 > 0) { 927 valx2 = valx2 ? B.CreateFMul(valx2, valx2, "__powx2") : opr0; 928 if (abs_opr1 & 1) { 929 nval = nval ? B.CreateFMul(nval, valx2, "__powprod") : valx2; 930 } 931 abs_opr1 >>= 1; 932 } 933 } 934 935 if (ci_opr1 < 0) { 936 cnval = ConstantFP::get(eltType, 1.0); 937 if (getVecSize(FInfo) > 1) { 938 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 939 } 940 nval = B.CreateFDiv(cnval, nval, "__1powprod"); 941 } 942 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " 943 << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr0 944 << ")\n"); 945 replaceCall(FPOp, nval); 946 return true; 947 } 948 949 // powr ---> exp2(y * log2(x)) 950 // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31)) 951 FunctionCallee ExpExpr = 952 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo)); 953 if (!ExpExpr) 954 return false; 955 956 bool needlog = false; 957 bool needabs = false; 958 bool needcopysign = false; 959 Constant *cnval = nullptr; 960 if (getVecSize(FInfo) == 1) { 961 CF = dyn_cast<ConstantFP>(opr0); 962 963 if (CF) { 964 double V = (getArgType(FInfo) == AMDGPULibFunc::F32) 965 ? (double)CF->getValueAPF().convertToFloat() 966 : CF->getValueAPF().convertToDouble(); 967 968 V = log2(std::abs(V)); 969 cnval = ConstantFP::get(eltType, V); 970 needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR) && 971 CF->isNegative(); 972 } else { 973 needlog = true; 974 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR && 975 (!CF || CF->isNegative()); 976 } 977 } else { 978 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr0); 979 980 if (!CDV) { 981 needlog = true; 982 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR; 983 } else { 984 assert ((int)CDV->getNumElements() == getVecSize(FInfo) && 985 "Wrong vector size detected"); 986 987 SmallVector<double, 0> DVal; 988 for (int i=0; i < getVecSize(FInfo); ++i) { 989 double V = (getArgType(FInfo) == AMDGPULibFunc::F32) 990 ? (double)CDV->getElementAsFloat(i) 991 : CDV->getElementAsDouble(i); 992 if (V < 0.0) needcopysign = true; 993 V = log2(std::abs(V)); 994 DVal.push_back(V); 995 } 996 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 997 SmallVector<float, 0> FVal; 998 for (unsigned i=0; i < DVal.size(); ++i) { 999 FVal.push_back((float)DVal[i]); 1000 } 1001 ArrayRef<float> tmp(FVal); 1002 cnval = ConstantDataVector::get(M->getContext(), tmp); 1003 } else { 1004 ArrayRef<double> tmp(DVal); 1005 cnval = ConstantDataVector::get(M->getContext(), tmp); 1006 } 1007 } 1008 } 1009 1010 if (needcopysign && (FInfo.getId() == AMDGPULibFunc::EI_POW)) { 1011 // We cannot handle corner cases for a general pow() function, give up 1012 // unless y is a constant integral value. Then proceed as if it were pown. 1013 if (getVecSize(FInfo) == 1) { 1014 if (const ConstantFP *CF = dyn_cast<ConstantFP>(opr1)) { 1015 double y = (getArgType(FInfo) == AMDGPULibFunc::F32) 1016 ? (double)CF->getValueAPF().convertToFloat() 1017 : CF->getValueAPF().convertToDouble(); 1018 if (y != (double)(int64_t)y) 1019 return false; 1020 } else 1021 return false; 1022 } else { 1023 if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1)) { 1024 for (int i=0; i < getVecSize(FInfo); ++i) { 1025 double y = (getArgType(FInfo) == AMDGPULibFunc::F32) 1026 ? (double)CDV->getElementAsFloat(i) 1027 : CDV->getElementAsDouble(i); 1028 if (y != (double)(int64_t)y) 1029 return false; 1030 } 1031 } else 1032 return false; 1033 } 1034 } 1035 1036 Value *nval; 1037 if (needabs) { 1038 FunctionCallee AbsExpr = 1039 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_FABS, FInfo)); 1040 if (!AbsExpr) 1041 return false; 1042 nval = CreateCallEx(B, AbsExpr, opr0, "__fabs"); 1043 } else { 1044 nval = cnval ? cnval : opr0; 1045 } 1046 if (needlog) { 1047 FunctionCallee LogExpr = 1048 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo)); 1049 if (!LogExpr) 1050 return false; 1051 nval = CreateCallEx(B,LogExpr, nval, "__log2"); 1052 } 1053 1054 if (FInfo.getId() == AMDGPULibFunc::EI_POWN) { 1055 // convert int(32) to fp(f32 or f64) 1056 opr1 = B.CreateSIToFP(opr1, nval->getType(), "pownI2F"); 1057 } 1058 nval = B.CreateFMul(opr1, nval, "__ylogx"); 1059 nval = CreateCallEx(B,ExpExpr, nval, "__exp2"); 1060 1061 if (needcopysign) { 1062 Value *opr_n; 1063 Type* rTy = opr0->getType(); 1064 Type* nTyS = eltType->isDoubleTy() ? B.getInt64Ty() : B.getInt32Ty(); 1065 Type *nTy = nTyS; 1066 if (const auto *vTy = dyn_cast<FixedVectorType>(rTy)) 1067 nTy = FixedVectorType::get(nTyS, vTy); 1068 unsigned size = nTy->getScalarSizeInBits(); 1069 opr_n = FPOp->getOperand(1); 1070 if (opr_n->getType()->isIntegerTy()) 1071 opr_n = B.CreateZExtOrBitCast(opr_n, nTy, "__ytou"); 1072 else 1073 opr_n = B.CreateFPToSI(opr1, nTy, "__ytou"); 1074 1075 Value *sign = B.CreateShl(opr_n, size-1, "__yeven"); 1076 sign = B.CreateAnd(B.CreateBitCast(opr0, nTy), sign, "__pow_sign"); 1077 nval = B.CreateOr(B.CreateBitCast(nval, nTy), sign); 1078 nval = B.CreateBitCast(nval, opr0->getType()); 1079 } 1080 1081 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " 1082 << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n"); 1083 replaceCall(FPOp, nval); 1084 1085 return true; 1086 } 1087 1088 bool AMDGPULibCalls::fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B, 1089 const FuncInfo &FInfo) { 1090 // skip vector function 1091 if (getVecSize(FInfo) != 1) 1092 return false; 1093 1094 Value *opr0 = FPOp->getOperand(0); 1095 Value *opr1 = FPOp->getOperand(1); 1096 1097 ConstantInt *CINT = dyn_cast<ConstantInt>(opr1); 1098 if (!CINT) { 1099 return false; 1100 } 1101 int ci_opr1 = (int)CINT->getSExtValue(); 1102 if (ci_opr1 == 1) { // rootn(x, 1) = x 1103 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << "\n"); 1104 replaceCall(FPOp, opr0); 1105 return true; 1106 } 1107 1108 Module *M = B.GetInsertBlock()->getModule(); 1109 if (ci_opr1 == 2) { // rootn(x, 2) = sqrt(x) 1110 if (FunctionCallee FPExpr = 1111 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) { 1112 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> sqrt(" << *opr0 1113 << ")\n"); 1114 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2sqrt"); 1115 replaceCall(FPOp, nval); 1116 return true; 1117 } 1118 } else if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x) 1119 if (FunctionCallee FPExpr = 1120 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) { 1121 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> cbrt(" << *opr0 1122 << ")\n"); 1123 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2cbrt"); 1124 replaceCall(FPOp, nval); 1125 return true; 1126 } 1127 } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x 1128 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1.0 / " << *opr0 << "\n"); 1129 Value *nval = B.CreateFDiv(ConstantFP::get(opr0->getType(), 1.0), 1130 opr0, 1131 "__rootn2div"); 1132 replaceCall(FPOp, nval); 1133 return true; 1134 } else if (ci_opr1 == -2) { // rootn(x, -2) = rsqrt(x) 1135 if (FunctionCallee FPExpr = 1136 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_RSQRT, FInfo))) { 1137 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> rsqrt(" << *opr0 1138 << ")\n"); 1139 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2rsqrt"); 1140 replaceCall(FPOp, nval); 1141 return true; 1142 } 1143 } 1144 return false; 1145 } 1146 1147 bool AMDGPULibCalls::fold_fma_mad(CallInst *CI, IRBuilder<> &B, 1148 const FuncInfo &FInfo) { 1149 Value *opr0 = CI->getArgOperand(0); 1150 Value *opr1 = CI->getArgOperand(1); 1151 Value *opr2 = CI->getArgOperand(2); 1152 1153 ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0); 1154 ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1); 1155 if ((CF0 && CF0->isZero()) || (CF1 && CF1->isZero())) { 1156 // fma/mad(a, b, c) = c if a=0 || b=0 1157 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr2 << "\n"); 1158 replaceCall(CI, opr2); 1159 return true; 1160 } 1161 if (CF0 && CF0->isExactlyValue(1.0f)) { 1162 // fma/mad(a, b, c) = b+c if a=1 1163 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr1 << " + " << *opr2 1164 << "\n"); 1165 Value *nval = B.CreateFAdd(opr1, opr2, "fmaadd"); 1166 replaceCall(CI, nval); 1167 return true; 1168 } 1169 if (CF1 && CF1->isExactlyValue(1.0f)) { 1170 // fma/mad(a, b, c) = a+c if b=1 1171 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " + " << *opr2 1172 << "\n"); 1173 Value *nval = B.CreateFAdd(opr0, opr2, "fmaadd"); 1174 replaceCall(CI, nval); 1175 return true; 1176 } 1177 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr2)) { 1178 if (CF->isZero()) { 1179 // fma/mad(a, b, c) = a*b if c=0 1180 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * " 1181 << *opr1 << "\n"); 1182 Value *nval = B.CreateFMul(opr0, opr1, "fmamul"); 1183 replaceCall(CI, nval); 1184 return true; 1185 } 1186 } 1187 1188 return false; 1189 } 1190 1191 // Get a scalar native builtin single argument FP function 1192 FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M, 1193 const FuncInfo &FInfo) { 1194 if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId())) 1195 return nullptr; 1196 FuncInfo nf = FInfo; 1197 nf.setPrefix(AMDGPULibFunc::NATIVE); 1198 return getFunction(M, nf); 1199 } 1200 1201 // fold sqrt -> native_sqrt (x) 1202 bool AMDGPULibCalls::fold_sqrt(FPMathOperator *FPOp, IRBuilder<> &B, 1203 const FuncInfo &FInfo) { 1204 if (!isUnsafeMath(FPOp)) 1205 return false; 1206 1207 if (getArgType(FInfo) == AMDGPULibFunc::F32 && (getVecSize(FInfo) == 1) && 1208 (FInfo.getPrefix() != AMDGPULibFunc::NATIVE)) { 1209 Module *M = B.GetInsertBlock()->getModule(); 1210 1211 if (FunctionCallee FPExpr = getNativeFunction( 1212 M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) { 1213 Value *opr0 = FPOp->getOperand(0); 1214 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " 1215 << "sqrt(" << *opr0 << ")\n"); 1216 Value *nval = CreateCallEx(B,FPExpr, opr0, "__sqrt"); 1217 replaceCall(FPOp, nval); 1218 return true; 1219 } 1220 } 1221 return false; 1222 } 1223 1224 // fold sin, cos -> sincos. 1225 bool AMDGPULibCalls::fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B, 1226 const FuncInfo &fInfo, AliasAnalysis *AA) { 1227 assert(fInfo.getId() == AMDGPULibFunc::EI_SIN || 1228 fInfo.getId() == AMDGPULibFunc::EI_COS); 1229 1230 if ((getArgType(fInfo) != AMDGPULibFunc::F32 && 1231 getArgType(fInfo) != AMDGPULibFunc::F64) || 1232 fInfo.getPrefix() != AMDGPULibFunc::NOPFX) 1233 return false; 1234 1235 bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN; 1236 1237 Value *CArgVal = FPOp->getOperand(0); 1238 CallInst *CI = cast<CallInst>(FPOp); 1239 BasicBlock * const CBB = CI->getParent(); 1240 1241 int const MaxScan = 30; 1242 bool Changed = false; 1243 1244 { // fold in load value. 1245 LoadInst *LI = dyn_cast<LoadInst>(CArgVal); 1246 if (LI && LI->getParent() == CBB) { 1247 BasicBlock::iterator BBI = LI->getIterator(); 1248 Value *AvailableVal = FindAvailableLoadedValue(LI, CBB, BBI, MaxScan, AA); 1249 if (AvailableVal) { 1250 Changed = true; 1251 CArgVal->replaceAllUsesWith(AvailableVal); 1252 if (CArgVal->getNumUses() == 0) 1253 LI->eraseFromParent(); 1254 CArgVal = FPOp->getOperand(0); 1255 } 1256 } 1257 } 1258 1259 Module *M = CI->getModule(); 1260 FuncInfo PartnerInfo(isSin ? AMDGPULibFunc::EI_COS : AMDGPULibFunc::EI_SIN, 1261 fInfo); 1262 const std::string PairName = PartnerInfo.mangle(); 1263 1264 CallInst *UI = nullptr; 1265 for (User* U : CArgVal->users()) { 1266 CallInst *XI = dyn_cast_or_null<CallInst>(U); 1267 if (!XI || XI == CI || XI->getParent() != CBB) 1268 continue; 1269 1270 Function *UCallee = XI->getCalledFunction(); 1271 if (!UCallee || !UCallee->getName().equals(PairName)) 1272 continue; 1273 1274 BasicBlock::iterator BBI = CI->getIterator(); 1275 if (BBI == CI->getParent()->begin()) 1276 break; 1277 --BBI; 1278 for (int I = MaxScan; I > 0 && BBI != CBB->begin(); --BBI, --I) { 1279 if (cast<Instruction>(BBI) == XI) { 1280 UI = XI; 1281 break; 1282 } 1283 } 1284 if (UI) break; 1285 } 1286 1287 if (!UI) 1288 return Changed; 1289 1290 // Merge the sin and cos. 1291 1292 // for OpenCL 2.0 we have only generic implementation of sincos 1293 // function. 1294 AMDGPULibFunc nf(AMDGPULibFunc::EI_SINCOS, fInfo); 1295 nf.getLeads()[0].PtrKind = AMDGPULibFunc::getEPtrKindFromAddrSpace(AMDGPUAS::FLAT_ADDRESS); 1296 FunctionCallee Fsincos = getFunction(M, nf); 1297 if (!Fsincos) 1298 return Changed; 1299 1300 BasicBlock::iterator ItOld = B.GetInsertPoint(); 1301 AllocaInst *Alloc = insertAlloca(UI, B, "__sincos_"); 1302 B.SetInsertPoint(UI); 1303 1304 Value *P = Alloc; 1305 Type *PTy = Fsincos.getFunctionType()->getParamType(1); 1306 // The allocaInst allocates the memory in private address space. This need 1307 // to be bitcasted to point to the address space of cos pointer type. 1308 // In OpenCL 2.0 this is generic, while in 1.2 that is private. 1309 if (PTy->getPointerAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) 1310 P = B.CreateAddrSpaceCast(Alloc, PTy); 1311 CallInst *Call = CreateCallEx2(B, Fsincos, UI->getArgOperand(0), P); 1312 1313 LLVM_DEBUG(errs() << "AMDIC: fold_sincos (" << *CI << ", " << *UI << ") with " 1314 << *Call << "\n"); 1315 1316 if (!isSin) { // CI->cos, UI->sin 1317 B.SetInsertPoint(&*ItOld); 1318 UI->replaceAllUsesWith(&*Call); 1319 Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc); 1320 CI->replaceAllUsesWith(Reload); 1321 UI->eraseFromParent(); 1322 CI->eraseFromParent(); 1323 } else { // CI->sin, UI->cos 1324 Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc); 1325 UI->replaceAllUsesWith(Reload); 1326 CI->replaceAllUsesWith(Call); 1327 UI->eraseFromParent(); 1328 CI->eraseFromParent(); 1329 } 1330 return true; 1331 } 1332 1333 bool AMDGPULibCalls::fold_wavefrontsize(CallInst *CI, IRBuilder<> &B) { 1334 if (!TM) 1335 return false; 1336 1337 StringRef CPU = TM->getTargetCPU(); 1338 StringRef Features = TM->getTargetFeatureString(); 1339 if ((CPU.empty() || CPU.equals_insensitive("generic")) && 1340 (Features.empty() || !Features.contains_insensitive("wavefrontsize"))) 1341 return false; 1342 1343 Function *F = CI->getParent()->getParent(); 1344 const GCNSubtarget &ST = TM->getSubtarget<GCNSubtarget>(*F); 1345 unsigned N = ST.getWavefrontSize(); 1346 1347 LLVM_DEBUG(errs() << "AMDIC: fold_wavefrontsize (" << *CI << ") with " 1348 << N << "\n"); 1349 1350 CI->replaceAllUsesWith(ConstantInt::get(B.getInt32Ty(), N)); 1351 CI->eraseFromParent(); 1352 return true; 1353 } 1354 1355 // Get insertion point at entry. 1356 BasicBlock::iterator AMDGPULibCalls::getEntryIns(CallInst * UI) { 1357 Function * Func = UI->getParent()->getParent(); 1358 BasicBlock * BB = &Func->getEntryBlock(); 1359 assert(BB && "Entry block not found!"); 1360 BasicBlock::iterator ItNew = BB->begin(); 1361 return ItNew; 1362 } 1363 1364 // Insert a AllocsInst at the beginning of function entry block. 1365 AllocaInst* AMDGPULibCalls::insertAlloca(CallInst *UI, IRBuilder<> &B, 1366 const char *prefix) { 1367 BasicBlock::iterator ItNew = getEntryIns(UI); 1368 Function *UCallee = UI->getCalledFunction(); 1369 Type *RetType = UCallee->getReturnType(); 1370 B.SetInsertPoint(&*ItNew); 1371 AllocaInst *Alloc = 1372 B.CreateAlloca(RetType, nullptr, std::string(prefix) + UI->getName()); 1373 Alloc->setAlignment( 1374 Align(UCallee->getParent()->getDataLayout().getTypeAllocSize(RetType))); 1375 return Alloc; 1376 } 1377 1378 bool AMDGPULibCalls::evaluateScalarMathFunc(const FuncInfo &FInfo, 1379 double& Res0, double& Res1, 1380 Constant *copr0, Constant *copr1, 1381 Constant *copr2) { 1382 // By default, opr0/opr1/opr3 holds values of float/double type. 1383 // If they are not float/double, each function has to its 1384 // operand separately. 1385 double opr0=0.0, opr1=0.0, opr2=0.0; 1386 ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(copr0); 1387 ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(copr1); 1388 ConstantFP *fpopr2 = dyn_cast_or_null<ConstantFP>(copr2); 1389 if (fpopr0) { 1390 opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1391 ? fpopr0->getValueAPF().convertToDouble() 1392 : (double)fpopr0->getValueAPF().convertToFloat(); 1393 } 1394 1395 if (fpopr1) { 1396 opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1397 ? fpopr1->getValueAPF().convertToDouble() 1398 : (double)fpopr1->getValueAPF().convertToFloat(); 1399 } 1400 1401 if (fpopr2) { 1402 opr2 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1403 ? fpopr2->getValueAPF().convertToDouble() 1404 : (double)fpopr2->getValueAPF().convertToFloat(); 1405 } 1406 1407 switch (FInfo.getId()) { 1408 default : return false; 1409 1410 case AMDGPULibFunc::EI_ACOS: 1411 Res0 = acos(opr0); 1412 return true; 1413 1414 case AMDGPULibFunc::EI_ACOSH: 1415 // acosh(x) == log(x + sqrt(x*x - 1)) 1416 Res0 = log(opr0 + sqrt(opr0*opr0 - 1.0)); 1417 return true; 1418 1419 case AMDGPULibFunc::EI_ACOSPI: 1420 Res0 = acos(opr0) / MATH_PI; 1421 return true; 1422 1423 case AMDGPULibFunc::EI_ASIN: 1424 Res0 = asin(opr0); 1425 return true; 1426 1427 case AMDGPULibFunc::EI_ASINH: 1428 // asinh(x) == log(x + sqrt(x*x + 1)) 1429 Res0 = log(opr0 + sqrt(opr0*opr0 + 1.0)); 1430 return true; 1431 1432 case AMDGPULibFunc::EI_ASINPI: 1433 Res0 = asin(opr0) / MATH_PI; 1434 return true; 1435 1436 case AMDGPULibFunc::EI_ATAN: 1437 Res0 = atan(opr0); 1438 return true; 1439 1440 case AMDGPULibFunc::EI_ATANH: 1441 // atanh(x) == (log(x+1) - log(x-1))/2; 1442 Res0 = (log(opr0 + 1.0) - log(opr0 - 1.0))/2.0; 1443 return true; 1444 1445 case AMDGPULibFunc::EI_ATANPI: 1446 Res0 = atan(opr0) / MATH_PI; 1447 return true; 1448 1449 case AMDGPULibFunc::EI_CBRT: 1450 Res0 = (opr0 < 0.0) ? -pow(-opr0, 1.0/3.0) : pow(opr0, 1.0/3.0); 1451 return true; 1452 1453 case AMDGPULibFunc::EI_COS: 1454 Res0 = cos(opr0); 1455 return true; 1456 1457 case AMDGPULibFunc::EI_COSH: 1458 Res0 = cosh(opr0); 1459 return true; 1460 1461 case AMDGPULibFunc::EI_COSPI: 1462 Res0 = cos(MATH_PI * opr0); 1463 return true; 1464 1465 case AMDGPULibFunc::EI_EXP: 1466 Res0 = exp(opr0); 1467 return true; 1468 1469 case AMDGPULibFunc::EI_EXP2: 1470 Res0 = pow(2.0, opr0); 1471 return true; 1472 1473 case AMDGPULibFunc::EI_EXP10: 1474 Res0 = pow(10.0, opr0); 1475 return true; 1476 1477 case AMDGPULibFunc::EI_EXPM1: 1478 Res0 = exp(opr0) - 1.0; 1479 return true; 1480 1481 case AMDGPULibFunc::EI_LOG: 1482 Res0 = log(opr0); 1483 return true; 1484 1485 case AMDGPULibFunc::EI_LOG2: 1486 Res0 = log(opr0) / log(2.0); 1487 return true; 1488 1489 case AMDGPULibFunc::EI_LOG10: 1490 Res0 = log(opr0) / log(10.0); 1491 return true; 1492 1493 case AMDGPULibFunc::EI_RSQRT: 1494 Res0 = 1.0 / sqrt(opr0); 1495 return true; 1496 1497 case AMDGPULibFunc::EI_SIN: 1498 Res0 = sin(opr0); 1499 return true; 1500 1501 case AMDGPULibFunc::EI_SINH: 1502 Res0 = sinh(opr0); 1503 return true; 1504 1505 case AMDGPULibFunc::EI_SINPI: 1506 Res0 = sin(MATH_PI * opr0); 1507 return true; 1508 1509 case AMDGPULibFunc::EI_SQRT: 1510 Res0 = sqrt(opr0); 1511 return true; 1512 1513 case AMDGPULibFunc::EI_TAN: 1514 Res0 = tan(opr0); 1515 return true; 1516 1517 case AMDGPULibFunc::EI_TANH: 1518 Res0 = tanh(opr0); 1519 return true; 1520 1521 case AMDGPULibFunc::EI_TANPI: 1522 Res0 = tan(MATH_PI * opr0); 1523 return true; 1524 1525 case AMDGPULibFunc::EI_RECIP: 1526 Res0 = 1.0 / opr0; 1527 return true; 1528 1529 // two-arg functions 1530 case AMDGPULibFunc::EI_DIVIDE: 1531 Res0 = opr0 / opr1; 1532 return true; 1533 1534 case AMDGPULibFunc::EI_POW: 1535 case AMDGPULibFunc::EI_POWR: 1536 Res0 = pow(opr0, opr1); 1537 return true; 1538 1539 case AMDGPULibFunc::EI_POWN: { 1540 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) { 1541 double val = (double)iopr1->getSExtValue(); 1542 Res0 = pow(opr0, val); 1543 return true; 1544 } 1545 return false; 1546 } 1547 1548 case AMDGPULibFunc::EI_ROOTN: { 1549 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) { 1550 double val = (double)iopr1->getSExtValue(); 1551 Res0 = pow(opr0, 1.0 / val); 1552 return true; 1553 } 1554 return false; 1555 } 1556 1557 // with ptr arg 1558 case AMDGPULibFunc::EI_SINCOS: 1559 Res0 = sin(opr0); 1560 Res1 = cos(opr0); 1561 return true; 1562 1563 // three-arg functions 1564 case AMDGPULibFunc::EI_FMA: 1565 case AMDGPULibFunc::EI_MAD: 1566 Res0 = opr0 * opr1 + opr2; 1567 return true; 1568 } 1569 1570 return false; 1571 } 1572 1573 bool AMDGPULibCalls::evaluateCall(CallInst *aCI, const FuncInfo &FInfo) { 1574 int numArgs = (int)aCI->arg_size(); 1575 if (numArgs > 3) 1576 return false; 1577 1578 Constant *copr0 = nullptr; 1579 Constant *copr1 = nullptr; 1580 Constant *copr2 = nullptr; 1581 if (numArgs > 0) { 1582 if ((copr0 = dyn_cast<Constant>(aCI->getArgOperand(0))) == nullptr) 1583 return false; 1584 } 1585 1586 if (numArgs > 1) { 1587 if ((copr1 = dyn_cast<Constant>(aCI->getArgOperand(1))) == nullptr) { 1588 if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS) 1589 return false; 1590 } 1591 } 1592 1593 if (numArgs > 2) { 1594 if ((copr2 = dyn_cast<Constant>(aCI->getArgOperand(2))) == nullptr) 1595 return false; 1596 } 1597 1598 // At this point, all arguments to aCI are constants. 1599 1600 // max vector size is 16, and sincos will generate two results. 1601 double DVal0[16], DVal1[16]; 1602 int FuncVecSize = getVecSize(FInfo); 1603 bool hasTwoResults = (FInfo.getId() == AMDGPULibFunc::EI_SINCOS); 1604 if (FuncVecSize == 1) { 1605 if (!evaluateScalarMathFunc(FInfo, DVal0[0], 1606 DVal1[0], copr0, copr1, copr2)) { 1607 return false; 1608 } 1609 } else { 1610 ConstantDataVector *CDV0 = dyn_cast_or_null<ConstantDataVector>(copr0); 1611 ConstantDataVector *CDV1 = dyn_cast_or_null<ConstantDataVector>(copr1); 1612 ConstantDataVector *CDV2 = dyn_cast_or_null<ConstantDataVector>(copr2); 1613 for (int i = 0; i < FuncVecSize; ++i) { 1614 Constant *celt0 = CDV0 ? CDV0->getElementAsConstant(i) : nullptr; 1615 Constant *celt1 = CDV1 ? CDV1->getElementAsConstant(i) : nullptr; 1616 Constant *celt2 = CDV2 ? CDV2->getElementAsConstant(i) : nullptr; 1617 if (!evaluateScalarMathFunc(FInfo, DVal0[i], 1618 DVal1[i], celt0, celt1, celt2)) { 1619 return false; 1620 } 1621 } 1622 } 1623 1624 LLVMContext &context = aCI->getContext(); 1625 Constant *nval0, *nval1; 1626 if (FuncVecSize == 1) { 1627 nval0 = ConstantFP::get(aCI->getType(), DVal0[0]); 1628 if (hasTwoResults) 1629 nval1 = ConstantFP::get(aCI->getType(), DVal1[0]); 1630 } else { 1631 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 1632 SmallVector <float, 0> FVal0, FVal1; 1633 for (int i = 0; i < FuncVecSize; ++i) 1634 FVal0.push_back((float)DVal0[i]); 1635 ArrayRef<float> tmp0(FVal0); 1636 nval0 = ConstantDataVector::get(context, tmp0); 1637 if (hasTwoResults) { 1638 for (int i = 0; i < FuncVecSize; ++i) 1639 FVal1.push_back((float)DVal1[i]); 1640 ArrayRef<float> tmp1(FVal1); 1641 nval1 = ConstantDataVector::get(context, tmp1); 1642 } 1643 } else { 1644 ArrayRef<double> tmp0(DVal0); 1645 nval0 = ConstantDataVector::get(context, tmp0); 1646 if (hasTwoResults) { 1647 ArrayRef<double> tmp1(DVal1); 1648 nval1 = ConstantDataVector::get(context, tmp1); 1649 } 1650 } 1651 } 1652 1653 if (hasTwoResults) { 1654 // sincos 1655 assert(FInfo.getId() == AMDGPULibFunc::EI_SINCOS && 1656 "math function with ptr arg not supported yet"); 1657 new StoreInst(nval1, aCI->getArgOperand(1), aCI); 1658 } 1659 1660 replaceCall(aCI, nval0); 1661 return true; 1662 } 1663 1664 // Public interface to the Simplify LibCalls pass. 1665 FunctionPass *llvm::createAMDGPUSimplifyLibCallsPass(const TargetMachine *TM) { 1666 return new AMDGPUSimplifyLibCalls(TM); 1667 } 1668 1669 FunctionPass *llvm::createAMDGPUUseNativeCallsPass() { 1670 return new AMDGPUUseNativeCalls(); 1671 } 1672 1673 bool AMDGPUSimplifyLibCalls::runOnFunction(Function &F) { 1674 if (skipFunction(F)) 1675 return false; 1676 1677 Simplifier.initFunction(F); 1678 1679 bool Changed = false; 1680 auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); 1681 1682 LLVM_DEBUG(dbgs() << "AMDIC: process function "; 1683 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';); 1684 1685 for (auto &BB : F) { 1686 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) { 1687 // Ignore non-calls. 1688 CallInst *CI = dyn_cast<CallInst>(I); 1689 ++I; 1690 if (CI) { 1691 if (Simplifier.fold(CI, AA)) 1692 Changed = true; 1693 } 1694 } 1695 } 1696 return Changed; 1697 } 1698 1699 PreservedAnalyses AMDGPUSimplifyLibCallsPass::run(Function &F, 1700 FunctionAnalysisManager &AM) { 1701 AMDGPULibCalls Simplifier(&TM); 1702 Simplifier.initNativeFuncs(); 1703 Simplifier.initFunction(F); 1704 1705 bool Changed = false; 1706 auto AA = &AM.getResult<AAManager>(F); 1707 1708 LLVM_DEBUG(dbgs() << "AMDIC: process function "; 1709 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';); 1710 1711 for (auto &BB : F) { 1712 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) { 1713 // Ignore non-calls. 1714 CallInst *CI = dyn_cast<CallInst>(I); 1715 ++I; 1716 1717 if (CI) { 1718 if (Simplifier.fold(CI, AA)) 1719 Changed = true; 1720 } 1721 } 1722 } 1723 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 1724 } 1725 1726 bool AMDGPUUseNativeCalls::runOnFunction(Function &F) { 1727 if (skipFunction(F) || UseNative.empty()) 1728 return false; 1729 1730 Simplifier.initFunction(F); 1731 1732 bool Changed = false; 1733 for (auto &BB : F) { 1734 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) { 1735 // Ignore non-calls. 1736 CallInst *CI = dyn_cast<CallInst>(I); 1737 ++I; 1738 if (CI && Simplifier.useNative(CI)) 1739 Changed = true; 1740 } 1741 } 1742 return Changed; 1743 } 1744 1745 PreservedAnalyses AMDGPUUseNativeCallsPass::run(Function &F, 1746 FunctionAnalysisManager &AM) { 1747 if (UseNative.empty()) 1748 return PreservedAnalyses::all(); 1749 1750 AMDGPULibCalls Simplifier; 1751 Simplifier.initNativeFuncs(); 1752 Simplifier.initFunction(F); 1753 1754 bool Changed = false; 1755 for (auto &BB : F) { 1756 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) { 1757 // Ignore non-calls. 1758 CallInst *CI = dyn_cast<CallInst>(I); 1759 ++I; 1760 if (CI && Simplifier.useNative(CI)) 1761 Changed = true; 1762 } 1763 } 1764 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 1765 } 1766