1 //===- AMDGPULibCalls.cpp -------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file does AMD library function optimizations. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPU.h" 15 #include "AMDGPULibFunc.h" 16 #include "GCNSubtarget.h" 17 #include "llvm/Analysis/AliasAnalysis.h" 18 #include "llvm/Analysis/Loads.h" 19 #include "llvm/IR/IRBuilder.h" 20 #include "llvm/IR/IntrinsicInst.h" 21 #include "llvm/IR/IntrinsicsAMDGPU.h" 22 #include "llvm/InitializePasses.h" 23 #include "llvm/Target/TargetMachine.h" 24 #include <cmath> 25 26 #define DEBUG_TYPE "amdgpu-simplifylib" 27 28 using namespace llvm; 29 30 static cl::opt<bool> EnablePreLink("amdgpu-prelink", 31 cl::desc("Enable pre-link mode optimizations"), 32 cl::init(false), 33 cl::Hidden); 34 35 static cl::list<std::string> UseNative("amdgpu-use-native", 36 cl::desc("Comma separated list of functions to replace with native, or all"), 37 cl::CommaSeparated, cl::ValueOptional, 38 cl::Hidden); 39 40 #define MATH_PI numbers::pi 41 #define MATH_E numbers::e 42 #define MATH_SQRT2 numbers::sqrt2 43 #define MATH_SQRT1_2 numbers::inv_sqrt2 44 45 namespace llvm { 46 47 class AMDGPULibCalls { 48 private: 49 50 typedef llvm::AMDGPULibFunc FuncInfo; 51 52 const TargetMachine *TM; 53 54 bool UnsafeFPMath = false; 55 56 // -fuse-native. 57 bool AllNative = false; 58 59 bool useNativeFunc(const StringRef F) const; 60 61 // Return a pointer (pointer expr) to the function if function definition with 62 // "FuncName" exists. It may create a new function prototype in pre-link mode. 63 FunctionCallee getFunction(Module *M, const FuncInfo &fInfo); 64 65 bool parseFunctionName(const StringRef &FMangledName, FuncInfo &FInfo); 66 67 bool TDOFold(CallInst *CI, const FuncInfo &FInfo); 68 69 /* Specialized optimizations */ 70 71 // recip (half or native) 72 bool fold_recip(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 73 74 // divide (half or native) 75 bool fold_divide(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 76 77 // pow/powr/pown 78 bool fold_pow(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo); 79 80 // rootn 81 bool fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo); 82 83 // fma/mad 84 bool fold_fma_mad(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 85 86 // -fuse-native for sincos 87 bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo); 88 89 // evaluate calls if calls' arguments are constants. 90 bool evaluateScalarMathFunc(const FuncInfo &FInfo, double& Res0, 91 double& Res1, Constant *copr0, Constant *copr1, Constant *copr2); 92 bool evaluateCall(CallInst *aCI, const FuncInfo &FInfo); 93 94 // sqrt 95 bool fold_sqrt(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo); 96 97 // sin/cos 98 bool fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo, 99 AliasAnalysis *AA); 100 101 // __read_pipe/__write_pipe 102 bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, 103 const FuncInfo &FInfo); 104 105 // llvm.amdgcn.wavefrontsize 106 bool fold_wavefrontsize(CallInst *CI, IRBuilder<> &B); 107 108 // Get insertion point at entry. 109 BasicBlock::iterator getEntryIns(CallInst * UI); 110 // Insert an Alloc instruction. 111 AllocaInst* insertAlloca(CallInst * UI, IRBuilder<> &B, const char *prefix); 112 // Get a scalar native builtin single argument FP function 113 FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo); 114 115 protected: 116 bool isUnsafeMath(const FPMathOperator *FPOp) const; 117 118 bool canIncreasePrecisionOfConstantFold(const FPMathOperator *FPOp) const; 119 120 static void replaceCall(Instruction *I, Value *With) { 121 I->replaceAllUsesWith(With); 122 I->eraseFromParent(); 123 } 124 125 static void replaceCall(FPMathOperator *I, Value *With) { 126 replaceCall(cast<Instruction>(I), With); 127 } 128 129 public: 130 AMDGPULibCalls(const TargetMachine *TM_ = nullptr) : TM(TM_) {} 131 132 bool fold(CallInst *CI, AliasAnalysis *AA = nullptr); 133 134 void initFunction(const Function &F); 135 void initNativeFuncs(); 136 137 // Replace a normal math function call with that native version 138 bool useNative(CallInst *CI); 139 }; 140 141 } // end llvm namespace 142 143 namespace { 144 145 class AMDGPUSimplifyLibCalls : public FunctionPass { 146 147 AMDGPULibCalls Simplifier; 148 149 public: 150 static char ID; // Pass identification 151 152 AMDGPUSimplifyLibCalls(const TargetMachine *TM = nullptr) 153 : FunctionPass(ID), Simplifier(TM) { 154 initializeAMDGPUSimplifyLibCallsPass(*PassRegistry::getPassRegistry()); 155 } 156 157 void getAnalysisUsage(AnalysisUsage &AU) const override { 158 AU.addRequired<AAResultsWrapperPass>(); 159 } 160 161 bool runOnFunction(Function &M) override; 162 }; 163 164 class AMDGPUUseNativeCalls : public FunctionPass { 165 166 AMDGPULibCalls Simplifier; 167 168 public: 169 static char ID; // Pass identification 170 171 AMDGPUUseNativeCalls() : FunctionPass(ID) { 172 initializeAMDGPUUseNativeCallsPass(*PassRegistry::getPassRegistry()); 173 Simplifier.initNativeFuncs(); 174 } 175 176 bool runOnFunction(Function &F) override; 177 }; 178 179 } // end anonymous namespace. 180 181 char AMDGPUSimplifyLibCalls::ID = 0; 182 char AMDGPUUseNativeCalls::ID = 0; 183 184 INITIALIZE_PASS_BEGIN(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", 185 "Simplify well-known AMD library calls", false, false) 186 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 187 INITIALIZE_PASS_END(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", 188 "Simplify well-known AMD library calls", false, false) 189 190 INITIALIZE_PASS(AMDGPUUseNativeCalls, "amdgpu-usenative", 191 "Replace builtin math calls with that native versions.", 192 false, false) 193 194 template <typename IRB> 195 static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg, 196 const Twine &Name = "") { 197 CallInst *R = B.CreateCall(Callee, Arg, Name); 198 if (Function *F = dyn_cast<Function>(Callee.getCallee())) 199 R->setCallingConv(F->getCallingConv()); 200 return R; 201 } 202 203 template <typename IRB> 204 static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1, 205 Value *Arg2, const Twine &Name = "") { 206 CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name); 207 if (Function *F = dyn_cast<Function>(Callee.getCallee())) 208 R->setCallingConv(F->getCallingConv()); 209 return R; 210 } 211 212 // Data structures for table-driven optimizations. 213 // FuncTbl works for both f32 and f64 functions with 1 input argument 214 215 struct TableEntry { 216 double result; 217 double input; 218 }; 219 220 /* a list of {result, input} */ 221 static const TableEntry tbl_acos[] = { 222 {MATH_PI / 2.0, 0.0}, 223 {MATH_PI / 2.0, -0.0}, 224 {0.0, 1.0}, 225 {MATH_PI, -1.0} 226 }; 227 static const TableEntry tbl_acosh[] = { 228 {0.0, 1.0} 229 }; 230 static const TableEntry tbl_acospi[] = { 231 {0.5, 0.0}, 232 {0.5, -0.0}, 233 {0.0, 1.0}, 234 {1.0, -1.0} 235 }; 236 static const TableEntry tbl_asin[] = { 237 {0.0, 0.0}, 238 {-0.0, -0.0}, 239 {MATH_PI / 2.0, 1.0}, 240 {-MATH_PI / 2.0, -1.0} 241 }; 242 static const TableEntry tbl_asinh[] = { 243 {0.0, 0.0}, 244 {-0.0, -0.0} 245 }; 246 static const TableEntry tbl_asinpi[] = { 247 {0.0, 0.0}, 248 {-0.0, -0.0}, 249 {0.5, 1.0}, 250 {-0.5, -1.0} 251 }; 252 static const TableEntry tbl_atan[] = { 253 {0.0, 0.0}, 254 {-0.0, -0.0}, 255 {MATH_PI / 4.0, 1.0}, 256 {-MATH_PI / 4.0, -1.0} 257 }; 258 static const TableEntry tbl_atanh[] = { 259 {0.0, 0.0}, 260 {-0.0, -0.0} 261 }; 262 static const TableEntry tbl_atanpi[] = { 263 {0.0, 0.0}, 264 {-0.0, -0.0}, 265 {0.25, 1.0}, 266 {-0.25, -1.0} 267 }; 268 static const TableEntry tbl_cbrt[] = { 269 {0.0, 0.0}, 270 {-0.0, -0.0}, 271 {1.0, 1.0}, 272 {-1.0, -1.0}, 273 }; 274 static const TableEntry tbl_cos[] = { 275 {1.0, 0.0}, 276 {1.0, -0.0} 277 }; 278 static const TableEntry tbl_cosh[] = { 279 {1.0, 0.0}, 280 {1.0, -0.0} 281 }; 282 static const TableEntry tbl_cospi[] = { 283 {1.0, 0.0}, 284 {1.0, -0.0} 285 }; 286 static const TableEntry tbl_erfc[] = { 287 {1.0, 0.0}, 288 {1.0, -0.0} 289 }; 290 static const TableEntry tbl_erf[] = { 291 {0.0, 0.0}, 292 {-0.0, -0.0} 293 }; 294 static const TableEntry tbl_exp[] = { 295 {1.0, 0.0}, 296 {1.0, -0.0}, 297 {MATH_E, 1.0} 298 }; 299 static const TableEntry tbl_exp2[] = { 300 {1.0, 0.0}, 301 {1.0, -0.0}, 302 {2.0, 1.0} 303 }; 304 static const TableEntry tbl_exp10[] = { 305 {1.0, 0.0}, 306 {1.0, -0.0}, 307 {10.0, 1.0} 308 }; 309 static const TableEntry tbl_expm1[] = { 310 {0.0, 0.0}, 311 {-0.0, -0.0} 312 }; 313 static const TableEntry tbl_log[] = { 314 {0.0, 1.0}, 315 {1.0, MATH_E} 316 }; 317 static const TableEntry tbl_log2[] = { 318 {0.0, 1.0}, 319 {1.0, 2.0} 320 }; 321 static const TableEntry tbl_log10[] = { 322 {0.0, 1.0}, 323 {1.0, 10.0} 324 }; 325 static const TableEntry tbl_rsqrt[] = { 326 {1.0, 1.0}, 327 {MATH_SQRT1_2, 2.0} 328 }; 329 static const TableEntry tbl_sin[] = { 330 {0.0, 0.0}, 331 {-0.0, -0.0} 332 }; 333 static const TableEntry tbl_sinh[] = { 334 {0.0, 0.0}, 335 {-0.0, -0.0} 336 }; 337 static const TableEntry tbl_sinpi[] = { 338 {0.0, 0.0}, 339 {-0.0, -0.0} 340 }; 341 static const TableEntry tbl_sqrt[] = { 342 {0.0, 0.0}, 343 {1.0, 1.0}, 344 {MATH_SQRT2, 2.0} 345 }; 346 static const TableEntry tbl_tan[] = { 347 {0.0, 0.0}, 348 {-0.0, -0.0} 349 }; 350 static const TableEntry tbl_tanh[] = { 351 {0.0, 0.0}, 352 {-0.0, -0.0} 353 }; 354 static const TableEntry tbl_tanpi[] = { 355 {0.0, 0.0}, 356 {-0.0, -0.0} 357 }; 358 static const TableEntry tbl_tgamma[] = { 359 {1.0, 1.0}, 360 {1.0, 2.0}, 361 {2.0, 3.0}, 362 {6.0, 4.0} 363 }; 364 365 static bool HasNative(AMDGPULibFunc::EFuncId id) { 366 switch(id) { 367 case AMDGPULibFunc::EI_DIVIDE: 368 case AMDGPULibFunc::EI_COS: 369 case AMDGPULibFunc::EI_EXP: 370 case AMDGPULibFunc::EI_EXP2: 371 case AMDGPULibFunc::EI_EXP10: 372 case AMDGPULibFunc::EI_LOG: 373 case AMDGPULibFunc::EI_LOG2: 374 case AMDGPULibFunc::EI_LOG10: 375 case AMDGPULibFunc::EI_POWR: 376 case AMDGPULibFunc::EI_RECIP: 377 case AMDGPULibFunc::EI_RSQRT: 378 case AMDGPULibFunc::EI_SIN: 379 case AMDGPULibFunc::EI_SINCOS: 380 case AMDGPULibFunc::EI_SQRT: 381 case AMDGPULibFunc::EI_TAN: 382 return true; 383 default:; 384 } 385 return false; 386 } 387 388 using TableRef = ArrayRef<TableEntry>; 389 390 static TableRef getOptTable(AMDGPULibFunc::EFuncId id) { 391 switch(id) { 392 case AMDGPULibFunc::EI_ACOS: return TableRef(tbl_acos); 393 case AMDGPULibFunc::EI_ACOSH: return TableRef(tbl_acosh); 394 case AMDGPULibFunc::EI_ACOSPI: return TableRef(tbl_acospi); 395 case AMDGPULibFunc::EI_ASIN: return TableRef(tbl_asin); 396 case AMDGPULibFunc::EI_ASINH: return TableRef(tbl_asinh); 397 case AMDGPULibFunc::EI_ASINPI: return TableRef(tbl_asinpi); 398 case AMDGPULibFunc::EI_ATAN: return TableRef(tbl_atan); 399 case AMDGPULibFunc::EI_ATANH: return TableRef(tbl_atanh); 400 case AMDGPULibFunc::EI_ATANPI: return TableRef(tbl_atanpi); 401 case AMDGPULibFunc::EI_CBRT: return TableRef(tbl_cbrt); 402 case AMDGPULibFunc::EI_NCOS: 403 case AMDGPULibFunc::EI_COS: return TableRef(tbl_cos); 404 case AMDGPULibFunc::EI_COSH: return TableRef(tbl_cosh); 405 case AMDGPULibFunc::EI_COSPI: return TableRef(tbl_cospi); 406 case AMDGPULibFunc::EI_ERFC: return TableRef(tbl_erfc); 407 case AMDGPULibFunc::EI_ERF: return TableRef(tbl_erf); 408 case AMDGPULibFunc::EI_EXP: return TableRef(tbl_exp); 409 case AMDGPULibFunc::EI_NEXP2: 410 case AMDGPULibFunc::EI_EXP2: return TableRef(tbl_exp2); 411 case AMDGPULibFunc::EI_EXP10: return TableRef(tbl_exp10); 412 case AMDGPULibFunc::EI_EXPM1: return TableRef(tbl_expm1); 413 case AMDGPULibFunc::EI_LOG: return TableRef(tbl_log); 414 case AMDGPULibFunc::EI_NLOG2: 415 case AMDGPULibFunc::EI_LOG2: return TableRef(tbl_log2); 416 case AMDGPULibFunc::EI_LOG10: return TableRef(tbl_log10); 417 case AMDGPULibFunc::EI_NRSQRT: 418 case AMDGPULibFunc::EI_RSQRT: return TableRef(tbl_rsqrt); 419 case AMDGPULibFunc::EI_NSIN: 420 case AMDGPULibFunc::EI_SIN: return TableRef(tbl_sin); 421 case AMDGPULibFunc::EI_SINH: return TableRef(tbl_sinh); 422 case AMDGPULibFunc::EI_SINPI: return TableRef(tbl_sinpi); 423 case AMDGPULibFunc::EI_NSQRT: 424 case AMDGPULibFunc::EI_SQRT: return TableRef(tbl_sqrt); 425 case AMDGPULibFunc::EI_TAN: return TableRef(tbl_tan); 426 case AMDGPULibFunc::EI_TANH: return TableRef(tbl_tanh); 427 case AMDGPULibFunc::EI_TANPI: return TableRef(tbl_tanpi); 428 case AMDGPULibFunc::EI_TGAMMA: return TableRef(tbl_tgamma); 429 default:; 430 } 431 return TableRef(); 432 } 433 434 static inline int getVecSize(const AMDGPULibFunc& FInfo) { 435 return FInfo.getLeads()[0].VectorSize; 436 } 437 438 static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) { 439 return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType; 440 } 441 442 FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) { 443 // If we are doing PreLinkOpt, the function is external. So it is safe to 444 // use getOrInsertFunction() at this stage. 445 446 return EnablePreLink ? AMDGPULibFunc::getOrInsertFunction(M, fInfo) 447 : AMDGPULibFunc::getFunction(M, fInfo); 448 } 449 450 bool AMDGPULibCalls::parseFunctionName(const StringRef &FMangledName, 451 FuncInfo &FInfo) { 452 return AMDGPULibFunc::parse(FMangledName, FInfo); 453 } 454 455 bool AMDGPULibCalls::isUnsafeMath(const FPMathOperator *FPOp) const { 456 return UnsafeFPMath || FPOp->isFast(); 457 } 458 459 bool AMDGPULibCalls::canIncreasePrecisionOfConstantFold( 460 const FPMathOperator *FPOp) const { 461 // TODO: Refine to approxFunc or contract 462 return isUnsafeMath(FPOp); 463 } 464 465 void AMDGPULibCalls::initFunction(const Function &F) { 466 UnsafeFPMath = F.getFnAttribute("unsafe-fp-math").getValueAsBool(); 467 } 468 469 bool AMDGPULibCalls::useNativeFunc(const StringRef F) const { 470 return AllNative || llvm::is_contained(UseNative, F); 471 } 472 473 void AMDGPULibCalls::initNativeFuncs() { 474 AllNative = useNativeFunc("all") || 475 (UseNative.getNumOccurrences() && UseNative.size() == 1 && 476 UseNative.begin()->empty()); 477 } 478 479 bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) { 480 bool native_sin = useNativeFunc("sin"); 481 bool native_cos = useNativeFunc("cos"); 482 483 if (native_sin && native_cos) { 484 Module *M = aCI->getModule(); 485 Value *opr0 = aCI->getArgOperand(0); 486 487 AMDGPULibFunc nf; 488 nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType; 489 nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize; 490 491 nf.setPrefix(AMDGPULibFunc::NATIVE); 492 nf.setId(AMDGPULibFunc::EI_SIN); 493 FunctionCallee sinExpr = getFunction(M, nf); 494 495 nf.setPrefix(AMDGPULibFunc::NATIVE); 496 nf.setId(AMDGPULibFunc::EI_COS); 497 FunctionCallee cosExpr = getFunction(M, nf); 498 if (sinExpr && cosExpr) { 499 Value *sinval = CallInst::Create(sinExpr, opr0, "splitsin", aCI); 500 Value *cosval = CallInst::Create(cosExpr, opr0, "splitcos", aCI); 501 new StoreInst(cosval, aCI->getArgOperand(1), aCI); 502 503 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI 504 << " with native version of sin/cos"); 505 506 replaceCall(aCI, sinval); 507 return true; 508 } 509 } 510 return false; 511 } 512 513 bool AMDGPULibCalls::useNative(CallInst *aCI) { 514 Function *Callee = aCI->getCalledFunction(); 515 if (!Callee || aCI->isNoBuiltin()) 516 return false; 517 518 FuncInfo FInfo; 519 if (!parseFunctionName(Callee->getName(), FInfo) || !FInfo.isMangled() || 520 FInfo.getPrefix() != AMDGPULibFunc::NOPFX || 521 getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()) || 522 !(AllNative || useNativeFunc(FInfo.getName()))) { 523 return false; 524 } 525 526 if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS) 527 return sincosUseNative(aCI, FInfo); 528 529 FInfo.setPrefix(AMDGPULibFunc::NATIVE); 530 FunctionCallee F = getFunction(aCI->getModule(), FInfo); 531 if (!F) 532 return false; 533 534 aCI->setCalledFunction(F); 535 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI 536 << " with native version"); 537 return true; 538 } 539 540 // Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe 541 // builtin, with appended type size and alignment arguments, where 2 or 4 542 // indicates the original number of arguments. The library has optimized version 543 // of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same 544 // power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N 545 // for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ..., 546 // 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4. 547 bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, 548 const FuncInfo &FInfo) { 549 auto *Callee = CI->getCalledFunction(); 550 if (!Callee->isDeclaration()) 551 return false; 552 553 assert(Callee->hasName() && "Invalid read_pipe/write_pipe function"); 554 auto *M = Callee->getParent(); 555 std::string Name = std::string(Callee->getName()); 556 auto NumArg = CI->arg_size(); 557 if (NumArg != 4 && NumArg != 6) 558 return false; 559 ConstantInt *PacketSize = 560 dyn_cast<ConstantInt>(CI->getArgOperand(NumArg - 2)); 561 ConstantInt *PacketAlign = 562 dyn_cast<ConstantInt>(CI->getArgOperand(NumArg - 1)); 563 if (!PacketSize || !PacketAlign) 564 return false; 565 566 unsigned Size = PacketSize->getZExtValue(); 567 Align Alignment = PacketAlign->getAlignValue(); 568 if (Alignment != Size) 569 return false; 570 571 unsigned PtrArgLoc = CI->arg_size() - 3; 572 Value *PtrArg = CI->getArgOperand(PtrArgLoc); 573 Type *PtrTy = PtrArg->getType(); 574 575 SmallVector<llvm::Type *, 6> ArgTys; 576 for (unsigned I = 0; I != PtrArgLoc; ++I) 577 ArgTys.push_back(CI->getArgOperand(I)->getType()); 578 ArgTys.push_back(PtrTy); 579 580 Name = Name + "_" + std::to_string(Size); 581 auto *FTy = FunctionType::get(Callee->getReturnType(), 582 ArrayRef<Type *>(ArgTys), false); 583 AMDGPULibFunc NewLibFunc(Name, FTy); 584 FunctionCallee F = AMDGPULibFunc::getOrInsertFunction(M, NewLibFunc); 585 if (!F) 586 return false; 587 588 auto *BCast = B.CreatePointerCast(PtrArg, PtrTy); 589 SmallVector<Value *, 6> Args; 590 for (unsigned I = 0; I != PtrArgLoc; ++I) 591 Args.push_back(CI->getArgOperand(I)); 592 Args.push_back(BCast); 593 594 auto *NCI = B.CreateCall(F, Args); 595 NCI->setAttributes(CI->getAttributes()); 596 CI->replaceAllUsesWith(NCI); 597 CI->dropAllReferences(); 598 CI->eraseFromParent(); 599 600 return true; 601 } 602 603 // This function returns false if no change; return true otherwise. 604 bool AMDGPULibCalls::fold(CallInst *CI, AliasAnalysis *AA) { 605 Function *Callee = CI->getCalledFunction(); 606 // Ignore indirect calls. 607 if (!Callee || CI->isNoBuiltin()) 608 return false; 609 610 IRBuilder<> B(CI); 611 switch (Callee->getIntrinsicID()) { 612 case Intrinsic::not_intrinsic: 613 break; 614 case Intrinsic::amdgcn_wavefrontsize: 615 return !EnablePreLink && fold_wavefrontsize(CI, B); 616 default: 617 return false; 618 } 619 620 FuncInfo FInfo; 621 if (!parseFunctionName(Callee->getName(), FInfo)) 622 return false; 623 624 // Further check the number of arguments to see if they match. 625 // TODO: Check calling convention matches too 626 if (!FInfo.isCompatibleSignature(CI->getFunctionType())) 627 return false; 628 629 LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << '\n'); 630 631 if (TDOFold(CI, FInfo)) 632 return true; 633 634 if (FPMathOperator *FPOp = dyn_cast<FPMathOperator>(CI)) { 635 // Under unsafe-math, evaluate calls if possible. 636 // According to Brian Sumner, we can do this for all f32 function calls 637 // using host's double function calls. 638 if (canIncreasePrecisionOfConstantFold(FPOp) && evaluateCall(CI, FInfo)) 639 return true; 640 641 // Copy fast flags from the original call. 642 B.setFastMathFlags(FPOp->getFastMathFlags()); 643 644 // Specialized optimizations for each function call 645 switch (FInfo.getId()) { 646 case AMDGPULibFunc::EI_POW: 647 case AMDGPULibFunc::EI_POWR: 648 case AMDGPULibFunc::EI_POWN: 649 return fold_pow(FPOp, B, FInfo); 650 case AMDGPULibFunc::EI_ROOTN: 651 return fold_rootn(FPOp, B, FInfo); 652 case AMDGPULibFunc::EI_SQRT: 653 return fold_sqrt(FPOp, B, FInfo); 654 case AMDGPULibFunc::EI_COS: 655 case AMDGPULibFunc::EI_SIN: 656 return fold_sincos(FPOp, B, FInfo, AA); 657 case AMDGPULibFunc::EI_RECIP: 658 // skip vector function 659 assert((FInfo.getPrefix() == AMDGPULibFunc::NATIVE || 660 FInfo.getPrefix() == AMDGPULibFunc::HALF) && 661 "recip must be an either native or half function"); 662 return (getVecSize(FInfo) != 1) ? false : fold_recip(CI, B, FInfo); 663 664 case AMDGPULibFunc::EI_DIVIDE: 665 // skip vector function 666 assert((FInfo.getPrefix() == AMDGPULibFunc::NATIVE || 667 FInfo.getPrefix() == AMDGPULibFunc::HALF) && 668 "divide must be an either native or half function"); 669 return (getVecSize(FInfo) != 1) ? false : fold_divide(CI, B, FInfo); 670 case AMDGPULibFunc::EI_FMA: 671 case AMDGPULibFunc::EI_MAD: 672 case AMDGPULibFunc::EI_NFMA: 673 // skip vector function 674 return (getVecSize(FInfo) != 1) ? false : fold_fma_mad(CI, B, FInfo); 675 default: 676 break; 677 } 678 } else { 679 // Specialized optimizations for each function call 680 switch (FInfo.getId()) { 681 case AMDGPULibFunc::EI_READ_PIPE_2: 682 case AMDGPULibFunc::EI_READ_PIPE_4: 683 case AMDGPULibFunc::EI_WRITE_PIPE_2: 684 case AMDGPULibFunc::EI_WRITE_PIPE_4: 685 return fold_read_write_pipe(CI, B, FInfo); 686 default: 687 break; 688 } 689 } 690 691 return false; 692 } 693 694 bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) { 695 // Table-Driven optimization 696 const TableRef tr = getOptTable(FInfo.getId()); 697 if (tr.empty()) 698 return false; 699 700 int const sz = (int)tr.size(); 701 Value *opr0 = CI->getArgOperand(0); 702 703 if (getVecSize(FInfo) > 1) { 704 if (ConstantDataVector *CV = dyn_cast<ConstantDataVector>(opr0)) { 705 SmallVector<double, 0> DVal; 706 for (int eltNo = 0; eltNo < getVecSize(FInfo); ++eltNo) { 707 ConstantFP *eltval = dyn_cast<ConstantFP>( 708 CV->getElementAsConstant((unsigned)eltNo)); 709 assert(eltval && "Non-FP arguments in math function!"); 710 bool found = false; 711 for (int i=0; i < sz; ++i) { 712 if (eltval->isExactlyValue(tr[i].input)) { 713 DVal.push_back(tr[i].result); 714 found = true; 715 break; 716 } 717 } 718 if (!found) { 719 // This vector constants not handled yet. 720 return false; 721 } 722 } 723 LLVMContext &context = CI->getParent()->getParent()->getContext(); 724 Constant *nval; 725 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 726 SmallVector<float, 0> FVal; 727 for (unsigned i = 0; i < DVal.size(); ++i) { 728 FVal.push_back((float)DVal[i]); 729 } 730 ArrayRef<float> tmp(FVal); 731 nval = ConstantDataVector::get(context, tmp); 732 } else { // F64 733 ArrayRef<double> tmp(DVal); 734 nval = ConstantDataVector::get(context, tmp); 735 } 736 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 737 replaceCall(CI, nval); 738 return true; 739 } 740 } else { 741 // Scalar version 742 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) { 743 for (int i = 0; i < sz; ++i) { 744 if (CF->isExactlyValue(tr[i].input)) { 745 Value *nval = ConstantFP::get(CF->getType(), tr[i].result); 746 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 747 replaceCall(CI, nval); 748 return true; 749 } 750 } 751 } 752 } 753 754 return false; 755 } 756 757 // [native_]half_recip(c) ==> 1.0/c 758 bool AMDGPULibCalls::fold_recip(CallInst *CI, IRBuilder<> &B, 759 const FuncInfo &FInfo) { 760 Value *opr0 = CI->getArgOperand(0); 761 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) { 762 // Just create a normal div. Later, InstCombine will be able 763 // to compute the divide into a constant (avoid check float infinity 764 // or subnormal at this point). 765 Value *nval = B.CreateFDiv(ConstantFP::get(CF->getType(), 1.0), 766 opr0, 767 "recip2div"); 768 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 769 replaceCall(CI, nval); 770 return true; 771 } 772 return false; 773 } 774 775 // [native_]half_divide(x, c) ==> x/c 776 bool AMDGPULibCalls::fold_divide(CallInst *CI, IRBuilder<> &B, 777 const FuncInfo &FInfo) { 778 Value *opr0 = CI->getArgOperand(0); 779 Value *opr1 = CI->getArgOperand(1); 780 ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0); 781 ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1); 782 783 if ((CF0 && CF1) || // both are constants 784 (CF1 && (getArgType(FInfo) == AMDGPULibFunc::F32))) 785 // CF1 is constant && f32 divide 786 { 787 Value *nval1 = B.CreateFDiv(ConstantFP::get(opr1->getType(), 1.0), 788 opr1, "__div2recip"); 789 Value *nval = B.CreateFMul(opr0, nval1, "__div2mul"); 790 replaceCall(CI, nval); 791 return true; 792 } 793 return false; 794 } 795 796 namespace llvm { 797 static double log2(double V) { 798 #if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L 799 return ::log2(V); 800 #else 801 return log(V) / numbers::ln2; 802 #endif 803 } 804 } 805 806 bool AMDGPULibCalls::fold_pow(FPMathOperator *FPOp, IRBuilder<> &B, 807 const FuncInfo &FInfo) { 808 assert((FInfo.getId() == AMDGPULibFunc::EI_POW || 809 FInfo.getId() == AMDGPULibFunc::EI_POWR || 810 FInfo.getId() == AMDGPULibFunc::EI_POWN) && 811 "fold_pow: encounter a wrong function call"); 812 813 Module *M = B.GetInsertBlock()->getModule(); 814 ConstantFP *CF; 815 ConstantInt *CINT; 816 Type *eltType; 817 Value *opr0 = FPOp->getOperand(0); 818 Value *opr1 = FPOp->getOperand(1); 819 ConstantAggregateZero *CZero = dyn_cast<ConstantAggregateZero>(opr1); 820 821 if (getVecSize(FInfo) == 1) { 822 eltType = opr0->getType(); 823 CF = dyn_cast<ConstantFP>(opr1); 824 CINT = dyn_cast<ConstantInt>(opr1); 825 } else { 826 VectorType *VTy = dyn_cast<VectorType>(opr0->getType()); 827 assert(VTy && "Oprand of vector function should be of vectortype"); 828 eltType = VTy->getElementType(); 829 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1); 830 831 // Now, only Handle vector const whose elements have the same value. 832 CF = CDV ? dyn_cast_or_null<ConstantFP>(CDV->getSplatValue()) : nullptr; 833 CINT = CDV ? dyn_cast_or_null<ConstantInt>(CDV->getSplatValue()) : nullptr; 834 } 835 836 // No unsafe math , no constant argument, do nothing 837 if (!isUnsafeMath(FPOp) && !CF && !CINT && !CZero) 838 return false; 839 840 // 0x1111111 means that we don't do anything for this call. 841 int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111); 842 843 if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0) || CZero) { 844 // pow/powr/pown(x, 0) == 1 845 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1\n"); 846 Constant *cnval = ConstantFP::get(eltType, 1.0); 847 if (getVecSize(FInfo) > 1) { 848 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 849 } 850 replaceCall(FPOp, cnval); 851 return true; 852 } 853 if ((CF && CF->isExactlyValue(1.0)) || (CINT && ci_opr1 == 1)) { 854 // pow/powr/pown(x, 1.0) = x 855 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << "\n"); 856 replaceCall(FPOp, opr0); 857 return true; 858 } 859 if ((CF && CF->isExactlyValue(2.0)) || (CINT && ci_opr1 == 2)) { 860 // pow/powr/pown(x, 2.0) = x*x 861 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << " * " 862 << *opr0 << "\n"); 863 Value *nval = B.CreateFMul(opr0, opr0, "__pow2"); 864 replaceCall(FPOp, nval); 865 return true; 866 } 867 if ((CF && CF->isExactlyValue(-1.0)) || (CINT && ci_opr1 == -1)) { 868 // pow/powr/pown(x, -1.0) = 1.0/x 869 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1 / " << *opr0 << "\n"); 870 Constant *cnval = ConstantFP::get(eltType, 1.0); 871 if (getVecSize(FInfo) > 1) { 872 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 873 } 874 Value *nval = B.CreateFDiv(cnval, opr0, "__powrecip"); 875 replaceCall(FPOp, nval); 876 return true; 877 } 878 879 if (CF && (CF->isExactlyValue(0.5) || CF->isExactlyValue(-0.5))) { 880 // pow[r](x, [-]0.5) = sqrt(x) 881 bool issqrt = CF->isExactlyValue(0.5); 882 if (FunctionCallee FPExpr = 883 getFunction(M, AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT 884 : AMDGPULibFunc::EI_RSQRT, 885 FInfo))) { 886 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << FInfo.getName() 887 << '(' << *opr0 << ")\n"); 888 Value *nval = CreateCallEx(B,FPExpr, opr0, issqrt ? "__pow2sqrt" 889 : "__pow2rsqrt"); 890 replaceCall(FPOp, nval); 891 return true; 892 } 893 } 894 895 if (!isUnsafeMath(FPOp)) 896 return false; 897 898 // Unsafe Math optimization 899 900 // Remember that ci_opr1 is set if opr1 is integral 901 if (CF) { 902 double dval = (getArgType(FInfo) == AMDGPULibFunc::F32) 903 ? (double)CF->getValueAPF().convertToFloat() 904 : CF->getValueAPF().convertToDouble(); 905 int ival = (int)dval; 906 if ((double)ival == dval) { 907 ci_opr1 = ival; 908 } else 909 ci_opr1 = 0x11111111; 910 } 911 912 // pow/powr/pown(x, c) = [1/](x*x*..x); where 913 // trunc(c) == c && the number of x == c && |c| <= 12 914 unsigned abs_opr1 = (ci_opr1 < 0) ? -ci_opr1 : ci_opr1; 915 if (abs_opr1 <= 12) { 916 Constant *cnval; 917 Value *nval; 918 if (abs_opr1 == 0) { 919 cnval = ConstantFP::get(eltType, 1.0); 920 if (getVecSize(FInfo) > 1) { 921 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 922 } 923 nval = cnval; 924 } else { 925 Value *valx2 = nullptr; 926 nval = nullptr; 927 while (abs_opr1 > 0) { 928 valx2 = valx2 ? B.CreateFMul(valx2, valx2, "__powx2") : opr0; 929 if (abs_opr1 & 1) { 930 nval = nval ? B.CreateFMul(nval, valx2, "__powprod") : valx2; 931 } 932 abs_opr1 >>= 1; 933 } 934 } 935 936 if (ci_opr1 < 0) { 937 cnval = ConstantFP::get(eltType, 1.0); 938 if (getVecSize(FInfo) > 1) { 939 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 940 } 941 nval = B.CreateFDiv(cnval, nval, "__1powprod"); 942 } 943 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " 944 << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr0 945 << ")\n"); 946 replaceCall(FPOp, nval); 947 return true; 948 } 949 950 // powr ---> exp2(y * log2(x)) 951 // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31)) 952 FunctionCallee ExpExpr = 953 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo)); 954 if (!ExpExpr) 955 return false; 956 957 bool needlog = false; 958 bool needabs = false; 959 bool needcopysign = false; 960 Constant *cnval = nullptr; 961 if (getVecSize(FInfo) == 1) { 962 CF = dyn_cast<ConstantFP>(opr0); 963 964 if (CF) { 965 double V = (getArgType(FInfo) == AMDGPULibFunc::F32) 966 ? (double)CF->getValueAPF().convertToFloat() 967 : CF->getValueAPF().convertToDouble(); 968 969 V = log2(std::abs(V)); 970 cnval = ConstantFP::get(eltType, V); 971 needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR) && 972 CF->isNegative(); 973 } else { 974 needlog = true; 975 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR && 976 (!CF || CF->isNegative()); 977 } 978 } else { 979 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr0); 980 981 if (!CDV) { 982 needlog = true; 983 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR; 984 } else { 985 assert ((int)CDV->getNumElements() == getVecSize(FInfo) && 986 "Wrong vector size detected"); 987 988 SmallVector<double, 0> DVal; 989 for (int i=0; i < getVecSize(FInfo); ++i) { 990 double V = (getArgType(FInfo) == AMDGPULibFunc::F32) 991 ? (double)CDV->getElementAsFloat(i) 992 : CDV->getElementAsDouble(i); 993 if (V < 0.0) needcopysign = true; 994 V = log2(std::abs(V)); 995 DVal.push_back(V); 996 } 997 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 998 SmallVector<float, 0> FVal; 999 for (unsigned i=0; i < DVal.size(); ++i) { 1000 FVal.push_back((float)DVal[i]); 1001 } 1002 ArrayRef<float> tmp(FVal); 1003 cnval = ConstantDataVector::get(M->getContext(), tmp); 1004 } else { 1005 ArrayRef<double> tmp(DVal); 1006 cnval = ConstantDataVector::get(M->getContext(), tmp); 1007 } 1008 } 1009 } 1010 1011 if (needcopysign && (FInfo.getId() == AMDGPULibFunc::EI_POW)) { 1012 // We cannot handle corner cases for a general pow() function, give up 1013 // unless y is a constant integral value. Then proceed as if it were pown. 1014 if (getVecSize(FInfo) == 1) { 1015 if (const ConstantFP *CF = dyn_cast<ConstantFP>(opr1)) { 1016 double y = (getArgType(FInfo) == AMDGPULibFunc::F32) 1017 ? (double)CF->getValueAPF().convertToFloat() 1018 : CF->getValueAPF().convertToDouble(); 1019 if (y != (double)(int64_t)y) 1020 return false; 1021 } else 1022 return false; 1023 } else { 1024 if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1)) { 1025 for (int i=0; i < getVecSize(FInfo); ++i) { 1026 double y = (getArgType(FInfo) == AMDGPULibFunc::F32) 1027 ? (double)CDV->getElementAsFloat(i) 1028 : CDV->getElementAsDouble(i); 1029 if (y != (double)(int64_t)y) 1030 return false; 1031 } 1032 } else 1033 return false; 1034 } 1035 } 1036 1037 Value *nval; 1038 if (needabs) { 1039 nval = B.CreateUnaryIntrinsic(Intrinsic::fabs, opr0, nullptr, "__fabs"); 1040 } else { 1041 nval = cnval ? cnval : opr0; 1042 } 1043 if (needlog) { 1044 FunctionCallee LogExpr = 1045 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo)); 1046 if (!LogExpr) 1047 return false; 1048 nval = CreateCallEx(B,LogExpr, nval, "__log2"); 1049 } 1050 1051 if (FInfo.getId() == AMDGPULibFunc::EI_POWN) { 1052 // convert int(32) to fp(f32 or f64) 1053 opr1 = B.CreateSIToFP(opr1, nval->getType(), "pownI2F"); 1054 } 1055 nval = B.CreateFMul(opr1, nval, "__ylogx"); 1056 nval = CreateCallEx(B,ExpExpr, nval, "__exp2"); 1057 1058 if (needcopysign) { 1059 Value *opr_n; 1060 Type* rTy = opr0->getType(); 1061 Type* nTyS = eltType->isDoubleTy() ? B.getInt64Ty() : B.getInt32Ty(); 1062 Type *nTy = nTyS; 1063 if (const auto *vTy = dyn_cast<FixedVectorType>(rTy)) 1064 nTy = FixedVectorType::get(nTyS, vTy); 1065 unsigned size = nTy->getScalarSizeInBits(); 1066 opr_n = FPOp->getOperand(1); 1067 if (opr_n->getType()->isIntegerTy()) 1068 opr_n = B.CreateZExtOrBitCast(opr_n, nTy, "__ytou"); 1069 else 1070 opr_n = B.CreateFPToSI(opr1, nTy, "__ytou"); 1071 1072 Value *sign = B.CreateShl(opr_n, size-1, "__yeven"); 1073 sign = B.CreateAnd(B.CreateBitCast(opr0, nTy), sign, "__pow_sign"); 1074 nval = B.CreateOr(B.CreateBitCast(nval, nTy), sign); 1075 nval = B.CreateBitCast(nval, opr0->getType()); 1076 } 1077 1078 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " 1079 << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n"); 1080 replaceCall(FPOp, nval); 1081 1082 return true; 1083 } 1084 1085 bool AMDGPULibCalls::fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B, 1086 const FuncInfo &FInfo) { 1087 // skip vector function 1088 if (getVecSize(FInfo) != 1) 1089 return false; 1090 1091 Value *opr0 = FPOp->getOperand(0); 1092 Value *opr1 = FPOp->getOperand(1); 1093 1094 ConstantInt *CINT = dyn_cast<ConstantInt>(opr1); 1095 if (!CINT) { 1096 return false; 1097 } 1098 int ci_opr1 = (int)CINT->getSExtValue(); 1099 if (ci_opr1 == 1) { // rootn(x, 1) = x 1100 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << "\n"); 1101 replaceCall(FPOp, opr0); 1102 return true; 1103 } 1104 1105 Module *M = B.GetInsertBlock()->getModule(); 1106 if (ci_opr1 == 2) { // rootn(x, 2) = sqrt(x) 1107 if (FunctionCallee FPExpr = 1108 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) { 1109 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> sqrt(" << *opr0 1110 << ")\n"); 1111 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2sqrt"); 1112 replaceCall(FPOp, nval); 1113 return true; 1114 } 1115 } else if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x) 1116 if (FunctionCallee FPExpr = 1117 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) { 1118 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> cbrt(" << *opr0 1119 << ")\n"); 1120 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2cbrt"); 1121 replaceCall(FPOp, nval); 1122 return true; 1123 } 1124 } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x 1125 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1.0 / " << *opr0 << "\n"); 1126 Value *nval = B.CreateFDiv(ConstantFP::get(opr0->getType(), 1.0), 1127 opr0, 1128 "__rootn2div"); 1129 replaceCall(FPOp, nval); 1130 return true; 1131 } else if (ci_opr1 == -2) { // rootn(x, -2) = rsqrt(x) 1132 if (FunctionCallee FPExpr = 1133 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_RSQRT, FInfo))) { 1134 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> rsqrt(" << *opr0 1135 << ")\n"); 1136 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2rsqrt"); 1137 replaceCall(FPOp, nval); 1138 return true; 1139 } 1140 } 1141 return false; 1142 } 1143 1144 bool AMDGPULibCalls::fold_fma_mad(CallInst *CI, IRBuilder<> &B, 1145 const FuncInfo &FInfo) { 1146 Value *opr0 = CI->getArgOperand(0); 1147 Value *opr1 = CI->getArgOperand(1); 1148 Value *opr2 = CI->getArgOperand(2); 1149 1150 ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0); 1151 ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1); 1152 if ((CF0 && CF0->isZero()) || (CF1 && CF1->isZero())) { 1153 // fma/mad(a, b, c) = c if a=0 || b=0 1154 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr2 << "\n"); 1155 replaceCall(CI, opr2); 1156 return true; 1157 } 1158 if (CF0 && CF0->isExactlyValue(1.0f)) { 1159 // fma/mad(a, b, c) = b+c if a=1 1160 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr1 << " + " << *opr2 1161 << "\n"); 1162 Value *nval = B.CreateFAdd(opr1, opr2, "fmaadd"); 1163 replaceCall(CI, nval); 1164 return true; 1165 } 1166 if (CF1 && CF1->isExactlyValue(1.0f)) { 1167 // fma/mad(a, b, c) = a+c if b=1 1168 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " + " << *opr2 1169 << "\n"); 1170 Value *nval = B.CreateFAdd(opr0, opr2, "fmaadd"); 1171 replaceCall(CI, nval); 1172 return true; 1173 } 1174 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr2)) { 1175 if (CF->isZero()) { 1176 // fma/mad(a, b, c) = a*b if c=0 1177 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * " 1178 << *opr1 << "\n"); 1179 Value *nval = B.CreateFMul(opr0, opr1, "fmamul"); 1180 replaceCall(CI, nval); 1181 return true; 1182 } 1183 } 1184 1185 return false; 1186 } 1187 1188 // Get a scalar native builtin single argument FP function 1189 FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M, 1190 const FuncInfo &FInfo) { 1191 if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId())) 1192 return nullptr; 1193 FuncInfo nf = FInfo; 1194 nf.setPrefix(AMDGPULibFunc::NATIVE); 1195 return getFunction(M, nf); 1196 } 1197 1198 // fold sqrt -> native_sqrt (x) 1199 bool AMDGPULibCalls::fold_sqrt(FPMathOperator *FPOp, IRBuilder<> &B, 1200 const FuncInfo &FInfo) { 1201 if (!isUnsafeMath(FPOp)) 1202 return false; 1203 1204 if (getArgType(FInfo) == AMDGPULibFunc::F32 && (getVecSize(FInfo) == 1) && 1205 (FInfo.getPrefix() != AMDGPULibFunc::NATIVE)) { 1206 Module *M = B.GetInsertBlock()->getModule(); 1207 1208 if (FunctionCallee FPExpr = getNativeFunction( 1209 M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) { 1210 Value *opr0 = FPOp->getOperand(0); 1211 LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " 1212 << "sqrt(" << *opr0 << ")\n"); 1213 Value *nval = CreateCallEx(B,FPExpr, opr0, "__sqrt"); 1214 replaceCall(FPOp, nval); 1215 return true; 1216 } 1217 } 1218 return false; 1219 } 1220 1221 // fold sin, cos -> sincos. 1222 bool AMDGPULibCalls::fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B, 1223 const FuncInfo &fInfo, AliasAnalysis *AA) { 1224 assert(fInfo.getId() == AMDGPULibFunc::EI_SIN || 1225 fInfo.getId() == AMDGPULibFunc::EI_COS); 1226 1227 if ((getArgType(fInfo) != AMDGPULibFunc::F32 && 1228 getArgType(fInfo) != AMDGPULibFunc::F64) || 1229 fInfo.getPrefix() != AMDGPULibFunc::NOPFX) 1230 return false; 1231 1232 bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN; 1233 1234 Value *CArgVal = FPOp->getOperand(0); 1235 CallInst *CI = cast<CallInst>(FPOp); 1236 BasicBlock * const CBB = CI->getParent(); 1237 1238 int const MaxScan = 30; 1239 bool Changed = false; 1240 1241 { // fold in load value. 1242 LoadInst *LI = dyn_cast<LoadInst>(CArgVal); 1243 if (LI && LI->getParent() == CBB) { 1244 BasicBlock::iterator BBI = LI->getIterator(); 1245 Value *AvailableVal = FindAvailableLoadedValue(LI, CBB, BBI, MaxScan, AA); 1246 if (AvailableVal) { 1247 Changed = true; 1248 CArgVal->replaceAllUsesWith(AvailableVal); 1249 if (CArgVal->getNumUses() == 0) 1250 LI->eraseFromParent(); 1251 CArgVal = FPOp->getOperand(0); 1252 } 1253 } 1254 } 1255 1256 Module *M = CI->getModule(); 1257 FuncInfo PartnerInfo(isSin ? AMDGPULibFunc::EI_COS : AMDGPULibFunc::EI_SIN, 1258 fInfo); 1259 const std::string PairName = PartnerInfo.mangle(); 1260 1261 CallInst *UI = nullptr; 1262 for (User* U : CArgVal->users()) { 1263 CallInst *XI = dyn_cast_or_null<CallInst>(U); 1264 if (!XI || XI == CI || XI->getParent() != CBB) 1265 continue; 1266 1267 Function *UCallee = XI->getCalledFunction(); 1268 if (!UCallee || !UCallee->getName().equals(PairName)) 1269 continue; 1270 1271 BasicBlock::iterator BBI = CI->getIterator(); 1272 if (BBI == CI->getParent()->begin()) 1273 break; 1274 --BBI; 1275 for (int I = MaxScan; I > 0 && BBI != CBB->begin(); --BBI, --I) { 1276 if (cast<Instruction>(BBI) == XI) { 1277 UI = XI; 1278 break; 1279 } 1280 } 1281 if (UI) break; 1282 } 1283 1284 if (!UI) 1285 return Changed; 1286 1287 // Merge the sin and cos. 1288 1289 // for OpenCL 2.0 we have only generic implementation of sincos 1290 // function. 1291 AMDGPULibFunc nf(AMDGPULibFunc::EI_SINCOS, fInfo); 1292 nf.getLeads()[0].PtrKind = AMDGPULibFunc::getEPtrKindFromAddrSpace(AMDGPUAS::FLAT_ADDRESS); 1293 FunctionCallee Fsincos = getFunction(M, nf); 1294 if (!Fsincos) 1295 return Changed; 1296 1297 BasicBlock::iterator ItOld = B.GetInsertPoint(); 1298 AllocaInst *Alloc = insertAlloca(UI, B, "__sincos_"); 1299 B.SetInsertPoint(UI); 1300 1301 Value *P = Alloc; 1302 Type *PTy = Fsincos.getFunctionType()->getParamType(1); 1303 // The allocaInst allocates the memory in private address space. This need 1304 // to be bitcasted to point to the address space of cos pointer type. 1305 // In OpenCL 2.0 this is generic, while in 1.2 that is private. 1306 if (PTy->getPointerAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) 1307 P = B.CreateAddrSpaceCast(Alloc, PTy); 1308 CallInst *Call = CreateCallEx2(B, Fsincos, UI->getArgOperand(0), P); 1309 1310 LLVM_DEBUG(errs() << "AMDIC: fold_sincos (" << *CI << ", " << *UI << ") with " 1311 << *Call << "\n"); 1312 1313 if (!isSin) { // CI->cos, UI->sin 1314 B.SetInsertPoint(&*ItOld); 1315 UI->replaceAllUsesWith(&*Call); 1316 Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc); 1317 CI->replaceAllUsesWith(Reload); 1318 UI->eraseFromParent(); 1319 CI->eraseFromParent(); 1320 } else { // CI->sin, UI->cos 1321 Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc); 1322 UI->replaceAllUsesWith(Reload); 1323 CI->replaceAllUsesWith(Call); 1324 UI->eraseFromParent(); 1325 CI->eraseFromParent(); 1326 } 1327 return true; 1328 } 1329 1330 bool AMDGPULibCalls::fold_wavefrontsize(CallInst *CI, IRBuilder<> &B) { 1331 if (!TM) 1332 return false; 1333 1334 StringRef CPU = TM->getTargetCPU(); 1335 StringRef Features = TM->getTargetFeatureString(); 1336 if ((CPU.empty() || CPU.equals_insensitive("generic")) && 1337 (Features.empty() || !Features.contains_insensitive("wavefrontsize"))) 1338 return false; 1339 1340 Function *F = CI->getParent()->getParent(); 1341 const GCNSubtarget &ST = TM->getSubtarget<GCNSubtarget>(*F); 1342 unsigned N = ST.getWavefrontSize(); 1343 1344 LLVM_DEBUG(errs() << "AMDIC: fold_wavefrontsize (" << *CI << ") with " 1345 << N << "\n"); 1346 1347 CI->replaceAllUsesWith(ConstantInt::get(B.getInt32Ty(), N)); 1348 CI->eraseFromParent(); 1349 return true; 1350 } 1351 1352 // Get insertion point at entry. 1353 BasicBlock::iterator AMDGPULibCalls::getEntryIns(CallInst * UI) { 1354 Function * Func = UI->getParent()->getParent(); 1355 BasicBlock * BB = &Func->getEntryBlock(); 1356 assert(BB && "Entry block not found!"); 1357 BasicBlock::iterator ItNew = BB->begin(); 1358 return ItNew; 1359 } 1360 1361 // Insert a AllocsInst at the beginning of function entry block. 1362 AllocaInst* AMDGPULibCalls::insertAlloca(CallInst *UI, IRBuilder<> &B, 1363 const char *prefix) { 1364 BasicBlock::iterator ItNew = getEntryIns(UI); 1365 Function *UCallee = UI->getCalledFunction(); 1366 Type *RetType = UCallee->getReturnType(); 1367 B.SetInsertPoint(&*ItNew); 1368 AllocaInst *Alloc = 1369 B.CreateAlloca(RetType, nullptr, std::string(prefix) + UI->getName()); 1370 Alloc->setAlignment( 1371 Align(UCallee->getParent()->getDataLayout().getTypeAllocSize(RetType))); 1372 return Alloc; 1373 } 1374 1375 bool AMDGPULibCalls::evaluateScalarMathFunc(const FuncInfo &FInfo, 1376 double& Res0, double& Res1, 1377 Constant *copr0, Constant *copr1, 1378 Constant *copr2) { 1379 // By default, opr0/opr1/opr3 holds values of float/double type. 1380 // If they are not float/double, each function has to its 1381 // operand separately. 1382 double opr0=0.0, opr1=0.0, opr2=0.0; 1383 ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(copr0); 1384 ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(copr1); 1385 ConstantFP *fpopr2 = dyn_cast_or_null<ConstantFP>(copr2); 1386 if (fpopr0) { 1387 opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1388 ? fpopr0->getValueAPF().convertToDouble() 1389 : (double)fpopr0->getValueAPF().convertToFloat(); 1390 } 1391 1392 if (fpopr1) { 1393 opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1394 ? fpopr1->getValueAPF().convertToDouble() 1395 : (double)fpopr1->getValueAPF().convertToFloat(); 1396 } 1397 1398 if (fpopr2) { 1399 opr2 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1400 ? fpopr2->getValueAPF().convertToDouble() 1401 : (double)fpopr2->getValueAPF().convertToFloat(); 1402 } 1403 1404 switch (FInfo.getId()) { 1405 default : return false; 1406 1407 case AMDGPULibFunc::EI_ACOS: 1408 Res0 = acos(opr0); 1409 return true; 1410 1411 case AMDGPULibFunc::EI_ACOSH: 1412 // acosh(x) == log(x + sqrt(x*x - 1)) 1413 Res0 = log(opr0 + sqrt(opr0*opr0 - 1.0)); 1414 return true; 1415 1416 case AMDGPULibFunc::EI_ACOSPI: 1417 Res0 = acos(opr0) / MATH_PI; 1418 return true; 1419 1420 case AMDGPULibFunc::EI_ASIN: 1421 Res0 = asin(opr0); 1422 return true; 1423 1424 case AMDGPULibFunc::EI_ASINH: 1425 // asinh(x) == log(x + sqrt(x*x + 1)) 1426 Res0 = log(opr0 + sqrt(opr0*opr0 + 1.0)); 1427 return true; 1428 1429 case AMDGPULibFunc::EI_ASINPI: 1430 Res0 = asin(opr0) / MATH_PI; 1431 return true; 1432 1433 case AMDGPULibFunc::EI_ATAN: 1434 Res0 = atan(opr0); 1435 return true; 1436 1437 case AMDGPULibFunc::EI_ATANH: 1438 // atanh(x) == (log(x+1) - log(x-1))/2; 1439 Res0 = (log(opr0 + 1.0) - log(opr0 - 1.0))/2.0; 1440 return true; 1441 1442 case AMDGPULibFunc::EI_ATANPI: 1443 Res0 = atan(opr0) / MATH_PI; 1444 return true; 1445 1446 case AMDGPULibFunc::EI_CBRT: 1447 Res0 = (opr0 < 0.0) ? -pow(-opr0, 1.0/3.0) : pow(opr0, 1.0/3.0); 1448 return true; 1449 1450 case AMDGPULibFunc::EI_COS: 1451 Res0 = cos(opr0); 1452 return true; 1453 1454 case AMDGPULibFunc::EI_COSH: 1455 Res0 = cosh(opr0); 1456 return true; 1457 1458 case AMDGPULibFunc::EI_COSPI: 1459 Res0 = cos(MATH_PI * opr0); 1460 return true; 1461 1462 case AMDGPULibFunc::EI_EXP: 1463 Res0 = exp(opr0); 1464 return true; 1465 1466 case AMDGPULibFunc::EI_EXP2: 1467 Res0 = pow(2.0, opr0); 1468 return true; 1469 1470 case AMDGPULibFunc::EI_EXP10: 1471 Res0 = pow(10.0, opr0); 1472 return true; 1473 1474 case AMDGPULibFunc::EI_EXPM1: 1475 Res0 = exp(opr0) - 1.0; 1476 return true; 1477 1478 case AMDGPULibFunc::EI_LOG: 1479 Res0 = log(opr0); 1480 return true; 1481 1482 case AMDGPULibFunc::EI_LOG2: 1483 Res0 = log(opr0) / log(2.0); 1484 return true; 1485 1486 case AMDGPULibFunc::EI_LOG10: 1487 Res0 = log(opr0) / log(10.0); 1488 return true; 1489 1490 case AMDGPULibFunc::EI_RSQRT: 1491 Res0 = 1.0 / sqrt(opr0); 1492 return true; 1493 1494 case AMDGPULibFunc::EI_SIN: 1495 Res0 = sin(opr0); 1496 return true; 1497 1498 case AMDGPULibFunc::EI_SINH: 1499 Res0 = sinh(opr0); 1500 return true; 1501 1502 case AMDGPULibFunc::EI_SINPI: 1503 Res0 = sin(MATH_PI * opr0); 1504 return true; 1505 1506 case AMDGPULibFunc::EI_SQRT: 1507 Res0 = sqrt(opr0); 1508 return true; 1509 1510 case AMDGPULibFunc::EI_TAN: 1511 Res0 = tan(opr0); 1512 return true; 1513 1514 case AMDGPULibFunc::EI_TANH: 1515 Res0 = tanh(opr0); 1516 return true; 1517 1518 case AMDGPULibFunc::EI_TANPI: 1519 Res0 = tan(MATH_PI * opr0); 1520 return true; 1521 1522 case AMDGPULibFunc::EI_RECIP: 1523 Res0 = 1.0 / opr0; 1524 return true; 1525 1526 // two-arg functions 1527 case AMDGPULibFunc::EI_DIVIDE: 1528 Res0 = opr0 / opr1; 1529 return true; 1530 1531 case AMDGPULibFunc::EI_POW: 1532 case AMDGPULibFunc::EI_POWR: 1533 Res0 = pow(opr0, opr1); 1534 return true; 1535 1536 case AMDGPULibFunc::EI_POWN: { 1537 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) { 1538 double val = (double)iopr1->getSExtValue(); 1539 Res0 = pow(opr0, val); 1540 return true; 1541 } 1542 return false; 1543 } 1544 1545 case AMDGPULibFunc::EI_ROOTN: { 1546 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) { 1547 double val = (double)iopr1->getSExtValue(); 1548 Res0 = pow(opr0, 1.0 / val); 1549 return true; 1550 } 1551 return false; 1552 } 1553 1554 // with ptr arg 1555 case AMDGPULibFunc::EI_SINCOS: 1556 Res0 = sin(opr0); 1557 Res1 = cos(opr0); 1558 return true; 1559 1560 // three-arg functions 1561 case AMDGPULibFunc::EI_FMA: 1562 case AMDGPULibFunc::EI_MAD: 1563 Res0 = opr0 * opr1 + opr2; 1564 return true; 1565 } 1566 1567 return false; 1568 } 1569 1570 bool AMDGPULibCalls::evaluateCall(CallInst *aCI, const FuncInfo &FInfo) { 1571 int numArgs = (int)aCI->arg_size(); 1572 if (numArgs > 3) 1573 return false; 1574 1575 Constant *copr0 = nullptr; 1576 Constant *copr1 = nullptr; 1577 Constant *copr2 = nullptr; 1578 if (numArgs > 0) { 1579 if ((copr0 = dyn_cast<Constant>(aCI->getArgOperand(0))) == nullptr) 1580 return false; 1581 } 1582 1583 if (numArgs > 1) { 1584 if ((copr1 = dyn_cast<Constant>(aCI->getArgOperand(1))) == nullptr) { 1585 if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS) 1586 return false; 1587 } 1588 } 1589 1590 if (numArgs > 2) { 1591 if ((copr2 = dyn_cast<Constant>(aCI->getArgOperand(2))) == nullptr) 1592 return false; 1593 } 1594 1595 // At this point, all arguments to aCI are constants. 1596 1597 // max vector size is 16, and sincos will generate two results. 1598 double DVal0[16], DVal1[16]; 1599 int FuncVecSize = getVecSize(FInfo); 1600 bool hasTwoResults = (FInfo.getId() == AMDGPULibFunc::EI_SINCOS); 1601 if (FuncVecSize == 1) { 1602 if (!evaluateScalarMathFunc(FInfo, DVal0[0], 1603 DVal1[0], copr0, copr1, copr2)) { 1604 return false; 1605 } 1606 } else { 1607 ConstantDataVector *CDV0 = dyn_cast_or_null<ConstantDataVector>(copr0); 1608 ConstantDataVector *CDV1 = dyn_cast_or_null<ConstantDataVector>(copr1); 1609 ConstantDataVector *CDV2 = dyn_cast_or_null<ConstantDataVector>(copr2); 1610 for (int i = 0; i < FuncVecSize; ++i) { 1611 Constant *celt0 = CDV0 ? CDV0->getElementAsConstant(i) : nullptr; 1612 Constant *celt1 = CDV1 ? CDV1->getElementAsConstant(i) : nullptr; 1613 Constant *celt2 = CDV2 ? CDV2->getElementAsConstant(i) : nullptr; 1614 if (!evaluateScalarMathFunc(FInfo, DVal0[i], 1615 DVal1[i], celt0, celt1, celt2)) { 1616 return false; 1617 } 1618 } 1619 } 1620 1621 LLVMContext &context = aCI->getContext(); 1622 Constant *nval0, *nval1; 1623 if (FuncVecSize == 1) { 1624 nval0 = ConstantFP::get(aCI->getType(), DVal0[0]); 1625 if (hasTwoResults) 1626 nval1 = ConstantFP::get(aCI->getType(), DVal1[0]); 1627 } else { 1628 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 1629 SmallVector <float, 0> FVal0, FVal1; 1630 for (int i = 0; i < FuncVecSize; ++i) 1631 FVal0.push_back((float)DVal0[i]); 1632 ArrayRef<float> tmp0(FVal0); 1633 nval0 = ConstantDataVector::get(context, tmp0); 1634 if (hasTwoResults) { 1635 for (int i = 0; i < FuncVecSize; ++i) 1636 FVal1.push_back((float)DVal1[i]); 1637 ArrayRef<float> tmp1(FVal1); 1638 nval1 = ConstantDataVector::get(context, tmp1); 1639 } 1640 } else { 1641 ArrayRef<double> tmp0(DVal0); 1642 nval0 = ConstantDataVector::get(context, tmp0); 1643 if (hasTwoResults) { 1644 ArrayRef<double> tmp1(DVal1); 1645 nval1 = ConstantDataVector::get(context, tmp1); 1646 } 1647 } 1648 } 1649 1650 if (hasTwoResults) { 1651 // sincos 1652 assert(FInfo.getId() == AMDGPULibFunc::EI_SINCOS && 1653 "math function with ptr arg not supported yet"); 1654 new StoreInst(nval1, aCI->getArgOperand(1), aCI); 1655 } 1656 1657 replaceCall(aCI, nval0); 1658 return true; 1659 } 1660 1661 // Public interface to the Simplify LibCalls pass. 1662 FunctionPass *llvm::createAMDGPUSimplifyLibCallsPass(const TargetMachine *TM) { 1663 return new AMDGPUSimplifyLibCalls(TM); 1664 } 1665 1666 FunctionPass *llvm::createAMDGPUUseNativeCallsPass() { 1667 return new AMDGPUUseNativeCalls(); 1668 } 1669 1670 bool AMDGPUSimplifyLibCalls::runOnFunction(Function &F) { 1671 if (skipFunction(F)) 1672 return false; 1673 1674 Simplifier.initFunction(F); 1675 1676 bool Changed = false; 1677 auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); 1678 1679 LLVM_DEBUG(dbgs() << "AMDIC: process function "; 1680 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';); 1681 1682 for (auto &BB : F) { 1683 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) { 1684 // Ignore non-calls. 1685 CallInst *CI = dyn_cast<CallInst>(I); 1686 ++I; 1687 if (CI) { 1688 if (Simplifier.fold(CI, AA)) 1689 Changed = true; 1690 } 1691 } 1692 } 1693 return Changed; 1694 } 1695 1696 PreservedAnalyses AMDGPUSimplifyLibCallsPass::run(Function &F, 1697 FunctionAnalysisManager &AM) { 1698 AMDGPULibCalls Simplifier(&TM); 1699 Simplifier.initNativeFuncs(); 1700 Simplifier.initFunction(F); 1701 1702 bool Changed = false; 1703 auto AA = &AM.getResult<AAManager>(F); 1704 1705 LLVM_DEBUG(dbgs() << "AMDIC: process function "; 1706 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';); 1707 1708 for (auto &BB : F) { 1709 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) { 1710 // Ignore non-calls. 1711 CallInst *CI = dyn_cast<CallInst>(I); 1712 ++I; 1713 1714 if (CI) { 1715 if (Simplifier.fold(CI, AA)) 1716 Changed = true; 1717 } 1718 } 1719 } 1720 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 1721 } 1722 1723 bool AMDGPUUseNativeCalls::runOnFunction(Function &F) { 1724 if (skipFunction(F) || UseNative.empty()) 1725 return false; 1726 1727 Simplifier.initFunction(F); 1728 1729 bool Changed = false; 1730 for (auto &BB : F) { 1731 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) { 1732 // Ignore non-calls. 1733 CallInst *CI = dyn_cast<CallInst>(I); 1734 ++I; 1735 if (CI && Simplifier.useNative(CI)) 1736 Changed = true; 1737 } 1738 } 1739 return Changed; 1740 } 1741 1742 PreservedAnalyses AMDGPUUseNativeCallsPass::run(Function &F, 1743 FunctionAnalysisManager &AM) { 1744 if (UseNative.empty()) 1745 return PreservedAnalyses::all(); 1746 1747 AMDGPULibCalls Simplifier; 1748 Simplifier.initNativeFuncs(); 1749 Simplifier.initFunction(F); 1750 1751 bool Changed = false; 1752 for (auto &BB : F) { 1753 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) { 1754 // Ignore non-calls. 1755 CallInst *CI = dyn_cast<CallInst>(I); 1756 ++I; 1757 if (CI && Simplifier.useNative(CI)) 1758 Changed = true; 1759 } 1760 } 1761 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 1762 } 1763