1 //===- AMDGPULibCalls.cpp -------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file does AMD library function optimizations. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPU.h" 15 #include "AMDGPULibFunc.h" 16 #include "AMDGPUSubtarget.h" 17 #include "llvm/ADT/StringRef.h" 18 #include "llvm/ADT/StringSet.h" 19 #include "llvm/Analysis/AliasAnalysis.h" 20 #include "llvm/Analysis/Loads.h" 21 #include "llvm/IR/Constants.h" 22 #include "llvm/IR/DerivedTypes.h" 23 #include "llvm/IR/Function.h" 24 #include "llvm/IR/IRBuilder.h" 25 #include "llvm/IR/Instructions.h" 26 #include "llvm/IR/Intrinsics.h" 27 #include "llvm/IR/LLVMContext.h" 28 #include "llvm/IR/Module.h" 29 #include "llvm/IR/PassManager.h" 30 #include "llvm/IR/ValueSymbolTable.h" 31 #include "llvm/InitializePasses.h" 32 #include "llvm/Support/Debug.h" 33 #include "llvm/Support/MathExtras.h" 34 #include "llvm/Support/raw_ostream.h" 35 #include "llvm/Target/TargetMachine.h" 36 #include <cmath> 37 #include <vector> 38 39 #define DEBUG_TYPE "amdgpu-simplifylib" 40 41 using namespace llvm; 42 43 static cl::opt<bool> EnablePreLink("amdgpu-prelink", 44 cl::desc("Enable pre-link mode optimizations"), 45 cl::init(false), 46 cl::Hidden); 47 48 static cl::list<std::string> UseNative("amdgpu-use-native", 49 cl::desc("Comma separated list of functions to replace with native, or all"), 50 cl::CommaSeparated, cl::ValueOptional, 51 cl::Hidden); 52 53 #define MATH_PI numbers::pi 54 #define MATH_E numbers::e 55 #define MATH_SQRT2 numbers::sqrt2 56 #define MATH_SQRT1_2 numbers::inv_sqrt2 57 58 namespace llvm { 59 60 class AMDGPULibCalls { 61 private: 62 63 typedef llvm::AMDGPULibFunc FuncInfo; 64 65 const TargetMachine *TM; 66 67 // -fuse-native. 68 bool AllNative = false; 69 70 bool useNativeFunc(const StringRef F) const; 71 72 // Return a pointer (pointer expr) to the function if function defintion with 73 // "FuncName" exists. It may create a new function prototype in pre-link mode. 74 FunctionCallee getFunction(Module *M, const FuncInfo &fInfo); 75 76 // Replace a normal function with its native version. 77 bool replaceWithNative(CallInst *CI, const FuncInfo &FInfo); 78 79 bool parseFunctionName(const StringRef& FMangledName, 80 FuncInfo *FInfo=nullptr /*out*/); 81 82 bool TDOFold(CallInst *CI, const FuncInfo &FInfo); 83 84 /* Specialized optimizations */ 85 86 // recip (half or native) 87 bool fold_recip(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 88 89 // divide (half or native) 90 bool fold_divide(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 91 92 // pow/powr/pown 93 bool fold_pow(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 94 95 // rootn 96 bool fold_rootn(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 97 98 // fma/mad 99 bool fold_fma_mad(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 100 101 // -fuse-native for sincos 102 bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo); 103 104 // evaluate calls if calls' arguments are constants. 105 bool evaluateScalarMathFunc(FuncInfo &FInfo, double& Res0, 106 double& Res1, Constant *copr0, Constant *copr1, Constant *copr2); 107 bool evaluateCall(CallInst *aCI, FuncInfo &FInfo); 108 109 // exp 110 bool fold_exp(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 111 112 // exp2 113 bool fold_exp2(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 114 115 // exp10 116 bool fold_exp10(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 117 118 // log 119 bool fold_log(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 120 121 // log2 122 bool fold_log2(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 123 124 // log10 125 bool fold_log10(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 126 127 // sqrt 128 bool fold_sqrt(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo); 129 130 // sin/cos 131 bool fold_sincos(CallInst * CI, IRBuilder<> &B, AliasAnalysis * AA); 132 133 // __read_pipe/__write_pipe 134 bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, FuncInfo &FInfo); 135 136 // llvm.amdgcn.wavefrontsize 137 bool fold_wavefrontsize(CallInst *CI, IRBuilder<> &B); 138 139 // Get insertion point at entry. 140 BasicBlock::iterator getEntryIns(CallInst * UI); 141 // Insert an Alloc instruction. 142 AllocaInst* insertAlloca(CallInst * UI, IRBuilder<> &B, const char *prefix); 143 // Get a scalar native builtin signle argument FP function 144 FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo); 145 146 protected: 147 CallInst *CI; 148 149 bool isUnsafeMath(const CallInst *CI) const; 150 151 void replaceCall(Value *With) { 152 CI->replaceAllUsesWith(With); 153 CI->eraseFromParent(); 154 } 155 156 public: 157 AMDGPULibCalls(const TargetMachine *TM_ = nullptr) : TM(TM_) {} 158 159 bool fold(CallInst *CI, AliasAnalysis *AA = nullptr); 160 161 void initNativeFuncs(); 162 163 // Replace a normal math function call with that native version 164 bool useNative(CallInst *CI); 165 }; 166 167 } // end llvm namespace 168 169 namespace { 170 171 class AMDGPUSimplifyLibCalls : public FunctionPass { 172 173 AMDGPULibCalls Simplifier; 174 175 public: 176 static char ID; // Pass identification 177 178 AMDGPUSimplifyLibCalls(const TargetMachine *TM = nullptr) 179 : FunctionPass(ID), Simplifier(TM) { 180 initializeAMDGPUSimplifyLibCallsPass(*PassRegistry::getPassRegistry()); 181 } 182 183 void getAnalysisUsage(AnalysisUsage &AU) const override { 184 AU.addRequired<AAResultsWrapperPass>(); 185 } 186 187 bool runOnFunction(Function &M) override; 188 }; 189 190 class AMDGPUUseNativeCalls : public FunctionPass { 191 192 AMDGPULibCalls Simplifier; 193 194 public: 195 static char ID; // Pass identification 196 197 AMDGPUUseNativeCalls() : FunctionPass(ID) { 198 initializeAMDGPUUseNativeCallsPass(*PassRegistry::getPassRegistry()); 199 Simplifier.initNativeFuncs(); 200 } 201 202 bool runOnFunction(Function &F) override; 203 }; 204 205 } // end anonymous namespace. 206 207 char AMDGPUSimplifyLibCalls::ID = 0; 208 char AMDGPUUseNativeCalls::ID = 0; 209 210 INITIALIZE_PASS_BEGIN(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", 211 "Simplify well-known AMD library calls", false, false) 212 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 213 INITIALIZE_PASS_END(AMDGPUSimplifyLibCalls, "amdgpu-simplifylib", 214 "Simplify well-known AMD library calls", false, false) 215 216 INITIALIZE_PASS(AMDGPUUseNativeCalls, "amdgpu-usenative", 217 "Replace builtin math calls with that native versions.", 218 false, false) 219 220 template <typename IRB> 221 static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg, 222 const Twine &Name = "") { 223 CallInst *R = B.CreateCall(Callee, Arg, Name); 224 if (Function *F = dyn_cast<Function>(Callee.getCallee())) 225 R->setCallingConv(F->getCallingConv()); 226 return R; 227 } 228 229 template <typename IRB> 230 static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1, 231 Value *Arg2, const Twine &Name = "") { 232 CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name); 233 if (Function *F = dyn_cast<Function>(Callee.getCallee())) 234 R->setCallingConv(F->getCallingConv()); 235 return R; 236 } 237 238 // Data structures for table-driven optimizations. 239 // FuncTbl works for both f32 and f64 functions with 1 input argument 240 241 struct TableEntry { 242 double result; 243 double input; 244 }; 245 246 /* a list of {result, input} */ 247 static const TableEntry tbl_acos[] = { 248 {MATH_PI / 2.0, 0.0}, 249 {MATH_PI / 2.0, -0.0}, 250 {0.0, 1.0}, 251 {MATH_PI, -1.0} 252 }; 253 static const TableEntry tbl_acosh[] = { 254 {0.0, 1.0} 255 }; 256 static const TableEntry tbl_acospi[] = { 257 {0.5, 0.0}, 258 {0.5, -0.0}, 259 {0.0, 1.0}, 260 {1.0, -1.0} 261 }; 262 static const TableEntry tbl_asin[] = { 263 {0.0, 0.0}, 264 {-0.0, -0.0}, 265 {MATH_PI / 2.0, 1.0}, 266 {-MATH_PI / 2.0, -1.0} 267 }; 268 static const TableEntry tbl_asinh[] = { 269 {0.0, 0.0}, 270 {-0.0, -0.0} 271 }; 272 static const TableEntry tbl_asinpi[] = { 273 {0.0, 0.0}, 274 {-0.0, -0.0}, 275 {0.5, 1.0}, 276 {-0.5, -1.0} 277 }; 278 static const TableEntry tbl_atan[] = { 279 {0.0, 0.0}, 280 {-0.0, -0.0}, 281 {MATH_PI / 4.0, 1.0}, 282 {-MATH_PI / 4.0, -1.0} 283 }; 284 static const TableEntry tbl_atanh[] = { 285 {0.0, 0.0}, 286 {-0.0, -0.0} 287 }; 288 static const TableEntry tbl_atanpi[] = { 289 {0.0, 0.0}, 290 {-0.0, -0.0}, 291 {0.25, 1.0}, 292 {-0.25, -1.0} 293 }; 294 static const TableEntry tbl_cbrt[] = { 295 {0.0, 0.0}, 296 {-0.0, -0.0}, 297 {1.0, 1.0}, 298 {-1.0, -1.0}, 299 }; 300 static const TableEntry tbl_cos[] = { 301 {1.0, 0.0}, 302 {1.0, -0.0} 303 }; 304 static const TableEntry tbl_cosh[] = { 305 {1.0, 0.0}, 306 {1.0, -0.0} 307 }; 308 static const TableEntry tbl_cospi[] = { 309 {1.0, 0.0}, 310 {1.0, -0.0} 311 }; 312 static const TableEntry tbl_erfc[] = { 313 {1.0, 0.0}, 314 {1.0, -0.0} 315 }; 316 static const TableEntry tbl_erf[] = { 317 {0.0, 0.0}, 318 {-0.0, -0.0} 319 }; 320 static const TableEntry tbl_exp[] = { 321 {1.0, 0.0}, 322 {1.0, -0.0}, 323 {MATH_E, 1.0} 324 }; 325 static const TableEntry tbl_exp2[] = { 326 {1.0, 0.0}, 327 {1.0, -0.0}, 328 {2.0, 1.0} 329 }; 330 static const TableEntry tbl_exp10[] = { 331 {1.0, 0.0}, 332 {1.0, -0.0}, 333 {10.0, 1.0} 334 }; 335 static const TableEntry tbl_expm1[] = { 336 {0.0, 0.0}, 337 {-0.0, -0.0} 338 }; 339 static const TableEntry tbl_log[] = { 340 {0.0, 1.0}, 341 {1.0, MATH_E} 342 }; 343 static const TableEntry tbl_log2[] = { 344 {0.0, 1.0}, 345 {1.0, 2.0} 346 }; 347 static const TableEntry tbl_log10[] = { 348 {0.0, 1.0}, 349 {1.0, 10.0} 350 }; 351 static const TableEntry tbl_rsqrt[] = { 352 {1.0, 1.0}, 353 {MATH_SQRT1_2, 2.0} 354 }; 355 static const TableEntry tbl_sin[] = { 356 {0.0, 0.0}, 357 {-0.0, -0.0} 358 }; 359 static const TableEntry tbl_sinh[] = { 360 {0.0, 0.0}, 361 {-0.0, -0.0} 362 }; 363 static const TableEntry tbl_sinpi[] = { 364 {0.0, 0.0}, 365 {-0.0, -0.0} 366 }; 367 static const TableEntry tbl_sqrt[] = { 368 {0.0, 0.0}, 369 {1.0, 1.0}, 370 {MATH_SQRT2, 2.0} 371 }; 372 static const TableEntry tbl_tan[] = { 373 {0.0, 0.0}, 374 {-0.0, -0.0} 375 }; 376 static const TableEntry tbl_tanh[] = { 377 {0.0, 0.0}, 378 {-0.0, -0.0} 379 }; 380 static const TableEntry tbl_tanpi[] = { 381 {0.0, 0.0}, 382 {-0.0, -0.0} 383 }; 384 static const TableEntry tbl_tgamma[] = { 385 {1.0, 1.0}, 386 {1.0, 2.0}, 387 {2.0, 3.0}, 388 {6.0, 4.0} 389 }; 390 391 static bool HasNative(AMDGPULibFunc::EFuncId id) { 392 switch(id) { 393 case AMDGPULibFunc::EI_DIVIDE: 394 case AMDGPULibFunc::EI_COS: 395 case AMDGPULibFunc::EI_EXP: 396 case AMDGPULibFunc::EI_EXP2: 397 case AMDGPULibFunc::EI_EXP10: 398 case AMDGPULibFunc::EI_LOG: 399 case AMDGPULibFunc::EI_LOG2: 400 case AMDGPULibFunc::EI_LOG10: 401 case AMDGPULibFunc::EI_POWR: 402 case AMDGPULibFunc::EI_RECIP: 403 case AMDGPULibFunc::EI_RSQRT: 404 case AMDGPULibFunc::EI_SIN: 405 case AMDGPULibFunc::EI_SINCOS: 406 case AMDGPULibFunc::EI_SQRT: 407 case AMDGPULibFunc::EI_TAN: 408 return true; 409 default:; 410 } 411 return false; 412 } 413 414 struct TableRef { 415 size_t size; 416 const TableEntry *table; // variable size: from 0 to (size - 1) 417 418 TableRef() : size(0), table(nullptr) {} 419 420 template <size_t N> 421 TableRef(const TableEntry (&tbl)[N]) : size(N), table(&tbl[0]) {} 422 }; 423 424 static TableRef getOptTable(AMDGPULibFunc::EFuncId id) { 425 switch(id) { 426 case AMDGPULibFunc::EI_ACOS: return TableRef(tbl_acos); 427 case AMDGPULibFunc::EI_ACOSH: return TableRef(tbl_acosh); 428 case AMDGPULibFunc::EI_ACOSPI: return TableRef(tbl_acospi); 429 case AMDGPULibFunc::EI_ASIN: return TableRef(tbl_asin); 430 case AMDGPULibFunc::EI_ASINH: return TableRef(tbl_asinh); 431 case AMDGPULibFunc::EI_ASINPI: return TableRef(tbl_asinpi); 432 case AMDGPULibFunc::EI_ATAN: return TableRef(tbl_atan); 433 case AMDGPULibFunc::EI_ATANH: return TableRef(tbl_atanh); 434 case AMDGPULibFunc::EI_ATANPI: return TableRef(tbl_atanpi); 435 case AMDGPULibFunc::EI_CBRT: return TableRef(tbl_cbrt); 436 case AMDGPULibFunc::EI_NCOS: 437 case AMDGPULibFunc::EI_COS: return TableRef(tbl_cos); 438 case AMDGPULibFunc::EI_COSH: return TableRef(tbl_cosh); 439 case AMDGPULibFunc::EI_COSPI: return TableRef(tbl_cospi); 440 case AMDGPULibFunc::EI_ERFC: return TableRef(tbl_erfc); 441 case AMDGPULibFunc::EI_ERF: return TableRef(tbl_erf); 442 case AMDGPULibFunc::EI_EXP: return TableRef(tbl_exp); 443 case AMDGPULibFunc::EI_NEXP2: 444 case AMDGPULibFunc::EI_EXP2: return TableRef(tbl_exp2); 445 case AMDGPULibFunc::EI_EXP10: return TableRef(tbl_exp10); 446 case AMDGPULibFunc::EI_EXPM1: return TableRef(tbl_expm1); 447 case AMDGPULibFunc::EI_LOG: return TableRef(tbl_log); 448 case AMDGPULibFunc::EI_NLOG2: 449 case AMDGPULibFunc::EI_LOG2: return TableRef(tbl_log2); 450 case AMDGPULibFunc::EI_LOG10: return TableRef(tbl_log10); 451 case AMDGPULibFunc::EI_NRSQRT: 452 case AMDGPULibFunc::EI_RSQRT: return TableRef(tbl_rsqrt); 453 case AMDGPULibFunc::EI_NSIN: 454 case AMDGPULibFunc::EI_SIN: return TableRef(tbl_sin); 455 case AMDGPULibFunc::EI_SINH: return TableRef(tbl_sinh); 456 case AMDGPULibFunc::EI_SINPI: return TableRef(tbl_sinpi); 457 case AMDGPULibFunc::EI_NSQRT: 458 case AMDGPULibFunc::EI_SQRT: return TableRef(tbl_sqrt); 459 case AMDGPULibFunc::EI_TAN: return TableRef(tbl_tan); 460 case AMDGPULibFunc::EI_TANH: return TableRef(tbl_tanh); 461 case AMDGPULibFunc::EI_TANPI: return TableRef(tbl_tanpi); 462 case AMDGPULibFunc::EI_TGAMMA: return TableRef(tbl_tgamma); 463 default:; 464 } 465 return TableRef(); 466 } 467 468 static inline int getVecSize(const AMDGPULibFunc& FInfo) { 469 return FInfo.getLeads()[0].VectorSize; 470 } 471 472 static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) { 473 return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType; 474 } 475 476 FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) { 477 // If we are doing PreLinkOpt, the function is external. So it is safe to 478 // use getOrInsertFunction() at this stage. 479 480 return EnablePreLink ? AMDGPULibFunc::getOrInsertFunction(M, fInfo) 481 : AMDGPULibFunc::getFunction(M, fInfo); 482 } 483 484 bool AMDGPULibCalls::parseFunctionName(const StringRef& FMangledName, 485 FuncInfo *FInfo) { 486 return AMDGPULibFunc::parse(FMangledName, *FInfo); 487 } 488 489 bool AMDGPULibCalls::isUnsafeMath(const CallInst *CI) const { 490 if (auto Op = dyn_cast<FPMathOperator>(CI)) 491 if (Op->isFast()) 492 return true; 493 const Function *F = CI->getParent()->getParent(); 494 Attribute Attr = F->getFnAttribute("unsafe-fp-math"); 495 return Attr.getValueAsString() == "true"; 496 } 497 498 bool AMDGPULibCalls::useNativeFunc(const StringRef F) const { 499 return AllNative || llvm::is_contained(UseNative, F); 500 } 501 502 void AMDGPULibCalls::initNativeFuncs() { 503 AllNative = useNativeFunc("all") || 504 (UseNative.getNumOccurrences() && UseNative.size() == 1 && 505 UseNative.begin()->empty()); 506 } 507 508 bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) { 509 bool native_sin = useNativeFunc("sin"); 510 bool native_cos = useNativeFunc("cos"); 511 512 if (native_sin && native_cos) { 513 Module *M = aCI->getModule(); 514 Value *opr0 = aCI->getArgOperand(0); 515 516 AMDGPULibFunc nf; 517 nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType; 518 nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize; 519 520 nf.setPrefix(AMDGPULibFunc::NATIVE); 521 nf.setId(AMDGPULibFunc::EI_SIN); 522 FunctionCallee sinExpr = getFunction(M, nf); 523 524 nf.setPrefix(AMDGPULibFunc::NATIVE); 525 nf.setId(AMDGPULibFunc::EI_COS); 526 FunctionCallee cosExpr = getFunction(M, nf); 527 if (sinExpr && cosExpr) { 528 Value *sinval = CallInst::Create(sinExpr, opr0, "splitsin", aCI); 529 Value *cosval = CallInst::Create(cosExpr, opr0, "splitcos", aCI); 530 new StoreInst(cosval, aCI->getArgOperand(1), aCI); 531 532 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI 533 << " with native version of sin/cos"); 534 535 replaceCall(sinval); 536 return true; 537 } 538 } 539 return false; 540 } 541 542 bool AMDGPULibCalls::useNative(CallInst *aCI) { 543 CI = aCI; 544 Function *Callee = aCI->getCalledFunction(); 545 546 FuncInfo FInfo; 547 if (!parseFunctionName(Callee->getName(), &FInfo) || !FInfo.isMangled() || 548 FInfo.getPrefix() != AMDGPULibFunc::NOPFX || 549 getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()) || 550 !(AllNative || useNativeFunc(FInfo.getName()))) { 551 return false; 552 } 553 554 if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS) 555 return sincosUseNative(aCI, FInfo); 556 557 FInfo.setPrefix(AMDGPULibFunc::NATIVE); 558 FunctionCallee F = getFunction(aCI->getModule(), FInfo); 559 if (!F) 560 return false; 561 562 aCI->setCalledFunction(F); 563 DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI 564 << " with native version"); 565 return true; 566 } 567 568 // Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe 569 // builtin, with appended type size and alignment arguments, where 2 or 4 570 // indicates the original number of arguments. The library has optimized version 571 // of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same 572 // power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N 573 // for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ..., 574 // 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4. 575 bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B, 576 FuncInfo &FInfo) { 577 auto *Callee = CI->getCalledFunction(); 578 if (!Callee->isDeclaration()) 579 return false; 580 581 assert(Callee->hasName() && "Invalid read_pipe/write_pipe function"); 582 auto *M = Callee->getParent(); 583 auto &Ctx = M->getContext(); 584 std::string Name = std::string(Callee->getName()); 585 auto NumArg = CI->getNumArgOperands(); 586 if (NumArg != 4 && NumArg != 6) 587 return false; 588 auto *PacketSize = CI->getArgOperand(NumArg - 2); 589 auto *PacketAlign = CI->getArgOperand(NumArg - 1); 590 if (!isa<ConstantInt>(PacketSize) || !isa<ConstantInt>(PacketAlign)) 591 return false; 592 unsigned Size = cast<ConstantInt>(PacketSize)->getZExtValue(); 593 Align Alignment = cast<ConstantInt>(PacketAlign)->getAlignValue(); 594 if (Alignment != Size) 595 return false; 596 597 Type *PtrElemTy; 598 if (Size <= 8) 599 PtrElemTy = Type::getIntNTy(Ctx, Size * 8); 600 else 601 PtrElemTy = FixedVectorType::get(Type::getInt64Ty(Ctx), Size / 8); 602 unsigned PtrArgLoc = CI->getNumArgOperands() - 3; 603 auto PtrArg = CI->getArgOperand(PtrArgLoc); 604 unsigned PtrArgAS = PtrArg->getType()->getPointerAddressSpace(); 605 auto *PtrTy = llvm::PointerType::get(PtrElemTy, PtrArgAS); 606 607 SmallVector<llvm::Type *, 6> ArgTys; 608 for (unsigned I = 0; I != PtrArgLoc; ++I) 609 ArgTys.push_back(CI->getArgOperand(I)->getType()); 610 ArgTys.push_back(PtrTy); 611 612 Name = Name + "_" + std::to_string(Size); 613 auto *FTy = FunctionType::get(Callee->getReturnType(), 614 ArrayRef<Type *>(ArgTys), false); 615 AMDGPULibFunc NewLibFunc(Name, FTy); 616 FunctionCallee F = AMDGPULibFunc::getOrInsertFunction(M, NewLibFunc); 617 if (!F) 618 return false; 619 620 auto *BCast = B.CreatePointerCast(PtrArg, PtrTy); 621 SmallVector<Value *, 6> Args; 622 for (unsigned I = 0; I != PtrArgLoc; ++I) 623 Args.push_back(CI->getArgOperand(I)); 624 Args.push_back(BCast); 625 626 auto *NCI = B.CreateCall(F, Args); 627 NCI->setAttributes(CI->getAttributes()); 628 CI->replaceAllUsesWith(NCI); 629 CI->dropAllReferences(); 630 CI->eraseFromParent(); 631 632 return true; 633 } 634 635 // This function returns false if no change; return true otherwise. 636 bool AMDGPULibCalls::fold(CallInst *CI, AliasAnalysis *AA) { 637 this->CI = CI; 638 Function *Callee = CI->getCalledFunction(); 639 640 // Ignore indirect calls. 641 if (Callee == 0) return false; 642 643 BasicBlock *BB = CI->getParent(); 644 LLVMContext &Context = CI->getParent()->getContext(); 645 IRBuilder<> B(Context); 646 647 // Set the builder to the instruction after the call. 648 B.SetInsertPoint(BB, CI->getIterator()); 649 650 // Copy fast flags from the original call. 651 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(CI)) 652 B.setFastMathFlags(FPOp->getFastMathFlags()); 653 654 switch (Callee->getIntrinsicID()) { 655 default: 656 break; 657 case Intrinsic::amdgcn_wavefrontsize: 658 return !EnablePreLink && fold_wavefrontsize(CI, B); 659 } 660 661 FuncInfo FInfo; 662 if (!parseFunctionName(Callee->getName(), &FInfo)) 663 return false; 664 665 // Further check the number of arguments to see if they match. 666 if (CI->getNumArgOperands() != FInfo.getNumArgs()) 667 return false; 668 669 if (TDOFold(CI, FInfo)) 670 return true; 671 672 // Under unsafe-math, evaluate calls if possible. 673 // According to Brian Sumner, we can do this for all f32 function calls 674 // using host's double function calls. 675 if (isUnsafeMath(CI) && evaluateCall(CI, FInfo)) 676 return true; 677 678 // Specilized optimizations for each function call 679 switch (FInfo.getId()) { 680 case AMDGPULibFunc::EI_RECIP: 681 // skip vector function 682 assert ((FInfo.getPrefix() == AMDGPULibFunc::NATIVE || 683 FInfo.getPrefix() == AMDGPULibFunc::HALF) && 684 "recip must be an either native or half function"); 685 return (getVecSize(FInfo) != 1) ? false : fold_recip(CI, B, FInfo); 686 687 case AMDGPULibFunc::EI_DIVIDE: 688 // skip vector function 689 assert ((FInfo.getPrefix() == AMDGPULibFunc::NATIVE || 690 FInfo.getPrefix() == AMDGPULibFunc::HALF) && 691 "divide must be an either native or half function"); 692 return (getVecSize(FInfo) != 1) ? false : fold_divide(CI, B, FInfo); 693 694 case AMDGPULibFunc::EI_POW: 695 case AMDGPULibFunc::EI_POWR: 696 case AMDGPULibFunc::EI_POWN: 697 return fold_pow(CI, B, FInfo); 698 699 case AMDGPULibFunc::EI_ROOTN: 700 // skip vector function 701 return (getVecSize(FInfo) != 1) ? false : fold_rootn(CI, B, FInfo); 702 703 case AMDGPULibFunc::EI_FMA: 704 case AMDGPULibFunc::EI_MAD: 705 case AMDGPULibFunc::EI_NFMA: 706 // skip vector function 707 return (getVecSize(FInfo) != 1) ? false : fold_fma_mad(CI, B, FInfo); 708 709 case AMDGPULibFunc::EI_SQRT: 710 return isUnsafeMath(CI) && fold_sqrt(CI, B, FInfo); 711 case AMDGPULibFunc::EI_COS: 712 case AMDGPULibFunc::EI_SIN: 713 if ((getArgType(FInfo) == AMDGPULibFunc::F32 || 714 getArgType(FInfo) == AMDGPULibFunc::F64) 715 && (FInfo.getPrefix() == AMDGPULibFunc::NOPFX)) 716 return fold_sincos(CI, B, AA); 717 718 break; 719 case AMDGPULibFunc::EI_READ_PIPE_2: 720 case AMDGPULibFunc::EI_READ_PIPE_4: 721 case AMDGPULibFunc::EI_WRITE_PIPE_2: 722 case AMDGPULibFunc::EI_WRITE_PIPE_4: 723 return fold_read_write_pipe(CI, B, FInfo); 724 725 default: 726 break; 727 } 728 729 return false; 730 } 731 732 bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) { 733 // Table-Driven optimization 734 const TableRef tr = getOptTable(FInfo.getId()); 735 if (tr.size==0) 736 return false; 737 738 int const sz = (int)tr.size; 739 const TableEntry * const ftbl = tr.table; 740 Value *opr0 = CI->getArgOperand(0); 741 742 if (getVecSize(FInfo) > 1) { 743 if (ConstantDataVector *CV = dyn_cast<ConstantDataVector>(opr0)) { 744 SmallVector<double, 0> DVal; 745 for (int eltNo = 0; eltNo < getVecSize(FInfo); ++eltNo) { 746 ConstantFP *eltval = dyn_cast<ConstantFP>( 747 CV->getElementAsConstant((unsigned)eltNo)); 748 assert(eltval && "Non-FP arguments in math function!"); 749 bool found = false; 750 for (int i=0; i < sz; ++i) { 751 if (eltval->isExactlyValue(ftbl[i].input)) { 752 DVal.push_back(ftbl[i].result); 753 found = true; 754 break; 755 } 756 } 757 if (!found) { 758 // This vector constants not handled yet. 759 return false; 760 } 761 } 762 LLVMContext &context = CI->getParent()->getParent()->getContext(); 763 Constant *nval; 764 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 765 SmallVector<float, 0> FVal; 766 for (unsigned i = 0; i < DVal.size(); ++i) { 767 FVal.push_back((float)DVal[i]); 768 } 769 ArrayRef<float> tmp(FVal); 770 nval = ConstantDataVector::get(context, tmp); 771 } else { // F64 772 ArrayRef<double> tmp(DVal); 773 nval = ConstantDataVector::get(context, tmp); 774 } 775 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 776 replaceCall(nval); 777 return true; 778 } 779 } else { 780 // Scalar version 781 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) { 782 for (int i = 0; i < sz; ++i) { 783 if (CF->isExactlyValue(ftbl[i].input)) { 784 Value *nval = ConstantFP::get(CF->getType(), ftbl[i].result); 785 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 786 replaceCall(nval); 787 return true; 788 } 789 } 790 } 791 } 792 793 return false; 794 } 795 796 bool AMDGPULibCalls::replaceWithNative(CallInst *CI, const FuncInfo &FInfo) { 797 Module *M = CI->getModule(); 798 if (getArgType(FInfo) != AMDGPULibFunc::F32 || 799 FInfo.getPrefix() != AMDGPULibFunc::NOPFX || 800 !HasNative(FInfo.getId())) 801 return false; 802 803 AMDGPULibFunc nf = FInfo; 804 nf.setPrefix(AMDGPULibFunc::NATIVE); 805 if (FunctionCallee FPExpr = getFunction(M, nf)) { 806 LLVM_DEBUG(dbgs() << "AMDIC: " << *CI << " ---> "); 807 808 CI->setCalledFunction(FPExpr); 809 810 LLVM_DEBUG(dbgs() << *CI << '\n'); 811 812 return true; 813 } 814 return false; 815 } 816 817 // [native_]half_recip(c) ==> 1.0/c 818 bool AMDGPULibCalls::fold_recip(CallInst *CI, IRBuilder<> &B, 819 const FuncInfo &FInfo) { 820 Value *opr0 = CI->getArgOperand(0); 821 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) { 822 // Just create a normal div. Later, InstCombine will be able 823 // to compute the divide into a constant (avoid check float infinity 824 // or subnormal at this point). 825 Value *nval = B.CreateFDiv(ConstantFP::get(CF->getType(), 1.0), 826 opr0, 827 "recip2div"); 828 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n"); 829 replaceCall(nval); 830 return true; 831 } 832 return false; 833 } 834 835 // [native_]half_divide(x, c) ==> x/c 836 bool AMDGPULibCalls::fold_divide(CallInst *CI, IRBuilder<> &B, 837 const FuncInfo &FInfo) { 838 Value *opr0 = CI->getArgOperand(0); 839 Value *opr1 = CI->getArgOperand(1); 840 ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0); 841 ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1); 842 843 if ((CF0 && CF1) || // both are constants 844 (CF1 && (getArgType(FInfo) == AMDGPULibFunc::F32))) 845 // CF1 is constant && f32 divide 846 { 847 Value *nval1 = B.CreateFDiv(ConstantFP::get(opr1->getType(), 1.0), 848 opr1, "__div2recip"); 849 Value *nval = B.CreateFMul(opr0, nval1, "__div2mul"); 850 replaceCall(nval); 851 return true; 852 } 853 return false; 854 } 855 856 namespace llvm { 857 static double log2(double V) { 858 #if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L 859 return ::log2(V); 860 #else 861 return log(V) / numbers::ln2; 862 #endif 863 } 864 } 865 866 bool AMDGPULibCalls::fold_pow(CallInst *CI, IRBuilder<> &B, 867 const FuncInfo &FInfo) { 868 assert((FInfo.getId() == AMDGPULibFunc::EI_POW || 869 FInfo.getId() == AMDGPULibFunc::EI_POWR || 870 FInfo.getId() == AMDGPULibFunc::EI_POWN) && 871 "fold_pow: encounter a wrong function call"); 872 873 Value *opr0, *opr1; 874 ConstantFP *CF; 875 ConstantInt *CINT; 876 ConstantAggregateZero *CZero; 877 Type *eltType; 878 879 opr0 = CI->getArgOperand(0); 880 opr1 = CI->getArgOperand(1); 881 CZero = dyn_cast<ConstantAggregateZero>(opr1); 882 if (getVecSize(FInfo) == 1) { 883 eltType = opr0->getType(); 884 CF = dyn_cast<ConstantFP>(opr1); 885 CINT = dyn_cast<ConstantInt>(opr1); 886 } else { 887 VectorType *VTy = dyn_cast<VectorType>(opr0->getType()); 888 assert(VTy && "Oprand of vector function should be of vectortype"); 889 eltType = VTy->getElementType(); 890 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1); 891 892 // Now, only Handle vector const whose elements have the same value. 893 CF = CDV ? dyn_cast_or_null<ConstantFP>(CDV->getSplatValue()) : nullptr; 894 CINT = CDV ? dyn_cast_or_null<ConstantInt>(CDV->getSplatValue()) : nullptr; 895 } 896 897 // No unsafe math , no constant argument, do nothing 898 if (!isUnsafeMath(CI) && !CF && !CINT && !CZero) 899 return false; 900 901 // 0x1111111 means that we don't do anything for this call. 902 int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111); 903 904 if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0) || CZero) { 905 // pow/powr/pown(x, 0) == 1 906 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1\n"); 907 Constant *cnval = ConstantFP::get(eltType, 1.0); 908 if (getVecSize(FInfo) > 1) { 909 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 910 } 911 replaceCall(cnval); 912 return true; 913 } 914 if ((CF && CF->isExactlyValue(1.0)) || (CINT && ci_opr1 == 1)) { 915 // pow/powr/pown(x, 1.0) = x 916 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n"); 917 replaceCall(opr0); 918 return true; 919 } 920 if ((CF && CF->isExactlyValue(2.0)) || (CINT && ci_opr1 == 2)) { 921 // pow/powr/pown(x, 2.0) = x*x 922 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * " << *opr0 923 << "\n"); 924 Value *nval = B.CreateFMul(opr0, opr0, "__pow2"); 925 replaceCall(nval); 926 return true; 927 } 928 if ((CF && CF->isExactlyValue(-1.0)) || (CINT && ci_opr1 == -1)) { 929 // pow/powr/pown(x, -1.0) = 1.0/x 930 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1 / " << *opr0 << "\n"); 931 Constant *cnval = ConstantFP::get(eltType, 1.0); 932 if (getVecSize(FInfo) > 1) { 933 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 934 } 935 Value *nval = B.CreateFDiv(cnval, opr0, "__powrecip"); 936 replaceCall(nval); 937 return true; 938 } 939 940 Module *M = CI->getModule(); 941 if (CF && (CF->isExactlyValue(0.5) || CF->isExactlyValue(-0.5))) { 942 // pow[r](x, [-]0.5) = sqrt(x) 943 bool issqrt = CF->isExactlyValue(0.5); 944 if (FunctionCallee FPExpr = 945 getFunction(M, AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT 946 : AMDGPULibFunc::EI_RSQRT, 947 FInfo))) { 948 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " 949 << FInfo.getName().c_str() << "(" << *opr0 << ")\n"); 950 Value *nval = CreateCallEx(B,FPExpr, opr0, issqrt ? "__pow2sqrt" 951 : "__pow2rsqrt"); 952 replaceCall(nval); 953 return true; 954 } 955 } 956 957 if (!isUnsafeMath(CI)) 958 return false; 959 960 // Unsafe Math optimization 961 962 // Remember that ci_opr1 is set if opr1 is integral 963 if (CF) { 964 double dval = (getArgType(FInfo) == AMDGPULibFunc::F32) 965 ? (double)CF->getValueAPF().convertToFloat() 966 : CF->getValueAPF().convertToDouble(); 967 int ival = (int)dval; 968 if ((double)ival == dval) { 969 ci_opr1 = ival; 970 } else 971 ci_opr1 = 0x11111111; 972 } 973 974 // pow/powr/pown(x, c) = [1/](x*x*..x); where 975 // trunc(c) == c && the number of x == c && |c| <= 12 976 unsigned abs_opr1 = (ci_opr1 < 0) ? -ci_opr1 : ci_opr1; 977 if (abs_opr1 <= 12) { 978 Constant *cnval; 979 Value *nval; 980 if (abs_opr1 == 0) { 981 cnval = ConstantFP::get(eltType, 1.0); 982 if (getVecSize(FInfo) > 1) { 983 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 984 } 985 nval = cnval; 986 } else { 987 Value *valx2 = nullptr; 988 nval = nullptr; 989 while (abs_opr1 > 0) { 990 valx2 = valx2 ? B.CreateFMul(valx2, valx2, "__powx2") : opr0; 991 if (abs_opr1 & 1) { 992 nval = nval ? B.CreateFMul(nval, valx2, "__powprod") : valx2; 993 } 994 abs_opr1 >>= 1; 995 } 996 } 997 998 if (ci_opr1 < 0) { 999 cnval = ConstantFP::get(eltType, 1.0); 1000 if (getVecSize(FInfo) > 1) { 1001 cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval); 1002 } 1003 nval = B.CreateFDiv(cnval, nval, "__1powprod"); 1004 } 1005 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " 1006 << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr0 1007 << ")\n"); 1008 replaceCall(nval); 1009 return true; 1010 } 1011 1012 // powr ---> exp2(y * log2(x)) 1013 // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31)) 1014 FunctionCallee ExpExpr = 1015 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo)); 1016 if (!ExpExpr) 1017 return false; 1018 1019 bool needlog = false; 1020 bool needabs = false; 1021 bool needcopysign = false; 1022 Constant *cnval = nullptr; 1023 if (getVecSize(FInfo) == 1) { 1024 CF = dyn_cast<ConstantFP>(opr0); 1025 1026 if (CF) { 1027 double V = (getArgType(FInfo) == AMDGPULibFunc::F32) 1028 ? (double)CF->getValueAPF().convertToFloat() 1029 : CF->getValueAPF().convertToDouble(); 1030 1031 V = log2(std::abs(V)); 1032 cnval = ConstantFP::get(eltType, V); 1033 needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR) && 1034 CF->isNegative(); 1035 } else { 1036 needlog = true; 1037 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR && 1038 (!CF || CF->isNegative()); 1039 } 1040 } else { 1041 ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr0); 1042 1043 if (!CDV) { 1044 needlog = true; 1045 needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR; 1046 } else { 1047 assert ((int)CDV->getNumElements() == getVecSize(FInfo) && 1048 "Wrong vector size detected"); 1049 1050 SmallVector<double, 0> DVal; 1051 for (int i=0; i < getVecSize(FInfo); ++i) { 1052 double V = (getArgType(FInfo) == AMDGPULibFunc::F32) 1053 ? (double)CDV->getElementAsFloat(i) 1054 : CDV->getElementAsDouble(i); 1055 if (V < 0.0) needcopysign = true; 1056 V = log2(std::abs(V)); 1057 DVal.push_back(V); 1058 } 1059 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 1060 SmallVector<float, 0> FVal; 1061 for (unsigned i=0; i < DVal.size(); ++i) { 1062 FVal.push_back((float)DVal[i]); 1063 } 1064 ArrayRef<float> tmp(FVal); 1065 cnval = ConstantDataVector::get(M->getContext(), tmp); 1066 } else { 1067 ArrayRef<double> tmp(DVal); 1068 cnval = ConstantDataVector::get(M->getContext(), tmp); 1069 } 1070 } 1071 } 1072 1073 if (needcopysign && (FInfo.getId() == AMDGPULibFunc::EI_POW)) { 1074 // We cannot handle corner cases for a general pow() function, give up 1075 // unless y is a constant integral value. Then proceed as if it were pown. 1076 if (getVecSize(FInfo) == 1) { 1077 if (const ConstantFP *CF = dyn_cast<ConstantFP>(opr1)) { 1078 double y = (getArgType(FInfo) == AMDGPULibFunc::F32) 1079 ? (double)CF->getValueAPF().convertToFloat() 1080 : CF->getValueAPF().convertToDouble(); 1081 if (y != (double)(int64_t)y) 1082 return false; 1083 } else 1084 return false; 1085 } else { 1086 if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1)) { 1087 for (int i=0; i < getVecSize(FInfo); ++i) { 1088 double y = (getArgType(FInfo) == AMDGPULibFunc::F32) 1089 ? (double)CDV->getElementAsFloat(i) 1090 : CDV->getElementAsDouble(i); 1091 if (y != (double)(int64_t)y) 1092 return false; 1093 } 1094 } else 1095 return false; 1096 } 1097 } 1098 1099 Value *nval; 1100 if (needabs) { 1101 FunctionCallee AbsExpr = 1102 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_FABS, FInfo)); 1103 if (!AbsExpr) 1104 return false; 1105 nval = CreateCallEx(B, AbsExpr, opr0, "__fabs"); 1106 } else { 1107 nval = cnval ? cnval : opr0; 1108 } 1109 if (needlog) { 1110 FunctionCallee LogExpr = 1111 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo)); 1112 if (!LogExpr) 1113 return false; 1114 nval = CreateCallEx(B,LogExpr, nval, "__log2"); 1115 } 1116 1117 if (FInfo.getId() == AMDGPULibFunc::EI_POWN) { 1118 // convert int(32) to fp(f32 or f64) 1119 opr1 = B.CreateSIToFP(opr1, nval->getType(), "pownI2F"); 1120 } 1121 nval = B.CreateFMul(opr1, nval, "__ylogx"); 1122 nval = CreateCallEx(B,ExpExpr, nval, "__exp2"); 1123 1124 if (needcopysign) { 1125 Value *opr_n; 1126 Type* rTy = opr0->getType(); 1127 Type* nTyS = eltType->isDoubleTy() ? B.getInt64Ty() : B.getInt32Ty(); 1128 Type *nTy = nTyS; 1129 if (const auto *vTy = dyn_cast<FixedVectorType>(rTy)) 1130 nTy = FixedVectorType::get(nTyS, vTy); 1131 unsigned size = nTy->getScalarSizeInBits(); 1132 opr_n = CI->getArgOperand(1); 1133 if (opr_n->getType()->isIntegerTy()) 1134 opr_n = B.CreateZExtOrBitCast(opr_n, nTy, "__ytou"); 1135 else 1136 opr_n = B.CreateFPToSI(opr1, nTy, "__ytou"); 1137 1138 Value *sign = B.CreateShl(opr_n, size-1, "__yeven"); 1139 sign = B.CreateAnd(B.CreateBitCast(opr0, nTy), sign, "__pow_sign"); 1140 nval = B.CreateOr(B.CreateBitCast(nval, nTy), sign); 1141 nval = B.CreateBitCast(nval, opr0->getType()); 1142 } 1143 1144 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " 1145 << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n"); 1146 replaceCall(nval); 1147 1148 return true; 1149 } 1150 1151 bool AMDGPULibCalls::fold_rootn(CallInst *CI, IRBuilder<> &B, 1152 const FuncInfo &FInfo) { 1153 Value *opr0 = CI->getArgOperand(0); 1154 Value *opr1 = CI->getArgOperand(1); 1155 1156 ConstantInt *CINT = dyn_cast<ConstantInt>(opr1); 1157 if (!CINT) { 1158 return false; 1159 } 1160 int ci_opr1 = (int)CINT->getSExtValue(); 1161 if (ci_opr1 == 1) { // rootn(x, 1) = x 1162 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n"); 1163 replaceCall(opr0); 1164 return true; 1165 } 1166 if (ci_opr1 == 2) { // rootn(x, 2) = sqrt(x) 1167 Module *M = CI->getModule(); 1168 if (FunctionCallee FPExpr = 1169 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) { 1170 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> sqrt(" << *opr0 << ")\n"); 1171 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2sqrt"); 1172 replaceCall(nval); 1173 return true; 1174 } 1175 } else if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x) 1176 Module *M = CI->getModule(); 1177 if (FunctionCallee FPExpr = 1178 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) { 1179 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> cbrt(" << *opr0 << ")\n"); 1180 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2cbrt"); 1181 replaceCall(nval); 1182 return true; 1183 } 1184 } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x 1185 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1.0 / " << *opr0 << "\n"); 1186 Value *nval = B.CreateFDiv(ConstantFP::get(opr0->getType(), 1.0), 1187 opr0, 1188 "__rootn2div"); 1189 replaceCall(nval); 1190 return true; 1191 } else if (ci_opr1 == -2) { // rootn(x, -2) = rsqrt(x) 1192 Module *M = CI->getModule(); 1193 if (FunctionCallee FPExpr = 1194 getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_RSQRT, FInfo))) { 1195 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> rsqrt(" << *opr0 1196 << ")\n"); 1197 Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2rsqrt"); 1198 replaceCall(nval); 1199 return true; 1200 } 1201 } 1202 return false; 1203 } 1204 1205 bool AMDGPULibCalls::fold_fma_mad(CallInst *CI, IRBuilder<> &B, 1206 const FuncInfo &FInfo) { 1207 Value *opr0 = CI->getArgOperand(0); 1208 Value *opr1 = CI->getArgOperand(1); 1209 Value *opr2 = CI->getArgOperand(2); 1210 1211 ConstantFP *CF0 = dyn_cast<ConstantFP>(opr0); 1212 ConstantFP *CF1 = dyn_cast<ConstantFP>(opr1); 1213 if ((CF0 && CF0->isZero()) || (CF1 && CF1->isZero())) { 1214 // fma/mad(a, b, c) = c if a=0 || b=0 1215 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr2 << "\n"); 1216 replaceCall(opr2); 1217 return true; 1218 } 1219 if (CF0 && CF0->isExactlyValue(1.0f)) { 1220 // fma/mad(a, b, c) = b+c if a=1 1221 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr1 << " + " << *opr2 1222 << "\n"); 1223 Value *nval = B.CreateFAdd(opr1, opr2, "fmaadd"); 1224 replaceCall(nval); 1225 return true; 1226 } 1227 if (CF1 && CF1->isExactlyValue(1.0f)) { 1228 // fma/mad(a, b, c) = a+c if b=1 1229 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " + " << *opr2 1230 << "\n"); 1231 Value *nval = B.CreateFAdd(opr0, opr2, "fmaadd"); 1232 replaceCall(nval); 1233 return true; 1234 } 1235 if (ConstantFP *CF = dyn_cast<ConstantFP>(opr2)) { 1236 if (CF->isZero()) { 1237 // fma/mad(a, b, c) = a*b if c=0 1238 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << " * " 1239 << *opr1 << "\n"); 1240 Value *nval = B.CreateFMul(opr0, opr1, "fmamul"); 1241 replaceCall(nval); 1242 return true; 1243 } 1244 } 1245 1246 return false; 1247 } 1248 1249 // Get a scalar native builtin signle argument FP function 1250 FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M, 1251 const FuncInfo &FInfo) { 1252 if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId())) 1253 return nullptr; 1254 FuncInfo nf = FInfo; 1255 nf.setPrefix(AMDGPULibFunc::NATIVE); 1256 return getFunction(M, nf); 1257 } 1258 1259 // fold sqrt -> native_sqrt (x) 1260 bool AMDGPULibCalls::fold_sqrt(CallInst *CI, IRBuilder<> &B, 1261 const FuncInfo &FInfo) { 1262 if (getArgType(FInfo) == AMDGPULibFunc::F32 && (getVecSize(FInfo) == 1) && 1263 (FInfo.getPrefix() != AMDGPULibFunc::NATIVE)) { 1264 if (FunctionCallee FPExpr = getNativeFunction( 1265 CI->getModule(), AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) { 1266 Value *opr0 = CI->getArgOperand(0); 1267 LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " 1268 << "sqrt(" << *opr0 << ")\n"); 1269 Value *nval = CreateCallEx(B,FPExpr, opr0, "__sqrt"); 1270 replaceCall(nval); 1271 return true; 1272 } 1273 } 1274 return false; 1275 } 1276 1277 // fold sin, cos -> sincos. 1278 bool AMDGPULibCalls::fold_sincos(CallInst *CI, IRBuilder<> &B, 1279 AliasAnalysis *AA) { 1280 AMDGPULibFunc fInfo; 1281 if (!AMDGPULibFunc::parse(CI->getCalledFunction()->getName(), fInfo)) 1282 return false; 1283 1284 assert(fInfo.getId() == AMDGPULibFunc::EI_SIN || 1285 fInfo.getId() == AMDGPULibFunc::EI_COS); 1286 bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN; 1287 1288 Value *CArgVal = CI->getArgOperand(0); 1289 BasicBlock * const CBB = CI->getParent(); 1290 1291 int const MaxScan = 30; 1292 bool Changed = false; 1293 1294 { // fold in load value. 1295 LoadInst *LI = dyn_cast<LoadInst>(CArgVal); 1296 if (LI && LI->getParent() == CBB) { 1297 BasicBlock::iterator BBI = LI->getIterator(); 1298 Value *AvailableVal = FindAvailableLoadedValue(LI, CBB, BBI, MaxScan, AA); 1299 if (AvailableVal) { 1300 Changed = true; 1301 CArgVal->replaceAllUsesWith(AvailableVal); 1302 if (CArgVal->getNumUses() == 0) 1303 LI->eraseFromParent(); 1304 CArgVal = CI->getArgOperand(0); 1305 } 1306 } 1307 } 1308 1309 Module *M = CI->getModule(); 1310 fInfo.setId(isSin ? AMDGPULibFunc::EI_COS : AMDGPULibFunc::EI_SIN); 1311 std::string const PairName = fInfo.mangle(); 1312 1313 CallInst *UI = nullptr; 1314 for (User* U : CArgVal->users()) { 1315 CallInst *XI = dyn_cast_or_null<CallInst>(U); 1316 if (!XI || XI == CI || XI->getParent() != CBB) 1317 continue; 1318 1319 Function *UCallee = XI->getCalledFunction(); 1320 if (!UCallee || !UCallee->getName().equals(PairName)) 1321 continue; 1322 1323 BasicBlock::iterator BBI = CI->getIterator(); 1324 if (BBI == CI->getParent()->begin()) 1325 break; 1326 --BBI; 1327 for (int I = MaxScan; I > 0 && BBI != CBB->begin(); --BBI, --I) { 1328 if (cast<Instruction>(BBI) == XI) { 1329 UI = XI; 1330 break; 1331 } 1332 } 1333 if (UI) break; 1334 } 1335 1336 if (!UI) 1337 return Changed; 1338 1339 // Merge the sin and cos. 1340 1341 // for OpenCL 2.0 we have only generic implementation of sincos 1342 // function. 1343 AMDGPULibFunc nf(AMDGPULibFunc::EI_SINCOS, fInfo); 1344 nf.getLeads()[0].PtrKind = AMDGPULibFunc::getEPtrKindFromAddrSpace(AMDGPUAS::FLAT_ADDRESS); 1345 FunctionCallee Fsincos = getFunction(M, nf); 1346 if (!Fsincos) 1347 return Changed; 1348 1349 BasicBlock::iterator ItOld = B.GetInsertPoint(); 1350 AllocaInst *Alloc = insertAlloca(UI, B, "__sincos_"); 1351 B.SetInsertPoint(UI); 1352 1353 Value *P = Alloc; 1354 Type *PTy = Fsincos.getFunctionType()->getParamType(1); 1355 // The allocaInst allocates the memory in private address space. This need 1356 // to be bitcasted to point to the address space of cos pointer type. 1357 // In OpenCL 2.0 this is generic, while in 1.2 that is private. 1358 if (PTy->getPointerAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS) 1359 P = B.CreateAddrSpaceCast(Alloc, PTy); 1360 CallInst *Call = CreateCallEx2(B, Fsincos, UI->getArgOperand(0), P); 1361 1362 LLVM_DEBUG(errs() << "AMDIC: fold_sincos (" << *CI << ", " << *UI << ") with " 1363 << *Call << "\n"); 1364 1365 if (!isSin) { // CI->cos, UI->sin 1366 B.SetInsertPoint(&*ItOld); 1367 UI->replaceAllUsesWith(&*Call); 1368 Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc); 1369 CI->replaceAllUsesWith(Reload); 1370 UI->eraseFromParent(); 1371 CI->eraseFromParent(); 1372 } else { // CI->sin, UI->cos 1373 Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc); 1374 UI->replaceAllUsesWith(Reload); 1375 CI->replaceAllUsesWith(Call); 1376 UI->eraseFromParent(); 1377 CI->eraseFromParent(); 1378 } 1379 return true; 1380 } 1381 1382 bool AMDGPULibCalls::fold_wavefrontsize(CallInst *CI, IRBuilder<> &B) { 1383 if (!TM) 1384 return false; 1385 1386 StringRef CPU = TM->getTargetCPU(); 1387 StringRef Features = TM->getTargetFeatureString(); 1388 if ((CPU.empty() || CPU.equals_lower("generic")) && 1389 (Features.empty() || 1390 Features.find_lower("wavefrontsize") == StringRef::npos)) 1391 return false; 1392 1393 Function *F = CI->getParent()->getParent(); 1394 const GCNSubtarget &ST = TM->getSubtarget<GCNSubtarget>(*F); 1395 unsigned N = ST.getWavefrontSize(); 1396 1397 LLVM_DEBUG(errs() << "AMDIC: fold_wavefrontsize (" << *CI << ") with " 1398 << N << "\n"); 1399 1400 CI->replaceAllUsesWith(ConstantInt::get(B.getInt32Ty(), N)); 1401 CI->eraseFromParent(); 1402 return true; 1403 } 1404 1405 // Get insertion point at entry. 1406 BasicBlock::iterator AMDGPULibCalls::getEntryIns(CallInst * UI) { 1407 Function * Func = UI->getParent()->getParent(); 1408 BasicBlock * BB = &Func->getEntryBlock(); 1409 assert(BB && "Entry block not found!"); 1410 BasicBlock::iterator ItNew = BB->begin(); 1411 return ItNew; 1412 } 1413 1414 // Insert a AllocsInst at the beginning of function entry block. 1415 AllocaInst* AMDGPULibCalls::insertAlloca(CallInst *UI, IRBuilder<> &B, 1416 const char *prefix) { 1417 BasicBlock::iterator ItNew = getEntryIns(UI); 1418 Function *UCallee = UI->getCalledFunction(); 1419 Type *RetType = UCallee->getReturnType(); 1420 B.SetInsertPoint(&*ItNew); 1421 AllocaInst *Alloc = B.CreateAlloca(RetType, 0, 1422 std::string(prefix) + UI->getName()); 1423 Alloc->setAlignment( 1424 Align(UCallee->getParent()->getDataLayout().getTypeAllocSize(RetType))); 1425 return Alloc; 1426 } 1427 1428 bool AMDGPULibCalls::evaluateScalarMathFunc(FuncInfo &FInfo, 1429 double& Res0, double& Res1, 1430 Constant *copr0, Constant *copr1, 1431 Constant *copr2) { 1432 // By default, opr0/opr1/opr3 holds values of float/double type. 1433 // If they are not float/double, each function has to its 1434 // operand separately. 1435 double opr0=0.0, opr1=0.0, opr2=0.0; 1436 ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(copr0); 1437 ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(copr1); 1438 ConstantFP *fpopr2 = dyn_cast_or_null<ConstantFP>(copr2); 1439 if (fpopr0) { 1440 opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1441 ? fpopr0->getValueAPF().convertToDouble() 1442 : (double)fpopr0->getValueAPF().convertToFloat(); 1443 } 1444 1445 if (fpopr1) { 1446 opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1447 ? fpopr1->getValueAPF().convertToDouble() 1448 : (double)fpopr1->getValueAPF().convertToFloat(); 1449 } 1450 1451 if (fpopr2) { 1452 opr2 = (getArgType(FInfo) == AMDGPULibFunc::F64) 1453 ? fpopr2->getValueAPF().convertToDouble() 1454 : (double)fpopr2->getValueAPF().convertToFloat(); 1455 } 1456 1457 switch (FInfo.getId()) { 1458 default : return false; 1459 1460 case AMDGPULibFunc::EI_ACOS: 1461 Res0 = acos(opr0); 1462 return true; 1463 1464 case AMDGPULibFunc::EI_ACOSH: 1465 // acosh(x) == log(x + sqrt(x*x - 1)) 1466 Res0 = log(opr0 + sqrt(opr0*opr0 - 1.0)); 1467 return true; 1468 1469 case AMDGPULibFunc::EI_ACOSPI: 1470 Res0 = acos(opr0) / MATH_PI; 1471 return true; 1472 1473 case AMDGPULibFunc::EI_ASIN: 1474 Res0 = asin(opr0); 1475 return true; 1476 1477 case AMDGPULibFunc::EI_ASINH: 1478 // asinh(x) == log(x + sqrt(x*x + 1)) 1479 Res0 = log(opr0 + sqrt(opr0*opr0 + 1.0)); 1480 return true; 1481 1482 case AMDGPULibFunc::EI_ASINPI: 1483 Res0 = asin(opr0) / MATH_PI; 1484 return true; 1485 1486 case AMDGPULibFunc::EI_ATAN: 1487 Res0 = atan(opr0); 1488 return true; 1489 1490 case AMDGPULibFunc::EI_ATANH: 1491 // atanh(x) == (log(x+1) - log(x-1))/2; 1492 Res0 = (log(opr0 + 1.0) - log(opr0 - 1.0))/2.0; 1493 return true; 1494 1495 case AMDGPULibFunc::EI_ATANPI: 1496 Res0 = atan(opr0) / MATH_PI; 1497 return true; 1498 1499 case AMDGPULibFunc::EI_CBRT: 1500 Res0 = (opr0 < 0.0) ? -pow(-opr0, 1.0/3.0) : pow(opr0, 1.0/3.0); 1501 return true; 1502 1503 case AMDGPULibFunc::EI_COS: 1504 Res0 = cos(opr0); 1505 return true; 1506 1507 case AMDGPULibFunc::EI_COSH: 1508 Res0 = cosh(opr0); 1509 return true; 1510 1511 case AMDGPULibFunc::EI_COSPI: 1512 Res0 = cos(MATH_PI * opr0); 1513 return true; 1514 1515 case AMDGPULibFunc::EI_EXP: 1516 Res0 = exp(opr0); 1517 return true; 1518 1519 case AMDGPULibFunc::EI_EXP2: 1520 Res0 = pow(2.0, opr0); 1521 return true; 1522 1523 case AMDGPULibFunc::EI_EXP10: 1524 Res0 = pow(10.0, opr0); 1525 return true; 1526 1527 case AMDGPULibFunc::EI_EXPM1: 1528 Res0 = exp(opr0) - 1.0; 1529 return true; 1530 1531 case AMDGPULibFunc::EI_LOG: 1532 Res0 = log(opr0); 1533 return true; 1534 1535 case AMDGPULibFunc::EI_LOG2: 1536 Res0 = log(opr0) / log(2.0); 1537 return true; 1538 1539 case AMDGPULibFunc::EI_LOG10: 1540 Res0 = log(opr0) / log(10.0); 1541 return true; 1542 1543 case AMDGPULibFunc::EI_RSQRT: 1544 Res0 = 1.0 / sqrt(opr0); 1545 return true; 1546 1547 case AMDGPULibFunc::EI_SIN: 1548 Res0 = sin(opr0); 1549 return true; 1550 1551 case AMDGPULibFunc::EI_SINH: 1552 Res0 = sinh(opr0); 1553 return true; 1554 1555 case AMDGPULibFunc::EI_SINPI: 1556 Res0 = sin(MATH_PI * opr0); 1557 return true; 1558 1559 case AMDGPULibFunc::EI_SQRT: 1560 Res0 = sqrt(opr0); 1561 return true; 1562 1563 case AMDGPULibFunc::EI_TAN: 1564 Res0 = tan(opr0); 1565 return true; 1566 1567 case AMDGPULibFunc::EI_TANH: 1568 Res0 = tanh(opr0); 1569 return true; 1570 1571 case AMDGPULibFunc::EI_TANPI: 1572 Res0 = tan(MATH_PI * opr0); 1573 return true; 1574 1575 case AMDGPULibFunc::EI_RECIP: 1576 Res0 = 1.0 / opr0; 1577 return true; 1578 1579 // two-arg functions 1580 case AMDGPULibFunc::EI_DIVIDE: 1581 Res0 = opr0 / opr1; 1582 return true; 1583 1584 case AMDGPULibFunc::EI_POW: 1585 case AMDGPULibFunc::EI_POWR: 1586 Res0 = pow(opr0, opr1); 1587 return true; 1588 1589 case AMDGPULibFunc::EI_POWN: { 1590 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) { 1591 double val = (double)iopr1->getSExtValue(); 1592 Res0 = pow(opr0, val); 1593 return true; 1594 } 1595 return false; 1596 } 1597 1598 case AMDGPULibFunc::EI_ROOTN: { 1599 if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) { 1600 double val = (double)iopr1->getSExtValue(); 1601 Res0 = pow(opr0, 1.0 / val); 1602 return true; 1603 } 1604 return false; 1605 } 1606 1607 // with ptr arg 1608 case AMDGPULibFunc::EI_SINCOS: 1609 Res0 = sin(opr0); 1610 Res1 = cos(opr0); 1611 return true; 1612 1613 // three-arg functions 1614 case AMDGPULibFunc::EI_FMA: 1615 case AMDGPULibFunc::EI_MAD: 1616 Res0 = opr0 * opr1 + opr2; 1617 return true; 1618 } 1619 1620 return false; 1621 } 1622 1623 bool AMDGPULibCalls::evaluateCall(CallInst *aCI, FuncInfo &FInfo) { 1624 int numArgs = (int)aCI->getNumArgOperands(); 1625 if (numArgs > 3) 1626 return false; 1627 1628 Constant *copr0 = nullptr; 1629 Constant *copr1 = nullptr; 1630 Constant *copr2 = nullptr; 1631 if (numArgs > 0) { 1632 if ((copr0 = dyn_cast<Constant>(aCI->getArgOperand(0))) == nullptr) 1633 return false; 1634 } 1635 1636 if (numArgs > 1) { 1637 if ((copr1 = dyn_cast<Constant>(aCI->getArgOperand(1))) == nullptr) { 1638 if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS) 1639 return false; 1640 } 1641 } 1642 1643 if (numArgs > 2) { 1644 if ((copr2 = dyn_cast<Constant>(aCI->getArgOperand(2))) == nullptr) 1645 return false; 1646 } 1647 1648 // At this point, all arguments to aCI are constants. 1649 1650 // max vector size is 16, and sincos will generate two results. 1651 double DVal0[16], DVal1[16]; 1652 bool hasTwoResults = (FInfo.getId() == AMDGPULibFunc::EI_SINCOS); 1653 if (getVecSize(FInfo) == 1) { 1654 if (!evaluateScalarMathFunc(FInfo, DVal0[0], 1655 DVal1[0], copr0, copr1, copr2)) { 1656 return false; 1657 } 1658 } else { 1659 ConstantDataVector *CDV0 = dyn_cast_or_null<ConstantDataVector>(copr0); 1660 ConstantDataVector *CDV1 = dyn_cast_or_null<ConstantDataVector>(copr1); 1661 ConstantDataVector *CDV2 = dyn_cast_or_null<ConstantDataVector>(copr2); 1662 for (int i=0; i < getVecSize(FInfo); ++i) { 1663 Constant *celt0 = CDV0 ? CDV0->getElementAsConstant(i) : nullptr; 1664 Constant *celt1 = CDV1 ? CDV1->getElementAsConstant(i) : nullptr; 1665 Constant *celt2 = CDV2 ? CDV2->getElementAsConstant(i) : nullptr; 1666 if (!evaluateScalarMathFunc(FInfo, DVal0[i], 1667 DVal1[i], celt0, celt1, celt2)) { 1668 return false; 1669 } 1670 } 1671 } 1672 1673 LLVMContext &context = CI->getParent()->getParent()->getContext(); 1674 Constant *nval0, *nval1; 1675 if (getVecSize(FInfo) == 1) { 1676 nval0 = ConstantFP::get(CI->getType(), DVal0[0]); 1677 if (hasTwoResults) 1678 nval1 = ConstantFP::get(CI->getType(), DVal1[0]); 1679 } else { 1680 if (getArgType(FInfo) == AMDGPULibFunc::F32) { 1681 SmallVector <float, 0> FVal0, FVal1; 1682 for (int i=0; i < getVecSize(FInfo); ++i) 1683 FVal0.push_back((float)DVal0[i]); 1684 ArrayRef<float> tmp0(FVal0); 1685 nval0 = ConstantDataVector::get(context, tmp0); 1686 if (hasTwoResults) { 1687 for (int i=0; i < getVecSize(FInfo); ++i) 1688 FVal1.push_back((float)DVal1[i]); 1689 ArrayRef<float> tmp1(FVal1); 1690 nval1 = ConstantDataVector::get(context, tmp1); 1691 } 1692 } else { 1693 ArrayRef<double> tmp0(DVal0); 1694 nval0 = ConstantDataVector::get(context, tmp0); 1695 if (hasTwoResults) { 1696 ArrayRef<double> tmp1(DVal1); 1697 nval1 = ConstantDataVector::get(context, tmp1); 1698 } 1699 } 1700 } 1701 1702 if (hasTwoResults) { 1703 // sincos 1704 assert(FInfo.getId() == AMDGPULibFunc::EI_SINCOS && 1705 "math function with ptr arg not supported yet"); 1706 new StoreInst(nval1, aCI->getArgOperand(1), aCI); 1707 } 1708 1709 replaceCall(nval0); 1710 return true; 1711 } 1712 1713 // Public interface to the Simplify LibCalls pass. 1714 FunctionPass *llvm::createAMDGPUSimplifyLibCallsPass(const TargetMachine *TM) { 1715 return new AMDGPUSimplifyLibCalls(TM); 1716 } 1717 1718 FunctionPass *llvm::createAMDGPUUseNativeCallsPass() { 1719 return new AMDGPUUseNativeCalls(); 1720 } 1721 1722 bool AMDGPUSimplifyLibCalls::runOnFunction(Function &F) { 1723 if (skipFunction(F)) 1724 return false; 1725 1726 bool Changed = false; 1727 auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); 1728 1729 LLVM_DEBUG(dbgs() << "AMDIC: process function "; 1730 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';); 1731 1732 for (auto &BB : F) { 1733 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) { 1734 // Ignore non-calls. 1735 CallInst *CI = dyn_cast<CallInst>(I); 1736 ++I; 1737 // Ignore intrinsics that do not become real instructions. 1738 if (!CI || isa<DbgInfoIntrinsic>(CI) || CI->isLifetimeStartOrEnd()) 1739 continue; 1740 1741 // Ignore indirect calls. 1742 Function *Callee = CI->getCalledFunction(); 1743 if (Callee == 0) continue; 1744 1745 LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << "\n"; 1746 dbgs().flush()); 1747 if(Simplifier.fold(CI, AA)) 1748 Changed = true; 1749 } 1750 } 1751 return Changed; 1752 } 1753 1754 PreservedAnalyses AMDGPUSimplifyLibCallsPass::run(Function &F, 1755 FunctionAnalysisManager &AM) { 1756 AMDGPULibCalls Simplifier; 1757 Simplifier.initNativeFuncs(); 1758 1759 bool Changed = false; 1760 auto AA = &AM.getResult<AAManager>(F); 1761 1762 LLVM_DEBUG(dbgs() << "AMDIC: process function "; 1763 F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';); 1764 1765 for (auto &BB : F) { 1766 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) { 1767 // Ignore non-calls. 1768 CallInst *CI = dyn_cast<CallInst>(I); 1769 ++I; 1770 // Ignore intrinsics that do not become real instructions. 1771 if (!CI || isa<DbgInfoIntrinsic>(CI) || CI->isLifetimeStartOrEnd()) 1772 continue; 1773 1774 // Ignore indirect calls. 1775 Function *Callee = CI->getCalledFunction(); 1776 if (Callee == 0) 1777 continue; 1778 1779 LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << "\n"; 1780 dbgs().flush()); 1781 if (Simplifier.fold(CI, AA)) 1782 Changed = true; 1783 } 1784 } 1785 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 1786 } 1787 1788 bool AMDGPUUseNativeCalls::runOnFunction(Function &F) { 1789 if (skipFunction(F) || UseNative.empty()) 1790 return false; 1791 1792 bool Changed = false; 1793 for (auto &BB : F) { 1794 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) { 1795 // Ignore non-calls. 1796 CallInst *CI = dyn_cast<CallInst>(I); 1797 ++I; 1798 if (!CI) continue; 1799 1800 // Ignore indirect calls. 1801 Function *Callee = CI->getCalledFunction(); 1802 if (Callee == 0) continue; 1803 1804 if(Simplifier.useNative(CI)) 1805 Changed = true; 1806 } 1807 } 1808 return Changed; 1809 } 1810 1811 PreservedAnalyses AMDGPUUseNativeCallsPass::run(Function &F, 1812 FunctionAnalysisManager &AM) { 1813 if (UseNative.empty()) 1814 return PreservedAnalyses::all(); 1815 1816 AMDGPULibCalls Simplifier; 1817 Simplifier.initNativeFuncs(); 1818 1819 bool Changed = false; 1820 for (auto &BB : F) { 1821 for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) { 1822 // Ignore non-calls. 1823 CallInst *CI = dyn_cast<CallInst>(I); 1824 ++I; 1825 if (!CI) 1826 continue; 1827 1828 // Ignore indirect calls. 1829 Function *Callee = CI->getCalledFunction(); 1830 if (Callee == 0) 1831 continue; 1832 1833 if (Simplifier.useNative(CI)) 1834 Changed = true; 1835 } 1836 } 1837 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 1838 } 1839