xref: /llvm-project/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp (revision dac8f974b51b076773ec49603df4e627f4d9089c)
1 //===- AMDGPULibCalls.cpp -------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file does AMD library function optimizations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AMDGPU.h"
15 #include "AMDGPULibFunc.h"
16 #include "GCNSubtarget.h"
17 #include "llvm/Analysis/AssumptionCache.h"
18 #include "llvm/Analysis/TargetLibraryInfo.h"
19 #include "llvm/Analysis/ValueTracking.h"
20 #include "llvm/IR/Dominators.h"
21 #include "llvm/IR/IRBuilder.h"
22 #include "llvm/IR/IntrinsicInst.h"
23 #include "llvm/IR/IntrinsicsAMDGPU.h"
24 #include "llvm/IR/PatternMatch.h"
25 #include "llvm/InitializePasses.h"
26 #include <cmath>
27 
28 #define DEBUG_TYPE "amdgpu-simplifylib"
29 
30 using namespace llvm;
31 using namespace llvm::PatternMatch;
32 
33 static cl::opt<bool> EnablePreLink("amdgpu-prelink",
34   cl::desc("Enable pre-link mode optimizations"),
35   cl::init(false),
36   cl::Hidden);
37 
38 static cl::list<std::string> UseNative("amdgpu-use-native",
39   cl::desc("Comma separated list of functions to replace with native, or all"),
40   cl::CommaSeparated, cl::ValueOptional,
41   cl::Hidden);
42 
43 #define MATH_PI      numbers::pi
44 #define MATH_E       numbers::e
45 #define MATH_SQRT2   numbers::sqrt2
46 #define MATH_SQRT1_2 numbers::inv_sqrt2
47 
48 namespace llvm {
49 
50 class AMDGPULibCalls {
51 private:
52   const TargetLibraryInfo *TLInfo = nullptr;
53   AssumptionCache *AC = nullptr;
54   DominatorTree *DT = nullptr;
55 
56   typedef llvm::AMDGPULibFunc FuncInfo;
57 
58   bool UnsafeFPMath = false;
59 
60   // -fuse-native.
61   bool AllNative = false;
62 
63   bool useNativeFunc(const StringRef F) const;
64 
65   // Return a pointer (pointer expr) to the function if function definition with
66   // "FuncName" exists. It may create a new function prototype in pre-link mode.
67   FunctionCallee getFunction(Module *M, const FuncInfo &fInfo);
68 
69   bool parseFunctionName(const StringRef &FMangledName, FuncInfo &FInfo);
70 
71   bool TDOFold(CallInst *CI, const FuncInfo &FInfo);
72 
73   /* Specialized optimizations */
74 
75   // pow/powr/pown
76   bool fold_pow(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);
77 
78   // rootn
79   bool fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);
80 
81   // -fuse-native for sincos
82   bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo);
83 
84   // evaluate calls if calls' arguments are constants.
85   bool evaluateScalarMathFunc(const FuncInfo &FInfo, double &Res0, double &Res1,
86                               Constant *copr0, Constant *copr1);
87   bool evaluateCall(CallInst *aCI, const FuncInfo &FInfo);
88 
89   // sqrt
90   bool fold_sqrt(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);
91 
92   /// Insert a value to sincos function \p Fsincos. Returns (value of sin, value
93   /// of cos, sincos call).
94   std::tuple<Value *, Value *, Value *> insertSinCos(Value *Arg,
95                                                      FastMathFlags FMF,
96                                                      IRBuilder<> &B,
97                                                      FunctionCallee Fsincos);
98 
99   // sin/cos
100   bool fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);
101 
102   // __read_pipe/__write_pipe
103   bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
104                             const FuncInfo &FInfo);
105 
106   // Get a scalar native builtin single argument FP function
107   FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo);
108 
109   /// Substitute a call to a known libcall with an intrinsic call. If \p
110   /// AllowMinSize is true, allow the replacement in a minsize function.
111   bool shouldReplaceLibcallWithIntrinsic(const CallInst *CI,
112                                          bool AllowMinSizeF32 = false,
113                                          bool AllowF64 = false,
114                                          bool AllowStrictFP = false);
115   void replaceLibCallWithSimpleIntrinsic(IRBuilder<> &B, CallInst *CI,
116                                          Intrinsic::ID IntrID);
117 
118   bool tryReplaceLibcallWithSimpleIntrinsic(IRBuilder<> &B, CallInst *CI,
119                                             Intrinsic::ID IntrID,
120                                             bool AllowMinSizeF32 = false,
121                                             bool AllowF64 = false,
122                                             bool AllowStrictFP = false);
123 
124 protected:
125   bool isUnsafeMath(const FPMathOperator *FPOp) const;
126   bool isUnsafeFiniteOnlyMath(const FPMathOperator *FPOp) const;
127 
128   bool canIncreasePrecisionOfConstantFold(const FPMathOperator *FPOp) const;
129 
130   static void replaceCall(Instruction *I, Value *With) {
131     I->replaceAllUsesWith(With);
132     I->eraseFromParent();
133   }
134 
135   static void replaceCall(FPMathOperator *I, Value *With) {
136     replaceCall(cast<Instruction>(I), With);
137   }
138 
139 public:
140   AMDGPULibCalls() {}
141 
142   bool fold(CallInst *CI);
143 
144   void initFunction(Function &F, FunctionAnalysisManager &FAM);
145   void initNativeFuncs();
146 
147   // Replace a normal math function call with that native version
148   bool useNative(CallInst *CI);
149 };
150 
151 } // end llvm namespace
152 
153 template <typename IRB>
154 static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg,
155                               const Twine &Name = "") {
156   CallInst *R = B.CreateCall(Callee, Arg, Name);
157   if (Function *F = dyn_cast<Function>(Callee.getCallee()))
158     R->setCallingConv(F->getCallingConv());
159   return R;
160 }
161 
162 template <typename IRB>
163 static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1,
164                                Value *Arg2, const Twine &Name = "") {
165   CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name);
166   if (Function *F = dyn_cast<Function>(Callee.getCallee()))
167     R->setCallingConv(F->getCallingConv());
168   return R;
169 }
170 
171 //  Data structures for table-driven optimizations.
172 //  FuncTbl works for both f32 and f64 functions with 1 input argument
173 
174 struct TableEntry {
175   double   result;
176   double   input;
177 };
178 
179 /* a list of {result, input} */
180 static const TableEntry tbl_acos[] = {
181   {MATH_PI / 2.0, 0.0},
182   {MATH_PI / 2.0, -0.0},
183   {0.0, 1.0},
184   {MATH_PI, -1.0}
185 };
186 static const TableEntry tbl_acosh[] = {
187   {0.0, 1.0}
188 };
189 static const TableEntry tbl_acospi[] = {
190   {0.5, 0.0},
191   {0.5, -0.0},
192   {0.0, 1.0},
193   {1.0, -1.0}
194 };
195 static const TableEntry tbl_asin[] = {
196   {0.0, 0.0},
197   {-0.0, -0.0},
198   {MATH_PI / 2.0, 1.0},
199   {-MATH_PI / 2.0, -1.0}
200 };
201 static const TableEntry tbl_asinh[] = {
202   {0.0, 0.0},
203   {-0.0, -0.0}
204 };
205 static const TableEntry tbl_asinpi[] = {
206   {0.0, 0.0},
207   {-0.0, -0.0},
208   {0.5, 1.0},
209   {-0.5, -1.0}
210 };
211 static const TableEntry tbl_atan[] = {
212   {0.0, 0.0},
213   {-0.0, -0.0},
214   {MATH_PI / 4.0, 1.0},
215   {-MATH_PI / 4.0, -1.0}
216 };
217 static const TableEntry tbl_atanh[] = {
218   {0.0, 0.0},
219   {-0.0, -0.0}
220 };
221 static const TableEntry tbl_atanpi[] = {
222   {0.0, 0.0},
223   {-0.0, -0.0},
224   {0.25, 1.0},
225   {-0.25, -1.0}
226 };
227 static const TableEntry tbl_cbrt[] = {
228   {0.0, 0.0},
229   {-0.0, -0.0},
230   {1.0, 1.0},
231   {-1.0, -1.0},
232 };
233 static const TableEntry tbl_cos[] = {
234   {1.0, 0.0},
235   {1.0, -0.0}
236 };
237 static const TableEntry tbl_cosh[] = {
238   {1.0, 0.0},
239   {1.0, -0.0}
240 };
241 static const TableEntry tbl_cospi[] = {
242   {1.0, 0.0},
243   {1.0, -0.0}
244 };
245 static const TableEntry tbl_erfc[] = {
246   {1.0, 0.0},
247   {1.0, -0.0}
248 };
249 static const TableEntry tbl_erf[] = {
250   {0.0, 0.0},
251   {-0.0, -0.0}
252 };
253 static const TableEntry tbl_exp[] = {
254   {1.0, 0.0},
255   {1.0, -0.0},
256   {MATH_E, 1.0}
257 };
258 static const TableEntry tbl_exp2[] = {
259   {1.0, 0.0},
260   {1.0, -0.0},
261   {2.0, 1.0}
262 };
263 static const TableEntry tbl_exp10[] = {
264   {1.0, 0.0},
265   {1.0, -0.0},
266   {10.0, 1.0}
267 };
268 static const TableEntry tbl_expm1[] = {
269   {0.0, 0.0},
270   {-0.0, -0.0}
271 };
272 static const TableEntry tbl_log[] = {
273   {0.0, 1.0},
274   {1.0, MATH_E}
275 };
276 static const TableEntry tbl_log2[] = {
277   {0.0, 1.0},
278   {1.0, 2.0}
279 };
280 static const TableEntry tbl_log10[] = {
281   {0.0, 1.0},
282   {1.0, 10.0}
283 };
284 static const TableEntry tbl_rsqrt[] = {
285   {1.0, 1.0},
286   {MATH_SQRT1_2, 2.0}
287 };
288 static const TableEntry tbl_sin[] = {
289   {0.0, 0.0},
290   {-0.0, -0.0}
291 };
292 static const TableEntry tbl_sinh[] = {
293   {0.0, 0.0},
294   {-0.0, -0.0}
295 };
296 static const TableEntry tbl_sinpi[] = {
297   {0.0, 0.0},
298   {-0.0, -0.0}
299 };
300 static const TableEntry tbl_sqrt[] = {
301   {0.0, 0.0},
302   {1.0, 1.0},
303   {MATH_SQRT2, 2.0}
304 };
305 static const TableEntry tbl_tan[] = {
306   {0.0, 0.0},
307   {-0.0, -0.0}
308 };
309 static const TableEntry tbl_tanh[] = {
310   {0.0, 0.0},
311   {-0.0, -0.0}
312 };
313 static const TableEntry tbl_tanpi[] = {
314   {0.0, 0.0},
315   {-0.0, -0.0}
316 };
317 static const TableEntry tbl_tgamma[] = {
318   {1.0, 1.0},
319   {1.0, 2.0},
320   {2.0, 3.0},
321   {6.0, 4.0}
322 };
323 
324 static bool HasNative(AMDGPULibFunc::EFuncId id) {
325   switch(id) {
326   case AMDGPULibFunc::EI_DIVIDE:
327   case AMDGPULibFunc::EI_COS:
328   case AMDGPULibFunc::EI_EXP:
329   case AMDGPULibFunc::EI_EXP2:
330   case AMDGPULibFunc::EI_EXP10:
331   case AMDGPULibFunc::EI_LOG:
332   case AMDGPULibFunc::EI_LOG2:
333   case AMDGPULibFunc::EI_LOG10:
334   case AMDGPULibFunc::EI_POWR:
335   case AMDGPULibFunc::EI_RECIP:
336   case AMDGPULibFunc::EI_RSQRT:
337   case AMDGPULibFunc::EI_SIN:
338   case AMDGPULibFunc::EI_SINCOS:
339   case AMDGPULibFunc::EI_SQRT:
340   case AMDGPULibFunc::EI_TAN:
341     return true;
342   default:;
343   }
344   return false;
345 }
346 
347 using TableRef = ArrayRef<TableEntry>;
348 
349 static TableRef getOptTable(AMDGPULibFunc::EFuncId id) {
350   switch(id) {
351   case AMDGPULibFunc::EI_ACOS:    return TableRef(tbl_acos);
352   case AMDGPULibFunc::EI_ACOSH:   return TableRef(tbl_acosh);
353   case AMDGPULibFunc::EI_ACOSPI:  return TableRef(tbl_acospi);
354   case AMDGPULibFunc::EI_ASIN:    return TableRef(tbl_asin);
355   case AMDGPULibFunc::EI_ASINH:   return TableRef(tbl_asinh);
356   case AMDGPULibFunc::EI_ASINPI:  return TableRef(tbl_asinpi);
357   case AMDGPULibFunc::EI_ATAN:    return TableRef(tbl_atan);
358   case AMDGPULibFunc::EI_ATANH:   return TableRef(tbl_atanh);
359   case AMDGPULibFunc::EI_ATANPI:  return TableRef(tbl_atanpi);
360   case AMDGPULibFunc::EI_CBRT:    return TableRef(tbl_cbrt);
361   case AMDGPULibFunc::EI_NCOS:
362   case AMDGPULibFunc::EI_COS:     return TableRef(tbl_cos);
363   case AMDGPULibFunc::EI_COSH:    return TableRef(tbl_cosh);
364   case AMDGPULibFunc::EI_COSPI:   return TableRef(tbl_cospi);
365   case AMDGPULibFunc::EI_ERFC:    return TableRef(tbl_erfc);
366   case AMDGPULibFunc::EI_ERF:     return TableRef(tbl_erf);
367   case AMDGPULibFunc::EI_EXP:     return TableRef(tbl_exp);
368   case AMDGPULibFunc::EI_NEXP2:
369   case AMDGPULibFunc::EI_EXP2:    return TableRef(tbl_exp2);
370   case AMDGPULibFunc::EI_EXP10:   return TableRef(tbl_exp10);
371   case AMDGPULibFunc::EI_EXPM1:   return TableRef(tbl_expm1);
372   case AMDGPULibFunc::EI_LOG:     return TableRef(tbl_log);
373   case AMDGPULibFunc::EI_NLOG2:
374   case AMDGPULibFunc::EI_LOG2:    return TableRef(tbl_log2);
375   case AMDGPULibFunc::EI_LOG10:   return TableRef(tbl_log10);
376   case AMDGPULibFunc::EI_NRSQRT:
377   case AMDGPULibFunc::EI_RSQRT:   return TableRef(tbl_rsqrt);
378   case AMDGPULibFunc::EI_NSIN:
379   case AMDGPULibFunc::EI_SIN:     return TableRef(tbl_sin);
380   case AMDGPULibFunc::EI_SINH:    return TableRef(tbl_sinh);
381   case AMDGPULibFunc::EI_SINPI:   return TableRef(tbl_sinpi);
382   case AMDGPULibFunc::EI_NSQRT:
383   case AMDGPULibFunc::EI_SQRT:    return TableRef(tbl_sqrt);
384   case AMDGPULibFunc::EI_TAN:     return TableRef(tbl_tan);
385   case AMDGPULibFunc::EI_TANH:    return TableRef(tbl_tanh);
386   case AMDGPULibFunc::EI_TANPI:   return TableRef(tbl_tanpi);
387   case AMDGPULibFunc::EI_TGAMMA:  return TableRef(tbl_tgamma);
388   default:;
389   }
390   return TableRef();
391 }
392 
393 static inline int getVecSize(const AMDGPULibFunc& FInfo) {
394   return FInfo.getLeads()[0].VectorSize;
395 }
396 
397 static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) {
398   return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType;
399 }
400 
401 FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) {
402   // If we are doing PreLinkOpt, the function is external. So it is safe to
403   // use getOrInsertFunction() at this stage.
404 
405   return EnablePreLink ? AMDGPULibFunc::getOrInsertFunction(M, fInfo)
406                        : AMDGPULibFunc::getFunction(M, fInfo);
407 }
408 
409 bool AMDGPULibCalls::parseFunctionName(const StringRef &FMangledName,
410                                        FuncInfo &FInfo) {
411   return AMDGPULibFunc::parse(FMangledName, FInfo);
412 }
413 
414 bool AMDGPULibCalls::isUnsafeMath(const FPMathOperator *FPOp) const {
415   return UnsafeFPMath || FPOp->isFast();
416 }
417 
418 bool AMDGPULibCalls::isUnsafeFiniteOnlyMath(const FPMathOperator *FPOp) const {
419   return UnsafeFPMath ||
420          (FPOp->hasApproxFunc() && FPOp->hasNoNaNs() && FPOp->hasNoInfs());
421 }
422 
423 bool AMDGPULibCalls::canIncreasePrecisionOfConstantFold(
424     const FPMathOperator *FPOp) const {
425   // TODO: Refine to approxFunc or contract
426   return isUnsafeMath(FPOp);
427 }
428 
429 void AMDGPULibCalls::initFunction(Function &F, FunctionAnalysisManager &FAM) {
430   UnsafeFPMath = F.getFnAttribute("unsafe-fp-math").getValueAsBool();
431   AC = &FAM.getResult<AssumptionAnalysis>(F);
432   TLInfo = &FAM.getResult<TargetLibraryAnalysis>(F);
433   DT = FAM.getCachedResult<DominatorTreeAnalysis>(F);
434 }
435 
436 bool AMDGPULibCalls::useNativeFunc(const StringRef F) const {
437   return AllNative || llvm::is_contained(UseNative, F);
438 }
439 
440 void AMDGPULibCalls::initNativeFuncs() {
441   AllNative = useNativeFunc("all") ||
442               (UseNative.getNumOccurrences() && UseNative.size() == 1 &&
443                UseNative.begin()->empty());
444 }
445 
446 bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) {
447   bool native_sin = useNativeFunc("sin");
448   bool native_cos = useNativeFunc("cos");
449 
450   if (native_sin && native_cos) {
451     Module *M = aCI->getModule();
452     Value *opr0 = aCI->getArgOperand(0);
453 
454     AMDGPULibFunc nf;
455     nf.getLeads()[0].ArgType = FInfo.getLeads()[0].ArgType;
456     nf.getLeads()[0].VectorSize = FInfo.getLeads()[0].VectorSize;
457 
458     nf.setPrefix(AMDGPULibFunc::NATIVE);
459     nf.setId(AMDGPULibFunc::EI_SIN);
460     FunctionCallee sinExpr = getFunction(M, nf);
461 
462     nf.setPrefix(AMDGPULibFunc::NATIVE);
463     nf.setId(AMDGPULibFunc::EI_COS);
464     FunctionCallee cosExpr = getFunction(M, nf);
465     if (sinExpr && cosExpr) {
466       Value *sinval = CallInst::Create(sinExpr, opr0, "splitsin", aCI);
467       Value *cosval = CallInst::Create(cosExpr, opr0, "splitcos", aCI);
468       new StoreInst(cosval, aCI->getArgOperand(1), aCI);
469 
470       DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
471                                           << " with native version of sin/cos");
472 
473       replaceCall(aCI, sinval);
474       return true;
475     }
476   }
477   return false;
478 }
479 
480 bool AMDGPULibCalls::useNative(CallInst *aCI) {
481   Function *Callee = aCI->getCalledFunction();
482   if (!Callee || aCI->isNoBuiltin())
483     return false;
484 
485   FuncInfo FInfo;
486   if (!parseFunctionName(Callee->getName(), FInfo) || !FInfo.isMangled() ||
487       FInfo.getPrefix() != AMDGPULibFunc::NOPFX ||
488       getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()) ||
489       !(AllNative || useNativeFunc(FInfo.getName()))) {
490     return false;
491   }
492 
493   if (FInfo.getId() == AMDGPULibFunc::EI_SINCOS)
494     return sincosUseNative(aCI, FInfo);
495 
496   FInfo.setPrefix(AMDGPULibFunc::NATIVE);
497   FunctionCallee F = getFunction(aCI->getModule(), FInfo);
498   if (!F)
499     return false;
500 
501   aCI->setCalledFunction(F);
502   DEBUG_WITH_TYPE("usenative", dbgs() << "<useNative> replace " << *aCI
503                                       << " with native version");
504   return true;
505 }
506 
507 // Clang emits call of __read_pipe_2 or __read_pipe_4 for OpenCL read_pipe
508 // builtin, with appended type size and alignment arguments, where 2 or 4
509 // indicates the original number of arguments. The library has optimized version
510 // of __read_pipe_2/__read_pipe_4 when the type size and alignment has the same
511 // power of 2 value. This function transforms __read_pipe_2 to __read_pipe_2_N
512 // for such cases where N is the size in bytes of the type (N = 1, 2, 4, 8, ...,
513 // 128). The same for __read_pipe_4, write_pipe_2, and write_pipe_4.
514 bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
515                                           const FuncInfo &FInfo) {
516   auto *Callee = CI->getCalledFunction();
517   if (!Callee->isDeclaration())
518     return false;
519 
520   assert(Callee->hasName() && "Invalid read_pipe/write_pipe function");
521   auto *M = Callee->getParent();
522   std::string Name = std::string(Callee->getName());
523   auto NumArg = CI->arg_size();
524   if (NumArg != 4 && NumArg != 6)
525     return false;
526   ConstantInt *PacketSize =
527       dyn_cast<ConstantInt>(CI->getArgOperand(NumArg - 2));
528   ConstantInt *PacketAlign =
529       dyn_cast<ConstantInt>(CI->getArgOperand(NumArg - 1));
530   if (!PacketSize || !PacketAlign)
531     return false;
532 
533   unsigned Size = PacketSize->getZExtValue();
534   Align Alignment = PacketAlign->getAlignValue();
535   if (Alignment != Size)
536     return false;
537 
538   unsigned PtrArgLoc = CI->arg_size() - 3;
539   Value *PtrArg = CI->getArgOperand(PtrArgLoc);
540   Type *PtrTy = PtrArg->getType();
541 
542   SmallVector<llvm::Type *, 6> ArgTys;
543   for (unsigned I = 0; I != PtrArgLoc; ++I)
544     ArgTys.push_back(CI->getArgOperand(I)->getType());
545   ArgTys.push_back(PtrTy);
546 
547   Name = Name + "_" + std::to_string(Size);
548   auto *FTy = FunctionType::get(Callee->getReturnType(),
549                                 ArrayRef<Type *>(ArgTys), false);
550   AMDGPULibFunc NewLibFunc(Name, FTy);
551   FunctionCallee F = AMDGPULibFunc::getOrInsertFunction(M, NewLibFunc);
552   if (!F)
553     return false;
554 
555   auto *BCast = B.CreatePointerCast(PtrArg, PtrTy);
556   SmallVector<Value *, 6> Args;
557   for (unsigned I = 0; I != PtrArgLoc; ++I)
558     Args.push_back(CI->getArgOperand(I));
559   Args.push_back(BCast);
560 
561   auto *NCI = B.CreateCall(F, Args);
562   NCI->setAttributes(CI->getAttributes());
563   CI->replaceAllUsesWith(NCI);
564   CI->dropAllReferences();
565   CI->eraseFromParent();
566 
567   return true;
568 }
569 
570 static bool isKnownIntegral(const Value *V, const DataLayout &DL,
571                             FastMathFlags FMF) {
572   if (isa<UndefValue>(V))
573     return true;
574 
575   if (const ConstantFP *CF = dyn_cast<ConstantFP>(V))
576     return CF->getValueAPF().isInteger();
577 
578   if (const ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(V)) {
579     for (unsigned i = 0, e = CDV->getNumElements(); i != e; ++i) {
580       Constant *ConstElt = CDV->getElementAsConstant(i);
581       if (isa<UndefValue>(ConstElt))
582         continue;
583       const ConstantFP *CFP = dyn_cast<ConstantFP>(ConstElt);
584       if (!CFP || !CFP->getValue().isInteger())
585         return false;
586     }
587 
588     return true;
589   }
590 
591   const Instruction *I = dyn_cast<Instruction>(V);
592   if (!I)
593     return false;
594 
595   switch (I->getOpcode()) {
596   case Instruction::SIToFP:
597   case Instruction::UIToFP:
598     // TODO: Could check nofpclass(inf) on incoming argument
599     if (FMF.noInfs())
600       return true;
601 
602     // Need to check int size cannot produce infinity, which computeKnownFPClass
603     // knows how to do already.
604     return isKnownNeverInfinity(I, DL);
605   default:
606     break;
607   }
608 
609   return false;
610 }
611 
612 // This function returns false if no change; return true otherwise.
613 bool AMDGPULibCalls::fold(CallInst *CI) {
614   Function *Callee = CI->getCalledFunction();
615   // Ignore indirect calls.
616   if (!Callee || Callee->isIntrinsic() || CI->isNoBuiltin())
617     return false;
618 
619   FuncInfo FInfo;
620   if (!parseFunctionName(Callee->getName(), FInfo))
621     return false;
622 
623   // Further check the number of arguments to see if they match.
624   // TODO: Check calling convention matches too
625   if (!FInfo.isCompatibleSignature(CI->getFunctionType()))
626     return false;
627 
628   LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << '\n');
629 
630   if (TDOFold(CI, FInfo))
631     return true;
632 
633   IRBuilder<> B(CI);
634 
635   if (FPMathOperator *FPOp = dyn_cast<FPMathOperator>(CI)) {
636     // Under unsafe-math, evaluate calls if possible.
637     // According to Brian Sumner, we can do this for all f32 function calls
638     // using host's double function calls.
639     if (canIncreasePrecisionOfConstantFold(FPOp) && evaluateCall(CI, FInfo))
640       return true;
641 
642     // Copy fast flags from the original call.
643     FastMathFlags FMF = FPOp->getFastMathFlags();
644     B.setFastMathFlags(FMF);
645 
646     // Specialized optimizations for each function call.
647     //
648     // TODO: Handle other simple intrinsic wrappers. Sqrt.
649     //
650     // TODO: Handle native functions
651     switch (FInfo.getId()) {
652     case AMDGPULibFunc::EI_EXP:
653       if (FMF.none())
654         return false;
655       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::exp,
656                                                   FMF.approxFunc());
657     case AMDGPULibFunc::EI_EXP2:
658       if (FMF.none())
659         return false;
660       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::exp2,
661                                                   FMF.approxFunc());
662     case AMDGPULibFunc::EI_LOG:
663       if (FMF.none())
664         return false;
665       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::log,
666                                                   FMF.approxFunc());
667     case AMDGPULibFunc::EI_LOG2:
668       if (FMF.none())
669         return false;
670       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::log2,
671                                                   FMF.approxFunc());
672     case AMDGPULibFunc::EI_LOG10:
673       if (FMF.none())
674         return false;
675       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::log10,
676                                                   FMF.approxFunc());
677     case AMDGPULibFunc::EI_FMIN:
678       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::minnum,
679                                                   true, true);
680     case AMDGPULibFunc::EI_FMAX:
681       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::maxnum,
682                                                   true, true);
683     case AMDGPULibFunc::EI_FMA:
684       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::fma, true,
685                                                   true);
686     case AMDGPULibFunc::EI_MAD:
687       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::fmuladd,
688                                                   true, true);
689     case AMDGPULibFunc::EI_FABS:
690       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::fabs, true,
691                                                   true, true);
692     case AMDGPULibFunc::EI_COPYSIGN:
693       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::copysign,
694                                                   true, true, true);
695     case AMDGPULibFunc::EI_FLOOR:
696       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::floor, true,
697                                                   true);
698     case AMDGPULibFunc::EI_CEIL:
699       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::ceil, true,
700                                                   true);
701     case AMDGPULibFunc::EI_TRUNC:
702       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::trunc, true,
703                                                   true);
704     case AMDGPULibFunc::EI_RINT:
705       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::rint, true,
706                                                   true);
707     case AMDGPULibFunc::EI_ROUND:
708       return tryReplaceLibcallWithSimpleIntrinsic(B, CI, Intrinsic::round, true,
709                                                   true);
710     case AMDGPULibFunc::EI_LDEXP: {
711       if (!shouldReplaceLibcallWithIntrinsic(CI, true, true))
712         return false;
713 
714       Value *Arg1 = CI->getArgOperand(1);
715       if (VectorType *VecTy = dyn_cast<VectorType>(CI->getType());
716           VecTy && !isa<VectorType>(Arg1->getType())) {
717         Value *SplatArg1 = B.CreateVectorSplat(VecTy->getElementCount(), Arg1);
718         CI->setArgOperand(1, SplatArg1);
719       }
720 
721       CI->setCalledFunction(Intrinsic::getDeclaration(
722           CI->getModule(), Intrinsic::ldexp,
723           {CI->getType(), CI->getArgOperand(1)->getType()}));
724       return true;
725     }
726     case AMDGPULibFunc::EI_POW: {
727       Module *M = Callee->getParent();
728       AMDGPULibFunc PowrInfo(AMDGPULibFunc::EI_POWR, FInfo);
729       FunctionCallee PowrFunc = getFunction(M, PowrInfo);
730       CallInst *Call = cast<CallInst>(FPOp);
731 
732       // pow(x, y) -> powr(x, y) for x >= -0.0
733       // TODO: Account for flags on current call
734       if (PowrFunc &&
735           cannotBeOrderedLessThanZero(FPOp->getOperand(0), M->getDataLayout(),
736                                       TLInfo, 0, AC, Call, DT)) {
737         Call->setCalledFunction(PowrFunc);
738         return fold_pow(FPOp, B, PowrInfo) || true;
739       }
740 
741       return fold_pow(FPOp, B, FInfo);
742     }
743     case AMDGPULibFunc::EI_POWR:
744     case AMDGPULibFunc::EI_POWN:
745       return fold_pow(FPOp, B, FInfo);
746     case AMDGPULibFunc::EI_ROOTN:
747       return fold_rootn(FPOp, B, FInfo);
748     case AMDGPULibFunc::EI_SQRT:
749       return fold_sqrt(FPOp, B, FInfo);
750     case AMDGPULibFunc::EI_COS:
751     case AMDGPULibFunc::EI_SIN:
752       return fold_sincos(FPOp, B, FInfo);
753     default:
754       break;
755     }
756   } else {
757     // Specialized optimizations for each function call
758     switch (FInfo.getId()) {
759     case AMDGPULibFunc::EI_READ_PIPE_2:
760     case AMDGPULibFunc::EI_READ_PIPE_4:
761     case AMDGPULibFunc::EI_WRITE_PIPE_2:
762     case AMDGPULibFunc::EI_WRITE_PIPE_4:
763       return fold_read_write_pipe(CI, B, FInfo);
764     default:
765       break;
766     }
767   }
768 
769   return false;
770 }
771 
772 bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) {
773   // Table-Driven optimization
774   const TableRef tr = getOptTable(FInfo.getId());
775   if (tr.empty())
776     return false;
777 
778   int const sz = (int)tr.size();
779   Value *opr0 = CI->getArgOperand(0);
780 
781   if (getVecSize(FInfo) > 1) {
782     if (ConstantDataVector *CV = dyn_cast<ConstantDataVector>(opr0)) {
783       SmallVector<double, 0> DVal;
784       for (int eltNo = 0; eltNo < getVecSize(FInfo); ++eltNo) {
785         ConstantFP *eltval = dyn_cast<ConstantFP>(
786                                CV->getElementAsConstant((unsigned)eltNo));
787         assert(eltval && "Non-FP arguments in math function!");
788         bool found = false;
789         for (int i=0; i < sz; ++i) {
790           if (eltval->isExactlyValue(tr[i].input)) {
791             DVal.push_back(tr[i].result);
792             found = true;
793             break;
794           }
795         }
796         if (!found) {
797           // This vector constants not handled yet.
798           return false;
799         }
800       }
801       LLVMContext &context = CI->getParent()->getParent()->getContext();
802       Constant *nval;
803       if (getArgType(FInfo) == AMDGPULibFunc::F32) {
804         SmallVector<float, 0> FVal;
805         for (unsigned i = 0; i < DVal.size(); ++i) {
806           FVal.push_back((float)DVal[i]);
807         }
808         ArrayRef<float> tmp(FVal);
809         nval = ConstantDataVector::get(context, tmp);
810       } else { // F64
811         ArrayRef<double> tmp(DVal);
812         nval = ConstantDataVector::get(context, tmp);
813       }
814       LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
815       replaceCall(CI, nval);
816       return true;
817     }
818   } else {
819     // Scalar version
820     if (ConstantFP *CF = dyn_cast<ConstantFP>(opr0)) {
821       for (int i = 0; i < sz; ++i) {
822         if (CF->isExactlyValue(tr[i].input)) {
823           Value *nval = ConstantFP::get(CF->getType(), tr[i].result);
824           LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *nval << "\n");
825           replaceCall(CI, nval);
826           return true;
827         }
828       }
829     }
830   }
831 
832   return false;
833 }
834 
835 namespace llvm {
836 static double log2(double V) {
837 #if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L
838   return ::log2(V);
839 #else
840   return log(V) / numbers::ln2;
841 #endif
842 }
843 }
844 
845 bool AMDGPULibCalls::fold_pow(FPMathOperator *FPOp, IRBuilder<> &B,
846                               const FuncInfo &FInfo) {
847   assert((FInfo.getId() == AMDGPULibFunc::EI_POW ||
848           FInfo.getId() == AMDGPULibFunc::EI_POWR ||
849           FInfo.getId() == AMDGPULibFunc::EI_POWN) &&
850          "fold_pow: encounter a wrong function call");
851 
852   Module *M = B.GetInsertBlock()->getModule();
853   Type *eltType = FPOp->getType()->getScalarType();
854   Value *opr0 = FPOp->getOperand(0);
855   Value *opr1 = FPOp->getOperand(1);
856 
857   const APFloat *CF = nullptr;
858   const APInt *CINT = nullptr;
859   if (!match(opr1, m_APFloatAllowUndef(CF)))
860     match(opr1, m_APIntAllowUndef(CINT));
861 
862   // 0x1111111 means that we don't do anything for this call.
863   int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111);
864 
865   if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0)) {
866     //  pow/powr/pown(x, 0) == 1
867     LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1\n");
868     Constant *cnval = ConstantFP::get(eltType, 1.0);
869     if (getVecSize(FInfo) > 1) {
870       cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
871     }
872     replaceCall(FPOp, cnval);
873     return true;
874   }
875   if ((CF && CF->isExactlyValue(1.0)) || (CINT && ci_opr1 == 1)) {
876     // pow/powr/pown(x, 1.0) = x
877     LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << "\n");
878     replaceCall(FPOp, opr0);
879     return true;
880   }
881   if ((CF && CF->isExactlyValue(2.0)) || (CINT && ci_opr1 == 2)) {
882     // pow/powr/pown(x, 2.0) = x*x
883     LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << " * "
884                       << *opr0 << "\n");
885     Value *nval = B.CreateFMul(opr0, opr0, "__pow2");
886     replaceCall(FPOp, nval);
887     return true;
888   }
889   if ((CF && CF->isExactlyValue(-1.0)) || (CINT && ci_opr1 == -1)) {
890     // pow/powr/pown(x, -1.0) = 1.0/x
891     LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1 / " << *opr0 << "\n");
892     Constant *cnval = ConstantFP::get(eltType, 1.0);
893     if (getVecSize(FInfo) > 1) {
894       cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
895     }
896     Value *nval = B.CreateFDiv(cnval, opr0, "__powrecip");
897     replaceCall(FPOp, nval);
898     return true;
899   }
900 
901   if (CF && (CF->isExactlyValue(0.5) || CF->isExactlyValue(-0.5))) {
902     // pow[r](x, [-]0.5) = sqrt(x)
903     bool issqrt = CF->isExactlyValue(0.5);
904     if (FunctionCallee FPExpr =
905             getFunction(M, AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT
906                                                 : AMDGPULibFunc::EI_RSQRT,
907                                          FInfo))) {
908       LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << FInfo.getName()
909                         << '(' << *opr0 << ")\n");
910       Value *nval = CreateCallEx(B,FPExpr, opr0, issqrt ? "__pow2sqrt"
911                                                         : "__pow2rsqrt");
912       replaceCall(FPOp, nval);
913       return true;
914     }
915   }
916 
917   if (!isUnsafeFiniteOnlyMath(FPOp))
918     return false;
919 
920   // Unsafe Math optimization
921 
922   // Remember that ci_opr1 is set if opr1 is integral
923   if (CF) {
924     double dval = (getArgType(FInfo) == AMDGPULibFunc::F32)
925                       ? (double)CF->convertToFloat()
926                       : CF->convertToDouble();
927     int ival = (int)dval;
928     if ((double)ival == dval) {
929       ci_opr1 = ival;
930     } else
931       ci_opr1 = 0x11111111;
932   }
933 
934   // pow/powr/pown(x, c) = [1/](x*x*..x); where
935   //   trunc(c) == c && the number of x == c && |c| <= 12
936   unsigned abs_opr1 = (ci_opr1 < 0) ? -ci_opr1 : ci_opr1;
937   if (abs_opr1 <= 12) {
938     Constant *cnval;
939     Value *nval;
940     if (abs_opr1 == 0) {
941       cnval = ConstantFP::get(eltType, 1.0);
942       if (getVecSize(FInfo) > 1) {
943         cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
944       }
945       nval = cnval;
946     } else {
947       Value *valx2 = nullptr;
948       nval = nullptr;
949       while (abs_opr1 > 0) {
950         valx2 = valx2 ? B.CreateFMul(valx2, valx2, "__powx2") : opr0;
951         if (abs_opr1 & 1) {
952           nval = nval ? B.CreateFMul(nval, valx2, "__powprod") : valx2;
953         }
954         abs_opr1 >>= 1;
955       }
956     }
957 
958     if (ci_opr1 < 0) {
959       cnval = ConstantFP::get(eltType, 1.0);
960       if (getVecSize(FInfo) > 1) {
961         cnval = ConstantDataVector::getSplat(getVecSize(FInfo), cnval);
962       }
963       nval = B.CreateFDiv(cnval, nval, "__1powprod");
964     }
965     LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> "
966                       << ((ci_opr1 < 0) ? "1/prod(" : "prod(") << *opr0
967                       << ")\n");
968     replaceCall(FPOp, nval);
969     return true;
970   }
971 
972   // powr ---> exp2(y * log2(x))
973   // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31))
974   FunctionCallee ExpExpr =
975       getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo));
976   if (!ExpExpr)
977     return false;
978 
979   bool needlog = false;
980   bool needabs = false;
981   bool needcopysign = false;
982   Constant *cnval = nullptr;
983   if (getVecSize(FInfo) == 1) {
984     CF = nullptr;
985     match(opr0, m_APFloatAllowUndef(CF));
986 
987     if (CF) {
988       double V = (getArgType(FInfo) == AMDGPULibFunc::F32)
989                      ? (double)CF->convertToFloat()
990                      : CF->convertToDouble();
991 
992       V = log2(std::abs(V));
993       cnval = ConstantFP::get(eltType, V);
994       needcopysign = (FInfo.getId() != AMDGPULibFunc::EI_POWR) &&
995                      CF->isNegative();
996     } else {
997       needlog = true;
998       needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR &&
999                                (!CF || CF->isNegative());
1000     }
1001   } else {
1002     ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr0);
1003 
1004     if (!CDV) {
1005       needlog = true;
1006       needcopysign = needabs = FInfo.getId() != AMDGPULibFunc::EI_POWR;
1007     } else {
1008       assert ((int)CDV->getNumElements() == getVecSize(FInfo) &&
1009               "Wrong vector size detected");
1010 
1011       SmallVector<double, 0> DVal;
1012       for (int i=0; i < getVecSize(FInfo); ++i) {
1013         double V = CDV->getElementAsAPFloat(i).convertToDouble();
1014         if (V < 0.0) needcopysign = true;
1015         V = log2(std::abs(V));
1016         DVal.push_back(V);
1017       }
1018       if (getArgType(FInfo) == AMDGPULibFunc::F32) {
1019         SmallVector<float, 0> FVal;
1020         for (unsigned i=0; i < DVal.size(); ++i) {
1021           FVal.push_back((float)DVal[i]);
1022         }
1023         ArrayRef<float> tmp(FVal);
1024         cnval = ConstantDataVector::get(M->getContext(), tmp);
1025       } else {
1026         ArrayRef<double> tmp(DVal);
1027         cnval = ConstantDataVector::get(M->getContext(), tmp);
1028       }
1029     }
1030   }
1031 
1032   if (needcopysign && (FInfo.getId() == AMDGPULibFunc::EI_POW)) {
1033     // We cannot handle corner cases for a general pow() function, give up
1034     // unless y is a constant integral value. Then proceed as if it were pown.
1035     if (!isKnownIntegral(opr1, M->getDataLayout(), FPOp->getFastMathFlags()))
1036       return false;
1037   }
1038 
1039   Value *nval;
1040   if (needabs) {
1041     nval = B.CreateUnaryIntrinsic(Intrinsic::fabs, opr0, nullptr, "__fabs");
1042   } else {
1043     nval = cnval ? cnval : opr0;
1044   }
1045   if (needlog) {
1046     FunctionCallee LogExpr =
1047         getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo));
1048     if (!LogExpr)
1049       return false;
1050     nval = CreateCallEx(B,LogExpr, nval, "__log2");
1051   }
1052 
1053   if (FInfo.getId() == AMDGPULibFunc::EI_POWN) {
1054     // convert int(32) to fp(f32 or f64)
1055     opr1 = B.CreateSIToFP(opr1, nval->getType(), "pownI2F");
1056   }
1057   nval = B.CreateFMul(opr1, nval, "__ylogx");
1058   nval = CreateCallEx(B,ExpExpr, nval, "__exp2");
1059 
1060   if (needcopysign) {
1061     Value *opr_n;
1062     Type* rTy = opr0->getType();
1063     Type* nTyS = B.getIntNTy(eltType->getPrimitiveSizeInBits());
1064     Type *nTy = nTyS;
1065     if (const auto *vTy = dyn_cast<FixedVectorType>(rTy))
1066       nTy = FixedVectorType::get(nTyS, vTy);
1067     unsigned size = nTy->getScalarSizeInBits();
1068     opr_n = FPOp->getOperand(1);
1069     if (opr_n->getType()->isIntegerTy())
1070       opr_n = B.CreateZExtOrTrunc(opr_n, nTy, "__ytou");
1071     else
1072       opr_n = B.CreateFPToSI(opr1, nTy, "__ytou");
1073 
1074     Value *sign = B.CreateShl(opr_n, size-1, "__yeven");
1075     sign = B.CreateAnd(B.CreateBitCast(opr0, nTy), sign, "__pow_sign");
1076     nval = B.CreateOr(B.CreateBitCast(nval, nTy), sign);
1077     nval = B.CreateBitCast(nval, opr0->getType());
1078   }
1079 
1080   LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> "
1081                     << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n");
1082   replaceCall(FPOp, nval);
1083 
1084   return true;
1085 }
1086 
1087 bool AMDGPULibCalls::fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B,
1088                                 const FuncInfo &FInfo) {
1089   // skip vector function
1090   if (getVecSize(FInfo) != 1)
1091     return false;
1092 
1093   Value *opr0 = FPOp->getOperand(0);
1094   Value *opr1 = FPOp->getOperand(1);
1095 
1096   ConstantInt *CINT = dyn_cast<ConstantInt>(opr1);
1097   if (!CINT) {
1098     return false;
1099   }
1100   int ci_opr1 = (int)CINT->getSExtValue();
1101   if (ci_opr1 == 1) {  // rootn(x, 1) = x
1102     LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> " << *opr0 << "\n");
1103     replaceCall(FPOp, opr0);
1104     return true;
1105   }
1106 
1107   Module *M = B.GetInsertBlock()->getModule();
1108   if (ci_opr1 == 2) { // rootn(x, 2) = sqrt(x)
1109     if (FunctionCallee FPExpr =
1110             getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) {
1111       LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> sqrt(" << *opr0
1112                         << ")\n");
1113       Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2sqrt");
1114       replaceCall(FPOp, nval);
1115       return true;
1116     }
1117   } else if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x)
1118     if (FunctionCallee FPExpr =
1119             getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) {
1120       LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> cbrt(" << *opr0
1121                         << ")\n");
1122       Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2cbrt");
1123       replaceCall(FPOp, nval);
1124       return true;
1125     }
1126   } else if (ci_opr1 == -1) { // rootn(x, -1) = 1.0/x
1127     LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> 1.0 / " << *opr0 << "\n");
1128     Value *nval = B.CreateFDiv(ConstantFP::get(opr0->getType(), 1.0),
1129                                opr0,
1130                                "__rootn2div");
1131     replaceCall(FPOp, nval);
1132     return true;
1133   } else if (ci_opr1 == -2) { // rootn(x, -2) = rsqrt(x)
1134     if (FunctionCallee FPExpr =
1135             getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_RSQRT, FInfo))) {
1136       LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> rsqrt(" << *opr0
1137                         << ")\n");
1138       Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2rsqrt");
1139       replaceCall(FPOp, nval);
1140       return true;
1141     }
1142   }
1143   return false;
1144 }
1145 
1146 // Get a scalar native builtin single argument FP function
1147 FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M,
1148                                                  const FuncInfo &FInfo) {
1149   if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()))
1150     return nullptr;
1151   FuncInfo nf = FInfo;
1152   nf.setPrefix(AMDGPULibFunc::NATIVE);
1153   return getFunction(M, nf);
1154 }
1155 
1156 // Some library calls are just wrappers around llvm intrinsics, but compiled
1157 // conservatively. Preserve the flags from the original call site by
1158 // substituting them with direct calls with all the flags.
1159 bool AMDGPULibCalls::shouldReplaceLibcallWithIntrinsic(const CallInst *CI,
1160                                                        bool AllowMinSizeF32,
1161                                                        bool AllowF64,
1162                                                        bool AllowStrictFP) {
1163   Type *FltTy = CI->getType()->getScalarType();
1164   const bool IsF32 = FltTy->isFloatTy();
1165 
1166   // f64 intrinsics aren't implemented for most operations.
1167   if (!IsF32 && !FltTy->isHalfTy() && (!AllowF64 || !FltTy->isDoubleTy()))
1168     return false;
1169 
1170   // We're implicitly inlining by replacing the libcall with the intrinsic, so
1171   // don't do it for noinline call sites.
1172   if (CI->isNoInline())
1173     return false;
1174 
1175   const Function *ParentF = CI->getFunction();
1176   // TODO: Handle strictfp
1177   if (!AllowStrictFP && ParentF->hasFnAttribute(Attribute::StrictFP))
1178     return false;
1179 
1180   if (IsF32 && !AllowMinSizeF32 && ParentF->hasMinSize())
1181     return false;
1182   return true;
1183 }
1184 
1185 void AMDGPULibCalls::replaceLibCallWithSimpleIntrinsic(IRBuilder<> &B,
1186                                                        CallInst *CI,
1187                                                        Intrinsic::ID IntrID) {
1188   if (CI->arg_size() == 2) {
1189     Value *Arg0 = CI->getArgOperand(0);
1190     Value *Arg1 = CI->getArgOperand(1);
1191     VectorType *Arg0VecTy = dyn_cast<VectorType>(Arg0->getType());
1192     VectorType *Arg1VecTy = dyn_cast<VectorType>(Arg1->getType());
1193     if (Arg0VecTy && !Arg1VecTy) {
1194       Value *SplatRHS = B.CreateVectorSplat(Arg0VecTy->getElementCount(), Arg1);
1195       CI->setArgOperand(1, SplatRHS);
1196     } else if (!Arg0VecTy && Arg1VecTy) {
1197       Value *SplatLHS = B.CreateVectorSplat(Arg1VecTy->getElementCount(), Arg0);
1198       CI->setArgOperand(0, SplatLHS);
1199     }
1200   }
1201 
1202   CI->setCalledFunction(
1203       Intrinsic::getDeclaration(CI->getModule(), IntrID, {CI->getType()}));
1204 }
1205 
1206 bool AMDGPULibCalls::tryReplaceLibcallWithSimpleIntrinsic(
1207     IRBuilder<> &B, CallInst *CI, Intrinsic::ID IntrID, bool AllowMinSizeF32,
1208     bool AllowF64, bool AllowStrictFP) {
1209   if (!shouldReplaceLibcallWithIntrinsic(CI, AllowMinSizeF32, AllowF64,
1210                                          AllowStrictFP))
1211     return false;
1212   replaceLibCallWithSimpleIntrinsic(B, CI, IntrID);
1213   return true;
1214 }
1215 
1216 // fold sqrt -> native_sqrt (x)
1217 bool AMDGPULibCalls::fold_sqrt(FPMathOperator *FPOp, IRBuilder<> &B,
1218                                const FuncInfo &FInfo) {
1219   if (!isUnsafeMath(FPOp))
1220     return false;
1221 
1222   if (getArgType(FInfo) == AMDGPULibFunc::F32 && (getVecSize(FInfo) == 1) &&
1223       (FInfo.getPrefix() != AMDGPULibFunc::NATIVE)) {
1224     Module *M = B.GetInsertBlock()->getModule();
1225 
1226     if (FunctionCallee FPExpr = getNativeFunction(
1227             M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) {
1228       Value *opr0 = FPOp->getOperand(0);
1229       LLVM_DEBUG(errs() << "AMDIC: " << *FPOp << " ---> "
1230                         << "sqrt(" << *opr0 << ")\n");
1231       Value *nval = CreateCallEx(B,FPExpr, opr0, "__sqrt");
1232       replaceCall(FPOp, nval);
1233       return true;
1234     }
1235   }
1236   return false;
1237 }
1238 
1239 std::tuple<Value *, Value *, Value *>
1240 AMDGPULibCalls::insertSinCos(Value *Arg, FastMathFlags FMF, IRBuilder<> &B,
1241                              FunctionCallee Fsincos) {
1242   DebugLoc DL = B.getCurrentDebugLocation();
1243   Function *F = B.GetInsertBlock()->getParent();
1244   B.SetInsertPointPastAllocas(F);
1245 
1246   AllocaInst *Alloc = B.CreateAlloca(Arg->getType(), nullptr, "__sincos_");
1247 
1248   if (Instruction *ArgInst = dyn_cast<Instruction>(Arg)) {
1249     // If the argument is an instruction, it must dominate all uses so put our
1250     // sincos call there. Otherwise, right after the allocas works well enough
1251     // if it's an argument or constant.
1252 
1253     B.SetInsertPoint(ArgInst->getParent(), ++ArgInst->getIterator());
1254 
1255     // SetInsertPoint unwelcomely always tries to set the debug loc.
1256     B.SetCurrentDebugLocation(DL);
1257   }
1258 
1259   Type *CosPtrTy = Fsincos.getFunctionType()->getParamType(1);
1260 
1261   // The allocaInst allocates the memory in private address space. This need
1262   // to be addrspacecasted to point to the address space of cos pointer type.
1263   // In OpenCL 2.0 this is generic, while in 1.2 that is private.
1264   Value *CastAlloc = B.CreateAddrSpaceCast(Alloc, CosPtrTy);
1265 
1266   CallInst *SinCos = CreateCallEx2(B, Fsincos, Arg, CastAlloc);
1267 
1268   // TODO: Is it worth trying to preserve the location for the cos calls for the
1269   // load?
1270 
1271   LoadInst *LoadCos = B.CreateLoad(Alloc->getAllocatedType(), Alloc);
1272   return {SinCos, LoadCos, SinCos};
1273 }
1274 
1275 // fold sin, cos -> sincos.
1276 bool AMDGPULibCalls::fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B,
1277                                  const FuncInfo &fInfo) {
1278   assert(fInfo.getId() == AMDGPULibFunc::EI_SIN ||
1279          fInfo.getId() == AMDGPULibFunc::EI_COS);
1280 
1281   if ((getArgType(fInfo) != AMDGPULibFunc::F32 &&
1282        getArgType(fInfo) != AMDGPULibFunc::F64) ||
1283       fInfo.getPrefix() != AMDGPULibFunc::NOPFX)
1284     return false;
1285 
1286   bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN;
1287 
1288   Value *CArgVal = FPOp->getOperand(0);
1289   CallInst *CI = cast<CallInst>(FPOp);
1290 
1291   Function *F = B.GetInsertBlock()->getParent();
1292   Module *M = F->getParent();
1293 
1294   // Merge the sin and cos. For OpenCL 2.0, there may only be a generic pointer
1295   // implementation. Prefer the private form if available.
1296   AMDGPULibFunc SinCosLibFuncPrivate(AMDGPULibFunc::EI_SINCOS, fInfo);
1297   SinCosLibFuncPrivate.getLeads()[0].PtrKind =
1298       AMDGPULibFunc::getEPtrKindFromAddrSpace(AMDGPUAS::PRIVATE_ADDRESS);
1299 
1300   AMDGPULibFunc SinCosLibFuncGeneric(AMDGPULibFunc::EI_SINCOS, fInfo);
1301   SinCosLibFuncGeneric.getLeads()[0].PtrKind =
1302       AMDGPULibFunc::getEPtrKindFromAddrSpace(AMDGPUAS::FLAT_ADDRESS);
1303 
1304   FunctionCallee FSinCosPrivate = getFunction(M, SinCosLibFuncPrivate);
1305   FunctionCallee FSinCosGeneric = getFunction(M, SinCosLibFuncGeneric);
1306   FunctionCallee FSinCos = FSinCosPrivate ? FSinCosPrivate : FSinCosGeneric;
1307   if (!FSinCos)
1308     return false;
1309 
1310   SmallVector<CallInst *> SinCalls;
1311   SmallVector<CallInst *> CosCalls;
1312   SmallVector<CallInst *> SinCosCalls;
1313   FuncInfo PartnerInfo(isSin ? AMDGPULibFunc::EI_COS : AMDGPULibFunc::EI_SIN,
1314                        fInfo);
1315   const std::string PairName = PartnerInfo.mangle();
1316 
1317   StringRef SinName = isSin ? CI->getCalledFunction()->getName() : PairName;
1318   StringRef CosName = isSin ? PairName : CI->getCalledFunction()->getName();
1319   const std::string SinCosPrivateName = SinCosLibFuncPrivate.mangle();
1320   const std::string SinCosGenericName = SinCosLibFuncGeneric.mangle();
1321 
1322   // Intersect the two sets of flags.
1323   FastMathFlags FMF = FPOp->getFastMathFlags();
1324   MDNode *FPMath = CI->getMetadata(LLVMContext::MD_fpmath);
1325 
1326   SmallVector<DILocation *> MergeDbgLocs = {CI->getDebugLoc()};
1327 
1328   for (User* U : CArgVal->users()) {
1329     CallInst *XI = dyn_cast<CallInst>(U);
1330     if (!XI || XI->getFunction() != F || XI->isNoBuiltin())
1331       continue;
1332 
1333     Function *UCallee = XI->getCalledFunction();
1334     if (!UCallee)
1335       continue;
1336 
1337     bool Handled = true;
1338 
1339     if (UCallee->getName() == SinName)
1340       SinCalls.push_back(XI);
1341     else if (UCallee->getName() == CosName)
1342       CosCalls.push_back(XI);
1343     else if (UCallee->getName() == SinCosPrivateName ||
1344              UCallee->getName() == SinCosGenericName)
1345       SinCosCalls.push_back(XI);
1346     else
1347       Handled = false;
1348 
1349     if (Handled) {
1350       MergeDbgLocs.push_back(XI->getDebugLoc());
1351       auto *OtherOp = cast<FPMathOperator>(XI);
1352       FMF &= OtherOp->getFastMathFlags();
1353       FPMath = MDNode::getMostGenericFPMath(
1354           FPMath, XI->getMetadata(LLVMContext::MD_fpmath));
1355     }
1356   }
1357 
1358   if (SinCalls.empty() || CosCalls.empty())
1359     return false;
1360 
1361   B.setFastMathFlags(FMF);
1362   B.setDefaultFPMathTag(FPMath);
1363   DILocation *DbgLoc = DILocation::getMergedLocations(MergeDbgLocs);
1364   B.SetCurrentDebugLocation(DbgLoc);
1365 
1366   auto [Sin, Cos, SinCos] = insertSinCos(CArgVal, FMF, B, FSinCos);
1367 
1368   auto replaceTrigInsts = [](ArrayRef<CallInst *> Calls, Value *Res) {
1369     for (CallInst *C : Calls)
1370       C->replaceAllUsesWith(Res);
1371 
1372     // Leave the other dead instructions to avoid clobbering iterators.
1373   };
1374 
1375   replaceTrigInsts(SinCalls, Sin);
1376   replaceTrigInsts(CosCalls, Cos);
1377   replaceTrigInsts(SinCosCalls, SinCos);
1378 
1379   // It's safe to delete the original now.
1380   CI->eraseFromParent();
1381   return true;
1382 }
1383 
1384 bool AMDGPULibCalls::evaluateScalarMathFunc(const FuncInfo &FInfo, double &Res0,
1385                                             double &Res1, Constant *copr0,
1386                                             Constant *copr1) {
1387   // By default, opr0/opr1/opr3 holds values of float/double type.
1388   // If they are not float/double, each function has to its
1389   // operand separately.
1390   double opr0 = 0.0, opr1 = 0.0;
1391   ConstantFP *fpopr0 = dyn_cast_or_null<ConstantFP>(copr0);
1392   ConstantFP *fpopr1 = dyn_cast_or_null<ConstantFP>(copr1);
1393   if (fpopr0) {
1394     opr0 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1395              ? fpopr0->getValueAPF().convertToDouble()
1396              : (double)fpopr0->getValueAPF().convertToFloat();
1397   }
1398 
1399   if (fpopr1) {
1400     opr1 = (getArgType(FInfo) == AMDGPULibFunc::F64)
1401              ? fpopr1->getValueAPF().convertToDouble()
1402              : (double)fpopr1->getValueAPF().convertToFloat();
1403   }
1404 
1405   switch (FInfo.getId()) {
1406   default : return false;
1407 
1408   case AMDGPULibFunc::EI_ACOS:
1409     Res0 = acos(opr0);
1410     return true;
1411 
1412   case AMDGPULibFunc::EI_ACOSH:
1413     // acosh(x) == log(x + sqrt(x*x - 1))
1414     Res0 = log(opr0 + sqrt(opr0*opr0 - 1.0));
1415     return true;
1416 
1417   case AMDGPULibFunc::EI_ACOSPI:
1418     Res0 = acos(opr0) / MATH_PI;
1419     return true;
1420 
1421   case AMDGPULibFunc::EI_ASIN:
1422     Res0 = asin(opr0);
1423     return true;
1424 
1425   case AMDGPULibFunc::EI_ASINH:
1426     // asinh(x) == log(x + sqrt(x*x + 1))
1427     Res0 = log(opr0 + sqrt(opr0*opr0 + 1.0));
1428     return true;
1429 
1430   case AMDGPULibFunc::EI_ASINPI:
1431     Res0 = asin(opr0) / MATH_PI;
1432     return true;
1433 
1434   case AMDGPULibFunc::EI_ATAN:
1435     Res0 = atan(opr0);
1436     return true;
1437 
1438   case AMDGPULibFunc::EI_ATANH:
1439     // atanh(x) == (log(x+1) - log(x-1))/2;
1440     Res0 = (log(opr0 + 1.0) - log(opr0 - 1.0))/2.0;
1441     return true;
1442 
1443   case AMDGPULibFunc::EI_ATANPI:
1444     Res0 = atan(opr0) / MATH_PI;
1445     return true;
1446 
1447   case AMDGPULibFunc::EI_CBRT:
1448     Res0 = (opr0 < 0.0) ? -pow(-opr0, 1.0/3.0) : pow(opr0, 1.0/3.0);
1449     return true;
1450 
1451   case AMDGPULibFunc::EI_COS:
1452     Res0 = cos(opr0);
1453     return true;
1454 
1455   case AMDGPULibFunc::EI_COSH:
1456     Res0 = cosh(opr0);
1457     return true;
1458 
1459   case AMDGPULibFunc::EI_COSPI:
1460     Res0 = cos(MATH_PI * opr0);
1461     return true;
1462 
1463   case AMDGPULibFunc::EI_EXP:
1464     Res0 = exp(opr0);
1465     return true;
1466 
1467   case AMDGPULibFunc::EI_EXP2:
1468     Res0 = pow(2.0, opr0);
1469     return true;
1470 
1471   case AMDGPULibFunc::EI_EXP10:
1472     Res0 = pow(10.0, opr0);
1473     return true;
1474 
1475   case AMDGPULibFunc::EI_LOG:
1476     Res0 = log(opr0);
1477     return true;
1478 
1479   case AMDGPULibFunc::EI_LOG2:
1480     Res0 = log(opr0) / log(2.0);
1481     return true;
1482 
1483   case AMDGPULibFunc::EI_LOG10:
1484     Res0 = log(opr0) / log(10.0);
1485     return true;
1486 
1487   case AMDGPULibFunc::EI_RSQRT:
1488     Res0 = 1.0 / sqrt(opr0);
1489     return true;
1490 
1491   case AMDGPULibFunc::EI_SIN:
1492     Res0 = sin(opr0);
1493     return true;
1494 
1495   case AMDGPULibFunc::EI_SINH:
1496     Res0 = sinh(opr0);
1497     return true;
1498 
1499   case AMDGPULibFunc::EI_SINPI:
1500     Res0 = sin(MATH_PI * opr0);
1501     return true;
1502 
1503   case AMDGPULibFunc::EI_TAN:
1504     Res0 = tan(opr0);
1505     return true;
1506 
1507   case AMDGPULibFunc::EI_TANH:
1508     Res0 = tanh(opr0);
1509     return true;
1510 
1511   case AMDGPULibFunc::EI_TANPI:
1512     Res0 = tan(MATH_PI * opr0);
1513     return true;
1514 
1515   // two-arg functions
1516   case AMDGPULibFunc::EI_POW:
1517   case AMDGPULibFunc::EI_POWR:
1518     Res0 = pow(opr0, opr1);
1519     return true;
1520 
1521   case AMDGPULibFunc::EI_POWN: {
1522     if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) {
1523       double val = (double)iopr1->getSExtValue();
1524       Res0 = pow(opr0, val);
1525       return true;
1526     }
1527     return false;
1528   }
1529 
1530   case AMDGPULibFunc::EI_ROOTN: {
1531     if (ConstantInt *iopr1 = dyn_cast_or_null<ConstantInt>(copr1)) {
1532       double val = (double)iopr1->getSExtValue();
1533       Res0 = pow(opr0, 1.0 / val);
1534       return true;
1535     }
1536     return false;
1537   }
1538 
1539   // with ptr arg
1540   case AMDGPULibFunc::EI_SINCOS:
1541     Res0 = sin(opr0);
1542     Res1 = cos(opr0);
1543     return true;
1544   }
1545 
1546   return false;
1547 }
1548 
1549 bool AMDGPULibCalls::evaluateCall(CallInst *aCI, const FuncInfo &FInfo) {
1550   int numArgs = (int)aCI->arg_size();
1551   if (numArgs > 3)
1552     return false;
1553 
1554   Constant *copr0 = nullptr;
1555   Constant *copr1 = nullptr;
1556   if (numArgs > 0) {
1557     if ((copr0 = dyn_cast<Constant>(aCI->getArgOperand(0))) == nullptr)
1558       return false;
1559   }
1560 
1561   if (numArgs > 1) {
1562     if ((copr1 = dyn_cast<Constant>(aCI->getArgOperand(1))) == nullptr) {
1563       if (FInfo.getId() != AMDGPULibFunc::EI_SINCOS)
1564         return false;
1565     }
1566   }
1567 
1568   // At this point, all arguments to aCI are constants.
1569 
1570   // max vector size is 16, and sincos will generate two results.
1571   double DVal0[16], DVal1[16];
1572   int FuncVecSize = getVecSize(FInfo);
1573   bool hasTwoResults = (FInfo.getId() == AMDGPULibFunc::EI_SINCOS);
1574   if (FuncVecSize == 1) {
1575     if (!evaluateScalarMathFunc(FInfo, DVal0[0], DVal1[0], copr0, copr1)) {
1576       return false;
1577     }
1578   } else {
1579     ConstantDataVector *CDV0 = dyn_cast_or_null<ConstantDataVector>(copr0);
1580     ConstantDataVector *CDV1 = dyn_cast_or_null<ConstantDataVector>(copr1);
1581     for (int i = 0; i < FuncVecSize; ++i) {
1582       Constant *celt0 = CDV0 ? CDV0->getElementAsConstant(i) : nullptr;
1583       Constant *celt1 = CDV1 ? CDV1->getElementAsConstant(i) : nullptr;
1584       if (!evaluateScalarMathFunc(FInfo, DVal0[i], DVal1[i], celt0, celt1)) {
1585         return false;
1586       }
1587     }
1588   }
1589 
1590   LLVMContext &context = aCI->getContext();
1591   Constant *nval0, *nval1;
1592   if (FuncVecSize == 1) {
1593     nval0 = ConstantFP::get(aCI->getType(), DVal0[0]);
1594     if (hasTwoResults)
1595       nval1 = ConstantFP::get(aCI->getType(), DVal1[0]);
1596   } else {
1597     if (getArgType(FInfo) == AMDGPULibFunc::F32) {
1598       SmallVector <float, 0> FVal0, FVal1;
1599       for (int i = 0; i < FuncVecSize; ++i)
1600         FVal0.push_back((float)DVal0[i]);
1601       ArrayRef<float> tmp0(FVal0);
1602       nval0 = ConstantDataVector::get(context, tmp0);
1603       if (hasTwoResults) {
1604         for (int i = 0; i < FuncVecSize; ++i)
1605           FVal1.push_back((float)DVal1[i]);
1606         ArrayRef<float> tmp1(FVal1);
1607         nval1 = ConstantDataVector::get(context, tmp1);
1608       }
1609     } else {
1610       ArrayRef<double> tmp0(DVal0);
1611       nval0 = ConstantDataVector::get(context, tmp0);
1612       if (hasTwoResults) {
1613         ArrayRef<double> tmp1(DVal1);
1614         nval1 = ConstantDataVector::get(context, tmp1);
1615       }
1616     }
1617   }
1618 
1619   if (hasTwoResults) {
1620     // sincos
1621     assert(FInfo.getId() == AMDGPULibFunc::EI_SINCOS &&
1622            "math function with ptr arg not supported yet");
1623     new StoreInst(nval1, aCI->getArgOperand(1), aCI);
1624   }
1625 
1626   replaceCall(aCI, nval0);
1627   return true;
1628 }
1629 
1630 PreservedAnalyses AMDGPUSimplifyLibCallsPass::run(Function &F,
1631                                                   FunctionAnalysisManager &AM) {
1632   AMDGPULibCalls Simplifier;
1633   Simplifier.initNativeFuncs();
1634   Simplifier.initFunction(F, AM);
1635 
1636   bool Changed = false;
1637 
1638   LLVM_DEBUG(dbgs() << "AMDIC: process function ";
1639              F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';);
1640 
1641   for (auto &BB : F) {
1642     for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) {
1643       // Ignore non-calls.
1644       CallInst *CI = dyn_cast<CallInst>(I);
1645       ++I;
1646 
1647       if (CI) {
1648         if (Simplifier.fold(CI))
1649           Changed = true;
1650       }
1651     }
1652   }
1653   return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
1654 }
1655 
1656 PreservedAnalyses AMDGPUUseNativeCallsPass::run(Function &F,
1657                                                 FunctionAnalysisManager &AM) {
1658   if (UseNative.empty())
1659     return PreservedAnalyses::all();
1660 
1661   AMDGPULibCalls Simplifier;
1662   Simplifier.initNativeFuncs();
1663   Simplifier.initFunction(F, AM);
1664 
1665   bool Changed = false;
1666   for (auto &BB : F) {
1667     for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) {
1668       // Ignore non-calls.
1669       CallInst *CI = dyn_cast<CallInst>(I);
1670       ++I;
1671       if (CI && Simplifier.useNative(CI))
1672         Changed = true;
1673     }
1674   }
1675   return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
1676 }
1677