xref: /llvm-project/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp (revision 04377a81aee14fc8807aad90b0c1fdfdcb4a3883)
1 //===-- PPCTargetTransformInfo.cpp - PPC specific TTI ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "PPCTargetTransformInfo.h"
10 #include "llvm/Analysis/CodeMetrics.h"
11 #include "llvm/Analysis/TargetTransformInfo.h"
12 #include "llvm/CodeGen/BasicTTIImpl.h"
13 #include "llvm/CodeGen/CostTable.h"
14 #include "llvm/CodeGen/TargetLowering.h"
15 #include "llvm/CodeGen/TargetSchedule.h"
16 #include "llvm/Support/CommandLine.h"
17 #include "llvm/Support/Debug.h"
18 using namespace llvm;
19 
20 #define DEBUG_TYPE "ppctti"
21 
22 static cl::opt<bool> DisablePPCConstHoist("disable-ppc-constant-hoisting",
23 cl::desc("disable constant hoisting on PPC"), cl::init(false), cl::Hidden);
24 
25 // This is currently only used for the data prefetch pass which is only enabled
26 // for BG/Q by default.
27 static cl::opt<unsigned>
28 CacheLineSize("ppc-loop-prefetch-cache-line", cl::Hidden, cl::init(64),
29               cl::desc("The loop prefetch cache line size"));
30 
31 static cl::opt<bool>
32 EnablePPCColdCC("ppc-enable-coldcc", cl::Hidden, cl::init(false),
33                 cl::desc("Enable using coldcc calling conv for cold "
34                          "internal functions"));
35 
36 static cl::opt<bool>
37 LsrNoInsnsCost("ppc-lsr-no-insns-cost", cl::Hidden, cl::init(false),
38                cl::desc("Do not add instruction count to lsr cost model"));
39 
40 // The latency of mtctr is only justified if there are more than 4
41 // comparisons that will be removed as a result.
42 static cl::opt<unsigned>
43 SmallCTRLoopThreshold("min-ctr-loop-threshold", cl::init(4), cl::Hidden,
44                       cl::desc("Loops with a constant trip count smaller than "
45                                "this value will not use the count register."));
46 
47 //===----------------------------------------------------------------------===//
48 //
49 // PPC cost model.
50 //
51 //===----------------------------------------------------------------------===//
52 
53 TargetTransformInfo::PopcntSupportKind
54 PPCTTIImpl::getPopcntSupport(unsigned TyWidth) {
55   assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
56   if (ST->hasPOPCNTD() != PPCSubtarget::POPCNTD_Unavailable && TyWidth <= 64)
57     return ST->hasPOPCNTD() == PPCSubtarget::POPCNTD_Slow ?
58              TTI::PSK_SlowHardware : TTI::PSK_FastHardware;
59   return TTI::PSK_Software;
60 }
61 
62 int PPCTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty) {
63   if (DisablePPCConstHoist)
64     return BaseT::getIntImmCost(Imm, Ty);
65 
66   assert(Ty->isIntegerTy());
67 
68   unsigned BitSize = Ty->getPrimitiveSizeInBits();
69   if (BitSize == 0)
70     return ~0U;
71 
72   if (Imm == 0)
73     return TTI::TCC_Free;
74 
75   if (Imm.getBitWidth() <= 64) {
76     if (isInt<16>(Imm.getSExtValue()))
77       return TTI::TCC_Basic;
78 
79     if (isInt<32>(Imm.getSExtValue())) {
80       // A constant that can be materialized using lis.
81       if ((Imm.getZExtValue() & 0xFFFF) == 0)
82         return TTI::TCC_Basic;
83 
84       return 2 * TTI::TCC_Basic;
85     }
86   }
87 
88   return 4 * TTI::TCC_Basic;
89 }
90 
91 int PPCTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
92                                     const APInt &Imm, Type *Ty) {
93   if (DisablePPCConstHoist)
94     return BaseT::getIntImmCostIntrin(IID, Idx, Imm, Ty);
95 
96   assert(Ty->isIntegerTy());
97 
98   unsigned BitSize = Ty->getPrimitiveSizeInBits();
99   if (BitSize == 0)
100     return ~0U;
101 
102   switch (IID) {
103   default:
104     return TTI::TCC_Free;
105   case Intrinsic::sadd_with_overflow:
106   case Intrinsic::uadd_with_overflow:
107   case Intrinsic::ssub_with_overflow:
108   case Intrinsic::usub_with_overflow:
109     if ((Idx == 1) && Imm.getBitWidth() <= 64 && isInt<16>(Imm.getSExtValue()))
110       return TTI::TCC_Free;
111     break;
112   case Intrinsic::experimental_stackmap:
113     if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
114       return TTI::TCC_Free;
115     break;
116   case Intrinsic::experimental_patchpoint_void:
117   case Intrinsic::experimental_patchpoint_i64:
118     if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
119       return TTI::TCC_Free;
120     break;
121   }
122   return PPCTTIImpl::getIntImmCost(Imm, Ty);
123 }
124 
125 int PPCTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
126                                   const APInt &Imm, Type *Ty) {
127   if (DisablePPCConstHoist)
128     return BaseT::getIntImmCostInst(Opcode, Idx, Imm, Ty);
129 
130   assert(Ty->isIntegerTy());
131 
132   unsigned BitSize = Ty->getPrimitiveSizeInBits();
133   if (BitSize == 0)
134     return ~0U;
135 
136   unsigned ImmIdx = ~0U;
137   bool ShiftedFree = false, RunFree = false, UnsignedFree = false,
138        ZeroFree = false;
139   switch (Opcode) {
140   default:
141     return TTI::TCC_Free;
142   case Instruction::GetElementPtr:
143     // Always hoist the base address of a GetElementPtr. This prevents the
144     // creation of new constants for every base constant that gets constant
145     // folded with the offset.
146     if (Idx == 0)
147       return 2 * TTI::TCC_Basic;
148     return TTI::TCC_Free;
149   case Instruction::And:
150     RunFree = true; // (for the rotate-and-mask instructions)
151     LLVM_FALLTHROUGH;
152   case Instruction::Add:
153   case Instruction::Or:
154   case Instruction::Xor:
155     ShiftedFree = true;
156     LLVM_FALLTHROUGH;
157   case Instruction::Sub:
158   case Instruction::Mul:
159   case Instruction::Shl:
160   case Instruction::LShr:
161   case Instruction::AShr:
162     ImmIdx = 1;
163     break;
164   case Instruction::ICmp:
165     UnsignedFree = true;
166     ImmIdx = 1;
167     // Zero comparisons can use record-form instructions.
168     LLVM_FALLTHROUGH;
169   case Instruction::Select:
170     ZeroFree = true;
171     break;
172   case Instruction::PHI:
173   case Instruction::Call:
174   case Instruction::Ret:
175   case Instruction::Load:
176   case Instruction::Store:
177     break;
178   }
179 
180   if (ZeroFree && Imm == 0)
181     return TTI::TCC_Free;
182 
183   if (Idx == ImmIdx && Imm.getBitWidth() <= 64) {
184     if (isInt<16>(Imm.getSExtValue()))
185       return TTI::TCC_Free;
186 
187     if (RunFree) {
188       if (Imm.getBitWidth() <= 32 &&
189           (isShiftedMask_32(Imm.getZExtValue()) ||
190            isShiftedMask_32(~Imm.getZExtValue())))
191         return TTI::TCC_Free;
192 
193       if (ST->isPPC64() &&
194           (isShiftedMask_64(Imm.getZExtValue()) ||
195            isShiftedMask_64(~Imm.getZExtValue())))
196         return TTI::TCC_Free;
197     }
198 
199     if (UnsignedFree && isUInt<16>(Imm.getZExtValue()))
200       return TTI::TCC_Free;
201 
202     if (ShiftedFree && (Imm.getZExtValue() & 0xFFFF) == 0)
203       return TTI::TCC_Free;
204   }
205 
206   return PPCTTIImpl::getIntImmCost(Imm, Ty);
207 }
208 
209 unsigned PPCTTIImpl::getUserCost(const User *U,
210                                  ArrayRef<const Value *> Operands) {
211   if (U->getType()->isVectorTy()) {
212     // Instructions that need to be split should cost more.
213     std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, U->getType());
214     return LT.first * BaseT::getUserCost(U, Operands);
215   }
216 
217   return BaseT::getUserCost(U, Operands);
218 }
219 
220 bool PPCTTIImpl::mightUseCTR(BasicBlock *BB,
221                              TargetLibraryInfo *LibInfo) {
222   const PPCTargetMachine &TM = ST->getTargetMachine();
223 
224   // Loop through the inline asm constraints and look for something that
225   // clobbers ctr.
226   auto asmClobbersCTR = [](InlineAsm *IA) {
227     InlineAsm::ConstraintInfoVector CIV = IA->ParseConstraints();
228     for (unsigned i = 0, ie = CIV.size(); i < ie; ++i) {
229       InlineAsm::ConstraintInfo &C = CIV[i];
230       if (C.Type != InlineAsm::isInput)
231         for (unsigned j = 0, je = C.Codes.size(); j < je; ++j)
232           if (StringRef(C.Codes[j]).equals_lower("{ctr}"))
233             return true;
234     }
235     return false;
236   };
237 
238   // Determining the address of a TLS variable results in a function call in
239   // certain TLS models.
240   std::function<bool(const Value*)> memAddrUsesCTR =
241     [&memAddrUsesCTR, &TM](const Value *MemAddr) -> bool {
242     const auto *GV = dyn_cast<GlobalValue>(MemAddr);
243     if (!GV) {
244       // Recurse to check for constants that refer to TLS global variables.
245       if (const auto *CV = dyn_cast<Constant>(MemAddr))
246         for (const auto &CO : CV->operands())
247           if (memAddrUsesCTR(CO))
248             return true;
249 
250       return false;
251     }
252 
253     if (!GV->isThreadLocal())
254       return false;
255     TLSModel::Model Model = TM.getTLSModel(GV);
256     return Model == TLSModel::GeneralDynamic ||
257       Model == TLSModel::LocalDynamic;
258   };
259 
260   auto isLargeIntegerTy = [](bool Is32Bit, Type *Ty) {
261     if (IntegerType *ITy = dyn_cast<IntegerType>(Ty))
262       return ITy->getBitWidth() > (Is32Bit ? 32U : 64U);
263 
264     return false;
265   };
266 
267   for (BasicBlock::iterator J = BB->begin(), JE = BB->end();
268        J != JE; ++J) {
269     if (CallInst *CI = dyn_cast<CallInst>(J)) {
270       // Inline ASM is okay, unless it clobbers the ctr register.
271       if (InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledValue())) {
272         if (asmClobbersCTR(IA))
273           return true;
274         continue;
275       }
276 
277       if (Function *F = CI->getCalledFunction()) {
278         // Most intrinsics don't become function calls, but some might.
279         // sin, cos, exp and log are always calls.
280         unsigned Opcode = 0;
281         if (F->getIntrinsicID() != Intrinsic::not_intrinsic) {
282           switch (F->getIntrinsicID()) {
283           default: continue;
284           // If we have a call to ppc_is_decremented_ctr_nonzero, or ppc_mtctr
285           // we're definitely using CTR.
286           case Intrinsic::set_loop_iterations:
287           case Intrinsic::loop_decrement:
288             return true;
289 
290           // Exclude eh_sjlj_setjmp; we don't need to exclude eh_sjlj_longjmp
291           // because, although it does clobber the counter register, the
292           // control can't then return to inside the loop unless there is also
293           // an eh_sjlj_setjmp.
294           case Intrinsic::eh_sjlj_setjmp:
295 
296           case Intrinsic::memcpy:
297           case Intrinsic::memmove:
298           case Intrinsic::memset:
299           case Intrinsic::powi:
300           case Intrinsic::log:
301           case Intrinsic::log2:
302           case Intrinsic::log10:
303           case Intrinsic::exp:
304           case Intrinsic::exp2:
305           case Intrinsic::pow:
306           case Intrinsic::sin:
307           case Intrinsic::cos:
308             return true;
309           case Intrinsic::copysign:
310             if (CI->getArgOperand(0)->getType()->getScalarType()->
311                 isPPC_FP128Ty())
312               return true;
313             else
314               continue; // ISD::FCOPYSIGN is never a library call.
315           case Intrinsic::sqrt:               Opcode = ISD::FSQRT;      break;
316           case Intrinsic::floor:              Opcode = ISD::FFLOOR;     break;
317           case Intrinsic::ceil:               Opcode = ISD::FCEIL;      break;
318           case Intrinsic::trunc:              Opcode = ISD::FTRUNC;     break;
319           case Intrinsic::rint:               Opcode = ISD::FRINT;      break;
320           case Intrinsic::lrint:              Opcode = ISD::LRINT;      break;
321           case Intrinsic::llrint:             Opcode = ISD::LLRINT;     break;
322           case Intrinsic::nearbyint:          Opcode = ISD::FNEARBYINT; break;
323           case Intrinsic::round:              Opcode = ISD::FROUND;     break;
324           case Intrinsic::lround:             Opcode = ISD::LROUND;     break;
325           case Intrinsic::llround:            Opcode = ISD::LLROUND;    break;
326           case Intrinsic::minnum:             Opcode = ISD::FMINNUM;    break;
327           case Intrinsic::maxnum:             Opcode = ISD::FMAXNUM;    break;
328           case Intrinsic::umul_with_overflow: Opcode = ISD::UMULO;      break;
329           case Intrinsic::smul_with_overflow: Opcode = ISD::SMULO;      break;
330           }
331         }
332 
333         // PowerPC does not use [US]DIVREM or other library calls for
334         // operations on regular types which are not otherwise library calls
335         // (i.e. soft float or atomics). If adapting for targets that do,
336         // additional care is required here.
337 
338         LibFunc Func;
339         if (!F->hasLocalLinkage() && F->hasName() && LibInfo &&
340             LibInfo->getLibFunc(F->getName(), Func) &&
341             LibInfo->hasOptimizedCodeGen(Func)) {
342           // Non-read-only functions are never treated as intrinsics.
343           if (!CI->onlyReadsMemory())
344             return true;
345 
346           // Conversion happens only for FP calls.
347           if (!CI->getArgOperand(0)->getType()->isFloatingPointTy())
348             return true;
349 
350           switch (Func) {
351           default: return true;
352           case LibFunc_copysign:
353           case LibFunc_copysignf:
354             continue; // ISD::FCOPYSIGN is never a library call.
355           case LibFunc_copysignl:
356             return true;
357           case LibFunc_fabs:
358           case LibFunc_fabsf:
359           case LibFunc_fabsl:
360             continue; // ISD::FABS is never a library call.
361           case LibFunc_sqrt:
362           case LibFunc_sqrtf:
363           case LibFunc_sqrtl:
364             Opcode = ISD::FSQRT; break;
365           case LibFunc_floor:
366           case LibFunc_floorf:
367           case LibFunc_floorl:
368             Opcode = ISD::FFLOOR; break;
369           case LibFunc_nearbyint:
370           case LibFunc_nearbyintf:
371           case LibFunc_nearbyintl:
372             Opcode = ISD::FNEARBYINT; break;
373           case LibFunc_ceil:
374           case LibFunc_ceilf:
375           case LibFunc_ceill:
376             Opcode = ISD::FCEIL; break;
377           case LibFunc_rint:
378           case LibFunc_rintf:
379           case LibFunc_rintl:
380             Opcode = ISD::FRINT; break;
381           case LibFunc_round:
382           case LibFunc_roundf:
383           case LibFunc_roundl:
384             Opcode = ISD::FROUND; break;
385           case LibFunc_trunc:
386           case LibFunc_truncf:
387           case LibFunc_truncl:
388             Opcode = ISD::FTRUNC; break;
389           case LibFunc_fmin:
390           case LibFunc_fminf:
391           case LibFunc_fminl:
392             Opcode = ISD::FMINNUM; break;
393           case LibFunc_fmax:
394           case LibFunc_fmaxf:
395           case LibFunc_fmaxl:
396             Opcode = ISD::FMAXNUM; break;
397           }
398         }
399 
400         if (Opcode) {
401           EVT EVTy =
402               TLI->getValueType(DL, CI->getArgOperand(0)->getType(), true);
403 
404           if (EVTy == MVT::Other)
405             return true;
406 
407           if (TLI->isOperationLegalOrCustom(Opcode, EVTy))
408             continue;
409           else if (EVTy.isVector() &&
410                    TLI->isOperationLegalOrCustom(Opcode, EVTy.getScalarType()))
411             continue;
412 
413           return true;
414         }
415       }
416 
417       return true;
418     } else if (isa<BinaryOperator>(J) &&
419                J->getType()->getScalarType()->isPPC_FP128Ty()) {
420       // Most operations on ppc_f128 values become calls.
421       return true;
422     } else if (isa<UIToFPInst>(J) || isa<SIToFPInst>(J) ||
423                isa<FPToUIInst>(J) || isa<FPToSIInst>(J)) {
424       CastInst *CI = cast<CastInst>(J);
425       if (CI->getSrcTy()->getScalarType()->isPPC_FP128Ty() ||
426           CI->getDestTy()->getScalarType()->isPPC_FP128Ty() ||
427           isLargeIntegerTy(!TM.isPPC64(), CI->getSrcTy()->getScalarType()) ||
428           isLargeIntegerTy(!TM.isPPC64(), CI->getDestTy()->getScalarType()))
429         return true;
430     } else if (isLargeIntegerTy(!TM.isPPC64(),
431                                 J->getType()->getScalarType()) &&
432                (J->getOpcode() == Instruction::UDiv ||
433                 J->getOpcode() == Instruction::SDiv ||
434                 J->getOpcode() == Instruction::URem ||
435                 J->getOpcode() == Instruction::SRem)) {
436       return true;
437     } else if (!TM.isPPC64() &&
438                isLargeIntegerTy(false, J->getType()->getScalarType()) &&
439                (J->getOpcode() == Instruction::Shl ||
440                 J->getOpcode() == Instruction::AShr ||
441                 J->getOpcode() == Instruction::LShr)) {
442       // Only on PPC32, for 128-bit integers (specifically not 64-bit
443       // integers), these might be runtime calls.
444       return true;
445     } else if (isa<IndirectBrInst>(J) || isa<InvokeInst>(J)) {
446       // On PowerPC, indirect jumps use the counter register.
447       return true;
448     } else if (SwitchInst *SI = dyn_cast<SwitchInst>(J)) {
449       if (SI->getNumCases() + 1 >= (unsigned)TLI->getMinimumJumpTableEntries())
450         return true;
451     }
452 
453     // FREM is always a call.
454     if (J->getOpcode() == Instruction::FRem)
455       return true;
456 
457     if (ST->useSoftFloat()) {
458       switch(J->getOpcode()) {
459       case Instruction::FAdd:
460       case Instruction::FSub:
461       case Instruction::FMul:
462       case Instruction::FDiv:
463       case Instruction::FPTrunc:
464       case Instruction::FPExt:
465       case Instruction::FPToUI:
466       case Instruction::FPToSI:
467       case Instruction::UIToFP:
468       case Instruction::SIToFP:
469       case Instruction::FCmp:
470         return true;
471       }
472     }
473 
474     for (Value *Operand : J->operands())
475       if (memAddrUsesCTR(Operand))
476         return true;
477   }
478 
479   return false;
480 }
481 
482 bool PPCTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
483                                           AssumptionCache &AC,
484                                           TargetLibraryInfo *LibInfo,
485                                           HardwareLoopInfo &HWLoopInfo) {
486   const PPCTargetMachine &TM = ST->getTargetMachine();
487   TargetSchedModel SchedModel;
488   SchedModel.init(ST);
489 
490   // Do not convert small short loops to CTR loop.
491   unsigned ConstTripCount = SE.getSmallConstantTripCount(L);
492   if (ConstTripCount && ConstTripCount < SmallCTRLoopThreshold) {
493     SmallPtrSet<const Value *, 32> EphValues;
494     CodeMetrics::collectEphemeralValues(L, &AC, EphValues);
495     CodeMetrics Metrics;
496     for (BasicBlock *BB : L->blocks())
497       Metrics.analyzeBasicBlock(BB, *this, EphValues);
498     // 6 is an approximate latency for the mtctr instruction.
499     if (Metrics.NumInsts <= (6 * SchedModel.getIssueWidth()))
500       return false;
501   }
502 
503   // We don't want to spill/restore the counter register, and so we don't
504   // want to use the counter register if the loop contains calls.
505   for (Loop::block_iterator I = L->block_begin(), IE = L->block_end();
506        I != IE; ++I)
507     if (mightUseCTR(*I, LibInfo))
508       return false;
509 
510   SmallVector<BasicBlock*, 4> ExitingBlocks;
511   L->getExitingBlocks(ExitingBlocks);
512 
513   // If there is an exit edge known to be frequently taken,
514   // we should not transform this loop.
515   for (auto &BB : ExitingBlocks) {
516     Instruction *TI = BB->getTerminator();
517     if (!TI) continue;
518 
519     if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
520       uint64_t TrueWeight = 0, FalseWeight = 0;
521       if (!BI->isConditional() ||
522           !BI->extractProfMetadata(TrueWeight, FalseWeight))
523         continue;
524 
525       // If the exit path is more frequent than the loop path,
526       // we return here without further analysis for this loop.
527       bool TrueIsExit = !L->contains(BI->getSuccessor(0));
528       if (( TrueIsExit && FalseWeight < TrueWeight) ||
529           (!TrueIsExit && FalseWeight > TrueWeight))
530         return false;
531     }
532   }
533 
534   LLVMContext &C = L->getHeader()->getContext();
535   HWLoopInfo.CountType = TM.isPPC64() ?
536     Type::getInt64Ty(C) : Type::getInt32Ty(C);
537   HWLoopInfo.LoopDecrement = ConstantInt::get(HWLoopInfo.CountType, 1);
538   return true;
539 }
540 
541 void PPCTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
542                                          TTI::UnrollingPreferences &UP) {
543   if (ST->getCPUDirective() == PPC::DIR_A2) {
544     // The A2 is in-order with a deep pipeline, and concatenation unrolling
545     // helps expose latency-hiding opportunities to the instruction scheduler.
546     UP.Partial = UP.Runtime = true;
547 
548     // We unroll a lot on the A2 (hundreds of instructions), and the benefits
549     // often outweigh the cost of a division to compute the trip count.
550     UP.AllowExpensiveTripCount = true;
551   }
552 
553   BaseT::getUnrollingPreferences(L, SE, UP);
554 }
555 
556 // This function returns true to allow using coldcc calling convention.
557 // Returning true results in coldcc being used for functions which are cold at
558 // all call sites when the callers of the functions are not calling any other
559 // non coldcc functions.
560 bool PPCTTIImpl::useColdCCForColdCall(Function &F) {
561   return EnablePPCColdCC;
562 }
563 
564 bool PPCTTIImpl::enableAggressiveInterleaving(bool LoopHasReductions) {
565   // On the A2, always unroll aggressively. For QPX unaligned loads, we depend
566   // on combining the loads generated for consecutive accesses, and failure to
567   // do so is particularly expensive. This makes it much more likely (compared
568   // to only using concatenation unrolling).
569   if (ST->getCPUDirective() == PPC::DIR_A2)
570     return true;
571 
572   return LoopHasReductions;
573 }
574 
575 PPCTTIImpl::TTI::MemCmpExpansionOptions
576 PPCTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
577   TTI::MemCmpExpansionOptions Options;
578   Options.LoadSizes = {8, 4, 2, 1};
579   Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
580   return Options;
581 }
582 
583 bool PPCTTIImpl::enableInterleavedAccessVectorization() {
584   return true;
585 }
586 
587 unsigned PPCTTIImpl::getNumberOfRegisters(unsigned ClassID) const {
588   assert(ClassID == GPRRC || ClassID == FPRRC ||
589          ClassID == VRRC || ClassID == VSXRC);
590   if (ST->hasVSX()) {
591     assert(ClassID == GPRRC || ClassID == VSXRC || ClassID == VRRC);
592     return ClassID == VSXRC ? 64 : 32;
593   }
594   assert(ClassID == GPRRC || ClassID == FPRRC || ClassID == VRRC);
595   return 32;
596 }
597 
598 unsigned PPCTTIImpl::getRegisterClassForType(bool Vector, Type *Ty) const {
599   if (Vector)
600     return ST->hasVSX() ? VSXRC : VRRC;
601   else if (Ty && (Ty->getScalarType()->isFloatTy() ||
602                   Ty->getScalarType()->isDoubleTy()))
603     return ST->hasVSX() ? VSXRC : FPRRC;
604   else if (Ty && (Ty->getScalarType()->isFP128Ty() ||
605                   Ty->getScalarType()->isPPC_FP128Ty()))
606     return VRRC;
607   else if (Ty && Ty->getScalarType()->isHalfTy())
608     return VSXRC;
609   else
610     return GPRRC;
611 }
612 
613 const char* PPCTTIImpl::getRegisterClassName(unsigned ClassID) const {
614 
615   switch (ClassID) {
616     default:
617       llvm_unreachable("unknown register class");
618       return "PPC::unknown register class";
619     case GPRRC:       return "PPC::GPRRC";
620     case FPRRC:       return "PPC::FPRRC";
621     case VRRC:        return "PPC::VRRC";
622     case VSXRC:       return "PPC::VSXRC";
623   }
624 }
625 
626 unsigned PPCTTIImpl::getRegisterBitWidth(bool Vector) const {
627   if (Vector) {
628     if (ST->hasQPX()) return 256;
629     if (ST->hasAltivec()) return 128;
630     return 0;
631   }
632 
633   if (ST->isPPC64())
634     return 64;
635   return 32;
636 
637 }
638 
639 unsigned PPCTTIImpl::getCacheLineSize() const {
640   // Check first if the user specified a custom line size.
641   if (CacheLineSize.getNumOccurrences() > 0)
642     return CacheLineSize;
643 
644   // On P7, P8 or P9 we have a cache line size of 128.
645   unsigned Directive = ST->getCPUDirective();
646   // Assume that Future CPU has the same cache line size as the others.
647   if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8 ||
648       Directive == PPC::DIR_PWR9 || Directive == PPC::DIR_PWR_FUTURE)
649     return 128;
650 
651   // On other processors return a default of 64 bytes.
652   return 64;
653 }
654 
655 unsigned PPCTTIImpl::getPrefetchDistance() const {
656   // This seems like a reasonable default for the BG/Q (this pass is enabled, by
657   // default, only on the BG/Q).
658   return 300;
659 }
660 
661 unsigned PPCTTIImpl::getMaxInterleaveFactor(unsigned VF) {
662   unsigned Directive = ST->getCPUDirective();
663   // The 440 has no SIMD support, but floating-point instructions
664   // have a 5-cycle latency, so unroll by 5x for latency hiding.
665   if (Directive == PPC::DIR_440)
666     return 5;
667 
668   // The A2 has no SIMD support, but floating-point instructions
669   // have a 6-cycle latency, so unroll by 6x for latency hiding.
670   if (Directive == PPC::DIR_A2)
671     return 6;
672 
673   // FIXME: For lack of any better information, do no harm...
674   if (Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500)
675     return 1;
676 
677   // For P7 and P8, floating-point instructions have a 6-cycle latency and
678   // there are two execution units, so unroll by 12x for latency hiding.
679   // FIXME: the same for P9 as previous gen until POWER9 scheduling is ready
680   // Assume that future is the same as the others.
681   if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8 ||
682       Directive == PPC::DIR_PWR9 || Directive == PPC::DIR_PWR_FUTURE)
683     return 12;
684 
685   // For most things, modern systems have two execution units (and
686   // out-of-order execution).
687   return 2;
688 }
689 
690 // Adjust the cost of vector instructions on targets which there is overlap
691 // between the vector and scalar units, thereby reducing the overall throughput
692 // of vector code wrt. scalar code.
693 int PPCTTIImpl::vectorCostAdjustment(int Cost, unsigned Opcode, Type *Ty1,
694                                      Type *Ty2) {
695   if (!ST->vectorsUseTwoUnits() || !Ty1->isVectorTy())
696     return Cost;
697 
698   std::pair<int, MVT> LT1 = TLI->getTypeLegalizationCost(DL, Ty1);
699   // If type legalization involves splitting the vector, we don't want to
700   // double the cost at every step - only the last step.
701   if (LT1.first != 1 || !LT1.second.isVector())
702     return Cost;
703 
704   int ISD = TLI->InstructionOpcodeToISD(Opcode);
705   if (TLI->isOperationExpand(ISD, LT1.second))
706     return Cost;
707 
708   if (Ty2) {
709     std::pair<int, MVT> LT2 = TLI->getTypeLegalizationCost(DL, Ty2);
710     if (LT2.first != 1 || !LT2.second.isVector())
711       return Cost;
712   }
713 
714   return Cost * 2;
715 }
716 
717 int PPCTTIImpl::getArithmeticInstrCost(unsigned Opcode, Type *Ty,
718                                        TTI::OperandValueKind Op1Info,
719                                        TTI::OperandValueKind Op2Info,
720                                        TTI::OperandValueProperties Opd1PropInfo,
721                                        TTI::OperandValueProperties Opd2PropInfo,
722                                        ArrayRef<const Value *> Args,
723                                        const Instruction *CxtI) {
724   assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode");
725 
726   // Fallback to the default implementation.
727   int Cost = BaseT::getArithmeticInstrCost(Opcode, Ty, Op1Info, Op2Info,
728                                            Opd1PropInfo, Opd2PropInfo);
729   return vectorCostAdjustment(Cost, Opcode, Ty, nullptr);
730 }
731 
732 int PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
733                                Type *SubTp) {
734   // Legalize the type.
735   std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp);
736 
737   // PPC, for both Altivec/VSX and QPX, support cheap arbitrary permutations
738   // (at least in the sense that there need only be one non-loop-invariant
739   // instruction). We need one such shuffle instruction for each actual
740   // register (this is not true for arbitrary shuffles, but is true for the
741   // structured types of shuffles covered by TTI::ShuffleKind).
742   return vectorCostAdjustment(LT.first, Instruction::ShuffleVector, Tp,
743                               nullptr);
744 }
745 
746 int PPCTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
747                                  const Instruction *I) {
748   assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode");
749 
750   int Cost = BaseT::getCastInstrCost(Opcode, Dst, Src);
751   return vectorCostAdjustment(Cost, Opcode, Dst, Src);
752 }
753 
754 int PPCTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
755                                    const Instruction *I) {
756   int Cost = BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, I);
757   return vectorCostAdjustment(Cost, Opcode, ValTy, nullptr);
758 }
759 
760 int PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) {
761   assert(Val->isVectorTy() && "This must be a vector type");
762 
763   int ISD = TLI->InstructionOpcodeToISD(Opcode);
764   assert(ISD && "Invalid opcode");
765 
766   int Cost = BaseT::getVectorInstrCost(Opcode, Val, Index);
767   Cost = vectorCostAdjustment(Cost, Opcode, Val, nullptr);
768 
769   if (ST->hasVSX() && Val->getScalarType()->isDoubleTy()) {
770     // Double-precision scalars are already located in index #0 (or #1 if LE).
771     if (ISD == ISD::EXTRACT_VECTOR_ELT &&
772         Index == (ST->isLittleEndian() ? 1 : 0))
773       return 0;
774 
775     return Cost;
776 
777   } else if (ST->hasQPX() && Val->getScalarType()->isFloatingPointTy()) {
778     // Floating point scalars are already located in index #0.
779     if (Index == 0)
780       return 0;
781 
782     return Cost;
783 
784   } else if (Val->getScalarType()->isIntegerTy() && Index != -1U) {
785     if (ST->hasP9Altivec()) {
786       if (ISD == ISD::INSERT_VECTOR_ELT)
787         // A move-to VSR and a permute/insert.  Assume vector operation cost
788         // for both (cost will be 2x on P9).
789         return vectorCostAdjustment(2, Opcode, Val, nullptr);
790 
791       // It's an extract.  Maybe we can do a cheap move-from VSR.
792       unsigned EltSize = Val->getScalarSizeInBits();
793       if (EltSize == 64) {
794         unsigned MfvsrdIndex = ST->isLittleEndian() ? 1 : 0;
795         if (Index == MfvsrdIndex)
796           return 1;
797       } else if (EltSize == 32) {
798         unsigned MfvsrwzIndex = ST->isLittleEndian() ? 2 : 1;
799         if (Index == MfvsrwzIndex)
800           return 1;
801       }
802 
803       // We need a vector extract (or mfvsrld).  Assume vector operation cost.
804       // The cost of the load constant for a vector extract is disregarded
805       // (invariant, easily schedulable).
806       return vectorCostAdjustment(1, Opcode, Val, nullptr);
807 
808     } else if (ST->hasDirectMove())
809       // Assume permute has standard cost.
810       // Assume move-to/move-from VSR have 2x standard cost.
811       return 3;
812   }
813 
814   // Estimated cost of a load-hit-store delay.  This was obtained
815   // experimentally as a minimum needed to prevent unprofitable
816   // vectorization for the paq8p benchmark.  It may need to be
817   // raised further if other unprofitable cases remain.
818   unsigned LHSPenalty = 2;
819   if (ISD == ISD::INSERT_VECTOR_ELT)
820     LHSPenalty += 7;
821 
822   // Vector element insert/extract with Altivec is very expensive,
823   // because they require store and reload with the attendant
824   // processor stall for load-hit-store.  Until VSX is available,
825   // these need to be estimated as very costly.
826   if (ISD == ISD::EXTRACT_VECTOR_ELT ||
827       ISD == ISD::INSERT_VECTOR_ELT)
828     return LHSPenalty + Cost;
829 
830   return Cost;
831 }
832 
833 int PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
834                                 MaybeAlign Alignment, unsigned AddressSpace,
835                                 const Instruction *I) {
836   // Legalize the type.
837   std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Src);
838   assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
839          "Invalid Opcode");
840 
841   int Cost = BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace);
842   Cost = vectorCostAdjustment(Cost, Opcode, Src, nullptr);
843 
844   bool IsAltivecType = ST->hasAltivec() &&
845                        (LT.second == MVT::v16i8 || LT.second == MVT::v8i16 ||
846                         LT.second == MVT::v4i32 || LT.second == MVT::v4f32);
847   bool IsVSXType = ST->hasVSX() &&
848                    (LT.second == MVT::v2f64 || LT.second == MVT::v2i64);
849   bool IsQPXType = ST->hasQPX() &&
850                    (LT.second == MVT::v4f64 || LT.second == MVT::v4f32);
851 
852   // VSX has 32b/64b load instructions. Legalization can handle loading of
853   // 32b/64b to VSR correctly and cheaply. But BaseT::getMemoryOpCost and
854   // PPCTargetLowering can't compute the cost appropriately. So here we
855   // explicitly check this case.
856   unsigned MemBytes = Src->getPrimitiveSizeInBits();
857   if (Opcode == Instruction::Load && ST->hasVSX() && IsAltivecType &&
858       (MemBytes == 64 || (ST->hasP8Vector() && MemBytes == 32)))
859     return 1;
860 
861   // Aligned loads and stores are easy.
862   unsigned SrcBytes = LT.second.getStoreSize();
863   if (!SrcBytes || !Alignment || Alignment >= SrcBytes)
864     return Cost;
865 
866   // If we can use the permutation-based load sequence, then this is also
867   // relatively cheap (not counting loop-invariant instructions): one load plus
868   // one permute (the last load in a series has extra cost, but we're
869   // neglecting that here). Note that on the P7, we could do unaligned loads
870   // for Altivec types using the VSX instructions, but that's more expensive
871   // than using the permutation-based load sequence. On the P8, that's no
872   // longer true.
873   if (Opcode == Instruction::Load &&
874       ((!ST->hasP8Vector() && IsAltivecType) || IsQPXType) &&
875       Alignment >= LT.second.getScalarType().getStoreSize())
876     return Cost + LT.first; // Add the cost of the permutations.
877 
878   // For VSX, we can do unaligned loads and stores on Altivec/VSX types. On the
879   // P7, unaligned vector loads are more expensive than the permutation-based
880   // load sequence, so that might be used instead, but regardless, the net cost
881   // is about the same (not counting loop-invariant instructions).
882   if (IsVSXType || (ST->hasVSX() && IsAltivecType))
883     return Cost;
884 
885   // Newer PPC supports unaligned memory access.
886   if (TLI->allowsMisalignedMemoryAccesses(LT.second, 0))
887     return Cost;
888 
889   // PPC in general does not support unaligned loads and stores. They'll need
890   // to be decomposed based on the alignment factor.
891 
892   // Add the cost of each scalar load or store.
893   assert(Alignment);
894   Cost += LT.first * ((SrcBytes / Alignment->value()) - 1);
895 
896   // For a vector type, there is also scalarization overhead (only for
897   // stores, loads are expanded using the vector-load + permutation sequence,
898   // which is much less expensive).
899   if (Src->isVectorTy() && Opcode == Instruction::Store)
900     for (int i = 0, e = Src->getVectorNumElements(); i < e; ++i)
901       Cost += getVectorInstrCost(Instruction::ExtractElement, Src, i);
902 
903   return Cost;
904 }
905 
906 int PPCTTIImpl::getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
907                                            unsigned Factor,
908                                            ArrayRef<unsigned> Indices,
909                                            unsigned Alignment,
910                                            unsigned AddressSpace,
911                                            bool UseMaskForCond,
912                                            bool UseMaskForGaps) {
913   if (UseMaskForCond || UseMaskForGaps)
914     return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
915                                              Alignment, AddressSpace,
916                                              UseMaskForCond, UseMaskForGaps);
917 
918   assert(isa<VectorType>(VecTy) &&
919          "Expect a vector type for interleaved memory op");
920 
921   // Legalize the type.
922   std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, VecTy);
923 
924   // Firstly, the cost of load/store operation.
925   int Cost =
926       getMemoryOpCost(Opcode, VecTy, MaybeAlign(Alignment), AddressSpace);
927 
928   // PPC, for both Altivec/VSX and QPX, support cheap arbitrary permutations
929   // (at least in the sense that there need only be one non-loop-invariant
930   // instruction). For each result vector, we need one shuffle per incoming
931   // vector (except that the first shuffle can take two incoming vectors
932   // because it does not need to take itself).
933   Cost += Factor*(LT.first-1);
934 
935   return Cost;
936 }
937 
938 unsigned PPCTTIImpl::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
939       ArrayRef<Value*> Args, FastMathFlags FMF, unsigned VF) {
940   return BaseT::getIntrinsicInstrCost(ID, RetTy, Args, FMF, VF);
941 }
942 
943 unsigned PPCTTIImpl::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
944       ArrayRef<Type*> Tys, FastMathFlags FMF,
945       unsigned ScalarizationCostPassed) {
946   if (ID == Intrinsic::bswap && ST->hasP9Vector())
947     return TLI->getTypeLegalizationCost(DL, RetTy).first;
948   return BaseT::getIntrinsicInstrCost(ID, RetTy, Tys, FMF,
949                                       ScalarizationCostPassed);
950 }
951 
952 bool PPCTTIImpl::canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE,
953                             LoopInfo *LI, DominatorTree *DT,
954                             AssumptionCache *AC, TargetLibraryInfo *LibInfo) {
955   // Process nested loops first.
956   for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I)
957     if (canSaveCmp(*I, BI, SE, LI, DT, AC, LibInfo))
958       return false; // Stop search.
959 
960   HardwareLoopInfo HWLoopInfo(L);
961 
962   if (!HWLoopInfo.canAnalyze(*LI))
963     return false;
964 
965   if (!isHardwareLoopProfitable(L, *SE, *AC, LibInfo, HWLoopInfo))
966     return false;
967 
968   if (!HWLoopInfo.isHardwareLoopCandidate(*SE, *LI, *DT))
969     return false;
970 
971   *BI = HWLoopInfo.ExitBranch;
972   return true;
973 }
974 
975 bool PPCTTIImpl::isLSRCostLess(TargetTransformInfo::LSRCost &C1,
976                                TargetTransformInfo::LSRCost &C2) {
977   // PowerPC default behaviour here is "instruction number 1st priority".
978   // If LsrNoInsnsCost is set, call default implementation.
979   if (!LsrNoInsnsCost)
980     return std::tie(C1.Insns, C1.NumRegs, C1.AddRecCost, C1.NumIVMuls,
981                     C1.NumBaseAdds, C1.ScaleCost, C1.ImmCost, C1.SetupCost) <
982            std::tie(C2.Insns, C2.NumRegs, C2.AddRecCost, C2.NumIVMuls,
983                     C2.NumBaseAdds, C2.ScaleCost, C2.ImmCost, C2.SetupCost);
984   else
985     return TargetTransformInfoImplBase::isLSRCostLess(C1, C2);
986 }
987