1 //===-- PPCTargetTransformInfo.cpp - PPC specific TTI ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "PPCTargetTransformInfo.h" 10 #include "llvm/Analysis/CodeMetrics.h" 11 #include "llvm/Analysis/TargetTransformInfo.h" 12 #include "llvm/CodeGen/BasicTTIImpl.h" 13 #include "llvm/CodeGen/CostTable.h" 14 #include "llvm/CodeGen/TargetLowering.h" 15 #include "llvm/CodeGen/TargetSchedule.h" 16 #include "llvm/Support/CommandLine.h" 17 #include "llvm/Support/Debug.h" 18 using namespace llvm; 19 20 #define DEBUG_TYPE "ppctti" 21 22 static cl::opt<bool> DisablePPCConstHoist("disable-ppc-constant-hoisting", 23 cl::desc("disable constant hoisting on PPC"), cl::init(false), cl::Hidden); 24 25 // This is currently only used for the data prefetch pass which is only enabled 26 // for BG/Q by default. 27 static cl::opt<unsigned> 28 CacheLineSize("ppc-loop-prefetch-cache-line", cl::Hidden, cl::init(64), 29 cl::desc("The loop prefetch cache line size")); 30 31 static cl::opt<bool> 32 EnablePPCColdCC("ppc-enable-coldcc", cl::Hidden, cl::init(false), 33 cl::desc("Enable using coldcc calling conv for cold " 34 "internal functions")); 35 36 static cl::opt<bool> 37 LsrNoInsnsCost("ppc-lsr-no-insns-cost", cl::Hidden, cl::init(false), 38 cl::desc("Do not add instruction count to lsr cost model")); 39 40 // The latency of mtctr is only justified if there are more than 4 41 // comparisons that will be removed as a result. 42 static cl::opt<unsigned> 43 SmallCTRLoopThreshold("min-ctr-loop-threshold", cl::init(4), cl::Hidden, 44 cl::desc("Loops with a constant trip count smaller than " 45 "this value will not use the count register.")); 46 47 //===----------------------------------------------------------------------===// 48 // 49 // PPC cost model. 50 // 51 //===----------------------------------------------------------------------===// 52 53 TargetTransformInfo::PopcntSupportKind 54 PPCTTIImpl::getPopcntSupport(unsigned TyWidth) { 55 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2"); 56 if (ST->hasPOPCNTD() != PPCSubtarget::POPCNTD_Unavailable && TyWidth <= 64) 57 return ST->hasPOPCNTD() == PPCSubtarget::POPCNTD_Slow ? 58 TTI::PSK_SlowHardware : TTI::PSK_FastHardware; 59 return TTI::PSK_Software; 60 } 61 62 int PPCTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty) { 63 if (DisablePPCConstHoist) 64 return BaseT::getIntImmCost(Imm, Ty); 65 66 assert(Ty->isIntegerTy()); 67 68 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 69 if (BitSize == 0) 70 return ~0U; 71 72 if (Imm == 0) 73 return TTI::TCC_Free; 74 75 if (Imm.getBitWidth() <= 64) { 76 if (isInt<16>(Imm.getSExtValue())) 77 return TTI::TCC_Basic; 78 79 if (isInt<32>(Imm.getSExtValue())) { 80 // A constant that can be materialized using lis. 81 if ((Imm.getZExtValue() & 0xFFFF) == 0) 82 return TTI::TCC_Basic; 83 84 return 2 * TTI::TCC_Basic; 85 } 86 } 87 88 return 4 * TTI::TCC_Basic; 89 } 90 91 int PPCTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, 92 const APInt &Imm, Type *Ty) { 93 if (DisablePPCConstHoist) 94 return BaseT::getIntImmCostIntrin(IID, Idx, Imm, Ty); 95 96 assert(Ty->isIntegerTy()); 97 98 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 99 if (BitSize == 0) 100 return ~0U; 101 102 switch (IID) { 103 default: 104 return TTI::TCC_Free; 105 case Intrinsic::sadd_with_overflow: 106 case Intrinsic::uadd_with_overflow: 107 case Intrinsic::ssub_with_overflow: 108 case Intrinsic::usub_with_overflow: 109 if ((Idx == 1) && Imm.getBitWidth() <= 64 && isInt<16>(Imm.getSExtValue())) 110 return TTI::TCC_Free; 111 break; 112 case Intrinsic::experimental_stackmap: 113 if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))) 114 return TTI::TCC_Free; 115 break; 116 case Intrinsic::experimental_patchpoint_void: 117 case Intrinsic::experimental_patchpoint_i64: 118 if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))) 119 return TTI::TCC_Free; 120 break; 121 } 122 return PPCTTIImpl::getIntImmCost(Imm, Ty); 123 } 124 125 int PPCTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx, 126 const APInt &Imm, Type *Ty) { 127 if (DisablePPCConstHoist) 128 return BaseT::getIntImmCostInst(Opcode, Idx, Imm, Ty); 129 130 assert(Ty->isIntegerTy()); 131 132 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 133 if (BitSize == 0) 134 return ~0U; 135 136 unsigned ImmIdx = ~0U; 137 bool ShiftedFree = false, RunFree = false, UnsignedFree = false, 138 ZeroFree = false; 139 switch (Opcode) { 140 default: 141 return TTI::TCC_Free; 142 case Instruction::GetElementPtr: 143 // Always hoist the base address of a GetElementPtr. This prevents the 144 // creation of new constants for every base constant that gets constant 145 // folded with the offset. 146 if (Idx == 0) 147 return 2 * TTI::TCC_Basic; 148 return TTI::TCC_Free; 149 case Instruction::And: 150 RunFree = true; // (for the rotate-and-mask instructions) 151 LLVM_FALLTHROUGH; 152 case Instruction::Add: 153 case Instruction::Or: 154 case Instruction::Xor: 155 ShiftedFree = true; 156 LLVM_FALLTHROUGH; 157 case Instruction::Sub: 158 case Instruction::Mul: 159 case Instruction::Shl: 160 case Instruction::LShr: 161 case Instruction::AShr: 162 ImmIdx = 1; 163 break; 164 case Instruction::ICmp: 165 UnsignedFree = true; 166 ImmIdx = 1; 167 // Zero comparisons can use record-form instructions. 168 LLVM_FALLTHROUGH; 169 case Instruction::Select: 170 ZeroFree = true; 171 break; 172 case Instruction::PHI: 173 case Instruction::Call: 174 case Instruction::Ret: 175 case Instruction::Load: 176 case Instruction::Store: 177 break; 178 } 179 180 if (ZeroFree && Imm == 0) 181 return TTI::TCC_Free; 182 183 if (Idx == ImmIdx && Imm.getBitWidth() <= 64) { 184 if (isInt<16>(Imm.getSExtValue())) 185 return TTI::TCC_Free; 186 187 if (RunFree) { 188 if (Imm.getBitWidth() <= 32 && 189 (isShiftedMask_32(Imm.getZExtValue()) || 190 isShiftedMask_32(~Imm.getZExtValue()))) 191 return TTI::TCC_Free; 192 193 if (ST->isPPC64() && 194 (isShiftedMask_64(Imm.getZExtValue()) || 195 isShiftedMask_64(~Imm.getZExtValue()))) 196 return TTI::TCC_Free; 197 } 198 199 if (UnsignedFree && isUInt<16>(Imm.getZExtValue())) 200 return TTI::TCC_Free; 201 202 if (ShiftedFree && (Imm.getZExtValue() & 0xFFFF) == 0) 203 return TTI::TCC_Free; 204 } 205 206 return PPCTTIImpl::getIntImmCost(Imm, Ty); 207 } 208 209 unsigned PPCTTIImpl::getUserCost(const User *U, 210 ArrayRef<const Value *> Operands) { 211 if (U->getType()->isVectorTy()) { 212 // Instructions that need to be split should cost more. 213 std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, U->getType()); 214 return LT.first * BaseT::getUserCost(U, Operands); 215 } 216 217 return BaseT::getUserCost(U, Operands); 218 } 219 220 bool PPCTTIImpl::mightUseCTR(BasicBlock *BB, 221 TargetLibraryInfo *LibInfo) { 222 const PPCTargetMachine &TM = ST->getTargetMachine(); 223 224 // Loop through the inline asm constraints and look for something that 225 // clobbers ctr. 226 auto asmClobbersCTR = [](InlineAsm *IA) { 227 InlineAsm::ConstraintInfoVector CIV = IA->ParseConstraints(); 228 for (unsigned i = 0, ie = CIV.size(); i < ie; ++i) { 229 InlineAsm::ConstraintInfo &C = CIV[i]; 230 if (C.Type != InlineAsm::isInput) 231 for (unsigned j = 0, je = C.Codes.size(); j < je; ++j) 232 if (StringRef(C.Codes[j]).equals_lower("{ctr}")) 233 return true; 234 } 235 return false; 236 }; 237 238 // Determining the address of a TLS variable results in a function call in 239 // certain TLS models. 240 std::function<bool(const Value*)> memAddrUsesCTR = 241 [&memAddrUsesCTR, &TM](const Value *MemAddr) -> bool { 242 const auto *GV = dyn_cast<GlobalValue>(MemAddr); 243 if (!GV) { 244 // Recurse to check for constants that refer to TLS global variables. 245 if (const auto *CV = dyn_cast<Constant>(MemAddr)) 246 for (const auto &CO : CV->operands()) 247 if (memAddrUsesCTR(CO)) 248 return true; 249 250 return false; 251 } 252 253 if (!GV->isThreadLocal()) 254 return false; 255 TLSModel::Model Model = TM.getTLSModel(GV); 256 return Model == TLSModel::GeneralDynamic || 257 Model == TLSModel::LocalDynamic; 258 }; 259 260 auto isLargeIntegerTy = [](bool Is32Bit, Type *Ty) { 261 if (IntegerType *ITy = dyn_cast<IntegerType>(Ty)) 262 return ITy->getBitWidth() > (Is32Bit ? 32U : 64U); 263 264 return false; 265 }; 266 267 for (BasicBlock::iterator J = BB->begin(), JE = BB->end(); 268 J != JE; ++J) { 269 if (CallInst *CI = dyn_cast<CallInst>(J)) { 270 // Inline ASM is okay, unless it clobbers the ctr register. 271 if (InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledValue())) { 272 if (asmClobbersCTR(IA)) 273 return true; 274 continue; 275 } 276 277 if (Function *F = CI->getCalledFunction()) { 278 // Most intrinsics don't become function calls, but some might. 279 // sin, cos, exp and log are always calls. 280 unsigned Opcode = 0; 281 if (F->getIntrinsicID() != Intrinsic::not_intrinsic) { 282 switch (F->getIntrinsicID()) { 283 default: continue; 284 // If we have a call to ppc_is_decremented_ctr_nonzero, or ppc_mtctr 285 // we're definitely using CTR. 286 case Intrinsic::set_loop_iterations: 287 case Intrinsic::loop_decrement: 288 return true; 289 290 // Exclude eh_sjlj_setjmp; we don't need to exclude eh_sjlj_longjmp 291 // because, although it does clobber the counter register, the 292 // control can't then return to inside the loop unless there is also 293 // an eh_sjlj_setjmp. 294 case Intrinsic::eh_sjlj_setjmp: 295 296 case Intrinsic::memcpy: 297 case Intrinsic::memmove: 298 case Intrinsic::memset: 299 case Intrinsic::powi: 300 case Intrinsic::log: 301 case Intrinsic::log2: 302 case Intrinsic::log10: 303 case Intrinsic::exp: 304 case Intrinsic::exp2: 305 case Intrinsic::pow: 306 case Intrinsic::sin: 307 case Intrinsic::cos: 308 return true; 309 case Intrinsic::copysign: 310 if (CI->getArgOperand(0)->getType()->getScalarType()-> 311 isPPC_FP128Ty()) 312 return true; 313 else 314 continue; // ISD::FCOPYSIGN is never a library call. 315 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break; 316 case Intrinsic::floor: Opcode = ISD::FFLOOR; break; 317 case Intrinsic::ceil: Opcode = ISD::FCEIL; break; 318 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break; 319 case Intrinsic::rint: Opcode = ISD::FRINT; break; 320 case Intrinsic::lrint: Opcode = ISD::LRINT; break; 321 case Intrinsic::llrint: Opcode = ISD::LLRINT; break; 322 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break; 323 case Intrinsic::round: Opcode = ISD::FROUND; break; 324 case Intrinsic::lround: Opcode = ISD::LROUND; break; 325 case Intrinsic::llround: Opcode = ISD::LLROUND; break; 326 case Intrinsic::minnum: Opcode = ISD::FMINNUM; break; 327 case Intrinsic::maxnum: Opcode = ISD::FMAXNUM; break; 328 case Intrinsic::umul_with_overflow: Opcode = ISD::UMULO; break; 329 case Intrinsic::smul_with_overflow: Opcode = ISD::SMULO; break; 330 } 331 } 332 333 // PowerPC does not use [US]DIVREM or other library calls for 334 // operations on regular types which are not otherwise library calls 335 // (i.e. soft float or atomics). If adapting for targets that do, 336 // additional care is required here. 337 338 LibFunc Func; 339 if (!F->hasLocalLinkage() && F->hasName() && LibInfo && 340 LibInfo->getLibFunc(F->getName(), Func) && 341 LibInfo->hasOptimizedCodeGen(Func)) { 342 // Non-read-only functions are never treated as intrinsics. 343 if (!CI->onlyReadsMemory()) 344 return true; 345 346 // Conversion happens only for FP calls. 347 if (!CI->getArgOperand(0)->getType()->isFloatingPointTy()) 348 return true; 349 350 switch (Func) { 351 default: return true; 352 case LibFunc_copysign: 353 case LibFunc_copysignf: 354 continue; // ISD::FCOPYSIGN is never a library call. 355 case LibFunc_copysignl: 356 return true; 357 case LibFunc_fabs: 358 case LibFunc_fabsf: 359 case LibFunc_fabsl: 360 continue; // ISD::FABS is never a library call. 361 case LibFunc_sqrt: 362 case LibFunc_sqrtf: 363 case LibFunc_sqrtl: 364 Opcode = ISD::FSQRT; break; 365 case LibFunc_floor: 366 case LibFunc_floorf: 367 case LibFunc_floorl: 368 Opcode = ISD::FFLOOR; break; 369 case LibFunc_nearbyint: 370 case LibFunc_nearbyintf: 371 case LibFunc_nearbyintl: 372 Opcode = ISD::FNEARBYINT; break; 373 case LibFunc_ceil: 374 case LibFunc_ceilf: 375 case LibFunc_ceill: 376 Opcode = ISD::FCEIL; break; 377 case LibFunc_rint: 378 case LibFunc_rintf: 379 case LibFunc_rintl: 380 Opcode = ISD::FRINT; break; 381 case LibFunc_round: 382 case LibFunc_roundf: 383 case LibFunc_roundl: 384 Opcode = ISD::FROUND; break; 385 case LibFunc_trunc: 386 case LibFunc_truncf: 387 case LibFunc_truncl: 388 Opcode = ISD::FTRUNC; break; 389 case LibFunc_fmin: 390 case LibFunc_fminf: 391 case LibFunc_fminl: 392 Opcode = ISD::FMINNUM; break; 393 case LibFunc_fmax: 394 case LibFunc_fmaxf: 395 case LibFunc_fmaxl: 396 Opcode = ISD::FMAXNUM; break; 397 } 398 } 399 400 if (Opcode) { 401 EVT EVTy = 402 TLI->getValueType(DL, CI->getArgOperand(0)->getType(), true); 403 404 if (EVTy == MVT::Other) 405 return true; 406 407 if (TLI->isOperationLegalOrCustom(Opcode, EVTy)) 408 continue; 409 else if (EVTy.isVector() && 410 TLI->isOperationLegalOrCustom(Opcode, EVTy.getScalarType())) 411 continue; 412 413 return true; 414 } 415 } 416 417 return true; 418 } else if (isa<BinaryOperator>(J) && 419 J->getType()->getScalarType()->isPPC_FP128Ty()) { 420 // Most operations on ppc_f128 values become calls. 421 return true; 422 } else if (isa<UIToFPInst>(J) || isa<SIToFPInst>(J) || 423 isa<FPToUIInst>(J) || isa<FPToSIInst>(J)) { 424 CastInst *CI = cast<CastInst>(J); 425 if (CI->getSrcTy()->getScalarType()->isPPC_FP128Ty() || 426 CI->getDestTy()->getScalarType()->isPPC_FP128Ty() || 427 isLargeIntegerTy(!TM.isPPC64(), CI->getSrcTy()->getScalarType()) || 428 isLargeIntegerTy(!TM.isPPC64(), CI->getDestTy()->getScalarType())) 429 return true; 430 } else if (isLargeIntegerTy(!TM.isPPC64(), 431 J->getType()->getScalarType()) && 432 (J->getOpcode() == Instruction::UDiv || 433 J->getOpcode() == Instruction::SDiv || 434 J->getOpcode() == Instruction::URem || 435 J->getOpcode() == Instruction::SRem)) { 436 return true; 437 } else if (!TM.isPPC64() && 438 isLargeIntegerTy(false, J->getType()->getScalarType()) && 439 (J->getOpcode() == Instruction::Shl || 440 J->getOpcode() == Instruction::AShr || 441 J->getOpcode() == Instruction::LShr)) { 442 // Only on PPC32, for 128-bit integers (specifically not 64-bit 443 // integers), these might be runtime calls. 444 return true; 445 } else if (isa<IndirectBrInst>(J) || isa<InvokeInst>(J)) { 446 // On PowerPC, indirect jumps use the counter register. 447 return true; 448 } else if (SwitchInst *SI = dyn_cast<SwitchInst>(J)) { 449 if (SI->getNumCases() + 1 >= (unsigned)TLI->getMinimumJumpTableEntries()) 450 return true; 451 } 452 453 // FREM is always a call. 454 if (J->getOpcode() == Instruction::FRem) 455 return true; 456 457 if (ST->useSoftFloat()) { 458 switch(J->getOpcode()) { 459 case Instruction::FAdd: 460 case Instruction::FSub: 461 case Instruction::FMul: 462 case Instruction::FDiv: 463 case Instruction::FPTrunc: 464 case Instruction::FPExt: 465 case Instruction::FPToUI: 466 case Instruction::FPToSI: 467 case Instruction::UIToFP: 468 case Instruction::SIToFP: 469 case Instruction::FCmp: 470 return true; 471 } 472 } 473 474 for (Value *Operand : J->operands()) 475 if (memAddrUsesCTR(Operand)) 476 return true; 477 } 478 479 return false; 480 } 481 482 bool PPCTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE, 483 AssumptionCache &AC, 484 TargetLibraryInfo *LibInfo, 485 HardwareLoopInfo &HWLoopInfo) { 486 const PPCTargetMachine &TM = ST->getTargetMachine(); 487 TargetSchedModel SchedModel; 488 SchedModel.init(ST); 489 490 // Do not convert small short loops to CTR loop. 491 unsigned ConstTripCount = SE.getSmallConstantTripCount(L); 492 if (ConstTripCount && ConstTripCount < SmallCTRLoopThreshold) { 493 SmallPtrSet<const Value *, 32> EphValues; 494 CodeMetrics::collectEphemeralValues(L, &AC, EphValues); 495 CodeMetrics Metrics; 496 for (BasicBlock *BB : L->blocks()) 497 Metrics.analyzeBasicBlock(BB, *this, EphValues); 498 // 6 is an approximate latency for the mtctr instruction. 499 if (Metrics.NumInsts <= (6 * SchedModel.getIssueWidth())) 500 return false; 501 } 502 503 // We don't want to spill/restore the counter register, and so we don't 504 // want to use the counter register if the loop contains calls. 505 for (Loop::block_iterator I = L->block_begin(), IE = L->block_end(); 506 I != IE; ++I) 507 if (mightUseCTR(*I, LibInfo)) 508 return false; 509 510 SmallVector<BasicBlock*, 4> ExitingBlocks; 511 L->getExitingBlocks(ExitingBlocks); 512 513 // If there is an exit edge known to be frequently taken, 514 // we should not transform this loop. 515 for (auto &BB : ExitingBlocks) { 516 Instruction *TI = BB->getTerminator(); 517 if (!TI) continue; 518 519 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) { 520 uint64_t TrueWeight = 0, FalseWeight = 0; 521 if (!BI->isConditional() || 522 !BI->extractProfMetadata(TrueWeight, FalseWeight)) 523 continue; 524 525 // If the exit path is more frequent than the loop path, 526 // we return here without further analysis for this loop. 527 bool TrueIsExit = !L->contains(BI->getSuccessor(0)); 528 if (( TrueIsExit && FalseWeight < TrueWeight) || 529 (!TrueIsExit && FalseWeight > TrueWeight)) 530 return false; 531 } 532 } 533 534 LLVMContext &C = L->getHeader()->getContext(); 535 HWLoopInfo.CountType = TM.isPPC64() ? 536 Type::getInt64Ty(C) : Type::getInt32Ty(C); 537 HWLoopInfo.LoopDecrement = ConstantInt::get(HWLoopInfo.CountType, 1); 538 return true; 539 } 540 541 void PPCTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, 542 TTI::UnrollingPreferences &UP) { 543 if (ST->getCPUDirective() == PPC::DIR_A2) { 544 // The A2 is in-order with a deep pipeline, and concatenation unrolling 545 // helps expose latency-hiding opportunities to the instruction scheduler. 546 UP.Partial = UP.Runtime = true; 547 548 // We unroll a lot on the A2 (hundreds of instructions), and the benefits 549 // often outweigh the cost of a division to compute the trip count. 550 UP.AllowExpensiveTripCount = true; 551 } 552 553 BaseT::getUnrollingPreferences(L, SE, UP); 554 } 555 556 // This function returns true to allow using coldcc calling convention. 557 // Returning true results in coldcc being used for functions which are cold at 558 // all call sites when the callers of the functions are not calling any other 559 // non coldcc functions. 560 bool PPCTTIImpl::useColdCCForColdCall(Function &F) { 561 return EnablePPCColdCC; 562 } 563 564 bool PPCTTIImpl::enableAggressiveInterleaving(bool LoopHasReductions) { 565 // On the A2, always unroll aggressively. For QPX unaligned loads, we depend 566 // on combining the loads generated for consecutive accesses, and failure to 567 // do so is particularly expensive. This makes it much more likely (compared 568 // to only using concatenation unrolling). 569 if (ST->getCPUDirective() == PPC::DIR_A2) 570 return true; 571 572 return LoopHasReductions; 573 } 574 575 PPCTTIImpl::TTI::MemCmpExpansionOptions 576 PPCTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const { 577 TTI::MemCmpExpansionOptions Options; 578 Options.LoadSizes = {8, 4, 2, 1}; 579 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize); 580 return Options; 581 } 582 583 bool PPCTTIImpl::enableInterleavedAccessVectorization() { 584 return true; 585 } 586 587 unsigned PPCTTIImpl::getNumberOfRegisters(unsigned ClassID) const { 588 assert(ClassID == GPRRC || ClassID == FPRRC || 589 ClassID == VRRC || ClassID == VSXRC); 590 if (ST->hasVSX()) { 591 assert(ClassID == GPRRC || ClassID == VSXRC || ClassID == VRRC); 592 return ClassID == VSXRC ? 64 : 32; 593 } 594 assert(ClassID == GPRRC || ClassID == FPRRC || ClassID == VRRC); 595 return 32; 596 } 597 598 unsigned PPCTTIImpl::getRegisterClassForType(bool Vector, Type *Ty) const { 599 if (Vector) 600 return ST->hasVSX() ? VSXRC : VRRC; 601 else if (Ty && (Ty->getScalarType()->isFloatTy() || 602 Ty->getScalarType()->isDoubleTy())) 603 return ST->hasVSX() ? VSXRC : FPRRC; 604 else if (Ty && (Ty->getScalarType()->isFP128Ty() || 605 Ty->getScalarType()->isPPC_FP128Ty())) 606 return VRRC; 607 else if (Ty && Ty->getScalarType()->isHalfTy()) 608 return VSXRC; 609 else 610 return GPRRC; 611 } 612 613 const char* PPCTTIImpl::getRegisterClassName(unsigned ClassID) const { 614 615 switch (ClassID) { 616 default: 617 llvm_unreachable("unknown register class"); 618 return "PPC::unknown register class"; 619 case GPRRC: return "PPC::GPRRC"; 620 case FPRRC: return "PPC::FPRRC"; 621 case VRRC: return "PPC::VRRC"; 622 case VSXRC: return "PPC::VSXRC"; 623 } 624 } 625 626 unsigned PPCTTIImpl::getRegisterBitWidth(bool Vector) const { 627 if (Vector) { 628 if (ST->hasQPX()) return 256; 629 if (ST->hasAltivec()) return 128; 630 return 0; 631 } 632 633 if (ST->isPPC64()) 634 return 64; 635 return 32; 636 637 } 638 639 unsigned PPCTTIImpl::getCacheLineSize() const { 640 // Check first if the user specified a custom line size. 641 if (CacheLineSize.getNumOccurrences() > 0) 642 return CacheLineSize; 643 644 // On P7, P8 or P9 we have a cache line size of 128. 645 unsigned Directive = ST->getCPUDirective(); 646 // Assume that Future CPU has the same cache line size as the others. 647 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8 || 648 Directive == PPC::DIR_PWR9 || Directive == PPC::DIR_PWR_FUTURE) 649 return 128; 650 651 // On other processors return a default of 64 bytes. 652 return 64; 653 } 654 655 unsigned PPCTTIImpl::getPrefetchDistance() const { 656 // This seems like a reasonable default for the BG/Q (this pass is enabled, by 657 // default, only on the BG/Q). 658 return 300; 659 } 660 661 unsigned PPCTTIImpl::getMaxInterleaveFactor(unsigned VF) { 662 unsigned Directive = ST->getCPUDirective(); 663 // The 440 has no SIMD support, but floating-point instructions 664 // have a 5-cycle latency, so unroll by 5x for latency hiding. 665 if (Directive == PPC::DIR_440) 666 return 5; 667 668 // The A2 has no SIMD support, but floating-point instructions 669 // have a 6-cycle latency, so unroll by 6x for latency hiding. 670 if (Directive == PPC::DIR_A2) 671 return 6; 672 673 // FIXME: For lack of any better information, do no harm... 674 if (Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) 675 return 1; 676 677 // For P7 and P8, floating-point instructions have a 6-cycle latency and 678 // there are two execution units, so unroll by 12x for latency hiding. 679 // FIXME: the same for P9 as previous gen until POWER9 scheduling is ready 680 // Assume that future is the same as the others. 681 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8 || 682 Directive == PPC::DIR_PWR9 || Directive == PPC::DIR_PWR_FUTURE) 683 return 12; 684 685 // For most things, modern systems have two execution units (and 686 // out-of-order execution). 687 return 2; 688 } 689 690 // Adjust the cost of vector instructions on targets which there is overlap 691 // between the vector and scalar units, thereby reducing the overall throughput 692 // of vector code wrt. scalar code. 693 int PPCTTIImpl::vectorCostAdjustment(int Cost, unsigned Opcode, Type *Ty1, 694 Type *Ty2) { 695 if (!ST->vectorsUseTwoUnits() || !Ty1->isVectorTy()) 696 return Cost; 697 698 std::pair<int, MVT> LT1 = TLI->getTypeLegalizationCost(DL, Ty1); 699 // If type legalization involves splitting the vector, we don't want to 700 // double the cost at every step - only the last step. 701 if (LT1.first != 1 || !LT1.second.isVector()) 702 return Cost; 703 704 int ISD = TLI->InstructionOpcodeToISD(Opcode); 705 if (TLI->isOperationExpand(ISD, LT1.second)) 706 return Cost; 707 708 if (Ty2) { 709 std::pair<int, MVT> LT2 = TLI->getTypeLegalizationCost(DL, Ty2); 710 if (LT2.first != 1 || !LT2.second.isVector()) 711 return Cost; 712 } 713 714 return Cost * 2; 715 } 716 717 int PPCTTIImpl::getArithmeticInstrCost(unsigned Opcode, Type *Ty, 718 TTI::OperandValueKind Op1Info, 719 TTI::OperandValueKind Op2Info, 720 TTI::OperandValueProperties Opd1PropInfo, 721 TTI::OperandValueProperties Opd2PropInfo, 722 ArrayRef<const Value *> Args, 723 const Instruction *CxtI) { 724 assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode"); 725 726 // Fallback to the default implementation. 727 int Cost = BaseT::getArithmeticInstrCost(Opcode, Ty, Op1Info, Op2Info, 728 Opd1PropInfo, Opd2PropInfo); 729 return vectorCostAdjustment(Cost, Opcode, Ty, nullptr); 730 } 731 732 int PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, 733 Type *SubTp) { 734 // Legalize the type. 735 std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp); 736 737 // PPC, for both Altivec/VSX and QPX, support cheap arbitrary permutations 738 // (at least in the sense that there need only be one non-loop-invariant 739 // instruction). We need one such shuffle instruction for each actual 740 // register (this is not true for arbitrary shuffles, but is true for the 741 // structured types of shuffles covered by TTI::ShuffleKind). 742 return vectorCostAdjustment(LT.first, Instruction::ShuffleVector, Tp, 743 nullptr); 744 } 745 746 int PPCTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, 747 const Instruction *I) { 748 assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode"); 749 750 int Cost = BaseT::getCastInstrCost(Opcode, Dst, Src); 751 return vectorCostAdjustment(Cost, Opcode, Dst, Src); 752 } 753 754 int PPCTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy, 755 const Instruction *I) { 756 int Cost = BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, I); 757 return vectorCostAdjustment(Cost, Opcode, ValTy, nullptr); 758 } 759 760 int PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) { 761 assert(Val->isVectorTy() && "This must be a vector type"); 762 763 int ISD = TLI->InstructionOpcodeToISD(Opcode); 764 assert(ISD && "Invalid opcode"); 765 766 int Cost = BaseT::getVectorInstrCost(Opcode, Val, Index); 767 Cost = vectorCostAdjustment(Cost, Opcode, Val, nullptr); 768 769 if (ST->hasVSX() && Val->getScalarType()->isDoubleTy()) { 770 // Double-precision scalars are already located in index #0 (or #1 if LE). 771 if (ISD == ISD::EXTRACT_VECTOR_ELT && 772 Index == (ST->isLittleEndian() ? 1 : 0)) 773 return 0; 774 775 return Cost; 776 777 } else if (ST->hasQPX() && Val->getScalarType()->isFloatingPointTy()) { 778 // Floating point scalars are already located in index #0. 779 if (Index == 0) 780 return 0; 781 782 return Cost; 783 784 } else if (Val->getScalarType()->isIntegerTy() && Index != -1U) { 785 if (ST->hasP9Altivec()) { 786 if (ISD == ISD::INSERT_VECTOR_ELT) 787 // A move-to VSR and a permute/insert. Assume vector operation cost 788 // for both (cost will be 2x on P9). 789 return vectorCostAdjustment(2, Opcode, Val, nullptr); 790 791 // It's an extract. Maybe we can do a cheap move-from VSR. 792 unsigned EltSize = Val->getScalarSizeInBits(); 793 if (EltSize == 64) { 794 unsigned MfvsrdIndex = ST->isLittleEndian() ? 1 : 0; 795 if (Index == MfvsrdIndex) 796 return 1; 797 } else if (EltSize == 32) { 798 unsigned MfvsrwzIndex = ST->isLittleEndian() ? 2 : 1; 799 if (Index == MfvsrwzIndex) 800 return 1; 801 } 802 803 // We need a vector extract (or mfvsrld). Assume vector operation cost. 804 // The cost of the load constant for a vector extract is disregarded 805 // (invariant, easily schedulable). 806 return vectorCostAdjustment(1, Opcode, Val, nullptr); 807 808 } else if (ST->hasDirectMove()) 809 // Assume permute has standard cost. 810 // Assume move-to/move-from VSR have 2x standard cost. 811 return 3; 812 } 813 814 // Estimated cost of a load-hit-store delay. This was obtained 815 // experimentally as a minimum needed to prevent unprofitable 816 // vectorization for the paq8p benchmark. It may need to be 817 // raised further if other unprofitable cases remain. 818 unsigned LHSPenalty = 2; 819 if (ISD == ISD::INSERT_VECTOR_ELT) 820 LHSPenalty += 7; 821 822 // Vector element insert/extract with Altivec is very expensive, 823 // because they require store and reload with the attendant 824 // processor stall for load-hit-store. Until VSX is available, 825 // these need to be estimated as very costly. 826 if (ISD == ISD::EXTRACT_VECTOR_ELT || 827 ISD == ISD::INSERT_VECTOR_ELT) 828 return LHSPenalty + Cost; 829 830 return Cost; 831 } 832 833 int PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, 834 MaybeAlign Alignment, unsigned AddressSpace, 835 const Instruction *I) { 836 // Legalize the type. 837 std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Src); 838 assert((Opcode == Instruction::Load || Opcode == Instruction::Store) && 839 "Invalid Opcode"); 840 841 int Cost = BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace); 842 Cost = vectorCostAdjustment(Cost, Opcode, Src, nullptr); 843 844 bool IsAltivecType = ST->hasAltivec() && 845 (LT.second == MVT::v16i8 || LT.second == MVT::v8i16 || 846 LT.second == MVT::v4i32 || LT.second == MVT::v4f32); 847 bool IsVSXType = ST->hasVSX() && 848 (LT.second == MVT::v2f64 || LT.second == MVT::v2i64); 849 bool IsQPXType = ST->hasQPX() && 850 (LT.second == MVT::v4f64 || LT.second == MVT::v4f32); 851 852 // VSX has 32b/64b load instructions. Legalization can handle loading of 853 // 32b/64b to VSR correctly and cheaply. But BaseT::getMemoryOpCost and 854 // PPCTargetLowering can't compute the cost appropriately. So here we 855 // explicitly check this case. 856 unsigned MemBytes = Src->getPrimitiveSizeInBits(); 857 if (Opcode == Instruction::Load && ST->hasVSX() && IsAltivecType && 858 (MemBytes == 64 || (ST->hasP8Vector() && MemBytes == 32))) 859 return 1; 860 861 // Aligned loads and stores are easy. 862 unsigned SrcBytes = LT.second.getStoreSize(); 863 if (!SrcBytes || !Alignment || Alignment >= SrcBytes) 864 return Cost; 865 866 // If we can use the permutation-based load sequence, then this is also 867 // relatively cheap (not counting loop-invariant instructions): one load plus 868 // one permute (the last load in a series has extra cost, but we're 869 // neglecting that here). Note that on the P7, we could do unaligned loads 870 // for Altivec types using the VSX instructions, but that's more expensive 871 // than using the permutation-based load sequence. On the P8, that's no 872 // longer true. 873 if (Opcode == Instruction::Load && 874 ((!ST->hasP8Vector() && IsAltivecType) || IsQPXType) && 875 Alignment >= LT.second.getScalarType().getStoreSize()) 876 return Cost + LT.first; // Add the cost of the permutations. 877 878 // For VSX, we can do unaligned loads and stores on Altivec/VSX types. On the 879 // P7, unaligned vector loads are more expensive than the permutation-based 880 // load sequence, so that might be used instead, but regardless, the net cost 881 // is about the same (not counting loop-invariant instructions). 882 if (IsVSXType || (ST->hasVSX() && IsAltivecType)) 883 return Cost; 884 885 // Newer PPC supports unaligned memory access. 886 if (TLI->allowsMisalignedMemoryAccesses(LT.second, 0)) 887 return Cost; 888 889 // PPC in general does not support unaligned loads and stores. They'll need 890 // to be decomposed based on the alignment factor. 891 892 // Add the cost of each scalar load or store. 893 assert(Alignment); 894 Cost += LT.first * ((SrcBytes / Alignment->value()) - 1); 895 896 // For a vector type, there is also scalarization overhead (only for 897 // stores, loads are expanded using the vector-load + permutation sequence, 898 // which is much less expensive). 899 if (Src->isVectorTy() && Opcode == Instruction::Store) 900 for (int i = 0, e = Src->getVectorNumElements(); i < e; ++i) 901 Cost += getVectorInstrCost(Instruction::ExtractElement, Src, i); 902 903 return Cost; 904 } 905 906 int PPCTTIImpl::getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, 907 unsigned Factor, 908 ArrayRef<unsigned> Indices, 909 unsigned Alignment, 910 unsigned AddressSpace, 911 bool UseMaskForCond, 912 bool UseMaskForGaps) { 913 if (UseMaskForCond || UseMaskForGaps) 914 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, 915 Alignment, AddressSpace, 916 UseMaskForCond, UseMaskForGaps); 917 918 assert(isa<VectorType>(VecTy) && 919 "Expect a vector type for interleaved memory op"); 920 921 // Legalize the type. 922 std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, VecTy); 923 924 // Firstly, the cost of load/store operation. 925 int Cost = 926 getMemoryOpCost(Opcode, VecTy, MaybeAlign(Alignment), AddressSpace); 927 928 // PPC, for both Altivec/VSX and QPX, support cheap arbitrary permutations 929 // (at least in the sense that there need only be one non-loop-invariant 930 // instruction). For each result vector, we need one shuffle per incoming 931 // vector (except that the first shuffle can take two incoming vectors 932 // because it does not need to take itself). 933 Cost += Factor*(LT.first-1); 934 935 return Cost; 936 } 937 938 unsigned PPCTTIImpl::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, 939 ArrayRef<Value*> Args, FastMathFlags FMF, unsigned VF) { 940 return BaseT::getIntrinsicInstrCost(ID, RetTy, Args, FMF, VF); 941 } 942 943 unsigned PPCTTIImpl::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, 944 ArrayRef<Type*> Tys, FastMathFlags FMF, 945 unsigned ScalarizationCostPassed) { 946 if (ID == Intrinsic::bswap && ST->hasP9Vector()) 947 return TLI->getTypeLegalizationCost(DL, RetTy).first; 948 return BaseT::getIntrinsicInstrCost(ID, RetTy, Tys, FMF, 949 ScalarizationCostPassed); 950 } 951 952 bool PPCTTIImpl::canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, 953 LoopInfo *LI, DominatorTree *DT, 954 AssumptionCache *AC, TargetLibraryInfo *LibInfo) { 955 // Process nested loops first. 956 for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) 957 if (canSaveCmp(*I, BI, SE, LI, DT, AC, LibInfo)) 958 return false; // Stop search. 959 960 HardwareLoopInfo HWLoopInfo(L); 961 962 if (!HWLoopInfo.canAnalyze(*LI)) 963 return false; 964 965 if (!isHardwareLoopProfitable(L, *SE, *AC, LibInfo, HWLoopInfo)) 966 return false; 967 968 if (!HWLoopInfo.isHardwareLoopCandidate(*SE, *LI, *DT)) 969 return false; 970 971 *BI = HWLoopInfo.ExitBranch; 972 return true; 973 } 974 975 bool PPCTTIImpl::isLSRCostLess(TargetTransformInfo::LSRCost &C1, 976 TargetTransformInfo::LSRCost &C2) { 977 // PowerPC default behaviour here is "instruction number 1st priority". 978 // If LsrNoInsnsCost is set, call default implementation. 979 if (!LsrNoInsnsCost) 980 return std::tie(C1.Insns, C1.NumRegs, C1.AddRecCost, C1.NumIVMuls, 981 C1.NumBaseAdds, C1.ScaleCost, C1.ImmCost, C1.SetupCost) < 982 std::tie(C2.Insns, C2.NumRegs, C2.AddRecCost, C2.NumIVMuls, 983 C2.NumBaseAdds, C2.ScaleCost, C2.ImmCost, C2.SetupCost); 984 else 985 return TargetTransformInfoImplBase::isLSRCostLess(C1, C2); 986 } 987