1 //===-- PPCTargetTransformInfo.cpp - PPC specific TTI ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "PPCTargetTransformInfo.h" 10 #include "llvm/Analysis/CodeMetrics.h" 11 #include "llvm/Analysis/TargetLibraryInfo.h" 12 #include "llvm/Analysis/TargetTransformInfo.h" 13 #include "llvm/CodeGen/BasicTTIImpl.h" 14 #include "llvm/CodeGen/CostTable.h" 15 #include "llvm/CodeGen/TargetLowering.h" 16 #include "llvm/CodeGen/TargetSchedule.h" 17 #include "llvm/IR/IntrinsicsPowerPC.h" 18 #include "llvm/Support/CommandLine.h" 19 #include "llvm/Support/Debug.h" 20 #include "llvm/Support/KnownBits.h" 21 #include "llvm/Transforms/InstCombine/InstCombiner.h" 22 #include "llvm/Transforms/Utils/Local.h" 23 24 using namespace llvm; 25 26 #define DEBUG_TYPE "ppctti" 27 28 static cl::opt<bool> DisablePPCConstHoist("disable-ppc-constant-hoisting", 29 cl::desc("disable constant hoisting on PPC"), cl::init(false), cl::Hidden); 30 31 // This is currently only used for the data prefetch pass 32 static cl::opt<unsigned> 33 CacheLineSize("ppc-loop-prefetch-cache-line", cl::Hidden, cl::init(64), 34 cl::desc("The loop prefetch cache line size")); 35 36 static cl::opt<bool> 37 EnablePPCColdCC("ppc-enable-coldcc", cl::Hidden, cl::init(false), 38 cl::desc("Enable using coldcc calling conv for cold " 39 "internal functions")); 40 41 static cl::opt<bool> 42 LsrNoInsnsCost("ppc-lsr-no-insns-cost", cl::Hidden, cl::init(false), 43 cl::desc("Do not add instruction count to lsr cost model")); 44 45 // The latency of mtctr is only justified if there are more than 4 46 // comparisons that will be removed as a result. 47 static cl::opt<unsigned> 48 SmallCTRLoopThreshold("min-ctr-loop-threshold", cl::init(4), cl::Hidden, 49 cl::desc("Loops with a constant trip count smaller than " 50 "this value will not use the count register.")); 51 52 //===----------------------------------------------------------------------===// 53 // 54 // PPC cost model. 55 // 56 //===----------------------------------------------------------------------===// 57 58 TargetTransformInfo::PopcntSupportKind 59 PPCTTIImpl::getPopcntSupport(unsigned TyWidth) { 60 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2"); 61 if (ST->hasPOPCNTD() != PPCSubtarget::POPCNTD_Unavailable && TyWidth <= 64) 62 return ST->hasPOPCNTD() == PPCSubtarget::POPCNTD_Slow ? 63 TTI::PSK_SlowHardware : TTI::PSK_FastHardware; 64 return TTI::PSK_Software; 65 } 66 67 Optional<Instruction *> 68 PPCTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { 69 Intrinsic::ID IID = II.getIntrinsicID(); 70 switch (IID) { 71 default: 72 break; 73 case Intrinsic::ppc_altivec_lvx: 74 case Intrinsic::ppc_altivec_lvxl: 75 // Turn PPC lvx -> load if the pointer is known aligned. 76 if (getOrEnforceKnownAlignment( 77 II.getArgOperand(0), Align(16), IC.getDataLayout(), &II, 78 &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) { 79 Value *Ptr = IC.Builder.CreateBitCast( 80 II.getArgOperand(0), PointerType::getUnqual(II.getType())); 81 return new LoadInst(II.getType(), Ptr, "", false, Align(16)); 82 } 83 break; 84 case Intrinsic::ppc_vsx_lxvw4x: 85 case Intrinsic::ppc_vsx_lxvd2x: { 86 // Turn PPC VSX loads into normal loads. 87 Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(0), 88 PointerType::getUnqual(II.getType())); 89 return new LoadInst(II.getType(), Ptr, Twine(""), false, Align(1)); 90 } 91 case Intrinsic::ppc_altivec_stvx: 92 case Intrinsic::ppc_altivec_stvxl: 93 // Turn stvx -> store if the pointer is known aligned. 94 if (getOrEnforceKnownAlignment( 95 II.getArgOperand(1), Align(16), IC.getDataLayout(), &II, 96 &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) { 97 Type *OpPtrTy = PointerType::getUnqual(II.getArgOperand(0)->getType()); 98 Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy); 99 return new StoreInst(II.getArgOperand(0), Ptr, false, Align(16)); 100 } 101 break; 102 case Intrinsic::ppc_vsx_stxvw4x: 103 case Intrinsic::ppc_vsx_stxvd2x: { 104 // Turn PPC VSX stores into normal stores. 105 Type *OpPtrTy = PointerType::getUnqual(II.getArgOperand(0)->getType()); 106 Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy); 107 return new StoreInst(II.getArgOperand(0), Ptr, false, Align(1)); 108 } 109 case Intrinsic::ppc_altivec_vperm: 110 // Turn vperm(V1,V2,mask) -> shuffle(V1,V2,mask) if mask is a constant. 111 // Note that ppc_altivec_vperm has a big-endian bias, so when creating 112 // a vectorshuffle for little endian, we must undo the transformation 113 // performed on vec_perm in altivec.h. That is, we must complement 114 // the permutation mask with respect to 31 and reverse the order of 115 // V1 and V2. 116 if (Constant *Mask = dyn_cast<Constant>(II.getArgOperand(2))) { 117 assert(cast<FixedVectorType>(Mask->getType())->getNumElements() == 16 && 118 "Bad type for intrinsic!"); 119 120 // Check that all of the elements are integer constants or undefs. 121 bool AllEltsOk = true; 122 for (unsigned i = 0; i != 16; ++i) { 123 Constant *Elt = Mask->getAggregateElement(i); 124 if (!Elt || !(isa<ConstantInt>(Elt) || isa<UndefValue>(Elt))) { 125 AllEltsOk = false; 126 break; 127 } 128 } 129 130 if (AllEltsOk) { 131 // Cast the input vectors to byte vectors. 132 Value *Op0 = 133 IC.Builder.CreateBitCast(II.getArgOperand(0), Mask->getType()); 134 Value *Op1 = 135 IC.Builder.CreateBitCast(II.getArgOperand(1), Mask->getType()); 136 Value *Result = UndefValue::get(Op0->getType()); 137 138 // Only extract each element once. 139 Value *ExtractedElts[32]; 140 memset(ExtractedElts, 0, sizeof(ExtractedElts)); 141 142 for (unsigned i = 0; i != 16; ++i) { 143 if (isa<UndefValue>(Mask->getAggregateElement(i))) 144 continue; 145 unsigned Idx = 146 cast<ConstantInt>(Mask->getAggregateElement(i))->getZExtValue(); 147 Idx &= 31; // Match the hardware behavior. 148 if (DL.isLittleEndian()) 149 Idx = 31 - Idx; 150 151 if (!ExtractedElts[Idx]) { 152 Value *Op0ToUse = (DL.isLittleEndian()) ? Op1 : Op0; 153 Value *Op1ToUse = (DL.isLittleEndian()) ? Op0 : Op1; 154 ExtractedElts[Idx] = IC.Builder.CreateExtractElement( 155 Idx < 16 ? Op0ToUse : Op1ToUse, IC.Builder.getInt32(Idx & 15)); 156 } 157 158 // Insert this value into the result vector. 159 Result = IC.Builder.CreateInsertElement(Result, ExtractedElts[Idx], 160 IC.Builder.getInt32(i)); 161 } 162 return CastInst::Create(Instruction::BitCast, Result, II.getType()); 163 } 164 } 165 break; 166 } 167 return None; 168 } 169 170 int PPCTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty, 171 TTI::TargetCostKind CostKind) { 172 if (DisablePPCConstHoist) 173 return BaseT::getIntImmCost(Imm, Ty, CostKind); 174 175 assert(Ty->isIntegerTy()); 176 177 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 178 if (BitSize == 0) 179 return ~0U; 180 181 if (Imm == 0) 182 return TTI::TCC_Free; 183 184 if (Imm.getBitWidth() <= 64) { 185 if (isInt<16>(Imm.getSExtValue())) 186 return TTI::TCC_Basic; 187 188 if (isInt<32>(Imm.getSExtValue())) { 189 // A constant that can be materialized using lis. 190 if ((Imm.getZExtValue() & 0xFFFF) == 0) 191 return TTI::TCC_Basic; 192 193 return 2 * TTI::TCC_Basic; 194 } 195 } 196 197 return 4 * TTI::TCC_Basic; 198 } 199 200 int PPCTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, 201 const APInt &Imm, Type *Ty, 202 TTI::TargetCostKind CostKind) { 203 if (DisablePPCConstHoist) 204 return BaseT::getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind); 205 206 assert(Ty->isIntegerTy()); 207 208 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 209 if (BitSize == 0) 210 return ~0U; 211 212 switch (IID) { 213 default: 214 return TTI::TCC_Free; 215 case Intrinsic::sadd_with_overflow: 216 case Intrinsic::uadd_with_overflow: 217 case Intrinsic::ssub_with_overflow: 218 case Intrinsic::usub_with_overflow: 219 if ((Idx == 1) && Imm.getBitWidth() <= 64 && isInt<16>(Imm.getSExtValue())) 220 return TTI::TCC_Free; 221 break; 222 case Intrinsic::experimental_stackmap: 223 if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))) 224 return TTI::TCC_Free; 225 break; 226 case Intrinsic::experimental_patchpoint_void: 227 case Intrinsic::experimental_patchpoint_i64: 228 if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))) 229 return TTI::TCC_Free; 230 break; 231 } 232 return PPCTTIImpl::getIntImmCost(Imm, Ty, CostKind); 233 } 234 235 int PPCTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx, 236 const APInt &Imm, Type *Ty, 237 TTI::TargetCostKind CostKind) { 238 if (DisablePPCConstHoist) 239 return BaseT::getIntImmCostInst(Opcode, Idx, Imm, Ty, CostKind); 240 241 assert(Ty->isIntegerTy()); 242 243 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 244 if (BitSize == 0) 245 return ~0U; 246 247 unsigned ImmIdx = ~0U; 248 bool ShiftedFree = false, RunFree = false, UnsignedFree = false, 249 ZeroFree = false; 250 switch (Opcode) { 251 default: 252 return TTI::TCC_Free; 253 case Instruction::GetElementPtr: 254 // Always hoist the base address of a GetElementPtr. This prevents the 255 // creation of new constants for every base constant that gets constant 256 // folded with the offset. 257 if (Idx == 0) 258 return 2 * TTI::TCC_Basic; 259 return TTI::TCC_Free; 260 case Instruction::And: 261 RunFree = true; // (for the rotate-and-mask instructions) 262 LLVM_FALLTHROUGH; 263 case Instruction::Add: 264 case Instruction::Or: 265 case Instruction::Xor: 266 ShiftedFree = true; 267 LLVM_FALLTHROUGH; 268 case Instruction::Sub: 269 case Instruction::Mul: 270 case Instruction::Shl: 271 case Instruction::LShr: 272 case Instruction::AShr: 273 ImmIdx = 1; 274 break; 275 case Instruction::ICmp: 276 UnsignedFree = true; 277 ImmIdx = 1; 278 // Zero comparisons can use record-form instructions. 279 LLVM_FALLTHROUGH; 280 case Instruction::Select: 281 ZeroFree = true; 282 break; 283 case Instruction::PHI: 284 case Instruction::Call: 285 case Instruction::Ret: 286 case Instruction::Load: 287 case Instruction::Store: 288 break; 289 } 290 291 if (ZeroFree && Imm == 0) 292 return TTI::TCC_Free; 293 294 if (Idx == ImmIdx && Imm.getBitWidth() <= 64) { 295 if (isInt<16>(Imm.getSExtValue())) 296 return TTI::TCC_Free; 297 298 if (RunFree) { 299 if (Imm.getBitWidth() <= 32 && 300 (isShiftedMask_32(Imm.getZExtValue()) || 301 isShiftedMask_32(~Imm.getZExtValue()))) 302 return TTI::TCC_Free; 303 304 if (ST->isPPC64() && 305 (isShiftedMask_64(Imm.getZExtValue()) || 306 isShiftedMask_64(~Imm.getZExtValue()))) 307 return TTI::TCC_Free; 308 } 309 310 if (UnsignedFree && isUInt<16>(Imm.getZExtValue())) 311 return TTI::TCC_Free; 312 313 if (ShiftedFree && (Imm.getZExtValue() & 0xFFFF) == 0) 314 return TTI::TCC_Free; 315 } 316 317 return PPCTTIImpl::getIntImmCost(Imm, Ty, CostKind); 318 } 319 320 unsigned 321 PPCTTIImpl::getUserCost(const User *U, ArrayRef<const Value *> Operands, 322 TTI::TargetCostKind CostKind) { 323 // We already implement getCastInstrCost and getMemoryOpCost where we perform 324 // the vector adjustment there. 325 if (isa<CastInst>(U) || isa<LoadInst>(U) || isa<StoreInst>(U)) 326 return BaseT::getUserCost(U, Operands, CostKind); 327 328 if (U->getType()->isVectorTy()) { 329 // Instructions that need to be split should cost more. 330 std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, U->getType()); 331 return LT.first * BaseT::getUserCost(U, Operands, CostKind); 332 } 333 334 return BaseT::getUserCost(U, Operands, CostKind); 335 } 336 337 bool PPCTTIImpl::mightUseCTR(BasicBlock *BB, TargetLibraryInfo *LibInfo, 338 SmallPtrSetImpl<const Value *> &Visited) { 339 const PPCTargetMachine &TM = ST->getTargetMachine(); 340 341 // Loop through the inline asm constraints and look for something that 342 // clobbers ctr. 343 auto asmClobbersCTR = [](InlineAsm *IA) { 344 InlineAsm::ConstraintInfoVector CIV = IA->ParseConstraints(); 345 for (unsigned i = 0, ie = CIV.size(); i < ie; ++i) { 346 InlineAsm::ConstraintInfo &C = CIV[i]; 347 if (C.Type != InlineAsm::isInput) 348 for (unsigned j = 0, je = C.Codes.size(); j < je; ++j) 349 if (StringRef(C.Codes[j]).equals_lower("{ctr}")) 350 return true; 351 } 352 return false; 353 }; 354 355 // Determining the address of a TLS variable results in a function call in 356 // certain TLS models. 357 std::function<bool(const Value *)> memAddrUsesCTR = 358 [&memAddrUsesCTR, &TM, &Visited](const Value *MemAddr) -> bool { 359 // No need to traverse again if we already checked this operand. 360 if (!Visited.insert(MemAddr).second) 361 return false; 362 const auto *GV = dyn_cast<GlobalValue>(MemAddr); 363 if (!GV) { 364 // Recurse to check for constants that refer to TLS global variables. 365 if (const auto *CV = dyn_cast<Constant>(MemAddr)) 366 for (const auto &CO : CV->operands()) 367 if (memAddrUsesCTR(CO)) 368 return true; 369 370 return false; 371 } 372 373 if (!GV->isThreadLocal()) 374 return false; 375 TLSModel::Model Model = TM.getTLSModel(GV); 376 return Model == TLSModel::GeneralDynamic || 377 Model == TLSModel::LocalDynamic; 378 }; 379 380 auto isLargeIntegerTy = [](bool Is32Bit, Type *Ty) { 381 if (IntegerType *ITy = dyn_cast<IntegerType>(Ty)) 382 return ITy->getBitWidth() > (Is32Bit ? 32U : 64U); 383 384 return false; 385 }; 386 387 for (BasicBlock::iterator J = BB->begin(), JE = BB->end(); 388 J != JE; ++J) { 389 if (CallInst *CI = dyn_cast<CallInst>(J)) { 390 // Inline ASM is okay, unless it clobbers the ctr register. 391 if (InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledOperand())) { 392 if (asmClobbersCTR(IA)) 393 return true; 394 continue; 395 } 396 397 if (Function *F = CI->getCalledFunction()) { 398 // Most intrinsics don't become function calls, but some might. 399 // sin, cos, exp and log are always calls. 400 unsigned Opcode = 0; 401 if (F->getIntrinsicID() != Intrinsic::not_intrinsic) { 402 switch (F->getIntrinsicID()) { 403 default: continue; 404 // If we have a call to loop_decrement or set_loop_iterations, 405 // we're definitely using CTR. 406 case Intrinsic::set_loop_iterations: 407 case Intrinsic::loop_decrement: 408 return true; 409 410 // Binary operations on 128-bit value will use CTR. 411 case Intrinsic::experimental_constrained_fadd: 412 case Intrinsic::experimental_constrained_fsub: 413 case Intrinsic::experimental_constrained_fmul: 414 case Intrinsic::experimental_constrained_fdiv: 415 case Intrinsic::experimental_constrained_frem: 416 if (F->getType()->getScalarType()->isFP128Ty() || 417 F->getType()->getScalarType()->isPPC_FP128Ty()) 418 return true; 419 break; 420 421 case Intrinsic::experimental_constrained_fptosi: 422 case Intrinsic::experimental_constrained_fptoui: 423 case Intrinsic::experimental_constrained_sitofp: 424 case Intrinsic::experimental_constrained_uitofp: { 425 Type *SrcType = CI->getArgOperand(0)->getType()->getScalarType(); 426 Type *DstType = CI->getType()->getScalarType(); 427 if (SrcType->isPPC_FP128Ty() || DstType->isPPC_FP128Ty() || 428 isLargeIntegerTy(!TM.isPPC64(), SrcType) || 429 isLargeIntegerTy(!TM.isPPC64(), DstType)) 430 return true; 431 break; 432 } 433 434 // Exclude eh_sjlj_setjmp; we don't need to exclude eh_sjlj_longjmp 435 // because, although it does clobber the counter register, the 436 // control can't then return to inside the loop unless there is also 437 // an eh_sjlj_setjmp. 438 case Intrinsic::eh_sjlj_setjmp: 439 440 case Intrinsic::memcpy: 441 case Intrinsic::memmove: 442 case Intrinsic::memset: 443 case Intrinsic::powi: 444 case Intrinsic::log: 445 case Intrinsic::log2: 446 case Intrinsic::log10: 447 case Intrinsic::exp: 448 case Intrinsic::exp2: 449 case Intrinsic::pow: 450 case Intrinsic::sin: 451 case Intrinsic::cos: 452 case Intrinsic::experimental_constrained_powi: 453 case Intrinsic::experimental_constrained_log: 454 case Intrinsic::experimental_constrained_log2: 455 case Intrinsic::experimental_constrained_log10: 456 case Intrinsic::experimental_constrained_exp: 457 case Intrinsic::experimental_constrained_exp2: 458 case Intrinsic::experimental_constrained_pow: 459 case Intrinsic::experimental_constrained_sin: 460 case Intrinsic::experimental_constrained_cos: 461 return true; 462 case Intrinsic::copysign: 463 if (CI->getArgOperand(0)->getType()->getScalarType()-> 464 isPPC_FP128Ty()) 465 return true; 466 else 467 continue; // ISD::FCOPYSIGN is never a library call. 468 case Intrinsic::fma: Opcode = ISD::FMA; break; 469 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break; 470 case Intrinsic::floor: Opcode = ISD::FFLOOR; break; 471 case Intrinsic::ceil: Opcode = ISD::FCEIL; break; 472 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break; 473 case Intrinsic::rint: Opcode = ISD::FRINT; break; 474 case Intrinsic::lrint: Opcode = ISD::LRINT; break; 475 case Intrinsic::llrint: Opcode = ISD::LLRINT; break; 476 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break; 477 case Intrinsic::round: Opcode = ISD::FROUND; break; 478 case Intrinsic::lround: Opcode = ISD::LROUND; break; 479 case Intrinsic::llround: Opcode = ISD::LLROUND; break; 480 case Intrinsic::minnum: Opcode = ISD::FMINNUM; break; 481 case Intrinsic::maxnum: Opcode = ISD::FMAXNUM; break; 482 case Intrinsic::experimental_constrained_fcmp: 483 Opcode = ISD::STRICT_FSETCC; 484 break; 485 case Intrinsic::experimental_constrained_fcmps: 486 Opcode = ISD::STRICT_FSETCCS; 487 break; 488 case Intrinsic::experimental_constrained_fma: 489 Opcode = ISD::STRICT_FMA; 490 break; 491 case Intrinsic::experimental_constrained_sqrt: 492 Opcode = ISD::STRICT_FSQRT; 493 break; 494 case Intrinsic::experimental_constrained_floor: 495 Opcode = ISD::STRICT_FFLOOR; 496 break; 497 case Intrinsic::experimental_constrained_ceil: 498 Opcode = ISD::STRICT_FCEIL; 499 break; 500 case Intrinsic::experimental_constrained_trunc: 501 Opcode = ISD::STRICT_FTRUNC; 502 break; 503 case Intrinsic::experimental_constrained_rint: 504 Opcode = ISD::STRICT_FRINT; 505 break; 506 case Intrinsic::experimental_constrained_lrint: 507 Opcode = ISD::STRICT_LRINT; 508 break; 509 case Intrinsic::experimental_constrained_llrint: 510 Opcode = ISD::STRICT_LLRINT; 511 break; 512 case Intrinsic::experimental_constrained_nearbyint: 513 Opcode = ISD::STRICT_FNEARBYINT; 514 break; 515 case Intrinsic::experimental_constrained_round: 516 Opcode = ISD::STRICT_FROUND; 517 break; 518 case Intrinsic::experimental_constrained_lround: 519 Opcode = ISD::STRICT_LROUND; 520 break; 521 case Intrinsic::experimental_constrained_llround: 522 Opcode = ISD::STRICT_LLROUND; 523 break; 524 case Intrinsic::experimental_constrained_minnum: 525 Opcode = ISD::STRICT_FMINNUM; 526 break; 527 case Intrinsic::experimental_constrained_maxnum: 528 Opcode = ISD::STRICT_FMAXNUM; 529 break; 530 case Intrinsic::umul_with_overflow: Opcode = ISD::UMULO; break; 531 case Intrinsic::smul_with_overflow: Opcode = ISD::SMULO; break; 532 } 533 } 534 535 // PowerPC does not use [US]DIVREM or other library calls for 536 // operations on regular types which are not otherwise library calls 537 // (i.e. soft float or atomics). If adapting for targets that do, 538 // additional care is required here. 539 540 LibFunc Func; 541 if (!F->hasLocalLinkage() && F->hasName() && LibInfo && 542 LibInfo->getLibFunc(F->getName(), Func) && 543 LibInfo->hasOptimizedCodeGen(Func)) { 544 // Non-read-only functions are never treated as intrinsics. 545 if (!CI->onlyReadsMemory()) 546 return true; 547 548 // Conversion happens only for FP calls. 549 if (!CI->getArgOperand(0)->getType()->isFloatingPointTy()) 550 return true; 551 552 switch (Func) { 553 default: return true; 554 case LibFunc_copysign: 555 case LibFunc_copysignf: 556 continue; // ISD::FCOPYSIGN is never a library call. 557 case LibFunc_copysignl: 558 return true; 559 case LibFunc_fabs: 560 case LibFunc_fabsf: 561 case LibFunc_fabsl: 562 continue; // ISD::FABS is never a library call. 563 case LibFunc_sqrt: 564 case LibFunc_sqrtf: 565 case LibFunc_sqrtl: 566 Opcode = ISD::FSQRT; break; 567 case LibFunc_floor: 568 case LibFunc_floorf: 569 case LibFunc_floorl: 570 Opcode = ISD::FFLOOR; break; 571 case LibFunc_nearbyint: 572 case LibFunc_nearbyintf: 573 case LibFunc_nearbyintl: 574 Opcode = ISD::FNEARBYINT; break; 575 case LibFunc_ceil: 576 case LibFunc_ceilf: 577 case LibFunc_ceill: 578 Opcode = ISD::FCEIL; break; 579 case LibFunc_rint: 580 case LibFunc_rintf: 581 case LibFunc_rintl: 582 Opcode = ISD::FRINT; break; 583 case LibFunc_round: 584 case LibFunc_roundf: 585 case LibFunc_roundl: 586 Opcode = ISD::FROUND; break; 587 case LibFunc_trunc: 588 case LibFunc_truncf: 589 case LibFunc_truncl: 590 Opcode = ISD::FTRUNC; break; 591 case LibFunc_fmin: 592 case LibFunc_fminf: 593 case LibFunc_fminl: 594 Opcode = ISD::FMINNUM; break; 595 case LibFunc_fmax: 596 case LibFunc_fmaxf: 597 case LibFunc_fmaxl: 598 Opcode = ISD::FMAXNUM; break; 599 } 600 } 601 602 if (Opcode) { 603 EVT EVTy = 604 TLI->getValueType(DL, CI->getArgOperand(0)->getType(), true); 605 606 if (EVTy == MVT::Other) 607 return true; 608 609 if (TLI->isOperationLegalOrCustom(Opcode, EVTy)) 610 continue; 611 else if (EVTy.isVector() && 612 TLI->isOperationLegalOrCustom(Opcode, EVTy.getScalarType())) 613 continue; 614 615 return true; 616 } 617 } 618 619 return true; 620 } else if (isa<BinaryOperator>(J) && 621 (J->getType()->getScalarType()->isFP128Ty() || 622 J->getType()->getScalarType()->isPPC_FP128Ty())) { 623 // Most operations on f128 or ppc_f128 values become calls. 624 return true; 625 } else if (isa<UIToFPInst>(J) || isa<SIToFPInst>(J) || 626 isa<FPToUIInst>(J) || isa<FPToSIInst>(J)) { 627 CastInst *CI = cast<CastInst>(J); 628 if (CI->getSrcTy()->getScalarType()->isPPC_FP128Ty() || 629 CI->getDestTy()->getScalarType()->isPPC_FP128Ty() || 630 isLargeIntegerTy(!TM.isPPC64(), CI->getSrcTy()->getScalarType()) || 631 isLargeIntegerTy(!TM.isPPC64(), CI->getDestTy()->getScalarType())) 632 return true; 633 } else if (isLargeIntegerTy(!TM.isPPC64(), 634 J->getType()->getScalarType()) && 635 (J->getOpcode() == Instruction::UDiv || 636 J->getOpcode() == Instruction::SDiv || 637 J->getOpcode() == Instruction::URem || 638 J->getOpcode() == Instruction::SRem)) { 639 return true; 640 } else if (!TM.isPPC64() && 641 isLargeIntegerTy(false, J->getType()->getScalarType()) && 642 (J->getOpcode() == Instruction::Shl || 643 J->getOpcode() == Instruction::AShr || 644 J->getOpcode() == Instruction::LShr)) { 645 // Only on PPC32, for 128-bit integers (specifically not 64-bit 646 // integers), these might be runtime calls. 647 return true; 648 } else if (isa<IndirectBrInst>(J) || isa<InvokeInst>(J)) { 649 // On PowerPC, indirect jumps use the counter register. 650 return true; 651 } else if (SwitchInst *SI = dyn_cast<SwitchInst>(J)) { 652 if (SI->getNumCases() + 1 >= (unsigned)TLI->getMinimumJumpTableEntries()) 653 return true; 654 } 655 656 // FREM is always a call. 657 if (J->getOpcode() == Instruction::FRem) 658 return true; 659 660 if (ST->useSoftFloat()) { 661 switch(J->getOpcode()) { 662 case Instruction::FAdd: 663 case Instruction::FSub: 664 case Instruction::FMul: 665 case Instruction::FDiv: 666 case Instruction::FPTrunc: 667 case Instruction::FPExt: 668 case Instruction::FPToUI: 669 case Instruction::FPToSI: 670 case Instruction::UIToFP: 671 case Instruction::SIToFP: 672 case Instruction::FCmp: 673 return true; 674 } 675 } 676 677 for (Value *Operand : J->operands()) 678 if (memAddrUsesCTR(Operand)) 679 return true; 680 } 681 682 return false; 683 } 684 685 bool PPCTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE, 686 AssumptionCache &AC, 687 TargetLibraryInfo *LibInfo, 688 HardwareLoopInfo &HWLoopInfo) { 689 const PPCTargetMachine &TM = ST->getTargetMachine(); 690 TargetSchedModel SchedModel; 691 SchedModel.init(ST); 692 693 // Do not convert small short loops to CTR loop. 694 unsigned ConstTripCount = SE.getSmallConstantTripCount(L); 695 if (ConstTripCount && ConstTripCount < SmallCTRLoopThreshold) { 696 SmallPtrSet<const Value *, 32> EphValues; 697 CodeMetrics::collectEphemeralValues(L, &AC, EphValues); 698 CodeMetrics Metrics; 699 for (BasicBlock *BB : L->blocks()) 700 Metrics.analyzeBasicBlock(BB, *this, EphValues); 701 // 6 is an approximate latency for the mtctr instruction. 702 if (Metrics.NumInsts <= (6 * SchedModel.getIssueWidth())) 703 return false; 704 } 705 706 // We don't want to spill/restore the counter register, and so we don't 707 // want to use the counter register if the loop contains calls. 708 SmallPtrSet<const Value *, 4> Visited; 709 for (Loop::block_iterator I = L->block_begin(), IE = L->block_end(); 710 I != IE; ++I) 711 if (mightUseCTR(*I, LibInfo, Visited)) 712 return false; 713 714 SmallVector<BasicBlock*, 4> ExitingBlocks; 715 L->getExitingBlocks(ExitingBlocks); 716 717 // If there is an exit edge known to be frequently taken, 718 // we should not transform this loop. 719 for (auto &BB : ExitingBlocks) { 720 Instruction *TI = BB->getTerminator(); 721 if (!TI) continue; 722 723 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) { 724 uint64_t TrueWeight = 0, FalseWeight = 0; 725 if (!BI->isConditional() || 726 !BI->extractProfMetadata(TrueWeight, FalseWeight)) 727 continue; 728 729 // If the exit path is more frequent than the loop path, 730 // we return here without further analysis for this loop. 731 bool TrueIsExit = !L->contains(BI->getSuccessor(0)); 732 if (( TrueIsExit && FalseWeight < TrueWeight) || 733 (!TrueIsExit && FalseWeight > TrueWeight)) 734 return false; 735 } 736 } 737 738 LLVMContext &C = L->getHeader()->getContext(); 739 HWLoopInfo.CountType = TM.isPPC64() ? 740 Type::getInt64Ty(C) : Type::getInt32Ty(C); 741 HWLoopInfo.LoopDecrement = ConstantInt::get(HWLoopInfo.CountType, 1); 742 return true; 743 } 744 745 void PPCTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, 746 TTI::UnrollingPreferences &UP) { 747 if (ST->getCPUDirective() == PPC::DIR_A2) { 748 // The A2 is in-order with a deep pipeline, and concatenation unrolling 749 // helps expose latency-hiding opportunities to the instruction scheduler. 750 UP.Partial = UP.Runtime = true; 751 752 // We unroll a lot on the A2 (hundreds of instructions), and the benefits 753 // often outweigh the cost of a division to compute the trip count. 754 UP.AllowExpensiveTripCount = true; 755 } 756 757 BaseT::getUnrollingPreferences(L, SE, UP); 758 } 759 760 void PPCTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE, 761 TTI::PeelingPreferences &PP) { 762 BaseT::getPeelingPreferences(L, SE, PP); 763 } 764 // This function returns true to allow using coldcc calling convention. 765 // Returning true results in coldcc being used for functions which are cold at 766 // all call sites when the callers of the functions are not calling any other 767 // non coldcc functions. 768 bool PPCTTIImpl::useColdCCForColdCall(Function &F) { 769 return EnablePPCColdCC; 770 } 771 772 bool PPCTTIImpl::enableAggressiveInterleaving(bool LoopHasReductions) { 773 // On the A2, always unroll aggressively. 774 if (ST->getCPUDirective() == PPC::DIR_A2) 775 return true; 776 777 return LoopHasReductions; 778 } 779 780 PPCTTIImpl::TTI::MemCmpExpansionOptions 781 PPCTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const { 782 TTI::MemCmpExpansionOptions Options; 783 Options.LoadSizes = {8, 4, 2, 1}; 784 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize); 785 return Options; 786 } 787 788 bool PPCTTIImpl::enableInterleavedAccessVectorization() { 789 return true; 790 } 791 792 unsigned PPCTTIImpl::getNumberOfRegisters(unsigned ClassID) const { 793 assert(ClassID == GPRRC || ClassID == FPRRC || 794 ClassID == VRRC || ClassID == VSXRC); 795 if (ST->hasVSX()) { 796 assert(ClassID == GPRRC || ClassID == VSXRC || ClassID == VRRC); 797 return ClassID == VSXRC ? 64 : 32; 798 } 799 assert(ClassID == GPRRC || ClassID == FPRRC || ClassID == VRRC); 800 return 32; 801 } 802 803 unsigned PPCTTIImpl::getRegisterClassForType(bool Vector, Type *Ty) const { 804 if (Vector) 805 return ST->hasVSX() ? VSXRC : VRRC; 806 else if (Ty && (Ty->getScalarType()->isFloatTy() || 807 Ty->getScalarType()->isDoubleTy())) 808 return ST->hasVSX() ? VSXRC : FPRRC; 809 else if (Ty && (Ty->getScalarType()->isFP128Ty() || 810 Ty->getScalarType()->isPPC_FP128Ty())) 811 return VRRC; 812 else if (Ty && Ty->getScalarType()->isHalfTy()) 813 return VSXRC; 814 else 815 return GPRRC; 816 } 817 818 const char* PPCTTIImpl::getRegisterClassName(unsigned ClassID) const { 819 820 switch (ClassID) { 821 default: 822 llvm_unreachable("unknown register class"); 823 return "PPC::unknown register class"; 824 case GPRRC: return "PPC::GPRRC"; 825 case FPRRC: return "PPC::FPRRC"; 826 case VRRC: return "PPC::VRRC"; 827 case VSXRC: return "PPC::VSXRC"; 828 } 829 } 830 831 unsigned PPCTTIImpl::getRegisterBitWidth(bool Vector) const { 832 if (Vector) { 833 if (ST->hasAltivec()) return 128; 834 return 0; 835 } 836 837 if (ST->isPPC64()) 838 return 64; 839 return 32; 840 841 } 842 843 unsigned PPCTTIImpl::getCacheLineSize() const { 844 // Check first if the user specified a custom line size. 845 if (CacheLineSize.getNumOccurrences() > 0) 846 return CacheLineSize; 847 848 // Starting with P7 we have a cache line size of 128. 849 unsigned Directive = ST->getCPUDirective(); 850 // Assume that Future CPU has the same cache line size as the others. 851 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8 || 852 Directive == PPC::DIR_PWR9 || Directive == PPC::DIR_PWR10 || 853 Directive == PPC::DIR_PWR_FUTURE) 854 return 128; 855 856 // On other processors return a default of 64 bytes. 857 return 64; 858 } 859 860 unsigned PPCTTIImpl::getPrefetchDistance() const { 861 return 300; 862 } 863 864 unsigned PPCTTIImpl::getMaxInterleaveFactor(unsigned VF) { 865 unsigned Directive = ST->getCPUDirective(); 866 // The 440 has no SIMD support, but floating-point instructions 867 // have a 5-cycle latency, so unroll by 5x for latency hiding. 868 if (Directive == PPC::DIR_440) 869 return 5; 870 871 // The A2 has no SIMD support, but floating-point instructions 872 // have a 6-cycle latency, so unroll by 6x for latency hiding. 873 if (Directive == PPC::DIR_A2) 874 return 6; 875 876 // FIXME: For lack of any better information, do no harm... 877 if (Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) 878 return 1; 879 880 // For P7 and P8, floating-point instructions have a 6-cycle latency and 881 // there are two execution units, so unroll by 12x for latency hiding. 882 // FIXME: the same for P9 as previous gen until POWER9 scheduling is ready 883 // FIXME: the same for P10 as previous gen until POWER10 scheduling is ready 884 // Assume that future is the same as the others. 885 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8 || 886 Directive == PPC::DIR_PWR9 || Directive == PPC::DIR_PWR10 || 887 Directive == PPC::DIR_PWR_FUTURE) 888 return 12; 889 890 // For most things, modern systems have two execution units (and 891 // out-of-order execution). 892 return 2; 893 } 894 895 // Adjust the cost of vector instructions on targets which there is overlap 896 // between the vector and scalar units, thereby reducing the overall throughput 897 // of vector code wrt. scalar code. 898 int PPCTTIImpl::vectorCostAdjustment(int Cost, unsigned Opcode, Type *Ty1, 899 Type *Ty2) { 900 if (!ST->vectorsUseTwoUnits() || !Ty1->isVectorTy()) 901 return Cost; 902 903 std::pair<int, MVT> LT1 = TLI->getTypeLegalizationCost(DL, Ty1); 904 // If type legalization involves splitting the vector, we don't want to 905 // double the cost at every step - only the last step. 906 if (LT1.first != 1 || !LT1.second.isVector()) 907 return Cost; 908 909 int ISD = TLI->InstructionOpcodeToISD(Opcode); 910 if (TLI->isOperationExpand(ISD, LT1.second)) 911 return Cost; 912 913 if (Ty2) { 914 std::pair<int, MVT> LT2 = TLI->getTypeLegalizationCost(DL, Ty2); 915 if (LT2.first != 1 || !LT2.second.isVector()) 916 return Cost; 917 } 918 919 return Cost * 2; 920 } 921 922 int PPCTTIImpl::getArithmeticInstrCost(unsigned Opcode, Type *Ty, 923 TTI::TargetCostKind CostKind, 924 TTI::OperandValueKind Op1Info, 925 TTI::OperandValueKind Op2Info, 926 TTI::OperandValueProperties Opd1PropInfo, 927 TTI::OperandValueProperties Opd2PropInfo, 928 ArrayRef<const Value *> Args, 929 const Instruction *CxtI) { 930 assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode"); 931 // TODO: Handle more cost kinds. 932 if (CostKind != TTI::TCK_RecipThroughput) 933 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, 934 Op2Info, Opd1PropInfo, 935 Opd2PropInfo, Args, CxtI); 936 937 // Fallback to the default implementation. 938 int Cost = BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, 939 Op2Info, 940 Opd1PropInfo, Opd2PropInfo); 941 return vectorCostAdjustment(Cost, Opcode, Ty, nullptr); 942 } 943 944 int PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, 945 Type *SubTp) { 946 // Legalize the type. 947 std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp); 948 949 // PPC, for both Altivec/VSX, support cheap arbitrary permutations 950 // (at least in the sense that there need only be one non-loop-invariant 951 // instruction). We need one such shuffle instruction for each actual 952 // register (this is not true for arbitrary shuffles, but is true for the 953 // structured types of shuffles covered by TTI::ShuffleKind). 954 return vectorCostAdjustment(LT.first, Instruction::ShuffleVector, Tp, 955 nullptr); 956 } 957 958 int PPCTTIImpl::getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind) { 959 if (CostKind != TTI::TCK_RecipThroughput) 960 return Opcode == Instruction::PHI ? 0 : 1; 961 // Branches are assumed to be predicted. 962 return CostKind == TTI::TCK_RecipThroughput ? 0 : 1; 963 } 964 965 int PPCTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, 966 TTI::CastContextHint CCH, 967 TTI::TargetCostKind CostKind, 968 const Instruction *I) { 969 assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode"); 970 971 int Cost = BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); 972 Cost = vectorCostAdjustment(Cost, Opcode, Dst, Src); 973 // TODO: Allow non-throughput costs that aren't binary. 974 if (CostKind != TTI::TCK_RecipThroughput) 975 return Cost == 0 ? 0 : 1; 976 return Cost; 977 } 978 979 int PPCTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy, 980 TTI::TargetCostKind CostKind, 981 const Instruction *I) { 982 int Cost = BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, CostKind, I); 983 // TODO: Handle other cost kinds. 984 if (CostKind != TTI::TCK_RecipThroughput) 985 return Cost; 986 return vectorCostAdjustment(Cost, Opcode, ValTy, nullptr); 987 } 988 989 int PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) { 990 assert(Val->isVectorTy() && "This must be a vector type"); 991 992 int ISD = TLI->InstructionOpcodeToISD(Opcode); 993 assert(ISD && "Invalid opcode"); 994 995 int Cost = BaseT::getVectorInstrCost(Opcode, Val, Index); 996 Cost = vectorCostAdjustment(Cost, Opcode, Val, nullptr); 997 998 if (ST->hasVSX() && Val->getScalarType()->isDoubleTy()) { 999 // Double-precision scalars are already located in index #0 (or #1 if LE). 1000 if (ISD == ISD::EXTRACT_VECTOR_ELT && 1001 Index == (ST->isLittleEndian() ? 1 : 0)) 1002 return 0; 1003 1004 return Cost; 1005 1006 } else if (Val->getScalarType()->isIntegerTy() && Index != -1U) { 1007 if (ST->hasP9Altivec()) { 1008 if (ISD == ISD::INSERT_VECTOR_ELT) 1009 // A move-to VSR and a permute/insert. Assume vector operation cost 1010 // for both (cost will be 2x on P9). 1011 return vectorCostAdjustment(2, Opcode, Val, nullptr); 1012 1013 // It's an extract. Maybe we can do a cheap move-from VSR. 1014 unsigned EltSize = Val->getScalarSizeInBits(); 1015 if (EltSize == 64) { 1016 unsigned MfvsrdIndex = ST->isLittleEndian() ? 1 : 0; 1017 if (Index == MfvsrdIndex) 1018 return 1; 1019 } else if (EltSize == 32) { 1020 unsigned MfvsrwzIndex = ST->isLittleEndian() ? 2 : 1; 1021 if (Index == MfvsrwzIndex) 1022 return 1; 1023 } 1024 1025 // We need a vector extract (or mfvsrld). Assume vector operation cost. 1026 // The cost of the load constant for a vector extract is disregarded 1027 // (invariant, easily schedulable). 1028 return vectorCostAdjustment(1, Opcode, Val, nullptr); 1029 1030 } else if (ST->hasDirectMove()) 1031 // Assume permute has standard cost. 1032 // Assume move-to/move-from VSR have 2x standard cost. 1033 return 3; 1034 } 1035 1036 // Estimated cost of a load-hit-store delay. This was obtained 1037 // experimentally as a minimum needed to prevent unprofitable 1038 // vectorization for the paq8p benchmark. It may need to be 1039 // raised further if other unprofitable cases remain. 1040 unsigned LHSPenalty = 2; 1041 if (ISD == ISD::INSERT_VECTOR_ELT) 1042 LHSPenalty += 7; 1043 1044 // Vector element insert/extract with Altivec is very expensive, 1045 // because they require store and reload with the attendant 1046 // processor stall for load-hit-store. Until VSX is available, 1047 // these need to be estimated as very costly. 1048 if (ISD == ISD::EXTRACT_VECTOR_ELT || 1049 ISD == ISD::INSERT_VECTOR_ELT) 1050 return LHSPenalty + Cost; 1051 1052 return Cost; 1053 } 1054 1055 int PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, 1056 MaybeAlign Alignment, unsigned AddressSpace, 1057 TTI::TargetCostKind CostKind, 1058 const Instruction *I) { 1059 if (TLI->getValueType(DL, Src, true) == MVT::Other) 1060 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, 1061 CostKind); 1062 // Legalize the type. 1063 std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Src); 1064 assert((Opcode == Instruction::Load || Opcode == Instruction::Store) && 1065 "Invalid Opcode"); 1066 1067 int Cost = BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, 1068 CostKind); 1069 // TODO: Handle other cost kinds. 1070 if (CostKind != TTI::TCK_RecipThroughput) 1071 return Cost; 1072 1073 Cost = vectorCostAdjustment(Cost, Opcode, Src, nullptr); 1074 1075 bool IsAltivecType = ST->hasAltivec() && 1076 (LT.second == MVT::v16i8 || LT.second == MVT::v8i16 || 1077 LT.second == MVT::v4i32 || LT.second == MVT::v4f32); 1078 bool IsVSXType = ST->hasVSX() && 1079 (LT.second == MVT::v2f64 || LT.second == MVT::v2i64); 1080 1081 // VSX has 32b/64b load instructions. Legalization can handle loading of 1082 // 32b/64b to VSR correctly and cheaply. But BaseT::getMemoryOpCost and 1083 // PPCTargetLowering can't compute the cost appropriately. So here we 1084 // explicitly check this case. 1085 unsigned MemBytes = Src->getPrimitiveSizeInBits(); 1086 if (Opcode == Instruction::Load && ST->hasVSX() && IsAltivecType && 1087 (MemBytes == 64 || (ST->hasP8Vector() && MemBytes == 32))) 1088 return 1; 1089 1090 // Aligned loads and stores are easy. 1091 unsigned SrcBytes = LT.second.getStoreSize(); 1092 if (!SrcBytes || !Alignment || *Alignment >= SrcBytes) 1093 return Cost; 1094 1095 // If we can use the permutation-based load sequence, then this is also 1096 // relatively cheap (not counting loop-invariant instructions): one load plus 1097 // one permute (the last load in a series has extra cost, but we're 1098 // neglecting that here). Note that on the P7, we could do unaligned loads 1099 // for Altivec types using the VSX instructions, but that's more expensive 1100 // than using the permutation-based load sequence. On the P8, that's no 1101 // longer true. 1102 if (Opcode == Instruction::Load && (!ST->hasP8Vector() && IsAltivecType) && 1103 *Alignment >= LT.second.getScalarType().getStoreSize()) 1104 return Cost + LT.first; // Add the cost of the permutations. 1105 1106 // For VSX, we can do unaligned loads and stores on Altivec/VSX types. On the 1107 // P7, unaligned vector loads are more expensive than the permutation-based 1108 // load sequence, so that might be used instead, but regardless, the net cost 1109 // is about the same (not counting loop-invariant instructions). 1110 if (IsVSXType || (ST->hasVSX() && IsAltivecType)) 1111 return Cost; 1112 1113 // Newer PPC supports unaligned memory access. 1114 if (TLI->allowsMisalignedMemoryAccesses(LT.second, 0)) 1115 return Cost; 1116 1117 // PPC in general does not support unaligned loads and stores. They'll need 1118 // to be decomposed based on the alignment factor. 1119 1120 // Add the cost of each scalar load or store. 1121 assert(Alignment); 1122 Cost += LT.first * ((SrcBytes / Alignment->value()) - 1); 1123 1124 // For a vector type, there is also scalarization overhead (only for 1125 // stores, loads are expanded using the vector-load + permutation sequence, 1126 // which is much less expensive). 1127 if (Src->isVectorTy() && Opcode == Instruction::Store) 1128 for (int i = 0, e = cast<FixedVectorType>(Src)->getNumElements(); i < e; 1129 ++i) 1130 Cost += getVectorInstrCost(Instruction::ExtractElement, Src, i); 1131 1132 return Cost; 1133 } 1134 1135 int PPCTTIImpl::getInterleavedMemoryOpCost( 1136 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, 1137 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, 1138 bool UseMaskForCond, bool UseMaskForGaps) { 1139 if (UseMaskForCond || UseMaskForGaps) 1140 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, 1141 Alignment, AddressSpace, CostKind, 1142 UseMaskForCond, UseMaskForGaps); 1143 1144 assert(isa<VectorType>(VecTy) && 1145 "Expect a vector type for interleaved memory op"); 1146 1147 // Legalize the type. 1148 std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, VecTy); 1149 1150 // Firstly, the cost of load/store operation. 1151 int Cost = 1152 getMemoryOpCost(Opcode, VecTy, MaybeAlign(Alignment), AddressSpace, 1153 CostKind); 1154 1155 // PPC, for both Altivec/VSX, support cheap arbitrary permutations 1156 // (at least in the sense that there need only be one non-loop-invariant 1157 // instruction). For each result vector, we need one shuffle per incoming 1158 // vector (except that the first shuffle can take two incoming vectors 1159 // because it does not need to take itself). 1160 Cost += Factor*(LT.first-1); 1161 1162 return Cost; 1163 } 1164 1165 unsigned PPCTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, 1166 TTI::TargetCostKind CostKind) { 1167 return BaseT::getIntrinsicInstrCost(ICA, CostKind); 1168 } 1169 1170 bool PPCTTIImpl::canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, 1171 LoopInfo *LI, DominatorTree *DT, 1172 AssumptionCache *AC, TargetLibraryInfo *LibInfo) { 1173 // Process nested loops first. 1174 for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) 1175 if (canSaveCmp(*I, BI, SE, LI, DT, AC, LibInfo)) 1176 return false; // Stop search. 1177 1178 HardwareLoopInfo HWLoopInfo(L); 1179 1180 if (!HWLoopInfo.canAnalyze(*LI)) 1181 return false; 1182 1183 if (!isHardwareLoopProfitable(L, *SE, *AC, LibInfo, HWLoopInfo)) 1184 return false; 1185 1186 if (!HWLoopInfo.isHardwareLoopCandidate(*SE, *LI, *DT)) 1187 return false; 1188 1189 *BI = HWLoopInfo.ExitBranch; 1190 return true; 1191 } 1192 1193 bool PPCTTIImpl::isLSRCostLess(TargetTransformInfo::LSRCost &C1, 1194 TargetTransformInfo::LSRCost &C2) { 1195 // PowerPC default behaviour here is "instruction number 1st priority". 1196 // If LsrNoInsnsCost is set, call default implementation. 1197 if (!LsrNoInsnsCost) 1198 return std::tie(C1.Insns, C1.NumRegs, C1.AddRecCost, C1.NumIVMuls, 1199 C1.NumBaseAdds, C1.ScaleCost, C1.ImmCost, C1.SetupCost) < 1200 std::tie(C2.Insns, C2.NumRegs, C2.AddRecCost, C2.NumIVMuls, 1201 C2.NumBaseAdds, C2.ScaleCost, C2.ImmCost, C2.SetupCost); 1202 else 1203 return TargetTransformInfoImplBase::isLSRCostLess(C1, C2); 1204 } 1205