1 //===-- PPCTargetTransformInfo.cpp - PPC specific TTI ---------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "PPCTargetTransformInfo.h" 10 #include "llvm/Analysis/CodeMetrics.h" 11 #include "llvm/Analysis/TargetLibraryInfo.h" 12 #include "llvm/Analysis/TargetTransformInfo.h" 13 #include "llvm/CodeGen/BasicTTIImpl.h" 14 #include "llvm/CodeGen/CostTable.h" 15 #include "llvm/CodeGen/TargetLowering.h" 16 #include "llvm/CodeGen/TargetSchedule.h" 17 #include "llvm/IR/IntrinsicsPowerPC.h" 18 #include "llvm/IR/ProfDataUtils.h" 19 #include "llvm/Support/CommandLine.h" 20 #include "llvm/Support/Debug.h" 21 #include "llvm/Support/KnownBits.h" 22 #include "llvm/Transforms/InstCombine/InstCombiner.h" 23 #include "llvm/Transforms/Utils/Local.h" 24 25 using namespace llvm; 26 27 #define DEBUG_TYPE "ppctti" 28 29 static cl::opt<bool> DisablePPCConstHoist("disable-ppc-constant-hoisting", 30 cl::desc("disable constant hoisting on PPC"), cl::init(false), cl::Hidden); 31 32 static cl::opt<bool> 33 EnablePPCColdCC("ppc-enable-coldcc", cl::Hidden, cl::init(false), 34 cl::desc("Enable using coldcc calling conv for cold " 35 "internal functions")); 36 37 static cl::opt<bool> 38 LsrNoInsnsCost("ppc-lsr-no-insns-cost", cl::Hidden, cl::init(false), 39 cl::desc("Do not add instruction count to lsr cost model")); 40 41 // The latency of mtctr is only justified if there are more than 4 42 // comparisons that will be removed as a result. 43 static cl::opt<unsigned> 44 SmallCTRLoopThreshold("min-ctr-loop-threshold", cl::init(4), cl::Hidden, 45 cl::desc("Loops with a constant trip count smaller than " 46 "this value will not use the count register.")); 47 48 //===----------------------------------------------------------------------===// 49 // 50 // PPC cost model. 51 // 52 //===----------------------------------------------------------------------===// 53 54 TargetTransformInfo::PopcntSupportKind 55 PPCTTIImpl::getPopcntSupport(unsigned TyWidth) { 56 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2"); 57 if (ST->hasPOPCNTD() != PPCSubtarget::POPCNTD_Unavailable && TyWidth <= 64) 58 return ST->hasPOPCNTD() == PPCSubtarget::POPCNTD_Slow ? 59 TTI::PSK_SlowHardware : TTI::PSK_FastHardware; 60 return TTI::PSK_Software; 61 } 62 63 Optional<Instruction *> 64 PPCTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { 65 Intrinsic::ID IID = II.getIntrinsicID(); 66 switch (IID) { 67 default: 68 break; 69 case Intrinsic::ppc_altivec_lvx: 70 case Intrinsic::ppc_altivec_lvxl: 71 // Turn PPC lvx -> load if the pointer is known aligned. 72 if (getOrEnforceKnownAlignment( 73 II.getArgOperand(0), Align(16), IC.getDataLayout(), &II, 74 &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) { 75 Value *Ptr = IC.Builder.CreateBitCast( 76 II.getArgOperand(0), PointerType::getUnqual(II.getType())); 77 return new LoadInst(II.getType(), Ptr, "", false, Align(16)); 78 } 79 break; 80 case Intrinsic::ppc_vsx_lxvw4x: 81 case Intrinsic::ppc_vsx_lxvd2x: { 82 // Turn PPC VSX loads into normal loads. 83 Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(0), 84 PointerType::getUnqual(II.getType())); 85 return new LoadInst(II.getType(), Ptr, Twine(""), false, Align(1)); 86 } 87 case Intrinsic::ppc_altivec_stvx: 88 case Intrinsic::ppc_altivec_stvxl: 89 // Turn stvx -> store if the pointer is known aligned. 90 if (getOrEnforceKnownAlignment( 91 II.getArgOperand(1), Align(16), IC.getDataLayout(), &II, 92 &IC.getAssumptionCache(), &IC.getDominatorTree()) >= 16) { 93 Type *OpPtrTy = PointerType::getUnqual(II.getArgOperand(0)->getType()); 94 Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy); 95 return new StoreInst(II.getArgOperand(0), Ptr, false, Align(16)); 96 } 97 break; 98 case Intrinsic::ppc_vsx_stxvw4x: 99 case Intrinsic::ppc_vsx_stxvd2x: { 100 // Turn PPC VSX stores into normal stores. 101 Type *OpPtrTy = PointerType::getUnqual(II.getArgOperand(0)->getType()); 102 Value *Ptr = IC.Builder.CreateBitCast(II.getArgOperand(1), OpPtrTy); 103 return new StoreInst(II.getArgOperand(0), Ptr, false, Align(1)); 104 } 105 case Intrinsic::ppc_altivec_vperm: 106 // Turn vperm(V1,V2,mask) -> shuffle(V1,V2,mask) if mask is a constant. 107 // Note that ppc_altivec_vperm has a big-endian bias, so when creating 108 // a vectorshuffle for little endian, we must undo the transformation 109 // performed on vec_perm in altivec.h. That is, we must complement 110 // the permutation mask with respect to 31 and reverse the order of 111 // V1 and V2. 112 if (Constant *Mask = dyn_cast<Constant>(II.getArgOperand(2))) { 113 assert(cast<FixedVectorType>(Mask->getType())->getNumElements() == 16 && 114 "Bad type for intrinsic!"); 115 116 // Check that all of the elements are integer constants or undefs. 117 bool AllEltsOk = true; 118 for (unsigned i = 0; i != 16; ++i) { 119 Constant *Elt = Mask->getAggregateElement(i); 120 if (!Elt || !(isa<ConstantInt>(Elt) || isa<UndefValue>(Elt))) { 121 AllEltsOk = false; 122 break; 123 } 124 } 125 126 if (AllEltsOk) { 127 // Cast the input vectors to byte vectors. 128 Value *Op0 = 129 IC.Builder.CreateBitCast(II.getArgOperand(0), Mask->getType()); 130 Value *Op1 = 131 IC.Builder.CreateBitCast(II.getArgOperand(1), Mask->getType()); 132 Value *Result = UndefValue::get(Op0->getType()); 133 134 // Only extract each element once. 135 Value *ExtractedElts[32]; 136 memset(ExtractedElts, 0, sizeof(ExtractedElts)); 137 138 for (unsigned i = 0; i != 16; ++i) { 139 if (isa<UndefValue>(Mask->getAggregateElement(i))) 140 continue; 141 unsigned Idx = 142 cast<ConstantInt>(Mask->getAggregateElement(i))->getZExtValue(); 143 Idx &= 31; // Match the hardware behavior. 144 if (DL.isLittleEndian()) 145 Idx = 31 - Idx; 146 147 if (!ExtractedElts[Idx]) { 148 Value *Op0ToUse = (DL.isLittleEndian()) ? Op1 : Op0; 149 Value *Op1ToUse = (DL.isLittleEndian()) ? Op0 : Op1; 150 ExtractedElts[Idx] = IC.Builder.CreateExtractElement( 151 Idx < 16 ? Op0ToUse : Op1ToUse, IC.Builder.getInt32(Idx & 15)); 152 } 153 154 // Insert this value into the result vector. 155 Result = IC.Builder.CreateInsertElement(Result, ExtractedElts[Idx], 156 IC.Builder.getInt32(i)); 157 } 158 return CastInst::Create(Instruction::BitCast, Result, II.getType()); 159 } 160 } 161 break; 162 } 163 return None; 164 } 165 166 InstructionCost PPCTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty, 167 TTI::TargetCostKind CostKind) { 168 if (DisablePPCConstHoist) 169 return BaseT::getIntImmCost(Imm, Ty, CostKind); 170 171 assert(Ty->isIntegerTy()); 172 173 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 174 if (BitSize == 0) 175 return ~0U; 176 177 if (Imm == 0) 178 return TTI::TCC_Free; 179 180 if (Imm.getBitWidth() <= 64) { 181 if (isInt<16>(Imm.getSExtValue())) 182 return TTI::TCC_Basic; 183 184 if (isInt<32>(Imm.getSExtValue())) { 185 // A constant that can be materialized using lis. 186 if ((Imm.getZExtValue() & 0xFFFF) == 0) 187 return TTI::TCC_Basic; 188 189 return 2 * TTI::TCC_Basic; 190 } 191 } 192 193 return 4 * TTI::TCC_Basic; 194 } 195 196 InstructionCost PPCTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, 197 const APInt &Imm, Type *Ty, 198 TTI::TargetCostKind CostKind) { 199 if (DisablePPCConstHoist) 200 return BaseT::getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind); 201 202 assert(Ty->isIntegerTy()); 203 204 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 205 if (BitSize == 0) 206 return ~0U; 207 208 switch (IID) { 209 default: 210 return TTI::TCC_Free; 211 case Intrinsic::sadd_with_overflow: 212 case Intrinsic::uadd_with_overflow: 213 case Intrinsic::ssub_with_overflow: 214 case Intrinsic::usub_with_overflow: 215 if ((Idx == 1) && Imm.getBitWidth() <= 64 && isInt<16>(Imm.getSExtValue())) 216 return TTI::TCC_Free; 217 break; 218 case Intrinsic::experimental_stackmap: 219 if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))) 220 return TTI::TCC_Free; 221 break; 222 case Intrinsic::experimental_patchpoint_void: 223 case Intrinsic::experimental_patchpoint_i64: 224 if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue()))) 225 return TTI::TCC_Free; 226 break; 227 } 228 return PPCTTIImpl::getIntImmCost(Imm, Ty, CostKind); 229 } 230 231 InstructionCost PPCTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx, 232 const APInt &Imm, Type *Ty, 233 TTI::TargetCostKind CostKind, 234 Instruction *Inst) { 235 if (DisablePPCConstHoist) 236 return BaseT::getIntImmCostInst(Opcode, Idx, Imm, Ty, CostKind, Inst); 237 238 assert(Ty->isIntegerTy()); 239 240 unsigned BitSize = Ty->getPrimitiveSizeInBits(); 241 if (BitSize == 0) 242 return ~0U; 243 244 unsigned ImmIdx = ~0U; 245 bool ShiftedFree = false, RunFree = false, UnsignedFree = false, 246 ZeroFree = false; 247 switch (Opcode) { 248 default: 249 return TTI::TCC_Free; 250 case Instruction::GetElementPtr: 251 // Always hoist the base address of a GetElementPtr. This prevents the 252 // creation of new constants for every base constant that gets constant 253 // folded with the offset. 254 if (Idx == 0) 255 return 2 * TTI::TCC_Basic; 256 return TTI::TCC_Free; 257 case Instruction::And: 258 RunFree = true; // (for the rotate-and-mask instructions) 259 LLVM_FALLTHROUGH; 260 case Instruction::Add: 261 case Instruction::Or: 262 case Instruction::Xor: 263 ShiftedFree = true; 264 LLVM_FALLTHROUGH; 265 case Instruction::Sub: 266 case Instruction::Mul: 267 case Instruction::Shl: 268 case Instruction::LShr: 269 case Instruction::AShr: 270 ImmIdx = 1; 271 break; 272 case Instruction::ICmp: 273 UnsignedFree = true; 274 ImmIdx = 1; 275 // Zero comparisons can use record-form instructions. 276 LLVM_FALLTHROUGH; 277 case Instruction::Select: 278 ZeroFree = true; 279 break; 280 case Instruction::PHI: 281 case Instruction::Call: 282 case Instruction::Ret: 283 case Instruction::Load: 284 case Instruction::Store: 285 break; 286 } 287 288 if (ZeroFree && Imm == 0) 289 return TTI::TCC_Free; 290 291 if (Idx == ImmIdx && Imm.getBitWidth() <= 64) { 292 if (isInt<16>(Imm.getSExtValue())) 293 return TTI::TCC_Free; 294 295 if (RunFree) { 296 if (Imm.getBitWidth() <= 32 && 297 (isShiftedMask_32(Imm.getZExtValue()) || 298 isShiftedMask_32(~Imm.getZExtValue()))) 299 return TTI::TCC_Free; 300 301 if (ST->isPPC64() && 302 (isShiftedMask_64(Imm.getZExtValue()) || 303 isShiftedMask_64(~Imm.getZExtValue()))) 304 return TTI::TCC_Free; 305 } 306 307 if (UnsignedFree && isUInt<16>(Imm.getZExtValue())) 308 return TTI::TCC_Free; 309 310 if (ShiftedFree && (Imm.getZExtValue() & 0xFFFF) == 0) 311 return TTI::TCC_Free; 312 } 313 314 return PPCTTIImpl::getIntImmCost(Imm, Ty, CostKind); 315 } 316 317 // Check if the current Type is an MMA vector type. Valid MMA types are 318 // v256i1 and v512i1 respectively. 319 static bool isMMAType(Type *Ty) { 320 return Ty->isVectorTy() && (Ty->getScalarSizeInBits() == 1) && 321 (Ty->getPrimitiveSizeInBits() > 128); 322 } 323 324 InstructionCost PPCTTIImpl::getUserCost(const User *U, 325 ArrayRef<const Value *> Operands, 326 TTI::TargetCostKind CostKind) { 327 // We already implement getCastInstrCost and getMemoryOpCost where we perform 328 // the vector adjustment there. 329 if (isa<CastInst>(U) || isa<LoadInst>(U) || isa<StoreInst>(U)) 330 return BaseT::getUserCost(U, Operands, CostKind); 331 332 if (U->getType()->isVectorTy()) { 333 // Instructions that need to be split should cost more. 334 std::pair<InstructionCost, MVT> LT = 335 TLI->getTypeLegalizationCost(DL, U->getType()); 336 return LT.first * BaseT::getUserCost(U, Operands, CostKind); 337 } 338 339 return BaseT::getUserCost(U, Operands, CostKind); 340 } 341 342 // Determining the address of a TLS variable results in a function call in 343 // certain TLS models. 344 static bool memAddrUsesCTR(const Value *MemAddr, const PPCTargetMachine &TM, 345 SmallPtrSetImpl<const Value *> &Visited) { 346 // No need to traverse again if we already checked this operand. 347 if (!Visited.insert(MemAddr).second) 348 return false; 349 const auto *GV = dyn_cast<GlobalValue>(MemAddr); 350 if (!GV) { 351 // Recurse to check for constants that refer to TLS global variables. 352 if (const auto *CV = dyn_cast<Constant>(MemAddr)) 353 for (const auto &CO : CV->operands()) 354 if (memAddrUsesCTR(CO, TM, Visited)) 355 return true; 356 return false; 357 } 358 359 if (!GV->isThreadLocal()) 360 return false; 361 TLSModel::Model Model = TM.getTLSModel(GV); 362 return Model == TLSModel::GeneralDynamic || Model == TLSModel::LocalDynamic; 363 } 364 365 bool PPCTTIImpl::mightUseCTR(BasicBlock *BB, TargetLibraryInfo *LibInfo, 366 SmallPtrSetImpl<const Value *> &Visited) { 367 const PPCTargetMachine &TM = ST->getTargetMachine(); 368 369 // Loop through the inline asm constraints and look for something that 370 // clobbers ctr. 371 auto asmClobbersCTR = [](InlineAsm *IA) { 372 InlineAsm::ConstraintInfoVector CIV = IA->ParseConstraints(); 373 for (const InlineAsm::ConstraintInfo &C : CIV) { 374 if (C.Type != InlineAsm::isInput) 375 for (const auto &Code : C.Codes) 376 if (StringRef(Code).equals_insensitive("{ctr}")) 377 return true; 378 } 379 return false; 380 }; 381 382 auto isLargeIntegerTy = [](bool Is32Bit, Type *Ty) { 383 if (IntegerType *ITy = dyn_cast<IntegerType>(Ty)) 384 return ITy->getBitWidth() > (Is32Bit ? 32U : 64U); 385 386 return false; 387 }; 388 389 auto supportedHalfPrecisionOp = [](Instruction *Inst) { 390 switch (Inst->getOpcode()) { 391 default: 392 return false; 393 case Instruction::FPTrunc: 394 case Instruction::FPExt: 395 case Instruction::Load: 396 case Instruction::Store: 397 case Instruction::FPToUI: 398 case Instruction::UIToFP: 399 case Instruction::FPToSI: 400 case Instruction::SIToFP: 401 return true; 402 } 403 }; 404 405 for (BasicBlock::iterator J = BB->begin(), JE = BB->end(); 406 J != JE; ++J) { 407 // There are no direct operations on half precision so assume that 408 // anything with that type requires a call except for a few select 409 // operations with Power9. 410 if (Instruction *CurrInst = dyn_cast<Instruction>(J)) { 411 for (const auto &Op : CurrInst->operands()) { 412 if (Op->getType()->getScalarType()->isHalfTy() || 413 CurrInst->getType()->getScalarType()->isHalfTy()) 414 return !(ST->isISA3_0() && supportedHalfPrecisionOp(CurrInst)); 415 } 416 } 417 if (CallInst *CI = dyn_cast<CallInst>(J)) { 418 // Inline ASM is okay, unless it clobbers the ctr register. 419 if (InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledOperand())) { 420 if (asmClobbersCTR(IA)) 421 return true; 422 continue; 423 } 424 425 if (Function *F = CI->getCalledFunction()) { 426 // Most intrinsics don't become function calls, but some might. 427 // sin, cos, exp and log are always calls. 428 unsigned Opcode = 0; 429 if (F->getIntrinsicID() != Intrinsic::not_intrinsic) { 430 switch (F->getIntrinsicID()) { 431 default: continue; 432 // If we have a call to loop_decrement or set_loop_iterations, 433 // we're definitely using CTR. 434 case Intrinsic::set_loop_iterations: 435 case Intrinsic::loop_decrement: 436 return true; 437 438 // Binary operations on 128-bit value will use CTR. 439 case Intrinsic::experimental_constrained_fadd: 440 case Intrinsic::experimental_constrained_fsub: 441 case Intrinsic::experimental_constrained_fmul: 442 case Intrinsic::experimental_constrained_fdiv: 443 case Intrinsic::experimental_constrained_frem: 444 if (F->getType()->getScalarType()->isFP128Ty() || 445 F->getType()->getScalarType()->isPPC_FP128Ty()) 446 return true; 447 break; 448 449 case Intrinsic::experimental_constrained_fptosi: 450 case Intrinsic::experimental_constrained_fptoui: 451 case Intrinsic::experimental_constrained_sitofp: 452 case Intrinsic::experimental_constrained_uitofp: { 453 Type *SrcType = CI->getArgOperand(0)->getType()->getScalarType(); 454 Type *DstType = CI->getType()->getScalarType(); 455 if (SrcType->isPPC_FP128Ty() || DstType->isPPC_FP128Ty() || 456 isLargeIntegerTy(!TM.isPPC64(), SrcType) || 457 isLargeIntegerTy(!TM.isPPC64(), DstType)) 458 return true; 459 break; 460 } 461 462 // Exclude eh_sjlj_setjmp; we don't need to exclude eh_sjlj_longjmp 463 // because, although it does clobber the counter register, the 464 // control can't then return to inside the loop unless there is also 465 // an eh_sjlj_setjmp. 466 case Intrinsic::eh_sjlj_setjmp: 467 468 case Intrinsic::memcpy: 469 case Intrinsic::memmove: 470 case Intrinsic::memset: 471 case Intrinsic::powi: 472 case Intrinsic::log: 473 case Intrinsic::log2: 474 case Intrinsic::log10: 475 case Intrinsic::exp: 476 case Intrinsic::exp2: 477 case Intrinsic::pow: 478 case Intrinsic::sin: 479 case Intrinsic::cos: 480 case Intrinsic::experimental_constrained_powi: 481 case Intrinsic::experimental_constrained_log: 482 case Intrinsic::experimental_constrained_log2: 483 case Intrinsic::experimental_constrained_log10: 484 case Intrinsic::experimental_constrained_exp: 485 case Intrinsic::experimental_constrained_exp2: 486 case Intrinsic::experimental_constrained_pow: 487 case Intrinsic::experimental_constrained_sin: 488 case Intrinsic::experimental_constrained_cos: 489 return true; 490 case Intrinsic::copysign: 491 if (CI->getArgOperand(0)->getType()->getScalarType()-> 492 isPPC_FP128Ty()) 493 return true; 494 else 495 continue; // ISD::FCOPYSIGN is never a library call. 496 case Intrinsic::fmuladd: 497 case Intrinsic::fma: Opcode = ISD::FMA; break; 498 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break; 499 case Intrinsic::floor: Opcode = ISD::FFLOOR; break; 500 case Intrinsic::ceil: Opcode = ISD::FCEIL; break; 501 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break; 502 case Intrinsic::rint: Opcode = ISD::FRINT; break; 503 case Intrinsic::lrint: Opcode = ISD::LRINT; break; 504 case Intrinsic::llrint: Opcode = ISD::LLRINT; break; 505 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break; 506 case Intrinsic::round: Opcode = ISD::FROUND; break; 507 case Intrinsic::lround: Opcode = ISD::LROUND; break; 508 case Intrinsic::llround: Opcode = ISD::LLROUND; break; 509 case Intrinsic::minnum: Opcode = ISD::FMINNUM; break; 510 case Intrinsic::maxnum: Opcode = ISD::FMAXNUM; break; 511 case Intrinsic::experimental_constrained_fcmp: 512 Opcode = ISD::STRICT_FSETCC; 513 break; 514 case Intrinsic::experimental_constrained_fcmps: 515 Opcode = ISD::STRICT_FSETCCS; 516 break; 517 case Intrinsic::experimental_constrained_fma: 518 Opcode = ISD::STRICT_FMA; 519 break; 520 case Intrinsic::experimental_constrained_sqrt: 521 Opcode = ISD::STRICT_FSQRT; 522 break; 523 case Intrinsic::experimental_constrained_floor: 524 Opcode = ISD::STRICT_FFLOOR; 525 break; 526 case Intrinsic::experimental_constrained_ceil: 527 Opcode = ISD::STRICT_FCEIL; 528 break; 529 case Intrinsic::experimental_constrained_trunc: 530 Opcode = ISD::STRICT_FTRUNC; 531 break; 532 case Intrinsic::experimental_constrained_rint: 533 Opcode = ISD::STRICT_FRINT; 534 break; 535 case Intrinsic::experimental_constrained_lrint: 536 Opcode = ISD::STRICT_LRINT; 537 break; 538 case Intrinsic::experimental_constrained_llrint: 539 Opcode = ISD::STRICT_LLRINT; 540 break; 541 case Intrinsic::experimental_constrained_nearbyint: 542 Opcode = ISD::STRICT_FNEARBYINT; 543 break; 544 case Intrinsic::experimental_constrained_round: 545 Opcode = ISD::STRICT_FROUND; 546 break; 547 case Intrinsic::experimental_constrained_lround: 548 Opcode = ISD::STRICT_LROUND; 549 break; 550 case Intrinsic::experimental_constrained_llround: 551 Opcode = ISD::STRICT_LLROUND; 552 break; 553 case Intrinsic::experimental_constrained_minnum: 554 Opcode = ISD::STRICT_FMINNUM; 555 break; 556 case Intrinsic::experimental_constrained_maxnum: 557 Opcode = ISD::STRICT_FMAXNUM; 558 break; 559 case Intrinsic::umul_with_overflow: Opcode = ISD::UMULO; break; 560 case Intrinsic::smul_with_overflow: Opcode = ISD::SMULO; break; 561 } 562 } 563 564 // PowerPC does not use [US]DIVREM or other library calls for 565 // operations on regular types which are not otherwise library calls 566 // (i.e. soft float or atomics). If adapting for targets that do, 567 // additional care is required here. 568 569 LibFunc Func; 570 if (!F->hasLocalLinkage() && F->hasName() && LibInfo && 571 LibInfo->getLibFunc(F->getName(), Func) && 572 LibInfo->hasOptimizedCodeGen(Func)) { 573 // Non-read-only functions are never treated as intrinsics. 574 if (!CI->onlyReadsMemory()) 575 return true; 576 577 // Conversion happens only for FP calls. 578 if (!CI->getArgOperand(0)->getType()->isFloatingPointTy()) 579 return true; 580 581 switch (Func) { 582 default: return true; 583 case LibFunc_copysign: 584 case LibFunc_copysignf: 585 continue; // ISD::FCOPYSIGN is never a library call. 586 case LibFunc_copysignl: 587 return true; 588 case LibFunc_fabs: 589 case LibFunc_fabsf: 590 case LibFunc_fabsl: 591 continue; // ISD::FABS is never a library call. 592 case LibFunc_sqrt: 593 case LibFunc_sqrtf: 594 case LibFunc_sqrtl: 595 Opcode = ISD::FSQRT; break; 596 case LibFunc_floor: 597 case LibFunc_floorf: 598 case LibFunc_floorl: 599 Opcode = ISD::FFLOOR; break; 600 case LibFunc_nearbyint: 601 case LibFunc_nearbyintf: 602 case LibFunc_nearbyintl: 603 Opcode = ISD::FNEARBYINT; break; 604 case LibFunc_ceil: 605 case LibFunc_ceilf: 606 case LibFunc_ceill: 607 Opcode = ISD::FCEIL; break; 608 case LibFunc_rint: 609 case LibFunc_rintf: 610 case LibFunc_rintl: 611 Opcode = ISD::FRINT; break; 612 case LibFunc_round: 613 case LibFunc_roundf: 614 case LibFunc_roundl: 615 Opcode = ISD::FROUND; break; 616 case LibFunc_trunc: 617 case LibFunc_truncf: 618 case LibFunc_truncl: 619 Opcode = ISD::FTRUNC; break; 620 case LibFunc_fmin: 621 case LibFunc_fminf: 622 case LibFunc_fminl: 623 Opcode = ISD::FMINNUM; break; 624 case LibFunc_fmax: 625 case LibFunc_fmaxf: 626 case LibFunc_fmaxl: 627 Opcode = ISD::FMAXNUM; break; 628 } 629 } 630 631 if (Opcode) { 632 EVT EVTy = 633 TLI->getValueType(DL, CI->getArgOperand(0)->getType(), true); 634 635 if (EVTy == MVT::Other) 636 return true; 637 638 if (TLI->isOperationLegalOrCustom(Opcode, EVTy)) 639 continue; 640 else if (EVTy.isVector() && 641 TLI->isOperationLegalOrCustom(Opcode, EVTy.getScalarType())) 642 continue; 643 644 return true; 645 } 646 } 647 648 return true; 649 } else if ((J->getType()->getScalarType()->isFP128Ty() || 650 J->getType()->getScalarType()->isPPC_FP128Ty())) { 651 // Most operations on f128 or ppc_f128 values become calls. 652 return true; 653 } else if (isa<FCmpInst>(J) && 654 J->getOperand(0)->getType()->getScalarType()->isFP128Ty()) { 655 return true; 656 } else if ((isa<FPTruncInst>(J) || isa<FPExtInst>(J)) && 657 (cast<CastInst>(J)->getSrcTy()->getScalarType()->isFP128Ty() || 658 cast<CastInst>(J)->getDestTy()->getScalarType()->isFP128Ty())) { 659 return true; 660 } else if (isa<UIToFPInst>(J) || isa<SIToFPInst>(J) || 661 isa<FPToUIInst>(J) || isa<FPToSIInst>(J)) { 662 CastInst *CI = cast<CastInst>(J); 663 if (CI->getSrcTy()->getScalarType()->isPPC_FP128Ty() || 664 CI->getDestTy()->getScalarType()->isPPC_FP128Ty() || 665 isLargeIntegerTy(!TM.isPPC64(), CI->getSrcTy()->getScalarType()) || 666 isLargeIntegerTy(!TM.isPPC64(), CI->getDestTy()->getScalarType())) 667 return true; 668 } else if (isLargeIntegerTy(!TM.isPPC64(), 669 J->getType()->getScalarType()) && 670 (J->getOpcode() == Instruction::UDiv || 671 J->getOpcode() == Instruction::SDiv || 672 J->getOpcode() == Instruction::URem || 673 J->getOpcode() == Instruction::SRem)) { 674 return true; 675 } else if (!TM.isPPC64() && 676 isLargeIntegerTy(false, J->getType()->getScalarType()) && 677 (J->getOpcode() == Instruction::Shl || 678 J->getOpcode() == Instruction::AShr || 679 J->getOpcode() == Instruction::LShr)) { 680 // Only on PPC32, for 128-bit integers (specifically not 64-bit 681 // integers), these might be runtime calls. 682 return true; 683 } else if (isa<IndirectBrInst>(J) || isa<InvokeInst>(J)) { 684 // On PowerPC, indirect jumps use the counter register. 685 return true; 686 } else if (SwitchInst *SI = dyn_cast<SwitchInst>(J)) { 687 if (SI->getNumCases() + 1 >= (unsigned)TLI->getMinimumJumpTableEntries()) 688 return true; 689 } 690 691 // FREM is always a call. 692 if (J->getOpcode() == Instruction::FRem) 693 return true; 694 695 if (ST->useSoftFloat()) { 696 switch(J->getOpcode()) { 697 case Instruction::FAdd: 698 case Instruction::FSub: 699 case Instruction::FMul: 700 case Instruction::FDiv: 701 case Instruction::FPTrunc: 702 case Instruction::FPExt: 703 case Instruction::FPToUI: 704 case Instruction::FPToSI: 705 case Instruction::UIToFP: 706 case Instruction::SIToFP: 707 case Instruction::FCmp: 708 return true; 709 } 710 } 711 712 for (Value *Operand : J->operands()) 713 if (memAddrUsesCTR(Operand, TM, Visited)) 714 return true; 715 } 716 717 return false; 718 } 719 720 bool PPCTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE, 721 AssumptionCache &AC, 722 TargetLibraryInfo *LibInfo, 723 HardwareLoopInfo &HWLoopInfo) { 724 const PPCTargetMachine &TM = ST->getTargetMachine(); 725 TargetSchedModel SchedModel; 726 SchedModel.init(ST); 727 728 // Do not convert small short loops to CTR loop. 729 unsigned ConstTripCount = SE.getSmallConstantTripCount(L); 730 if (ConstTripCount && ConstTripCount < SmallCTRLoopThreshold) { 731 SmallPtrSet<const Value *, 32> EphValues; 732 CodeMetrics::collectEphemeralValues(L, &AC, EphValues); 733 CodeMetrics Metrics; 734 for (BasicBlock *BB : L->blocks()) 735 Metrics.analyzeBasicBlock(BB, *this, EphValues); 736 // 6 is an approximate latency for the mtctr instruction. 737 if (Metrics.NumInsts <= (6 * SchedModel.getIssueWidth())) 738 return false; 739 } 740 741 // We don't want to spill/restore the counter register, and so we don't 742 // want to use the counter register if the loop contains calls. 743 SmallPtrSet<const Value *, 4> Visited; 744 for (Loop::block_iterator I = L->block_begin(), IE = L->block_end(); 745 I != IE; ++I) 746 if (mightUseCTR(*I, LibInfo, Visited)) 747 return false; 748 749 SmallVector<BasicBlock*, 4> ExitingBlocks; 750 L->getExitingBlocks(ExitingBlocks); 751 752 // If there is an exit edge known to be frequently taken, 753 // we should not transform this loop. 754 for (auto &BB : ExitingBlocks) { 755 Instruction *TI = BB->getTerminator(); 756 if (!TI) continue; 757 758 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) { 759 uint64_t TrueWeight = 0, FalseWeight = 0; 760 if (!BI->isConditional() || 761 !extractBranchWeights(BI, TrueWeight, FalseWeight)) 762 continue; 763 764 // If the exit path is more frequent than the loop path, 765 // we return here without further analysis for this loop. 766 bool TrueIsExit = !L->contains(BI->getSuccessor(0)); 767 if (( TrueIsExit && FalseWeight < TrueWeight) || 768 (!TrueIsExit && FalseWeight > TrueWeight)) 769 return false; 770 } 771 } 772 773 // If an exit block has a PHI that accesses a TLS variable as one of the 774 // incoming values from the loop, we cannot produce a CTR loop because the 775 // address for that value will be computed in the loop. 776 SmallVector<BasicBlock *, 4> ExitBlocks; 777 L->getExitBlocks(ExitBlocks); 778 for (auto &BB : ExitBlocks) { 779 for (auto &PHI : BB->phis()) { 780 for (int Idx = 0, EndIdx = PHI.getNumIncomingValues(); Idx < EndIdx; 781 Idx++) { 782 const BasicBlock *IncomingBB = PHI.getIncomingBlock(Idx); 783 const Value *IncomingValue = PHI.getIncomingValue(Idx); 784 if (L->contains(IncomingBB) && 785 memAddrUsesCTR(IncomingValue, TM, Visited)) 786 return false; 787 } 788 } 789 } 790 791 LLVMContext &C = L->getHeader()->getContext(); 792 HWLoopInfo.CountType = TM.isPPC64() ? 793 Type::getInt64Ty(C) : Type::getInt32Ty(C); 794 HWLoopInfo.LoopDecrement = ConstantInt::get(HWLoopInfo.CountType, 1); 795 return true; 796 } 797 798 void PPCTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, 799 TTI::UnrollingPreferences &UP, 800 OptimizationRemarkEmitter *ORE) { 801 if (ST->getCPUDirective() == PPC::DIR_A2) { 802 // The A2 is in-order with a deep pipeline, and concatenation unrolling 803 // helps expose latency-hiding opportunities to the instruction scheduler. 804 UP.Partial = UP.Runtime = true; 805 806 // We unroll a lot on the A2 (hundreds of instructions), and the benefits 807 // often outweigh the cost of a division to compute the trip count. 808 UP.AllowExpensiveTripCount = true; 809 } 810 811 BaseT::getUnrollingPreferences(L, SE, UP, ORE); 812 } 813 814 void PPCTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE, 815 TTI::PeelingPreferences &PP) { 816 BaseT::getPeelingPreferences(L, SE, PP); 817 } 818 // This function returns true to allow using coldcc calling convention. 819 // Returning true results in coldcc being used for functions which are cold at 820 // all call sites when the callers of the functions are not calling any other 821 // non coldcc functions. 822 bool PPCTTIImpl::useColdCCForColdCall(Function &F) { 823 return EnablePPCColdCC; 824 } 825 826 bool PPCTTIImpl::enableAggressiveInterleaving(bool LoopHasReductions) { 827 // On the A2, always unroll aggressively. 828 if (ST->getCPUDirective() == PPC::DIR_A2) 829 return true; 830 831 return LoopHasReductions; 832 } 833 834 PPCTTIImpl::TTI::MemCmpExpansionOptions 835 PPCTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const { 836 TTI::MemCmpExpansionOptions Options; 837 Options.LoadSizes = {8, 4, 2, 1}; 838 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize); 839 return Options; 840 } 841 842 bool PPCTTIImpl::enableInterleavedAccessVectorization() { 843 return true; 844 } 845 846 unsigned PPCTTIImpl::getNumberOfRegisters(unsigned ClassID) const { 847 assert(ClassID == GPRRC || ClassID == FPRRC || 848 ClassID == VRRC || ClassID == VSXRC); 849 if (ST->hasVSX()) { 850 assert(ClassID == GPRRC || ClassID == VSXRC || ClassID == VRRC); 851 return ClassID == VSXRC ? 64 : 32; 852 } 853 assert(ClassID == GPRRC || ClassID == FPRRC || ClassID == VRRC); 854 return 32; 855 } 856 857 unsigned PPCTTIImpl::getRegisterClassForType(bool Vector, Type *Ty) const { 858 if (Vector) 859 return ST->hasVSX() ? VSXRC : VRRC; 860 else if (Ty && (Ty->getScalarType()->isFloatTy() || 861 Ty->getScalarType()->isDoubleTy())) 862 return ST->hasVSX() ? VSXRC : FPRRC; 863 else if (Ty && (Ty->getScalarType()->isFP128Ty() || 864 Ty->getScalarType()->isPPC_FP128Ty())) 865 return VRRC; 866 else if (Ty && Ty->getScalarType()->isHalfTy()) 867 return VSXRC; 868 else 869 return GPRRC; 870 } 871 872 const char* PPCTTIImpl::getRegisterClassName(unsigned ClassID) const { 873 874 switch (ClassID) { 875 default: 876 llvm_unreachable("unknown register class"); 877 return "PPC::unknown register class"; 878 case GPRRC: return "PPC::GPRRC"; 879 case FPRRC: return "PPC::FPRRC"; 880 case VRRC: return "PPC::VRRC"; 881 case VSXRC: return "PPC::VSXRC"; 882 } 883 } 884 885 TypeSize 886 PPCTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const { 887 switch (K) { 888 case TargetTransformInfo::RGK_Scalar: 889 return TypeSize::getFixed(ST->isPPC64() ? 64 : 32); 890 case TargetTransformInfo::RGK_FixedWidthVector: 891 return TypeSize::getFixed(ST->hasAltivec() ? 128 : 0); 892 case TargetTransformInfo::RGK_ScalableVector: 893 return TypeSize::getScalable(0); 894 } 895 896 llvm_unreachable("Unsupported register kind"); 897 } 898 899 unsigned PPCTTIImpl::getCacheLineSize() const { 900 // Starting with P7 we have a cache line size of 128. 901 unsigned Directive = ST->getCPUDirective(); 902 // Assume that Future CPU has the same cache line size as the others. 903 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8 || 904 Directive == PPC::DIR_PWR9 || Directive == PPC::DIR_PWR10 || 905 Directive == PPC::DIR_PWR_FUTURE) 906 return 128; 907 908 // On other processors return a default of 64 bytes. 909 return 64; 910 } 911 912 unsigned PPCTTIImpl::getPrefetchDistance() const { 913 return 300; 914 } 915 916 unsigned PPCTTIImpl::getMaxInterleaveFactor(unsigned VF) { 917 unsigned Directive = ST->getCPUDirective(); 918 // The 440 has no SIMD support, but floating-point instructions 919 // have a 5-cycle latency, so unroll by 5x for latency hiding. 920 if (Directive == PPC::DIR_440) 921 return 5; 922 923 // The A2 has no SIMD support, but floating-point instructions 924 // have a 6-cycle latency, so unroll by 6x for latency hiding. 925 if (Directive == PPC::DIR_A2) 926 return 6; 927 928 // FIXME: For lack of any better information, do no harm... 929 if (Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) 930 return 1; 931 932 // For P7 and P8, floating-point instructions have a 6-cycle latency and 933 // there are two execution units, so unroll by 12x for latency hiding. 934 // FIXME: the same for P9 as previous gen until POWER9 scheduling is ready 935 // FIXME: the same for P10 as previous gen until POWER10 scheduling is ready 936 // Assume that future is the same as the others. 937 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8 || 938 Directive == PPC::DIR_PWR9 || Directive == PPC::DIR_PWR10 || 939 Directive == PPC::DIR_PWR_FUTURE) 940 return 12; 941 942 // For most things, modern systems have two execution units (and 943 // out-of-order execution). 944 return 2; 945 } 946 947 // Returns a cost adjustment factor to adjust the cost of vector instructions 948 // on targets which there is overlap between the vector and scalar units, 949 // thereby reducing the overall throughput of vector code wrt. scalar code. 950 // An invalid instruction cost is returned if the type is an MMA vector type. 951 InstructionCost PPCTTIImpl::vectorCostAdjustmentFactor(unsigned Opcode, 952 Type *Ty1, Type *Ty2) { 953 // If the vector type is of an MMA type (v256i1, v512i1), an invalid 954 // instruction cost is returned. This is to signify to other cost computing 955 // functions to return the maximum instruction cost in order to prevent any 956 // opportunities for the optimizer to produce MMA types within the IR. 957 if (isMMAType(Ty1)) 958 return InstructionCost::getInvalid(); 959 960 if (!ST->vectorsUseTwoUnits() || !Ty1->isVectorTy()) 961 return InstructionCost(1); 962 963 std::pair<InstructionCost, MVT> LT1 = TLI->getTypeLegalizationCost(DL, Ty1); 964 // If type legalization involves splitting the vector, we don't want to 965 // double the cost at every step - only the last step. 966 if (LT1.first != 1 || !LT1.second.isVector()) 967 return InstructionCost(1); 968 969 int ISD = TLI->InstructionOpcodeToISD(Opcode); 970 if (TLI->isOperationExpand(ISD, LT1.second)) 971 return InstructionCost(1); 972 973 if (Ty2) { 974 std::pair<InstructionCost, MVT> LT2 = TLI->getTypeLegalizationCost(DL, Ty2); 975 if (LT2.first != 1 || !LT2.second.isVector()) 976 return InstructionCost(1); 977 } 978 979 return InstructionCost(2); 980 } 981 982 InstructionCost PPCTTIImpl::getArithmeticInstrCost( 983 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, 984 TTI::OperandValueKind Op1Info, TTI::OperandValueKind Op2Info, 985 TTI::OperandValueProperties Opd1PropInfo, 986 TTI::OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args, 987 const Instruction *CxtI) { 988 assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode"); 989 990 InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Ty, nullptr); 991 if (!CostFactor.isValid()) 992 return InstructionCost::getMax(); 993 994 // TODO: Handle more cost kinds. 995 if (CostKind != TTI::TCK_RecipThroughput) 996 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, 997 Op2Info, Opd1PropInfo, 998 Opd2PropInfo, Args, CxtI); 999 1000 // Fallback to the default implementation. 1001 InstructionCost Cost = BaseT::getArithmeticInstrCost( 1002 Opcode, Ty, CostKind, Op1Info, Op2Info, Opd1PropInfo, Opd2PropInfo); 1003 return Cost * CostFactor; 1004 } 1005 1006 InstructionCost PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, 1007 ArrayRef<int> Mask, int Index, 1008 Type *SubTp, 1009 ArrayRef<const Value *> Args) { 1010 1011 InstructionCost CostFactor = 1012 vectorCostAdjustmentFactor(Instruction::ShuffleVector, Tp, nullptr); 1013 if (!CostFactor.isValid()) 1014 return InstructionCost::getMax(); 1015 1016 // Legalize the type. 1017 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Tp); 1018 1019 // PPC, for both Altivec/VSX, support cheap arbitrary permutations 1020 // (at least in the sense that there need only be one non-loop-invariant 1021 // instruction). We need one such shuffle instruction for each actual 1022 // register (this is not true for arbitrary shuffles, but is true for the 1023 // structured types of shuffles covered by TTI::ShuffleKind). 1024 return LT.first * CostFactor; 1025 } 1026 1027 InstructionCost PPCTTIImpl::getCFInstrCost(unsigned Opcode, 1028 TTI::TargetCostKind CostKind, 1029 const Instruction *I) { 1030 if (CostKind != TTI::TCK_RecipThroughput) 1031 return Opcode == Instruction::PHI ? 0 : 1; 1032 // Branches are assumed to be predicted. 1033 return 0; 1034 } 1035 1036 InstructionCost PPCTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, 1037 Type *Src, 1038 TTI::CastContextHint CCH, 1039 TTI::TargetCostKind CostKind, 1040 const Instruction *I) { 1041 assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode"); 1042 1043 InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Dst, Src); 1044 if (!CostFactor.isValid()) 1045 return InstructionCost::getMax(); 1046 1047 InstructionCost Cost = 1048 BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I); 1049 Cost *= CostFactor; 1050 // TODO: Allow non-throughput costs that aren't binary. 1051 if (CostKind != TTI::TCK_RecipThroughput) 1052 return Cost == 0 ? 0 : 1; 1053 return Cost; 1054 } 1055 1056 InstructionCost PPCTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, 1057 Type *CondTy, 1058 CmpInst::Predicate VecPred, 1059 TTI::TargetCostKind CostKind, 1060 const Instruction *I) { 1061 InstructionCost CostFactor = 1062 vectorCostAdjustmentFactor(Opcode, ValTy, nullptr); 1063 if (!CostFactor.isValid()) 1064 return InstructionCost::getMax(); 1065 1066 InstructionCost Cost = 1067 BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); 1068 // TODO: Handle other cost kinds. 1069 if (CostKind != TTI::TCK_RecipThroughput) 1070 return Cost; 1071 return Cost * CostFactor; 1072 } 1073 1074 InstructionCost PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, 1075 unsigned Index) { 1076 assert(Val->isVectorTy() && "This must be a vector type"); 1077 1078 int ISD = TLI->InstructionOpcodeToISD(Opcode); 1079 assert(ISD && "Invalid opcode"); 1080 1081 InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Val, nullptr); 1082 if (!CostFactor.isValid()) 1083 return InstructionCost::getMax(); 1084 1085 InstructionCost Cost = BaseT::getVectorInstrCost(Opcode, Val, Index); 1086 Cost *= CostFactor; 1087 1088 if (ST->hasVSX() && Val->getScalarType()->isDoubleTy()) { 1089 // Double-precision scalars are already located in index #0 (or #1 if LE). 1090 if (ISD == ISD::EXTRACT_VECTOR_ELT && 1091 Index == (ST->isLittleEndian() ? 1 : 0)) 1092 return 0; 1093 1094 return Cost; 1095 1096 } else if (Val->getScalarType()->isIntegerTy() && Index != -1U) { 1097 if (ST->hasP9Altivec()) { 1098 if (ISD == ISD::INSERT_VECTOR_ELT) 1099 // A move-to VSR and a permute/insert. Assume vector operation cost 1100 // for both (cost will be 2x on P9). 1101 return 2 * CostFactor; 1102 1103 // It's an extract. Maybe we can do a cheap move-from VSR. 1104 unsigned EltSize = Val->getScalarSizeInBits(); 1105 if (EltSize == 64) { 1106 unsigned MfvsrdIndex = ST->isLittleEndian() ? 1 : 0; 1107 if (Index == MfvsrdIndex) 1108 return 1; 1109 } else if (EltSize == 32) { 1110 unsigned MfvsrwzIndex = ST->isLittleEndian() ? 2 : 1; 1111 if (Index == MfvsrwzIndex) 1112 return 1; 1113 } 1114 1115 // We need a vector extract (or mfvsrld). Assume vector operation cost. 1116 // The cost of the load constant for a vector extract is disregarded 1117 // (invariant, easily schedulable). 1118 return CostFactor; 1119 1120 } else if (ST->hasDirectMove()) 1121 // Assume permute has standard cost. 1122 // Assume move-to/move-from VSR have 2x standard cost. 1123 return 3; 1124 } 1125 1126 // Estimated cost of a load-hit-store delay. This was obtained 1127 // experimentally as a minimum needed to prevent unprofitable 1128 // vectorization for the paq8p benchmark. It may need to be 1129 // raised further if other unprofitable cases remain. 1130 unsigned LHSPenalty = 2; 1131 if (ISD == ISD::INSERT_VECTOR_ELT) 1132 LHSPenalty += 7; 1133 1134 // Vector element insert/extract with Altivec is very expensive, 1135 // because they require store and reload with the attendant 1136 // processor stall for load-hit-store. Until VSX is available, 1137 // these need to be estimated as very costly. 1138 if (ISD == ISD::EXTRACT_VECTOR_ELT || 1139 ISD == ISD::INSERT_VECTOR_ELT) 1140 return LHSPenalty + Cost; 1141 1142 return Cost; 1143 } 1144 1145 InstructionCost PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, 1146 MaybeAlign Alignment, 1147 unsigned AddressSpace, 1148 TTI::TargetCostKind CostKind, 1149 const Instruction *I) { 1150 1151 InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Src, nullptr); 1152 if (!CostFactor.isValid()) 1153 return InstructionCost::getMax(); 1154 1155 if (TLI->getValueType(DL, Src, true) == MVT::Other) 1156 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, 1157 CostKind); 1158 // Legalize the type. 1159 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Src); 1160 assert((Opcode == Instruction::Load || Opcode == Instruction::Store) && 1161 "Invalid Opcode"); 1162 1163 InstructionCost Cost = 1164 BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); 1165 // TODO: Handle other cost kinds. 1166 if (CostKind != TTI::TCK_RecipThroughput) 1167 return Cost; 1168 1169 Cost *= CostFactor; 1170 1171 bool IsAltivecType = ST->hasAltivec() && 1172 (LT.second == MVT::v16i8 || LT.second == MVT::v8i16 || 1173 LT.second == MVT::v4i32 || LT.second == MVT::v4f32); 1174 bool IsVSXType = ST->hasVSX() && 1175 (LT.second == MVT::v2f64 || LT.second == MVT::v2i64); 1176 1177 // VSX has 32b/64b load instructions. Legalization can handle loading of 1178 // 32b/64b to VSR correctly and cheaply. But BaseT::getMemoryOpCost and 1179 // PPCTargetLowering can't compute the cost appropriately. So here we 1180 // explicitly check this case. 1181 unsigned MemBytes = Src->getPrimitiveSizeInBits(); 1182 if (Opcode == Instruction::Load && ST->hasVSX() && IsAltivecType && 1183 (MemBytes == 64 || (ST->hasP8Vector() && MemBytes == 32))) 1184 return 1; 1185 1186 // Aligned loads and stores are easy. 1187 unsigned SrcBytes = LT.second.getStoreSize(); 1188 if (!SrcBytes || !Alignment || *Alignment >= SrcBytes) 1189 return Cost; 1190 1191 // If we can use the permutation-based load sequence, then this is also 1192 // relatively cheap (not counting loop-invariant instructions): one load plus 1193 // one permute (the last load in a series has extra cost, but we're 1194 // neglecting that here). Note that on the P7, we could do unaligned loads 1195 // for Altivec types using the VSX instructions, but that's more expensive 1196 // than using the permutation-based load sequence. On the P8, that's no 1197 // longer true. 1198 if (Opcode == Instruction::Load && (!ST->hasP8Vector() && IsAltivecType) && 1199 *Alignment >= LT.second.getScalarType().getStoreSize()) 1200 return Cost + LT.first; // Add the cost of the permutations. 1201 1202 // For VSX, we can do unaligned loads and stores on Altivec/VSX types. On the 1203 // P7, unaligned vector loads are more expensive than the permutation-based 1204 // load sequence, so that might be used instead, but regardless, the net cost 1205 // is about the same (not counting loop-invariant instructions). 1206 if (IsVSXType || (ST->hasVSX() && IsAltivecType)) 1207 return Cost; 1208 1209 // Newer PPC supports unaligned memory access. 1210 if (TLI->allowsMisalignedMemoryAccesses(LT.second, 0)) 1211 return Cost; 1212 1213 // PPC in general does not support unaligned loads and stores. They'll need 1214 // to be decomposed based on the alignment factor. 1215 1216 // Add the cost of each scalar load or store. 1217 assert(Alignment); 1218 Cost += LT.first * ((SrcBytes / Alignment->value()) - 1); 1219 1220 // For a vector type, there is also scalarization overhead (only for 1221 // stores, loads are expanded using the vector-load + permutation sequence, 1222 // which is much less expensive). 1223 if (Src->isVectorTy() && Opcode == Instruction::Store) 1224 for (int i = 0, e = cast<FixedVectorType>(Src)->getNumElements(); i < e; 1225 ++i) 1226 Cost += getVectorInstrCost(Instruction::ExtractElement, Src, i); 1227 1228 return Cost; 1229 } 1230 1231 InstructionCost PPCTTIImpl::getInterleavedMemoryOpCost( 1232 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, 1233 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, 1234 bool UseMaskForCond, bool UseMaskForGaps) { 1235 InstructionCost CostFactor = 1236 vectorCostAdjustmentFactor(Opcode, VecTy, nullptr); 1237 if (!CostFactor.isValid()) 1238 return InstructionCost::getMax(); 1239 1240 if (UseMaskForCond || UseMaskForGaps) 1241 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, 1242 Alignment, AddressSpace, CostKind, 1243 UseMaskForCond, UseMaskForGaps); 1244 1245 assert(isa<VectorType>(VecTy) && 1246 "Expect a vector type for interleaved memory op"); 1247 1248 // Legalize the type. 1249 std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, VecTy); 1250 1251 // Firstly, the cost of load/store operation. 1252 InstructionCost Cost = getMemoryOpCost(Opcode, VecTy, MaybeAlign(Alignment), 1253 AddressSpace, CostKind); 1254 1255 // PPC, for both Altivec/VSX, support cheap arbitrary permutations 1256 // (at least in the sense that there need only be one non-loop-invariant 1257 // instruction). For each result vector, we need one shuffle per incoming 1258 // vector (except that the first shuffle can take two incoming vectors 1259 // because it does not need to take itself). 1260 Cost += Factor*(LT.first-1); 1261 1262 return Cost; 1263 } 1264 1265 InstructionCost 1266 PPCTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, 1267 TTI::TargetCostKind CostKind) { 1268 return BaseT::getIntrinsicInstrCost(ICA, CostKind); 1269 } 1270 1271 bool PPCTTIImpl::areTypesABICompatible(const Function *Caller, 1272 const Function *Callee, 1273 const ArrayRef<Type *> &Types) const { 1274 1275 // We need to ensure that argument promotion does not 1276 // attempt to promote pointers to MMA types (__vector_pair 1277 // and __vector_quad) since these types explicitly cannot be 1278 // passed as arguments. Both of these types are larger than 1279 // the 128-bit Altivec vectors and have a scalar size of 1 bit. 1280 if (!BaseT::areTypesABICompatible(Caller, Callee, Types)) 1281 return false; 1282 1283 return llvm::none_of(Types, [](Type *Ty) { 1284 if (Ty->isSized()) 1285 return Ty->isIntOrIntVectorTy(1) && Ty->getPrimitiveSizeInBits() > 128; 1286 return false; 1287 }); 1288 } 1289 1290 bool PPCTTIImpl::canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, 1291 LoopInfo *LI, DominatorTree *DT, 1292 AssumptionCache *AC, TargetLibraryInfo *LibInfo) { 1293 // Process nested loops first. 1294 for (Loop *I : *L) 1295 if (canSaveCmp(I, BI, SE, LI, DT, AC, LibInfo)) 1296 return false; // Stop search. 1297 1298 HardwareLoopInfo HWLoopInfo(L); 1299 1300 if (!HWLoopInfo.canAnalyze(*LI)) 1301 return false; 1302 1303 if (!isHardwareLoopProfitable(L, *SE, *AC, LibInfo, HWLoopInfo)) 1304 return false; 1305 1306 if (!HWLoopInfo.isHardwareLoopCandidate(*SE, *LI, *DT)) 1307 return false; 1308 1309 *BI = HWLoopInfo.ExitBranch; 1310 return true; 1311 } 1312 1313 bool PPCTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1, 1314 const TargetTransformInfo::LSRCost &C2) { 1315 // PowerPC default behaviour here is "instruction number 1st priority". 1316 // If LsrNoInsnsCost is set, call default implementation. 1317 if (!LsrNoInsnsCost) 1318 return std::tie(C1.Insns, C1.NumRegs, C1.AddRecCost, C1.NumIVMuls, 1319 C1.NumBaseAdds, C1.ScaleCost, C1.ImmCost, C1.SetupCost) < 1320 std::tie(C2.Insns, C2.NumRegs, C2.AddRecCost, C2.NumIVMuls, 1321 C2.NumBaseAdds, C2.ScaleCost, C2.ImmCost, C2.SetupCost); 1322 else 1323 return TargetTransformInfoImplBase::isLSRCostLess(C1, C2); 1324 } 1325 1326 bool PPCTTIImpl::isNumRegsMajorCostOfLSR() { 1327 return false; 1328 } 1329 1330 bool PPCTTIImpl::shouldBuildRelLookupTables() const { 1331 const PPCTargetMachine &TM = ST->getTargetMachine(); 1332 // XCOFF hasn't implemented lowerRelativeReference, disable non-ELF for now. 1333 if (!TM.isELFv2ABI()) 1334 return false; 1335 return BaseT::shouldBuildRelLookupTables(); 1336 } 1337 1338 bool PPCTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst, 1339 MemIntrinsicInfo &Info) { 1340 switch (Inst->getIntrinsicID()) { 1341 case Intrinsic::ppc_altivec_lvx: 1342 case Intrinsic::ppc_altivec_lvxl: 1343 case Intrinsic::ppc_altivec_lvebx: 1344 case Intrinsic::ppc_altivec_lvehx: 1345 case Intrinsic::ppc_altivec_lvewx: 1346 case Intrinsic::ppc_vsx_lxvd2x: 1347 case Intrinsic::ppc_vsx_lxvw4x: 1348 case Intrinsic::ppc_vsx_lxvd2x_be: 1349 case Intrinsic::ppc_vsx_lxvw4x_be: 1350 case Intrinsic::ppc_vsx_lxvl: 1351 case Intrinsic::ppc_vsx_lxvll: 1352 case Intrinsic::ppc_vsx_lxvp: { 1353 Info.PtrVal = Inst->getArgOperand(0); 1354 Info.ReadMem = true; 1355 Info.WriteMem = false; 1356 return true; 1357 } 1358 case Intrinsic::ppc_altivec_stvx: 1359 case Intrinsic::ppc_altivec_stvxl: 1360 case Intrinsic::ppc_altivec_stvebx: 1361 case Intrinsic::ppc_altivec_stvehx: 1362 case Intrinsic::ppc_altivec_stvewx: 1363 case Intrinsic::ppc_vsx_stxvd2x: 1364 case Intrinsic::ppc_vsx_stxvw4x: 1365 case Intrinsic::ppc_vsx_stxvd2x_be: 1366 case Intrinsic::ppc_vsx_stxvw4x_be: 1367 case Intrinsic::ppc_vsx_stxvl: 1368 case Intrinsic::ppc_vsx_stxvll: 1369 case Intrinsic::ppc_vsx_stxvp: { 1370 Info.PtrVal = Inst->getArgOperand(1); 1371 Info.ReadMem = false; 1372 Info.WriteMem = true; 1373 return true; 1374 } 1375 default: 1376 break; 1377 } 1378 1379 return false; 1380 } 1381 1382 bool PPCTTIImpl::hasActiveVectorLength(unsigned Opcode, Type *DataType, 1383 Align Alignment) const { 1384 // Only load and stores instructions can have variable vector length on Power. 1385 if (Opcode != Instruction::Load && Opcode != Instruction::Store) 1386 return false; 1387 // Loads/stores with length instructions use bits 0-7 of the GPR operand and 1388 // therefore cannot be used in 32-bit mode. 1389 if ((!ST->hasP9Vector() && !ST->hasP10Vector()) || !ST->isPPC64()) 1390 return false; 1391 if (isa<FixedVectorType>(DataType)) { 1392 unsigned VecWidth = DataType->getPrimitiveSizeInBits(); 1393 return VecWidth == 128; 1394 } 1395 Type *ScalarTy = DataType->getScalarType(); 1396 1397 if (ScalarTy->isPointerTy()) 1398 return true; 1399 1400 if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy()) 1401 return true; 1402 1403 if (!ScalarTy->isIntegerTy()) 1404 return false; 1405 1406 unsigned IntWidth = ScalarTy->getIntegerBitWidth(); 1407 return IntWidth == 8 || IntWidth == 16 || IntWidth == 32 || IntWidth == 64; 1408 } 1409 1410 InstructionCost PPCTTIImpl::getVPMemoryOpCost(unsigned Opcode, Type *Src, 1411 Align Alignment, 1412 unsigned AddressSpace, 1413 TTI::TargetCostKind CostKind, 1414 const Instruction *I) { 1415 InstructionCost Cost = BaseT::getVPMemoryOpCost(Opcode, Src, Alignment, 1416 AddressSpace, CostKind, I); 1417 if (TLI->getValueType(DL, Src, true) == MVT::Other) 1418 return Cost; 1419 // TODO: Handle other cost kinds. 1420 if (CostKind != TTI::TCK_RecipThroughput) 1421 return Cost; 1422 1423 assert((Opcode == Instruction::Load || Opcode == Instruction::Store) && 1424 "Invalid Opcode"); 1425 1426 auto *SrcVTy = dyn_cast<FixedVectorType>(Src); 1427 assert(SrcVTy && "Expected a vector type for VP memory operations"); 1428 1429 if (hasActiveVectorLength(Opcode, Src, Alignment)) { 1430 std::pair<InstructionCost, MVT> LT = 1431 TLI->getTypeLegalizationCost(DL, SrcVTy); 1432 1433 InstructionCost CostFactor = 1434 vectorCostAdjustmentFactor(Opcode, Src, nullptr); 1435 if (!CostFactor.isValid()) 1436 return InstructionCost::getMax(); 1437 1438 InstructionCost Cost = LT.first * CostFactor; 1439 assert(Cost.isValid() && "Expected valid cost"); 1440 1441 // On P9 but not on P10, if the op is misaligned then it will cause a 1442 // pipeline flush. Otherwise the VSX masked memops cost the same as unmasked 1443 // ones. 1444 const Align DesiredAlignment(16); 1445 if (Alignment >= DesiredAlignment || ST->getCPUDirective() != PPC::DIR_PWR9) 1446 return Cost; 1447 1448 // Since alignment may be under estimated, we try to compute the probability 1449 // that the actual address is aligned to the desired boundary. For example 1450 // an 8-byte aligned load is assumed to be actually 16-byte aligned half the 1451 // time, while a 4-byte aligned load has a 25% chance of being 16-byte 1452 // aligned. 1453 float AlignmentProb = ((float)Alignment.value()) / DesiredAlignment.value(); 1454 float MisalignmentProb = 1.0 - AlignmentProb; 1455 return (MisalignmentProb * P9PipelineFlushEstimate) + 1456 (AlignmentProb * *Cost.getValue()); 1457 } 1458 1459 // Usually we should not get to this point, but the following is an attempt to 1460 // model the cost of legalization. Currently we can only lower intrinsics with 1461 // evl but no mask, on Power 9/10. Otherwise, we must scalarize. 1462 return getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); 1463 } 1464