1 //===-- SystemZTargetTransformInfo.cpp - SystemZ-specific TTI -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements a TargetTransformInfo analysis pass specific to the
10 // SystemZ target machine. It uses the target's detailed information to provide
11 // more precise answers to certain TTI queries, while letting the target
12 // independent and default TTI implementations handle the rest.
13 //
14 //===----------------------------------------------------------------------===//
15
16 #include "SystemZTargetTransformInfo.h"
17 #include "llvm/Analysis/TargetTransformInfo.h"
18 #include "llvm/CodeGen/BasicTTIImpl.h"
19 #include "llvm/CodeGen/CostTable.h"
20 #include "llvm/CodeGen/TargetLowering.h"
21 #include "llvm/IR/IntrinsicInst.h"
22 #include "llvm/Support/Debug.h"
23 using namespace llvm;
24
25 #define DEBUG_TYPE "systemztti"
26
27 //===----------------------------------------------------------------------===//
28 //
29 // SystemZ cost model.
30 //
31 //===----------------------------------------------------------------------===//
32
getIntImmCost(const APInt & Imm,Type * Ty,TTI::TargetCostKind CostKind)33 InstructionCost SystemZTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
34 TTI::TargetCostKind CostKind) {
35 assert(Ty->isIntegerTy());
36
37 unsigned BitSize = Ty->getPrimitiveSizeInBits();
38 // There is no cost model for constants with a bit size of 0. Return TCC_Free
39 // here, so that constant hoisting will ignore this constant.
40 if (BitSize == 0)
41 return TTI::TCC_Free;
42 // No cost model for operations on integers larger than 64 bit implemented yet.
43 if (BitSize > 64)
44 return TTI::TCC_Free;
45
46 if (Imm == 0)
47 return TTI::TCC_Free;
48
49 if (Imm.getBitWidth() <= 64) {
50 // Constants loaded via lgfi.
51 if (isInt<32>(Imm.getSExtValue()))
52 return TTI::TCC_Basic;
53 // Constants loaded via llilf.
54 if (isUInt<32>(Imm.getZExtValue()))
55 return TTI::TCC_Basic;
56 // Constants loaded via llihf:
57 if ((Imm.getZExtValue() & 0xffffffff) == 0)
58 return TTI::TCC_Basic;
59
60 return 2 * TTI::TCC_Basic;
61 }
62
63 return 4 * TTI::TCC_Basic;
64 }
65
getIntImmCostInst(unsigned Opcode,unsigned Idx,const APInt & Imm,Type * Ty,TTI::TargetCostKind CostKind,Instruction * Inst)66 InstructionCost SystemZTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
67 const APInt &Imm, Type *Ty,
68 TTI::TargetCostKind CostKind,
69 Instruction *Inst) {
70 assert(Ty->isIntegerTy());
71
72 unsigned BitSize = Ty->getPrimitiveSizeInBits();
73 // There is no cost model for constants with a bit size of 0. Return TCC_Free
74 // here, so that constant hoisting will ignore this constant.
75 if (BitSize == 0)
76 return TTI::TCC_Free;
77 // No cost model for operations on integers larger than 64 bit implemented yet.
78 if (BitSize > 64)
79 return TTI::TCC_Free;
80
81 switch (Opcode) {
82 default:
83 return TTI::TCC_Free;
84 case Instruction::GetElementPtr:
85 // Always hoist the base address of a GetElementPtr. This prevents the
86 // creation of new constants for every base constant that gets constant
87 // folded with the offset.
88 if (Idx == 0)
89 return 2 * TTI::TCC_Basic;
90 return TTI::TCC_Free;
91 case Instruction::Store:
92 if (Idx == 0 && Imm.getBitWidth() <= 64) {
93 // Any 8-bit immediate store can by implemented via mvi.
94 if (BitSize == 8)
95 return TTI::TCC_Free;
96 // 16-bit immediate values can be stored via mvhhi/mvhi/mvghi.
97 if (isInt<16>(Imm.getSExtValue()))
98 return TTI::TCC_Free;
99 }
100 break;
101 case Instruction::ICmp:
102 if (Idx == 1 && Imm.getBitWidth() <= 64) {
103 // Comparisons against signed 32-bit immediates implemented via cgfi.
104 if (isInt<32>(Imm.getSExtValue()))
105 return TTI::TCC_Free;
106 // Comparisons against unsigned 32-bit immediates implemented via clgfi.
107 if (isUInt<32>(Imm.getZExtValue()))
108 return TTI::TCC_Free;
109 }
110 break;
111 case Instruction::Add:
112 case Instruction::Sub:
113 if (Idx == 1 && Imm.getBitWidth() <= 64) {
114 // We use algfi/slgfi to add/subtract 32-bit unsigned immediates.
115 if (isUInt<32>(Imm.getZExtValue()))
116 return TTI::TCC_Free;
117 // Or their negation, by swapping addition vs. subtraction.
118 if (isUInt<32>(-Imm.getSExtValue()))
119 return TTI::TCC_Free;
120 }
121 break;
122 case Instruction::Mul:
123 if (Idx == 1 && Imm.getBitWidth() <= 64) {
124 // We use msgfi to multiply by 32-bit signed immediates.
125 if (isInt<32>(Imm.getSExtValue()))
126 return TTI::TCC_Free;
127 }
128 break;
129 case Instruction::Or:
130 case Instruction::Xor:
131 if (Idx == 1 && Imm.getBitWidth() <= 64) {
132 // Masks supported by oilf/xilf.
133 if (isUInt<32>(Imm.getZExtValue()))
134 return TTI::TCC_Free;
135 // Masks supported by oihf/xihf.
136 if ((Imm.getZExtValue() & 0xffffffff) == 0)
137 return TTI::TCC_Free;
138 }
139 break;
140 case Instruction::And:
141 if (Idx == 1 && Imm.getBitWidth() <= 64) {
142 // Any 32-bit AND operation can by implemented via nilf.
143 if (BitSize <= 32)
144 return TTI::TCC_Free;
145 // 64-bit masks supported by nilf.
146 if (isUInt<32>(~Imm.getZExtValue()))
147 return TTI::TCC_Free;
148 // 64-bit masks supported by nilh.
149 if ((Imm.getZExtValue() & 0xffffffff) == 0xffffffff)
150 return TTI::TCC_Free;
151 // Some 64-bit AND operations can be implemented via risbg.
152 const SystemZInstrInfo *TII = ST->getInstrInfo();
153 unsigned Start, End;
154 if (TII->isRxSBGMask(Imm.getZExtValue(), BitSize, Start, End))
155 return TTI::TCC_Free;
156 }
157 break;
158 case Instruction::Shl:
159 case Instruction::LShr:
160 case Instruction::AShr:
161 // Always return TCC_Free for the shift value of a shift instruction.
162 if (Idx == 1)
163 return TTI::TCC_Free;
164 break;
165 case Instruction::UDiv:
166 case Instruction::SDiv:
167 case Instruction::URem:
168 case Instruction::SRem:
169 case Instruction::Trunc:
170 case Instruction::ZExt:
171 case Instruction::SExt:
172 case Instruction::IntToPtr:
173 case Instruction::PtrToInt:
174 case Instruction::BitCast:
175 case Instruction::PHI:
176 case Instruction::Call:
177 case Instruction::Select:
178 case Instruction::Ret:
179 case Instruction::Load:
180 break;
181 }
182
183 return SystemZTTIImpl::getIntImmCost(Imm, Ty, CostKind);
184 }
185
186 InstructionCost
getIntImmCostIntrin(Intrinsic::ID IID,unsigned Idx,const APInt & Imm,Type * Ty,TTI::TargetCostKind CostKind)187 SystemZTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
188 const APInt &Imm, Type *Ty,
189 TTI::TargetCostKind CostKind) {
190 assert(Ty->isIntegerTy());
191
192 unsigned BitSize = Ty->getPrimitiveSizeInBits();
193 // There is no cost model for constants with a bit size of 0. Return TCC_Free
194 // here, so that constant hoisting will ignore this constant.
195 if (BitSize == 0)
196 return TTI::TCC_Free;
197 // No cost model for operations on integers larger than 64 bit implemented yet.
198 if (BitSize > 64)
199 return TTI::TCC_Free;
200
201 switch (IID) {
202 default:
203 return TTI::TCC_Free;
204 case Intrinsic::sadd_with_overflow:
205 case Intrinsic::uadd_with_overflow:
206 case Intrinsic::ssub_with_overflow:
207 case Intrinsic::usub_with_overflow:
208 // These get expanded to include a normal addition/subtraction.
209 if (Idx == 1 && Imm.getBitWidth() <= 64) {
210 if (isUInt<32>(Imm.getZExtValue()))
211 return TTI::TCC_Free;
212 if (isUInt<32>(-Imm.getSExtValue()))
213 return TTI::TCC_Free;
214 }
215 break;
216 case Intrinsic::smul_with_overflow:
217 case Intrinsic::umul_with_overflow:
218 // These get expanded to include a normal multiplication.
219 if (Idx == 1 && Imm.getBitWidth() <= 64) {
220 if (isInt<32>(Imm.getSExtValue()))
221 return TTI::TCC_Free;
222 }
223 break;
224 case Intrinsic::experimental_stackmap:
225 if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
226 return TTI::TCC_Free;
227 break;
228 case Intrinsic::experimental_patchpoint_void:
229 case Intrinsic::experimental_patchpoint_i64:
230 if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
231 return TTI::TCC_Free;
232 break;
233 }
234 return SystemZTTIImpl::getIntImmCost(Imm, Ty, CostKind);
235 }
236
237 TargetTransformInfo::PopcntSupportKind
getPopcntSupport(unsigned TyWidth)238 SystemZTTIImpl::getPopcntSupport(unsigned TyWidth) {
239 assert(isPowerOf2_32(TyWidth) && "Type width must be power of 2");
240 if (ST->hasPopulationCount() && TyWidth <= 64)
241 return TTI::PSK_FastHardware;
242 return TTI::PSK_Software;
243 }
244
getUnrollingPreferences(Loop * L,ScalarEvolution & SE,TTI::UnrollingPreferences & UP)245 void SystemZTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
246 TTI::UnrollingPreferences &UP) {
247 // Find out if L contains a call, what the machine instruction count
248 // estimate is, and how many stores there are.
249 bool HasCall = false;
250 InstructionCost NumStores = 0;
251 for (auto &BB : L->blocks())
252 for (auto &I : *BB) {
253 if (isa<CallInst>(&I) || isa<InvokeInst>(&I)) {
254 if (const Function *F = cast<CallBase>(I).getCalledFunction()) {
255 if (isLoweredToCall(F))
256 HasCall = true;
257 if (F->getIntrinsicID() == Intrinsic::memcpy ||
258 F->getIntrinsicID() == Intrinsic::memset)
259 NumStores++;
260 } else { // indirect call.
261 HasCall = true;
262 }
263 }
264 if (isa<StoreInst>(&I)) {
265 Type *MemAccessTy = I.getOperand(0)->getType();
266 NumStores += getMemoryOpCost(Instruction::Store, MemAccessTy, None, 0,
267 TTI::TCK_RecipThroughput);
268 }
269 }
270
271 // The z13 processor will run out of store tags if too many stores
272 // are fed into it too quickly. Therefore make sure there are not
273 // too many stores in the resulting unrolled loop.
274 unsigned const NumStoresVal = *NumStores.getValue();
275 unsigned const Max = (NumStoresVal ? (12 / NumStoresVal) : UINT_MAX);
276
277 if (HasCall) {
278 // Only allow full unrolling if loop has any calls.
279 UP.FullUnrollMaxCount = Max;
280 UP.MaxCount = 1;
281 return;
282 }
283
284 UP.MaxCount = Max;
285 if (UP.MaxCount <= 1)
286 return;
287
288 // Allow partial and runtime trip count unrolling.
289 UP.Partial = UP.Runtime = true;
290
291 UP.PartialThreshold = 75;
292 UP.DefaultUnrollRuntimeCount = 4;
293
294 // Allow expensive instructions in the pre-header of the loop.
295 UP.AllowExpensiveTripCount = true;
296
297 UP.Force = true;
298 }
299
getPeelingPreferences(Loop * L,ScalarEvolution & SE,TTI::PeelingPreferences & PP)300 void SystemZTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
301 TTI::PeelingPreferences &PP) {
302 BaseT::getPeelingPreferences(L, SE, PP);
303 }
304
isLSRCostLess(TargetTransformInfo::LSRCost & C1,TargetTransformInfo::LSRCost & C2)305 bool SystemZTTIImpl::isLSRCostLess(TargetTransformInfo::LSRCost &C1,
306 TargetTransformInfo::LSRCost &C2) {
307 // SystemZ specific: check instruction count (first), and don't care about
308 // ImmCost, since offsets are checked explicitly.
309 return std::tie(C1.Insns, C1.NumRegs, C1.AddRecCost,
310 C1.NumIVMuls, C1.NumBaseAdds,
311 C1.ScaleCost, C1.SetupCost) <
312 std::tie(C2.Insns, C2.NumRegs, C2.AddRecCost,
313 C2.NumIVMuls, C2.NumBaseAdds,
314 C2.ScaleCost, C2.SetupCost);
315 }
316
getNumberOfRegisters(unsigned ClassID) const317 unsigned SystemZTTIImpl::getNumberOfRegisters(unsigned ClassID) const {
318 bool Vector = (ClassID == 1);
319 if (!Vector)
320 // Discount the stack pointer. Also leave out %r0, since it can't
321 // be used in an address.
322 return 14;
323 if (ST->hasVector())
324 return 32;
325 return 0;
326 }
327
328 TypeSize
getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const329 SystemZTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
330 switch (K) {
331 case TargetTransformInfo::RGK_Scalar:
332 return TypeSize::getFixed(64);
333 case TargetTransformInfo::RGK_FixedWidthVector:
334 return TypeSize::getFixed(ST->hasVector() ? 128 : 0);
335 case TargetTransformInfo::RGK_ScalableVector:
336 return TypeSize::getScalable(0);
337 }
338
339 llvm_unreachable("Unsupported register kind");
340 }
341
getMinPrefetchStride(unsigned NumMemAccesses,unsigned NumStridedMemAccesses,unsigned NumPrefetches,bool HasCall) const342 unsigned SystemZTTIImpl::getMinPrefetchStride(unsigned NumMemAccesses,
343 unsigned NumStridedMemAccesses,
344 unsigned NumPrefetches,
345 bool HasCall) const {
346 // Don't prefetch a loop with many far apart accesses.
347 if (NumPrefetches > 16)
348 return UINT_MAX;
349
350 // Emit prefetch instructions for smaller strides in cases where we think
351 // the hardware prefetcher might not be able to keep up.
352 if (NumStridedMemAccesses > 32 && !HasCall &&
353 (NumMemAccesses - NumStridedMemAccesses) * 32 <= NumStridedMemAccesses)
354 return 1;
355
356 return ST->hasMiscellaneousExtensions3() ? 8192 : 2048;
357 }
358
hasDivRemOp(Type * DataType,bool IsSigned)359 bool SystemZTTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) {
360 EVT VT = TLI->getValueType(DL, DataType);
361 return (VT.isScalarInteger() && TLI->isTypeLegal(VT));
362 }
363
364 // Return the bit size for the scalar type or vector element
365 // type. getScalarSizeInBits() returns 0 for a pointer type.
getScalarSizeInBits(Type * Ty)366 static unsigned getScalarSizeInBits(Type *Ty) {
367 unsigned Size =
368 (Ty->isPtrOrPtrVectorTy() ? 64U : Ty->getScalarSizeInBits());
369 assert(Size > 0 && "Element must have non-zero size.");
370 return Size;
371 }
372
373 // getNumberOfParts() calls getTypeLegalizationCost() which splits the vector
374 // type until it is legal. This would e.g. return 4 for <6 x i64>, instead of
375 // 3.
getNumVectorRegs(Type * Ty)376 static unsigned getNumVectorRegs(Type *Ty) {
377 auto *VTy = cast<FixedVectorType>(Ty);
378 unsigned WideBits = getScalarSizeInBits(Ty) * VTy->getNumElements();
379 assert(WideBits > 0 && "Could not compute size of vector");
380 return ((WideBits % 128U) ? ((WideBits / 128U) + 1) : (WideBits / 128U));
381 }
382
getArithmeticInstrCost(unsigned Opcode,Type * Ty,TTI::TargetCostKind CostKind,TTI::OperandValueKind Op1Info,TTI::OperandValueKind Op2Info,TTI::OperandValueProperties Opd1PropInfo,TTI::OperandValueProperties Opd2PropInfo,ArrayRef<const Value * > Args,const Instruction * CxtI)383 InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
384 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
385 TTI::OperandValueKind Op1Info, TTI::OperandValueKind Op2Info,
386 TTI::OperandValueProperties Opd1PropInfo,
387 TTI::OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args,
388 const Instruction *CxtI) {
389
390 // TODO: Handle more cost kinds.
391 if (CostKind != TTI::TCK_RecipThroughput)
392 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
393 Op2Info, Opd1PropInfo,
394 Opd2PropInfo, Args, CxtI);
395
396 // TODO: return a good value for BB-VECTORIZER that includes the
397 // immediate loads, which we do not want to count for the loop
398 // vectorizer, since they are hopefully hoisted out of the loop. This
399 // would require a new parameter 'InLoop', but not sure if constant
400 // args are common enough to motivate this.
401
402 unsigned ScalarBits = Ty->getScalarSizeInBits();
403
404 // There are thre cases of division and remainder: Dividing with a register
405 // needs a divide instruction. A divisor which is a power of two constant
406 // can be implemented with a sequence of shifts. Any other constant needs a
407 // multiply and shifts.
408 const unsigned DivInstrCost = 20;
409 const unsigned DivMulSeqCost = 10;
410 const unsigned SDivPow2Cost = 4;
411
412 bool SignedDivRem =
413 Opcode == Instruction::SDiv || Opcode == Instruction::SRem;
414 bool UnsignedDivRem =
415 Opcode == Instruction::UDiv || Opcode == Instruction::URem;
416
417 // Check for a constant divisor.
418 bool DivRemConst = false;
419 bool DivRemConstPow2 = false;
420 if ((SignedDivRem || UnsignedDivRem) && Args.size() == 2) {
421 if (const Constant *C = dyn_cast<Constant>(Args[1])) {
422 const ConstantInt *CVal =
423 (C->getType()->isVectorTy()
424 ? dyn_cast_or_null<const ConstantInt>(C->getSplatValue())
425 : dyn_cast<const ConstantInt>(C));
426 if (CVal != nullptr &&
427 (CVal->getValue().isPowerOf2() || (-CVal->getValue()).isPowerOf2()))
428 DivRemConstPow2 = true;
429 else
430 DivRemConst = true;
431 }
432 }
433
434 if (!Ty->isVectorTy()) {
435 // These FP operations are supported with a dedicated instruction for
436 // float, double and fp128 (base implementation assumes float generally
437 // costs 2).
438 if (Opcode == Instruction::FAdd || Opcode == Instruction::FSub ||
439 Opcode == Instruction::FMul || Opcode == Instruction::FDiv)
440 return 1;
441
442 // There is no native support for FRem.
443 if (Opcode == Instruction::FRem)
444 return LIBCALL_COST;
445
446 // Give discount for some combined logical operations if supported.
447 if (Args.size() == 2 && ST->hasMiscellaneousExtensions3()) {
448 if (Opcode == Instruction::Xor) {
449 for (const Value *A : Args) {
450 if (const Instruction *I = dyn_cast<Instruction>(A))
451 if (I->hasOneUse() &&
452 (I->getOpcode() == Instruction::And ||
453 I->getOpcode() == Instruction::Or ||
454 I->getOpcode() == Instruction::Xor))
455 return 0;
456 }
457 }
458 else if (Opcode == Instruction::Or || Opcode == Instruction::And) {
459 for (const Value *A : Args) {
460 if (const Instruction *I = dyn_cast<Instruction>(A))
461 if (I->hasOneUse() && I->getOpcode() == Instruction::Xor)
462 return 0;
463 }
464 }
465 }
466
467 // Or requires one instruction, although it has custom handling for i64.
468 if (Opcode == Instruction::Or)
469 return 1;
470
471 if (Opcode == Instruction::Xor && ScalarBits == 1) {
472 if (ST->hasLoadStoreOnCond2())
473 return 5; // 2 * (li 0; loc 1); xor
474 return 7; // 2 * ipm sequences ; xor ; shift ; compare
475 }
476
477 if (DivRemConstPow2)
478 return (SignedDivRem ? SDivPow2Cost : 1);
479 if (DivRemConst)
480 return DivMulSeqCost;
481 if (SignedDivRem || UnsignedDivRem)
482 return DivInstrCost;
483 }
484 else if (ST->hasVector()) {
485 auto *VTy = cast<FixedVectorType>(Ty);
486 unsigned VF = VTy->getNumElements();
487 unsigned NumVectors = getNumVectorRegs(Ty);
488
489 // These vector operations are custom handled, but are still supported
490 // with one instruction per vector, regardless of element size.
491 if (Opcode == Instruction::Shl || Opcode == Instruction::LShr ||
492 Opcode == Instruction::AShr) {
493 return NumVectors;
494 }
495
496 if (DivRemConstPow2)
497 return (NumVectors * (SignedDivRem ? SDivPow2Cost : 1));
498 if (DivRemConst) {
499 SmallVector<Type *> Tys(Args.size(), Ty);
500 return VF * DivMulSeqCost + getScalarizationOverhead(VTy, Args, Tys);
501 }
502 if ((SignedDivRem || UnsignedDivRem) && VF > 4)
503 // Temporary hack: disable high vectorization factors with integer
504 // division/remainder, which will get scalarized and handled with
505 // GR128 registers. The mischeduler is not clever enough to avoid
506 // spilling yet.
507 return 1000;
508
509 // These FP operations are supported with a single vector instruction for
510 // double (base implementation assumes float generally costs 2). For
511 // FP128, the scalar cost is 1, and there is no overhead since the values
512 // are already in scalar registers.
513 if (Opcode == Instruction::FAdd || Opcode == Instruction::FSub ||
514 Opcode == Instruction::FMul || Opcode == Instruction::FDiv) {
515 switch (ScalarBits) {
516 case 32: {
517 // The vector enhancements facility 1 provides v4f32 instructions.
518 if (ST->hasVectorEnhancements1())
519 return NumVectors;
520 // Return the cost of multiple scalar invocation plus the cost of
521 // inserting and extracting the values.
522 InstructionCost ScalarCost =
523 getArithmeticInstrCost(Opcode, Ty->getScalarType(), CostKind);
524 SmallVector<Type *> Tys(Args.size(), Ty);
525 InstructionCost Cost =
526 (VF * ScalarCost) + getScalarizationOverhead(VTy, Args, Tys);
527 // FIXME: VF 2 for these FP operations are currently just as
528 // expensive as for VF 4.
529 if (VF == 2)
530 Cost *= 2;
531 return Cost;
532 }
533 case 64:
534 case 128:
535 return NumVectors;
536 default:
537 break;
538 }
539 }
540
541 // There is no native support for FRem.
542 if (Opcode == Instruction::FRem) {
543 SmallVector<Type *> Tys(Args.size(), Ty);
544 InstructionCost Cost =
545 (VF * LIBCALL_COST) + getScalarizationOverhead(VTy, Args, Tys);
546 // FIXME: VF 2 for float is currently just as expensive as for VF 4.
547 if (VF == 2 && ScalarBits == 32)
548 Cost *= 2;
549 return Cost;
550 }
551 }
552
553 // Fallback to the default implementation.
554 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
555 Opd1PropInfo, Opd2PropInfo, Args, CxtI);
556 }
557
getShuffleCost(TTI::ShuffleKind Kind,VectorType * Tp,ArrayRef<int> Mask,int Index,VectorType * SubTp)558 InstructionCost SystemZTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
559 VectorType *Tp,
560 ArrayRef<int> Mask, int Index,
561 VectorType *SubTp) {
562 Kind = improveShuffleKindFromMask(Kind, Mask);
563 if (ST->hasVector()) {
564 unsigned NumVectors = getNumVectorRegs(Tp);
565
566 // TODO: Since fp32 is expanded, the shuffle cost should always be 0.
567
568 // FP128 values are always in scalar registers, so there is no work
569 // involved with a shuffle, except for broadcast. In that case register
570 // moves are done with a single instruction per element.
571 if (Tp->getScalarType()->isFP128Ty())
572 return (Kind == TargetTransformInfo::SK_Broadcast ? NumVectors - 1 : 0);
573
574 switch (Kind) {
575 case TargetTransformInfo::SK_ExtractSubvector:
576 // ExtractSubvector Index indicates start offset.
577
578 // Extracting a subvector from first index is a noop.
579 return (Index == 0 ? 0 : NumVectors);
580
581 case TargetTransformInfo::SK_Broadcast:
582 // Loop vectorizer calls here to figure out the extra cost of
583 // broadcasting a loaded value to all elements of a vector. Since vlrep
584 // loads and replicates with a single instruction, adjust the returned
585 // value.
586 return NumVectors - 1;
587
588 default:
589
590 // SystemZ supports single instruction permutation / replication.
591 return NumVectors;
592 }
593 }
594
595 return BaseT::getShuffleCost(Kind, Tp, Mask, Index, SubTp);
596 }
597
598 // Return the log2 difference of the element sizes of the two vector types.
getElSizeLog2Diff(Type * Ty0,Type * Ty1)599 static unsigned getElSizeLog2Diff(Type *Ty0, Type *Ty1) {
600 unsigned Bits0 = Ty0->getScalarSizeInBits();
601 unsigned Bits1 = Ty1->getScalarSizeInBits();
602
603 if (Bits1 > Bits0)
604 return (Log2_32(Bits1) - Log2_32(Bits0));
605
606 return (Log2_32(Bits0) - Log2_32(Bits1));
607 }
608
609 // Return the number of instructions needed to truncate SrcTy to DstTy.
610 unsigned SystemZTTIImpl::
getVectorTruncCost(Type * SrcTy,Type * DstTy)611 getVectorTruncCost(Type *SrcTy, Type *DstTy) {
612 assert (SrcTy->isVectorTy() && DstTy->isVectorTy());
613 assert(SrcTy->getPrimitiveSizeInBits().getFixedSize() >
614 DstTy->getPrimitiveSizeInBits().getFixedSize() &&
615 "Packing must reduce size of vector type.");
616 assert(cast<FixedVectorType>(SrcTy)->getNumElements() ==
617 cast<FixedVectorType>(DstTy)->getNumElements() &&
618 "Packing should not change number of elements.");
619
620 // TODO: Since fp32 is expanded, the extract cost should always be 0.
621
622 unsigned NumParts = getNumVectorRegs(SrcTy);
623 if (NumParts <= 2)
624 // Up to 2 vector registers can be truncated efficiently with pack or
625 // permute. The latter requires an immediate mask to be loaded, which
626 // typically gets hoisted out of a loop. TODO: return a good value for
627 // BB-VECTORIZER that includes the immediate loads, which we do not want
628 // to count for the loop vectorizer.
629 return 1;
630
631 unsigned Cost = 0;
632 unsigned Log2Diff = getElSizeLog2Diff(SrcTy, DstTy);
633 unsigned VF = cast<FixedVectorType>(SrcTy)->getNumElements();
634 for (unsigned P = 0; P < Log2Diff; ++P) {
635 if (NumParts > 1)
636 NumParts /= 2;
637 Cost += NumParts;
638 }
639
640 // Currently, a general mix of permutes and pack instructions is output by
641 // isel, which follow the cost computation above except for this case which
642 // is one instruction less:
643 if (VF == 8 && SrcTy->getScalarSizeInBits() == 64 &&
644 DstTy->getScalarSizeInBits() == 8)
645 Cost--;
646
647 return Cost;
648 }
649
650 // Return the cost of converting a vector bitmask produced by a compare
651 // (SrcTy), to the type of the select or extend instruction (DstTy).
652 unsigned SystemZTTIImpl::
getVectorBitmaskConversionCost(Type * SrcTy,Type * DstTy)653 getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy) {
654 assert (SrcTy->isVectorTy() && DstTy->isVectorTy() &&
655 "Should only be called with vector types.");
656
657 unsigned PackCost = 0;
658 unsigned SrcScalarBits = SrcTy->getScalarSizeInBits();
659 unsigned DstScalarBits = DstTy->getScalarSizeInBits();
660 unsigned Log2Diff = getElSizeLog2Diff(SrcTy, DstTy);
661 if (SrcScalarBits > DstScalarBits)
662 // The bitmask will be truncated.
663 PackCost = getVectorTruncCost(SrcTy, DstTy);
664 else if (SrcScalarBits < DstScalarBits) {
665 unsigned DstNumParts = getNumVectorRegs(DstTy);
666 // Each vector select needs its part of the bitmask unpacked.
667 PackCost = Log2Diff * DstNumParts;
668 // Extra cost for moving part of mask before unpacking.
669 PackCost += DstNumParts - 1;
670 }
671
672 return PackCost;
673 }
674
675 // Return the type of the compared operands. This is needed to compute the
676 // cost for a Select / ZExt or SExt instruction.
getCmpOpsType(const Instruction * I,unsigned VF=1)677 static Type *getCmpOpsType(const Instruction *I, unsigned VF = 1) {
678 Type *OpTy = nullptr;
679 if (CmpInst *CI = dyn_cast<CmpInst>(I->getOperand(0)))
680 OpTy = CI->getOperand(0)->getType();
681 else if (Instruction *LogicI = dyn_cast<Instruction>(I->getOperand(0)))
682 if (LogicI->getNumOperands() == 2)
683 if (CmpInst *CI0 = dyn_cast<CmpInst>(LogicI->getOperand(0)))
684 if (isa<CmpInst>(LogicI->getOperand(1)))
685 OpTy = CI0->getOperand(0)->getType();
686
687 if (OpTy != nullptr) {
688 if (VF == 1) {
689 assert (!OpTy->isVectorTy() && "Expected scalar type");
690 return OpTy;
691 }
692 // Return the potentially vectorized type based on 'I' and 'VF'. 'I' may
693 // be either scalar or already vectorized with a same or lesser VF.
694 Type *ElTy = OpTy->getScalarType();
695 return FixedVectorType::get(ElTy, VF);
696 }
697
698 return nullptr;
699 }
700
701 // Get the cost of converting a boolean vector to a vector with same width
702 // and element size as Dst, plus the cost of zero extending if needed.
703 unsigned SystemZTTIImpl::
getBoolVecToIntConversionCost(unsigned Opcode,Type * Dst,const Instruction * I)704 getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst,
705 const Instruction *I) {
706 auto *DstVTy = cast<FixedVectorType>(Dst);
707 unsigned VF = DstVTy->getNumElements();
708 unsigned Cost = 0;
709 // If we know what the widths of the compared operands, get any cost of
710 // converting it to match Dst. Otherwise assume same widths.
711 Type *CmpOpTy = ((I != nullptr) ? getCmpOpsType(I, VF) : nullptr);
712 if (CmpOpTy != nullptr)
713 Cost = getVectorBitmaskConversionCost(CmpOpTy, Dst);
714 if (Opcode == Instruction::ZExt || Opcode == Instruction::UIToFP)
715 // One 'vn' per dst vector with an immediate mask.
716 Cost += getNumVectorRegs(Dst);
717 return Cost;
718 }
719
getCastInstrCost(unsigned Opcode,Type * Dst,Type * Src,TTI::CastContextHint CCH,TTI::TargetCostKind CostKind,const Instruction * I)720 InstructionCost SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
721 Type *Src,
722 TTI::CastContextHint CCH,
723 TTI::TargetCostKind CostKind,
724 const Instruction *I) {
725 // FIXME: Can the logic below also be used for these cost kinds?
726 if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency) {
727 auto BaseCost = BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
728 return BaseCost == 0 ? BaseCost : 1;
729 }
730
731 unsigned DstScalarBits = Dst->getScalarSizeInBits();
732 unsigned SrcScalarBits = Src->getScalarSizeInBits();
733
734 if (!Src->isVectorTy()) {
735 assert (!Dst->isVectorTy());
736
737 if (Opcode == Instruction::SIToFP || Opcode == Instruction::UIToFP) {
738 if (SrcScalarBits >= 32 ||
739 (I != nullptr && isa<LoadInst>(I->getOperand(0))))
740 return 1;
741 return SrcScalarBits > 1 ? 2 /*i8/i16 extend*/ : 5 /*branch seq.*/;
742 }
743
744 if ((Opcode == Instruction::ZExt || Opcode == Instruction::SExt) &&
745 Src->isIntegerTy(1)) {
746 if (ST->hasLoadStoreOnCond2())
747 return 2; // li 0; loc 1
748
749 // This should be extension of a compare i1 result, which is done with
750 // ipm and a varying sequence of instructions.
751 unsigned Cost = 0;
752 if (Opcode == Instruction::SExt)
753 Cost = (DstScalarBits < 64 ? 3 : 4);
754 if (Opcode == Instruction::ZExt)
755 Cost = 3;
756 Type *CmpOpTy = ((I != nullptr) ? getCmpOpsType(I) : nullptr);
757 if (CmpOpTy != nullptr && CmpOpTy->isFloatingPointTy())
758 // If operands of an fp-type was compared, this costs +1.
759 Cost++;
760 return Cost;
761 }
762 }
763 else if (ST->hasVector()) {
764 // Vector to scalar cast.
765 auto *SrcVecTy = cast<FixedVectorType>(Src);
766 auto *DstVecTy = dyn_cast<FixedVectorType>(Dst);
767 if (!DstVecTy) {
768 // TODO: tune vector-to-scalar cast.
769 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
770 }
771 unsigned VF = SrcVecTy->getNumElements();
772 unsigned NumDstVectors = getNumVectorRegs(Dst);
773 unsigned NumSrcVectors = getNumVectorRegs(Src);
774
775 if (Opcode == Instruction::Trunc) {
776 if (Src->getScalarSizeInBits() == Dst->getScalarSizeInBits())
777 return 0; // Check for NOOP conversions.
778 return getVectorTruncCost(Src, Dst);
779 }
780
781 if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt) {
782 if (SrcScalarBits >= 8) {
783 // ZExt/SExt will be handled with one unpack per doubling of width.
784 unsigned NumUnpacks = getElSizeLog2Diff(Src, Dst);
785
786 // For types that spans multiple vector registers, some additional
787 // instructions are used to setup the unpacking.
788 unsigned NumSrcVectorOps =
789 (NumUnpacks > 1 ? (NumDstVectors - NumSrcVectors)
790 : (NumDstVectors / 2));
791
792 return (NumUnpacks * NumDstVectors) + NumSrcVectorOps;
793 }
794 else if (SrcScalarBits == 1)
795 return getBoolVecToIntConversionCost(Opcode, Dst, I);
796 }
797
798 if (Opcode == Instruction::SIToFP || Opcode == Instruction::UIToFP ||
799 Opcode == Instruction::FPToSI || Opcode == Instruction::FPToUI) {
800 // TODO: Fix base implementation which could simplify things a bit here
801 // (seems to miss on differentiating on scalar/vector types).
802
803 // Only 64 bit vector conversions are natively supported before z15.
804 if (DstScalarBits == 64 || ST->hasVectorEnhancements2()) {
805 if (SrcScalarBits == DstScalarBits)
806 return NumDstVectors;
807
808 if (SrcScalarBits == 1)
809 return getBoolVecToIntConversionCost(Opcode, Dst, I) + NumDstVectors;
810 }
811
812 // Return the cost of multiple scalar invocation plus the cost of
813 // inserting and extracting the values. Base implementation does not
814 // realize float->int gets scalarized.
815 InstructionCost ScalarCost = getCastInstrCost(
816 Opcode, Dst->getScalarType(), Src->getScalarType(), CCH, CostKind);
817 InstructionCost TotCost = VF * ScalarCost;
818 bool NeedsInserts = true, NeedsExtracts = true;
819 // FP128 registers do not get inserted or extracted.
820 if (DstScalarBits == 128 &&
821 (Opcode == Instruction::SIToFP || Opcode == Instruction::UIToFP))
822 NeedsInserts = false;
823 if (SrcScalarBits == 128 &&
824 (Opcode == Instruction::FPToSI || Opcode == Instruction::FPToUI))
825 NeedsExtracts = false;
826
827 TotCost += getScalarizationOverhead(SrcVecTy, false, NeedsExtracts);
828 TotCost += getScalarizationOverhead(DstVecTy, NeedsInserts, false);
829
830 // FIXME: VF 2 for float<->i32 is currently just as expensive as for VF 4.
831 if (VF == 2 && SrcScalarBits == 32 && DstScalarBits == 32)
832 TotCost *= 2;
833
834 return TotCost;
835 }
836
837 if (Opcode == Instruction::FPTrunc) {
838 if (SrcScalarBits == 128) // fp128 -> double/float + inserts of elements.
839 return VF /*ldxbr/lexbr*/ +
840 getScalarizationOverhead(DstVecTy, true, false);
841 else // double -> float
842 return VF / 2 /*vledb*/ + std::max(1U, VF / 4 /*vperm*/);
843 }
844
845 if (Opcode == Instruction::FPExt) {
846 if (SrcScalarBits == 32 && DstScalarBits == 64) {
847 // float -> double is very rare and currently unoptimized. Instead of
848 // using vldeb, which can do two at a time, all conversions are
849 // scalarized.
850 return VF * 2;
851 }
852 // -> fp128. VF * lxdb/lxeb + extraction of elements.
853 return VF + getScalarizationOverhead(SrcVecTy, false, true);
854 }
855 }
856
857 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
858 }
859
860 // Scalar i8 / i16 operations will typically be made after first extending
861 // the operands to i32.
getOperandsExtensionCost(const Instruction * I)862 static unsigned getOperandsExtensionCost(const Instruction *I) {
863 unsigned ExtCost = 0;
864 for (Value *Op : I->operands())
865 // A load of i8 or i16 sign/zero extends to i32.
866 if (!isa<LoadInst>(Op) && !isa<ConstantInt>(Op))
867 ExtCost++;
868
869 return ExtCost;
870 }
871
getCmpSelInstrCost(unsigned Opcode,Type * ValTy,Type * CondTy,CmpInst::Predicate VecPred,TTI::TargetCostKind CostKind,const Instruction * I)872 InstructionCost SystemZTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
873 Type *CondTy,
874 CmpInst::Predicate VecPred,
875 TTI::TargetCostKind CostKind,
876 const Instruction *I) {
877 if (CostKind != TTI::TCK_RecipThroughput)
878 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind);
879
880 if (!ValTy->isVectorTy()) {
881 switch (Opcode) {
882 case Instruction::ICmp: {
883 // A loaded value compared with 0 with multiple users becomes Load and
884 // Test. The load is then not foldable, so return 0 cost for the ICmp.
885 unsigned ScalarBits = ValTy->getScalarSizeInBits();
886 if (I != nullptr && ScalarBits >= 32)
887 if (LoadInst *Ld = dyn_cast<LoadInst>(I->getOperand(0)))
888 if (const ConstantInt *C = dyn_cast<ConstantInt>(I->getOperand(1)))
889 if (!Ld->hasOneUse() && Ld->getParent() == I->getParent() &&
890 C->isZero())
891 return 0;
892
893 unsigned Cost = 1;
894 if (ValTy->isIntegerTy() && ValTy->getScalarSizeInBits() <= 16)
895 Cost += (I != nullptr ? getOperandsExtensionCost(I) : 2);
896 return Cost;
897 }
898 case Instruction::Select:
899 if (ValTy->isFloatingPointTy())
900 return 4; // No load on condition for FP - costs a conditional jump.
901 return 1; // Load On Condition / Select Register.
902 }
903 }
904 else if (ST->hasVector()) {
905 unsigned VF = cast<FixedVectorType>(ValTy)->getNumElements();
906
907 // Called with a compare instruction.
908 if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) {
909 unsigned PredicateExtraCost = 0;
910 if (I != nullptr) {
911 // Some predicates cost one or two extra instructions.
912 switch (cast<CmpInst>(I)->getPredicate()) {
913 case CmpInst::Predicate::ICMP_NE:
914 case CmpInst::Predicate::ICMP_UGE:
915 case CmpInst::Predicate::ICMP_ULE:
916 case CmpInst::Predicate::ICMP_SGE:
917 case CmpInst::Predicate::ICMP_SLE:
918 PredicateExtraCost = 1;
919 break;
920 case CmpInst::Predicate::FCMP_ONE:
921 case CmpInst::Predicate::FCMP_ORD:
922 case CmpInst::Predicate::FCMP_UEQ:
923 case CmpInst::Predicate::FCMP_UNO:
924 PredicateExtraCost = 2;
925 break;
926 default:
927 break;
928 }
929 }
930
931 // Float is handled with 2*vmr[lh]f + 2*vldeb + vfchdb for each pair of
932 // floats. FIXME: <2 x float> generates same code as <4 x float>.
933 unsigned CmpCostPerVector = (ValTy->getScalarType()->isFloatTy() ? 10 : 1);
934 unsigned NumVecs_cmp = getNumVectorRegs(ValTy);
935
936 unsigned Cost = (NumVecs_cmp * (CmpCostPerVector + PredicateExtraCost));
937 return Cost;
938 }
939 else { // Called with a select instruction.
940 assert (Opcode == Instruction::Select);
941
942 // We can figure out the extra cost of packing / unpacking if the
943 // instruction was passed and the compare instruction is found.
944 unsigned PackCost = 0;
945 Type *CmpOpTy = ((I != nullptr) ? getCmpOpsType(I, VF) : nullptr);
946 if (CmpOpTy != nullptr)
947 PackCost =
948 getVectorBitmaskConversionCost(CmpOpTy, ValTy);
949
950 return getNumVectorRegs(ValTy) /*vsel*/ + PackCost;
951 }
952 }
953
954 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind);
955 }
956
getVectorInstrCost(unsigned Opcode,Type * Val,unsigned Index)957 InstructionCost SystemZTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
958 unsigned Index) {
959 // vlvgp will insert two grs into a vector register, so only count half the
960 // number of instructions.
961 if (Opcode == Instruction::InsertElement && Val->isIntOrIntVectorTy(64))
962 return ((Index % 2 == 0) ? 1 : 0);
963
964 if (Opcode == Instruction::ExtractElement) {
965 int Cost = ((getScalarSizeInBits(Val) == 1) ? 2 /*+test-under-mask*/ : 1);
966
967 // Give a slight penalty for moving out of vector pipeline to FXU unit.
968 if (Index == 0 && Val->isIntOrIntVectorTy())
969 Cost += 1;
970
971 return Cost;
972 }
973
974 return BaseT::getVectorInstrCost(Opcode, Val, Index);
975 }
976
977 // Check if a load may be folded as a memory operand in its user.
978 bool SystemZTTIImpl::
isFoldableLoad(const LoadInst * Ld,const Instruction * & FoldedValue)979 isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue) {
980 if (!Ld->hasOneUse())
981 return false;
982 FoldedValue = Ld;
983 const Instruction *UserI = cast<Instruction>(*Ld->user_begin());
984 unsigned LoadedBits = getScalarSizeInBits(Ld->getType());
985 unsigned TruncBits = 0;
986 unsigned SExtBits = 0;
987 unsigned ZExtBits = 0;
988 if (UserI->hasOneUse()) {
989 unsigned UserBits = UserI->getType()->getScalarSizeInBits();
990 if (isa<TruncInst>(UserI))
991 TruncBits = UserBits;
992 else if (isa<SExtInst>(UserI))
993 SExtBits = UserBits;
994 else if (isa<ZExtInst>(UserI))
995 ZExtBits = UserBits;
996 }
997 if (TruncBits || SExtBits || ZExtBits) {
998 FoldedValue = UserI;
999 UserI = cast<Instruction>(*UserI->user_begin());
1000 // Load (single use) -> trunc/extend (single use) -> UserI
1001 }
1002 if ((UserI->getOpcode() == Instruction::Sub ||
1003 UserI->getOpcode() == Instruction::SDiv ||
1004 UserI->getOpcode() == Instruction::UDiv) &&
1005 UserI->getOperand(1) != FoldedValue)
1006 return false; // Not commutative, only RHS foldable.
1007 // LoadOrTruncBits holds the number of effectively loaded bits, but 0 if an
1008 // extension was made of the load.
1009 unsigned LoadOrTruncBits =
1010 ((SExtBits || ZExtBits) ? 0 : (TruncBits ? TruncBits : LoadedBits));
1011 switch (UserI->getOpcode()) {
1012 case Instruction::Add: // SE: 16->32, 16/32->64, z14:16->64. ZE: 32->64
1013 case Instruction::Sub:
1014 case Instruction::ICmp:
1015 if (LoadedBits == 32 && ZExtBits == 64)
1016 return true;
1017 LLVM_FALLTHROUGH;
1018 case Instruction::Mul: // SE: 16->32, 32->64, z14:16->64
1019 if (UserI->getOpcode() != Instruction::ICmp) {
1020 if (LoadedBits == 16 &&
1021 (SExtBits == 32 ||
1022 (SExtBits == 64 && ST->hasMiscellaneousExtensions2())))
1023 return true;
1024 if (LoadOrTruncBits == 16)
1025 return true;
1026 }
1027 LLVM_FALLTHROUGH;
1028 case Instruction::SDiv:// SE: 32->64
1029 if (LoadedBits == 32 && SExtBits == 64)
1030 return true;
1031 LLVM_FALLTHROUGH;
1032 case Instruction::UDiv:
1033 case Instruction::And:
1034 case Instruction::Or:
1035 case Instruction::Xor:
1036 // This also makes sense for float operations, but disabled for now due
1037 // to regressions.
1038 // case Instruction::FCmp:
1039 // case Instruction::FAdd:
1040 // case Instruction::FSub:
1041 // case Instruction::FMul:
1042 // case Instruction::FDiv:
1043
1044 // All possible extensions of memory checked above.
1045
1046 // Comparison between memory and immediate.
1047 if (UserI->getOpcode() == Instruction::ICmp)
1048 if (ConstantInt *CI = dyn_cast<ConstantInt>(UserI->getOperand(1)))
1049 if (CI->getValue().isIntN(16))
1050 return true;
1051 return (LoadOrTruncBits == 32 || LoadOrTruncBits == 64);
1052 break;
1053 }
1054 return false;
1055 }
1056
isBswapIntrinsicCall(const Value * V)1057 static bool isBswapIntrinsicCall(const Value *V) {
1058 if (const Instruction *I = dyn_cast<Instruction>(V))
1059 if (auto *CI = dyn_cast<CallInst>(I))
1060 if (auto *F = CI->getCalledFunction())
1061 if (F->getIntrinsicID() == Intrinsic::bswap)
1062 return true;
1063 return false;
1064 }
1065
getMemoryOpCost(unsigned Opcode,Type * Src,MaybeAlign Alignment,unsigned AddressSpace,TTI::TargetCostKind CostKind,const Instruction * I)1066 InstructionCost SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
1067 MaybeAlign Alignment,
1068 unsigned AddressSpace,
1069 TTI::TargetCostKind CostKind,
1070 const Instruction *I) {
1071 assert(!Src->isVoidTy() && "Invalid type");
1072
1073 // TODO: Handle other cost kinds.
1074 if (CostKind != TTI::TCK_RecipThroughput)
1075 return 1;
1076
1077 if (!Src->isVectorTy() && Opcode == Instruction::Load && I != nullptr) {
1078 // Store the load or its truncated or extended value in FoldedValue.
1079 const Instruction *FoldedValue = nullptr;
1080 if (isFoldableLoad(cast<LoadInst>(I), FoldedValue)) {
1081 const Instruction *UserI = cast<Instruction>(*FoldedValue->user_begin());
1082 assert (UserI->getNumOperands() == 2 && "Expected a binop.");
1083
1084 // UserI can't fold two loads, so in that case return 0 cost only
1085 // half of the time.
1086 for (unsigned i = 0; i < 2; ++i) {
1087 if (UserI->getOperand(i) == FoldedValue)
1088 continue;
1089
1090 if (Instruction *OtherOp = dyn_cast<Instruction>(UserI->getOperand(i))){
1091 LoadInst *OtherLoad = dyn_cast<LoadInst>(OtherOp);
1092 if (!OtherLoad &&
1093 (isa<TruncInst>(OtherOp) || isa<SExtInst>(OtherOp) ||
1094 isa<ZExtInst>(OtherOp)))
1095 OtherLoad = dyn_cast<LoadInst>(OtherOp->getOperand(0));
1096 if (OtherLoad && isFoldableLoad(OtherLoad, FoldedValue/*dummy*/))
1097 return i == 0; // Both operands foldable.
1098 }
1099 }
1100
1101 return 0; // Only I is foldable in user.
1102 }
1103 }
1104
1105 unsigned NumOps =
1106 (Src->isVectorTy() ? getNumVectorRegs(Src) : getNumberOfParts(Src));
1107
1108 // Store/Load reversed saves one instruction.
1109 if (((!Src->isVectorTy() && NumOps == 1) || ST->hasVectorEnhancements2()) &&
1110 I != nullptr) {
1111 if (Opcode == Instruction::Load && I->hasOneUse()) {
1112 const Instruction *LdUser = cast<Instruction>(*I->user_begin());
1113 // In case of load -> bswap -> store, return normal cost for the load.
1114 if (isBswapIntrinsicCall(LdUser) &&
1115 (!LdUser->hasOneUse() || !isa<StoreInst>(*LdUser->user_begin())))
1116 return 0;
1117 }
1118 else if (const StoreInst *SI = dyn_cast<StoreInst>(I)) {
1119 const Value *StoredVal = SI->getValueOperand();
1120 if (StoredVal->hasOneUse() && isBswapIntrinsicCall(StoredVal))
1121 return 0;
1122 }
1123 }
1124
1125 if (Src->getScalarSizeInBits() == 128)
1126 // 128 bit scalars are held in a pair of two 64 bit registers.
1127 NumOps *= 2;
1128
1129 return NumOps;
1130 }
1131
1132 // The generic implementation of getInterleavedMemoryOpCost() is based on
1133 // adding costs of the memory operations plus all the extracts and inserts
1134 // needed for using / defining the vector operands. The SystemZ version does
1135 // roughly the same but bases the computations on vector permutations
1136 // instead.
getInterleavedMemoryOpCost(unsigned Opcode,Type * VecTy,unsigned Factor,ArrayRef<unsigned> Indices,Align Alignment,unsigned AddressSpace,TTI::TargetCostKind CostKind,bool UseMaskForCond,bool UseMaskForGaps)1137 InstructionCost SystemZTTIImpl::getInterleavedMemoryOpCost(
1138 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1139 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1140 bool UseMaskForCond, bool UseMaskForGaps) {
1141 if (UseMaskForCond || UseMaskForGaps)
1142 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
1143 Alignment, AddressSpace, CostKind,
1144 UseMaskForCond, UseMaskForGaps);
1145 assert(isa<VectorType>(VecTy) &&
1146 "Expect a vector type for interleaved memory op");
1147
1148 unsigned NumElts = cast<FixedVectorType>(VecTy)->getNumElements();
1149 assert(Factor > 1 && NumElts % Factor == 0 && "Invalid interleave factor");
1150 unsigned VF = NumElts / Factor;
1151 unsigned NumEltsPerVecReg = (128U / getScalarSizeInBits(VecTy));
1152 unsigned NumVectorMemOps = getNumVectorRegs(VecTy);
1153 unsigned NumPermutes = 0;
1154
1155 if (Opcode == Instruction::Load) {
1156 // Loading interleave groups may have gaps, which may mean fewer
1157 // loads. Find out how many vectors will be loaded in total, and in how
1158 // many of them each value will be in.
1159 BitVector UsedInsts(NumVectorMemOps, false);
1160 std::vector<BitVector> ValueVecs(Factor, BitVector(NumVectorMemOps, false));
1161 for (unsigned Index : Indices)
1162 for (unsigned Elt = 0; Elt < VF; ++Elt) {
1163 unsigned Vec = (Index + Elt * Factor) / NumEltsPerVecReg;
1164 UsedInsts.set(Vec);
1165 ValueVecs[Index].set(Vec);
1166 }
1167 NumVectorMemOps = UsedInsts.count();
1168
1169 for (unsigned Index : Indices) {
1170 // Estimate that each loaded source vector containing this Index
1171 // requires one operation, except that vperm can handle two input
1172 // registers first time for each dst vector.
1173 unsigned NumSrcVecs = ValueVecs[Index].count();
1174 unsigned NumDstVecs = divideCeil(VF * getScalarSizeInBits(VecTy), 128U);
1175 assert (NumSrcVecs >= NumDstVecs && "Expected at least as many sources");
1176 NumPermutes += std::max(1U, NumSrcVecs - NumDstVecs);
1177 }
1178 } else {
1179 // Estimate the permutes for each stored vector as the smaller of the
1180 // number of elements and the number of source vectors. Subtract one per
1181 // dst vector for vperm (S.A.).
1182 unsigned NumSrcVecs = std::min(NumEltsPerVecReg, Factor);
1183 unsigned NumDstVecs = NumVectorMemOps;
1184 assert (NumSrcVecs > 1 && "Expected at least two source vectors.");
1185 NumPermutes += (NumDstVecs * NumSrcVecs) - NumDstVecs;
1186 }
1187
1188 // Cost of load/store operations and the permutations needed.
1189 return NumVectorMemOps + NumPermutes;
1190 }
1191
getVectorIntrinsicInstrCost(Intrinsic::ID ID,Type * RetTy)1192 static int getVectorIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy) {
1193 if (RetTy->isVectorTy() && ID == Intrinsic::bswap)
1194 return getNumVectorRegs(RetTy); // VPERM
1195 return -1;
1196 }
1197
1198 InstructionCost
getIntrinsicInstrCost(const IntrinsicCostAttributes & ICA,TTI::TargetCostKind CostKind)1199 SystemZTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
1200 TTI::TargetCostKind CostKind) {
1201 InstructionCost Cost =
1202 getVectorIntrinsicInstrCost(ICA.getID(), ICA.getReturnType());
1203 if (Cost != -1)
1204 return Cost;
1205 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
1206 }
1207