1 //===----------- VectorUtils.cpp - Vectorizer utility functions -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines vectorizer utilities. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/Analysis/VectorUtils.h" 14 #include "llvm/ADT/EquivalenceClasses.h" 15 #include "llvm/ADT/SmallVector.h" 16 #include "llvm/Analysis/DemandedBits.h" 17 #include "llvm/Analysis/LoopInfo.h" 18 #include "llvm/Analysis/LoopIterator.h" 19 #include "llvm/Analysis/ScalarEvolution.h" 20 #include "llvm/Analysis/ScalarEvolutionExpressions.h" 21 #include "llvm/Analysis/TargetTransformInfo.h" 22 #include "llvm/Analysis/ValueTracking.h" 23 #include "llvm/IR/Constants.h" 24 #include "llvm/IR/DerivedTypes.h" 25 #include "llvm/IR/IRBuilder.h" 26 #include "llvm/IR/MemoryModelRelaxationAnnotations.h" 27 #include "llvm/IR/PatternMatch.h" 28 #include "llvm/IR/Value.h" 29 #include "llvm/Support/CommandLine.h" 30 31 #define DEBUG_TYPE "vectorutils" 32 33 using namespace llvm; 34 using namespace llvm::PatternMatch; 35 36 /// Maximum factor for an interleaved memory access. 37 static cl::opt<unsigned> MaxInterleaveGroupFactor( 38 "max-interleave-group-factor", cl::Hidden, 39 cl::desc("Maximum factor for an interleaved access group (default = 8)"), 40 cl::init(8)); 41 42 /// Return true if all of the intrinsic's arguments and return type are scalars 43 /// for the scalar form of the intrinsic, and vectors for the vector form of the 44 /// intrinsic (except operands that are marked as always being scalar by 45 /// isVectorIntrinsicWithScalarOpAtArg). 46 bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) { 47 switch (ID) { 48 case Intrinsic::abs: // Begin integer bit-manipulation. 49 case Intrinsic::bswap: 50 case Intrinsic::bitreverse: 51 case Intrinsic::ctpop: 52 case Intrinsic::ctlz: 53 case Intrinsic::cttz: 54 case Intrinsic::fshl: 55 case Intrinsic::fshr: 56 case Intrinsic::smax: 57 case Intrinsic::smin: 58 case Intrinsic::umax: 59 case Intrinsic::umin: 60 case Intrinsic::sadd_sat: 61 case Intrinsic::ssub_sat: 62 case Intrinsic::uadd_sat: 63 case Intrinsic::usub_sat: 64 case Intrinsic::smul_fix: 65 case Intrinsic::smul_fix_sat: 66 case Intrinsic::umul_fix: 67 case Intrinsic::umul_fix_sat: 68 case Intrinsic::sqrt: // Begin floating-point. 69 case Intrinsic::asin: 70 case Intrinsic::acos: 71 case Intrinsic::atan: 72 case Intrinsic::atan2: 73 case Intrinsic::sin: 74 case Intrinsic::cos: 75 case Intrinsic::tan: 76 case Intrinsic::sinh: 77 case Intrinsic::cosh: 78 case Intrinsic::tanh: 79 case Intrinsic::exp: 80 case Intrinsic::exp10: 81 case Intrinsic::exp2: 82 case Intrinsic::log: 83 case Intrinsic::log10: 84 case Intrinsic::log2: 85 case Intrinsic::fabs: 86 case Intrinsic::minnum: 87 case Intrinsic::maxnum: 88 case Intrinsic::minimum: 89 case Intrinsic::maximum: 90 case Intrinsic::copysign: 91 case Intrinsic::floor: 92 case Intrinsic::ceil: 93 case Intrinsic::trunc: 94 case Intrinsic::rint: 95 case Intrinsic::nearbyint: 96 case Intrinsic::round: 97 case Intrinsic::roundeven: 98 case Intrinsic::pow: 99 case Intrinsic::fma: 100 case Intrinsic::fmuladd: 101 case Intrinsic::is_fpclass: 102 case Intrinsic::powi: 103 case Intrinsic::canonicalize: 104 case Intrinsic::fptosi_sat: 105 case Intrinsic::fptoui_sat: 106 case Intrinsic::lrint: 107 case Intrinsic::llrint: 108 case Intrinsic::ucmp: 109 case Intrinsic::scmp: 110 return true; 111 default: 112 return false; 113 } 114 } 115 116 /// Identifies if the vector form of the intrinsic has a scalar operand. 117 bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, 118 unsigned ScalarOpdIdx) { 119 switch (ID) { 120 case Intrinsic::abs: 121 case Intrinsic::ctlz: 122 case Intrinsic::cttz: 123 case Intrinsic::is_fpclass: 124 case Intrinsic::powi: 125 return (ScalarOpdIdx == 1); 126 case Intrinsic::smul_fix: 127 case Intrinsic::smul_fix_sat: 128 case Intrinsic::umul_fix: 129 case Intrinsic::umul_fix_sat: 130 return (ScalarOpdIdx == 2); 131 default: 132 return false; 133 } 134 } 135 136 bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, 137 int OpdIdx) { 138 assert(ID != Intrinsic::not_intrinsic && "Not an intrinsic!"); 139 140 switch (ID) { 141 case Intrinsic::fptosi_sat: 142 case Intrinsic::fptoui_sat: 143 case Intrinsic::lrint: 144 case Intrinsic::llrint: 145 case Intrinsic::ucmp: 146 case Intrinsic::scmp: 147 return OpdIdx == -1 || OpdIdx == 0; 148 case Intrinsic::is_fpclass: 149 return OpdIdx == 0; 150 case Intrinsic::powi: 151 return OpdIdx == -1 || OpdIdx == 1; 152 default: 153 return OpdIdx == -1; 154 } 155 } 156 157 bool llvm::isVectorIntrinsicWithStructReturnOverloadAtField(Intrinsic::ID ID, 158 int RetIdx) { 159 switch (ID) { 160 case Intrinsic::frexp: 161 return RetIdx == 0 || RetIdx == 1; 162 default: 163 return RetIdx == 0; 164 } 165 } 166 167 /// Returns intrinsic ID for call. 168 /// For the input call instruction it finds mapping intrinsic and returns 169 /// its ID, in case it does not found it return not_intrinsic. 170 Intrinsic::ID llvm::getVectorIntrinsicIDForCall(const CallInst *CI, 171 const TargetLibraryInfo *TLI) { 172 Intrinsic::ID ID = getIntrinsicForCallSite(*CI, TLI); 173 if (ID == Intrinsic::not_intrinsic) 174 return Intrinsic::not_intrinsic; 175 176 if (isTriviallyVectorizable(ID) || ID == Intrinsic::lifetime_start || 177 ID == Intrinsic::lifetime_end || ID == Intrinsic::assume || 178 ID == Intrinsic::experimental_noalias_scope_decl || 179 ID == Intrinsic::sideeffect || ID == Intrinsic::pseudoprobe) 180 return ID; 181 return Intrinsic::not_intrinsic; 182 } 183 184 /// Given a vector and an element number, see if the scalar value is 185 /// already around as a register, for example if it were inserted then extracted 186 /// from the vector. 187 Value *llvm::findScalarElement(Value *V, unsigned EltNo) { 188 assert(V->getType()->isVectorTy() && "Not looking at a vector?"); 189 VectorType *VTy = cast<VectorType>(V->getType()); 190 // For fixed-length vector, return poison for out of range access. 191 if (auto *FVTy = dyn_cast<FixedVectorType>(VTy)) { 192 unsigned Width = FVTy->getNumElements(); 193 if (EltNo >= Width) 194 return PoisonValue::get(FVTy->getElementType()); 195 } 196 197 if (Constant *C = dyn_cast<Constant>(V)) 198 return C->getAggregateElement(EltNo); 199 200 if (InsertElementInst *III = dyn_cast<InsertElementInst>(V)) { 201 // If this is an insert to a variable element, we don't know what it is. 202 if (!isa<ConstantInt>(III->getOperand(2))) 203 return nullptr; 204 unsigned IIElt = cast<ConstantInt>(III->getOperand(2))->getZExtValue(); 205 206 // If this is an insert to the element we are looking for, return the 207 // inserted value. 208 if (EltNo == IIElt) 209 return III->getOperand(1); 210 211 // Guard against infinite loop on malformed, unreachable IR. 212 if (III == III->getOperand(0)) 213 return nullptr; 214 215 // Otherwise, the insertelement doesn't modify the value, recurse on its 216 // vector input. 217 return findScalarElement(III->getOperand(0), EltNo); 218 } 219 220 ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(V); 221 // Restrict the following transformation to fixed-length vector. 222 if (SVI && isa<FixedVectorType>(SVI->getType())) { 223 unsigned LHSWidth = 224 cast<FixedVectorType>(SVI->getOperand(0)->getType())->getNumElements(); 225 int InEl = SVI->getMaskValue(EltNo); 226 if (InEl < 0) 227 return PoisonValue::get(VTy->getElementType()); 228 if (InEl < (int)LHSWidth) 229 return findScalarElement(SVI->getOperand(0), InEl); 230 return findScalarElement(SVI->getOperand(1), InEl - LHSWidth); 231 } 232 233 // Extract a value from a vector add operation with a constant zero. 234 // TODO: Use getBinOpIdentity() to generalize this. 235 Value *Val; Constant *C; 236 if (match(V, m_Add(m_Value(Val), m_Constant(C)))) 237 if (Constant *Elt = C->getAggregateElement(EltNo)) 238 if (Elt->isNullValue()) 239 return findScalarElement(Val, EltNo); 240 241 // If the vector is a splat then we can trivially find the scalar element. 242 if (isa<ScalableVectorType>(VTy)) 243 if (Value *Splat = getSplatValue(V)) 244 if (EltNo < VTy->getElementCount().getKnownMinValue()) 245 return Splat; 246 247 // Otherwise, we don't know. 248 return nullptr; 249 } 250 251 int llvm::getSplatIndex(ArrayRef<int> Mask) { 252 int SplatIndex = -1; 253 for (int M : Mask) { 254 // Ignore invalid (undefined) mask elements. 255 if (M < 0) 256 continue; 257 258 // There can be only 1 non-negative mask element value if this is a splat. 259 if (SplatIndex != -1 && SplatIndex != M) 260 return -1; 261 262 // Initialize the splat index to the 1st non-negative mask element. 263 SplatIndex = M; 264 } 265 assert((SplatIndex == -1 || SplatIndex >= 0) && "Negative index?"); 266 return SplatIndex; 267 } 268 269 /// Get splat value if the input is a splat vector or return nullptr. 270 /// This function is not fully general. It checks only 2 cases: 271 /// the input value is (1) a splat constant vector or (2) a sequence 272 /// of instructions that broadcasts a scalar at element 0. 273 Value *llvm::getSplatValue(const Value *V) { 274 if (isa<VectorType>(V->getType())) 275 if (auto *C = dyn_cast<Constant>(V)) 276 return C->getSplatValue(); 277 278 // shuf (inselt ?, Splat, 0), ?, <0, undef, 0, ...> 279 Value *Splat; 280 if (match(V, 281 m_Shuffle(m_InsertElt(m_Value(), m_Value(Splat), m_ZeroInt()), 282 m_Value(), m_ZeroMask()))) 283 return Splat; 284 285 return nullptr; 286 } 287 288 bool llvm::isSplatValue(const Value *V, int Index, unsigned Depth) { 289 assert(Depth <= MaxAnalysisRecursionDepth && "Limit Search Depth"); 290 291 if (isa<VectorType>(V->getType())) { 292 if (isa<UndefValue>(V)) 293 return true; 294 // FIXME: We can allow undefs, but if Index was specified, we may want to 295 // check that the constant is defined at that index. 296 if (auto *C = dyn_cast<Constant>(V)) 297 return C->getSplatValue() != nullptr; 298 } 299 300 if (auto *Shuf = dyn_cast<ShuffleVectorInst>(V)) { 301 // FIXME: We can safely allow undefs here. If Index was specified, we will 302 // check that the mask elt is defined at the required index. 303 if (!all_equal(Shuf->getShuffleMask())) 304 return false; 305 306 // Match any index. 307 if (Index == -1) 308 return true; 309 310 // Match a specific element. The mask should be defined at and match the 311 // specified index. 312 return Shuf->getMaskValue(Index) == Index; 313 } 314 315 // The remaining tests are all recursive, so bail out if we hit the limit. 316 if (Depth++ == MaxAnalysisRecursionDepth) 317 return false; 318 319 // If both operands of a binop are splats, the result is a splat. 320 Value *X, *Y, *Z; 321 if (match(V, m_BinOp(m_Value(X), m_Value(Y)))) 322 return isSplatValue(X, Index, Depth) && isSplatValue(Y, Index, Depth); 323 324 // If all operands of a select are splats, the result is a splat. 325 if (match(V, m_Select(m_Value(X), m_Value(Y), m_Value(Z)))) 326 return isSplatValue(X, Index, Depth) && isSplatValue(Y, Index, Depth) && 327 isSplatValue(Z, Index, Depth); 328 329 // TODO: Add support for unary ops (fneg), casts, intrinsics (overflow ops). 330 331 return false; 332 } 333 334 bool llvm::getShuffleDemandedElts(int SrcWidth, ArrayRef<int> Mask, 335 const APInt &DemandedElts, APInt &DemandedLHS, 336 APInt &DemandedRHS, bool AllowUndefElts) { 337 DemandedLHS = DemandedRHS = APInt::getZero(SrcWidth); 338 339 // Early out if we don't demand any elements. 340 if (DemandedElts.isZero()) 341 return true; 342 343 // Simple case of a shuffle with zeroinitializer. 344 if (all_of(Mask, [](int Elt) { return Elt == 0; })) { 345 DemandedLHS.setBit(0); 346 return true; 347 } 348 349 for (unsigned I = 0, E = Mask.size(); I != E; ++I) { 350 int M = Mask[I]; 351 assert((-1 <= M) && (M < (SrcWidth * 2)) && 352 "Invalid shuffle mask constant"); 353 354 if (!DemandedElts[I] || (AllowUndefElts && (M < 0))) 355 continue; 356 357 // For undef elements, we don't know anything about the common state of 358 // the shuffle result. 359 if (M < 0) 360 return false; 361 362 if (M < SrcWidth) 363 DemandedLHS.setBit(M); 364 else 365 DemandedRHS.setBit(M - SrcWidth); 366 } 367 368 return true; 369 } 370 371 void llvm::narrowShuffleMaskElts(int Scale, ArrayRef<int> Mask, 372 SmallVectorImpl<int> &ScaledMask) { 373 assert(Scale > 0 && "Unexpected scaling factor"); 374 375 // Fast-path: if no scaling, then it is just a copy. 376 if (Scale == 1) { 377 ScaledMask.assign(Mask.begin(), Mask.end()); 378 return; 379 } 380 381 ScaledMask.clear(); 382 for (int MaskElt : Mask) { 383 if (MaskElt >= 0) { 384 assert(((uint64_t)Scale * MaskElt + (Scale - 1)) <= INT32_MAX && 385 "Overflowed 32-bits"); 386 } 387 for (int SliceElt = 0; SliceElt != Scale; ++SliceElt) 388 ScaledMask.push_back(MaskElt < 0 ? MaskElt : Scale * MaskElt + SliceElt); 389 } 390 } 391 392 bool llvm::widenShuffleMaskElts(int Scale, ArrayRef<int> Mask, 393 SmallVectorImpl<int> &ScaledMask) { 394 assert(Scale > 0 && "Unexpected scaling factor"); 395 396 // Fast-path: if no scaling, then it is just a copy. 397 if (Scale == 1) { 398 ScaledMask.assign(Mask.begin(), Mask.end()); 399 return true; 400 } 401 402 // We must map the original elements down evenly to a type with less elements. 403 int NumElts = Mask.size(); 404 if (NumElts % Scale != 0) 405 return false; 406 407 ScaledMask.clear(); 408 ScaledMask.reserve(NumElts / Scale); 409 410 // Step through the input mask by splitting into Scale-sized slices. 411 do { 412 ArrayRef<int> MaskSlice = Mask.take_front(Scale); 413 assert((int)MaskSlice.size() == Scale && "Expected Scale-sized slice."); 414 415 // The first element of the slice determines how we evaluate this slice. 416 int SliceFront = MaskSlice.front(); 417 if (SliceFront < 0) { 418 // Negative values (undef or other "sentinel" values) must be equal across 419 // the entire slice. 420 if (!all_equal(MaskSlice)) 421 return false; 422 ScaledMask.push_back(SliceFront); 423 } else { 424 // A positive mask element must be cleanly divisible. 425 if (SliceFront % Scale != 0) 426 return false; 427 // Elements of the slice must be consecutive. 428 for (int i = 1; i < Scale; ++i) 429 if (MaskSlice[i] != SliceFront + i) 430 return false; 431 ScaledMask.push_back(SliceFront / Scale); 432 } 433 Mask = Mask.drop_front(Scale); 434 } while (!Mask.empty()); 435 436 assert((int)ScaledMask.size() * Scale == NumElts && "Unexpected scaled mask"); 437 438 // All elements of the original mask can be scaled down to map to the elements 439 // of a mask with wider elements. 440 return true; 441 } 442 443 bool llvm::scaleShuffleMaskElts(unsigned NumDstElts, ArrayRef<int> Mask, 444 SmallVectorImpl<int> &ScaledMask) { 445 unsigned NumSrcElts = Mask.size(); 446 assert(NumSrcElts > 0 && NumDstElts > 0 && "Unexpected scaling factor"); 447 448 // Fast-path: if no scaling, then it is just a copy. 449 if (NumSrcElts == NumDstElts) { 450 ScaledMask.assign(Mask.begin(), Mask.end()); 451 return true; 452 } 453 454 // Ensure we can find a whole scale factor. 455 assert(((NumSrcElts % NumDstElts) == 0 || (NumDstElts % NumSrcElts) == 0) && 456 "Unexpected scaling factor"); 457 458 if (NumSrcElts > NumDstElts) { 459 int Scale = NumSrcElts / NumDstElts; 460 return widenShuffleMaskElts(Scale, Mask, ScaledMask); 461 } 462 463 int Scale = NumDstElts / NumSrcElts; 464 narrowShuffleMaskElts(Scale, Mask, ScaledMask); 465 return true; 466 } 467 468 void llvm::getShuffleMaskWithWidestElts(ArrayRef<int> Mask, 469 SmallVectorImpl<int> &ScaledMask) { 470 std::array<SmallVector<int, 16>, 2> TmpMasks; 471 SmallVectorImpl<int> *Output = &TmpMasks[0], *Tmp = &TmpMasks[1]; 472 ArrayRef<int> InputMask = Mask; 473 for (unsigned Scale = 2; Scale <= InputMask.size(); ++Scale) { 474 while (widenShuffleMaskElts(Scale, InputMask, *Output)) { 475 InputMask = *Output; 476 std::swap(Output, Tmp); 477 } 478 } 479 ScaledMask.assign(InputMask.begin(), InputMask.end()); 480 } 481 482 void llvm::processShuffleMasks( 483 ArrayRef<int> Mask, unsigned NumOfSrcRegs, unsigned NumOfDestRegs, 484 unsigned NumOfUsedRegs, function_ref<void()> NoInputAction, 485 function_ref<void(ArrayRef<int>, unsigned, unsigned)> SingleInputAction, 486 function_ref<void(ArrayRef<int>, unsigned, unsigned)> ManyInputsAction) { 487 SmallVector<SmallVector<SmallVector<int>>> Res(NumOfDestRegs); 488 // Try to perform better estimation of the permutation. 489 // 1. Split the source/destination vectors into real registers. 490 // 2. Do the mask analysis to identify which real registers are 491 // permuted. 492 int Sz = Mask.size(); 493 unsigned SzDest = Sz / NumOfDestRegs; 494 unsigned SzSrc = Sz / NumOfSrcRegs; 495 for (unsigned I = 0; I < NumOfDestRegs; ++I) { 496 auto &RegMasks = Res[I]; 497 RegMasks.assign(NumOfSrcRegs, {}); 498 // Check that the values in dest registers are in the one src 499 // register. 500 for (unsigned K = 0; K < SzDest; ++K) { 501 int Idx = I * SzDest + K; 502 if (Idx == Sz) 503 break; 504 if (Mask[Idx] >= Sz || Mask[Idx] == PoisonMaskElem) 505 continue; 506 int SrcRegIdx = Mask[Idx] / SzSrc; 507 // Add a cost of PermuteTwoSrc for each new source register permute, 508 // if we have more than one source registers. 509 if (RegMasks[SrcRegIdx].empty()) 510 RegMasks[SrcRegIdx].assign(SzDest, PoisonMaskElem); 511 RegMasks[SrcRegIdx][K] = Mask[Idx] % SzSrc; 512 } 513 } 514 // Process split mask. 515 for (unsigned I = 0; I < NumOfUsedRegs; ++I) { 516 auto &Dest = Res[I]; 517 int NumSrcRegs = 518 count_if(Dest, [](ArrayRef<int> Mask) { return !Mask.empty(); }); 519 switch (NumSrcRegs) { 520 case 0: 521 // No input vectors were used! 522 NoInputAction(); 523 break; 524 case 1: { 525 // Find the only mask with at least single undef mask elem. 526 auto *It = 527 find_if(Dest, [](ArrayRef<int> Mask) { return !Mask.empty(); }); 528 unsigned SrcReg = std::distance(Dest.begin(), It); 529 SingleInputAction(*It, SrcReg, I); 530 break; 531 } 532 default: { 533 // The first mask is a permutation of a single register. Since we have >2 534 // input registers to shuffle, we merge the masks for 2 first registers 535 // and generate a shuffle of 2 registers rather than the reordering of the 536 // first register and then shuffle with the second register. Next, 537 // generate the shuffles of the resulting register + the remaining 538 // registers from the list. 539 auto &&CombineMasks = [](MutableArrayRef<int> FirstMask, 540 ArrayRef<int> SecondMask) { 541 for (int Idx = 0, VF = FirstMask.size(); Idx < VF; ++Idx) { 542 if (SecondMask[Idx] != PoisonMaskElem) { 543 assert(FirstMask[Idx] == PoisonMaskElem && 544 "Expected undefined mask element."); 545 FirstMask[Idx] = SecondMask[Idx] + VF; 546 } 547 } 548 }; 549 auto &&NormalizeMask = [](MutableArrayRef<int> Mask) { 550 for (int Idx = 0, VF = Mask.size(); Idx < VF; ++Idx) { 551 if (Mask[Idx] != PoisonMaskElem) 552 Mask[Idx] = Idx; 553 } 554 }; 555 int SecondIdx; 556 do { 557 int FirstIdx = -1; 558 SecondIdx = -1; 559 MutableArrayRef<int> FirstMask, SecondMask; 560 for (unsigned I = 0; I < NumOfDestRegs; ++I) { 561 SmallVectorImpl<int> &RegMask = Dest[I]; 562 if (RegMask.empty()) 563 continue; 564 565 if (FirstIdx == SecondIdx) { 566 FirstIdx = I; 567 FirstMask = RegMask; 568 continue; 569 } 570 SecondIdx = I; 571 SecondMask = RegMask; 572 CombineMasks(FirstMask, SecondMask); 573 ManyInputsAction(FirstMask, FirstIdx, SecondIdx); 574 NormalizeMask(FirstMask); 575 RegMask.clear(); 576 SecondMask = FirstMask; 577 SecondIdx = FirstIdx; 578 } 579 if (FirstIdx != SecondIdx && SecondIdx >= 0) { 580 CombineMasks(SecondMask, FirstMask); 581 ManyInputsAction(SecondMask, SecondIdx, FirstIdx); 582 Dest[FirstIdx].clear(); 583 NormalizeMask(SecondMask); 584 } 585 } while (SecondIdx >= 0); 586 break; 587 } 588 } 589 } 590 } 591 592 void llvm::getHorizDemandedEltsForFirstOperand(unsigned VectorBitWidth, 593 const APInt &DemandedElts, 594 APInt &DemandedLHS, 595 APInt &DemandedRHS) { 596 assert(VectorBitWidth >= 128 && "Vectors smaller than 128 bit not supported"); 597 int NumLanes = VectorBitWidth / 128; 598 int NumElts = DemandedElts.getBitWidth(); 599 int NumEltsPerLane = NumElts / NumLanes; 600 int HalfEltsPerLane = NumEltsPerLane / 2; 601 602 DemandedLHS = APInt::getZero(NumElts); 603 DemandedRHS = APInt::getZero(NumElts); 604 605 // Map DemandedElts to the horizontal operands. 606 for (int Idx = 0; Idx != NumElts; ++Idx) { 607 if (!DemandedElts[Idx]) 608 continue; 609 int LaneIdx = (Idx / NumEltsPerLane) * NumEltsPerLane; 610 int LocalIdx = Idx % NumEltsPerLane; 611 if (LocalIdx < HalfEltsPerLane) { 612 DemandedLHS.setBit(LaneIdx + 2 * LocalIdx); 613 } else { 614 LocalIdx -= HalfEltsPerLane; 615 DemandedRHS.setBit(LaneIdx + 2 * LocalIdx); 616 } 617 } 618 } 619 620 MapVector<Instruction *, uint64_t> 621 llvm::computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB, 622 const TargetTransformInfo *TTI) { 623 624 // DemandedBits will give us every value's live-out bits. But we want 625 // to ensure no extra casts would need to be inserted, so every DAG 626 // of connected values must have the same minimum bitwidth. 627 EquivalenceClasses<Value *> ECs; 628 SmallVector<Value *, 16> Worklist; 629 SmallPtrSet<Value *, 4> Roots; 630 SmallPtrSet<Value *, 16> Visited; 631 DenseMap<Value *, uint64_t> DBits; 632 SmallPtrSet<Instruction *, 4> InstructionSet; 633 MapVector<Instruction *, uint64_t> MinBWs; 634 635 // Determine the roots. We work bottom-up, from truncs or icmps. 636 bool SeenExtFromIllegalType = false; 637 for (auto *BB : Blocks) 638 for (auto &I : *BB) { 639 InstructionSet.insert(&I); 640 641 if (TTI && (isa<ZExtInst>(&I) || isa<SExtInst>(&I)) && 642 !TTI->isTypeLegal(I.getOperand(0)->getType())) 643 SeenExtFromIllegalType = true; 644 645 // Only deal with non-vector integers up to 64-bits wide. 646 if ((isa<TruncInst>(&I) || isa<ICmpInst>(&I)) && 647 !I.getType()->isVectorTy() && 648 I.getOperand(0)->getType()->getScalarSizeInBits() <= 64) { 649 // Don't make work for ourselves. If we know the loaded type is legal, 650 // don't add it to the worklist. 651 if (TTI && isa<TruncInst>(&I) && TTI->isTypeLegal(I.getType())) 652 continue; 653 654 Worklist.push_back(&I); 655 Roots.insert(&I); 656 } 657 } 658 // Early exit. 659 if (Worklist.empty() || (TTI && !SeenExtFromIllegalType)) 660 return MinBWs; 661 662 // Now proceed breadth-first, unioning values together. 663 while (!Worklist.empty()) { 664 Value *Val = Worklist.pop_back_val(); 665 Value *Leader = ECs.getOrInsertLeaderValue(Val); 666 667 if (!Visited.insert(Val).second) 668 continue; 669 670 // Non-instructions terminate a chain successfully. 671 if (!isa<Instruction>(Val)) 672 continue; 673 Instruction *I = cast<Instruction>(Val); 674 675 // If we encounter a type that is larger than 64 bits, we can't represent 676 // it so bail out. 677 if (DB.getDemandedBits(I).getBitWidth() > 64) 678 return MapVector<Instruction *, uint64_t>(); 679 680 uint64_t V = DB.getDemandedBits(I).getZExtValue(); 681 DBits[Leader] |= V; 682 DBits[I] = V; 683 684 // Casts, loads and instructions outside of our range terminate a chain 685 // successfully. 686 if (isa<SExtInst>(I) || isa<ZExtInst>(I) || isa<LoadInst>(I) || 687 !InstructionSet.count(I)) 688 continue; 689 690 // Unsafe casts terminate a chain unsuccessfully. We can't do anything 691 // useful with bitcasts, ptrtoints or inttoptrs and it'd be unsafe to 692 // transform anything that relies on them. 693 if (isa<BitCastInst>(I) || isa<PtrToIntInst>(I) || isa<IntToPtrInst>(I) || 694 !I->getType()->isIntegerTy()) { 695 DBits[Leader] |= ~0ULL; 696 continue; 697 } 698 699 // We don't modify the types of PHIs. Reductions will already have been 700 // truncated if possible, and inductions' sizes will have been chosen by 701 // indvars. 702 if (isa<PHINode>(I)) 703 continue; 704 705 if (DBits[Leader] == ~0ULL) 706 // All bits demanded, no point continuing. 707 continue; 708 709 for (Value *O : cast<User>(I)->operands()) { 710 ECs.unionSets(Leader, O); 711 Worklist.push_back(O); 712 } 713 } 714 715 // Now we've discovered all values, walk them to see if there are 716 // any users we didn't see. If there are, we can't optimize that 717 // chain. 718 for (auto &I : DBits) 719 for (auto *U : I.first->users()) 720 if (U->getType()->isIntegerTy() && DBits.count(U) == 0) 721 DBits[ECs.getOrInsertLeaderValue(I.first)] |= ~0ULL; 722 723 for (auto I = ECs.begin(), E = ECs.end(); I != E; ++I) { 724 uint64_t LeaderDemandedBits = 0; 725 for (Value *M : llvm::make_range(ECs.member_begin(I), ECs.member_end())) 726 LeaderDemandedBits |= DBits[M]; 727 728 uint64_t MinBW = llvm::bit_width(LeaderDemandedBits); 729 // Round up to a power of 2 730 MinBW = llvm::bit_ceil(MinBW); 731 732 // We don't modify the types of PHIs. Reductions will already have been 733 // truncated if possible, and inductions' sizes will have been chosen by 734 // indvars. 735 // If we are required to shrink a PHI, abandon this entire equivalence class. 736 bool Abort = false; 737 for (Value *M : llvm::make_range(ECs.member_begin(I), ECs.member_end())) 738 if (isa<PHINode>(M) && MinBW < M->getType()->getScalarSizeInBits()) { 739 Abort = true; 740 break; 741 } 742 if (Abort) 743 continue; 744 745 for (Value *M : llvm::make_range(ECs.member_begin(I), ECs.member_end())) { 746 auto *MI = dyn_cast<Instruction>(M); 747 if (!MI) 748 continue; 749 Type *Ty = M->getType(); 750 if (Roots.count(M)) 751 Ty = MI->getOperand(0)->getType(); 752 753 if (MinBW >= Ty->getScalarSizeInBits()) 754 continue; 755 756 // If any of M's operands demand more bits than MinBW then M cannot be 757 // performed safely in MinBW. 758 if (any_of(MI->operands(), [&DB, MinBW](Use &U) { 759 auto *CI = dyn_cast<ConstantInt>(U); 760 // For constants shift amounts, check if the shift would result in 761 // poison. 762 if (CI && 763 isa<ShlOperator, LShrOperator, AShrOperator>(U.getUser()) && 764 U.getOperandNo() == 1) 765 return CI->uge(MinBW); 766 uint64_t BW = bit_width(DB.getDemandedBits(&U).getZExtValue()); 767 return bit_ceil(BW) > MinBW; 768 })) 769 continue; 770 771 MinBWs[MI] = MinBW; 772 } 773 } 774 775 return MinBWs; 776 } 777 778 /// Add all access groups in @p AccGroups to @p List. 779 template <typename ListT> 780 static void addToAccessGroupList(ListT &List, MDNode *AccGroups) { 781 // Interpret an access group as a list containing itself. 782 if (AccGroups->getNumOperands() == 0) { 783 assert(isValidAsAccessGroup(AccGroups) && "Node must be an access group"); 784 List.insert(AccGroups); 785 return; 786 } 787 788 for (const auto &AccGroupListOp : AccGroups->operands()) { 789 auto *Item = cast<MDNode>(AccGroupListOp.get()); 790 assert(isValidAsAccessGroup(Item) && "List item must be an access group"); 791 List.insert(Item); 792 } 793 } 794 795 MDNode *llvm::uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2) { 796 if (!AccGroups1) 797 return AccGroups2; 798 if (!AccGroups2) 799 return AccGroups1; 800 if (AccGroups1 == AccGroups2) 801 return AccGroups1; 802 803 SmallSetVector<Metadata *, 4> Union; 804 addToAccessGroupList(Union, AccGroups1); 805 addToAccessGroupList(Union, AccGroups2); 806 807 if (Union.size() == 0) 808 return nullptr; 809 if (Union.size() == 1) 810 return cast<MDNode>(Union.front()); 811 812 LLVMContext &Ctx = AccGroups1->getContext(); 813 return MDNode::get(Ctx, Union.getArrayRef()); 814 } 815 816 MDNode *llvm::intersectAccessGroups(const Instruction *Inst1, 817 const Instruction *Inst2) { 818 bool MayAccessMem1 = Inst1->mayReadOrWriteMemory(); 819 bool MayAccessMem2 = Inst2->mayReadOrWriteMemory(); 820 821 if (!MayAccessMem1 && !MayAccessMem2) 822 return nullptr; 823 if (!MayAccessMem1) 824 return Inst2->getMetadata(LLVMContext::MD_access_group); 825 if (!MayAccessMem2) 826 return Inst1->getMetadata(LLVMContext::MD_access_group); 827 828 MDNode *MD1 = Inst1->getMetadata(LLVMContext::MD_access_group); 829 MDNode *MD2 = Inst2->getMetadata(LLVMContext::MD_access_group); 830 if (!MD1 || !MD2) 831 return nullptr; 832 if (MD1 == MD2) 833 return MD1; 834 835 // Use set for scalable 'contains' check. 836 SmallPtrSet<Metadata *, 4> AccGroupSet2; 837 addToAccessGroupList(AccGroupSet2, MD2); 838 839 SmallVector<Metadata *, 4> Intersection; 840 if (MD1->getNumOperands() == 0) { 841 assert(isValidAsAccessGroup(MD1) && "Node must be an access group"); 842 if (AccGroupSet2.count(MD1)) 843 Intersection.push_back(MD1); 844 } else { 845 for (const MDOperand &Node : MD1->operands()) { 846 auto *Item = cast<MDNode>(Node.get()); 847 assert(isValidAsAccessGroup(Item) && "List item must be an access group"); 848 if (AccGroupSet2.count(Item)) 849 Intersection.push_back(Item); 850 } 851 } 852 853 if (Intersection.size() == 0) 854 return nullptr; 855 if (Intersection.size() == 1) 856 return cast<MDNode>(Intersection.front()); 857 858 LLVMContext &Ctx = Inst1->getContext(); 859 return MDNode::get(Ctx, Intersection); 860 } 861 862 /// \returns \p I after propagating metadata from \p VL. 863 Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) { 864 if (VL.empty()) 865 return Inst; 866 Instruction *I0 = cast<Instruction>(VL[0]); 867 SmallVector<std::pair<unsigned, MDNode *>, 4> Metadata; 868 I0->getAllMetadataOtherThanDebugLoc(Metadata); 869 870 for (auto Kind : {LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope, 871 LLVMContext::MD_noalias, LLVMContext::MD_fpmath, 872 LLVMContext::MD_nontemporal, LLVMContext::MD_invariant_load, 873 LLVMContext::MD_access_group, LLVMContext::MD_mmra}) { 874 MDNode *MD = I0->getMetadata(Kind); 875 for (int J = 1, E = VL.size(); MD && J != E; ++J) { 876 const Instruction *IJ = cast<Instruction>(VL[J]); 877 MDNode *IMD = IJ->getMetadata(Kind); 878 879 switch (Kind) { 880 case LLVMContext::MD_mmra: { 881 MD = MMRAMetadata::combine(Inst->getContext(), MD, IMD); 882 break; 883 } 884 case LLVMContext::MD_tbaa: 885 MD = MDNode::getMostGenericTBAA(MD, IMD); 886 break; 887 case LLVMContext::MD_alias_scope: 888 MD = MDNode::getMostGenericAliasScope(MD, IMD); 889 break; 890 case LLVMContext::MD_fpmath: 891 MD = MDNode::getMostGenericFPMath(MD, IMD); 892 break; 893 case LLVMContext::MD_noalias: 894 case LLVMContext::MD_nontemporal: 895 case LLVMContext::MD_invariant_load: 896 MD = MDNode::intersect(MD, IMD); 897 break; 898 case LLVMContext::MD_access_group: 899 MD = intersectAccessGroups(Inst, IJ); 900 break; 901 default: 902 llvm_unreachable("unhandled metadata"); 903 } 904 } 905 906 Inst->setMetadata(Kind, MD); 907 } 908 909 return Inst; 910 } 911 912 Constant * 913 llvm::createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, 914 const InterleaveGroup<Instruction> &Group) { 915 // All 1's means mask is not needed. 916 if (Group.getNumMembers() == Group.getFactor()) 917 return nullptr; 918 919 // TODO: support reversed access. 920 assert(!Group.isReverse() && "Reversed group not supported."); 921 922 SmallVector<Constant *, 16> Mask; 923 for (unsigned i = 0; i < VF; i++) 924 for (unsigned j = 0; j < Group.getFactor(); ++j) { 925 unsigned HasMember = Group.getMember(j) ? 1 : 0; 926 Mask.push_back(Builder.getInt1(HasMember)); 927 } 928 929 return ConstantVector::get(Mask); 930 } 931 932 llvm::SmallVector<int, 16> 933 llvm::createReplicatedMask(unsigned ReplicationFactor, unsigned VF) { 934 SmallVector<int, 16> MaskVec; 935 for (unsigned i = 0; i < VF; i++) 936 for (unsigned j = 0; j < ReplicationFactor; j++) 937 MaskVec.push_back(i); 938 939 return MaskVec; 940 } 941 942 llvm::SmallVector<int, 16> llvm::createInterleaveMask(unsigned VF, 943 unsigned NumVecs) { 944 SmallVector<int, 16> Mask; 945 for (unsigned i = 0; i < VF; i++) 946 for (unsigned j = 0; j < NumVecs; j++) 947 Mask.push_back(j * VF + i); 948 949 return Mask; 950 } 951 952 llvm::SmallVector<int, 16> 953 llvm::createStrideMask(unsigned Start, unsigned Stride, unsigned VF) { 954 SmallVector<int, 16> Mask; 955 for (unsigned i = 0; i < VF; i++) 956 Mask.push_back(Start + i * Stride); 957 958 return Mask; 959 } 960 961 llvm::SmallVector<int, 16> llvm::createSequentialMask(unsigned Start, 962 unsigned NumInts, 963 unsigned NumUndefs) { 964 SmallVector<int, 16> Mask; 965 for (unsigned i = 0; i < NumInts; i++) 966 Mask.push_back(Start + i); 967 968 for (unsigned i = 0; i < NumUndefs; i++) 969 Mask.push_back(-1); 970 971 return Mask; 972 } 973 974 llvm::SmallVector<int, 16> llvm::createUnaryMask(ArrayRef<int> Mask, 975 unsigned NumElts) { 976 // Avoid casts in the loop and make sure we have a reasonable number. 977 int NumEltsSigned = NumElts; 978 assert(NumEltsSigned > 0 && "Expected smaller or non-zero element count"); 979 980 // If the mask chooses an element from operand 1, reduce it to choose from the 981 // corresponding element of operand 0. Undef mask elements are unchanged. 982 SmallVector<int, 16> UnaryMask; 983 for (int MaskElt : Mask) { 984 assert((MaskElt < NumEltsSigned * 2) && "Expected valid shuffle mask"); 985 int UnaryElt = MaskElt >= NumEltsSigned ? MaskElt - NumEltsSigned : MaskElt; 986 UnaryMask.push_back(UnaryElt); 987 } 988 return UnaryMask; 989 } 990 991 /// A helper function for concatenating vectors. This function concatenates two 992 /// vectors having the same element type. If the second vector has fewer 993 /// elements than the first, it is padded with undefs. 994 static Value *concatenateTwoVectors(IRBuilderBase &Builder, Value *V1, 995 Value *V2) { 996 VectorType *VecTy1 = dyn_cast<VectorType>(V1->getType()); 997 VectorType *VecTy2 = dyn_cast<VectorType>(V2->getType()); 998 assert(VecTy1 && VecTy2 && 999 VecTy1->getScalarType() == VecTy2->getScalarType() && 1000 "Expect two vectors with the same element type"); 1001 1002 unsigned NumElts1 = cast<FixedVectorType>(VecTy1)->getNumElements(); 1003 unsigned NumElts2 = cast<FixedVectorType>(VecTy2)->getNumElements(); 1004 assert(NumElts1 >= NumElts2 && "Unexpect the first vector has less elements"); 1005 1006 if (NumElts1 > NumElts2) { 1007 // Extend with UNDEFs. 1008 V2 = Builder.CreateShuffleVector( 1009 V2, createSequentialMask(0, NumElts2, NumElts1 - NumElts2)); 1010 } 1011 1012 return Builder.CreateShuffleVector( 1013 V1, V2, createSequentialMask(0, NumElts1 + NumElts2, 0)); 1014 } 1015 1016 Value *llvm::concatenateVectors(IRBuilderBase &Builder, 1017 ArrayRef<Value *> Vecs) { 1018 unsigned NumVecs = Vecs.size(); 1019 assert(NumVecs > 1 && "Should be at least two vectors"); 1020 1021 SmallVector<Value *, 8> ResList; 1022 ResList.append(Vecs.begin(), Vecs.end()); 1023 do { 1024 SmallVector<Value *, 8> TmpList; 1025 for (unsigned i = 0; i < NumVecs - 1; i += 2) { 1026 Value *V0 = ResList[i], *V1 = ResList[i + 1]; 1027 assert((V0->getType() == V1->getType() || i == NumVecs - 2) && 1028 "Only the last vector may have a different type"); 1029 1030 TmpList.push_back(concatenateTwoVectors(Builder, V0, V1)); 1031 } 1032 1033 // Push the last vector if the total number of vectors is odd. 1034 if (NumVecs % 2 != 0) 1035 TmpList.push_back(ResList[NumVecs - 1]); 1036 1037 ResList = TmpList; 1038 NumVecs = ResList.size(); 1039 } while (NumVecs > 1); 1040 1041 return ResList[0]; 1042 } 1043 1044 bool llvm::maskIsAllZeroOrUndef(Value *Mask) { 1045 assert(isa<VectorType>(Mask->getType()) && 1046 isa<IntegerType>(Mask->getType()->getScalarType()) && 1047 cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1048 1 && 1049 "Mask must be a vector of i1"); 1050 1051 auto *ConstMask = dyn_cast<Constant>(Mask); 1052 if (!ConstMask) 1053 return false; 1054 if (ConstMask->isNullValue() || isa<UndefValue>(ConstMask)) 1055 return true; 1056 if (isa<ScalableVectorType>(ConstMask->getType())) 1057 return false; 1058 for (unsigned 1059 I = 0, 1060 E = cast<FixedVectorType>(ConstMask->getType())->getNumElements(); 1061 I != E; ++I) { 1062 if (auto *MaskElt = ConstMask->getAggregateElement(I)) 1063 if (MaskElt->isNullValue() || isa<UndefValue>(MaskElt)) 1064 continue; 1065 return false; 1066 } 1067 return true; 1068 } 1069 1070 bool llvm::maskIsAllOneOrUndef(Value *Mask) { 1071 assert(isa<VectorType>(Mask->getType()) && 1072 isa<IntegerType>(Mask->getType()->getScalarType()) && 1073 cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1074 1 && 1075 "Mask must be a vector of i1"); 1076 1077 auto *ConstMask = dyn_cast<Constant>(Mask); 1078 if (!ConstMask) 1079 return false; 1080 if (ConstMask->isAllOnesValue() || isa<UndefValue>(ConstMask)) 1081 return true; 1082 if (isa<ScalableVectorType>(ConstMask->getType())) 1083 return false; 1084 for (unsigned 1085 I = 0, 1086 E = cast<FixedVectorType>(ConstMask->getType())->getNumElements(); 1087 I != E; ++I) { 1088 if (auto *MaskElt = ConstMask->getAggregateElement(I)) 1089 if (MaskElt->isAllOnesValue() || isa<UndefValue>(MaskElt)) 1090 continue; 1091 return false; 1092 } 1093 return true; 1094 } 1095 1096 bool llvm::maskContainsAllOneOrUndef(Value *Mask) { 1097 assert(isa<VectorType>(Mask->getType()) && 1098 isa<IntegerType>(Mask->getType()->getScalarType()) && 1099 cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1100 1 && 1101 "Mask must be a vector of i1"); 1102 1103 auto *ConstMask = dyn_cast<Constant>(Mask); 1104 if (!ConstMask) 1105 return false; 1106 if (ConstMask->isAllOnesValue() || isa<UndefValue>(ConstMask)) 1107 return true; 1108 if (isa<ScalableVectorType>(ConstMask->getType())) 1109 return false; 1110 for (unsigned 1111 I = 0, 1112 E = cast<FixedVectorType>(ConstMask->getType())->getNumElements(); 1113 I != E; ++I) { 1114 if (auto *MaskElt = ConstMask->getAggregateElement(I)) 1115 if (MaskElt->isAllOnesValue() || isa<UndefValue>(MaskElt)) 1116 return true; 1117 } 1118 return false; 1119 } 1120 1121 /// TODO: This is a lot like known bits, but for 1122 /// vectors. Is there something we can common this with? 1123 APInt llvm::possiblyDemandedEltsInMask(Value *Mask) { 1124 assert(isa<FixedVectorType>(Mask->getType()) && 1125 isa<IntegerType>(Mask->getType()->getScalarType()) && 1126 cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() == 1127 1 && 1128 "Mask must be a fixed width vector of i1"); 1129 1130 const unsigned VWidth = 1131 cast<FixedVectorType>(Mask->getType())->getNumElements(); 1132 APInt DemandedElts = APInt::getAllOnes(VWidth); 1133 if (auto *CV = dyn_cast<ConstantVector>(Mask)) 1134 for (unsigned i = 0; i < VWidth; i++) 1135 if (CV->getAggregateElement(i)->isNullValue()) 1136 DemandedElts.clearBit(i); 1137 return DemandedElts; 1138 } 1139 1140 bool InterleavedAccessInfo::isStrided(int Stride) { 1141 unsigned Factor = std::abs(Stride); 1142 return Factor >= 2 && Factor <= MaxInterleaveGroupFactor; 1143 } 1144 1145 void InterleavedAccessInfo::collectConstStrideAccesses( 1146 MapVector<Instruction *, StrideDescriptor> &AccessStrideInfo, 1147 const DenseMap<Value*, const SCEV*> &Strides) { 1148 auto &DL = TheLoop->getHeader()->getDataLayout(); 1149 1150 // Since it's desired that the load/store instructions be maintained in 1151 // "program order" for the interleaved access analysis, we have to visit the 1152 // blocks in the loop in reverse postorder (i.e., in a topological order). 1153 // Such an ordering will ensure that any load/store that may be executed 1154 // before a second load/store will precede the second load/store in 1155 // AccessStrideInfo. 1156 LoopBlocksDFS DFS(TheLoop); 1157 DFS.perform(LI); 1158 for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO())) 1159 for (auto &I : *BB) { 1160 Value *Ptr = getLoadStorePointerOperand(&I); 1161 if (!Ptr) 1162 continue; 1163 Type *ElementTy = getLoadStoreType(&I); 1164 1165 // Currently, codegen doesn't support cases where the type size doesn't 1166 // match the alloc size. Skip them for now. 1167 uint64_t Size = DL.getTypeAllocSize(ElementTy); 1168 if (Size * 8 != DL.getTypeSizeInBits(ElementTy)) 1169 continue; 1170 1171 // We don't check wrapping here because we don't know yet if Ptr will be 1172 // part of a full group or a group with gaps. Checking wrapping for all 1173 // pointers (even those that end up in groups with no gaps) will be overly 1174 // conservative. For full groups, wrapping should be ok since if we would 1175 // wrap around the address space we would do a memory access at nullptr 1176 // even without the transformation. The wrapping checks are therefore 1177 // deferred until after we've formed the interleaved groups. 1178 int64_t Stride = 1179 getPtrStride(PSE, ElementTy, Ptr, TheLoop, Strides, 1180 /*Assume=*/true, /*ShouldCheckWrap=*/false).value_or(0); 1181 1182 const SCEV *Scev = replaceSymbolicStrideSCEV(PSE, Strides, Ptr); 1183 AccessStrideInfo[&I] = StrideDescriptor(Stride, Scev, Size, 1184 getLoadStoreAlignment(&I)); 1185 } 1186 } 1187 1188 // Analyze interleaved accesses and collect them into interleaved load and 1189 // store groups. 1190 // 1191 // When generating code for an interleaved load group, we effectively hoist all 1192 // loads in the group to the location of the first load in program order. When 1193 // generating code for an interleaved store group, we sink all stores to the 1194 // location of the last store. This code motion can change the order of load 1195 // and store instructions and may break dependences. 1196 // 1197 // The code generation strategy mentioned above ensures that we won't violate 1198 // any write-after-read (WAR) dependences. 1199 // 1200 // E.g., for the WAR dependence: a = A[i]; // (1) 1201 // A[i] = b; // (2) 1202 // 1203 // The store group of (2) is always inserted at or below (2), and the load 1204 // group of (1) is always inserted at or above (1). Thus, the instructions will 1205 // never be reordered. All other dependences are checked to ensure the 1206 // correctness of the instruction reordering. 1207 // 1208 // The algorithm visits all memory accesses in the loop in bottom-up program 1209 // order. Program order is established by traversing the blocks in the loop in 1210 // reverse postorder when collecting the accesses. 1211 // 1212 // We visit the memory accesses in bottom-up order because it can simplify the 1213 // construction of store groups in the presence of write-after-write (WAW) 1214 // dependences. 1215 // 1216 // E.g., for the WAW dependence: A[i] = a; // (1) 1217 // A[i] = b; // (2) 1218 // A[i + 1] = c; // (3) 1219 // 1220 // We will first create a store group with (3) and (2). (1) can't be added to 1221 // this group because it and (2) are dependent. However, (1) can be grouped 1222 // with other accesses that may precede it in program order. Note that a 1223 // bottom-up order does not imply that WAW dependences should not be checked. 1224 void InterleavedAccessInfo::analyzeInterleaving( 1225 bool EnablePredicatedInterleavedMemAccesses) { 1226 LLVM_DEBUG(dbgs() << "LV: Analyzing interleaved accesses...\n"); 1227 const auto &Strides = LAI->getSymbolicStrides(); 1228 1229 // Holds all accesses with a constant stride. 1230 MapVector<Instruction *, StrideDescriptor> AccessStrideInfo; 1231 collectConstStrideAccesses(AccessStrideInfo, Strides); 1232 1233 if (AccessStrideInfo.empty()) 1234 return; 1235 1236 // Collect the dependences in the loop. 1237 collectDependences(); 1238 1239 // Holds all interleaved store groups temporarily. 1240 SmallSetVector<InterleaveGroup<Instruction> *, 4> StoreGroups; 1241 // Holds all interleaved load groups temporarily. 1242 SmallSetVector<InterleaveGroup<Instruction> *, 4> LoadGroups; 1243 // Groups added to this set cannot have new members added. 1244 SmallPtrSet<InterleaveGroup<Instruction> *, 4> CompletedLoadGroups; 1245 1246 // Search in bottom-up program order for pairs of accesses (A and B) that can 1247 // form interleaved load or store groups. In the algorithm below, access A 1248 // precedes access B in program order. We initialize a group for B in the 1249 // outer loop of the algorithm, and then in the inner loop, we attempt to 1250 // insert each A into B's group if: 1251 // 1252 // 1. A and B have the same stride, 1253 // 2. A and B have the same memory object size, and 1254 // 3. A belongs in B's group according to its distance from B. 1255 // 1256 // Special care is taken to ensure group formation will not break any 1257 // dependences. 1258 for (auto BI = AccessStrideInfo.rbegin(), E = AccessStrideInfo.rend(); 1259 BI != E; ++BI) { 1260 Instruction *B = BI->first; 1261 StrideDescriptor DesB = BI->second; 1262 1263 // Initialize a group for B if it has an allowable stride. Even if we don't 1264 // create a group for B, we continue with the bottom-up algorithm to ensure 1265 // we don't break any of B's dependences. 1266 InterleaveGroup<Instruction> *GroupB = nullptr; 1267 if (isStrided(DesB.Stride) && 1268 (!isPredicated(B->getParent()) || EnablePredicatedInterleavedMemAccesses)) { 1269 GroupB = getInterleaveGroup(B); 1270 if (!GroupB) { 1271 LLVM_DEBUG(dbgs() << "LV: Creating an interleave group with:" << *B 1272 << '\n'); 1273 GroupB = createInterleaveGroup(B, DesB.Stride, DesB.Alignment); 1274 if (B->mayWriteToMemory()) 1275 StoreGroups.insert(GroupB); 1276 else 1277 LoadGroups.insert(GroupB); 1278 } 1279 } 1280 1281 for (auto AI = std::next(BI); AI != E; ++AI) { 1282 Instruction *A = AI->first; 1283 StrideDescriptor DesA = AI->second; 1284 1285 // Our code motion strategy implies that we can't have dependences 1286 // between accesses in an interleaved group and other accesses located 1287 // between the first and last member of the group. Note that this also 1288 // means that a group can't have more than one member at a given offset. 1289 // The accesses in a group can have dependences with other accesses, but 1290 // we must ensure we don't extend the boundaries of the group such that 1291 // we encompass those dependent accesses. 1292 // 1293 // For example, assume we have the sequence of accesses shown below in a 1294 // stride-2 loop: 1295 // 1296 // (1, 2) is a group | A[i] = a; // (1) 1297 // | A[i-1] = b; // (2) | 1298 // A[i-3] = c; // (3) 1299 // A[i] = d; // (4) | (2, 4) is not a group 1300 // 1301 // Because accesses (2) and (3) are dependent, we can group (2) with (1) 1302 // but not with (4). If we did, the dependent access (3) would be within 1303 // the boundaries of the (2, 4) group. 1304 auto DependentMember = [&](InterleaveGroup<Instruction> *Group, 1305 StrideEntry *A) -> Instruction * { 1306 for (uint32_t Index = 0; Index < Group->getFactor(); ++Index) { 1307 Instruction *MemberOfGroupB = Group->getMember(Index); 1308 if (MemberOfGroupB && !canReorderMemAccessesForInterleavedGroups( 1309 A, &*AccessStrideInfo.find(MemberOfGroupB))) 1310 return MemberOfGroupB; 1311 } 1312 return nullptr; 1313 }; 1314 1315 auto GroupA = getInterleaveGroup(A); 1316 // If A is a load, dependencies are tolerable, there's nothing to do here. 1317 // If both A and B belong to the same (store) group, they are independent, 1318 // even if dependencies have not been recorded. 1319 // If both GroupA and GroupB are null, there's nothing to do here. 1320 if (A->mayWriteToMemory() && GroupA != GroupB) { 1321 Instruction *DependentInst = nullptr; 1322 // If GroupB is a load group, we have to compare AI against all 1323 // members of GroupB because if any load within GroupB has a dependency 1324 // on AI, we need to mark GroupB as complete and also release the 1325 // store GroupA (if A belongs to one). The former prevents incorrect 1326 // hoisting of load B above store A while the latter prevents incorrect 1327 // sinking of store A below load B. 1328 if (GroupB && LoadGroups.contains(GroupB)) 1329 DependentInst = DependentMember(GroupB, &*AI); 1330 else if (!canReorderMemAccessesForInterleavedGroups(&*AI, &*BI)) 1331 DependentInst = B; 1332 1333 if (DependentInst) { 1334 // A has a store dependence on B (or on some load within GroupB) and 1335 // is part of a store group. Release A's group to prevent illegal 1336 // sinking of A below B. A will then be free to form another group 1337 // with instructions that precede it. 1338 if (GroupA && StoreGroups.contains(GroupA)) { 1339 LLVM_DEBUG(dbgs() << "LV: Invalidated store group due to " 1340 "dependence between " 1341 << *A << " and " << *DependentInst << '\n'); 1342 StoreGroups.remove(GroupA); 1343 releaseGroup(GroupA); 1344 } 1345 // If B is a load and part of an interleave group, no earlier loads 1346 // can be added to B's interleave group, because this would mean the 1347 // DependentInst would move across store A. Mark the interleave group 1348 // as complete. 1349 if (GroupB && LoadGroups.contains(GroupB)) { 1350 LLVM_DEBUG(dbgs() << "LV: Marking interleave group for " << *B 1351 << " as complete.\n"); 1352 CompletedLoadGroups.insert(GroupB); 1353 } 1354 } 1355 } 1356 if (CompletedLoadGroups.contains(GroupB)) { 1357 // Skip trying to add A to B, continue to look for other conflicting A's 1358 // in groups to be released. 1359 continue; 1360 } 1361 1362 // At this point, we've checked for illegal code motion. If either A or B 1363 // isn't strided, there's nothing left to do. 1364 if (!isStrided(DesA.Stride) || !isStrided(DesB.Stride)) 1365 continue; 1366 1367 // Ignore A if it's already in a group or isn't the same kind of memory 1368 // operation as B. 1369 // Note that mayReadFromMemory() isn't mutually exclusive to 1370 // mayWriteToMemory in the case of atomic loads. We shouldn't see those 1371 // here, canVectorizeMemory() should have returned false - except for the 1372 // case we asked for optimization remarks. 1373 if (isInterleaved(A) || 1374 (A->mayReadFromMemory() != B->mayReadFromMemory()) || 1375 (A->mayWriteToMemory() != B->mayWriteToMemory())) 1376 continue; 1377 1378 // Check rules 1 and 2. Ignore A if its stride or size is different from 1379 // that of B. 1380 if (DesA.Stride != DesB.Stride || DesA.Size != DesB.Size) 1381 continue; 1382 1383 // Ignore A if the memory object of A and B don't belong to the same 1384 // address space 1385 if (getLoadStoreAddressSpace(A) != getLoadStoreAddressSpace(B)) 1386 continue; 1387 1388 // Calculate the distance from A to B. 1389 const SCEVConstant *DistToB = dyn_cast<SCEVConstant>( 1390 PSE.getSE()->getMinusSCEV(DesA.Scev, DesB.Scev)); 1391 if (!DistToB) 1392 continue; 1393 int64_t DistanceToB = DistToB->getAPInt().getSExtValue(); 1394 1395 // Check rule 3. Ignore A if its distance to B is not a multiple of the 1396 // size. 1397 if (DistanceToB % static_cast<int64_t>(DesB.Size)) 1398 continue; 1399 1400 // All members of a predicated interleave-group must have the same predicate, 1401 // and currently must reside in the same BB. 1402 BasicBlock *BlockA = A->getParent(); 1403 BasicBlock *BlockB = B->getParent(); 1404 if ((isPredicated(BlockA) || isPredicated(BlockB)) && 1405 (!EnablePredicatedInterleavedMemAccesses || BlockA != BlockB)) 1406 continue; 1407 1408 // The index of A is the index of B plus A's distance to B in multiples 1409 // of the size. 1410 int IndexA = 1411 GroupB->getIndex(B) + DistanceToB / static_cast<int64_t>(DesB.Size); 1412 1413 // Try to insert A into B's group. 1414 if (GroupB->insertMember(A, IndexA, DesA.Alignment)) { 1415 LLVM_DEBUG(dbgs() << "LV: Inserted:" << *A << '\n' 1416 << " into the interleave group with" << *B 1417 << '\n'); 1418 InterleaveGroupMap[A] = GroupB; 1419 1420 // Set the first load in program order as the insert position. 1421 if (A->mayReadFromMemory()) 1422 GroupB->setInsertPos(A); 1423 } 1424 } // Iteration over A accesses. 1425 } // Iteration over B accesses. 1426 1427 auto InvalidateGroupIfMemberMayWrap = [&](InterleaveGroup<Instruction> *Group, 1428 int Index, 1429 const char *FirstOrLast) -> bool { 1430 Instruction *Member = Group->getMember(Index); 1431 assert(Member && "Group member does not exist"); 1432 Value *MemberPtr = getLoadStorePointerOperand(Member); 1433 Type *AccessTy = getLoadStoreType(Member); 1434 if (getPtrStride(PSE, AccessTy, MemberPtr, TheLoop, Strides, 1435 /*Assume=*/false, /*ShouldCheckWrap=*/true).value_or(0)) 1436 return false; 1437 LLVM_DEBUG(dbgs() << "LV: Invalidate candidate interleaved group due to " 1438 << FirstOrLast 1439 << " group member potentially pointer-wrapping.\n"); 1440 releaseGroup(Group); 1441 return true; 1442 }; 1443 1444 // Remove interleaved groups with gaps whose memory 1445 // accesses may wrap around. We have to revisit the getPtrStride analysis, 1446 // this time with ShouldCheckWrap=true, since collectConstStrideAccesses does 1447 // not check wrapping (see documentation there). 1448 // FORNOW we use Assume=false; 1449 // TODO: Change to Assume=true but making sure we don't exceed the threshold 1450 // of runtime SCEV assumptions checks (thereby potentially failing to 1451 // vectorize altogether). 1452 // Additional optional optimizations: 1453 // TODO: If we are peeling the loop and we know that the first pointer doesn't 1454 // wrap then we can deduce that all pointers in the group don't wrap. 1455 // This means that we can forcefully peel the loop in order to only have to 1456 // check the first pointer for no-wrap. When we'll change to use Assume=true 1457 // we'll only need at most one runtime check per interleaved group. 1458 for (auto *Group : LoadGroups) { 1459 // Case 1: A full group. Can Skip the checks; For full groups, if the wide 1460 // load would wrap around the address space we would do a memory access at 1461 // nullptr even without the transformation. 1462 if (Group->getNumMembers() == Group->getFactor()) 1463 continue; 1464 1465 // Case 2: If first and last members of the group don't wrap this implies 1466 // that all the pointers in the group don't wrap. 1467 // So we check only group member 0 (which is always guaranteed to exist), 1468 // and group member Factor - 1; If the latter doesn't exist we rely on 1469 // peeling (if it is a non-reversed access -- see Case 3). 1470 if (InvalidateGroupIfMemberMayWrap(Group, 0, "first")) 1471 continue; 1472 if (Group->getMember(Group->getFactor() - 1)) 1473 InvalidateGroupIfMemberMayWrap(Group, Group->getFactor() - 1, "last"); 1474 else { 1475 // Case 3: A non-reversed interleaved load group with gaps: We need 1476 // to execute at least one scalar epilogue iteration. This will ensure 1477 // we don't speculatively access memory out-of-bounds. We only need 1478 // to look for a member at index factor - 1, since every group must have 1479 // a member at index zero. 1480 if (Group->isReverse()) { 1481 LLVM_DEBUG( 1482 dbgs() << "LV: Invalidate candidate interleaved group due to " 1483 "a reverse access with gaps.\n"); 1484 releaseGroup(Group); 1485 continue; 1486 } 1487 LLVM_DEBUG( 1488 dbgs() << "LV: Interleaved group requires epilogue iteration.\n"); 1489 RequiresScalarEpilogue = true; 1490 } 1491 } 1492 1493 for (auto *Group : StoreGroups) { 1494 // Case 1: A full group. Can Skip the checks; For full groups, if the wide 1495 // store would wrap around the address space we would do a memory access at 1496 // nullptr even without the transformation. 1497 if (Group->getNumMembers() == Group->getFactor()) 1498 continue; 1499 1500 // Interleave-store-group with gaps is implemented using masked wide store. 1501 // Remove interleaved store groups with gaps if 1502 // masked-interleaved-accesses are not enabled by the target. 1503 if (!EnablePredicatedInterleavedMemAccesses) { 1504 LLVM_DEBUG( 1505 dbgs() << "LV: Invalidate candidate interleaved store group due " 1506 "to gaps.\n"); 1507 releaseGroup(Group); 1508 continue; 1509 } 1510 1511 // Case 2: If first and last members of the group don't wrap this implies 1512 // that all the pointers in the group don't wrap. 1513 // So we check only group member 0 (which is always guaranteed to exist), 1514 // and the last group member. Case 3 (scalar epilog) is not relevant for 1515 // stores with gaps, which are implemented with masked-store (rather than 1516 // speculative access, as in loads). 1517 if (InvalidateGroupIfMemberMayWrap(Group, 0, "first")) 1518 continue; 1519 for (int Index = Group->getFactor() - 1; Index > 0; Index--) 1520 if (Group->getMember(Index)) { 1521 InvalidateGroupIfMemberMayWrap(Group, Index, "last"); 1522 break; 1523 } 1524 } 1525 } 1526 1527 void InterleavedAccessInfo::invalidateGroupsRequiringScalarEpilogue() { 1528 // If no group had triggered the requirement to create an epilogue loop, 1529 // there is nothing to do. 1530 if (!requiresScalarEpilogue()) 1531 return; 1532 1533 // Release groups requiring scalar epilogues. Note that this also removes them 1534 // from InterleaveGroups. 1535 bool ReleasedGroup = InterleaveGroups.remove_if([&](auto *Group) { 1536 if (!Group->requiresScalarEpilogue()) 1537 return false; 1538 LLVM_DEBUG( 1539 dbgs() 1540 << "LV: Invalidate candidate interleaved group due to gaps that " 1541 "require a scalar epilogue (not allowed under optsize) and cannot " 1542 "be masked (not enabled). \n"); 1543 releaseGroupWithoutRemovingFromSet(Group); 1544 return true; 1545 }); 1546 assert(ReleasedGroup && "At least one group must be invalidated, as a " 1547 "scalar epilogue was required"); 1548 (void)ReleasedGroup; 1549 RequiresScalarEpilogue = false; 1550 } 1551 1552 template <typename InstT> 1553 void InterleaveGroup<InstT>::addMetadata(InstT *NewInst) const { 1554 llvm_unreachable("addMetadata can only be used for Instruction"); 1555 } 1556 1557 namespace llvm { 1558 template <> 1559 void InterleaveGroup<Instruction>::addMetadata(Instruction *NewInst) const { 1560 SmallVector<Value *, 4> VL; 1561 std::transform(Members.begin(), Members.end(), std::back_inserter(VL), 1562 [](std::pair<int, Instruction *> p) { return p.second; }); 1563 propagateMetadata(NewInst, VL); 1564 } 1565 } // namespace llvm 1566