Lines Matching full:vs
133 const VectorSplit &VS, ValueVector *cachePtr = nullptr);
139 unsigned size() const { return VS.NumFragments; }
145 VectorSplit VS;
212 VectorSplit VS;
222 /// described in @p VS.
224 const VectorSplit &VS, Twine Name) {
225 unsigned NumElements = VS.VecTy->getNumElements();
229 if (VS.NumPacked > 1) {
233 for (unsigned I = 0; I < VS.NumPacked; ++I)
241 Value *Res = PoisonValue::get(VS.VecTy);
242 for (unsigned I = 0; I < VS.NumFragments; ++I) {
245 unsigned NumPacked = VS.NumPacked;
246 if (I == VS.NumFragments - 1 && VS.RemainderTy) {
247 if (auto *RemVecTy = dyn_cast<FixedVectorType>(VS.RemainderTy))
254 Res = Builder.CreateInsertElement(Res, Fragment, I * VS.NumPacked,
262 InsertMask[I * VS.NumPacked + J] = NumElements + J;
266 InsertMask[I * VS.NumPacked + J] = I * VS.NumPacked + J;
315 Scatterer scatter(Instruction *Point, Value *V, const VectorSplit &VS);
316 void gather(Instruction *Op, const ValueVector &CV, const VectorSplit &VS);
346 const VectorSplit &VS, ValueVector *cachePtr)
347 : BB(bb), BBI(bbi), V(v), VS(VS), CachePtr(cachePtr) {
350 Tmp.resize(VS.NumFragments, nullptr);
352 assert((CachePtr->empty() || VS.NumFragments == CachePtr->size() ||
355 if (VS.NumFragments > CachePtr->size())
356 CachePtr->resize(VS.NumFragments, nullptr);
371 CV[Frag] = Builder.CreateConstGEP1_32(VS.SplitTy, V, Frag,
376 Type *FragmentTy = VS.getFragmentType(Frag);
381 Mask.push_back(Frag * VS.NumPacked + J);
398 if (Frag * VS.NumPacked == J) {
403 if (VS.NumPacked == 1 && !CV[J]) {
410 CV[Frag] = Builder.CreateExtractElement(V, Frag * VS.NumPacked,
440 const VectorSplit &VS) {
446 return Scatterer(BB, BB->begin(), V, VS, &Scattered[{V, VS.SplitTy}]);
457 PoisonValue::get(V->getType()), VS);
462 BB, skipPastPhiNodesAndDbg(std::next(BasicBlock::iterator(VOp))), V, VS,
463 &Scattered[{V, VS.SplitTy}]);
467 return Scatterer(Point->getParent(), Point->getIterator(), V, VS);
475 const VectorSplit &VS) {
480 ValueVector &SV = Scattered[{Op, VS.SplitTy}];
577 std::optional<VectorSplit> VS = getVectorSplit(Ty);
578 if (!VS)
582 Layout.VS = *VS;
584 if (!DL.typeSizeEqualsStoreSize(VS->SplitTy) ||
585 (VS->RemainderTy && !DL.typeSizeEqualsStoreSize(VS->RemainderTy)))
588 Layout.SplitSize = DL.getTypeStoreSize(VS->SplitTy);
596 std::optional<VectorSplit> VS = getVectorSplit(I.getType());
597 if (!VS)
602 OpVS = VS;
605 if (!OpVS || VS->NumPacked != OpVS->NumPacked)
611 assert(Op.size() == VS->NumFragments && "Mismatched unary operation");
613 Res.resize(VS->NumFragments);
614 for (unsigned Frag = 0; Frag < VS->NumFragments; ++Frag)
616 gather(&I, Res, *VS);
624 std::optional<VectorSplit> VS = getVectorSplit(I.getType());
625 if (!VS)
630 OpVS = VS;
633 if (!OpVS || VS->NumPacked != OpVS->NumPacked)
640 assert(VOp0.size() == VS->NumFragments && "Mismatched binary operation");
641 assert(VOp1.size() == VS->NumFragments && "Mismatched binary operation");
643 Res.resize(VS->NumFragments);
644 for (unsigned Frag = 0; Frag < VS->NumFragments; ++Frag) {
649 gather(&I, Res, *VS);
660 std::optional<VectorSplit> VS = getVectorSplit(CI.getType());
661 if (!VS)
682 Tys.push_back(VS->SplitTy);
690 assert(OpVecTy->getNumElements() == VS->VecTy->getNumElements());
692 if (!OpVS || OpVS->NumPacked != VS->NumPacked) {
715 ValueVector Res(VS->NumFragments);
722 for (unsigned I = 0; I < VS->NumFragments; ++I) {
723 bool IsRemainder = I == VS->NumFragments - 1 && VS->RemainderTy;
727 Tys[0] = VS->RemainderTy;
746 gather(&CI, Res, *VS);
751 std::optional<VectorSplit> VS = getVectorSplit(SI.getType());
752 if (!VS)
758 if (!CondVS || CondVS->NumPacked != VS->NumPacked) {
765 Scatterer VOp1 = scatter(&SI, SI.getOperand(1), *VS);
766 Scatterer VOp2 = scatter(&SI, SI.getOperand(2), *VS);
767 assert(VOp1.size() == VS->NumFragments && "Mismatched select");
768 assert(VOp2.size() == VS->NumFragments && "Mismatched select");
770 Res.resize(VS->NumFragments);
775 for (unsigned I = 0; I < VS->NumFragments; ++I) {
784 for (unsigned I = 0; I < VS->NumFragments; ++I) {
791 gather(&SI, Res, *VS);
812 std::optional<VectorSplit> VS = getVectorSplit(GEPI.getType());
813 if (!VS)
827 if (!OpVS || OpVS->NumPacked != VS->NumPacked) {
838 Res.resize(VS->NumFragments);
839 for (unsigned I = 0; I < VS->NumFragments; ++I) {
855 gather(&GEPI, Res, *VS);
963 std::optional<VectorSplit> VS = getVectorSplit(IEI.getType());
964 if (!VS)
968 Scatterer Op0 = scatter(&IEI, IEI.getOperand(0), *VS);
973 Res.resize(VS->NumFragments);
977 unsigned Fragment = Idx / VS->NumPacked;
978 for (unsigned I = 0; I < VS->NumFragments; ++I) {
980 bool IsPacked = VS->NumPacked > 1;
981 if (Fragment == VS->NumFragments - 1 && VS->RemainderTy &&
982 !VS->RemainderTy->isVectorTy())
986 Builder.CreateInsertElement(Op0[I], NewElt, Idx % VS->NumPacked);
996 if (!ScalarizeVariableInsertExtract || VS->NumPacked > 1)
999 for (unsigned I = 0; I < VS->NumFragments; ++I) {
1009 gather(&IEI, Res, *VS);
1014 std::optional<VectorSplit> VS = getVectorSplit(EEI.getOperand(0)->getType());
1015 if (!VS)
1019 Scatterer Op0 = scatter(&EEI, EEI.getOperand(0), *VS);
1024 unsigned Fragment = Idx / VS->NumPacked;
1026 bool IsPacked = VS->NumPacked > 1;
1027 if (Fragment == VS->NumFragments - 1 && VS->RemainderTy &&
1028 !VS->RemainderTy->isVectorTy())
1031 Res = Builder.CreateExtractElement(Res, Idx % VS->NumPacked);
1037 if (!ScalarizeVariableInsertExtract || VS->NumPacked > 1)
1040 Value *Res = PoisonValue::get(VS->VecTy->getElementType());
1041 for (unsigned I = 0; I < VS->NumFragments; ++I) {
1054 std::optional<VectorSplit> VS = getVectorSplit(SVI.getType());
1057 if (!VS || !VSOp || VS->NumPacked > 1 || VSOp->NumPacked > 1)
1063 Res.resize(VS->NumFragments);
1065 for (unsigned I = 0; I < VS->NumFragments; ++I) {
1068 Res[I] = PoisonValue::get(VS->VecTy->getElementType());
1074 gather(&SVI, Res, *VS);
1079 std::optional<VectorSplit> VS = getVectorSplit(PHI.getType());
1080 if (!VS)
1085 Res.resize(VS->NumFragments);
1088 for (unsigned I = 0; I < VS->NumFragments; ++I) {
1089 Res[I] = Builder.CreatePHI(VS->getFragmentType(I), NumOps,
1094 Scatterer Op = scatter(&PHI, PHI.getIncomingValue(I), *VS);
1096 for (unsigned J = 0; J < VS->NumFragments; ++J)
1099 gather(&PHI, Res, *VS);
1115 Scatterer Ptr = scatter(&LI, LI.getPointerOperand(), Layout->VS);
1117 Res.resize(Layout->VS.NumFragments);
1119 for (unsigned I = 0; I < Layout->VS.NumFragments; ++I) {
1120 Res[I] = Builder.CreateAlignedLoad(Layout->VS.getFragmentType(I), Ptr[I],
1124 gather(&LI, Res, Layout->VS);
1141 Scatterer VPtr = scatter(&SI, SI.getPointerOperand(), Layout->VS);
1142 Scatterer VVal = scatter(&SI, FullValue, Layout->VS);
1145 Stores.resize(Layout->VS.NumFragments);
1146 for (unsigned I = 0; I < Layout->VS.NumFragments; ++I) {
1186 VectorSplit VS = *getVectorSplit(Ty);
1187 assert(VS.NumFragments == CV.size());
1189 Res = concatenate(Builder, CV, VS, Op->getName());