Lines Matching full:vs
112 const VectorSplit &VS, ValueVector *cachePtr = nullptr);
118 unsigned size() const { return VS.NumFragments; }
124 VectorSplit VS;
191 VectorSplit VS;
219 /// described in @p VS.
221 const VectorSplit &VS, Twine Name) {
222 unsigned NumElements = VS.VecTy->getNumElements();
226 if (VS.NumPacked > 1) {
230 for (unsigned I = 0; I < VS.NumPacked; ++I)
238 Value *Res = PoisonValue::get(VS.VecTy);
239 for (unsigned I = 0; I < VS.NumFragments; ++I) {
242 unsigned NumPacked = VS.NumPacked;
243 if (I == VS.NumFragments - 1 && VS.RemainderTy) {
244 if (auto *RemVecTy = dyn_cast<FixedVectorType>(VS.RemainderTy))
251 Res = Builder.CreateInsertElement(Res, Fragment, I * VS.NumPacked,
259 InsertMask[I * VS.NumPacked + J] = NumElements + J;
263 InsertMask[I * VS.NumPacked + J] = I * VS.NumPacked + J;
304 Scatterer scatter(Instruction *Point, Value *V, const VectorSplit &VS);
305 void gather(Instruction *Op, const ValueVector &CV, const VectorSplit &VS);
362 const VectorSplit &VS, ValueVector *cachePtr)
363 : BB(bb), BBI(bbi), V(v), VS(VS), CachePtr(cachePtr) {
366 Tmp.resize(VS.NumFragments, nullptr);
368 assert((CachePtr->empty() || VS.NumFragments == CachePtr->size() ||
371 if (VS.NumFragments > CachePtr->size())
372 CachePtr->resize(VS.NumFragments, nullptr);
387 CV[Frag] = Builder.CreateConstGEP1_32(VS.SplitTy, V, Frag,
392 Type *FragmentTy = VS.getFragmentType(Frag);
397 Mask.push_back(Frag * VS.NumPacked + J);
414 if (Frag * VS.NumPacked == J) {
419 if (VS.NumPacked == 1 && !CV[J]) {
426 CV[Frag] = Builder.CreateExtractElement(V, Frag * VS.NumPacked,
471 const VectorSplit &VS) {
477 return Scatterer(BB, BB->begin(), V, VS, &Scattered[{V, VS.SplitTy}]);
488 PoisonValue::get(V->getType()), VS);
493 BB, skipPastPhiNodesAndDbg(std::next(BasicBlock::iterator(VOp))), V, VS,
494 &Scattered[{V, VS.SplitTy}]);
498 return Scatterer(Point->getParent(), Point->getIterator(), V, VS);
506 const VectorSplit &VS) {
511 ValueVector &SV = Scattered[{Op, VS.SplitTy}];
608 std::optional<VectorSplit> VS = getVectorSplit(Ty);
609 if (!VS)
613 Layout.VS = *VS;
615 if (!DL.typeSizeEqualsStoreSize(VS->SplitTy) ||
616 (VS->RemainderTy && !DL.typeSizeEqualsStoreSize(VS->RemainderTy)))
619 Layout.SplitSize = DL.getTypeStoreSize(VS->SplitTy);
627 std::optional<VectorSplit> VS = getVectorSplit(I.getType());
628 if (!VS)
633 OpVS = VS;
636 if (!OpVS || VS->NumPacked != OpVS->NumPacked)
642 assert(Op.size() == VS->NumFragments && "Mismatched unary operation");
644 Res.resize(VS->NumFragments);
645 for (unsigned Frag = 0; Frag < VS->NumFragments; ++Frag)
647 gather(&I, Res, *VS);
655 std::optional<VectorSplit> VS = getVectorSplit(I.getType());
656 if (!VS)
661 OpVS = VS;
664 if (!OpVS || VS->NumPacked != OpVS->NumPacked)
671 assert(VOp0.size() == VS->NumFragments && "Mismatched binary operation");
672 assert(VOp1.size() == VS->NumFragments && "Mismatched binary operation");
674 Res.resize(VS->NumFragments);
675 for (unsigned Frag = 0; Frag < VS->NumFragments; ++Frag) {
680 gather(&I, Res, *VS);
689 std::optional<VectorSplit> VS;
691 VS = getVectorSplit(CallType->getContainedType(0));
693 VS = getVectorSplit(CallType);
694 if (!VS)
716 Tys.push_back(VS->SplitTy);
729 if (!CurrVS || CurrVS->NumPacked != VS->NumPacked)
741 assert(OpVecTy->getNumElements() == VS->VecTy->getNumElements());
743 if (!OpVS || OpVS->NumPacked != VS->NumPacked) {
766 ValueVector Res(VS->NumFragments);
774 for (unsigned I = 0; I < VS->NumFragments; ++I) {
775 bool IsRemainder = I == VS->NumFragments - 1 && VS->RemainderTy;
779 Tys[0] = VS->RemainderTy;
798 gather(&CI, Res, *VS);
803 std::optional<VectorSplit> VS = getVectorSplit(SI.getType());
804 if (!VS)
810 if (!CondVS || CondVS->NumPacked != VS->NumPacked) {
817 Scatterer VOp1 = scatter(&SI, SI.getOperand(1), *VS);
818 Scatterer VOp2 = scatter(&SI, SI.getOperand(2), *VS);
819 assert(VOp1.size() == VS->NumFragments && "Mismatched select");
820 assert(VOp2.size() == VS->NumFragments && "Mismatched select");
822 Res.resize(VS->NumFragments);
827 for (unsigned I = 0; I < VS->NumFragments; ++I) {
836 for (unsigned I = 0; I < VS->NumFragments; ++I) {
843 gather(&SI, Res, *VS);
864 std::optional<VectorSplit> VS = getVectorSplit(GEPI.getType());
865 if (!VS)
879 if (!OpVS || OpVS->NumPacked != VS->NumPacked) {
890 Res.resize(VS->NumFragments);
891 for (unsigned I = 0; I < VS->NumFragments; ++I) {
907 gather(&GEPI, Res, *VS);
1015 std::optional<VectorSplit> VS = getVectorSplit(IEI.getType());
1016 if (!VS)
1020 Scatterer Op0 = scatter(&IEI, IEI.getOperand(0), *VS);
1025 Res.resize(VS->NumFragments);
1029 unsigned Fragment = Idx / VS->NumPacked;
1030 for (unsigned I = 0; I < VS->NumFragments; ++I) {
1032 bool IsPacked = VS->NumPacked > 1;
1033 if (Fragment == VS->NumFragments - 1 && VS->RemainderTy &&
1034 !VS->RemainderTy->isVectorTy())
1038 Builder.CreateInsertElement(Op0[I], NewElt, Idx % VS->NumPacked);
1048 if (!ScalarizeVariableInsertExtract || VS->NumPacked > 1)
1051 for (unsigned I = 0; I < VS->NumFragments; ++I) {
1061 gather(&IEI, Res, *VS);
1083 std::optional<VectorSplit> VS = getVectorSplit(VecType);
1084 if (!VS)
1087 Scatterer Op0 = scatter(&EVI, Op, *VS);
1097 gather(&EVI, Res, *VS);
1102 std::optional<VectorSplit> VS = getVectorSplit(EEI.getOperand(0)->getType());
1103 if (!VS)
1107 Scatterer Op0 = scatter(&EEI, EEI.getOperand(0), *VS);
1112 unsigned Fragment = Idx / VS->NumPacked;
1114 bool IsPacked = VS->NumPacked > 1;
1115 if (Fragment == VS->NumFragments - 1 && VS->RemainderTy &&
1116 !VS->RemainderTy->isVectorTy())
1119 Res = Builder.CreateExtractElement(Res, Idx % VS->NumPacked);
1125 if (!ScalarizeVariableInsertExtract || VS->NumPacked > 1)
1128 Value *Res = PoisonValue::get(VS->VecTy->getElementType());
1129 for (unsigned I = 0; I < VS->NumFragments; ++I) {
1142 std::optional<VectorSplit> VS = getVectorSplit(SVI.getType());
1145 if (!VS || !VSOp || VS->NumPacked > 1 || VSOp->NumPacked > 1)
1151 Res.resize(VS->NumFragments);
1153 for (unsigned I = 0; I < VS->NumFragments; ++I) {
1156 Res[I] = PoisonValue::get(VS->VecTy->getElementType());
1162 gather(&SVI, Res, *VS);
1167 std::optional<VectorSplit> VS = getVectorSplit(PHI.getType());
1168 if (!VS)
1173 Res.resize(VS->NumFragments);
1176 for (unsigned I = 0; I < VS->NumFragments; ++I) {
1177 Res[I] = Builder.CreatePHI(VS->getFragmentType(I), NumOps,
1182 Scatterer Op = scatter(&PHI, PHI.getIncomingValue(I), *VS);
1184 for (unsigned J = 0; J < VS->NumFragments; ++J)
1187 gather(&PHI, Res, *VS);
1203 Scatterer Ptr = scatter(&LI, LI.getPointerOperand(), Layout->VS);
1205 Res.resize(Layout->VS.NumFragments);
1207 for (unsigned I = 0; I < Layout->VS.NumFragments; ++I) {
1208 Res[I] = Builder.CreateAlignedLoad(Layout->VS.getFragmentType(I), Ptr[I],
1212 gather(&LI, Res, Layout->VS);
1229 Scatterer VPtr = scatter(&SI, SI.getPointerOperand(), Layout->VS);
1230 Scatterer VVal = scatter(&SI, FullValue, Layout->VS);
1233 Stores.resize(Layout->VS.NumFragments);
1234 for (unsigned I = 0; I < Layout->VS.NumFragments; ++I) {
1274 VectorSplit VS = *getVectorSplit(Ty);
1275 assert(VS.NumFragments == CV.size());
1277 Res = concatenate(Builder, CV, VS, Op->getName());
1301 VectorSplit VS = *getVectorSplit(ElemTy);
1302 assert(VS.NumFragments == CV.size());
1305 concatenate(Builder, ElemCV[I], VS, Op->getName());