Lines Matching full:part
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
309 // VPReplicateRecipe may be cloned as part of an existing VPlan-to-VPlan
382 Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) {
387 Value *A = State.get(getOperand(0), Part, OnlyFirstLaneUsed);
388 Value *B = State.get(getOperand(1), Part, OnlyFirstLaneUsed);
398 Value *A = State.get(getOperand(0), Part);
403 Value *A = State.get(getOperand(0), Part, OnlyFirstLaneUsed);
404 Value *B = State.get(getOperand(1), Part, OnlyFirstLaneUsed);
408 Value *Cond = State.get(getOperand(0), Part);
409 Value *Op1 = State.get(getOperand(1), Part);
410 Value *Op2 = State.get(getOperand(2), Part);
415 Value *VIVElem0 = State.get(getOperand(0), VPIteration(Part, 0));
417 Value *ScalarTC = State.get(getOperand(1), VPIteration(Part, 0));
419 // If this part of the active lane mask is scalar, generate the CMP directly
444 // For the first part, use the recurrence phi (v1), otherwise v2.
446 Value *PartMinus1 = Part == 0 ? V1 : State.get(getOperand(1), Part - 1);
449 Value *V2 = State.get(getOperand(1), Part);
453 if (Part != 0)
481 assert(Part == 0 && "No unrolling expected for predicated vectorization.");
491 if (Part == 0)
495 // elements) times the unroll part.
496 Value *Step = createStepForVF(Builder, IV->getType(), State.VF, Part);
501 if (Part != 0)
504 Value *Cond = State.get(getOperand(0), VPIteration(Part, 0));
522 if (Part != 0)
525 Value *IV = State.get(getOperand(0), Part, /*IsScalar*/ true);
526 Value *TC = State.get(getOperand(1), Part, /*IsScalar*/ true);
546 if (Part != 0)
561 for (unsigned Part = 0; Part < State.UF; ++Part)
562 RdxParts[Part] = State.get(LoopExitingDef, Part, PhiR->isInLoop());
570 for (unsigned Part = 0; Part < State.UF; ++Part)
571 RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
585 for (unsigned Part = 1; Part < State.UF; ++Part) {
586 Value *RdxPart = RdxParts[Part];
621 if (Part != 0)
645 Value *A = State.get(getOperand(0), Part);
646 Value *B = State.get(getOperand(1), Part);
652 Value *Ptr = State.get(getOperand(0), Part, /* IsScalar */ true);
653 Value *Addend = State.get(getOperand(1), Part, /* IsScalar */ true);
657 if (Part != 0)
660 State.get(getOperand(0), Part, /* IsScalar */ true);
662 State.get(getOperand(1), Part, /* IsScalar */ true);
716 for (unsigned Part = 0; Part < State.UF; ++Part) {
720 Value *GeneratedValue = generatePerLane(State, VPIteration(Part, Lane));
722 State.set(this, GeneratedValue, VPIteration(Part, Lane));
727 if (Part != 0 && OnlyFirstPartUsed && hasResult()) {
729 State.set(this, Part0, Part,
734 Value *GeneratedValue = generatePerPart(State, Part);
742 State.set(this, GeneratedValue, Part,
834 O << "VF * Part +";
876 for (unsigned Part = 0; Part < State.UF; ++Part) {
893 // from the start of the respective part when interleaving.
895 Arg = State.get(I.value(), VPIteration(Part, 0));
897 Arg = State.get(I.value(), Part);
928 State.set(this, V, Part);
986 for (unsigned Part = 0; Part < State.UF; ++Part) {
987 Value *Cond = InvarCond ? InvarCond : State.get(getCond(), Part);
988 Value *Op0 = State.get(getOperand(1), Part);
989 Value *Op1 = State.get(getOperand(2), Part);
991 State.set(this, Sel, Part);
1076 for (unsigned Part = 0; Part < State.UF; ++Part) {
1079 Ops.push_back(State.get(VPOp, Part));
1087 State.set(this, V, Part);
1094 for (unsigned Part = 0; Part < State.UF; ++Part) {
1095 Value *Op = State.get(getOperand(0), Part);
1098 State.set(this, Freeze, Part);
1106 for (unsigned Part = 0; Part < State.UF; ++Part) {
1107 Value *A = State.get(getOperand(0), Part);
1108 Value *B = State.get(getOperand(1), Part);
1119 State.set(this, C, Part);
1135 for (unsigned Part = 0; Part < State.UF; ++Part) {
1137 State.VF) == State.get(this, Part)->getType() &&
1161 for (unsigned Part = 0; Part < State.UF; ++Part) {
1162 if (Part > 0 && Op->isLiveIn()) {
1164 State.set(this, State.get(this, 0), Part);
1167 Value *A = State.get(Op, Part);
1169 State.set(this, Cast, Part);
1334 for (unsigned Part = 0; Part < State.UF; ++Part) {
1335 State.set(this, LastInduction, Part);
1449 StartPart = State.Instance->Part;
1454 for (unsigned Part = StartPart; Part < EndPart; ++Part) {
1455 Value *StartIdx0 = createStepForVF(Builder, IntStepTy, State.VF, Part);
1464 State.set(this, Add, Part);
1483 State.set(this, Add, VPIteration(Part, Lane));
1527 for (unsigned Part = 0; Part < State.UF; ++Part) {
1529 State.set(this, EntryPart, Part);
1535 // to produce a scalar GEP for each unroll part. Thus, the GEP we
1540 for (unsigned Part = 0; Part < State.UF; ++Part) {
1545 : State.get(getOperand(0), Part);
1555 Indices.push_back(State.get(Operand, Part));
1564 State.set(this, NewGEP, Part);
1589 for (unsigned Part = 0; Part < State.UF; ++Part) {
1590 // Calculate the pointer for the specific unroll-part.
1596 Type *IndexTy = State.VF.isScalable() && (IsReverse || Part > 0)
1607 // NumElt = -Part * RunTimeVF
1609 ConstantInt::get(IndexTy, -(int64_t)Part), RunTimeVF);
1616 Value *Increment = createStepForVF(Builder, IndexTy, State.VF, Part);
1620 State.set(this, PartPtr, Part, /*IsScalar*/ true);
1658 for (unsigned Part = 0; Part < State.UF; ++Part) {
1661 Value *In0 = State.get(getIncomingValue(In), Part, OnlyFirstLaneUsed);
1663 Entry[Part] = In0; // Initialize with the first incoming value.
1667 Value *Cond = State.get(getMask(In), Part, OnlyFirstLaneUsed);
1668 Entry[Part] =
1669 State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi");
1673 for (unsigned Part = 0; Part < State.UF; ++Part)
1674 State.set(this, Entry[Part], Part, OnlyFirstLaneUsed);
1708 for (unsigned Part = 0; Part < State.UF; ++Part) {
1709 Value *NewVecOp = State.get(getVecOp(), Part);
1711 Value *NewCond = State.get(Cond, Part, State.VF.isScalar());
1735 PrevInChain = State.get(getChainOp(), Part, /*IsScalar*/ true);
1746 State.set(this, NextInChain, Part, /*IsScalar*/ true);
1889 Value *VPScalarCastRecipe ::generate(VPTransformState &State, unsigned Part) {
1897 Value *Op = State.get(getOperand(0), VPIteration(Part, 0));
1907 for (unsigned Part = 0; Part != State.UF; ++Part) {
1911 if (Part > 0 && IsUniformAcrossVFsAndUFs)
1914 Res = generate(State, Part);
1915 State.set(this, Res, VPIteration(Part, 0));
1933 unsigned Part = State.Instance->Part;
1939 ConditionBit = State.get(BlockInMask, Part);
1972 unsigned Part = State.Instance->Part;
1973 if (State.hasVectorValue(getOperand(0), Part)) {
1974 Value *VectorValue = State.get(getOperand(0), Part);
1979 if (State.hasVectorValue(this, Part))
1980 State.reset(this, VPhi, Part);
1982 State.set(this, VPhi, Part);
1985 State.reset(getOperand(0), VPhi, Part);
2167 for (unsigned Part = 0; Part < State.UF; Part++) {
2168 Value *AddrPart = State.get(Addr, VPIteration(Part, 0));
2195 unsigned Part, Value *MaskForGaps) -> Value * {
2200 auto *BlockInMaskPart = State.get(BlockInMask, Part);
2212 Value *BlockInMaskPart = State.get(BlockInMask, Part);
2232 // For each unroll part, create a wide load for the group.
2234 for (unsigned Part = 0; Part < State.UF; Part++) {
2237 Value *GroupMask = CreateGroupMask(Part, MaskForGaps);
2238 NewLoad = State.Builder.CreateMaskedLoad(VecTy, AddrParts[Part],
2243 VecTy, AddrParts[Part], Group->getAlign(), "wide.vec");
2254 for (unsigned Part = 0; Part < State.UF; ++Part) {
2258 Intrinsic::vector_deinterleave2, VecTy, NewLoads[Part],
2279 State.set(VPDefs[J], StridedVec, Part);
2299 for (unsigned Part = 0; Part < State.UF; Part++) {
2301 NewLoads[Part], StrideMask, "strided.vec");
2314 State.set(VPDefs[J], StridedVec, Part);
2330 for (unsigned Part = 0; Part < State.UF; Part++) {
2346 Value *StoredVec = State.get(StoredValues[StoredIdx], Part);
2365 Value *GroupMask = CreateGroupMask(Part, MaskForGaps);
2367 IVec, AddrParts[Part], Group->getAlign(), GroupMask);
2369 NewStoreInstr = State.Builder.CreateAlignedStore(IVec, AddrParts[Part],
2415 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
2416 State.set(this, EntryPart, Part, /*IsScalar*/ true);
2473 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
2474 State.set(this, Res, {Part, 0});
2494 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
2495 Value *VStep = createStepForVF(Builder, STy, VF, Part);
2502 State.set(this, CanonicalVectorIV, Part);
2574 for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
2577 State.set(this, EntryPart, Part, IsInLoop);
2608 for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
2609 Value *EntryPart = State.get(this, Part, IsInLoop);
2611 // first unroll part.
2612 Value *StartVal = (Part == 0) ? StartV : Iden;
2663 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
2664 Value *StartMask = State.get(getOperand(0), Part);
2669 State.set(this, EntryPart, Part);