Lines Matching defs:Lane
1217 for (unsigned Lane : seq<unsigned>(VL.size())) {
1218 if (isa<PoisonValue>(VL[Lane]))
1220 if (cast<Instruction>(VL[Lane])->getOpcode() == Opcode1)
1221 OpcodeMask.set(Lane * ScalarTyNumElements,
1222 Lane * ScalarTyNumElements + ScalarTyNumElements);
2061 /// \returns the operand data at \p OpIdx and \p Lane.
2062 OperandData &getData(unsigned OpIdx, unsigned Lane) {
2063 return OpsVec[OpIdx][Lane];
2066 /// \returns the operand data at \p OpIdx and \p Lane. Const version.
2067 const OperandData &getData(unsigned OpIdx, unsigned Lane) const {
2068 return OpsVec[OpIdx][Lane];
2075 for (unsigned Lane = 0, NumLanes = getNumLanes(); Lane != NumLanes;
2076 ++Lane)
2077 OpsVec[OpIdx][Lane].IsUsed = false;
2081 void swap(unsigned OpIdx1, unsigned OpIdx2, unsigned Lane) {
2082 std::swap(OpsVec[OpIdx1][Lane], OpsVec[OpIdx2][Lane]);
2085 /// \param Lane lane of the operands under analysis.
2086 /// \param OpIdx operand index in \p Lane lane we're looking the best
2094 int getSplatScore(unsigned Lane, unsigned OpIdx, unsigned Idx,
2096 Value *IdxLaneV = getData(Idx, Lane).V;
2097 if (!isa<Instruction>(IdxLaneV) || IdxLaneV == getData(OpIdx, Lane).V ||
2102 if (Ln == Lane)
2113 Value *OpIdxLaneV = getData(OpIdx, Lane).V;
2128 /// \param Lane lane of the operands under analysis.
2129 /// \param OpIdx operand index in \p Lane lane we're looking the best
2134 int getExternalUseScore(unsigned Lane, unsigned OpIdx, unsigned Idx) const {
2135 Value *IdxLaneV = getData(Idx, Lane).V;
2136 Value *OpIdxLaneV = getData(OpIdx, Lane).V;
2164 int Lane, unsigned OpIdx, unsigned Idx,
2174 int SplatScore = getSplatScore(Lane, OpIdx, Idx, UsedLanes);
2186 Score += getExternalUseScore(Lane, OpIdx, Idx);
2195 /// The key - {Operand Index, Lane}.
2201 // Search all operands in Ops[*][Lane] for the one that matches best
2205 getBestOperand(unsigned OpIdx, int Lane, int LastLane,
2219 // The linearized opcode of the operand at OpIdx, Lane.
2220 bool OpIdxAPO = getData(OpIdx, Lane).APO;
2230 BestScoresPerLanes.try_emplace(std::make_pair(OpIdx, Lane), 0)
2241 // Get the operand at Idx and Lane.
2242 OperandData &OpData = getData(Idx, Lane);
2260 bool LeftToRight = Lane > LastLane;
2263 int Score = getLookAheadScore(OpLeft, OpRight, MainAltOps, Lane,
2270 BestScoresPerLanes[std::make_pair(OpIdx, Lane)] = Score;
2280 BestScoresPerLanes[std::make_pair(OpIdx, Lane)] =
2292 BestScoresPerLanes[std::make_pair(OpIdx, Lane)] =
2304 getData(*BestOp.Idx, Lane).IsUsed = IsUsed;
2329 unsigned Lane = I - 1;
2331 getMaxNumOperandsThatCanBeReordered(Lane);
2338 HashMap[NumFreeOpsHash.Hash] = std::make_pair(1, Lane);
2344 HashMap[NumFreeOpsHash.Hash] = std::make_pair(1, Lane);
2348 HashMap.try_emplace(NumFreeOpsHash.Hash, 1, Lane);
2383 /// for \p Lane and the number of compatible instructions(with the same
2386 OperandsOrderData getMaxNumOperandsThatCanBeReordered(unsigned Lane) const {
2404 const OperandData &OpData = getData(OpIdx, Lane);
2452 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
2453 assert((isa<Instruction>(VL[Lane]) || isa<PoisonValue>(VL[Lane])) &&
2457 // opcode of VL[Lane] and whether the operand at OpIdx is the LHS or
2460 // is false. The RHS is true only if VL[Lane] is an inverse operation.
2465 if (isa<PoisonValue>(VL[Lane])) {
2468 OpsVec[OpIdx][Lane] = {EI->getVectorOperand(), true, false};
2473 OpsVec[OpIdx][Lane] = {EV->getAggregateOperand(), true, false};
2477 OpsVec[OpIdx][Lane] = {
2482 bool IsInverseOperation = !isCommutative(cast<Instruction>(VL[Lane]));
2484 OpsVec[OpIdx][Lane] = {cast<Instruction>(VL[Lane])->getOperand(OpIdx),
2496 /// \returns the operand value at \p OpIdx and \p Lane.
2497 Value *getValue(unsigned OpIdx, unsigned Lane) const {
2498 return getData(OpIdx, Lane).V;
2510 bool shouldBroadcast(Value *Op, unsigned OpIdx, unsigned Lane) {
2511 assert(Op == getValue(OpIdx, Lane) &&
2512 "Op is expected to be getValue(OpIdx, Lane).");
2516 bool OpAPO = getData(OpIdx, Lane).APO;
2520 if (Ln == Lane)
2522 // This is set to true if we found a candidate for broadcast at Lane.
2528 Value *OpILane = getValue(OpI, Lane);
2567 /// than \p Lane, compatible with the operand \p Op.
2568 bool canBeVectorized(Instruction *Op, unsigned OpIdx, unsigned Lane) const {
2569 assert(Op == getValue(OpIdx, Lane) &&
2570 "Op is expected to be getValue(OpIdx, Lane).");
2571 bool OpAPO = getData(OpIdx, Lane).APO;
2573 if (Ln == Lane)
2605 for (unsigned Lane = 0, Lanes = getNumLanes(); Lane != Lanes; ++Lane)
2606 OpVL[Lane] = OpsVec[OpIdx][Lane].V;
2611 // The original operands are in OrigOps[OpIdx][Lane].
2612 // The reordered operands are returned in 'SortedOps[OpIdx][Lane]'.
2626 // Lane 0 : A[0] = B[0] + C[0] // Visited 3rd
2627 // Lane 1 : A[1] = C[1] - B[1] // Visited 1st
2628 // Lane 2 : A[2] = B[2] + C[2] // Visited 2nd
2629 // Lane 3 : A[3] = C[3] - B[3] // Visited 4th
2630 // we will start at Lane 1, since the operands of the subtraction cannot
2632 // fashion. That is, Lanes 2, then Lane 0, and finally Lane 3.
2712 int Lane = FirstLane + Direction * Distance;
2713 if (Lane < 0 || Lane >= (int)NumLanes)
2715 UsedLanes.set(Lane);
2716 int LastLane = Lane - Direction;
2721 // Search for the operand that matches SortedOps[OpIdx][Lane-1].
2723 getBestOperand(OpIdx, Lane, LastLane, ReorderingModes,
2731 swap(OpIdx, *BestIdx, Lane);
2738 OperandData &AltOp = getData(OpIdx, Lane);
3807 : Scalar(S), User(U), E(E), Lane(L) {}
3819 int Lane;
4175 int Lane = std::distance(TE->Scalars.begin(),
4177 assert(Lane >= 0 && "Lane not set");
4192 if (auto *I = dyn_cast<Instruction>(TE->getOperand(OpIdx)[Lane]))
6567 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
6568 Value *Scalar = Entry->Scalars[Lane];
6645 for (unsigned Lane : seq<unsigned>(0, TE->Scalars.size())) {
6646 Value *V = TE->Scalars[Lane];
6672 if (StoresVec.size() > Lane)
12501 ScalarUserAndIdx.emplace_back(EU.Scalar, EU.User, EU.Lane);
12591 Mask[InIdx] = EU.Lane;
12613 VecTy, EU.Lane);
12617 EU.Lane, EU.Scalar, ScalarUserAndIdx);
16458 Value *Lane = Builder.getInt32(ExternalUse.Lane);
16512 Ex = Builder.CreateExtractElement(Vec, Lane);
16521 ExternalUse.Lane * VecTyNumElements);
16523 Ex = Builder.CreateExtractElement(Vec, Lane);
16670 Mask[Idx] = ExternalUse.Lane;
16828 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
16829 Value *Scalar = Entry->Scalars[Lane];
20308 // Lane[0] = <2 x Ty> <a, e>
20309 // Lane[1] = <2 x Ty> <b, f>
20310 // Lane[2] = <2 x Ty> <c, g>
20311 // Lane[3] = <2 x Ty> <d, h>
20312 // result[0] = reduce add Lane[0]
20313 // result[1] = reduce add Lane[1]
20314 // result[2] = reduce add Lane[2]
20315 // result[3] = reduce add Lane[3]
20318 Value *Lane = Builder.CreateShuffleVector(VectorizedRoot, Mask);
20321 emitReduction(Lane, Builder, TTI, RdxRootInst->getType()), I);