Lines Matching +full:test +full:- +full:part1
1 //===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
30 static cl::opt<unsigned> HvxWidenThreshold("hexagon-hvx-widen",
40 // For a float scalar type, return (exp-bits, exp-bias, fraction-bits) in getIEEEProperties()
70 // The idea is to rely on target-independent operations as much in initializeHVXLowering()
108 // Note: v16i1 -> i16 is handled in type legalization instead of op in initializeHVXLowering()
141 // Custom-lower BUILD_VECTOR. The standard (target-independent) in initializeHVXLowering()
172 // Custom-lower BUILD_VECTOR. The standard (target-independent) in initializeHVXLowering()
176 // Make concat-vectors custom to handle concats of more than 2 vectors. in initializeHVXLowering()
230 // Make concat-vectors custom to handle concats of more than 2 vectors. in initializeHVXLowering()
270 // Custom-lower BUILD_VECTOR for vector pairs. The standard (target- in initializeHVXLowering()
274 // Make concat-vectors custom to handle concats of more than 2 vectors. in initializeHVXLowering()
277 // Custom-lower these operations for pairs. Expand them into a concat in initializeHVXLowering()
335 setOperationAction(ISD::MULHS, WordV, Custom); // -> _LOHI in initializeHVXLowering()
336 setOperationAction(ISD::MULHU, WordV, Custom); // -> _LOHI in initializeHVXLowering()
371 // v64i8 -> v64i1 (single) in initializeHVXLowering()
372 // v64i16 -> v64i1 (pair) in initializeHVXLowering()
519 assert((NumElem % 2) == 0 && "Expecting even-sized vector type"); in typeSplit()
743 SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32); in getIndexInWord32()
767 ByteMask.push_back(-1); in getByteShuffle()
860 // should be of a sub-vector type of the source vector type. in buildHvxVectorReg()
866 SrcIdx.push_back(-1); in buildHvxVectorReg()
879 int I = C->getSExtValue(); in buildHvxVectorReg()
912 if (!Used.test(I)) in buildHvxVectorReg()
952 // performed post-loop to position the last element. in buildHvxVectorReg()
1002 // uses. The scaling down will pick every 2nd or 4th (every Scale-th in createHvxPrefixPred()
1068 SDValue S4 = DAG.getConstant(HwLen-4, dl, MVT::i32); in createHvxPrefixPred()
1091 return !N->isZero(); in buildHvxVectorPred()
1096 return N->isZero(); in buildHvxVectorPred()
1121 // Find the first non-undef value in this group. in buildHvxVectorPred()
1167 // of the desired element in the 32-bit word. in extractHvxElementReg()
1207 {ByteIdxV, DAG.getConstant(-4, dl, MVT::i32)}); in insertHvxElementReg()
1220 // If this is not inserting a 32-bit word, convert it into such a thing. in insertHvxElementReg()
1227 // 2. Treating the extracted word as a 32-bit vector, insert the given in insertHvxElementReg()
1234 // 3. Insert the 32-bit word back into the original vector. in insertHvxElementReg()
1259 unsigned Idx = IdxV.getNode()->getAsZExtVal(); in extractHvxSubvectorReg()
1268 Idx -= VecTy.getVectorNumElements() / 2; in extractHvxSubvectorReg()
1302 unsigned Idx = IdxV.getNode()->getAsZExtVal(); in extractHvxSubvectorPred()
1332 // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the in extractHvxSubvectorPred()
1337 // 8-byte groups as many times as necessary. in extractHvxSubvectorPred()
1348 // Combine the two low words from ShuffV into a v8i8, and byte-compare in extractHvxSubvectorPred()
1383 unsigned Idx = CN->getZExtValue(); in insertHvxSubvectorReg()
1408 if (!IdxN || !IdxN->isZero()) { in insertHvxSubvectorReg()
1414 // would be by HwLen-Idx, but if two words are inserted, it will need to be in insertHvxSubvectorReg()
1415 // by (HwLen-4)-Idx. in insertHvxSubvectorReg()
1428 RolBase = HwLen-4; in insertHvxSubvectorReg()
1431 if (RolBase != 4 || !IdxN || !IdxN->isZero()) { in insertHvxSubvectorReg()
1468 if (!IdxN || !IdxN->isZero()) { in insertHvxSubvectorPred()
1483 if (!IdxN || !IdxN->isZero()) { in insertHvxSubvectorPred()
1494 // Sign- and any-extending of a vector predicate to a vector register is in extendHvxVectorPred()
1495 // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and in extendHvxVectorPred()
1511 // Given a predicate register VecQ, transfer bits VecQ[0..HwLen-1] in compressHvxPred()
1512 // (i.e. the entire predicate register) to bits [0..HwLen-1] of a in compressHvxPred()
1666 // Vector concatenation of two integer (non-bool) vectors does not need in LowerHvxConcatVectors()
1667 // special lowering. Custom-lower concats of bool vectors and expand in LowerHvxConcatVectors()
1675 // Expand the other cases into a build-vector. in LowerHvxConcatVectors()
1677 for (SDValue V : Op.getNode()->ops()) in LowerHvxConcatVectors()
1681 // all operations are expected to be type-legalized, and i16 is not in LowerHvxConcatVectors()
1683 // type, sign-extend it to a valid one. in LowerHvxConcatVectors()
1727 ArrayRef<SDUse> U(Op.getNode()->ops()); in LowerHvxConcatVectors()
1744 for (SDValue V : Op.getNode()->op_values()) { in LowerHvxConcatVectors()
1755 Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]); in LowerHvxConcatVectors()
1804 unsigned Idx = IdxV.getNode()->getAsZExtVal(); in LowerHvxExtractSubvector()
1835 // Lower any-extends of boolean vectors to sign-extends, since they in LowerHvxAnyExt()
1836 // translate directly to Q2V. Zero-extending could also be done equally in LowerHvxAnyExt()
1838 // For all other vectors, use zero-extend. in LowerHvxAnyExt()
1870 // cttz(x) = bitwidth(x) - ctlz(~x & (x-1)) in LowerHvxCttz()
1885 DAG.getConstant(-1, dl, MVT::i32)); in LowerHvxCttz()
1939 // Legal on HVX v62+, but lower it here because patterns can't handle multi- in LowerHvxMulLoHi()
2000 // Handle bitcast from i128 -> v128i1 and i64 -> v64i1. in LowerHvxBitcast()
2038 // Sign- and zero-extends are legal. in LowerHvxExtend()
2098 SDValue Mask = DAG.getConstant(ElemWidth - 1, dl, MVT::i32); in LowerHvxFunnelShift()
2109 SDValue Part1 = in LowerHvxFunnelShift() local
2113 SDValue Or = DAG.getNode(ISD::OR, dl, InpTy, {Part1, Part2}); in LowerHvxFunnelShift()
2120 InpTy, dl, DAG.getConstant(ElemWidth - 1, dl, ElemTy)); in LowerHvxFunnelShift()
2131 SmallVector<SDValue> Ops(Op->ops().begin(), Op->ops().end()); in LowerHvxIntrinsic()
2150 return Swap(DAG.getNode(HexagonISD::SMUL_LOHI, dl, Op->getVTList(), in LowerHvxIntrinsic()
2154 return Swap(DAG.getNode(HexagonISD::UMUL_LOHI, dl, Op->getVTList(), in LowerHvxIntrinsic()
2158 return Swap(DAG.getNode(HexagonISD::USMUL_LOHI, dl, Op->getVTList(), in LowerHvxIntrinsic()
2172 SDValue Mask = MaskN->getMask(); in LowerHvxMaskedOp()
2173 SDValue Chain = MaskN->getChain(); in LowerHvxMaskedOp()
2174 SDValue Base = MaskN->getBasePtr(); in LowerHvxMaskedOp()
2175 auto *MemOp = MF.getMachineMemOperand(MaskN->getMemOperand(), 0, HwLen); in LowerHvxMaskedOp()
2177 unsigned Opc = Op->getOpcode(); in LowerHvxMaskedOp()
2183 SDValue Thru = cast<MaskedLoadSDNode>(MaskN)->getPassThru(); in LowerHvxMaskedOp()
2195 SDValue Value = cast<MaskedStoreSDNode>(MaskN)->getValue(); in LowerHvxMaskedOp()
2198 if (MaskN->getAlign().value() % HwLen == 0) { in LowerHvxMaskedOp()
2242 assert(Op->getOpcode() == ISD::FP_EXTEND); in LowerHvxFpExtend()
2267 {HiVec, LoVec, DAG.getConstant(-4, dl, MVT::i32)}, DAG); in LowerHvxFpExtend()
2314 // i8, i16 -> f16 is legal. in LowerHvxIntToFp()
2419 DAG.getNode(ISD::XOR, dl, ResTy, {A, DAG.getConstant(-1, dl, ResTy)}); in emitHvxAddWithOverflow()
2432 // tie-break by rounding halves to even integer. in emitHvxShiftRightRnd()
2439 // uint tmp0 = inp + ((1 << (Amt-1)) - 1); in emitHvxShiftRightRnd()
2443 // uint tmp1 = inp >> (Amt-1); // tmp1 == tmp2 iff in emitHvxShiftRightRnd()
2444 // uint tmp2 = tmp0 >> (Amt-1); // the Amt-1 lower bits were all 0 in emitHvxShiftRightRnd()
2454 SDValue LowBits = DAG.getConstant((1ull << (Amt - 1)) - 1, dl, IntTy); in emitHvxShiftRightRnd()
2463 SDValue AmtM1 = DAG.getConstant(Amt - 1, dl, IntTy); in emitHvxShiftRightRnd()
2495 // The final additions need to make sure to properly maintain any carry- in emitHvxMulHsV60()
2500 // -------------- in emitHvxMulHsV60()
2513 // We need to add T0+T2, recording the carry-out, which will be 1<<16 in emitHvxMulHsV60()
2515 // P1 = interleaved even/odd 32-bit (unsigned) sums of 16-bit halves in emitHvxMulHsV60()
2517 // P2 = interleaved even/odd 32-bit (signed) sums of 16-bit halves in emitHvxMulHsV60()
2519 // T3 = full-precision(T0+T2) >> 16 in emitHvxMulHsV60()
2520 // The low halves are added-unsigned, the high ones are added-signed. in emitHvxMulHsV60()
2549 // Do halfword-wise multiplications for unsigned*unsigned product, then in emitHvxMulLoHiV60()
2597 // mulhus(A.uw,B.w) = mulhu(A.uw,B.uw) - (A.w if B < 0) in emitHvxMulLoHiV60()
2666 // Rewrite conversion between integer and floating-point in such a way that in EqualizeFpIntConversion()
2668 // floating-point type, combined with additional integer-integer extensions in EqualizeFpIntConversion()
2670 // E.g. f32 -> i8 ==> f32 -> i32 -> i8 in EqualizeFpIntConversion()
2713 // // "bias+1". When the encoded exp is "all-1" (i.e. inf/nan), this will in ExpandHvxFpToInt()
2716 // // represented in bias-width bits (i.e. 011..1). in ExpandHvxFpToInt()
2717 // int32_t expm1 = (inp << 1) - 0x80000000; in ExpandHvxFpToInt()
2731 // // amount as a 6-bit signed value (so 33 is same as -31, i.e. shift in ExpandHvxFpToInt()
2733 // int32_t rsh = min(31 - (expm1 + 1), 32); in ExpandHvxFpToInt()
2741 // frac = -frac; // in ExpandHvxFpToInt()
2784 assert((1ull << (ExpWidth - 1)) == (1 + ExpBias)); in ExpandHvxFpToInt()
2789 SDValue M80 = DAG.getConstant(1ull << (ElemWidth - 1), dl, ResTy); in ExpandHvxFpToInt()
2790 SDValue M7F = DAG.getConstant((1ull << (ElemWidth - 1)) - 1, dl, ResTy); in ExpandHvxFpToInt()
2794 SDValue MNE = DAG.getConstant(ElemWidth - ExpWidth, dl, ResTy); in ExpandHvxFpToInt()
2801 SDValue MN2 = DAG.getConstant(ElemWidth - 2, dl, ResTy); in ExpandHvxFpToInt()
2845 // uint32_t frac1 = frac0 + ((1 << 8) - 1); in ExpandHvxIntToFp()
2853 // int32_t exp = 32 - norm_left + renorm + 127; in ExpandHvxIntToFp()
2878 SDValue M80 = DAG.getConstant(1ull << (ElemWidth - 1), dl, InpTy); in ExpandHvxIntToFp()
2936 MVT Ty = typeSplit(N->getVT().getSimpleVT()).first; in SplitVectorOp()
2941 for (SDValue A : Op.getNode()->ops()) { in SplitVectorOp()
2968 MVT MemTy = MemN->getMemoryVT().getSimpleVT(); in SplitHvxMemOp()
2975 SDValue Chain = MemN->getChain(); in SplitHvxMemOp()
2976 SDValue Base0 = MemN->getBasePtr(); in SplitHvxMemOp()
2979 unsigned MemOpc = MemN->getOpcode(); in SplitHvxMemOp()
2982 if (MachineMemOperand *MMO = MemN->getMemOperand()) { in SplitHvxMemOp()
2992 assert(cast<LoadSDNode>(Op)->isUnindexed()); in SplitHvxMemOp()
3001 assert(cast<StoreSDNode>(Op)->isUnindexed()); in SplitHvxMemOp()
3002 VectorPair Vals = opSplit(cast<StoreSDNode>(Op)->getValue(), dl, DAG); in SplitHvxMemOp()
3011 assert(MaskN->isUnindexed()); in SplitHvxMemOp()
3012 VectorPair Masks = opSplit(MaskN->getMask(), dl, DAG); in SplitHvxMemOp()
3017 opSplit(cast<MaskedLoadSDNode>(Op)->getPassThru(), dl, DAG); in SplitHvxMemOp()
3032 VectorPair Vals = opSplit(cast<MaskedStoreSDNode>(Op)->getValue(), dl, DAG); in SplitHvxMemOp()
3042 std::string Name = "Unexpected operation: " + Op->getOperationName(&DAG); in SplitHvxMemOp()
3050 assert(LoadN->isUnindexed() && "Not widening indexed loads yet"); in WidenHvxLoad()
3051 assert(LoadN->getMemoryVT().getVectorElementType() != MVT::i1 && in WidenHvxLoad()
3054 SDValue Chain = LoadN->getChain(); in WidenHvxLoad()
3055 SDValue Base = LoadN->getBasePtr(); in WidenHvxLoad()
3069 auto *MemOp = MF.getMachineMemOperand(LoadN->getMemOperand(), 0, HwLen); in WidenHvxLoad()
3082 assert(StoreN->isUnindexed() && "Not widening indexed stores yet"); in WidenHvxStore()
3083 assert(StoreN->getMemoryVT().getVectorElementType() != MVT::i1 && in WidenHvxStore()
3086 SDValue Chain = StoreN->getChain(); in WidenHvxStore()
3087 SDValue Base = StoreN->getBasePtr(); in WidenHvxStore()
3090 SDValue Value = opCastElem(StoreN->getValue(), MVT::i8, DAG); in WidenHvxStore()
3107 auto *MemOp = MF.getMachineMemOperand(StoreN->getMemOperand(), 0, HwLen); in WidenHvxStore()
3141 llvm::any_of(Op.getNode()->ops(), [this] (SDValue V) { in LowerHvxOperation()
3192 // In general, sign- and zero-extends can't be split and still in LowerHvxOperation()
3255 // E.g. i8 -> i16 remains unchanged, but i8 -> i32 ==> i8 -> i16 -> i32. in ExpandHvxResizeIntoSteps()
3270 llvm_unreachable("ISD:: ops will be auto-folded"); in ExpandHvxResizeIntoSteps()
3360 unsigned Opc = N->getOpcode(); in LowerHvxOperationWrapper()
3363 if (N->getNumOperands() > 0) in LowerHvxOperationWrapper()
3383 if (shouldWidenToHvx(ty(cast<StoreSDNode>(N)->getValue()), DAG)) { in LowerHvxOperationWrapper()
3392 assert(S->getOpcode() == ISD::MERGE_VALUES); in LowerHvxOperationWrapper()
3398 if (isHvxPairTy(ty(Op->getOperand(1)))) { // Stored value in LowerHvxOperationWrapper()
3426 unsigned Opc = N->getOpcode(); in ReplaceHvxNodeResults()
3429 if (N->getNumOperands() > 0) in ReplaceHvxNodeResults()
3451 assert(Load->getOpcode() == ISD::MERGE_VALUES); in ReplaceHvxNodeResults()
3484 // Simplify V:v2NiB --(bitcast)--> vNi2B --(truncate)--> vNiB in combineTruncateBeforeLegal()
3485 // to extract-subvector (shuffle V, pick even, pick odd) in combineTruncateBeforeLegal()
3556 ArrayRef<int> Mask = SV->getMask(); in combineConcatVectorsBeforeLegal()
3560 if (M == -1) { in combineConcatVectorsBeforeLegal()
3566 M -= InpLen; in combineConcatVectorsBeforeLegal()
3592 SmallVector<SDValue, 4> Ops(N->ops().begin(), N->ops().end()); in PerformHvxDAGCombine()
3604 // (vselect (xor x, qtrue), v0, v1) -> (vselect x, v1, v0) in PerformHvxDAGCombine()
3606 if (Cond->getOpcode() == ISD::XOR) { in PerformHvxDAGCombine()
3608 if (C1->getOpcode() == HexagonISD::QTRUE) in PerformHvxDAGCombine()
3616 return C->isZero() ? DAG.getNode(HexagonISD::QFALSE, dl, ty(Op)) in PerformHvxDAGCombine()
3623 DAG.getConstant(-1, dl, MVT::i32)); in PerformHvxDAGCombine()
3678 if (llvm::any_of(N->values(), IsHvxTy) || llvm::any_of(N->ops(), IsHvxOp)) in isHvxOperation()
3689 for (int i = 0, e = N->getNumValues(); i != e; ++i) { in isHvxOperation()
3693 return llvm::any_of(N->ops(), IsWidenedToHvx); in isHvxOperation()