Lines Matching refs:CI
1653 static Value *upgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallBase &CI,
1655 Type *Ty = CI.getType();
1699 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
1700 CI.getArgOperand(2) };
1706 Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
1709 : Builder.CreateBitCast(CI.getArgOperand(1),
1711 return emitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
1714 static Value *upgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallBase &CI,
1716 Type *Ty = CI.getType();
1717 Value *Op0 = CI.getOperand(0);
1718 Value *Op1 = CI.getOperand(1);
1719 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1722 if (CI.arg_size() == 4) { // For masked intrinsics.
1723 Value *VecSrc = CI.getOperand(2);
1724 Value *Mask = CI.getOperand(3);
1730 static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallBase &CI,
1732 Type *Ty = CI.getType();
1733 Value *Src = CI.getArgOperand(0);
1734 Value *Amt = CI.getArgOperand(1);
1746 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1749 if (CI.arg_size() == 4) { // For masked intrinsics.
1750 Value *VecSrc = CI.getOperand(2);
1751 Value *Mask = CI.getOperand(3);
1757 static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallBase &CI, unsigned Imm,
1759 Type *Ty = CI.getType();
1760 Value *LHS = CI.getArgOperand(0);
1761 Value *RHS = CI.getArgOperand(1);
1796 static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallBase &CI,
1798 Type *Ty = CI.getType();
1799 Value *Op0 = CI.getArgOperand(0);
1800 Value *Op1 = CI.getArgOperand(1);
1801 Value *Amt = CI.getArgOperand(2);
1816 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1819 unsigned NumArgs = CI.arg_size();
1821 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
1822 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
1823 CI.getArgOperand(0);
1824 Value *Mask = CI.getOperand(NumArgs - 1);
1874 static Value *upgradeAbs(IRBuilder<> &Builder, CallBase &CI) {
1875 Type *Ty = CI.getType();
1876 Value *Op0 = CI.getArgOperand(0);
1877 Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty);
1879 if (CI.arg_size() == 3)
1880 Res = emitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
1884 static Value *upgradePMULDQ(IRBuilder<> &Builder, CallBase &CI, bool IsSigned) {
1885 Type *Ty = CI.getType();
1888 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
1889 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
1907 if (CI.arg_size() == 4)
1908 Res = emitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
1936 static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallBase &CI,
1938 Value *Op0 = CI.getArgOperand(0);
1959 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
1962 Value *Mask = CI.getArgOperand(CI.arg_size() - 1);
1968 static Value *upgradeX86MaskedShift(IRBuilder<> &Builder, CallBase &CI,
1970 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID);
1972 { CI.getArgOperand(0), CI.getArgOperand(1) });
1973 return emitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
1976 static Value *upgradeMaskedMove(IRBuilder<> &Builder, CallBase &CI) {
1977 Value* A = CI.getArgOperand(0);
1978 Value* B = CI.getArgOperand(1);
1979 Value* Src = CI.getArgOperand(2);
1980 Value* Mask = CI.getArgOperand(3);
1990 static Value *upgradeMaskToInt(IRBuilder<> &Builder, CallBase &CI) {
1991 Value* Op = CI.getArgOperand(0);
1992 Type* ReturnOp = CI.getType();
1993 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
2000 CallBase &CI, Value *&Rep) {
2003 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
2004 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
2144 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
2226 SmallVector<Value *, 4> Args(CI.args());
2229 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
2231 unsigned NumArgs = CI.arg_size();
2232 Rep = emitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
2233 CI.getArgOperand(NumArgs - 2));
2248 static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
2259 Value *Arg0 = CI->getArgOperand(0);
2260 Value *Arg1 = CI->getArgOperand(1);
2279 Value *Arg0 = CI->getArgOperand(0);
2280 Value *Arg1 = CI->getArgOperand(1);
2290 Value *Arg0 = CI->getArgOperand(0);
2291 Value *Arg1 = CI->getArgOperand(1);
2302 Value *Arg0 = CI->getArgOperand(0);
2303 Value *Arg1 = CI->getArgOperand(1);
2309 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
2310 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
2315 upgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
2316 CI->getArgOperand(2), Aligned);
2322 CI->getArgOperand(0), CI->getArgOperand(1));
2323 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
2326 if (CI->getOperand(0)->getType()->isIntegerTy(8))
2328 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
2330 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
2333 Value *Vec = CI->getArgOperand(0);
2344 F->getParent(), Intrinsic::sqrt, CI->getType()),
2345 {CI->getArgOperand(0)});
2347 if (CI->arg_size() == 4 &&
2348 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
2349 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
2353 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(3)};
2354 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
2359 CI->getType()),
2360 {CI->getArgOperand(0)});
2363 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
2366 Value *Op0 = CI->getArgOperand(0);
2367 Value *Op1 = CI->getArgOperand(1);
2368 Value *Mask = CI->getArgOperand(2);
2378 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
2380 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
2382 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
2384 unsigned NumElts = CI->getType()->getScalarSizeInBits();
2385 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
2386 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
2398 Rep = Builder.CreateBitCast(Rep, CI->getType());
2400 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2401 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2403 Rep = Builder.CreateBitCast(Rep, CI->getType());
2405 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2406 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2409 Rep = Builder.CreateBitCast(Rep, CI->getType());
2411 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2412 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2414 Rep = Builder.CreateBitCast(Rep, CI->getType());
2416 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2417 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2419 Rep = Builder.CreateBitCast(Rep, CI->getType());
2421 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2422 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2425 Rep = Builder.CreateBitCast(Rep, CI->getType());
2427 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2429 Rep = Builder.CreateBitCast(Rep, CI->getType());
2431 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2432 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2447 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
2449 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
2460 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
2465 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
2467 Type *OpTy = CI->getArgOperand(0)->getType();
2485 {CI->getOperand(0), CI->getArgOperand(1)});
2486 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
2488 Type *OpTy = CI->getArgOperand(0)->getType();
2508 {CI->getOperand(0), CI->getArgOperand(1)});
2509 Rep = applyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
2511 SmallVector<Value *, 4> Args(CI->args());
2531 Value *Mask = Constant::getAllOnesValue(CI->getType());
2540 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2541 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
2543 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2544 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
2549 Value *Op = CI->getArgOperand(0);
2556 Rep = upgradeAbs(Builder, *CI);
2560 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
2564 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
2568 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
2572 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
2576 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ false);
2580 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/ true);
2584 Builder.CreateSIToFP(CI->getArgOperand(1),
2585 cast<VectorType>(CI->getType())->getElementType());
2586 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2589 Builder.CreateUIToFP(CI->getArgOperand(1),
2590 cast<VectorType>(CI->getType())->getElementType());
2591 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2593 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
2595 Rep, cast<VectorType>(CI->getType())->getElementType());
2596 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2612 auto *DstTy = cast<FixedVectorType>(CI->getType());
2613 Rep = CI->getArgOperand(0);
2626 else if (CI->arg_size() == 4 &&
2627 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
2628 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
2632 Intrinsic::getDeclaration(CI->getModule(), IID, {DstTy, SrcTy});
2633 Rep = Builder.CreateCall(F, {Rep, CI->getArgOperand(3)});
2639 if (CI->arg_size() >= 3)
2640 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
2641 CI->getArgOperand(1));
2644 auto *DstTy = cast<FixedVectorType>(CI->getType());
2645 Rep = CI->getArgOperand(0);
2655 if (CI->arg_size() >= 3)
2656 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
2657 CI->getArgOperand(1));
2661 Rep = upgradeMaskedLoad(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
2662 CI->getArgOperand(2), Aligned);
2664 auto *ResultTy = cast<FixedVectorType>(CI->getType());
2668 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2671 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2676 Rep = Builder.CreateCall(ELd, {Ptr, MaskVec, CI->getOperand(1)});
2678 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
2682 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2686 getX86MaskVec(Builder, CI->getArgOperand(2),
2691 Rep = Builder.CreateCall(CSt, {CI->getArgOperand(1), Ptr, MaskVec});
2694 auto *ResultTy = cast<FixedVectorType>(CI->getType());
2696 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2704 {CI->getOperand(0), CI->getOperand(1), MaskVec});
2717 if (CI->arg_size() == 3) {
2718 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2741 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
2743 Value *Sel = CI->getArgOperand(2);
2745 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
2746 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
2750 Rep = upgradeX86Rotate(Builder, *CI, false);
2753 Rep = upgradeX86Rotate(Builder, *CI, true);
2758 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
2763 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
2768 Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
2769 Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});
2770 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
2774 auto *VecTy = cast<FixedVectorType>(CI->getType());
2777 Value *Load = Builder.CreateLoad(EltTy, CI->getArgOperand(0));
2788 auto *DstTy = cast<FixedVectorType>(CI->getType());
2796 Value *SV = Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
2802 if (CI->arg_size() == 3)
2803 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
2804 CI->getArgOperand(1));
2809 Type *Ty = CI->getArgOperand(1)->getType();
2810 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
2812 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
2816 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
2819 Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
2829 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2830 Type *VT = CI->getType();
2845 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2846 CI->getArgOperand(1), ShuffleMask);
2848 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
2851 unsigned NumSrcElts = cast<FixedVectorType>(CI->getArgOperand(0)->getType())
2854 cast<FixedVectorType>(CI->getType())->getNumElements();
2860 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2861 CI->getArgOperand(0), ShuffleMask);
2863 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
2869 Value *Op = CI->getArgOperand(0);
2870 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
2876 if (CI->arg_size() == 3)
2877 Rep = emitX86Select(Builder, CI->getArgOperand(2), Rep,
2878 CI->getArgOperand(1));
2883 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
2888 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
2892 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
2896 Rep = upgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
2898 Rep = upgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2899 CI->getArgOperand(1), CI->getArgOperand(2),
2900 CI->getArgOperand(3), CI->getArgOperand(4),
2904 Builder, CI->getArgOperand(0), CI->getArgOperand(1),
2905 CI->getArgOperand(2), CI->getArgOperand(3), CI->getArgOperand(4), true);
2908 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2909 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
2913 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2914 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
2919 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2920 Rep = upgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2924 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2925 Rep = upgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2929 Value *Op0 = CI->getArgOperand(0);
2930 Value *Op1 = CI->getArgOperand(1);
2931 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2932 auto *VecTy = cast<FixedVectorType>(CI->getType());
2943 Value *Op0 = CI->getArgOperand(0);
2944 Value *Op1 = CI->getArgOperand(1);
2945 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2947 cast<FixedVectorType>(CI->getType())->getNumElements();
2983 if (CI->arg_size() == 5)
2984 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep,
2985 CI->getArgOperand(3));
2989 Value *Op0 = CI->getArgOperand(0);
2990 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2992 cast<FixedVectorType>(CI->getType())->getNumElements();
3008 if (CI->arg_size() == 4)
3009 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3010 CI->getArgOperand(2));
3013 Value *Op0 = CI->getArgOperand(0);
3014 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3015 auto *VecTy = cast<FixedVectorType>(CI->getType());
3024 if (CI->arg_size() == 4)
3025 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3026 CI->getArgOperand(2));
3036 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3038 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3043 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3044 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
3047 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
3048 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
3065 Value *Op0 = CI->getArgOperand(0);
3066 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3067 auto *VecTy = cast<FixedVectorType>(CI->getType());
3082 if (CI->arg_size() == 4)
3083 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3084 CI->getArgOperand(2));
3087 Value *Op0 = CI->getArgOperand(0);
3088 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3089 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3101 if (CI->arg_size() == 4)
3102 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3103 CI->getArgOperand(2));
3106 Value *Op0 = CI->getArgOperand(0);
3107 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
3108 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3120 if (CI->arg_size() == 4)
3121 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep,
3122 CI->getArgOperand(2));
3124 Value *Op0 = CI->getArgOperand(0);
3125 Value *Op1 = CI->getArgOperand(1);
3126 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
3127 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3129 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3147 emitX86Select(Builder, CI->getArgOperand(4), Rep, CI->getArgOperand(3));
3151 Value *Op0 = CI->getArgOperand(0);
3152 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3153 unsigned NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3169 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3172 Value *Op0 = CI->getArgOperand(0);
3173 Value *Op1 = CI->getArgOperand(1);
3174 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3175 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3185 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3188 Value *Op0 = CI->getArgOperand(0);
3189 Value *Op1 = CI->getArgOperand(1);
3190 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3191 int NumLaneElts = 128 / CI->getType()->getScalarSizeInBits();
3201 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3204 VectorType *FTy = cast<VectorType>(CI->getType());
3206 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3207 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3210 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3213 VectorType *FTy = cast<VectorType>(CI->getType());
3215 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
3217 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3220 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3223 VectorType *FTy = cast<VectorType>(CI->getType());
3225 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3226 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3229 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3232 VectorType *FTy = cast<VectorType>(CI->getType());
3234 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
3235 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
3238 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3240 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
3242 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3244 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
3246 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3248 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
3250 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3261 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
3263 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
3266 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3277 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
3279 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
3282 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3293 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
3295 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
3298 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3309 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
3311 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
3314 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3327 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(4)});
3329 emitX86Select(Builder, CI->getArgOperand(3), Rep, CI->getArgOperand(2));
3333 F->getParent(), Intrinsic::ctlz, CI->getType()),
3334 {CI->getArgOperand(0), Builder.getInt1(false)});
3336 emitX86Select(Builder, CI->getArgOperand(2), Rep, CI->getArgOperand(1));
3403 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
3470 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
3535 Rep = upgradeX86MaskedShift(Builder, *CI, IID);
3537 Rep = upgradeMaskedMove(Builder, *CI);
3539 Rep = upgradeMaskToInt(Builder, *CI);
3544 Value *Ptr = CI->getArgOperand(0);
3548 Ptr, PointerType::getUnqual(CI->getType()), "cast");
3550 CI->getType(), BC,
3551 Align(CI->getType()->getPrimitiveSizeInBits().getFixedValue() / 8));
3562 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
3563 CI->getArgOperand(2)};
3578 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3584 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
3586 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
3587 CI->getArgOperand(2)};
3593 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3598 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
3612 Value *A = CI->getArgOperand(0);
3613 Value *B = CI->getArgOperand(1);
3614 Value *C = CI->getArgOperand(2);
3627 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3628 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
3629 Value *Ops[] = {A, B, C, CI->getArgOperand(4)};
3636 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID);
3639 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
3652 Builder.CreateExtractElement(CI->getArgOperand(2), (uint64_t)0);
3654 Rep = emitX86ScalarSelect(Builder, CI->getArgOperand(3), Rep, PassThru);
3655 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), Rep,
3671 Value *A = CI->getArgOperand(0);
3672 Value *B = CI->getArgOperand(1);
3673 Value *C = CI->getArgOperand(2);
3682 if (CI->arg_size() == 5 &&
3683 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3684 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
3693 {A, B, C, CI->getArgOperand(4)});
3695 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
3700 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
3701 : IsMask3 ? CI->getArgOperand(2)
3702 : CI->getArgOperand(0);
3704 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3706 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3707 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
3720 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
3721 CI->getArgOperand(2)};
3734 if (CI->arg_size() == 5) {
3742 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
3743 CI->getArgOperand(2), CI->getArgOperand(4)};
3750 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3752 Value *Ops[] = {CI->getArgOperand(0), CI->getArgOperand(1),
3753 CI->getArgOperand(2)};
3755 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
3771 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType())
3772 : IsMask3 ? CI->getArgOperand(2)
3773 : CI->getArgOperand(0);
3775 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3779 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3780 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
3797 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
3798 CI->getArgOperand(2), CI->getArgOperand(3)};
3799 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3801 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3802 : CI->getArgOperand(0);
3803 Rep = emitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
3808 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3825 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
3826 CI->getArgOperand(2)};
3827 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3829 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3830 : CI->getArgOperand(0);
3831 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3837 Rep = upgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
3844 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3861 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
3862 CI->getArgOperand(2)};
3863 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3865 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3866 : CI->getArgOperand(0);
3867 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3874 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3891 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
3892 CI->getArgOperand(2)};
3893 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3895 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3896 : CI->getArgOperand(0);
3897 Rep = emitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3914 Value *Args[] = {CI->getArgOperand(0), CI->getArgOperand(1),
3915 CI->getArgOperand(2)};
3917 Intrinsic::getDeclaration(CI->getModule(), IID), Args);
3923 CI->getArgOperand(3), llvm::PointerType::getUnqual(Data->getType()));
3928 CI->replaceAllUsesWith(CF);
3931 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
3938 static Value *upgradeARMIntrinsicCall(StringRef Name, CallBase *CI, Function *F,
3945 CI->getArgOperand(0), CI->getName());
3975 unsigned ID = CI->getIntrinsicID();
3981 Tys = {CI->getType(), CI->getOperand(0)->getType(), V2I1Ty};
3986 Tys = {CI->getOperand(0)->getType(), CI->getOperand(0)->getType(),
3990 Tys = {CI->getType(), CI->getOperand(0)->getType(),
3991 CI->getOperand(1)->getType(), V2I1Ty};
3994 Tys = {CI->getOperand(0)->getType(), CI->getOperand(1)->getType(),
3995 CI->getOperand(2)->getType(), V2I1Ty};
4003 Tys = {CI->getOperand(1)->getType(), V2I1Ty};
4010 for (Value *Op : CI->args()) {
4027 return Builder.CreateCall(Fn, Ops, CI->getName());
4037 static Value *upgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
4047 unsigned NumOperands = CI->getNumOperands();
4051 Value *Ptr = CI->getArgOperand(0);
4056 Value *Val = CI->getArgOperand(1);
4057 if (Val->getType() != CI->getType()) // Malformed.
4066 OrderArg = dyn_cast<ConstantInt>(CI->getArgOperand(2));
4071 ConstantInt *VolatileArg = dyn_cast<ConstantInt>(CI->getArgOperand(4));
4084 Type *RetTy = CI->getType();
4112 static MDType *unwrapMAVOp(CallBase *CI, unsigned Op) {
4113 if (MetadataAsValue *MAV = dyn_cast<MetadataAsValue>(CI->getArgOperand(Op)))
4120 /// \p CI - The debug intrinsic call.
4121 static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) {
4124 DR = new DbgLabelRecord(unwrapMAVOp<DILabel>(CI, 0), CI->getDebugLoc());
4127 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, 1),
4128 unwrapMAVOp<DIExpression>(CI, 2), unwrapMAVOp<DIAssignID>(CI, 3),
4129 unwrapMAVOp<Metadata>(CI, 4), unwrapMAVOp<DIExpression>(CI, 5),
4130 CI->getDebugLoc());
4133 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, 1),
4134 unwrapMAVOp<DIExpression>(CI, 2), CI->getDebugLoc(),
4138 DIExpression *Expr = unwrapMAVOp<DIExpression>(CI, 2);
4140 DR = new DbgVariableRecord(unwrapMAVOp<Metadata>(CI, 0),
4141 unwrapMAVOp<DILocalVariable>(CI, 1), Expr,
4142 CI->getDebugLoc());
4147 if (CI->arg_size() == 4) {
4148 auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1));
4156 unwrapMAVOp<Metadata>(CI, 0), unwrapMAVOp<DILocalVariable>(CI, VarOp),
4157 unwrapMAVOp<DIExpression>(CI, ExprOp), CI->getDebugLoc());
4160 CI->getParent()->insertDbgRecordBefore(DR, CI->getIterator());
4165 void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
4169 Function *F = dyn_cast<Function>(CI->getCalledOperand());
4173 LLVMContext &C = CI->getContext();
4175 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
4195 Value *Arg = CI->getArgOperand(0);
4202 Value *Ptr = CI->getArgOperand(0);
4203 Value *Val = CI->getArgOperand(1);
4209 Value *Arg0 = CI->getArgOperand(0);
4210 Value *Arg1 = CI->getArgOperand(1);
4218 Value *Arg0 = CI->getArgOperand(0);
4219 Value *Arg1 = CI->getArgOperand(1);
4226 Value *Arg = CI->getArgOperand(0);
4235 Value *Arg = CI->getArgOperand(0);
4247 CI->getArgOperand(0), "h2f");
4256 Value *Arg = CI->getArgOperand(I);
4270 Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
4272 Rep = upgradeARMIntrinsicCall(Name, CI, F, Builder);
4274 Rep = upgradeAMDGCNIntrinsicCall(Name, CI, F, Builder);
4280 if (!CI->getModule()->IsNewDbgInfoFormat) {
4282 upgradeIntrinsicFunction1(CI->getCalledFunction(), NewFn, false);
4287 upgradeDbgIntrinsicToDbgRecord(Name, CI);
4295 CI->replaceAllUsesWith(Rep);
4296 CI->eraseFromParent();
4302 if (CI->getFunctionType() == NewFn->getFunctionType()) {
4305 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
4307 CI->setCalledFunction(NewFn);
4312 if (auto *OldST = dyn_cast<StructType>(CI->getType())) {
4319 SmallVector<Value *> Args(CI->args());
4326 CI->replaceAllUsesWith(Res);
4327 CI->eraseFromParent();
4333 CI->setCalledOperand(
4334 ConstantExpr::getPointerCast(NewFn, CI->getCalledOperand()->getType()));
4350 SmallVector<Value *, 4> Args(CI->args());
4358 SmallVector<Value *, 4> Args(CI->args());
4376 SmallVector<Value *, 2> Args(CI->args());
4389 SmallVector<Value *, 3> Args(CI->args());
4390 Args.push_back(ConstantTokenNone::get(CI->getContext()));
4404 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4406 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0), NewIdx});
4418 unsigned I = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
4419 auto *Ty = cast<ScalableVectorType>(CI->getArgOperand(2)->getType());
4423 NewFn, {CI->getArgOperand(0), CI->getArgOperand(2), NewIdx});
4438 Value *V = CI->getArgOperand(I);
4455 assert(CI->arg_size() == 3 &&
4458 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
4462 auto Iter = CI->args().begin();
4471 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
4476 assert(CI->arg_size() == 1 &&
4479 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
4484 CI->arg_size() == 2 ? Builder.getFalse() : CI->getArgOperand(2);
4486 CI->arg_size() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
4488 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
4493 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
4497 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
4506 cast<MetadataAsValue>(CI->getArgOperand(2))->getMetadata());
4509 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
4515 assert(CI->arg_size() == 4);
4517 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
4521 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
4524 CI->eraseFromParent();
4530 if (CI->arg_size() != 4) {
4537 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
4538 CI->getArgOperand(2), CI->getArgOperand(3),
4540 NewCall->takeName(CI);
4541 CI->replaceAllUsesWith(NewCall);
4542 CI->eraseFromParent();
4547 if (CI->arg_size() != 4) {
4553 Builder.CreateCall(NewFn, {CI->getArgOperand(0), CI->getArgOperand(1),
4554 CI->getArgOperand(2), CI->getArgOperand(3),
4556 NewCall->takeName(CI);
4557 CI->replaceAllUsesWith(NewCall);
4558 CI->eraseFromParent();
4569 Value *Arg2 = CI->getArgOperand(2);
4570 if (Arg2->getType()->isIntegerTy(32) && !CI->getType()->isIntegerTy(64))
4573 Value *Arg0 = CI->getArgOperand(0);
4574 Value *Arg1 = CI->getArgOperand(1);
4575 if (CI->getType()->isIntegerTy(64)) {
4585 if (Res->getType() != CI->getType())
4586 Res = Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
4587 NewCall->takeName(CI);
4588 CI->replaceAllUsesWith(Res);
4589 CI->eraseFromParent();
4600 if (!CI->getType()->isIntegerTy(64))
4604 Builder.CreateTrunc(CI->getArgOperand(0), Builder.getInt32Ty());
4608 Builder.CreateIntCast(NewCall, CI->getType(), /*isSigned*/ true);
4609 NewCall->takeName(CI);
4610 CI->replaceAllUsesWith(Res);
4611 CI->eraseFromParent();
4617 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
4624 SmallVector<Value *, 4> Args(CI->args());
4639 Value *Arg0 = CI->getArgOperand(0);
4644 Value *Arg1 = CI->getArgOperand(1);
4658 if (CI->getNumOperands() == 0)
4665 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0),
4671 NewCall->takeName(CI);
4672 CI->replaceAllUsesWith(TSC);
4673 CI->eraseFromParent();
4685 SmallVector<Value *, 4> Args(CI->args());
4699 SmallVector<Value *, 4> Args(CI->args());
4707 NewCall->takeName(CI);
4708 CI->replaceAllUsesWith(Res);
4709 CI->eraseFromParent();
4719 SmallVector<Value *, 4> Args(CI->args());
4720 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
4730 NewCall->takeName(CI);
4731 CI->replaceAllUsesWith(Res);
4732 CI->eraseFromParent();
4738 SmallVector<Value *, 4> Args(CI->args());
4740 cast<FixedVectorType>(CI->getType())->getNumElements() * 2;
4765 if (CI->arg_size() != 5) {
4771 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
4772 CI->getArgOperand(2), CI->getArgOperand(4)};
4774 AttributeList OldAttrs = CI->getAttributes();
4782 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
4792 NewCall->takeName(CI);
4793 CI->replaceAllUsesWith(NewCall);
4794 CI->eraseFromParent();
4951 CallInst *CI = dyn_cast<CallInst>(U);
4952 if (!CI || CI->getCalledFunction() != Fn)
4955 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
4961 if (NewFuncTy->getReturnType() != CI->getType() &&
4962 !CastInst::castIsValid(Instruction::BitCast, CI,
4968 for (unsigned I = 0, E = CI->arg_size(); I != E; ++I) {
4969 Value *Arg = CI->getArgOperand(I);
4991 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
4992 NewCall->takeName(CI);
4995 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
4997 if (!CI->use_empty())
4998 CI->replaceAllUsesWith(NewRetVal);
4999 CI->eraseFromParent();