Lines Matching defs:Intrinsic
210 /// Utility to insert an atomic instruction based on Intrinsic::ID
261 /// Utility to insert an atomic instruction based Intrinsic::ID and
529 CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID,
530 llvm::Intrinsic::ID ConstrainedIntrinsicID) {
631 llvm::Intrinsic::ID IntrinsicID) {
650 Function *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType());
725 /// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
731 const llvm::Intrinsic::ID IntrinsicID,
802 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
1116 CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
1408 CGF.CGM.getIntrinsic(Intrinsic::sponentry, CGF.AllocaInt8PtrTy));
1411 CGF.CGM.getIntrinsic(Intrinsic::frameaddress, CGF.AllocaInt8PtrTy),
1841 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1849 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
2022 Intrinsic::abs, ArgValue,
2040 Intrinsic::ssub_with_overflow, Zero, ArgValue);
2275 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2345 EmitOverflowIntrinsic(CGF, llvm::Intrinsic::umul_with_overflow, AbsSigned,
2440 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2674 *this, E, Intrinsic::acos, Intrinsic::experimental_constrained_acos));
2685 *this, E, Intrinsic::asin, Intrinsic::experimental_constrained_asin));
2696 *this, E, Intrinsic::atan, Intrinsic::experimental_constrained_atan));
2707 Intrinsic::ceil,
2708 Intrinsic::experimental_constrained_ceil));
2719 emitBuiltinWithOneOverloadedType<2>(*this, E, Intrinsic::copysign));
2730 Intrinsic::cos,
2731 Intrinsic::experimental_constrained_cos));
2742 *this, E, Intrinsic::cosh, Intrinsic::experimental_constrained_cosh));
2753 Intrinsic::exp,
2754 Intrinsic::experimental_constrained_exp));
2765 Intrinsic::exp2,
2766 Intrinsic::experimental_constrained_exp2));
2776 emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::exp10));
2787 emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::fabs));
2798 Intrinsic::floor,
2799 Intrinsic::experimental_constrained_floor));
2810 Intrinsic::fma,
2811 Intrinsic::experimental_constrained_fma));
2822 Intrinsic::maxnum,
2823 Intrinsic::experimental_constrained_maxnum));
2834 Intrinsic::minnum,
2835 Intrinsic::experimental_constrained_minnum));
2862 Intrinsic::log,
2863 Intrinsic::experimental_constrained_log));
2874 Intrinsic::log10,
2875 Intrinsic::experimental_constrained_log10));
2886 Intrinsic::log2,
2887 Intrinsic::experimental_constrained_log2));
2897 Intrinsic::nearbyint,
2898 Intrinsic::experimental_constrained_nearbyint));
2909 Intrinsic::pow,
2910 Intrinsic::experimental_constrained_pow));
2921 Intrinsic::rint,
2922 Intrinsic::experimental_constrained_rint));
2933 Intrinsic::round,
2934 Intrinsic::experimental_constrained_round));
2945 Intrinsic::roundeven,
2946 Intrinsic::experimental_constrained_roundeven));
2957 Intrinsic::sin,
2958 Intrinsic::experimental_constrained_sin));
2969 *this, E, Intrinsic::sinh, Intrinsic::experimental_constrained_sinh));
2981 *this, E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
2995 *this, E, Intrinsic::tan, Intrinsic::experimental_constrained_tan));
3006 *this, E, Intrinsic::tanh, Intrinsic::experimental_constrained_tanh));
3017 Intrinsic::trunc,
3018 Intrinsic::experimental_constrained_trunc));
3028 *this, E, Intrinsic::lround,
3029 Intrinsic::experimental_constrained_lround));
3039 *this, E, Intrinsic::llround,
3040 Intrinsic::experimental_constrained_llround));
3050 *this, E, Intrinsic::lrint,
3051 Intrinsic::experimental_constrained_lrint));
3061 *this, E, Intrinsic::llrint,
3062 Intrinsic::experimental_constrained_llrint));
3069 *this, E, Intrinsic::ldexp,
3070 Intrinsic::experimental_constrained_ldexp));
3115 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy, {DstPtr->getType()}),
3210 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3236 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
3268 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3294 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
3315 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
3331 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
3350 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
3376 Function *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
3403 CGM.getIntrinsic(Intrinsic::expect_with_probability, ArgType);
3431 Function *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
3452 CGM.getIntrinsic(llvm::Intrinsic::allow_runtime_check),
3493 emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::bswap));
3500 emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::bitreverse));
3550 CGM.getIntrinsic(Intrinsic::is_constant, ConvertType(ArgType));
3576 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
3581 Function *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
3585 Function *F = CGM.getIntrinsic(Intrinsic::readsteadycounter);
3591 Function *F = CGM.getIntrinsic(Intrinsic::clear_cache);
3595 EmitTrapCall(Intrinsic::trap);
3606 EmitTrapCall(Intrinsic::trap);
3610 EmitTrapCall(Intrinsic::debugtrap);
3631 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_powi,
3636 Function *F = CGM.getIntrinsic(Intrinsic::powi,
3652 return RValue::get(emitFrexpBuiltin(*this, E, Intrinsic::frexp));
3786 llvm::Intrinsic::abs, EmitScalarExpr(E->getArg(0)),
3790 *this, E, llvm::Intrinsic::fabs, "elt.abs");
3796 *this, E, llvm::Intrinsic::acos, "elt.acos"));
3799 *this, E, llvm::Intrinsic::asin, "elt.asin"));
3802 *this, E, llvm::Intrinsic::atan, "elt.atan"));
3805 *this, E, llvm::Intrinsic::ceil, "elt.ceil"));
3808 *this, E, llvm::Intrinsic::exp, "elt.exp"));
3811 *this, E, llvm::Intrinsic::exp2, "elt.exp2"));
3814 *this, E, llvm::Intrinsic::log, "elt.log"));
3817 *this, E, llvm::Intrinsic::log2, "elt.log2"));
3820 *this, E, llvm::Intrinsic::log10, "elt.log10"));
3823 emitBuiltinWithOneOverloadedType<2>(*this, E, llvm::Intrinsic::pow));
3827 *this, E, llvm::Intrinsic::bitreverse, "elt.bitreverse"));
3830 *this, E, llvm::Intrinsic::cos, "elt.cos"));
3833 *this, E, llvm::Intrinsic::cosh, "elt.cosh"));
3836 *this, E, llvm::Intrinsic::floor, "elt.floor"));
3839 *this, E, llvm::Intrinsic::roundeven, "elt.roundeven"));
3842 *this, E, llvm::Intrinsic::round, "elt.round"));
3845 *this, E, llvm::Intrinsic::rint, "elt.rint"));
3848 *this, E, llvm::Intrinsic::nearbyint, "elt.nearbyint"));
3851 *this, E, llvm::Intrinsic::sin, "elt.sin"));
3854 *this, E, llvm::Intrinsic::sinh, "elt.sinh"));
3857 *this, E, llvm::Intrinsic::tan, "elt.tan"));
3860 *this, E, llvm::Intrinsic::tanh, "elt.tanh"));
3863 *this, E, llvm::Intrinsic::trunc, "elt.trunc"));
3866 *this, E, llvm::Intrinsic::canonicalize, "elt.canonicalize"));
3869 *this, E, llvm::Intrinsic::copysign));
3872 emitBuiltinWithOneOverloadedType<3>(*this, E, llvm::Intrinsic::fma));
3885 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
3887 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
3901 ? llvm::Intrinsic::smax
3902 : llvm::Intrinsic::umax,
3917 ? llvm::Intrinsic::smin
3918 : llvm::Intrinsic::umin,
3933 return llvm::Intrinsic::vector_reduce_smax;
3935 return llvm::Intrinsic::vector_reduce_umax;
3937 return llvm::Intrinsic::vector_reduce_fmax;
3951 return llvm::Intrinsic::vector_reduce_smin;
3953 return llvm::Intrinsic::vector_reduce_umin;
3955 return llvm::Intrinsic::vector_reduce_fmin;
3964 *this, E, llvm::Intrinsic::vector_reduce_add, "rdx.add"));
3967 *this, E, llvm::Intrinsic::vector_reduce_mul, "rdx.mul"));
3970 *this, E, llvm::Intrinsic::vector_reduce_xor, "rdx.xor"));
3973 *this, E, llvm::Intrinsic::vector_reduce_or, "rdx.or"));
3976 *this, E, llvm::Intrinsic::vector_reduce_and, "rdx.and"));
4048 Function *F = CGM.getIntrinsic(Intrinsic::get_rounding);
4059 Function *F = CGM.getIntrinsic(Intrinsic::set_rounding);
4444 Function *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
4451 Function *F = CGM.getIntrinsic(Intrinsic::returnaddress);
4455 Function *F = CGM.getIntrinsic(Intrinsic::returnaddress);
4461 Function *F = CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy);
4498 CGM.getIntrinsic(IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32
4499 : Intrinsic::eh_return_i64);
4509 Function *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
4544 CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy),
4556 Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
4563 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
5010 CGM.getIntrinsic(llvm::Intrinsic::codeview_annotation, {});
5018 CGM.getIntrinsic(llvm::Intrinsic::annotation,
5063 llvm::Intrinsic::ID IntrinsicId;
5071 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5078 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5133 llvm::Intrinsic::ID IntrinsicId;
5139 ? llvm::Intrinsic::sadd_with_overflow
5140 : llvm::Intrinsic::uadd_with_overflow;
5144 ? llvm::Intrinsic::ssub_with_overflow
5145 : llvm::Intrinsic::usub_with_overflow;
5149 ? llvm::Intrinsic::smul_with_overflow
5150 : llvm::Intrinsic::umul_with_overflow;
5217 llvm::Intrinsic::ID IntrinsicId;
5223 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
5228 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
5233 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
5238 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
5243 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
5248 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
5438 return llvm::Intrinsic::ptrauth_auth;
5440 return llvm::Intrinsic::ptrauth_resign;
5442 return llvm::Intrinsic::ptrauth_blend;
5444 return llvm::Intrinsic::ptrauth_sign_generic;
5446 return llvm::Intrinsic::ptrauth_sign;
5448 return llvm::Intrinsic::ptrauth_strip;
5452 auto Intrinsic = CGM.getIntrinsic(IntrinsicID);
5453 llvm::Value *Result = EmitRuntimeCall(Intrinsic, Args);
5506 return EmitCoroutineIntrinsic(E, Intrinsic::coro_id);
5508 return EmitCoroutineIntrinsic(E, Intrinsic::coro_promise);
5510 EmitCoroutineIntrinsic(E, Intrinsic::coro_resume);
5513 return EmitCoroutineIntrinsic(E, Intrinsic::coro_frame);
5515 return EmitCoroutineIntrinsic(E, Intrinsic::coro_noop);
5517 return EmitCoroutineIntrinsic(E, Intrinsic::coro_free);
5519 EmitCoroutineIntrinsic(E, Intrinsic::coro_destroy);
5522 return EmitCoroutineIntrinsic(E, Intrinsic::coro_done);
5524 return EmitCoroutineIntrinsic(E, Intrinsic::coro_alloc);
5526 return EmitCoroutineIntrinsic(E, Intrinsic::coro_begin);
5528 return EmitCoroutineIntrinsic(E, Intrinsic::coro_end);
5530 return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend);
5532 return EmitCoroutineIntrinsic(E, Intrinsic::coro_size);
5534 return EmitCoroutineIntrinsic(E, Intrinsic::coro_align);
5999 emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::canonicalize));
6022 Function *F = CGM.getIntrinsic(Intrinsic::xray_customevent);
6056 Function *F = CGM.getIntrinsic(Intrinsic::xray_typedevent);
6134 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
6138 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name);
6139 if (IntrinsicID == Intrinsic::not_intrinsic && Prefix == "spv" &&
6141 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin("amdgcn", Name);
6145 if (IntrinsicID == Intrinsic::not_intrinsic)
6146 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
6149 if (IntrinsicID != Intrinsic::not_intrinsic) {
6181 ArgValue = Builder.CreateIntrinsic(Intrinsic::x86_cast_vector_to_tile,
6210 V = Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector, {RetTy},
6505 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
6509 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
7498 #NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7516 #NameBase, SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
7712 Int == Intrinsic::aarch64_neon_addp)
7713 Int = Intrinsic::aarch64_neon_faddp;
7718 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs");
7939 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
8020 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
8021 Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
8092 ? Intrinsic::experimental_constrained_nearbyint
8093 : Intrinsic::nearbyint;
8448 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
8466 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_ballot, {RegisterType});
8514 AccessKind == VolatileRead ? llvm::Intrinsic::read_volatile_register
8515 : llvm::Intrinsic::read_register,
8530 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
8620 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
8631 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
8638 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
8644 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Arg->getType());
8654 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls), Arg, "cls");
8658 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls64), Arg,
8681 F = CGM.getIntrinsic(Intrinsic::arm_mcrr);
8684 F = CGM.getIntrinsic(Intrinsic::arm_mcrr2);
8715 F = CGM.getIntrinsic(Intrinsic::arm_mrrc);
8718 F = CGM.getIntrinsic(Intrinsic::arm_mrrc2);
8752 F = CGM.getIntrinsic(Intrinsic::arm_ldaexd);
8757 F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
8785 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
8786 : Intrinsic::arm_ldrex,
8807 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
8808 : Intrinsic::arm_strexd);
8844 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
8845 : Intrinsic::arm_strex,
8855 Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
8860 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
8863 CRCIntrinsicID = Intrinsic::arm_crc32b; break;
8865 CRCIntrinsicID = Intrinsic::arm_crc32cb; break;
8867 CRCIntrinsicID = Intrinsic::arm_crc32h; break;
8869 CRCIntrinsicID = Intrinsic::arm_crc32ch; break;
8872 CRCIntrinsicID = Intrinsic::arm_crc32w; break;
8875 CRCIntrinsicID = Intrinsic::arm_crc32cw; break;
8878 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
8937 llvm::Function *F = CGM.getIntrinsic(Intrinsic::sponentry, AllocaInt8PtrTy);
9061 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vrintn, Tys);
9079 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops,
9082 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops,
9085 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops,
9088 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
9092 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvtbfp2bf), Ops,
9101 ? Intrinsic::arm_mcr
9102 : Intrinsic::arm_mcr2);
9127 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
9171 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys);
9187 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
9191 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
9194 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
9198 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
9202 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty),
9205 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
9212 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
9222 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
9238 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
9249 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
9252 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2),
9255 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3),
9258 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4),
9261 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1),
9264 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2),
9267 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3),
9270 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4),
9335 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vreinterpretq,
9400 Intrinsic::ID IRIntr;
9567 Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1");
9571 Ty, Intrinsic::aarch64_neon_tbl1, "vtbl1");
9575 Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2");
9579 Ty, Intrinsic::aarch64_neon_tbl2, "vtbl2");
9584 Intrinsic::aarch64_neon_tbl1, "vtbl1");
9596 Ty, Intrinsic::aarch64_neon_tbx1, "vtbx1");
9601 Intrinsic::aarch64_neon_tbl2, "vtbl2");
9614 Ty, Intrinsic::aarch64_neon_tbx2, "vtbx2");
9618 Int = Intrinsic::aarch64_neon_tbl1; s = "vtbl1"; break;
9621 Int = Intrinsic::aarch64_neon_tbl2; s = "vtbl2"; break;
9624 Int = Intrinsic::aarch64_neon_tbl3; s = "vtbl3"; break;
9627 Int = Intrinsic::aarch64_neon_tbl4; s = "vtbl4"; break;
9630 Int = Intrinsic::aarch64_neon_tbx1; s = "vtbx1"; break;
9633 Int = Intrinsic::aarch64_neon_tbx2; s = "vtbx2"; break;
9636 Int = Intrinsic::aarch64_neon_tbx3; s = "vtbx3"; break;
9639 Int = Intrinsic::aarch64_neon_tbx4; s = "vtbx4"; break;
9787 CGM.getIntrinsic(Intrinsic::aarch64_sve_ptrue, getSVEPredType(TypeFlags));
9820 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
9824 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
9986 case Intrinsic::aarch64_sve_ld2_sret:
9987 case Intrinsic::aarch64_sve_ld1_pn_x2:
9988 case Intrinsic::aarch64_sve_ldnt1_pn_x2:
9989 case Intrinsic::aarch64_sve_ld2q_sret:
9992 case Intrinsic::aarch64_sve_ld3_sret:
9993 case Intrinsic::aarch64_sve_ld3q_sret:
9996 case Intrinsic::aarch64_sve_ld4_sret:
9997 case Intrinsic::aarch64_sve_ld1_pn_x4:
9998 case Intrinsic::aarch64_sve_ldnt1_pn_x4:
9999 case Intrinsic::aarch64_sve_ld4q_sret:
10034 case Intrinsic::aarch64_sve_st2:
10035 case Intrinsic::aarch64_sve_st1_pn_x2:
10036 case Intrinsic::aarch64_sve_stnt1_pn_x2:
10037 case Intrinsic::aarch64_sve_st2q:
10040 case Intrinsic::aarch64_sve_st3:
10041 case Intrinsic::aarch64_sve_st3q:
10044 case Intrinsic::aarch64_sve_st4:
10045 case Intrinsic::aarch64_sve_st1_pn_x4:
10046 case Intrinsic::aarch64_sve_stnt1_pn_x4:
10047 case Intrinsic::aarch64_sve_st4q:
10136 case Intrinsic::aarch64_sve_ld1uwq:
10137 case Intrinsic::aarch64_sve_ld1udq:
10186 case Intrinsic::aarch64_sve_st1wq:
10187 case Intrinsic::aarch64_sve_st1dq:
10231 CGM.getIntrinsic(Intrinsic::aarch64_sme_cntsb);
10573 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
10580 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
10599 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy);
10601 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy);
10604 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_psel, OverloadedTy);
10615 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_and_z, OverloadedTy);
10623 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_eor_z, OverloadedTy);
10630 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
10635 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
10640 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
10645 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
10651 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
10657 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
10710 CGM.getIntrinsic(Intrinsic::aarch64_sve_dupq_lane, OverloadedTy);
10721 F = CGM.getIntrinsic(NumOpnds == 2 ? Intrinsic::aarch64_sve_cmpne
10722 : Intrinsic::aarch64_sve_cmpne_wide,
10735 CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, Ty);
10756 Function *F = CGM.getIntrinsic(Intrinsic::vscale, Ty);
10774 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_sve_tbl2, VTy);
10822 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
10942 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint);
10947 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_break);
10975 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
10982 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
10988 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Arg->getType());
10997 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls), Arg,
11002 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls64), Arg,
11010 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint32z, Ty),
11018 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint64z, Ty),
11026 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint32x, Ty),
11034 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint64x, Ty),
11043 CGM.getIntrinsic(Intrinsic::aarch64_fjcvtzs), Arg);
11056 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_ld64b);
11081 ? Intrinsic::aarch64_st64b
11083 ? Intrinsic::aarch64_st64bv
11084 : Intrinsic::aarch64_st64bv0);
11094 ? Intrinsic::aarch64_rndr
11095 : Intrinsic::aarch64_rndrrs);
11124 ? Intrinsic::aarch64_ldaxp
11125 : Intrinsic::aarch64_ldxp);
11151 ? Intrinsic::aarch64_ldaxr
11152 : Intrinsic::aarch64_ldxr,
11172 ? Intrinsic::aarch64_stlxp
11173 : Intrinsic::aarch64_stxp);
11209 ? Intrinsic::aarch64_stlxr
11210 : Intrinsic::aarch64_stxr,
11232 CGM.getIntrinsic(llvm::Intrinsic::read_register, {Int64Ty});
11241 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::aarch64_break);
11246 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
11255 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
11258 CRCIntrinsicID = Intrinsic::aarch64_crc32b; break;
11260 CRCIntrinsicID = Intrinsic::aarch64_crc32cb; break;
11262 CRCIntrinsicID = Intrinsic::aarch64_crc32h; break;
11264 CRCIntrinsicID = Intrinsic::aarch64_crc32ch; break;
11266 CRCIntrinsicID = Intrinsic::aarch64_crc32w; break;
11268 CRCIntrinsicID = Intrinsic::aarch64_crc32cw; break;
11270 CRCIntrinsicID = Intrinsic::aarch64_crc32x; break;
11272 CRCIntrinsicID = Intrinsic::aarch64_crc32cx; break;
11275 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
11295 CGM.getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
11299 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
11302 MTEIntrinsicID = Intrinsic::aarch64_irg; break;
11304 MTEIntrinsicID = Intrinsic::aarch64_addg; break;
11306 MTEIntrinsicID = Intrinsic::aarch64_gmi; break;
11308 MTEIntrinsicID = Intrinsic::aarch64_ldg; break;
11310 MTEIntrinsicID = Intrinsic::aarch64_stg; break;
11312 MTEIntrinsicID = Intrinsic::aarch64_subp; break;
11315 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
11318 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
11328 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
11338 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
11350 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
11360 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
11366 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
11443 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
11448 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
11456 CGM.getIntrinsic(Intrinsic::addressofreturnaddress, AllocaInt8PtrTy);
11461 llvm::Function *F = CGM.getIntrinsic(Intrinsic::sponentry, AllocaInt8PtrTy);
11499 CGM.getIntrinsic(llvm::Intrinsic::read_register, {Int64Ty});
11523 CGM.getIntrinsic(llvm::Intrinsic::read_register, {Int64Ty});
11554 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
11568 ? CGM.getIntrinsic(Intrinsic::aarch64_cls)
11569 : CGM.getIntrinsic(Intrinsic::aarch64_cls64);
11581 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
11594 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
11599 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hlt);
11682 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, HalfTy), Ops, "vabs");
11757 Int = Intrinsic::aarch64_neon_fcvtau; break;
11759 Int = Intrinsic::aarch64_neon_fcvtmu; break;
11761 Int = Intrinsic::aarch64_neon_fcvtnu; break;
11763 Int = Intrinsic::aarch64_neon_fcvtpu; break;
11765 Int = Intrinsic::aarch64_neon_fcvtzu; break;
11767 Int = Intrinsic::aarch64_neon_fcvtas; break;
11769 Int = Intrinsic::aarch64_neon_fcvtms; break;
11771 Int = Intrinsic::aarch64_neon_fcvtns; break;
11773 Int = Intrinsic::aarch64_neon_fcvtps; break;
11775 Int = Intrinsic::aarch64_neon_fcvtzs; break;
11792 Int = Intrinsic::aarch64_neon_facge; break;
11794 Int = Intrinsic::aarch64_neon_facgt; break;
11796 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]); break;
11798 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]); break;
11813 Int = Intrinsic::aarch64_neon_vcvtfp2fxs; break;
11815 Int = Intrinsic::aarch64_neon_vcvtfp2fxu; break;
11830 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
11834 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
12155 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, HalfTy,
12162 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, HalfTy,
12177 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
12183 ? Intrinsic::aarch64_neon_sqadd
12184 : Intrinsic::aarch64_neon_sqsub;
12190 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty),
12196 ? Intrinsic::aarch64_neon_uqshl
12197 : Intrinsic::aarch64_neon_sqshl;
12205 ? Intrinsic::aarch64_neon_urshl
12206 : Intrinsic::aarch64_neon_srshl;
12215 ? Intrinsic::aarch64_neon_urshl
12216 : Intrinsic::aarch64_neon_srshl;
12274 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
12282 ? Intrinsic::aarch64_neon_sqadd
12283 : Intrinsic::aarch64_neon_sqsub;
12292 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
12296 ? Intrinsic::aarch64_neon_sqadd
12297 : Intrinsic::aarch64_neon_sqsub;
12310 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
12316 ? Intrinsic::aarch64_neon_sqadd
12317 : Intrinsic::aarch64_neon_sqsub;
12400 Int = Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
12401 : Intrinsic::fma;
12416 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
12431 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12441 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12454 *this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
12459 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
12460 if (Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
12465 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
12466 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
12470 Int = Intrinsic::aarch64_neon_fmax;
12476 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
12477 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
12481 Int = Intrinsic::aarch64_neon_fmin;
12487 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
12488 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
12498 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
12509 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
12510 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
12515 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
12516 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
12520 Int = Intrinsic::aarch64_neon_fminnm;
12524 Int = Intrinsic::aarch64_neon_fminnm;
12528 Int = Intrinsic::aarch64_neon_fmaxnm;
12532 Int = Intrinsic::aarch64_neon_fmaxnm;
12536 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy),
12541 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy),
12545 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, HalfTy),
12548 Int = Intrinsic::aarch64_neon_sqshrun;
12551 Int = Intrinsic::aarch64_neon_sqrshrun;
12554 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
12557 Int = Intrinsic::aarch64_neon_rshrn;
12560 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
12565 ? Intrinsic::experimental_constrained_round
12566 : Intrinsic::round;
12572 ? Intrinsic::experimental_constrained_round
12573 : Intrinsic::round;
12579 ? Intrinsic::experimental_constrained_nearbyint
12580 : Intrinsic::nearbyint;
12586 ? Intrinsic::experimental_constrained_floor
12587 : Intrinsic::floor;
12593 ? Intrinsic::experimental_constrained_floor
12594 : Intrinsic::floor;
12600 ? Intrinsic::experimental_constrained_roundeven
12601 : Intrinsic::roundeven;
12607 ? Intrinsic::experimental_constrained_roundeven
12608 : Intrinsic::roundeven;
12614 ? Intrinsic::experimental_constrained_roundeven
12615 : Intrinsic::roundeven;
12621 ? Intrinsic::experimental_constrained_ceil
12622 : Intrinsic::ceil;
12628 ? Intrinsic::experimental_constrained_ceil
12629 : Intrinsic::ceil;
12635 ? Intrinsic::experimental_constrained_rint
12636 : Intrinsic::rint;
12642 ? Intrinsic::experimental_constrained_rint
12643 : Intrinsic::rint;
12649 ? Intrinsic::experimental_constrained_trunc
12650 : Intrinsic::trunc;
12658 Int = Intrinsic::aarch64_neon_frint32x;
12666 Int = Intrinsic::aarch64_neon_frint32z;
12674 Int = Intrinsic::aarch64_neon_frint64x;
12682 Int = Intrinsic::aarch64_neon_frint64z;
12688 ? Intrinsic::experimental_constrained_trunc
12689 : Intrinsic::trunc;
12727 usgn ? Intrinsic::aarch64_neon_fcvtzu : Intrinsic::aarch64_neon_fcvtzs;
12743 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
12759 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
12775 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
12791 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
12797 Int = Intrinsic::aarch64_neon_fmulx;
12807 Int = Intrinsic::aarch64_neon_fmulx;
12830 Int = Intrinsic::aarch64_neon_fmaxnmp;
12835 Int = Intrinsic::aarch64_neon_fminnmp;
12841 ? Intrinsic::experimental_constrained_sqrt
12842 : Intrinsic::sqrt;
12848 ? Intrinsic::experimental_constrained_sqrt
12849 : Intrinsic::sqrt;
12855 Int = Intrinsic::bitreverse;
12863 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12875 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12887 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12899 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
12908 Int = Intrinsic::aarch64_neon_umaxv;
12917 Int = Intrinsic::aarch64_neon_umaxv;
12926 Int = Intrinsic::aarch64_neon_umaxv;
12935 Int = Intrinsic::aarch64_neon_umaxv;
12944 Int = Intrinsic::aarch64_neon_smaxv;
12953 Int = Intrinsic::aarch64_neon_smaxv;
12962 Int = Intrinsic::aarch64_neon_smaxv;
12971 Int = Intrinsic::aarch64_neon_smaxv;
12980 Int = Intrinsic::aarch64_neon_fmaxv;
12989 Int = Intrinsic::aarch64_neon_fmaxv;
12998 Int = Intrinsic::aarch64_neon_uminv;
13007 Int = Intrinsic::aarch64_neon_uminv;
13016 Int = Intrinsic::aarch64_neon_uminv;
13025 Int = Intrinsic::aarch64_neon_uminv;
13034 Int = Intrinsic::aarch64_neon_sminv;
13043 Int = Intrinsic::aarch64_neon_sminv;
13052 Int = Intrinsic::aarch64_neon_sminv;
13061 Int = Intrinsic::aarch64_neon_sminv;
13070 Int = Intrinsic::aarch64_neon_fminv;
13079 Int = Intrinsic::aarch64_neon_fminv;
13088 Int = Intrinsic::aarch64_neon_fmaxnmv;
13097 Int = Intrinsic::aarch64_neon_fmaxnmv;
13106 Int = Intrinsic::aarch64_neon_fminnmv;
13115 Int = Intrinsic::aarch64_neon_fminnmv;
13129 Int = Intrinsic::aarch64_neon_uaddlv;
13138 Int = Intrinsic::aarch64_neon_uaddlv;
13146 Int = Intrinsic::aarch64_neon_uaddlv;
13155 Int = Intrinsic::aarch64_neon_uaddlv;
13163 Int = Intrinsic::aarch64_neon_saddlv;
13172 Int = Intrinsic::aarch64_neon_saddlv;
13180 Int = Intrinsic::aarch64_neon_saddlv;
13189 Int = Intrinsic::aarch64_neon_saddlv;
13198 Int = Intrinsic::aarch64_neon_vsri;
13204 Int = Intrinsic::aarch64_neon_vsli;
13215 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
13274 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys);
13281 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys);
13288 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys);
13295 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys);
13302 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys);
13309 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys);
13316 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
13327 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
13339 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
13353 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys),
13361 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys),
13368 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys),
13376 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys),
13383 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys),
13391 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys),
13448 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty),
13452 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty),
13456 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty),
13460 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty),
13464 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty),
13468 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty),
13472 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty),
13476 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty),
13481 Int = Intrinsic::aarch64_neon_usqadd;
13486 Int = Intrinsic::aarch64_neon_suqadd;
13530 llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getDeclaration(
13531 &CGM.getModule(), llvm::Intrinsic::bpf_preserve_field_info,
13552 FnDecl = llvm::Intrinsic::getDeclaration(
13553 &CGM.getModule(), llvm::Intrinsic::bpf_btf_type_id, {});
13555 FnDecl = llvm::Intrinsic::getDeclaration(
13556 &CGM.getModule(), llvm::Intrinsic::bpf_preserve_type_info, {});
13590 llvm::Function *IntrinsicFn = llvm::Intrinsic::getDeclaration(
13591 &CGM.getModule(), llvm::Intrinsic::bpf_preserve_enum_value, {});
13677 llvm::Function *F = CGF.CGM.getIntrinsic(Intrinsic::masked_expandload,
13689 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
13690 : Intrinsic::x86_avx512_mask_expand;
13702 llvm::Function *F = CGF.CGM.getIntrinsic(Intrinsic::masked_compressstore,
13734 unsigned IID = IsRight ? Intrinsic::fshr : Intrinsic::fshl;
13877 Intrinsic::ID IID = IsSigned ? Intrinsic::x86_avx512_sitofp_round
13878 : Intrinsic::x86_avx512_uitofp_round;
13896 Intrinsic::ID IID = Intrinsic::not_intrinsic;
13905 IID = llvm::Intrinsic::x86_avx512fp16_vfmadd_ph_512;
13913 IID = llvm::Intrinsic::x86_avx512fp16_vfmaddsub_ph_512;
13921 IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512; break;
13928 IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512; break;
13935 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
13943 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
13957 if (IID != Intrinsic::not_intrinsic &&
13967 FMA = CGF.CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, Ty);
13970 FMA = CGF.CGM.getIntrinsic(Intrinsic::fma, Ty);
14032 Intrinsic::ID IID;
14036 IID = Intrinsic::x86_avx512fp16_vfmadd_f16;
14039 IID = Intrinsic::x86_avx512_vfmadd_f32;
14042 IID = Intrinsic::x86_avx512_vfmadd_f64;
14052 Intrinsic::experimental_constrained_fma, Ops[0]->getType());
14055 Function *FMA = CGF.CGM.getIntrinsic(Intrinsic::fma, Ops[0]->getType());
14109 Intrinsic::ID IID;
14111 IID = Intrinsic::x86_avx512_pternlog_d_128;
14113 IID = Intrinsic::x86_avx512_pternlog_d_256;
14115 IID = Intrinsic::x86_avx512_pternlog_d_512;
14117 IID = Intrinsic::x86_avx512_pternlog_q_128;
14119 IID = Intrinsic::x86_avx512_pternlog_q_256;
14121 IID = Intrinsic::x86_avx512_pternlog_q_512;
14155 CGF.CGM.getIntrinsic(Intrinsic::x86_avx512_mask_vcvtph2ps_512);
14392 auto getCmpIntrinsicCall = [this, &Ops](Intrinsic::ID ID, unsigned Imm) {
14425 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
14429 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_clflush),
14433 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_lfence));
14436 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_mfence));
14439 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_sfence));
14442 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_pause));
14445 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtsc));
14448 Value *Call = Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtscp));
14456 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
14462 Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType());
14517 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
14523 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
14541 Intrinsic::ID ID;
14544 ID = Intrinsic::x86_##NAME; \
14562 ID = Intrinsic::x86_xsetbv;
14575 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_xgetbv), Ops);
14614 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
14877 Intrinsic::ID IID;
14881 IID = Intrinsic::x86_avx512_mask_gather3div2_df;
14884 IID = Intrinsic::x86_avx512_mask_gather3div2_di;
14887 IID = Intrinsic::x86_avx512_mask_gather3div4_df;
14890 IID = Intrinsic::x86_avx512_mask_gather3div4_di;
14893 IID = Intrinsic::x86_avx512_mask_gather3div4_sf;
14896 IID = Intrinsic::x86_avx512_mask_gather3div4_si;
14899 IID = Intrinsic::x86_avx512_mask_gather3div8_sf;
14902 IID = Intrinsic::x86_avx512_mask_gather3div8_si;
14905 IID = Intrinsic::x86_avx512_mask_gather3siv2_df;
14908 IID = Intrinsic::x86_avx512_mask_gather3siv2_di;
14911 IID = Intrinsic::x86_avx512_mask_gather3siv4_df;
14914 IID = Intrinsic::x86_avx512_mask_gather3siv4_di;
14917 IID = Intrinsic::x86_avx512_mask_gather3siv4_sf;
14920 IID = Intrinsic::x86_avx512_mask_gather3siv4_si;
14923 IID = Intrinsic::x86_avx512_mask_gather3siv8_sf;
14926 IID = Intrinsic::x86_avx512_mask_gather3siv8_si;
14929 IID = Intrinsic::x86_avx512_mask_gather_dpd_512;
14932 IID = Intrinsic::x86_avx512_mask_gather_dps_512;
14935 IID = Intrinsic::x86_avx512_mask_gather_qpd_512;
14938 IID = Intrinsic::x86_avx512_mask_gather_qps_512;
14941 IID = Intrinsic::x86_avx512_mask_gather_dpq_512;
14944 IID = Intrinsic::x86_avx512_mask_gather_dpi_512;
14947 IID = Intrinsic::x86_avx512_mask_gather_qpq_512;
14950 IID = Intrinsic::x86_avx512_mask_gather_qpi_512;
14986 Intrinsic::ID IID;
14990 IID = Intrinsic::x86_avx512_mask_scatter_dpd_512;
14993 IID = Intrinsic::x86_avx512_mask_scatter_dps_512;
14996 IID = Intrinsic::x86_avx512_mask_scatter_qpd_512;
14999 IID = Intrinsic::x86_avx512_mask_scatter_qps_512;
15002 IID = Intrinsic::x86_avx512_mask_scatter_dpq_512;
15005 IID = Intrinsic::x86_avx512_mask_scatter_dpi_512;
15008 IID = Intrinsic::x86_avx512_mask_scatter_qpq_512;
15011 IID = Intrinsic::x86_avx512_mask_scatter_qpi_512;
15014 IID = Intrinsic::x86_avx512_mask_scatterdiv2_df;
15017 IID = Intrinsic::x86_avx512_mask_scatterdiv2_di;
15020 IID = Intrinsic::x86_avx512_mask_scatterdiv4_df;
15023 IID = Intrinsic::x86_avx512_mask_scatterdiv4_di;
15026 IID = Intrinsic::x86_avx512_mask_scatterdiv4_sf;
15029 IID = Intrinsic::x86_avx512_mask_scatterdiv4_si;
15032 IID = Intrinsic::x86_avx512_mask_scatterdiv8_sf;
15035 IID = Intrinsic::x86_avx512_mask_scatterdiv8_si;
15038 IID = Intrinsic::x86_avx512_mask_scattersiv2_df;
15041 IID = Intrinsic::x86_avx512_mask_scattersiv2_di;
15044 IID = Intrinsic::x86_avx512_mask_scattersiv4_df;
15047 IID = Intrinsic::x86_avx512_mask_scattersiv4_di;
15050 IID = Intrinsic::x86_avx512_mask_scattersiv4_sf;
15053 IID = Intrinsic::x86_avx512_mask_scattersiv4_si;
15056 IID = Intrinsic::x86_avx512_mask_scattersiv8_sf;
15059 IID = Intrinsic::x86_avx512_mask_scattersiv8_si;
15164 Intrinsic::ID IID;
15168 IID = Intrinsic::x86_avx512_mask_pmov_db_512;
15171 IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
15174 IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
15705 Intrinsic::ID IID;
15709 IID = Intrinsic::x86_avx512_ktestc_b;
15712 IID = Intrinsic::x86_avx512_ktestz_b;
15715 IID = Intrinsic::x86_avx512_ktestc_w;
15718 IID = Intrinsic::x86_avx512_ktestz_w;
15721 IID = Intrinsic::x86_avx512_ktestc_d;
15724 IID = Intrinsic::x86_avx512_ktestz_d;
15727 IID = Intrinsic::x86_avx512_ktestc_q;
15730 IID = Intrinsic::x86_avx512_ktestz_q;
15745 Intrinsic::ID IID;
15749 IID = Intrinsic::x86_avx512_kadd_b;
15752 IID = Intrinsic::x86_avx512_kadd_w;
15755 IID = Intrinsic::x86_avx512_kadd_d;
15758 IID = Intrinsic::x86_avx512_kadd_q;
15842 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
15851 F = CGM.getIntrinsic(Intrinsic::experimental_constrained_sqrt,
15855 F = CGM.getIntrinsic(Intrinsic::sqrt, A->getType());
15867 Intrinsic::ID IID;
15873 IID = Intrinsic::x86_avx512fp16_mask_sqrt_sh;
15876 IID = Intrinsic::x86_avx512_mask_sqrt_sd;
15879 IID = Intrinsic::x86_avx512_mask_sqrt_ss;
15888 F = CGM.getIntrinsic(Intrinsic::experimental_constrained_sqrt,
15892 F = CGM.getIntrinsic(Intrinsic::sqrt, A->getType());
15913 Intrinsic::ID IID;
15919 IID = Intrinsic::x86_avx512fp16_sqrt_ph_512;
15922 IID = Intrinsic::x86_avx512_sqrt_ps_512;
15925 IID = Intrinsic::x86_avx512_sqrt_pd_512;
15933 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_sqrt,
15937 Function *F = CGM.getIntrinsic(Intrinsic::sqrt, Ops[0]->getType());
16021 CGM.getIntrinsic(Intrinsic::vector_reduce_fadd, Ops[1]->getType());
16032 CGM.getIntrinsic(Intrinsic::vector_reduce_fmul, Ops[1]->getType());
16043 CGM.getIntrinsic(Intrinsic::vector_reduce_fmax, Ops[0]->getType());
16054 CGM.getIntrinsic(Intrinsic::vector_reduce_fmin, Ops[0]->getType());
16066 Intrinsic::ID ID;
16070 ID = Intrinsic::x86_rdrand_16;
16073 ID = Intrinsic::x86_rdrand_32;
16076 ID = Intrinsic::x86_rdrand_64;
16079 ID = Intrinsic::x86_rdseed_16;
16082 ID = Intrinsic::x86_rdseed_32;
16085 ID = Intrinsic::x86_rdseed_64;
16098 Intrinsic::ID IID;
16102 IID = Intrinsic::x86_addcarry_32;
16105 IID = Intrinsic::x86_addcarry_64;
16108 IID = Intrinsic::x86_subborrow_32;
16111 IID = Intrinsic::x86_subborrow_64;
16136 Intrinsic::ID ID;
16140 ID = Intrinsic::x86_avx512fp16_fpclass_ph_128;
16143 ID = Intrinsic::x86_avx512fp16_fpclass_ph_256;
16146 ID = Intrinsic::x86_avx512fp16_fpclass_ph_512;
16149 ID = Intrinsic::x86_avx512_fpclass_ps_128;
16152 ID = Intrinsic::x86_avx512_fpclass_ps_256;
16155 ID = Intrinsic::x86_avx512_fpclass_ps_512;
16158 ID = Intrinsic::x86_avx512_fpclass_pd_128;
16161 ID = Intrinsic::x86_avx512_fpclass_pd_256;
16164 ID = Intrinsic::x86_avx512_fpclass_pd_512;
16180 Intrinsic::ID ID;
16185 ID = Intrinsic::x86_avx512_vp2intersect_q_512;
16188 ID = Intrinsic::x86_avx512_vp2intersect_q_256;
16191 ID = Intrinsic::x86_avx512_vp2intersect_q_128;
16194 ID = Intrinsic::x86_avx512_vp2intersect_d_512;
16197 ID = Intrinsic::x86_avx512_vp2intersect_d_256;
16200 ID = Intrinsic::x86_avx512_vp2intersect_d_128;
16217 Intrinsic::ID ID;
16221 ID = Intrinsic::x86_avx512_pmultishift_qb_128;
16224 ID = Intrinsic::x86_avx512_pmultishift_qb_256;
16227 ID = Intrinsic::x86_avx512_pmultishift_qb_512;
16242 Intrinsic::ID ID;
16246 ID = Intrinsic::x86_avx512_vpshufbitqmb_128;
16249 ID = Intrinsic::x86_avx512_vpshufbitqmb_256;
16252 ID = Intrinsic::x86_avx512_vpshufbitqmb_512;
16349 Intrinsic::ID IID;
16353 IID = Intrinsic::x86_sse_cmp_ps;
16356 IID = Intrinsic::x86_avx_cmp_ps_256;
16359 IID = Intrinsic::x86_sse2_cmp_pd;
16362 IID = Intrinsic::x86_avx_cmp_pd_256;
16365 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_128;
16368 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_256;
16371 IID = Intrinsic::x86_avx512fp16_mask_cmp_ph_512;
16374 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
16377 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
16380 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
16383 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
16386 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
16389 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
16427 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
16429 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
16431 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
16433 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
16435 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
16437 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
16439 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
16441 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
16443 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
16445 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
16447 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
16449 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
16451 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
16453 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
16455 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
16457 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
16474 Intrinsic::ID IID = Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128;
16482 Intrinsic::ID IID;
16486 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_256;
16489 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_512;
16580 BuiltinID == X86::BI__shiftleft128 ? Intrinsic::fshl : Intrinsic::fshr,
16598 CGM.getIntrinsic(Intrinsic::addressofreturnaddress, AllocaInt8PtrTy);
16608 return EmitTrapCall(Intrinsic::trap);
16646 Intrinsic::ID IID = Intrinsic::x86_encodekey128;
16659 Intrinsic::ID IID = Intrinsic::x86_encodekey256;
16676 Intrinsic::ID IID;
16682 IID = Intrinsic::x86_aesenc128kl;
16686 IID = Intrinsic::x86_aesdec128kl;
16690 IID = Intrinsic::x86_aesenc256kl;
16694 IID = Intrinsic::x86_aesdec256kl;
16727 Intrinsic::ID IID;
16731 IID = Intrinsic::x86_aesencwide128kl;
16735 IID = Intrinsic::x86_aesdecwide128kl;
16739 IID = Intrinsic::x86_aesencwide256kl;
16743 IID = Intrinsic::x86_aesdecwide256kl;
16791 Intrinsic::ID IID = IsConjFMA
16792 ? Intrinsic::x86_avx512fp16_mask_vfcmadd_cph_512
16793 : Intrinsic::x86_avx512fp16_mask_vfmadd_cph_512;
16801 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
16802 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
16811 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
16812 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
16819 CGM.getIntrinsic(Intrinsic::prefetch, Ops[0]->getType()),
16839 Intrinsic::ID ID = Intrinsic::not_intrinsic;
16929 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_fixed_addr_ld);
16969 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_fixed_addr_ld);
16982 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter));
17011 ID = Intrinsic::ppc_altivec_lvx;
17014 ID = Intrinsic::ppc_altivec_lvxl;
17017 ID = Intrinsic::ppc_altivec_lvebx;
17020 ID = Intrinsic::ppc_altivec_lvehx;
17023 ID = Intrinsic::ppc_altivec_lvewx;
17026 ID = Intrinsic::ppc_altivec_lvsl;
17029 ID = Intrinsic::ppc_altivec_lvsr;
17032 ID = Intrinsic::ppc_vsx_lxvd2x;
17035 ID = Intrinsic::ppc_vsx_lxvw4x;
17038 ID = Intrinsic::ppc_vsx_lxvd2x_be;
17041 ID = Intrinsic::ppc_vsx_lxvw4x_be;
17044 ID = Intrinsic::ppc_vsx_lxvl;
17047 ID = Intrinsic::ppc_vsx_lxvll;
17080 ID = Intrinsic::ppc_altivec_stvx;
17083 ID = Intrinsic::ppc_altivec_stvxl;
17086 ID = Intrinsic::ppc_altivec_stvebx;
17089 ID = Intrinsic::ppc_altivec_stvehx;
17092 ID = Intrinsic::ppc_altivec_stvewx;
17095 ID = Intrinsic::ppc_vsx_stxvd2x;
17098 ID = Intrinsic::ppc_vsx_stxvw4x;
17101 ID = Intrinsic::ppc_vsx_stxvd2x_be;
17104 ID = Intrinsic::ppc_vsx_stxvw4x_be;
17107 ID = Intrinsic::ppc_vsx_stxvl;
17110 ID = Intrinsic::ppc_vsx_stxvll;
17140 llvm::Function *Lvx = CGM.getIntrinsic(Intrinsic::ppc_altivec_lvx);
17141 llvm::Function *Lvs = CGM.getIntrinsic(IsLE ? Intrinsic::ppc_altivec_lvsr
17142 : Intrinsic::ppc_altivec_lvsl);
17143 llvm::Function *Vperm = CGM.getIntrinsic(Intrinsic::ppc_altivec_vperm);
17220 Function *F = CGM.getIntrinsic(Intrinsic::bswap, ConvTy);
17260 Intrinsic::experimental_constrained_sqrt, ResultType);
17263 llvm::Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType);
17275 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
17285 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
17337 ID = Is32bit ? Intrinsic::ppc_altivec_vinsw : Intrinsic::ppc_altivec_vinsd;
17354 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
17379 ? Intrinsic::ppc_altivec_vaddcuq
17380 : Intrinsic::ppc_altivec_vsubcuq;
17400 ID = Intrinsic::ppc_altivec_vaddeuqm;
17403 ID = Intrinsic::ppc_altivec_vaddecuq;
17406 ID = Intrinsic::ppc_altivec_vsubeuqm;
17409 ID = Intrinsic::ppc_altivec_vsubecuq;
17424 Function *F = CGM.getIntrinsic(Intrinsic::fshl, Op0->getType());
17432 ? Intrinsic::ppc_rldimi
17433 : Intrinsic::ppc_rlwimi),
17440 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_rlwnm),
17447 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
17462 CGM.getIntrinsic(Intrinsic::ppc_cmpb, {Int64Ty, Int64Ty, Int64Ty});
17480 CGM.getIntrinsic(Intrinsic::ppc_cmpb, {Int32Ty, Int32Ty, Int32Ty});
17501 ID = Intrinsic::copysign;
17521 ? Intrinsic::experimental_constrained_floor
17522 : Intrinsic::floor;
17526 ? Intrinsic::experimental_constrained_round
17527 : Intrinsic::round;
17531 ? Intrinsic::experimental_constrained_rint
17532 : Intrinsic::rint;
17536 ? Intrinsic::experimental_constrained_ceil
17537 : Intrinsic::ceil;
17541 ? Intrinsic::experimental_constrained_trunc
17542 : Intrinsic::trunc;
17553 llvm::Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
17575 llvm::Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType);
17599 Function *F = CGM.getIntrinsic(Intrinsic::fshl, Ty);
17604 Function *F = CGM.getIntrinsic(Intrinsic::ppc_load2r);
17626 F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
17628 F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
17661 CGM.getIntrinsic(Intrinsic::ppc_fnmsub, ResultType), {X, Y, Z});
17671 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxinsertw);
17701 // Intrinsic expects the first arg to be a vector of int.
17710 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
17712 // Intrinsic expects the first argument to be a vector of doublewords.
17830 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_sthcx);
17861 auto Intrinsic = Intrinsic::ppc_vsx_disassemble_pair;
17864 Intrinsic = Intrinsic::ppc_mma_disassemble_acc;
17866 llvm::Function *F = CGM.getIntrinsic(Intrinsic);
17895 ID = Intrinsic::ppc_##Intr; \
17980 Function *F = CGM.getIntrinsic(Intrinsic::ppc_mfspr, RetType);
17989 Function *F = CGM.getIntrinsic(Intrinsic::ppc_mtspr, RetType);
17995 Function *F = CGM.getIntrinsic(Intrinsic::ppc_popcntb, {ArgType, ArgType});
18004 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_mtfsf);
18020 *this, E, Intrinsic::rint,
18021 Intrinsic::experimental_constrained_rint))
18026 *this, E, Intrinsic::floor,
18027 Intrinsic::experimental_constrained_floor))
18032 *this, E, Intrinsic::round,
18033 Intrinsic::experimental_constrained_round))
18038 *this, E, Intrinsic::ceil,
18039 Intrinsic::experimental_constrained_ceil))
18044 *this, E, Intrinsic::trunc,
18045 Intrinsic::experimental_constrained_trunc))
18050 *this, E, Intrinsic::sqrt,
18051 Intrinsic::experimental_constrained_sqrt))
18057 CGM.getIntrinsic(Intrinsic::ppc_test_data_class, Op0->getType()),
18065 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_maxfe),
18073 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_maxfl),
18081 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_maxfs),
18089 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_minfe),
18097 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_minfl),
18105 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_minfs),
18115 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_setrnd),
18118 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_readflm));
18127 auto *F = CGF.CGM.getIntrinsic(Intrinsic::amdgcn_dispatch_ptr);
18142 auto *F = CGF.CGM.getIntrinsic(Intrinsic::amdgcn_implicitarg_ptr);
18317 Intrinsic::ID getDotProductIntrinsic(QualType QT, int elementCount) {
18321 return Intrinsic::dx_dot2;
18323 return Intrinsic::dx_dot3;
18325 return Intrinsic::dx_dot4;
18329 return Intrinsic::dx_sdot;
18332 return Intrinsic::dx_udot;
18367 IsUnsigned ? Intrinsic::dx_uclamp : Intrinsic::dx_clamp,
18422 /*ReturnType=*/Op0->getType(), Intrinsic::dx_frac,
18436 return Builder.CreateIntrinsic(retType, Intrinsic::dx_isinf,
18445 /*ReturnType*/ M->getType(), Intrinsic::fmuladd,
18451 /*ReturnType*/ M->getType(), Intrinsic::dx_imad,
18460 /*ReturnType=*/M->getType(), Intrinsic::dx_umad,
18536 llvm::Function *Callee = CGM.getIntrinsic(Intrinsic::amdgcn_div_scale,
18557 llvm::Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_div_fmas,
18565 Intrinsic::amdgcn_ds_swizzle);
18568 Intrinsic::amdgcn_mov_dpp8);
18585 CGM.getIntrinsic(Intrinsic::amdgcn_update_dpp, Args[0]->getType());
18593 ? Intrinsic::amdgcn_permlane16
18594 : Intrinsic::amdgcn_permlanex16);
18597 Intrinsic::amdgcn_permlane64);
18600 Intrinsic::amdgcn_readlane);
18603 Intrinsic::amdgcn_readfirstlane);
18608 Intrinsic::amdgcn_div_fixup);
18611 return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_trig_preop);
18615 return emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::amdgcn_rcp);
18620 Intrinsic::amdgcn_sqrt);
18624 return emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::amdgcn_rsq);
18628 Intrinsic::amdgcn_rsq_clamp);
18631 return emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::amdgcn_sin);
18634 return emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::amdgcn_cos);
18638 return emitBuiltinWithOneOverloadedType<1>(*this, E, Intrinsic::amdgcn_log);
18641 Intrinsic::amdgcn_exp2);
18644 Intrinsic::amdgcn_log_clamp);
18650 CGM.getIntrinsic(Intrinsic::ldexp, {Src0->getType(), Src1->getType()});
18659 CGM.getIntrinsic(Intrinsic::ldexp, {Src0->getType(), Int16Ty});
18666 Intrinsic::amdgcn_frexp_mant);
18670 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp,
18676 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp,
18684 Intrinsic::amdgcn_fract);
18687 Intrinsic::amdgcn_lerp);
18690 Intrinsic::amdgcn_ubfe);
18693 Intrinsic::amdgcn_sbfe);
18698 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_ballot, { ResultType });
18710 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_icmp,
18721 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_fcmp,
18728 return emitFPIntBuiltin(*this, E, Intrinsic::amdgcn_class);
18732 Intrinsic::amdgcn_fmed3);
18735 Intrinsic::ID Intrin = BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_append ?
18736 Intrinsic::amdgcn_ds_append : Intrinsic::amdgcn_ds_consume;
18751 Intrinsic::ID IID;
18756 IID = Intrinsic::amdgcn_global_atomic_fadd;
18761 IID = Intrinsic::amdgcn_global_atomic_fadd;
18764 IID = Intrinsic::amdgcn_global_atomic_fadd;
18767 IID = Intrinsic::amdgcn_global_atomic_fmin;
18770 IID = Intrinsic::amdgcn_global_atomic_fmax;
18773 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18776 IID = Intrinsic::amdgcn_flat_atomic_fmin;
18779 IID = Intrinsic::amdgcn_flat_atomic_fmax;
18783 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18788 IID = Intrinsic::amdgcn_flat_atomic_fadd;
18799 Intrinsic::ID IID;
18802 IID = Intrinsic::amdgcn_global_atomic_fadd_v2bf16;
18805 IID = Intrinsic::amdgcn_flat_atomic_fadd_v2bf16;
18822 Intrinsic::ID IID;
18826 IID = Intrinsic::amdgcn_global_load_tr_b64;
18834 IID = Intrinsic::amdgcn_global_load_tr_b128;
18843 Function *F = CGM.getIntrinsic(Intrinsic::get_fpenv,
18848 Function *F = CGM.getIntrinsic(Intrinsic::set_fpenv,
18879 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_image_bvh_intersect_ray,
18890 Function *F = CGM.getIntrinsic(Intrinsic::amdgcn_ds_bvh_stack_rtn);
18981 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_f16;
18988 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf16;
18997 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16;
19006 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16;
19011 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied;
19016 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16_tied;
19023 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu8;
19030 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu4;
19035 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_fp8;
19040 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_bf8;
19045 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_fp8;
19050 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_bf8;
19055 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x32_iu4;
19060 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_f16;
19065 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16;
19070 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f16_16x16x32_f16;
19075 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16;
19080 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8;
19085 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4;
19090 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4;
19095 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8;
19100 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8;
19105 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8;
19110 BuiltinWMMAOp = Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8;
19130 return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_x, 0, 1024);
19132 return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_y, 0, 1024);
19134 return emitRangedBuiltin(*this, Intrinsic::amdgcn_workitem_id_z, 0, 1024);
19156 Intrinsic::r600_recipsqrt_ieee);
19158 return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_x, 0, 1024);
19160 return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_y, 0, 1024);
19162 return emitRangedBuiltin(*this, Intrinsic::r600_read_tidig_z, 0, 1024);
19167 Function *F = CGM.getIntrinsic(Intrinsic::fshr, Src0->getType());
19262 CGM.getIntrinsic(Intrinsic::amdgcn_s_sendmsg_rtn, {ResultType});
19267 *this, E, Intrinsic::amdgcn_make_buffer_rsrc);
19275 *this, E, Intrinsic::amdgcn_raw_ptr_buffer_store);
19304 CGM.getIntrinsic(Intrinsic::amdgcn_raw_ptr_buffer_load, RetTy);
19338 Function *F = CGM.getIntrinsic(Intrinsic::s390_tbegin);
19344 Function *F = CGM.getIntrinsic(Intrinsic::s390_tbegin_nofloat);
19350 Function *F = CGM.getIntrinsic(Intrinsic::s390_tbeginc);
19355 Function *F = CGM.getIntrinsic(Intrinsic::s390_tabort);
19361 Function *F = CGM.getIntrinsic(Intrinsic::s390_ntstg);
19376 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
19387 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
19398 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
19413 Function *F = CGM.getIntrinsic(Intrinsic::fshl, ResultType);
19424 Function *F = CGM.getIntrinsic(Intrinsic::fshl, ResultType);
19433 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_sqrt, ResultType);
19436 Function *F = CGM.getIntrinsic(Intrinsic::sqrt, ResultType);
19447 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
19450 Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
19461 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
19464 Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
19475 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
19478 Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
19489 Function *F = CGM.getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
19493 Function *F = CGM.getIntrinsic(Intrinsic::fma, ResultType);
19502 Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
19509 Function *F = CGM.getIntrinsic(Intrinsic::fabs, ResultType);
19521 Intrinsic::ID ID = Intrinsic::not_intrinsic;
19522 Intrinsic::ID CI;
19528 case 0: ID = Intrinsic::rint;
19529 CI = Intrinsic::experimental_constrained_rint; break;
19535 case 0: ID = Intrinsic::nearbyint;
19536 CI = Intrinsic::experimental_constrained_nearbyint; break;
19537 case 1: ID = Intrinsic::round;
19538 CI = Intrinsic::experimental_constrained_round; break;
19539 case 5: ID = Intrinsic::trunc;
19540 CI = Intrinsic::experimental_constrained_trunc; break;
19541 case 6: ID = Intrinsic::ceil;
19542 CI = Intrinsic::experimental_constrained_ceil; break;
19543 case 7: ID = Intrinsic::floor;
19544 CI = Intrinsic::experimental_constrained_floor; break;
19548 if (ID != Intrinsic::not_intrinsic) {
19558 case SystemZ::BI__builtin_s390_vfisb: ID = Intrinsic::s390_vfisb; break;
19559 case SystemZ::BI__builtin_s390_vfidb: ID = Intrinsic::s390_vfidb; break;
19576 Intrinsic::ID ID = Intrinsic::not_intrinsic;
19577 Intrinsic::ID CI;
19580 case 4: ID = Intrinsic::maxnum;
19581 CI = Intrinsic::experimental_constrained_maxnum; break;
19583 if (ID != Intrinsic::not_intrinsic) {
19593 case SystemZ::BI__builtin_s390_vfmaxsb: ID = Intrinsic::s390_vfmaxsb; break;
19594 case SystemZ::BI__builtin_s390_vfmaxdb: ID = Intrinsic::s390_vfmaxdb; break;
19610 Intrinsic::ID ID = Intrinsic::not_intrinsic;
19611 Intrinsic::ID CI;
19614 case 4: ID = Intrinsic::minnum;
19615 CI = Intrinsic::experimental_constrained_minnum; break;
19617 if (ID != Intrinsic::not_intrinsic) {
19627 case SystemZ::BI__builtin_s390_vfminsb: ID = Intrinsic::s390_vfminsb; break;
19628 case SystemZ::BI__builtin_s390_vfmindb: ID = Intrinsic::s390_vfmindb; break;
19641 Function *F = CGM.getIntrinsic(Intrinsic::bswap, ResultType);
19649 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
19739 // Intrinsic IDs for row/col variants. 0 if particular layout is unsupported.
19745 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
19927 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
19928 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
19929 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
19930 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
19933 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
19934 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
19935 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
19936 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
19940 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
19944 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
19950 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
19959 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
20087 if (IntrinsicID == Intrinsic::nvvm_ldg_global_f ||
20088 IntrinsicID == Intrinsic::nvvm_ldu_global_f)
20184 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_inc_32, Ptr->getType());
20192 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_dec_32, Ptr->getType());
20228 return MakeLdgLdu(Intrinsic::nvvm_ldg_global_i, *this, E);
20234 return MakeLdgLdu(Intrinsic::nvvm_ldg_global_f, *this, E);
20265 return MakeLdgLdu(Intrinsic::nvvm_ldu_global_i, *this, E);
20271 return MakeLdgLdu(Intrinsic::nvvm_ldu_global_f, *this, E);
20276 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta, *this, E);
20280 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys, *this, E);
20283 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta, *this, E);
20286 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys, *this, E);
20290 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta, *this, E);
20294 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys, *this, E);
20301 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta, *this, E);
20308 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys, *this, E);
20315 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta, *this, E);
20322 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys, *this, E);
20324 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta, *this, E);
20326 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta, *this, E);
20328 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys, *this, E);
20330 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys, *this, E);
20334 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta, *this, E);
20338 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys, *this, E);
20342 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta, *this, E);
20346 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys, *this, E);
20350 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta, *this, E);
20354 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys, *this, E);
20363 Intrinsic::nvvm_atomic_cas_gen_i_cta, {ElemTy, Ptr->getType()}),
20374 Intrinsic::nvvm_atomic_cas_gen_i_sys, {ElemTy, Ptr->getType()}),
20384 ? Intrinsic::nvvm_match_all_sync_i32p
20385 : Intrinsic::nvvm_match_all_sync_i64p),
20504 Function *Intrinsic =
20506 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
20517 Value *Result = Builder.CreateCall(Intrinsic, Values);
20522 // Intrinsic::nvvm_wmma_m16n16k16_mma_sync<layout A,B><DType><CType><Satf>
20577 Function *Intrinsic = CGM.getIntrinsic(IID);
20578 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
20589 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
20600 Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
20609 Value *Result = Builder.CreateCall(Intrinsic, Values);
20621 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16, BuiltinID, E, *this);
20623 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16x2, BuiltinID, E, *this);
20625 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn, BuiltinID, E, *this);
20627 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn_relu, BuiltinID, E, *this);
20629 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz, BuiltinID, E, *this);
20631 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz_relu, BuiltinID, E, *this);
20633 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16, BuiltinID, E, *this);
20635 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16x2, BuiltinID, E, *this);
20637 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16, BuiltinID, E, *this);
20639 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16x2, BuiltinID, E, *this);
20641 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16, BuiltinID, E,
20644 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16x2, BuiltinID, E,
20647 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16, BuiltinID, E,
20650 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16x2, BuiltinID, E,
20653 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16, BuiltinID, E, *this);
20655 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16x2, BuiltinID, E, *this);
20657 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16, BuiltinID, E, *this);
20659 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16x2, BuiltinID, E, *this);
20661 return MakeHalfType(Intrinsic::nvvm_fmax_f16, BuiltinID, E, *this);
20663 return MakeHalfType(Intrinsic::nvvm_fmax_f16x2, BuiltinID, E, *this);
20665 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16, BuiltinID, E, *this);
20667 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16x2, BuiltinID, E, *this);
20669 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16, BuiltinID, E, *this);
20671 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16x2, BuiltinID, E,
20674 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16, BuiltinID,
20677 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16x2,
20680 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16, BuiltinID, E,
20683 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16x2, BuiltinID,
20686 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16, BuiltinID, E, *this);
20688 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16x2, BuiltinID, E, *this);
20690 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16, BuiltinID, E,
20693 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16x2, BuiltinID,
20696 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16, BuiltinID, E,
20699 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16x2, BuiltinID, E,
20702 return MakeHalfType(Intrinsic::nvvm_fmin_f16, BuiltinID, E, *this);
20704 return MakeHalfType(Intrinsic::nvvm_fmin_f16x2, BuiltinID, E, *this);
20706 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16, BuiltinID, E, *this);
20708 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16x2, BuiltinID, E, *this);
20710 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16, BuiltinID, E, *this);
20712 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16x2, BuiltinID, E,
20715 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16, BuiltinID,
20718 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16x2,
20721 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16, BuiltinID, E,
20724 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16x2, BuiltinID,
20727 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16, BuiltinID, E, *this);
20729 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16x2, BuiltinID, E, *this);
20731 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16, BuiltinID, E,
20734 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16x2, BuiltinID,
20737 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16, BuiltinID, E,
20740 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID, E,
20743 return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID, E, *this);
20745 return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID, E, *this);
20747 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID, E, *this);
20749 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID, E, *this);
20752 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_4,
20753 Intrinsic::nvvm_cp_async_ca_shared_global_4_s, *this, E,
20756 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_8,
20757 Intrinsic::nvvm_cp_async_ca_shared_global_8_s, *this, E,
20760 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_16,
20761 Intrinsic::nvvm_cp_async_ca_shared_global_16_s, *this, E,
20764 return MakeCpAsync(Intrinsic::nvvm_cp_async_cg_shared_global_16,
20765 Intrinsic::nvvm_cp_async_cg_shared_global_16_s, *this, E,
20769 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_x));
20772 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_y));
20775 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_z));
20778 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_clusterid_w));
20781 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_x));
20784 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_y));
20787 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_z));
20790 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_nclusterid_w));
20793 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_x));
20796 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_y));
20799 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_z));
20802 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctaid_w));
20805 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_x));
20808 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_y));
20811 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_z));
20814 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctaid_w));
20817 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_ctarank));
20820 CGM.getIntrinsic(Intrinsic::nvvm_read_ptx_sreg_cluster_nctarank));
20823 CGM.getIntrinsic(Intrinsic::nvvm_is_explicit_cluster));
20826 CGM.getIntrinsic(Intrinsic::nvvm_isspacep_shared_cluster),
20830 CGM.getIntrinsic(Intrinsic::nvvm_mapa),
20834 CGM.getIntrinsic(Intrinsic::nvvm_mapa_shared_cluster),
20838 CGM.getIntrinsic(Intrinsic::nvvm_getctarank),
20842 CGM.getIntrinsic(Intrinsic::nvvm_getctarank_shared_cluster),
20846 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cluster_arrive));
20849 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cluster_arrive_relaxed));
20852 CGM.getIntrinsic(Intrinsic::nvvm_barrier_cluster_wait));
20855 CGM.getIntrinsic(Intrinsic::nvvm_fence_sc_cluster));
20933 Intrinsic::ptrmask, {Args.SrcType, Args.IntType},
20949 CGM.getIntrinsic(Intrinsic::wasm_memory_size, ResultType);
20957 CGM.getIntrinsic(Intrinsic::wasm_memory_grow, ResultType);
20962 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_tls_size, ResultType);
20967 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_tls_align, ResultType);
20971 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_tls_base);
20977 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_throw);
20981 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_rethrow);
20988 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_atomic_wait32);
20995 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_atomic_wait64);
21001 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_atomic_notify);
21011 CGM.getIntrinsic(Intrinsic::wasm_trunc_signed, {ResT, Src->getType()});
21020 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_trunc_unsigned,
21032 CGM.getIntrinsic(Intrinsic::fptosi_sat, {ResT, Src->getType()});
21043 CGM.getIntrinsic(Intrinsic::fptoui_sat, {ResT, Src->getType()});
21054 CGM.getIntrinsic(Intrinsic::minimum, ConvertType(E->getType()));
21065 CGM.getIntrinsic(Intrinsic::maximum, ConvertType(E->getType()));
21074 CGM.getIntrinsic(Intrinsic::wasm_pmin, ConvertType(E->getType()));
21083 CGM.getIntrinsic(Intrinsic::wasm_pmax, ConvertType(E->getType()));
21098 IntNo = Intrinsic::ceil;
21102 IntNo = Intrinsic::floor;
21106 IntNo = Intrinsic::trunc;
21110 IntNo = Intrinsic::nearbyint;
21120 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_ref_null_extern);
21124 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_ref_null_func);
21130 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_swizzle);
21145 IntNo = Intrinsic::sadd_sat;
21149 IntNo = Intrinsic::uadd_sat;
21153 IntNo = Intrinsic::wasm_sub_sat_signed;
21157 IntNo = Intrinsic::wasm_sub_sat_unsigned;
21222 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_avgr_unsigned,
21229 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_q15mulr_sat_signed);
21241 IntNo = Intrinsic::wasm_extadd_pairwise_signed;
21245 IntNo = Intrinsic::wasm_extadd_pairwise_unsigned;
21259 CGM.getIntrinsic(Intrinsic::wasm_bitselect, ConvertType(E->getType()));
21265 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_dot);
21271 CGM.getIntrinsic(Intrinsic::ctpop, ConvertType(E->getType()));
21282 IntNo = Intrinsic::wasm_anytrue;
21288 IntNo = Intrinsic::wasm_alltrue;
21303 CGM.getIntrinsic(Intrinsic::wasm_bitmask, Vec->getType());
21309 Function *Callee = CGM.getIntrinsic(Intrinsic::fabs, Vec->getType());
21315 Function *Callee = CGM.getIntrinsic(Intrinsic::sqrt, Vec->getType());
21328 IntNo = Intrinsic::wasm_narrow_signed;
21332 IntNo = Intrinsic::wasm_narrow_unsigned;
21347 IntNo = Intrinsic::fptosi_sat;
21350 IntNo = Intrinsic::fptoui_sat;
21373 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_shuffle);
21390 IntNo = Intrinsic::wasm_relaxed_madd;
21395 IntNo = Intrinsic::wasm_relaxed_nmadd;
21411 CGM.getIntrinsic(Intrinsic::wasm_relaxed_laneselect, A->getType());
21417 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_relaxed_swizzle);
21430 IntNo = Intrinsic::wasm_relaxed_min;
21434 IntNo = Intrinsic::wasm_relaxed_max;
21450 IntNo = Intrinsic::wasm_relaxed_trunc_signed;
21453 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned;
21456 IntNo = Intrinsic::wasm_relaxed_trunc_signed_zero;
21459 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned_zero;
21470 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_relaxed_q15mulr_signed);
21477 CGM.getIntrinsic(Intrinsic::wasm_relaxed_dot_i8x16_i7x16_signed);
21485 CGM.getIntrinsic(Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed);
21493 CGM.getIntrinsic(Intrinsic::wasm_relaxed_dot_bf16x8_add_f32);
21498 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_loadf16_f32);
21504 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_storef16_f32);
21509 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_splat_f16x8);
21515 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_extract_lane_f16x8);
21524 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_get_externref);
21526 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_get_funcref);
21539 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_set_externref);
21541 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_set_funcref);
21550 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_table_size);
21561 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_grow_externref);
21563 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_fill_funcref);
21579 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_fill_externref);
21581 Callee = CGM.getIntrinsic(Intrinsic::wasm_table_fill_funcref);
21596 Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_table_copy);
21605 static std::pair<Intrinsic::ID, unsigned>
21609 Intrinsic::ID IntrinsicID;
21614 { Hexagon::BI__builtin_HEXAGON_##x, Intrinsic::hexagon_##x, s },
21656 return {Intrinsic::not_intrinsic, 0};
21663 Intrinsic::ID ID;
21734 Intrinsic::ID ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandvrt_128B
21735 : Intrinsic::hexagon_V6_vandvrt;
21740 Intrinsic::ID ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandqrt_128B
21741 : Intrinsic::hexagon_V6_vandqrt;
21833 return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr, Int8Ty);
21835 return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr, Int8Ty);
21837 return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr, Int16Ty);
21839 return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr, Int16Ty);
21841 return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr, Int32Ty);
21843 return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr, Int64Ty);
21887 Intrinsic::ID ID = Intrinsic::not_intrinsic;
21925 ID = Intrinsic::riscv_orc_b;
21929 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
21938 Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType());
21949 ID = Intrinsic::riscv_clmul;
21953 ID = Intrinsic::riscv_clmulh;
21957 ID = Intrinsic::riscv_clmulr;
21963 ID = Intrinsic::riscv_xperm8;
21967 ID = Intrinsic::riscv_xperm4;
21973 ID = Intrinsic::riscv_brev8;
21976 ID = Intrinsic::riscv_zip;
21979 ID = Intrinsic::riscv_unzip;
21991 ID = Intrinsic::riscv_sha256sig0;
21994 ID = Intrinsic::riscv_sha256sig1;
21997 ID = Intrinsic::riscv_sha256sum0;
22000 ID = Intrinsic::riscv_sha256sum1;
22005 ID = Intrinsic::riscv_sm4ks;
22008 ID = Intrinsic::riscv_sm4ed;
22013 ID = Intrinsic::riscv_sm3p0;
22016 ID = Intrinsic::riscv_sm3p1;
22074 assert(ID != Intrinsic::not_intrinsic);