Lines Matching refs:Ops
5571 Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops, in EmitNeonCall() argument
5581 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift); in EmitNeonCall()
5583 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name); in EmitNeonCall()
5587 return Builder.CreateConstrainedFPCall(F, Ops, name); in EmitNeonCall()
5589 return Builder.CreateCall(F, Ops, name); in EmitNeonCall()
6737 SmallVectorImpl<Value *> &Ops, const CallExpr *E) { in EmitCommonNeonSISDBuiltinExpr() argument
6759 std::swap(Ops[0], Ops[1]); in EmitCommonNeonSISDBuiltinExpr()
6775 if (Ops[j]->getType()->getPrimitiveSizeInBits() == in EmitCommonNeonSISDBuiltinExpr()
6779 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy()); in EmitCommonNeonSISDBuiltinExpr()
6782 Ops[j] = CGF.Builder.CreateTruncOrBitCast( in EmitCommonNeonSISDBuiltinExpr()
6783 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType()); in EmitCommonNeonSISDBuiltinExpr()
6784 Ops[j] = in EmitCommonNeonSISDBuiltinExpr()
6785 CGF.Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0); in EmitCommonNeonSISDBuiltinExpr()
6788 Value *Result = CGF.EmitNeonCall(F, Ops, s); in EmitCommonNeonSISDBuiltinExpr()
6800 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1, in EmitCommonNeonBuiltinExpr() argument
6843 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
6844 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements); in EmitCommonNeonBuiltinExpr()
6856 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, Ty), Ops, "vabs"); in EmitCommonNeonBuiltinExpr()
6857 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops, "vabs"); in EmitCommonNeonBuiltinExpr()
6861 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
6862 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitCommonNeonBuiltinExpr()
6863 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
6864 return Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
6871 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
6872 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); in EmitCommonNeonBuiltinExpr()
6873 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn"); in EmitCommonNeonBuiltinExpr()
6878 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn"); in EmitCommonNeonBuiltinExpr()
6881 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn"); in EmitCommonNeonBuiltinExpr()
6887 std::swap(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
6909 return EmitNeonCall(F, Ops, NameHint); in EmitCommonNeonBuiltinExpr()
6913 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ, in EmitCommonNeonBuiltinExpr()
6917 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE, in EmitCommonNeonBuiltinExpr()
6921 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE, in EmitCommonNeonBuiltinExpr()
6925 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT, in EmitCommonNeonBuiltinExpr()
6929 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT, in EmitCommonNeonBuiltinExpr()
6935 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef())); in EmitCommonNeonBuiltinExpr()
6939 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
6942 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
6943 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
6948 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
6951 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
6952 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
6959 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
6968 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
6984 return EmitNeonCall(F, Ops, "vcvt_n"); in EmitCommonNeonBuiltinExpr()
6998 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type)); in EmitCommonNeonBuiltinExpr()
6999 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") in EmitCommonNeonBuiltinExpr()
7000 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt"); in EmitCommonNeonBuiltinExpr()
7051 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7055 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7060 int CV = cast<ConstantInt>(Ops[2])->getSExtValue(); in EmitCommonNeonBuiltinExpr()
7065 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7066 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7067 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext"); in EmitCommonNeonBuiltinExpr()
7071 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7072 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7073 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
7078 {Ops[1], Ops[2], Ops[0]}); in EmitCommonNeonBuiltinExpr()
7083 Ops.push_back(getAlignmentValue32(PtrOp0)); in EmitCommonNeonBuiltinExpr()
7084 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vld1"); in EmitCommonNeonBuiltinExpr()
7093 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitCommonNeonBuiltinExpr()
7096 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN"); in EmitCommonNeonBuiltinExpr()
7097 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitCommonNeonBuiltinExpr()
7098 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7099 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
7116 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint); in EmitCommonNeonBuiltinExpr()
7117 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitCommonNeonBuiltinExpr()
7118 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7119 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
7127 Ops[0] = Builder.CreateInsertElement(V, Ld, CI); in EmitCommonNeonBuiltinExpr()
7128 return EmitNeonSplat(Ops[0], CI); in EmitCommonNeonBuiltinExpr()
7138 for (unsigned I = 2; I < Ops.size() - 1; ++I) in EmitCommonNeonBuiltinExpr()
7139 Ops[I] = Builder.CreateBitCast(Ops[I], Ty); in EmitCommonNeonBuiltinExpr()
7140 Ops.push_back(getAlignmentValue32(PtrOp1)); in EmitCommonNeonBuiltinExpr()
7141 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), NameHint); in EmitCommonNeonBuiltinExpr()
7142 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitCommonNeonBuiltinExpr()
7143 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7144 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitCommonNeonBuiltinExpr()
7149 Ops[0] = Builder.CreateBitCast(Ops[0], DTy); in EmitCommonNeonBuiltinExpr()
7151 return Builder.CreateZExt(Ops[0], Ty, "vmovl"); in EmitCommonNeonBuiltinExpr()
7152 return Builder.CreateSExt(Ops[0], Ty, "vmovl"); in EmitCommonNeonBuiltinExpr()
7157 Ops[0] = Builder.CreateBitCast(Ops[0], QTy); in EmitCommonNeonBuiltinExpr()
7158 return Builder.CreateTrunc(Ops[0], Ty, "vmovn"); in EmitCommonNeonBuiltinExpr()
7168 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); in EmitCommonNeonBuiltinExpr()
7178 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7188 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vpaddl"); in EmitCommonNeonBuiltinExpr()
7192 SmallVector<Value *, 2> MulOps(Ops.begin() + 1, Ops.end()); in EmitCommonNeonBuiltinExpr()
7193 Ops[1] = in EmitCommonNeonBuiltinExpr()
7195 Ops.resize(2); in EmitCommonNeonBuiltinExpr()
7196 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7210 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7219 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7223 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshl_n", in EmitCommonNeonBuiltinExpr()
7227 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshlu_n", in EmitCommonNeonBuiltinExpr()
7234 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7240 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7243 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshr_n", in EmitCommonNeonBuiltinExpr()
7250 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
7254 Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false); in EmitCommonNeonBuiltinExpr()
7255 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], in EmitCommonNeonBuiltinExpr()
7260 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
7262 Ops[0] = Builder.CreateZExt(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
7264 Ops[0] = Builder.CreateSExt(Ops[0], VTy); in EmitCommonNeonBuiltinExpr()
7265 Ops[1] = EmitNeonShiftVector(Ops[1], VTy, false); in EmitCommonNeonBuiltinExpr()
7266 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n"); in EmitCommonNeonBuiltinExpr()
7271 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
7272 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy, false); in EmitCommonNeonBuiltinExpr()
7274 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
7276 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
7277 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n"); in EmitCommonNeonBuiltinExpr()
7281 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn, "vshr_n"); in EmitCommonNeonBuiltinExpr()
7297 Ops.push_back(getAlignmentValue32(PtrOp0)); in EmitCommonNeonBuiltinExpr()
7298 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
7306 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
7313 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitCommonNeonBuiltinExpr()
7314 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
7328 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitCommonNeonBuiltinExpr()
7329 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
7332 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, ""); in EmitCommonNeonBuiltinExpr()
7339 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy); in EmitCommonNeonBuiltinExpr()
7340 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy); in EmitCommonNeonBuiltinExpr()
7341 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn"); in EmitCommonNeonBuiltinExpr()
7346 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn"); in EmitCommonNeonBuiltinExpr()
7349 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn"); in EmitCommonNeonBuiltinExpr()
7353 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
7354 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7355 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
7364 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
7365 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); in EmitCommonNeonBuiltinExpr()
7372 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitCommonNeonBuiltinExpr()
7373 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7374 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); in EmitCommonNeonBuiltinExpr()
7375 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], in EmitCommonNeonBuiltinExpr()
7377 return Builder.CreateSExt(Ops[0], Ty, "vtst"); in EmitCommonNeonBuiltinExpr()
7381 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
7382 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7383 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
7391 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
7392 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); in EmitCommonNeonBuiltinExpr()
7399 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitCommonNeonBuiltinExpr()
7400 return EmitNeonCall(F, Ops, ""); in EmitCommonNeonBuiltinExpr()
7404 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitCommonNeonBuiltinExpr()
7405 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitCommonNeonBuiltinExpr()
7406 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitCommonNeonBuiltinExpr()
7415 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitCommonNeonBuiltinExpr()
7416 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); in EmitCommonNeonBuiltinExpr()
7428 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vdot"); in EmitCommonNeonBuiltinExpr()
7435 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_low"); in EmitCommonNeonBuiltinExpr()
7442 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_low"); in EmitCommonNeonBuiltinExpr()
7449 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlal_high"); in EmitCommonNeonBuiltinExpr()
7456 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vfmlsl_high"); in EmitCommonNeonBuiltinExpr()
7463 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vmmla"); in EmitCommonNeonBuiltinExpr()
7469 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusmmla"); in EmitCommonNeonBuiltinExpr()
7476 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vusdot"); in EmitCommonNeonBuiltinExpr()
7483 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vbfdot"); in EmitCommonNeonBuiltinExpr()
7488 return EmitNeonCall(F, Ops, "vcvtfp2bf"); in EmitCommonNeonBuiltinExpr()
7498 Value *Result = EmitNeonCall(F, Ops, NameHint); in EmitCommonNeonBuiltinExpr()
7529 static Value *packTBLDVectorList(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in packTBLDVectorList() argument
7539 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType()); in packTBLDVectorList()
7545 int PairPos = 0, End = Ops.size() - 1; in packTBLDVectorList()
7547 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], in packTBLDVectorList()
7548 Ops[PairPos+1], Indices, in packTBLDVectorList()
7557 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos], in packTBLDVectorList()
7631 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) }; in EmitSpecialRegisterBuiltin() local
7632 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitSpecialRegisterBuiltin()
7784 Value *Ops[2]; in EmitARMBuiltinExpr() local
7786 Ops[i] = EmitScalarExpr(E->getArg(i)); in EmitARMBuiltinExpr()
7790 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); in EmitARMBuiltinExpr()
8096 SmallVector<Value*, 4> Ops; in EmitARMBuiltinExpr() local
8127 Ops.push_back(PtrOp0.getPointer()); in EmitARMBuiltinExpr()
8154 Ops.push_back(PtrOp1.getPointer()); in EmitARMBuiltinExpr()
8160 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitARMBuiltinExpr()
8164 Ops.push_back(llvm::ConstantInt::get( in EmitARMBuiltinExpr()
8187 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane"); in EmitARMBuiltinExpr()
8207 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitARMBuiltinExpr()
8210 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops, in EmitARMBuiltinExpr()
8213 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops, in EmitARMBuiltinExpr()
8216 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops, in EmitARMBuiltinExpr()
8219 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops, in EmitARMBuiltinExpr()
8223 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vcvtbfp2bf), Ops, in EmitARMBuiltinExpr()
8234 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0], in EmitARMBuiltinExpr()
8235 Ops[3], Ops[4], Ops[5]}); in EmitARMBuiltinExpr()
8262 return Builder.CreateCall(F, Ops, "vcvtr"); in EmitARMBuiltinExpr()
8285 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch); in EmitARMBuiltinExpr()
8295 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8296 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitARMBuiltinExpr()
8298 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); in EmitARMBuiltinExpr()
8304 Value *Ld = Builder.CreateCall(F, {Ops[0], Align}); in EmitARMBuiltinExpr()
8307 return Builder.CreateShuffleVector(Ops[1], Ld, Indices, "vld1q_lane"); in EmitARMBuiltinExpr()
8311 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8314 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane"); in EmitARMBuiltinExpr()
8319 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n", in EmitARMBuiltinExpr()
8323 Ops, "vqrshrun_n", 1, true); in EmitARMBuiltinExpr()
8326 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n", in EmitARMBuiltinExpr()
8330 Ops, "vqshrun_n", 1, true); in EmitARMBuiltinExpr()
8334 Ops, "vrecpe"); in EmitARMBuiltinExpr()
8337 Ops, "vrshrn_n", 1, true); in EmitARMBuiltinExpr()
8340 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
8341 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8342 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true); in EmitARMBuiltinExpr()
8344 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]}); in EmitARMBuiltinExpr()
8345 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n"); in EmitARMBuiltinExpr()
8352 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift); in EmitARMBuiltinExpr()
8354 Ops, "vsli_n"); in EmitARMBuiltinExpr()
8357 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitARMBuiltinExpr()
8358 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); in EmitARMBuiltinExpr()
8359 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitARMBuiltinExpr()
8364 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8365 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2])); in EmitARMBuiltinExpr()
8366 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV); in EmitARMBuiltinExpr()
8367 Ops[2] = getAlignmentValue32(PtrOp0); in EmitARMBuiltinExpr()
8368 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()}; in EmitARMBuiltinExpr()
8370 Tys), Ops); in EmitARMBuiltinExpr()
8374 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitARMBuiltinExpr()
8375 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitARMBuiltinExpr()
8377 Ops[1], Builder.CreateElementBitCast(PtrOp0, Ops[1]->getType())); in EmitARMBuiltinExpr()
8382 Ops, "vtbl1"); in EmitARMBuiltinExpr()
8385 Ops, "vtbl2"); in EmitARMBuiltinExpr()
8388 Ops, "vtbl3"); in EmitARMBuiltinExpr()
8391 Ops, "vtbl4"); in EmitARMBuiltinExpr()
8394 Ops, "vtbx1"); in EmitARMBuiltinExpr()
8397 Ops, "vtbx2"); in EmitARMBuiltinExpr()
8400 Ops, "vtbx3"); in EmitARMBuiltinExpr()
8403 Ops, "vtbx4"); in EmitARMBuiltinExpr()
8551 llvm::SmallVector<Value *, 4> Ops; in EmitARMMVEBuiltinExpr() local
8570 Ops.push_back(EmitScalarExpr(Addr)); in EmitARMMVEBuiltinExpr()
8574 Value *LoadResult = Builder.CreateCall(F, Ops); in EmitARMMVEBuiltinExpr()
8588 llvm::SmallVector<Value *, 4> Ops; in EmitARMMVEBuiltinExpr() local
8592 Ops.push_back(EmitScalarExpr(Addr)); in EmitARMMVEBuiltinExpr()
8613 Ops.push_back(Builder.CreateExtractValue(Mvec, {0, i})); in EmitARMMVEBuiltinExpr()
8618 Ops.push_back(llvm::ConstantInt::get(Int32Ty, i)); in EmitARMMVEBuiltinExpr()
8619 ToReturn = Builder.CreateCall(F, Ops); in EmitARMMVEBuiltinExpr()
8620 Ops.pop_back(); in EmitARMMVEBuiltinExpr()
8641 SmallVectorImpl<Value *> &Ops, in EmitAArch64TblBuiltinExpr() argument
8698 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 1), nullptr, Ops[1], in EmitAArch64TblBuiltinExpr()
8702 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 2), nullptr, Ops[2], in EmitAArch64TblBuiltinExpr()
8706 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 3), nullptr, Ops[3], in EmitAArch64TblBuiltinExpr()
8710 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(0, 4), nullptr, Ops[4], in EmitAArch64TblBuiltinExpr()
8715 packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 1), nullptr, Ops[2], Ty, in EmitAArch64TblBuiltinExpr()
8719 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV); in EmitAArch64TblBuiltinExpr()
8722 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); in EmitAArch64TblBuiltinExpr()
8727 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 2), Ops[0], Ops[3], in EmitAArch64TblBuiltinExpr()
8732 packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 3), nullptr, Ops[4], Ty, in EmitAArch64TblBuiltinExpr()
8736 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4], in EmitAArch64TblBuiltinExpr()
8740 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]); in EmitAArch64TblBuiltinExpr()
8745 return packTBLDVectorList(CGF, ArrayRef(Ops).slice(1, 4), Ops[0], Ops[5], in EmitAArch64TblBuiltinExpr()
8779 return CGF.EmitNeonCall(F, Ops, s); in EmitAArch64TblBuiltinExpr()
8960 SmallVectorImpl<Value *> &Ops, in EmitSVEGatherLoad() argument
8971 Ops[0] = EmitSVEPredicateCast(Ops[0], OverloadedTy); in EmitSVEGatherLoad()
8974 if (Ops[1]->getType()->isVectorTy()) in EmitSVEGatherLoad()
8978 F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[1]->getType()}); in EmitSVEGatherLoad()
8989 if (Ops.size() == 2) { in EmitSVEGatherLoad()
8990 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset"); in EmitSVEGatherLoad()
8991 Ops.push_back(ConstantInt::get(Int64Ty, 0)); in EmitSVEGatherLoad()
8996 if (!TypeFlags.isByteIndexed() && Ops[1]->getType()->isVectorTy()) { in EmitSVEGatherLoad()
8999 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt)); in EmitSVEGatherLoad()
9002 Value *Call = Builder.CreateCall(F, Ops); in EmitSVEGatherLoad()
9011 SmallVectorImpl<Value *> &Ops, in EmitSVEScatterStore() argument
9019 Ops.insert(Ops.begin(), Ops.pop_back_val()); in EmitSVEScatterStore()
9022 if (Ops[2]->getType()->isVectorTy()) in EmitSVEScatterStore()
9026 F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[2]->getType()}); in EmitSVEScatterStore()
9037 if (Ops.size() == 3) { in EmitSVEScatterStore()
9038 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset"); in EmitSVEScatterStore()
9039 Ops.push_back(ConstantInt::get(Int64Ty, 0)); in EmitSVEScatterStore()
9044 Ops[0] = Builder.CreateTrunc(Ops[0], OverloadedTy); in EmitSVEScatterStore()
9051 Ops[1] = EmitSVEPredicateCast(Ops[1], OverloadedTy); in EmitSVEScatterStore()
9055 if (!TypeFlags.isByteIndexed() && Ops[2]->getType()->isVectorTy()) { in EmitSVEScatterStore()
9058 Ops[3] = Builder.CreateShl(Ops[3], Log2_32(BytesPerElt)); in EmitSVEScatterStore()
9061 return Builder.CreateCall(F, Ops); in EmitSVEScatterStore()
9065 SmallVectorImpl<Value *> &Ops, in EmitSVEGatherPrefetch() argument
9069 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType()); in EmitSVEGatherPrefetch()
9071 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType()); in EmitSVEGatherPrefetch()
9074 Ops[0] = EmitSVEPredicateCast(Ops[0], OverloadedTy); in EmitSVEGatherPrefetch()
9077 if (Ops[1]->getType()->isVectorTy()) { in EmitSVEGatherPrefetch()
9078 if (Ops.size() == 3) { in EmitSVEGatherPrefetch()
9080 Ops.push_back(ConstantInt::get(Int64Ty, 0)); in EmitSVEGatherPrefetch()
9083 std::swap(Ops[2], Ops[3]); in EmitSVEGatherPrefetch()
9089 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt)); in EmitSVEGatherPrefetch()
9094 return Builder.CreateCall(F, Ops); in EmitSVEGatherPrefetch()
9098 SmallVectorImpl<Value*> &Ops, in EmitSVEStructLoad() argument
9121 Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy); in EmitSVEStructLoad()
9122 Value *BasePtr= Builder.CreateBitCast(Ops[1], VecPtrTy); in EmitSVEStructLoad()
9125 if (Ops.size() > 2) in EmitSVEStructLoad()
9126 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]); in EmitSVEStructLoad()
9142 SmallVectorImpl<Value*> &Ops, in EmitSVEStructStore() argument
9163 Value *Predicate = EmitSVEPredicateCast(Ops[0], VTy); in EmitSVEStructStore()
9164 Value *BasePtr = Builder.CreateBitCast(Ops[1], VecPtrTy); in EmitSVEStructStore()
9167 if (Ops.size() > 3) in EmitSVEStructStore()
9168 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]); in EmitSVEStructStore()
9171 Value *Val = Ops.back(); in EmitSVEStructStore()
9191 SmallVectorImpl<Value *> &Ops, in EmitSVEPMull() argument
9196 Ops[OpNo] = EmitSVEDupX(Ops[OpNo]); in EmitSVEPMull()
9200 Function *F = CGM.getIntrinsic(BuiltinID, Ops[0]->getType()); in EmitSVEPMull()
9201 Value *Call = Builder.CreateCall(F, {Ops[0], Ops[1]}); in EmitSVEPMull()
9209 ArrayRef<Value *> Ops, unsigned BuiltinID) { in EmitSVEMovl() argument
9212 return Builder.CreateCall(F, {Ops[0], Builder.getInt32(0)}); in EmitSVEMovl()
9216 SmallVectorImpl<Value *> &Ops, in EmitSVEPrefetchLoad() argument
9222 Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy); in EmitSVEPrefetchLoad()
9223 Value *BasePtr = Ops[1]; in EmitSVEPrefetchLoad()
9226 if (Ops.size() > 3) { in EmitSVEPrefetchLoad()
9228 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]); in EmitSVEPrefetchLoad()
9233 Value *PrfOp = Ops.back(); in EmitSVEPrefetchLoad()
9241 SmallVectorImpl<Value *> &Ops, in EmitSVEMaskedLoad() argument
9253 Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy); in EmitSVEMaskedLoad()
9254 Value *BasePtr = Builder.CreateBitCast(Ops[1], MemoryTy->getPointerTo()); in EmitSVEMaskedLoad()
9257 if (Ops.size() > 2) in EmitSVEMaskedLoad()
9258 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]); in EmitSVEMaskedLoad()
9272 SmallVectorImpl<Value *> &Ops, in EmitSVEMaskedStore() argument
9280 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType()); in EmitSVEMaskedStore()
9283 Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy); in EmitSVEMaskedStore()
9284 Value *BasePtr = Builder.CreateBitCast(Ops[1], MemoryTy->getPointerTo()); in EmitSVEMaskedStore()
9287 if (Ops.size() == 4) in EmitSVEMaskedStore()
9288 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]); in EmitSVEMaskedStore()
9291 llvm::Value *Val = Builder.CreateTrunc(Ops.back(), MemoryTy); in EmitSVEMaskedStore()
9324 SmallVectorImpl<Value *> &Ops) { in InsertExplicitZeroOperand() argument
9326 Ops.insert(Ops.begin(), SplatZero); in InsertExplicitZeroOperand()
9330 SmallVectorImpl<Value *> &Ops) { in InsertExplicitUndefOperand() argument
9332 Ops.insert(Ops.begin(), SplatUndef); in InsertExplicitUndefOperand()
9338 ArrayRef<Value *> Ops) { in getSVEOverloadTypes() argument
9345 return {DefaultType, Ops[1]->getType()}; in getSVEOverloadTypes()
9348 return {getSVEPredType(TypeFlags), Ops[0]->getType()}; in getSVEOverloadTypes()
9351 return {Ops[0]->getType(), Ops.back()->getType()}; in getSVEOverloadTypes()
9359 ArrayRef<Value *> Ops) { in EmitSVETupleSetOrGet() argument
9363 unsigned I = cast<ConstantInt>(Ops[1])->getSExtValue(); in EmitSVETupleSetOrGet()
9365 TypeFlags.isTupleSet() ? Ops[2]->getType() : Ty); in EmitSVETupleSetOrGet()
9370 return Builder.CreateInsertVector(Ty, Ops[0], Ops[2], Idx); in EmitSVETupleSetOrGet()
9371 return Builder.CreateExtractVector(Ty, Ops[0], Idx); in EmitSVETupleSetOrGet()
9376 ArrayRef<Value *> Ops) { in EmitSVETupleCreate() argument
9379 auto *SrcTy = dyn_cast<llvm::ScalableVectorType>(Ops[0]->getType()); in EmitSVETupleCreate()
9382 for (unsigned I = 0; I < Ops.size(); I++) { in EmitSVETupleCreate()
9384 Call = Builder.CreateInsertVector(Ty, Call, Ops[I], Idx); in EmitSVETupleCreate()
9405 llvm::SmallVector<Value *, 4> Ops; in EmitAArch64SVEBuiltinExpr() local
9408 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitAArch64SVEBuiltinExpr()
9420 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), *Result)); in EmitAArch64SVEBuiltinExpr()
9428 return EmitSVEMaskedLoad(E, Ty, Ops, Builtin->LLVMIntrinsic, in EmitAArch64SVEBuiltinExpr()
9431 return EmitSVEMaskedStore(E, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
9433 return EmitSVEGatherLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
9435 return EmitSVEScatterStore(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
9437 return EmitSVEPrefetchLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
9439 return EmitSVEGatherPrefetch(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
9441 return EmitSVEStructLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
9443 return EmitSVEStructStore(TypeFlags, Ops, Builtin->LLVMIntrinsic); in EmitAArch64SVEBuiltinExpr()
9445 return EmitSVETupleSetOrGet(TypeFlags, Ty, Ops); in EmitAArch64SVEBuiltinExpr()
9447 return EmitSVETupleCreate(TypeFlags, Ty, Ops); in EmitAArch64SVEBuiltinExpr()
9452 InsertExplicitZeroOperand(Builder, Ty, Ops); in EmitAArch64SVEBuiltinExpr()
9455 InsertExplicitUndefOperand(Builder, Ty, Ops); in EmitAArch64SVEBuiltinExpr()
9460 Ops.push_back(Builder.getInt32(/*SV_ALL*/ 31)); in EmitAArch64SVEBuiltinExpr()
9462 Ops.insert(&Ops[1], Builder.getInt32(/*SV_ALL*/ 31)); in EmitAArch64SVEBuiltinExpr()
9465 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in EmitAArch64SVEBuiltinExpr()
9466 if (auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType())) in EmitAArch64SVEBuiltinExpr()
9468 Ops[i] = EmitSVEPredicateCast(Ops[i], getSVEType(TypeFlags)); in EmitAArch64SVEBuiltinExpr()
9473 Ops[OpNo] = EmitSVEDupX(Ops[OpNo]); in EmitAArch64SVEBuiltinExpr()
9477 std::swap(Ops[1], Ops[2]); in EmitAArch64SVEBuiltinExpr()
9480 std::swap(Ops[1], Ops[2]); in EmitAArch64SVEBuiltinExpr()
9484 llvm::Type *OpndTy = Ops[1]->getType(); in EmitAArch64SVEBuiltinExpr()
9486 Ops[1] = Builder.CreateSelect(Ops[0], Ops[1], SplatZero); in EmitAArch64SVEBuiltinExpr()
9490 getSVEOverloadTypes(TypeFlags, Ty, Ops)); in EmitAArch64SVEBuiltinExpr()
9491 Value *Call = Builder.CreateCall(F, Ops); in EmitAArch64SVEBuiltinExpr()
9510 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]}); in EmitAArch64SVEBuiltinExpr()
9518 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]}); in EmitAArch64SVEBuiltinExpr()
9524 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb); in EmitAArch64SVEBuiltinExpr()
9529 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb); in EmitAArch64SVEBuiltinExpr()
9534 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt); in EmitAArch64SVEBuiltinExpr()
9539 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt); in EmitAArch64SVEBuiltinExpr()
9545 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair); in EmitAArch64SVEBuiltinExpr()
9551 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair); in EmitAArch64SVEBuiltinExpr()
9558 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType())); in EmitAArch64SVEBuiltinExpr()
9582 unsigned NumOpnds = Ops.size(); in EmitAArch64SVEBuiltinExpr()
9590 llvm::Type *EltTy = Ops[0]->getType(); in EmitAArch64SVEBuiltinExpr()
9596 VecOps.push_back(Builder.CreateZExt(Ops[I], EltTy)); in EmitAArch64SVEBuiltinExpr()
9661 Value *V0 = Builder.CreateExtractVector(VTy, Ops[0], in EmitAArch64SVEBuiltinExpr()
9665 VTy, Ops[0], ConstantInt::get(CGM.Int64Ty, MinElts)); in EmitAArch64SVEBuiltinExpr()
9667 return Builder.CreateCall(F, {V0, V1, Ops[1]}); in EmitAArch64SVEBuiltinExpr()
9682 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], Builder.getInt64(0)); in EmitAArch64SVEBuiltinExpr()
9697 return Builder.CreateExtractVector(Ty, Ops[0], Builder.getInt64(0)); in EmitAArch64SVEBuiltinExpr()
9712 Value *Insert = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0], in EmitAArch64SVEBuiltinExpr()
9893 Value *Ops[2]; in EmitAArch64BuiltinExpr() local
9895 Ops[i] = EmitScalarExpr(E->getArg(i)); in EmitAArch64BuiltinExpr()
9899 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops); in EmitAArch64BuiltinExpr()
10015 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)}; in EmitAArch64BuiltinExpr() local
10016 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
10223 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) }; in EmitAArch64BuiltinExpr() local
10224 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
10285 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")}; in EmitAArch64BuiltinExpr() local
10286 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
10310 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, "x18")}; in EmitAArch64BuiltinExpr() local
10311 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops); in EmitAArch64BuiltinExpr()
10346 llvm::SmallVector<Value*, 4> Ops; in EmitAArch64BuiltinExpr() local
10364 Ops.push_back(PtrOp0.getPointer()); in EmitAArch64BuiltinExpr()
10369 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitAArch64BuiltinExpr()
10373 Ops.push_back(llvm::ConstantInt::get( in EmitAArch64BuiltinExpr()
10384 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1))); in EmitAArch64BuiltinExpr()
10385 Value *Result = EmitCommonNeonSISDBuiltinExpr(*this, *Builtin, Ops, E); in EmitAArch64BuiltinExpr()
10404 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10405 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::fabs, HalfTy), Ops, "vabs"); in EmitAArch64BuiltinExpr()
10408 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10409 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
10410 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
10411 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10413 return Builder.CreateBitCast(Ops[0], Int128Ty); in EmitAArch64BuiltinExpr()
10424 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy); in EmitAArch64BuiltinExpr()
10433 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10434 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64; in EmitAArch64BuiltinExpr()
10437 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); in EmitAArch64BuiltinExpr()
10439 return Builder.CreateUIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
10440 return Builder.CreateSIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
10450 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10453 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64) in EmitAArch64BuiltinExpr()
10455 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32) in EmitAArch64BuiltinExpr()
10459 Ops[0] = Builder.CreateBitCast(Ops[0], InTy); in EmitAArch64BuiltinExpr()
10461 return Builder.CreateUIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
10462 return Builder.CreateSIToFP(Ops[0], FTy); in EmitAArch64BuiltinExpr()
10478 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10502 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvt"); in EmitAArch64BuiltinExpr()
10503 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
10513 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10521 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]); break; in EmitAArch64BuiltinExpr()
10523 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]); break; in EmitAArch64BuiltinExpr()
10525 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "facg"); in EmitAArch64BuiltinExpr()
10526 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
10534 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10542 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n"); in EmitAArch64BuiltinExpr()
10543 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
10551 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10556 Ops[0] = Builder.CreateSExt(Ops[0], InTy, "sext"); in EmitAArch64BuiltinExpr()
10560 Ops[0] = Builder.CreateZExt(Ops[0], InTy); in EmitAArch64BuiltinExpr()
10563 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fcvth_n"); in EmitAArch64BuiltinExpr()
10605 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10607 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
10613 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10615 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
10621 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10623 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
10629 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10631 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
10637 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10639 Ops[0], ConvertType(E->getCallReturnType(getContext())), in EmitAArch64BuiltinExpr()
10643 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
10644 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
10645 Ops[0] = in EmitAArch64BuiltinExpr()
10646 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty)); in EmitAArch64BuiltinExpr()
10647 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd"); in EmitAArch64BuiltinExpr()
10663 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10664 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
10665 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); in EmitAArch64BuiltinExpr()
10667 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10669 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10670 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
10686 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10687 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy); in EmitAArch64BuiltinExpr()
10688 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy); in EmitAArch64BuiltinExpr()
10690 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10692 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10693 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
10709 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10710 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
10711 Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy); in EmitAArch64BuiltinExpr()
10713 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10715 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10716 return Builder.CreateSExt(Ops[0], Int16Ty, "vcmpd"); in EmitAArch64BuiltinExpr()
10742 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10743 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
10744 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
10745 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10746 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd"); in EmitAArch64BuiltinExpr()
10750 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10751 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty); in EmitAArch64BuiltinExpr()
10752 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
10753 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10754 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], in EmitAArch64BuiltinExpr()
10756 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd"); in EmitAArch64BuiltinExpr()
10770 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
10771 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
10774 Ops[1] = in EmitAArch64BuiltinExpr()
10775 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 1)); in EmitAArch64BuiltinExpr()
10776 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
10777 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
10780 Ops[1] = in EmitAArch64BuiltinExpr()
10781 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 2)); in EmitAArch64BuiltinExpr()
10782 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
10783 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane"); in EmitAArch64BuiltinExpr()
10787 Ops[0] = in EmitAArch64BuiltinExpr()
10788 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 8)); in EmitAArch64BuiltinExpr()
10789 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10793 Ops[0] = in EmitAArch64BuiltinExpr()
10794 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 16)); in EmitAArch64BuiltinExpr()
10795 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10799 Ops[0] = in EmitAArch64BuiltinExpr()
10800 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 4)); in EmitAArch64BuiltinExpr()
10801 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10805 Ops[0] = in EmitAArch64BuiltinExpr()
10806 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 8)); in EmitAArch64BuiltinExpr()
10807 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10811 Ops[0] = in EmitAArch64BuiltinExpr()
10812 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 2)); in EmitAArch64BuiltinExpr()
10813 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10816 Ops[0] = in EmitAArch64BuiltinExpr()
10817 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2)); in EmitAArch64BuiltinExpr()
10818 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10822 Ops[0] = in EmitAArch64BuiltinExpr()
10823 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 4)); in EmitAArch64BuiltinExpr()
10824 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10828 Ops[0] = in EmitAArch64BuiltinExpr()
10829 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 1)); in EmitAArch64BuiltinExpr()
10830 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10833 Ops[0] = in EmitAArch64BuiltinExpr()
10834 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1)); in EmitAArch64BuiltinExpr()
10835 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10839 Ops[0] = in EmitAArch64BuiltinExpr()
10840 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 2)); in EmitAArch64BuiltinExpr()
10841 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10844 Ops[0] = in EmitAArch64BuiltinExpr()
10845 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2)); in EmitAArch64BuiltinExpr()
10846 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10849 Ops[0] = in EmitAArch64BuiltinExpr()
10850 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1)); in EmitAArch64BuiltinExpr()
10851 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10855 Ops[0] = in EmitAArch64BuiltinExpr()
10856 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 4)); in EmitAArch64BuiltinExpr()
10857 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10861 Ops[0] = in EmitAArch64BuiltinExpr()
10862 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 2)); in EmitAArch64BuiltinExpr()
10863 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
10866 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10867 return Builder.CreateFAdd(Ops[0], Ops[1], "vaddh"); in EmitAArch64BuiltinExpr()
10869 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10870 return Builder.CreateFSub(Ops[0], Ops[1], "vsubh"); in EmitAArch64BuiltinExpr()
10872 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10873 return Builder.CreateFMul(Ops[0], Ops[1], "vmulh"); in EmitAArch64BuiltinExpr()
10875 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10876 return Builder.CreateFDiv(Ops[0], Ops[1], "vdivh"); in EmitAArch64BuiltinExpr()
10881 {EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2)), Ops[0]}); in EmitAArch64BuiltinExpr()
10890 {Sub, EmitScalarExpr(E->getArg(2)), Ops[0]}); in EmitAArch64BuiltinExpr()
10894 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd"); in EmitAArch64BuiltinExpr()
10897 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd"); in EmitAArch64BuiltinExpr()
10901 ProductOps.push_back(vectorWrapScalar16(Ops[1])); in EmitAArch64BuiltinExpr()
10904 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), in EmitAArch64BuiltinExpr()
10907 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); in EmitAArch64BuiltinExpr()
10912 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
10915 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10916 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
10918 Ops, "vqshlu_n"); in EmitAArch64BuiltinExpr()
10925 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10926 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
10927 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vqshl_n"); in EmitAArch64BuiltinExpr()
10934 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
10935 int SV = cast<ConstantInt>(Ops[1])->getSExtValue(); in EmitAArch64BuiltinExpr()
10936 Ops[1] = ConstantInt::get(Int64Ty, -SV); in EmitAArch64BuiltinExpr()
10937 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops, "vrshr_n"); in EmitAArch64BuiltinExpr()
10944 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty); in EmitAArch64BuiltinExpr()
10945 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2)))); in EmitAArch64BuiltinExpr()
10946 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty), in EmitAArch64BuiltinExpr()
10947 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)}); in EmitAArch64BuiltinExpr()
10948 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty)); in EmitAArch64BuiltinExpr()
10954 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n"); in EmitAArch64BuiltinExpr()
10959 Ops[0], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), in EmitAArch64BuiltinExpr()
10969 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt), in EmitAArch64BuiltinExpr()
10974 Ops[1] = Builder.CreateAShr( in EmitAArch64BuiltinExpr()
10975 Ops[1], ConstantInt::get(Int64Ty, std::min(static_cast<uint64_t>(63), in EmitAArch64BuiltinExpr()
10978 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10986 return Ops[0]; in EmitAArch64BuiltinExpr()
10987 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt), in EmitAArch64BuiltinExpr()
10989 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
10995 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), in EmitAArch64BuiltinExpr()
10998 ProductOps.push_back(vectorWrapScalar16(Ops[1])); in EmitAArch64BuiltinExpr()
10999 ProductOps.push_back(vectorWrapScalar16(Ops[2])); in EmitAArch64BuiltinExpr()
11001 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy), in EmitAArch64BuiltinExpr()
11004 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0"); in EmitAArch64BuiltinExpr()
11005 Ops.pop_back(); in EmitAArch64BuiltinExpr()
11011 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
11016 ProductOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
11018 Ops[1] = in EmitAArch64BuiltinExpr()
11025 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
11031 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)), in EmitAArch64BuiltinExpr()
11034 ProductOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
11035 ProductOps.push_back(Ops[2]); in EmitAArch64BuiltinExpr()
11036 Ops[1] = in EmitAArch64BuiltinExpr()
11039 Ops.pop_back(); in EmitAArch64BuiltinExpr()
11045 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops, "vqdmlXl"); in EmitAArch64BuiltinExpr()
11050 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11056 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)), in EmitAArch64BuiltinExpr()
11083 Builtin->NameHint, Builtin->TypeModifier, E, Ops, in EmitAArch64BuiltinExpr()
11086 if (Value *V = EmitAArch64TblBuiltinExpr(*this, BuiltinID, E, Ops, Arch)) in EmitAArch64BuiltinExpr()
11095 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
11096 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
11097 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl"); in EmitAArch64BuiltinExpr()
11099 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl"); in EmitAArch64BuiltinExpr()
11100 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl"); in EmitAArch64BuiltinExpr()
11101 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl"); in EmitAArch64BuiltinExpr()
11102 return Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11108 Value *Addend = Ops[0]; in EmitAArch64BuiltinExpr()
11109 Value *Multiplicand = Ops[1]; in EmitAArch64BuiltinExpr()
11110 Value *LaneSource = Ops[2]; in EmitAArch64BuiltinExpr()
11111 Ops[0] = Multiplicand; in EmitAArch64BuiltinExpr()
11112 Ops[1] = LaneSource; in EmitAArch64BuiltinExpr()
11113 Ops[2] = Addend; in EmitAArch64BuiltinExpr()
11120 llvm::Constant *cst = cast<Constant>(Ops[3]); in EmitAArch64BuiltinExpr()
11122 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy); in EmitAArch64BuiltinExpr()
11123 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane"); in EmitAArch64BuiltinExpr()
11125 Ops.pop_back(); in EmitAArch64BuiltinExpr()
11128 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fmla"); in EmitAArch64BuiltinExpr()
11134 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
11135 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy); in EmitAArch64BuiltinExpr()
11138 Ops[2] = Builder.CreateBitCast(Ops[2], VTy); in EmitAArch64BuiltinExpr()
11139 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); in EmitAArch64BuiltinExpr()
11143 DoubleTy, {Ops[1], Ops[2], Ops[0]}); in EmitAArch64BuiltinExpr()
11146 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11147 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
11151 Ops[2] = Builder.CreateBitCast(Ops[2], STy); in EmitAArch64BuiltinExpr()
11153 cast<ConstantInt>(Ops[3])); in EmitAArch64BuiltinExpr()
11154 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane"); in EmitAArch64BuiltinExpr()
11158 {Ops[2], Ops[1], Ops[0]}); in EmitAArch64BuiltinExpr()
11161 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11162 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
11164 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
11165 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3])); in EmitAArch64BuiltinExpr()
11168 {Ops[2], Ops[1], Ops[0]}); in EmitAArch64BuiltinExpr()
11176 Ops.push_back(EmitScalarExpr(E->getArg(3))); in EmitAArch64BuiltinExpr()
11178 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract"); in EmitAArch64BuiltinExpr()
11181 {Ops[1], Ops[2], Ops[0]}); in EmitAArch64BuiltinExpr()
11187 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull"); in EmitAArch64BuiltinExpr()
11193 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmax"); in EmitAArch64BuiltinExpr()
11195 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11197 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmax"); in EmitAArch64BuiltinExpr()
11204 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmin"); in EmitAArch64BuiltinExpr()
11206 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11208 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmin"); in EmitAArch64BuiltinExpr()
11215 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vabd"); in EmitAArch64BuiltinExpr()
11226 TmpOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
11229 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType()); in EmitAArch64BuiltinExpr()
11237 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmin"); in EmitAArch64BuiltinExpr()
11243 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmax"); in EmitAArch64BuiltinExpr()
11247 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vminnm"); in EmitAArch64BuiltinExpr()
11249 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11251 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vminnm"); in EmitAArch64BuiltinExpr()
11255 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmaxnm"); in EmitAArch64BuiltinExpr()
11257 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11259 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmaxnm"); in EmitAArch64BuiltinExpr()
11261 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11263 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
11266 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11268 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
11270 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitAArch64BuiltinExpr()
11272 Ops, "vrecps"); in EmitAArch64BuiltinExpr()
11275 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrun_n"); in EmitAArch64BuiltinExpr()
11278 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrun_n"); in EmitAArch64BuiltinExpr()
11281 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqshrn_n"); in EmitAArch64BuiltinExpr()
11284 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrshrn_n"); in EmitAArch64BuiltinExpr()
11287 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n"); in EmitAArch64BuiltinExpr()
11289 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11293 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrnda"); in EmitAArch64BuiltinExpr()
11300 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnda"); in EmitAArch64BuiltinExpr()
11303 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11307 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndi"); in EmitAArch64BuiltinExpr()
11310 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11314 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndm"); in EmitAArch64BuiltinExpr()
11321 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndm"); in EmitAArch64BuiltinExpr()
11324 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11328 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
11335 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
11338 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11342 return EmitNeonCall(CGM.getIntrinsic(Int, FloatTy), Ops, "vrndn"); in EmitAArch64BuiltinExpr()
11345 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11349 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndp"); in EmitAArch64BuiltinExpr()
11356 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndp"); in EmitAArch64BuiltinExpr()
11359 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11363 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndx"); in EmitAArch64BuiltinExpr()
11370 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndx"); in EmitAArch64BuiltinExpr()
11373 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11377 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vrndz"); in EmitAArch64BuiltinExpr()
11381 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11383 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd32x"); in EmitAArch64BuiltinExpr()
11387 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11389 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd32z"); in EmitAArch64BuiltinExpr()
11393 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11395 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd64x"); in EmitAArch64BuiltinExpr()
11399 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11401 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrnd64z"); in EmitAArch64BuiltinExpr()
11408 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrndz"); in EmitAArch64BuiltinExpr()
11412 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11414 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") in EmitAArch64BuiltinExpr()
11415 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
11420 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); in EmitAArch64BuiltinExpr()
11422 return Builder.CreateFPExt(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
11428 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag)); in EmitAArch64BuiltinExpr()
11430 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt"); in EmitAArch64BuiltinExpr()
11447 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtz"); in EmitAArch64BuiltinExpr()
11463 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvta"); in EmitAArch64BuiltinExpr()
11479 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtm"); in EmitAArch64BuiltinExpr()
11495 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtn"); in EmitAArch64BuiltinExpr()
11511 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vcvtp"); in EmitAArch64BuiltinExpr()
11516 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmulx"); in EmitAArch64BuiltinExpr()
11522 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitAArch64BuiltinExpr()
11523 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); in EmitAArch64BuiltinExpr()
11524 Ops.pop_back(); in EmitAArch64BuiltinExpr()
11526 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vmulx"); in EmitAArch64BuiltinExpr()
11534 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
11537 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
11538 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract"); in EmitAArch64BuiltinExpr()
11539 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11549 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpmaxnm"); in EmitAArch64BuiltinExpr()
11554 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vpminnm"); in EmitAArch64BuiltinExpr()
11557 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11561 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops, "vsqrt"); in EmitAArch64BuiltinExpr()
11568 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11569 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqrt"); in EmitAArch64BuiltinExpr()
11574 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vrbit"); in EmitAArch64BuiltinExpr()
11585 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11586 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
11587 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11597 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11598 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
11599 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11609 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11610 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
11611 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11621 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11622 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddv"); in EmitAArch64BuiltinExpr()
11623 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11630 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11631 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11632 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11639 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11640 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11641 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11648 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11649 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11650 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11657 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11658 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11659 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11666 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11667 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11668 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11675 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11676 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11677 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11684 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11685 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11686 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11693 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11694 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11695 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11702 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11703 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11704 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11711 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11712 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxv"); in EmitAArch64BuiltinExpr()
11713 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11720 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11721 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11722 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11729 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11730 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11731 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11738 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11739 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11740 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11747 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11748 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11749 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11756 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11757 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11758 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11765 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11766 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11767 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11774 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11775 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11776 return Builder.CreateTrunc(Ops[0], Int8Ty); in EmitAArch64BuiltinExpr()
11783 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11784 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11785 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11792 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11793 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11794 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11801 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11802 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminv"); in EmitAArch64BuiltinExpr()
11803 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11810 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11811 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv"); in EmitAArch64BuiltinExpr()
11812 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11819 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11820 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vmaxnmv"); in EmitAArch64BuiltinExpr()
11821 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11828 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11829 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv"); in EmitAArch64BuiltinExpr()
11830 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11837 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11838 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vminnmv"); in EmitAArch64BuiltinExpr()
11839 return Builder.CreateTrunc(Ops[0], HalfTy); in EmitAArch64BuiltinExpr()
11842 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy); in EmitAArch64BuiltinExpr()
11844 return Builder.CreateFMul(Ops[0], RHS); in EmitAArch64BuiltinExpr()
11851 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11852 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11853 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11860 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11861 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11868 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11869 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11870 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11877 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11878 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11885 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11886 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11887 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11894 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11895 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11902 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11903 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11904 return Builder.CreateTrunc(Ops[0], Int16Ty); in EmitAArch64BuiltinExpr()
11911 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitAArch64BuiltinExpr()
11912 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "vaddlv"); in EmitAArch64BuiltinExpr()
11918 return EmitNeonCall(Intrin, Ops, "vsri_n"); in EmitAArch64BuiltinExpr()
11924 return EmitNeonCall(Intrin, Ops, "vsli_n"); in EmitAArch64BuiltinExpr()
11928 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11929 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn, "vsra_n"); in EmitAArch64BuiltinExpr()
11930 return Builder.CreateAdd(Ops[0], Ops[1]); in EmitAArch64BuiltinExpr()
11935 TmpOps.push_back(Ops[1]); in EmitAArch64BuiltinExpr()
11936 TmpOps.push_back(Ops[2]); in EmitAArch64BuiltinExpr()
11939 Ops[0] = Builder.CreateBitCast(Ops[0], VTy); in EmitAArch64BuiltinExpr()
11940 return Builder.CreateAdd(Ops[0], tmp); in EmitAArch64BuiltinExpr()
11944 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); in EmitAArch64BuiltinExpr()
11945 return Builder.CreateAlignedLoad(VTy, Ops[0], PtrOp0.getAlignment()); in EmitAArch64BuiltinExpr()
11949 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy)); in EmitAArch64BuiltinExpr()
11950 Ops[1] = Builder.CreateBitCast(Ops[1], VTy); in EmitAArch64BuiltinExpr()
11951 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment()); in EmitAArch64BuiltinExpr()
11954 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
11956 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11957 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], in EmitAArch64BuiltinExpr()
11959 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane"); in EmitAArch64BuiltinExpr()
11965 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
11966 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], in EmitAArch64BuiltinExpr()
11969 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI); in EmitAArch64BuiltinExpr()
11970 return EmitNeonSplat(Ops[0], CI); in EmitAArch64BuiltinExpr()
11974 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
11975 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]); in EmitAArch64BuiltinExpr()
11976 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
11977 return Builder.CreateAlignedStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty), in EmitAArch64BuiltinExpr()
11982 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
11985 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); in EmitAArch64BuiltinExpr()
11986 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
11987 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
11988 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
11993 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
11996 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); in EmitAArch64BuiltinExpr()
11997 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
11998 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
11999 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
12004 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
12007 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); in EmitAArch64BuiltinExpr()
12008 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
12009 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
12010 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
12016 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
12019 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2"); in EmitAArch64BuiltinExpr()
12020 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
12021 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
12022 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
12028 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
12031 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3"); in EmitAArch64BuiltinExpr()
12032 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
12033 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
12034 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
12040 Ops[1] = Builder.CreateBitCast(Ops[1], PTy); in EmitAArch64BuiltinExpr()
12043 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4"); in EmitAArch64BuiltinExpr()
12044 Ops[0] = Builder.CreateBitCast(Ops[0], in EmitAArch64BuiltinExpr()
12045 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitAArch64BuiltinExpr()
12046 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
12050 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
12052 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end()); in EmitAArch64BuiltinExpr()
12053 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
12054 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
12055 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitAArch64BuiltinExpr()
12056 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld2_lane"); in EmitAArch64BuiltinExpr()
12057 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
12058 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
12059 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
12063 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
12065 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end()); in EmitAArch64BuiltinExpr()
12066 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
12067 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
12068 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); in EmitAArch64BuiltinExpr()
12069 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); in EmitAArch64BuiltinExpr()
12070 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld3_lane"); in EmitAArch64BuiltinExpr()
12071 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
12072 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
12073 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
12077 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() }; in EmitAArch64BuiltinExpr()
12079 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end()); in EmitAArch64BuiltinExpr()
12080 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
12081 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
12082 Ops[3] = Builder.CreateBitCast(Ops[3], Ty); in EmitAArch64BuiltinExpr()
12083 Ops[4] = Builder.CreateBitCast(Ops[4], Ty); in EmitAArch64BuiltinExpr()
12084 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty); in EmitAArch64BuiltinExpr()
12085 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld4_lane"); in EmitAArch64BuiltinExpr()
12086 Ty = llvm::PointerType::getUnqual(Ops[1]->getType()); in EmitAArch64BuiltinExpr()
12087 Ops[0] = Builder.CreateBitCast(Ops[0], Ty); in EmitAArch64BuiltinExpr()
12088 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); in EmitAArch64BuiltinExpr()
12092 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
12093 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() }; in EmitAArch64BuiltinExpr()
12095 Ops, ""); in EmitAArch64BuiltinExpr()
12099 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
12100 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitAArch64BuiltinExpr()
12101 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; in EmitAArch64BuiltinExpr()
12103 Ops, ""); in EmitAArch64BuiltinExpr()
12107 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
12108 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() }; in EmitAArch64BuiltinExpr()
12110 Ops, ""); in EmitAArch64BuiltinExpr()
12114 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
12115 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty); in EmitAArch64BuiltinExpr()
12116 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; in EmitAArch64BuiltinExpr()
12118 Ops, ""); in EmitAArch64BuiltinExpr()
12122 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
12123 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() }; in EmitAArch64BuiltinExpr()
12125 Ops, ""); in EmitAArch64BuiltinExpr()
12129 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); in EmitAArch64BuiltinExpr()
12130 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty); in EmitAArch64BuiltinExpr()
12131 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() }; in EmitAArch64BuiltinExpr()
12133 Ops, ""); in EmitAArch64BuiltinExpr()
12137 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
12138 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
12139 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
12148 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
12149 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn"); in EmitAArch64BuiltinExpr()
12156 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
12157 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
12158 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
12166 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
12167 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp"); in EmitAArch64BuiltinExpr()
12174 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitAArch64BuiltinExpr()
12175 Ops[1] = Builder.CreateBitCast(Ops[1], Ty); in EmitAArch64BuiltinExpr()
12176 Ops[2] = Builder.CreateBitCast(Ops[2], Ty); in EmitAArch64BuiltinExpr()
12185 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi); in EmitAArch64BuiltinExpr()
12186 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip"); in EmitAArch64BuiltinExpr()
12193 Ops, "vtbl1"); in EmitAArch64BuiltinExpr()
12197 Ops, "vtbl2"); in EmitAArch64BuiltinExpr()
12201 Ops, "vtbl3"); in EmitAArch64BuiltinExpr()
12205 Ops, "vtbl4"); in EmitAArch64BuiltinExpr()
12209 Ops, "vtbx1"); in EmitAArch64BuiltinExpr()
12213 Ops, "vtbx2"); in EmitAArch64BuiltinExpr()
12217 Ops, "vtbx3"); in EmitAArch64BuiltinExpr()
12221 Ops, "vtbx4"); in EmitAArch64BuiltinExpr()
12226 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vsqadd"); in EmitAArch64BuiltinExpr()
12231 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vuqadd"); in EmitAArch64BuiltinExpr()
12345 BuildVector(ArrayRef<llvm::Value*> Ops) { in BuildVector() argument
12346 assert((Ops.size() & (Ops.size() - 1)) == 0 && in BuildVector()
12349 for (unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i) in BuildVector()
12350 AllConstants &= isa<Constant>(Ops[i]); in BuildVector()
12355 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in BuildVector()
12356 CstOps.push_back(cast<Constant>(Ops[i])); in BuildVector()
12362 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size())); in BuildVector()
12364 for (unsigned i = 0, e = Ops.size(); i != e; ++i) in BuildVector()
12365 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt64(i)); in BuildVector()
12391 static Value *EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in EmitX86MaskedStore() argument
12394 Value *Ptr = CGF.Builder.CreateBitCast(Ops[0], in EmitX86MaskedStore()
12395 llvm::PointerType::getUnqual(Ops[1]->getType())); in EmitX86MaskedStore()
12398 CGF, Ops[2], in EmitX86MaskedStore()
12399 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements()); in EmitX86MaskedStore()
12401 return CGF.Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec); in EmitX86MaskedStore()
12404 static Value *EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in EmitX86MaskedLoad() argument
12407 llvm::Type *Ty = Ops[1]->getType(); in EmitX86MaskedLoad()
12409 CGF.Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty)); in EmitX86MaskedLoad()
12412 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements()); in EmitX86MaskedLoad()
12414 return CGF.Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]); in EmitX86MaskedLoad()
12418 ArrayRef<Value *> Ops) { in EmitX86ExpandLoad() argument
12419 auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType()); in EmitX86ExpandLoad()
12423 Value *Ptr = CGF.Builder.CreateBitCast(Ops[0], in EmitX86ExpandLoad()
12427 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements()); in EmitX86ExpandLoad()
12431 return CGF.Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] }); in EmitX86ExpandLoad()
12435 ArrayRef<Value *> Ops, in EmitX86CompressExpand() argument
12437 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType()); in EmitX86CompressExpand()
12439 Value *MaskVec = getMaskVecValue(CGF, Ops[2], ResultTy->getNumElements()); in EmitX86CompressExpand()
12444 return CGF.Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec }); in EmitX86CompressExpand()
12448 ArrayRef<Value *> Ops) { in EmitX86CompressStore() argument
12449 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType()); in EmitX86CompressStore()
12453 Value *Ptr = CGF.Builder.CreateBitCast(Ops[0], in EmitX86CompressStore()
12456 Value *MaskVec = getMaskVecValue(CGF, Ops[2], ResultTy->getNumElements()); in EmitX86CompressStore()
12460 return CGF.Builder.CreateCall(F, { Ops[1], Ptr, MaskVec }); in EmitX86CompressStore()
12464 ArrayRef<Value *> Ops, in EmitX86MaskLogic() argument
12466 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86MaskLogic()
12467 Value *LHS = getMaskVecValue(CGF, Ops[0], NumElts); in EmitX86MaskLogic()
12468 Value *RHS = getMaskVecValue(CGF, Ops[1], NumElts); in EmitX86MaskLogic()
12474 Ops[0]->getType()); in EmitX86MaskLogic()
12495 static Value *EmitX86vpcom(CodeGenFunction &CGF, ArrayRef<Value *> Ops, in EmitX86vpcom() argument
12497 Value *Op0 = Ops[0]; in EmitX86vpcom()
12498 Value *Op1 = Ops[1]; in EmitX86vpcom()
12500 uint64_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86vpcom()
12587 bool Signed, ArrayRef<Value *> Ops) { in EmitX86MaskedCompare() argument
12588 assert((Ops.size() == 2 || Ops.size() == 4) && in EmitX86MaskedCompare()
12591 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86MaskedCompare()
12611 Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]); in EmitX86MaskedCompare()
12615 if (Ops.size() == 4) in EmitX86MaskedCompare()
12616 MaskIn = Ops[3]; in EmitX86MaskedCompare()
12627 ArrayRef<Value *> Ops, bool IsSigned) { in EmitX86ConvertIntToFp() argument
12628 unsigned Rnd = cast<llvm::ConstantInt>(Ops[3])->getZExtValue(); in EmitX86ConvertIntToFp()
12629 llvm::Type *Ty = Ops[1]->getType(); in EmitX86ConvertIntToFp()
12635 Function *F = CGF.CGM.getIntrinsic(IID, { Ty, Ops[0]->getType() }); in EmitX86ConvertIntToFp()
12636 Res = CGF.Builder.CreateCall(F, { Ops[0], Ops[3] }); in EmitX86ConvertIntToFp()
12639 Res = IsSigned ? CGF.Builder.CreateSIToFP(Ops[0], Ty) in EmitX86ConvertIntToFp()
12640 : CGF.Builder.CreateUIToFP(Ops[0], Ty); in EmitX86ConvertIntToFp()
12643 return EmitX86Select(CGF, Ops[2], Res, Ops[1]); in EmitX86ConvertIntToFp()
12648 ArrayRef<Value *> Ops, unsigned BuiltinID, in EmitX86FMAExpr() argument
12703 Value *A = Ops[0]; in EmitX86FMAExpr()
12704 Value *B = Ops[1]; in EmitX86FMAExpr()
12705 Value *C = Ops[2]; in EmitX86FMAExpr()
12714 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4 || in EmitX86FMAExpr()
12717 Res = CGF.Builder.CreateCall(Intr, {A, B, C, Ops.back() }); in EmitX86FMAExpr()
12740 MaskFalseVal = Ops[0]; in EmitX86FMAExpr()
12748 MaskFalseVal = Constant::getNullValue(Ops[0]->getType()); in EmitX86FMAExpr()
12762 MaskFalseVal = Ops[2]; in EmitX86FMAExpr()
12767 return EmitX86Select(CGF, Ops[3], Res, MaskFalseVal); in EmitX86FMAExpr()
12773 MutableArrayRef<Value *> Ops, Value *Upper, in EmitScalarFMAExpr() argument
12777 if (Ops.size() > 4) in EmitScalarFMAExpr()
12778 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue(); in EmitScalarFMAExpr()
12781 Ops[2] = CGF.Builder.CreateFNeg(Ops[2]); in EmitScalarFMAExpr()
12783 Ops[0] = CGF.Builder.CreateExtractElement(Ops[0], (uint64_t)0); in EmitScalarFMAExpr()
12784 Ops[1] = CGF.Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitScalarFMAExpr()
12785 Ops[2] = CGF.Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitScalarFMAExpr()
12790 switch (Ops[0]->getType()->getPrimitiveSizeInBits()) { in EmitScalarFMAExpr()
12804 {Ops[0], Ops[1], Ops[2], Ops[4]}); in EmitScalarFMAExpr()
12808 Intrinsic::experimental_constrained_fma, Ops[0]->getType()); in EmitScalarFMAExpr()
12809 Res = CGF.Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3)); in EmitScalarFMAExpr()
12811 Function *FMA = CGF.CGM.getIntrinsic(Intrinsic::fma, Ops[0]->getType()); in EmitScalarFMAExpr()
12812 Res = CGF.Builder.CreateCall(FMA, Ops.slice(0, 3)); in EmitScalarFMAExpr()
12815 if (Ops.size() > 3) { in EmitScalarFMAExpr()
12817 : Ops[PTIdx]; in EmitScalarFMAExpr()
12825 Res = EmitX86ScalarSelect(CGF, Ops[3], Res, PassThru); in EmitScalarFMAExpr()
12831 ArrayRef<Value *> Ops) { in EmitX86Muldq() argument
12832 llvm::Type *Ty = Ops[0]->getType(); in EmitX86Muldq()
12836 Value *LHS = CGF.Builder.CreateBitCast(Ops[0], Ty); in EmitX86Muldq()
12837 Value *RHS = CGF.Builder.CreateBitCast(Ops[1], Ty); in EmitX86Muldq()
12860 ArrayRef<Value *> Ops) { in EmitX86Ternlog() argument
12861 llvm::Type *Ty = Ops[0]->getType(); in EmitX86Ternlog()
12882 Ops.drop_back()); in EmitX86Ternlog()
12883 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0]; in EmitX86Ternlog()
12884 return EmitX86Select(CGF, Ops[4], Ternlog, PassThru); in EmitX86Ternlog()
12903 ArrayRef<Value *> Ops, in EmitX86CvtF16ToFloatExpr() argument
12905 assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) && in EmitX86CvtF16ToFloatExpr()
12909 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) { in EmitX86CvtF16ToFloatExpr()
12912 return CGF.Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]}); in EmitX86CvtF16ToFloatExpr()
12916 Value *Src = Ops[0]; in EmitX86CvtF16ToFloatExpr()
12933 if (Ops.size() >= 3) in EmitX86CvtF16ToFloatExpr()
12934 Res = EmitX86Select(CGF, Ops[2], Res, Ops[1]); in EmitX86CvtF16ToFloatExpr()
13110 SmallVector<Value*, 4> Ops; in EmitX86BuiltinExpr() local
13123 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitX86BuiltinExpr()
13129 Ops.push_back(llvm::ConstantInt::get( in EmitX86BuiltinExpr()
13139 auto getCmpIntrinsicCall = [this, &Ops](Intrinsic::ID ID, unsigned Imm) { in EmitX86BuiltinExpr()
13140 Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm)); in EmitX86BuiltinExpr()
13142 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
13150 auto getVectorFCmpIR = [this, &Ops, E](CmpInst::Predicate Pred, in EmitX86BuiltinExpr()
13155 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
13157 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
13158 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
13167 Value *Address = Ops[0]; in EmitX86BuiltinExpr()
13168 ConstantInt *C = cast<ConstantInt>(Ops[1]); in EmitX86BuiltinExpr()
13177 Ops[0]); in EmitX86BuiltinExpr()
13197 Ops[0]); in EmitX86BuiltinExpr()
13203 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitX86BuiltinExpr()
13204 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitX86BuiltinExpr()
13209 Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType()); in EmitX86BuiltinExpr()
13210 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitX86BuiltinExpr()
13224 return Builder.CreateBitCast(BuildVector(Ops), in EmitX86BuiltinExpr()
13237 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
13238 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
13242 return Builder.CreateExtractElement(Ops[0], Index); in EmitX86BuiltinExpr()
13253 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
13254 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
13258 return Builder.CreateInsertElement(Ops[0], Ops[1], Index); in EmitX86BuiltinExpr()
13263 Builder.CreateStore(Ops[0], Tmp); in EmitX86BuiltinExpr()
13314 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty); in EmitX86BuiltinExpr()
13315 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty); in EmitX86BuiltinExpr()
13316 Ops[1] = Mhi; in EmitX86BuiltinExpr()
13317 Ops.push_back(Mlo); in EmitX86BuiltinExpr()
13318 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
13322 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_xgetbv), Ops); in EmitX86BuiltinExpr()
13341 return EmitX86MaskedStore(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
13346 return EmitX86MaskedStore(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
13362 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
13376 return EmitX86SExtMask(*this, Ops[0], ConvertType(E->getType())); in EmitX86BuiltinExpr()
13390 return EmitX86ConvertToMask(*this, Ops[0]); in EmitX86BuiltinExpr()
13398 return EmitX86ConvertIntToFp(*this, E, Ops, /*IsSigned*/ true); in EmitX86BuiltinExpr()
13405 return EmitX86ConvertIntToFp(*this, E, Ops, /*IsSigned*/ false); in EmitX86BuiltinExpr()
13412 return EmitScalarFMAExpr(*this, E, Ops, Ops[0]); in EmitX86BuiltinExpr()
13415 return EmitScalarFMAExpr(*this, E, Ops, in EmitX86BuiltinExpr()
13416 Constant::getNullValue(Ops[0]->getType())); in EmitX86BuiltinExpr()
13420 return EmitScalarFMAExpr(*this, E, Ops, Ops[0], /*ZeroMask*/ true); in EmitX86BuiltinExpr()
13424 return EmitScalarFMAExpr(*this, E, Ops, Ops[2], /*ZeroMask*/ false, 2); in EmitX86BuiltinExpr()
13428 return EmitScalarFMAExpr(*this, E, Ops, Ops[2], /*ZeroMask*/ false, 2, in EmitX86BuiltinExpr()
13448 return EmitX86FMAExpr(*this, E, Ops, BuiltinID, /*IsAddSub*/ false); in EmitX86BuiltinExpr()
13461 return EmitX86FMAExpr(*this, E, Ops, BuiltinID, /*IsAddSub*/ true); in EmitX86BuiltinExpr()
13476 *this, Ops, in EmitX86BuiltinExpr()
13497 return EmitX86MaskedLoad(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
13502 return EmitX86MaskedLoad(*this, Ops, Align(1)); in EmitX86BuiltinExpr()
13517 *this, Ops, in EmitX86BuiltinExpr()
13538 return EmitX86ExpandLoad(*this, Ops); in EmitX86BuiltinExpr()
13558 return EmitX86CompressStore(*this, Ops); in EmitX86BuiltinExpr()
13578 return EmitX86CompressExpand(*this, Ops, /*IsCompress*/false); in EmitX86BuiltinExpr()
13598 return EmitX86CompressExpand(*this, Ops, /*IsCompress*/true); in EmitX86BuiltinExpr()
13702 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(), in EmitX86BuiltinExpr()
13703 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements()); in EmitX86BuiltinExpr()
13704 Ops[3] = getMaskVecValue(*this, Ops[3], MinElts); in EmitX86BuiltinExpr()
13706 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
13811 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements(), in EmitX86BuiltinExpr()
13812 cast<llvm::FixedVectorType>(Ops[3]->getType())->getNumElements()); in EmitX86BuiltinExpr()
13813 Ops[1] = getMaskVecValue(*this, Ops[1], MinElts); in EmitX86BuiltinExpr()
13815 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
13837 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
13839 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
13848 Value *Res = Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
13851 if (Ops.size() == 4) in EmitX86BuiltinExpr()
13852 Res = EmitX86Select(*this, Ops[3], Res, Ops[2]); in EmitX86BuiltinExpr()
13873 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
13875 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements(); in EmitX86BuiltinExpr()
13877 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
13887 Ops[1], ArrayRef(Indices, DstNumElts), "widen"); in EmitX86BuiltinExpr()
13896 return Builder.CreateShuffleVector(Ops[0], Op1, in EmitX86BuiltinExpr()
13901 Value *Res = Builder.CreateTrunc(Ops[0], Ops[1]->getType()); in EmitX86BuiltinExpr()
13902 return EmitX86Select(*this, Ops[2], Res, Ops[1]); in EmitX86BuiltinExpr()
13907 if (const auto *C = dyn_cast<Constant>(Ops[2])) in EmitX86BuiltinExpr()
13909 return Builder.CreateTrunc(Ops[0], Ops[1]->getType()); in EmitX86BuiltinExpr()
13926 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
13937 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
13938 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
13946 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
13952 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
13953 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
13969 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
13975 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
13976 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
13992 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
14004 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
14005 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
14021 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
14030 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
14031 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
14050 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
14057 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
14058 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
14067 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts), in EmitX86BuiltinExpr()
14073 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
14076 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14088 Ops[1] = Ops[0]; in EmitX86BuiltinExpr()
14089 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
14103 return Builder.CreateShuffleVector(Ops[1], Ops[0], in EmitX86BuiltinExpr()
14113 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14114 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
14123 return Builder.CreateShuffleVector(Ops[1], Ops[0], in EmitX86BuiltinExpr()
14134 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
14135 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
14151 return Builder.CreateShuffleVector(Ops[0], Ops[1], in EmitX86BuiltinExpr()
14159 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue(); in EmitX86BuiltinExpr()
14161 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14173 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType()); in EmitX86BuiltinExpr()
14175 OutOps[l] = Ops[1]; in EmitX86BuiltinExpr()
14177 OutOps[l] = Ops[0]; in EmitX86BuiltinExpr()
14197 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
14198 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
14217 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast"); in EmitX86BuiltinExpr()
14221 return Builder.CreateBitCast(SV, Ops[0]->getType(), "cast"); in EmitX86BuiltinExpr()
14226 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
14227 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType()); in EmitX86BuiltinExpr()
14246 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast"); in EmitX86BuiltinExpr()
14256 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
14257 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
14260 return llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
14262 Value *In = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
14271 return Builder.CreateBitCast(SV, Ops[0]->getType()); in EmitX86BuiltinExpr()
14277 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff; in EmitX86BuiltinExpr()
14278 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
14281 return llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
14283 Value *In = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
14292 return Builder.CreateBitCast(SV, Ops[0]->getType()); in EmitX86BuiltinExpr()
14301 Value *Ptr = Ops[0]; in EmitX86BuiltinExpr()
14302 Value *Src = Ops[1]; in EmitX86BuiltinExpr()
14340 return EmitX86FunnelShift(*this, Ops[0], Ops[0], Ops[1], false); in EmitX86BuiltinExpr()
14353 return EmitX86FunnelShift(*this, Ops[0], Ops[0], Ops[1], true); in EmitX86BuiltinExpr()
14378 return EmitX86Select(*this, Ops[0], Ops[1], Ops[2]); in EmitX86BuiltinExpr()
14383 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitX86BuiltinExpr()
14384 Value *B = Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitX86BuiltinExpr()
14385 A = EmitX86ScalarSelect(*this, Ops[0], A, B); in EmitX86BuiltinExpr()
14386 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0); in EmitX86BuiltinExpr()
14400 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86BuiltinExpr()
14401 return EmitX86MaskedCompare(*this, CC, true, Ops); in EmitX86BuiltinExpr()
14415 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7; in EmitX86BuiltinExpr()
14416 return EmitX86MaskedCompare(*this, CC, false, Ops); in EmitX86BuiltinExpr()
14422 return EmitX86vpcom(*this, Ops, true); in EmitX86BuiltinExpr()
14427 return EmitX86vpcom(*this, Ops, false); in EmitX86BuiltinExpr()
14433 Value *Or = EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
14434 Value *C = llvm::Constant::getAllOnesValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
14442 Value *Or = EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
14443 Value *C = llvm::Constant::getNullValue(Ops[0]->getType()); in EmitX86BuiltinExpr()
14485 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
14486 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
14487 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
14513 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
14514 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
14515 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
14518 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
14524 return EmitX86MaskLogic(*this, Instruction::And, Ops); in EmitX86BuiltinExpr()
14529 return EmitX86MaskLogic(*this, Instruction::And, Ops, true); in EmitX86BuiltinExpr()
14534 return EmitX86MaskLogic(*this, Instruction::Or, Ops); in EmitX86BuiltinExpr()
14539 return EmitX86MaskLogic(*this, Instruction::Xor, Ops, true); in EmitX86BuiltinExpr()
14544 return EmitX86MaskLogic(*this, Instruction::Xor, Ops); in EmitX86BuiltinExpr()
14549 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
14550 Value *Res = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
14552 Ops[0]->getType()); in EmitX86BuiltinExpr()
14561 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
14562 Value *Res = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
14563 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
14569 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth(); in EmitX86BuiltinExpr()
14570 Value *LHS = getMaskVecValue(*this, Ops[0], NumElts); in EmitX86BuiltinExpr()
14571 Value *RHS = getMaskVecValue(*this, Ops[1], NumElts); in EmitX86BuiltinExpr()
14584 return Builder.CreateBitCast(Res, Ops[0]->getType()); in EmitX86BuiltinExpr()
14593 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitX86BuiltinExpr()
14594 return Builder.CreateCall(F, {Ops[0],Builder.getInt1(false)}); in EmitX86BuiltinExpr()
14598 Value *A = Builder.CreateExtractElement(Ops[0], (uint64_t)0); in EmitX86BuiltinExpr()
14609 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0); in EmitX86BuiltinExpr()
14614 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue(); in EmitX86BuiltinExpr()
14633 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
14635 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0); in EmitX86BuiltinExpr()
14646 Value *Src = Builder.CreateExtractElement(Ops[2], (uint64_t)0); in EmitX86BuiltinExpr()
14647 A = EmitX86ScalarSelect(*this, Ops[3], A, Src); in EmitX86BuiltinExpr()
14648 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0); in EmitX86BuiltinExpr()
14659 if (Ops.size() == 2) { in EmitX86BuiltinExpr()
14660 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue(); in EmitX86BuiltinExpr()
14679 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
14685 Ops[0]->getType()); in EmitX86BuiltinExpr()
14686 return Builder.CreateConstrainedFPCall(F, Ops[0]); in EmitX86BuiltinExpr()
14688 Function *F = CGM.getIntrinsic(Intrinsic::sqrt, Ops[0]->getType()); in EmitX86BuiltinExpr()
14689 return Builder.CreateCall(F, Ops[0]); in EmitX86BuiltinExpr()
14696 return EmitX86Muldq(*this, /*IsSigned*/false, Ops); in EmitX86BuiltinExpr()
14701 return EmitX86Muldq(*this, /*IsSigned*/true, Ops); in EmitX86BuiltinExpr()
14709 return EmitX86Ternlog(*this, /*ZeroMask*/false, Ops); in EmitX86BuiltinExpr()
14717 return EmitX86Ternlog(*this, /*ZeroMask*/true, Ops); in EmitX86BuiltinExpr()
14728 return EmitX86FunnelShift(*this, Ops[0], Ops[1], Ops[2], false); in EmitX86BuiltinExpr()
14740 return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true); in EmitX86BuiltinExpr()
14751 return EmitX86FunnelShift(*this, Ops[0], Ops[1], Ops[2], false); in EmitX86BuiltinExpr()
14763 return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true); in EmitX86BuiltinExpr()
14772 CGM.getIntrinsic(Intrinsic::vector_reduce_fadd, Ops[1]->getType()); in EmitX86BuiltinExpr()
14775 return Builder.CreateCall(F, {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
14783 CGM.getIntrinsic(Intrinsic::vector_reduce_fmul, Ops[1]->getType()); in EmitX86BuiltinExpr()
14786 return Builder.CreateCall(F, {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
14794 CGM.getIntrinsic(Intrinsic::vector_reduce_fmax, Ops[0]->getType()); in EmitX86BuiltinExpr()
14797 return Builder.CreateCall(F, {Ops[0]}); in EmitX86BuiltinExpr()
14805 CGM.getIntrinsic(Intrinsic::vector_reduce_fmin, Ops[0]->getType()); in EmitX86BuiltinExpr()
14808 return Builder.CreateCall(F, {Ops[0]}); in EmitX86BuiltinExpr()
14815 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy, "cast"); in EmitX86BuiltinExpr()
14817 return Builder.CreateCall(F, Ops, "pswapd"); in EmitX86BuiltinExpr()
14850 Ops[0]); in EmitX86BuiltinExpr()
14875 { Ops[0], Ops[1], Ops[2] }); in EmitX86BuiltinExpr()
14877 Ops[3]); in EmitX86BuiltinExpr()
14891 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14892 Value *MaskIn = Ops[2]; in EmitX86BuiltinExpr()
14893 Ops.erase(&Ops[2]); in EmitX86BuiltinExpr()
14927 Value *Fpclass = Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
14938 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14963 Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID), {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
14966 Builder.CreateDefaultAlignedStore(Result, Ops[2]); in EmitX86BuiltinExpr()
14970 return Builder.CreateDefaultAlignedStore(Result, Ops[3]); in EmitX86BuiltinExpr()
14990 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
14997 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
14998 Value *MaskIn = Ops[2]; in EmitX86BuiltinExpr()
14999 Ops.erase(&Ops[2]); in EmitX86BuiltinExpr()
15015 Value *Shufbit = Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitX86BuiltinExpr()
15066 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f; in EmitX86BuiltinExpr()
15146 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15147 Ops[3] = getMaskVecValue(*this, Ops[3], NumElts); in EmitX86BuiltinExpr()
15148 Value *Cmp = Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
15152 return Builder.CreateCall(Intr, Ops); in EmitX86BuiltinExpr()
15163 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(); in EmitX86BuiltinExpr()
15166 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
15168 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]); in EmitX86BuiltinExpr()
15169 return EmitX86MaskedCompareResult(*this, Cmp, NumElts, Ops[3]); in EmitX86BuiltinExpr()
15216 return EmitX86CvtF16ToFloatExpr(*this, Ops, ConvertType(E->getType())); in EmitX86BuiltinExpr()
15221 Ops[2] = getMaskVecValue( in EmitX86BuiltinExpr()
15222 *this, Ops[2], in EmitX86BuiltinExpr()
15223 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements()); in EmitX86BuiltinExpr()
15225 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
15228 return Builder.CreateFPExt(Ops[0], Builder.getFloatTy()); in EmitX86BuiltinExpr()
15242 Value *Res = Builder.CreateCall(CGM.getIntrinsic(IID), Ops[0]); in EmitX86BuiltinExpr()
15243 return EmitX86Select(*this, Ops[2], Res, Ops[1]); in EmitX86BuiltinExpr()
15290 Value *LHS = Builder.CreateIntCast(Ops[0], Int64Ty, isSigned); in EmitX86BuiltinExpr()
15291 Value *RHS = Builder.CreateIntCast(Ops[1], Int64Ty, isSigned); in EmitX86BuiltinExpr()
15302 Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned); in EmitX86BuiltinExpr()
15303 Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned); in EmitX86BuiltinExpr()
15335 std::swap(Ops[0], Ops[1]); in EmitX86BuiltinExpr()
15336 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty); in EmitX86BuiltinExpr()
15337 return Builder.CreateCall(F, Ops); in EmitX86BuiltinExpr()
15354 return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], Align(1), true); in EmitX86BuiltinExpr()
15377 Builder.CreateIntToPtr(Ops[0], llvm::PointerType::get(IntTy, 257)); in EmitX86BuiltinExpr()
15389 Builder.CreateIntToPtr(Ops[0], llvm::PointerType::get(IntTy, 256)); in EmitX86BuiltinExpr()
15398 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[0], Ops[1]}); in EmitX86BuiltinExpr()
15402 Value *Ptr = Builder.CreateConstGEP1_32(Int8Ty, Ops[2], i * 16); in EmitX86BuiltinExpr()
15414 Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[0], Ops[1], Ops[2]}); in EmitX86BuiltinExpr()
15418 Value *Ptr = Builder.CreateConstGEP1_32(Int8Ty, Ops[3], i * 16); in EmitX86BuiltinExpr()
15453 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[1], Ops[2]}); in EmitX86BuiltinExpr()
15466 Builder.CreateDefaultAlignedStore(Out, Ops[0]); in EmitX86BuiltinExpr()
15471 Builder.CreateDefaultAlignedStore(Zero, Ops[0]); in EmitX86BuiltinExpr()
15504 InOps[0] = Ops[2]; in EmitX86BuiltinExpr()
15506 Value *Ptr = Builder.CreateConstGEP1_32(Ty, Ops[1], i); in EmitX86BuiltinExpr()
15524 Value *Ptr = Builder.CreateConstGEP1_32(Extract->getType(), Ops[0], i); in EmitX86BuiltinExpr()
15533 Value *Ptr = Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i); in EmitX86BuiltinExpr()
15548 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
15549 return EmitX86Select(*this, Ops[3], Call, Ops[0]); in EmitX86BuiltinExpr()
15557 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
15558 Value *And = Builder.CreateAnd(Ops[3], llvm::ConstantInt::get(Int8Ty, 1)); in EmitX86BuiltinExpr()
15559 return EmitX86Select(*this, And, Call, Ops[0]); in EmitX86BuiltinExpr()
15567 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops); in EmitX86BuiltinExpr()
15569 return Builder.CreateShuffleVector(Call, Ops[2], Mask); in EmitX86BuiltinExpr()
15573 CGM.getIntrinsic(Intrinsic::prefetch, Ops[0]->getType()), in EmitX86BuiltinExpr()
15574 {Ops[0], llvm::ConstantInt::get(Int32Ty, 0), Ops[1], in EmitX86BuiltinExpr()
15618 SmallVector<Value *, 2> Ops; in EmitPPCBuiltinExpr() local
15619 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitPPCBuiltinExpr()
15620 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitPPCBuiltinExpr()
15623 Ops[0] = Builder.CreateBitCast(Ops[0], Int8PtrTy); in EmitPPCBuiltinExpr()
15625 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); in EmitPPCBuiltinExpr()
15626 Ops[0] = Builder.CreateGEP(Int8Ty, Ops[1], Ops[0]); in EmitPPCBuiltinExpr()
15627 Ops.pop_back(); in EmitPPCBuiltinExpr()
15673 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
15689 SmallVector<Value *, 3> Ops; in EmitPPCBuiltinExpr() local
15690 Ops.push_back(EmitScalarExpr(E->getArg(0))); in EmitPPCBuiltinExpr()
15691 Ops.push_back(EmitScalarExpr(E->getArg(1))); in EmitPPCBuiltinExpr()
15692 Ops.push_back(EmitScalarExpr(E->getArg(2))); in EmitPPCBuiltinExpr()
15695 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); in EmitPPCBuiltinExpr()
15697 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); in EmitPPCBuiltinExpr()
15698 Ops[1] = Builder.CreateGEP(Int8Ty, Ops[2], Ops[1]); in EmitPPCBuiltinExpr()
15699 Ops.pop_back(); in EmitPPCBuiltinExpr()
15739 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
15999 SmallVector<Value *, 2> Ops; in EmitPPCBuiltinExpr() local
16004 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty)); in EmitPPCBuiltinExpr()
16005 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty)); in EmitPPCBuiltinExpr()
16009 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops, ""); in EmitPPCBuiltinExpr()
16015 SmallVector<Value *, 3> Ops; in EmitPPCBuiltinExpr() local
16021 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty)); in EmitPPCBuiltinExpr()
16022 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty)); in EmitPPCBuiltinExpr()
16023 Ops.push_back(Builder.CreateBitCast(Op2, V1I128Ty)); in EmitPPCBuiltinExpr()
16040 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops, ""); in EmitPPCBuiltinExpr()
16476 SmallVector<Value *, 4> Ops; in EmitPPCBuiltinExpr() local
16479 Ops.push_back(EmitArrayToPointerDecay(E->getArg(i)).getPointer()); in EmitPPCBuiltinExpr()
16481 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitPPCBuiltinExpr()
16500 Value *Ptr = Builder.CreateBitCast(Ops[0], VTy->getPointerTo()); in EmitPPCBuiltinExpr()
16518 std::reverse(Ops.begin() + 1, Ops.end()); in EmitPPCBuiltinExpr()
16535 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy); in EmitPPCBuiltinExpr()
16536 Ops[0] = Builder.CreateGEP(Int8Ty, Ops[1], Ops[0]); in EmitPPCBuiltinExpr()
16538 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy); in EmitPPCBuiltinExpr()
16539 Ops[1] = Builder.CreateGEP(Int8Ty, Ops[2], Ops[1]); in EmitPPCBuiltinExpr()
16541 Ops.pop_back(); in EmitPPCBuiltinExpr()
16543 return Builder.CreateCall(F, Ops, ""); in EmitPPCBuiltinExpr()
16551 for (unsigned i=1; i<Ops.size(); i++) in EmitPPCBuiltinExpr()
16552 CallOps.push_back(Ops[i]); in EmitPPCBuiltinExpr()
16555 return Builder.CreateAlignedStore(Call, Ops[0], MaybeAlign(64)); in EmitPPCBuiltinExpr()
19062 Value *Ops[18]; in EmitWebAssemblyBuiltinExpr() local
19064 Ops[OpIdx++] = EmitScalarExpr(E->getArg(0)); in EmitWebAssemblyBuiltinExpr()
19065 Ops[OpIdx++] = EmitScalarExpr(E->getArg(1)); in EmitWebAssemblyBuiltinExpr()
19070 Ops[OpIdx++] = llvm::ConstantInt::get(getLLVMContext(), *LaneConst); in EmitWebAssemblyBuiltinExpr()
19073 return Builder.CreateCall(Callee, Ops); in EmitWebAssemblyBuiltinExpr()
19272 SmallVector<llvm::Value*,5> Ops = { Base }; in EmitHexagonBuiltinExpr() local
19274 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitHexagonBuiltinExpr()
19276 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops); in EmitHexagonBuiltinExpr()
19392 SmallVector<llvm::Value*,4> Ops; in EmitHexagonBuiltinExpr() local
19398 Ops.push_back(V2Q(EmitScalarExpr(PredOp))); in EmitHexagonBuiltinExpr()
19401 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitHexagonBuiltinExpr()
19402 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops); in EmitHexagonBuiltinExpr()
19449 SmallVector<Value *, 4> Ops; in EmitRISCVBuiltinExpr() local
19471 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitRISCVBuiltinExpr()
19477 Ops.push_back(llvm::ConstantInt::get( in EmitRISCVBuiltinExpr()
19517 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType()); in EmitRISCVBuiltinExpr()
19518 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitRISCVBuiltinExpr()
19522 Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType()); in EmitRISCVBuiltinExpr()
19523 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)}); in EmitRISCVBuiltinExpr()
19677 return Builder.CreateCall(F, Ops, ""); in EmitRISCVBuiltinExpr()
19682 SmallVector<Value *, 4> Ops; in EmitLoongArchBuiltinExpr() local
19685 Ops.push_back(EmitScalarExpr(E->getArg(i))); in EmitLoongArchBuiltinExpr()
19803 return Builder.CreateCall(F, Ops); in EmitLoongArchBuiltinExpr()