Lines Matching full:builder
81 Byte = CGF.Builder.getInt8(0x00);
92 auto *I = CGF.Builder.CreateMemSet(AI, Byte, Size, AlignmentInBytes);
177 return CGF.Builder.CreatePtrToInt(V, IntType);
188 return CGF.Builder.CreateIntToPtr(V, ResultType);
232 CGF.Builder.CreateAtomicRMW(Kind, DestAddr, Val, Ordering);
284 llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
286 Result = CGF.Builder.CreateBinOp(Op, Result, Val);
289 CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,
322 Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
327 return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1),
331 return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T,
373 auto *Result = CGF.Builder.CreateAtomicCmpXchg(
376 return CGF.Builder.CreateExtractValue(Result, 0);
417 ExchangeHigh = CGF.Builder.CreateZExt(ExchangeHigh, Int128Ty);
418 ExchangeLow = CGF.Builder.CreateZExt(ExchangeLow, Int128Ty);
420 CGF.Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
421 llvm::Value *Exchange = CGF.Builder.CreateOr(ExchangeHigh, ExchangeLow);
424 llvm::Value *Comparand = CGF.Builder.CreateLoad(ComparandAddr);
426 auto *CXI = CGF.Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
436 CGF.Builder.CreateStore(CGF.Builder.CreateExtractValue(CXI, 0),
440 Value *Success = CGF.Builder.CreateExtractValue(CXI, 1);
441 return CGF.Builder.CreateZExt(Success, CGF.Int8Ty);
450 auto *Result = CGF.Builder.CreateAtomicRMW(
452 return CGF.Builder.CreateAdd(Result, ConstantInt::get(IntTy, 1));
462 auto *Result = CGF.Builder.CreateAtomicRMW(
464 return CGF.Builder.CreateSub(Result, ConstantInt::get(IntTy, 1));
474 llvm::LoadInst *Load = CGF.Builder.CreateAlignedLoad(ITy, Ptr, LoadSize);
486 CGF.Builder.CreateAlignedStore(Value, Ptr, StoreSize);
500 if (CGF.Builder.getIsFPConstrained()) {
502 return CGF.Builder.CreateConstrainedFPCall(F, { Src0 });
505 return CGF.Builder.CreateCall(F, Src0);
518 if (CGF.Builder.getIsFPConstrained()) {
520 return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
523 return CGF.Builder.CreateCall(F, { Src0, Src1 });
535 if (CGF.Builder.getIsFPConstrained()) {
538 return CGF.Builder.CreateConstrainedFPCall(F, {Src0, Src1});
543 return CGF.Builder.CreateCall(F, {Src0, Src1});
556 if (CGF.Builder.getIsFPConstrained()) {
558 return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
561 return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });
573 if (CGF.Builder.getIsFPConstrained())
578 if (CGF.Builder.getIsFPConstrained())
579 return CGF.Builder.CreateConstrainedFPCall(F, Args);
581 return CGF.Builder.CreateCall(F, Args);
596 return CGF.Builder.CreateCall(F, Args, Name);
607 return CGF.Builder.CreateCall(F, {Src0, Src1});
618 if (CGF.Builder.getIsFPConstrained()) {
622 return CGF.Builder.CreateConstrainedFPCall(F, {Src0});
626 return CGF.Builder.CreateCall(F, Src0);
639 llvm::Value *Call = CGF.Builder.CreateCall(F, Src0);
641 llvm::Value *Exp = CGF.Builder.CreateExtractValue(Call, 1);
645 return CGF.Builder.CreateExtractValue(Call, 0);
651 llvm::CallInst *Call = CGF.Builder.CreateCall(F, V);
664 V = CGF.Builder.CreateBitCast(V, IntTy);
676 V = CGF.Builder.CreateLShr(V, ShiftCst);
681 V = CGF.Builder.CreateTrunc(V, IntTy);
684 return CGF.Builder.CreateICmpSLT(V, Zero);
709 !CGF.Builder.getIsFPConstrained()) {
740 llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y});
741 Carry = CGF.Builder.CreateExtractValue(Tmp, 1);
742 return CGF.Builder.CreateExtractValue(Tmp, 0);
748 llvm::CallInst *Call = CGF.Builder.CreateCall(F);
803 return Builder.CreateCall(CGM.getIntrinsic(inst, {ArgValue->getType()}),
1028 CountedByInst = Builder.CreateIntCast(CountedByInst, ResType, IsSigned);
1039 IdxInst = Builder.CreateIntCast(IdxInst, ResType, IdxSigned);
1044 Builder.CreateSub(CountedByInst, IdxInst, "", !IsSigned, IsSigned);
1053 Builder.CreateMul(CountedByInst, ElemSize, "", !IsSigned, IsSigned);
1054 Res = Builder.CreateIntCast(Res, ResType, IsSigned);
1059 Value *Cmp = Builder.CreateIsNotNeg(CountedByInst);
1061 Cmp = Builder.CreateAnd(Builder.CreateIsNotNeg(IdxInst), Cmp);
1063 return Builder.CreateSelect(Cmp, Res, ConstantInt::get(ResType, 0, IsSigned));
1119 Value *Min = Builder.getInt1((Type & 2) != 0);
1121 Value *NullIsUnknown = Builder.getTrue();
1122 Value *Dynamic = Builder.getInt1(IsDynamic);
1123 return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic});
1236 return CGF.Builder.CreateCall(IA, {BitBase, BitPos});
1272 Value *ByteIndex = CGF.Builder.CreateAShr(
1274 Value *BitBaseI8 = CGF.Builder.CreatePointerCast(BitBase, CGF.Int8PtrTy);
1275 Address ByteAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, BitBaseI8,
1279 CGF.Builder.CreateAnd(CGF.Builder.CreateTrunc(BitPos, CGF.Int8Ty),
1285 Mask = CGF.Builder.CreateShl(llvm::ConstantInt::get(CGF.Int8Ty, 1), PosLow,
1298 Mask = CGF.Builder.CreateNot(Mask);
1301 OldByte = CGF.Builder.CreateAtomicRMW(RMWOp, ByteAddr, Mask, Ordering);
1304 OldByte = CGF.Builder.CreateLoad(ByteAddr, "bittest.byte");
1311 NewByte = CGF.Builder.CreateXor(OldByte, Mask);
1314 NewByte = CGF.Builder.CreateAnd(OldByte, CGF.Builder.CreateNot(Mask));
1317 NewByte = CGF.Builder.CreateOr(OldByte, Mask);
1321 CGF.Builder.CreateStore(NewByte, ByteAddr);
1326 Value *ShiftedByte = CGF.Builder.CreateLShr(OldByte, PosLow, "bittest.shr");
1327 return CGF.Builder.CreateAnd(
1375 llvm::CallInst *CI = CGF.Builder.CreateCall(IA, {Addr});
1407 Arg1 = CGF.Builder.CreateCall(
1410 Arg1 = CGF.Builder.CreateCall(
1424 llvm::Value *Buf = CGF.Builder.CreateBitOrPointerCast(
1827 BasicBlock *Begin = Builder.GetInsertBlock();
1829 Builder.SetInsertPoint(End);
1830 PHINode *Result = Builder.CreatePHI(ResultType, 2, "bitscan_result");
1832 Builder.SetInsertPoint(Begin);
1833 Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
1835 Builder.CreateCondBr(IsZero, End, NotZero);
1838 Builder.SetInsertPoint(NotZero);
1842 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1843 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
1844 Builder.CreateStore(ZeroCount, IndexAddress, false);
1850 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1851 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
1852 Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
1853 Builder.CreateStore(Index, IndexAddress, false);
1855 Builder.CreateBr(End);
1858 Builder.SetInsertPoint(End);
1980 llvm::CallInst *CI = Builder.CreateCall(IA, EmitScalarExpr(E->getArg(0)));
2010 Value *Cond = Builder.CreateICmpNE(
2015 llvm::ConstantInt::get(Builder.getInt8Ty(), Kind)},
2021 return CGF.Builder.CreateBinaryIntrinsic(
2023 ConstantInt::get(CGF.Builder.getInt1Ty(), HasNSW));
2039 Value *ResultAndOverflow = CGF.Builder.CreateBinaryIntrinsic(
2041 Value *Result = CGF.Builder.CreateExtractValue(ResultAndOverflow, 0);
2042 Value *NotOverflow = CGF.Builder.CreateNot(
2043 CGF.Builder.CreateExtractValue(ResultAndOverflow, 1));
2055 Value *CmpResult = CGF.Builder.CreateICmpSLT(ArgValue, Zero, "abscond");
2056 return CGF.Builder.CreateSelect(CmpResult, Result, ArgValue, "abs");
2132 Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"), Ctx.VoidTy,
2134 Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()),
2135 Builder.CreateConstByteGEP(BufAddr, Offset++, "summary"));
2136 Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()),
2137 Builder.CreateConstByteGEP(BufAddr, Offset++, "numArgs"));
2141 Builder.CreateStore(
2142 Builder.getInt8(Item.getDescriptorByte()),
2143 Builder.CreateConstByteGEP(BufAddr, Offset++, "argDescriptor"));
2144 Builder.CreateStore(
2145 Builder.getInt8(Item.getSizeByte()),
2146 Builder.CreateConstByteGEP(BufAddr, Offset++, "argSize"));
2153 Address Addr = Builder.CreateConstByteGEP(BufAddr, Offset, "argData");
2155 Builder.CreateStore(Builder.CreateLoad(Arg), Addr);
2219 Builder.CreateStore(ArgVal, Addr);
2231 ArgVal = Builder.getInt32(Item.getConstValue().getQuantity());
2238 ArgVal = Builder.CreateBitOrPointerCast(ArgVal, IntTy);
2241 ArgVal = Builder.CreateZExtOrBitCast(ArgVal, ConvertType(ArgTy));
2283 llvm::Value *IntMaxOverflow = CGF.Builder.CreateICmpUGT(Result, IntMaxValue);
2284 HasOverflow = CGF.Builder.CreateOr(HasOverflow, IntMaxOverflow);
2289 CGF.Builder.CreateStore(CGF.EmitToMemory(Result, ResultQTy), ResultPtr,
2326 Signed = CGF.Builder.CreateSExt(Signed, Unsigned->getType(), "op.sext");
2328 Unsigned = CGF.Builder.CreateZExt(Unsigned, Signed->getType(), "op.zext");
2337 llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(Signed, Zero);
2338 llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(Zero, Signed);
2340 CGF.Builder.CreateSelect(IsNegative, AbsOfNegative, Signed);
2355 CGF.Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2356 CGF.Builder.CreateZExt(IsNegative, OpTy));
2358 CGF.Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2359 Overflow = CGF.Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2362 llvm::Value *NegativeResult = CGF.Builder.CreateNeg(UnsignedResult);
2364 CGF.Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2365 Result = CGF.Builder.CreateTrunc(SignedResult, ResTy);
2368 llvm::Value *Underflow = CGF.Builder.CreateAnd(
2369 IsNegative, CGF.Builder.CreateIsNotNull(UnsignedResult));
2370 Overflow = CGF.Builder.CreateOr(UnsignedOverflow, Underflow);
2374 llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
2376 Overflow = CGF.Builder.CreateOr(Overflow, TruncOverflow);
2380 Result = CGF.Builder.CreateSelect(
2381 IsNegative, CGF.Builder.CreateNeg(UnsignedResult), UnsignedResult);
2383 Result = CGF.Builder.CreateTrunc(Result, ResTy);
2389 CGF.Builder.CreateStore(CGF.EmitToMemory(Result, ResultQTy), ResultPtr,
2437 ShiftAmt = Builder.CreateIntCast(ShiftAmt, Ty, false);
2442 return RValue::get(Builder.CreateCall(F, { Src, Src, ShiftAmt }));
2519 if (CGF.Builder.getIsFPConstrained() &&
2520 CGF.Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2522 CGF.getTargetHooks().testFPKind(V, BuiltinID, CGF.Builder, CGF.CGM))
2538 return RValue::get(CGF->Builder.CreateCall(UBF, Args));
2773 if (Builder.getIsFPConstrained())
2850 return RValue::get(Builder.CreateFRem(Arg1, Arg2, "fmod"));
3115 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy, {DstPtr->getType()}),
3159 Imag = Builder.CreateFNeg(Imag, "neg");
3214 Value *IsNeg = Builder.CreateICmpSLT(ArgValue, Zero, "isneg");
3215 Value *Inverse = Builder.CreateNot(ArgValue, "not");
3216 Value *Tmp = Builder.CreateSelect(IsNeg, Inverse, ArgValue);
3217 Value *Ctlz = Builder.CreateCall(F, {Tmp, Builder.getFalse()});
3218 Value *Result = Builder.CreateSub(Ctlz, llvm::ConstantInt::get(ArgType, 1));
3219 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3240 Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef());
3241 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
3244 Builder.CreateIntCast(Result, ResultType, /*isSigned*/ false, "cast");
3249 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
3252 Builder.CreateSelect(IsZero, FallbackValue, Result, "ctzg");
3272 Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef());
3273 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
3276 Builder.CreateIntCast(Result, ResultType, /*isSigned*/ false, "cast");
3281 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
3284 Builder.CreateSelect(IsZero, FallbackValue, Result, "clzg");
3298 Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
3301 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
3302 Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
3304 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3318 Value *Tmp = Builder.CreateCall(F, ArgValue);
3319 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
3321 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3334 Value *Result = Builder.CreateCall(F, {ArgValue, Builder.getFalse()});
3336 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
3353 Value *Result = Builder.CreateCall(F, ArgValue);
3356 Builder.CreateIntCast(Result, ResultType, /*isSigned*/ false, "cast");
3378 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval");
3404 Value *Result = Builder.CreateCall(
3432 Builder.CreateCall(FnAssume, ArgValue);
3444 Builder.CreateAssumption(ConstantInt::getTrue(getLLVMContext()), {OBD});
3451 llvm::Value *Allow = Builder.CreateCall(
3460 llvm::FastMathFlags FMF = Builder.getFastMathFlags();
3469 Value *Real = Builder.CreateArithmeticFence(ComplexVal.first,
3471 Value *Imag = Builder.CreateArithmeticFence(ComplexVal.second,
3483 Builder.CreateArithmeticFence(ArgValue, ConvertType(ArgType)));
3547 ArgValue = Builder.CreateBitCast(ArgValue, ConvertType(ArgType));
3551 Value *Result = Builder.CreateCall(F, ArgValue);
3553 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/false);
3577 Builder.CreateCall(F, {Address, RW, Locality, Data});
3582 return RValue::get(Builder.CreateCall(F));
3586 return RValue::get(Builder.CreateCall(F));
3592 return RValue::get(Builder.CreateCall(F, {Begin, End}));
3598 llvm::DILocation *TrapLocation = Builder.getCurrentDebugLocation();
3627 if (Builder.getIsFPConstrained()) {
3633 return RValue::get(Builder.CreateConstrainedFPCall(F, { Src0, Src1 }));
3638 return RValue::get(Builder.CreateCall(F, { Src0, Src1 }));
3668 LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp");
3671 LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp");
3674 LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp");
3677 LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp");
3680 LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp");
3683 LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp");
3687 return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
3696 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcNan),
3704 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcSNan),
3714 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcInf),
3730 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcFinite),
3738 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcNormal),
3746 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcSubnormal),
3754 Builder.CreateZExt(Builder.createIsFPClass(V, FPClassTest::fcZero),
3765 return RValue::get(Builder.CreateZExt(Builder.createIsFPClass(V, Test),
3773 Result = Builder.CreateFreeze(Result);
3785 Result = Builder.CreateBinaryIntrinsic(
3787 Builder.getFalse(), nullptr, "elt.abs");
3888 Result = Builder.CreateBinaryIntrinsic(Opc, Op0, Op1, nullptr, "elt.sat");
3900 Result = Builder.CreateBinaryIntrinsic(Ty->isSignedIntegerType()
3905 Result = Builder.CreateMaxNum(Op0, Op1, "elt.max");
3916 Result = Builder.CreateBinaryIntrinsic(Ty->isSignedIntegerType()
3921 Result = Builder.CreateMinNum(Op0, Op1, "elt.min");
3981 MatrixBuilder MB(Builder);
3988 MatrixBuilder MB(Builder);
4008 MatrixBuilder MB(Builder);
4034 Value *IsInf = Builder.CreateFCmpOEQ(
4042 Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
4043 Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero);
4051 Value *Result = Builder.CreateCall(F);
4053 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
4062 Builder.CreateCall(F, V);
4073 BasicBlock *Begin = Builder.GetInsertBlock();
4075 Builder.SetInsertPoint(End);
4077 Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4,
4081 Builder.SetInsertPoint(Begin);
4082 Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
4086 Builder.CreateCondBr(IsZero, End, NotZero);
4090 Builder.SetInsertPoint(NotZero);
4091 Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp");
4094 Builder.CreateCondBr(IsNan, End, NotNan);
4098 Builder.SetInsertPoint(NotNan);
4101 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
4105 Builder.CreateCondBr(IsInf, End, NotInf);
4109 Builder.SetInsertPoint(NotInf);
4113 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
4116 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
4118 Builder.CreateBr(End);
4122 Builder.SetInsertPoint(End);
4143 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
4165 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
4185 Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false);
4200 Builder.CreateMemMove(Dest, Src, SizeVal, false);
4213 Builder.CreateMemCpy(Dest, Src, SizeVal, false);
4216 return RValue::get(Builder.CreateInBoundsGEP(
4229 Builder.CreateMemCpyInline(Dest, Src, Size);
4249 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
4250 Builder.CreateMemCpy(Dest, Src, SizeVal, false);
4275 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
4276 Builder.CreateMemMove(Dest, Src, SizeVal, false);
4287 Builder.CreateMemMove(Dest, Src, SizeVal, false);
4293 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
4294 Builder.getInt8Ty());
4298 Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
4304 Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty());
4310 Builder.CreateMemSetInline(Dest, ByteVal, Size);
4324 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
4325 Builder.getInt8Ty());
4326 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
4327 Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
4341 BasicBlock *Entry = Builder.GetInsertBlock();
4345 Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(SizeTy, 0));
4346 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
4349 PHINode *StrPhi = Builder.CreatePHI(Str->getType(), 2);
4351 PHINode *SizePhi = Builder.CreatePHI(SizeTy, 2);
4355 Value *StrCh = Builder.CreateAlignedLoad(WCharTy, StrPhi, WCharAlign);
4356 Value *FoundChr = Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
4357 Value *StrEqChr = Builder.CreateICmpEQ(StrCh, Chr);
4358 Builder.CreateCondBr(StrEqChr, Exit, Next);
4361 Value *NextStr = Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
4362 Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(SizeTy, 1));
4364 Builder.CreateICmpEQ(NextSize, ConstantInt::get(SizeTy, 0));
4365 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
4370 PHINode *Ret = Builder.CreatePHI(Str->getType(), 3);
4388 BasicBlock *Entry = Builder.GetInsertBlock();
4393 Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(SizeTy, 0));
4394 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
4397 PHINode *DstPhi = Builder.CreatePHI(Dst->getType(), 2);
4399 PHINode *SrcPhi = Builder.CreatePHI(Src->getType(), 2);
4401 PHINode *SizePhi = Builder.CreatePHI(SizeTy, 2);
4405 Value *DstCh = Builder.CreateAlignedLoad(WCharTy, DstPhi, WCharAlign);
4406 Value *SrcCh = Builder.CreateAlignedLoad(WCharTy, SrcPhi, WCharAlign);
4407 Value *DstGtSrc = Builder.CreateICmpUGT(DstCh, SrcCh);
4408 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
4411 Value *DstLtSrc = Builder.CreateICmpULT(DstCh, SrcCh);
4412 Builder.CreateCondBr(DstLtSrc, Exit, Next);
4415 Value *NextDst = Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
4416 Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
4417 Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(SizeTy, 1));
4419 Builder.CreateICmpEQ(NextSize, ConstantInt::get(SizeTy, 0));
4420 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
4426 PHINode *Ret = Builder.CreatePHI(IntTy, 4);
4445 return RValue::get(Builder.CreateCall(F,
4452 return RValue::get(Builder.CreateCall(F, Depth));
4456 return RValue::get(Builder.CreateCall(F, Builder.getInt32(0)));
4462 return RValue::get(Builder.CreateCall(F, Depth));
4500 Builder.CreateCall(F, {Int, Ptr});
4501 Builder.CreateUnreachable();
4510 Builder.CreateCall(F);
4526 Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast");
4534 return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext"));
4536 return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext"));
4543 Value *FrameAddr = Builder.CreateCall(
4546 Builder.CreateStore(FrameAddr, Buf);
4549 Value *StackAddr = Builder.CreateStackSave();
4552 Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Buf, 2);
4553 Builder.CreateStore(StackAddr, StackSaveSlot);
4557 return RValue::get(Builder.CreateCall(F, Buf.emitRawPointer(*this)));
4563 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
4566 Builder.CreateUnreachable();
4578 Ptr = Builder.CreateLaunderInvariantGroup(Ptr);
4729 Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr);
4742 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
4783 Value *NewVal = Builder.getInt8(1);
4791 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4796 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4800 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4805 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4809 Result = Builder.CreateAtomicRMW(
4815 return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
4832 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
4833 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
4835 Builder.SetInsertPoint(ContBB);
4836 PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set");
4839 Builder.SetInsertPoint(BBs[i]);
4840 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
4844 Builder.CreateBr(ContBB);
4847 SI->addCase(Builder.getInt32(0), BBs[0]);
4848 SI->addCase(Builder.getInt32(1), BBs[1]);
4849 SI->addCase(Builder.getInt32(2), BBs[1]);
4850 SI->addCase(Builder.getInt32(3), BBs[2]);
4851 SI->addCase(Builder.getInt32(4), BBs[3]);
4852 SI->addCase(Builder.getInt32(5), BBs[4]);
4854 Builder.SetInsertPoint(ContBB);
4855 return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
4865 Value *NewVal = Builder.getInt8(0);
4869 StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
4896 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
4897 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
4900 Builder.SetInsertPoint(BBs[i]);
4901 StoreInst *Store = Builder.CreateStore(NewVal, Ptr, Volatile);
4903 Builder.CreateBr(ContBB);
4906 SI->addCase(Builder.getInt32(0), BBs[0]);
4907 SI->addCase(Builder.getInt32(3), BBs[1]);
4908 SI->addCase(Builder.getInt32(5), BBs[2]);
4910 Builder.SetInsertPoint(ContBB);
4933 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4936 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4939 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4942 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4955 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
4956 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
4958 Builder.SetInsertPoint(AcquireBB);
4959 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4960 Builder.CreateBr(ContBB);
4961 SI->addCase(Builder.getInt32(1), AcquireBB);
4962 SI->addCase(Builder.getInt32(2), AcquireBB);
4964 Builder.SetInsertPoint(ReleaseBB);
4965 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4966 Builder.CreateBr(ContBB);
4967 SI->addCase(Builder.getInt32(3), ReleaseBB);
4969 Builder.SetInsertPoint(AcqRelBB);
4970 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4971 Builder.CreateBr(ContBB);
4972 SI->addCase(Builder.getInt32(4), AcqRelBB);
4974 Builder.SetInsertPoint(SeqCstBB);
4975 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4976 Builder.CreateBr(ContBB);
4977 SI->addCase(Builder.getInt32(5), SeqCstBB);
4979 Builder.SetInsertPoint(ContBB);
4987 Builder.CreateZExt(EmitSignBit(*this, EmitScalarExpr(E->getArg(0))),
5012 Builder.CreateCall(F, MetadataAsValue::get(getLLVMContext(), StrTuple));
5089 llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2),
5091 Builder.CreateStore(CarryOut, CarryOutPtr);
5159 Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
5160 Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
5169 llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy);
5173 llvm::Value *ResultTruncExt = Builder.CreateIntCast(
5176 Builder.CreateICmpNE(Result, ResultTruncExt);
5178 Overflow = Builder.CreateOr(Overflow, TruncationOverflow);
5185 Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile);
5255 Builder.CreateStore(Sum, SumOutPtr);
5307 Exchange = Builder.CreatePtrToInt(Exchange, IntType);
5310 Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
5316 auto Result = Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
5320 return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
5410 Args[0] = Builder.CreatePtrToInt(Args[0], IntPtrTy);
5415 Args[4] = Builder.CreatePtrToInt(Args[4], IntPtrTy);
5421 Args[2] = Builder.CreatePtrToInt(Args[2], IntPtrTy);
5426 Args[1] = Builder.CreatePtrToInt(Args[1], IntPtrTy);
5458 Result = Builder.CreateIntToPtr(Result, OrigValueType);
5559 Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy);
5575 Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy);
5579 Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty);
5622 Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty);
5704 NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT);
5706 NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT);
5710 return RValue::get(Builder.CreateBitOrPointerCast(NewCall,
5727 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
5748 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5750 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5752 AttrBuilder B(Builder.getContext());
5784 auto *GEP = Builder.CreateGEP(Tmp.getElementType(), TmpPtr,
5789 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(I)), SizeTy);
5790 Builder.CreateAlignedStore(
5803 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5804 auto *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5832 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(3)), Int32Ty);
5847 EventWaitList = Builder.CreatePointerCast(EventWaitList, PtrTy);
5855 Builder.CreatePointerCast(EmitScalarExpr(E->getArg(5)), PtrTy);
5861 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5863 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5907 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
5912 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5913 Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5922 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
5927 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5928 Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5938 llvm::Type *GenericVoidPtrTy = Builder.getPtrTy(
5945 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5946 Value *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5963 Value *HalfVal = Builder.CreateFPTrunc(Val, Builder.getHalfTy());
5964 Builder.CreateStore(HalfVal, Address);
5969 Value *HalfVal = Builder.CreateLoad(Address);
5970 return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getDoubleTy()));
5974 Value *HalfVal = Builder.CreateLoad(Address);
5975 return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getFloatTy()));
6032 Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0);
6037 Arg1 = Builder.CreateTruncOrBitCast(Arg1, PTy1);
6038 return RValue::get(Builder.CreateCall(F, {Arg0Val, Arg1}));
6061 Arg0 = Builder.CreateTruncOrBitCast(Arg0, PTy0);
6070 Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1);
6075 Arg2 = Builder.CreateTruncOrBitCast(Arg2, PTy2);
6076 return RValue::get(Builder.CreateCall(F, {Arg0, Arg1Val, Arg2}));
6098 Value *ArgPtr = Builder.CreateLoad(SrcAddr, "ap.val");
6099 return RValue::get(Builder.CreateStore(ArgPtr, DestAddr));
6172 ArgValue = Builder.CreateAddrSpaceCast(
6181 ArgValue = Builder.CreateIntrinsic(Intrinsic::x86_cast_vector_to_tile,
6184 ArgValue = Builder.CreateBitCast(ArgValue, PTy);
6190 Value *V = Builder.CreateCall(F, Args);
6201 V = Builder.CreateAddrSpaceCast(
6210 V = Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector, {RetTy},
6213 V = Builder.CreateBitCast(V, RetTy);
6394 return Builder.CreateShuffleVector(V, V, SV, "lane");
6414 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
6418 return Builder.CreateConstrainedFPCall(F, Ops, name);
6420 return Builder.CreateCall(F, Ops, name);
6438 Vec = Builder.CreateBitCast(Vec, Ty);
6456 return Builder.CreateLShr(Vec, Shift, name);
6458 return Builder.CreateAShr(Vec, Shift, name);
7644 Ops[j] = CGF.Builder.CreateTruncOrBitCast(
7647 CGF.Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0);
7654 return CGF.Builder.CreateExtractElement(Result, C0);
7656 return CGF.Builder.CreateBitCast(Result, ResultType, s);
7686 return Builder.getInt32(addr.getAlignment().getQuantity());
7705 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
7723 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
7724 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
7725 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]);
7726 return Builder.CreateBitCast(Ops[0], Ty);
7733 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
7734 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
7735 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1], "vaddhn");
7740 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vaddhn");
7743 return Builder.CreateTrunc(Ops[0], VTy, "vaddhn");
7797 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
7801 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7804 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
7805 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
7810 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7813 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
7814 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
7860 Ops[0] = Builder.CreateBitCast(Ops[0], GetFloatNeonType(this, Type));
7861 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
7862 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
7927 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7928 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7929 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices, "vext");
7933 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7934 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7935 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7956 Ops[1] = Builder.CreateCall(F, Ops[1], "vld1xN");
7957 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7974 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
7975 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7981 LoadInst *Ld = Builder.CreateLoad(PtrOp0);
7983 Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
7995 Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
7997 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), NameHint);
7998 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8003 Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
8005 return Builder.CreateZExt(Ops[0], Ty, "vmovl");
8006 return Builder.CreateSExt(Ops[0], Ty, "vmovl");
8011 Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
8012 return Builder.CreateTrunc(Ops[0], Ty, "vmovn");
8091 Int = Builder.getIsFPConstrained()
8109 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
8114 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
8116 Ops[0] = Builder.CreateZExt(Ops[0], VTy);
8118 Ops[0] = Builder.CreateSExt(Ops[0], VTy);
8120 return Builder.CreateShl(Ops[0], Ops[1], "vshll_n");
8125 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
8128 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
8130 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
8131 return Builder.CreateTrunc(Ops[0], Ty, "vshrn_n");
8167 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
8192 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
8193 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
8194 Ops[0] = Builder.CreateSub(Ops[0], Ops[1], "vsubhn");
8199 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt, "vsubhn");
8202 return Builder.CreateTrunc(Ops[0], VTy, "vsubhn");
8206 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8207 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8216 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8217 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
8218 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8224 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8225 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8226 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
8227 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
8229 return Builder.CreateSExt(Ops[0], Ty, "vtst");
8233 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8234 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8242 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8243 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
8244 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8250 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
8255 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8256 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8265 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8266 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
8267 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8352 return Builder.CreateBitCast(Result, ResultType, NameHint);
8367 Op = Builder.CreateBitCast(Op, OTy);
8370 Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
8372 Op = Builder.CreateFCmpS(Fp, Op, Constant::getNullValue(OTy));
8374 Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
8376 return Builder.CreateSExt(Op, Ty, Name);
8397 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
8407 TblOps.push_back(CGF.Builder.CreateShuffleVector(Ops[PairPos],
8448 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
8463 CodeGen::CGBuilderTy &Builder = CGF.Builder;
8467 llvm::Value *Call = Builder.CreateCall(F, {Builder.getInt1(true)});
8470 Value *Rt2 = Builder.CreateLShr(Call, 32);
8471 Rt2 = Builder.CreateTrunc(Rt2, CGF.Int32Ty);
8492 CodeGen::CGBuilderTy &Builder = CGF.Builder;
8517 llvm::Value *Call = Builder.CreateCall(F, Metadata);
8521 return Builder.CreateTrunc(Call, ValueType);
8525 return Builder.CreateIntToPtr(Call, ValueType);
8534 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
8535 return Builder.CreateCall(F, { Metadata, ArgValue });
8540 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
8541 return Builder.CreateCall(F, { Metadata, ArgValue });
8544 return Builder.CreateCall(F, { Metadata, ArgValue });
8615 return Builder.CreateCall(Emit);
8620 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
8632 return Builder.CreateCall(F, {Address, RW, Locality, IsData});
8637 return Builder.CreateCall(
8645 Value *Res = Builder.CreateCall(F, {Arg, Builder.getInt1(false)});
8647 Res = Builder.CreateTrunc(Res, Builder.getInt32Ty());
8654 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls), Arg, "cls");
8658 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_cls64), Arg,
8701 Value *Rt = Builder.CreateTruncOrBitCast(RtAndRt2, Int32Ty);
8702 Value *Rt2 = Builder.CreateLShr(RtAndRt2, C1);
8703 Rt2 = Builder.CreateTruncOrBitCast(Rt2, Int32Ty);
8705 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
8725 Value *RtAndRt2 = Builder.CreateCall(F, {Coproc, Opc1, CRm});
8730 Value *Rt = Builder.CreateExtractValue(RtAndRt2, 1);
8731 Value *Rt1 = Builder.CreateExtractValue(RtAndRt2, 0);
8732 Rt = Builder.CreateZExt(Rt, Int64Ty);
8733 Rt1 = Builder.CreateZExt(Rt1, Int64Ty);
8736 RtAndRt2 = Builder.CreateShl(Rt, ShiftCast, "shl", true);
8737 RtAndRt2 = Builder.CreateOr(RtAndRt2, Rt1);
8739 return Builder.CreateBitCast(RtAndRt2, ConvertType(E->getType()));
8762 Value *Val = Builder.CreateCall(F, LdPtr, "ldrexd");
8764 Value *Val0 = Builder.CreateExtractValue(Val, 1);
8765 Value *Val1 = Builder.CreateExtractValue(Val, 0);
8766 Val0 = Builder.CreateZExt(Val0, Int64Ty);
8767 Val1 = Builder.CreateZExt(Val1, Int64Ty);
8770 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
8771 Val = Builder.CreateOr(Val, Val1);
8772 return Builder.CreateBitCast(Val, ConvertType(E->getType()));
8788 CallInst *Val = Builder.CreateCall(F, LoadAddr, "ldrex");
8793 return Builder.CreateIntToPtr(Val, RealResTy);
8797 return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
8813 Builder.CreateStore(Val, Tmp);
8816 Val = Builder.CreateLoad(LdPtr);
8818 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
8819 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
8821 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "strexd");
8834 StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty);
8839 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
8840 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty);
8848 CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr}, "strex");
8856 return Builder.CreateCall(F);
8887 Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty);
8888 Value *Arg1b = Builder.CreateLShr(Arg1, C1);
8889 Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty);
8892 Value *Res = Builder.CreateCall(F, {Arg0, Arg1a});
8893 return Builder.CreateCall(F, {Res, Arg1b});
8895 Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty);
8898 return Builder.CreateCall(F, {Arg0, Arg1});
8938 return Builder.CreateCall(F);
8968 return Builder.getInt32(addr.getAlignment().getQuantity());
9056 return Builder.CreateExtractElement(Ops[0], Ops[1], "vget_lane");
9062 return Builder.CreateCall(F, {Arg}, "vrndn"); }
9076 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
9103 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
9131 return Builder.CreateCall(F, Ops, "vcvtr");
9164 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9167 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9173 Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
9176 return Builder.CreateShuffleVector(Ops[1], Ld, Indices, "vld1q_lane");
9180 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9182 Value *Ld = Builder.CreateLoad(PtrOp0);
9183 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
9209 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9210 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9213 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]});
9214 return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
9226 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
9228 return Builder.CreateAdd(Ops[0], Ops[1]);
9233 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9235 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
9238 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
9243 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
9244 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
9245 return Builder.CreateStore(Ops[1],
9280 static llvm::Value *SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V,
9284 return Unsigned ? Builder.CreateZExt(V, T) : Builder.CreateSExt(V, T);
9287 static llvm::Value *MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V,
9306 return Unsigned ? Builder.CreateLShr(V, Shift) : Builder.CreateAShr(V, Shift);
9309 static llvm::Value *ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V) {
9314 return Builder.CreateVectorSplat(Elements, V);
9317 static llvm::Value *ARMMVEVectorReinterpret(CGBuilderTy &Builder,
9334 return Builder.CreateCall(
9339 return Builder.CreateBitCast(V, DestType);
9343 static llvm::Value *VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd) {
9351 return Builder.CreateShuffleVector(V, Indices);
9354 static llvm::Value *VectorZip(CGBuilderTy &Builder, llvm::Value *V0,
9365 return Builder.CreateShuffleVector(V0, V1, Indices);
9369 static llvm::Value *ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT) {
9378 return ARMMVEVectorSplat(Builder, Lane);
9381 static llvm::Value *ARMMVEVectorElementReverse(CGBuilderTy &Builder,
9392 return Builder.CreateShuffleVector(V, Indices);
9442 Value *LoadResult = Builder.CreateCall(F, Ops);
9445 Value *Vec = Builder.CreateExtractValue(LoadResult, i);
9446 MvecOut = Builder.CreateInsertValue(MvecOut, Vec, {0, i});
9452 return Builder.CreateStore(MvecOut, ReturnValue.getAddress());
9479 auto Mvec = Builder.CreateLoad(MvecSlot.getAddress());
9481 Ops.push_back(Builder.CreateExtractValue(Mvec, {0, i}));
9487 ToReturn = Builder.CreateCall(F, Ops);
9560 CodeGen::CGBuilderTy &Builder = CGF.Builder;
9587 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
9588 CmpRes = Builder.CreateSExt(CmpRes, Ty);
9590 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
9591 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
9592 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
9604 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
9606 CmpRes = Builder.CreateSExt(CmpRes, Ty);
9608 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
9609 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
9610 return Builder.CreateOr(EltsFromInput, EltsFromTbl, "vtbx");
9652 Op = Builder.CreateBitCast(Op, Int16Ty);
9655 Op = Builder.CreateInsertElement(V, Op, CI);
9667 return Builder.getInt8Ty();
9669 return Builder.getInt16Ty();
9671 return Builder.getInt32Ty();
9673 return Builder.getInt64Ty();
9684 return Builder.getInt8Ty();
9686 return Builder.getInt16Ty();
9688 return Builder.getInt32Ty();
9690 return Builder.getInt64Ty();
9692 return Builder.getInt128Ty();
9695 return Builder.getHalfTy();
9697 return Builder.getFloatTy();
9699 return Builder.getDoubleTy();
9702 return Builder.getBFloatTy();
9708 return Builder.getInt1Ty();
9720 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
9722 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9724 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
9726 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
9729 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9731 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9733 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
9735 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
9738 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
9740 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9742 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
9744 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
9756 return llvm::ScalableVectorType::get(Builder.getInt8Ty(), 16);
9758 return llvm::ScalableVectorType::get(Builder.getInt16Ty(), 8);
9760 return llvm::ScalableVectorType::get(Builder.getInt32Ty(), 4);
9762 return llvm::ScalableVectorType::get(Builder.getInt64Ty(), 2);
9765 return llvm::ScalableVectorType::get(Builder.getHalfTy(), 8);
9767 return llvm::ScalableVectorType::get(Builder.getBFloatTy(), 8);
9769 return llvm::ScalableVectorType::get(Builder.getFloatTy(), 4);
9771 return llvm::ScalableVectorType::get(Builder.getDoubleTy(), 2);
9774 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
9776 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9778 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
9780 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
9788 return Builder.CreateCall(Ptrue, {Builder.getInt32(/*SV_ALL*/ 31)});
9830 Value *C = Builder.CreateCall(F, Pred);
9878 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
9881 Value *Call = Builder.CreateCall(F, Ops);
9885 return TypeFlags.isZExtReturn() ? Builder.CreateZExt(Call, ResultTy)
9886 : Builder.CreateSExt(Call, ResultTy);
9923 Ops[0] = Builder.CreateTrunc(Ops[0], OverloadedTy);
9940 Ops[3] = Builder.CreateShl(Ops[3], Log2_32(BytesPerElt));
9943 return Builder.CreateCall(F, Ops);
9971 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
9976 return Builder.CreateCall(F, Ops);
10013 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]);
10016 Value *Call = Builder.CreateCall(F, {Predicate, BasePtr});
10021 Value *SRet = Builder.CreateExtractValue(Call, I);
10022 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, Idx);
10059 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]);
10069 return Builder.CreateCall(F, Operands);
10086 Value *Call = Builder.CreateCall(F, {Ops[0], Ops[1]});
10097 return Builder.CreateCall(F, {Ops[0], Builder.getInt32(0)});
10112 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
10117 return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
10154 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
10158 cast<llvm::Instruction>(Builder.CreateCall(F, {Predicate, BasePtr}));
10165 return IsZExtReturn ? Builder.CreateZExt(Load, VectorTy)
10166 : Builder.CreateSExt(Load, VectorTy);
10201 BasePtr = Builder.CreateGEP(AddrMemoryTy, BasePtr, Ops[2]);
10205 IsQuadStore ? Ops.back() : Builder.CreateTrunc(Ops.back(), MemoryTy);
10210 cast<llvm::Instruction>(Builder.CreateCall(F, {Val, Predicate, BasePtr}));
10233 Builder.CreateCall(StreamingVectorLength);
10235 Builder.CreateMul(StreamingVectorLengthCall, Ops[4], "mulvl");
10237 BasePtr = Builder.CreateGEP(Int8Ty, Ops[3], Mulvl);
10243 return Builder.CreateCall(F, NewOps);
10255 return Builder.CreateCall(F, Ops);
10265 return Builder.CreateCall(F, Ops);
10272 Ops.push_back(Builder.getInt32(0));
10274 Ops[2] = Builder.CreateIntCast(Ops[2], Int32Ty, true);
10276 return Builder.CreateCall(F, Ops);
10282 return Builder.CreateVectorSplat(
10297 return Builder.CreateBitCast(Val, Ty);
10300 static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty,
10306 static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty,
10355 return Builder.CreateInsertVector(Ty, Ops[0], Ops[2], Idx);
10356 return Builder.CreateExtractVector(Ty, Ops[0], Idx);
10373 Call = Builder.CreateInsertVector(Ty, Call, Ops[I], Idx);
10399 Value *SRet = Builder.CreateExtractValue(Call, I);
10405 SRet, ScalableVectorType::get(Builder.getInt1Ty(), 16));
10407 Ret = Builder.CreateInsertVector(WideVTy, Ret, SRet, Idx);
10465 Ops.push_back(Builder.CreateExtractVector(NewVTy, Arg, Idx));
10511 InsertExplicitZeroOperand(Builder, Ty, Ops);
10514 InsertExplicitUndefOperand(Builder, Ty, Ops);
10519 Ops.push_back(Builder.getInt32(/*SV_ALL*/ 31));
10521 Ops.insert(&Ops[1], Builder.getInt32(/*SV_ALL*/ 31));
10550 Ops[1] = Builder.CreateSelect(Ops[0], Ops[1], SplatZero);
10555 Value *Call = Builder.CreateCall(F, Ops);
10574 return Builder.CreateCall(CastFromSVCountF, Ops[0]);
10581 return Builder.CreateCall(CastToSVCountF, Ops[0]);
10606 IsSVCount ? Builder.CreateCall(CastFromSVCountF, Ops[0]) : Ops[0];
10608 llvm::Value *PSel = Builder.CreateCall(F, {Ops0, Ops1, Ops[2]});
10609 return IsSVCount ? Builder.CreateCall(CastToSVCountF, PSel) : PSel;
10616 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
10624 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
10664 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType()));
10702 VecOps.push_back(Builder.CreateZExt(Ops[I], EltTy));
10706 Value *InsertSubVec = Builder.CreateInsertVector(
10707 OverloadedTy, PoisonValue::get(OverloadedTy), Vec, Builder.getInt64(0));
10712 Builder.CreateCall(F, {InsertSubVec, Builder.getInt64(0)});
10724 Value *Call = Builder.CreateCall(
10725 F, {Pred, DupQLane, EmitSVEDupX(Builder.getInt64(0))});
10733 auto SVBoolTy = ScalableVectorType::get(Builder.getInt1Ty(), 16);
10736 return Builder.CreateCall(CastToSVCountF, ConstantInt::getFalse(SVBoolTy));
10757 return Builder.CreateMul(NumEls, Builder.CreateCall(F));
10775 return Builder.CreateCall(F, Ops);
10790 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], Builder.getInt64(0));
10805 return Builder.CreateExtractVector(Ty, Ops[0], Builder.getInt64(0));
10820 Value *Insert = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0],
10821 Builder.getInt64(0));
10822 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
10823 {Insert, Builder.getInt64(0)});
10894 Value *Call = Builder.CreateCall(F, Ops);
10943 return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
10949 return Builder.CreateCall(F, Builder.CreateZExt(Arg, CGM.Int32Ty));
10964 Builder.CreateStore(Builder.CreateExtractValue(CI, 0),
10966 return Builder.CreateStore(Builder.CreateExtractValue(CI, 1),
10974 return Builder.CreateCall(
10981 return Builder.CreateCall(
10989 Value *Res = Builder.CreateCall(F, {Arg, Builder.getInt1(false)});
10991 Res = Builder.CreateTrunc(Res, Builder.getInt32Ty());
10997 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls), Arg,
11002 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_cls64), Arg,
11010 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint32z, Ty),
11018 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint64z, Ty),
11026 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint32x, Ty),
11034 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::aarch64_frint64x, Ty),
11042 return Builder.CreateCall(
11057 llvm::Value *Val = Builder.CreateCall(F, MemAddr);
11061 Builder.CreateGEP(Int64Ty, ValPtr, Builder.getInt32(i));
11064 ToRet = Builder.CreateStore(Builder.CreateExtractValue(Val, i), Addr);
11074 Builder.CreateGEP(Int64Ty, ValPtr, Builder.getInt32(i));
11077 Args.push_back(Builder.CreateLoad(Addr));
11086 return Builder.CreateCall(F, Args);
11097 llvm::Value *Val = Builder.CreateCall(F);
11098 Value *RandomValue = Builder.CreateExtractValue(Val, 0);
11099 Value *Status = Builder.CreateExtractValue(Val, 1);
11102 Builder.CreateStore(RandomValue, MemAddress);
11103 Status = Builder.CreateZExt(Status, Int32Ty);
11128 Value *Val = Builder.CreateCall(F, LdPtr, "ldxp");
11130 Value *Val0 = Builder.CreateExtractValue(Val, 1);
11131 Value *Val1 = Builder.CreateExtractValue(Val, 0);
11133 Val0 = Builder.CreateZExt(Val0, Int128Ty);
11134 Val1 = Builder.CreateZExt(Val1, Int128Ty);
11137 Val = Builder.CreateShl(Val0, ShiftCst, "shl", true /* nuw */);
11138 Val = Builder.CreateOr(Val, Val1);
11139 return Builder.CreateBitCast(Val, ConvertType(E->getType()));
11154 CallInst *Val = Builder.CreateCall(F, LoadAddr, "ldxr");
11159 return Builder.CreateIntToPtr(Val, RealResTy);
11163 return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
11180 llvm::Value *Val = Builder.CreateLoad(Tmp);
11182 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
11183 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
11185 return Builder.CreateCall(F, {Arg0, Arg1, StPtr}, "stxp");
11198 StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty);
11203 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
11204 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
11212 CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
11233 return Builder.CreateCall(F, Metadata);
11242 return Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0))});
11247 return Builder.CreateCall(F);
11251 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
11281 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
11283 return Builder.CreateCall(F, {Arg0, Arg1});
11291 Dst = Builder.CreatePointerCast(Dst, Int8PtrTy);
11292 Val = Builder.CreateTrunc(Val, Int8Ty);
11293 Size = Builder.CreateIntCast(Size, Int64Ty, false);
11294 return Builder.CreateCall(
11322 Pointer = Builder.CreatePointerCast(Pointer, Int8PtrTy);
11323 Mask = Builder.CreateZExt(Mask, Int64Ty);
11324 Value *RV = Builder.CreateCall(
11326 return Builder.CreatePointerCast(RV, T);
11332 Pointer = Builder.CreatePointerCast(Pointer, Int8PtrTy);
11333 TagOffset = Builder.CreateZExt(TagOffset, Int64Ty);
11334 Value *RV = Builder.CreateCall(
11336 return Builder.CreatePointerCast(RV, T);
11342 ExcludedMask = Builder.CreateZExt(ExcludedMask, Int64Ty);
11343 Pointer = Builder.CreatePointerCast(Pointer, Int8PtrTy);
11344 return Builder.CreateCall(
11352 TagAddress = Builder.CreatePointerCast(TagAddress, Int8PtrTy);
11353 Value *RV = Builder.CreateCall(
11355 return Builder.CreatePointerCast(RV, T);
11362 TagAddress = Builder.CreatePointerCast(TagAddress, Int8PtrTy);
11363 return Builder.CreateCall(
11369 PointerA = Builder.CreatePointerCast(PointerA, Int8PtrTy);
11370 PointerB = Builder.CreatePointerCast(PointerB, Int8PtrTy);
11371 return Builder.CreateCall(
11445 return Builder.CreateCall(F, Metadata);
11451 return Builder.CreateCall(F, { Metadata, ArgValue });
11457 return Builder.CreateCall(F);
11462 return Builder.CreateCall(F);
11472 Builder.CreateIntCast(EmitScalarExpr(E->getArg(0)), Int128Ty, IsSigned);
11474 Builder.CreateIntCast(EmitScalarExpr(E->getArg(1)), Int128Ty, IsSigned);
11478 MulResult = Builder.CreateNSWMul(LHS, RHS);
11479 HigherBits = Builder.CreateAShr(MulResult, 64);
11481 MulResult = Builder.CreateNUWMul(LHS, RHS);
11482 HigherBits = Builder.CreateLShr(MulResult, 64);
11484 HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
11500 llvm::Value *X18 = Builder.CreateCall(F, Metadata);
11501 X18 = Builder.CreateIntToPtr(X18, Int8PtrTy);
11504 Value *Offset = Builder.CreateZExt(EmitScalarExpr(E->getArg(0)), Int64Ty);
11505 Value *Ptr = Builder.CreateGEP(Int8Ty, X18, Offset);
11507 StoreInst *Store = Builder.CreateAlignedStore(Val, Ptr, CharUnits::One());
11524 llvm::Value *X18 = Builder.CreateCall(F, Metadata);
11525 X18 = Builder.CreateIntToPtr(X18, Int8PtrTy);
11528 Value *Offset = Builder.CreateZExt(EmitScalarExpr(E->getArg(0)), Int64Ty);
11529 Value *Ptr = Builder.CreateGEP(Int8Ty, X18, Offset);
11530 LoadInst *Load = Builder.CreateAlignedLoad(IntTy, Ptr, CharUnits::One());
11540 return Builder.CreateBitCast(Arg, RetTy);
11552 Arg = Builder.CreateXor(Arg, Constant::getAllOnesValue(ArgType));
11555 Value *Result = Builder.CreateCall(F, {Arg, Builder.getInt1(false)});
11559 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
11571 Value *Result = Builder.CreateCall(F, Arg, "cls");
11573 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
11583 Value *Result = Builder.CreateCall(F, ArgValue);
11585 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
11595 return Builder.CreateCall(F, {Address, RW, Locality, Data});
11600 Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0))});
11604 return ConstantInt::get(Builder.getInt32Ty(), 0);
11686 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
11687 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
11688 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]);
11690 return Builder.CreateBitCast(Ops[0], Int128Ty);
11695 return Builder.CreateAlignedLoad(Int128Ty, Ptr,
11700 return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->getArg(1)), Ptr);
11712 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
11714 return Builder.CreateUIToFP(Ops[0], FTy);
11715 return Builder.CreateSIToFP(Ops[0], FTy);
11734 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
11736 return Builder.CreateUIToFP(Ops[0], FTy);
11737 return Builder.CreateSIToFP(Ops[0], FTy);
11778 return Builder.CreateTrunc(Ops[0], Int16Ty);
11801 return Builder.CreateTrunc(Ops[0], Int16Ty);
11818 return Builder.CreateTrunc(Ops[0], Int16Ty);
11831 Ops[0] = Builder.CreateSExt(Ops[0], InTy, "sext");
11835 Ops[0] = Builder.CreateZExt(Ops[0], InTy);
11844 Vec = Builder.CreateBitCast(Vec, Ty, "v2i64");
11847 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
11848 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
11850 return Builder.CreateAdd(Op0, Op1, "vpaddd");
11856 Vec = Builder.CreateBitCast(Vec, Ty, "v2f64");
11859 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
11860 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
11862 return Builder.CreateFAdd(Op0, Op1, "vpaddd");
11868 Vec = Builder.CreateBitCast(Vec, Ty, "v2f32");
11871 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0, "lane0");
11872 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1, "lane1");
11874 return Builder.CreateFAdd(Op0, Op1, "vpaddd");
11919 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
11921 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty));
11922 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqzd");
11939 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
11940 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
11942 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
11944 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
11945 return Builder.CreateSExt(Ops[0], Int64Ty, "vcmpd");
11962 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
11963 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
11965 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
11967 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
11968 return Builder.CreateSExt(Ops[0], Int32Ty, "vcmpd");
11985 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
11986 Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy);
11988 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
11990 Ops[0] = Builder.CreateFCmpS(P, Ops[0], Ops[1]);
11991 return Builder.CreateSExt(Ops[0], Int16Ty, "vcmpd");
12018 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
12019 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
12020 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
12021 return Builder.CreateSExt(Ops[0], Int64Ty, "vceqd");
12026 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
12027 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
12028 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
12029 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
12031 return Builder.CreateSExt(Ops[0], Int64Ty, "vtstd");
12046 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
12050 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 1));
12052 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
12056 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(DoubleTy, 2));
12058 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vset_lane");
12063 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 8));
12064 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12069 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int8Ty, 16));
12070 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12075 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 4));
12076 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12081 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int16Ty, 8));
12082 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12087 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 2));
12088 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12092 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2));
12093 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12098 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int32Ty, 4));
12099 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12104 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 1));
12105 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12109 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1));
12110 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12115 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(Int64Ty, 2));
12116 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12120 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 2));
12121 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12125 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 1));
12126 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12131 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(FloatTy, 4));
12132 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12137 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(DoubleTy, 2));
12138 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12142 return Builder.CreateFAdd(Ops[0], Ops[1], "vaddh");
12145 return Builder.CreateFSub(Ops[0], Ops[1], "vsubh");
12148 return Builder.CreateFMul(Ops[0], Ops[1], "vmulh");
12151 return Builder.CreateFDiv(Ops[0], Ops[1], "vdivh");
12158 Value* Neg = Builder.CreateFNeg(EmitScalarExpr(E->getArg(1)), "vsubh");
12167 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd");
12170 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd");
12180 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
12189 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
12199 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
12217 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
12218 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2))));
12219 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty),
12220 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
12221 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
12226 return Builder.CreateShl(
12231 return Builder.CreateAShr(
12242 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt),
12247 Ops[1] = Builder.CreateAShr(
12251 return Builder.CreateAdd(Ops[0], Ops[1]);
12260 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
12262 return Builder.CreateAdd(Ops[0], Ops[1]);
12268 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
12277 Ops[1] = Builder.CreateExtractElement(Ops[1], CI, "lane0");
12304 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
12323 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12329 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12338 Builder.CreateAtomicRMW(AtomicRMWInst::Add, DestAddr, Val,
12340 return Builder.CreateAdd(RMWI, Val);
12369 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy, "vbsl");
12370 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy, "vbsl");
12371 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy, "vbsl");
12373 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1], "vbsl");
12374 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2], "vbsl");
12375 Ops[0] = Builder.CreateOr(Ops[1], Ops[2], "vbsl");
12376 return Builder.CreateBitCast(Ops[0], Ty);
12396 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
12397 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV, "lane");
12400 Int = Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
12408 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
12409 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
12412 Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
12413 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
12418 return Builder.CreateBitCast(Result, Ty);
12420 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12421 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
12425 Ops[2] = Builder.CreateBitCast(Ops[2], STy);
12428 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV, "lane");
12435 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12436 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
12438 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
12452 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3], "extract");
12503 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
12504 return Builder.CreateAdd(tmp, addend);
12564 Int = Builder.getIsFPConstrained()
12571 Int = Builder.getIsFPConstrained()
12578 Int = Builder.getIsFPConstrained()
12585 Int = Builder.getIsFPConstrained()
12592 Int = Builder.getIsFPConstrained()
12599 Int = Builder.getIsFPConstrained()
12606 Int = Builder.getIsFPConstrained()
12613 Int = Builder.getIsFPConstrained()
12620 Int = Builder.getIsFPConstrained()
12627 Int = Builder.getIsFPConstrained()
12634 Int = Builder.getIsFPConstrained()
12641 Int = Builder.getIsFPConstrained()
12648 Int = Builder.getIsFPConstrained()
12687 Int = Builder.getIsFPConstrained()
12694 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12696 return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
12697 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
12702 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
12704 return Builder.CreateFPExt(Ops[0], Ty, "vcvt");
12710 Ops[0] = Builder.CreateBitCast(Ops[0], GetNeonType(this, SrcFlag));
12712 return Builder.CreateFPTrunc(Ops[0], Ty, "vcvt");
12805 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
12816 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
12819 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
12820 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2], "extract");
12821 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
12822 return Builder.CreateBitCast(Result, Ty);
12825 return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd");
12827 return Builder.CreateFNeg(EmitScalarExpr(E->getArg(0)), "vnegh");
12840 Int = Builder.getIsFPConstrained()
12847 Int = Builder.getIsFPConstrained()
12850 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12869 return Builder.CreateTrunc(Ops[0], Int8Ty);
12881 return Builder.CreateTrunc(Ops[0], Int16Ty);
12893 return Builder.CreateTrunc(Ops[0], Int8Ty);
12905 return Builder.CreateTrunc(Ops[0], Int16Ty);
12914 return Builder.CreateTrunc(Ops[0], Int8Ty);
12923 return Builder.CreateTrunc(Ops[0], Int16Ty);
12932 return Builder.CreateTrunc(Ops[0], Int8Ty);
12941 return Builder.CreateTrunc(Ops[0], Int16Ty);
12950 return Builder.CreateTrunc(Ops[0], Int8Ty);
12959 return Builder.CreateTrunc(Ops[0], Int16Ty);
12968 return Builder.CreateTrunc(Ops[0], Int8Ty);
12977 return Builder.CreateTrunc(Ops[0], Int16Ty);
12986 return Builder.CreateTrunc(Ops[0], HalfTy);
12995 return Builder.CreateTrunc(Ops[0], HalfTy);
13004 return Builder.CreateTrunc(Ops[0], Int8Ty);
13013 return Builder.CreateTrunc(Ops[0], Int16Ty);
13022 return Builder.CreateTrunc(Ops[0], Int8Ty);
13031 return Builder.CreateTrunc(Ops[0], Int16Ty);
13040 return Builder.CreateTrunc(Ops[0], Int8Ty);
13049 return Builder.CreateTrunc(Ops[0], Int16Ty);
13058 return Builder.CreateTrunc(Ops[0], Int8Ty);
13067 return Builder.CreateTrunc(Ops[0], Int16Ty);
13076 return Builder.CreateTrunc(Ops[0], HalfTy);
13085 return Builder.CreateTrunc(Ops[0], HalfTy);
13094 return Builder.CreateTrunc(Ops[0], HalfTy);
13103 return Builder.CreateTrunc(Ops[0], HalfTy);
13112 return Builder.CreateTrunc(Ops[0], HalfTy);
13121 return Builder.CreateTrunc(Ops[0], HalfTy);
13124 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
13125 Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy);
13126 return Builder.CreateFMul(Ops[0], RHS);
13135 return Builder.CreateTrunc(Ops[0], Int16Ty);
13152 return Builder.CreateTrunc(Ops[0], Int16Ty);
13169 return Builder.CreateTrunc(Ops[0], Int16Ty);
13186 return Builder.CreateTrunc(Ops[0], Int16Ty);
13210 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
13212 return Builder.CreateAdd(Ops[0], Ops[1]);
13221 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
13222 return Builder.CreateAdd(Ops[0], tmp);
13226 return Builder.CreateAlignedLoad(VTy, Ops[0], PtrOp0.getAlignment());
13230 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
13231 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment());
13234 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13235 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
13237 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vld1_lane");
13241 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13242 llvm::LoadInst *LI = Builder.CreateAlignedLoad(
13246 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2], "vldap1_lane");
13251 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
13254 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
13259 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13260 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
13261 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment());
13264 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13265 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
13267 Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.getAlignment());
13275 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
13276 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13282 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
13283 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13289 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
13290 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13296 Ops[1] = Builder.CreateCall(F, Ops[1], "vld2");
13297 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13303 Ops[1] = Builder.CreateCall(F, Ops[1], "vld3");
13304 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13310 Ops[1] = Builder.CreateCall(F, Ops[1], "vld4");
13311 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13318 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13319 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
13320 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
13321 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld2_lane");
13322 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13329 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13330 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
13331 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
13332 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
13333 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld3_lane");
13334 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13341 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13342 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
13343 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
13344 Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
13345 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty);
13346 Ops[1] = Builder.CreateCall(F, ArrayRef(Ops).slice(1), "vld4_lane");
13347 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
13359 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
13374 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
13389 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
13396 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13397 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
13406 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13407 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vtrn");
13408 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
13414 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13415 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
13423 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13424 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vuzp");
13425 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
13431 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
13432 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
13441 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
13442 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices, "vzip");
13443 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
13533 return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
13557 CallInst *Fn = Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
13584 Value *EnumStrVal = Builder.CreateGlobalStringPtr(EnumStr);
13593 Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
13621 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt64(i));
13631 CGF.Builder.getInt1Ty(),
13633 Value *MaskVec = CGF.Builder.CreateBitCast(Mask, MaskTy);
13641 MaskVec = CGF.Builder.CreateShuffleVector(
13655 return CGF.Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec);
13666 return CGF.Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]);
13679 return CGF.Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
13692 return CGF.Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec });
13704 return CGF.Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
13715 LHS = CGF.Builder.CreateNot(LHS);
13717 return CGF.Builder.CreateBitCast(CGF.Builder.CreateBinOp(Opc, LHS, RHS),
13730 Amt = CGF.Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
13731 Amt = CGF.Builder.CreateVectorSplat(NumElts, Amt);
13736 return CGF.Builder.CreateCall(F, {Op0, Op1, Amt});
13774 Value *Cmp = CGF.Builder.CreateICmp(Pred, Op0, Op1);
13775 Value *Res = CGF.Builder.CreateSExt(Cmp, Ty);
13790 return CGF.Builder.CreateSelect(Mask, Op0, Op1);
13801 CGF.Builder.getInt1Ty(), Mask->getType()->getIntegerBitWidth());
13802 Mask = CGF.Builder.CreateBitCast(Mask, MaskTy);
13803 Mask = CGF.Builder.CreateExtractElement(Mask, (uint64_t)0);
13804 return CGF.Builder.CreateSelect(Mask, Op0, Op1);
13812 Cmp = CGF.Builder.CreateAnd(Cmp, getMaskVecValue(CGF, MaskIn, NumElts));
13821 Cmp = CGF.Builder.CreateShuffleVector(
13825 return CGF.Builder.CreateBitCast(Cmp,
13840 llvm::FixedVectorType::get(CGF.Builder.getInt1Ty(), NumElts));
13843 llvm::FixedVectorType::get(CGF.Builder.getInt1Ty(), NumElts));
13855 Cmp = CGF.Builder.CreateICmp(Pred, Ops[0], Ops[1]);
13880 Res = CGF.Builder.CreateCall(F, { Ops[0], Ops[3] });
13883 Res = IsSigned ? CGF.Builder.CreateSIToFP(Ops[0], Ty)
13884 : CGF.Builder.CreateUIToFP(Ops[0], Ty);
13952 C = CGF.Builder.CreateFNeg(C);
13961 Res = CGF.Builder.CreateCall(Intr, {A, B, C, Ops.back() });
13965 if (CGF.Builder.getIsFPConstrained()) {
13968 Res = CGF.Builder.CreateConstrainedFPCall(FMA, {A, B, C});
13971 Res = CGF.Builder.CreateCall(FMA, {A, B, C});
14025 Ops[2] = CGF.Builder.CreateFNeg(Ops[2]);
14027 Ops[0] = CGF.Builder.CreateExtractElement(Ops[0], (uint64_t)0);
14028 Ops[1] = CGF.Builder.CreateExtractElement(Ops[1], (uint64_t)0);
14029 Ops[2] = CGF.Builder.CreateExtractElement(Ops[2], (uint64_t)0);
14047 Res = CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(IID),
14049 } else if (CGF.Builder.getIsFPConstrained()) {
14053 Res = CGF.Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3));
14056 Res = CGF.Builder.CreateCall(FMA, Ops.slice(0, 3));
14067 PassThru = CGF.Builder.CreateExtractElement(Upper, (uint64_t)0);
14071 return CGF.Builder.CreateInsertElement(Upper, Res, (uint64_t)0);
14080 Value *LHS = CGF.Builder.CreateBitCast(Ops[0], Ty);
14081 Value *RHS = CGF.Builder.CreateBitCast(Ops[1], Ty);
14086 LHS = CGF.Builder.CreateShl(LHS, ShiftAmt);
14087 LHS = CGF.Builder.CreateAShr(LHS, ShiftAmt);
14088 RHS = CGF.Builder.CreateShl(RHS, ShiftAmt);
14089 RHS = CGF.Builder.CreateAShr(RHS, ShiftAmt);
14093 LHS = CGF.Builder.CreateAnd(LHS, Mask);
14094 RHS = CGF.Builder.CreateAnd(RHS, Mask);
14097 return CGF.Builder.CreateMul(LHS, RHS);
14125 Value *Ternlog = CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(IID),
14136 return CGF.Builder.CreateSExt(Mask, DstTy, "vpmovm2");
14156 return CGF.Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]});
14166 Src = CGF.Builder.CreateShuffleVector(Src, ArrayRef<int>{0, 1, 2, 3});
14172 Src = CGF.Builder.CreateBitCast(Src, HalfTy);
14175 Value *Res = CGF.Builder.CreateFPExt(Src, DstTy, "cvtph2ps");
14184 llvm::Type *Int32Ty = Builder.getInt32Ty();
14221 llvm::Value *CpuValue = Builder.CreateInBoundsGEP(STy, CpuModel, Idxs);
14222 CpuValue = Builder.CreateAlignedLoad(Int32Ty, CpuValue,
14226 return Builder.CreateICmpEQ(CpuValue,
14234 return Builder.getFalse();
14244 Value *Result = Builder.getTrue();
14261 Value *Idxs[] = {Builder.getInt32(0), Builder.getInt32(3),
14262 Builder.getInt32(0)};
14263 Value *CpuFeatures = Builder.CreateInBoundsGEP(STy, CpuModel, Idxs);
14264 Value *Features = Builder.CreateAlignedLoad(Int32Ty, CpuFeatures,
14268 Value *Mask = Builder.getInt32(FeatureMask[0]);
14269 Value *Bitset = Builder.CreateAnd(Features, Mask);
14270 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
14271 Result = Builder.CreateAnd(Result, Cmp);
14282 Value *Idxs[] = {Builder.getInt32(0), Builder.getInt32(i - 1)};
14283 Value *Features = Builder.CreateAlignedLoad(
14284 Int32Ty, Builder.CreateInBoundsGEP(ATy, CpuFeatures2, Idxs),
14287 Value *Mask = Builder.getInt32(M);
14288 Value *Bitset = Builder.CreateAnd(Features, Mask);
14289 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
14290 Result = Builder.CreateAnd(Result, Cmp);
14303 return Builder.CreateCall(Func);
14314 return Builder.CreateCall(Func);
14325 return Builder.getFalse();
14335 Value *Result = Builder.getTrue();
14345 llvm::Value *CpuFeatures = Builder.CreateGEP(
14348 Value *Features = Builder.CreateAlignedLoad(Int64Ty, CpuFeatures,
14350 Value *Mask = Builder.getInt64(FeaturesMask);
14351 Value *Bitset = Builder.CreateAnd(Features, Mask);
14352 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
14353 Result = Builder.CreateAnd(Result, Cmp);
14395 return Builder.CreateCall(F, Ops);
14408 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
14410 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
14413 Value *Sext = Builder.CreateSExt(Cmp, IntVecTy);
14414 return Builder.CreateBitCast(Sext, FPVecTy);
14426 return Builder.CreateCall(F, {Address, RW, Locality, Data});
14429 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_clflush),
14433 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_lfence));
14436 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_mfence));
14439 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_sfence));
14442 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_pause));
14445 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtsc));
14448 Value *Call = Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtscp));
14449 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 1),
14451 return Builder.CreateExtractValue(Call, 0);
14457 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)});
14463 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)});
14477 return Builder.CreateBitCast(BuildVector(Ops),
14495 return Builder.CreateExtractElement(Ops[0], Index);
14511 return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
14516 Builder.CreateStore(Ops[0], Tmp);
14517 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
14523 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
14525 return Builder.CreateLoad(Tmp, "stmxcsr");
14566 Value *Mhi = Builder.CreateTrunc(
14567 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty);
14568 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty);
14571 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
14575 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_xgetbv), Ops);
14615 return Builder.CreateCall(F, Ops);
14959 return Builder.CreateCall(Intr, Ops);
15068 return Builder.CreateCall(Intr, Ops);
15101 Value *Res = Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts),
15139 Value *Op1 = Builder.CreateShuffleVector(
15149 return Builder.CreateShuffleVector(Ops[0], Op1,
15154 Value *Res = Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15162 return Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15179 return Builder.CreateCall(Intr, Ops);
15199 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15222 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts),
15245 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts),
15274 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts),
15303 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15320 return Builder.CreateShuffleVector(Ops[0], ArrayRef(Indices, NumElts),
15356 return Builder.CreateShuffleVector(Ops[1], Ops[0],
15376 return Builder.CreateShuffleVector(Ops[1], Ops[0],
15404 return Builder.CreateShuffleVector(Ops[0], Ops[1],
15443 return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
15470 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast");
15472 Value *SV = Builder.CreateShuffleVector(
15474 return Builder.CreateBitCast(SV, Ops[0]->getType(), "cast");
15499 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy, "cast");
15501 Value *SV = Builder.CreateShuffleVector(
15503 return Builder.CreateBitCast(SV, ResultType, "cast");
15522 Value *SV = Builder.CreateShuffleVector(
15524 return Builder.CreateBitCast(SV, Ops[0]->getType());
15543 Value *SV = Builder.CreateShuffleVector(
15545 return Builder.CreateBitCast(SV, Ops[0]->getType());
15552 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
15560 Src = Builder.CreateExtractElement(Src, (uint64_t)0, "extract");
15563 StoreInst *SI = Builder.CreateDefaultAlignedStore(Src, Ptr);
15632 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
15633 Value *B = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
15635 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0);
15684 Value *Cmp = Builder.CreateICmpEQ(Or, C);
15685 return Builder.CreateZExt(Cmp, ConvertType(E->getType()));
15693 Value *Cmp = Builder.CreateICmpEQ(Or, C);
15694 return Builder.CreateZExt(Cmp, ConvertType(E->getType()));
15738 return Builder.CreateCall(Intr, {LHS, RHS});
15766 Value *Res = Builder.CreateCall(Intr, {LHS, RHS});
15767 return Builder.CreateBitCast(Res, Ops[0]->getType());
15800 return Builder.CreateBitCast(Builder.CreateNot(Res),
15812 return Builder.CreateBitCast(Res, Ops[0]->getType());
15827 LHS = Builder.CreateShuffleVector(LHS, LHS, ArrayRef(Indices, NumElts / 2));
15828 RHS = Builder.CreateShuffleVector(RHS, RHS, ArrayRef(Indices, NumElts / 2));
15832 Builder.CreateShuffleVector(RHS, LHS, ArrayRef(Indices, NumElts));
15833 return Builder.CreateBitCast(Res, Ops[0]->getType());
15843 return Builder.CreateCall(F, {Ops[0],Builder.getInt1(false)});
15847 Value *A = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
15849 if (Builder.getIsFPConstrained()) {
15853 A = Builder.CreateConstrainedFPCall(F, {A});
15856 A = Builder.CreateCall(F, {A});
15858 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
15882 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
15884 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
15886 if (Builder.getIsFPConstrained()) {
15890 A = Builder.CreateConstrainedFPCall(F, A);
15893 A = Builder.CreateCall(F, A);
15895 Value *Src = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
15897 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
15928 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
15931 if (Builder.getIsFPConstrained()) {
15935 return Builder.CreateConstrainedFPCall(F, Ops[0]);
15938 return Builder.CreateCall(F, Ops[0]);
16022 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
16023 Builder.getFastMathFlags().setAllowReassoc();
16024 return Builder.CreateCall(F, {Ops[0], Ops[1]});
16033 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
16034 Builder.getFastMathFlags().setAllowReassoc();
16035 return Builder.CreateCall(F, {Ops[0], Ops[1]});
16044 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
16045 Builder.getFastMathFlags().setNoNaNs();
16046 return Builder.CreateCall(F, {Ops[0]});
16055 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
16056 Builder.getFastMathFlags().setNoNaNs();
16057 return Builder.CreateCall(F, {Ops[0]});
16089 Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID));
16090 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0),
16092 return Builder.CreateExtractValue(Call, 1);
16115 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID),
16117 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 1),
16119 return Builder.CreateExtractValue(Call, 0);
16168 Value *Fpclass = Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
16204 Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID), {Ops[0], Ops[1]});
16205 Value *Result = Builder.CreateExtractValue(Call, 0);
16207 Builder.CreateDefaultAlignedStore(Result, Ops[2]);
16209 Result = Builder.CreateExtractValue(Call, 1);
16211 return Builder.CreateDefaultAlignedStore(Result, Ops[3]);
16231 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
16256 Value *Shufbit = Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
16345 if (Builder.getIsFPConstrained() &&
16398 Value *Cmp = Builder.CreateCall(Intr, Ops);
16402 return Builder.CreateCall(Intr, Ops);
16416 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
16418 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
16475 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
16478 return Builder.CreateFPExt(Ops[0], Builder.getFloatTy());
16492 Value *Res = Builder.CreateCall(CGM.getIntrinsic(IID), Ops[0]);
16522 Value *IACall = Builder.CreateCall(IA, {FuncId, SubFuncId});
16526 Value *Extracted = Builder.CreateExtractValue(IACall, i);
16527 Value *StorePtr = Builder.CreateConstInBoundsGEP1_32(Int32Ty, BasePtr, i);
16528 Store = Builder.CreateAlignedStore(Extracted, StorePtr, getIntAlign());
16540 Value *LHS = Builder.CreateIntCast(Ops[0], Int64Ty, isSigned);
16541 Value *RHS = Builder.CreateIntCast(Ops[1], Int64Ty, isSigned);
16542 return Builder.CreateMul(LHS, RHS, "", !isSigned, isSigned);
16552 Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
16553 Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
16557 MulResult = Builder.CreateNSWMul(LHS, RHS);
16558 HigherBits = Builder.CreateAShr(MulResult, 64);
16560 MulResult = Builder.CreateNUWMul(LHS, RHS);
16561 HigherBits = Builder.CreateLShr(MulResult, 64);
16563 HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
16569 Builder.CreateStore(HigherBits, HighBitsAddress);
16570 return Builder.CreateIntCast(MulResult, ResType, IsSigned);
16574 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
16586 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
16587 return Builder.CreateCall(F, Ops);
16592 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
16599 return Builder.CreateCall(F);
16604 return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], Align(1), true);
16617 llvm::CallInst *CI = Builder.CreateCall(IA);
16626 Value *Ptr = Builder.CreateIntToPtr(
16628 LoadInst *Load = Builder.CreateAlignedLoad(
16638 Value *Ptr = Builder.CreateIntToPtr(
16640 LoadInst *Load = Builder.CreateAlignedLoad(
16648 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[0], Ops[1]});
16651 Value *Extract = Builder.CreateExtractValue(Call, i + 1);
16652 Value *Ptr = Builder.CreateConstGEP1_32(Int8Ty, Ops[2], i * 16);
16653 Builder.CreateAlignedStore(Extract, Ptr, Align(1));
16656 return Builder.CreateExtractValue(Call, 0);
16662 Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[0], Ops[1], Ops[2]});
16665 Value *Extract = Builder.CreateExtractValue(Call, i + 1);
16666 Value *Ptr = Builder.CreateConstGEP1_32(Int8Ty, Ops[3], i * 16);
16667 Builder.CreateAlignedStore(Extract, Ptr, Align(1));
16670 return Builder.CreateExtractValue(Call, 0);
16699 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), {Ops[1], Ops[2]});
16706 Value *Ret = Builder.CreateExtractValue(Call, 0);
16707 Value *Succ = Builder.CreateTrunc(Ret, Builder.getInt1Ty());
16708 Value *Out = Builder.CreateExtractValue(Call, 1);
16709 Builder.CreateCondBr(Succ, NoError, Error);
16711 Builder.SetInsertPoint(NoError);
16712 Builder.CreateDefaultAlignedStore(Out, Ops[0]);
16713 Builder.CreateBr(End);
16715 Builder.SetInsertPoint(Error);
16717 Builder.CreateDefaultAlignedStore(Zero, Ops[0]);
16718 Builder.CreateBr(End);
16720 Builder.SetInsertPoint(End);
16721 return Builder.CreateExtractValue(Call, 0);
16748 llvm::Type *Ty = FixedVectorType::get(Builder.getInt64Ty(), 2);
16752 Value *Ptr = Builder.CreateConstGEP1_32(Ty, Ops[1], i);
16753 InOps[i + 1] = Builder.CreateAlignedLoad(Ty, Ptr, Align(16));
16756 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), InOps);
16763 Value *Ret = Builder.CreateExtractValue(Call, 0);
16764 Value *Succ = Builder.CreateTrunc(Ret, Builder.getInt1Ty());
16765 Builder.CreateCondBr(Succ, NoError, Error);
16767 Builder.SetInsertPoint(NoError);
16769 Value *Extract = Builder.CreateExtractValue(Call, i + 1);
16770 Value *Ptr = Builder.CreateConstGEP1_32(Extract->getType(), Ops[0], i);
16771 Builder.CreateAlignedStore(Extract, Ptr, Align(16));
16773 Builder.CreateBr(End);
16775 Builder.SetInsertPoint(Error);
16777 Value *Out = Builder.CreateExtractValue(Call, i + 1);
16779 Value *Ptr = Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i);
16780 Builder.CreateAlignedStore(Zero, Ptr, Align(16));
16782 Builder.CreateBr(End);
16784 Builder.SetInsertPoint(End);
16785 return Builder.CreateExtractValue(Call, 0);
16794 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
16803 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
16804 Value *And = Builder.CreateAnd(Ops[3], llvm::ConstantInt::get(Int8Ty, 1));
16813 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
16815 return Builder.CreateShuffleVector(Call, Ops[2], Mask);
16818 return Builder.CreateCall(
16832 // return Builder.CreateFDiv(EmitScalarExpr(E->getArg(0)),
16837 // return Builder.CreateFDiv(Op0, Op1, "swdiv")
16863 FieldValue = Builder.CreateInBoundsGEP(STy, SysConf, Idxs);
16864 FieldValue = Builder.CreateAlignedLoad(Int32Ty, FieldValue,
16873 Builder.CreateCall(Func, {ConstantInt::get(Int32Ty, FieldIdx)});
16879 FieldValue = Builder.CreateAnd(FieldValue, Mask);
16887 return Builder.CreateICmp(
16930 Value *TheCall = Builder.CreateCall(F, {Op0}, "cpu_is");
16931 return Builder.CreateICmpEQ(TheCall,
16967 return Builder.getFalse();
16970 Value *TheCall = Builder.CreateCall(F, {Op0}, "cpu_supports");
16972 Builder.CreateAnd(TheCall, llvm::ConstantInt::get(Int32Ty, BitMask));
16973 return Builder.CreateICmpNE(Mask, llvm::Constant::getNullValue(Int32Ty));
16982 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter));
17004 Ops[0] = Builder.CreateGEP(Int8Ty, Ops[1], Ops[0]);
17051 return Builder.CreateCall(F, Ops, "");
17073 Ops[1] = Builder.CreateGEP(Int8Ty, Ops[2], Ops[1]);
17114 return Builder.CreateCall(F, Ops, "");
17129 Builder.CreateLoad(Address(Op0, ResTy, CharUnits::fromQuantity(1)));
17137 return Builder.CreateShuffleVector(LD, LD, RevMask);
17144 Value *HiMem = Builder.CreateGEP(
17146 Value *LoLd = Builder.CreateCall(Lvx, Op0, "ld.lo");
17147 Value *HiLd = Builder.CreateCall(Lvx, HiMem, "ld.hi");
17148 Value *Mask1 = Builder.CreateCall(Lvs, Op0, "mask1");
17152 Value *AllElts = Builder.CreateCall(Vperm, {Op0, Op1, Mask1}, "shuffle1");
17162 return Builder.CreateShuffleVector(Builder.CreateBitCast(AllElts, ResTy),
17167 Consts.push_back(Builder.getInt8(NumBytes + Idx));
17169 return Builder.CreateBitCast(
17170 Builder.CreateCall(Vperm, {Zero, AllElts, Mask2}, "shuffle2"), ResTy);
17187 StVec = Builder.CreateShuffleVector(Op2, Op2, RevMask);
17189 return Builder.CreateStore(
17214 Value *Vec = Builder.CreateBitCast(
17217 Builder.CreateGEP(Int8Ty, Op0, ConstantInt::get(Int64Ty, Offset));
17218 Value *Elt = Builder.CreateExtractElement(Vec, EltNo);
17221 Elt = Builder.CreateCall(F, Elt);
17223 return Builder.CreateStore(
17258 if (Builder.getIsFPConstrained()) {
17261 return Builder.CreateConstrainedFPCall(F, X);
17264 return Builder.CreateCall(F, X);
17274 Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
17276 return Builder.CreateCall(F, {X, Undef});
17284 Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
17286 return Builder.CreateCall(F, {X, Undef});
17342 ? Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int32Ty, 4))
17343 : Builder.CreateBitCast(Op0,
17345 return Builder.CreateBitCast(
17346 Builder.CreateCall(CGM.getIntrinsic(ID), {Op0, Op1, Op2}), ResultType);
17355 return Builder.CreateCall(F, X);
17362 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int128Ty, 1));
17363 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int128Ty, 1));
17365 return Builder.CreateAdd(Op0, Op1, "vadduqm");
17367 return Builder.CreateSub(Op0, Op1, "vsubuqm");
17376 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty));
17377 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty));
17381 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops, "");
17393 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty));
17394 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty));
17395 Ops.push_back(Builder.CreateBitCast(Op2, V1I128Ty));
17412 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops, "");
17425 Op2 = Builder.CreateZExt(Op2, Int64Ty);
17426 Value *Shift = Builder.CreateCall(F, {Op0, Op0, Op2});
17427 return Builder.CreateOr(Builder.CreateAnd(Shift, Op3),
17428 Builder.CreateAnd(Op1, Builder.CreateNot(Op3)));
17430 return Builder.CreateCall(
17440 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_rlwnm),
17448 Value *Tmp = Builder.CreateCall(F, Op0);
17451 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
17453 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
17463 return Builder.CreateCall(F, {Op0, Op1}, "cmpb");
17481 Value *ArgOneLo = Builder.CreateTrunc(Op0, Int32Ty);
17482 Value *ArgTwoLo = Builder.CreateTrunc(Op1, Int32Ty);
17485 Builder.CreateTrunc(Builder.CreateLShr(Op0, ShiftAmt), Int32Ty);
17487 Builder.CreateTrunc(Builder.CreateLShr(Op1, ShiftAmt), Int32Ty);
17488 Value *ResLo = Builder.CreateZExt(
17489 Builder.CreateCall(F, {ArgOneLo, ArgTwoLo}, "cmpb"), Int64Ty);
17490 Value *ResHiShift = Builder.CreateZExt(
17491 Builder.CreateCall(F, {ArgOneHi, ArgTwoHi}, "cmpb"), Int64Ty);
17492 Value *ResHi = Builder.CreateShl(ResHiShift, ShiftAmt);
17493 return Builder.CreateOr(ResLo, ResHi);
17503 return Builder.CreateCall(F, {X, Y});
17520 ID = Builder.getIsFPConstrained()
17525 ID = Builder.getIsFPConstrained()
17530 ID = Builder.getIsFPConstrained()
17535 ID = Builder.getIsFPConstrained()
17540 ID = Builder.getIsFPConstrained()
17544 return Builder.getIsFPConstrained() ? Builder.CreateConstrainedFPCall(F, X)
17545 : Builder.CreateCall(F, X);
17554 return Builder.CreateCall(F, X);
17562 FastMathFlags FMF = Builder.getFastMathFlags();
17563 Builder.getFastMathFlags().setFast();
17570 Value *FDiv = Builder.CreateFDiv(X, Y, "recipdiv");
17571 Builder.getFastMathFlags() &= (FMF);
17576 Value *FDiv = Builder.CreateFDiv(One, Builder.CreateCall(F, X), "rsqrt");
17577 Builder.getFastMathFlags() &= (FMF);
17598 Value *ShiftAmt = Builder.CreateIntCast(Op1, Ty, false);
17600 Value *Rotate = Builder.CreateCall(F, {Op0, Op0, ShiftAmt});
17601 return Builder.CreateAnd(Rotate, Op2);
17606 Value *LoadIntrinsic = Builder.CreateCall(F, {Op0});
17607 return Builder.CreateTrunc(LoadIntrinsic, Int16Ty);
17625 if (Builder.getIsFPConstrained())
17632 if (Builder.getIsFPConstrained())
17633 return Builder.CreateConstrainedFPCall(F, {X, Y, Z});
17635 return Builder.CreateCall(F, {X, Y, Z});
17638 if (Builder.getIsFPConstrained())
17639 return Builder.CreateFNeg(
17640 Builder.CreateConstrainedFPCall(F, {X, Y, Z}), "neg");
17642 return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, Z}), "neg");
17645 if (Builder.getIsFPConstrained())
17646 return Builder.CreateConstrainedFPCall(
17647 F, {X, Y, Builder.CreateFNeg(Z, "neg")});
17649 return Builder.CreateCall(F, {X, Y, Builder.CreateFNeg(Z, "neg")});
17654 if (Builder.getIsFPConstrained())
17655 return Builder.CreateFNeg(
17656 Builder.CreateConstrainedFPCall(
17657 F, {X, Y, Builder.CreateFNeg(Z, "neg")}),
17660 return Builder.CreateCall(
17690 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int64Ty, 2));
17694 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int64Ty, 2));
17695 Op0 = Builder.CreateShuffleVector(Op0, Op0, ArrayRef<int>{1, 0});
17702 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int32Ty, 4));
17704 return Builder.CreateCall(F, {Op0, Op1, Op2});
17713 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int64Ty, 2));
17729 Value *Call = Builder.CreateCall(F, {Op0, Op1});
17732 Builder.CreateShuffleVector(Call, Call, ArrayRef<int>{1, 0});
17736 return Builder.CreateCall(F, {Op0, Op1});
17748 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int64Ty, 2));
17749 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int64Ty, 2));
17758 Value *ShuffleCall = Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
17761 return Builder.CreateBitCast(ShuffleCall, RetTy);
17771 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int32Ty, 4));
17772 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int32Ty, 4));
17796 Value *ShuffleCall = Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
17799 return Builder.CreateBitCast(ShuffleCall, RetTy);
17808 Value *Res = Builder.CreateInsertElement(
17810 Res = Builder.CreateInsertElement(Res, Op1,
17812 return Builder.CreateBitCast(Res, ConvertType(E->getType()));
17819 Value *Unpacked = Builder.CreateBitCast(
17826 return Builder.CreateExtractElement(Unpacked, Index);
17832 Value *Op1 = Builder.CreateSExt(EmitScalarExpr(E->getArg(1)), Int32Ty);
17833 return Builder.CreateCall(F, {Op0, Op1});
17868 Value *Vec = Builder.CreateLoad(Addr);
17869 Value *Call = Builder.CreateCall(F, {Vec});
17873 Value *Vec = Builder.CreateExtractValue(Call, i);
17875 Value *GEP = Builder.CreateInBoundsGEP(VTy, Ptr, Index);
17876 Builder.CreateAlignedStore(Vec, GEP, MaybeAlign(16));
17906 Ops[0] = Builder.CreateGEP(Int8Ty, Ops[1], Ops[0]);
17908 Ops[1] = Builder.CreateGEP(Int8Ty, Ops[2], Ops[1]);
17912 return Builder.CreateCall(F, Ops, "");
17917 Value *Acc = Builder.CreateLoad(Addr);
17923 Value *Call = Builder.CreateCall(F, CallOps);
17924 return Builder.CreateAlignedStore(Call, Ops[0], MaybeAlign(64));
17931 Value *OldVal = Builder.CreateLoad(OldValAddr);
17946 Builder.CreateStore(LoadedVal, OldValAddr);
17947 return Builder.CreateZExt(Pair.second, Builder.getInt32Ty());
17981 return Builder.CreateCall(F, {Op0});
17990 return Builder.CreateCall(F, {Op0, Op1});
17996 return Builder.CreateCall(F, {ArgValue}, "popcntb");
18003 Value *Cast = Builder.CreateUIToFP(Op1, DoubleTy);
18005 return Builder.CreateCall(F, {Op0, Cast}, "");
18012 FastMathFlags FMF = Builder.getFastMathFlags();
18013 Builder.getFastMathFlags().setFast();
18014 Value *FDiv = Builder.CreateFDiv(Op0, Op1, "swdiv_nochk");
18015 Builder.getFastMathFlags() &= (FMF);
18056 return Builder.CreateCall(
18065 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_maxfe),
18073 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_maxfl),
18081 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_maxfs),
18089 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_minfe),
18097 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_minfl),
18105 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_minfs),
18112 return Builder.CreateFDiv(Op0, Op1, "swdiv");
18115 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_setrnd),
18118 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_readflm));
18128 auto *Call = CGF.Builder.CreateCall(F);
18138 return CGF.Builder.CreateAddrSpaceCast(Call, RetTy);
18143 auto *Call = CGF.Builder.CreateCall(F);
18178 Value *ABIVersion = CGF.Builder.CreateAlignedLoad(CGF.Int32Ty, ABIVersionC,
18181 Value *IsCOV5 = CGF.Builder.CreateICmpSGE(
18186 Value *ImplicitGEP = CGF.Builder.CreateConstGEP1_32(
18190 Value *DispatchGEP = CGF.Builder.CreateConstGEP1_32(
18193 auto Result = CGF.Builder.CreateSelect(IsCOV5, ImplicitGEP, DispatchGEP);
18194 LD = CGF.Builder.CreateLoad(
18200 GEP = CGF.Builder.CreateConstGEP1_32(
18204 GEP = CGF.Builder.CreateConstGEP1_32(
18207 LD = CGF.Builder.CreateLoad(
18228 auto *GEP = CGF.Builder.CreateGEP(CGF.Int8Ty, DP, Offset);
18229 auto *LD = CGF.Builder.CreateLoad(
18343 return Builder.CreateIntrinsic(
18350 return Builder.CreateIntrinsic(
18365 return Builder.CreateIntrinsic(
18377 return Builder.CreateFMul(Op0, Op1, "dx.dot");
18380 return Builder.CreateMul(Op0, Op1, "dx.dot");
18401 return Builder.CreateIntrinsic(
18413 return Builder.CreateIntrinsic(
18421 return Builder.CreateIntrinsic(
18436 return Builder.CreateIntrinsic(retType, Intrinsic::dx_isinf,
18444 return Builder.CreateIntrinsic(
18450 return Builder.CreateIntrinsic(
18454 Value *Mul = Builder.CreateNSWMul(M, A);
18455 return Builder.CreateNSWAdd(Mul, B);
18459 return Builder.CreateIntrinsic(
18463 Value *Mul = Builder.CreateNUWMul(M, A);
18464 return Builder.CreateNUWAdd(Mul, B);
18478 return Builder.CreateFDiv(One, Op0, "hlsl.rcp");
18484 return Builder.CreateIntrinsic(
18539 llvm::Value *Tmp = Builder.CreateCall(Callee, {X, Y, Z});
18541 llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0);
18542 llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1);
18546 llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType);
18547 Builder.CreateStore(FlagExt, FlagOutPtr);
18559 llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3);
18560 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
18586 return Builder.CreateCall(F, Args);
18651 return Builder.CreateCall(F, {Src0, Src1});
18660 return Builder.CreateCall(F, {Src0, Builder.CreateTrunc(Src1, Int16Ty)});
18671 { Builder.getInt32Ty(), Src0->getType() });
18672 return Builder.CreateCall(F, Src0);
18677 { Builder.getInt16Ty(), Src0->getType() });
18678 return Builder.CreateCall(F, Src0);
18699 return Builder.CreateCall(F, { Src });
18711 { Builder.getInt64Ty(), Src0->getType() });
18712 return Builder.CreateCall(F, { Src0, Src1, Src2 });
18722 { Builder.getInt64Ty(), Src0->getType() });
18723 return Builder.CreateCall(F, { Src0, Src1, Src2 });
18739 return Builder.CreateCall(F, { Src0, Builder.getFalse() });
18795 return Builder.CreateCall(F, {Addr, Val});
18811 return Builder.CreateCall(F, {Addr, Val});
18840 return Builder.CreateCall(F, {Addr});
18845 return Builder.CreateCall(F);
18851 return Builder.CreateCall(F, {Env});
18872 RayOrigin = Builder.CreateShuffleVector(RayOrigin, RayOrigin,
18875 Builder.CreateShuffleVector(RayDir, RayDir, ArrayRef<int>{0, 1, 2});
18876 RayInverseDir = Builder.CreateShuffleVector(RayInverseDir, RayInverseDir,
18881 return Builder.CreateCall(F, {NodePtr, RayExtent, RayOrigin, RayDir,
18891 Value *Call = Builder.CreateCall(F, Args);
18892 Value *Rtn = Builder.CreateExtractValue(Call, 0);
18893 Value *A = Builder.CreateExtractValue(Call, 1);
18895 Value *I0 = Builder.CreateInsertElement(PoisonValue::get(RetTy), Rtn,
18897 return Builder.CreateInsertElement(I0, A, 1);
19118 Args.push_back(Builder.getFalse());
19125 return Builder.CreateCall(F, Args);
19168 return Builder.CreateCall(F, { Src0, Src1, Src2 });
19173 FenceInst *Fence = Builder.CreateFence(AO, SSID);
19245 llvm::Type::getBFloatTy(Builder.getContext()), 2);
19246 Val = Builder.CreateBitCast(Val, V2BF16Ty);
19251 Builder.CreateAtomicRMW(BinOp, Ptr, Val, AO, SSID);
19254 return Builder.CreateBitCast(RMW, OrigTy);
19263 return Builder.CreateCall(F, {Arg});
19305 return Builder.CreateCall(
19326 Value *Call = CGF.Builder.CreateCall(F, Args);
19327 Value *CC = CGF.Builder.CreateExtractValue(Call, 1);
19328 CGF.Builder.CreateStore(CC, CCPtr);
19329 return CGF.Builder.CreateExtractValue(Call, 0);
19339 return Builder.CreateCall(F, {TDB, Control});
19345 return Builder.CreateCall(F, {TDB, Control});
19351 return Builder.CreateCall(F, {TDB, Control});
19356 return Builder.CreateCall(F, Builder.CreateSExt(Data, Int64Ty, "tabort"));
19362 return Builder.CreateCall(F, {Data, Address});
19377 return Builder.CreateCall(F, X);
19386 Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
19388 return Builder.CreateCall(F, {X, Undef});
19397 Value *Undef = ConstantInt::get(Builder.getInt1Ty(), false);
19399 return Builder.CreateCall(F, {X, Undef});
19411 Amt = Builder.CreateIntCast(Amt, ResultType->getScalarType(), false);
19412 Amt = Builder.CreateVectorSplat(NumElts, Amt);
19414 return Builder.CreateCall(F, { Src, Src, Amt });
19425 return Builder.CreateCall(F, { Src, Src, Amt });
19432 if (Builder.getIsFPConstrained()) {
19434 return Builder.CreateConstrainedFPCall(F, { X });
19437 return Builder.CreateCall(F, X);
19446 if (Builder.getIsFPConstrained()) {
19448 return Builder.CreateConstrainedFPCall(F, {X, Y, Z});
19451 return Builder.CreateCall(F, {X, Y, Z});
19460 if (Builder.getIsFPConstrained()) {
19462 return Builder.CreateConstrainedFPCall(F, {X, Y, Builder.CreateFNeg(Z, "neg")});
19465 return Builder.CreateCall(F, {X, Y, Builder.CreateFNeg(Z, "neg")});
19474 if (Builder.getIsFPConstrained()) {
19476 return Builder.CreateFNeg(Builder.CreateConstrainedFPCall(F, {X, Y, Z}), "neg");
19479 return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, Z}), "neg");
19488 if (Builder.getIsFPConstrained()) {
19490 Value *NegZ = Builder.CreateFNeg(Z, "sub");
19491 return Builder.CreateFNeg(Builder.CreateConstrainedFPCall(F, {X, Y, NegZ}));
19494 Value *NegZ = Builder.CreateFNeg(Z, "neg");
19495 return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, NegZ}));
19503 return Builder.CreateCall(F, X);
19510 return Builder.CreateFNeg(Builder.CreateCall(F, X), "neg");
19549 if (Builder.getIsFPConstrained()) {
19551 return Builder.CreateConstrainedFPCall(F, X);
19554 return Builder.CreateCall(F, X);
19565 return Builder.CreateCall(F, {X, M4Value, M5Value});
19584 if (Builder.getIsFPConstrained()) {
19586 return Builder.CreateConstrainedFPCall(F, {X, Y});
19589 return Builder.CreateCall(F, {X, Y});
19599 return Builder.CreateCall(F, {X, Y, M4Value});
19618 if (Builder.getIsFPConstrained()) {
19620 return Builder.CreateConstrainedFPCall(F, {X, Y});
19623 return Builder.CreateCall(F, {X, Y});
19633 return Builder.CreateCall(F, {X, Y, M4Value});
19642 return Builder.CreateCall(F, X);
20049 return CGF.Builder.CreateCall(
20051 {Ptr, ConstantInt::get(CGF.Builder.getInt32Ty(), Align.getQuantity())});
20059 return CGF.Builder.CreateCall(
20068 ? CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(IntrinsicIDS),
20072 : CGF.Builder.CreateCall(CGF.CGM.getIntrinsic(IntrinsicID),
20103 ArgValue = CGF.Builder.CreateBitCast(ArgValue, PTy);
20107 return CGF.Builder.CreateCall(F, Args);
20176 return Builder.CreateAtomicRMW(llvm::AtomicRMWInst::FAdd, DestAddr, Val,
20185 return Builder.CreateCall(FnALI32, {Ptr, Val});
20193 return Builder.CreateCall(FnALD32, {Ptr, Val});
20361 return Builder.CreateCall(
20372 return Builder.CreateCall(
20382 Value *ResultPair = Builder.CreateCall(
20387 Value *Pred = Builder.CreateZExt(Builder.CreateExtractValue(ResultPair, 1),
20389 Builder.CreateStore(Pred, PredOutPtr);
20390 return Builder.CreateExtractValue(ResultPair, 0);
20459 Builder.CreateCall(CGM.getIntrinsic(IID, Src->getType()), {Src, Ldm});
20464 Builder.CreateAlignedStore(Result, Dst.emitRawPointer(*this),
20468 Builder.CreateAlignedStore(
20469 Builder.CreateBitCast(Builder.CreateExtractValue(Result, i),
20471 Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this),
20509 Value *V = Builder.CreateAlignedLoad(
20511 Builder.CreateGEP(Src.getElementType(), Src.emitRawPointer(*this),
20514 Values.push_back(Builder.CreateBitCast(V, ParamType));
20517 Value *Result = Builder.CreateCall(Intrinsic, Values);
20581 Value *V = Builder.CreateAlignedLoad(
20583 Builder.CreateGEP(SrcA.getElementType(), SrcA.emitRawPointer(*this),
20586 Values.push_back(Builder.CreateBitCast(V, AType));
20591 Value *V = Builder.CreateAlignedLoad(
20593 Builder.CreateGEP(SrcB.getElementType(), SrcB.emitRawPointer(*this),
20596 Values.push_back(Builder.CreateBitCast(V, BType));
20602 Value *V = Builder.CreateAlignedLoad(
20604 Builder.CreateGEP(SrcC.getElementType(), SrcC.emitRawPointer(*this),
20607 Values.push_back(Builder.CreateBitCast(V, CType));
20609 Value *Result = Builder.CreateCall(Intrinsic, Values);
20612 Builder.CreateAlignedStore(
20613 Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), DType),
20614 Builder.CreateGEP(Dst.getElementType(), Dst.emitRawPointer(*this),
20768 return Builder.CreateCall(
20771 return Builder.CreateCall(
20774 return Builder.CreateCall(
20777 return Builder.CreateCall(
20780 return Builder.CreateCall(
20783 return Builder.CreateCall(
20786 return Builder.CreateCall(
20789 return Builder.CreateCall(
20792 return Builder.CreateCall(
20795 return Builder.CreateCall(
20798 return Builder.CreateCall(
20801 return Builder.CreateCall(
20804 return Builder.CreateCall(
20807 return Builder.CreateCall(
20810 return Builder.CreateCall(
20813 return Builder.CreateCall(
20816 return Builder.CreateCall(
20819 return Builder.CreateCall(
20822 return Builder.CreateCall(
20825 return Builder.CreateCall(
20829 return Builder.CreateCall(
20833 return Builder.CreateCall(
20837 return Builder.CreateCall(
20841 return Builder.CreateCall(
20845 return Builder.CreateCall(
20848 return Builder.CreateCall(
20851 return Builder.CreateCall(
20854 return Builder.CreateCall(
20885 Alignment = CGF.Builder.CreateZExtOrTrunc(Alignment, IntType, "alignment");
20887 Mask = CGF.Builder.CreateSub(Alignment, One, "mask");
20898 Builder.CreateBitOrPointerCast(Args.Src, Args.IntType, "src_addr");
20899 return RValue::get(Builder.CreateICmpEQ(
20900 Builder.CreateAnd(SrcAddress, Args.Mask, "set_bits"),
20918 Builder.CreateGEP(Int8Ty, SrcForMask, Args.Mask, "over_boundary");
20925 SrcForMask = Builder.CreateAdd(SrcForMask, Args.Mask, "over_boundary");
20929 llvm::Value *InvertedMask = Builder.CreateNot(Args.Mask, "inverted_mask");
20932 Result = Builder.CreateIntrinsic(
20936 Result = Builder.CreateAnd(SrcForMask, InvertedMask, "aligned_result");
20950 return Builder.CreateCall(Callee, I);
20958 return Builder.CreateCall(Callee, Args);
20963 return Builder.CreateCall(Callee);
20968 return Builder.CreateCall(Callee);
20972 return Builder.CreateCall(Callee);
20978 return Builder.CreateCall(Callee, {Tag, Obj});
20982 return Builder.CreateCall(Callee);
20989 return Builder.CreateCall(Callee, {Addr, Expected, Timeout});
20996 return Builder.CreateCall(Callee, {Addr, Expected, Timeout});
21002 return Builder.CreateCall(Callee, {Addr, Count});
21012 return Builder.CreateCall(Callee, {Src});
21022 return Builder.CreateCall(Callee, {Src});
21033 return Builder.CreateCall(Callee, {Src});
21044 return Builder.CreateCall(Callee, {Src});
21055 return Builder.CreateCall(Callee, {LHS, RHS});
21066 return Builder.CreateCall(Callee, {LHS, RHS});
21075 return Builder.CreateCall(Callee, {LHS, RHS});
21084 return Builder.CreateCall(Callee, {LHS, RHS});
21117 return Builder.CreateCall(Callee, Value);
21121 return Builder.CreateCall(Callee);
21125 return Builder.CreateCall(Callee);
21131 return Builder.CreateCall(Callee, {Src, Indices});
21165 return Builder.CreateCall(Callee, {LHS, RHS});
21172 Value *Neg = Builder.CreateNeg(Vec, "neg");
21174 Value *ICmp = Builder.CreateICmpSLT(Vec, Zero, "abscond");
21175 return Builder.CreateSelect(ICmp, Neg, Vec, "abs");
21196 ICmp = Builder.CreateICmpSLT(LHS, RHS);
21201 ICmp = Builder.CreateICmpULT(LHS, RHS);
21206 ICmp = Builder.CreateICmpSGT(LHS, RHS);
21211 ICmp = Builder.CreateICmpUGT(LHS, RHS);
21216 return Builder.CreateSelect(ICmp, LHS, RHS);
21224 return Builder.CreateCall(Callee, {LHS, RHS});
21230 return Builder.CreateCall(Callee, {LHS, RHS});
21252 return Builder.CreateCall(Callee, Vec);
21260 return Builder.CreateCall(Callee, {V1, V2, C});
21266 return Builder.CreateCall(Callee, {LHS, RHS});
21272 return Builder.CreateCall(Callee, {Vec});
21295 return Builder.CreateCall(Callee, {Vec});
21304 return Builder.CreateCall(Callee, {Vec});
21310 return Builder.CreateCall(Callee, {Vec});
21316 return Builder.CreateCall(Callee, {Vec});
21339 return Builder.CreateCall(Callee, {Low, High});
21356 llvm::Type *TruncT = SrcT->getWithNewType(Builder.getInt32Ty());
21358 Value *Trunc = Builder.CreateCall(Callee, Vec);
21360 return Builder.CreateShuffleVector(Trunc, Splat, ArrayRef<int>{0, 1, 2, 3});
21374 return Builder.CreateCall(Callee, Ops);
21401 return Builder.CreateCall(Callee, {A, B, C});
21412 return Builder.CreateCall(Callee, {A, B, C});
21418 return Builder.CreateCall(Callee, {Src, Indices});
21440 return Builder.CreateCall(Callee, {LHS, RHS});
21465 return Builder.CreateCall(Callee, {Vec});
21471 return Builder.CreateCall(Callee, {LHS, RHS});
21478 return Builder.CreateCall(Callee, {LHS, RHS});
21486 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
21494 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
21499 return Builder.CreateCall(Callee, {Addr});
21505 return Builder.CreateCall(Callee, {Val, Addr});
21510 return Builder.CreateCall(Callee, {Val});
21516 return Builder.CreateCall(Callee, {Vector, Index});
21530 return Builder.CreateCall(Callee, {Table, Index});
21545 return Builder.CreateCall(Callee, {Table, Index, Val});
21551 return Builder.CreateCall(Callee, Value);
21568 return Builder.CreateCall(Callee, {Table, Val, NElems});
21586 return Builder.CreateCall(Callee, {Table, Index, Val, NElems});
21598 return Builder.CreateCall(Callee, {TableX, TableY, SrcIdx, DstIdx, NElems});
21671 llvm::Value *Base = Builder.CreateLoad(BP);
21684 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops);
21687 llvm::Value *NewBase = IsLoad ? Builder.CreateExtractValue(Result, 1)
21692 Builder.CreateAlignedStore(NewBase, LV, Dest.getAlignment());
21694 RetVal = Builder.CreateExtractValue(Result, 0);
21717 llvm::Value *Result = Builder.CreateCall(
21721 llvm::Value *DestVal = Builder.CreateExtractValue(Result, 0);
21726 DestVal = Builder.CreateTrunc(DestVal, DestTy);
21728 Builder.CreateAlignedStore(DestVal, DestAddress, DestAddr.getAlignment());
21730 return Builder.CreateExtractValue(Result, 1);
21736 return Builder.CreateCall(CGM.getIntrinsic(ID),
21737 {Vec, Builder.getInt32(-1)});
21742 return Builder.CreateCall(CGM.getIntrinsic(ID),
21743 {Pred, Builder.getInt32(-1)});
21758 llvm::Value *PredIn = V2Q(Builder.CreateLoad(PredAddr));
21759 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(ID),
21762 llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1);
21763 Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.emitRawPointer(*this),
21765 return Builder.CreateExtractValue(Result, 0);
21778 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(ID),
21781 llvm::Value *PredOut = Builder.CreateExtractValue(Result, 1);
21782 Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.emitRawPointer(*this),
21784 return Builder.CreateExtractValue(Result, 0);
21805 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
21880 llvm::Value *AggValue = Builder.CreateLoad(L.getAddress());
21930 Value *Result = Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)});
21932 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
21939 Value *Result = Builder.CreateCall(F, {Ops[0], Builder.getInt1(false)});
21941 Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
22028 llvm::ConstantAsMetadata::get(Builder.getInt32(DomainVal)));
22030 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
22040 LoadInst *Load = Builder.CreateLoad(
22056 llvm::ConstantAsMetadata::get(Builder.getInt32(DomainVal)));
22058 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
22060 StoreInst *Store = Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
22077 return Builder.CreateCall(F, Ops, "");