Lines Matching +full:region +full:- +full:freeze +full:- +full:timeout +full:- +full:us
1 //===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
11 //===----------------------------------------------------------------------===//
93 I->addAnnotationMetadata("auto-init");
96 /// getBuiltinLibFunction - Given a builtin id for a function like
106 // TODO: This list should be expanded or refactored after all GCC-compatible
134 // The AIX library functions frexpl, ldexpl, and modfl are for 128-bit
136 // if it is 64-bit 'long double' mode.
146 if (FD->hasAttr<AsmLabelAttr>())
150 // PPC, after backend supports IEEE 128-bit style libcalls.
165 cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
176 if (V->getType()->isPointerTy())
179 assert(V->getType() == IntType);
187 if (ResultType->isPointerTy())
190 assert(V->getType() == ResultType);
196 Address Ptr = CGF.EmitPointerWithAlignment(E->getArg(0));
197 unsigned Bytes = Ptr.getElementType()->isPointerTy()
199 : Ptr.getElementType()->getScalarSizeInBits() / 8;
203 Diags.Report(E->getBeginLoc(), diag::warn_sync_op_misaligned);
204 // Force address to be at least naturally-aligned.
216 QualType T = E->getType();
217 assert(E->getArg(0)->getType()->isPointerType());
219 E->getArg(0)->getType()->getPointeeType()));
220 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
227 llvm::Value *Val = CGF.EmitScalarExpr(E->getArg(1));
228 llvm::Type *ValueType = Val->getType();
237 Value *Val = CGF.EmitScalarExpr(E->getArg(0));
238 Address Addr = CGF.EmitPointerWithAlignment(E->getArg(1));
240 Val = CGF.EmitToMemory(Val, E->getArg(0)->getType());
241 LValue LV = CGF.MakeAddrLValue(Addr, E->getArg(0)->getType());
248 Address Addr = CGF.EmitPointerWithAlignment(E->getArg(0));
250 LValue LV = CGF.MakeAddrLValue(Addr, E->getType());
252 return CGF.EmitLoadOfScalar(LV, E->getExprLoc());
269 QualType T = E->getType();
270 assert(E->getArg(0)->getType()->isPointerType());
272 E->getArg(0)->getType()->getPointeeType()));
273 assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
280 llvm::Value *Val = CGF.EmitScalarExpr(E->getArg(1));
281 llvm::Type *ValueType = Val->getType();
299 /// arg0 - address to operate on
300 /// arg1 - value to compare with
301 /// arg2 - new value
311 QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType();
317 Value *Cmp = CGF.EmitScalarExpr(E->getArg(1));
318 llvm::Type *ValueType = Cmp->getType();
320 Value *New = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);
328 CGF.ConvertType(E->getType()));
351 assert(E->getArg(0)->getType()->isPointerType());
353 E->getType(), E->getArg(0)->getType()->getPointeeType()));
354 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
355 E->getArg(1)->getType()));
356 assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
357 E->getArg(2)->getType()));
361 auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
362 auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));
375 Result->setVolatile(true);
379 // 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
388 // Note that Destination is assumed to be at least 16-byte aligned, despite
394 assert(E->getNumArgs() == 4);
395 llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
396 llvm::Value *ExchangeHigh = CGF.EmitScalarExpr(E->getArg(1));
397 llvm::Value *ExchangeLow = CGF.EmitScalarExpr(E->getArg(2));
398 Address ComparandAddr = CGF.EmitPointerWithAlignment(E->getArg(3));
400 assert(DestPtr->getType()->isPointerTy());
401 assert(!ExchangeHigh->getType()->isPointerTy());
402 assert(!ExchangeLow->getType()->isPointerTy());
433 CXI->setVolatile(true);
446 assert(E->getArg(0)->getType()->isPointerType());
448 auto *IntTy = CGF.ConvertType(E->getType());
458 assert(E->getArg(0)->getType()->isPointerType());
460 auto *IntTy = CGF.ConvertType(E->getType());
469 Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
470 QualType ElTy = E->getArg(0)->getType()->getPointeeType();
475 Load->setVolatile(true);
481 Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
482 Value *Value = CGF.EmitScalarExpr(E->getArg(1));
483 QualType ElTy = E->getArg(0)->getType()->getPointeeType();
487 Store->setVolatile(true);
493 // floating-point intrinsic.
497 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
501 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
504 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
510 // Depending on mode, this may be a constrained floating-point intrinsic.
514 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
515 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
519 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
522 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
531 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
532 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
537 {Src0->getType(), Src1->getType()});
542 CGF.CGM.getIntrinsic(IntrinsicID, {Src0->getType(), Src1->getType()});
547 // Depending on mode, this may be a constrained floating-point intrinsic.
551 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
552 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
553 llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
557 Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
560 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
566 // Depending on mode, this may be a constrained floating-point intrinsic.
591 static_assert(N, "expect non-empty argument");
594 Args.push_back(CGF.EmitScalarExpr(E->getArg(I)));
595 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Args[0]->getType());
603 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
604 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
606 Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
615 llvm::Type *ResultType = CGF.ConvertType(E->getType());
616 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
621 {ResultType, Src0->getType()});
625 CGF.CGM.getIntrinsic(IntrinsicID, {ResultType, Src0->getType()});
632 llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
633 llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
635 QualType IntPtrTy = E->getArg(1)->getType()->getPointeeType();
638 CGF.CGM.getIntrinsic(IntrinsicID, {Src0->getType(), IntTy});
648 /// EmitFAbs - Emit a call to @llvm.fabs().
650 Function *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType());
652 Call->setDoesNotAccessMemory();
661 llvm::Type *Ty = V->getType();
662 int Width = Ty->getPrimitiveSizeInBits();
665 if (Ty->isPPC_FP128Ty()) {
666 // We want the sign bit of the higher-order double. The bitcast we just
667 // did works as if the double-double was stored to memory and then
668 // read as an i128. The "store" will put the higher-order double in the
669 // lower address in both little- and big-Endian modes, but the "load"
671 // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
678 // We are truncating value in order to extract the higher-order
692 CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot());
695 if (unsigned BuiltinID = FD->getBuiltinID()) {
696 auto IsErrnoIntrinsic = [&]() -> unsigned {
735 assert(X->getType() == Y->getType() &&
739 Function *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType());
750 Call->addRangeRetAttr(CR);
751 Call->addRetAttr(llvm::Attribute::AttrKind::NoUndef);
765 assert(Type->isIntegerType() && "Given type is not an integer.");
766 unsigned Width = Type->isBooleanType() ? 1
767 : Type->isBitIntType() ? context.getIntWidth(Type)
769 bool Signed = Type->isSignedIntegerType();
803 return Builder.CreateCall(CGM.getIntrinsic(inst, {ArgValue->getType()}),
818 return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true);
827 if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type))
839 if (RD->isImplicit())
842 for (const FieldDecl *FD : RD->fields()) {
845 Ctx, FD, FD->getType(), StrictFlexArraysLevel,
852 QualType Ty = FD->getType();
853 if (Ty->isRecordType()) {
855 Ctx, Ty->getAsRecordDecl(), FAMDecl, Offset)) {
862 if (!RD->isUnion())
872 for (const FieldDecl *FD : RD->fields()) {
873 if (FD->getType()->isCountAttributedType())
876 QualType Ty = FD->getType();
877 if (Ty->isRecordType())
878 Num += CountCountedByAttrs(Ty->getAsRecordDecl());
899 // __builtin_dynamic_object_size(p->array, 1) ==
900 // p->count * sizeof(*p->array)
904 // __builtin_dynamic_object_size(&p->array[42], 1) ==
905 // (p->count - 42) * sizeof(*p->array)
911 // offsetof(struct s, array) + p->count * sizeof(*p->array))
914 const Expr *Base = E->IgnoreParenImpCasts();
918 UO && UO->getOpcode() == UO_AddrOf) {
919 Expr *SubExpr = UO->getSubExpr()->IgnoreParenImpCasts();
921 Base = ASE->getBase()->IgnoreParenImpCasts();
922 Idx = ASE->getIdx()->IgnoreParenImpCasts();
925 int64_t Val = IL->getValue().getSExtValue();
944 const ValueDecl *VD = ME->getMemberDecl();
945 OuterRD = VD->getDeclContext()->getOuterLexicalRecordContext();
951 QualType Ty = DRE->getDecl()->getType();
952 if (Ty->isPointerType())
953 Ty = Ty->getPointeeType();
954 OuterRD = Ty->getAsRecordDecl();
987 // We call FindFlexibleArrayMemberAndOffset even if FAMDecl is non-null to
994 if (!FAMDecl || !FAMDecl->getType()->isCountAttributedType())
1023 bool IsSigned = CountedByFD->getType()->isSignedIntegerType();
1033 if (Idx->HasSideEffects(getContext()))
1034 // We can't have side-effects.
1037 bool IdxSigned = Idx->getType()->isSignedIntegerType();
1048 const ArrayType *ArrayTy = Ctx.getAsArrayType(FAMDecl->getType());
1049 CharUnits Size = Ctx.getTypeSizeInChars(ArrayTy->getElementType());
1068 /// - A llvm::Argument (if E is a param with the pass_object_size attribute on
1070 /// - A call to the @llvm.objectsize intrinsic
1072 /// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
1081 if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) {
1082 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
1083 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
1085 areBOSTypesCompatible(PS->getType(), Type)) {
1089 const ImplicitParamDecl *D = Iter->second;
1093 return EmitLoadOfScalar(DIter->second, /*Volatile=*/false,
1094 getContext().getSizeType(), E->getBeginLoc());
1106 // evaluate E for side-effects. In either case, we shouldn't lower to
1108 if (Type == 3 || (!EmittedE && E->HasSideEffects(getContext())))
1112 assert(Ptr->getType()->isPointerTy() &&
1113 "Non-pointer passed to __builtin_object_size?");
1116 CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
1163 // X86-specific 64-bit variants.
1177 // ARM/AArch64-specific ordering variants.
1230 CGF.getContext().getTypeSize(E->getArg(1)->getType()));
1258 Value *BitBase = CGF.EmitScalarExpr(E->getArg(0));
1259 Value *BitPos = CGF.EmitScalarExpr(E->getArg(1));
1273 BitPos, llvm::ConstantInt::get(BitPos->getType(), 3), "bittest.byteidx");
1303 // Emit a plain load for the non-interlocked intrinsics.
1334 Value *Addr = CGF.EmitScalarExpr(E->getArg(0));
1376 CI->addParamAttr(
1390 /// architecture except 32-bit x86, the frame address is passed. On x86, extra
1425 CGF.EmitScalarExpr(E->getArg(0)), CGF.Int8PtrTy);
1428 CB->setAttributes(ReturnsTwiceAttr);
1816 Address IndexAddress(EmitPointerWithAlignment(E->getArg(0)));
1817 Value *ArgValue = EmitScalarExpr(E->getArg(1));
1819 llvm::Type *ArgType = ArgValue->getType();
1821 llvm::Type *ResultType = ConvertType(E->getType());
1828 BasicBlock *End = createBasicBlock("bitscan_end", this->CurFn);
1834 BasicBlock *NotZero = createBasicBlock("bitscan_not_zero", this->CurFn);
1836 Result->addIncoming(ResZero, Begin);
1846 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
1847 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
1856 Result->addIncoming(ResOne, NotZero);
1954 // https://msdn.microsoft.com/en-us/library/dn774154.aspx
1980 llvm::CallInst *CI = Builder.CreateCall(IA, EmitScalarExpr(E->getArg(0)));
1981 CI->setAttributes(NoReturnAttr);
2011 ArgValue, llvm::Constant::getNullValue(ArgValue->getType()));
2014 {EmitCheckSourceLocation(E->getExprLoc()),
2028 Value *ArgValue = CGF.EmitScalarExpr(E->getArg(0));
2032 if (!VCI->isMinSignedValue())
2038 Constant *Zero = Constant::getNullValue(ArgValue->getType());
2045 // TODO: support -ftrapv-handler.
2049 {CGF.EmitCheckSourceLocation(E->getArg(0)->getExprLoc()),
2050 CGF.EmitCheckTypeDescriptor(E->getType())},
2109 // attached to the function when compiling with -Oz.
2115 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
2118 Fn->setDoesNotThrow();
2120 // Attach 'noinline' at -Oz.
2122 Fn->addFnAttr(llvm::Attribute::NoInline);
2194 // expression is passed, push a lifetime-extended cleanup to extend its
2197 E = E->IgnoreParenCasts();
2202 // created including arguments of non-ARC types (e.g., C++
2209 if (TheExpr->getType()->isObjCRetainableType() &&
2211 assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
2215 QualType Ty = TheExpr->getType();
2235 CGM.getDataLayout().getTypeSizeInBits(ArgVal->getType());
2240 // If ArgVal has type x86_fp80, zero-extend ArgVal.
2281 llvm::Value *IntMaxValue = llvm::ConstantInt::get(Result->getType(), IntMax);
2287 ResultArg->getType()->getPointeeType().isVolatileQualified();
2294 /// Determine if a binop is a checked mixed-sign multiply we can specialize.
2304 /// Emit a checked mixed-sign multiply. This is a cheaper specialization of
2305 /// the generic checked-binop irgen.
2314 "Not a mixed-sign multipliction we can specialize");
2326 Signed = CGF.Builder.CreateSExt(Signed, Unsigned->getType(), "op.sext");
2328 Unsigned = CGF.Builder.CreateZExt(Unsigned, Signed->getType(), "op.zext");
2330 llvm::Type *OpTy = Signed->getType();
2388 ResultArg->getType()->getPointeeType().isVolatileQualified();
2400 const auto *Record = Ty->getAsCXXRecordDecl();
2408 assert(Record->hasDefinition() &&
2411 if (Record->isDynamicClass())
2414 for (FieldDecl *F : Record->fields()) {
2415 if (TypeRequiresBuiltinLaunderImp(Ctx, F->getType(), Seen))
2431 llvm::Value *Src = EmitScalarExpr(E->getArg(0));
2432 llvm::Value *ShiftAmt = EmitScalarExpr(E->getArg(1));
2436 llvm::Type *Ty = Src->getType();
2439 // Rotate is a special case of LLVM funnel shift - 1st 2 args are the same.
2445 // Map math builtins for long-double to f128 version.
2530 auto Name = FD->getNameAsString() + "__hipstdpar_unsupported";
2531 auto FnTy = CGF->CGM.getTypes().GetFunctionType(FD);
2532 auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, FnTy);
2535 for (auto &&FormalTy : FnTy->params())
2538 return RValue::get(CGF->Builder.CreateCall(UBF, Args));
2544 const FunctionDecl *FD = GD.getDecl()->getAsFunction();
2546 // TODO: Extend this handling to all builtin calls that we can constant-fold.
2548 if (E->isPRValue() && E->EvaluateAsRValue(Result, CGM.getContext()) &&
2558 // If current long-double semantics is IEEE 128-bit, replace math builtins
2559 // of long-double with f128 equivalent.
2561 // after backend supports IEEE 128-bit style libcalls.
2571 FD->hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2574 // ErrnoOverriden is true if math-errno is overriden via the
2575 // '#pragma float_control(precise, on)'. This pragma disables fast-math,
2576 // which implies math-errno.
2577 if (E->hasStoredFPFeatures()) {
2578 FPOptionsOverride OP = E->getFPFeatures();
2583 // fast-math which implies math-errno.
2584 bool OptNone = CurFuncDecl && CurFuncDecl->hasAttr<OptimizeNoneAttr>();
2586 // True if we are compiling at -O2 and errno has been disabled
2588 // attribute opt-none hasn't been seen.
2630 // ConstAttr is enabled in fast-math mode. In fast-math mode, math-errno is
2632 // Math intrinsics are generated only when math-errno is disabled. Any pragmas
2633 // or attributes that affect math-errno should prevent or allow math
2635 // 1- In fast math mode, unless math-errno is overriden
2638 // 2- If math-errno was enabled on command line but overriden
2641 // 3- If we are compiling with optimization and errno has been disabled
2837 // fmod() is a special-case. It maps to the frem instruction rather than an
2848 Value *Arg1 = EmitScalarExpr(E->getArg(0));
2849 Value *Arg2 = EmitScalarExpr(E->getArg(1));
3081 EmitNonNullArgCheck(RValue::get(Val), Arg->getType(), Arg->getExprLoc(), FD,
3088 SourceLocation Loc = Arg->getExprLoc();
3091 if (CE->getCastKind() == CK_BitCast)
3092 Arg = CE->getSubExpr();
3093 EmitTypeCheck(Kind, Loc, Val, Arg->getType(), A.getAlignment(),
3102 return RValue::get(ConstantEmitter(*this).emitAbstract(E, E->getType()));
3108 ? EmitScalarExpr(E->getArg(0))
3109 : EmitVAListRef(E->getArg(0)).emitRawPointer(*this),
3113 Value *DstPtr = EmitVAListRef(E->getArg(0)).emitRawPointer(*this);
3114 Value *SrcPtr = EmitVAListRef(E->getArg(1)).emitRawPointer(*this);
3115 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy, {DstPtr->getType()}),
3130 Result = EmitAbs(*this, EmitScalarExpr(E->getArg(0)), false);
3134 Result = EmitAbs(*this, EmitScalarExpr(E->getArg(0)), true);
3146 Value *Real = EmitScalarExpr(E->getArg(0));
3147 Value *Imag = EmitScalarExpr(E->getArg(1));
3156 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3168 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3173 // Only enabled preserved access index region when debuginfo
3174 // is available as debuginfo is needed to preserve user-level
3177 CGM.Error(E->getExprLoc(), "using builtin_preserve_access_index() without -g");
3178 return RValue::get(EmitScalarExpr(E->getArg(0)));
3183 CGM.Error(E->getExprLoc(), "nested builtin_preserve_access_index() not supported");
3184 return RValue::get(EmitScalarExpr(E->getArg(0)));
3188 Value *Res = EmitScalarExpr(E->getArg(0));
3199 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3206 // clrsb(x) -> clz(x < 0 ? ~x : x) - 1 or
3207 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3209 llvm::Type *ArgType = ArgValue->getType();
3212 llvm::Type *ResultType = ConvertType(E->getType());
3229 E->getNumArgs() > 1;
3232 HasFallback ? EmitScalarExpr(E->getArg(0))
3233 : EmitCheckedArgForBuiltin(E->getArg(0), BCK_CTZPassedZero);
3235 llvm::Type *ArgType = ArgValue->getType();
3238 llvm::Type *ResultType = ConvertType(E->getType());
3242 if (Result->getType() != ResultType)
3250 Value *FallbackValue = EmitScalarExpr(E->getArg(1));
3261 E->getNumArgs() > 1;
3264 HasFallback ? EmitScalarExpr(E->getArg(0))
3265 : EmitCheckedArgForBuiltin(E->getArg(0), BCK_CLZPassedZero);
3267 llvm::Type *ArgType = ArgValue->getType();
3270 llvm::Type *ResultType = ConvertType(E->getType());
3274 if (Result->getType() != ResultType)
3282 Value *FallbackValue = EmitScalarExpr(E->getArg(1));
3290 // ffs(x) -> x ? cttz(x) + 1 : 0
3291 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3293 llvm::Type *ArgType = ArgValue->getType();
3296 llvm::Type *ResultType = ConvertType(E->getType());
3303 if (Result->getType() != ResultType)
3311 // parity(x) -> ctpop(x) & 1
3312 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3314 llvm::Type *ArgType = ArgValue->getType();
3317 llvm::Type *ResultType = ConvertType(E->getType());
3320 if (Result->getType() != ResultType)
3328 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3330 llvm::Type *ArgType = ArgValue->getType();
3333 llvm::Type *ResultType = ConvertType(E->getType());
3335 if (Result->getType() != ResultType)
3347 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3349 llvm::Type *ArgType = ArgValue->getType();
3352 llvm::Type *ResultType = ConvertType(E->getType());
3354 if (Result->getType() != ResultType)
3363 return RValue::get(EmitScalarExpr(E->getArg(0)));
3366 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3367 llvm::Type *ArgType = ArgValue->getType();
3369 Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
3370 // Don't generate llvm.expect on -O0 as the backend won't use it for
3372 // Note, we still IRGen ExpectedValue because it could have side-effects.
3382 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3383 llvm::Type *ArgType = ArgValue->getType();
3385 Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
3387 const Expr *ProbArg = E->getArg(2);
3388 bool EvalSucceed = ProbArg->EvaluateAsFloat(Probability, CGM.getContext());
3394 llvm::Type *Ty = ConvertType(ProbArg->getType());
3396 // Don't generate llvm.expect.with.probability on -O0 as the backend
3398 // Note, we still IRGen ExpectedValue because it could have side-effects.
3409 const Expr *Ptr = E->getArg(0);
3412 (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr;
3414 Value *AlignmentValue = EmitScalarExpr(E->getArg(1));
3416 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
3417 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
3427 if (E->getArg(0)->HasSideEffects(getContext()))
3430 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3436 const Expr *Arg0 = E->getArg(0);
3437 const Expr *Arg1 = E->getArg(1);
3449 cast<StringLiteral>(E->getArg(0)->IgnoreParenCasts())->getString();
3464 QualType ArgType = E->getArg(0)->getType();
3465 if (ArgType->isComplexType()) {
3467 QualType ElementType = ArgType->castAs<ComplexType>()->getElementType();
3468 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3475 ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
3480 Value *ArgValue = EmitScalarExpr(E->getArg(0));
3525 llvm::Type *ResultType = ConvertType(E->getType());
3527 const Expr *Arg = E->getArg(0);
3528 QualType ArgType = Arg->getType();
3529 // FIXME: The allowance for Obj-C pointers and block pointers is historical
3531 if (!ArgType->isIntegralOrEnumerationType() && !ArgType->isFloatingType() &&
3532 !ArgType->isObjCObjectPointerType() && !ArgType->isBlockPointerType())
3537 if (Arg->HasSideEffects(getContext()))
3539 // side-effects.
3543 if (ArgType->isObjCObjectPointerType()) {
3544 // Convert Objective-C objects to id because we cannot distinguish between
3545 // LLVM types for Obj-C classes as they are opaque.
3552 if (Result->getType() != ResultType)
3559 E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue();
3560 auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType()));
3565 return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType,
3569 Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
3571 RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) :
3573 Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) :
3576 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
3589 Value *Begin = EmitScalarExpr(E->getArg(0));
3590 Value *End = EmitScalarExpr(E->getArg(1));
3600 TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor(
3601 TrapLocation, *E->getArg(0)->tryEvaluateString(getContext()),
3602 *E->getArg(1)->tryEvaluateString(getContext()));
3613 EmitUnreachable(E->getExprLoc());
3624 llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
3625 llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
3632 Src0->getType());
3637 { Src0->getType(), Src1->getType() });
3662 Value *LHS = EmitScalarExpr(E->getArg(0));
3663 Value *RHS = EmitScalarExpr(E->getArg(1));
3687 return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
3692 Value *V = EmitScalarExpr(E->getArg(0));
3697 ConvertType(E->getType())));
3702 Value *V = EmitScalarExpr(E->getArg(0));
3705 ConvertType(E->getType())));
3710 Value *V = EmitScalarExpr(E->getArg(0));
3715 ConvertType(E->getType())));
3726 Value *V = EmitScalarExpr(E->getArg(0));
3731 ConvertType(E->getType())));
3736 Value *V = EmitScalarExpr(E->getArg(0));
3739 ConvertType(E->getType())));
3744 Value *V = EmitScalarExpr(E->getArg(0));
3747 ConvertType(E->getType())));
3752 Value *V = EmitScalarExpr(E->getArg(0));
3755 ConvertType(E->getType())));
3760 if (!E->getArg(1)->EvaluateAsInt(Result, CGM.getContext()))
3764 Value *V = EmitScalarExpr(E->getArg(0));
3766 ConvertType(E->getType())));
3770 llvm::Type *Ty = ConvertType(E->getArg(0)->getType());
3780 QualType QT = E->getArg(0)->getType();
3782 if (auto *VecTy = QT->getAs<VectorType>())
3783 QT = VecTy->getElementType();
3784 if (QT->isIntegerType())
3786 llvm::Intrinsic::abs, EmitScalarExpr(E->getArg(0)),
3875 Value *Op0 = EmitScalarExpr(E->getArg(0));
3876 Value *Op1 = EmitScalarExpr(E->getArg(1));
3878 assert(Op0->getType()->isIntOrIntVectorTy() && "integer type expected");
3879 QualType Ty = E->getArg(0)->getType();
3880 if (auto *VecTy = Ty->getAs<VectorType>())
3881 Ty = VecTy->getElementType();
3882 bool IsSigned = Ty->isSignedIntegerType();
3893 Value *Op0 = EmitScalarExpr(E->getArg(0));
3894 Value *Op1 = EmitScalarExpr(E->getArg(1));
3896 if (Op0->getType()->isIntOrIntVectorTy()) {
3897 QualType Ty = E->getArg(0)->getType();
3898 if (auto *VecTy = Ty->getAs<VectorType>())
3899 Ty = VecTy->getElementType();
3900 Result = Builder.CreateBinaryIntrinsic(Ty->isSignedIntegerType()
3909 Value *Op0 = EmitScalarExpr(E->getArg(0));
3910 Value *Op1 = EmitScalarExpr(E->getArg(1));
3912 if (Op0->getType()->isIntOrIntVectorTy()) {
3913 QualType Ty = E->getArg(0)->getType();
3914 if (auto *VecTy = Ty->getAs<VectorType>())
3915 Ty = VecTy->getElementType();
3916 Result = Builder.CreateBinaryIntrinsic(Ty->isSignedIntegerType()
3927 if (auto *VecTy = QT->getAs<VectorType>())
3928 QT = VecTy->getElementType();
3929 else if (QT->isSizelessVectorType())
3930 QT = QT->getSizelessVectorEltType(CGM.getContext());
3932 if (QT->isSignedIntegerType())
3934 if (QT->isUnsignedIntegerType())
3936 assert(QT->isFloatingType() && "must have a float here");
3940 *this, E, GetIntrinsicID(E->getArg(0)->getType()), "rdx.min"));
3945 if (auto *VecTy = QT->getAs<VectorType>())
3946 QT = VecTy->getElementType();
3947 else if (QT->isSizelessVectorType())
3948 QT = QT->getSizelessVectorEltType(CGM.getContext());
3950 if (QT->isSignedIntegerType())
3952 if (QT->isUnsignedIntegerType())
3954 assert(QT->isFloatingType() && "must have a float here");
3959 *this, E, GetIntrinsicID(E->getArg(0)->getType()), "rdx.min"));
3979 auto *MatrixTy = E->getArg(0)->getType()->castAs<ConstantMatrixType>();
3980 Value *MatValue = EmitScalarExpr(E->getArg(0));
3982 Value *Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
3983 MatrixTy->getNumColumns());
3990 Value *Stride = EmitScalarExpr(E->getArg(3));
3991 const auto *ResultTy = E->getType()->getAs<ConstantMatrixType>();
3992 auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
3994 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
3996 Address Src = EmitPointerWithAlignment(E->getArg(0));
3998 E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD,
4003 ResultTy->getNumRows(), ResultTy->getNumColumns(), "matrix");
4009 Value *Matrix = EmitScalarExpr(E->getArg(0));
4010 Address Dst = EmitPointerWithAlignment(E->getArg(1));
4011 Value *Stride = EmitScalarExpr(E->getArg(2));
4013 const auto *MatrixTy = E->getArg(0)->getType()->getAs<ConstantMatrixType>();
4014 auto *PtrTy = E->getArg(1)->getType()->getAs<PointerType>();
4016 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
4019 E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD,
4024 MatrixTy->getNumRows(), MatrixTy->getNumColumns());
4029 // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
4031 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4032 Value *Arg = EmitScalarExpr(E->getArg(0));
4035 AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf");
4038 llvm::Type *IntTy = ConvertType(E->getType());
4041 Value *NegativeOne = ConstantInt::get(IntTy, -1);
4050 llvm::Type *ResultType = ConvertType(E->getType());
4052 if (Result->getType() != ResultType)
4061 Value *V = EmitScalarExpr(E->getArg(0));
4068 // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
4069 Value *V = EmitScalarExpr(E->getArg(5));
4070 llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
4074 BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn);
4077 Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4,
4084 Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
4085 BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn);
4087 Result->addIncoming(ZeroLiteral, Begin);
4092 Value *NanLiteral = EmitScalarExpr(E->getArg(0));
4093 BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn);
4095 Result->addIncoming(NanLiteral, NotZero);
4101 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
4103 Value *InfLiteral = EmitScalarExpr(E->getArg(1));
4104 BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn);
4106 Result->addIncoming(InfLiteral, NotNan);
4111 getContext().getFloatTypeSemantics(E->getArg(5)->getType()));
4113 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
4116 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
4117 EmitScalarExpr(E->getArg(3)));
4119 Result->addIncoming(NormalResult, NotInf);
4136 Value *Size = EmitScalarExpr(E->getArg(0));
4144 AI->setAlignment(SuitableAlignmentInBytes);
4148 LangAS EAS = E->getType()->getPointeeType().getAddressSpace();
4150 llvm::Type *Ty = CGM.getTypes().ConvertType(E->getType());
4159 Value *Size = EmitScalarExpr(E->getArg(0));
4160 Value *AlignmentInBitsValue = EmitScalarExpr(E->getArg(1));
4162 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
4166 AI->setAlignment(AlignmentInBytes);
4170 LangAS EAS = E->getType()->getPointeeType().getAddressSpace();
4172 llvm::Type *Ty = CGM.getTypes().ConvertType(E->getType());
4181 Address Dest = EmitPointerWithAlignment(E->getArg(0));
4182 Value *SizeVal = EmitScalarExpr(E->getArg(1));
4183 EmitNonNullArgCheck(Dest, E->getArg(0)->getType(),
4184 E->getArg(0)->getExprLoc(), FD, 0);
4191 Address Src = EmitPointerWithAlignment(E->getArg(0));
4192 Address Dest = EmitPointerWithAlignment(E->getArg(1));
4193 Value *SizeVal = EmitScalarExpr(E->getArg(2));
4195 E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD,
4198 E->getArg(1)->getType(), E->getArg(1)->getExprLoc(), FD,
4208 Address Dest = EmitPointerWithAlignment(E->getArg(0));
4209 Address Src = EmitPointerWithAlignment(E->getArg(1));
4210 Value *SizeVal = EmitScalarExpr(E->getArg(2));
4211 EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0);
4212 EmitArgCheck(TCK_Load, Src, E->getArg(1), 1);
4223 Address Dest = EmitPointerWithAlignment(E->getArg(0));
4224 Address Src = EmitPointerWithAlignment(E->getArg(1));
4226 E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue();
4227 EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0);
4228 EmitArgCheck(TCK_Load, Src, E->getArg(1), 1);
4240 if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
4241 !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
4247 Address Dest = EmitPointerWithAlignment(E->getArg(0));
4248 Address Src = EmitPointerWithAlignment(E->getArg(1));
4255 Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
4256 Address SrcAddr = EmitPointerWithAlignment(E->getArg(1));
4257 Value *SizeVal = EmitScalarExpr(E->getArg(2));
4266 if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
4267 !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
4273 Address Dest = EmitPointerWithAlignment(E->getArg(0));
4274 Address Src = EmitPointerWithAlignment(E->getArg(1));
4282 Address Dest = EmitPointerWithAlignment(E->getArg(0));
4283 Address Src = EmitPointerWithAlignment(E->getArg(1));
4284 Value *SizeVal = EmitScalarExpr(E->getArg(2));
4285 EmitArgCheck(TCK_Store, Dest, E->getArg(0), 0);
4286 EmitArgCheck(TCK_Load, Src, E->getArg(1), 1);
4292 Address Dest = EmitPointerWithAlignment(E->getArg(0));
4293 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
4295 Value *SizeVal = EmitScalarExpr(E->getArg(2));
4296 EmitNonNullArgCheck(Dest, E->getArg(0)->getType(),
4297 E->getArg(0)->getExprLoc(), FD, 0);
4302 Address Dest = EmitPointerWithAlignment(E->getArg(0));
4304 Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)), Builder.getInt8Ty());
4306 E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue();
4308 E->getArg(0)->getType(), E->getArg(0)->getExprLoc(), FD,
4316 if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
4317 !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
4323 Address Dest = EmitPointerWithAlignment(E->getArg(0));
4324 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
4337 Value *Str = EmitScalarExpr(E->getArg(0));
4338 Value *Chr = EmitScalarExpr(E->getArg(1));
4339 Value *Size = EmitScalarExpr(E->getArg(2));
4349 PHINode *StrPhi = Builder.CreatePHI(Str->getType(), 2);
4350 StrPhi->addIncoming(Str, Entry);
4352 SizePhi->addIncoming(Size, Entry);
4366 StrPhi->addIncoming(NextStr, Next);
4367 SizePhi->addIncoming(NextSize, Next);
4370 PHINode *Ret = Builder.CreatePHI(Str->getType(), 3);
4371 Ret->addIncoming(llvm::Constant::getNullValue(Str->getType()), Entry);
4372 Ret->addIncoming(llvm::Constant::getNullValue(Str->getType()), Next);
4373 Ret->addIncoming(FoundChr, CmpEq);
4384 Value *Dst = EmitScalarExpr(E->getArg(0));
4385 Value *Src = EmitScalarExpr(E->getArg(1));
4386 Value *Size = EmitScalarExpr(E->getArg(2));
4397 PHINode *DstPhi = Builder.CreatePHI(Dst->getType(), 2);
4398 DstPhi->addIncoming(Dst, Entry);
4399 PHINode *SrcPhi = Builder.CreatePHI(Src->getType(), 2);
4400 SrcPhi->addIncoming(Src, Entry);
4402 SizePhi->addIncoming(Size, Entry);
4421 DstPhi->addIncoming(NextDst, Next);
4422 SrcPhi->addIncoming(NextSrc, Next);
4423 SizePhi->addIncoming(NextSize, Next);
4427 Ret->addIncoming(ConstantInt::get(IntTy, 0), Entry);
4428 Ret->addIncoming(ConstantInt::get(IntTy, 1), CmpGT);
4429 Ret->addIncoming(ConstantInt::get(IntTy, -1), CmpLT);
4430 Ret->addIncoming(ConstantInt::get(IntTy, 0), Next);
4441 // this instead of hard-coding 0, which is correct for most targets.
4449 Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
4459 Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
4465 Value *Address = EmitScalarExpr(E->getArg(0));
4470 Value *Address = EmitScalarExpr(E->getArg(0));
4476 = cast<llvm::IntegerType>(ConvertType(E->getType()));
4478 if (Column == -1) {
4485 Value *Address = EmitScalarExpr(E->getArg(0));
4488 return RValue::get(llvm::UndefValue::get(ConvertType(E->getType())));
4491 Value *Int = EmitScalarExpr(E->getArg(0));
4492 Value *Ptr = EmitScalarExpr(E->getArg(1));
4494 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
4495 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
4496 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
4498 CGM.getIntrinsic(IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32
4518 // doesn't implicitly ignore high-order bits when doing
4522 // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
4525 Value *Ptr = EmitScalarExpr(E->getArg(0));
4529 if (IntPtrTy->getBitWidth() == 64)
4540 Address Buf = EmitPointerWithAlignment(E->getArg(0));
4550 assert(Buf.emitRawPointer(*this)->getType() == StackAddr->getType());
4560 Value *Buf = EmitScalarExpr(E->getArg(0));
4574 const Expr *Arg = E->getArg(0);
4575 QualType ArgTy = Arg->getType()->getPointeeType();
4724 QualType ElTy = E->getArg(0)->getType()->getPointeeType();
4730 Store->setAtomic(llvm::AtomicOrdering::Release);
4735 // We assume this is supposed to correspond to a C++0x-style
4736 // sequentially-consistent fence (i.e. this is only usable for
4740 // to use it with non-atomic loads and stores to get acquire/release
4753 // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
4754 // _Atomic(T) is always properly-aligned.
4757 Args.add(RValue::get(EmitScalarExpr(E->getArg(0))),
4760 Args.add(RValue::get(EmitScalarExpr(E->getArg(1))),
4766 CGM.getTypes().arrangeBuiltinFunctionCall(E->getType(), Args);
4776 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
4778 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
4781 EmitPointerWithAlignment(E->getArg(0)).withElementType(Int8Ty);
4784 Value *Order = EmitScalarExpr(E->getArg(1));
4786 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4814 Result->setVolatile(Volatile);
4842 RMW->setVolatile(Volatile);
4843 Result->addIncoming(RMW, BBs[i]);
4847 SI->addCase(Builder.getInt32(0), BBs[0]);
4848 SI->addCase(Builder.getInt32(1), BBs[1]);
4849 SI->addCase(Builder.getInt32(2), BBs[1]);
4850 SI->addCase(Builder.getInt32(3), BBs[2]);
4851 SI->addCase(Builder.getInt32(4), BBs[3]);
4852 SI->addCase(Builder.getInt32(5), BBs[4]);
4859 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
4861 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
4863 Address Ptr = EmitPointerWithAlignment(E->getArg(0));
4866 Value *Order = EmitScalarExpr(E->getArg(1));
4868 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4873 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
4876 Store->setOrdering(llvm::AtomicOrdering::Release);
4879 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
4902 Store->setOrdering(Orders[i]);
4906 SI->addCase(Builder.getInt32(0), BBs[0]);
4907 SI->addCase(Builder.getInt32(3), BBs[1]);
4908 SI->addCase(Builder.getInt32(5), BBs[2]);
4924 Value *Order = EmitScalarExpr(E->getArg(0));
4926 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
4961 SI->addCase(Builder.getInt32(1), AcquireBB);
4962 SI->addCase(Builder.getInt32(2), AcquireBB);
4967 SI->addCase(Builder.getInt32(3), ReleaseBB);
4972 SI->addCase(Builder.getInt32(4), AcqRelBB);
4977 SI->addCase(Builder.getInt32(5), SeqCstBB);
4987 Builder.CreateZExt(EmitSignBit(*this, EmitScalarExpr(E->getArg(0))),
4988 ConvertType(E->getType())));
4993 // Re-encode each wide string to UTF8 and make an MDString.
4995 for (const Expr *Arg : E->arguments()) {
4996 const auto *Str = cast<StringLiteral>(Arg->IgnoreParenCasts());
4997 assert(Str->getCharByteWidth() == 2);
4998 StringRef WideBytes = Str->getBytes();
5002 CGM.ErrorUnsupported(E, "non-UTF16 __annotation argument");
5016 llvm::Value *AnnVal = EmitScalarExpr(E->getArg(0));
5019 {AnnVal->getType(), CGM.ConstGlobalsPtrTy});
5022 // non-wide string literal, potentially casted, so the cast<> is safe.
5023 const Expr *AnnotationStrExpr = E->getArg(1)->IgnoreParenCasts();
5024 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
5026 EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc(), nullptr));
5057 llvm::Value *X = EmitScalarExpr(E->getArg(0));
5058 llvm::Value *Y = EmitScalarExpr(E->getArg(1));
5059 llvm::Value *Carryin = EmitScalarExpr(E->getArg(2));
5060 Address CarryOutPtr = EmitPointerWithAlignment(E->getArg(3));
5090 X->getType());
5098 const clang::Expr *LeftArg = E->getArg(0);
5099 const clang::Expr *RightArg = E->getArg(1);
5100 const clang::Expr *ResultArg = E->getArg(2);
5103 ResultArg->getType()->castAs<PointerType>()->getPointeeType();
5106 getIntegerWidthAndSignedness(CGM.getContext(), LeftArg->getType());
5108 getIntegerWidthAndSignedness(CGM.getContext(), RightArg->getType());
5112 // Handle mixed-sign multiplication as a special case, because adding
5184 ResultArg->getType()->getPointeeType().isVolatileQualified();
5212 llvm::Value *X = EmitScalarExpr(E->getArg(0));
5213 llvm::Value *Y = EmitScalarExpr(E->getArg(1));
5214 Address SumOutPtr = EmitPointerWithAlignment(E->getArg(2));
5262 return RValue::get(EmitLValue(E->getArg(0)).getPointer(*this));
5265 E->getArg(0)->getAsBuiltinConstantDeclRef(CGM.getContext())));
5268 E->getCallee()->getType()->castAs<FunctionProtoType>(), E, false);
5271 E->getCallee()->getType()->castAs<FunctionProtoType>(), E, true);
5285 const CallExpr *Call = cast<CallExpr>(E->getArg(0));
5286 const Expr *Chain = E->getArg(1);
5287 return EmitCall(Call->getCallee()->getType(),
5288 EmitCallee(Call->getCallee()), Call, ReturnValue,
5301 getLLVMContext(), getContext().getTypeSize(E->getType()));
5305 llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
5306 RTy = Exchange->getType();
5310 Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
5318 Result->setVolatile(true);
5381 // affected by the -fms-volatile setting.
5394 return RValue::get(ConstantEmitter(*this).emitAbstract(E, E->getType()));
5404 for (auto argExpr : E->arguments())
5408 llvm::Type *OrigValueType = Args[0]->getType();
5409 if (OrigValueType->isPointerTy())
5414 if (Args[4]->getType()->isPointerTy())
5420 if (Args[2]->getType()->isPointerTy())
5425 if (Args[1]->getType()->isPointerTy())
5435 auto IntrinsicID = [&]() -> unsigned {
5457 OrigValueType->isPointerTy()) {
5473 if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
5474 E->getArg(0)->getType()->isPointerType())
5478 if (getTarget().getTriple().isOSMSVCRT() && E->getNumArgs() == 1 &&
5479 E->getArg(0)->getType()->isPointerType()) {
5494 return RValue::get(EmitLValue(E->getArg(0)).getPointer(*this));
5497 CGM.getCXXABI().getThrowInfo(FD->getParamDecl(0)->getType()))
5536 // OpenCL v2.0 s6.13.16.2, Built-in pipe read and write functions
5539 Value *Arg0 = EmitScalarExpr(E->getArg(0)),
5540 *Arg1 = EmitScalarExpr(E->getArg(1));
5542 Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
5543 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
5551 if (2U == E->getNumArgs()) {
5556 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
5564 assert(4 == E->getNumArgs() &&
5569 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
5571 Value *Arg2 = EmitScalarExpr(E->getArg(2)),
5572 *Arg3 = EmitScalarExpr(E->getArg(3));
5578 if (Arg2->getType() != Int32Ty)
5585 // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write
5608 Value *Arg0 = EmitScalarExpr(E->getArg(0)),
5609 *Arg1 = EmitScalarExpr(E->getArg(1));
5612 Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
5613 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
5616 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
5621 if (Arg1->getType() != Int32Ty)
5626 // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write
5648 Value *Arg0 = EmitScalarExpr(E->getArg(0)),
5649 *Arg1 = EmitScalarExpr(E->getArg(1));
5651 Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
5652 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
5655 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
5663 // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions
5667 const auto *PipeTy = E->getArg(0)->getType()->castAs<PipeType>();
5673 std::string(PipeTy->isReadOnly() ? "_ro" : "_wo");
5676 Value *Arg0 = EmitScalarExpr(E->getArg(0));
5678 Value *PacketSize = OpenCLRT.getPipeElemSize(E->getArg(0));
5679 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->getArg(0));
5680 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
5688 // OpenCL v2.0 s6.13.9 - Address space qualifier functions.
5692 auto Arg0 = EmitScalarExpr(E->getArg(0));
5699 E->getType()->getPointeeType().getAddressSpace()));
5702 if (Arg0->getType()->getPointerAddressSpace() !=
5703 NewArgT->getPointerAddressSpace())
5707 auto NewName = std::string("__") + E->getDirectCallee()->getName().str();
5711 ConvertType(E->getType())));
5714 // OpenCL v2.0, s6.13.17 - Enqueue kernel function.
5724 unsigned NumArgs = E->getNumArgs();
5730 llvm::Value *Queue = EmitScalarExpr(E->getArg(0));
5731 llvm::Value *Flags = EmitScalarExpr(E->getArg(1));
5732 LValue NDRangeL = EmitAggExprToLValue(E->getArg(2));
5746 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(3));
5760 RTCall->setAttributes(ByValAttrSet);
5768 -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
5769 llvm::APInt ArraySize(32, NumArgs - First);
5783 auto *Index = llvm::ConstantInt::get(IntTy, I - First);
5789 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(I)), SizeTy);
5797 if (E->getArg(3)->getType()->isBlockPointerType()) {
5801 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(3));
5812 Block, ConstantInt::get(IntTy, NumArgs - 4),
5816 GenericVoidPtrTy, IntTy, ElemPtr->getType()};
5832 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->getArg(3)), Int32Ty);
5838 if (E->getArg(4)->isNullPointerConstant(
5843 E->getArg(4)->getType()->isArrayType()
5844 ? EmitArrayToPointerDecay(E->getArg(4)).emitRawPointer(*this)
5845 : EmitScalarExpr(E->getArg(4));
5850 if (E->getArg(5)->isNullPointerConstant(
5855 Builder.CreatePointerCast(EmitScalarExpr(E->getArg(5)), PtrTy);
5859 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(6));
5884 Args.push_back(ConstantInt::get(Int32Ty, NumArgs - 7));
5891 ArgTys.push_back(ElemPtr->getType());
5904 // OpenCL v2.0 s6.13.17.6 - Kernel query functions need bitcast of block
5910 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(0));
5925 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(0));
5940 LValue NDRangeL = EmitAggExprToLValue(E->getArg(0));
5943 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(1));
5954 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
5961 Value *Val = EmitScalarExpr(E->getArg(0));
5962 Address Address = EmitPointerWithAlignment(E->getArg(1));
5968 Address Address = EmitPointerWithAlignment(E->getArg(0));
5973 Address Address = EmitPointerWithAlignment(E->getArg(0));
6004 // Fall through - it's already mapped to the intrinsic by ClangBuiltin.
6018 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6019 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayCustomEvents())
6023 auto FTy = F->getFunctionType();
6024 auto Arg0 = E->getArg(0);
6026 auto Arg0Ty = Arg0->getType();
6027 auto PTy0 = FTy->getParamType(0);
6028 if (PTy0 != Arg0Val->getType()) {
6029 if (Arg0Ty->isArrayType())
6034 auto Arg1 = EmitScalarExpr(E->getArg(1));
6035 auto PTy1 = FTy->getParamType(1);
6036 if (PTy1 != Arg1->getType())
6052 if (const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
6053 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayTypedEvents())
6057 auto FTy = F->getFunctionType();
6058 auto Arg0 = EmitScalarExpr(E->getArg(0));
6059 auto PTy0 = FTy->getParamType(0);
6060 if (PTy0 != Arg0->getType())
6062 auto Arg1 = E->getArg(1);
6064 auto Arg1Ty = Arg1->getType();
6065 auto PTy1 = FTy->getParamType(1);
6066 if (PTy1 != Arg1Val->getType()) {
6067 if (Arg1Ty->isArrayType())
6072 auto Arg2 = EmitScalarExpr(E->getArg(2));
6073 auto PTy2 = FTy->getParamType(2);
6074 if (PTy2 != Arg2->getType())
6082 EmitVAStartEnd(EmitMSVAListRef(E->getArg(0)).emitRawPointer(*this),
6092 Address DestAddr = EmitMSVAListRef(E->getArg(0));
6093 Address SrcAddr = EmitMSVAListRef(E->getArg(1));
6104 cast<DeclRefExpr>(E->getArg(0)->IgnoreImpCasts())->getDecl());
6124 // This is down here to avoid non-target specific builtins, however, if
6160 llvm::FunctionType *FTy = F->getFunctionType();
6162 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
6166 llvm::Type *PTy = FTy->getParamType(i);
6167 if (PTy != ArgValue->getType()) {
6168 // XXX - vector of pointers?
6170 if (PtrTy->getAddressSpace() !=
6171 ArgValue->getType()->getPointerAddressSpace()) {
6174 PtrTy->getAddressSpace()));
6180 if (PTy->isX86_AMXTy())
6182 {ArgValue->getType()}, {ArgValue});
6191 QualType BuiltinRetType = E->getType();
6194 if (!BuiltinRetType->isVoidType())
6197 if (RetTy != V->getType()) {
6198 // XXX - vector of pointers?
6200 if (PtrTy->getAddressSpace() != V->getType()->getPointerAddressSpace()) {
6203 PtrTy->getAddressSpace()));
6209 if (V->getType()->isX86_AMXTy())
6216 if (RetTy->isVoidTy())
6222 // Some target-specific builtins can have aggregate return values, e.g.
6224 // ReturnValue to be non-null, so that the target-specific emission code can
6226 TypeEvaluationKind EvalKind = getEvaluationKind(E->getType());
6228 Address DestPtr = CreateMemTemp(E->getType(), "agg.tmp");
6232 // Now see if we can emit a target-specific builtin.
6236 if (V->getType()->isVoidTy())
6258 return GetUndefRValue(E->getType());
6271 if (CGF->getLangOpts().HIPStdPar && CGF->getLangOpts().CUDAIsDevice &&
6272 Arch != CGF->getTarget().getTriple().getArch())
6280 return CGF->EmitARMBuiltinExpr(BuiltinID, E, ReturnValue, Arch);
6284 return CGF->EmitAArch64BuiltinExpr(BuiltinID, E, Arch);
6287 return CGF->EmitBPFBuiltinExpr(BuiltinID, E);
6290 return CGF->EmitX86BuiltinExpr(BuiltinID, E);
6295 return CGF->EmitPPCBuiltinExpr(BuiltinID, E);
6298 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
6300 return CGF->EmitSystemZBuiltinExpr(BuiltinID, E);
6303 return CGF->EmitNVPTXBuiltinExpr(BuiltinID, E);
6306 return CGF->EmitWebAssemblyBuiltinExpr(BuiltinID, E);
6308 return CGF->EmitHexagonBuiltinExpr(BuiltinID, E);
6311 return CGF->EmitRISCVBuiltinExpr(BuiltinID, E, ReturnValue);
6313 if (CGF->getTarget().getTriple().getOS() != llvm::Triple::OSType::AMDHSA)
6315 return CGF->EmitAMDGPUBuiltinExpr(BuiltinID, E);
6328 ReturnValue, getContext().getAuxTargetInfo()->getTriple().getArch());
6344 return llvm::FixedVectorType::get(CGF->Int8Ty, V1Ty ? 1 : (8 << IsQuad));
6347 return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6350 return llvm::FixedVectorType::get(CGF->BFloatTy, V1Ty ? 1 : (4 << IsQuad));
6352 return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6355 return llvm::FixedVectorType::get(CGF->HalfTy, V1Ty ? 1 : (4 << IsQuad));
6357 return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
6359 return llvm::FixedVectorType::get(CGF->Int32Ty, V1Ty ? 1 : (2 << IsQuad));
6362 return llvm::FixedVectorType::get(CGF->Int64Ty, V1Ty ? 1 : (1 << IsQuad));
6367 return llvm::FixedVectorType::get(CGF->Int8Ty, 16);
6369 return llvm::FixedVectorType::get(CGF->FloatTy, V1Ty ? 1 : (2 << IsQuad));
6371 return llvm::FixedVectorType::get(CGF->DoubleTy, V1Ty ? 1 : (1 << IsQuad));
6381 return llvm::FixedVectorType::get(CGF->HalfTy, (4 << IsQuad));
6383 return llvm::FixedVectorType::get(CGF->FloatTy, (2 << IsQuad));
6385 return llvm::FixedVectorType::get(CGF->DoubleTy, (1 << IsQuad));
6387 llvm_unreachable("Type can't be converted to floating-point!");
6398 ElementCount EC = cast<llvm::VectorType>(V->getType())->getElementCount();
6406 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
6408 if (F->isConstrainedFPIntrinsic())
6409 if (ai->getType()->isMetadataTy())
6412 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
6414 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
6417 if (F->isConstrainedFPIntrinsic())
6425 int SV = cast<ConstantInt>(V)->getSExtValue();
6426 return ConstantInt::get(Ty, neg ? -SV : SV);
6429 // Right-shift a vector by a constant.
6435 int ShiftAmt = cast<ConstantInt>(Shift)->getSExtValue();
6436 int EltSize = VTy->getScalarSizeInBits();
6444 // Right-shifting an unsigned value by its size yields 0.
6447 // Right-shifting a signed value by its size is equivalent
6448 // to a shift of size-1.
6449 --ShiftAmt;
6450 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
7475 // The mangling rules cause us to have one ID for each type for vldap1(q)_lane
7552 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
7571 llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
7574 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
7581 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
7619 // with swapped operands. The table gives us the right intrinsic but we
7628 const Expr *Arg = E->getArg(0);
7629 llvm::Type *ArgTy = CGF.ConvertType(Arg->getType());
7634 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
7636 llvm::Type *ArgTy = ai->getType();
7637 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
7638 ArgTy->getPrimitiveSizeInBits())
7641 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
7645 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType());
7651 llvm::Type *ResultType = CGF.ConvertType(E->getType());
7652 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
7653 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
7665 const Expr *Arg = E->getArg(E->getNumArgs() - 1);
7667 Arg->getIntegerConstantExpr(getContext());
7672 NeonTypeFlags Type(NeonTypeConst->getZExtValue());
7685 auto getAlignmentValue32 = [&](Address addr) -> Value* {
7699 auto NumElements = VTy->getElementCount();
7711 if (VTy->getElementType()->isFloatingPointTy() &&
7717 if (VTy->getElementType()->isFloatingPointTy())
7739 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
7756 switch (VTy->getScalarSizeInBits()) {
7768 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
7795 // We generate target-independent intrinsic, which needs a second argument
7916 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
7922 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
7924 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
7980 PtrOp0 = PtrOp0.withElementType(VTy->getElementType());
7994 for (unsigned I = 2; I < Ops.size() - 1; ++I)
8026 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
8030 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
8037 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
8040 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
8059 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
8060 RTy->getNumElements() * 2);
8087 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
8198 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
8212 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8239 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8261 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8276 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8283 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8290 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8297 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8304 llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
8311 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8317 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8324 llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
8331 llvm::FixedVectorType::get(BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
8349 llvm::Type *ResultType = ConvertType(E->getType());
8350 // AArch64 intrinsic one-element vector type cast to
8358 llvm::Type *OTy = Op->getType();
8362 // currently gives us exactly the same calls for vceqz_f32 and vceqz_s32
8365 OTy = BI->getOperand(0)->getType();
8368 if (OTy->getScalarType()->isFloatingPointTy()) {
8389 auto *TblTy = cast<llvm::FixedVectorType>(Ops[0]->getType());
8390 for (unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
8395 int PairPos = 0, End = Ops.size() - 1;
8403 // If there's an odd number of 64-bit lookup table, fill the high 64-bit
8404 // of the 128-bit lookup table with zero.
8488 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64) ||
8489 RegisterType->isIntegerTy(128)) &&
8497 const Expr *SysRegStrExpr = E->getArg(0)->IgnoreParenCasts();
8498 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
8507 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
8508 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
8509 && "Can't fit 64-bit value in 32-bit register");
8523 if (ValueType->isPointerTy())
8531 llvm::Value *ArgValue = CGF.EmitScalarExpr(E->getArg(1));
8538 if (ValueType->isPointerTy()) {
8603 if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
8619 Value *Option = EmitScalarExpr(E->getArg(0));
8624 Value *Address = EmitScalarExpr(E->getArg(0));
8625 Value *RW = EmitScalarExpr(E->getArg(1));
8626 Value *IsData = EmitScalarExpr(E->getArg(2));
8631 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
8636 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
8638 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
8643 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
8644 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Arg->getType());
8653 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
8657 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
8663 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
8664 const FunctionDecl *FD = E->getDirectCallee();
8667 Ops[i] = EmitScalarExpr(E->getArg(i));
8668 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
8670 StringRef Name = FD->getName();
8695 Value *Coproc = EmitScalarExpr(E->getArg(0));
8696 Value *Opc1 = EmitScalarExpr(E->getArg(1));
8697 Value *RtAndRt2 = EmitScalarExpr(E->getArg(2));
8698 Value *CRm = EmitScalarExpr(E->getArg(3));
8722 Value *Coproc = EmitScalarExpr(E->getArg(0));
8723 Value *Opc1 = EmitScalarExpr(E->getArg(1));
8724 Value *CRm = EmitScalarExpr(E->getArg(2));
8739 return Builder.CreateBitCast(RtAndRt2, ConvertType(E->getType()));
8745 getContext().getTypeSize(E->getType()) == 64) ||
8761 Value *LdPtr = EmitScalarExpr(E->getArg(0));
8772 return Builder.CreateBitCast(Val, ConvertType(E->getType()));
8777 Value *LoadAddr = EmitScalarExpr(E->getArg(0));
8779 QualType Ty = E->getType();
8789 Val->addParamAttr(
8792 if (RealResTy->isPointerTy())
8805 getContext().getTypeSize(E->getArg(0)->getType()) == 64)) {
8811 Address Tmp = CreateMemTemp(E->getArg(0)->getType());
8812 Value *Val = EmitScalarExpr(E->getArg(0));
8820 Value *StPtr = EmitScalarExpr(E->getArg(1));
8826 Value *StoreVal = EmitScalarExpr(E->getArg(0));
8827 Value *StoreAddr = EmitScalarExpr(E->getArg(1));
8829 QualType Ty = E->getArg(0)->getType();
8833 if (StoreVal->getType()->isPointerTy())
8838 CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
8846 StoreAddr->getType());
8849 CI->addParamAttr(
8879 Value *Arg0 = EmitScalarExpr(E->getArg(0));
8880 Value *Arg1 = EmitScalarExpr(E->getArg(1));
8953 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
8958 BuiltinID = It->second;
8967 auto getAlignmentValue32 = [&](Address addr) -> Value* {
8975 unsigned NumArgs = E->getNumArgs() - (HasExtraArg ? 1 : 0);
9003 PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
9030 PtrOp1 = EmitPointerWithAlignment(E->getArg(1));
9059 Value *Arg = EmitScalarExpr(E->getArg(0));
9060 llvm::Type *Tys[] = {Arg->getType()};
9110 const Expr *Arg = E->getArg(E->getNumArgs()-1);
9112 Arg->getIntegerConstantExpr(getContext());
9126 bool usgn = Result->getZExtValue() == 1;
9135 NeonTypeFlags Type = Result->getZExtValue();
9153 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
9154 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
9160 // Handle 64-bit integer elements as a special case. Use shuffles of
9161 // one-element vectors to avoid poor code for i64 in the backend.
9162 if (VTy->getElementType()->isIntegerTy(64)) {
9165 int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
9166 Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
9168 // Load the value as a one-element vector.
9169 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
9175 int Indices[] = {1 - Lane, Lane};
9181 PtrOp0 = PtrOp0.withElementType(VTy->getElementType());
9230 // Handle 64-bit integer elements as a special case. Use a shuffle to get
9231 // a one-element vector and avoid poor code for i64 in the backend.
9232 if (VTy->getElementType()->isIntegerTy(64)) {
9237 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
9246 PtrOp0.withElementType(Ops[1]->getType()));
9277 return E->getIntegerConstantExpr(Context)->getExtValue();
9282 // Helper function called by Tablegen-constructed ARM MVE builtin codegen,
9294 unsigned LaneBits = cast<llvm::VectorType>(V->getType())
9295 ->getElementType()
9296 ->getPrimitiveSizeInBits();
9302 return llvm::Constant::getNullValue(V->getType());
9304 --Shift;
9310 // MVE-specific helper function for a vector splat, which infers the element
9313 unsigned Elements = 128 / V->getType()->getPrimitiveSizeInBits();
9321 // Convert one MVE vector type into another by reinterpreting its in-register
9324 // Little-endian, this is identical to a bitcast (which reinterprets the
9325 // memory format). But big-endian, they're not necessarily the same, because
9329 // We generate a bitcast whenever we can (if we're little-endian, or if the
9332 if (CGF->getTarget().isBigEndian() &&
9333 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
9335 CGF->CGM.getIntrinsic(Intrinsic::arm_mve_vreinterpretq,
9336 {DestType, V->getType()}),
9348 cast<llvm::FixedVectorType>(V->getType())->getNumElements();
9357 assert(V0->getType() == V1->getType() && "Can't zip different vector types");
9360 cast<llvm::FixedVectorType>(V0->getType())->getNumElements();
9370 // MVE-specific helper function to make a vector splat of a constant such as
9372 llvm::Type *T = cast<llvm::VectorType>(VT)->getElementType();
9373 unsigned LaneBits = T->getPrimitiveSizeInBits();
9374 uint32_t Value = HighBit << (LaneBits - 1);
9376 Value |= (1UL << (LaneBits - 1)) - 1;
9384 // MVE-specific helper function which reverses the elements of a
9385 // vector within every (ReverseWidth)-bit collection of lanes.
9387 unsigned LaneSize = V->getType()->getScalarSizeInBits();
9389 unsigned Mask = ReverseWidth / LaneSize - 1;
9422 auto MvecCType = E->getType();
9424 assert(MvecLType->isStructTy() &&
9426 assert(MvecLType->getStructNumElements() == 1 &&
9427 "Return-type struct for vld[24]q should have one element");
9428 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9429 assert(MvecLTypeInner->isArrayTy() &&
9430 "Return-type struct for vld[24]q should contain an array");
9431 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9432 "Array member of return-type struct vld[24]q has wrong length");
9433 auto VecLType = MvecLTypeInner->getArrayElementType();
9437 auto Addr = E->getArg(0);
9439 Tys.push_back(ConvertType(Addr->getType()));
9459 auto Addr = E->getArg(0);
9461 Tys.push_back(ConvertType(Addr->getType()));
9463 auto MvecCType = E->getArg(1)->getType();
9465 assert(MvecLType->isStructTy() && "Data type for vst2q should be a struct");
9466 assert(MvecLType->getStructNumElements() == 1 &&
9467 "Data-type struct for vst2q should have one element");
9468 auto MvecLTypeInner = MvecLType->getStructElementType(0);
9469 assert(MvecLTypeInner->isArrayTy() &&
9470 "Data-type struct for vst2q should contain an array");
9471 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
9472 "Array member of return-type struct vld[24]q has wrong length");
9473 auto VecLType = MvecLTypeInner->getArrayElementType();
9478 EmitAggExpr(E->getArg(1), MvecSlot);
9545 assert(E->getNumArgs() >= 3);
9548 const Expr *Arg = E->getArg(E->getNumArgs() - 1);
9550 Arg->getIntegerConstantExpr(CGF.getContext());
9555 NeonTypeFlags Type = Result->getZExtValue();
9659 /// SVEBuiltinMemEltTy - Returns the memory element type for this memory
9794 unsigned NumElts = SVEBitsPerBlock / EltTy->getScalarSizeInBits();
9803 if (isa<TargetExtType>(Pred->getType()) &&
9804 cast<TargetExtType>(Pred->getType())->getName() == "aarch64.svcount")
9808 if (Pred->getType() == RTy)
9813 switch (VTy->getMinNumElements()) {
9825 IntrinsicTy = Pred->getType();
9831 assert(C->getType() == RTy && "Unexpected return type!");
9843 if (Ops[1]->getType()->isVectorTy())
9845 // map this built-in to an LLVM IR intrinsic, we need both the return type
9847 F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[1]->getType()});
9851 // return type in order to uniquely map this built-in to an LLVM IR
9863 Ops[0], cast<llvm::ScalableVectorType>(F->getArg(0)->getType()));
9869 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
9875 if (!TypeFlags.isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
9877 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
9901 if (Ops[2]->getType()->isVectorTy())
9903 // map this built-in to an LLVM IR intrinsic, we need both the return type
9905 F = CGM.getIntrinsic(IntID, {OverloadedTy, Ops[2]->getType()});
9909 // return type in order to uniquely map this built-in to an LLVM IR
9917 assert(Ops[1]->getType()->isVectorTy() && "Scalar base requires an offset");
9933 Ops[1], cast<llvm::ScalableVectorType>(F->getArg(1)->getType()));
9937 if (!TypeFlags.isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
9939 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
9949 // The gather prefetches are overloaded on the vector input - this can either
9951 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->getType());
9953 OverloadedTy = cast<llvm::ScalableVectorType>(Ops[2]->getType());
9959 if (Ops[1]->getType()->isVectorTy()) {
9969 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
10005 auto RetTy = llvm::VectorType::get(VTy->getElementType(),
10006 VTy->getElementCount() * N);
10017 unsigned MinElts = VTy->getMinNumElements();
10064 for (unsigned I = Ops.size() - N; I < Ops.size(); ++I)
10084 // The pair-wise function has a narrower overloaded type.
10085 Function *F = CGM.getIntrinsic(BuiltinID, Ops[0]->getType());
10116 Function *F = CGM.getIntrinsic(BuiltinID, Predicate->getType());
10125 QualType LangPTy = E->getArg(1)->getType();
10127 LangPTy->castAs<PointerType>()->getPointeeType());
10159 auto TBAAInfo = CGM.getTBAAAccessInfo(LangPTy->getPointeeType());
10172 QualType LangPTy = E->getArg(1)->getType();
10174 LangPTy->castAs<PointerType>()->getPointeeType());
10178 auto VectorTy = cast<llvm::ScalableVectorType>(Ops.back()->getType());
10211 auto TBAAInfo = CGM.getTBAAAccessInfo(LangPTy->getPointeeType());
10283 cast<llvm::VectorType>(Ty)->getElementCount(), Scalar);
10287 return EmitSVEDupX(Scalar, getSVEVectorForElementType(Scalar->getType()));
10292 // intrinsic that is code-generated as a no-op, because the LLVM bitcast
10296 // LLVM CodeGen for a bitcast must add an explicit REV for big-endian.
10322 return {DefaultType, Ops[1]->getType()};
10325 return {getSVEPredType(TypeFlags), Ops[0]->getType()};
10328 return {Ops[0]->getType(), Ops.back()->getType()};
10330 if (TypeFlags.isReductionQV() && !ResultType->isScalableTy() &&
10331 ResultType->isVectorTy())
10332 return {ResultType, Ops[1]->getType()};
10344 unsigned I = cast<ConstantInt>(Ops[1])->getSExtValue();
10346 TypeFlags.isTupleSet() ? Ops[2]->getType() : Ty);
10352 I * SingleVecTy->getMinNumElements());
10364 auto *SrcTy = dyn_cast<llvm::ScalableVectorType>(Ops[0]->getType());
10369 unsigned MinElts = SrcTy->getMinNumElements();
10380 // Multi-vector results should be broken up into a single (wide) result
10382 auto *StructTy = dyn_cast<StructType>(Call->getType());
10386 auto *VTy = dyn_cast<ScalableVectorType>(StructTy->getTypeAtIndex(0U));
10389 unsigned N = StructTy->getNumElements();
10392 bool IsPredTy = VTy->getElementType()->isIntegerTy(1);
10393 unsigned MinElts = IsPredTy ? 16 : VTy->getMinNumElements();
10396 ScalableVectorType::get(VTy->getElementType(), MinElts * N);
10400 assert(SRet->getType() == VTy && "Unexpected type for result value");
10427 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
10429 Value *Arg = EmitScalarExpr(E->getArg(i));
10435 E->getArg(i)->getIntegerConstantExpr(getContext());
10441 *Result = Result->extOrTrunc(32);
10446 if (IsTupleGetOrSet || !isa<ScalableVectorType>(Arg->getType())) {
10451 auto *VTy = cast<ScalableVectorType>(Arg->getType());
10452 unsigned MinElts = VTy->getMinNumElements();
10453 bool IsPred = VTy->getElementType()->isIntegerTy(1);
10454 unsigned N = (MinElts * VTy->getScalarSizeInBits()) / (IsPred ? 16 : 128);
10464 ScalableVectorType::get(VTy->getElementType(), MinElts / N);
10472 llvm::Type *Ty = ConvertType(E->getType());
10475 Value *Val = EmitScalarExpr(E->getArg(0));
10483 SVETypeFlags TypeFlags(Builtin->TypeModifier);
10487 return EmitSVEMaskedLoad(E, Ty, Ops, Builtin->LLVMIntrinsic,
10490 return EmitSVEMaskedStore(E, Ops, Builtin->LLVMIntrinsic);
10492 return EmitSVEGatherLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10494 return EmitSVEScatterStore(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10496 return EmitSVEPrefetchLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10498 return EmitSVEGatherPrefetch(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10500 return EmitSVEStructLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10502 return EmitSVEStructStore(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10509 else if (Builtin->LLVMIntrinsic != 0) {
10525 if (auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10526 if (PredTy->getElementType()->isIntegerTy(1))
10548 llvm::Type *OpndTy = Ops[1]->getType();
10553 Function *F = CGM.getIntrinsic(Builtin->LLVMIntrinsic,
10558 if (auto PredTy = dyn_cast<llvm::VectorType>(Call->getType()))
10559 if (PredTy->getScalarType()->isIntegerTy(1))
10592 bool IsSVCount = isa<TargetExtType>(Ops[0]->getType());
10593 assert(((!IsSVCount || cast<TargetExtType>(Ops[0]->getType())->getName() ==
10603 auto OverloadedTy = getSVEType(SVETypeFlags(Builtin->TypeModifier));
10613 SVETypeFlags TypeFlags(Builtin->TypeModifier);
10621 SVETypeFlags TypeFlags(Builtin->TypeModifier);
10664 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->getType()));
10691 cast<llvm::VectorType>(Ty)->getElementType()->isIntegerTy(1);
10696 llvm::Type *EltTy = Ops[0]->getType();
10717 SVETypeFlags TypeFlags(Builtin->TypeModifier);
10751 SVETypeFlags TF(Builtin->TypeModifier);
10754 llvm::ConstantInt::get(Ty, VTy->getElementCount().getKnownMinValue());
10772 SVETypeFlags TF(Builtin->TypeModifier);
10861 SVETypeFlags TypeFlags(Builtin->TypeModifier);
10865 return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10867 return EmitSMEReadWrite(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10870 return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10875 return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic);
10877 // Handle builtins which require their multi-vector operands to be swapped
10881 if (Builtin->LLVMIntrinsic == 0)
10886 if (auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->getType()))
10887 if (PredTy->getElementType()->isIntegerTy(1))
10892 ? CGM.getIntrinsic(Builtin->LLVMIntrinsic)
10893 : CGM.getIntrinsic(Builtin->LLVMIntrinsic, {getSVEType(TypeFlags)});
10913 unsigned HintID = static_cast<unsigned>(-1);
10941 if (HintID != static_cast<unsigned>(-1)) {
10948 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
10960 CI->setAttributes(Attrs);
10961 CI->setCallingConv(
10965 EmitPointerWithAlignment(E->getArg(0)));
10967 EmitPointerWithAlignment(E->getArg(1)));
10971 assert((getContext().getTypeSize(E->getType()) == 32) &&
10973 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
10975 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
10978 assert((getContext().getTypeSize(E->getType()) == 64) &&
10980 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
10982 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
10987 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
10988 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Arg->getType());
10996 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
11001 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
11008 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
11009 llvm::Type *Ty = Arg->getType();
11016 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
11017 llvm::Type *Ty = Arg->getType();
11024 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
11025 llvm::Type *Ty = Arg->getType();
11032 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
11033 llvm::Type *Ty = Arg->getType();
11039 assert((getContext().getTypeSize(E->getType()) == 32) &&
11041 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
11050 llvm::Value *MemAddr = EmitScalarExpr(E->getArg(0));
11051 llvm::Value *ValPtr = EmitScalarExpr(E->getArg(1));
11101 Address MemAddress = EmitPointerWithAlignment(E->getArg(0));
11108 assert(E->getNumArgs() == 2 && "__clear_cache takes 2 arguments");
11109 const FunctionDecl *FD = E->getDirectCallee();
11112 Ops[i] = EmitScalarExpr(E->getArg(i));
11113 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
11115 StringRef Name = FD->getName();
11121 getContext().getTypeSize(E->getType()) == 128) {
11127 Value *LdPtr = EmitScalarExpr(E->getArg(0));
11139 return Builder.CreateBitCast(Val, ConvertType(E->getType()));
11142 Value *LoadAddr = EmitScalarExpr(E->getArg(0));
11144 QualType Ty = E->getType();
11155 Val->addParamAttr(
11158 if (RealResTy->isPointerTy())
11169 getContext().getTypeSize(E->getArg(0)->getType()) == 128) {
11176 Address Tmp = CreateMemTemp(E->getArg(0)->getType());
11177 EmitAnyExprToMem(E->getArg(0), Tmp, Qualifiers(), /*init*/ true);
11184 Value *StPtr = EmitScalarExpr(E->getArg(1));
11190 Value *StoreVal = EmitScalarExpr(E->getArg(0));
11191 Value *StoreAddr = EmitScalarExpr(E->getArg(1));
11193 QualType Ty = E->getArg(0)->getType();
11197 if (StoreVal->getType()->isPointerTy())
11202 CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
11211 StoreAddr->getType());
11213 CI->addParamAttr(
11220 if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
11238 if (!E->getArg(0)->EvaluateAsInt(Result, CGM.getContext()))
11242 return Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0))});
11276 Value *Arg0 = EmitScalarExpr(E->getArg(0));
11277 Value *Arg1 = EmitScalarExpr(E->getArg(1));
11280 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
11288 Value *Dst = EmitScalarExpr(E->getArg(0));
11289 Value *Val = EmitScalarExpr(E->getArg(1));
11290 Value *Size = EmitScalarExpr(E->getArg(2));
11316 llvm::Type *T = ConvertType(E->getType());
11319 Value *Pointer = EmitScalarExpr(E->getArg(0));
11320 Value *Mask = EmitScalarExpr(E->getArg(1));
11329 Value *Pointer = EmitScalarExpr(E->getArg(0));
11330 Value *TagOffset = EmitScalarExpr(E->getArg(1));
11339 Value *Pointer = EmitScalarExpr(E->getArg(0));
11340 Value *ExcludedMask = EmitScalarExpr(E->getArg(1));
11351 Value *TagAddress = EmitScalarExpr(E->getArg(0));
11361 Value *TagAddress = EmitScalarExpr(E->getArg(0));
11367 Value *PointerA = EmitScalarExpr(E->getArg(0));
11368 Value *PointerB = EmitScalarExpr(E->getArg(1));
11425 E->getArg(0)->EvaluateKnownConstInt(getContext()).getZExtValue();
11449 llvm::Value *ArgValue = EmitScalarExpr(E->getArg(1));
11467 llvm::Type *ResType = ConvertType(E->getType());
11472 Builder.CreateIntCast(EmitScalarExpr(E->getArg(0)), Int128Ty, IsSigned);
11474 Builder.CreateIntCast(EmitScalarExpr(E->getArg(1)), Int128Ty, IsSigned);
11504 Value *Offset = Builder.CreateZExt(EmitScalarExpr(E->getArg(0)), Int64Ty);
11506 Value *Val = EmitScalarExpr(E->getArg(1));
11515 llvm::Type *IntTy = ConvertType(E->getType());
11528 Value *Offset = Builder.CreateZExt(EmitScalarExpr(E->getArg(0)), Int64Ty);
11538 Value *Arg = EmitScalarExpr(E->getArg(0));
11539 llvm::Type *RetTy = ConvertType(E->getType());
11547 Value *Arg = EmitScalarExpr(E->getArg(0));
11548 llvm::Type *ArgType = Arg->getType();
11565 Value *Arg = EmitScalarExpr(E->getArg(0));
11579 Value *ArgValue = EmitScalarExpr(E->getArg(0));
11580 llvm::Type *ArgType = ArgValue->getType();
11590 Value *Address = EmitScalarExpr(E->getArg(0));
11594 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
11600 Builder.CreateCall(F, {EmitScalarExpr(E->getArg(0))});
11613 // Some intrinsics are equivalent - if they are use the base intrinsic ID.
11618 BuiltinID = It->second;
11629 for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) {
11648 PtrOp0 = EmitPointerWithAlignment(E->getArg(0));
11661 Ops.push_back(EmitScalarExpr(E->getArg(E->getNumArgs() - 1)));
11667 const Expr *Arg = E->getArg(E->getNumArgs()-1);
11670 Arg->getIntegerConstantExpr(getContext()))
11672 Type = NeonTypeFlags(Result->getZExtValue());
11677 // Handle non-overloaded intrinsics first.
11681 Ops.push_back(EmitScalarExpr(E->getArg(0)));
11685 Ops.push_back(EmitScalarExpr(E->getArg(1)));
11694 Value *Ptr = EmitScalarExpr(E->getArg(0));
11700 return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->getArg(1)), Ptr);
11708 Ops.push_back(EmitScalarExpr(E->getArg(0)));
11709 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
11725 Ops.push_back(EmitScalarExpr(E->getArg(0)));
11728 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
11730 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
11753 Ops.push_back(EmitScalarExpr(E->getArg(0)));
11788 Ops.push_back(EmitScalarExpr(E->getArg(1)));
11809 Ops.push_back(EmitScalarExpr(E->getArg(1)));
11826 Ops.push_back(EmitScalarExpr(E->getArg(1)));
11842 Value *Vec = EmitScalarExpr(E->getArg(0));
11854 Value *Vec = EmitScalarExpr(E->getArg(0));
11866 Value *Vec = EmitScalarExpr(E->getArg(0));
11880 Ops.push_back(EmitScalarExpr(E->getArg(0)));
11882 Ops[0], ConvertType(E->getCallReturnType(getContext())),
11888 Ops.push_back(EmitScalarExpr(E->getArg(0)));
11890 Ops[0], ConvertType(E->getCallReturnType(getContext())),
11896 Ops.push_back(EmitScalarExpr(E->getArg(0)));
11898 Ops[0], ConvertType(E->getCallReturnType(getContext())),
11904 Ops.push_back(EmitScalarExpr(E->getArg(0)));
11906 Ops[0], ConvertType(E->getCallReturnType(getContext())),
11912 Ops.push_back(EmitScalarExpr(E->getArg(0)));
11914 Ops[0], ConvertType(E->getCallReturnType(getContext())),
11918 Ops.push_back(EmitScalarExpr(E->getArg(0)));
11938 Ops.push_back(EmitScalarExpr(E->getArg(1)));
11961 Ops.push_back(EmitScalarExpr(E->getArg(1)));
11984 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12017 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12025 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12045 Ops.push_back(EmitScalarExpr(E->getArg(2)));
12051 Ops.push_back(EmitScalarExpr(E->getArg(2)));
12057 Ops.push_back(EmitScalarExpr(E->getArg(2)));
12064 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12070 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12076 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12082 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12088 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12093 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12099 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12105 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12110 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12116 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12121 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12126 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12132 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12138 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12141 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12144 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12147 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12150 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12156 {EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2)), Ops[0]});
12158 Value* Neg = Builder.CreateFNeg(EmitScalarExpr(E->getArg(1)), "vsubh");
12163 {Neg, EmitScalarExpr(E->getArg(2)), Ops[0]});
12167 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->getArg(1)), "vaddd");
12170 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->getArg(1)), "vsubd");
12175 ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->getArg(2))));
12188 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12198 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12207 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12208 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
12209 Ops[1] = ConstantInt::get(Int64Ty, -SV);
12218 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->getArg(2))));
12225 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
12227 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()), "shld_n");
12230 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
12233 Amt->getZExtValue())),
12237 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
12238 uint64_t ShiftAmt = Amt->getZExtValue();
12239 // Right-shifting an unsigned value by its size yields 0.
12246 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2)));
12249 Amt->getZExtValue())),
12254 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->getArg(2)));
12255 uint64_t ShiftAmt = Amt->getZExtValue();
12256 // Right-shifting an unsigned value by its size yields 0.
12268 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
12290 ProductOps.push_back(EmitScalarExpr(E->getArg(2)));
12304 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->getArg(3)),
12323 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12329 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->getArg(1)),
12336 Value *Val = EmitScalarExpr(E->getArg(1));
12356 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
12357 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
12391 ? llvm::FixedVectorType::get(VTy->getElementType(),
12392 VTy->getNumElements() / 2)
12395 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
12407 if (VTy && VTy->getElementType() == DoubleTy) {
12423 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
12424 VTy->getNumElements() * 2);
12426 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
12450 Ops.push_back(EmitScalarExpr(E->getArg(3)));
12451 llvm::Type *Ty = ConvertType(E->getCallReturnType(getContext()));
12466 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
12469 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12477 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
12480 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12488 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
12492 unsigned ArgElts = VTy->getNumElements();
12493 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
12494 unsigned BitWidth = EltTy->getBitWidth();
12503 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
12510 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
12516 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
12523 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12531 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12535 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12540 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12544 Ops.push_back(EmitScalarExpr(E->getArg(1)));
12563 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12577 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12584 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12598 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12612 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12619 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12633 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12647 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12657 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12665 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12673 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12681 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12804 Ops.push_back(EmitScalarExpr(E->getArg(2)));
12825 return Builder.CreateNeg(EmitScalarExpr(E->getArg(0)), "vnegd");
12827 return Builder.CreateFNeg(EmitScalarExpr(E->getArg(0)), "vnegh");
12839 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12867 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12879 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12891 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12903 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12912 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12921 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12930 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12939 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12948 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12957 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12966 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12975 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12984 Ops.push_back(EmitScalarExpr(E->getArg(0)));
12993 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13002 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13011 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13020 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13029 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13038 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13047 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13056 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13065 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13074 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13083 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13092 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13101 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13110 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13119 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13125 Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->getArg(1)), DoubleTy);
13133 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13142 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13150 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13159 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13167 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13176 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13184 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13193 Ops.push_back(EmitScalarExpr(E->getArg(0)));
13235 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
13243 VTy->getElementType(), Ops[0], PtrOp0.getAlignment());
13244 LI->setAtomic(llvm::AtomicOrdering::Acquire);
13251 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
13268 SI->setAtomic(llvm::AtomicOrdering::Release);
13315 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13326 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13338 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
13352 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
13360 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13367 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
13375 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13382 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
13390 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
13402 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13420 for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
13437 for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
13509 const Expr *Arg = E->getArg(0);
13510 bool IsBitField = Arg->IgnoreParens()->getObjectKind() == OK_BitField;
13513 CGM.Error(E->getExprLoc(),
13514 "using __builtin_preserve_field_info() without -g");
13526 ConstantInt *C = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
13527 Value *InfoKind = ConstantInt::get(Int64Ty, C->getSExtValue());
13532 {FieldAddr->getType()});
13538 CGM.Error(E->getExprLoc(), "using builtin function without -g");
13542 const Expr *Arg0 = E->getArg(0);
13543 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(
13544 Arg0->getType(), Arg0->getExprLoc());
13546 ConstantInt *Flag = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
13547 Value *FlagValue = ConstantInt::get(Int64Ty, Flag->getSExtValue());
13558 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
13563 CGM.Error(E->getExprLoc(), "using builtin function without -g");
13567 const Expr *Arg0 = E->getArg(0);
13568 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(
13569 Arg0->getType(), Arg0->getExprLoc());
13572 const auto *UO = cast<UnaryOperator>(Arg0->IgnoreParens());
13573 const auto *CE = cast<CStyleCastExpr>(UO->getSubExpr());
13574 const auto *DR = cast<DeclRefExpr>(CE->getSubExpr());
13575 const auto *Enumerator = cast<EnumConstantDecl>(DR->getDecl());
13577 auto InitVal = Enumerator->getInitVal();
13583 std::string EnumStr = Enumerator->getNameAsString() + ":" + InitValStr;
13586 ConstantInt *Flag = cast<ConstantInt>(EmitScalarExpr(E->getArg(1)));
13587 Value *FlagValue = ConstantInt::get(Int64Ty, Flag->getSExtValue());
13594 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
13602 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
13603 "Not a power-of-two sized vector!");
13618 llvm::FixedVectorType::get(Ops[0]->getType(), Ops.size()));
13632 cast<IntegerType>(Mask->getType())->getBitWidth());
13653 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements());
13660 llvm::Type *Ty = Ops[1]->getType();
13664 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements());
13671 auto *ResultTy = cast<llvm::VectorType>(Ops[1]->getType());
13675 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements());
13685 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
13687 Value *MaskVec = getMaskVecValue(CGF, Ops[2], ResultTy->getNumElements());
13697 auto *ResultTy = cast<llvm::FixedVectorType>(Ops[1]->getType());
13700 Value *MaskVec = getMaskVecValue(CGF, Ops[2], ResultTy->getNumElements());
13710 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
13718 Ops[0]->getType());
13723 llvm::Type *Ty = Op0->getType();
13726 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
13728 if (Amt->getType() != Ty) {
13729 unsigned NumElts = cast<llvm::FixedVectorType>(Ty)->getNumElements();
13730 Amt = CGF.Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
13743 llvm::Type *Ty = Op0->getType();
13744 uint64_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
13784 if (C->isAllOnesValue())
13788 CGF, Mask, cast<llvm::FixedVectorType>(Op0->getType())->getNumElements());
13797 if (C->isAllOnesValue())
13801 CGF.Builder.getInt1Ty(), Mask->getType()->getIntegerBitWidth());
13811 if (!C || !C->isAllOnesValue())
13822 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
13835 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
13866 Value *Zero = Constant::getNullValue(In->getType());
13872 unsigned Rnd = cast<llvm::ConstantInt>(Ops[3])->getZExtValue();
13873 llvm::Type *Ty = Ops[1]->getType();
13879 Function *F = CGF.CGM.getIntrinsic(IID, { Ty, Ops[0]->getType() });
13958 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4 ||
13963 llvm::Type *Ty = A->getType();
13992 MaskFalseVal = Constant::getNullValue(Ops[0]->getType());
14022 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
14034 switch (Ops[0]->getType()->getPrimitiveSizeInBits()) {
14052 Intrinsic::experimental_constrained_fma, Ops[0]->getType());
14055 Function *FMA = CGF.CGM.getIntrinsic(Intrinsic::fma, Ops[0]->getType());
14060 Value *PassThru = ZeroMask ? Constant::getNullValue(Res->getType())
14076 llvm::Type *Ty = Ops[0]->getType();
14079 Ty->getPrimitiveSizeInBits() / 64);
14105 llvm::Type *Ty = Ops[0]->getType();
14107 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
14108 unsigned EltWidth = Ty->getScalarSizeInBits();
14134 cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14140 const Expr *CPUExpr = E->getArg(0)->IgnoreParenCasts();
14141 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
14153 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) {
14159 unsigned NumDstElts = cast<llvm::FixedVectorType>(DstTy)->getNumElements();
14164 cast<llvm::FixedVectorType>(Src->getType())->getNumElements()) {
14174 // Perform the fp-extension.
14186 // Matching the struct layout from the compiler-rt/libgcc structure that is
14197 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(true);
14231 const Expr *FeatureExpr = E->getArg(0)->IgnoreParenCasts();
14232 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
14246 // Matching the struct layout from the compiler-rt/libgcc structure that is
14257 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(true);
14277 cast<llvm::GlobalValue>(CpuFeatures2)->setDSOLocal(true);
14282 Value *Idxs[] = {Builder.getInt32(0), Builder.getInt32(i - 1)};
14300 cast<llvm::GlobalValue>(Func.getCallee())->setDSOLocal(true);
14302 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14311 cast<llvm::GlobalValue>(Func.getCallee())->setDSOLocal(true);
14313 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
14318 const Expr *ArgExpr = E->getArg(0)->IgnoreParenCasts();
14319 StringRef ArgStr = cast<StringLiteral>(ArgExpr)->getString();
14344 cast<llvm::GlobalValue>(AArch64CPUFeatures)->setDSOLocal(true);
14382 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
14411 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
14422 Value *RW = ConstantInt::get(Int32Ty, (C->getZExtValue() >> 2) & 0x1);
14423 Value *Locality = ConstantInt::get(Int32Ty, C->getZExtValue() & 0x3);
14425 Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
14456 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
14462 Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType());
14471 // TODO: If we had a "freeze" IR instruction to generate a fixed undef
14473 return llvm::Constant::getNullValue(ConvertType(E->getType()));
14490 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14491 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue();
14492 Index &= NumElts - 1;
14506 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
14507 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
14508 Index &= NumElts - 1;
14515 RawAddress Tmp = CreateMemTemp(E->getArg(0)->getType());
14522 RawAddress Tmp = CreateMemTemp(E->getType());
14613 llvm::Type *ResultType = ConvertType(E->getType());
14629 return EmitX86SExtMask(*this, Ops[0], ConvertType(E->getType()));
14669 Constant::getNullValue(Ops[0]->getType()));
14730 getContext().getTypeAlignInChars(E->getArg(1)->getType()).getAsAlign());
14771 getContext().getTypeAlignInChars(E->getArg(1)->getType()).getAsAlign());
14955 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements(),
14956 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements());
15064 cast<llvm::FixedVectorType>(Ops[2]->getType())->getNumElements(),
15065 cast<llvm::FixedVectorType>(Ops[3]->getType())->getNumElements());
15087 auto *DstTy = cast<llvm::FixedVectorType>(ConvertType(E->getType()));
15088 unsigned NumElts = DstTy->getNumElements();
15090 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15092 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue();
15094 Index &= SubVectors - 1; // Remove any extra bits.
15126 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15128 cast<llvm::FixedVectorType>(Ops[1]->getType())->getNumElements();
15130 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
15132 Index &= SubVectors - 1; // Remove any extra bits.
15144 Indices[i] = (i - Index) + DstNumElts;
15154 Value *Res = Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15161 if (C->isAllOnesValue())
15162 return Builder.CreateTrunc(Ops[0], Ops[1]->getType());
15190 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15191 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15205 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15206 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15207 unsigned NumElts = Ty->getNumElements();
15209 // Splat the 8-bits of immediate 4 times to help the loop wrap around.
15228 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15229 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15230 unsigned NumElts = Ty->getNumElements();
15232 // Splat the 8-bits of immediate 4 times to help the loop wrap around.
15257 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15258 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15259 unsigned NumElts = Ty->getNumElements();
15260 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
15263 // Splat the 8-bits of immediate 4 times to help the loop wrap around.
15283 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15284 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15285 unsigned NumElts = Ty->getNumElements();
15286 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
15289 // Splat the 8-bits of immediate 4 times to help the loop wrap around.
15310 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15311 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15312 unsigned NumElts = Ty->getNumElements();
15314 // These intrinsics operate on 256-bit lanes of four 64-bit elements.
15326 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
15329 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15335 return llvm::Constant::getNullValue(ConvertType(E->getType()));
15340 ShiftVal -= 16;
15342 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
15346 // 256-bit palignr operates on 128-bit lanes so we need to handle that
15351 Idx += NumElts - 16; // End of lane, switch operand.
15366 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15367 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
15370 ShiftVal &= NumElts - 1;
15387 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15388 auto *Ty = cast<llvm::FixedVectorType>(Ops[0]->getType());
15389 unsigned NumElts = Ty->getNumElements();
15390 unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
15412 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
15414 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
15426 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType());
15450 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15451 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
15453 unsigned NumElts = ResultType->getNumElements() * 8;
15460 // 256/512-bit pslldq operates on 128-bit lanes so we need to handle that
15463 unsigned Idx = NumElts + i - ShiftVal;
15464 if (Idx < NumElts) Idx -= NumElts - 16; // end of lane, switch operand.
15474 return Builder.CreateBitCast(SV, Ops[0]->getType(), "cast");
15479 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15480 auto *ResultType = cast<llvm::FixedVectorType>(Ops[0]->getType());
15482 unsigned NumElts = ResultType->getNumElements() * 8;
15489 // 256/512-bit psrldq operates on 128-bit lanes so we need to handle that
15493 if (Idx >= 16) Idx += NumElts - 16; // end of lane, switch operand.
15509 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15510 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15513 return llvm::Constant::getNullValue(Ops[0]->getType());
15519 Indices[i] = NumElts + i - ShiftVal;
15521 Value *Zero = llvm::Constant::getNullValue(In->getType());
15524 return Builder.CreateBitCast(SV, Ops[0]->getType());
15530 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
15531 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15534 return llvm::Constant::getNullValue(Ops[0]->getType());
15542 Value *Zero = llvm::Constant::getNullValue(In->getType());
15545 return Builder.CreateBitCast(SV, Ops[0]->getType());
15564 SI->setMetadata(llvm::LLVMContext::MD_nontemporal, Node);
15565 SI->setAlignment(llvm::Align(1));
15568 // Rotate is a special case of funnel shift - 1st 2 args are the same.
15649 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
15664 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
15683 Value *C = llvm::Constant::getAllOnesValue(Ops[0]->getType());
15685 return Builder.CreateZExt(Cmp, ConvertType(E->getType()));
15692 Value *C = llvm::Constant::getNullValue(Ops[0]->getType());
15694 return Builder.CreateZExt(Cmp, ConvertType(E->getType()));
15734 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15762 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15767 return Builder.CreateBitCast(Res, Ops[0]->getType());
15798 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15801 Ops[0]->getType());
15810 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15812 return Builder.CreateBitCast(Res, Ops[0]->getType());
15818 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
15833 return Builder.CreateBitCast(Res, Ops[0]->getType());
15842 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
15852 A->getType());
15855 F = CGM.getIntrinsic(Intrinsic::sqrt, A->getType());
15863 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
15889 A->getType());
15892 F = CGM.getIntrinsic(Intrinsic::sqrt, A->getType());
15909 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
15934 Ops[0]->getType());
15937 Function *F = CGM.getIntrinsic(Intrinsic::sqrt, Ops[0]->getType());
16021 CGM.getIntrinsic(Intrinsic::vector_reduce_fadd, Ops[1]->getType());
16032 CGM.getIntrinsic(Intrinsic::vector_reduce_fmul, Ops[1]->getType());
16043 CGM.getIntrinsic(Intrinsic::vector_reduce_fmax, Ops[0]->getType());
16054 CGM.getIntrinsic(Intrinsic::vector_reduce_fmin, Ops[0]->getType());
16132 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16179 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16238 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16303 // This is only possible if fp-model is not strict and FENV_ACCESS is off.
16307 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f;
16314 // Predicates for 16-31 repeat the 0-15 predicates. Only the signalling
16336 // Invert the signalling behavior for 16-31.
16396 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16413 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements();
16466 return EmitX86CvtF16ToFloatExpr(*this, Ops, ConvertType(E->getType()));
16473 cast<llvm::FixedVectorType>(Ops[0]->getType())->getNumElements());
16498 Value *FuncId = EmitScalarExpr(E->getArg(1));
16500 ? EmitScalarExpr(E->getArg(2))
16513 // x86-64 uses %rbx as the base register, so preserve it.
16523 Value *BasePtr = EmitScalarExpr(E->getArg(0));
16548 llvm::Type *ResType = ConvertType(E->getType());
16568 Address HighBitsAddress = EmitPointerWithAlignment(E->getArg(2));
16582 // Flip low/high ops and zero-extend amount to matching type.
16583 // shiftleft128(Low, High, Amt) -> fshl(High, Low, Amt)
16584 // shiftright128(Low, High, Amt) -> fshr(High, Low, Amt)
16602 // We treat __stosb as a volatile memset - it may not generate "rep stosb"
16618 CI->setAttributes(NoReturnAttr);
16625 llvm::Type *IntTy = ConvertType(E->getType());
16629 IntTy, Ptr, getContext().getTypeAlignInChars(E->getType()));
16630 Load->setVolatile(true);
16637 llvm::Type *IntTy = ConvertType(E->getType());
16641 IntTy, Ptr, getContext().getTypeAlignInChars(E->getType()));
16642 Load->setVolatile(true);
16702 createBasicBlock(BlockName + "_no_error", this->CurFn);
16703 BasicBlock *Error = createBasicBlock(BlockName + "_error", this->CurFn);
16704 BasicBlock *End = createBasicBlock(BlockName + "_end", this->CurFn);
16716 Constant *Zero = llvm::Constant::getNullValue(Out->getType());
16759 createBasicBlock(BlockName + "_no_error", this->CurFn);
16760 BasicBlock *Error = createBasicBlock(BlockName + "_error", this->CurFn);
16761 BasicBlock *End = createBasicBlock(BlockName + "_end", this->CurFn);
16770 Value *Ptr = Builder.CreateConstGEP1_32(Extract->getType(), Ops[0], i);
16778 Constant *Zero = llvm::Constant::getNullValue(Out->getType());
16779 Value *Ptr = Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i);
16819 CGM.getIntrinsic(Intrinsic::prefetch, Ops[0]->getType()),
16832 // return Builder.CreateFDiv(EmitScalarExpr(E->getArg(0)),
16833 // EmitScalarExpr(E->getArg(1)), "swdiv");
16835 // Value *Op0 = EmitScalarExpr(E->getArg(0));
16836 // Value *Op1 = EmitScalarExpr(E->getArg(1));
16844 unsigned OpValue) -> Value * {
16846 return llvm::ConstantInt::getFalse(ConvertType(E->getType()));
16849 return llvm::ConstantInt::getTrue(ConvertType(E->getType()));
16881 llvm::Type *ValueType = FieldValue->getType();
16882 bool IsValueType64Bit = ValueType->isIntegerTy(64);
16884 (IsValueType64Bit || ValueType->isIntegerTy(32)) &&
16885 "Only 32/64-bit integers are supported in GenAIXPPCBuiltinCpuExpr().");
16896 const Expr *CPUExpr = E->getArg(0)->IgnoreParenCasts();
16897 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
16926 return llvm::ConstantInt::getFalse(ConvertType(E->getType()));
16936 const Expr *CPUExpr = E->getArg(0)->IgnoreParenCasts();
16937 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
16979 // __builtin_ppc_get_timebase is GCC 4.8+'s PowerPC-specific name for what we
17000 Ops.push_back(EmitScalarExpr(E->getArg(0)));
17001 Ops.push_back(EmitScalarExpr(E->getArg(1)));
17068 Ops.push_back(EmitScalarExpr(E->getArg(0)));
17069 Ops.push_back(EmitScalarExpr(E->getArg(1)));
17070 Ops.push_back(EmitScalarExpr(E->getArg(2)));
17120 Value *Op0 = EmitScalarExpr(E->getArg(0));
17121 Value *Op1 = EmitScalarExpr(E->getArg(1));
17122 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
17123 llvm::Type *ResTy = ConvertType(E->getType());
17136 RevMask.push_back(15 - Idx);
17145 Int8Ty, Op0, ConstantInt::get(Op1->getType(), NumBytes - 1));
17153 Constant *Zero = llvm::Constant::getNullValue(IsLE ? ResTy : AllElts->getType());
17158 int Val = (NumBytes - Idx - 1 >= 0) ? (NumBytes - Idx - 1)
17159 : 16 - (NumBytes - Idx);
17173 Value *Op0 = EmitScalarExpr(E->getArg(0));
17174 Value *Op1 = EmitScalarExpr(E->getArg(1));
17175 Value *Op2 = EmitScalarExpr(E->getArg(2));
17176 int64_t NumBytes = cast<ConstantInt>(Op1)->getZExtValue();
17186 RevMask.push_back(15 - Idx);
17190 StVec, Address(Op0, Op2->getType(), CharUnits::fromQuantity(1)));
17232 Result = StoreSubVec(8, NumBytes - 8, IsLE ? 0 : 1);
17233 RemainingBytes -= 8;
17237 Result = StoreSubVec(4, NumBytes - Stored - 4,
17238 IsLE ? (Stored >> 2) : 3 - (Stored >> 2));
17239 RemainingBytes -= 4;
17243 Result = StoreSubVec(2, NumBytes - Stored - 2,
17244 IsLE ? (Stored >> 1) : 7 - (Stored >> 1));
17245 RemainingBytes -= 2;
17250 StoreSubVec(1, NumBytes - Stored - 1, IsLE ? Stored : 15 - Stored);
17256 llvm::Type *ResultType = ConvertType(E->getType());
17257 Value *X = EmitScalarExpr(E->getArg(0));
17272 llvm::Type *ResultType = ConvertType(E->getType());
17273 Value *X = EmitScalarExpr(E->getArg(0));
17282 llvm::Type *ResultType = ConvertType(E->getType());
17283 Value *X = EmitScalarExpr(E->getArg(0));
17292 llvm::Type *ResultType = ConvertType(E->getType());
17293 Value *Op0 = EmitScalarExpr(E->getArg(0));
17294 Value *Op1 = EmitScalarExpr(E->getArg(1));
17295 Value *Op2 = EmitScalarExpr(E->getArg(2));
17317 int64_t ConstArg = ArgCI->getSExtValue();
17327 CGM.Error(E->getExprLoc(), RangeErrMsg);
17334 ConstArg = (Is32bit ? 12 : 8) - ConstArg;
17352 llvm::Type *ResultType = ConvertType(E->getType());
17353 Value *X = EmitScalarExpr(E->getArg(0));
17359 Value *Op0 = EmitScalarExpr(E->getArg(0));
17360 Value *Op1 = EmitScalarExpr(E->getArg(1));
17372 Value *Op0 = EmitScalarExpr(E->getArg(0));
17373 Value *Op1 = EmitScalarExpr(E->getArg(1));
17388 Value *Op0 = EmitScalarExpr(E->getArg(0));
17389 Value *Op1 = EmitScalarExpr(E->getArg(1));
17390 Value *Op2 = EmitScalarExpr(E->getArg(2));
17416 Value *Op0 = EmitScalarExpr(E->getArg(0));
17417 Value *Op1 = EmitScalarExpr(E->getArg(1));
17418 Value *Op2 = EmitScalarExpr(E->getArg(2));
17419 Value *Op3 = EmitScalarExpr(E->getArg(3));
17420 // rldimi is 64-bit instruction, expand the intrinsic before isel to
17424 Function *F = CGM.getIntrinsic(Intrinsic::fshl, Op0->getType());
17437 Value *Op0 = EmitScalarExpr(E->getArg(0));
17438 Value *Op1 = EmitScalarExpr(E->getArg(1));
17439 Value *Op2 = EmitScalarExpr(E->getArg(2));
17445 Value *Op0 = EmitScalarExpr(E->getArg(0));
17446 llvm::Type *ArgType = Op0->getType();
17450 llvm::Type *ResultType = ConvertType(E->getType());
17452 if (Result->getType() != ResultType)
17458 Value *Op0 = EmitScalarExpr(E->getArg(0));
17459 Value *Op1 = EmitScalarExpr(E->getArg(1));
17498 llvm::Type *ResultType = ConvertType(E->getType());
17499 Value *X = EmitScalarExpr(E->getArg(0));
17500 Value *Y = EmitScalarExpr(E->getArg(1));
17516 llvm::Type *ResultType = ConvertType(E->getType());
17517 Value *X = EmitScalarExpr(E->getArg(0));
17551 llvm::Type *ResultType = ConvertType(E->getType());
17552 Value *X = EmitScalarExpr(E->getArg(0));
17564 llvm::Type *ResultType = ConvertType(E->getType());
17565 Value *X = EmitScalarExpr(E->getArg(0));
17569 Value *Y = EmitScalarExpr(E->getArg(1));
17581 Value *Op0 = EmitScalarExpr(E->getArg(0));
17582 Value *Op1 = EmitScalarExpr(E->getArg(1));
17584 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
17585 AlignmentCI = ConstantInt::get(AlignmentCI->getIntegerType(),
17588 emitAlignmentAssumption(Op1, E->getArg(1),
17594 Value *Op0 = EmitScalarExpr(E->getArg(0));
17595 Value *Op1 = EmitScalarExpr(E->getArg(1));
17596 Value *Op2 = EmitScalarExpr(E->getArg(2));
17597 llvm::Type *Ty = Op0->getType();
17605 Value *Op0 = EmitScalarExpr(E->getArg(0));
17620 llvm::Type *ResultType = ConvertType(E->getType());
17621 Value *X = EmitScalarExpr(E->getArg(0));
17622 Value *Y = EmitScalarExpr(E->getArg(1));
17623 Value *Z = EmitScalarExpr(E->getArg(2));
17664 return nullptr; // Suppress no-return warning
17668 Value *Op0 = EmitScalarExpr(E->getArg(0));
17669 Value *Op1 = EmitScalarExpr(E->getArg(1));
17670 Value *Op2 = EmitScalarExpr(E->getArg(2));
17679 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
17698 Index = MaxIndex - Index;
17708 Value *Op0 = EmitScalarExpr(E->getArg(0));
17709 Value *Op1 = EmitScalarExpr(E->getArg(1));
17721 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
17725 Index = MaxIndex - Index;
17741 Value *Op0 = EmitScalarExpr(E->getArg(0));
17742 Value *Op1 = EmitScalarExpr(E->getArg(1));
17743 Value *Op2 = EmitScalarExpr(E->getArg(2));
17747 unsigned Index = ArgCI->getZExtValue();
17759 QualType BIRetType = E->getType();
17765 Value *Op0 = EmitScalarExpr(E->getArg(0));
17766 Value *Op1 = EmitScalarExpr(E->getArg(1));
17767 Value *Op2 = EmitScalarExpr(E->getArg(2));
17770 unsigned Index = ArgCI->getZExtValue() & 0x3;
17780 // Little endian element N comes from element 8+N-Index of the
17783 ElemIdx0 = (8 - Index) % 8;
17784 ElemIdx1 = (9 - Index) % 8;
17785 ElemIdx2 = (10 - Index) % 8;
17786 ElemIdx3 = (11 - Index) % 8;
17797 QualType BIRetType = E->getType();
17803 Value *Op0 = EmitScalarExpr(E->getArg(0));
17804 Value *Op1 = EmitScalarExpr(E->getArg(1));
17807 llvm::PoisonValue::get(llvm::FixedVectorType::get(Op0->getType(), 2));
17812 return Builder.CreateBitCast(Res, ConvertType(E->getType()));
17816 Value *Op0 = EmitScalarExpr(E->getArg(0));
17817 Value *Op1 = EmitScalarExpr(E->getArg(1));
17820 Op0, llvm::FixedVectorType::get(ConvertType(E->getType()), 2));
17824 ConstantInt::get(Index->getIntegerType(), 1 - Index->getZExtValue());
17831 Value *Op0 = EmitScalarExpr(E->getArg(0));
17832 Value *Op1 = Builder.CreateSExt(EmitScalarExpr(E->getArg(1)), Int32Ty);
17847 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++)
17848 if (E->getArg(i)->getType()->isArrayType())
17850 EmitArrayToPointerDecay(E->getArg(i)).emitRawPointer(*this));
17852 Ops.push_back(EmitScalarExpr(E->getArg(i)));
17867 Address Addr = EmitPointerWithAlignment(E->getArg(1));
17916 Address Addr = EmitPointerWithAlignment(E->getArg(0));
17929 Address Addr = EmitPointerWithAlignment(E->getArg(0));
17930 Address OldValAddr = EmitPointerWithAlignment(E->getArg(1));
17932 QualType AtomicTy = E->getArg(0)->getType()->getPointeeType();
17934 Value *Op2 = EmitScalarExpr(E->getArg(2));
17936 LV, RValue::get(OldVal), RValue::get(Op2), E->getExprLoc(),
17939 // https://www.ibm.com/docs/en/xl-c-and-cpp-aix/16.1?topic=functions-compare-swap-compare-swaplp
17976 Value *Op0 = EmitScalarExpr(E->getArg(0));
17984 Value *Op0 = EmitScalarExpr(E->getArg(0));
17985 Value *Op1 = EmitScalarExpr(E->getArg(1));
17993 Value *ArgValue = EmitScalarExpr(E->getArg(0));
17994 llvm::Type *ArgType = ArgValue->getType();
18001 Value *Op0 = EmitScalarExpr(E->getArg(0));
18002 Value *Op1 = EmitScalarExpr(E->getArg(1));
18010 Value *Op0 = EmitScalarExpr(E->getArg(0));
18011 Value *Op1 = EmitScalarExpr(E->getArg(1));
18054 Value *Op0 = EmitScalarExpr(E->getArg(0));
18055 Value *Op1 = EmitScalarExpr(E->getArg(1));
18057 CGM.getIntrinsic(Intrinsic::ppc_test_data_class, Op0->getType()),
18061 Value *Op0 = EmitScalarExpr(E->getArg(0));
18062 Value *Op1 = EmitScalarExpr(E->getArg(1));
18063 Value *Op2 = EmitScalarExpr(E->getArg(2));
18064 Value *Op3 = EmitScalarExpr(E->getArg(3));
18069 Value *Op0 = EmitScalarExpr(E->getArg(0));
18070 Value *Op1 = EmitScalarExpr(E->getArg(1));
18071 Value *Op2 = EmitScalarExpr(E->getArg(2));
18072 Value *Op3 = EmitScalarExpr(E->getArg(3));
18077 Value *Op0 = EmitScalarExpr(E->getArg(0));
18078 Value *Op1 = EmitScalarExpr(E->getArg(1));
18079 Value *Op2 = EmitScalarExpr(E->getArg(2));
18080 Value *Op3 = EmitScalarExpr(E->getArg(3));
18085 Value *Op0 = EmitScalarExpr(E->getArg(0));
18086 Value *Op1 = EmitScalarExpr(E->getArg(1));
18087 Value *Op2 = EmitScalarExpr(E->getArg(2));
18088 Value *Op3 = EmitScalarExpr(E->getArg(3));
18093 Value *Op0 = EmitScalarExpr(E->getArg(0));
18094 Value *Op1 = EmitScalarExpr(E->getArg(1));
18095 Value *Op2 = EmitScalarExpr(E->getArg(2));
18096 Value *Op3 = EmitScalarExpr(E->getArg(3));
18101 Value *Op0 = EmitScalarExpr(E->getArg(0));
18102 Value *Op1 = EmitScalarExpr(E->getArg(1));
18103 Value *Op2 = EmitScalarExpr(E->getArg(2));
18104 Value *Op3 = EmitScalarExpr(E->getArg(3));
18110 Value *Op0 = EmitScalarExpr(E->getArg(0));
18111 Value *Op1 = EmitScalarExpr(E->getArg(1));
18116 {EmitScalarExpr(E->getArg(0))});
18129 Call->addRetAttr(
18130 Attribute::getWithDereferenceableBytes(Call->getContext(), 64));
18131 Call->addRetAttr(Attribute::getWithAlignment(Call->getContext(), Align(4)));
18134 QualType BuiltinRetType = E->getType();
18136 if (RetTy == Call->getType())
18144 Call->addRetAttr(
18145 Attribute::getWithDereferenceableBytes(Call->getContext(), 256));
18146 Call->addRetAttr(Attribute::getWithAlignment(Call->getContext(), Align(8)));
18156 /// compiling device libraries in an ABI-agnostic way.
18214 LD->setMetadata(llvm::LLVMContext::MD_range, RNode);
18215 LD->setMetadata(llvm::LLVMContext::MD_noundef,
18217 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
18231 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
18239 // \p Order takes a C++11 comptabile memory-ordering specifier and converts
18246 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
18277 int scope = cast<llvm::ConstantInt>(Scope)->getZExtValue();
18305 Arg = EmitScalarExpr(E->getArg(Idx));
18310 E->getArg(Idx)->getIntegerConstantExpr(getContext());
18318 if (QT->hasFloatingRepresentation()) {
18328 if (QT->hasSignedIntegerRepresentation())
18331 assert(QT->hasUnsignedIntegerRepresentation());
18342 Value *Op0 = EmitScalarExpr(E->getArg(0));
18349 Value *Op0 = EmitScalarExpr(E->getArg(0));
18356 Value *OpX = EmitScalarExpr(E->getArg(0));
18357 Value *OpMin = EmitScalarExpr(E->getArg(1));
18358 Value *OpMax = EmitScalarExpr(E->getArg(2));
18360 QualType Ty = E->getArg(0)->getType();
18362 if (auto *VecTy = Ty->getAs<VectorType>())
18363 Ty = VecTy->getElementType();
18364 IsUnsigned = Ty->isUnsignedIntegerType();
18366 /*ReturnType=*/OpX->getType(),
18371 Value *Op0 = EmitScalarExpr(E->getArg(0));
18372 Value *Op1 = EmitScalarExpr(E->getArg(1));
18373 llvm::Type *T0 = Op0->getType();
18374 llvm::Type *T1 = Op1->getType();
18375 if (!T0->isVectorTy() && !T1->isVectorTy()) {
18376 if (T0->isFloatingPointTy())
18379 if (T0->isIntegerTy())
18387 assert(T0->isVectorTy() && T1->isVectorTy() &&
18391 assert(T0->getScalarType() == T1->getScalarType() &&
18394 auto *VecTy0 = E->getArg(0)->getType()->getAs<VectorType>();
18396 E->getArg(1)->getType()->getAs<VectorType>();
18398 assert(VecTy0->getNumElements() == VecTy1->getNumElements() &&
18402 /*ReturnType=*/T0->getScalarType(),
18403 getDotProductIntrinsic(E->getArg(0)->getType(),
18404 VecTy0->getNumElements()),
18408 Value *X = EmitScalarExpr(E->getArg(0));
18409 Value *Y = EmitScalarExpr(E->getArg(1));
18410 Value *S = EmitScalarExpr(E->getArg(2));
18411 if (!E->getArg(0)->getType()->hasFloatingRepresentation())
18414 /*ReturnType=*/X->getType(), CGM.getHLSLRuntime().getLerpIntrinsic(),
18418 Value *Op0 = EmitScalarExpr(E->getArg(0));
18419 if (!E->getArg(0)->getType()->hasFloatingRepresentation())
18422 /*ReturnType=*/Op0->getType(), Intrinsic::dx_frac,
18426 Value *Op0 = EmitScalarExpr(E->getArg(0));
18427 llvm::Type *Xty = Op0->getType();
18428 llvm::Type *retType = llvm::Type::getInt1Ty(this->getLLVMContext());
18429 if (Xty->isVectorTy()) {
18430 auto *XVecTy = E->getArg(0)->getType()->getAs<VectorType>();
18432 retType, ElementCount::getFixed(XVecTy->getNumElements()));
18434 if (!E->getArg(0)->getType()->hasFloatingRepresentation())
18440 Value *M = EmitScalarExpr(E->getArg(0));
18441 Value *A = EmitScalarExpr(E->getArg(1));
18442 Value *B = EmitScalarExpr(E->getArg(2));
18443 if (E->getArg(0)->getType()->hasFloatingRepresentation())
18445 /*ReturnType*/ M->getType(), Intrinsic::fmuladd,
18448 if (E->getArg(0)->getType()->hasSignedIntegerRepresentation()) {
18451 /*ReturnType*/ M->getType(), Intrinsic::dx_imad,
18457 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation());
18460 /*ReturnType=*/M->getType(), Intrinsic::dx_umad,
18467 Value *Op0 = EmitScalarExpr(E->getArg(0));
18468 if (!E->getArg(0)->getType()->hasFloatingRepresentation())
18470 llvm::Type *Ty = Op0->getType();
18471 llvm::Type *EltTy = Ty->getScalarType();
18472 Constant *One = Ty->isVectorTy()
18475 cast<FixedVectorType>(Ty)->getNumElements()),
18481 Value *Op0 = EmitScalarExpr(E->getArg(0));
18482 if (!E->getArg(0)->getType()->hasFloatingRepresentation())
18485 /*ReturnType=*/Op0->getType(), CGM.getHLSLRuntime().getRsqrtIntrinsic(),
18499 constexpr const char *Tag = "amdgpu-as";
18501 LLVMContext &Ctx = Inst->getContext();
18503 for (unsigned K = 2; K < E->getNumArgs(); ++K) {
18504 llvm::Value *V = EmitScalarExpr(E->getArg(K));
18511 CGM.Error(E->getExprLoc(),
18517 Inst->setMetadata(LLVMContext::MD_mmra, MMRAMetadata::getMD(Ctx, MMRAs));
18530 Address FlagOutPtr = EmitPointerWithAlignment(E->getArg(3));
18532 llvm::Value *X = EmitScalarExpr(E->getArg(0));
18533 llvm::Value *Y = EmitScalarExpr(E->getArg(1));
18534 llvm::Value *Z = EmitScalarExpr(E->getArg(2));
18537 X->getType());
18552 llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
18553 llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
18554 llvm::Value *Src2 = EmitScalarExpr(E->getArg(2));
18555 llvm::Value *Src3 = EmitScalarExpr(E->getArg(3));
18558 Src0->getType());
18578 for (unsigned I = 0; I != E->getNumArgs(); ++I) {
18583 Args.insert(Args.begin(), llvm::PoisonValue::get(Args[0]->getType()));
18585 CGM.getIntrinsic(Intrinsic::amdgcn_update_dpp, Args[0]->getType());
18647 llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
18648 llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
18650 CGM.getIntrinsic(Intrinsic::ldexp, {Src0->getType(), Src1->getType()});
18656 llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
18657 llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
18659 CGM.getIntrinsic(Intrinsic::ldexp, {Src0->getType(), Int16Ty});
18669 Value *Src0 = EmitScalarExpr(E->getArg(0));
18671 { Builder.getInt32Ty(), Src0->getType() });
18675 Value *Src0 = EmitScalarExpr(E->getArg(0));
18677 { Builder.getInt16Ty(), Src0->getType() });
18696 llvm::Type *ResultType = ConvertType(E->getType());
18697 llvm::Value *Src = EmitScalarExpr(E->getArg(0));
18705 llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
18706 llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
18707 llvm::Value *Src2 = EmitScalarExpr(E->getArg(2));
18709 // FIXME-GFX10: How should 32 bit mask be handled?
18711 { Builder.getInt64Ty(), Src0->getType() });
18716 llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
18717 llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
18718 llvm::Value *Src2 = EmitScalarExpr(E->getArg(2));
18720 // FIXME-GFX10: How should 32 bit mask be handled?
18722 { Builder.getInt64Ty(), Src0->getType() });
18737 Value *Src0 = EmitScalarExpr(E->getArg(0));
18738 Function *F = CGM.getIntrinsic(Intrin, { Src0->getType() });
18791 llvm::Value *Addr = EmitScalarExpr(E->getArg(0));
18792 llvm::Value *Val = EmitScalarExpr(E->getArg(1));
18794 CGM.getIntrinsic(IID, {ArgTy, Addr->getType(), Val->getType()});
18808 llvm::Value *Addr = EmitScalarExpr(E->getArg(0));
18809 llvm::Value *Val = EmitScalarExpr(E->getArg(1));
18810 llvm::Function *F = CGM.getIntrinsic(IID, {Addr->getType()});
18837 llvm::Type *LoadTy = ConvertType(E->getType());
18838 llvm::Value *Addr = EmitScalarExpr(E->getArg(0));
18850 llvm::Value *Env = EmitScalarExpr(E->getArg(0));
18863 llvm::Value *NodePtr = EmitScalarExpr(E->getArg(0));
18864 llvm::Value *RayExtent = EmitScalarExpr(E->getArg(1));
18865 llvm::Value *RayOrigin = EmitScalarExpr(E->getArg(2));
18866 llvm::Value *RayDir = EmitScalarExpr(E->getArg(3));
18867 llvm::Value *RayInverseDir = EmitScalarExpr(E->getArg(4));
18868 llvm::Value *TextureDescr = EmitScalarExpr(E->getArg(5));
18880 {NodePtr->getType(), RayDir->getType()});
18887 for (int i = 0, e = E->getNumArgs(); i != e; ++i)
18888 Args.push_back(EmitScalarExpr(E->getArg(i)));
18894 llvm::Type *RetTy = ConvertType(E->getType());
18969 // On GFX12, the intrinsics with 16-bit accumulator use a packed layout.
19115 for (int i = 0, e = E->getNumArgs(); i != e; ++i)
19116 Args.push_back(EmitScalarExpr(E->getArg(i)));
19122 ArgTypes.push_back(Args[ArgIdx]->getType());
19164 llvm::Value *Src0 = EmitScalarExpr(E->getArg(0));
19165 llvm::Value *Src1 = EmitScalarExpr(E->getArg(1));
19166 llvm::Value *Src2 = EmitScalarExpr(E->getArg(2));
19167 Function *F = CGM.getIntrinsic(Intrinsic::fshr, Src0->getType());
19171 ProcessOrderScopeAMDGCN(EmitScalarExpr(E->getArg(0)),
19172 EmitScalarExpr(E->getArg(1)), AO, SSID);
19174 if (E->getNumArgs() > 2)
19215 Value *Val = EmitScalarExpr(E->getArg(1));
19216 llvm::Type *OrigTy = Val->getType();
19217 QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
19226 cast<ConstantInt>(EmitScalarExpr(E->getArg(4)))->getZExtValue();
19230 PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
19233 if (E->getNumArgs() >= 4) {
19235 ProcessOrderScopeAMDGCN(EmitScalarExpr(E->getArg(2)),
19236 EmitScalarExpr(E->getArg(3)), AO, SSID);
19253 RMW->setVolatile(true);
19258 llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
19259 llvm::Type *ResultType = ConvertType(E->getType());
19306 F, {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)),
19307 EmitScalarExpr(E->getArg(2)), EmitScalarExpr(E->getArg(3))});
19315 /// to an int that receives the post-instruction CC value. At the LLVM level
19320 unsigned NumArgs = E->getNumArgs() - 1;
19323 Args[I] = CGF.EmitScalarExpr(E->getArg(I));
19324 Address CCPtr = CGF.EmitPointerWithAlignment(E->getArg(NumArgs));
19336 Value *TDB = EmitScalarExpr(E->getArg(0));
19342 Value *TDB = EmitScalarExpr(E->getArg(0));
19354 Value *Data = EmitScalarExpr(E->getArg(0));
19359 Value *Address = EmitScalarExpr(E->getArg(0));
19360 Value *Data = EmitScalarExpr(E->getArg(1));
19366 // to target-specific LLVM intrinsics. The ones handled specially here can
19374 llvm::Type *ResultType = ConvertType(E->getType());
19375 Value *X = EmitScalarExpr(E->getArg(0));
19384 llvm::Type *ResultType = ConvertType(E->getType());
19385 Value *X = EmitScalarExpr(E->getArg(0));
19395 llvm::Type *ResultType = ConvertType(E->getType());
19396 Value *X = EmitScalarExpr(E->getArg(0));
19406 llvm::Type *ResultType = ConvertType(E->getType());
19407 llvm::Value *Src = EmitScalarExpr(E->getArg(0));
19408 llvm::Value *Amt = EmitScalarExpr(E->getArg(1));
19410 unsigned NumElts = cast<llvm::FixedVectorType>(ResultType)->getNumElements();
19411 Amt = Builder.CreateIntCast(Amt, ResultType->getScalarType(), false);
19421 llvm::Type *ResultType = ConvertType(E->getType());
19422 llvm::Value *Src = EmitScalarExpr(E->getArg(0));
19423 llvm::Value *Amt = EmitScalarExpr(E->getArg(1));
19430 llvm::Type *ResultType = ConvertType(E->getType());
19431 Value *X = EmitScalarExpr(E->getArg(0));
19442 llvm::Type *ResultType = ConvertType(E->getType());
19443 Value *X = EmitScalarExpr(E->getArg(0));
19444 Value *Y = EmitScalarExpr(E->getArg(1));
19445 Value *Z = EmitScalarExpr(E->getArg(2));
19456 llvm::Type *ResultType = ConvertType(E->getType());
19457 Value *X = EmitScalarExpr(E->getArg(0));
19458 Value *Y = EmitScalarExpr(E->getArg(1));
19459 Value *Z = EmitScalarExpr(E->getArg(2));
19470 llvm::Type *ResultType = ConvertType(E->getType());
19471 Value *X = EmitScalarExpr(E->getArg(0));
19472 Value *Y = EmitScalarExpr(E->getArg(1));
19473 Value *Z = EmitScalarExpr(E->getArg(2));
19484 llvm::Type *ResultType = ConvertType(E->getType());
19485 Value *X = EmitScalarExpr(E->getArg(0));
19486 Value *Y = EmitScalarExpr(E->getArg(1));
19487 Value *Z = EmitScalarExpr(E->getArg(2));
19500 llvm::Type *ResultType = ConvertType(E->getType());
19501 Value *X = EmitScalarExpr(E->getArg(0));
19507 llvm::Type *ResultType = ConvertType(E->getType());
19508 Value *X = EmitScalarExpr(E->getArg(0));
19514 llvm::Type *ResultType = ConvertType(E->getType());
19515 Value *X = EmitScalarExpr(E->getArg(0));
19516 // Constant-fold the M4 and M5 mask arguments.
19517 llvm::APSInt M4 = *E->getArg(1)->getIntegerConstantExpr(getContext());
19518 llvm::APSInt M5 = *E->getArg(2)->getIntegerConstantExpr(getContext());
19525 case 0: // IEEE-inexact exception allowed
19532 case 4: // IEEE-inexact exception suppressed
19569 llvm::Type *ResultType = ConvertType(E->getType());
19570 Value *X = EmitScalarExpr(E->getArg(0));
19571 Value *Y = EmitScalarExpr(E->getArg(1));
19572 // Constant-fold the M4 mask argument.
19573 llvm::APSInt M4 = *E->getArg(2)->getIntegerConstantExpr(getContext());
19603 llvm::Type *ResultType = ConvertType(E->getType());
19604 Value *X = EmitScalarExpr(E->getArg(0));
19605 Value *Y = EmitScalarExpr(E->getArg(1));
19606 // Constant-fold the M4 mask argument.
19607 llvm::APSInt M4 = *E->getArg(2)->getIntegerConstantExpr(getContext());
19639 llvm::Type *ResultType = ConvertType(E->getType());
19640 Value *X = EmitScalarExpr(E->getArg(0));
19645 // Vector intrinsics that output the post-instruction CC value.
19809 // Sub-integer MMA loads.
19873 // Integer and sub-integer MMA stores.
19909 // Variants are ordered by layout-A/layout-B/satf, where 'row' has priority
19910 // over 'col' for layout. The index of non-satf variants is expected to match
19925 // clang-format off
19937 // Sub-integer MMA only supports row.col layout.
19966 // clang-format on
20010 // Sub-integer MMA
20045 Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
20046 QualType ArgType = E->getArg(0)->getType();
20048 llvm::Type *ElemTy = CGF.ConvertTypeForMem(ArgType->getPointeeType());
20050 CGF.CGM.getIntrinsic(IntrinsicID, {ElemTy, Ptr->getType()}),
20056 Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
20058 CGF.ConvertTypeForMem(E->getArg(0)->getType()->getPointeeType());
20060 CGF.CGM.getIntrinsic(IntrinsicID, {ElemTy, Ptr->getType()}),
20061 {Ptr, CGF.EmitScalarExpr(E->getArg(1))});
20067 return E->getNumArgs() == 3
20069 {CGF.EmitScalarExpr(E->getArg(0)),
20070 CGF.EmitScalarExpr(E->getArg(1)),
20071 CGF.EmitScalarExpr(E->getArg(2))})
20073 {CGF.EmitScalarExpr(E->getArg(0)),
20074 CGF.EmitScalarExpr(E->getArg(1))});
20082 CGF.CGM.Error(E->getExprLoc(), C.BuiltinInfo.getName(BuiltinID).str() +
20093 auto *FTy = F->getFunctionType();
20098 for (unsigned i = 0, e = E->getNumArgs(); i != e; ++i) {
20100 auto *ArgValue = CGF.EmitScalarExpr(E->getArg(i));
20101 auto *PTy = FTy->getParamType(i);
20102 if (PTy != ArgValue->getType())
20173 Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
20174 Value *Val = EmitScalarExpr(E->getArg(1));
20181 Value *Ptr = EmitScalarExpr(E->getArg(0));
20182 Value *Val = EmitScalarExpr(E->getArg(1));
20184 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_inc_32, Ptr->getType());
20189 Value *Ptr = EmitScalarExpr(E->getArg(0));
20190 Value *Val = EmitScalarExpr(E->getArg(1));
20192 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_dec_32, Ptr->getType());
20358 Value *Ptr = EmitScalarExpr(E->getArg(0));
20360 ConvertTypeForMem(E->getArg(0)->getType()->getPointeeType());
20363 Intrinsic::nvvm_atomic_cas_gen_i_cta, {ElemTy, Ptr->getType()}),
20364 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
20369 Value *Ptr = EmitScalarExpr(E->getArg(0));
20371 ConvertTypeForMem(E->getArg(0)->getType()->getPointeeType());
20374 Intrinsic::nvvm_atomic_cas_gen_i_sys, {ElemTy, Ptr->getType()}),
20375 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
20379 Value *Mask = EmitScalarExpr(E->getArg(0));
20380 Value *Val = EmitScalarExpr(E->getArg(1));
20381 Address PredOutPtr = EmitPointerWithAlignment(E->getArg(2));
20422 // Sub-integer MMA loads.
20445 Address Dst = EmitPointerWithAlignment(E->getArg(0));
20446 Value *Src = EmitScalarExpr(E->getArg(1));
20447 Value *Ldm = EmitScalarExpr(E->getArg(2));
20449 E->getArg(3)->getIntegerConstantExpr(getContext());
20452 bool isColMajor = isColMajorArg->getSExtValue();
20459 Builder.CreateCall(CGM.getIntrinsic(IID, Src->getType()), {Src, Ldm});
20492 Value *Dst = EmitScalarExpr(E->getArg(0));
20493 Address Src = EmitPointerWithAlignment(E->getArg(1));
20494 Value *Ldm = EmitScalarExpr(E->getArg(2));
20496 E->getArg(3)->getIntegerConstantExpr(getContext());
20499 bool isColMajor = isColMajorArg->getSExtValue();
20505 CGM.getIntrinsic(IID, Dst->getType());
20506 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
20521 // BI__hmma_m16n16k16_mma_<Dtype><CType>(d, a, b, c, layout, satf) -->
20550 Address Dst = EmitPointerWithAlignment(E->getArg(0));
20551 Address SrcA = EmitPointerWithAlignment(E->getArg(1));
20552 Address SrcB = EmitPointerWithAlignment(E->getArg(2));
20553 Address SrcC = EmitPointerWithAlignment(E->getArg(3));
20555 E->getArg(4)->getIntegerConstantExpr(getContext());
20558 int Layout = LayoutArg->getSExtValue();
20566 E->getArg(5)->getIntegerConstantExpr(getContext()))
20578 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
20589 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
20600 Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
20827 EmitScalarExpr(E->getArg(0)));
20831 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
20835 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
20839 EmitScalarExpr(E->getArg(0)));
20843 EmitScalarExpr(E->getArg(0)));
20870 QualType AstType = E->getArg(0)->getType();
20871 if (AstType->isArrayType())
20872 Src = CGF.EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(CGF);
20874 Src = CGF.EmitScalarExpr(E->getArg(0));
20875 SrcType = Src->getType();
20876 if (SrcType->isPointerTy()) {
20881 assert(SrcType->isIntegerTy());
20884 Alignment = CGF.EmitScalarExpr(E->getArg(1));
20892 /// Generate (x & (y-1)) == 0.
20896 if (Args.SrcType->isPointerTy())
20904 /// Generate (x & ~(y-1)) to align down or ((x+(y-1)) & ~(y-1)) to align up.
20915 if (Args.Src->getType()->isPointerTy()) {
20923 E->getExprLoc(), "over_boundary");
20931 if (Args.Src->getType()->isPointerTy()) {
20938 assert(Result->getType() == Args.SrcType);
20946 llvm::Type *ResultType = ConvertType(E->getType());
20947 Value *I = EmitScalarExpr(E->getArg(0));
20953 llvm::Type *ResultType = ConvertType(E->getType());
20954 Value *Args[] = {EmitScalarExpr(E->getArg(0)),
20955 EmitScalarExpr(E->getArg(1))};
20961 llvm::Type *ResultType = ConvertType(E->getType());
20966 llvm::Type *ResultType = ConvertType(E->getType());
20975 Value *Tag = EmitScalarExpr(E->getArg(0));
20976 Value *Obj = EmitScalarExpr(E->getArg(1));
20985 Value *Addr = EmitScalarExpr(E->getArg(0));
20986 Value *Expected = EmitScalarExpr(E->getArg(1));
20987 Value *Timeout = EmitScalarExpr(E->getArg(2));
20989 return Builder.CreateCall(Callee, {Addr, Expected, Timeout});
20992 Value *Addr = EmitScalarExpr(E->getArg(0));
20993 Value *Expected = EmitScalarExpr(E->getArg(1));
20994 Value *Timeout = EmitScalarExpr(E->getArg(2));
20996 return Builder.CreateCall(Callee, {Addr, Expected, Timeout});
20999 Value *Addr = EmitScalarExpr(E->getArg(0));
21000 Value *Count = EmitScalarExpr(E->getArg(1));
21008 Value *Src = EmitScalarExpr(E->getArg(0));
21009 llvm::Type *ResT = ConvertType(E->getType());
21011 CGM.getIntrinsic(Intrinsic::wasm_trunc_signed, {ResT, Src->getType()});
21018 Value *Src = EmitScalarExpr(E->getArg(0));
21019 llvm::Type *ResT = ConvertType(E->getType());
21021 {ResT, Src->getType()});
21029 Value *Src = EmitScalarExpr(E->getArg(0));
21030 llvm::Type *ResT = ConvertType(E->getType());
21032 CGM.getIntrinsic(Intrinsic::fptosi_sat, {ResT, Src->getType()});
21040 Value *Src = EmitScalarExpr(E->getArg(0));
21041 llvm::Type *ResT = ConvertType(E->getType());
21043 CGM.getIntrinsic(Intrinsic::fptoui_sat, {ResT, Src->getType()});
21051 Value *LHS = EmitScalarExpr(E->getArg(0));
21052 Value *RHS = EmitScalarExpr(E->getArg(1));
21054 CGM.getIntrinsic(Intrinsic::minimum, ConvertType(E->getType()));
21062 Value *LHS = EmitScalarExpr(E->getArg(0));
21063 Value *RHS = EmitScalarExpr(E->getArg(1));
21065 CGM.getIntrinsic(Intrinsic::maximum, ConvertType(E->getType()));
21071 Value *LHS = EmitScalarExpr(E->getArg(0));
21072 Value *RHS = EmitScalarExpr(E->getArg(1));
21074 CGM.getIntrinsic(Intrinsic::wasm_pmin, ConvertType(E->getType()));
21080 Value *LHS = EmitScalarExpr(E->getArg(0));
21081 Value *RHS = EmitScalarExpr(E->getArg(1));
21083 CGM.getIntrinsic(Intrinsic::wasm_pmax, ConvertType(E->getType()));
21115 Value *Value = EmitScalarExpr(E->getArg(0));
21116 Function *Callee = CGM.getIntrinsic(IntNo, ConvertType(E->getType()));
21128 Value *Src = EmitScalarExpr(E->getArg(0));
21129 Value *Indices = EmitScalarExpr(E->getArg(1));
21162 Value *LHS = EmitScalarExpr(E->getArg(0));
21163 Value *RHS = EmitScalarExpr(E->getArg(1));
21164 Function *Callee = CGM.getIntrinsic(IntNo, ConvertType(E->getType()));
21171 Value *Vec = EmitScalarExpr(E->getArg(0));
21173 Constant *Zero = llvm::Constant::getNullValue(Vec->getType());
21189 Value *LHS = EmitScalarExpr(E->getArg(0));
21190 Value *RHS = EmitScalarExpr(E->getArg(1));
21220 Value *LHS = EmitScalarExpr(E->getArg(0));
21221 Value *RHS = EmitScalarExpr(E->getArg(1));
21223 ConvertType(E->getType()));
21227 Value *LHS = EmitScalarExpr(E->getArg(0));
21228 Value *RHS = EmitScalarExpr(E->getArg(1));
21236 Value *Vec = EmitScalarExpr(E->getArg(0));
21251 Function *Callee = CGM.getIntrinsic(IntNo, ConvertType(E->getType()));
21255 Value *V1 = EmitScalarExpr(E->getArg(0));
21256 Value *V2 = EmitScalarExpr(E->getArg(1));
21257 Value *C = EmitScalarExpr(E->getArg(2));
21259 CGM.getIntrinsic(Intrinsic::wasm_bitselect, ConvertType(E->getType()));
21263 Value *LHS = EmitScalarExpr(E->getArg(0));
21264 Value *RHS = EmitScalarExpr(E->getArg(1));
21269 Value *Vec = EmitScalarExpr(E->getArg(0));
21271 CGM.getIntrinsic(Intrinsic::ctpop, ConvertType(E->getType()));
21293 Value *Vec = EmitScalarExpr(E->getArg(0));
21294 Function *Callee = CGM.getIntrinsic(IntNo, Vec->getType());
21301 Value *Vec = EmitScalarExpr(E->getArg(0));
21303 CGM.getIntrinsic(Intrinsic::wasm_bitmask, Vec->getType());
21308 Value *Vec = EmitScalarExpr(E->getArg(0));
21309 Function *Callee = CGM.getIntrinsic(Intrinsic::fabs, Vec->getType());
21314 Value *Vec = EmitScalarExpr(E->getArg(0));
21315 Function *Callee = CGM.getIntrinsic(Intrinsic::sqrt, Vec->getType());
21322 Value *Low = EmitScalarExpr(E->getArg(0));
21323 Value *High = EmitScalarExpr(E->getArg(1));
21338 CGM.getIntrinsic(IntNo, {ConvertType(E->getType()), Low->getType()});
21343 Value *Vec = EmitScalarExpr(E->getArg(0));
21355 llvm::Type *SrcT = Vec->getType();
21356 llvm::Type *TruncT = SrcT->getWithNewType(Builder.getInt32Ty());
21365 Ops[OpIdx++] = EmitScalarExpr(E->getArg(0));
21366 Ops[OpIdx++] = EmitScalarExpr(E->getArg(1));
21369 E->getArg(OpIdx)->getIntegerConstantExpr(getContext());
21382 Value *A = EmitScalarExpr(E->getArg(0));
21383 Value *B = EmitScalarExpr(E->getArg(1));
21384 Value *C = EmitScalarExpr(E->getArg(2));
21400 Function *Callee = CGM.getIntrinsic(IntNo, A->getType());
21407 Value *A = EmitScalarExpr(E->getArg(0));
21408 Value *B = EmitScalarExpr(E->getArg(1));
21409 Value *C = EmitScalarExpr(E->getArg(2));
21411 CGM.getIntrinsic(Intrinsic::wasm_relaxed_laneselect, A->getType());
21415 Value *Src = EmitScalarExpr(E->getArg(0));
21416 Value *Indices = EmitScalarExpr(E->getArg(1));
21424 Value *LHS = EmitScalarExpr(E->getArg(0));
21425 Value *RHS = EmitScalarExpr(E->getArg(1));
21439 Function *Callee = CGM.getIntrinsic(IntNo, LHS->getType());
21446 Value *Vec = EmitScalarExpr(E->getArg(0));
21468 Value *LHS = EmitScalarExpr(E->getArg(0));
21469 Value *RHS = EmitScalarExpr(E->getArg(1));
21474 Value *LHS = EmitScalarExpr(E->getArg(0));
21475 Value *RHS = EmitScalarExpr(E->getArg(1));
21481 Value *LHS = EmitScalarExpr(E->getArg(0));
21482 Value *RHS = EmitScalarExpr(E->getArg(1));
21483 Value *Acc = EmitScalarExpr(E->getArg(2));
21489 Value *LHS = EmitScalarExpr(E->getArg(0));
21490 Value *RHS = EmitScalarExpr(E->getArg(1));
21491 Value *Acc = EmitScalarExpr(E->getArg(2));
21497 Value *Addr = EmitScalarExpr(E->getArg(0));
21502 Value *Val = EmitScalarExpr(E->getArg(0));
21503 Value *Addr = EmitScalarExpr(E->getArg(1));
21508 Value *Val = EmitScalarExpr(E->getArg(0));
21513 Value *Vector = EmitScalarExpr(E->getArg(0));
21514 Value *Index = EmitScalarExpr(E->getArg(1));
21519 assert(E->getArg(0)->getType()->isArrayType());
21520 Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
21521 Value *Index = EmitScalarExpr(E->getArg(1));
21523 if (E->getType().isWebAssemblyExternrefType())
21525 else if (E->getType().isWebAssemblyFuncrefType())
21533 assert(E->getArg(0)->getType()->isArrayType());
21534 Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
21535 Value *Index = EmitScalarExpr(E->getArg(1));
21536 Value *Val = EmitScalarExpr(E->getArg(2));
21538 if (E->getArg(2)->getType().isWebAssemblyExternrefType())
21540 else if (E->getArg(2)->getType().isWebAssemblyFuncrefType())
21548 assert(E->getArg(0)->getType()->isArrayType());
21549 Value *Value = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
21554 assert(E->getArg(0)->getType()->isArrayType());
21555 Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
21556 Value *Val = EmitScalarExpr(E->getArg(1));
21557 Value *NElems = EmitScalarExpr(E->getArg(2));
21560 if (E->getArg(1)->getType().isWebAssemblyExternrefType())
21562 else if (E->getArg(2)->getType().isWebAssemblyFuncrefType())
21571 assert(E->getArg(0)->getType()->isArrayType());
21572 Value *Table = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
21573 Value *Index = EmitScalarExpr(E->getArg(1));
21574 Value *Val = EmitScalarExpr(E->getArg(2));
21575 Value *NElems = EmitScalarExpr(E->getArg(3));
21578 if (E->getArg(2)->getType().isWebAssemblyExternrefType())
21580 else if (E->getArg(2)->getType().isWebAssemblyFuncrefType())
21589 assert(E->getArg(0)->getType()->isArrayType());
21590 Value *TableX = EmitArrayToPointerDecay(E->getArg(0)).emitRawPointer(*this);
21591 Value *TableY = EmitArrayToPointerDecay(E->getArg(1)).emitRawPointer(*this);
21592 Value *DstIdx = EmitScalarExpr(E->getArg(2));
21593 Value *SrcIdx = EmitScalarExpr(E->getArg(3));
21594 Value *NElems = EmitScalarExpr(E->getArg(4));
21655 if (F == std::end(Infos) || F->BuiltinID != BuiltinID)
21658 return {F->IntrinsicID, F->VecLen};
21669 Address A = EmitPointerWithAlignment(E->getArg(0));
21675 // builtin(Base, Inc, Mod, Start) -> intr(Base, Inc, Mod, Start)
21676 // builtin(Base, Mod, Start) -> intr(Base, Mod, Start)
21678 // builtin(Base, Inc, Mod, Val, Start) -> intr(Base, Inc, Mod, Val, Start)
21679 // builtin(Base, Mod, Val, Start) -> intr(Base, Mod, Val, Start)
21681 for (unsigned i = 1, e = E->getNumArgs(); i != e; ++i)
21682 Ops.push_back(EmitScalarExpr(E->getArg(i)));
21689 llvm::Value *LV = EmitScalarExpr(E->getArg(0));
21690 Address Dest = EmitPointerWithAlignment(E->getArg(0));
21698 // Handle the conversion of bit-reverse load intrinsics to bit code.
21705 llvm::Value *BaseAddress = EmitScalarExpr(E->getArg(0));
21710 Address DestAddr = EmitPointerWithAlignment(E->getArg(1));
21718 CGM.getIntrinsic(IntID), {BaseAddress, EmitScalarExpr(E->getArg(2))});
21737 {Vec, Builder.getInt32(-1)});
21743 {Pred, Builder.getInt32(-1)});
21754 // Get the type from the 0-th argument.
21755 llvm::Type *VecType = ConvertType(E->getArg(0)->getType());
21757 EmitPointerWithAlignment(E->getArg(2)).withElementType(VecType);
21760 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
21768 // input carry, only generate carry-out. Since they still produce two
21774 // Get the type from the 0-th argument.
21775 llvm::Type *VecType = ConvertType(E->getArg(0)->getType());
21777 EmitPointerWithAlignment(E->getArg(2)).withElementType(VecType);
21779 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
21796 const Expr *PredOp = E->getArg(0);
21799 if (Cast->getCastKind() == CK_BitCast)
21800 PredOp = Cast->getSubExpr();
21803 for (int i = 1, e = E->getNumArgs(); i != e; ++i)
21804 Ops.push_back(EmitScalarExpr(E->getArg(i)));
21853 llvm::Type *ResultType = ConvertType(E->getType());
21876 for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) {
21878 if (hasAggregateEvaluationKind(E->getArg(i)->getType())) {
21879 LValue L = EmitAggExprToLValue(E->getArg(i));
21929 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
21931 if (Result->getType() != ResultType)
21938 Function *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType());
21940 if (Result->getType() != ResultType)
22021 llvm::Type *ResTy = ConvertType(E->getType());
22024 DomainVal = cast<ConstantInt>(Ops[1])->getZExtValue();
22033 if(ResTy->isScalableTy()) {
22035 llvm::Type *ScalarTy = ResTy->getScalarType();
22036 Width = ScalarTy->getPrimitiveSizeInBits() *
22037 SVTy->getElementCount().getKnownMinValue();
22039 Width = ResTy->getPrimitiveSizeInBits();
22043 Load->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
22044 Load->setMetadata(CGM.getModule().getMDKindID("riscv-nontemporal-domain"),
22052 DomainVal = cast<ConstantInt>(Ops[2])->getZExtValue();
22061 Store->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
22062 Store->setMetadata(CGM.getModule().getMDKindID("riscv-nontemporal-domain"),