xref: /freebsd-src/contrib/llvm-project/llvm/lib/IR/IRBuilder.cpp (revision 0fca6ea1d4eea4c934cfff25ac9ee8ad6fe95583)
10b57cec5SDimitry Andric //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric // This file implements the IRBuilder class, which is used as a convenient way
100b57cec5SDimitry Andric // to create LLVM instructions with a consistent and simplified interface.
110b57cec5SDimitry Andric //
120b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
130b57cec5SDimitry Andric 
140b57cec5SDimitry Andric #include "llvm/IR/IRBuilder.h"
150b57cec5SDimitry Andric #include "llvm/ADT/ArrayRef.h"
160b57cec5SDimitry Andric #include "llvm/IR/Constant.h"
170b57cec5SDimitry Andric #include "llvm/IR/Constants.h"
1881ad6265SDimitry Andric #include "llvm/IR/DebugInfoMetadata.h"
190b57cec5SDimitry Andric #include "llvm/IR/DerivedTypes.h"
200b57cec5SDimitry Andric #include "llvm/IR/Function.h"
210b57cec5SDimitry Andric #include "llvm/IR/GlobalValue.h"
220b57cec5SDimitry Andric #include "llvm/IR/GlobalVariable.h"
230b57cec5SDimitry Andric #include "llvm/IR/IntrinsicInst.h"
240b57cec5SDimitry Andric #include "llvm/IR/Intrinsics.h"
250b57cec5SDimitry Andric #include "llvm/IR/LLVMContext.h"
26*0fca6ea1SDimitry Andric #include "llvm/IR/Module.h"
275ffd83dbSDimitry Andric #include "llvm/IR/NoFolder.h"
280b57cec5SDimitry Andric #include "llvm/IR/Operator.h"
290b57cec5SDimitry Andric #include "llvm/IR/Statepoint.h"
300b57cec5SDimitry Andric #include "llvm/IR/Type.h"
310b57cec5SDimitry Andric #include "llvm/IR/Value.h"
320b57cec5SDimitry Andric #include "llvm/Support/Casting.h"
330b57cec5SDimitry Andric #include <cassert>
340b57cec5SDimitry Andric #include <cstdint>
35bdd1243dSDimitry Andric #include <optional>
360b57cec5SDimitry Andric #include <vector>
370b57cec5SDimitry Andric 
380b57cec5SDimitry Andric using namespace llvm;
390b57cec5SDimitry Andric 
400b57cec5SDimitry Andric /// CreateGlobalString - Make a new global variable with an initializer that
410b57cec5SDimitry Andric /// has array of i8 type filled in with the nul terminated string value
420b57cec5SDimitry Andric /// specified.  If Name is specified, it is the name of the global variable
430b57cec5SDimitry Andric /// created.
440b57cec5SDimitry Andric GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
450b57cec5SDimitry Andric                                                   const Twine &Name,
4616d6b3b3SDimitry Andric                                                   unsigned AddressSpace,
47*0fca6ea1SDimitry Andric                                                   Module *M, bool AddNull) {
48*0fca6ea1SDimitry Andric   Constant *StrConstant = ConstantDataArray::getString(Context, Str, AddNull);
4916d6b3b3SDimitry Andric   if (!M)
5016d6b3b3SDimitry Andric     M = BB->getParent()->getParent();
5116d6b3b3SDimitry Andric   auto *GV = new GlobalVariable(
5216d6b3b3SDimitry Andric       *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
5316d6b3b3SDimitry Andric       StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
540b57cec5SDimitry Andric   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
555ffd83dbSDimitry Andric   GV->setAlignment(Align(1));
560b57cec5SDimitry Andric   return GV;
570b57cec5SDimitry Andric }
580b57cec5SDimitry Andric 
590b57cec5SDimitry Andric Type *IRBuilderBase::getCurrentFunctionReturnType() const {
600b57cec5SDimitry Andric   assert(BB && BB->getParent() && "No current function!");
610b57cec5SDimitry Andric   return BB->getParent()->getReturnType();
620b57cec5SDimitry Andric }
630b57cec5SDimitry Andric 
6481ad6265SDimitry Andric DebugLoc IRBuilderBase::getCurrentDebugLocation() const {
6581ad6265SDimitry Andric   for (auto &KV : MetadataToCopy)
6681ad6265SDimitry Andric     if (KV.first == LLVMContext::MD_dbg)
6781ad6265SDimitry Andric       return {cast<DILocation>(KV.second)};
6881ad6265SDimitry Andric 
6981ad6265SDimitry Andric   return {};
7081ad6265SDimitry Andric }
7181ad6265SDimitry Andric void IRBuilderBase::SetInstDebugLocation(Instruction *I) const {
7281ad6265SDimitry Andric   for (const auto &KV : MetadataToCopy)
7381ad6265SDimitry Andric     if (KV.first == LLVMContext::MD_dbg) {
7481ad6265SDimitry Andric       I->setDebugLoc(DebugLoc(KV.second));
7581ad6265SDimitry Andric       return;
7681ad6265SDimitry Andric     }
7781ad6265SDimitry Andric }
7881ad6265SDimitry Andric 
79bdd1243dSDimitry Andric CallInst *
80bdd1243dSDimitry Andric IRBuilderBase::createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
81bdd1243dSDimitry Andric                                 const Twine &Name, Instruction *FMFSource,
82bdd1243dSDimitry Andric                                 ArrayRef<OperandBundleDef> OpBundles) {
83bdd1243dSDimitry Andric   CallInst *CI = CreateCall(Callee, Ops, OpBundles, Name);
840b57cec5SDimitry Andric   if (FMFSource)
850b57cec5SDimitry Andric     CI->copyFastMathFlags(FMFSource);
860b57cec5SDimitry Andric   return CI;
870b57cec5SDimitry Andric }
880b57cec5SDimitry Andric 
89e8d8bef9SDimitry Andric Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) {
90e8d8bef9SDimitry Andric   assert(isa<ConstantInt>(Scaling) && "Expected constant integer");
91fe6060f1SDimitry Andric   if (cast<ConstantInt>(Scaling)->isZero())
92fe6060f1SDimitry Andric     return Scaling;
93fe6060f1SDimitry Andric   Module *M = GetInsertBlock()->getParent()->getParent();
94e8d8bef9SDimitry Andric   Function *TheFn =
95e8d8bef9SDimitry Andric       Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()});
96bdd1243dSDimitry Andric   CallInst *CI = CreateCall(TheFn, {}, {}, Name);
9706c3fb27SDimitry Andric   return cast<ConstantInt>(Scaling)->isOne() ? CI : CreateMul(CI, Scaling);
9806c3fb27SDimitry Andric }
9906c3fb27SDimitry Andric 
10006c3fb27SDimitry Andric Value *IRBuilderBase::CreateElementCount(Type *DstType, ElementCount EC) {
10106c3fb27SDimitry Andric   Constant *MinEC = ConstantInt::get(DstType, EC.getKnownMinValue());
10206c3fb27SDimitry Andric   return EC.isScalable() ? CreateVScale(MinEC) : MinEC;
10306c3fb27SDimitry Andric }
10406c3fb27SDimitry Andric 
10506c3fb27SDimitry Andric Value *IRBuilderBase::CreateTypeSize(Type *DstType, TypeSize Size) {
10606c3fb27SDimitry Andric   Constant *MinSize = ConstantInt::get(DstType, Size.getKnownMinValue());
10706c3fb27SDimitry Andric   return Size.isScalable() ? CreateVScale(MinSize) : MinSize;
108e8d8bef9SDimitry Andric }
109e8d8bef9SDimitry Andric 
110fe6060f1SDimitry Andric Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {
111fe6060f1SDimitry Andric   Type *STy = DstType->getScalarType();
112349cc55cSDimitry Andric   if (isa<ScalableVectorType>(DstType)) {
113349cc55cSDimitry Andric     Type *StepVecType = DstType;
114349cc55cSDimitry Andric     // TODO: We expect this special case (element type < 8 bits) to be
115349cc55cSDimitry Andric     // temporary - once the intrinsic properly supports < 8 bits this code
116349cc55cSDimitry Andric     // can be removed.
117349cc55cSDimitry Andric     if (STy->getScalarSizeInBits() < 8)
118349cc55cSDimitry Andric       StepVecType =
119349cc55cSDimitry Andric           VectorType::get(getInt8Ty(), cast<ScalableVectorType>(DstType));
120349cc55cSDimitry Andric     Value *Res = CreateIntrinsic(Intrinsic::experimental_stepvector,
121349cc55cSDimitry Andric                                  {StepVecType}, {}, nullptr, Name);
122349cc55cSDimitry Andric     if (StepVecType != DstType)
123349cc55cSDimitry Andric       Res = CreateTrunc(Res, DstType);
124349cc55cSDimitry Andric     return Res;
125349cc55cSDimitry Andric   }
126349cc55cSDimitry Andric 
127fe6060f1SDimitry Andric   unsigned NumEls = cast<FixedVectorType>(DstType)->getNumElements();
128fe6060f1SDimitry Andric 
129fe6060f1SDimitry Andric   // Create a vector of consecutive numbers from zero to VF.
130fe6060f1SDimitry Andric   SmallVector<Constant *, 8> Indices;
131fe6060f1SDimitry Andric   for (unsigned i = 0; i < NumEls; ++i)
132fe6060f1SDimitry Andric     Indices.push_back(ConstantInt::get(STy, i));
133fe6060f1SDimitry Andric 
134fe6060f1SDimitry Andric   // Add the consecutive indices to the vector value.
135fe6060f1SDimitry Andric   return ConstantVector::get(Indices);
136fe6060f1SDimitry Andric }
137fe6060f1SDimitry Andric 
138480093f4SDimitry Andric CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
139480093f4SDimitry Andric                                       MaybeAlign Align, bool isVolatile,
140480093f4SDimitry Andric                                       MDNode *TBAATag, MDNode *ScopeTag,
1410b57cec5SDimitry Andric                                       MDNode *NoAliasTag) {
1420b57cec5SDimitry Andric   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
1430b57cec5SDimitry Andric   Type *Tys[] = { Ptr->getType(), Size->getType() };
1440b57cec5SDimitry Andric   Module *M = BB->getParent()->getParent();
1450b57cec5SDimitry Andric   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
1460b57cec5SDimitry Andric 
147bdd1243dSDimitry Andric   CallInst *CI = CreateCall(TheFn, Ops);
1480b57cec5SDimitry Andric 
149480093f4SDimitry Andric   if (Align)
15081ad6265SDimitry Andric     cast<MemSetInst>(CI)->setDestAlignment(*Align);
15181ad6265SDimitry Andric 
15281ad6265SDimitry Andric   // Set the TBAA info if present.
15381ad6265SDimitry Andric   if (TBAATag)
15481ad6265SDimitry Andric     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
15581ad6265SDimitry Andric 
15681ad6265SDimitry Andric   if (ScopeTag)
15781ad6265SDimitry Andric     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
15881ad6265SDimitry Andric 
15981ad6265SDimitry Andric   if (NoAliasTag)
16081ad6265SDimitry Andric     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
16181ad6265SDimitry Andric 
16281ad6265SDimitry Andric   return CI;
16381ad6265SDimitry Andric }
16481ad6265SDimitry Andric 
16581ad6265SDimitry Andric CallInst *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,
16681ad6265SDimitry Andric                                             Value *Val, Value *Size,
16781ad6265SDimitry Andric                                             bool IsVolatile, MDNode *TBAATag,
16881ad6265SDimitry Andric                                             MDNode *ScopeTag,
16981ad6265SDimitry Andric                                             MDNode *NoAliasTag) {
17081ad6265SDimitry Andric   Value *Ops[] = {Dst, Val, Size, getInt1(IsVolatile)};
17181ad6265SDimitry Andric   Type *Tys[] = {Dst->getType(), Size->getType()};
17281ad6265SDimitry Andric   Module *M = BB->getParent()->getParent();
17381ad6265SDimitry Andric   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset_inline, Tys);
17481ad6265SDimitry Andric 
175bdd1243dSDimitry Andric   CallInst *CI = CreateCall(TheFn, Ops);
17681ad6265SDimitry Andric 
17781ad6265SDimitry Andric   if (DstAlign)
17881ad6265SDimitry Andric     cast<MemSetInlineInst>(CI)->setDestAlignment(*DstAlign);
1790b57cec5SDimitry Andric 
1800b57cec5SDimitry Andric   // Set the TBAA info if present.
1810b57cec5SDimitry Andric   if (TBAATag)
1820b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
1830b57cec5SDimitry Andric 
1840b57cec5SDimitry Andric   if (ScopeTag)
1850b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
1860b57cec5SDimitry Andric 
1870b57cec5SDimitry Andric   if (NoAliasTag)
1880b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
1890b57cec5SDimitry Andric 
1900b57cec5SDimitry Andric   return CI;
1910b57cec5SDimitry Andric }
1920b57cec5SDimitry Andric 
1930b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
194480093f4SDimitry Andric     Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
1950b57cec5SDimitry Andric     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
1960b57cec5SDimitry Andric 
1970b57cec5SDimitry Andric   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
1980b57cec5SDimitry Andric   Type *Tys[] = {Ptr->getType(), Size->getType()};
1990b57cec5SDimitry Andric   Module *M = BB->getParent()->getParent();
2000b57cec5SDimitry Andric   Function *TheFn = Intrinsic::getDeclaration(
2010b57cec5SDimitry Andric       M, Intrinsic::memset_element_unordered_atomic, Tys);
2020b57cec5SDimitry Andric 
203bdd1243dSDimitry Andric   CallInst *CI = CreateCall(TheFn, Ops);
2040b57cec5SDimitry Andric 
205480093f4SDimitry Andric   cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment);
2060b57cec5SDimitry Andric 
2070b57cec5SDimitry Andric   // Set the TBAA info if present.
2080b57cec5SDimitry Andric   if (TBAATag)
2090b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
2100b57cec5SDimitry Andric 
2110b57cec5SDimitry Andric   if (ScopeTag)
2120b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
2130b57cec5SDimitry Andric 
2140b57cec5SDimitry Andric   if (NoAliasTag)
2150b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
2160b57cec5SDimitry Andric 
2170b57cec5SDimitry Andric   return CI;
2180b57cec5SDimitry Andric }
2190b57cec5SDimitry Andric 
220e8d8bef9SDimitry Andric CallInst *IRBuilderBase::CreateMemTransferInst(
221e8d8bef9SDimitry Andric     Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src,
222e8d8bef9SDimitry Andric     MaybeAlign SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag,
223e8d8bef9SDimitry Andric     MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
2245f757f3fSDimitry Andric   assert((IntrID == Intrinsic::memcpy || IntrID == Intrinsic::memcpy_inline ||
2255f757f3fSDimitry Andric           IntrID == Intrinsic::memmove) &&
2265f757f3fSDimitry Andric          "Unexpected intrinsic ID");
2270b57cec5SDimitry Andric   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
2280b57cec5SDimitry Andric   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
2290b57cec5SDimitry Andric   Module *M = BB->getParent()->getParent();
230e8d8bef9SDimitry Andric   Function *TheFn = Intrinsic::getDeclaration(M, IntrID, Tys);
2310b57cec5SDimitry Andric 
232bdd1243dSDimitry Andric   CallInst *CI = CreateCall(TheFn, Ops);
2330b57cec5SDimitry Andric 
234e8d8bef9SDimitry Andric   auto* MCI = cast<MemTransferInst>(CI);
235480093f4SDimitry Andric   if (DstAlign)
236480093f4SDimitry Andric     MCI->setDestAlignment(*DstAlign);
237480093f4SDimitry Andric   if (SrcAlign)
238480093f4SDimitry Andric     MCI->setSourceAlignment(*SrcAlign);
2390b57cec5SDimitry Andric 
2400b57cec5SDimitry Andric   // Set the TBAA info if present.
2410b57cec5SDimitry Andric   if (TBAATag)
2420b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
2430b57cec5SDimitry Andric 
2440b57cec5SDimitry Andric   // Set the TBAA Struct info if present.
2450b57cec5SDimitry Andric   if (TBAAStructTag)
2460b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
2470b57cec5SDimitry Andric 
2480b57cec5SDimitry Andric   if (ScopeTag)
2490b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
2500b57cec5SDimitry Andric 
2510b57cec5SDimitry Andric   if (NoAliasTag)
2520b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
2530b57cec5SDimitry Andric 
2540b57cec5SDimitry Andric   return CI;
2550b57cec5SDimitry Andric }
2560b57cec5SDimitry Andric 
2570b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
2585ffd83dbSDimitry Andric     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
2590b57cec5SDimitry Andric     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
2600b57cec5SDimitry Andric     MDNode *ScopeTag, MDNode *NoAliasTag) {
2610b57cec5SDimitry Andric   assert(DstAlign >= ElementSize &&
2620b57cec5SDimitry Andric          "Pointer alignment must be at least element size");
2630b57cec5SDimitry Andric   assert(SrcAlign >= ElementSize &&
2640b57cec5SDimitry Andric          "Pointer alignment must be at least element size");
2650b57cec5SDimitry Andric   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
2660b57cec5SDimitry Andric   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
2670b57cec5SDimitry Andric   Module *M = BB->getParent()->getParent();
2680b57cec5SDimitry Andric   Function *TheFn = Intrinsic::getDeclaration(
2690b57cec5SDimitry Andric       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
2700b57cec5SDimitry Andric 
271bdd1243dSDimitry Andric   CallInst *CI = CreateCall(TheFn, Ops);
2720b57cec5SDimitry Andric 
2730b57cec5SDimitry Andric   // Set the alignment of the pointer args.
2740b57cec5SDimitry Andric   auto *AMCI = cast<AtomicMemCpyInst>(CI);
2750b57cec5SDimitry Andric   AMCI->setDestAlignment(DstAlign);
2760b57cec5SDimitry Andric   AMCI->setSourceAlignment(SrcAlign);
2770b57cec5SDimitry Andric 
2780b57cec5SDimitry Andric   // Set the TBAA info if present.
2790b57cec5SDimitry Andric   if (TBAATag)
2800b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
2810b57cec5SDimitry Andric 
2820b57cec5SDimitry Andric   // Set the TBAA Struct info if present.
2830b57cec5SDimitry Andric   if (TBAAStructTag)
2840b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
2850b57cec5SDimitry Andric 
2860b57cec5SDimitry Andric   if (ScopeTag)
2870b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
2880b57cec5SDimitry Andric 
2890b57cec5SDimitry Andric   if (NoAliasTag)
2900b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
2910b57cec5SDimitry Andric 
2920b57cec5SDimitry Andric   return CI;
2930b57cec5SDimitry Andric }
2940b57cec5SDimitry Andric 
2955f757f3fSDimitry Andric /// isConstantOne - Return true only if val is constant int 1
2965f757f3fSDimitry Andric static bool isConstantOne(const Value *Val) {
2975f757f3fSDimitry Andric   assert(Val && "isConstantOne does not work with nullptr Val");
2985f757f3fSDimitry Andric   const ConstantInt *CVal = dyn_cast<ConstantInt>(Val);
2995f757f3fSDimitry Andric   return CVal && CVal->isOne();
3005f757f3fSDimitry Andric }
3015f757f3fSDimitry Andric 
3025f757f3fSDimitry Andric CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Type *AllocTy,
3035f757f3fSDimitry Andric                                       Value *AllocSize, Value *ArraySize,
3045f757f3fSDimitry Andric                                       ArrayRef<OperandBundleDef> OpB,
3055f757f3fSDimitry Andric                                       Function *MallocF, const Twine &Name) {
3065f757f3fSDimitry Andric   // malloc(type) becomes:
3075f757f3fSDimitry Andric   //       i8* malloc(typeSize)
3085f757f3fSDimitry Andric   // malloc(type, arraySize) becomes:
3095f757f3fSDimitry Andric   //       i8* malloc(typeSize*arraySize)
3105f757f3fSDimitry Andric   if (!ArraySize)
3115f757f3fSDimitry Andric     ArraySize = ConstantInt::get(IntPtrTy, 1);
3125f757f3fSDimitry Andric   else if (ArraySize->getType() != IntPtrTy)
3135f757f3fSDimitry Andric     ArraySize = CreateIntCast(ArraySize, IntPtrTy, false);
3145f757f3fSDimitry Andric 
3155f757f3fSDimitry Andric   if (!isConstantOne(ArraySize)) {
3165f757f3fSDimitry Andric     if (isConstantOne(AllocSize)) {
3175f757f3fSDimitry Andric       AllocSize = ArraySize; // Operand * 1 = Operand
3185f757f3fSDimitry Andric     } else {
3195f757f3fSDimitry Andric       // Multiply type size by the array size...
3205f757f3fSDimitry Andric       AllocSize = CreateMul(ArraySize, AllocSize, "mallocsize");
3215f757f3fSDimitry Andric     }
3225f757f3fSDimitry Andric   }
3235f757f3fSDimitry Andric 
3245f757f3fSDimitry Andric   assert(AllocSize->getType() == IntPtrTy && "malloc arg is wrong size");
3255f757f3fSDimitry Andric   // Create the call to Malloc.
3260b57cec5SDimitry Andric   Module *M = BB->getParent()->getParent();
3275f757f3fSDimitry Andric   Type *BPTy = PointerType::getUnqual(Context);
3285f757f3fSDimitry Andric   FunctionCallee MallocFunc = MallocF;
3295f757f3fSDimitry Andric   if (!MallocFunc)
3305f757f3fSDimitry Andric     // prototype malloc as "void *malloc(size_t)"
3315f757f3fSDimitry Andric     MallocFunc = M->getOrInsertFunction("malloc", BPTy, IntPtrTy);
3325f757f3fSDimitry Andric   CallInst *MCall = CreateCall(MallocFunc, AllocSize, OpB, Name);
3330b57cec5SDimitry Andric 
3345f757f3fSDimitry Andric   MCall->setTailCall();
3355f757f3fSDimitry Andric   if (Function *F = dyn_cast<Function>(MallocFunc.getCallee())) {
3365f757f3fSDimitry Andric     MCall->setCallingConv(F->getCallingConv());
3375f757f3fSDimitry Andric     F->setReturnDoesNotAlias();
3385f757f3fSDimitry Andric   }
3390b57cec5SDimitry Andric 
3405f757f3fSDimitry Andric   assert(!MCall->getType()->isVoidTy() && "Malloc has void return type");
3410b57cec5SDimitry Andric 
3425f757f3fSDimitry Andric   return MCall;
3435f757f3fSDimitry Andric }
3440b57cec5SDimitry Andric 
3455f757f3fSDimitry Andric CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Type *AllocTy,
3465f757f3fSDimitry Andric                                       Value *AllocSize, Value *ArraySize,
3475f757f3fSDimitry Andric                                       Function *MallocF, const Twine &Name) {
3480b57cec5SDimitry Andric 
3495f757f3fSDimitry Andric   return CreateMalloc(IntPtrTy, AllocTy, AllocSize, ArraySize, std::nullopt,
3505f757f3fSDimitry Andric                       MallocF, Name);
3515f757f3fSDimitry Andric }
3520b57cec5SDimitry Andric 
3535f757f3fSDimitry Andric /// CreateFree - Generate the IR for a call to the builtin free function.
3545f757f3fSDimitry Andric CallInst *IRBuilderBase::CreateFree(Value *Source,
3555f757f3fSDimitry Andric                                     ArrayRef<OperandBundleDef> Bundles) {
3565f757f3fSDimitry Andric   assert(Source->getType()->isPointerTy() &&
3575f757f3fSDimitry Andric          "Can not free something of nonpointer type!");
3585f757f3fSDimitry Andric 
3595f757f3fSDimitry Andric   Module *M = BB->getParent()->getParent();
3605f757f3fSDimitry Andric 
3615f757f3fSDimitry Andric   Type *VoidTy = Type::getVoidTy(M->getContext());
3625f757f3fSDimitry Andric   Type *VoidPtrTy = PointerType::getUnqual(M->getContext());
3635f757f3fSDimitry Andric   // prototype free as "void free(void*)"
3645f757f3fSDimitry Andric   FunctionCallee FreeFunc = M->getOrInsertFunction("free", VoidTy, VoidPtrTy);
3655f757f3fSDimitry Andric   CallInst *Result = CreateCall(FreeFunc, Source, Bundles, "");
3665f757f3fSDimitry Andric   Result->setTailCall();
3675f757f3fSDimitry Andric   if (Function *F = dyn_cast<Function>(FreeFunc.getCallee()))
3685f757f3fSDimitry Andric     Result->setCallingConv(F->getCallingConv());
3695f757f3fSDimitry Andric 
3705f757f3fSDimitry Andric   return Result;
3710b57cec5SDimitry Andric }
3720b57cec5SDimitry Andric 
3730b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
3745ffd83dbSDimitry Andric     Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
3750b57cec5SDimitry Andric     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
3760b57cec5SDimitry Andric     MDNode *ScopeTag, MDNode *NoAliasTag) {
3770b57cec5SDimitry Andric   assert(DstAlign >= ElementSize &&
3780b57cec5SDimitry Andric          "Pointer alignment must be at least element size");
3790b57cec5SDimitry Andric   assert(SrcAlign >= ElementSize &&
3800b57cec5SDimitry Andric          "Pointer alignment must be at least element size");
3810b57cec5SDimitry Andric   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
3820b57cec5SDimitry Andric   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
3830b57cec5SDimitry Andric   Module *M = BB->getParent()->getParent();
3840b57cec5SDimitry Andric   Function *TheFn = Intrinsic::getDeclaration(
3850b57cec5SDimitry Andric       M, Intrinsic::memmove_element_unordered_atomic, Tys);
3860b57cec5SDimitry Andric 
387bdd1243dSDimitry Andric   CallInst *CI = CreateCall(TheFn, Ops);
3880b57cec5SDimitry Andric 
3890b57cec5SDimitry Andric   // Set the alignment of the pointer args.
3905ffd83dbSDimitry Andric   CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
3915ffd83dbSDimitry Andric   CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
3920b57cec5SDimitry Andric 
3930b57cec5SDimitry Andric   // Set the TBAA info if present.
3940b57cec5SDimitry Andric   if (TBAATag)
3950b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
3960b57cec5SDimitry Andric 
3970b57cec5SDimitry Andric   // Set the TBAA Struct info if present.
3980b57cec5SDimitry Andric   if (TBAAStructTag)
3990b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
4000b57cec5SDimitry Andric 
4010b57cec5SDimitry Andric   if (ScopeTag)
4020b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
4030b57cec5SDimitry Andric 
4040b57cec5SDimitry Andric   if (NoAliasTag)
4050b57cec5SDimitry Andric     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
4060b57cec5SDimitry Andric 
4070b57cec5SDimitry Andric   return CI;
4080b57cec5SDimitry Andric }
4090b57cec5SDimitry Andric 
410bdd1243dSDimitry Andric CallInst *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
411bdd1243dSDimitry Andric   Module *M = GetInsertBlock()->getParent()->getParent();
4120b57cec5SDimitry Andric   Value *Ops[] = {Src};
4130b57cec5SDimitry Andric   Type *Tys[] = { Src->getType() };
4140b57cec5SDimitry Andric   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
415bdd1243dSDimitry Andric   return CreateCall(Decl, Ops);
4160b57cec5SDimitry Andric }
4170b57cec5SDimitry Andric 
4180b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
4190b57cec5SDimitry Andric   Module *M = GetInsertBlock()->getParent()->getParent();
4200b57cec5SDimitry Andric   Value *Ops[] = {Acc, Src};
421e8d8bef9SDimitry Andric   auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fadd,
422e8d8bef9SDimitry Andric                                         {Src->getType()});
423bdd1243dSDimitry Andric   return CreateCall(Decl, Ops);
4240b57cec5SDimitry Andric }
4250b57cec5SDimitry Andric 
4260b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
4270b57cec5SDimitry Andric   Module *M = GetInsertBlock()->getParent()->getParent();
4280b57cec5SDimitry Andric   Value *Ops[] = {Acc, Src};
429e8d8bef9SDimitry Andric   auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fmul,
430e8d8bef9SDimitry Andric                                         {Src->getType()});
431bdd1243dSDimitry Andric   return CreateCall(Decl, Ops);
4320b57cec5SDimitry Andric }
4330b57cec5SDimitry Andric 
4340b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
435bdd1243dSDimitry Andric   return getReductionIntrinsic(Intrinsic::vector_reduce_add, Src);
4360b57cec5SDimitry Andric }
4370b57cec5SDimitry Andric 
4380b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
439bdd1243dSDimitry Andric   return getReductionIntrinsic(Intrinsic::vector_reduce_mul, Src);
4400b57cec5SDimitry Andric }
4410b57cec5SDimitry Andric 
4420b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
443bdd1243dSDimitry Andric   return getReductionIntrinsic(Intrinsic::vector_reduce_and, Src);
4440b57cec5SDimitry Andric }
4450b57cec5SDimitry Andric 
4460b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
447bdd1243dSDimitry Andric   return getReductionIntrinsic(Intrinsic::vector_reduce_or, Src);
4480b57cec5SDimitry Andric }
4490b57cec5SDimitry Andric 
4500b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
451bdd1243dSDimitry Andric   return getReductionIntrinsic(Intrinsic::vector_reduce_xor, Src);
4520b57cec5SDimitry Andric }
4530b57cec5SDimitry Andric 
4540b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
455e8d8bef9SDimitry Andric   auto ID =
456e8d8bef9SDimitry Andric       IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
457bdd1243dSDimitry Andric   return getReductionIntrinsic(ID, Src);
4580b57cec5SDimitry Andric }
4590b57cec5SDimitry Andric 
4600b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
461e8d8bef9SDimitry Andric   auto ID =
462e8d8bef9SDimitry Andric       IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
463bdd1243dSDimitry Andric   return getReductionIntrinsic(ID, Src);
4640b57cec5SDimitry Andric }
4650b57cec5SDimitry Andric 
466e8d8bef9SDimitry Andric CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
467bdd1243dSDimitry Andric   return getReductionIntrinsic(Intrinsic::vector_reduce_fmax, Src);
4680b57cec5SDimitry Andric }
4690b57cec5SDimitry Andric 
470e8d8bef9SDimitry Andric CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src) {
471bdd1243dSDimitry Andric   return getReductionIntrinsic(Intrinsic::vector_reduce_fmin, Src);
4720b57cec5SDimitry Andric }
4730b57cec5SDimitry Andric 
47406c3fb27SDimitry Andric CallInst *IRBuilderBase::CreateFPMaximumReduce(Value *Src) {
47506c3fb27SDimitry Andric   return getReductionIntrinsic(Intrinsic::vector_reduce_fmaximum, Src);
47606c3fb27SDimitry Andric }
47706c3fb27SDimitry Andric 
47806c3fb27SDimitry Andric CallInst *IRBuilderBase::CreateFPMinimumReduce(Value *Src) {
47906c3fb27SDimitry Andric   return getReductionIntrinsic(Intrinsic::vector_reduce_fminimum, Src);
48006c3fb27SDimitry Andric }
48106c3fb27SDimitry Andric 
4820b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
4830b57cec5SDimitry Andric   assert(isa<PointerType>(Ptr->getType()) &&
4840b57cec5SDimitry Andric          "lifetime.start only applies to pointers.");
4850b57cec5SDimitry Andric   if (!Size)
4860b57cec5SDimitry Andric     Size = getInt64(-1);
4870b57cec5SDimitry Andric   else
4880b57cec5SDimitry Andric     assert(Size->getType() == getInt64Ty() &&
4890b57cec5SDimitry Andric            "lifetime.start requires the size to be an i64");
4900b57cec5SDimitry Andric   Value *Ops[] = { Size, Ptr };
4910b57cec5SDimitry Andric   Module *M = BB->getParent()->getParent();
4920b57cec5SDimitry Andric   Function *TheFn =
4930b57cec5SDimitry Andric       Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
494bdd1243dSDimitry Andric   return CreateCall(TheFn, Ops);
4950b57cec5SDimitry Andric }
4960b57cec5SDimitry Andric 
4970b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
4980b57cec5SDimitry Andric   assert(isa<PointerType>(Ptr->getType()) &&
4990b57cec5SDimitry Andric          "lifetime.end only applies to pointers.");
5000b57cec5SDimitry Andric   if (!Size)
5010b57cec5SDimitry Andric     Size = getInt64(-1);
5020b57cec5SDimitry Andric   else
5030b57cec5SDimitry Andric     assert(Size->getType() == getInt64Ty() &&
5040b57cec5SDimitry Andric            "lifetime.end requires the size to be an i64");
5050b57cec5SDimitry Andric   Value *Ops[] = { Size, Ptr };
5060b57cec5SDimitry Andric   Module *M = BB->getParent()->getParent();
5070b57cec5SDimitry Andric   Function *TheFn =
5080b57cec5SDimitry Andric       Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
509bdd1243dSDimitry Andric   return CreateCall(TheFn, Ops);
5100b57cec5SDimitry Andric }
5110b57cec5SDimitry Andric 
5120b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
5130b57cec5SDimitry Andric 
5140b57cec5SDimitry Andric   assert(isa<PointerType>(Ptr->getType()) &&
5150b57cec5SDimitry Andric          "invariant.start only applies to pointers.");
5160b57cec5SDimitry Andric   if (!Size)
5170b57cec5SDimitry Andric     Size = getInt64(-1);
5180b57cec5SDimitry Andric   else
5190b57cec5SDimitry Andric     assert(Size->getType() == getInt64Ty() &&
5200b57cec5SDimitry Andric            "invariant.start requires the size to be an i64");
5210b57cec5SDimitry Andric 
5220b57cec5SDimitry Andric   Value *Ops[] = {Size, Ptr};
5230b57cec5SDimitry Andric   // Fill in the single overloaded type: memory object type.
5240b57cec5SDimitry Andric   Type *ObjectPtr[1] = {Ptr->getType()};
5250b57cec5SDimitry Andric   Module *M = BB->getParent()->getParent();
5260b57cec5SDimitry Andric   Function *TheFn =
5270b57cec5SDimitry Andric       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
528bdd1243dSDimitry Andric   return CreateCall(TheFn, Ops);
529bdd1243dSDimitry Andric }
530bdd1243dSDimitry Andric 
531bdd1243dSDimitry Andric static MaybeAlign getAlign(Value *Ptr) {
532bdd1243dSDimitry Andric   if (auto *O = dyn_cast<GlobalObject>(Ptr))
533bdd1243dSDimitry Andric     return O->getAlign();
534bdd1243dSDimitry Andric   if (auto *A = dyn_cast<GlobalAlias>(Ptr))
535bdd1243dSDimitry Andric     return A->getAliaseeObject()->getAlign();
536bdd1243dSDimitry Andric   return {};
537bdd1243dSDimitry Andric }
538bdd1243dSDimitry Andric 
539bdd1243dSDimitry Andric CallInst *IRBuilderBase::CreateThreadLocalAddress(Value *Ptr) {
5405f757f3fSDimitry Andric   assert(isa<GlobalValue>(Ptr) && cast<GlobalValue>(Ptr)->isThreadLocal() &&
541bdd1243dSDimitry Andric          "threadlocal_address only applies to thread local variables.");
542bdd1243dSDimitry Andric   CallInst *CI = CreateIntrinsic(llvm::Intrinsic::threadlocal_address,
543bdd1243dSDimitry Andric                                  {Ptr->getType()}, {Ptr});
544bdd1243dSDimitry Andric   if (MaybeAlign A = getAlign(Ptr)) {
545bdd1243dSDimitry Andric     CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), *A));
546bdd1243dSDimitry Andric     CI->addRetAttr(Attribute::getWithAlignment(CI->getContext(), *A));
547bdd1243dSDimitry Andric   }
548bdd1243dSDimitry Andric   return CI;
5490b57cec5SDimitry Andric }
5500b57cec5SDimitry Andric 
551e8d8bef9SDimitry Andric CallInst *
552e8d8bef9SDimitry Andric IRBuilderBase::CreateAssumption(Value *Cond,
553e8d8bef9SDimitry Andric                                 ArrayRef<OperandBundleDef> OpBundles) {
5540b57cec5SDimitry Andric   assert(Cond->getType() == getInt1Ty() &&
5550b57cec5SDimitry Andric          "an assumption condition must be of type i1");
5560b57cec5SDimitry Andric 
5570b57cec5SDimitry Andric   Value *Ops[] = { Cond };
5580b57cec5SDimitry Andric   Module *M = BB->getParent()->getParent();
5590b57cec5SDimitry Andric   Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
560bdd1243dSDimitry Andric   return CreateCall(FnAssume, Ops, OpBundles);
561e8d8bef9SDimitry Andric }
562e8d8bef9SDimitry Andric 
563e8d8bef9SDimitry Andric Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
564e8d8bef9SDimitry Andric   Module *M = BB->getModule();
565e8d8bef9SDimitry Andric   auto *FnIntrinsic = Intrinsic::getDeclaration(
566e8d8bef9SDimitry Andric       M, Intrinsic::experimental_noalias_scope_decl, {});
567bdd1243dSDimitry Andric   return CreateCall(FnIntrinsic, {Scope});
5680b57cec5SDimitry Andric }
5690b57cec5SDimitry Andric 
5700b57cec5SDimitry Andric /// Create a call to a Masked Load intrinsic.
571fe6060f1SDimitry Andric /// \p Ty        - vector type to load
5720b57cec5SDimitry Andric /// \p Ptr       - base pointer for the load
5735ffd83dbSDimitry Andric /// \p Alignment - alignment of the source location
5740b57cec5SDimitry Andric /// \p Mask      - vector of booleans which indicates what vector lanes should
5750b57cec5SDimitry Andric ///                be accessed in memory
5760b57cec5SDimitry Andric /// \p PassThru  - pass-through value that is used to fill the masked-off lanes
5770b57cec5SDimitry Andric ///                of the result
5780b57cec5SDimitry Andric /// \p Name      - name of the result variable
579fe6060f1SDimitry Andric CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
5800b57cec5SDimitry Andric                                           Value *Mask, Value *PassThru,
5810b57cec5SDimitry Andric                                           const Twine &Name) {
5820b57cec5SDimitry Andric   auto *PtrTy = cast<PointerType>(Ptr->getType());
583fe6060f1SDimitry Andric   assert(Ty->isVectorTy() && "Type should be vector");
5840b57cec5SDimitry Andric   assert(Mask && "Mask should not be all-ones (null)");
5850b57cec5SDimitry Andric   if (!PassThru)
586bdd1243dSDimitry Andric     PassThru = PoisonValue::get(Ty);
587fe6060f1SDimitry Andric   Type *OverloadedTypes[] = { Ty, PtrTy };
5885ffd83dbSDimitry Andric   Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru};
5890b57cec5SDimitry Andric   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
5900b57cec5SDimitry Andric                                OverloadedTypes, Name);
5910b57cec5SDimitry Andric }
5920b57cec5SDimitry Andric 
5930b57cec5SDimitry Andric /// Create a call to a Masked Store intrinsic.
5940b57cec5SDimitry Andric /// \p Val       - data to be stored,
5950b57cec5SDimitry Andric /// \p Ptr       - base pointer for the store
5965ffd83dbSDimitry Andric /// \p Alignment - alignment of the destination location
5970b57cec5SDimitry Andric /// \p Mask      - vector of booleans which indicates what vector lanes should
5980b57cec5SDimitry Andric ///                be accessed in memory
5990b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
6005ffd83dbSDimitry Andric                                            Align Alignment, Value *Mask) {
6010b57cec5SDimitry Andric   auto *PtrTy = cast<PointerType>(Ptr->getType());
602fe6060f1SDimitry Andric   Type *DataTy = Val->getType();
603fe6060f1SDimitry Andric   assert(DataTy->isVectorTy() && "Val should be a vector");
6040b57cec5SDimitry Andric   assert(Mask && "Mask should not be all-ones (null)");
6050b57cec5SDimitry Andric   Type *OverloadedTypes[] = { DataTy, PtrTy };
6065ffd83dbSDimitry Andric   Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask};
6070b57cec5SDimitry Andric   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
6080b57cec5SDimitry Andric }
6090b57cec5SDimitry Andric 
6100b57cec5SDimitry Andric /// Create a call to a Masked intrinsic, with given intrinsic Id,
6110b57cec5SDimitry Andric /// an array of operands - Ops, and an array of overloaded types -
6120b57cec5SDimitry Andric /// OverloadedTypes.
6130b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
6140b57cec5SDimitry Andric                                                ArrayRef<Value *> Ops,
6150b57cec5SDimitry Andric                                                ArrayRef<Type *> OverloadedTypes,
6160b57cec5SDimitry Andric                                                const Twine &Name) {
6170b57cec5SDimitry Andric   Module *M = BB->getParent()->getParent();
6180b57cec5SDimitry Andric   Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
619bdd1243dSDimitry Andric   return CreateCall(TheFn, Ops, {}, Name);
6200b57cec5SDimitry Andric }
6210b57cec5SDimitry Andric 
6220b57cec5SDimitry Andric /// Create a call to a Masked Gather intrinsic.
623fe6060f1SDimitry Andric /// \p Ty       - vector type to gather
6240b57cec5SDimitry Andric /// \p Ptrs     - vector of pointers for loading
6250b57cec5SDimitry Andric /// \p Align    - alignment for one element
6260b57cec5SDimitry Andric /// \p Mask     - vector of booleans which indicates what vector lanes should
6270b57cec5SDimitry Andric ///               be accessed in memory
6280b57cec5SDimitry Andric /// \p PassThru - pass-through value that is used to fill the masked-off lanes
6290b57cec5SDimitry Andric ///               of the result
6300b57cec5SDimitry Andric /// \p Name     - name of the result variable
631fe6060f1SDimitry Andric CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
632fe6060f1SDimitry Andric                                             Align Alignment, Value *Mask,
633fe6060f1SDimitry Andric                                             Value *PassThru,
6340b57cec5SDimitry Andric                                             const Twine &Name) {
635fe6060f1SDimitry Andric   auto *VecTy = cast<VectorType>(Ty);
636fe6060f1SDimitry Andric   ElementCount NumElts = VecTy->getElementCount();
637fe6060f1SDimitry Andric   auto *PtrsTy = cast<VectorType>(Ptrs->getType());
638fe6060f1SDimitry Andric   assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");
6390b57cec5SDimitry Andric 
6400b57cec5SDimitry Andric   if (!Mask)
64106c3fb27SDimitry Andric     Mask = getAllOnesMask(NumElts);
6420b57cec5SDimitry Andric 
6430b57cec5SDimitry Andric   if (!PassThru)
644bdd1243dSDimitry Andric     PassThru = PoisonValue::get(Ty);
6450b57cec5SDimitry Andric 
646fe6060f1SDimitry Andric   Type *OverloadedTypes[] = {Ty, PtrsTy};
6475ffd83dbSDimitry Andric   Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru};
6480b57cec5SDimitry Andric 
6490b57cec5SDimitry Andric   // We specify only one type when we create this intrinsic. Types of other
6500b57cec5SDimitry Andric   // arguments are derived from this type.
6510b57cec5SDimitry Andric   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
6520b57cec5SDimitry Andric                                Name);
6530b57cec5SDimitry Andric }
6540b57cec5SDimitry Andric 
6550b57cec5SDimitry Andric /// Create a call to a Masked Scatter intrinsic.
6560b57cec5SDimitry Andric /// \p Data  - data to be stored,
6570b57cec5SDimitry Andric /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
6580b57cec5SDimitry Andric ///            stored
6590b57cec5SDimitry Andric /// \p Align - alignment for one element
6600b57cec5SDimitry Andric /// \p Mask  - vector of booleans which indicates what vector lanes should
6610b57cec5SDimitry Andric ///            be accessed in memory
6620b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
6635ffd83dbSDimitry Andric                                              Align Alignment, Value *Mask) {
664fe6060f1SDimitry Andric   auto *PtrsTy = cast<VectorType>(Ptrs->getType());
665fe6060f1SDimitry Andric   auto *DataTy = cast<VectorType>(Data->getType());
666fe6060f1SDimitry Andric   ElementCount NumElts = PtrsTy->getElementCount();
6670b57cec5SDimitry Andric 
6680b57cec5SDimitry Andric   if (!Mask)
66906c3fb27SDimitry Andric     Mask = getAllOnesMask(NumElts);
6700b57cec5SDimitry Andric 
6710b57cec5SDimitry Andric   Type *OverloadedTypes[] = {DataTy, PtrsTy};
6725ffd83dbSDimitry Andric   Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask};
6730b57cec5SDimitry Andric 
6740b57cec5SDimitry Andric   // We specify only one type when we create this intrinsic. Types of other
6750b57cec5SDimitry Andric   // arguments are derived from this type.
6760b57cec5SDimitry Andric   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
6770b57cec5SDimitry Andric }
6780b57cec5SDimitry Andric 
679bdd1243dSDimitry Andric /// Create a call to Masked Expand Load intrinsic
680bdd1243dSDimitry Andric /// \p Ty        - vector type to load
681bdd1243dSDimitry Andric /// \p Ptr       - base pointer for the load
682bdd1243dSDimitry Andric /// \p Mask      - vector of booleans which indicates what vector lanes should
683bdd1243dSDimitry Andric ///                be accessed in memory
684bdd1243dSDimitry Andric /// \p PassThru  - pass-through value that is used to fill the masked-off lanes
685bdd1243dSDimitry Andric ///                of the result
686bdd1243dSDimitry Andric /// \p Name      - name of the result variable
687bdd1243dSDimitry Andric CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
688bdd1243dSDimitry Andric                                                 Value *Mask, Value *PassThru,
689bdd1243dSDimitry Andric                                                 const Twine &Name) {
690bdd1243dSDimitry Andric   assert(Ty->isVectorTy() && "Type should be vector");
691bdd1243dSDimitry Andric   assert(Mask && "Mask should not be all-ones (null)");
692bdd1243dSDimitry Andric   if (!PassThru)
693bdd1243dSDimitry Andric     PassThru = PoisonValue::get(Ty);
694bdd1243dSDimitry Andric   Type *OverloadedTypes[] = {Ty};
695bdd1243dSDimitry Andric   Value *Ops[] = {Ptr, Mask, PassThru};
696bdd1243dSDimitry Andric   return CreateMaskedIntrinsic(Intrinsic::masked_expandload, Ops,
697bdd1243dSDimitry Andric                                OverloadedTypes, Name);
698bdd1243dSDimitry Andric }
699bdd1243dSDimitry Andric 
700bdd1243dSDimitry Andric /// Create a call to Masked Compress Store intrinsic
701bdd1243dSDimitry Andric /// \p Val       - data to be stored,
702bdd1243dSDimitry Andric /// \p Ptr       - base pointer for the store
703bdd1243dSDimitry Andric /// \p Mask      - vector of booleans which indicates what vector lanes should
704bdd1243dSDimitry Andric ///                be accessed in memory
705bdd1243dSDimitry Andric CallInst *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr,
706bdd1243dSDimitry Andric                                                    Value *Mask) {
707bdd1243dSDimitry Andric   Type *DataTy = Val->getType();
708bdd1243dSDimitry Andric   assert(DataTy->isVectorTy() && "Val should be a vector");
709bdd1243dSDimitry Andric   assert(Mask && "Mask should not be all-ones (null)");
710bdd1243dSDimitry Andric   Type *OverloadedTypes[] = {DataTy};
711bdd1243dSDimitry Andric   Value *Ops[] = {Val, Ptr, Mask};
712bdd1243dSDimitry Andric   return CreateMaskedIntrinsic(Intrinsic::masked_compressstore, Ops,
713bdd1243dSDimitry Andric                                OverloadedTypes);
714bdd1243dSDimitry Andric }
715bdd1243dSDimitry Andric 
7165ffd83dbSDimitry Andric template <typename T0>
7170b57cec5SDimitry Andric static std::vector<Value *>
7180b57cec5SDimitry Andric getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
7195ffd83dbSDimitry Andric                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
7200b57cec5SDimitry Andric   std::vector<Value *> Args;
7210b57cec5SDimitry Andric   Args.push_back(B.getInt64(ID));
7220b57cec5SDimitry Andric   Args.push_back(B.getInt32(NumPatchBytes));
7230b57cec5SDimitry Andric   Args.push_back(ActualCallee);
7240b57cec5SDimitry Andric   Args.push_back(B.getInt32(CallArgs.size()));
7250b57cec5SDimitry Andric   Args.push_back(B.getInt32(Flags));
726e8d8bef9SDimitry Andric   llvm::append_range(Args, CallArgs);
7275ffd83dbSDimitry Andric   // GC Transition and Deopt args are now always handled via operand bundle.
7285ffd83dbSDimitry Andric   // They will be removed from the signature of gc.statepoint shortly.
7295ffd83dbSDimitry Andric   Args.push_back(B.getInt32(0));
7305ffd83dbSDimitry Andric   Args.push_back(B.getInt32(0));
7315ffd83dbSDimitry Andric   // GC args are now encoded in the gc-live operand bundle
7320b57cec5SDimitry Andric   return Args;
7330b57cec5SDimitry Andric }
7340b57cec5SDimitry Andric 
7355ffd83dbSDimitry Andric template<typename T1, typename T2, typename T3>
7365ffd83dbSDimitry Andric static std::vector<OperandBundleDef>
737bdd1243dSDimitry Andric getStatepointBundles(std::optional<ArrayRef<T1>> TransitionArgs,
738bdd1243dSDimitry Andric                      std::optional<ArrayRef<T2>> DeoptArgs,
7395ffd83dbSDimitry Andric                      ArrayRef<T3> GCArgs) {
7405ffd83dbSDimitry Andric   std::vector<OperandBundleDef> Rval;
7415ffd83dbSDimitry Andric   if (DeoptArgs) {
7425ffd83dbSDimitry Andric     SmallVector<Value*, 16> DeoptValues;
743e8d8bef9SDimitry Andric     llvm::append_range(DeoptValues, *DeoptArgs);
7445ffd83dbSDimitry Andric     Rval.emplace_back("deopt", DeoptValues);
7455ffd83dbSDimitry Andric   }
7465ffd83dbSDimitry Andric   if (TransitionArgs) {
7475ffd83dbSDimitry Andric     SmallVector<Value*, 16> TransitionValues;
748e8d8bef9SDimitry Andric     llvm::append_range(TransitionValues, *TransitionArgs);
7495ffd83dbSDimitry Andric     Rval.emplace_back("gc-transition", TransitionValues);
7505ffd83dbSDimitry Andric   }
7515ffd83dbSDimitry Andric   if (GCArgs.size()) {
7525ffd83dbSDimitry Andric     SmallVector<Value*, 16> LiveValues;
753e8d8bef9SDimitry Andric     llvm::append_range(LiveValues, GCArgs);
7545ffd83dbSDimitry Andric     Rval.emplace_back("gc-live", LiveValues);
7555ffd83dbSDimitry Andric   }
7565ffd83dbSDimitry Andric   return Rval;
7575ffd83dbSDimitry Andric }
7585ffd83dbSDimitry Andric 
7590b57cec5SDimitry Andric template <typename T0, typename T1, typename T2, typename T3>
7600b57cec5SDimitry Andric static CallInst *CreateGCStatepointCallCommon(
7610b57cec5SDimitry Andric     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
76281ad6265SDimitry Andric     FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
763bdd1243dSDimitry Andric     std::optional<ArrayRef<T1>> TransitionArgs,
764bdd1243dSDimitry Andric     std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
765bdd1243dSDimitry Andric     const Twine &Name) {
7660b57cec5SDimitry Andric   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
7670b57cec5SDimitry Andric   // Fill in the one generic type'd argument (the function is also vararg)
7680b57cec5SDimitry Andric   Function *FnStatepoint =
7690b57cec5SDimitry Andric       Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
77081ad6265SDimitry Andric                                 {ActualCallee.getCallee()->getType()});
7710b57cec5SDimitry Andric 
77281ad6265SDimitry Andric   std::vector<Value *> Args = getStatepointArgs(
77381ad6265SDimitry Andric       *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs);
7745ffd83dbSDimitry Andric 
77581ad6265SDimitry Andric   CallInst *CI = Builder->CreateCall(
77681ad6265SDimitry Andric       FnStatepoint, Args,
77781ad6265SDimitry Andric       getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
77881ad6265SDimitry Andric   CI->addParamAttr(2,
77981ad6265SDimitry Andric                    Attribute::get(Builder->getContext(), Attribute::ElementType,
78081ad6265SDimitry Andric                                   ActualCallee.getFunctionType()));
78181ad6265SDimitry Andric   return CI;
7820b57cec5SDimitry Andric }
7830b57cec5SDimitry Andric 
7840b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateGCStatepointCall(
78581ad6265SDimitry Andric     uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
786bdd1243dSDimitry Andric     ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
7870b57cec5SDimitry Andric     ArrayRef<Value *> GCArgs, const Twine &Name) {
7880b57cec5SDimitry Andric   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
7890b57cec5SDimitry Andric       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
790bdd1243dSDimitry Andric       CallArgs, std::nullopt /* No Transition Args */, DeoptArgs, GCArgs, Name);
7910b57cec5SDimitry Andric }
7920b57cec5SDimitry Andric 
7930b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateGCStatepointCall(
79481ad6265SDimitry Andric     uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
79581ad6265SDimitry Andric     uint32_t Flags, ArrayRef<Value *> CallArgs,
796bdd1243dSDimitry Andric     std::optional<ArrayRef<Use>> TransitionArgs,
797bdd1243dSDimitry Andric     std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
798bdd1243dSDimitry Andric     const Twine &Name) {
799e8d8bef9SDimitry Andric   return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
8000b57cec5SDimitry Andric       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
8010b57cec5SDimitry Andric       DeoptArgs, GCArgs, Name);
8020b57cec5SDimitry Andric }
8030b57cec5SDimitry Andric 
8040b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateGCStatepointCall(
80581ad6265SDimitry Andric     uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
806bdd1243dSDimitry Andric     ArrayRef<Use> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
8070b57cec5SDimitry Andric     ArrayRef<Value *> GCArgs, const Twine &Name) {
8080b57cec5SDimitry Andric   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
8090b57cec5SDimitry Andric       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
810bdd1243dSDimitry Andric       CallArgs, std::nullopt, DeoptArgs, GCArgs, Name);
8110b57cec5SDimitry Andric }
8120b57cec5SDimitry Andric 
8130b57cec5SDimitry Andric template <typename T0, typename T1, typename T2, typename T3>
8140b57cec5SDimitry Andric static InvokeInst *CreateGCStatepointInvokeCommon(
8150b57cec5SDimitry Andric     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
81681ad6265SDimitry Andric     FunctionCallee ActualInvokee, BasicBlock *NormalDest,
81781ad6265SDimitry Andric     BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<T0> InvokeArgs,
818bdd1243dSDimitry Andric     std::optional<ArrayRef<T1>> TransitionArgs,
819bdd1243dSDimitry Andric     std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
820bdd1243dSDimitry Andric     const Twine &Name) {
8210b57cec5SDimitry Andric   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
8220b57cec5SDimitry Andric   // Fill in the one generic type'd argument (the function is also vararg)
82381ad6265SDimitry Andric   Function *FnStatepoint =
82481ad6265SDimitry Andric       Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
82581ad6265SDimitry Andric                                 {ActualInvokee.getCallee()->getType()});
8260b57cec5SDimitry Andric 
8270b57cec5SDimitry Andric   std::vector<Value *> Args =
82881ad6265SDimitry Andric       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(),
82981ad6265SDimitry Andric                         Flags, InvokeArgs);
8305ffd83dbSDimitry Andric 
83181ad6265SDimitry Andric   InvokeInst *II = Builder->CreateInvoke(
83281ad6265SDimitry Andric       FnStatepoint, NormalDest, UnwindDest, Args,
83381ad6265SDimitry Andric       getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
83481ad6265SDimitry Andric   II->addParamAttr(2,
83581ad6265SDimitry Andric                    Attribute::get(Builder->getContext(), Attribute::ElementType,
83681ad6265SDimitry Andric                                   ActualInvokee.getFunctionType()));
83781ad6265SDimitry Andric   return II;
8380b57cec5SDimitry Andric }
8390b57cec5SDimitry Andric 
8400b57cec5SDimitry Andric InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
84181ad6265SDimitry Andric     uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
8420b57cec5SDimitry Andric     BasicBlock *NormalDest, BasicBlock *UnwindDest,
843bdd1243dSDimitry Andric     ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
8440b57cec5SDimitry Andric     ArrayRef<Value *> GCArgs, const Twine &Name) {
8450b57cec5SDimitry Andric   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
8460b57cec5SDimitry Andric       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
847bdd1243dSDimitry Andric       uint32_t(StatepointFlags::None), InvokeArgs,
848bdd1243dSDimitry Andric       std::nullopt /* No Transition Args*/, DeoptArgs, GCArgs, Name);
8490b57cec5SDimitry Andric }
8500b57cec5SDimitry Andric 
8510b57cec5SDimitry Andric InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
85281ad6265SDimitry Andric     uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
8530b57cec5SDimitry Andric     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
854bdd1243dSDimitry Andric     ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Use>> TransitionArgs,
855bdd1243dSDimitry Andric     std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
85681ad6265SDimitry Andric     const Twine &Name) {
857e8d8bef9SDimitry Andric   return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
8580b57cec5SDimitry Andric       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
8590b57cec5SDimitry Andric       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
8600b57cec5SDimitry Andric }
8610b57cec5SDimitry Andric 
8620b57cec5SDimitry Andric InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
86381ad6265SDimitry Andric     uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
8640b57cec5SDimitry Andric     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
865bdd1243dSDimitry Andric     std::optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs,
86681ad6265SDimitry Andric     const Twine &Name) {
8670b57cec5SDimitry Andric   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
8680b57cec5SDimitry Andric       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
869bdd1243dSDimitry Andric       uint32_t(StatepointFlags::None), InvokeArgs, std::nullopt, DeoptArgs,
870bdd1243dSDimitry Andric       GCArgs, Name);
8710b57cec5SDimitry Andric }
8720b57cec5SDimitry Andric 
8730b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
87481ad6265SDimitry Andric                                         Type *ResultType, const Twine &Name) {
8750b57cec5SDimitry Andric   Intrinsic::ID ID = Intrinsic::experimental_gc_result;
8760b57cec5SDimitry Andric   Module *M = BB->getParent()->getParent();
8770b57cec5SDimitry Andric   Type *Types[] = {ResultType};
8780b57cec5SDimitry Andric   Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
8790b57cec5SDimitry Andric 
8800b57cec5SDimitry Andric   Value *Args[] = {Statepoint};
881bdd1243dSDimitry Andric   return CreateCall(FnGCResult, Args, {}, Name);
8820b57cec5SDimitry Andric }
8830b57cec5SDimitry Andric 
8840b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
88581ad6265SDimitry Andric                                           int BaseOffset, int DerivedOffset,
88681ad6265SDimitry Andric                                           Type *ResultType, const Twine &Name) {
8870b57cec5SDimitry Andric   Module *M = BB->getParent()->getParent();
8880b57cec5SDimitry Andric   Type *Types[] = {ResultType};
8890b57cec5SDimitry Andric   Function *FnGCRelocate =
8900b57cec5SDimitry Andric       Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
8910b57cec5SDimitry Andric 
89281ad6265SDimitry Andric   Value *Args[] = {Statepoint, getInt32(BaseOffset), getInt32(DerivedOffset)};
893bdd1243dSDimitry Andric   return CreateCall(FnGCRelocate, Args, {}, Name);
8940b57cec5SDimitry Andric }
8950b57cec5SDimitry Andric 
896fe6060f1SDimitry Andric CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr,
897fe6060f1SDimitry Andric                                                 const Twine &Name) {
898fe6060f1SDimitry Andric   Module *M = BB->getParent()->getParent();
899fe6060f1SDimitry Andric   Type *PtrTy = DerivedPtr->getType();
900fe6060f1SDimitry Andric   Function *FnGCFindBase = Intrinsic::getDeclaration(
901fe6060f1SDimitry Andric       M, Intrinsic::experimental_gc_get_pointer_base, {PtrTy, PtrTy});
902bdd1243dSDimitry Andric   return CreateCall(FnGCFindBase, {DerivedPtr}, {}, Name);
903fe6060f1SDimitry Andric }
904fe6060f1SDimitry Andric 
905fe6060f1SDimitry Andric CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr,
906fe6060f1SDimitry Andric                                                   const Twine &Name) {
907fe6060f1SDimitry Andric   Module *M = BB->getParent()->getParent();
908fe6060f1SDimitry Andric   Type *PtrTy = DerivedPtr->getType();
909fe6060f1SDimitry Andric   Function *FnGCGetOffset = Intrinsic::getDeclaration(
910fe6060f1SDimitry Andric       M, Intrinsic::experimental_gc_get_pointer_offset, {PtrTy});
911bdd1243dSDimitry Andric   return CreateCall(FnGCGetOffset, {DerivedPtr}, {}, Name);
912fe6060f1SDimitry Andric }
913fe6060f1SDimitry Andric 
9140b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
9150b57cec5SDimitry Andric                                               Instruction *FMFSource,
9160b57cec5SDimitry Andric                                               const Twine &Name) {
9170b57cec5SDimitry Andric   Module *M = BB->getModule();
9180b57cec5SDimitry Andric   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
919bdd1243dSDimitry Andric   return createCallHelper(Fn, {V}, Name, FMFSource);
9200b57cec5SDimitry Andric }
9210b57cec5SDimitry Andric 
922*0fca6ea1SDimitry Andric Value *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
923*0fca6ea1SDimitry Andric                                             Value *RHS, Instruction *FMFSource,
9240b57cec5SDimitry Andric                                             const Twine &Name) {
9250b57cec5SDimitry Andric   Module *M = BB->getModule();
9260b57cec5SDimitry Andric   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
927*0fca6ea1SDimitry Andric   if (Value *V = Folder.FoldBinaryIntrinsic(ID, LHS, RHS, Fn->getReturnType(),
928*0fca6ea1SDimitry Andric                                             FMFSource))
929*0fca6ea1SDimitry Andric     return V;
930bdd1243dSDimitry Andric   return createCallHelper(Fn, {LHS, RHS}, Name, FMFSource);
9310b57cec5SDimitry Andric }
9320b57cec5SDimitry Andric 
9330b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
9340b57cec5SDimitry Andric                                          ArrayRef<Type *> Types,
9350b57cec5SDimitry Andric                                          ArrayRef<Value *> Args,
9360b57cec5SDimitry Andric                                          Instruction *FMFSource,
9370b57cec5SDimitry Andric                                          const Twine &Name) {
9380b57cec5SDimitry Andric   Module *M = BB->getModule();
9390b57cec5SDimitry Andric   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
940bdd1243dSDimitry Andric   return createCallHelper(Fn, Args, Name, FMFSource);
941bdd1243dSDimitry Andric }
942bdd1243dSDimitry Andric 
943bdd1243dSDimitry Andric CallInst *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
944bdd1243dSDimitry Andric                                          ArrayRef<Value *> Args,
945bdd1243dSDimitry Andric                                          Instruction *FMFSource,
946bdd1243dSDimitry Andric                                          const Twine &Name) {
947bdd1243dSDimitry Andric   Module *M = BB->getModule();
948bdd1243dSDimitry Andric 
949bdd1243dSDimitry Andric   SmallVector<Intrinsic::IITDescriptor> Table;
950bdd1243dSDimitry Andric   Intrinsic::getIntrinsicInfoTableEntries(ID, Table);
951bdd1243dSDimitry Andric   ArrayRef<Intrinsic::IITDescriptor> TableRef(Table);
952bdd1243dSDimitry Andric 
953bdd1243dSDimitry Andric   SmallVector<Type *> ArgTys;
954bdd1243dSDimitry Andric   ArgTys.reserve(Args.size());
955bdd1243dSDimitry Andric   for (auto &I : Args)
956bdd1243dSDimitry Andric     ArgTys.push_back(I->getType());
957bdd1243dSDimitry Andric   FunctionType *FTy = FunctionType::get(RetTy, ArgTys, false);
958bdd1243dSDimitry Andric   SmallVector<Type *> OverloadTys;
959bdd1243dSDimitry Andric   Intrinsic::MatchIntrinsicTypesResult Res =
960bdd1243dSDimitry Andric       matchIntrinsicSignature(FTy, TableRef, OverloadTys);
961bdd1243dSDimitry Andric   (void)Res;
962bdd1243dSDimitry Andric   assert(Res == Intrinsic::MatchIntrinsicTypes_Match && TableRef.empty() &&
963bdd1243dSDimitry Andric          "Wrong types for intrinsic!");
964bdd1243dSDimitry Andric   // TODO: Handle varargs intrinsics.
965bdd1243dSDimitry Andric 
966bdd1243dSDimitry Andric   Function *Fn = Intrinsic::getDeclaration(M, ID, OverloadTys);
967bdd1243dSDimitry Andric   return createCallHelper(Fn, Args, Name, FMFSource);
9680b57cec5SDimitry Andric }
9695ffd83dbSDimitry Andric 
9705ffd83dbSDimitry Andric CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
9715ffd83dbSDimitry Andric     Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource,
9725ffd83dbSDimitry Andric     const Twine &Name, MDNode *FPMathTag,
973bdd1243dSDimitry Andric     std::optional<RoundingMode> Rounding,
974bdd1243dSDimitry Andric     std::optional<fp::ExceptionBehavior> Except) {
9755ffd83dbSDimitry Andric   Value *RoundingV = getConstrainedFPRounding(Rounding);
9765ffd83dbSDimitry Andric   Value *ExceptV = getConstrainedFPExcept(Except);
9775ffd83dbSDimitry Andric 
9785ffd83dbSDimitry Andric   FastMathFlags UseFMF = FMF;
9795ffd83dbSDimitry Andric   if (FMFSource)
9805ffd83dbSDimitry Andric     UseFMF = FMFSource->getFastMathFlags();
9815ffd83dbSDimitry Andric 
9825ffd83dbSDimitry Andric   CallInst *C = CreateIntrinsic(ID, {L->getType()},
9835ffd83dbSDimitry Andric                                 {L, R, RoundingV, ExceptV}, nullptr, Name);
9845ffd83dbSDimitry Andric   setConstrainedFPCallAttr(C);
9855ffd83dbSDimitry Andric   setFPAttrs(C, FPMathTag, UseFMF);
9865ffd83dbSDimitry Andric   return C;
9875ffd83dbSDimitry Andric }
9885ffd83dbSDimitry Andric 
98906c3fb27SDimitry Andric CallInst *IRBuilderBase::CreateConstrainedFPUnroundedBinOp(
99006c3fb27SDimitry Andric     Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource,
99106c3fb27SDimitry Andric     const Twine &Name, MDNode *FPMathTag,
99206c3fb27SDimitry Andric     std::optional<fp::ExceptionBehavior> Except) {
99306c3fb27SDimitry Andric   Value *ExceptV = getConstrainedFPExcept(Except);
99406c3fb27SDimitry Andric 
99506c3fb27SDimitry Andric   FastMathFlags UseFMF = FMF;
99606c3fb27SDimitry Andric   if (FMFSource)
99706c3fb27SDimitry Andric     UseFMF = FMFSource->getFastMathFlags();
99806c3fb27SDimitry Andric 
99906c3fb27SDimitry Andric   CallInst *C =
100006c3fb27SDimitry Andric       CreateIntrinsic(ID, {L->getType()}, {L, R, ExceptV}, nullptr, Name);
100106c3fb27SDimitry Andric   setConstrainedFPCallAttr(C);
100206c3fb27SDimitry Andric   setFPAttrs(C, FPMathTag, UseFMF);
100306c3fb27SDimitry Andric   return C;
100406c3fb27SDimitry Andric }
100506c3fb27SDimitry Andric 
10065ffd83dbSDimitry Andric Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
10075ffd83dbSDimitry Andric                                    const Twine &Name, MDNode *FPMathTag) {
10085ffd83dbSDimitry Andric   if (Instruction::isBinaryOp(Opc)) {
10095ffd83dbSDimitry Andric     assert(Ops.size() == 2 && "Invalid number of operands!");
10105ffd83dbSDimitry Andric     return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc),
10115ffd83dbSDimitry Andric                        Ops[0], Ops[1], Name, FPMathTag);
10125ffd83dbSDimitry Andric   }
10135ffd83dbSDimitry Andric   if (Instruction::isUnaryOp(Opc)) {
10145ffd83dbSDimitry Andric     assert(Ops.size() == 1 && "Invalid number of operands!");
10155ffd83dbSDimitry Andric     return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc),
10165ffd83dbSDimitry Andric                       Ops[0], Name, FPMathTag);
10175ffd83dbSDimitry Andric   }
10185ffd83dbSDimitry Andric   llvm_unreachable("Unexpected opcode!");
10195ffd83dbSDimitry Andric }
10205ffd83dbSDimitry Andric 
10215ffd83dbSDimitry Andric CallInst *IRBuilderBase::CreateConstrainedFPCast(
10225ffd83dbSDimitry Andric     Intrinsic::ID ID, Value *V, Type *DestTy,
10235ffd83dbSDimitry Andric     Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag,
1024bdd1243dSDimitry Andric     std::optional<RoundingMode> Rounding,
1025bdd1243dSDimitry Andric     std::optional<fp::ExceptionBehavior> Except) {
10265ffd83dbSDimitry Andric   Value *ExceptV = getConstrainedFPExcept(Except);
10275ffd83dbSDimitry Andric 
10285ffd83dbSDimitry Andric   FastMathFlags UseFMF = FMF;
10295ffd83dbSDimitry Andric   if (FMFSource)
10305ffd83dbSDimitry Andric     UseFMF = FMFSource->getFastMathFlags();
10315ffd83dbSDimitry Andric 
10325ffd83dbSDimitry Andric   CallInst *C;
1033*0fca6ea1SDimitry Andric   if (Intrinsic::hasConstrainedFPRoundingModeOperand(ID)) {
10345ffd83dbSDimitry Andric     Value *RoundingV = getConstrainedFPRounding(Rounding);
10355ffd83dbSDimitry Andric     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV},
10365ffd83dbSDimitry Andric                         nullptr, Name);
10375ffd83dbSDimitry Andric   } else
10385ffd83dbSDimitry Andric     C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr,
10395ffd83dbSDimitry Andric                         Name);
10405ffd83dbSDimitry Andric 
10415ffd83dbSDimitry Andric   setConstrainedFPCallAttr(C);
10425ffd83dbSDimitry Andric 
10435ffd83dbSDimitry Andric   if (isa<FPMathOperator>(C))
10445ffd83dbSDimitry Andric     setFPAttrs(C, FPMathTag, UseFMF);
10455ffd83dbSDimitry Andric   return C;
10465ffd83dbSDimitry Andric }
10475ffd83dbSDimitry Andric 
10485ffd83dbSDimitry Andric Value *IRBuilderBase::CreateFCmpHelper(
10495ffd83dbSDimitry Andric     CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name,
10505ffd83dbSDimitry Andric     MDNode *FPMathTag, bool IsSignaling) {
10515ffd83dbSDimitry Andric   if (IsFPConstrained) {
10525ffd83dbSDimitry Andric     auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
10535ffd83dbSDimitry Andric                           : Intrinsic::experimental_constrained_fcmp;
10545ffd83dbSDimitry Andric     return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name);
10555ffd83dbSDimitry Andric   }
10565ffd83dbSDimitry Andric 
1057*0fca6ea1SDimitry Andric   if (auto *V = Folder.FoldCmp(P, LHS, RHS))
1058*0fca6ea1SDimitry Andric     return V;
10595ffd83dbSDimitry Andric   return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name);
10605ffd83dbSDimitry Andric }
10615ffd83dbSDimitry Andric 
10625ffd83dbSDimitry Andric CallInst *IRBuilderBase::CreateConstrainedFPCmp(
10635ffd83dbSDimitry Andric     Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
1064bdd1243dSDimitry Andric     const Twine &Name, std::optional<fp::ExceptionBehavior> Except) {
10655ffd83dbSDimitry Andric   Value *PredicateV = getConstrainedFPPredicate(P);
10665ffd83dbSDimitry Andric   Value *ExceptV = getConstrainedFPExcept(Except);
10675ffd83dbSDimitry Andric 
10685ffd83dbSDimitry Andric   CallInst *C = CreateIntrinsic(ID, {L->getType()},
10695ffd83dbSDimitry Andric                                 {L, R, PredicateV, ExceptV}, nullptr, Name);
10705ffd83dbSDimitry Andric   setConstrainedFPCallAttr(C);
10715ffd83dbSDimitry Andric   return C;
10725ffd83dbSDimitry Andric }
10735ffd83dbSDimitry Andric 
10745ffd83dbSDimitry Andric CallInst *IRBuilderBase::CreateConstrainedFPCall(
10755ffd83dbSDimitry Andric     Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
1076bdd1243dSDimitry Andric     std::optional<RoundingMode> Rounding,
1077bdd1243dSDimitry Andric     std::optional<fp::ExceptionBehavior> Except) {
10785ffd83dbSDimitry Andric   llvm::SmallVector<Value *, 6> UseArgs;
10795ffd83dbSDimitry Andric 
1080fe6060f1SDimitry Andric   append_range(UseArgs, Args);
1081*0fca6ea1SDimitry Andric 
1082*0fca6ea1SDimitry Andric   if (Intrinsic::hasConstrainedFPRoundingModeOperand(Callee->getIntrinsicID()))
10835ffd83dbSDimitry Andric     UseArgs.push_back(getConstrainedFPRounding(Rounding));
10845ffd83dbSDimitry Andric   UseArgs.push_back(getConstrainedFPExcept(Except));
10855ffd83dbSDimitry Andric 
10865ffd83dbSDimitry Andric   CallInst *C = CreateCall(Callee, UseArgs, Name);
10875ffd83dbSDimitry Andric   setConstrainedFPCallAttr(C);
10885ffd83dbSDimitry Andric   return C;
10895ffd83dbSDimitry Andric }
10905ffd83dbSDimitry Andric 
10915ffd83dbSDimitry Andric Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,
10925ffd83dbSDimitry Andric                                    const Twine &Name, Instruction *MDFrom) {
109304eeddc0SDimitry Andric   if (auto *V = Folder.FoldSelect(C, True, False))
109404eeddc0SDimitry Andric     return V;
10955ffd83dbSDimitry Andric 
10965ffd83dbSDimitry Andric   SelectInst *Sel = SelectInst::Create(C, True, False);
10975ffd83dbSDimitry Andric   if (MDFrom) {
10985ffd83dbSDimitry Andric     MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof);
10995ffd83dbSDimitry Andric     MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable);
11005ffd83dbSDimitry Andric     Sel = addBranchMetadata(Sel, Prof, Unpred);
11015ffd83dbSDimitry Andric   }
11025ffd83dbSDimitry Andric   if (isa<FPMathOperator>(Sel))
11035ffd83dbSDimitry Andric     setFPAttrs(Sel, nullptr /* MDNode* */, FMF);
11045ffd83dbSDimitry Andric   return Insert(Sel, Name);
11055ffd83dbSDimitry Andric }
11065ffd83dbSDimitry Andric 
110704eeddc0SDimitry Andric Value *IRBuilderBase::CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS,
11085ffd83dbSDimitry Andric                                     const Twine &Name) {
11095ffd83dbSDimitry Andric   assert(LHS->getType() == RHS->getType() &&
11105ffd83dbSDimitry Andric          "Pointer subtraction operand types must match!");
11115ffd83dbSDimitry Andric   Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context));
11125ffd83dbSDimitry Andric   Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context));
11135ffd83dbSDimitry Andric   Value *Difference = CreateSub(LHS_int, RHS_int);
111404eeddc0SDimitry Andric   return CreateExactSDiv(Difference, ConstantExpr::getSizeOf(ElemTy),
11155ffd83dbSDimitry Andric                          Name);
11165ffd83dbSDimitry Andric }
11175ffd83dbSDimitry Andric 
11185ffd83dbSDimitry Andric Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
11195ffd83dbSDimitry Andric   assert(isa<PointerType>(Ptr->getType()) &&
11205ffd83dbSDimitry Andric          "launder.invariant.group only applies to pointers.");
11215ffd83dbSDimitry Andric   auto *PtrType = Ptr->getType();
11225ffd83dbSDimitry Andric   Module *M = BB->getParent()->getParent();
11235ffd83dbSDimitry Andric   Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration(
112406c3fb27SDimitry Andric       M, Intrinsic::launder_invariant_group, {PtrType});
11255ffd83dbSDimitry Andric 
112606c3fb27SDimitry Andric   assert(FnLaunderInvariantGroup->getReturnType() == PtrType &&
11275ffd83dbSDimitry Andric          FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
112806c3fb27SDimitry Andric              PtrType &&
11295ffd83dbSDimitry Andric          "LaunderInvariantGroup should take and return the same type");
11305ffd83dbSDimitry Andric 
113106c3fb27SDimitry Andric   return CreateCall(FnLaunderInvariantGroup, {Ptr});
11325ffd83dbSDimitry Andric }
11335ffd83dbSDimitry Andric 
11345ffd83dbSDimitry Andric Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
11355ffd83dbSDimitry Andric   assert(isa<PointerType>(Ptr->getType()) &&
11365ffd83dbSDimitry Andric          "strip.invariant.group only applies to pointers.");
11375ffd83dbSDimitry Andric 
11385ffd83dbSDimitry Andric   auto *PtrType = Ptr->getType();
11395ffd83dbSDimitry Andric   Module *M = BB->getParent()->getParent();
11405ffd83dbSDimitry Andric   Function *FnStripInvariantGroup = Intrinsic::getDeclaration(
114106c3fb27SDimitry Andric       M, Intrinsic::strip_invariant_group, {PtrType});
11425ffd83dbSDimitry Andric 
114306c3fb27SDimitry Andric   assert(FnStripInvariantGroup->getReturnType() == PtrType &&
11445ffd83dbSDimitry Andric          FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
114506c3fb27SDimitry Andric              PtrType &&
11465ffd83dbSDimitry Andric          "StripInvariantGroup should take and return the same type");
11475ffd83dbSDimitry Andric 
114806c3fb27SDimitry Andric   return CreateCall(FnStripInvariantGroup, {Ptr});
11495ffd83dbSDimitry Andric }
11505ffd83dbSDimitry Andric 
1151fe6060f1SDimitry Andric Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) {
1152fe6060f1SDimitry Andric   auto *Ty = cast<VectorType>(V->getType());
1153fe6060f1SDimitry Andric   if (isa<ScalableVectorType>(Ty)) {
1154fe6060f1SDimitry Andric     Module *M = BB->getParent()->getParent();
1155*0fca6ea1SDimitry Andric     Function *F = Intrinsic::getDeclaration(M, Intrinsic::vector_reverse, Ty);
1156fe6060f1SDimitry Andric     return Insert(CallInst::Create(F, V), Name);
1157fe6060f1SDimitry Andric   }
1158fe6060f1SDimitry Andric   // Keep the original behaviour for fixed vector
1159fe6060f1SDimitry Andric   SmallVector<int, 8> ShuffleMask;
1160fe6060f1SDimitry Andric   int NumElts = Ty->getElementCount().getKnownMinValue();
1161fe6060f1SDimitry Andric   for (int i = 0; i < NumElts; ++i)
1162fe6060f1SDimitry Andric     ShuffleMask.push_back(NumElts - i - 1);
1163fe6060f1SDimitry Andric   return CreateShuffleVector(V, ShuffleMask, Name);
1164fe6060f1SDimitry Andric }
1165fe6060f1SDimitry Andric 
1166fe6060f1SDimitry Andric Value *IRBuilderBase::CreateVectorSplice(Value *V1, Value *V2, int64_t Imm,
1167fe6060f1SDimitry Andric                                          const Twine &Name) {
1168fe6060f1SDimitry Andric   assert(isa<VectorType>(V1->getType()) && "Unexpected type");
1169fe6060f1SDimitry Andric   assert(V1->getType() == V2->getType() &&
1170fe6060f1SDimitry Andric          "Splice expects matching operand types!");
1171fe6060f1SDimitry Andric 
1172fe6060f1SDimitry Andric   if (auto *VTy = dyn_cast<ScalableVectorType>(V1->getType())) {
1173fe6060f1SDimitry Andric     Module *M = BB->getParent()->getParent();
1174*0fca6ea1SDimitry Andric     Function *F = Intrinsic::getDeclaration(M, Intrinsic::vector_splice, VTy);
1175fe6060f1SDimitry Andric 
1176fe6060f1SDimitry Andric     Value *Ops[] = {V1, V2, getInt32(Imm)};
1177fe6060f1SDimitry Andric     return Insert(CallInst::Create(F, Ops), Name);
1178fe6060f1SDimitry Andric   }
1179fe6060f1SDimitry Andric 
1180fe6060f1SDimitry Andric   unsigned NumElts = cast<FixedVectorType>(V1->getType())->getNumElements();
1181fe6060f1SDimitry Andric   assert(((-Imm <= NumElts) || (Imm < NumElts)) &&
1182fe6060f1SDimitry Andric          "Invalid immediate for vector splice!");
1183fe6060f1SDimitry Andric 
1184fe6060f1SDimitry Andric   // Keep the original behaviour for fixed vector
1185fe6060f1SDimitry Andric   unsigned Idx = (NumElts + Imm) % NumElts;
1186fe6060f1SDimitry Andric   SmallVector<int, 8> Mask;
1187fe6060f1SDimitry Andric   for (unsigned I = 0; I < NumElts; ++I)
1188fe6060f1SDimitry Andric     Mask.push_back(Idx + I);
1189fe6060f1SDimitry Andric 
1190fe6060f1SDimitry Andric   return CreateShuffleVector(V1, V2, Mask);
1191fe6060f1SDimitry Andric }
1192fe6060f1SDimitry Andric 
11935ffd83dbSDimitry Andric Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
11945ffd83dbSDimitry Andric                                         const Twine &Name) {
1195e8d8bef9SDimitry Andric   auto EC = ElementCount::getFixed(NumElts);
1196e8d8bef9SDimitry Andric   return CreateVectorSplat(EC, V, Name);
1197e8d8bef9SDimitry Andric }
11985ffd83dbSDimitry Andric 
1199e8d8bef9SDimitry Andric Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
1200e8d8bef9SDimitry Andric                                         const Twine &Name) {
1201e8d8bef9SDimitry Andric   assert(EC.isNonZero() && "Cannot splat to an empty vector!");
1202e8d8bef9SDimitry Andric 
1203e8d8bef9SDimitry Andric   // First insert it into a poison vector so we can shuffle it.
1204e8d8bef9SDimitry Andric   Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));
1205bdd1243dSDimitry Andric   V = CreateInsertElement(Poison, V, getInt64(0), Name + ".splatinsert");
12065ffd83dbSDimitry Andric 
12075ffd83dbSDimitry Andric   // Shuffle the value across the desired number of elements.
1208e8d8bef9SDimitry Andric   SmallVector<int, 16> Zeros;
1209e8d8bef9SDimitry Andric   Zeros.resize(EC.getKnownMinValue());
1210e8d8bef9SDimitry Andric   return CreateShuffleVector(V, Zeros, Name + ".splat");
12115ffd83dbSDimitry Andric }
12125ffd83dbSDimitry Andric 
12135ffd83dbSDimitry Andric Value *IRBuilderBase::CreatePreserveArrayAccessIndex(
12145ffd83dbSDimitry Andric     Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex,
12155ffd83dbSDimitry Andric     MDNode *DbgInfo) {
12165ffd83dbSDimitry Andric   auto *BaseType = Base->getType();
1217fe6060f1SDimitry Andric   assert(isa<PointerType>(BaseType) &&
1218fe6060f1SDimitry Andric          "Invalid Base ptr type for preserve.array.access.index.");
12195ffd83dbSDimitry Andric 
12205ffd83dbSDimitry Andric   Value *LastIndexV = getInt32(LastIndex);
12215ffd83dbSDimitry Andric   Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
1222e8d8bef9SDimitry Andric   SmallVector<Value *, 4> IdxList(Dimension, Zero);
12235ffd83dbSDimitry Andric   IdxList.push_back(LastIndexV);
12245ffd83dbSDimitry Andric 
122506c3fb27SDimitry Andric   Type *ResultType = GetElementPtrInst::getGEPReturnType(Base, IdxList);
12265ffd83dbSDimitry Andric 
12275ffd83dbSDimitry Andric   Module *M = BB->getParent()->getParent();
12285ffd83dbSDimitry Andric   Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration(
12295ffd83dbSDimitry Andric       M, Intrinsic::preserve_array_access_index, {ResultType, BaseType});
12305ffd83dbSDimitry Andric 
12315ffd83dbSDimitry Andric   Value *DimV = getInt32(Dimension);
12325ffd83dbSDimitry Andric   CallInst *Fn =
12335ffd83dbSDimitry Andric       CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV});
1234fe6060f1SDimitry Andric   Fn->addParamAttr(
1235fe6060f1SDimitry Andric       0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
12365ffd83dbSDimitry Andric   if (DbgInfo)
12375ffd83dbSDimitry Andric     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
12385ffd83dbSDimitry Andric 
12395ffd83dbSDimitry Andric   return Fn;
12405ffd83dbSDimitry Andric }
12415ffd83dbSDimitry Andric 
12425ffd83dbSDimitry Andric Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
12435ffd83dbSDimitry Andric     Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
12445ffd83dbSDimitry Andric   assert(isa<PointerType>(Base->getType()) &&
12455ffd83dbSDimitry Andric          "Invalid Base ptr type for preserve.union.access.index.");
12465ffd83dbSDimitry Andric   auto *BaseType = Base->getType();
12475ffd83dbSDimitry Andric 
12485ffd83dbSDimitry Andric   Module *M = BB->getParent()->getParent();
12495ffd83dbSDimitry Andric   Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration(
12505ffd83dbSDimitry Andric       M, Intrinsic::preserve_union_access_index, {BaseType, BaseType});
12515ffd83dbSDimitry Andric 
12525ffd83dbSDimitry Andric   Value *DIIndex = getInt32(FieldIndex);
12535ffd83dbSDimitry Andric   CallInst *Fn =
12545ffd83dbSDimitry Andric       CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex});
12555ffd83dbSDimitry Andric   if (DbgInfo)
12565ffd83dbSDimitry Andric     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
12575ffd83dbSDimitry Andric 
12585ffd83dbSDimitry Andric   return Fn;
12595ffd83dbSDimitry Andric }
12605ffd83dbSDimitry Andric 
12615ffd83dbSDimitry Andric Value *IRBuilderBase::CreatePreserveStructAccessIndex(
12625ffd83dbSDimitry Andric     Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
12635ffd83dbSDimitry Andric     MDNode *DbgInfo) {
12645ffd83dbSDimitry Andric   auto *BaseType = Base->getType();
1265fe6060f1SDimitry Andric   assert(isa<PointerType>(BaseType) &&
1266fe6060f1SDimitry Andric          "Invalid Base ptr type for preserve.struct.access.index.");
12675ffd83dbSDimitry Andric 
12685ffd83dbSDimitry Andric   Value *GEPIndex = getInt32(Index);
12695ffd83dbSDimitry Andric   Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
12705ffd83dbSDimitry Andric   Type *ResultType =
127106c3fb27SDimitry Andric       GetElementPtrInst::getGEPReturnType(Base, {Zero, GEPIndex});
12725ffd83dbSDimitry Andric 
12735ffd83dbSDimitry Andric   Module *M = BB->getParent()->getParent();
12745ffd83dbSDimitry Andric   Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration(
12755ffd83dbSDimitry Andric       M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType});
12765ffd83dbSDimitry Andric 
12775ffd83dbSDimitry Andric   Value *DIIndex = getInt32(FieldIndex);
12785ffd83dbSDimitry Andric   CallInst *Fn = CreateCall(FnPreserveStructAccessIndex,
12795ffd83dbSDimitry Andric                             {Base, GEPIndex, DIIndex});
1280fe6060f1SDimitry Andric   Fn->addParamAttr(
1281fe6060f1SDimitry Andric       0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
12825ffd83dbSDimitry Andric   if (DbgInfo)
12835ffd83dbSDimitry Andric     Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
12845ffd83dbSDimitry Andric 
12855ffd83dbSDimitry Andric   return Fn;
12865ffd83dbSDimitry Andric }
12875ffd83dbSDimitry Andric 
128806c3fb27SDimitry Andric Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) {
128906c3fb27SDimitry Andric   ConstantInt *TestV = getInt32(Test);
129006c3fb27SDimitry Andric   Module *M = BB->getParent()->getParent();
129106c3fb27SDimitry Andric   Function *FnIsFPClass =
129206c3fb27SDimitry Andric       Intrinsic::getDeclaration(M, Intrinsic::is_fpclass, {FPNum->getType()});
129306c3fb27SDimitry Andric   return CreateCall(FnIsFPClass, {FPNum, TestV});
129406c3fb27SDimitry Andric }
129506c3fb27SDimitry Andric 
1296e8d8bef9SDimitry Andric CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
1297e8d8bef9SDimitry Andric                                                          Value *PtrValue,
1298e8d8bef9SDimitry Andric                                                          Value *AlignValue,
1299e8d8bef9SDimitry Andric                                                          Value *OffsetValue) {
1300e8d8bef9SDimitry Andric   SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
1301e8d8bef9SDimitry Andric   if (OffsetValue)
1302e8d8bef9SDimitry Andric     Vals.push_back(OffsetValue);
1303e8d8bef9SDimitry Andric   OperandBundleDefT<Value *> AlignOpB("align", Vals);
1304e8d8bef9SDimitry Andric   return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB});
13055ffd83dbSDimitry Andric }
13065ffd83dbSDimitry Andric 
1307e8d8bef9SDimitry Andric CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1308e8d8bef9SDimitry Andric                                                    Value *PtrValue,
1309e8d8bef9SDimitry Andric                                                    unsigned Alignment,
1310e8d8bef9SDimitry Andric                                                    Value *OffsetValue) {
13115ffd83dbSDimitry Andric   assert(isa<PointerType>(PtrValue->getType()) &&
13125ffd83dbSDimitry Andric          "trying to create an alignment assumption on a non-pointer?");
13135ffd83dbSDimitry Andric   assert(Alignment != 0 && "Invalid Alignment");
13145ffd83dbSDimitry Andric   auto *PtrTy = cast<PointerType>(PtrValue->getType());
13155ffd83dbSDimitry Andric   Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
1316e8d8bef9SDimitry Andric   Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment);
1317e8d8bef9SDimitry Andric   return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
13185ffd83dbSDimitry Andric }
13195ffd83dbSDimitry Andric 
1320e8d8bef9SDimitry Andric CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
1321e8d8bef9SDimitry Andric                                                    Value *PtrValue,
1322e8d8bef9SDimitry Andric                                                    Value *Alignment,
1323e8d8bef9SDimitry Andric                                                    Value *OffsetValue) {
13245ffd83dbSDimitry Andric   assert(isa<PointerType>(PtrValue->getType()) &&
13255ffd83dbSDimitry Andric          "trying to create an alignment assumption on a non-pointer?");
1326e8d8bef9SDimitry Andric   return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
13275ffd83dbSDimitry Andric }
13285ffd83dbSDimitry Andric 
132981ad6265SDimitry Andric IRBuilderDefaultInserter::~IRBuilderDefaultInserter() = default;
133081ad6265SDimitry Andric IRBuilderCallbackInserter::~IRBuilderCallbackInserter() = default;
133181ad6265SDimitry Andric IRBuilderFolder::~IRBuilderFolder() = default;
13325ffd83dbSDimitry Andric void ConstantFolder::anchor() {}
13335ffd83dbSDimitry Andric void NoFolder::anchor() {}
1334