10b57cec5SDimitry Andric //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric // 90b57cec5SDimitry Andric // This file implements the IRBuilder class, which is used as a convenient way 100b57cec5SDimitry Andric // to create LLVM instructions with a consistent and simplified interface. 110b57cec5SDimitry Andric // 120b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 130b57cec5SDimitry Andric 140b57cec5SDimitry Andric #include "llvm/IR/IRBuilder.h" 150b57cec5SDimitry Andric #include "llvm/ADT/ArrayRef.h" 160b57cec5SDimitry Andric #include "llvm/IR/Constant.h" 170b57cec5SDimitry Andric #include "llvm/IR/Constants.h" 1881ad6265SDimitry Andric #include "llvm/IR/DebugInfoMetadata.h" 190b57cec5SDimitry Andric #include "llvm/IR/DerivedTypes.h" 200b57cec5SDimitry Andric #include "llvm/IR/Function.h" 210b57cec5SDimitry Andric #include "llvm/IR/GlobalValue.h" 220b57cec5SDimitry Andric #include "llvm/IR/GlobalVariable.h" 230b57cec5SDimitry Andric #include "llvm/IR/IntrinsicInst.h" 240b57cec5SDimitry Andric #include "llvm/IR/Intrinsics.h" 250b57cec5SDimitry Andric #include "llvm/IR/LLVMContext.h" 26*0fca6ea1SDimitry Andric #include "llvm/IR/Module.h" 275ffd83dbSDimitry Andric #include "llvm/IR/NoFolder.h" 280b57cec5SDimitry Andric #include "llvm/IR/Operator.h" 290b57cec5SDimitry Andric #include "llvm/IR/Statepoint.h" 300b57cec5SDimitry Andric #include "llvm/IR/Type.h" 310b57cec5SDimitry Andric #include "llvm/IR/Value.h" 320b57cec5SDimitry Andric #include "llvm/Support/Casting.h" 330b57cec5SDimitry Andric #include <cassert> 340b57cec5SDimitry Andric #include <cstdint> 35bdd1243dSDimitry Andric #include <optional> 360b57cec5SDimitry Andric #include <vector> 370b57cec5SDimitry Andric 380b57cec5SDimitry Andric using namespace llvm; 390b57cec5SDimitry Andric 400b57cec5SDimitry Andric /// CreateGlobalString - Make a new global variable with an initializer that 410b57cec5SDimitry Andric /// has array of i8 type filled in with the nul terminated string value 420b57cec5SDimitry Andric /// specified. If Name is specified, it is the name of the global variable 430b57cec5SDimitry Andric /// created. 440b57cec5SDimitry Andric GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str, 450b57cec5SDimitry Andric const Twine &Name, 4616d6b3b3SDimitry Andric unsigned AddressSpace, 47*0fca6ea1SDimitry Andric Module *M, bool AddNull) { 48*0fca6ea1SDimitry Andric Constant *StrConstant = ConstantDataArray::getString(Context, Str, AddNull); 4916d6b3b3SDimitry Andric if (!M) 5016d6b3b3SDimitry Andric M = BB->getParent()->getParent(); 5116d6b3b3SDimitry Andric auto *GV = new GlobalVariable( 5216d6b3b3SDimitry Andric *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage, 5316d6b3b3SDimitry Andric StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace); 540b57cec5SDimitry Andric GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); 555ffd83dbSDimitry Andric GV->setAlignment(Align(1)); 560b57cec5SDimitry Andric return GV; 570b57cec5SDimitry Andric } 580b57cec5SDimitry Andric 590b57cec5SDimitry Andric Type *IRBuilderBase::getCurrentFunctionReturnType() const { 600b57cec5SDimitry Andric assert(BB && BB->getParent() && "No current function!"); 610b57cec5SDimitry Andric return BB->getParent()->getReturnType(); 620b57cec5SDimitry Andric } 630b57cec5SDimitry Andric 6481ad6265SDimitry Andric DebugLoc IRBuilderBase::getCurrentDebugLocation() const { 6581ad6265SDimitry Andric for (auto &KV : MetadataToCopy) 6681ad6265SDimitry Andric if (KV.first == LLVMContext::MD_dbg) 6781ad6265SDimitry Andric return {cast<DILocation>(KV.second)}; 6881ad6265SDimitry Andric 6981ad6265SDimitry Andric return {}; 7081ad6265SDimitry Andric } 7181ad6265SDimitry Andric void IRBuilderBase::SetInstDebugLocation(Instruction *I) const { 7281ad6265SDimitry Andric for (const auto &KV : MetadataToCopy) 7381ad6265SDimitry Andric if (KV.first == LLVMContext::MD_dbg) { 7481ad6265SDimitry Andric I->setDebugLoc(DebugLoc(KV.second)); 7581ad6265SDimitry Andric return; 7681ad6265SDimitry Andric } 7781ad6265SDimitry Andric } 7881ad6265SDimitry Andric 79bdd1243dSDimitry Andric CallInst * 80bdd1243dSDimitry Andric IRBuilderBase::createCallHelper(Function *Callee, ArrayRef<Value *> Ops, 81bdd1243dSDimitry Andric const Twine &Name, Instruction *FMFSource, 82bdd1243dSDimitry Andric ArrayRef<OperandBundleDef> OpBundles) { 83bdd1243dSDimitry Andric CallInst *CI = CreateCall(Callee, Ops, OpBundles, Name); 840b57cec5SDimitry Andric if (FMFSource) 850b57cec5SDimitry Andric CI->copyFastMathFlags(FMFSource); 860b57cec5SDimitry Andric return CI; 870b57cec5SDimitry Andric } 880b57cec5SDimitry Andric 89e8d8bef9SDimitry Andric Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) { 90e8d8bef9SDimitry Andric assert(isa<ConstantInt>(Scaling) && "Expected constant integer"); 91fe6060f1SDimitry Andric if (cast<ConstantInt>(Scaling)->isZero()) 92fe6060f1SDimitry Andric return Scaling; 93fe6060f1SDimitry Andric Module *M = GetInsertBlock()->getParent()->getParent(); 94e8d8bef9SDimitry Andric Function *TheFn = 95e8d8bef9SDimitry Andric Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()}); 96bdd1243dSDimitry Andric CallInst *CI = CreateCall(TheFn, {}, {}, Name); 9706c3fb27SDimitry Andric return cast<ConstantInt>(Scaling)->isOne() ? CI : CreateMul(CI, Scaling); 9806c3fb27SDimitry Andric } 9906c3fb27SDimitry Andric 10006c3fb27SDimitry Andric Value *IRBuilderBase::CreateElementCount(Type *DstType, ElementCount EC) { 10106c3fb27SDimitry Andric Constant *MinEC = ConstantInt::get(DstType, EC.getKnownMinValue()); 10206c3fb27SDimitry Andric return EC.isScalable() ? CreateVScale(MinEC) : MinEC; 10306c3fb27SDimitry Andric } 10406c3fb27SDimitry Andric 10506c3fb27SDimitry Andric Value *IRBuilderBase::CreateTypeSize(Type *DstType, TypeSize Size) { 10606c3fb27SDimitry Andric Constant *MinSize = ConstantInt::get(DstType, Size.getKnownMinValue()); 10706c3fb27SDimitry Andric return Size.isScalable() ? CreateVScale(MinSize) : MinSize; 108e8d8bef9SDimitry Andric } 109e8d8bef9SDimitry Andric 110fe6060f1SDimitry Andric Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) { 111fe6060f1SDimitry Andric Type *STy = DstType->getScalarType(); 112349cc55cSDimitry Andric if (isa<ScalableVectorType>(DstType)) { 113349cc55cSDimitry Andric Type *StepVecType = DstType; 114349cc55cSDimitry Andric // TODO: We expect this special case (element type < 8 bits) to be 115349cc55cSDimitry Andric // temporary - once the intrinsic properly supports < 8 bits this code 116349cc55cSDimitry Andric // can be removed. 117349cc55cSDimitry Andric if (STy->getScalarSizeInBits() < 8) 118349cc55cSDimitry Andric StepVecType = 119349cc55cSDimitry Andric VectorType::get(getInt8Ty(), cast<ScalableVectorType>(DstType)); 120349cc55cSDimitry Andric Value *Res = CreateIntrinsic(Intrinsic::experimental_stepvector, 121349cc55cSDimitry Andric {StepVecType}, {}, nullptr, Name); 122349cc55cSDimitry Andric if (StepVecType != DstType) 123349cc55cSDimitry Andric Res = CreateTrunc(Res, DstType); 124349cc55cSDimitry Andric return Res; 125349cc55cSDimitry Andric } 126349cc55cSDimitry Andric 127fe6060f1SDimitry Andric unsigned NumEls = cast<FixedVectorType>(DstType)->getNumElements(); 128fe6060f1SDimitry Andric 129fe6060f1SDimitry Andric // Create a vector of consecutive numbers from zero to VF. 130fe6060f1SDimitry Andric SmallVector<Constant *, 8> Indices; 131fe6060f1SDimitry Andric for (unsigned i = 0; i < NumEls; ++i) 132fe6060f1SDimitry Andric Indices.push_back(ConstantInt::get(STy, i)); 133fe6060f1SDimitry Andric 134fe6060f1SDimitry Andric // Add the consecutive indices to the vector value. 135fe6060f1SDimitry Andric return ConstantVector::get(Indices); 136fe6060f1SDimitry Andric } 137fe6060f1SDimitry Andric 138480093f4SDimitry Andric CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size, 139480093f4SDimitry Andric MaybeAlign Align, bool isVolatile, 140480093f4SDimitry Andric MDNode *TBAATag, MDNode *ScopeTag, 1410b57cec5SDimitry Andric MDNode *NoAliasTag) { 1420b57cec5SDimitry Andric Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)}; 1430b57cec5SDimitry Andric Type *Tys[] = { Ptr->getType(), Size->getType() }; 1440b57cec5SDimitry Andric Module *M = BB->getParent()->getParent(); 1450b57cec5SDimitry Andric Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys); 1460b57cec5SDimitry Andric 147bdd1243dSDimitry Andric CallInst *CI = CreateCall(TheFn, Ops); 1480b57cec5SDimitry Andric 149480093f4SDimitry Andric if (Align) 15081ad6265SDimitry Andric cast<MemSetInst>(CI)->setDestAlignment(*Align); 15181ad6265SDimitry Andric 15281ad6265SDimitry Andric // Set the TBAA info if present. 15381ad6265SDimitry Andric if (TBAATag) 15481ad6265SDimitry Andric CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); 15581ad6265SDimitry Andric 15681ad6265SDimitry Andric if (ScopeTag) 15781ad6265SDimitry Andric CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); 15881ad6265SDimitry Andric 15981ad6265SDimitry Andric if (NoAliasTag) 16081ad6265SDimitry Andric CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); 16181ad6265SDimitry Andric 16281ad6265SDimitry Andric return CI; 16381ad6265SDimitry Andric } 16481ad6265SDimitry Andric 16581ad6265SDimitry Andric CallInst *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign, 16681ad6265SDimitry Andric Value *Val, Value *Size, 16781ad6265SDimitry Andric bool IsVolatile, MDNode *TBAATag, 16881ad6265SDimitry Andric MDNode *ScopeTag, 16981ad6265SDimitry Andric MDNode *NoAliasTag) { 17081ad6265SDimitry Andric Value *Ops[] = {Dst, Val, Size, getInt1(IsVolatile)}; 17181ad6265SDimitry Andric Type *Tys[] = {Dst->getType(), Size->getType()}; 17281ad6265SDimitry Andric Module *M = BB->getParent()->getParent(); 17381ad6265SDimitry Andric Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset_inline, Tys); 17481ad6265SDimitry Andric 175bdd1243dSDimitry Andric CallInst *CI = CreateCall(TheFn, Ops); 17681ad6265SDimitry Andric 17781ad6265SDimitry Andric if (DstAlign) 17881ad6265SDimitry Andric cast<MemSetInlineInst>(CI)->setDestAlignment(*DstAlign); 1790b57cec5SDimitry Andric 1800b57cec5SDimitry Andric // Set the TBAA info if present. 1810b57cec5SDimitry Andric if (TBAATag) 1820b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); 1830b57cec5SDimitry Andric 1840b57cec5SDimitry Andric if (ScopeTag) 1850b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); 1860b57cec5SDimitry Andric 1870b57cec5SDimitry Andric if (NoAliasTag) 1880b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); 1890b57cec5SDimitry Andric 1900b57cec5SDimitry Andric return CI; 1910b57cec5SDimitry Andric } 1920b57cec5SDimitry Andric 1930b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet( 194480093f4SDimitry Andric Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize, 1950b57cec5SDimitry Andric MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) { 1960b57cec5SDimitry Andric 1970b57cec5SDimitry Andric Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)}; 1980b57cec5SDimitry Andric Type *Tys[] = {Ptr->getType(), Size->getType()}; 1990b57cec5SDimitry Andric Module *M = BB->getParent()->getParent(); 2000b57cec5SDimitry Andric Function *TheFn = Intrinsic::getDeclaration( 2010b57cec5SDimitry Andric M, Intrinsic::memset_element_unordered_atomic, Tys); 2020b57cec5SDimitry Andric 203bdd1243dSDimitry Andric CallInst *CI = CreateCall(TheFn, Ops); 2040b57cec5SDimitry Andric 205480093f4SDimitry Andric cast<AtomicMemSetInst>(CI)->setDestAlignment(Alignment); 2060b57cec5SDimitry Andric 2070b57cec5SDimitry Andric // Set the TBAA info if present. 2080b57cec5SDimitry Andric if (TBAATag) 2090b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); 2100b57cec5SDimitry Andric 2110b57cec5SDimitry Andric if (ScopeTag) 2120b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); 2130b57cec5SDimitry Andric 2140b57cec5SDimitry Andric if (NoAliasTag) 2150b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); 2160b57cec5SDimitry Andric 2170b57cec5SDimitry Andric return CI; 2180b57cec5SDimitry Andric } 2190b57cec5SDimitry Andric 220e8d8bef9SDimitry Andric CallInst *IRBuilderBase::CreateMemTransferInst( 221e8d8bef9SDimitry Andric Intrinsic::ID IntrID, Value *Dst, MaybeAlign DstAlign, Value *Src, 222e8d8bef9SDimitry Andric MaybeAlign SrcAlign, Value *Size, bool isVolatile, MDNode *TBAATag, 223e8d8bef9SDimitry Andric MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) { 2245f757f3fSDimitry Andric assert((IntrID == Intrinsic::memcpy || IntrID == Intrinsic::memcpy_inline || 2255f757f3fSDimitry Andric IntrID == Intrinsic::memmove) && 2265f757f3fSDimitry Andric "Unexpected intrinsic ID"); 2270b57cec5SDimitry Andric Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)}; 2280b57cec5SDimitry Andric Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() }; 2290b57cec5SDimitry Andric Module *M = BB->getParent()->getParent(); 230e8d8bef9SDimitry Andric Function *TheFn = Intrinsic::getDeclaration(M, IntrID, Tys); 2310b57cec5SDimitry Andric 232bdd1243dSDimitry Andric CallInst *CI = CreateCall(TheFn, Ops); 2330b57cec5SDimitry Andric 234e8d8bef9SDimitry Andric auto* MCI = cast<MemTransferInst>(CI); 235480093f4SDimitry Andric if (DstAlign) 236480093f4SDimitry Andric MCI->setDestAlignment(*DstAlign); 237480093f4SDimitry Andric if (SrcAlign) 238480093f4SDimitry Andric MCI->setSourceAlignment(*SrcAlign); 2390b57cec5SDimitry Andric 2400b57cec5SDimitry Andric // Set the TBAA info if present. 2410b57cec5SDimitry Andric if (TBAATag) 2420b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); 2430b57cec5SDimitry Andric 2440b57cec5SDimitry Andric // Set the TBAA Struct info if present. 2450b57cec5SDimitry Andric if (TBAAStructTag) 2460b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag); 2470b57cec5SDimitry Andric 2480b57cec5SDimitry Andric if (ScopeTag) 2490b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); 2500b57cec5SDimitry Andric 2510b57cec5SDimitry Andric if (NoAliasTag) 2520b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); 2530b57cec5SDimitry Andric 2540b57cec5SDimitry Andric return CI; 2550b57cec5SDimitry Andric } 2560b57cec5SDimitry Andric 2570b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy( 2585ffd83dbSDimitry Andric Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, 2590b57cec5SDimitry Andric uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag, 2600b57cec5SDimitry Andric MDNode *ScopeTag, MDNode *NoAliasTag) { 2610b57cec5SDimitry Andric assert(DstAlign >= ElementSize && 2620b57cec5SDimitry Andric "Pointer alignment must be at least element size"); 2630b57cec5SDimitry Andric assert(SrcAlign >= ElementSize && 2640b57cec5SDimitry Andric "Pointer alignment must be at least element size"); 2650b57cec5SDimitry Andric Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)}; 2660b57cec5SDimitry Andric Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()}; 2670b57cec5SDimitry Andric Module *M = BB->getParent()->getParent(); 2680b57cec5SDimitry Andric Function *TheFn = Intrinsic::getDeclaration( 2690b57cec5SDimitry Andric M, Intrinsic::memcpy_element_unordered_atomic, Tys); 2700b57cec5SDimitry Andric 271bdd1243dSDimitry Andric CallInst *CI = CreateCall(TheFn, Ops); 2720b57cec5SDimitry Andric 2730b57cec5SDimitry Andric // Set the alignment of the pointer args. 2740b57cec5SDimitry Andric auto *AMCI = cast<AtomicMemCpyInst>(CI); 2750b57cec5SDimitry Andric AMCI->setDestAlignment(DstAlign); 2760b57cec5SDimitry Andric AMCI->setSourceAlignment(SrcAlign); 2770b57cec5SDimitry Andric 2780b57cec5SDimitry Andric // Set the TBAA info if present. 2790b57cec5SDimitry Andric if (TBAATag) 2800b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); 2810b57cec5SDimitry Andric 2820b57cec5SDimitry Andric // Set the TBAA Struct info if present. 2830b57cec5SDimitry Andric if (TBAAStructTag) 2840b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag); 2850b57cec5SDimitry Andric 2860b57cec5SDimitry Andric if (ScopeTag) 2870b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); 2880b57cec5SDimitry Andric 2890b57cec5SDimitry Andric if (NoAliasTag) 2900b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); 2910b57cec5SDimitry Andric 2920b57cec5SDimitry Andric return CI; 2930b57cec5SDimitry Andric } 2940b57cec5SDimitry Andric 2955f757f3fSDimitry Andric /// isConstantOne - Return true only if val is constant int 1 2965f757f3fSDimitry Andric static bool isConstantOne(const Value *Val) { 2975f757f3fSDimitry Andric assert(Val && "isConstantOne does not work with nullptr Val"); 2985f757f3fSDimitry Andric const ConstantInt *CVal = dyn_cast<ConstantInt>(Val); 2995f757f3fSDimitry Andric return CVal && CVal->isOne(); 3005f757f3fSDimitry Andric } 3015f757f3fSDimitry Andric 3025f757f3fSDimitry Andric CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Type *AllocTy, 3035f757f3fSDimitry Andric Value *AllocSize, Value *ArraySize, 3045f757f3fSDimitry Andric ArrayRef<OperandBundleDef> OpB, 3055f757f3fSDimitry Andric Function *MallocF, const Twine &Name) { 3065f757f3fSDimitry Andric // malloc(type) becomes: 3075f757f3fSDimitry Andric // i8* malloc(typeSize) 3085f757f3fSDimitry Andric // malloc(type, arraySize) becomes: 3095f757f3fSDimitry Andric // i8* malloc(typeSize*arraySize) 3105f757f3fSDimitry Andric if (!ArraySize) 3115f757f3fSDimitry Andric ArraySize = ConstantInt::get(IntPtrTy, 1); 3125f757f3fSDimitry Andric else if (ArraySize->getType() != IntPtrTy) 3135f757f3fSDimitry Andric ArraySize = CreateIntCast(ArraySize, IntPtrTy, false); 3145f757f3fSDimitry Andric 3155f757f3fSDimitry Andric if (!isConstantOne(ArraySize)) { 3165f757f3fSDimitry Andric if (isConstantOne(AllocSize)) { 3175f757f3fSDimitry Andric AllocSize = ArraySize; // Operand * 1 = Operand 3185f757f3fSDimitry Andric } else { 3195f757f3fSDimitry Andric // Multiply type size by the array size... 3205f757f3fSDimitry Andric AllocSize = CreateMul(ArraySize, AllocSize, "mallocsize"); 3215f757f3fSDimitry Andric } 3225f757f3fSDimitry Andric } 3235f757f3fSDimitry Andric 3245f757f3fSDimitry Andric assert(AllocSize->getType() == IntPtrTy && "malloc arg is wrong size"); 3255f757f3fSDimitry Andric // Create the call to Malloc. 3260b57cec5SDimitry Andric Module *M = BB->getParent()->getParent(); 3275f757f3fSDimitry Andric Type *BPTy = PointerType::getUnqual(Context); 3285f757f3fSDimitry Andric FunctionCallee MallocFunc = MallocF; 3295f757f3fSDimitry Andric if (!MallocFunc) 3305f757f3fSDimitry Andric // prototype malloc as "void *malloc(size_t)" 3315f757f3fSDimitry Andric MallocFunc = M->getOrInsertFunction("malloc", BPTy, IntPtrTy); 3325f757f3fSDimitry Andric CallInst *MCall = CreateCall(MallocFunc, AllocSize, OpB, Name); 3330b57cec5SDimitry Andric 3345f757f3fSDimitry Andric MCall->setTailCall(); 3355f757f3fSDimitry Andric if (Function *F = dyn_cast<Function>(MallocFunc.getCallee())) { 3365f757f3fSDimitry Andric MCall->setCallingConv(F->getCallingConv()); 3375f757f3fSDimitry Andric F->setReturnDoesNotAlias(); 3385f757f3fSDimitry Andric } 3390b57cec5SDimitry Andric 3405f757f3fSDimitry Andric assert(!MCall->getType()->isVoidTy() && "Malloc has void return type"); 3410b57cec5SDimitry Andric 3425f757f3fSDimitry Andric return MCall; 3435f757f3fSDimitry Andric } 3440b57cec5SDimitry Andric 3455f757f3fSDimitry Andric CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Type *AllocTy, 3465f757f3fSDimitry Andric Value *AllocSize, Value *ArraySize, 3475f757f3fSDimitry Andric Function *MallocF, const Twine &Name) { 3480b57cec5SDimitry Andric 3495f757f3fSDimitry Andric return CreateMalloc(IntPtrTy, AllocTy, AllocSize, ArraySize, std::nullopt, 3505f757f3fSDimitry Andric MallocF, Name); 3515f757f3fSDimitry Andric } 3520b57cec5SDimitry Andric 3535f757f3fSDimitry Andric /// CreateFree - Generate the IR for a call to the builtin free function. 3545f757f3fSDimitry Andric CallInst *IRBuilderBase::CreateFree(Value *Source, 3555f757f3fSDimitry Andric ArrayRef<OperandBundleDef> Bundles) { 3565f757f3fSDimitry Andric assert(Source->getType()->isPointerTy() && 3575f757f3fSDimitry Andric "Can not free something of nonpointer type!"); 3585f757f3fSDimitry Andric 3595f757f3fSDimitry Andric Module *M = BB->getParent()->getParent(); 3605f757f3fSDimitry Andric 3615f757f3fSDimitry Andric Type *VoidTy = Type::getVoidTy(M->getContext()); 3625f757f3fSDimitry Andric Type *VoidPtrTy = PointerType::getUnqual(M->getContext()); 3635f757f3fSDimitry Andric // prototype free as "void free(void*)" 3645f757f3fSDimitry Andric FunctionCallee FreeFunc = M->getOrInsertFunction("free", VoidTy, VoidPtrTy); 3655f757f3fSDimitry Andric CallInst *Result = CreateCall(FreeFunc, Source, Bundles, ""); 3665f757f3fSDimitry Andric Result->setTailCall(); 3675f757f3fSDimitry Andric if (Function *F = dyn_cast<Function>(FreeFunc.getCallee())) 3685f757f3fSDimitry Andric Result->setCallingConv(F->getCallingConv()); 3695f757f3fSDimitry Andric 3705f757f3fSDimitry Andric return Result; 3710b57cec5SDimitry Andric } 3720b57cec5SDimitry Andric 3730b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove( 3745ffd83dbSDimitry Andric Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size, 3750b57cec5SDimitry Andric uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag, 3760b57cec5SDimitry Andric MDNode *ScopeTag, MDNode *NoAliasTag) { 3770b57cec5SDimitry Andric assert(DstAlign >= ElementSize && 3780b57cec5SDimitry Andric "Pointer alignment must be at least element size"); 3790b57cec5SDimitry Andric assert(SrcAlign >= ElementSize && 3800b57cec5SDimitry Andric "Pointer alignment must be at least element size"); 3810b57cec5SDimitry Andric Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)}; 3820b57cec5SDimitry Andric Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()}; 3830b57cec5SDimitry Andric Module *M = BB->getParent()->getParent(); 3840b57cec5SDimitry Andric Function *TheFn = Intrinsic::getDeclaration( 3850b57cec5SDimitry Andric M, Intrinsic::memmove_element_unordered_atomic, Tys); 3860b57cec5SDimitry Andric 387bdd1243dSDimitry Andric CallInst *CI = CreateCall(TheFn, Ops); 3880b57cec5SDimitry Andric 3890b57cec5SDimitry Andric // Set the alignment of the pointer args. 3905ffd83dbSDimitry Andric CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign)); 3915ffd83dbSDimitry Andric CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign)); 3920b57cec5SDimitry Andric 3930b57cec5SDimitry Andric // Set the TBAA info if present. 3940b57cec5SDimitry Andric if (TBAATag) 3950b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); 3960b57cec5SDimitry Andric 3970b57cec5SDimitry Andric // Set the TBAA Struct info if present. 3980b57cec5SDimitry Andric if (TBAAStructTag) 3990b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag); 4000b57cec5SDimitry Andric 4010b57cec5SDimitry Andric if (ScopeTag) 4020b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag); 4030b57cec5SDimitry Andric 4040b57cec5SDimitry Andric if (NoAliasTag) 4050b57cec5SDimitry Andric CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); 4060b57cec5SDimitry Andric 4070b57cec5SDimitry Andric return CI; 4080b57cec5SDimitry Andric } 4090b57cec5SDimitry Andric 410bdd1243dSDimitry Andric CallInst *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) { 411bdd1243dSDimitry Andric Module *M = GetInsertBlock()->getParent()->getParent(); 4120b57cec5SDimitry Andric Value *Ops[] = {Src}; 4130b57cec5SDimitry Andric Type *Tys[] = { Src->getType() }; 4140b57cec5SDimitry Andric auto Decl = Intrinsic::getDeclaration(M, ID, Tys); 415bdd1243dSDimitry Andric return CreateCall(Decl, Ops); 4160b57cec5SDimitry Andric } 4170b57cec5SDimitry Andric 4180b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) { 4190b57cec5SDimitry Andric Module *M = GetInsertBlock()->getParent()->getParent(); 4200b57cec5SDimitry Andric Value *Ops[] = {Acc, Src}; 421e8d8bef9SDimitry Andric auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fadd, 422e8d8bef9SDimitry Andric {Src->getType()}); 423bdd1243dSDimitry Andric return CreateCall(Decl, Ops); 4240b57cec5SDimitry Andric } 4250b57cec5SDimitry Andric 4260b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) { 4270b57cec5SDimitry Andric Module *M = GetInsertBlock()->getParent()->getParent(); 4280b57cec5SDimitry Andric Value *Ops[] = {Acc, Src}; 429e8d8bef9SDimitry Andric auto Decl = Intrinsic::getDeclaration(M, Intrinsic::vector_reduce_fmul, 430e8d8bef9SDimitry Andric {Src->getType()}); 431bdd1243dSDimitry Andric return CreateCall(Decl, Ops); 4320b57cec5SDimitry Andric } 4330b57cec5SDimitry Andric 4340b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateAddReduce(Value *Src) { 435bdd1243dSDimitry Andric return getReductionIntrinsic(Intrinsic::vector_reduce_add, Src); 4360b57cec5SDimitry Andric } 4370b57cec5SDimitry Andric 4380b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateMulReduce(Value *Src) { 439bdd1243dSDimitry Andric return getReductionIntrinsic(Intrinsic::vector_reduce_mul, Src); 4400b57cec5SDimitry Andric } 4410b57cec5SDimitry Andric 4420b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateAndReduce(Value *Src) { 443bdd1243dSDimitry Andric return getReductionIntrinsic(Intrinsic::vector_reduce_and, Src); 4440b57cec5SDimitry Andric } 4450b57cec5SDimitry Andric 4460b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateOrReduce(Value *Src) { 447bdd1243dSDimitry Andric return getReductionIntrinsic(Intrinsic::vector_reduce_or, Src); 4480b57cec5SDimitry Andric } 4490b57cec5SDimitry Andric 4500b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateXorReduce(Value *Src) { 451bdd1243dSDimitry Andric return getReductionIntrinsic(Intrinsic::vector_reduce_xor, Src); 4520b57cec5SDimitry Andric } 4530b57cec5SDimitry Andric 4540b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) { 455e8d8bef9SDimitry Andric auto ID = 456e8d8bef9SDimitry Andric IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax; 457bdd1243dSDimitry Andric return getReductionIntrinsic(ID, Src); 4580b57cec5SDimitry Andric } 4590b57cec5SDimitry Andric 4600b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) { 461e8d8bef9SDimitry Andric auto ID = 462e8d8bef9SDimitry Andric IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin; 463bdd1243dSDimitry Andric return getReductionIntrinsic(ID, Src); 4640b57cec5SDimitry Andric } 4650b57cec5SDimitry Andric 466e8d8bef9SDimitry Andric CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src) { 467bdd1243dSDimitry Andric return getReductionIntrinsic(Intrinsic::vector_reduce_fmax, Src); 4680b57cec5SDimitry Andric } 4690b57cec5SDimitry Andric 470e8d8bef9SDimitry Andric CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src) { 471bdd1243dSDimitry Andric return getReductionIntrinsic(Intrinsic::vector_reduce_fmin, Src); 4720b57cec5SDimitry Andric } 4730b57cec5SDimitry Andric 47406c3fb27SDimitry Andric CallInst *IRBuilderBase::CreateFPMaximumReduce(Value *Src) { 47506c3fb27SDimitry Andric return getReductionIntrinsic(Intrinsic::vector_reduce_fmaximum, Src); 47606c3fb27SDimitry Andric } 47706c3fb27SDimitry Andric 47806c3fb27SDimitry Andric CallInst *IRBuilderBase::CreateFPMinimumReduce(Value *Src) { 47906c3fb27SDimitry Andric return getReductionIntrinsic(Intrinsic::vector_reduce_fminimum, Src); 48006c3fb27SDimitry Andric } 48106c3fb27SDimitry Andric 4820b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) { 4830b57cec5SDimitry Andric assert(isa<PointerType>(Ptr->getType()) && 4840b57cec5SDimitry Andric "lifetime.start only applies to pointers."); 4850b57cec5SDimitry Andric if (!Size) 4860b57cec5SDimitry Andric Size = getInt64(-1); 4870b57cec5SDimitry Andric else 4880b57cec5SDimitry Andric assert(Size->getType() == getInt64Ty() && 4890b57cec5SDimitry Andric "lifetime.start requires the size to be an i64"); 4900b57cec5SDimitry Andric Value *Ops[] = { Size, Ptr }; 4910b57cec5SDimitry Andric Module *M = BB->getParent()->getParent(); 4920b57cec5SDimitry Andric Function *TheFn = 4930b57cec5SDimitry Andric Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()}); 494bdd1243dSDimitry Andric return CreateCall(TheFn, Ops); 4950b57cec5SDimitry Andric } 4960b57cec5SDimitry Andric 4970b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) { 4980b57cec5SDimitry Andric assert(isa<PointerType>(Ptr->getType()) && 4990b57cec5SDimitry Andric "lifetime.end only applies to pointers."); 5000b57cec5SDimitry Andric if (!Size) 5010b57cec5SDimitry Andric Size = getInt64(-1); 5020b57cec5SDimitry Andric else 5030b57cec5SDimitry Andric assert(Size->getType() == getInt64Ty() && 5040b57cec5SDimitry Andric "lifetime.end requires the size to be an i64"); 5050b57cec5SDimitry Andric Value *Ops[] = { Size, Ptr }; 5060b57cec5SDimitry Andric Module *M = BB->getParent()->getParent(); 5070b57cec5SDimitry Andric Function *TheFn = 5080b57cec5SDimitry Andric Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()}); 509bdd1243dSDimitry Andric return CreateCall(TheFn, Ops); 5100b57cec5SDimitry Andric } 5110b57cec5SDimitry Andric 5120b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) { 5130b57cec5SDimitry Andric 5140b57cec5SDimitry Andric assert(isa<PointerType>(Ptr->getType()) && 5150b57cec5SDimitry Andric "invariant.start only applies to pointers."); 5160b57cec5SDimitry Andric if (!Size) 5170b57cec5SDimitry Andric Size = getInt64(-1); 5180b57cec5SDimitry Andric else 5190b57cec5SDimitry Andric assert(Size->getType() == getInt64Ty() && 5200b57cec5SDimitry Andric "invariant.start requires the size to be an i64"); 5210b57cec5SDimitry Andric 5220b57cec5SDimitry Andric Value *Ops[] = {Size, Ptr}; 5230b57cec5SDimitry Andric // Fill in the single overloaded type: memory object type. 5240b57cec5SDimitry Andric Type *ObjectPtr[1] = {Ptr->getType()}; 5250b57cec5SDimitry Andric Module *M = BB->getParent()->getParent(); 5260b57cec5SDimitry Andric Function *TheFn = 5270b57cec5SDimitry Andric Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr); 528bdd1243dSDimitry Andric return CreateCall(TheFn, Ops); 529bdd1243dSDimitry Andric } 530bdd1243dSDimitry Andric 531bdd1243dSDimitry Andric static MaybeAlign getAlign(Value *Ptr) { 532bdd1243dSDimitry Andric if (auto *O = dyn_cast<GlobalObject>(Ptr)) 533bdd1243dSDimitry Andric return O->getAlign(); 534bdd1243dSDimitry Andric if (auto *A = dyn_cast<GlobalAlias>(Ptr)) 535bdd1243dSDimitry Andric return A->getAliaseeObject()->getAlign(); 536bdd1243dSDimitry Andric return {}; 537bdd1243dSDimitry Andric } 538bdd1243dSDimitry Andric 539bdd1243dSDimitry Andric CallInst *IRBuilderBase::CreateThreadLocalAddress(Value *Ptr) { 5405f757f3fSDimitry Andric assert(isa<GlobalValue>(Ptr) && cast<GlobalValue>(Ptr)->isThreadLocal() && 541bdd1243dSDimitry Andric "threadlocal_address only applies to thread local variables."); 542bdd1243dSDimitry Andric CallInst *CI = CreateIntrinsic(llvm::Intrinsic::threadlocal_address, 543bdd1243dSDimitry Andric {Ptr->getType()}, {Ptr}); 544bdd1243dSDimitry Andric if (MaybeAlign A = getAlign(Ptr)) { 545bdd1243dSDimitry Andric CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), *A)); 546bdd1243dSDimitry Andric CI->addRetAttr(Attribute::getWithAlignment(CI->getContext(), *A)); 547bdd1243dSDimitry Andric } 548bdd1243dSDimitry Andric return CI; 5490b57cec5SDimitry Andric } 5500b57cec5SDimitry Andric 551e8d8bef9SDimitry Andric CallInst * 552e8d8bef9SDimitry Andric IRBuilderBase::CreateAssumption(Value *Cond, 553e8d8bef9SDimitry Andric ArrayRef<OperandBundleDef> OpBundles) { 5540b57cec5SDimitry Andric assert(Cond->getType() == getInt1Ty() && 5550b57cec5SDimitry Andric "an assumption condition must be of type i1"); 5560b57cec5SDimitry Andric 5570b57cec5SDimitry Andric Value *Ops[] = { Cond }; 5580b57cec5SDimitry Andric Module *M = BB->getParent()->getParent(); 5590b57cec5SDimitry Andric Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume); 560bdd1243dSDimitry Andric return CreateCall(FnAssume, Ops, OpBundles); 561e8d8bef9SDimitry Andric } 562e8d8bef9SDimitry Andric 563e8d8bef9SDimitry Andric Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) { 564e8d8bef9SDimitry Andric Module *M = BB->getModule(); 565e8d8bef9SDimitry Andric auto *FnIntrinsic = Intrinsic::getDeclaration( 566e8d8bef9SDimitry Andric M, Intrinsic::experimental_noalias_scope_decl, {}); 567bdd1243dSDimitry Andric return CreateCall(FnIntrinsic, {Scope}); 5680b57cec5SDimitry Andric } 5690b57cec5SDimitry Andric 5700b57cec5SDimitry Andric /// Create a call to a Masked Load intrinsic. 571fe6060f1SDimitry Andric /// \p Ty - vector type to load 5720b57cec5SDimitry Andric /// \p Ptr - base pointer for the load 5735ffd83dbSDimitry Andric /// \p Alignment - alignment of the source location 5740b57cec5SDimitry Andric /// \p Mask - vector of booleans which indicates what vector lanes should 5750b57cec5SDimitry Andric /// be accessed in memory 5760b57cec5SDimitry Andric /// \p PassThru - pass-through value that is used to fill the masked-off lanes 5770b57cec5SDimitry Andric /// of the result 5780b57cec5SDimitry Andric /// \p Name - name of the result variable 579fe6060f1SDimitry Andric CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, 5800b57cec5SDimitry Andric Value *Mask, Value *PassThru, 5810b57cec5SDimitry Andric const Twine &Name) { 5820b57cec5SDimitry Andric auto *PtrTy = cast<PointerType>(Ptr->getType()); 583fe6060f1SDimitry Andric assert(Ty->isVectorTy() && "Type should be vector"); 5840b57cec5SDimitry Andric assert(Mask && "Mask should not be all-ones (null)"); 5850b57cec5SDimitry Andric if (!PassThru) 586bdd1243dSDimitry Andric PassThru = PoisonValue::get(Ty); 587fe6060f1SDimitry Andric Type *OverloadedTypes[] = { Ty, PtrTy }; 5885ffd83dbSDimitry Andric Value *Ops[] = {Ptr, getInt32(Alignment.value()), Mask, PassThru}; 5890b57cec5SDimitry Andric return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, 5900b57cec5SDimitry Andric OverloadedTypes, Name); 5910b57cec5SDimitry Andric } 5920b57cec5SDimitry Andric 5930b57cec5SDimitry Andric /// Create a call to a Masked Store intrinsic. 5940b57cec5SDimitry Andric /// \p Val - data to be stored, 5950b57cec5SDimitry Andric /// \p Ptr - base pointer for the store 5965ffd83dbSDimitry Andric /// \p Alignment - alignment of the destination location 5970b57cec5SDimitry Andric /// \p Mask - vector of booleans which indicates what vector lanes should 5980b57cec5SDimitry Andric /// be accessed in memory 5990b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr, 6005ffd83dbSDimitry Andric Align Alignment, Value *Mask) { 6010b57cec5SDimitry Andric auto *PtrTy = cast<PointerType>(Ptr->getType()); 602fe6060f1SDimitry Andric Type *DataTy = Val->getType(); 603fe6060f1SDimitry Andric assert(DataTy->isVectorTy() && "Val should be a vector"); 6040b57cec5SDimitry Andric assert(Mask && "Mask should not be all-ones (null)"); 6050b57cec5SDimitry Andric Type *OverloadedTypes[] = { DataTy, PtrTy }; 6065ffd83dbSDimitry Andric Value *Ops[] = {Val, Ptr, getInt32(Alignment.value()), Mask}; 6070b57cec5SDimitry Andric return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes); 6080b57cec5SDimitry Andric } 6090b57cec5SDimitry Andric 6100b57cec5SDimitry Andric /// Create a call to a Masked intrinsic, with given intrinsic Id, 6110b57cec5SDimitry Andric /// an array of operands - Ops, and an array of overloaded types - 6120b57cec5SDimitry Andric /// OverloadedTypes. 6130b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id, 6140b57cec5SDimitry Andric ArrayRef<Value *> Ops, 6150b57cec5SDimitry Andric ArrayRef<Type *> OverloadedTypes, 6160b57cec5SDimitry Andric const Twine &Name) { 6170b57cec5SDimitry Andric Module *M = BB->getParent()->getParent(); 6180b57cec5SDimitry Andric Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes); 619bdd1243dSDimitry Andric return CreateCall(TheFn, Ops, {}, Name); 6200b57cec5SDimitry Andric } 6210b57cec5SDimitry Andric 6220b57cec5SDimitry Andric /// Create a call to a Masked Gather intrinsic. 623fe6060f1SDimitry Andric /// \p Ty - vector type to gather 6240b57cec5SDimitry Andric /// \p Ptrs - vector of pointers for loading 6250b57cec5SDimitry Andric /// \p Align - alignment for one element 6260b57cec5SDimitry Andric /// \p Mask - vector of booleans which indicates what vector lanes should 6270b57cec5SDimitry Andric /// be accessed in memory 6280b57cec5SDimitry Andric /// \p PassThru - pass-through value that is used to fill the masked-off lanes 6290b57cec5SDimitry Andric /// of the result 6300b57cec5SDimitry Andric /// \p Name - name of the result variable 631fe6060f1SDimitry Andric CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs, 632fe6060f1SDimitry Andric Align Alignment, Value *Mask, 633fe6060f1SDimitry Andric Value *PassThru, 6340b57cec5SDimitry Andric const Twine &Name) { 635fe6060f1SDimitry Andric auto *VecTy = cast<VectorType>(Ty); 636fe6060f1SDimitry Andric ElementCount NumElts = VecTy->getElementCount(); 637fe6060f1SDimitry Andric auto *PtrsTy = cast<VectorType>(Ptrs->getType()); 638fe6060f1SDimitry Andric assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch"); 6390b57cec5SDimitry Andric 6400b57cec5SDimitry Andric if (!Mask) 64106c3fb27SDimitry Andric Mask = getAllOnesMask(NumElts); 6420b57cec5SDimitry Andric 6430b57cec5SDimitry Andric if (!PassThru) 644bdd1243dSDimitry Andric PassThru = PoisonValue::get(Ty); 6450b57cec5SDimitry Andric 646fe6060f1SDimitry Andric Type *OverloadedTypes[] = {Ty, PtrsTy}; 6475ffd83dbSDimitry Andric Value *Ops[] = {Ptrs, getInt32(Alignment.value()), Mask, PassThru}; 6480b57cec5SDimitry Andric 6490b57cec5SDimitry Andric // We specify only one type when we create this intrinsic. Types of other 6500b57cec5SDimitry Andric // arguments are derived from this type. 6510b57cec5SDimitry Andric return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes, 6520b57cec5SDimitry Andric Name); 6530b57cec5SDimitry Andric } 6540b57cec5SDimitry Andric 6550b57cec5SDimitry Andric /// Create a call to a Masked Scatter intrinsic. 6560b57cec5SDimitry Andric /// \p Data - data to be stored, 6570b57cec5SDimitry Andric /// \p Ptrs - the vector of pointers, where the \p Data elements should be 6580b57cec5SDimitry Andric /// stored 6590b57cec5SDimitry Andric /// \p Align - alignment for one element 6600b57cec5SDimitry Andric /// \p Mask - vector of booleans which indicates what vector lanes should 6610b57cec5SDimitry Andric /// be accessed in memory 6620b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs, 6635ffd83dbSDimitry Andric Align Alignment, Value *Mask) { 664fe6060f1SDimitry Andric auto *PtrsTy = cast<VectorType>(Ptrs->getType()); 665fe6060f1SDimitry Andric auto *DataTy = cast<VectorType>(Data->getType()); 666fe6060f1SDimitry Andric ElementCount NumElts = PtrsTy->getElementCount(); 6670b57cec5SDimitry Andric 6680b57cec5SDimitry Andric if (!Mask) 66906c3fb27SDimitry Andric Mask = getAllOnesMask(NumElts); 6700b57cec5SDimitry Andric 6710b57cec5SDimitry Andric Type *OverloadedTypes[] = {DataTy, PtrsTy}; 6725ffd83dbSDimitry Andric Value *Ops[] = {Data, Ptrs, getInt32(Alignment.value()), Mask}; 6730b57cec5SDimitry Andric 6740b57cec5SDimitry Andric // We specify only one type when we create this intrinsic. Types of other 6750b57cec5SDimitry Andric // arguments are derived from this type. 6760b57cec5SDimitry Andric return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes); 6770b57cec5SDimitry Andric } 6780b57cec5SDimitry Andric 679bdd1243dSDimitry Andric /// Create a call to Masked Expand Load intrinsic 680bdd1243dSDimitry Andric /// \p Ty - vector type to load 681bdd1243dSDimitry Andric /// \p Ptr - base pointer for the load 682bdd1243dSDimitry Andric /// \p Mask - vector of booleans which indicates what vector lanes should 683bdd1243dSDimitry Andric /// be accessed in memory 684bdd1243dSDimitry Andric /// \p PassThru - pass-through value that is used to fill the masked-off lanes 685bdd1243dSDimitry Andric /// of the result 686bdd1243dSDimitry Andric /// \p Name - name of the result variable 687bdd1243dSDimitry Andric CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr, 688bdd1243dSDimitry Andric Value *Mask, Value *PassThru, 689bdd1243dSDimitry Andric const Twine &Name) { 690bdd1243dSDimitry Andric assert(Ty->isVectorTy() && "Type should be vector"); 691bdd1243dSDimitry Andric assert(Mask && "Mask should not be all-ones (null)"); 692bdd1243dSDimitry Andric if (!PassThru) 693bdd1243dSDimitry Andric PassThru = PoisonValue::get(Ty); 694bdd1243dSDimitry Andric Type *OverloadedTypes[] = {Ty}; 695bdd1243dSDimitry Andric Value *Ops[] = {Ptr, Mask, PassThru}; 696bdd1243dSDimitry Andric return CreateMaskedIntrinsic(Intrinsic::masked_expandload, Ops, 697bdd1243dSDimitry Andric OverloadedTypes, Name); 698bdd1243dSDimitry Andric } 699bdd1243dSDimitry Andric 700bdd1243dSDimitry Andric /// Create a call to Masked Compress Store intrinsic 701bdd1243dSDimitry Andric /// \p Val - data to be stored, 702bdd1243dSDimitry Andric /// \p Ptr - base pointer for the store 703bdd1243dSDimitry Andric /// \p Mask - vector of booleans which indicates what vector lanes should 704bdd1243dSDimitry Andric /// be accessed in memory 705bdd1243dSDimitry Andric CallInst *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr, 706bdd1243dSDimitry Andric Value *Mask) { 707bdd1243dSDimitry Andric Type *DataTy = Val->getType(); 708bdd1243dSDimitry Andric assert(DataTy->isVectorTy() && "Val should be a vector"); 709bdd1243dSDimitry Andric assert(Mask && "Mask should not be all-ones (null)"); 710bdd1243dSDimitry Andric Type *OverloadedTypes[] = {DataTy}; 711bdd1243dSDimitry Andric Value *Ops[] = {Val, Ptr, Mask}; 712bdd1243dSDimitry Andric return CreateMaskedIntrinsic(Intrinsic::masked_compressstore, Ops, 713bdd1243dSDimitry Andric OverloadedTypes); 714bdd1243dSDimitry Andric } 715bdd1243dSDimitry Andric 7165ffd83dbSDimitry Andric template <typename T0> 7170b57cec5SDimitry Andric static std::vector<Value *> 7180b57cec5SDimitry Andric getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes, 7195ffd83dbSDimitry Andric Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) { 7200b57cec5SDimitry Andric std::vector<Value *> Args; 7210b57cec5SDimitry Andric Args.push_back(B.getInt64(ID)); 7220b57cec5SDimitry Andric Args.push_back(B.getInt32(NumPatchBytes)); 7230b57cec5SDimitry Andric Args.push_back(ActualCallee); 7240b57cec5SDimitry Andric Args.push_back(B.getInt32(CallArgs.size())); 7250b57cec5SDimitry Andric Args.push_back(B.getInt32(Flags)); 726e8d8bef9SDimitry Andric llvm::append_range(Args, CallArgs); 7275ffd83dbSDimitry Andric // GC Transition and Deopt args are now always handled via operand bundle. 7285ffd83dbSDimitry Andric // They will be removed from the signature of gc.statepoint shortly. 7295ffd83dbSDimitry Andric Args.push_back(B.getInt32(0)); 7305ffd83dbSDimitry Andric Args.push_back(B.getInt32(0)); 7315ffd83dbSDimitry Andric // GC args are now encoded in the gc-live operand bundle 7320b57cec5SDimitry Andric return Args; 7330b57cec5SDimitry Andric } 7340b57cec5SDimitry Andric 7355ffd83dbSDimitry Andric template<typename T1, typename T2, typename T3> 7365ffd83dbSDimitry Andric static std::vector<OperandBundleDef> 737bdd1243dSDimitry Andric getStatepointBundles(std::optional<ArrayRef<T1>> TransitionArgs, 738bdd1243dSDimitry Andric std::optional<ArrayRef<T2>> DeoptArgs, 7395ffd83dbSDimitry Andric ArrayRef<T3> GCArgs) { 7405ffd83dbSDimitry Andric std::vector<OperandBundleDef> Rval; 7415ffd83dbSDimitry Andric if (DeoptArgs) { 7425ffd83dbSDimitry Andric SmallVector<Value*, 16> DeoptValues; 743e8d8bef9SDimitry Andric llvm::append_range(DeoptValues, *DeoptArgs); 7445ffd83dbSDimitry Andric Rval.emplace_back("deopt", DeoptValues); 7455ffd83dbSDimitry Andric } 7465ffd83dbSDimitry Andric if (TransitionArgs) { 7475ffd83dbSDimitry Andric SmallVector<Value*, 16> TransitionValues; 748e8d8bef9SDimitry Andric llvm::append_range(TransitionValues, *TransitionArgs); 7495ffd83dbSDimitry Andric Rval.emplace_back("gc-transition", TransitionValues); 7505ffd83dbSDimitry Andric } 7515ffd83dbSDimitry Andric if (GCArgs.size()) { 7525ffd83dbSDimitry Andric SmallVector<Value*, 16> LiveValues; 753e8d8bef9SDimitry Andric llvm::append_range(LiveValues, GCArgs); 7545ffd83dbSDimitry Andric Rval.emplace_back("gc-live", LiveValues); 7555ffd83dbSDimitry Andric } 7565ffd83dbSDimitry Andric return Rval; 7575ffd83dbSDimitry Andric } 7585ffd83dbSDimitry Andric 7590b57cec5SDimitry Andric template <typename T0, typename T1, typename T2, typename T3> 7600b57cec5SDimitry Andric static CallInst *CreateGCStatepointCallCommon( 7610b57cec5SDimitry Andric IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes, 76281ad6265SDimitry Andric FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs, 763bdd1243dSDimitry Andric std::optional<ArrayRef<T1>> TransitionArgs, 764bdd1243dSDimitry Andric std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs, 765bdd1243dSDimitry Andric const Twine &Name) { 7660b57cec5SDimitry Andric Module *M = Builder->GetInsertBlock()->getParent()->getParent(); 7670b57cec5SDimitry Andric // Fill in the one generic type'd argument (the function is also vararg) 7680b57cec5SDimitry Andric Function *FnStatepoint = 7690b57cec5SDimitry Andric Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint, 77081ad6265SDimitry Andric {ActualCallee.getCallee()->getType()}); 7710b57cec5SDimitry Andric 77281ad6265SDimitry Andric std::vector<Value *> Args = getStatepointArgs( 77381ad6265SDimitry Andric *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs); 7745ffd83dbSDimitry Andric 77581ad6265SDimitry Andric CallInst *CI = Builder->CreateCall( 77681ad6265SDimitry Andric FnStatepoint, Args, 77781ad6265SDimitry Andric getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name); 77881ad6265SDimitry Andric CI->addParamAttr(2, 77981ad6265SDimitry Andric Attribute::get(Builder->getContext(), Attribute::ElementType, 78081ad6265SDimitry Andric ActualCallee.getFunctionType())); 78181ad6265SDimitry Andric return CI; 7820b57cec5SDimitry Andric } 7830b57cec5SDimitry Andric 7840b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateGCStatepointCall( 78581ad6265SDimitry Andric uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee, 786bdd1243dSDimitry Andric ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs, 7870b57cec5SDimitry Andric ArrayRef<Value *> GCArgs, const Twine &Name) { 7880b57cec5SDimitry Andric return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>( 7890b57cec5SDimitry Andric this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None), 790bdd1243dSDimitry Andric CallArgs, std::nullopt /* No Transition Args */, DeoptArgs, GCArgs, Name); 7910b57cec5SDimitry Andric } 7920b57cec5SDimitry Andric 7930b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateGCStatepointCall( 79481ad6265SDimitry Andric uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee, 79581ad6265SDimitry Andric uint32_t Flags, ArrayRef<Value *> CallArgs, 796bdd1243dSDimitry Andric std::optional<ArrayRef<Use>> TransitionArgs, 797bdd1243dSDimitry Andric std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs, 798bdd1243dSDimitry Andric const Twine &Name) { 799e8d8bef9SDimitry Andric return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>( 8000b57cec5SDimitry Andric this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs, 8010b57cec5SDimitry Andric DeoptArgs, GCArgs, Name); 8020b57cec5SDimitry Andric } 8030b57cec5SDimitry Andric 8040b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateGCStatepointCall( 80581ad6265SDimitry Andric uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee, 806bdd1243dSDimitry Andric ArrayRef<Use> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs, 8070b57cec5SDimitry Andric ArrayRef<Value *> GCArgs, const Twine &Name) { 8080b57cec5SDimitry Andric return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>( 8090b57cec5SDimitry Andric this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None), 810bdd1243dSDimitry Andric CallArgs, std::nullopt, DeoptArgs, GCArgs, Name); 8110b57cec5SDimitry Andric } 8120b57cec5SDimitry Andric 8130b57cec5SDimitry Andric template <typename T0, typename T1, typename T2, typename T3> 8140b57cec5SDimitry Andric static InvokeInst *CreateGCStatepointInvokeCommon( 8150b57cec5SDimitry Andric IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes, 81681ad6265SDimitry Andric FunctionCallee ActualInvokee, BasicBlock *NormalDest, 81781ad6265SDimitry Andric BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<T0> InvokeArgs, 818bdd1243dSDimitry Andric std::optional<ArrayRef<T1>> TransitionArgs, 819bdd1243dSDimitry Andric std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs, 820bdd1243dSDimitry Andric const Twine &Name) { 8210b57cec5SDimitry Andric Module *M = Builder->GetInsertBlock()->getParent()->getParent(); 8220b57cec5SDimitry Andric // Fill in the one generic type'd argument (the function is also vararg) 82381ad6265SDimitry Andric Function *FnStatepoint = 82481ad6265SDimitry Andric Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint, 82581ad6265SDimitry Andric {ActualInvokee.getCallee()->getType()}); 8260b57cec5SDimitry Andric 8270b57cec5SDimitry Andric std::vector<Value *> Args = 82881ad6265SDimitry Andric getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(), 82981ad6265SDimitry Andric Flags, InvokeArgs); 8305ffd83dbSDimitry Andric 83181ad6265SDimitry Andric InvokeInst *II = Builder->CreateInvoke( 83281ad6265SDimitry Andric FnStatepoint, NormalDest, UnwindDest, Args, 83381ad6265SDimitry Andric getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name); 83481ad6265SDimitry Andric II->addParamAttr(2, 83581ad6265SDimitry Andric Attribute::get(Builder->getContext(), Attribute::ElementType, 83681ad6265SDimitry Andric ActualInvokee.getFunctionType())); 83781ad6265SDimitry Andric return II; 8380b57cec5SDimitry Andric } 8390b57cec5SDimitry Andric 8400b57cec5SDimitry Andric InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( 84181ad6265SDimitry Andric uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee, 8420b57cec5SDimitry Andric BasicBlock *NormalDest, BasicBlock *UnwindDest, 843bdd1243dSDimitry Andric ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Value *>> DeoptArgs, 8440b57cec5SDimitry Andric ArrayRef<Value *> GCArgs, const Twine &Name) { 8450b57cec5SDimitry Andric return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>( 8460b57cec5SDimitry Andric this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, 847bdd1243dSDimitry Andric uint32_t(StatepointFlags::None), InvokeArgs, 848bdd1243dSDimitry Andric std::nullopt /* No Transition Args*/, DeoptArgs, GCArgs, Name); 8490b57cec5SDimitry Andric } 8500b57cec5SDimitry Andric 8510b57cec5SDimitry Andric InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( 85281ad6265SDimitry Andric uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee, 8530b57cec5SDimitry Andric BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags, 854bdd1243dSDimitry Andric ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Use>> TransitionArgs, 855bdd1243dSDimitry Andric std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs, 85681ad6265SDimitry Andric const Twine &Name) { 857e8d8bef9SDimitry Andric return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>( 8580b57cec5SDimitry Andric this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags, 8590b57cec5SDimitry Andric InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name); 8600b57cec5SDimitry Andric } 8610b57cec5SDimitry Andric 8620b57cec5SDimitry Andric InvokeInst *IRBuilderBase::CreateGCStatepointInvoke( 86381ad6265SDimitry Andric uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee, 8640b57cec5SDimitry Andric BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs, 865bdd1243dSDimitry Andric std::optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs, 86681ad6265SDimitry Andric const Twine &Name) { 8670b57cec5SDimitry Andric return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>( 8680b57cec5SDimitry Andric this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, 869bdd1243dSDimitry Andric uint32_t(StatepointFlags::None), InvokeArgs, std::nullopt, DeoptArgs, 870bdd1243dSDimitry Andric GCArgs, Name); 8710b57cec5SDimitry Andric } 8720b57cec5SDimitry Andric 8730b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint, 87481ad6265SDimitry Andric Type *ResultType, const Twine &Name) { 8750b57cec5SDimitry Andric Intrinsic::ID ID = Intrinsic::experimental_gc_result; 8760b57cec5SDimitry Andric Module *M = BB->getParent()->getParent(); 8770b57cec5SDimitry Andric Type *Types[] = {ResultType}; 8780b57cec5SDimitry Andric Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types); 8790b57cec5SDimitry Andric 8800b57cec5SDimitry Andric Value *Args[] = {Statepoint}; 881bdd1243dSDimitry Andric return CreateCall(FnGCResult, Args, {}, Name); 8820b57cec5SDimitry Andric } 8830b57cec5SDimitry Andric 8840b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint, 88581ad6265SDimitry Andric int BaseOffset, int DerivedOffset, 88681ad6265SDimitry Andric Type *ResultType, const Twine &Name) { 8870b57cec5SDimitry Andric Module *M = BB->getParent()->getParent(); 8880b57cec5SDimitry Andric Type *Types[] = {ResultType}; 8890b57cec5SDimitry Andric Function *FnGCRelocate = 8900b57cec5SDimitry Andric Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types); 8910b57cec5SDimitry Andric 89281ad6265SDimitry Andric Value *Args[] = {Statepoint, getInt32(BaseOffset), getInt32(DerivedOffset)}; 893bdd1243dSDimitry Andric return CreateCall(FnGCRelocate, Args, {}, Name); 8940b57cec5SDimitry Andric } 8950b57cec5SDimitry Andric 896fe6060f1SDimitry Andric CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr, 897fe6060f1SDimitry Andric const Twine &Name) { 898fe6060f1SDimitry Andric Module *M = BB->getParent()->getParent(); 899fe6060f1SDimitry Andric Type *PtrTy = DerivedPtr->getType(); 900fe6060f1SDimitry Andric Function *FnGCFindBase = Intrinsic::getDeclaration( 901fe6060f1SDimitry Andric M, Intrinsic::experimental_gc_get_pointer_base, {PtrTy, PtrTy}); 902bdd1243dSDimitry Andric return CreateCall(FnGCFindBase, {DerivedPtr}, {}, Name); 903fe6060f1SDimitry Andric } 904fe6060f1SDimitry Andric 905fe6060f1SDimitry Andric CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr, 906fe6060f1SDimitry Andric const Twine &Name) { 907fe6060f1SDimitry Andric Module *M = BB->getParent()->getParent(); 908fe6060f1SDimitry Andric Type *PtrTy = DerivedPtr->getType(); 909fe6060f1SDimitry Andric Function *FnGCGetOffset = Intrinsic::getDeclaration( 910fe6060f1SDimitry Andric M, Intrinsic::experimental_gc_get_pointer_offset, {PtrTy}); 911bdd1243dSDimitry Andric return CreateCall(FnGCGetOffset, {DerivedPtr}, {}, Name); 912fe6060f1SDimitry Andric } 913fe6060f1SDimitry Andric 9140b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, 9150b57cec5SDimitry Andric Instruction *FMFSource, 9160b57cec5SDimitry Andric const Twine &Name) { 9170b57cec5SDimitry Andric Module *M = BB->getModule(); 9180b57cec5SDimitry Andric Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()}); 919bdd1243dSDimitry Andric return createCallHelper(Fn, {V}, Name, FMFSource); 9200b57cec5SDimitry Andric } 9210b57cec5SDimitry Andric 922*0fca6ea1SDimitry Andric Value *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, 923*0fca6ea1SDimitry Andric Value *RHS, Instruction *FMFSource, 9240b57cec5SDimitry Andric const Twine &Name) { 9250b57cec5SDimitry Andric Module *M = BB->getModule(); 9260b57cec5SDimitry Andric Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() }); 927*0fca6ea1SDimitry Andric if (Value *V = Folder.FoldBinaryIntrinsic(ID, LHS, RHS, Fn->getReturnType(), 928*0fca6ea1SDimitry Andric FMFSource)) 929*0fca6ea1SDimitry Andric return V; 930bdd1243dSDimitry Andric return createCallHelper(Fn, {LHS, RHS}, Name, FMFSource); 9310b57cec5SDimitry Andric } 9320b57cec5SDimitry Andric 9330b57cec5SDimitry Andric CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID, 9340b57cec5SDimitry Andric ArrayRef<Type *> Types, 9350b57cec5SDimitry Andric ArrayRef<Value *> Args, 9360b57cec5SDimitry Andric Instruction *FMFSource, 9370b57cec5SDimitry Andric const Twine &Name) { 9380b57cec5SDimitry Andric Module *M = BB->getModule(); 9390b57cec5SDimitry Andric Function *Fn = Intrinsic::getDeclaration(M, ID, Types); 940bdd1243dSDimitry Andric return createCallHelper(Fn, Args, Name, FMFSource); 941bdd1243dSDimitry Andric } 942bdd1243dSDimitry Andric 943bdd1243dSDimitry Andric CallInst *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID, 944bdd1243dSDimitry Andric ArrayRef<Value *> Args, 945bdd1243dSDimitry Andric Instruction *FMFSource, 946bdd1243dSDimitry Andric const Twine &Name) { 947bdd1243dSDimitry Andric Module *M = BB->getModule(); 948bdd1243dSDimitry Andric 949bdd1243dSDimitry Andric SmallVector<Intrinsic::IITDescriptor> Table; 950bdd1243dSDimitry Andric Intrinsic::getIntrinsicInfoTableEntries(ID, Table); 951bdd1243dSDimitry Andric ArrayRef<Intrinsic::IITDescriptor> TableRef(Table); 952bdd1243dSDimitry Andric 953bdd1243dSDimitry Andric SmallVector<Type *> ArgTys; 954bdd1243dSDimitry Andric ArgTys.reserve(Args.size()); 955bdd1243dSDimitry Andric for (auto &I : Args) 956bdd1243dSDimitry Andric ArgTys.push_back(I->getType()); 957bdd1243dSDimitry Andric FunctionType *FTy = FunctionType::get(RetTy, ArgTys, false); 958bdd1243dSDimitry Andric SmallVector<Type *> OverloadTys; 959bdd1243dSDimitry Andric Intrinsic::MatchIntrinsicTypesResult Res = 960bdd1243dSDimitry Andric matchIntrinsicSignature(FTy, TableRef, OverloadTys); 961bdd1243dSDimitry Andric (void)Res; 962bdd1243dSDimitry Andric assert(Res == Intrinsic::MatchIntrinsicTypes_Match && TableRef.empty() && 963bdd1243dSDimitry Andric "Wrong types for intrinsic!"); 964bdd1243dSDimitry Andric // TODO: Handle varargs intrinsics. 965bdd1243dSDimitry Andric 966bdd1243dSDimitry Andric Function *Fn = Intrinsic::getDeclaration(M, ID, OverloadTys); 967bdd1243dSDimitry Andric return createCallHelper(Fn, Args, Name, FMFSource); 9680b57cec5SDimitry Andric } 9695ffd83dbSDimitry Andric 9705ffd83dbSDimitry Andric CallInst *IRBuilderBase::CreateConstrainedFPBinOp( 9715ffd83dbSDimitry Andric Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource, 9725ffd83dbSDimitry Andric const Twine &Name, MDNode *FPMathTag, 973bdd1243dSDimitry Andric std::optional<RoundingMode> Rounding, 974bdd1243dSDimitry Andric std::optional<fp::ExceptionBehavior> Except) { 9755ffd83dbSDimitry Andric Value *RoundingV = getConstrainedFPRounding(Rounding); 9765ffd83dbSDimitry Andric Value *ExceptV = getConstrainedFPExcept(Except); 9775ffd83dbSDimitry Andric 9785ffd83dbSDimitry Andric FastMathFlags UseFMF = FMF; 9795ffd83dbSDimitry Andric if (FMFSource) 9805ffd83dbSDimitry Andric UseFMF = FMFSource->getFastMathFlags(); 9815ffd83dbSDimitry Andric 9825ffd83dbSDimitry Andric CallInst *C = CreateIntrinsic(ID, {L->getType()}, 9835ffd83dbSDimitry Andric {L, R, RoundingV, ExceptV}, nullptr, Name); 9845ffd83dbSDimitry Andric setConstrainedFPCallAttr(C); 9855ffd83dbSDimitry Andric setFPAttrs(C, FPMathTag, UseFMF); 9865ffd83dbSDimitry Andric return C; 9875ffd83dbSDimitry Andric } 9885ffd83dbSDimitry Andric 98906c3fb27SDimitry Andric CallInst *IRBuilderBase::CreateConstrainedFPUnroundedBinOp( 99006c3fb27SDimitry Andric Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource, 99106c3fb27SDimitry Andric const Twine &Name, MDNode *FPMathTag, 99206c3fb27SDimitry Andric std::optional<fp::ExceptionBehavior> Except) { 99306c3fb27SDimitry Andric Value *ExceptV = getConstrainedFPExcept(Except); 99406c3fb27SDimitry Andric 99506c3fb27SDimitry Andric FastMathFlags UseFMF = FMF; 99606c3fb27SDimitry Andric if (FMFSource) 99706c3fb27SDimitry Andric UseFMF = FMFSource->getFastMathFlags(); 99806c3fb27SDimitry Andric 99906c3fb27SDimitry Andric CallInst *C = 100006c3fb27SDimitry Andric CreateIntrinsic(ID, {L->getType()}, {L, R, ExceptV}, nullptr, Name); 100106c3fb27SDimitry Andric setConstrainedFPCallAttr(C); 100206c3fb27SDimitry Andric setFPAttrs(C, FPMathTag, UseFMF); 100306c3fb27SDimitry Andric return C; 100406c3fb27SDimitry Andric } 100506c3fb27SDimitry Andric 10065ffd83dbSDimitry Andric Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops, 10075ffd83dbSDimitry Andric const Twine &Name, MDNode *FPMathTag) { 10085ffd83dbSDimitry Andric if (Instruction::isBinaryOp(Opc)) { 10095ffd83dbSDimitry Andric assert(Ops.size() == 2 && "Invalid number of operands!"); 10105ffd83dbSDimitry Andric return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc), 10115ffd83dbSDimitry Andric Ops[0], Ops[1], Name, FPMathTag); 10125ffd83dbSDimitry Andric } 10135ffd83dbSDimitry Andric if (Instruction::isUnaryOp(Opc)) { 10145ffd83dbSDimitry Andric assert(Ops.size() == 1 && "Invalid number of operands!"); 10155ffd83dbSDimitry Andric return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc), 10165ffd83dbSDimitry Andric Ops[0], Name, FPMathTag); 10175ffd83dbSDimitry Andric } 10185ffd83dbSDimitry Andric llvm_unreachable("Unexpected opcode!"); 10195ffd83dbSDimitry Andric } 10205ffd83dbSDimitry Andric 10215ffd83dbSDimitry Andric CallInst *IRBuilderBase::CreateConstrainedFPCast( 10225ffd83dbSDimitry Andric Intrinsic::ID ID, Value *V, Type *DestTy, 10235ffd83dbSDimitry Andric Instruction *FMFSource, const Twine &Name, MDNode *FPMathTag, 1024bdd1243dSDimitry Andric std::optional<RoundingMode> Rounding, 1025bdd1243dSDimitry Andric std::optional<fp::ExceptionBehavior> Except) { 10265ffd83dbSDimitry Andric Value *ExceptV = getConstrainedFPExcept(Except); 10275ffd83dbSDimitry Andric 10285ffd83dbSDimitry Andric FastMathFlags UseFMF = FMF; 10295ffd83dbSDimitry Andric if (FMFSource) 10305ffd83dbSDimitry Andric UseFMF = FMFSource->getFastMathFlags(); 10315ffd83dbSDimitry Andric 10325ffd83dbSDimitry Andric CallInst *C; 1033*0fca6ea1SDimitry Andric if (Intrinsic::hasConstrainedFPRoundingModeOperand(ID)) { 10345ffd83dbSDimitry Andric Value *RoundingV = getConstrainedFPRounding(Rounding); 10355ffd83dbSDimitry Andric C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV}, 10365ffd83dbSDimitry Andric nullptr, Name); 10375ffd83dbSDimitry Andric } else 10385ffd83dbSDimitry Andric C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr, 10395ffd83dbSDimitry Andric Name); 10405ffd83dbSDimitry Andric 10415ffd83dbSDimitry Andric setConstrainedFPCallAttr(C); 10425ffd83dbSDimitry Andric 10435ffd83dbSDimitry Andric if (isa<FPMathOperator>(C)) 10445ffd83dbSDimitry Andric setFPAttrs(C, FPMathTag, UseFMF); 10455ffd83dbSDimitry Andric return C; 10465ffd83dbSDimitry Andric } 10475ffd83dbSDimitry Andric 10485ffd83dbSDimitry Andric Value *IRBuilderBase::CreateFCmpHelper( 10495ffd83dbSDimitry Andric CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name, 10505ffd83dbSDimitry Andric MDNode *FPMathTag, bool IsSignaling) { 10515ffd83dbSDimitry Andric if (IsFPConstrained) { 10525ffd83dbSDimitry Andric auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps 10535ffd83dbSDimitry Andric : Intrinsic::experimental_constrained_fcmp; 10545ffd83dbSDimitry Andric return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name); 10555ffd83dbSDimitry Andric } 10565ffd83dbSDimitry Andric 1057*0fca6ea1SDimitry Andric if (auto *V = Folder.FoldCmp(P, LHS, RHS)) 1058*0fca6ea1SDimitry Andric return V; 10595ffd83dbSDimitry Andric return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name); 10605ffd83dbSDimitry Andric } 10615ffd83dbSDimitry Andric 10625ffd83dbSDimitry Andric CallInst *IRBuilderBase::CreateConstrainedFPCmp( 10635ffd83dbSDimitry Andric Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R, 1064bdd1243dSDimitry Andric const Twine &Name, std::optional<fp::ExceptionBehavior> Except) { 10655ffd83dbSDimitry Andric Value *PredicateV = getConstrainedFPPredicate(P); 10665ffd83dbSDimitry Andric Value *ExceptV = getConstrainedFPExcept(Except); 10675ffd83dbSDimitry Andric 10685ffd83dbSDimitry Andric CallInst *C = CreateIntrinsic(ID, {L->getType()}, 10695ffd83dbSDimitry Andric {L, R, PredicateV, ExceptV}, nullptr, Name); 10705ffd83dbSDimitry Andric setConstrainedFPCallAttr(C); 10715ffd83dbSDimitry Andric return C; 10725ffd83dbSDimitry Andric } 10735ffd83dbSDimitry Andric 10745ffd83dbSDimitry Andric CallInst *IRBuilderBase::CreateConstrainedFPCall( 10755ffd83dbSDimitry Andric Function *Callee, ArrayRef<Value *> Args, const Twine &Name, 1076bdd1243dSDimitry Andric std::optional<RoundingMode> Rounding, 1077bdd1243dSDimitry Andric std::optional<fp::ExceptionBehavior> Except) { 10785ffd83dbSDimitry Andric llvm::SmallVector<Value *, 6> UseArgs; 10795ffd83dbSDimitry Andric 1080fe6060f1SDimitry Andric append_range(UseArgs, Args); 1081*0fca6ea1SDimitry Andric 1082*0fca6ea1SDimitry Andric if (Intrinsic::hasConstrainedFPRoundingModeOperand(Callee->getIntrinsicID())) 10835ffd83dbSDimitry Andric UseArgs.push_back(getConstrainedFPRounding(Rounding)); 10845ffd83dbSDimitry Andric UseArgs.push_back(getConstrainedFPExcept(Except)); 10855ffd83dbSDimitry Andric 10865ffd83dbSDimitry Andric CallInst *C = CreateCall(Callee, UseArgs, Name); 10875ffd83dbSDimitry Andric setConstrainedFPCallAttr(C); 10885ffd83dbSDimitry Andric return C; 10895ffd83dbSDimitry Andric } 10905ffd83dbSDimitry Andric 10915ffd83dbSDimitry Andric Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False, 10925ffd83dbSDimitry Andric const Twine &Name, Instruction *MDFrom) { 109304eeddc0SDimitry Andric if (auto *V = Folder.FoldSelect(C, True, False)) 109404eeddc0SDimitry Andric return V; 10955ffd83dbSDimitry Andric 10965ffd83dbSDimitry Andric SelectInst *Sel = SelectInst::Create(C, True, False); 10975ffd83dbSDimitry Andric if (MDFrom) { 10985ffd83dbSDimitry Andric MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof); 10995ffd83dbSDimitry Andric MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable); 11005ffd83dbSDimitry Andric Sel = addBranchMetadata(Sel, Prof, Unpred); 11015ffd83dbSDimitry Andric } 11025ffd83dbSDimitry Andric if (isa<FPMathOperator>(Sel)) 11035ffd83dbSDimitry Andric setFPAttrs(Sel, nullptr /* MDNode* */, FMF); 11045ffd83dbSDimitry Andric return Insert(Sel, Name); 11055ffd83dbSDimitry Andric } 11065ffd83dbSDimitry Andric 110704eeddc0SDimitry Andric Value *IRBuilderBase::CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS, 11085ffd83dbSDimitry Andric const Twine &Name) { 11095ffd83dbSDimitry Andric assert(LHS->getType() == RHS->getType() && 11105ffd83dbSDimitry Andric "Pointer subtraction operand types must match!"); 11115ffd83dbSDimitry Andric Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context)); 11125ffd83dbSDimitry Andric Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context)); 11135ffd83dbSDimitry Andric Value *Difference = CreateSub(LHS_int, RHS_int); 111404eeddc0SDimitry Andric return CreateExactSDiv(Difference, ConstantExpr::getSizeOf(ElemTy), 11155ffd83dbSDimitry Andric Name); 11165ffd83dbSDimitry Andric } 11175ffd83dbSDimitry Andric 11185ffd83dbSDimitry Andric Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) { 11195ffd83dbSDimitry Andric assert(isa<PointerType>(Ptr->getType()) && 11205ffd83dbSDimitry Andric "launder.invariant.group only applies to pointers."); 11215ffd83dbSDimitry Andric auto *PtrType = Ptr->getType(); 11225ffd83dbSDimitry Andric Module *M = BB->getParent()->getParent(); 11235ffd83dbSDimitry Andric Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration( 112406c3fb27SDimitry Andric M, Intrinsic::launder_invariant_group, {PtrType}); 11255ffd83dbSDimitry Andric 112606c3fb27SDimitry Andric assert(FnLaunderInvariantGroup->getReturnType() == PtrType && 11275ffd83dbSDimitry Andric FnLaunderInvariantGroup->getFunctionType()->getParamType(0) == 112806c3fb27SDimitry Andric PtrType && 11295ffd83dbSDimitry Andric "LaunderInvariantGroup should take and return the same type"); 11305ffd83dbSDimitry Andric 113106c3fb27SDimitry Andric return CreateCall(FnLaunderInvariantGroup, {Ptr}); 11325ffd83dbSDimitry Andric } 11335ffd83dbSDimitry Andric 11345ffd83dbSDimitry Andric Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) { 11355ffd83dbSDimitry Andric assert(isa<PointerType>(Ptr->getType()) && 11365ffd83dbSDimitry Andric "strip.invariant.group only applies to pointers."); 11375ffd83dbSDimitry Andric 11385ffd83dbSDimitry Andric auto *PtrType = Ptr->getType(); 11395ffd83dbSDimitry Andric Module *M = BB->getParent()->getParent(); 11405ffd83dbSDimitry Andric Function *FnStripInvariantGroup = Intrinsic::getDeclaration( 114106c3fb27SDimitry Andric M, Intrinsic::strip_invariant_group, {PtrType}); 11425ffd83dbSDimitry Andric 114306c3fb27SDimitry Andric assert(FnStripInvariantGroup->getReturnType() == PtrType && 11445ffd83dbSDimitry Andric FnStripInvariantGroup->getFunctionType()->getParamType(0) == 114506c3fb27SDimitry Andric PtrType && 11465ffd83dbSDimitry Andric "StripInvariantGroup should take and return the same type"); 11475ffd83dbSDimitry Andric 114806c3fb27SDimitry Andric return CreateCall(FnStripInvariantGroup, {Ptr}); 11495ffd83dbSDimitry Andric } 11505ffd83dbSDimitry Andric 1151fe6060f1SDimitry Andric Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) { 1152fe6060f1SDimitry Andric auto *Ty = cast<VectorType>(V->getType()); 1153fe6060f1SDimitry Andric if (isa<ScalableVectorType>(Ty)) { 1154fe6060f1SDimitry Andric Module *M = BB->getParent()->getParent(); 1155*0fca6ea1SDimitry Andric Function *F = Intrinsic::getDeclaration(M, Intrinsic::vector_reverse, Ty); 1156fe6060f1SDimitry Andric return Insert(CallInst::Create(F, V), Name); 1157fe6060f1SDimitry Andric } 1158fe6060f1SDimitry Andric // Keep the original behaviour for fixed vector 1159fe6060f1SDimitry Andric SmallVector<int, 8> ShuffleMask; 1160fe6060f1SDimitry Andric int NumElts = Ty->getElementCount().getKnownMinValue(); 1161fe6060f1SDimitry Andric for (int i = 0; i < NumElts; ++i) 1162fe6060f1SDimitry Andric ShuffleMask.push_back(NumElts - i - 1); 1163fe6060f1SDimitry Andric return CreateShuffleVector(V, ShuffleMask, Name); 1164fe6060f1SDimitry Andric } 1165fe6060f1SDimitry Andric 1166fe6060f1SDimitry Andric Value *IRBuilderBase::CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, 1167fe6060f1SDimitry Andric const Twine &Name) { 1168fe6060f1SDimitry Andric assert(isa<VectorType>(V1->getType()) && "Unexpected type"); 1169fe6060f1SDimitry Andric assert(V1->getType() == V2->getType() && 1170fe6060f1SDimitry Andric "Splice expects matching operand types!"); 1171fe6060f1SDimitry Andric 1172fe6060f1SDimitry Andric if (auto *VTy = dyn_cast<ScalableVectorType>(V1->getType())) { 1173fe6060f1SDimitry Andric Module *M = BB->getParent()->getParent(); 1174*0fca6ea1SDimitry Andric Function *F = Intrinsic::getDeclaration(M, Intrinsic::vector_splice, VTy); 1175fe6060f1SDimitry Andric 1176fe6060f1SDimitry Andric Value *Ops[] = {V1, V2, getInt32(Imm)}; 1177fe6060f1SDimitry Andric return Insert(CallInst::Create(F, Ops), Name); 1178fe6060f1SDimitry Andric } 1179fe6060f1SDimitry Andric 1180fe6060f1SDimitry Andric unsigned NumElts = cast<FixedVectorType>(V1->getType())->getNumElements(); 1181fe6060f1SDimitry Andric assert(((-Imm <= NumElts) || (Imm < NumElts)) && 1182fe6060f1SDimitry Andric "Invalid immediate for vector splice!"); 1183fe6060f1SDimitry Andric 1184fe6060f1SDimitry Andric // Keep the original behaviour for fixed vector 1185fe6060f1SDimitry Andric unsigned Idx = (NumElts + Imm) % NumElts; 1186fe6060f1SDimitry Andric SmallVector<int, 8> Mask; 1187fe6060f1SDimitry Andric for (unsigned I = 0; I < NumElts; ++I) 1188fe6060f1SDimitry Andric Mask.push_back(Idx + I); 1189fe6060f1SDimitry Andric 1190fe6060f1SDimitry Andric return CreateShuffleVector(V1, V2, Mask); 1191fe6060f1SDimitry Andric } 1192fe6060f1SDimitry Andric 11935ffd83dbSDimitry Andric Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V, 11945ffd83dbSDimitry Andric const Twine &Name) { 1195e8d8bef9SDimitry Andric auto EC = ElementCount::getFixed(NumElts); 1196e8d8bef9SDimitry Andric return CreateVectorSplat(EC, V, Name); 1197e8d8bef9SDimitry Andric } 11985ffd83dbSDimitry Andric 1199e8d8bef9SDimitry Andric Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V, 1200e8d8bef9SDimitry Andric const Twine &Name) { 1201e8d8bef9SDimitry Andric assert(EC.isNonZero() && "Cannot splat to an empty vector!"); 1202e8d8bef9SDimitry Andric 1203e8d8bef9SDimitry Andric // First insert it into a poison vector so we can shuffle it. 1204e8d8bef9SDimitry Andric Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC)); 1205bdd1243dSDimitry Andric V = CreateInsertElement(Poison, V, getInt64(0), Name + ".splatinsert"); 12065ffd83dbSDimitry Andric 12075ffd83dbSDimitry Andric // Shuffle the value across the desired number of elements. 1208e8d8bef9SDimitry Andric SmallVector<int, 16> Zeros; 1209e8d8bef9SDimitry Andric Zeros.resize(EC.getKnownMinValue()); 1210e8d8bef9SDimitry Andric return CreateShuffleVector(V, Zeros, Name + ".splat"); 12115ffd83dbSDimitry Andric } 12125ffd83dbSDimitry Andric 12135ffd83dbSDimitry Andric Value *IRBuilderBase::CreatePreserveArrayAccessIndex( 12145ffd83dbSDimitry Andric Type *ElTy, Value *Base, unsigned Dimension, unsigned LastIndex, 12155ffd83dbSDimitry Andric MDNode *DbgInfo) { 12165ffd83dbSDimitry Andric auto *BaseType = Base->getType(); 1217fe6060f1SDimitry Andric assert(isa<PointerType>(BaseType) && 1218fe6060f1SDimitry Andric "Invalid Base ptr type for preserve.array.access.index."); 12195ffd83dbSDimitry Andric 12205ffd83dbSDimitry Andric Value *LastIndexV = getInt32(LastIndex); 12215ffd83dbSDimitry Andric Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); 1222e8d8bef9SDimitry Andric SmallVector<Value *, 4> IdxList(Dimension, Zero); 12235ffd83dbSDimitry Andric IdxList.push_back(LastIndexV); 12245ffd83dbSDimitry Andric 122506c3fb27SDimitry Andric Type *ResultType = GetElementPtrInst::getGEPReturnType(Base, IdxList); 12265ffd83dbSDimitry Andric 12275ffd83dbSDimitry Andric Module *M = BB->getParent()->getParent(); 12285ffd83dbSDimitry Andric Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration( 12295ffd83dbSDimitry Andric M, Intrinsic::preserve_array_access_index, {ResultType, BaseType}); 12305ffd83dbSDimitry Andric 12315ffd83dbSDimitry Andric Value *DimV = getInt32(Dimension); 12325ffd83dbSDimitry Andric CallInst *Fn = 12335ffd83dbSDimitry Andric CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV}); 1234fe6060f1SDimitry Andric Fn->addParamAttr( 1235fe6060f1SDimitry Andric 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy)); 12365ffd83dbSDimitry Andric if (DbgInfo) 12375ffd83dbSDimitry Andric Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); 12385ffd83dbSDimitry Andric 12395ffd83dbSDimitry Andric return Fn; 12405ffd83dbSDimitry Andric } 12415ffd83dbSDimitry Andric 12425ffd83dbSDimitry Andric Value *IRBuilderBase::CreatePreserveUnionAccessIndex( 12435ffd83dbSDimitry Andric Value *Base, unsigned FieldIndex, MDNode *DbgInfo) { 12445ffd83dbSDimitry Andric assert(isa<PointerType>(Base->getType()) && 12455ffd83dbSDimitry Andric "Invalid Base ptr type for preserve.union.access.index."); 12465ffd83dbSDimitry Andric auto *BaseType = Base->getType(); 12475ffd83dbSDimitry Andric 12485ffd83dbSDimitry Andric Module *M = BB->getParent()->getParent(); 12495ffd83dbSDimitry Andric Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration( 12505ffd83dbSDimitry Andric M, Intrinsic::preserve_union_access_index, {BaseType, BaseType}); 12515ffd83dbSDimitry Andric 12525ffd83dbSDimitry Andric Value *DIIndex = getInt32(FieldIndex); 12535ffd83dbSDimitry Andric CallInst *Fn = 12545ffd83dbSDimitry Andric CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex}); 12555ffd83dbSDimitry Andric if (DbgInfo) 12565ffd83dbSDimitry Andric Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); 12575ffd83dbSDimitry Andric 12585ffd83dbSDimitry Andric return Fn; 12595ffd83dbSDimitry Andric } 12605ffd83dbSDimitry Andric 12615ffd83dbSDimitry Andric Value *IRBuilderBase::CreatePreserveStructAccessIndex( 12625ffd83dbSDimitry Andric Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex, 12635ffd83dbSDimitry Andric MDNode *DbgInfo) { 12645ffd83dbSDimitry Andric auto *BaseType = Base->getType(); 1265fe6060f1SDimitry Andric assert(isa<PointerType>(BaseType) && 1266fe6060f1SDimitry Andric "Invalid Base ptr type for preserve.struct.access.index."); 12675ffd83dbSDimitry Andric 12685ffd83dbSDimitry Andric Value *GEPIndex = getInt32(Index); 12695ffd83dbSDimitry Andric Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); 12705ffd83dbSDimitry Andric Type *ResultType = 127106c3fb27SDimitry Andric GetElementPtrInst::getGEPReturnType(Base, {Zero, GEPIndex}); 12725ffd83dbSDimitry Andric 12735ffd83dbSDimitry Andric Module *M = BB->getParent()->getParent(); 12745ffd83dbSDimitry Andric Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration( 12755ffd83dbSDimitry Andric M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType}); 12765ffd83dbSDimitry Andric 12775ffd83dbSDimitry Andric Value *DIIndex = getInt32(FieldIndex); 12785ffd83dbSDimitry Andric CallInst *Fn = CreateCall(FnPreserveStructAccessIndex, 12795ffd83dbSDimitry Andric {Base, GEPIndex, DIIndex}); 1280fe6060f1SDimitry Andric Fn->addParamAttr( 1281fe6060f1SDimitry Andric 0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy)); 12825ffd83dbSDimitry Andric if (DbgInfo) 12835ffd83dbSDimitry Andric Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); 12845ffd83dbSDimitry Andric 12855ffd83dbSDimitry Andric return Fn; 12865ffd83dbSDimitry Andric } 12875ffd83dbSDimitry Andric 128806c3fb27SDimitry Andric Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) { 128906c3fb27SDimitry Andric ConstantInt *TestV = getInt32(Test); 129006c3fb27SDimitry Andric Module *M = BB->getParent()->getParent(); 129106c3fb27SDimitry Andric Function *FnIsFPClass = 129206c3fb27SDimitry Andric Intrinsic::getDeclaration(M, Intrinsic::is_fpclass, {FPNum->getType()}); 129306c3fb27SDimitry Andric return CreateCall(FnIsFPClass, {FPNum, TestV}); 129406c3fb27SDimitry Andric } 129506c3fb27SDimitry Andric 1296e8d8bef9SDimitry Andric CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL, 1297e8d8bef9SDimitry Andric Value *PtrValue, 1298e8d8bef9SDimitry Andric Value *AlignValue, 1299e8d8bef9SDimitry Andric Value *OffsetValue) { 1300e8d8bef9SDimitry Andric SmallVector<Value *, 4> Vals({PtrValue, AlignValue}); 1301e8d8bef9SDimitry Andric if (OffsetValue) 1302e8d8bef9SDimitry Andric Vals.push_back(OffsetValue); 1303e8d8bef9SDimitry Andric OperandBundleDefT<Value *> AlignOpB("align", Vals); 1304e8d8bef9SDimitry Andric return CreateAssumption(ConstantInt::getTrue(getContext()), {AlignOpB}); 13055ffd83dbSDimitry Andric } 13065ffd83dbSDimitry Andric 1307e8d8bef9SDimitry Andric CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL, 1308e8d8bef9SDimitry Andric Value *PtrValue, 1309e8d8bef9SDimitry Andric unsigned Alignment, 1310e8d8bef9SDimitry Andric Value *OffsetValue) { 13115ffd83dbSDimitry Andric assert(isa<PointerType>(PtrValue->getType()) && 13125ffd83dbSDimitry Andric "trying to create an alignment assumption on a non-pointer?"); 13135ffd83dbSDimitry Andric assert(Alignment != 0 && "Invalid Alignment"); 13145ffd83dbSDimitry Andric auto *PtrTy = cast<PointerType>(PtrValue->getType()); 13155ffd83dbSDimitry Andric Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace()); 1316e8d8bef9SDimitry Andric Value *AlignValue = ConstantInt::get(IntPtrTy, Alignment); 1317e8d8bef9SDimitry Andric return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue); 13185ffd83dbSDimitry Andric } 13195ffd83dbSDimitry Andric 1320e8d8bef9SDimitry Andric CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL, 1321e8d8bef9SDimitry Andric Value *PtrValue, 1322e8d8bef9SDimitry Andric Value *Alignment, 1323e8d8bef9SDimitry Andric Value *OffsetValue) { 13245ffd83dbSDimitry Andric assert(isa<PointerType>(PtrValue->getType()) && 13255ffd83dbSDimitry Andric "trying to create an alignment assumption on a non-pointer?"); 1326e8d8bef9SDimitry Andric return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue); 13275ffd83dbSDimitry Andric } 13285ffd83dbSDimitry Andric 132981ad6265SDimitry Andric IRBuilderDefaultInserter::~IRBuilderDefaultInserter() = default; 133081ad6265SDimitry Andric IRBuilderCallbackInserter::~IRBuilderCallbackInserter() = default; 133181ad6265SDimitry Andric IRBuilderFolder::~IRBuilderFolder() = default; 13325ffd83dbSDimitry Andric void ConstantFolder::anchor() {} 13335ffd83dbSDimitry Andric void NoFolder::anchor() {} 1334