1 //===-- SPIRVEmitIntrinsics.cpp - emit SPIRV intrinsics ---------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // The pass emits SPIRV intrinsics keeping essential high-level information for 10 // the translation of LLVM IR to SPIR-V. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SPIRV.h" 15 #include "SPIRVBuiltins.h" 16 #include "SPIRVMetadata.h" 17 #include "SPIRVSubtarget.h" 18 #include "SPIRVTargetMachine.h" 19 #include "SPIRVUtils.h" 20 #include "llvm/ADT/DenseSet.h" 21 #include "llvm/IR/IRBuilder.h" 22 #include "llvm/IR/InstIterator.h" 23 #include "llvm/IR/InstVisitor.h" 24 #include "llvm/IR/IntrinsicsSPIRV.h" 25 #include "llvm/IR/TypedPointerType.h" 26 27 #include <queue> 28 #include <unordered_set> 29 30 // This pass performs the following transformation on LLVM IR level required 31 // for the following translation to SPIR-V: 32 // - replaces direct usages of aggregate constants with target-specific 33 // intrinsics; 34 // - replaces aggregates-related instructions (extract/insert, ld/st, etc) 35 // with a target-specific intrinsics; 36 // - emits intrinsics for the global variable initializers since IRTranslator 37 // doesn't handle them and it's not very convenient to translate them 38 // ourselves; 39 // - emits intrinsics to keep track of the string names assigned to the values; 40 // - emits intrinsics to keep track of constants (this is necessary to have an 41 // LLVM IR constant after the IRTranslation is completed) for their further 42 // deduplication; 43 // - emits intrinsics to keep track of original LLVM types of the values 44 // to be able to emit proper SPIR-V types eventually. 45 // 46 // TODO: consider removing spv.track.constant in favor of spv.assign.type. 47 48 using namespace llvm; 49 50 namespace llvm { 51 namespace SPIRV { 52 #define GET_BuiltinGroup_DECL 53 #include "SPIRVGenTables.inc" 54 } // namespace SPIRV 55 void initializeSPIRVEmitIntrinsicsPass(PassRegistry &); 56 } // namespace llvm 57 58 namespace { 59 60 inline MetadataAsValue *buildMD(Value *Arg) { 61 LLVMContext &Ctx = Arg->getContext(); 62 return MetadataAsValue::get( 63 Ctx, MDNode::get(Ctx, ValueAsMetadata::getConstant(Arg))); 64 } 65 66 class SPIRVEmitIntrinsics 67 : public ModulePass, 68 public InstVisitor<SPIRVEmitIntrinsics, Instruction *> { 69 SPIRVTargetMachine *TM = nullptr; 70 SPIRVGlobalRegistry *GR = nullptr; 71 Function *CurrF = nullptr; 72 bool TrackConstants = true; 73 bool HaveFunPtrs = false; 74 DenseMap<Instruction *, Constant *> AggrConsts; 75 DenseMap<Instruction *, Type *> AggrConstTypes; 76 DenseSet<Instruction *> AggrStores; 77 78 // map of function declarations to <pointer arg index => element type> 79 DenseMap<Function *, SmallVector<std::pair<unsigned, Type *>>> FDeclPtrTys; 80 81 // a register of Instructions that don't have a complete type definition 82 bool CanTodoType = true; 83 unsigned TodoTypeSz = 0; 84 DenseMap<Value *, bool> TodoType; 85 void insertTodoType(Value *Op) { 86 // TODO: add isa<CallInst>(Op) to no-insert 87 if (CanTodoType && !isa<GetElementPtrInst>(Op)) { 88 auto It = TodoType.try_emplace(Op, true); 89 if (It.second) 90 ++TodoTypeSz; 91 } 92 } 93 void eraseTodoType(Value *Op) { 94 auto It = TodoType.find(Op); 95 if (It != TodoType.end() && It->second) { 96 TodoType[Op] = false; 97 --TodoTypeSz; 98 } 99 } 100 bool isTodoType(Value *Op) { 101 if (isa<GetElementPtrInst>(Op)) 102 return false; 103 auto It = TodoType.find(Op); 104 return It != TodoType.end() && It->second; 105 } 106 // a register of Instructions that were visited by deduceOperandElementType() 107 // to validate operand types with an instruction 108 std::unordered_set<Instruction *> TypeValidated; 109 110 // well known result types of builtins 111 enum WellKnownTypes { Event }; 112 113 // deduce element type of untyped pointers 114 Type *deduceElementType(Value *I, bool UnknownElemTypeI8); 115 Type *deduceElementTypeHelper(Value *I, bool UnknownElemTypeI8); 116 Type *deduceElementTypeHelper(Value *I, std::unordered_set<Value *> &Visited, 117 bool UnknownElemTypeI8, 118 bool IgnoreKnownType = false); 119 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand, 120 bool UnknownElemTypeI8); 121 Type *deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand, 122 std::unordered_set<Value *> &Visited, 123 bool UnknownElemTypeI8); 124 Type *deduceElementTypeByUsersDeep(Value *Op, 125 std::unordered_set<Value *> &Visited, 126 bool UnknownElemTypeI8); 127 void maybeAssignPtrType(Type *&Ty, Value *I, Type *RefTy, 128 bool UnknownElemTypeI8); 129 130 // deduce nested types of composites 131 Type *deduceNestedTypeHelper(User *U, bool UnknownElemTypeI8); 132 Type *deduceNestedTypeHelper(User *U, Type *Ty, 133 std::unordered_set<Value *> &Visited, 134 bool UnknownElemTypeI8); 135 136 // deduce Types of operands of the Instruction if possible 137 void deduceOperandElementType(Instruction *I, 138 SmallPtrSet<Instruction *, 4> *UncompleteRets, 139 const SmallPtrSet<Value *, 4> *AskOps = nullptr, 140 bool IsPostprocessing = false); 141 142 void preprocessCompositeConstants(IRBuilder<> &B); 143 void preprocessUndefs(IRBuilder<> &B); 144 145 CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef<Type *> Types, 146 Value *Arg, Value *Arg2, ArrayRef<Constant *> Imms, 147 IRBuilder<> &B) { 148 SmallVector<Value *, 4> Args; 149 Args.push_back(Arg2); 150 Args.push_back(buildMD(Arg)); 151 for (auto *Imm : Imms) 152 Args.push_back(Imm); 153 return B.CreateIntrinsic(IntrID, {Types}, Args); 154 } 155 156 Type *reconstructType(Value *Op, bool UnknownElemTypeI8, 157 bool IsPostprocessing); 158 159 void buildAssignType(IRBuilder<> &B, Type *ElemTy, Value *Arg); 160 void buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg); 161 void updateAssignType(CallInst *AssignCI, Value *Arg, Value *OfType); 162 163 void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B); 164 void processInstrAfterVisit(Instruction *I, IRBuilder<> &B); 165 bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B, 166 bool UnknownElemTypeI8); 167 void insertAssignTypeIntrs(Instruction *I, IRBuilder<> &B); 168 void insertAssignPtrTypeTargetExt(TargetExtType *AssignedType, Value *V, 169 IRBuilder<> &B); 170 void replacePointerOperandWithPtrCast(Instruction *I, Value *Pointer, 171 Type *ExpectedElementType, 172 unsigned OperandToReplace, 173 IRBuilder<> &B); 174 void insertPtrCastOrAssignTypeInstr(Instruction *I, IRBuilder<> &B); 175 void insertSpirvDecorations(Instruction *I, IRBuilder<> &B); 176 void processGlobalValue(GlobalVariable &GV, IRBuilder<> &B); 177 void processParamTypes(Function *F, IRBuilder<> &B); 178 void processParamTypesByFunHeader(Function *F, IRBuilder<> &B); 179 Type *deduceFunParamElementType(Function *F, unsigned OpIdx); 180 Type *deduceFunParamElementType(Function *F, unsigned OpIdx, 181 std::unordered_set<Function *> &FVisited); 182 183 bool deduceOperandElementTypeCalledFunction( 184 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops, 185 Type *&KnownElemTy); 186 void deduceOperandElementTypeFunctionPointer( 187 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops, 188 Type *&KnownElemTy, bool IsPostprocessing); 189 bool deduceOperandElementTypeFunctionRet( 190 Instruction *I, SmallPtrSet<Instruction *, 4> *UncompleteRets, 191 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing, 192 Type *&KnownElemTy, Value *Op, Function *F); 193 194 CallInst *buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy); 195 void replaceUsesOfWithSpvPtrcast(Value *Op, Type *ElemTy, Instruction *I, 196 DenseMap<Function *, CallInst *> Ptrcasts); 197 void propagateElemType(Value *Op, Type *ElemTy, 198 DenseSet<std::pair<Value *, Value *>> &VisitedSubst); 199 void 200 propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy, 201 DenseSet<std::pair<Value *, Value *>> &VisitedSubst); 202 void propagateElemTypeRec(Value *Op, Type *PtrElemTy, Type *CastElemTy, 203 DenseSet<std::pair<Value *, Value *>> &VisitedSubst, 204 std::unordered_set<Value *> &Visited, 205 DenseMap<Function *, CallInst *> Ptrcasts); 206 207 void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true); 208 void replaceAllUsesWithAndErase(IRBuilder<> &B, Instruction *Src, 209 Instruction *Dest, bool DeleteOld = true); 210 211 void applyDemangledPtrArgTypes(IRBuilder<> &B); 212 213 bool runOnFunction(Function &F); 214 bool postprocessTypes(Module &M); 215 bool processFunctionPointers(Module &M); 216 void parseFunDeclarations(Module &M); 217 218 void useRoundingMode(ConstrainedFPIntrinsic *FPI, IRBuilder<> &B); 219 220 public: 221 static char ID; 222 SPIRVEmitIntrinsics() : ModulePass(ID) { 223 initializeSPIRVEmitIntrinsicsPass(*PassRegistry::getPassRegistry()); 224 } 225 SPIRVEmitIntrinsics(SPIRVTargetMachine *_TM) : ModulePass(ID), TM(_TM) { 226 initializeSPIRVEmitIntrinsicsPass(*PassRegistry::getPassRegistry()); 227 } 228 Instruction *visitInstruction(Instruction &I) { return &I; } 229 Instruction *visitSwitchInst(SwitchInst &I); 230 Instruction *visitGetElementPtrInst(GetElementPtrInst &I); 231 Instruction *visitBitCastInst(BitCastInst &I); 232 Instruction *visitInsertElementInst(InsertElementInst &I); 233 Instruction *visitExtractElementInst(ExtractElementInst &I); 234 Instruction *visitInsertValueInst(InsertValueInst &I); 235 Instruction *visitExtractValueInst(ExtractValueInst &I); 236 Instruction *visitLoadInst(LoadInst &I); 237 Instruction *visitStoreInst(StoreInst &I); 238 Instruction *visitAllocaInst(AllocaInst &I); 239 Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I); 240 Instruction *visitUnreachableInst(UnreachableInst &I); 241 Instruction *visitCallInst(CallInst &I); 242 243 StringRef getPassName() const override { return "SPIRV emit intrinsics"; } 244 245 bool runOnModule(Module &M) override; 246 247 void getAnalysisUsage(AnalysisUsage &AU) const override { 248 ModulePass::getAnalysisUsage(AU); 249 } 250 }; 251 252 bool isConvergenceIntrinsic(const Instruction *I) { 253 const auto *II = dyn_cast<IntrinsicInst>(I); 254 if (!II) 255 return false; 256 257 return II->getIntrinsicID() == Intrinsic::experimental_convergence_entry || 258 II->getIntrinsicID() == Intrinsic::experimental_convergence_loop || 259 II->getIntrinsicID() == Intrinsic::experimental_convergence_anchor; 260 } 261 262 bool expectIgnoredInIRTranslation(const Instruction *I) { 263 const auto *II = dyn_cast<IntrinsicInst>(I); 264 if (!II) 265 return false; 266 switch (II->getIntrinsicID()) { 267 case Intrinsic::invariant_start: 268 case Intrinsic::spv_resource_handlefrombinding: 269 case Intrinsic::spv_resource_getpointer: 270 return true; 271 default: 272 return false; 273 } 274 } 275 276 bool allowEmitFakeUse(const Value *Arg) { 277 if (isSpvIntrinsic(Arg)) 278 return false; 279 if (dyn_cast<AtomicCmpXchgInst>(Arg) || dyn_cast<InsertValueInst>(Arg) || 280 dyn_cast<UndefValue>(Arg)) 281 return false; 282 if (const auto *LI = dyn_cast<LoadInst>(Arg)) 283 if (LI->getType()->isAggregateType()) 284 return false; 285 return true; 286 } 287 288 } // namespace 289 290 char SPIRVEmitIntrinsics::ID = 0; 291 292 INITIALIZE_PASS(SPIRVEmitIntrinsics, "emit-intrinsics", "SPIRV emit intrinsics", 293 false, false) 294 295 static inline bool isAssignTypeInstr(const Instruction *I) { 296 return isa<IntrinsicInst>(I) && 297 cast<IntrinsicInst>(I)->getIntrinsicID() == Intrinsic::spv_assign_type; 298 } 299 300 static bool isMemInstrToReplace(Instruction *I) { 301 return isa<StoreInst>(I) || isa<LoadInst>(I) || isa<InsertValueInst>(I) || 302 isa<ExtractValueInst>(I) || isa<AtomicCmpXchgInst>(I); 303 } 304 305 static bool isAggrConstForceInt32(const Value *V) { 306 return isa<ConstantArray>(V) || isa<ConstantStruct>(V) || 307 isa<ConstantDataArray>(V) || 308 (isa<ConstantAggregateZero>(V) && !V->getType()->isVectorTy()); 309 } 310 311 static void setInsertPointSkippingPhis(IRBuilder<> &B, Instruction *I) { 312 if (isa<PHINode>(I)) 313 B.SetInsertPoint(I->getParent()->getFirstNonPHIOrDbgOrAlloca()); 314 else 315 B.SetInsertPoint(I); 316 } 317 318 static void setInsertPointAfterDef(IRBuilder<> &B, Instruction *I) { 319 B.SetCurrentDebugLocation(I->getDebugLoc()); 320 if (I->getType()->isVoidTy()) 321 B.SetInsertPoint(I->getNextNode()); 322 else 323 B.SetInsertPoint(*I->getInsertionPointAfterDef()); 324 } 325 326 static bool requireAssignType(Instruction *I) { 327 IntrinsicInst *Intr = dyn_cast<IntrinsicInst>(I); 328 if (Intr) { 329 switch (Intr->getIntrinsicID()) { 330 case Intrinsic::invariant_start: 331 case Intrinsic::invariant_end: 332 return false; 333 } 334 } 335 return true; 336 } 337 338 static inline void reportFatalOnTokenType(const Instruction *I) { 339 if (I->getType()->isTokenTy()) 340 report_fatal_error("A token is encountered but SPIR-V without extensions " 341 "does not support token type", 342 false); 343 } 344 345 static void emitAssignName(Instruction *I, IRBuilder<> &B) { 346 if (!I->hasName() || I->getType()->isAggregateType() || 347 expectIgnoredInIRTranslation(I)) 348 return; 349 reportFatalOnTokenType(I); 350 setInsertPointAfterDef(B, I); 351 std::vector<Value *> Args = {I}; 352 addStringImm(I->getName(), B, Args); 353 B.CreateIntrinsic(Intrinsic::spv_assign_name, {I->getType()}, Args); 354 } 355 356 void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest, 357 bool DeleteOld) { 358 Src->replaceAllUsesWith(Dest); 359 // Update deduced type records 360 GR->updateIfExistDeducedElementType(Src, Dest, DeleteOld); 361 GR->updateIfExistAssignPtrTypeInstr(Src, Dest, DeleteOld); 362 // Update uncomplete type records if any 363 if (isTodoType(Src)) { 364 if (DeleteOld) 365 eraseTodoType(Src); 366 insertTodoType(Dest); 367 } 368 } 369 370 void SPIRVEmitIntrinsics::replaceAllUsesWithAndErase(IRBuilder<> &B, 371 Instruction *Src, 372 Instruction *Dest, 373 bool DeleteOld) { 374 replaceAllUsesWith(Src, Dest, DeleteOld); 375 std::string Name = Src->hasName() ? Src->getName().str() : ""; 376 Src->eraseFromParent(); 377 if (!Name.empty()) { 378 Dest->setName(Name); 379 emitAssignName(Dest, B); 380 } 381 } 382 383 static bool IsKernelArgInt8(Function *F, StoreInst *SI) { 384 return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL && 385 isPointerTy(SI->getValueOperand()->getType()) && 386 isa<Argument>(SI->getValueOperand()); 387 } 388 389 // Maybe restore original function return type. 390 static inline Type *restoreMutatedType(SPIRVGlobalRegistry *GR, Instruction *I, 391 Type *Ty) { 392 CallInst *CI = dyn_cast<CallInst>(I); 393 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() || 394 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic()) 395 return Ty; 396 if (Type *OriginalTy = GR->findMutated(CI->getCalledFunction())) 397 return OriginalTy; 398 return Ty; 399 } 400 401 // Reconstruct type with nested element types according to deduced type info. 402 // Return nullptr if no detailed type info is available. 403 Type *SPIRVEmitIntrinsics::reconstructType(Value *Op, bool UnknownElemTypeI8, 404 bool IsPostprocessing) { 405 Type *Ty = Op->getType(); 406 if (auto *OpI = dyn_cast<Instruction>(Op)) 407 Ty = restoreMutatedType(GR, OpI, Ty); 408 if (!isUntypedPointerTy(Ty)) 409 return Ty; 410 // try to find the pointee type 411 if (Type *NestedTy = GR->findDeducedElementType(Op)) 412 return getTypedPointerWrapper(NestedTy, getPointerAddressSpace(Ty)); 413 // not a pointer according to the type info (e.g., Event object) 414 CallInst *CI = GR->findAssignPtrTypeInstr(Op); 415 if (CI) { 416 MetadataAsValue *MD = cast<MetadataAsValue>(CI->getArgOperand(1)); 417 return cast<ConstantAsMetadata>(MD->getMetadata())->getType(); 418 } 419 if (UnknownElemTypeI8) { 420 if (!IsPostprocessing) 421 insertTodoType(Op); 422 return getTypedPointerWrapper(IntegerType::getInt8Ty(Op->getContext()), 423 getPointerAddressSpace(Ty)); 424 } 425 return nullptr; 426 } 427 428 void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty, 429 Value *Arg) { 430 Value *OfType = PoisonValue::get(Ty); 431 CallInst *AssignCI = nullptr; 432 if (Arg->getType()->isAggregateType() && Ty->isAggregateType() && 433 allowEmitFakeUse(Arg)) { 434 LLVMContext &Ctx = Arg->getContext(); 435 SmallVector<Metadata *, 2> ArgMDs{ 436 MDNode::get(Ctx, ValueAsMetadata::getConstant(OfType)), 437 MDString::get(Ctx, Arg->getName())}; 438 B.CreateIntrinsic(Intrinsic::spv_value_md, {}, 439 {MetadataAsValue::get(Ctx, MDTuple::get(Ctx, ArgMDs))}); 440 AssignCI = B.CreateIntrinsic(Intrinsic::fake_use, {}, {Arg}); 441 } else { 442 AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {Arg->getType()}, 443 OfType, Arg, {}, B); 444 } 445 GR->addAssignPtrTypeInstr(Arg, AssignCI); 446 } 447 448 void SPIRVEmitIntrinsics::buildAssignPtr(IRBuilder<> &B, Type *ElemTy, 449 Value *Arg) { 450 Value *OfType = PoisonValue::get(ElemTy); 451 CallInst *AssignPtrTyCI = GR->findAssignPtrTypeInstr(Arg); 452 if (AssignPtrTyCI == nullptr || 453 AssignPtrTyCI->getParent()->getParent() != CurrF) { 454 AssignPtrTyCI = buildIntrWithMD( 455 Intrinsic::spv_assign_ptr_type, {Arg->getType()}, OfType, Arg, 456 {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B); 457 GR->addDeducedElementType(AssignPtrTyCI, ElemTy); 458 GR->addDeducedElementType(Arg, ElemTy); 459 GR->addAssignPtrTypeInstr(Arg, AssignPtrTyCI); 460 } else { 461 updateAssignType(AssignPtrTyCI, Arg, OfType); 462 } 463 } 464 465 void SPIRVEmitIntrinsics::updateAssignType(CallInst *AssignCI, Value *Arg, 466 Value *OfType) { 467 AssignCI->setArgOperand(1, buildMD(OfType)); 468 if (cast<IntrinsicInst>(AssignCI)->getIntrinsicID() != 469 Intrinsic::spv_assign_ptr_type) 470 return; 471 472 // update association with the pointee type 473 Type *ElemTy = OfType->getType(); 474 GR->addDeducedElementType(AssignCI, ElemTy); 475 GR->addDeducedElementType(Arg, ElemTy); 476 } 477 478 CallInst *SPIRVEmitIntrinsics::buildSpvPtrcast(Function *F, Value *Op, 479 Type *ElemTy) { 480 IRBuilder<> B(Op->getContext()); 481 if (auto *OpI = dyn_cast<Instruction>(Op)) { 482 // spv_ptrcast's argument Op denotes an instruction that generates 483 // a value, and we may use getInsertionPointAfterDef() 484 setInsertPointAfterDef(B, OpI); 485 } else if (auto *OpA = dyn_cast<Argument>(Op)) { 486 B.SetInsertPointPastAllocas(OpA->getParent()); 487 B.SetCurrentDebugLocation(DebugLoc()); 488 } else { 489 B.SetInsertPoint(F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca()); 490 } 491 Type *OpTy = Op->getType(); 492 SmallVector<Type *, 2> Types = {OpTy, OpTy}; 493 SmallVector<Value *, 2> Args = {Op, buildMD(PoisonValue::get(ElemTy)), 494 B.getInt32(getPointerAddressSpace(OpTy))}; 495 CallInst *PtrCasted = 496 B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args); 497 buildAssignPtr(B, ElemTy, PtrCasted); 498 return PtrCasted; 499 } 500 501 void SPIRVEmitIntrinsics::replaceUsesOfWithSpvPtrcast( 502 Value *Op, Type *ElemTy, Instruction *I, 503 DenseMap<Function *, CallInst *> Ptrcasts) { 504 Function *F = I->getParent()->getParent(); 505 CallInst *PtrCastedI = nullptr; 506 auto It = Ptrcasts.find(F); 507 if (It == Ptrcasts.end()) { 508 PtrCastedI = buildSpvPtrcast(F, Op, ElemTy); 509 Ptrcasts[F] = PtrCastedI; 510 } else { 511 PtrCastedI = It->second; 512 } 513 I->replaceUsesOfWith(Op, PtrCastedI); 514 } 515 516 void SPIRVEmitIntrinsics::propagateElemType( 517 Value *Op, Type *ElemTy, 518 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) { 519 DenseMap<Function *, CallInst *> Ptrcasts; 520 SmallVector<User *> Users(Op->users()); 521 for (auto *U : Users) { 522 if (!isa<Instruction>(U) || isSpvIntrinsic(U)) 523 continue; 524 if (!VisitedSubst.insert(std::make_pair(U, Op)).second) 525 continue; 526 Instruction *UI = dyn_cast<Instruction>(U); 527 // If the instruction was validated already, we need to keep it valid by 528 // keeping current Op type. 529 if (isa<GetElementPtrInst>(UI) || 530 TypeValidated.find(UI) != TypeValidated.end()) 531 replaceUsesOfWithSpvPtrcast(Op, ElemTy, UI, Ptrcasts); 532 } 533 } 534 535 void SPIRVEmitIntrinsics::propagateElemTypeRec( 536 Value *Op, Type *PtrElemTy, Type *CastElemTy, 537 DenseSet<std::pair<Value *, Value *>> &VisitedSubst) { 538 std::unordered_set<Value *> Visited; 539 DenseMap<Function *, CallInst *> Ptrcasts; 540 propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited, 541 Ptrcasts); 542 } 543 544 void SPIRVEmitIntrinsics::propagateElemTypeRec( 545 Value *Op, Type *PtrElemTy, Type *CastElemTy, 546 DenseSet<std::pair<Value *, Value *>> &VisitedSubst, 547 std::unordered_set<Value *> &Visited, 548 DenseMap<Function *, CallInst *> Ptrcasts) { 549 if (!Visited.insert(Op).second) 550 return; 551 SmallVector<User *> Users(Op->users()); 552 for (auto *U : Users) { 553 if (!isa<Instruction>(U) || isSpvIntrinsic(U)) 554 continue; 555 if (!VisitedSubst.insert(std::make_pair(U, Op)).second) 556 continue; 557 Instruction *UI = dyn_cast<Instruction>(U); 558 // If the instruction was validated already, we need to keep it valid by 559 // keeping current Op type. 560 if (isa<GetElementPtrInst>(UI) || 561 TypeValidated.find(UI) != TypeValidated.end()) 562 replaceUsesOfWithSpvPtrcast(Op, CastElemTy, UI, Ptrcasts); 563 } 564 } 565 566 // Set element pointer type to the given value of ValueTy and tries to 567 // specify this type further (recursively) by Operand value, if needed. 568 569 Type * 570 SPIRVEmitIntrinsics::deduceElementTypeByValueDeep(Type *ValueTy, Value *Operand, 571 bool UnknownElemTypeI8) { 572 std::unordered_set<Value *> Visited; 573 return deduceElementTypeByValueDeep(ValueTy, Operand, Visited, 574 UnknownElemTypeI8); 575 } 576 577 Type *SPIRVEmitIntrinsics::deduceElementTypeByValueDeep( 578 Type *ValueTy, Value *Operand, std::unordered_set<Value *> &Visited, 579 bool UnknownElemTypeI8) { 580 Type *Ty = ValueTy; 581 if (Operand) { 582 if (auto *PtrTy = dyn_cast<PointerType>(Ty)) { 583 if (Type *NestedTy = 584 deduceElementTypeHelper(Operand, Visited, UnknownElemTypeI8)) 585 Ty = getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace()); 586 } else { 587 Ty = deduceNestedTypeHelper(dyn_cast<User>(Operand), Ty, Visited, 588 UnknownElemTypeI8); 589 } 590 } 591 return Ty; 592 } 593 594 // Traverse User instructions to deduce an element pointer type of the operand. 595 Type *SPIRVEmitIntrinsics::deduceElementTypeByUsersDeep( 596 Value *Op, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8) { 597 if (!Op || !isPointerTy(Op->getType()) || isa<ConstantPointerNull>(Op) || 598 isa<UndefValue>(Op)) 599 return nullptr; 600 601 if (auto ElemTy = getPointeeType(Op->getType())) 602 return ElemTy; 603 604 // maybe we already know operand's element type 605 if (Type *KnownTy = GR->findDeducedElementType(Op)) 606 return KnownTy; 607 608 for (User *OpU : Op->users()) { 609 if (Instruction *Inst = dyn_cast<Instruction>(OpU)) { 610 if (Type *Ty = deduceElementTypeHelper(Inst, Visited, UnknownElemTypeI8)) 611 return Ty; 612 } 613 } 614 return nullptr; 615 } 616 617 // Implements what we know in advance about intrinsics and builtin calls 618 // TODO: consider feasibility of this particular case to be generalized by 619 // encoding knowledge about intrinsics and builtin calls by corresponding 620 // specification rules 621 static Type *getPointeeTypeByCallInst(StringRef DemangledName, 622 Function *CalledF, unsigned OpIdx) { 623 if ((DemangledName.starts_with("__spirv_ocl_printf(") || 624 DemangledName.starts_with("printf(")) && 625 OpIdx == 0) 626 return IntegerType::getInt8Ty(CalledF->getContext()); 627 return nullptr; 628 } 629 630 // Deduce and return a successfully deduced Type of the Instruction, 631 // or nullptr otherwise. 632 Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(Value *I, 633 bool UnknownElemTypeI8) { 634 std::unordered_set<Value *> Visited; 635 return deduceElementTypeHelper(I, Visited, UnknownElemTypeI8); 636 } 637 638 void SPIRVEmitIntrinsics::maybeAssignPtrType(Type *&Ty, Value *Op, Type *RefTy, 639 bool UnknownElemTypeI8) { 640 if (isUntypedPointerTy(RefTy)) { 641 if (!UnknownElemTypeI8) 642 return; 643 insertTodoType(Op); 644 } 645 Ty = RefTy; 646 } 647 648 Type *SPIRVEmitIntrinsics::deduceElementTypeHelper( 649 Value *I, std::unordered_set<Value *> &Visited, bool UnknownElemTypeI8, 650 bool IgnoreKnownType) { 651 // allow to pass nullptr as an argument 652 if (!I) 653 return nullptr; 654 655 // maybe already known 656 if (!IgnoreKnownType) 657 if (Type *KnownTy = GR->findDeducedElementType(I)) 658 return KnownTy; 659 660 // maybe a cycle 661 if (!Visited.insert(I).second) 662 return nullptr; 663 664 // fallback value in case when we fail to deduce a type 665 Type *Ty = nullptr; 666 // look for known basic patterns of type inference 667 if (auto *Ref = dyn_cast<AllocaInst>(I)) { 668 maybeAssignPtrType(Ty, I, Ref->getAllocatedType(), UnknownElemTypeI8); 669 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(I)) { 670 // TODO: not sure if GetElementPtrInst::getTypeAtIndex() does anything 671 // useful here 672 if (isNestedPointer(Ref->getSourceElementType())) { 673 Ty = Ref->getSourceElementType(); 674 for (Use &U : drop_begin(Ref->indices())) 675 Ty = GetElementPtrInst::getTypeAtIndex(Ty, U.get()); 676 } else { 677 Ty = Ref->getResultElementType(); 678 } 679 } else if (auto *Ref = dyn_cast<LoadInst>(I)) { 680 Value *Op = Ref->getPointerOperand(); 681 Type *KnownTy = GR->findDeducedElementType(Op); 682 if (!KnownTy) 683 KnownTy = Op->getType(); 684 if (Type *ElemTy = getPointeeType(KnownTy)) 685 maybeAssignPtrType(Ty, I, ElemTy, UnknownElemTypeI8); 686 } else if (auto *Ref = dyn_cast<GlobalValue>(I)) { 687 Ty = deduceElementTypeByValueDeep( 688 Ref->getValueType(), 689 Ref->getNumOperands() > 0 ? Ref->getOperand(0) : nullptr, Visited, 690 UnknownElemTypeI8); 691 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(I)) { 692 Type *RefTy = deduceElementTypeHelper(Ref->getPointerOperand(), Visited, 693 UnknownElemTypeI8); 694 maybeAssignPtrType(Ty, I, RefTy, UnknownElemTypeI8); 695 } else if (auto *Ref = dyn_cast<BitCastInst>(I)) { 696 if (Type *Src = Ref->getSrcTy(), *Dest = Ref->getDestTy(); 697 isPointerTy(Src) && isPointerTy(Dest)) 698 Ty = deduceElementTypeHelper(Ref->getOperand(0), Visited, 699 UnknownElemTypeI8); 700 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(I)) { 701 Value *Op = Ref->getNewValOperand(); 702 if (isPointerTy(Op->getType())) 703 Ty = deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8); 704 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(I)) { 705 Value *Op = Ref->getValOperand(); 706 if (isPointerTy(Op->getType())) 707 Ty = deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8); 708 } else if (auto *Ref = dyn_cast<PHINode>(I)) { 709 Type *BestTy = nullptr; 710 unsigned MaxN = 1; 711 DenseMap<Type *, unsigned> PhiTys; 712 for (int i = Ref->getNumIncomingValues() - 1; i >= 0; --i) { 713 Ty = deduceElementTypeByUsersDeep(Ref->getIncomingValue(i), Visited, 714 UnknownElemTypeI8); 715 if (!Ty) 716 continue; 717 auto It = PhiTys.try_emplace(Ty, 1); 718 if (!It.second) { 719 ++It.first->second; 720 if (It.first->second > MaxN) { 721 MaxN = It.first->second; 722 BestTy = Ty; 723 } 724 } 725 } 726 if (BestTy) 727 Ty = BestTy; 728 } else if (auto *Ref = dyn_cast<SelectInst>(I)) { 729 for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) { 730 Ty = deduceElementTypeByUsersDeep(Op, Visited, UnknownElemTypeI8); 731 if (Ty) 732 break; 733 } 734 } else if (auto *CI = dyn_cast<CallInst>(I)) { 735 static StringMap<unsigned> ResTypeByArg = { 736 {"to_global", 0}, 737 {"to_local", 0}, 738 {"to_private", 0}, 739 {"__spirv_GenericCastToPtr_ToGlobal", 0}, 740 {"__spirv_GenericCastToPtr_ToLocal", 0}, 741 {"__spirv_GenericCastToPtr_ToPrivate", 0}, 742 {"__spirv_GenericCastToPtrExplicit_ToGlobal", 0}, 743 {"__spirv_GenericCastToPtrExplicit_ToLocal", 0}, 744 {"__spirv_GenericCastToPtrExplicit_ToPrivate", 0}}; 745 // TODO: maybe improve performance by caching demangled names 746 747 auto *II = dyn_cast<IntrinsicInst>(I); 748 if (II && II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) { 749 auto *ImageType = cast<TargetExtType>(II->getOperand(0)->getType()); 750 assert(ImageType->getTargetExtName() == "spirv.Image"); 751 Ty = ImageType->getTypeParameter(0); 752 } else if (Function *CalledF = CI->getCalledFunction()) { 753 std::string DemangledName = 754 getOclOrSpirvBuiltinDemangledName(CalledF->getName()); 755 if (DemangledName.length() > 0) 756 DemangledName = SPIRV::lookupBuiltinNameHelper(DemangledName); 757 auto AsArgIt = ResTypeByArg.find(DemangledName); 758 if (AsArgIt != ResTypeByArg.end()) 759 Ty = deduceElementTypeHelper(CI->getArgOperand(AsArgIt->second), 760 Visited, UnknownElemTypeI8); 761 else if (Type *KnownRetTy = GR->findDeducedElementType(CalledF)) 762 Ty = KnownRetTy; 763 } 764 } 765 766 // remember the found relationship 767 if (Ty && !IgnoreKnownType) { 768 // specify nested types if needed, otherwise return unchanged 769 GR->addDeducedElementType(I, Ty); 770 } 771 772 return Ty; 773 } 774 775 // Re-create a type of the value if it has untyped pointer fields, also nested. 776 // Return the original value type if no corrections of untyped pointer 777 // information is found or needed. 778 Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(User *U, 779 bool UnknownElemTypeI8) { 780 std::unordered_set<Value *> Visited; 781 return deduceNestedTypeHelper(U, U->getType(), Visited, UnknownElemTypeI8); 782 } 783 784 Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper( 785 User *U, Type *OrigTy, std::unordered_set<Value *> &Visited, 786 bool UnknownElemTypeI8) { 787 if (!U) 788 return OrigTy; 789 790 // maybe already known 791 if (Type *KnownTy = GR->findDeducedCompositeType(U)) 792 return KnownTy; 793 794 // maybe a cycle 795 if (!Visited.insert(U).second) 796 return OrigTy; 797 798 if (dyn_cast<StructType>(OrigTy)) { 799 SmallVector<Type *> Tys; 800 bool Change = false; 801 for (unsigned i = 0; i < U->getNumOperands(); ++i) { 802 Value *Op = U->getOperand(i); 803 Type *OpTy = Op->getType(); 804 Type *Ty = OpTy; 805 if (Op) { 806 if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) { 807 if (Type *NestedTy = 808 deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8)) 809 Ty = getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace()); 810 } else { 811 Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited, 812 UnknownElemTypeI8); 813 } 814 } 815 Tys.push_back(Ty); 816 Change |= Ty != OpTy; 817 } 818 if (Change) { 819 Type *NewTy = StructType::create(Tys); 820 GR->addDeducedCompositeType(U, NewTy); 821 return NewTy; 822 } 823 } else if (auto *ArrTy = dyn_cast<ArrayType>(OrigTy)) { 824 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(0) : nullptr) { 825 Type *OpTy = ArrTy->getElementType(); 826 Type *Ty = OpTy; 827 if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) { 828 if (Type *NestedTy = 829 deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8)) 830 Ty = getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace()); 831 } else { 832 Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited, 833 UnknownElemTypeI8); 834 } 835 if (Ty != OpTy) { 836 Type *NewTy = ArrayType::get(Ty, ArrTy->getNumElements()); 837 GR->addDeducedCompositeType(U, NewTy); 838 return NewTy; 839 } 840 } 841 } else if (auto *VecTy = dyn_cast<VectorType>(OrigTy)) { 842 if (Value *Op = U->getNumOperands() > 0 ? U->getOperand(0) : nullptr) { 843 Type *OpTy = VecTy->getElementType(); 844 Type *Ty = OpTy; 845 if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) { 846 if (Type *NestedTy = 847 deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8)) 848 Ty = getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace()); 849 } else { 850 Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited, 851 UnknownElemTypeI8); 852 } 853 if (Ty != OpTy) { 854 Type *NewTy = VectorType::get(Ty, VecTy->getElementCount()); 855 GR->addDeducedCompositeType(U, NewTy); 856 return NewTy; 857 } 858 } 859 } 860 861 return OrigTy; 862 } 863 864 Type *SPIRVEmitIntrinsics::deduceElementType(Value *I, bool UnknownElemTypeI8) { 865 if (Type *Ty = deduceElementTypeHelper(I, UnknownElemTypeI8)) 866 return Ty; 867 if (!UnknownElemTypeI8) 868 return nullptr; 869 insertTodoType(I); 870 return IntegerType::getInt8Ty(I->getContext()); 871 } 872 873 static inline Type *getAtomicElemTy(SPIRVGlobalRegistry *GR, Instruction *I, 874 Value *PointerOperand) { 875 Type *PointeeTy = GR->findDeducedElementType(PointerOperand); 876 if (PointeeTy && !isUntypedPointerTy(PointeeTy)) 877 return nullptr; 878 auto *PtrTy = dyn_cast<PointerType>(I->getType()); 879 if (!PtrTy) 880 return I->getType(); 881 if (Type *NestedTy = GR->findDeducedElementType(I)) 882 return getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace()); 883 return nullptr; 884 } 885 886 // Try to deduce element type for a call base. Returns false if this is an 887 // indirect function invocation, and true otherwise. 888 bool SPIRVEmitIntrinsics::deduceOperandElementTypeCalledFunction( 889 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops, 890 Type *&KnownElemTy) { 891 Function *CalledF = CI->getCalledFunction(); 892 if (!CalledF) 893 return false; 894 std::string DemangledName = 895 getOclOrSpirvBuiltinDemangledName(CalledF->getName()); 896 if (DemangledName.length() > 0 && 897 !StringRef(DemangledName).starts_with("llvm.")) { 898 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(*CalledF); 899 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode( 900 DemangledName, ST.getPreferredInstructionSet()); 901 if (Opcode == SPIRV::OpGroupAsyncCopy) { 902 for (unsigned i = 0, PtrCnt = 0; i < CI->arg_size() && PtrCnt < 2; ++i) { 903 Value *Op = CI->getArgOperand(i); 904 if (!isPointerTy(Op->getType())) 905 continue; 906 ++PtrCnt; 907 if (Type *ElemTy = GR->findDeducedElementType(Op)) 908 KnownElemTy = ElemTy; // src will rewrite dest if both are defined 909 Ops.push_back(std::make_pair(Op, i)); 910 } 911 } else if (Grp == SPIRV::Atomic || Grp == SPIRV::AtomicFloating) { 912 if (CI->arg_size() < 2) 913 return true; 914 Value *Op = CI->getArgOperand(0); 915 if (!isPointerTy(Op->getType())) 916 return true; 917 switch (Opcode) { 918 case SPIRV::OpAtomicLoad: 919 case SPIRV::OpAtomicCompareExchangeWeak: 920 case SPIRV::OpAtomicCompareExchange: 921 case SPIRV::OpAtomicExchange: 922 case SPIRV::OpAtomicIAdd: 923 case SPIRV::OpAtomicISub: 924 case SPIRV::OpAtomicOr: 925 case SPIRV::OpAtomicXor: 926 case SPIRV::OpAtomicAnd: 927 case SPIRV::OpAtomicUMin: 928 case SPIRV::OpAtomicUMax: 929 case SPIRV::OpAtomicSMin: 930 case SPIRV::OpAtomicSMax: { 931 KnownElemTy = getAtomicElemTy(GR, CI, Op); 932 if (!KnownElemTy) 933 return true; 934 Ops.push_back(std::make_pair(Op, 0)); 935 } break; 936 } 937 } 938 } 939 return true; 940 } 941 942 // Try to deduce element type for a function pointer. 943 void SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionPointer( 944 CallInst *CI, SmallVector<std::pair<Value *, unsigned>> &Ops, 945 Type *&KnownElemTy, bool IsPostprocessing) { 946 Value *Op = CI->getCalledOperand(); 947 if (!Op || !isPointerTy(Op->getType())) 948 return; 949 Ops.push_back(std::make_pair(Op, std::numeric_limits<unsigned>::max())); 950 FunctionType *FTy = CI->getFunctionType(); 951 bool IsNewFTy = false, IsUncomplete = false; 952 SmallVector<Type *, 4> ArgTys; 953 for (Value *Arg : CI->args()) { 954 Type *ArgTy = Arg->getType(); 955 if (ArgTy->isPointerTy()) { 956 if (Type *ElemTy = GR->findDeducedElementType(Arg)) { 957 IsNewFTy = true; 958 ArgTy = getTypedPointerWrapper(ElemTy, getPointerAddressSpace(ArgTy)); 959 if (isTodoType(Arg)) 960 IsUncomplete = true; 961 } else { 962 IsUncomplete = true; 963 } 964 } 965 ArgTys.push_back(ArgTy); 966 } 967 Type *RetTy = FTy->getReturnType(); 968 if (CI->getType()->isPointerTy()) { 969 if (Type *ElemTy = GR->findDeducedElementType(CI)) { 970 IsNewFTy = true; 971 RetTy = 972 getTypedPointerWrapper(ElemTy, getPointerAddressSpace(CI->getType())); 973 if (isTodoType(CI)) 974 IsUncomplete = true; 975 } else { 976 IsUncomplete = true; 977 } 978 } 979 if (!IsPostprocessing && IsUncomplete) 980 insertTodoType(Op); 981 KnownElemTy = 982 IsNewFTy ? FunctionType::get(RetTy, ArgTys, FTy->isVarArg()) : FTy; 983 } 984 985 bool SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionRet( 986 Instruction *I, SmallPtrSet<Instruction *, 4> *UncompleteRets, 987 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing, 988 Type *&KnownElemTy, Value *Op, Function *F) { 989 KnownElemTy = GR->findDeducedElementType(F); 990 if (KnownElemTy) 991 return false; 992 if (Type *OpElemTy = GR->findDeducedElementType(Op)) { 993 GR->addDeducedElementType(F, OpElemTy); 994 GR->addReturnType( 995 F, TypedPointerType::get(OpElemTy, 996 getPointerAddressSpace(F->getReturnType()))); 997 // non-recursive update of types in function uses 998 DenseSet<std::pair<Value *, Value *>> VisitedSubst{std::make_pair(I, Op)}; 999 for (User *U : F->users()) { 1000 CallInst *CI = dyn_cast<CallInst>(U); 1001 if (!CI || CI->getCalledFunction() != F) 1002 continue; 1003 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(CI)) { 1004 if (Type *PrevElemTy = GR->findDeducedElementType(CI)) { 1005 updateAssignType(AssignCI, CI, PoisonValue::get(OpElemTy)); 1006 propagateElemType(CI, PrevElemTy, VisitedSubst); 1007 } 1008 } 1009 } 1010 // Non-recursive update of types in the function uncomplete returns. 1011 // This may happen just once per a function, the latch is a pair of 1012 // findDeducedElementType(F) / addDeducedElementType(F, ...). 1013 // With or without the latch it is a non-recursive call due to 1014 // UncompleteRets set to nullptr in this call. 1015 if (UncompleteRets) 1016 for (Instruction *UncompleteRetI : *UncompleteRets) 1017 deduceOperandElementType(UncompleteRetI, nullptr, AskOps, 1018 IsPostprocessing); 1019 } else if (UncompleteRets) { 1020 UncompleteRets->insert(I); 1021 } 1022 TypeValidated.insert(I); 1023 return true; 1024 } 1025 1026 // If the Instruction has Pointer operands with unresolved types, this function 1027 // tries to deduce them. If the Instruction has Pointer operands with known 1028 // types which differ from expected, this function tries to insert a bitcast to 1029 // resolve the issue. 1030 void SPIRVEmitIntrinsics::deduceOperandElementType( 1031 Instruction *I, SmallPtrSet<Instruction *, 4> *UncompleteRets, 1032 const SmallPtrSet<Value *, 4> *AskOps, bool IsPostprocessing) { 1033 SmallVector<std::pair<Value *, unsigned>> Ops; 1034 Type *KnownElemTy = nullptr; 1035 bool Uncomplete = false; 1036 // look for known basic patterns of type inference 1037 if (auto *Ref = dyn_cast<PHINode>(I)) { 1038 if (!isPointerTy(I->getType()) || 1039 !(KnownElemTy = GR->findDeducedElementType(I))) 1040 return; 1041 Uncomplete = isTodoType(I); 1042 for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) { 1043 Value *Op = Ref->getIncomingValue(i); 1044 if (isPointerTy(Op->getType())) 1045 Ops.push_back(std::make_pair(Op, i)); 1046 } 1047 } else if (auto *Ref = dyn_cast<AddrSpaceCastInst>(I)) { 1048 KnownElemTy = GR->findDeducedElementType(I); 1049 if (!KnownElemTy) 1050 return; 1051 Uncomplete = isTodoType(I); 1052 Ops.push_back(std::make_pair(Ref->getPointerOperand(), 0)); 1053 } else if (auto *Ref = dyn_cast<BitCastInst>(I)) { 1054 if (!isPointerTy(I->getType())) 1055 return; 1056 KnownElemTy = GR->findDeducedElementType(I); 1057 if (!KnownElemTy) 1058 return; 1059 Uncomplete = isTodoType(I); 1060 Ops.push_back(std::make_pair(Ref->getOperand(0), 0)); 1061 } else if (auto *Ref = dyn_cast<GetElementPtrInst>(I)) { 1062 if (GR->findDeducedElementType(Ref->getPointerOperand())) 1063 return; 1064 KnownElemTy = Ref->getSourceElementType(); 1065 Ops.push_back(std::make_pair(Ref->getPointerOperand(), 1066 GetElementPtrInst::getPointerOperandIndex())); 1067 } else if (auto *Ref = dyn_cast<LoadInst>(I)) { 1068 KnownElemTy = I->getType(); 1069 if (isUntypedPointerTy(KnownElemTy)) 1070 return; 1071 Type *PointeeTy = GR->findDeducedElementType(Ref->getPointerOperand()); 1072 if (PointeeTy && !isUntypedPointerTy(PointeeTy)) 1073 return; 1074 Ops.push_back(std::make_pair(Ref->getPointerOperand(), 1075 LoadInst::getPointerOperandIndex())); 1076 } else if (auto *Ref = dyn_cast<StoreInst>(I)) { 1077 if (!(KnownElemTy = 1078 reconstructType(Ref->getValueOperand(), false, IsPostprocessing))) 1079 return; 1080 Type *PointeeTy = GR->findDeducedElementType(Ref->getPointerOperand()); 1081 if (PointeeTy && !isUntypedPointerTy(PointeeTy)) 1082 return; 1083 Ops.push_back(std::make_pair(Ref->getPointerOperand(), 1084 StoreInst::getPointerOperandIndex())); 1085 } else if (auto *Ref = dyn_cast<AtomicCmpXchgInst>(I)) { 1086 KnownElemTy = getAtomicElemTy(GR, I, Ref->getPointerOperand()); 1087 if (!KnownElemTy) 1088 return; 1089 Ops.push_back(std::make_pair(Ref->getPointerOperand(), 1090 AtomicCmpXchgInst::getPointerOperandIndex())); 1091 } else if (auto *Ref = dyn_cast<AtomicRMWInst>(I)) { 1092 KnownElemTy = getAtomicElemTy(GR, I, Ref->getPointerOperand()); 1093 if (!KnownElemTy) 1094 return; 1095 Ops.push_back(std::make_pair(Ref->getPointerOperand(), 1096 AtomicRMWInst::getPointerOperandIndex())); 1097 } else if (auto *Ref = dyn_cast<SelectInst>(I)) { 1098 if (!isPointerTy(I->getType()) || 1099 !(KnownElemTy = GR->findDeducedElementType(I))) 1100 return; 1101 Uncomplete = isTodoType(I); 1102 for (unsigned i = 0; i < Ref->getNumOperands(); i++) { 1103 Value *Op = Ref->getOperand(i); 1104 if (isPointerTy(Op->getType())) 1105 Ops.push_back(std::make_pair(Op, i)); 1106 } 1107 } else if (auto *Ref = dyn_cast<ReturnInst>(I)) { 1108 if (!isPointerTy(CurrF->getReturnType())) 1109 return; 1110 Value *Op = Ref->getReturnValue(); 1111 if (!Op) 1112 return; 1113 if (deduceOperandElementTypeFunctionRet(I, UncompleteRets, AskOps, 1114 IsPostprocessing, KnownElemTy, Op, 1115 CurrF)) 1116 return; 1117 Uncomplete = isTodoType(CurrF); 1118 Ops.push_back(std::make_pair(Op, 0)); 1119 } else if (auto *Ref = dyn_cast<ICmpInst>(I)) { 1120 if (!isPointerTy(Ref->getOperand(0)->getType())) 1121 return; 1122 Value *Op0 = Ref->getOperand(0); 1123 Value *Op1 = Ref->getOperand(1); 1124 Type *ElemTy0 = GR->findDeducedElementType(Op0); 1125 Type *ElemTy1 = GR->findDeducedElementType(Op1); 1126 if (ElemTy0) { 1127 KnownElemTy = ElemTy0; 1128 Uncomplete = isTodoType(Op0); 1129 Ops.push_back(std::make_pair(Op1, 1)); 1130 } else if (ElemTy1) { 1131 KnownElemTy = ElemTy1; 1132 Uncomplete = isTodoType(Op1); 1133 Ops.push_back(std::make_pair(Op0, 0)); 1134 } 1135 } else if (CallInst *CI = dyn_cast<CallInst>(I)) { 1136 if (!CI->isIndirectCall()) 1137 deduceOperandElementTypeCalledFunction(CI, Ops, KnownElemTy); 1138 else if (HaveFunPtrs) 1139 deduceOperandElementTypeFunctionPointer(CI, Ops, KnownElemTy, 1140 IsPostprocessing); 1141 } 1142 1143 // There is no enough info to deduce types or all is valid. 1144 if (!KnownElemTy || Ops.size() == 0) 1145 return; 1146 1147 LLVMContext &Ctx = CurrF->getContext(); 1148 IRBuilder<> B(Ctx); 1149 for (auto &OpIt : Ops) { 1150 Value *Op = OpIt.first; 1151 if (Op->use_empty()) 1152 continue; 1153 if (AskOps && !AskOps->contains(Op)) 1154 continue; 1155 Type *AskTy = nullptr; 1156 CallInst *AskCI = nullptr; 1157 if (IsPostprocessing && AskOps) { 1158 AskTy = GR->findDeducedElementType(Op); 1159 AskCI = GR->findAssignPtrTypeInstr(Op); 1160 assert(AskTy && AskCI); 1161 } 1162 Type *Ty = AskTy ? AskTy : GR->findDeducedElementType(Op); 1163 if (Ty == KnownElemTy) 1164 continue; 1165 Value *OpTyVal = PoisonValue::get(KnownElemTy); 1166 Type *OpTy = Op->getType(); 1167 if (!Ty || AskTy || isUntypedPointerTy(Ty) || isTodoType(Op)) { 1168 Type *PrevElemTy = GR->findDeducedElementType(Op); 1169 GR->addDeducedElementType(Op, KnownElemTy); 1170 // check if KnownElemTy is complete 1171 if (!Uncomplete) 1172 eraseTodoType(Op); 1173 else if (!IsPostprocessing) 1174 insertTodoType(Op); 1175 // check if there is existing Intrinsic::spv_assign_ptr_type instruction 1176 CallInst *AssignCI = AskCI ? AskCI : GR->findAssignPtrTypeInstr(Op); 1177 if (AssignCI == nullptr) { 1178 Instruction *User = dyn_cast<Instruction>(Op->use_begin()->get()); 1179 setInsertPointSkippingPhis(B, User ? User->getNextNode() : I); 1180 CallInst *CI = 1181 buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {OpTy}, OpTyVal, Op, 1182 {B.getInt32(getPointerAddressSpace(OpTy))}, B); 1183 GR->addAssignPtrTypeInstr(Op, CI); 1184 } else { 1185 updateAssignType(AssignCI, Op, OpTyVal); 1186 DenseSet<std::pair<Value *, Value *>> VisitedSubst{ 1187 std::make_pair(I, Op)}; 1188 propagateElemTypeRec(Op, KnownElemTy, PrevElemTy, VisitedSubst); 1189 } 1190 } else { 1191 eraseTodoType(Op); 1192 CallInst *PtrCastI = 1193 buildSpvPtrcast(I->getParent()->getParent(), Op, KnownElemTy); 1194 if (OpIt.second == std::numeric_limits<unsigned>::max()) 1195 dyn_cast<CallInst>(I)->setCalledOperand(PtrCastI); 1196 else 1197 I->setOperand(OpIt.second, PtrCastI); 1198 } 1199 } 1200 TypeValidated.insert(I); 1201 } 1202 1203 void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old, 1204 Instruction *New, 1205 IRBuilder<> &B) { 1206 while (!Old->user_empty()) { 1207 auto *U = Old->user_back(); 1208 if (isAssignTypeInstr(U)) { 1209 B.SetInsertPoint(U); 1210 SmallVector<Value *, 2> Args = {New, U->getOperand(1)}; 1211 CallInst *AssignCI = 1212 B.CreateIntrinsic(Intrinsic::spv_assign_type, {New->getType()}, Args); 1213 GR->addAssignPtrTypeInstr(New, AssignCI); 1214 U->eraseFromParent(); 1215 } else if (isMemInstrToReplace(U) || isa<ReturnInst>(U) || 1216 isa<CallInst>(U)) { 1217 U->replaceUsesOfWith(Old, New); 1218 } else { 1219 llvm_unreachable("illegal aggregate intrinsic user"); 1220 } 1221 } 1222 Old->eraseFromParent(); 1223 } 1224 1225 void SPIRVEmitIntrinsics::preprocessUndefs(IRBuilder<> &B) { 1226 std::queue<Instruction *> Worklist; 1227 for (auto &I : instructions(CurrF)) 1228 Worklist.push(&I); 1229 1230 while (!Worklist.empty()) { 1231 Instruction *I = Worklist.front(); 1232 bool BPrepared = false; 1233 Worklist.pop(); 1234 1235 for (auto &Op : I->operands()) { 1236 auto *AggrUndef = dyn_cast<UndefValue>(Op); 1237 if (!AggrUndef || !Op->getType()->isAggregateType()) 1238 continue; 1239 1240 if (!BPrepared) { 1241 setInsertPointSkippingPhis(B, I); 1242 BPrepared = true; 1243 } 1244 auto *IntrUndef = B.CreateIntrinsic(Intrinsic::spv_undef, {}, {}); 1245 Worklist.push(IntrUndef); 1246 I->replaceUsesOfWith(Op, IntrUndef); 1247 AggrConsts[IntrUndef] = AggrUndef; 1248 AggrConstTypes[IntrUndef] = AggrUndef->getType(); 1249 } 1250 } 1251 } 1252 1253 void SPIRVEmitIntrinsics::preprocessCompositeConstants(IRBuilder<> &B) { 1254 std::queue<Instruction *> Worklist; 1255 for (auto &I : instructions(CurrF)) 1256 Worklist.push(&I); 1257 1258 while (!Worklist.empty()) { 1259 auto *I = Worklist.front(); 1260 bool IsPhi = isa<PHINode>(I), BPrepared = false; 1261 assert(I); 1262 bool KeepInst = false; 1263 for (const auto &Op : I->operands()) { 1264 Constant *AggrConst = nullptr; 1265 Type *ResTy = nullptr; 1266 if (auto *COp = dyn_cast<ConstantVector>(Op)) { 1267 AggrConst = cast<Constant>(COp); 1268 ResTy = COp->getType(); 1269 } else if (auto *COp = dyn_cast<ConstantArray>(Op)) { 1270 AggrConst = cast<Constant>(COp); 1271 ResTy = B.getInt32Ty(); 1272 } else if (auto *COp = dyn_cast<ConstantStruct>(Op)) { 1273 AggrConst = cast<Constant>(COp); 1274 ResTy = B.getInt32Ty(); 1275 } else if (auto *COp = dyn_cast<ConstantDataArray>(Op)) { 1276 AggrConst = cast<Constant>(COp); 1277 ResTy = B.getInt32Ty(); 1278 } else if (auto *COp = dyn_cast<ConstantAggregateZero>(Op)) { 1279 AggrConst = cast<Constant>(COp); 1280 ResTy = Op->getType()->isVectorTy() ? COp->getType() : B.getInt32Ty(); 1281 } 1282 if (AggrConst) { 1283 SmallVector<Value *> Args; 1284 if (auto *COp = dyn_cast<ConstantDataSequential>(Op)) 1285 for (unsigned i = 0; i < COp->getNumElements(); ++i) 1286 Args.push_back(COp->getElementAsConstant(i)); 1287 else 1288 for (auto &COp : AggrConst->operands()) 1289 Args.push_back(COp); 1290 if (!BPrepared) { 1291 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent()) 1292 : B.SetInsertPoint(I); 1293 BPrepared = true; 1294 } 1295 auto *CI = 1296 B.CreateIntrinsic(Intrinsic::spv_const_composite, {ResTy}, {Args}); 1297 Worklist.push(CI); 1298 I->replaceUsesOfWith(Op, CI); 1299 KeepInst = true; 1300 AggrConsts[CI] = AggrConst; 1301 AggrConstTypes[CI] = deduceNestedTypeHelper(AggrConst, false); 1302 } 1303 } 1304 if (!KeepInst) 1305 Worklist.pop(); 1306 } 1307 } 1308 1309 static void createDecorationIntrinsic(Instruction *I, MDNode *Node, 1310 IRBuilder<> &B) { 1311 LLVMContext &Ctx = I->getContext(); 1312 setInsertPointAfterDef(B, I); 1313 B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {I->getType()}, 1314 {I, MetadataAsValue::get(Ctx, MDNode::get(Ctx, {Node}))}); 1315 } 1316 1317 static void createRoundingModeDecoration(Instruction *I, 1318 unsigned RoundingModeDeco, 1319 IRBuilder<> &B) { 1320 LLVMContext &Ctx = I->getContext(); 1321 Type *Int32Ty = Type::getInt32Ty(Ctx); 1322 MDNode *RoundingModeNode = MDNode::get( 1323 Ctx, 1324 {ConstantAsMetadata::get( 1325 ConstantInt::get(Int32Ty, SPIRV::Decoration::FPRoundingMode)), 1326 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, RoundingModeDeco))}); 1327 createDecorationIntrinsic(I, RoundingModeNode, B); 1328 } 1329 1330 static void createSaturatedConversionDecoration(Instruction *I, 1331 IRBuilder<> &B) { 1332 LLVMContext &Ctx = I->getContext(); 1333 Type *Int32Ty = Type::getInt32Ty(Ctx); 1334 MDNode *SaturatedConversionNode = 1335 MDNode::get(Ctx, {ConstantAsMetadata::get(ConstantInt::get( 1336 Int32Ty, SPIRV::Decoration::SaturatedConversion))}); 1337 createDecorationIntrinsic(I, SaturatedConversionNode, B); 1338 } 1339 1340 Instruction *SPIRVEmitIntrinsics::visitCallInst(CallInst &Call) { 1341 if (!Call.isInlineAsm()) 1342 return &Call; 1343 1344 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand()); 1345 LLVMContext &Ctx = CurrF->getContext(); 1346 1347 Constant *TyC = UndefValue::get(IA->getFunctionType()); 1348 MDString *ConstraintString = MDString::get(Ctx, IA->getConstraintString()); 1349 SmallVector<Value *> Args = { 1350 buildMD(TyC), 1351 MetadataAsValue::get(Ctx, MDNode::get(Ctx, ConstraintString))}; 1352 for (unsigned OpIdx = 0; OpIdx < Call.arg_size(); OpIdx++) 1353 Args.push_back(Call.getArgOperand(OpIdx)); 1354 1355 IRBuilder<> B(Call.getParent()); 1356 B.SetInsertPoint(&Call); 1357 B.CreateIntrinsic(Intrinsic::spv_inline_asm, {}, {Args}); 1358 return &Call; 1359 } 1360 1361 // Use a tip about rounding mode to create a decoration. 1362 void SPIRVEmitIntrinsics::useRoundingMode(ConstrainedFPIntrinsic *FPI, 1363 IRBuilder<> &B) { 1364 std::optional<RoundingMode> RM = FPI->getRoundingMode(); 1365 if (!RM.has_value()) 1366 return; 1367 unsigned RoundingModeDeco = std::numeric_limits<unsigned>::max(); 1368 switch (RM.value()) { 1369 default: 1370 // ignore unknown rounding modes 1371 break; 1372 case RoundingMode::NearestTiesToEven: 1373 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTE; 1374 break; 1375 case RoundingMode::TowardNegative: 1376 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTN; 1377 break; 1378 case RoundingMode::TowardPositive: 1379 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTP; 1380 break; 1381 case RoundingMode::TowardZero: 1382 RoundingModeDeco = SPIRV::FPRoundingMode::FPRoundingMode::RTZ; 1383 break; 1384 case RoundingMode::Dynamic: 1385 case RoundingMode::NearestTiesToAway: 1386 // TODO: check if supported 1387 break; 1388 } 1389 if (RoundingModeDeco == std::numeric_limits<unsigned>::max()) 1390 return; 1391 // Convert the tip about rounding mode into a decoration record. 1392 createRoundingModeDecoration(FPI, RoundingModeDeco, B); 1393 } 1394 1395 Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) { 1396 BasicBlock *ParentBB = I.getParent(); 1397 IRBuilder<> B(ParentBB); 1398 B.SetInsertPoint(&I); 1399 SmallVector<Value *, 4> Args; 1400 SmallVector<BasicBlock *> BBCases; 1401 for (auto &Op : I.operands()) { 1402 if (Op.get()->getType()->isSized()) { 1403 Args.push_back(Op); 1404 } else if (BasicBlock *BB = dyn_cast<BasicBlock>(Op.get())) { 1405 BBCases.push_back(BB); 1406 Args.push_back(BlockAddress::get(BB->getParent(), BB)); 1407 } else { 1408 report_fatal_error("Unexpected switch operand"); 1409 } 1410 } 1411 CallInst *NewI = B.CreateIntrinsic(Intrinsic::spv_switch, 1412 {I.getOperand(0)->getType()}, {Args}); 1413 // remove switch to avoid its unneeded and undesirable unwrap into branches 1414 // and conditions 1415 replaceAllUsesWith(&I, NewI); 1416 I.eraseFromParent(); 1417 // insert artificial and temporary instruction to preserve valid CFG, 1418 // it will be removed after IR translation pass 1419 B.SetInsertPoint(ParentBB); 1420 IndirectBrInst *BrI = B.CreateIndirectBr( 1421 Constant::getNullValue(PointerType::getUnqual(ParentBB->getContext())), 1422 BBCases.size()); 1423 for (BasicBlock *BBCase : BBCases) 1424 BrI->addDestination(BBCase); 1425 return BrI; 1426 } 1427 1428 Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) { 1429 IRBuilder<> B(I.getParent()); 1430 B.SetInsertPoint(&I); 1431 SmallVector<Type *, 2> Types = {I.getType(), I.getOperand(0)->getType()}; 1432 SmallVector<Value *, 4> Args; 1433 Args.push_back(B.getInt1(I.isInBounds())); 1434 for (auto &Op : I.operands()) 1435 Args.push_back(Op); 1436 auto *NewI = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args}); 1437 replaceAllUsesWithAndErase(B, &I, NewI); 1438 return NewI; 1439 } 1440 1441 Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) { 1442 IRBuilder<> B(I.getParent()); 1443 B.SetInsertPoint(&I); 1444 Value *Source = I.getOperand(0); 1445 1446 // SPIR-V, contrary to LLVM 17+ IR, supports bitcasts between pointers of 1447 // varying element types. In case of IR coming from older versions of LLVM 1448 // such bitcasts do not provide sufficient information, should be just skipped 1449 // here, and handled in insertPtrCastOrAssignTypeInstr. 1450 if (isPointerTy(I.getType())) { 1451 replaceAllUsesWith(&I, Source); 1452 I.eraseFromParent(); 1453 return nullptr; 1454 } 1455 1456 SmallVector<Type *, 2> Types = {I.getType(), Source->getType()}; 1457 SmallVector<Value *> Args(I.op_begin(), I.op_end()); 1458 auto *NewI = B.CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args}); 1459 replaceAllUsesWithAndErase(B, &I, NewI); 1460 return NewI; 1461 } 1462 1463 void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt( 1464 TargetExtType *AssignedType, Value *V, IRBuilder<> &B) { 1465 Type *VTy = V->getType(); 1466 1467 // A couple of sanity checks. 1468 assert((isPointerTy(VTy)) && "Expect a pointer type!"); 1469 if (Type *ElemTy = getPointeeType(VTy)) 1470 if (ElemTy != AssignedType) 1471 report_fatal_error("Unexpected pointer element type!"); 1472 1473 CallInst *AssignCI = GR->findAssignPtrTypeInstr(V); 1474 if (!AssignCI) { 1475 buildAssignType(B, AssignedType, V); 1476 return; 1477 } 1478 1479 Type *CurrentType = 1480 dyn_cast<ConstantAsMetadata>( 1481 cast<MetadataAsValue>(AssignCI->getOperand(1))->getMetadata()) 1482 ->getType(); 1483 if (CurrentType == AssignedType) 1484 return; 1485 1486 // Builtin types cannot be redeclared or casted. 1487 if (CurrentType->isTargetExtTy()) 1488 report_fatal_error("Type mismatch " + CurrentType->getTargetExtName() + 1489 "/" + AssignedType->getTargetExtName() + 1490 " for value " + V->getName(), 1491 false); 1492 1493 // Our previous guess about the type seems to be wrong, let's update 1494 // inferred type according to a new, more precise type information. 1495 updateAssignType(AssignCI, V, PoisonValue::get(AssignedType)); 1496 } 1497 1498 void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( 1499 Instruction *I, Value *Pointer, Type *ExpectedElementType, 1500 unsigned OperandToReplace, IRBuilder<> &B) { 1501 TypeValidated.insert(I); 1502 1503 // Do not emit spv_ptrcast if Pointer's element type is ExpectedElementType 1504 Type *PointerElemTy = deduceElementTypeHelper(Pointer, false); 1505 if (PointerElemTy == ExpectedElementType || 1506 isEquivalentTypes(PointerElemTy, ExpectedElementType)) 1507 return; 1508 1509 setInsertPointSkippingPhis(B, I); 1510 Value *ExpectedElementVal = PoisonValue::get(ExpectedElementType); 1511 MetadataAsValue *VMD = buildMD(ExpectedElementVal); 1512 unsigned AddressSpace = getPointerAddressSpace(Pointer->getType()); 1513 bool FirstPtrCastOrAssignPtrType = true; 1514 1515 // Do not emit new spv_ptrcast if equivalent one already exists or when 1516 // spv_assign_ptr_type already targets this pointer with the same element 1517 // type. 1518 for (auto User : Pointer->users()) { 1519 auto *II = dyn_cast<IntrinsicInst>(User); 1520 if (!II || 1521 (II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type && 1522 II->getIntrinsicID() != Intrinsic::spv_ptrcast) || 1523 II->getOperand(0) != Pointer) 1524 continue; 1525 1526 // There is some spv_ptrcast/spv_assign_ptr_type already targeting this 1527 // pointer. 1528 FirstPtrCastOrAssignPtrType = false; 1529 if (II->getOperand(1) != VMD || 1530 dyn_cast<ConstantInt>(II->getOperand(2))->getSExtValue() != 1531 AddressSpace) 1532 continue; 1533 1534 // The spv_ptrcast/spv_assign_ptr_type targeting this pointer is of the same 1535 // element type and address space. 1536 if (II->getIntrinsicID() != Intrinsic::spv_ptrcast) 1537 return; 1538 1539 // This must be a spv_ptrcast, do not emit new if this one has the same BB 1540 // as I. Otherwise, search for other spv_ptrcast/spv_assign_ptr_type. 1541 if (II->getParent() != I->getParent()) 1542 continue; 1543 1544 I->setOperand(OperandToReplace, II); 1545 return; 1546 } 1547 1548 if (isa<Instruction>(Pointer) || isa<Argument>(Pointer)) { 1549 if (FirstPtrCastOrAssignPtrType) { 1550 // If this would be the first spv_ptrcast, do not emit spv_ptrcast and 1551 // emit spv_assign_ptr_type instead. 1552 buildAssignPtr(B, ExpectedElementType, Pointer); 1553 return; 1554 } else if (isTodoType(Pointer)) { 1555 eraseTodoType(Pointer); 1556 if (!isa<CallInst>(Pointer) && !isa<GetElementPtrInst>(Pointer)) { 1557 // If this wouldn't be the first spv_ptrcast but existing type info is 1558 // uncomplete, update spv_assign_ptr_type arguments. 1559 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Pointer)) { 1560 Type *PrevElemTy = GR->findDeducedElementType(Pointer); 1561 assert(PrevElemTy); 1562 DenseSet<std::pair<Value *, Value *>> VisitedSubst{ 1563 std::make_pair(I, Pointer)}; 1564 updateAssignType(AssignCI, Pointer, ExpectedElementVal); 1565 propagateElemType(Pointer, PrevElemTy, VisitedSubst); 1566 } else { 1567 buildAssignPtr(B, ExpectedElementType, Pointer); 1568 } 1569 return; 1570 } 1571 } 1572 } 1573 1574 // Emit spv_ptrcast 1575 SmallVector<Type *, 2> Types = {Pointer->getType(), Pointer->getType()}; 1576 SmallVector<Value *, 2> Args = {Pointer, VMD, B.getInt32(AddressSpace)}; 1577 auto *PtrCastI = B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args); 1578 I->setOperand(OperandToReplace, PtrCastI); 1579 // We need to set up a pointee type for the newly created spv_ptrcast. 1580 buildAssignPtr(B, ExpectedElementType, PtrCastI); 1581 } 1582 1583 void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I, 1584 IRBuilder<> &B) { 1585 // Handle basic instructions: 1586 StoreInst *SI = dyn_cast<StoreInst>(I); 1587 if (IsKernelArgInt8(CurrF, SI)) { 1588 replacePointerOperandWithPtrCast( 1589 I, SI->getValueOperand(), IntegerType::getInt8Ty(CurrF->getContext()), 1590 0, B); 1591 } 1592 if (SI) { 1593 Value *Op = SI->getValueOperand(); 1594 Value *Pointer = SI->getPointerOperand(); 1595 Type *OpTy = Op->getType(); 1596 if (auto *OpI = dyn_cast<Instruction>(Op)) 1597 OpTy = restoreMutatedType(GR, OpI, OpTy); 1598 if (OpTy == Op->getType()) 1599 OpTy = deduceElementTypeByValueDeep(OpTy, Op, false); 1600 replacePointerOperandWithPtrCast(I, Pointer, OpTy, 1, B); 1601 return; 1602 } 1603 if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 1604 Value *Pointer = LI->getPointerOperand(); 1605 Type *OpTy = LI->getType(); 1606 if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) { 1607 if (Type *ElemTy = GR->findDeducedElementType(LI)) { 1608 OpTy = getTypedPointerWrapper(ElemTy, PtrTy->getAddressSpace()); 1609 } else { 1610 Type *NewOpTy = OpTy; 1611 OpTy = deduceElementTypeByValueDeep(OpTy, LI, false); 1612 if (OpTy == NewOpTy) 1613 insertTodoType(Pointer); 1614 } 1615 } 1616 replacePointerOperandWithPtrCast(I, Pointer, OpTy, 0, B); 1617 return; 1618 } 1619 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) { 1620 Value *Pointer = GEPI->getPointerOperand(); 1621 Type *OpTy = GEPI->getSourceElementType(); 1622 replacePointerOperandWithPtrCast(I, Pointer, OpTy, 0, B); 1623 if (isNestedPointer(OpTy)) 1624 insertTodoType(Pointer); 1625 return; 1626 } 1627 1628 // TODO: review and merge with existing logics: 1629 // Handle calls to builtins (non-intrinsics): 1630 CallInst *CI = dyn_cast<CallInst>(I); 1631 if (!CI || CI->isIndirectCall() || CI->isInlineAsm() || 1632 !CI->getCalledFunction() || CI->getCalledFunction()->isIntrinsic()) 1633 return; 1634 1635 // collect information about formal parameter types 1636 std::string DemangledName = 1637 getOclOrSpirvBuiltinDemangledName(CI->getCalledFunction()->getName()); 1638 Function *CalledF = CI->getCalledFunction(); 1639 SmallVector<Type *, 4> CalledArgTys; 1640 bool HaveTypes = false; 1641 for (unsigned OpIdx = 0; OpIdx < CalledF->arg_size(); ++OpIdx) { 1642 Argument *CalledArg = CalledF->getArg(OpIdx); 1643 Type *ArgType = CalledArg->getType(); 1644 if (!isPointerTy(ArgType)) { 1645 CalledArgTys.push_back(nullptr); 1646 } else if (Type *ArgTypeElem = getPointeeType(ArgType)) { 1647 CalledArgTys.push_back(ArgTypeElem); 1648 HaveTypes = true; 1649 } else { 1650 Type *ElemTy = GR->findDeducedElementType(CalledArg); 1651 if (!ElemTy && hasPointeeTypeAttr(CalledArg)) 1652 ElemTy = getPointeeTypeByAttr(CalledArg); 1653 if (!ElemTy) { 1654 ElemTy = getPointeeTypeByCallInst(DemangledName, CalledF, OpIdx); 1655 if (ElemTy) { 1656 GR->addDeducedElementType(CalledArg, ElemTy); 1657 } else { 1658 for (User *U : CalledArg->users()) { 1659 if (Instruction *Inst = dyn_cast<Instruction>(U)) { 1660 if ((ElemTy = deduceElementTypeHelper(Inst, false)) != nullptr) 1661 break; 1662 } 1663 } 1664 } 1665 } 1666 HaveTypes |= ElemTy != nullptr; 1667 CalledArgTys.push_back(ElemTy); 1668 } 1669 } 1670 1671 if (DemangledName.empty() && !HaveTypes) 1672 return; 1673 1674 for (unsigned OpIdx = 0; OpIdx < CI->arg_size(); OpIdx++) { 1675 Value *ArgOperand = CI->getArgOperand(OpIdx); 1676 if (!isPointerTy(ArgOperand->getType())) 1677 continue; 1678 1679 // Constants (nulls/undefs) are handled in insertAssignPtrTypeIntrs() 1680 if (!isa<Instruction>(ArgOperand) && !isa<Argument>(ArgOperand)) { 1681 // However, we may have assumptions about the formal argument's type and 1682 // may have a need to insert a ptr cast for the actual parameter of this 1683 // call. 1684 Argument *CalledArg = CalledF->getArg(OpIdx); 1685 if (!GR->findDeducedElementType(CalledArg)) 1686 continue; 1687 } 1688 1689 Type *ExpectedType = 1690 OpIdx < CalledArgTys.size() ? CalledArgTys[OpIdx] : nullptr; 1691 if (!ExpectedType && !DemangledName.empty()) 1692 ExpectedType = SPIRV::parseBuiltinCallArgumentBaseType( 1693 DemangledName, OpIdx, I->getContext()); 1694 if (!ExpectedType || ExpectedType->isVoidTy()) 1695 continue; 1696 1697 if (ExpectedType->isTargetExtTy() && 1698 !isTypedPointerWrapper(cast<TargetExtType>(ExpectedType))) 1699 insertAssignPtrTypeTargetExt(cast<TargetExtType>(ExpectedType), 1700 ArgOperand, B); 1701 else 1702 replacePointerOperandWithPtrCast(CI, ArgOperand, ExpectedType, OpIdx, B); 1703 } 1704 } 1705 1706 Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) { 1707 SmallVector<Type *, 4> Types = {I.getType(), I.getOperand(0)->getType(), 1708 I.getOperand(1)->getType(), 1709 I.getOperand(2)->getType()}; 1710 IRBuilder<> B(I.getParent()); 1711 B.SetInsertPoint(&I); 1712 SmallVector<Value *> Args(I.op_begin(), I.op_end()); 1713 auto *NewI = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args}); 1714 replaceAllUsesWithAndErase(B, &I, NewI); 1715 return NewI; 1716 } 1717 1718 Instruction * 1719 SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) { 1720 IRBuilder<> B(I.getParent()); 1721 B.SetInsertPoint(&I); 1722 SmallVector<Type *, 3> Types = {I.getType(), I.getVectorOperandType(), 1723 I.getIndexOperand()->getType()}; 1724 SmallVector<Value *, 2> Args = {I.getVectorOperand(), I.getIndexOperand()}; 1725 auto *NewI = B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args}); 1726 replaceAllUsesWithAndErase(B, &I, NewI); 1727 return NewI; 1728 } 1729 1730 Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) { 1731 IRBuilder<> B(I.getParent()); 1732 B.SetInsertPoint(&I); 1733 SmallVector<Type *, 1> Types = {I.getInsertedValueOperand()->getType()}; 1734 SmallVector<Value *> Args; 1735 for (auto &Op : I.operands()) 1736 if (isa<UndefValue>(Op)) 1737 Args.push_back(UndefValue::get(B.getInt32Ty())); 1738 else 1739 Args.push_back(Op); 1740 for (auto &Op : I.indices()) 1741 Args.push_back(B.getInt32(Op)); 1742 Instruction *NewI = 1743 B.CreateIntrinsic(Intrinsic::spv_insertv, {Types}, {Args}); 1744 replaceMemInstrUses(&I, NewI, B); 1745 return NewI; 1746 } 1747 1748 Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) { 1749 if (I.getAggregateOperand()->getType()->isAggregateType()) 1750 return &I; 1751 IRBuilder<> B(I.getParent()); 1752 B.SetInsertPoint(&I); 1753 SmallVector<Value *> Args; 1754 for (auto &Op : I.operands()) 1755 Args.push_back(Op); 1756 for (auto &Op : I.indices()) 1757 Args.push_back(B.getInt32(Op)); 1758 auto *NewI = 1759 B.CreateIntrinsic(Intrinsic::spv_extractv, {I.getType()}, {Args}); 1760 replaceAllUsesWithAndErase(B, &I, NewI); 1761 return NewI; 1762 } 1763 1764 Instruction *SPIRVEmitIntrinsics::visitLoadInst(LoadInst &I) { 1765 if (!I.getType()->isAggregateType()) 1766 return &I; 1767 IRBuilder<> B(I.getParent()); 1768 B.SetInsertPoint(&I); 1769 TrackConstants = false; 1770 const auto *TLI = TM->getSubtargetImpl()->getTargetLowering(); 1771 MachineMemOperand::Flags Flags = 1772 TLI->getLoadMemOperandFlags(I, CurrF->getDataLayout()); 1773 auto *NewI = 1774 B.CreateIntrinsic(Intrinsic::spv_load, {I.getOperand(0)->getType()}, 1775 {I.getPointerOperand(), B.getInt16(Flags), 1776 B.getInt8(I.getAlign().value())}); 1777 replaceMemInstrUses(&I, NewI, B); 1778 return NewI; 1779 } 1780 1781 Instruction *SPIRVEmitIntrinsics::visitStoreInst(StoreInst &I) { 1782 if (!AggrStores.contains(&I)) 1783 return &I; 1784 IRBuilder<> B(I.getParent()); 1785 B.SetInsertPoint(&I); 1786 TrackConstants = false; 1787 const auto *TLI = TM->getSubtargetImpl()->getTargetLowering(); 1788 MachineMemOperand::Flags Flags = 1789 TLI->getStoreMemOperandFlags(I, CurrF->getDataLayout()); 1790 auto *PtrOp = I.getPointerOperand(); 1791 auto *NewI = B.CreateIntrinsic( 1792 Intrinsic::spv_store, {I.getValueOperand()->getType(), PtrOp->getType()}, 1793 {I.getValueOperand(), PtrOp, B.getInt16(Flags), 1794 B.getInt8(I.getAlign().value())}); 1795 I.eraseFromParent(); 1796 return NewI; 1797 } 1798 1799 Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) { 1800 Value *ArraySize = nullptr; 1801 if (I.isArrayAllocation()) { 1802 const SPIRVSubtarget *STI = TM->getSubtargetImpl(*I.getFunction()); 1803 if (!STI->canUseExtension( 1804 SPIRV::Extension::SPV_INTEL_variable_length_array)) 1805 report_fatal_error( 1806 "array allocation: this instruction requires the following " 1807 "SPIR-V extension: SPV_INTEL_variable_length_array", 1808 false); 1809 ArraySize = I.getArraySize(); 1810 } 1811 IRBuilder<> B(I.getParent()); 1812 B.SetInsertPoint(&I); 1813 TrackConstants = false; 1814 Type *PtrTy = I.getType(); 1815 auto *NewI = 1816 ArraySize 1817 ? B.CreateIntrinsic(Intrinsic::spv_alloca_array, 1818 {PtrTy, ArraySize->getType()}, 1819 {ArraySize, B.getInt8(I.getAlign().value())}) 1820 : B.CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, 1821 {B.getInt8(I.getAlign().value())}); 1822 replaceAllUsesWithAndErase(B, &I, NewI); 1823 return NewI; 1824 } 1825 1826 Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) { 1827 assert(I.getType()->isAggregateType() && "Aggregate result is expected"); 1828 IRBuilder<> B(I.getParent()); 1829 B.SetInsertPoint(&I); 1830 SmallVector<Value *> Args; 1831 for (auto &Op : I.operands()) 1832 Args.push_back(Op); 1833 Args.push_back(B.getInt32( 1834 static_cast<uint32_t>(getMemScope(I.getContext(), I.getSyncScopeID())))); 1835 Args.push_back(B.getInt32( 1836 static_cast<uint32_t>(getMemSemantics(I.getSuccessOrdering())))); 1837 Args.push_back(B.getInt32( 1838 static_cast<uint32_t>(getMemSemantics(I.getFailureOrdering())))); 1839 auto *NewI = B.CreateIntrinsic(Intrinsic::spv_cmpxchg, 1840 {I.getPointerOperand()->getType()}, {Args}); 1841 replaceMemInstrUses(&I, NewI, B); 1842 return NewI; 1843 } 1844 1845 Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) { 1846 IRBuilder<> B(I.getParent()); 1847 B.SetInsertPoint(&I); 1848 B.CreateIntrinsic(Intrinsic::spv_unreachable, {}, {}); 1849 return &I; 1850 } 1851 1852 void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV, 1853 IRBuilder<> &B) { 1854 // Skip special artifical variable llvm.global.annotations. 1855 if (GV.getName() == "llvm.global.annotations") 1856 return; 1857 Constant *Init = nullptr; 1858 if (hasInitializer(&GV)) { 1859 // Deduce element type and store results in Global Registry. 1860 // Result is ignored, because TypedPointerType is not supported 1861 // by llvm IR general logic. 1862 deduceElementTypeHelper(&GV, false); 1863 Init = GV.getInitializer(); 1864 Type *Ty = isAggrConstForceInt32(Init) ? B.getInt32Ty() : Init->getType(); 1865 Constant *Const = isAggrConstForceInt32(Init) ? B.getInt32(1) : Init; 1866 auto *InitInst = B.CreateIntrinsic(Intrinsic::spv_init_global, 1867 {GV.getType(), Ty}, {&GV, Const}); 1868 InitInst->setArgOperand(1, Init); 1869 } 1870 if (!Init && GV.getNumUses() == 0) 1871 B.CreateIntrinsic(Intrinsic::spv_unref_global, GV.getType(), &GV); 1872 } 1873 1874 // Return true, if we can't decide what is the pointee type now and will get 1875 // back to the question later. Return false is spv_assign_ptr_type is not needed 1876 // or can be inserted immediately. 1877 bool SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I, 1878 IRBuilder<> &B, 1879 bool UnknownElemTypeI8) { 1880 reportFatalOnTokenType(I); 1881 if (!isPointerTy(I->getType()) || !requireAssignType(I)) 1882 return false; 1883 1884 setInsertPointAfterDef(B, I); 1885 if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) { 1886 buildAssignPtr(B, ElemTy, I); 1887 return false; 1888 } 1889 return true; 1890 } 1891 1892 void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, 1893 IRBuilder<> &B) { 1894 // TODO: extend the list of functions with known result types 1895 static StringMap<unsigned> ResTypeWellKnown = { 1896 {"async_work_group_copy", WellKnownTypes::Event}, 1897 {"async_work_group_strided_copy", WellKnownTypes::Event}, 1898 {"__spirv_GroupAsyncCopy", WellKnownTypes::Event}}; 1899 1900 reportFatalOnTokenType(I); 1901 1902 bool IsKnown = false; 1903 if (auto *CI = dyn_cast<CallInst>(I)) { 1904 if (!CI->isIndirectCall() && !CI->isInlineAsm() && 1905 CI->getCalledFunction() && !CI->getCalledFunction()->isIntrinsic()) { 1906 Function *CalledF = CI->getCalledFunction(); 1907 std::string DemangledName = 1908 getOclOrSpirvBuiltinDemangledName(CalledF->getName()); 1909 FPDecorationId DecorationId = FPDecorationId::NONE; 1910 if (DemangledName.length() > 0) 1911 DemangledName = 1912 SPIRV::lookupBuiltinNameHelper(DemangledName, &DecorationId); 1913 auto ResIt = ResTypeWellKnown.find(DemangledName); 1914 if (ResIt != ResTypeWellKnown.end()) { 1915 IsKnown = true; 1916 setInsertPointAfterDef(B, I); 1917 switch (ResIt->second) { 1918 case WellKnownTypes::Event: 1919 buildAssignType(B, TargetExtType::get(I->getContext(), "spirv.Event"), 1920 I); 1921 break; 1922 } 1923 } 1924 // check if a floating rounding mode or saturation info is present 1925 switch (DecorationId) { 1926 default: 1927 break; 1928 case FPDecorationId::SAT: 1929 createSaturatedConversionDecoration(CI, B); 1930 break; 1931 case FPDecorationId::RTE: 1932 createRoundingModeDecoration( 1933 CI, SPIRV::FPRoundingMode::FPRoundingMode::RTE, B); 1934 break; 1935 case FPDecorationId::RTZ: 1936 createRoundingModeDecoration( 1937 CI, SPIRV::FPRoundingMode::FPRoundingMode::RTZ, B); 1938 break; 1939 case FPDecorationId::RTP: 1940 createRoundingModeDecoration( 1941 CI, SPIRV::FPRoundingMode::FPRoundingMode::RTP, B); 1942 break; 1943 case FPDecorationId::RTN: 1944 createRoundingModeDecoration( 1945 CI, SPIRV::FPRoundingMode::FPRoundingMode::RTN, B); 1946 break; 1947 } 1948 } 1949 } 1950 1951 Type *Ty = I->getType(); 1952 if (!IsKnown && !Ty->isVoidTy() && !isPointerTy(Ty) && requireAssignType(I)) { 1953 setInsertPointAfterDef(B, I); 1954 Type *TypeToAssign = Ty; 1955 if (auto *II = dyn_cast<IntrinsicInst>(I)) { 1956 if (II->getIntrinsicID() == Intrinsic::spv_const_composite || 1957 II->getIntrinsicID() == Intrinsic::spv_undef) { 1958 auto It = AggrConstTypes.find(II); 1959 if (It == AggrConstTypes.end()) 1960 report_fatal_error("Unknown composite intrinsic type"); 1961 TypeToAssign = It->second; 1962 } 1963 } 1964 TypeToAssign = restoreMutatedType(GR, I, TypeToAssign); 1965 buildAssignType(B, TypeToAssign, I); 1966 } 1967 for (const auto &Op : I->operands()) { 1968 if (isa<ConstantPointerNull>(Op) || isa<UndefValue>(Op) || 1969 // Check GetElementPtrConstantExpr case. 1970 (isa<ConstantExpr>(Op) && isa<GEPOperator>(Op))) { 1971 setInsertPointSkippingPhis(B, I); 1972 Type *OpTy = Op->getType(); 1973 if (isa<UndefValue>(Op) && OpTy->isAggregateType()) { 1974 CallInst *AssignCI = 1975 buildIntrWithMD(Intrinsic::spv_assign_type, {B.getInt32Ty()}, Op, 1976 UndefValue::get(B.getInt32Ty()), {}, B); 1977 GR->addAssignPtrTypeInstr(Op, AssignCI); 1978 } else if (!isa<Instruction>(Op)) { 1979 Type *OpTy = Op->getType(); 1980 Type *OpTyElem = getPointeeType(OpTy); 1981 if (OpTyElem) { 1982 buildAssignPtr(B, OpTyElem, Op); 1983 } else if (isPointerTy(OpTy)) { 1984 Type *ElemTy = GR->findDeducedElementType(Op); 1985 buildAssignPtr(B, ElemTy ? ElemTy : deduceElementType(Op, true), Op); 1986 } else { 1987 CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, 1988 {OpTy}, Op, Op, {}, B); 1989 GR->addAssignPtrTypeInstr(Op, AssignCI); 1990 } 1991 } 1992 } 1993 } 1994 } 1995 1996 void SPIRVEmitIntrinsics::insertSpirvDecorations(Instruction *I, 1997 IRBuilder<> &B) { 1998 if (MDNode *MD = I->getMetadata("spirv.Decorations")) { 1999 setInsertPointAfterDef(B, I); 2000 B.CreateIntrinsic(Intrinsic::spv_assign_decoration, {I->getType()}, 2001 {I, MetadataAsValue::get(I->getContext(), MD)}); 2002 } 2003 } 2004 2005 void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I, 2006 IRBuilder<> &B) { 2007 auto *II = dyn_cast<IntrinsicInst>(I); 2008 bool IsConstComposite = 2009 II && II->getIntrinsicID() == Intrinsic::spv_const_composite; 2010 if (IsConstComposite && TrackConstants) { 2011 setInsertPointAfterDef(B, I); 2012 auto t = AggrConsts.find(I); 2013 assert(t != AggrConsts.end()); 2014 auto *NewOp = 2015 buildIntrWithMD(Intrinsic::spv_track_constant, 2016 {II->getType(), II->getType()}, t->second, I, {}, B); 2017 replaceAllUsesWith(I, NewOp, false); 2018 NewOp->setArgOperand(0, I); 2019 } 2020 bool IsPhi = isa<PHINode>(I), BPrepared = false; 2021 for (const auto &Op : I->operands()) { 2022 if (isa<PHINode>(I) || isa<SwitchInst>(I)) 2023 TrackConstants = false; 2024 if ((isa<ConstantData>(Op) || isa<ConstantExpr>(Op)) && TrackConstants) { 2025 unsigned OpNo = Op.getOperandNo(); 2026 if (II && ((II->getIntrinsicID() == Intrinsic::spv_gep && OpNo == 0) || 2027 (II->paramHasAttr(OpNo, Attribute::ImmArg)))) 2028 continue; 2029 if (!BPrepared) { 2030 IsPhi ? B.SetInsertPointPastAllocas(I->getParent()->getParent()) 2031 : B.SetInsertPoint(I); 2032 BPrepared = true; 2033 } 2034 Type *OpTy = Op->getType(); 2035 Value *OpTyVal = Op; 2036 if (OpTy->isTargetExtTy()) 2037 OpTyVal = PoisonValue::get(OpTy); 2038 CallInst *NewOp = 2039 buildIntrWithMD(Intrinsic::spv_track_constant, 2040 {OpTy, OpTyVal->getType()}, Op, OpTyVal, {}, B); 2041 Type *OpElemTy = nullptr; 2042 if (!IsConstComposite && isPointerTy(OpTy) && 2043 (OpElemTy = GR->findDeducedElementType(Op)) != nullptr && 2044 OpElemTy != IntegerType::getInt8Ty(I->getContext())) { 2045 buildAssignPtr(B, IntegerType::getInt8Ty(I->getContext()), NewOp); 2046 SmallVector<Type *, 2> Types = {OpTy, OpTy}; 2047 SmallVector<Value *, 2> Args = { 2048 NewOp, buildMD(PoisonValue::get(OpElemTy)), 2049 B.getInt32(getPointerAddressSpace(OpTy))}; 2050 CallInst *PtrCasted = 2051 B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args); 2052 buildAssignPtr(B, OpElemTy, PtrCasted); 2053 NewOp = PtrCasted; 2054 } 2055 I->setOperand(OpNo, NewOp); 2056 } 2057 } 2058 emitAssignName(I, B); 2059 } 2060 2061 Type *SPIRVEmitIntrinsics::deduceFunParamElementType(Function *F, 2062 unsigned OpIdx) { 2063 std::unordered_set<Function *> FVisited; 2064 return deduceFunParamElementType(F, OpIdx, FVisited); 2065 } 2066 2067 Type *SPIRVEmitIntrinsics::deduceFunParamElementType( 2068 Function *F, unsigned OpIdx, std::unordered_set<Function *> &FVisited) { 2069 // maybe a cycle 2070 if (!FVisited.insert(F).second) 2071 return nullptr; 2072 2073 std::unordered_set<Value *> Visited; 2074 SmallVector<std::pair<Function *, unsigned>> Lookup; 2075 // search in function's call sites 2076 for (User *U : F->users()) { 2077 CallInst *CI = dyn_cast<CallInst>(U); 2078 if (!CI || OpIdx >= CI->arg_size()) 2079 continue; 2080 Value *OpArg = CI->getArgOperand(OpIdx); 2081 if (!isPointerTy(OpArg->getType())) 2082 continue; 2083 // maybe we already know operand's element type 2084 if (Type *KnownTy = GR->findDeducedElementType(OpArg)) 2085 return KnownTy; 2086 // try to deduce from the operand itself 2087 Visited.clear(); 2088 if (Type *Ty = deduceElementTypeHelper(OpArg, Visited, false)) 2089 return Ty; 2090 // search in actual parameter's users 2091 for (User *OpU : OpArg->users()) { 2092 Instruction *Inst = dyn_cast<Instruction>(OpU); 2093 if (!Inst || Inst == CI) 2094 continue; 2095 Visited.clear(); 2096 if (Type *Ty = deduceElementTypeHelper(Inst, Visited, false)) 2097 return Ty; 2098 } 2099 // check if it's a formal parameter of the outer function 2100 if (!CI->getParent() || !CI->getParent()->getParent()) 2101 continue; 2102 Function *OuterF = CI->getParent()->getParent(); 2103 if (FVisited.find(OuterF) != FVisited.end()) 2104 continue; 2105 for (unsigned i = 0; i < OuterF->arg_size(); ++i) { 2106 if (OuterF->getArg(i) == OpArg) { 2107 Lookup.push_back(std::make_pair(OuterF, i)); 2108 break; 2109 } 2110 } 2111 } 2112 2113 // search in function parameters 2114 for (auto &Pair : Lookup) { 2115 if (Type *Ty = deduceFunParamElementType(Pair.first, Pair.second, FVisited)) 2116 return Ty; 2117 } 2118 2119 return nullptr; 2120 } 2121 2122 void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F, 2123 IRBuilder<> &B) { 2124 B.SetInsertPointPastAllocas(F); 2125 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) { 2126 Argument *Arg = F->getArg(OpIdx); 2127 if (!isUntypedPointerTy(Arg->getType())) 2128 continue; 2129 Type *ElemTy = GR->findDeducedElementType(Arg); 2130 if (ElemTy) 2131 continue; 2132 if (hasPointeeTypeAttr(Arg) && 2133 (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) { 2134 buildAssignPtr(B, ElemTy, Arg); 2135 continue; 2136 } 2137 // search in function's call sites 2138 for (User *U : F->users()) { 2139 CallInst *CI = dyn_cast<CallInst>(U); 2140 if (!CI || OpIdx >= CI->arg_size()) 2141 continue; 2142 Value *OpArg = CI->getArgOperand(OpIdx); 2143 if (!isPointerTy(OpArg->getType())) 2144 continue; 2145 // maybe we already know operand's element type 2146 if ((ElemTy = GR->findDeducedElementType(OpArg)) != nullptr) 2147 break; 2148 } 2149 if (ElemTy) { 2150 buildAssignPtr(B, ElemTy, Arg); 2151 continue; 2152 } 2153 if (HaveFunPtrs) { 2154 for (User *U : Arg->users()) { 2155 CallInst *CI = dyn_cast<CallInst>(U); 2156 if (CI && !isa<IntrinsicInst>(CI) && CI->isIndirectCall() && 2157 CI->getCalledOperand() == Arg && 2158 CI->getParent()->getParent() == CurrF) { 2159 SmallVector<std::pair<Value *, unsigned>> Ops; 2160 deduceOperandElementTypeFunctionPointer(CI, Ops, ElemTy, false); 2161 if (ElemTy) { 2162 buildAssignPtr(B, ElemTy, Arg); 2163 break; 2164 } 2165 } 2166 } 2167 } 2168 } 2169 } 2170 2171 void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) { 2172 B.SetInsertPointPastAllocas(F); 2173 for (unsigned OpIdx = 0; OpIdx < F->arg_size(); ++OpIdx) { 2174 Argument *Arg = F->getArg(OpIdx); 2175 if (!isUntypedPointerTy(Arg->getType())) 2176 continue; 2177 Type *ElemTy = GR->findDeducedElementType(Arg); 2178 if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) { 2179 if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Arg)) { 2180 DenseSet<std::pair<Value *, Value *>> VisitedSubst; 2181 updateAssignType(AssignCI, Arg, PoisonValue::get(ElemTy)); 2182 propagateElemType(Arg, IntegerType::getInt8Ty(F->getContext()), 2183 VisitedSubst); 2184 } else { 2185 buildAssignPtr(B, ElemTy, Arg); 2186 } 2187 } 2188 } 2189 } 2190 2191 static FunctionType *getFunctionPointerElemType(Function *F, 2192 SPIRVGlobalRegistry *GR) { 2193 FunctionType *FTy = F->getFunctionType(); 2194 bool IsNewFTy = false; 2195 SmallVector<Type *, 4> ArgTys; 2196 for (Argument &Arg : F->args()) { 2197 Type *ArgTy = Arg.getType(); 2198 if (ArgTy->isPointerTy()) 2199 if (Type *ElemTy = GR->findDeducedElementType(&Arg)) { 2200 IsNewFTy = true; 2201 ArgTy = getTypedPointerWrapper(ElemTy, getPointerAddressSpace(ArgTy)); 2202 } 2203 ArgTys.push_back(ArgTy); 2204 } 2205 return IsNewFTy 2206 ? FunctionType::get(FTy->getReturnType(), ArgTys, FTy->isVarArg()) 2207 : FTy; 2208 } 2209 2210 bool SPIRVEmitIntrinsics::processFunctionPointers(Module &M) { 2211 SmallVector<Function *> Worklist; 2212 for (auto &F : M) { 2213 if (F.isIntrinsic()) 2214 continue; 2215 if (F.isDeclaration()) { 2216 for (User *U : F.users()) { 2217 CallInst *CI = dyn_cast<CallInst>(U); 2218 if (!CI || CI->getCalledFunction() != &F) { 2219 Worklist.push_back(&F); 2220 break; 2221 } 2222 } 2223 } else { 2224 if (F.user_empty()) 2225 continue; 2226 Type *FPElemTy = GR->findDeducedElementType(&F); 2227 if (!FPElemTy) 2228 FPElemTy = getFunctionPointerElemType(&F, GR); 2229 for (User *U : F.users()) { 2230 IntrinsicInst *II = dyn_cast<IntrinsicInst>(U); 2231 if (!II || II->arg_size() != 3 || II->getOperand(0) != &F) 2232 continue; 2233 if (II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type || 2234 II->getIntrinsicID() == Intrinsic::spv_ptrcast) { 2235 updateAssignType(II, &F, PoisonValue::get(FPElemTy)); 2236 break; 2237 } 2238 } 2239 } 2240 } 2241 if (Worklist.empty()) 2242 return false; 2243 2244 std::string ServiceFunName = SPIRV_BACKEND_SERVICE_FUN_NAME; 2245 if (!getVacantFunctionName(M, ServiceFunName)) 2246 report_fatal_error( 2247 "cannot allocate a name for the internal service function"); 2248 LLVMContext &Ctx = M.getContext(); 2249 Function *SF = 2250 Function::Create(FunctionType::get(Type::getVoidTy(Ctx), {}, false), 2251 GlobalValue::PrivateLinkage, ServiceFunName, M); 2252 SF->addFnAttr(SPIRV_BACKEND_SERVICE_FUN_NAME, ""); 2253 BasicBlock *BB = BasicBlock::Create(Ctx, "entry", SF); 2254 IRBuilder<> IRB(BB); 2255 2256 for (Function *F : Worklist) { 2257 SmallVector<Value *> Args; 2258 for (const auto &Arg : F->args()) 2259 Args.push_back(PoisonValue::get(Arg.getType())); 2260 IRB.CreateCall(F, Args); 2261 } 2262 IRB.CreateRetVoid(); 2263 2264 return true; 2265 } 2266 2267 // Apply types parsed from demangled function declarations. 2268 void SPIRVEmitIntrinsics::applyDemangledPtrArgTypes(IRBuilder<> &B) { 2269 for (auto It : FDeclPtrTys) { 2270 Function *F = It.first; 2271 for (auto *U : F->users()) { 2272 CallInst *CI = dyn_cast<CallInst>(U); 2273 if (!CI || CI->getCalledFunction() != F) 2274 continue; 2275 unsigned Sz = CI->arg_size(); 2276 for (auto [Idx, ElemTy] : It.second) { 2277 if (Idx >= Sz) 2278 continue; 2279 Value *Param = CI->getArgOperand(Idx); 2280 if (GR->findDeducedElementType(Param) || isa<GlobalValue>(Param)) 2281 continue; 2282 if (Argument *Arg = dyn_cast<Argument>(Param)) { 2283 if (!hasPointeeTypeAttr(Arg)) { 2284 B.SetInsertPointPastAllocas(Arg->getParent()); 2285 B.SetCurrentDebugLocation(DebugLoc()); 2286 buildAssignPtr(B, ElemTy, Arg); 2287 } 2288 } else if (isa<Instruction>(Param)) { 2289 GR->addDeducedElementType(Param, ElemTy); 2290 // insertAssignTypeIntrs() will complete buildAssignPtr() 2291 } else { 2292 B.SetInsertPoint(CI->getParent() 2293 ->getParent() 2294 ->getEntryBlock() 2295 .getFirstNonPHIOrDbgOrAlloca()); 2296 buildAssignPtr(B, ElemTy, Param); 2297 } 2298 CallInst *Ref = dyn_cast<CallInst>(Param); 2299 if (!Ref) 2300 continue; 2301 Function *RefF = Ref->getCalledFunction(); 2302 if (!RefF || !isPointerTy(RefF->getReturnType()) || 2303 GR->findDeducedElementType(RefF)) 2304 continue; 2305 GR->addDeducedElementType(RefF, ElemTy); 2306 GR->addReturnType( 2307 RefF, TypedPointerType::get( 2308 ElemTy, getPointerAddressSpace(RefF->getReturnType()))); 2309 } 2310 } 2311 } 2312 } 2313 2314 bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) { 2315 if (Func.isDeclaration()) 2316 return false; 2317 2318 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(Func); 2319 GR = ST.getSPIRVGlobalRegistry(); 2320 2321 if (!CurrF) 2322 HaveFunPtrs = 2323 ST.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers); 2324 2325 CurrF = &Func; 2326 IRBuilder<> B(Func.getContext()); 2327 AggrConsts.clear(); 2328 AggrConstTypes.clear(); 2329 AggrStores.clear(); 2330 2331 processParamTypesByFunHeader(CurrF, B); 2332 2333 // StoreInst's operand type can be changed during the next transformations, 2334 // so we need to store it in the set. Also store already transformed types. 2335 for (auto &I : instructions(Func)) { 2336 StoreInst *SI = dyn_cast<StoreInst>(&I); 2337 if (!SI) 2338 continue; 2339 Type *ElTy = SI->getValueOperand()->getType(); 2340 if (ElTy->isAggregateType() || ElTy->isVectorTy()) 2341 AggrStores.insert(&I); 2342 } 2343 2344 B.SetInsertPoint(&Func.getEntryBlock(), Func.getEntryBlock().begin()); 2345 for (auto &GV : Func.getParent()->globals()) 2346 processGlobalValue(GV, B); 2347 2348 preprocessUndefs(B); 2349 preprocessCompositeConstants(B); 2350 SmallVector<Instruction *> Worklist; 2351 for (auto &I : instructions(Func)) 2352 Worklist.push_back(&I); 2353 2354 applyDemangledPtrArgTypes(B); 2355 2356 // Pass forward: use operand to deduce instructions result. 2357 for (auto &I : Worklist) { 2358 // Don't emit intrinsincs for convergence intrinsics. 2359 if (isConvergenceIntrinsic(I)) 2360 continue; 2361 2362 bool Postpone = insertAssignPtrTypeIntrs(I, B, false); 2363 // if Postpone is true, we can't decide on pointee type yet 2364 insertAssignTypeIntrs(I, B); 2365 insertPtrCastOrAssignTypeInstr(I, B); 2366 insertSpirvDecorations(I, B); 2367 // if instruction requires a pointee type set, let's check if we know it 2368 // already, and force it to be i8 if not 2369 if (Postpone && !GR->findAssignPtrTypeInstr(I)) 2370 insertAssignPtrTypeIntrs(I, B, true); 2371 2372 if (auto *FPI = dyn_cast<ConstrainedFPIntrinsic>(I)) 2373 useRoundingMode(FPI, B); 2374 } 2375 2376 // Pass backward: use instructions results to specify/update/cast operands 2377 // where needed. 2378 SmallPtrSet<Instruction *, 4> UncompleteRets; 2379 for (auto &I : llvm::reverse(instructions(Func))) 2380 deduceOperandElementType(&I, &UncompleteRets); 2381 2382 // Pass forward for PHIs only, their operands are not preceed the instruction 2383 // in meaning of `instructions(Func)`. 2384 for (BasicBlock &BB : Func) 2385 for (PHINode &Phi : BB.phis()) 2386 if (isPointerTy(Phi.getType())) 2387 deduceOperandElementType(&Phi, nullptr); 2388 2389 for (auto *I : Worklist) { 2390 TrackConstants = true; 2391 if (!I->getType()->isVoidTy() || isa<StoreInst>(I)) 2392 setInsertPointAfterDef(B, I); 2393 // Visitors return either the original/newly created instruction for further 2394 // processing, nullptr otherwise. 2395 I = visit(*I); 2396 if (!I) 2397 continue; 2398 2399 // Don't emit intrinsics for convergence operations. 2400 if (isConvergenceIntrinsic(I)) 2401 continue; 2402 2403 processInstrAfterVisit(I, B); 2404 } 2405 2406 return true; 2407 } 2408 2409 // Try to deduce a better type for pointers to untyped ptr. 2410 bool SPIRVEmitIntrinsics::postprocessTypes(Module &M) { 2411 if (!GR || TodoTypeSz == 0) 2412 return false; 2413 2414 unsigned SzTodo = TodoTypeSz; 2415 DenseMap<Value *, SmallPtrSet<Value *, 4>> ToProcess; 2416 for (auto [Op, Enabled] : TodoType) { 2417 // TODO: add isa<CallInst>(Op) to continue 2418 if (!Enabled || isa<GetElementPtrInst>(Op)) 2419 continue; 2420 CallInst *AssignCI = GR->findAssignPtrTypeInstr(Op); 2421 Type *KnownTy = GR->findDeducedElementType(Op); 2422 if (!KnownTy || !AssignCI) 2423 continue; 2424 assert(Op == AssignCI->getArgOperand(0)); 2425 // Try to improve the type deduced after all Functions are processed. 2426 if (auto *CI = dyn_cast<Instruction>(Op)) { 2427 CurrF = CI->getParent()->getParent(); 2428 std::unordered_set<Value *> Visited; 2429 if (Type *ElemTy = deduceElementTypeHelper(Op, Visited, false, true)) { 2430 if (ElemTy != KnownTy) { 2431 DenseSet<std::pair<Value *, Value *>> VisitedSubst; 2432 propagateElemType(CI, ElemTy, VisitedSubst); 2433 eraseTodoType(Op); 2434 continue; 2435 } 2436 } 2437 } 2438 for (User *U : Op->users()) { 2439 Instruction *Inst = dyn_cast<Instruction>(U); 2440 if (Inst && !isa<IntrinsicInst>(Inst)) 2441 ToProcess[Inst].insert(Op); 2442 } 2443 } 2444 if (TodoTypeSz == 0) 2445 return true; 2446 2447 for (auto &F : M) { 2448 CurrF = &F; 2449 SmallPtrSet<Instruction *, 4> UncompleteRets; 2450 for (auto &I : llvm::reverse(instructions(F))) { 2451 auto It = ToProcess.find(&I); 2452 if (It == ToProcess.end()) 2453 continue; 2454 It->second.remove_if([this](Value *V) { return !isTodoType(V); }); 2455 if (It->second.size() == 0) 2456 continue; 2457 deduceOperandElementType(&I, &UncompleteRets, &It->second, true); 2458 if (TodoTypeSz == 0) 2459 return true; 2460 } 2461 } 2462 2463 return SzTodo > TodoTypeSz; 2464 } 2465 2466 // Parse and store argument types of function declarations where needed. 2467 void SPIRVEmitIntrinsics::parseFunDeclarations(Module &M) { 2468 for (auto &F : M) { 2469 if (!F.isDeclaration() || F.isIntrinsic()) 2470 continue; 2471 // get the demangled name 2472 std::string DemangledName = getOclOrSpirvBuiltinDemangledName(F.getName()); 2473 if (DemangledName.empty()) 2474 continue; 2475 // allow only OpGroupAsyncCopy use case at the moment 2476 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F); 2477 auto [Grp, Opcode, ExtNo] = SPIRV::mapBuiltinToOpcode( 2478 DemangledName, ST.getPreferredInstructionSet()); 2479 if (Opcode != SPIRV::OpGroupAsyncCopy) 2480 continue; 2481 // find pointer arguments 2482 SmallVector<unsigned> Idxs; 2483 for (unsigned OpIdx = 0; OpIdx < F.arg_size(); ++OpIdx) { 2484 Argument *Arg = F.getArg(OpIdx); 2485 if (isPointerTy(Arg->getType()) && !hasPointeeTypeAttr(Arg)) 2486 Idxs.push_back(OpIdx); 2487 } 2488 if (!Idxs.size()) 2489 continue; 2490 // parse function arguments 2491 LLVMContext &Ctx = F.getContext(); 2492 SmallVector<StringRef, 10> TypeStrs; 2493 SPIRV::parseBuiltinTypeStr(TypeStrs, DemangledName, Ctx); 2494 if (!TypeStrs.size()) 2495 continue; 2496 // find type info for pointer arguments 2497 for (unsigned Idx : Idxs) { 2498 if (Idx >= TypeStrs.size()) 2499 continue; 2500 if (Type *ElemTy = 2501 SPIRV::parseBuiltinCallArgumentType(TypeStrs[Idx].trim(), Ctx)) 2502 if (TypedPointerType::isValidElementType(ElemTy) && 2503 !ElemTy->isTargetExtTy()) 2504 FDeclPtrTys[&F].push_back(std::make_pair(Idx, ElemTy)); 2505 } 2506 } 2507 } 2508 2509 bool SPIRVEmitIntrinsics::runOnModule(Module &M) { 2510 bool Changed = false; 2511 2512 parseFunDeclarations(M); 2513 2514 TodoType.clear(); 2515 for (auto &F : M) 2516 Changed |= runOnFunction(F); 2517 2518 // Specify function parameters after all functions were processed. 2519 for (auto &F : M) { 2520 // check if function parameter types are set 2521 CurrF = &F; 2522 if (!F.isDeclaration() && !F.isIntrinsic()) { 2523 IRBuilder<> B(F.getContext()); 2524 processParamTypes(&F, B); 2525 } 2526 } 2527 2528 CanTodoType = false; 2529 Changed |= postprocessTypes(M); 2530 2531 if (HaveFunPtrs) 2532 Changed |= processFunctionPointers(M); 2533 2534 return Changed; 2535 } 2536 2537 ModulePass *llvm::createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM) { 2538 return new SPIRVEmitIntrinsics(TM); 2539 } 2540