1 //===-- AMDGPULowerModuleLDSPass.cpp ------------------------------*- C++ -*-=// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass eliminates LDS uses from non-kernel functions. 10 // 11 // The strategy is to create a new struct with a field for each LDS variable 12 // and allocate that struct at the same address for every kernel. Uses of the 13 // original LDS variables are then replaced with compile time offsets from that 14 // known address. AMDGPUMachineFunction allocates the LDS global. 15 // 16 // Local variables with constant annotation or non-undef initializer are passed 17 // through unchanged for simplification or error diagnostics in later passes. 18 // 19 // To reduce the memory overhead variables that are only used by kernels are 20 // excluded from this transform. The analysis to determine whether a variable 21 // is only used by a kernel is cheap and conservative so this may allocate 22 // a variable in every kernel when it was not strictly necessary to do so. 23 // 24 // A possible future refinement is to specialise the structure per-kernel, so 25 // that fields can be elided based on more expensive analysis. 26 // 27 //===----------------------------------------------------------------------===// 28 29 #include "AMDGPU.h" 30 #include "Utils/AMDGPUBaseInfo.h" 31 #include "Utils/AMDGPUMemoryUtils.h" 32 #include "llvm/ADT/BitVector.h" 33 #include "llvm/ADT/DenseMap.h" 34 #include "llvm/ADT/STLExtras.h" 35 #include "llvm/ADT/SetVector.h" 36 #include "llvm/Analysis/CallGraph.h" 37 #include "llvm/IR/Constants.h" 38 #include "llvm/IR/DerivedTypes.h" 39 #include "llvm/IR/IRBuilder.h" 40 #include "llvm/IR/InlineAsm.h" 41 #include "llvm/IR/Instructions.h" 42 #include "llvm/IR/MDBuilder.h" 43 #include "llvm/InitializePasses.h" 44 #include "llvm/Pass.h" 45 #include "llvm/Support/CommandLine.h" 46 #include "llvm/Support/Debug.h" 47 #include "llvm/Support/OptimizedStructLayout.h" 48 #include "llvm/Transforms/Utils/ModuleUtils.h" 49 #include <tuple> 50 #include <vector> 51 52 #define DEBUG_TYPE "amdgpu-lower-module-lds" 53 54 using namespace llvm; 55 56 static cl::opt<bool> SuperAlignLDSGlobals( 57 "amdgpu-super-align-lds-globals", 58 cl::desc("Increase alignment of LDS if it is not on align boundary"), 59 cl::init(true), cl::Hidden); 60 61 namespace { 62 class AMDGPULowerModuleLDS : public ModulePass { 63 64 static void removeFromUsedList(Module &M, StringRef Name, 65 SmallPtrSetImpl<Constant *> &ToRemove) { 66 GlobalVariable *GV = M.getNamedGlobal(Name); 67 if (!GV || ToRemove.empty()) { 68 return; 69 } 70 71 SmallVector<Constant *, 16> Init; 72 auto *CA = cast<ConstantArray>(GV->getInitializer()); 73 for (auto &Op : CA->operands()) { 74 // ModuleUtils::appendToUsed only inserts Constants 75 Constant *C = cast<Constant>(Op); 76 if (!ToRemove.contains(C->stripPointerCasts())) { 77 Init.push_back(C); 78 } 79 } 80 81 if (Init.size() == CA->getNumOperands()) { 82 return; // none to remove 83 } 84 85 GV->eraseFromParent(); 86 87 for (Constant *C : ToRemove) { 88 C->removeDeadConstantUsers(); 89 } 90 91 if (!Init.empty()) { 92 ArrayType *ATy = 93 ArrayType::get(Type::getInt8PtrTy(M.getContext()), Init.size()); 94 GV = 95 new llvm::GlobalVariable(M, ATy, false, GlobalValue::AppendingLinkage, 96 ConstantArray::get(ATy, Init), Name); 97 GV->setSection("llvm.metadata"); 98 } 99 } 100 101 static void 102 removeFromUsedLists(Module &M, 103 const std::vector<GlobalVariable *> &LocalVars) { 104 // The verifier rejects used lists containing an inttoptr of a constant 105 // so remove the variables from these lists before replaceAllUsesWith 106 107 SmallPtrSet<Constant *, 32> LocalVarsSet; 108 for (GlobalVariable *LocalVar : LocalVars) 109 if (Constant *C = dyn_cast<Constant>(LocalVar->stripPointerCasts())) 110 LocalVarsSet.insert(C); 111 removeFromUsedList(M, "llvm.used", LocalVarsSet); 112 removeFromUsedList(M, "llvm.compiler.used", LocalVarsSet); 113 } 114 115 static void markUsedByKernel(IRBuilder<> &Builder, Function *Func, 116 GlobalVariable *SGV) { 117 // The llvm.amdgcn.module.lds instance is implicitly used by all kernels 118 // that might call a function which accesses a field within it. This is 119 // presently approximated to 'all kernels' if there are any such functions 120 // in the module. This implicit use is redefined as an explicit use here so 121 // that later passes, specifically PromoteAlloca, account for the required 122 // memory without any knowledge of this transform. 123 124 // An operand bundle on llvm.donothing works because the call instruction 125 // survives until after the last pass that needs to account for LDS. It is 126 // better than inline asm as the latter survives until the end of codegen. A 127 // totally robust solution would be a function with the same semantics as 128 // llvm.donothing that takes a pointer to the instance and is lowered to a 129 // no-op after LDS is allocated, but that is not presently necessary. 130 131 LLVMContext &Ctx = Func->getContext(); 132 133 Builder.SetInsertPoint(Func->getEntryBlock().getFirstNonPHI()); 134 135 FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx), {}); 136 137 Function *Decl = 138 Intrinsic::getDeclaration(Func->getParent(), Intrinsic::donothing, {}); 139 140 Value *UseInstance[1] = {Builder.CreateInBoundsGEP( 141 SGV->getValueType(), SGV, ConstantInt::get(Type::getInt32Ty(Ctx), 0))}; 142 143 Builder.CreateCall(FTy, Decl, {}, 144 {OperandBundleDefT<Value *>("ExplicitUse", UseInstance)}, 145 ""); 146 } 147 148 static bool eliminateConstantExprUsesOfLDSFromAllInstructions(Module &M) { 149 // Constants are uniqued within LLVM. A ConstantExpr referring to a LDS 150 // global may have uses from multiple different functions as a result. 151 // This pass specialises LDS variables with respect to the kernel that 152 // allocates them. 153 154 // This is semantically equivalent to: 155 // for (auto &F : M.functions()) 156 // for (auto &BB : F) 157 // for (auto &I : BB) 158 // for (Use &Op : I.operands()) 159 // if (constantExprUsesLDS(Op)) 160 // replaceConstantExprInFunction(I, Op); 161 162 bool Changed = false; 163 164 // Find all ConstantExpr that are direct users of an LDS global 165 SmallVector<ConstantExpr *> Stack; 166 for (auto &GV : M.globals()) 167 if (AMDGPU::isLDSVariableToLower(GV)) 168 for (User *U : GV.users()) 169 if (ConstantExpr *C = dyn_cast<ConstantExpr>(U)) 170 Stack.push_back(C); 171 172 // Expand to include constexpr users of direct users 173 SetVector<ConstantExpr *> ConstExprUsersOfLDS; 174 while (!Stack.empty()) { 175 ConstantExpr *V = Stack.pop_back_val(); 176 if (ConstExprUsersOfLDS.contains(V)) 177 continue; 178 179 ConstExprUsersOfLDS.insert(V); 180 181 for (auto *Nested : V->users()) 182 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Nested)) 183 Stack.push_back(CE); 184 } 185 186 // Find all instructions that use any of the ConstExpr users of LDS 187 SetVector<Instruction *> InstructionWorklist; 188 for (ConstantExpr *CE : ConstExprUsersOfLDS) 189 for (User *U : CE->users()) 190 if (auto *I = dyn_cast<Instruction>(U)) 191 InstructionWorklist.insert(I); 192 193 // Replace those ConstExpr operands with instructions 194 while (!InstructionWorklist.empty()) { 195 Instruction *I = InstructionWorklist.pop_back_val(); 196 for (Use &U : I->operands()) { 197 198 auto *BI = I; 199 if (auto *Phi = dyn_cast<PHINode>(I)) { 200 BasicBlock *BB = Phi->getIncomingBlock(U); 201 BasicBlock::iterator It = BB->getFirstInsertionPt(); 202 assert(It != BB->end() && "Unexpected empty basic block"); 203 BI = &(*(It)); 204 } 205 206 if (ConstantExpr *C = dyn_cast<ConstantExpr>(U.get())) { 207 if (ConstExprUsersOfLDS.contains(C)) { 208 Changed = true; 209 Instruction *NI = C->getAsInstruction(BI); 210 InstructionWorklist.insert(NI); 211 U.set(NI); 212 C->removeDeadConstantUsers(); 213 } 214 } 215 } 216 } 217 218 return Changed; 219 } 220 221 public: 222 static char ID; 223 224 AMDGPULowerModuleLDS() : ModulePass(ID) { 225 initializeAMDGPULowerModuleLDSPass(*PassRegistry::getPassRegistry()); 226 } 227 228 bool runOnModule(Module &M) override { 229 LLVMContext &Ctx = M.getContext(); 230 CallGraph CG = CallGraph(M); 231 bool Changed = superAlignLDSGlobals(M); 232 233 Changed |= eliminateConstantExprUsesOfLDSFromAllInstructions(M); 234 235 // Move variables used by functions into amdgcn.module.lds 236 std::vector<GlobalVariable *> ModuleScopeVariables = 237 AMDGPU::findLDSVariablesToLower(M, nullptr); 238 if (!ModuleScopeVariables.empty()) { 239 std::string VarName = "llvm.amdgcn.module.lds"; 240 241 GlobalVariable *SGV; 242 DenseMap<GlobalVariable *, Constant *> LDSVarsToConstantGEP; 243 std::tie(SGV, LDSVarsToConstantGEP) = 244 createLDSVariableReplacement(M, VarName, ModuleScopeVariables); 245 246 appendToCompilerUsed( 247 M, {static_cast<GlobalValue *>( 248 ConstantExpr::getPointerBitCastOrAddrSpaceCast( 249 cast<Constant>(SGV), Type::getInt8PtrTy(Ctx)))}); 250 251 removeFromUsedLists(M, ModuleScopeVariables); 252 replaceLDSVariablesWithStruct(M, ModuleScopeVariables, SGV, 253 LDSVarsToConstantGEP, 254 [](Use &) { return true; }); 255 256 // This ensures the variable is allocated when called functions access it. 257 // It also lets other passes, specifically PromoteAlloca, accurately 258 // calculate how much LDS will be used by the kernel after lowering. 259 260 IRBuilder<> Builder(Ctx); 261 for (Function &Func : M.functions()) { 262 if (!Func.isDeclaration() && AMDGPU::isKernelCC(&Func)) { 263 const CallGraphNode *N = CG[&Func]; 264 const bool CalleesRequireModuleLDS = N->size() > 0; 265 266 if (CalleesRequireModuleLDS) { 267 // If a function this kernel might call requires module LDS, 268 // annotate the kernel to let later passes know it will allocate 269 // this structure, even if not apparent from the IR. 270 markUsedByKernel(Builder, &Func, SGV); 271 } else { 272 // However if we are certain this kernel cannot call a function that 273 // requires module LDS, annotate the kernel so the backend can elide 274 // the allocation without repeating callgraph walks. 275 Func.addFnAttr("amdgpu-elide-module-lds"); 276 } 277 } 278 } 279 280 Changed = true; 281 } 282 283 // Move variables used by kernels into per-kernel instances 284 for (Function &F : M.functions()) { 285 if (F.isDeclaration()) 286 continue; 287 288 // Only lower compute kernels' LDS. 289 if (!AMDGPU::isKernel(F.getCallingConv())) 290 continue; 291 292 std::vector<GlobalVariable *> KernelUsedVariables = 293 AMDGPU::findLDSVariablesToLower(M, &F); 294 295 if (!KernelUsedVariables.empty()) { 296 // The association between kernel function and LDS struct is done by 297 // symbol name, which only works if the function in question has a name 298 // This is not expected to be a problem in practice as kernels are 299 // called by name making anonymous ones (which are named by the backend) 300 // difficult to use. This does mean that llvm test cases need 301 // to name the kernels. 302 if (!F.hasName()) { 303 report_fatal_error("Anonymous kernels cannot use LDS variables"); 304 } 305 306 std::string VarName = 307 (Twine("llvm.amdgcn.kernel.") + F.getName() + ".lds").str(); 308 GlobalVariable *SGV; 309 DenseMap<GlobalVariable *, Constant *> LDSVarsToConstantGEP; 310 std::tie(SGV, LDSVarsToConstantGEP) = 311 createLDSVariableReplacement(M, VarName, KernelUsedVariables); 312 313 removeFromUsedLists(M, KernelUsedVariables); 314 replaceLDSVariablesWithStruct( 315 M, KernelUsedVariables, SGV, LDSVarsToConstantGEP, [&F](Use &U) { 316 Instruction *I = dyn_cast<Instruction>(U.getUser()); 317 return I && I->getFunction() == &F; 318 }); 319 Changed = true; 320 } 321 } 322 323 for (auto &GV : make_early_inc_range(M.globals())) 324 if (AMDGPU::isLDSVariableToLower(GV)) { 325 GV.removeDeadConstantUsers(); 326 if (GV.use_empty()) 327 GV.eraseFromParent(); 328 } 329 330 return Changed; 331 } 332 333 private: 334 // Increase the alignment of LDS globals if necessary to maximise the chance 335 // that we can use aligned LDS instructions to access them. 336 static bool superAlignLDSGlobals(Module &M) { 337 const DataLayout &DL = M.getDataLayout(); 338 bool Changed = false; 339 if (!SuperAlignLDSGlobals) { 340 return Changed; 341 } 342 343 for (auto &GV : M.globals()) { 344 if (GV.getType()->getPointerAddressSpace() != AMDGPUAS::LOCAL_ADDRESS) { 345 // Only changing alignment of LDS variables 346 continue; 347 } 348 if (!GV.hasInitializer()) { 349 // cuda/hip extern __shared__ variable, leave alignment alone 350 continue; 351 } 352 353 Align Alignment = AMDGPU::getAlign(DL, &GV); 354 TypeSize GVSize = DL.getTypeAllocSize(GV.getValueType()); 355 356 if (GVSize > 8) { 357 // We might want to use a b96 or b128 load/store 358 Alignment = std::max(Alignment, Align(16)); 359 } else if (GVSize > 4) { 360 // We might want to use a b64 load/store 361 Alignment = std::max(Alignment, Align(8)); 362 } else if (GVSize > 2) { 363 // We might want to use a b32 load/store 364 Alignment = std::max(Alignment, Align(4)); 365 } else if (GVSize > 1) { 366 // We might want to use a b16 load/store 367 Alignment = std::max(Alignment, Align(2)); 368 } 369 370 if (Alignment != AMDGPU::getAlign(DL, &GV)) { 371 Changed = true; 372 GV.setAlignment(Alignment); 373 } 374 } 375 return Changed; 376 } 377 378 std::tuple<GlobalVariable *, DenseMap<GlobalVariable *, Constant *>> 379 createLDSVariableReplacement( 380 Module &M, std::string VarName, 381 std::vector<GlobalVariable *> const &LDSVarsToTransform) { 382 // Create a struct instance containing LDSVarsToTransform and map from those 383 // variables to ConstantExprGEP 384 // Variables may be introduced to meet alignment requirements. No aliasing 385 // metadata is useful for these as they have no uses. Erased before return. 386 387 LLVMContext &Ctx = M.getContext(); 388 const DataLayout &DL = M.getDataLayout(); 389 assert(!LDSVarsToTransform.empty()); 390 391 SmallVector<OptimizedStructLayoutField, 8> LayoutFields; 392 LayoutFields.reserve(LDSVarsToTransform.size()); 393 for (GlobalVariable *GV : LDSVarsToTransform) { 394 OptimizedStructLayoutField F(GV, DL.getTypeAllocSize(GV->getValueType()), 395 AMDGPU::getAlign(DL, GV)); 396 LayoutFields.emplace_back(F); 397 } 398 399 performOptimizedStructLayout(LayoutFields); 400 401 std::vector<GlobalVariable *> LocalVars; 402 BitVector IsPaddingField; 403 LocalVars.reserve(LDSVarsToTransform.size()); // will be at least this large 404 IsPaddingField.reserve(LDSVarsToTransform.size()); 405 { 406 uint64_t CurrentOffset = 0; 407 for (size_t I = 0; I < LayoutFields.size(); I++) { 408 GlobalVariable *FGV = static_cast<GlobalVariable *>( 409 const_cast<void *>(LayoutFields[I].Id)); 410 Align DataAlign = LayoutFields[I].Alignment; 411 412 uint64_t DataAlignV = DataAlign.value(); 413 if (uint64_t Rem = CurrentOffset % DataAlignV) { 414 uint64_t Padding = DataAlignV - Rem; 415 416 // Append an array of padding bytes to meet alignment requested 417 // Note (o + (a - (o % a)) ) % a == 0 418 // (offset + Padding ) % align == 0 419 420 Type *ATy = ArrayType::get(Type::getInt8Ty(Ctx), Padding); 421 LocalVars.push_back(new GlobalVariable( 422 M, ATy, false, GlobalValue::InternalLinkage, UndefValue::get(ATy), 423 "", nullptr, GlobalValue::NotThreadLocal, AMDGPUAS::LOCAL_ADDRESS, 424 false)); 425 IsPaddingField.push_back(true); 426 CurrentOffset += Padding; 427 } 428 429 LocalVars.push_back(FGV); 430 IsPaddingField.push_back(false); 431 CurrentOffset += LayoutFields[I].Size; 432 } 433 } 434 435 std::vector<Type *> LocalVarTypes; 436 LocalVarTypes.reserve(LocalVars.size()); 437 std::transform( 438 LocalVars.cbegin(), LocalVars.cend(), std::back_inserter(LocalVarTypes), 439 [](const GlobalVariable *V) -> Type * { return V->getValueType(); }); 440 441 StructType *LDSTy = StructType::create(Ctx, LocalVarTypes, VarName + ".t"); 442 443 Align StructAlign = AMDGPU::getAlign(DL, LocalVars[0]); 444 445 GlobalVariable *SGV = new GlobalVariable( 446 M, LDSTy, false, GlobalValue::InternalLinkage, UndefValue::get(LDSTy), 447 VarName, nullptr, GlobalValue::NotThreadLocal, AMDGPUAS::LOCAL_ADDRESS, 448 false); 449 SGV->setAlignment(StructAlign); 450 451 DenseMap<GlobalVariable *, Constant *> Map; 452 Type *I32 = Type::getInt32Ty(Ctx); 453 for (size_t I = 0; I < LocalVars.size(); I++) { 454 GlobalVariable *GV = LocalVars[I]; 455 Constant *GEPIdx[] = {ConstantInt::get(I32, 0), ConstantInt::get(I32, I)}; 456 Constant *GEP = ConstantExpr::getGetElementPtr(LDSTy, SGV, GEPIdx, true); 457 if (IsPaddingField[I]) { 458 assert(GV->use_empty()); 459 GV->eraseFromParent(); 460 } else { 461 Map[GV] = GEP; 462 } 463 } 464 assert(Map.size() == LDSVarsToTransform.size()); 465 return std::make_tuple(SGV, std::move(Map)); 466 } 467 468 template <typename PredicateTy> 469 void replaceLDSVariablesWithStruct( 470 Module &M, std::vector<GlobalVariable *> const &LDSVarsToTransform, 471 GlobalVariable *SGV, 472 DenseMap<GlobalVariable *, Constant *> &LDSVarsToConstantGEP, 473 PredicateTy Predicate) { 474 LLVMContext &Ctx = M.getContext(); 475 const DataLayout &DL = M.getDataLayout(); 476 477 // Create alias.scope and their lists. Each field in the new structure 478 // does not alias with all other fields. 479 SmallVector<MDNode *> AliasScopes; 480 SmallVector<Metadata *> NoAliasList; 481 const size_t NumberVars = LDSVarsToTransform.size(); 482 if (NumberVars > 1) { 483 MDBuilder MDB(Ctx); 484 AliasScopes.reserve(NumberVars); 485 MDNode *Domain = MDB.createAnonymousAliasScopeDomain(); 486 for (size_t I = 0; I < NumberVars; I++) { 487 MDNode *Scope = MDB.createAnonymousAliasScope(Domain); 488 AliasScopes.push_back(Scope); 489 } 490 NoAliasList.append(&AliasScopes[1], AliasScopes.end()); 491 } 492 493 // Replace uses of ith variable with a constantexpr to the corresponding 494 // field of the instance that will be allocated by AMDGPUMachineFunction 495 for (size_t I = 0; I < NumberVars; I++) { 496 GlobalVariable *GV = LDSVarsToTransform[I]; 497 Constant *GEP = LDSVarsToConstantGEP[GV]; 498 499 GV->replaceUsesWithIf(GEP, Predicate); 500 if (GV->use_empty()) { 501 GV->eraseFromParent(); 502 } 503 504 APInt APOff(DL.getIndexTypeSizeInBits(GEP->getType()), 0); 505 GEP->stripAndAccumulateInBoundsConstantOffsets(DL, APOff); 506 uint64_t Offset = APOff.getZExtValue(); 507 508 Align A = commonAlignment(SGV->getAlign().valueOrOne(), Offset); 509 510 if (I) 511 NoAliasList[I - 1] = AliasScopes[I - 1]; 512 MDNode *NoAlias = 513 NoAliasList.empty() ? nullptr : MDNode::get(Ctx, NoAliasList); 514 MDNode *AliasScope = 515 AliasScopes.empty() ? nullptr : MDNode::get(Ctx, {AliasScopes[I]}); 516 517 refineUsesAlignmentAndAA(GEP, A, DL, AliasScope, NoAlias); 518 } 519 } 520 521 void refineUsesAlignmentAndAA(Value *Ptr, Align A, const DataLayout &DL, 522 MDNode *AliasScope, MDNode *NoAlias, 523 unsigned MaxDepth = 5) { 524 if (!MaxDepth || (A == 1 && !AliasScope)) 525 return; 526 527 for (User *U : Ptr->users()) { 528 if (auto *I = dyn_cast<Instruction>(U)) { 529 if (AliasScope && I->mayReadOrWriteMemory()) { 530 MDNode *AS = I->getMetadata(LLVMContext::MD_alias_scope); 531 AS = (AS ? MDNode::getMostGenericAliasScope(AS, AliasScope) 532 : AliasScope); 533 I->setMetadata(LLVMContext::MD_alias_scope, AS); 534 535 MDNode *NA = I->getMetadata(LLVMContext::MD_noalias); 536 NA = (NA ? MDNode::intersect(NA, NoAlias) : NoAlias); 537 I->setMetadata(LLVMContext::MD_noalias, NA); 538 } 539 } 540 541 if (auto *LI = dyn_cast<LoadInst>(U)) { 542 LI->setAlignment(std::max(A, LI->getAlign())); 543 continue; 544 } 545 if (auto *SI = dyn_cast<StoreInst>(U)) { 546 if (SI->getPointerOperand() == Ptr) 547 SI->setAlignment(std::max(A, SI->getAlign())); 548 continue; 549 } 550 if (auto *AI = dyn_cast<AtomicRMWInst>(U)) { 551 // None of atomicrmw operations can work on pointers, but let's 552 // check it anyway in case it will or we will process ConstantExpr. 553 if (AI->getPointerOperand() == Ptr) 554 AI->setAlignment(std::max(A, AI->getAlign())); 555 continue; 556 } 557 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(U)) { 558 if (AI->getPointerOperand() == Ptr) 559 AI->setAlignment(std::max(A, AI->getAlign())); 560 continue; 561 } 562 if (auto *GEP = dyn_cast<GetElementPtrInst>(U)) { 563 unsigned BitWidth = DL.getIndexTypeSizeInBits(GEP->getType()); 564 APInt Off(BitWidth, 0); 565 if (GEP->getPointerOperand() == Ptr) { 566 Align GA; 567 if (GEP->accumulateConstantOffset(DL, Off)) 568 GA = commonAlignment(A, Off.getLimitedValue()); 569 refineUsesAlignmentAndAA(GEP, GA, DL, AliasScope, NoAlias, 570 MaxDepth - 1); 571 } 572 continue; 573 } 574 if (auto *I = dyn_cast<Instruction>(U)) { 575 if (I->getOpcode() == Instruction::BitCast || 576 I->getOpcode() == Instruction::AddrSpaceCast) 577 refineUsesAlignmentAndAA(I, A, DL, AliasScope, NoAlias, MaxDepth - 1); 578 } 579 } 580 } 581 }; 582 583 } // namespace 584 char AMDGPULowerModuleLDS::ID = 0; 585 586 char &llvm::AMDGPULowerModuleLDSID = AMDGPULowerModuleLDS::ID; 587 588 INITIALIZE_PASS(AMDGPULowerModuleLDS, DEBUG_TYPE, 589 "Lower uses of LDS variables from non-kernel functions", false, 590 false) 591 592 ModulePass *llvm::createAMDGPULowerModuleLDSPass() { 593 return new AMDGPULowerModuleLDS(); 594 } 595 596 PreservedAnalyses AMDGPULowerModuleLDSPass::run(Module &M, 597 ModuleAnalysisManager &) { 598 return AMDGPULowerModuleLDS().runOnModule(M) ? PreservedAnalyses::none() 599 : PreservedAnalyses::all(); 600 } 601