1 //===- AMDGPUOpenCLEnqueuedBlockLowering.cpp - Lower enqueued block -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // \file
10 // This post-linking pass replaces the function pointer of enqueued
11 // block kernel with a global variable (runtime handle) and adds
12 // "runtime-handle" attribute to the enqueued block kernel.
13 //
14 // In LLVM CodeGen the runtime-handle metadata will be translated to
15 // RuntimeHandle metadata in code object. Runtime allocates a global buffer
16 // for each kernel with RuntimeHandle metadata and saves the kernel address
17 // required for the AQL packet into the buffer. __enqueue_kernel function
18 // in device library knows that the invoke function pointer in the block
19 // literal is actually runtime handle and loads the kernel address from it
20 // and put it into AQL packet for dispatching.
21 //
22 // This cannot be done in FE since FE cannot create a unique global variable
23 // with external linkage across LLVM modules. The global variable with internal
24 // linkage does not work since optimization passes will try to replace loads
25 // of the global variable with its initialization value.
26 //
27 // It also identifies the kernels directly or indirectly enqueues kernels
28 // and adds "calls-enqueue-kernel" function attribute to them, which will
29 // be used to determine whether to emit runtime metadata for the kernel
30 // enqueue related hidden kernel arguments.
31 //
32 //===----------------------------------------------------------------------===//
33
34 #include "AMDGPU.h"
35 #include "llvm/ADT/DenseSet.h"
36 #include "llvm/ADT/SmallString.h"
37 #include "llvm/IR/Constants.h"
38 #include "llvm/IR/Instructions.h"
39 #include "llvm/IR/Mangler.h"
40 #include "llvm/IR/Module.h"
41 #include "llvm/Pass.h"
42 #include "llvm/Support/Debug.h"
43
44 #define DEBUG_TYPE "amdgpu-lower-enqueued-block"
45
46 using namespace llvm;
47
48 namespace {
49
50 /// Lower enqueued blocks.
51 class AMDGPUOpenCLEnqueuedBlockLowering : public ModulePass {
52 public:
53 static char ID;
54
AMDGPUOpenCLEnqueuedBlockLowering()55 explicit AMDGPUOpenCLEnqueuedBlockLowering() : ModulePass(ID) {}
56
57 private:
58 bool runOnModule(Module &M) override;
59 };
60
61 } // end anonymous namespace
62
63 char AMDGPUOpenCLEnqueuedBlockLowering::ID = 0;
64
65 char &llvm::AMDGPUOpenCLEnqueuedBlockLoweringID =
66 AMDGPUOpenCLEnqueuedBlockLowering::ID;
67
68 INITIALIZE_PASS(AMDGPUOpenCLEnqueuedBlockLowering, DEBUG_TYPE,
69 "Lower OpenCL enqueued blocks", false, false)
70
createAMDGPUOpenCLEnqueuedBlockLoweringPass()71 ModulePass* llvm::createAMDGPUOpenCLEnqueuedBlockLoweringPass() {
72 return new AMDGPUOpenCLEnqueuedBlockLowering();
73 }
74
75 /// Collect direct or indirect callers of \p F and save them
76 /// to \p Callers.
collectCallers(Function * F,DenseSet<Function * > & Callers)77 static void collectCallers(Function *F, DenseSet<Function *> &Callers) {
78 for (auto *U : F->users()) {
79 if (auto *CI = dyn_cast<CallInst>(&*U)) {
80 auto *Caller = CI->getParent()->getParent();
81 if (Callers.insert(Caller).second)
82 collectCallers(Caller, Callers);
83 }
84 }
85 }
86
87 /// If \p U is instruction or constant, collect functions which directly or
88 /// indirectly use it.
collectFunctionUsers(User * U,DenseSet<Function * > & Funcs)89 static void collectFunctionUsers(User *U, DenseSet<Function *> &Funcs) {
90 if (auto *I = dyn_cast<Instruction>(U)) {
91 auto *F = I->getParent()->getParent();
92 if (Funcs.insert(F).second)
93 collectCallers(F, Funcs);
94 return;
95 }
96 for (User *U : U->users())
97 collectFunctionUsers(U, Funcs);
98 }
99
runOnModule(Module & M)100 bool AMDGPUOpenCLEnqueuedBlockLowering::runOnModule(Module &M) {
101 DenseSet<Function *> Callers;
102 auto &C = M.getContext();
103 bool Changed = false;
104
105 // ptr kernel_object, i32 private_segment_size, i32 group_segment_size
106 StructType *HandleTy = nullptr;
107
108 for (auto &F : M.functions()) {
109 if (F.hasFnAttribute("enqueued-block")) {
110 if (!F.hasName()) {
111 SmallString<64> Name;
112 Mangler::getNameWithPrefix(Name, "__amdgpu_enqueued_kernel",
113 M.getDataLayout());
114 F.setName(Name);
115 }
116 LLVM_DEBUG(dbgs() << "found enqueued kernel: " << F.getName() << '\n');
117 auto RuntimeHandle = (F.getName() + ".runtime_handle").str();
118 if (!HandleTy) {
119 Type *Int32 = Type::getInt32Ty(C);
120 HandleTy = StructType::create(
121 C, {Type::getInt8Ty(C)->getPointerTo(0), Int32, Int32},
122 "block.runtime.handle.t");
123 }
124
125 auto *GV = new GlobalVariable(
126 M, HandleTy,
127 /*isConstant=*/true, GlobalValue::ExternalLinkage,
128 /*Initializer=*/Constant::getNullValue(HandleTy), RuntimeHandle,
129 /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
130 AMDGPUAS::GLOBAL_ADDRESS,
131 /*isExternallyInitialized=*/true);
132 LLVM_DEBUG(dbgs() << "runtime handle created: " << *GV << '\n');
133
134 for (User *U : F.users())
135 collectFunctionUsers(U, Callers);
136
137 F.replaceAllUsesWith(ConstantExpr::getAddrSpaceCast(GV, F.getType()));
138 F.addFnAttr("runtime-handle", RuntimeHandle);
139 F.setLinkage(GlobalValue::ExternalLinkage);
140 Changed = true;
141 }
142 }
143
144 // FIXME: This call graph analysis is broken and should be
145 // removed. AMDGPUAttributor infers the individual implicit argument fields
146 // are needed or not, but the runtime crashes in cases where we fail to
147 // optimize these out at -O0.
148 for (auto *F : Callers) {
149 if (F->getCallingConv() != CallingConv::AMDGPU_KERNEL)
150 continue;
151 F->addFnAttr("calls-enqueue-kernel");
152 LLVM_DEBUG(dbgs() << "mark enqueue_kernel caller:" << F->getName() << '\n');
153 }
154 return Changed;
155 }
156