1 //===-- AMDGPUPromoteAlloca.cpp - Promote Allocas -------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass eliminates allocas by either converting them into vectors or
11 // by migrating them to local address space.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "AMDGPU.h"
16 #include "AMDGPUSubtarget.h"
17 #include "llvm/Analysis/ValueTracking.h"
18 #include "llvm/IR/IRBuilder.h"
19 #include "llvm/IR/InstVisitor.h"
20 #include "llvm/Support/Debug.h"
21
22 #define DEBUG_TYPE "amdgpu-promote-alloca"
23
24 using namespace llvm;
25
26 namespace {
27
28 class AMDGPUPromoteAlloca : public FunctionPass,
29 public InstVisitor<AMDGPUPromoteAlloca> {
30
31 static char ID;
32 Module *Mod;
33 const AMDGPUSubtarget &ST;
34 int LocalMemAvailable;
35
36 public:
AMDGPUPromoteAlloca(const AMDGPUSubtarget & st)37 AMDGPUPromoteAlloca(const AMDGPUSubtarget &st) : FunctionPass(ID), ST(st),
38 LocalMemAvailable(0) { }
39 bool doInitialization(Module &M) override;
40 bool runOnFunction(Function &F) override;
getPassName() const41 const char *getPassName() const override { return "AMDGPU Promote Alloca"; }
42 void visitAlloca(AllocaInst &I);
43 };
44
45 } // End anonymous namespace
46
47 char AMDGPUPromoteAlloca::ID = 0;
48
doInitialization(Module & M)49 bool AMDGPUPromoteAlloca::doInitialization(Module &M) {
50 Mod = &M;
51 return false;
52 }
53
runOnFunction(Function & F)54 bool AMDGPUPromoteAlloca::runOnFunction(Function &F) {
55
56 const FunctionType *FTy = F.getFunctionType();
57
58 LocalMemAvailable = ST.getLocalMemorySize();
59
60
61 // If the function has any arguments in the local address space, then it's
62 // possible these arguments require the entire local memory space, so
63 // we cannot use local memory in the pass.
64 for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) {
65 const Type *ParamTy = FTy->getParamType(i);
66 if (ParamTy->isPointerTy() &&
67 ParamTy->getPointerAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
68 LocalMemAvailable = 0;
69 DEBUG(dbgs() << "Function has local memory argument. Promoting to "
70 "local memory disabled.\n");
71 break;
72 }
73 }
74
75 if (LocalMemAvailable > 0) {
76 // Check how much local memory is being used by global objects
77 for (Module::global_iterator I = Mod->global_begin(),
78 E = Mod->global_end(); I != E; ++I) {
79 GlobalVariable *GV = I;
80 PointerType *GVTy = GV->getType();
81 if (GVTy->getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS)
82 continue;
83 for (Value::use_iterator U = GV->use_begin(),
84 UE = GV->use_end(); U != UE; ++U) {
85 Instruction *Use = dyn_cast<Instruction>(*U);
86 if (!Use)
87 continue;
88 if (Use->getParent()->getParent() == &F)
89 LocalMemAvailable -=
90 Mod->getDataLayout()->getTypeAllocSize(GVTy->getElementType());
91 }
92 }
93 }
94
95 LocalMemAvailable = std::max(0, LocalMemAvailable);
96 DEBUG(dbgs() << LocalMemAvailable << "bytes free in local memory.\n");
97
98 visit(F);
99
100 return false;
101 }
102
arrayTypeToVecType(const Type * ArrayTy)103 static VectorType *arrayTypeToVecType(const Type *ArrayTy) {
104 return VectorType::get(ArrayTy->getArrayElementType(),
105 ArrayTy->getArrayNumElements());
106 }
107
108 static Value *
calculateVectorIndex(Value * Ptr,const std::map<GetElementPtrInst *,Value * > & GEPIdx)109 calculateVectorIndex(Value *Ptr,
110 const std::map<GetElementPtrInst *, Value *> &GEPIdx) {
111 if (isa<AllocaInst>(Ptr))
112 return Constant::getNullValue(Type::getInt32Ty(Ptr->getContext()));
113
114 GetElementPtrInst *GEP = cast<GetElementPtrInst>(Ptr);
115
116 auto I = GEPIdx.find(GEP);
117 return I == GEPIdx.end() ? nullptr : I->second;
118 }
119
GEPToVectorIndex(GetElementPtrInst * GEP)120 static Value* GEPToVectorIndex(GetElementPtrInst *GEP) {
121 // FIXME we only support simple cases
122 if (GEP->getNumOperands() != 3)
123 return NULL;
124
125 ConstantInt *I0 = dyn_cast<ConstantInt>(GEP->getOperand(1));
126 if (!I0 || !I0->isZero())
127 return NULL;
128
129 return GEP->getOperand(2);
130 }
131
132 // Not an instruction handled below to turn into a vector.
133 //
134 // TODO: Check isTriviallyVectorizable for calls and handle other
135 // instructions.
canVectorizeInst(Instruction * Inst)136 static bool canVectorizeInst(Instruction *Inst) {
137 switch (Inst->getOpcode()) {
138 case Instruction::Load:
139 case Instruction::Store:
140 case Instruction::BitCast:
141 case Instruction::AddrSpaceCast:
142 return true;
143 default:
144 return false;
145 }
146 }
147
tryPromoteAllocaToVector(AllocaInst * Alloca)148 static bool tryPromoteAllocaToVector(AllocaInst *Alloca) {
149 Type *AllocaTy = Alloca->getAllocatedType();
150
151 DEBUG(dbgs() << "Alloca Candidate for vectorization \n");
152
153 // FIXME: There is no reason why we can't support larger arrays, we
154 // are just being conservative for now.
155 if (!AllocaTy->isArrayTy() ||
156 AllocaTy->getArrayElementType()->isVectorTy() ||
157 AllocaTy->getArrayNumElements() > 4) {
158
159 DEBUG(dbgs() << " Cannot convert type to vector");
160 return false;
161 }
162
163 std::map<GetElementPtrInst*, Value*> GEPVectorIdx;
164 std::vector<Value*> WorkList;
165 for (User *AllocaUser : Alloca->users()) {
166 GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(AllocaUser);
167 if (!GEP) {
168 if (!canVectorizeInst(cast<Instruction>(AllocaUser)))
169 return false;
170
171 WorkList.push_back(AllocaUser);
172 continue;
173 }
174
175 Value *Index = GEPToVectorIndex(GEP);
176
177 // If we can't compute a vector index from this GEP, then we can't
178 // promote this alloca to vector.
179 if (!Index) {
180 DEBUG(dbgs() << " Cannot compute vector index for GEP " << *GEP << '\n');
181 return false;
182 }
183
184 GEPVectorIdx[GEP] = Index;
185 for (User *GEPUser : AllocaUser->users()) {
186 if (!canVectorizeInst(cast<Instruction>(GEPUser)))
187 return false;
188
189 WorkList.push_back(GEPUser);
190 }
191 }
192
193 VectorType *VectorTy = arrayTypeToVecType(AllocaTy);
194
195 DEBUG(dbgs() << " Converting alloca to vector "
196 << *AllocaTy << " -> " << *VectorTy << '\n');
197
198 for (std::vector<Value*>::iterator I = WorkList.begin(),
199 E = WorkList.end(); I != E; ++I) {
200 Instruction *Inst = cast<Instruction>(*I);
201 IRBuilder<> Builder(Inst);
202 switch (Inst->getOpcode()) {
203 case Instruction::Load: {
204 Value *Ptr = Inst->getOperand(0);
205 Value *Index = calculateVectorIndex(Ptr, GEPVectorIdx);
206 Value *BitCast = Builder.CreateBitCast(Alloca, VectorTy->getPointerTo(0));
207 Value *VecValue = Builder.CreateLoad(BitCast);
208 Value *ExtractElement = Builder.CreateExtractElement(VecValue, Index);
209 Inst->replaceAllUsesWith(ExtractElement);
210 Inst->eraseFromParent();
211 break;
212 }
213 case Instruction::Store: {
214 Value *Ptr = Inst->getOperand(1);
215 Value *Index = calculateVectorIndex(Ptr, GEPVectorIdx);
216 Value *BitCast = Builder.CreateBitCast(Alloca, VectorTy->getPointerTo(0));
217 Value *VecValue = Builder.CreateLoad(BitCast);
218 Value *NewVecValue = Builder.CreateInsertElement(VecValue,
219 Inst->getOperand(0),
220 Index);
221 Builder.CreateStore(NewVecValue, BitCast);
222 Inst->eraseFromParent();
223 break;
224 }
225 case Instruction::BitCast:
226 case Instruction::AddrSpaceCast:
227 break;
228
229 default:
230 Inst->dump();
231 llvm_unreachable("Inconsistency in instructions promotable to vector");
232 }
233 }
234 return true;
235 }
236
collectUsesWithPtrTypes(Value * Val,std::vector<Value * > & WorkList)237 static bool collectUsesWithPtrTypes(Value *Val, std::vector<Value*> &WorkList) {
238 bool Success = true;
239 for (User *User : Val->users()) {
240 if(std::find(WorkList.begin(), WorkList.end(), User) != WorkList.end())
241 continue;
242 if (isa<CallInst>(User)) {
243 WorkList.push_back(User);
244 continue;
245 }
246
247 // FIXME: Correctly handle ptrtoint instructions.
248 Instruction *UseInst = dyn_cast<Instruction>(User);
249 if (UseInst && UseInst->getOpcode() == Instruction::PtrToInt)
250 return false;
251
252 if (!User->getType()->isPointerTy())
253 continue;
254
255 WorkList.push_back(User);
256
257 Success &= collectUsesWithPtrTypes(User, WorkList);
258 }
259 return Success;
260 }
261
visitAlloca(AllocaInst & I)262 void AMDGPUPromoteAlloca::visitAlloca(AllocaInst &I) {
263 IRBuilder<> Builder(&I);
264
265 // First try to replace the alloca with a vector
266 Type *AllocaTy = I.getAllocatedType();
267
268 DEBUG(dbgs() << "Trying to promote " << I << '\n');
269
270 if (tryPromoteAllocaToVector(&I))
271 return;
272
273 DEBUG(dbgs() << " alloca is not a candidate for vectorization.\n");
274
275 // FIXME: This is the maximum work group size. We should try to get
276 // value from the reqd_work_group_size function attribute if it is
277 // available.
278 unsigned WorkGroupSize = 256;
279 int AllocaSize = WorkGroupSize *
280 Mod->getDataLayout()->getTypeAllocSize(AllocaTy);
281
282 if (AllocaSize > LocalMemAvailable) {
283 DEBUG(dbgs() << " Not enough local memory to promote alloca.\n");
284 return;
285 }
286
287 std::vector<Value*> WorkList;
288
289 if (!collectUsesWithPtrTypes(&I, WorkList)) {
290 DEBUG(dbgs() << " Do not know how to convert all uses\n");
291 return;
292 }
293
294 DEBUG(dbgs() << "Promoting alloca to local memory\n");
295 LocalMemAvailable -= AllocaSize;
296
297 GlobalVariable *GV = new GlobalVariable(
298 *Mod, ArrayType::get(I.getAllocatedType(), 256), false,
299 GlobalValue::ExternalLinkage, 0, I.getName(), 0,
300 GlobalVariable::NotThreadLocal, AMDGPUAS::LOCAL_ADDRESS);
301
302 FunctionType *FTy = FunctionType::get(
303 Type::getInt32Ty(Mod->getContext()), false);
304 AttributeSet AttrSet;
305 AttrSet.addAttribute(Mod->getContext(), 0, Attribute::ReadNone);
306
307 Value *ReadLocalSizeY = Mod->getOrInsertFunction(
308 "llvm.r600.read.local.size.y", FTy, AttrSet);
309 Value *ReadLocalSizeZ = Mod->getOrInsertFunction(
310 "llvm.r600.read.local.size.z", FTy, AttrSet);
311 Value *ReadTIDIGX = Mod->getOrInsertFunction(
312 "llvm.r600.read.tidig.x", FTy, AttrSet);
313 Value *ReadTIDIGY = Mod->getOrInsertFunction(
314 "llvm.r600.read.tidig.y", FTy, AttrSet);
315 Value *ReadTIDIGZ = Mod->getOrInsertFunction(
316 "llvm.r600.read.tidig.z", FTy, AttrSet);
317
318
319 Value *TCntY = Builder.CreateCall(ReadLocalSizeY);
320 Value *TCntZ = Builder.CreateCall(ReadLocalSizeZ);
321 Value *TIdX = Builder.CreateCall(ReadTIDIGX);
322 Value *TIdY = Builder.CreateCall(ReadTIDIGY);
323 Value *TIdZ = Builder.CreateCall(ReadTIDIGZ);
324
325 Value *Tmp0 = Builder.CreateMul(TCntY, TCntZ);
326 Tmp0 = Builder.CreateMul(Tmp0, TIdX);
327 Value *Tmp1 = Builder.CreateMul(TIdY, TCntZ);
328 Value *TID = Builder.CreateAdd(Tmp0, Tmp1);
329 TID = Builder.CreateAdd(TID, TIdZ);
330
331 std::vector<Value*> Indices;
332 Indices.push_back(Constant::getNullValue(Type::getInt32Ty(Mod->getContext())));
333 Indices.push_back(TID);
334
335 Value *Offset = Builder.CreateGEP(GV, Indices);
336 I.mutateType(Offset->getType());
337 I.replaceAllUsesWith(Offset);
338 I.eraseFromParent();
339
340 for (std::vector<Value*>::iterator i = WorkList.begin(),
341 e = WorkList.end(); i != e; ++i) {
342 Value *V = *i;
343 CallInst *Call = dyn_cast<CallInst>(V);
344 if (!Call) {
345 Type *EltTy = V->getType()->getPointerElementType();
346 PointerType *NewTy = PointerType::get(EltTy, AMDGPUAS::LOCAL_ADDRESS);
347
348 // The operand's value should be corrected on its own.
349 if (isa<AddrSpaceCastInst>(V))
350 continue;
351
352 // FIXME: It doesn't really make sense to try to do this for all
353 // instructions.
354 V->mutateType(NewTy);
355 continue;
356 }
357
358 IntrinsicInst *Intr = dyn_cast<IntrinsicInst>(Call);
359 if (!Intr) {
360 std::vector<Type*> ArgTypes;
361 for (unsigned ArgIdx = 0, ArgEnd = Call->getNumArgOperands();
362 ArgIdx != ArgEnd; ++ArgIdx) {
363 ArgTypes.push_back(Call->getArgOperand(ArgIdx)->getType());
364 }
365 Function *F = Call->getCalledFunction();
366 FunctionType *NewType = FunctionType::get(Call->getType(), ArgTypes,
367 F->isVarArg());
368 Constant *C = Mod->getOrInsertFunction(StringRef(F->getName().str() + ".local"), NewType,
369 F->getAttributes());
370 Function *NewF = cast<Function>(C);
371 Call->setCalledFunction(NewF);
372 continue;
373 }
374
375 Builder.SetInsertPoint(Intr);
376 switch (Intr->getIntrinsicID()) {
377 case Intrinsic::lifetime_start:
378 case Intrinsic::lifetime_end:
379 // These intrinsics are for address space 0 only
380 Intr->eraseFromParent();
381 continue;
382 case Intrinsic::memcpy: {
383 MemCpyInst *MemCpy = cast<MemCpyInst>(Intr);
384 Builder.CreateMemCpy(MemCpy->getRawDest(), MemCpy->getRawSource(),
385 MemCpy->getLength(), MemCpy->getAlignment(),
386 MemCpy->isVolatile());
387 Intr->eraseFromParent();
388 continue;
389 }
390 case Intrinsic::memset: {
391 MemSetInst *MemSet = cast<MemSetInst>(Intr);
392 Builder.CreateMemSet(MemSet->getRawDest(), MemSet->getValue(),
393 MemSet->getLength(), MemSet->getAlignment(),
394 MemSet->isVolatile());
395 Intr->eraseFromParent();
396 continue;
397 }
398 default:
399 Intr->dump();
400 llvm_unreachable("Don't know how to promote alloca intrinsic use.");
401 }
402 }
403 }
404
createAMDGPUPromoteAlloca(const AMDGPUSubtarget & ST)405 FunctionPass *llvm::createAMDGPUPromoteAlloca(const AMDGPUSubtarget &ST) {
406 return new AMDGPUPromoteAlloca(ST);
407 }
408