xref: /llvm-project/llvm/lib/Target/AMDGPU/AMDGPUAsanInstrumentation.cpp (revision 416f1c465db62d829283f6902ef35e027e127aa7)
1 //===AMDGPUAsanInstrumentation.cpp - ASAN related helper functions===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===-------------------------------------------------------------===//
8 
9 #include "AMDGPUAsanInstrumentation.h"
10 
11 #define DEBUG_TYPE "amdgpu-asan-instrumentation"
12 
13 using namespace llvm;
14 
15 namespace llvm {
16 namespace AMDGPU {
17 
18 static uint64_t getRedzoneSizeForScale(int AsanScale) {
19   // Redzone used for stack and globals is at least 32 bytes.
20   // For scales 6 and 7, the redzone has to be 64 and 128 bytes respectively.
21   return std::max(32U, 1U << AsanScale);
22 }
23 
24 static uint64_t getMinRedzoneSizeForGlobal(int AsanScale) {
25   return getRedzoneSizeForScale(AsanScale);
26 }
27 
28 uint64_t getRedzoneSizeForGlobal(int AsanScale, uint64_t SizeInBytes) {
29   constexpr uint64_t kMaxRZ = 1 << 18;
30   const uint64_t MinRZ = getMinRedzoneSizeForGlobal(AsanScale);
31 
32   uint64_t RZ = 0;
33   if (SizeInBytes <= MinRZ / 2) {
34     // Reduce redzone size for small size objects, e.g. int, char[1]. MinRZ is
35     // at least 32 bytes, optimize when SizeInBytes is less than or equal to
36     // half of MinRZ.
37     RZ = MinRZ - SizeInBytes;
38   } else {
39     // Calculate RZ, where MinRZ <= RZ <= MaxRZ, and RZ ~ 1/4 * SizeInBytes.
40     RZ = std::clamp((SizeInBytes / MinRZ / 4) * MinRZ, MinRZ, kMaxRZ);
41 
42     // Round up to multiple of MinRZ.
43     if (SizeInBytes % MinRZ)
44       RZ += MinRZ - (SizeInBytes % MinRZ);
45   }
46 
47   assert((RZ + SizeInBytes) % MinRZ == 0);
48 
49   return RZ;
50 }
51 
52 static size_t TypeStoreSizeToSizeIndex(uint32_t TypeSize) {
53   size_t Res = llvm::countr_zero(TypeSize / 8);
54   return Res;
55 }
56 
57 static Instruction *genAMDGPUReportBlock(Module &M, IRBuilder<> &IRB,
58                                          Value *Cond, bool Recover) {
59   Value *ReportCond = Cond;
60   if (!Recover) {
61     auto *Ballot =
62         IRB.CreateIntrinsic(Intrinsic::amdgcn_ballot, IRB.getInt64Ty(), {Cond});
63     ReportCond = IRB.CreateIsNotNull(Ballot);
64   }
65 
66   auto *Trm = SplitBlockAndInsertIfThen(
67       ReportCond, &*IRB.GetInsertPoint(), false,
68       MDBuilder(M.getContext()).createUnlikelyBranchWeights());
69   Trm->getParent()->setName("asan.report");
70 
71   if (Recover)
72     return Trm;
73 
74   Trm = SplitBlockAndInsertIfThen(Cond, Trm, false);
75   IRB.SetInsertPoint(Trm);
76   return IRB.CreateIntrinsic(Intrinsic::amdgcn_unreachable, {}, {});
77 }
78 
79 static Value *createSlowPathCmp(Module &M, IRBuilder<> &IRB, Type *IntptrTy,
80                                 Value *AddrLong, Value *ShadowValue,
81                                 uint32_t TypeStoreSize, int AsanScale) {
82   uint64_t Granularity = static_cast<uint64_t>(1) << AsanScale;
83   // Addr & (Granularity - 1)
84   Value *LastAccessedByte =
85       IRB.CreateAnd(AddrLong, ConstantInt::get(IntptrTy, Granularity - 1));
86   // (Addr & (Granularity - 1)) + size - 1
87   if (TypeStoreSize / 8 > 1)
88     LastAccessedByte = IRB.CreateAdd(
89         LastAccessedByte, ConstantInt::get(IntptrTy, TypeStoreSize / 8 - 1));
90   // (uint8_t) ((Addr & (Granularity-1)) + size - 1)
91   LastAccessedByte =
92       IRB.CreateIntCast(LastAccessedByte, ShadowValue->getType(), false);
93   // ((uint8_t) ((Addr & (Granularity-1)) + size - 1)) >= ShadowValue
94   return IRB.CreateICmpSGE(LastAccessedByte, ShadowValue);
95 }
96 
97 static Instruction *generateCrashCode(Module &M, IRBuilder<> &IRB,
98                                       Type *IntptrTy, Instruction *InsertBefore,
99                                       Value *Addr, bool IsWrite,
100                                       size_t AccessSizeIndex,
101                                       Value *SizeArgument, bool Recover) {
102   IRB.SetInsertPoint(InsertBefore);
103   CallInst *Call = nullptr;
104   SmallString<128> kAsanReportErrorTemplate{"__asan_report_"};
105   SmallString<64> TypeStr{IsWrite ? "store" : "load"};
106   SmallString<64> EndingStr{Recover ? "_noabort" : ""};
107 
108   SmallString<128> AsanErrorCallbackSizedString;
109   raw_svector_ostream AsanErrorCallbackSizedOS(AsanErrorCallbackSizedString);
110   AsanErrorCallbackSizedOS << kAsanReportErrorTemplate << TypeStr << "_n"
111                            << EndingStr;
112 
113   SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
114   AttributeList AL2;
115   FunctionCallee AsanErrorCallbackSized = M.getOrInsertFunction(
116       AsanErrorCallbackSizedOS.str(),
117       FunctionType::get(IRB.getVoidTy(), Args2, false), AL2);
118   SmallVector<Type *, 2> Args1{1, IntptrTy};
119   AttributeList AL1;
120 
121   SmallString<128> AsanErrorCallbackString;
122   raw_svector_ostream AsanErrorCallbackOS(AsanErrorCallbackString);
123   AsanErrorCallbackOS << kAsanReportErrorTemplate << TypeStr
124                       << (1ULL << AccessSizeIndex) << EndingStr;
125 
126   FunctionCallee AsanErrorCallback = M.getOrInsertFunction(
127       AsanErrorCallbackOS.str(),
128       FunctionType::get(IRB.getVoidTy(), Args1, false), AL1);
129   if (SizeArgument) {
130     Call = IRB.CreateCall(AsanErrorCallbackSized, {Addr, SizeArgument});
131   } else {
132     Call = IRB.CreateCall(AsanErrorCallback, Addr);
133   }
134 
135   Call->setCannotMerge();
136   return Call;
137 }
138 
139 static Value *memToShadow(Module &M, IRBuilder<> &IRB, Type *IntptrTy,
140                           Value *Shadow, int AsanScale, uint32_t AsanOffset) {
141   // Shadow >> scale
142   Shadow = IRB.CreateLShr(Shadow, AsanScale);
143   if (AsanOffset == 0)
144     return Shadow;
145   // (Shadow >> scale) | offset
146   Value *ShadowBase = ConstantInt::get(IntptrTy, AsanOffset);
147   return IRB.CreateAdd(Shadow, ShadowBase);
148 }
149 
150 static void instrumentAddressImpl(Module &M, IRBuilder<> &IRB,
151                                   Instruction *OrigIns,
152                                   Instruction *InsertBefore, Value *Addr,
153                                   Align Alignment, uint32_t TypeStoreSize,
154                                   bool IsWrite, Value *SizeArgument,
155                                   bool UseCalls, bool Recover, int AsanScale,
156                                   int AsanOffset) {
157   Type *AddrTy = Addr->getType();
158   Type *IntptrTy = M.getDataLayout().getIntPtrType(
159       M.getContext(), AddrTy->getPointerAddressSpace());
160   IRB.SetInsertPoint(InsertBefore);
161   size_t AccessSizeIndex = TypeStoreSizeToSizeIndex(TypeStoreSize);
162   Type *ShadowTy = IntegerType::get(M.getContext(),
163                                     std::max(8U, TypeStoreSize >> AsanScale));
164   Type *ShadowPtrTy = PointerType::get(M.getContext(), 0);
165   Value *AddrLong = IRB.CreatePtrToInt(Addr, IntptrTy);
166   Value *ShadowPtr =
167       memToShadow(M, IRB, IntptrTy, AddrLong, AsanScale, AsanOffset);
168   const uint64_t ShadowAlign =
169       std::max<uint64_t>(Alignment.value() >> AsanScale, 1);
170   Value *ShadowValue = IRB.CreateAlignedLoad(
171       ShadowTy, IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy), Align(ShadowAlign));
172   Value *Cmp = IRB.CreateIsNotNull(ShadowValue);
173   auto *Cmp2 = createSlowPathCmp(M, IRB, IntptrTy, AddrLong, ShadowValue,
174                                  TypeStoreSize, AsanScale);
175   Cmp = IRB.CreateAnd(Cmp, Cmp2);
176   Instruction *CrashTerm = genAMDGPUReportBlock(M, IRB, Cmp, Recover);
177   Instruction *Crash =
178       generateCrashCode(M, IRB, IntptrTy, CrashTerm, AddrLong, IsWrite,
179                         AccessSizeIndex, SizeArgument, Recover);
180   Crash->setDebugLoc(OrigIns->getDebugLoc());
181 }
182 
183 void instrumentAddress(Module &M, IRBuilder<> &IRB, Instruction *OrigIns,
184                        Instruction *InsertBefore, Value *Addr, Align Alignment,
185                        TypeSize TypeStoreSize, bool IsWrite,
186                        Value *SizeArgument, bool UseCalls, bool Recover,
187                        int AsanScale, int AsanOffset) {
188   if (!TypeStoreSize.isScalable()) {
189     unsigned Granularity = 1 << AsanScale;
190     const auto FixedSize = TypeStoreSize.getFixedValue();
191     switch (FixedSize) {
192     case 8:
193     case 16:
194     case 32:
195     case 64:
196     case 128:
197       if (Alignment.value() >= Granularity ||
198           Alignment.value() >= FixedSize / 8)
199         return instrumentAddressImpl(
200             M, IRB, OrigIns, InsertBefore, Addr, Alignment, FixedSize, IsWrite,
201             SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);
202     }
203   }
204   // Instrument unusual size or unusual alignment.
205   IRB.SetInsertPoint(InsertBefore);
206   Type *AddrTy = Addr->getType();
207   Type *IntptrTy = M.getDataLayout().getIntPtrType(AddrTy);
208   Value *NumBits = IRB.CreateTypeSize(IntptrTy, TypeStoreSize);
209   Value *Size = IRB.CreateLShr(NumBits, ConstantInt::get(IntptrTy, 3));
210   Value *AddrLong = IRB.CreatePtrToInt(Addr, IntptrTy);
211   Value *SizeMinusOne = IRB.CreateAdd(Size, ConstantInt::get(IntptrTy, -1));
212   Value *LastByte =
213       IRB.CreateIntToPtr(IRB.CreateAdd(AddrLong, SizeMinusOne), AddrTy);
214   instrumentAddressImpl(M, IRB, OrigIns, InsertBefore, Addr, {}, 8, IsWrite,
215                         SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);
216   instrumentAddressImpl(M, IRB, OrigIns, InsertBefore, LastByte, {}, 8, IsWrite,
217                         SizeArgument, UseCalls, Recover, AsanScale, AsanOffset);
218 }
219 
220 void getInterestingMemoryOperands(
221     Module &M, Instruction *I,
222     SmallVectorImpl<InterestingMemoryOperand> &Interesting) {
223   const DataLayout &DL = M.getDataLayout();
224   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
225     Interesting.emplace_back(I, LI->getPointerOperandIndex(), false,
226                              LI->getType(), LI->getAlign());
227   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
228     Interesting.emplace_back(I, SI->getPointerOperandIndex(), true,
229                              SI->getValueOperand()->getType(), SI->getAlign());
230   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
231     Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true,
232                              RMW->getValOperand()->getType(), std::nullopt);
233   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
234     Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true,
235                              XCHG->getCompareOperand()->getType(),
236                              std::nullopt);
237   } else if (auto *CI = dyn_cast<CallInst>(I)) {
238     switch (CI->getIntrinsicID()) {
239     case Intrinsic::masked_load:
240     case Intrinsic::masked_store:
241     case Intrinsic::masked_gather:
242     case Intrinsic::masked_scatter: {
243       bool IsWrite = CI->getType()->isVoidTy();
244       // Masked store has an initial operand for the value.
245       unsigned OpOffset = IsWrite ? 1 : 0;
246       Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
247       MaybeAlign Alignment = Align(1);
248       // Otherwise no alignment guarantees. We probably got Undef.
249       if (auto *Op = dyn_cast<ConstantInt>(CI->getOperand(1 + OpOffset)))
250         Alignment = Op->getMaybeAlignValue();
251       Value *Mask = CI->getOperand(2 + OpOffset);
252       Interesting.emplace_back(I, OpOffset, IsWrite, Ty, Alignment, Mask);
253       break;
254     }
255     case Intrinsic::masked_expandload:
256     case Intrinsic::masked_compressstore: {
257       bool IsWrite = CI->getIntrinsicID() == Intrinsic::masked_compressstore;
258       unsigned OpOffset = IsWrite ? 1 : 0;
259       auto *BasePtr = CI->getOperand(OpOffset);
260       MaybeAlign Alignment = BasePtr->getPointerAlignment(DL);
261       Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
262       IRBuilder<> IB(I);
263       Value *Mask = CI->getOperand(1 + OpOffset);
264       Type *IntptrTy = M.getDataLayout().getIntPtrType(
265           M.getContext(), BasePtr->getType()->getPointerAddressSpace());
266       // Use the popcount of Mask as the effective vector length.
267       Type *ExtTy = VectorType::get(IntptrTy, cast<VectorType>(Ty));
268       Value *ExtMask = IB.CreateZExt(Mask, ExtTy);
269       Value *EVL = IB.CreateAddReduce(ExtMask);
270       Value *TrueMask = ConstantInt::get(Mask->getType(), 1);
271       Interesting.emplace_back(I, OpOffset, IsWrite, Ty, Alignment, TrueMask,
272                                EVL);
273       break;
274     }
275     case Intrinsic::vp_load:
276     case Intrinsic::vp_store:
277     case Intrinsic::experimental_vp_strided_load:
278     case Intrinsic::experimental_vp_strided_store: {
279       auto *VPI = cast<VPIntrinsic>(CI);
280       unsigned IID = CI->getIntrinsicID();
281       bool IsWrite = CI->getType()->isVoidTy();
282       unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);
283       Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
284       MaybeAlign Alignment = VPI->getOperand(PtrOpNo)->getPointerAlignment(DL);
285       Value *Stride = nullptr;
286       if (IID == Intrinsic::experimental_vp_strided_store ||
287           IID == Intrinsic::experimental_vp_strided_load) {
288         Stride = VPI->getOperand(PtrOpNo + 1);
289         // Use the pointer alignment as the element alignment if the stride is a
290         // mutiple of the pointer alignment. Otherwise, the element alignment
291         // should be Align(1).
292         unsigned PointerAlign = Alignment.valueOrOne().value();
293         if (!isa<ConstantInt>(Stride) ||
294             cast<ConstantInt>(Stride)->getZExtValue() % PointerAlign != 0)
295           Alignment = Align(1);
296       }
297       Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment,
298                                VPI->getMaskParam(), VPI->getVectorLengthParam(),
299                                Stride);
300       break;
301     }
302     case Intrinsic::vp_gather:
303     case Intrinsic::vp_scatter: {
304       auto *VPI = cast<VPIntrinsic>(CI);
305       unsigned IID = CI->getIntrinsicID();
306       bool IsWrite = IID == Intrinsic::vp_scatter;
307       unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID);
308       Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType();
309       MaybeAlign Alignment = VPI->getPointerAlignment();
310       Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment,
311                                VPI->getMaskParam(),
312                                VPI->getVectorLengthParam());
313       break;
314     }
315     case Intrinsic::amdgcn_raw_buffer_load:
316     case Intrinsic::amdgcn_raw_ptr_buffer_load:
317     case Intrinsic::amdgcn_raw_buffer_load_format:
318     case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
319     case Intrinsic::amdgcn_raw_tbuffer_load:
320     case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
321     case Intrinsic::amdgcn_struct_buffer_load:
322     case Intrinsic::amdgcn_struct_ptr_buffer_load:
323     case Intrinsic::amdgcn_struct_buffer_load_format:
324     case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
325     case Intrinsic::amdgcn_struct_tbuffer_load:
326     case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
327     case Intrinsic::amdgcn_s_buffer_load:
328     case Intrinsic::amdgcn_global_load_tr_b64:
329     case Intrinsic::amdgcn_global_load_tr_b128: {
330       unsigned PtrOpNo = 0;
331       bool IsWrite = false;
332       Type *Ty = CI->getType();
333       Value *Ptr = CI->getArgOperand(PtrOpNo);
334       MaybeAlign Alignment = Ptr->getPointerAlignment(DL);
335       Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment);
336       break;
337     }
338     case Intrinsic::amdgcn_raw_tbuffer_store:
339     case Intrinsic::amdgcn_raw_ptr_tbuffer_store:
340     case Intrinsic::amdgcn_raw_buffer_store:
341     case Intrinsic::amdgcn_raw_ptr_buffer_store:
342     case Intrinsic::amdgcn_raw_buffer_store_format:
343     case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
344     case Intrinsic::amdgcn_struct_buffer_store:
345     case Intrinsic::amdgcn_struct_ptr_buffer_store:
346     case Intrinsic::amdgcn_struct_buffer_store_format:
347     case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
348     case Intrinsic::amdgcn_struct_tbuffer_store:
349     case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
350       unsigned PtrOpNo = 1;
351       bool IsWrite = true;
352       Value *Ptr = CI->getArgOperand(PtrOpNo);
353       Type *Ty = Ptr->getType();
354       MaybeAlign Alignment = Ptr->getPointerAlignment(DL);
355       Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment);
356       break;
357     }
358     default:
359       for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) {
360         if (Type *Ty = CI->getParamByRefType(ArgNo)) {
361           Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
362         } else if (Type *Ty = CI->getParamByValType(ArgNo)) {
363           Interesting.emplace_back(I, ArgNo, false, Ty, Align(1));
364         }
365       }
366     }
367   }
368 }
369 } // end namespace AMDGPU
370 } // end namespace llvm
371