1 //===AMDGPUAsanInstrumentation.cpp - ASAN related helper functions===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===-------------------------------------------------------------===// 8 9 #include "AMDGPUAsanInstrumentation.h" 10 11 #define DEBUG_TYPE "amdgpu-asan-instrumentation" 12 13 using namespace llvm; 14 15 namespace llvm { 16 namespace AMDGPU { 17 18 static uint64_t getRedzoneSizeForScale(int AsanScale) { 19 // Redzone used for stack and globals is at least 32 bytes. 20 // For scales 6 and 7, the redzone has to be 64 and 128 bytes respectively. 21 return std::max(32U, 1U << AsanScale); 22 } 23 24 static uint64_t getMinRedzoneSizeForGlobal(int AsanScale) { 25 return getRedzoneSizeForScale(AsanScale); 26 } 27 28 uint64_t getRedzoneSizeForGlobal(int AsanScale, uint64_t SizeInBytes) { 29 constexpr uint64_t kMaxRZ = 1 << 18; 30 const uint64_t MinRZ = getMinRedzoneSizeForGlobal(AsanScale); 31 32 uint64_t RZ = 0; 33 if (SizeInBytes <= MinRZ / 2) { 34 // Reduce redzone size for small size objects, e.g. int, char[1]. MinRZ is 35 // at least 32 bytes, optimize when SizeInBytes is less than or equal to 36 // half of MinRZ. 37 RZ = MinRZ - SizeInBytes; 38 } else { 39 // Calculate RZ, where MinRZ <= RZ <= MaxRZ, and RZ ~ 1/4 * SizeInBytes. 40 RZ = std::clamp((SizeInBytes / MinRZ / 4) * MinRZ, MinRZ, kMaxRZ); 41 42 // Round up to multiple of MinRZ. 43 if (SizeInBytes % MinRZ) 44 RZ += MinRZ - (SizeInBytes % MinRZ); 45 } 46 47 assert((RZ + SizeInBytes) % MinRZ == 0); 48 49 return RZ; 50 } 51 52 static size_t TypeStoreSizeToSizeIndex(uint32_t TypeSize) { 53 size_t Res = llvm::countr_zero(TypeSize / 8); 54 return Res; 55 } 56 57 static Instruction *genAMDGPUReportBlock(Module &M, IRBuilder<> &IRB, 58 Value *Cond, bool Recover) { 59 Value *ReportCond = Cond; 60 if (!Recover) { 61 auto *Ballot = 62 IRB.CreateIntrinsic(Intrinsic::amdgcn_ballot, IRB.getInt64Ty(), {Cond}); 63 ReportCond = IRB.CreateIsNotNull(Ballot); 64 } 65 66 auto *Trm = SplitBlockAndInsertIfThen( 67 ReportCond, &*IRB.GetInsertPoint(), false, 68 MDBuilder(M.getContext()).createUnlikelyBranchWeights()); 69 Trm->getParent()->setName("asan.report"); 70 71 if (Recover) 72 return Trm; 73 74 Trm = SplitBlockAndInsertIfThen(Cond, Trm, false); 75 IRB.SetInsertPoint(Trm); 76 return IRB.CreateIntrinsic(Intrinsic::amdgcn_unreachable, {}, {}); 77 } 78 79 static Value *createSlowPathCmp(Module &M, IRBuilder<> &IRB, Type *IntptrTy, 80 Value *AddrLong, Value *ShadowValue, 81 uint32_t TypeStoreSize, int AsanScale) { 82 uint64_t Granularity = static_cast<uint64_t>(1) << AsanScale; 83 // Addr & (Granularity - 1) 84 Value *LastAccessedByte = 85 IRB.CreateAnd(AddrLong, ConstantInt::get(IntptrTy, Granularity - 1)); 86 // (Addr & (Granularity - 1)) + size - 1 87 if (TypeStoreSize / 8 > 1) 88 LastAccessedByte = IRB.CreateAdd( 89 LastAccessedByte, ConstantInt::get(IntptrTy, TypeStoreSize / 8 - 1)); 90 // (uint8_t) ((Addr & (Granularity-1)) + size - 1) 91 LastAccessedByte = 92 IRB.CreateIntCast(LastAccessedByte, ShadowValue->getType(), false); 93 // ((uint8_t) ((Addr & (Granularity-1)) + size - 1)) >= ShadowValue 94 return IRB.CreateICmpSGE(LastAccessedByte, ShadowValue); 95 } 96 97 static Instruction *generateCrashCode(Module &M, IRBuilder<> &IRB, 98 Type *IntptrTy, Instruction *InsertBefore, 99 Value *Addr, bool IsWrite, 100 size_t AccessSizeIndex, 101 Value *SizeArgument, bool Recover) { 102 IRB.SetInsertPoint(InsertBefore); 103 CallInst *Call = nullptr; 104 SmallString<128> kAsanReportErrorTemplate{"__asan_report_"}; 105 SmallString<64> TypeStr{IsWrite ? "store" : "load"}; 106 SmallString<64> EndingStr{Recover ? "_noabort" : ""}; 107 108 SmallString<128> AsanErrorCallbackSizedString; 109 raw_svector_ostream AsanErrorCallbackSizedOS(AsanErrorCallbackSizedString); 110 AsanErrorCallbackSizedOS << kAsanReportErrorTemplate << TypeStr << "_n" 111 << EndingStr; 112 113 SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy}; 114 AttributeList AL2; 115 FunctionCallee AsanErrorCallbackSized = M.getOrInsertFunction( 116 AsanErrorCallbackSizedOS.str(), 117 FunctionType::get(IRB.getVoidTy(), Args2, false), AL2); 118 SmallVector<Type *, 2> Args1{1, IntptrTy}; 119 AttributeList AL1; 120 121 SmallString<128> AsanErrorCallbackString; 122 raw_svector_ostream AsanErrorCallbackOS(AsanErrorCallbackString); 123 AsanErrorCallbackOS << kAsanReportErrorTemplate << TypeStr 124 << (1ULL << AccessSizeIndex) << EndingStr; 125 126 FunctionCallee AsanErrorCallback = M.getOrInsertFunction( 127 AsanErrorCallbackOS.str(), 128 FunctionType::get(IRB.getVoidTy(), Args1, false), AL1); 129 if (SizeArgument) { 130 Call = IRB.CreateCall(AsanErrorCallbackSized, {Addr, SizeArgument}); 131 } else { 132 Call = IRB.CreateCall(AsanErrorCallback, Addr); 133 } 134 135 Call->setCannotMerge(); 136 return Call; 137 } 138 139 static Value *memToShadow(Module &M, IRBuilder<> &IRB, Type *IntptrTy, 140 Value *Shadow, int AsanScale, uint32_t AsanOffset) { 141 // Shadow >> scale 142 Shadow = IRB.CreateLShr(Shadow, AsanScale); 143 if (AsanOffset == 0) 144 return Shadow; 145 // (Shadow >> scale) | offset 146 Value *ShadowBase = ConstantInt::get(IntptrTy, AsanOffset); 147 return IRB.CreateAdd(Shadow, ShadowBase); 148 } 149 150 static void instrumentAddressImpl(Module &M, IRBuilder<> &IRB, 151 Instruction *OrigIns, 152 Instruction *InsertBefore, Value *Addr, 153 Align Alignment, uint32_t TypeStoreSize, 154 bool IsWrite, Value *SizeArgument, 155 bool UseCalls, bool Recover, int AsanScale, 156 int AsanOffset) { 157 Type *AddrTy = Addr->getType(); 158 Type *IntptrTy = M.getDataLayout().getIntPtrType( 159 M.getContext(), AddrTy->getPointerAddressSpace()); 160 IRB.SetInsertPoint(InsertBefore); 161 size_t AccessSizeIndex = TypeStoreSizeToSizeIndex(TypeStoreSize); 162 Type *ShadowTy = IntegerType::get(M.getContext(), 163 std::max(8U, TypeStoreSize >> AsanScale)); 164 Type *ShadowPtrTy = PointerType::get(M.getContext(), 0); 165 Value *AddrLong = IRB.CreatePtrToInt(Addr, IntptrTy); 166 Value *ShadowPtr = 167 memToShadow(M, IRB, IntptrTy, AddrLong, AsanScale, AsanOffset); 168 const uint64_t ShadowAlign = 169 std::max<uint64_t>(Alignment.value() >> AsanScale, 1); 170 Value *ShadowValue = IRB.CreateAlignedLoad( 171 ShadowTy, IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy), Align(ShadowAlign)); 172 Value *Cmp = IRB.CreateIsNotNull(ShadowValue); 173 auto *Cmp2 = createSlowPathCmp(M, IRB, IntptrTy, AddrLong, ShadowValue, 174 TypeStoreSize, AsanScale); 175 Cmp = IRB.CreateAnd(Cmp, Cmp2); 176 Instruction *CrashTerm = genAMDGPUReportBlock(M, IRB, Cmp, Recover); 177 Instruction *Crash = 178 generateCrashCode(M, IRB, IntptrTy, CrashTerm, AddrLong, IsWrite, 179 AccessSizeIndex, SizeArgument, Recover); 180 Crash->setDebugLoc(OrigIns->getDebugLoc()); 181 } 182 183 void instrumentAddress(Module &M, IRBuilder<> &IRB, Instruction *OrigIns, 184 Instruction *InsertBefore, Value *Addr, Align Alignment, 185 TypeSize TypeStoreSize, bool IsWrite, 186 Value *SizeArgument, bool UseCalls, bool Recover, 187 int AsanScale, int AsanOffset) { 188 if (!TypeStoreSize.isScalable()) { 189 unsigned Granularity = 1 << AsanScale; 190 const auto FixedSize = TypeStoreSize.getFixedValue(); 191 switch (FixedSize) { 192 case 8: 193 case 16: 194 case 32: 195 case 64: 196 case 128: 197 if (Alignment.value() >= Granularity || 198 Alignment.value() >= FixedSize / 8) 199 return instrumentAddressImpl( 200 M, IRB, OrigIns, InsertBefore, Addr, Alignment, FixedSize, IsWrite, 201 SizeArgument, UseCalls, Recover, AsanScale, AsanOffset); 202 } 203 } 204 // Instrument unusual size or unusual alignment. 205 IRB.SetInsertPoint(InsertBefore); 206 Type *AddrTy = Addr->getType(); 207 Type *IntptrTy = M.getDataLayout().getIntPtrType(AddrTy); 208 Value *NumBits = IRB.CreateTypeSize(IntptrTy, TypeStoreSize); 209 Value *Size = IRB.CreateLShr(NumBits, ConstantInt::get(IntptrTy, 3)); 210 Value *AddrLong = IRB.CreatePtrToInt(Addr, IntptrTy); 211 Value *SizeMinusOne = IRB.CreateAdd(Size, ConstantInt::get(IntptrTy, -1)); 212 Value *LastByte = 213 IRB.CreateIntToPtr(IRB.CreateAdd(AddrLong, SizeMinusOne), AddrTy); 214 instrumentAddressImpl(M, IRB, OrigIns, InsertBefore, Addr, {}, 8, IsWrite, 215 SizeArgument, UseCalls, Recover, AsanScale, AsanOffset); 216 instrumentAddressImpl(M, IRB, OrigIns, InsertBefore, LastByte, {}, 8, IsWrite, 217 SizeArgument, UseCalls, Recover, AsanScale, AsanOffset); 218 } 219 220 void getInterestingMemoryOperands( 221 Module &M, Instruction *I, 222 SmallVectorImpl<InterestingMemoryOperand> &Interesting) { 223 const DataLayout &DL = M.getDataLayout(); 224 if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 225 Interesting.emplace_back(I, LI->getPointerOperandIndex(), false, 226 LI->getType(), LI->getAlign()); 227 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 228 Interesting.emplace_back(I, SI->getPointerOperandIndex(), true, 229 SI->getValueOperand()->getType(), SI->getAlign()); 230 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) { 231 Interesting.emplace_back(I, RMW->getPointerOperandIndex(), true, 232 RMW->getValOperand()->getType(), std::nullopt); 233 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) { 234 Interesting.emplace_back(I, XCHG->getPointerOperandIndex(), true, 235 XCHG->getCompareOperand()->getType(), 236 std::nullopt); 237 } else if (auto *CI = dyn_cast<CallInst>(I)) { 238 switch (CI->getIntrinsicID()) { 239 case Intrinsic::masked_load: 240 case Intrinsic::masked_store: 241 case Intrinsic::masked_gather: 242 case Intrinsic::masked_scatter: { 243 bool IsWrite = CI->getType()->isVoidTy(); 244 // Masked store has an initial operand for the value. 245 unsigned OpOffset = IsWrite ? 1 : 0; 246 Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType(); 247 MaybeAlign Alignment = Align(1); 248 // Otherwise no alignment guarantees. We probably got Undef. 249 if (auto *Op = dyn_cast<ConstantInt>(CI->getOperand(1 + OpOffset))) 250 Alignment = Op->getMaybeAlignValue(); 251 Value *Mask = CI->getOperand(2 + OpOffset); 252 Interesting.emplace_back(I, OpOffset, IsWrite, Ty, Alignment, Mask); 253 break; 254 } 255 case Intrinsic::masked_expandload: 256 case Intrinsic::masked_compressstore: { 257 bool IsWrite = CI->getIntrinsicID() == Intrinsic::masked_compressstore; 258 unsigned OpOffset = IsWrite ? 1 : 0; 259 auto *BasePtr = CI->getOperand(OpOffset); 260 MaybeAlign Alignment = BasePtr->getPointerAlignment(DL); 261 Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType(); 262 IRBuilder<> IB(I); 263 Value *Mask = CI->getOperand(1 + OpOffset); 264 Type *IntptrTy = M.getDataLayout().getIntPtrType( 265 M.getContext(), BasePtr->getType()->getPointerAddressSpace()); 266 // Use the popcount of Mask as the effective vector length. 267 Type *ExtTy = VectorType::get(IntptrTy, cast<VectorType>(Ty)); 268 Value *ExtMask = IB.CreateZExt(Mask, ExtTy); 269 Value *EVL = IB.CreateAddReduce(ExtMask); 270 Value *TrueMask = ConstantInt::get(Mask->getType(), 1); 271 Interesting.emplace_back(I, OpOffset, IsWrite, Ty, Alignment, TrueMask, 272 EVL); 273 break; 274 } 275 case Intrinsic::vp_load: 276 case Intrinsic::vp_store: 277 case Intrinsic::experimental_vp_strided_load: 278 case Intrinsic::experimental_vp_strided_store: { 279 auto *VPI = cast<VPIntrinsic>(CI); 280 unsigned IID = CI->getIntrinsicID(); 281 bool IsWrite = CI->getType()->isVoidTy(); 282 unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID); 283 Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType(); 284 MaybeAlign Alignment = VPI->getOperand(PtrOpNo)->getPointerAlignment(DL); 285 Value *Stride = nullptr; 286 if (IID == Intrinsic::experimental_vp_strided_store || 287 IID == Intrinsic::experimental_vp_strided_load) { 288 Stride = VPI->getOperand(PtrOpNo + 1); 289 // Use the pointer alignment as the element alignment if the stride is a 290 // mutiple of the pointer alignment. Otherwise, the element alignment 291 // should be Align(1). 292 unsigned PointerAlign = Alignment.valueOrOne().value(); 293 if (!isa<ConstantInt>(Stride) || 294 cast<ConstantInt>(Stride)->getZExtValue() % PointerAlign != 0) 295 Alignment = Align(1); 296 } 297 Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment, 298 VPI->getMaskParam(), VPI->getVectorLengthParam(), 299 Stride); 300 break; 301 } 302 case Intrinsic::vp_gather: 303 case Intrinsic::vp_scatter: { 304 auto *VPI = cast<VPIntrinsic>(CI); 305 unsigned IID = CI->getIntrinsicID(); 306 bool IsWrite = IID == Intrinsic::vp_scatter; 307 unsigned PtrOpNo = *VPI->getMemoryPointerParamPos(IID); 308 Type *Ty = IsWrite ? CI->getArgOperand(0)->getType() : CI->getType(); 309 MaybeAlign Alignment = VPI->getPointerAlignment(); 310 Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment, 311 VPI->getMaskParam(), 312 VPI->getVectorLengthParam()); 313 break; 314 } 315 case Intrinsic::amdgcn_raw_buffer_load: 316 case Intrinsic::amdgcn_raw_ptr_buffer_load: 317 case Intrinsic::amdgcn_raw_buffer_load_format: 318 case Intrinsic::amdgcn_raw_ptr_buffer_load_format: 319 case Intrinsic::amdgcn_raw_tbuffer_load: 320 case Intrinsic::amdgcn_raw_ptr_tbuffer_load: 321 case Intrinsic::amdgcn_struct_buffer_load: 322 case Intrinsic::amdgcn_struct_ptr_buffer_load: 323 case Intrinsic::amdgcn_struct_buffer_load_format: 324 case Intrinsic::amdgcn_struct_ptr_buffer_load_format: 325 case Intrinsic::amdgcn_struct_tbuffer_load: 326 case Intrinsic::amdgcn_struct_ptr_tbuffer_load: 327 case Intrinsic::amdgcn_s_buffer_load: 328 case Intrinsic::amdgcn_global_load_tr_b64: 329 case Intrinsic::amdgcn_global_load_tr_b128: { 330 unsigned PtrOpNo = 0; 331 bool IsWrite = false; 332 Type *Ty = CI->getType(); 333 Value *Ptr = CI->getArgOperand(PtrOpNo); 334 MaybeAlign Alignment = Ptr->getPointerAlignment(DL); 335 Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment); 336 break; 337 } 338 case Intrinsic::amdgcn_raw_tbuffer_store: 339 case Intrinsic::amdgcn_raw_ptr_tbuffer_store: 340 case Intrinsic::amdgcn_raw_buffer_store: 341 case Intrinsic::amdgcn_raw_ptr_buffer_store: 342 case Intrinsic::amdgcn_raw_buffer_store_format: 343 case Intrinsic::amdgcn_raw_ptr_buffer_store_format: 344 case Intrinsic::amdgcn_struct_buffer_store: 345 case Intrinsic::amdgcn_struct_ptr_buffer_store: 346 case Intrinsic::amdgcn_struct_buffer_store_format: 347 case Intrinsic::amdgcn_struct_ptr_buffer_store_format: 348 case Intrinsic::amdgcn_struct_tbuffer_store: 349 case Intrinsic::amdgcn_struct_ptr_tbuffer_store: { 350 unsigned PtrOpNo = 1; 351 bool IsWrite = true; 352 Value *Ptr = CI->getArgOperand(PtrOpNo); 353 Type *Ty = Ptr->getType(); 354 MaybeAlign Alignment = Ptr->getPointerAlignment(DL); 355 Interesting.emplace_back(I, PtrOpNo, IsWrite, Ty, Alignment); 356 break; 357 } 358 default: 359 for (unsigned ArgNo = 0; ArgNo < CI->arg_size(); ArgNo++) { 360 if (Type *Ty = CI->getParamByRefType(ArgNo)) { 361 Interesting.emplace_back(I, ArgNo, false, Ty, Align(1)); 362 } else if (Type *Ty = CI->getParamByValType(ArgNo)) { 363 Interesting.emplace_back(I, ArgNo, false, Ty, Align(1)); 364 } 365 } 366 } 367 } 368 } 369 } // end namespace AMDGPU 370 } // end namespace llvm 371