1 //===-- dfsan_allocator.cpp -------------------------- --------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of DataflowSanitizer. 10 // 11 // DataflowSanitizer allocator. 12 //===----------------------------------------------------------------------===// 13 14 #include "dfsan_allocator.h" 15 16 #include "dfsan.h" 17 #include "dfsan_flags.h" 18 #include "dfsan_thread.h" 19 #include "sanitizer_common/sanitizer_allocator.h" 20 #include "sanitizer_common/sanitizer_allocator_checks.h" 21 #include "sanitizer_common/sanitizer_allocator_interface.h" 22 #include "sanitizer_common/sanitizer_allocator_report.h" 23 #include "sanitizer_common/sanitizer_errno.h" 24 25 using namespace __dfsan; 26 27 namespace { 28 29 struct Metadata { 30 uptr requested_size; 31 }; 32 33 struct DFsanMapUnmapCallback { 34 void OnMap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); } 35 void OnMapSecondary(uptr p, uptr size, uptr user_begin, 36 uptr user_size) const { 37 OnMap(p, size); 38 } 39 void OnUnmap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); } 40 }; 41 42 // Note: to ensure that the allocator is compatible with the application memory 43 // layout (especially with high-entropy ASLR), kSpaceBeg and kSpaceSize must be 44 // duplicated as MappingDesc::ALLOCATOR in dfsan_platform.h. 45 #if defined(__aarch64__) 46 const uptr kAllocatorSpace = 0xE00000000000ULL; 47 #else 48 const uptr kAllocatorSpace = 0x700000000000ULL; 49 #endif 50 const uptr kMaxAllowedMallocSize = 1ULL << 40; 51 52 struct AP64 { // Allocator64 parameters. Deliberately using a short name. 53 static const uptr kSpaceBeg = kAllocatorSpace; 54 static const uptr kSpaceSize = 0x40000000000; // 4T. 55 static const uptr kMetadataSize = sizeof(Metadata); 56 typedef DefaultSizeClassMap SizeClassMap; 57 typedef DFsanMapUnmapCallback MapUnmapCallback; 58 static const uptr kFlags = 0; 59 using AddressSpaceView = LocalAddressSpaceView; 60 }; 61 62 typedef SizeClassAllocator64<AP64> PrimaryAllocator; 63 64 typedef CombinedAllocator<PrimaryAllocator> Allocator; 65 typedef Allocator::AllocatorCache AllocatorCache; 66 67 static Allocator allocator; 68 static AllocatorCache fallback_allocator_cache; 69 static StaticSpinMutex fallback_mutex; 70 71 static uptr max_malloc_size; 72 } // namespace 73 74 void __dfsan::dfsan_allocator_init() { 75 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); 76 allocator.Init(common_flags()->allocator_release_to_os_interval_ms); 77 if (common_flags()->max_allocation_size_mb) 78 max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20, 79 kMaxAllowedMallocSize); 80 else 81 max_malloc_size = kMaxAllowedMallocSize; 82 } 83 84 static AllocatorCache *GetAllocatorCache(DFsanThreadLocalMallocStorage *ms) { 85 CHECK(ms); 86 CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache)); 87 return reinterpret_cast<AllocatorCache *>(ms->allocator_cache); 88 } 89 90 void DFsanThreadLocalMallocStorage::CommitBack() { 91 allocator.SwallowCache(GetAllocatorCache(this)); 92 } 93 94 static void *DFsanAllocate(uptr size, uptr alignment, bool zeroise) { 95 if (size > max_malloc_size) { 96 if (AllocatorMayReturnNull()) { 97 Report("WARNING: DataflowSanitizer failed to allocate 0x%zx bytes\n", 98 size); 99 return nullptr; 100 } 101 UNINITIALIZED BufferedStackTrace stack; 102 ReportAllocationSizeTooBig(size, max_malloc_size, &stack); 103 } 104 if (UNLIKELY(IsRssLimitExceeded())) { 105 if (AllocatorMayReturnNull()) 106 return nullptr; 107 UNINITIALIZED BufferedStackTrace stack; 108 ReportRssLimitExceeded(&stack); 109 } 110 DFsanThread *t = GetCurrentThread(); 111 void *allocated; 112 if (t) { 113 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); 114 allocated = allocator.Allocate(cache, size, alignment); 115 } else { 116 SpinMutexLock l(&fallback_mutex); 117 AllocatorCache *cache = &fallback_allocator_cache; 118 allocated = allocator.Allocate(cache, size, alignment); 119 } 120 if (UNLIKELY(!allocated)) { 121 SetAllocatorOutOfMemory(); 122 if (AllocatorMayReturnNull()) 123 return nullptr; 124 UNINITIALIZED BufferedStackTrace stack; 125 ReportOutOfMemory(size, &stack); 126 } 127 Metadata *meta = 128 reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated)); 129 meta->requested_size = size; 130 if (zeroise) { 131 internal_memset(allocated, 0, size); 132 dfsan_set_label(0, allocated, size); 133 } else if (flags().zero_in_malloc) { 134 dfsan_set_label(0, allocated, size); 135 } 136 return allocated; 137 } 138 139 void __dfsan::dfsan_deallocate(void *p) { 140 CHECK(p); 141 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p)); 142 uptr size = meta->requested_size; 143 meta->requested_size = 0; 144 if (flags().zero_in_free) 145 dfsan_set_label(0, p, size); 146 DFsanThread *t = GetCurrentThread(); 147 if (t) { 148 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); 149 allocator.Deallocate(cache, p); 150 } else { 151 SpinMutexLock l(&fallback_mutex); 152 AllocatorCache *cache = &fallback_allocator_cache; 153 allocator.Deallocate(cache, p); 154 } 155 } 156 157 static void *DFsanReallocate(void *old_p, uptr new_size, uptr alignment) { 158 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(old_p)); 159 uptr old_size = meta->requested_size; 160 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p); 161 if (new_size <= actually_allocated_size) { 162 // We are not reallocating here. 163 meta->requested_size = new_size; 164 if (new_size > old_size && flags().zero_in_malloc) 165 dfsan_set_label(0, (char *)old_p + old_size, new_size - old_size); 166 return old_p; 167 } 168 uptr memcpy_size = Min(new_size, old_size); 169 void *new_p = DFsanAllocate(new_size, alignment, false /*zeroise*/); 170 if (new_p) { 171 dfsan_copy_memory(new_p, old_p, memcpy_size); 172 dfsan_deallocate(old_p); 173 } 174 return new_p; 175 } 176 177 static void *DFsanCalloc(uptr nmemb, uptr size) { 178 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { 179 if (AllocatorMayReturnNull()) 180 return nullptr; 181 UNINITIALIZED BufferedStackTrace stack; 182 ReportCallocOverflow(nmemb, size, &stack); 183 } 184 return DFsanAllocate(nmemb * size, sizeof(u64), true /*zeroise*/); 185 } 186 187 static const void *AllocationBegin(const void *p) { 188 if (!p) 189 return nullptr; 190 void *beg = allocator.GetBlockBegin(p); 191 if (!beg) 192 return nullptr; 193 Metadata *b = (Metadata *)allocator.GetMetaData(beg); 194 if (!b) 195 return nullptr; 196 if (b->requested_size == 0) 197 return nullptr; 198 return (const void *)beg; 199 } 200 201 static uptr AllocationSize(const void *p) { 202 if (!p) 203 return 0; 204 const void *beg = allocator.GetBlockBegin(p); 205 if (beg != p) 206 return 0; 207 Metadata *b = (Metadata *)allocator.GetMetaData(p); 208 return b->requested_size; 209 } 210 211 static uptr AllocationSizeFast(const void *p) { 212 return reinterpret_cast<Metadata *>(allocator.GetMetaData(p))->requested_size; 213 } 214 215 void *__dfsan::dfsan_malloc(uptr size) { 216 return SetErrnoOnNull(DFsanAllocate(size, sizeof(u64), false /*zeroise*/)); 217 } 218 219 void *__dfsan::dfsan_calloc(uptr nmemb, uptr size) { 220 return SetErrnoOnNull(DFsanCalloc(nmemb, size)); 221 } 222 223 void *__dfsan::dfsan_realloc(void *ptr, uptr size) { 224 if (!ptr) 225 return SetErrnoOnNull(DFsanAllocate(size, sizeof(u64), false /*zeroise*/)); 226 if (size == 0) { 227 dfsan_deallocate(ptr); 228 return nullptr; 229 } 230 return SetErrnoOnNull(DFsanReallocate(ptr, size, sizeof(u64))); 231 } 232 233 void *__dfsan::dfsan_reallocarray(void *ptr, uptr nmemb, uptr size) { 234 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) { 235 errno = errno_ENOMEM; 236 if (AllocatorMayReturnNull()) 237 return nullptr; 238 UNINITIALIZED BufferedStackTrace stack; 239 ReportReallocArrayOverflow(nmemb, size, &stack); 240 } 241 return dfsan_realloc(ptr, nmemb * size); 242 } 243 244 void *__dfsan::dfsan_valloc(uptr size) { 245 return SetErrnoOnNull( 246 DFsanAllocate(size, GetPageSizeCached(), false /*zeroise*/)); 247 } 248 249 void *__dfsan::dfsan_pvalloc(uptr size) { 250 uptr PageSize = GetPageSizeCached(); 251 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) { 252 errno = errno_ENOMEM; 253 if (AllocatorMayReturnNull()) 254 return nullptr; 255 UNINITIALIZED BufferedStackTrace stack; 256 ReportPvallocOverflow(size, &stack); 257 } 258 // pvalloc(0) should allocate one page. 259 size = size ? RoundUpTo(size, PageSize) : PageSize; 260 return SetErrnoOnNull(DFsanAllocate(size, PageSize, false /*zeroise*/)); 261 } 262 263 void *__dfsan::dfsan_aligned_alloc(uptr alignment, uptr size) { 264 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) { 265 errno = errno_EINVAL; 266 if (AllocatorMayReturnNull()) 267 return nullptr; 268 UNINITIALIZED BufferedStackTrace stack; 269 ReportInvalidAlignedAllocAlignment(size, alignment, &stack); 270 } 271 return SetErrnoOnNull(DFsanAllocate(size, alignment, false /*zeroise*/)); 272 } 273 274 void *__dfsan::dfsan_memalign(uptr alignment, uptr size) { 275 if (UNLIKELY(!IsPowerOfTwo(alignment))) { 276 errno = errno_EINVAL; 277 if (AllocatorMayReturnNull()) 278 return nullptr; 279 UNINITIALIZED BufferedStackTrace stack; 280 ReportInvalidAllocationAlignment(alignment, &stack); 281 } 282 return SetErrnoOnNull(DFsanAllocate(size, alignment, false /*zeroise*/)); 283 } 284 285 int __dfsan::dfsan_posix_memalign(void **memptr, uptr alignment, uptr size) { 286 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { 287 if (AllocatorMayReturnNull()) 288 return errno_EINVAL; 289 UNINITIALIZED BufferedStackTrace stack; 290 ReportInvalidPosixMemalignAlignment(alignment, &stack); 291 } 292 void *ptr = DFsanAllocate(size, alignment, false /*zeroise*/); 293 if (UNLIKELY(!ptr)) 294 // OOM error is already taken care of by DFsanAllocate. 295 return errno_ENOMEM; 296 CHECK(IsAligned((uptr)ptr, alignment)); 297 *memptr = ptr; 298 return 0; 299 } 300 301 extern "C" { 302 uptr __sanitizer_get_current_allocated_bytes() { 303 uptr stats[AllocatorStatCount]; 304 allocator.GetStats(stats); 305 return stats[AllocatorStatAllocated]; 306 } 307 308 uptr __sanitizer_get_heap_size() { 309 uptr stats[AllocatorStatCount]; 310 allocator.GetStats(stats); 311 return stats[AllocatorStatMapped]; 312 } 313 314 uptr __sanitizer_get_free_bytes() { return 1; } 315 316 uptr __sanitizer_get_unmapped_bytes() { return 1; } 317 318 uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } 319 320 int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; } 321 322 const void *__sanitizer_get_allocated_begin(const void *p) { 323 return AllocationBegin(p); 324 } 325 326 uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); } 327 328 uptr __sanitizer_get_allocated_size_fast(const void *p) { 329 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p)); 330 uptr ret = AllocationSizeFast(p); 331 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p)); 332 return ret; 333 } 334 } 335