1d89ec533Spatrick //===-- dfsan_allocator.cpp -------------------------- --------------------===//
2d89ec533Spatrick //
3d89ec533Spatrick // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4d89ec533Spatrick // See https://llvm.org/LICENSE.txt for license information.
5d89ec533Spatrick // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6d89ec533Spatrick //
7d89ec533Spatrick //===----------------------------------------------------------------------===//
8d89ec533Spatrick //
9d89ec533Spatrick // This file is a part of DataflowSanitizer.
10d89ec533Spatrick //
11d89ec533Spatrick // DataflowSanitizer allocator.
12d89ec533Spatrick //===----------------------------------------------------------------------===//
13d89ec533Spatrick
14d89ec533Spatrick #include "dfsan_allocator.h"
15d89ec533Spatrick
16d89ec533Spatrick #include "dfsan.h"
17d89ec533Spatrick #include "dfsan_flags.h"
18d89ec533Spatrick #include "dfsan_thread.h"
19d89ec533Spatrick #include "sanitizer_common/sanitizer_allocator.h"
20d89ec533Spatrick #include "sanitizer_common/sanitizer_allocator_checks.h"
21d89ec533Spatrick #include "sanitizer_common/sanitizer_allocator_interface.h"
22d89ec533Spatrick #include "sanitizer_common/sanitizer_allocator_report.h"
23d89ec533Spatrick #include "sanitizer_common/sanitizer_errno.h"
24d89ec533Spatrick
25d89ec533Spatrick namespace __dfsan {
26d89ec533Spatrick
27d89ec533Spatrick struct Metadata {
28d89ec533Spatrick uptr requested_size;
29d89ec533Spatrick };
30d89ec533Spatrick
31d89ec533Spatrick struct DFsanMapUnmapCallback {
OnMap__dfsan::DFsanMapUnmapCallback32d89ec533Spatrick void OnMap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); }
OnUnmap__dfsan::DFsanMapUnmapCallback33d89ec533Spatrick void OnUnmap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); }
34d89ec533Spatrick };
35d89ec533Spatrick
36*810390e3Srobert #if defined(__aarch64__)
37*810390e3Srobert const uptr kAllocatorSpace = 0xE00000000000ULL;
38*810390e3Srobert #else
39*810390e3Srobert const uptr kAllocatorSpace = 0x700000000000ULL;
40*810390e3Srobert #endif
41*810390e3Srobert const uptr kMaxAllowedMallocSize = 8UL << 30;
42d89ec533Spatrick
43d89ec533Spatrick struct AP64 { // Allocator64 parameters. Deliberately using a short name.
44d89ec533Spatrick static const uptr kSpaceBeg = kAllocatorSpace;
45d89ec533Spatrick static const uptr kSpaceSize = 0x40000000000; // 4T.
46d89ec533Spatrick static const uptr kMetadataSize = sizeof(Metadata);
47d89ec533Spatrick typedef DefaultSizeClassMap SizeClassMap;
48d89ec533Spatrick typedef DFsanMapUnmapCallback MapUnmapCallback;
49d89ec533Spatrick static const uptr kFlags = 0;
50d89ec533Spatrick using AddressSpaceView = LocalAddressSpaceView;
51d89ec533Spatrick };
52d89ec533Spatrick
53d89ec533Spatrick typedef SizeClassAllocator64<AP64> PrimaryAllocator;
54d89ec533Spatrick
55d89ec533Spatrick typedef CombinedAllocator<PrimaryAllocator> Allocator;
56d89ec533Spatrick typedef Allocator::AllocatorCache AllocatorCache;
57d89ec533Spatrick
58d89ec533Spatrick static Allocator allocator;
59d89ec533Spatrick static AllocatorCache fallback_allocator_cache;
60d89ec533Spatrick static StaticSpinMutex fallback_mutex;
61d89ec533Spatrick
62d89ec533Spatrick static uptr max_malloc_size;
63d89ec533Spatrick
dfsan_allocator_init()64d89ec533Spatrick void dfsan_allocator_init() {
65d89ec533Spatrick SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
66d89ec533Spatrick allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
67d89ec533Spatrick if (common_flags()->max_allocation_size_mb)
68d89ec533Spatrick max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20,
69d89ec533Spatrick kMaxAllowedMallocSize);
70d89ec533Spatrick else
71d89ec533Spatrick max_malloc_size = kMaxAllowedMallocSize;
72d89ec533Spatrick }
73d89ec533Spatrick
GetAllocatorCache(DFsanThreadLocalMallocStorage * ms)74d89ec533Spatrick AllocatorCache *GetAllocatorCache(DFsanThreadLocalMallocStorage *ms) {
75d89ec533Spatrick CHECK(ms);
76d89ec533Spatrick CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
77d89ec533Spatrick return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);
78d89ec533Spatrick }
79d89ec533Spatrick
CommitBack()80d89ec533Spatrick void DFsanThreadLocalMallocStorage::CommitBack() {
81d89ec533Spatrick allocator.SwallowCache(GetAllocatorCache(this));
82d89ec533Spatrick }
83d89ec533Spatrick
DFsanAllocate(uptr size,uptr alignment,bool zeroise)84d89ec533Spatrick static void *DFsanAllocate(uptr size, uptr alignment, bool zeroise) {
85d89ec533Spatrick if (size > max_malloc_size) {
86d89ec533Spatrick if (AllocatorMayReturnNull()) {
87d89ec533Spatrick Report("WARNING: DataflowSanitizer failed to allocate 0x%zx bytes\n",
88d89ec533Spatrick size);
89d89ec533Spatrick return nullptr;
90d89ec533Spatrick }
91d89ec533Spatrick BufferedStackTrace stack;
92d89ec533Spatrick ReportAllocationSizeTooBig(size, max_malloc_size, &stack);
93d89ec533Spatrick }
94*810390e3Srobert if (UNLIKELY(IsRssLimitExceeded())) {
95*810390e3Srobert if (AllocatorMayReturnNull())
96*810390e3Srobert return nullptr;
97*810390e3Srobert BufferedStackTrace stack;
98*810390e3Srobert ReportRssLimitExceeded(&stack);
99*810390e3Srobert }
100d89ec533Spatrick DFsanThread *t = GetCurrentThread();
101d89ec533Spatrick void *allocated;
102d89ec533Spatrick if (t) {
103d89ec533Spatrick AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
104d89ec533Spatrick allocated = allocator.Allocate(cache, size, alignment);
105d89ec533Spatrick } else {
106d89ec533Spatrick SpinMutexLock l(&fallback_mutex);
107d89ec533Spatrick AllocatorCache *cache = &fallback_allocator_cache;
108d89ec533Spatrick allocated = allocator.Allocate(cache, size, alignment);
109d89ec533Spatrick }
110d89ec533Spatrick if (UNLIKELY(!allocated)) {
111d89ec533Spatrick SetAllocatorOutOfMemory();
112d89ec533Spatrick if (AllocatorMayReturnNull())
113d89ec533Spatrick return nullptr;
114d89ec533Spatrick BufferedStackTrace stack;
115d89ec533Spatrick ReportOutOfMemory(size, &stack);
116d89ec533Spatrick }
117d89ec533Spatrick Metadata *meta =
118d89ec533Spatrick reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
119d89ec533Spatrick meta->requested_size = size;
120d89ec533Spatrick if (zeroise) {
121d89ec533Spatrick internal_memset(allocated, 0, size);
122d89ec533Spatrick dfsan_set_label(0, allocated, size);
123d89ec533Spatrick } else if (flags().zero_in_malloc) {
124d89ec533Spatrick dfsan_set_label(0, allocated, size);
125d89ec533Spatrick }
126d89ec533Spatrick return allocated;
127d89ec533Spatrick }
128d89ec533Spatrick
dfsan_deallocate(void * p)129d89ec533Spatrick void dfsan_deallocate(void *p) {
130d89ec533Spatrick CHECK(p);
131d89ec533Spatrick Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));
132d89ec533Spatrick uptr size = meta->requested_size;
133d89ec533Spatrick meta->requested_size = 0;
134d89ec533Spatrick if (flags().zero_in_free)
135d89ec533Spatrick dfsan_set_label(0, p, size);
136d89ec533Spatrick DFsanThread *t = GetCurrentThread();
137d89ec533Spatrick if (t) {
138d89ec533Spatrick AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
139d89ec533Spatrick allocator.Deallocate(cache, p);
140d89ec533Spatrick } else {
141d89ec533Spatrick SpinMutexLock l(&fallback_mutex);
142d89ec533Spatrick AllocatorCache *cache = &fallback_allocator_cache;
143d89ec533Spatrick allocator.Deallocate(cache, p);
144d89ec533Spatrick }
145d89ec533Spatrick }
146d89ec533Spatrick
DFsanReallocate(void * old_p,uptr new_size,uptr alignment)147d89ec533Spatrick void *DFsanReallocate(void *old_p, uptr new_size, uptr alignment) {
148d89ec533Spatrick Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(old_p));
149d89ec533Spatrick uptr old_size = meta->requested_size;
150d89ec533Spatrick uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p);
151d89ec533Spatrick if (new_size <= actually_allocated_size) {
152d89ec533Spatrick // We are not reallocating here.
153d89ec533Spatrick meta->requested_size = new_size;
154d89ec533Spatrick if (new_size > old_size && flags().zero_in_malloc)
155d89ec533Spatrick dfsan_set_label(0, (char *)old_p + old_size, new_size - old_size);
156d89ec533Spatrick return old_p;
157d89ec533Spatrick }
158d89ec533Spatrick uptr memcpy_size = Min(new_size, old_size);
159d89ec533Spatrick void *new_p = DFsanAllocate(new_size, alignment, false /*zeroise*/);
160d89ec533Spatrick if (new_p) {
161d89ec533Spatrick dfsan_copy_memory(new_p, old_p, memcpy_size);
162d89ec533Spatrick dfsan_deallocate(old_p);
163d89ec533Spatrick }
164d89ec533Spatrick return new_p;
165d89ec533Spatrick }
166d89ec533Spatrick
DFsanCalloc(uptr nmemb,uptr size)167d89ec533Spatrick void *DFsanCalloc(uptr nmemb, uptr size) {
168d89ec533Spatrick if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
169d89ec533Spatrick if (AllocatorMayReturnNull())
170d89ec533Spatrick return nullptr;
171d89ec533Spatrick BufferedStackTrace stack;
172d89ec533Spatrick ReportCallocOverflow(nmemb, size, &stack);
173d89ec533Spatrick }
174d89ec533Spatrick return DFsanAllocate(nmemb * size, sizeof(u64), true /*zeroise*/);
175d89ec533Spatrick }
176d89ec533Spatrick
AllocationSize(const void * p)177d89ec533Spatrick static uptr AllocationSize(const void *p) {
178d89ec533Spatrick if (!p)
179d89ec533Spatrick return 0;
180d89ec533Spatrick const void *beg = allocator.GetBlockBegin(p);
181d89ec533Spatrick if (beg != p)
182d89ec533Spatrick return 0;
183d89ec533Spatrick Metadata *b = (Metadata *)allocator.GetMetaData(p);
184d89ec533Spatrick return b->requested_size;
185d89ec533Spatrick }
186d89ec533Spatrick
dfsan_malloc(uptr size)187d89ec533Spatrick void *dfsan_malloc(uptr size) {
188d89ec533Spatrick return SetErrnoOnNull(DFsanAllocate(size, sizeof(u64), false /*zeroise*/));
189d89ec533Spatrick }
190d89ec533Spatrick
dfsan_calloc(uptr nmemb,uptr size)191d89ec533Spatrick void *dfsan_calloc(uptr nmemb, uptr size) {
192d89ec533Spatrick return SetErrnoOnNull(DFsanCalloc(nmemb, size));
193d89ec533Spatrick }
194d89ec533Spatrick
dfsan_realloc(void * ptr,uptr size)195d89ec533Spatrick void *dfsan_realloc(void *ptr, uptr size) {
196d89ec533Spatrick if (!ptr)
197d89ec533Spatrick return SetErrnoOnNull(DFsanAllocate(size, sizeof(u64), false /*zeroise*/));
198d89ec533Spatrick if (size == 0) {
199d89ec533Spatrick dfsan_deallocate(ptr);
200d89ec533Spatrick return nullptr;
201d89ec533Spatrick }
202d89ec533Spatrick return SetErrnoOnNull(DFsanReallocate(ptr, size, sizeof(u64)));
203d89ec533Spatrick }
204d89ec533Spatrick
dfsan_reallocarray(void * ptr,uptr nmemb,uptr size)205d89ec533Spatrick void *dfsan_reallocarray(void *ptr, uptr nmemb, uptr size) {
206d89ec533Spatrick if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
207d89ec533Spatrick errno = errno_ENOMEM;
208d89ec533Spatrick if (AllocatorMayReturnNull())
209d89ec533Spatrick return nullptr;
210d89ec533Spatrick BufferedStackTrace stack;
211d89ec533Spatrick ReportReallocArrayOverflow(nmemb, size, &stack);
212d89ec533Spatrick }
213d89ec533Spatrick return dfsan_realloc(ptr, nmemb * size);
214d89ec533Spatrick }
215d89ec533Spatrick
dfsan_valloc(uptr size)216d89ec533Spatrick void *dfsan_valloc(uptr size) {
217d89ec533Spatrick return SetErrnoOnNull(
218d89ec533Spatrick DFsanAllocate(size, GetPageSizeCached(), false /*zeroise*/));
219d89ec533Spatrick }
220d89ec533Spatrick
dfsan_pvalloc(uptr size)221d89ec533Spatrick void *dfsan_pvalloc(uptr size) {
222d89ec533Spatrick uptr PageSize = GetPageSizeCached();
223d89ec533Spatrick if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
224d89ec533Spatrick errno = errno_ENOMEM;
225d89ec533Spatrick if (AllocatorMayReturnNull())
226d89ec533Spatrick return nullptr;
227d89ec533Spatrick BufferedStackTrace stack;
228d89ec533Spatrick ReportPvallocOverflow(size, &stack);
229d89ec533Spatrick }
230d89ec533Spatrick // pvalloc(0) should allocate one page.
231d89ec533Spatrick size = size ? RoundUpTo(size, PageSize) : PageSize;
232d89ec533Spatrick return SetErrnoOnNull(DFsanAllocate(size, PageSize, false /*zeroise*/));
233d89ec533Spatrick }
234d89ec533Spatrick
dfsan_aligned_alloc(uptr alignment,uptr size)235d89ec533Spatrick void *dfsan_aligned_alloc(uptr alignment, uptr size) {
236d89ec533Spatrick if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
237d89ec533Spatrick errno = errno_EINVAL;
238d89ec533Spatrick if (AllocatorMayReturnNull())
239d89ec533Spatrick return nullptr;
240d89ec533Spatrick BufferedStackTrace stack;
241d89ec533Spatrick ReportInvalidAlignedAllocAlignment(size, alignment, &stack);
242d89ec533Spatrick }
243d89ec533Spatrick return SetErrnoOnNull(DFsanAllocate(size, alignment, false /*zeroise*/));
244d89ec533Spatrick }
245d89ec533Spatrick
dfsan_memalign(uptr alignment,uptr size)246d89ec533Spatrick void *dfsan_memalign(uptr alignment, uptr size) {
247d89ec533Spatrick if (UNLIKELY(!IsPowerOfTwo(alignment))) {
248d89ec533Spatrick errno = errno_EINVAL;
249d89ec533Spatrick if (AllocatorMayReturnNull())
250d89ec533Spatrick return nullptr;
251d89ec533Spatrick BufferedStackTrace stack;
252d89ec533Spatrick ReportInvalidAllocationAlignment(alignment, &stack);
253d89ec533Spatrick }
254d89ec533Spatrick return SetErrnoOnNull(DFsanAllocate(size, alignment, false /*zeroise*/));
255d89ec533Spatrick }
256d89ec533Spatrick
dfsan_posix_memalign(void ** memptr,uptr alignment,uptr size)257d89ec533Spatrick int dfsan_posix_memalign(void **memptr, uptr alignment, uptr size) {
258d89ec533Spatrick if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
259d89ec533Spatrick if (AllocatorMayReturnNull())
260d89ec533Spatrick return errno_EINVAL;
261d89ec533Spatrick BufferedStackTrace stack;
262d89ec533Spatrick ReportInvalidPosixMemalignAlignment(alignment, &stack);
263d89ec533Spatrick }
264d89ec533Spatrick void *ptr = DFsanAllocate(size, alignment, false /*zeroise*/);
265d89ec533Spatrick if (UNLIKELY(!ptr))
266d89ec533Spatrick // OOM error is already taken care of by DFsanAllocate.
267d89ec533Spatrick return errno_ENOMEM;
268d89ec533Spatrick CHECK(IsAligned((uptr)ptr, alignment));
269d89ec533Spatrick *memptr = ptr;
270d89ec533Spatrick return 0;
271d89ec533Spatrick }
272d89ec533Spatrick
273d89ec533Spatrick } // namespace __dfsan
274d89ec533Spatrick
275d89ec533Spatrick using namespace __dfsan;
276d89ec533Spatrick
__sanitizer_get_current_allocated_bytes()277d89ec533Spatrick uptr __sanitizer_get_current_allocated_bytes() {
278d89ec533Spatrick uptr stats[AllocatorStatCount];
279d89ec533Spatrick allocator.GetStats(stats);
280d89ec533Spatrick return stats[AllocatorStatAllocated];
281d89ec533Spatrick }
282d89ec533Spatrick
__sanitizer_get_heap_size()283d89ec533Spatrick uptr __sanitizer_get_heap_size() {
284d89ec533Spatrick uptr stats[AllocatorStatCount];
285d89ec533Spatrick allocator.GetStats(stats);
286d89ec533Spatrick return stats[AllocatorStatMapped];
287d89ec533Spatrick }
288d89ec533Spatrick
__sanitizer_get_free_bytes()289d89ec533Spatrick uptr __sanitizer_get_free_bytes() { return 1; }
290d89ec533Spatrick
__sanitizer_get_unmapped_bytes()291d89ec533Spatrick uptr __sanitizer_get_unmapped_bytes() { return 1; }
292d89ec533Spatrick
__sanitizer_get_estimated_allocated_size(uptr size)293d89ec533Spatrick uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
294d89ec533Spatrick
__sanitizer_get_ownership(const void * p)295d89ec533Spatrick int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
296d89ec533Spatrick
__sanitizer_get_allocated_size(const void * p)297d89ec533Spatrick uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
298