1 //===-- xray_allocator.h ---------------------------------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file is a part of XRay, a dynamic runtime instrumentation system.
11 //
12 // Defines the allocator interface for an arena allocator, used primarily for
13 // the profiling runtime.
14 //
15 //===----------------------------------------------------------------------===//
16 #ifndef XRAY_ALLOCATOR_H
17 #define XRAY_ALLOCATOR_H
18
19 #include "sanitizer_common/sanitizer_common.h"
20 #include "sanitizer_common/sanitizer_internal_defs.h"
21 #include "sanitizer_common/sanitizer_mutex.h"
22 #if SANITIZER_FUCHSIA
23 #include <zircon/process.h>
24 #include <zircon/status.h>
25 #include <zircon/syscalls.h>
26 #else
27 #include "sanitizer_common/sanitizer_posix.h"
28 #endif
29 #include "xray_defs.h"
30 #include "xray_utils.h"
31 #include <cstddef>
32 #include <cstdint>
33 #include <sys/mman.h>
34
35 namespace __xray {
36
37 // We implement our own memory allocation routine which will bypass the
38 // internal allocator. This allows us to manage the memory directly, using
39 // mmap'ed memory to back the allocators.
allocate()40 template <class T> T *allocate() XRAY_NEVER_INSTRUMENT {
41 uptr RoundedSize = RoundUpTo(sizeof(T), GetPageSizeCached());
42 #if SANITIZER_FUCHSIA
43 zx_handle_t Vmo;
44 zx_status_t Status = _zx_vmo_create(RoundedSize, 0, &Vmo);
45 if (Status != ZX_OK) {
46 if (Verbosity())
47 Report("XRay Profiling: Failed to create VMO of size %zu: %s\n",
48 sizeof(T), _zx_status_get_string(Status));
49 return nullptr;
50 }
51 uintptr_t B;
52 Status =
53 _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0,
54 Vmo, 0, sizeof(T), &B);
55 _zx_handle_close(Vmo);
56 if (Status != ZX_OK) {
57 if (Verbosity())
58 Report("XRay Profiling: Failed to map VMAR of size %zu: %s\n", sizeof(T),
59 _zx_status_get_string(Status));
60 return nullptr;
61 }
62 return reinterpret_cast<T *>(B);
63 #else
64 uptr B = internal_mmap(NULL, RoundedSize, PROT_READ | PROT_WRITE,
65 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
66 int ErrNo = 0;
67 if (UNLIKELY(internal_iserror(B, &ErrNo))) {
68 if (Verbosity())
69 Report(
70 "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n",
71 RoundedSize, B);
72 return nullptr;
73 }
74 #endif
75 return reinterpret_cast<T *>(B);
76 }
77
deallocate(T * B)78 template <class T> void deallocate(T *B) XRAY_NEVER_INSTRUMENT {
79 if (B == nullptr)
80 return;
81 uptr RoundedSize = RoundUpTo(sizeof(T), GetPageSizeCached());
82 #if SANITIZER_FUCHSIA
83 _zx_vmar_unmap(_zx_vmar_root_self(), reinterpret_cast<uintptr_t>(B),
84 RoundedSize);
85 #else
86 internal_munmap(B, RoundedSize);
87 #endif
88 }
89
90 template <class T = unsigned char>
allocateBuffer(size_t S)91 T *allocateBuffer(size_t S) XRAY_NEVER_INSTRUMENT {
92 uptr RoundedSize = RoundUpTo(S * sizeof(T), GetPageSizeCached());
93 #if SANITIZER_FUCHSIA
94 zx_handle_t Vmo;
95 zx_status_t Status = _zx_vmo_create(RoundedSize, 0, &Vmo);
96 if (Status != ZX_OK) {
97 if (Verbosity())
98 Report("XRay Profiling: Failed to create VMO of size %zu: %s\n", S,
99 _zx_status_get_string(Status));
100 return nullptr;
101 }
102 uintptr_t B;
103 Status = _zx_vmar_map(_zx_vmar_root_self(),
104 ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, Vmo, 0, S, &B);
105 _zx_handle_close(Vmo);
106 if (Status != ZX_OK) {
107 if (Verbosity())
108 Report("XRay Profiling: Failed to map VMAR of size %zu: %s\n", S,
109 _zx_status_get_string(Status));
110 return nullptr;
111 }
112 #else
113 uptr B = internal_mmap(NULL, RoundedSize, PROT_READ | PROT_WRITE,
114 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
115 int ErrNo = 0;
116 if (UNLIKELY(internal_iserror(B, &ErrNo))) {
117 if (Verbosity())
118 Report(
119 "XRay Profiling: Failed to allocate memory of size %d; Error = %d.\n",
120 RoundedSize, B);
121 return nullptr;
122 }
123 #endif
124 return reinterpret_cast<T *>(B);
125 }
126
deallocateBuffer(T * B,size_t S)127 template <class T> void deallocateBuffer(T *B, size_t S) XRAY_NEVER_INSTRUMENT {
128 if (B == nullptr)
129 return;
130 uptr RoundedSize = RoundUpTo(S * sizeof(T), GetPageSizeCached());
131 #if SANITIZER_FUCHSIA
132 _zx_vmar_unmap(_zx_vmar_root_self(), reinterpret_cast<uintptr_t>(B),
133 RoundedSize);
134 #else
135 internal_munmap(B, RoundedSize);
136 #endif
137 }
138
139 template <class T, class... U>
initArray(size_t N,U &&...Us)140 T *initArray(size_t N, U &&... Us) XRAY_NEVER_INSTRUMENT {
141 auto A = allocateBuffer<T>(N);
142 if (A != nullptr)
143 while (N > 0)
144 new (A + (--N)) T(std::forward<U>(Us)...);
145 return A;
146 }
147
148 /// The Allocator type hands out fixed-sized chunks of memory that are
149 /// cache-line aligned and sized. This is useful for placement of
150 /// performance-sensitive data in memory that's frequently accessed. The
151 /// allocator also self-limits the peak memory usage to a dynamically defined
152 /// maximum.
153 ///
154 /// N is the lower-bound size of the block of memory to return from the
155 /// allocation function. N is used to compute the size of a block, which is
156 /// cache-line-size multiples worth of memory. We compute the size of a block by
157 /// determining how many cache lines worth of memory is required to subsume N.
158 ///
159 /// The Allocator instance will manage its own memory acquired through mmap.
160 /// This severely constrains the platforms on which this can be used to POSIX
161 /// systems where mmap semantics are well-defined.
162 ///
163 /// FIXME: Isolate the lower-level memory management to a different abstraction
164 /// that can be platform-specific.
165 template <size_t N> struct Allocator {
166 // The Allocator returns memory as Block instances.
167 struct Block {
168 /// Compute the minimum cache-line size multiple that is >= N.
169 static constexpr auto Size = nearest_boundary(N, kCacheLineSize);
170 void *Data;
171 };
172
173 private:
174 size_t MaxMemory{0};
175 unsigned char *BackingStore = nullptr;
176 unsigned char *AlignedNextBlock = nullptr;
177 size_t AllocatedBlocks = 0;
178 bool Owned;
179 SpinMutex Mutex{};
180
AllocAllocator181 void *Alloc() XRAY_NEVER_INSTRUMENT {
182 SpinMutexLock Lock(&Mutex);
183 if (UNLIKELY(BackingStore == nullptr)) {
184 BackingStore = allocateBuffer(MaxMemory);
185 if (BackingStore == nullptr) {
186 if (Verbosity())
187 Report("XRay Profiling: Failed to allocate memory for allocator.\n");
188 return nullptr;
189 }
190
191 AlignedNextBlock = BackingStore;
192
193 // Ensure that NextBlock is aligned appropriately.
194 auto BackingStoreNum = reinterpret_cast<uintptr_t>(BackingStore);
195 auto AlignedNextBlockNum = nearest_boundary(
196 reinterpret_cast<uintptr_t>(AlignedNextBlock), kCacheLineSize);
197 if (diff(AlignedNextBlockNum, BackingStoreNum) > ptrdiff_t(MaxMemory)) {
198 deallocateBuffer(BackingStore, MaxMemory);
199 AlignedNextBlock = BackingStore = nullptr;
200 if (Verbosity())
201 Report("XRay Profiling: Cannot obtain enough memory from "
202 "preallocated region.\n");
203 return nullptr;
204 }
205
206 AlignedNextBlock = reinterpret_cast<unsigned char *>(AlignedNextBlockNum);
207
208 // Assert that AlignedNextBlock is cache-line aligned.
209 DCHECK_EQ(reinterpret_cast<uintptr_t>(AlignedNextBlock) % kCacheLineSize,
210 0);
211 }
212
213 if (((AllocatedBlocks + 1) * Block::Size) > MaxMemory)
214 return nullptr;
215
216 // Align the pointer we'd like to return to an appropriate alignment, then
217 // advance the pointer from where to start allocations.
218 void *Result = AlignedNextBlock;
219 AlignedNextBlock =
220 reinterpret_cast<unsigned char *>(AlignedNextBlock) + Block::Size;
221 ++AllocatedBlocks;
222 return Result;
223 }
224
225 public:
AllocatorAllocator226 explicit Allocator(size_t M) XRAY_NEVER_INSTRUMENT
227 : MaxMemory(RoundUpTo(M, kCacheLineSize)),
228 BackingStore(nullptr),
229 AlignedNextBlock(nullptr),
230 AllocatedBlocks(0),
231 Owned(true),
232 Mutex() {}
233
AllocatorAllocator234 explicit Allocator(void *P, size_t M) XRAY_NEVER_INSTRUMENT
235 : MaxMemory(M),
236 BackingStore(reinterpret_cast<unsigned char *>(P)),
237 AlignedNextBlock(reinterpret_cast<unsigned char *>(P)),
238 AllocatedBlocks(0),
239 Owned(false),
240 Mutex() {}
241
242 Allocator(const Allocator &) = delete;
243 Allocator &operator=(const Allocator &) = delete;
244
AllocatorAllocator245 Allocator(Allocator &&O) XRAY_NEVER_INSTRUMENT {
246 SpinMutexLock L0(&Mutex);
247 SpinMutexLock L1(&O.Mutex);
248 MaxMemory = O.MaxMemory;
249 O.MaxMemory = 0;
250 BackingStore = O.BackingStore;
251 O.BackingStore = nullptr;
252 AlignedNextBlock = O.AlignedNextBlock;
253 O.AlignedNextBlock = nullptr;
254 AllocatedBlocks = O.AllocatedBlocks;
255 O.AllocatedBlocks = 0;
256 Owned = O.Owned;
257 O.Owned = false;
258 }
259
260 Allocator &operator=(Allocator &&O) XRAY_NEVER_INSTRUMENT {
261 SpinMutexLock L0(&Mutex);
262 SpinMutexLock L1(&O.Mutex);
263 MaxMemory = O.MaxMemory;
264 O.MaxMemory = 0;
265 if (BackingStore != nullptr)
266 deallocateBuffer(BackingStore, MaxMemory);
267 BackingStore = O.BackingStore;
268 O.BackingStore = nullptr;
269 AlignedNextBlock = O.AlignedNextBlock;
270 O.AlignedNextBlock = nullptr;
271 AllocatedBlocks = O.AllocatedBlocks;
272 O.AllocatedBlocks = 0;
273 Owned = O.Owned;
274 O.Owned = false;
275 return *this;
276 }
277
AllocateAllocator278 Block Allocate() XRAY_NEVER_INSTRUMENT { return {Alloc()}; }
279
~AllocatorAllocator280 ~Allocator() NOEXCEPT XRAY_NEVER_INSTRUMENT {
281 if (Owned && BackingStore != nullptr) {
282 deallocateBuffer(BackingStore, MaxMemory);
283 }
284 }
285 };
286
287 } // namespace __xray
288
289 #endif // XRAY_ALLOCATOR_H
290