1 //===------------------------ malloc.c ------------------*- C -*-=============//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This library provides a cross-platform lock free thread caching malloc
10 // implementation in C11.
11 //
12 //
13 // This file provides overrides for the standard library malloc entry points for
14 // C and new/delete operators for C++ It also provides automatic
15 // initialization/finalization of process and threads
16 //
17 //===----------------------------------------------------------------------===//
18
19 #if defined(__TINYC__)
20 #include <sys/types.h>
21 #endif
22
23 #ifndef ARCH_64BIT
24 #if defined(__LLP64__) || defined(__LP64__) || defined(_WIN64)
25 #define ARCH_64BIT 1
26 _Static_assert(sizeof(size_t) == 8, "Data type size mismatch");
27 _Static_assert(sizeof(void *) == 8, "Data type size mismatch");
28 #else
29 #define ARCH_64BIT 0
30 _Static_assert(sizeof(size_t) == 4, "Data type size mismatch");
31 _Static_assert(sizeof(void *) == 4, "Data type size mismatch");
32 #endif
33 #endif
34
35 #if (defined(__GNUC__) || defined(__clang__))
36 #pragma GCC visibility push(default)
37 #endif
38
39 #define USE_IMPLEMENT 1
40 #define USE_INTERPOSE 0
41 #define USE_ALIAS 0
42
43 #if defined(__APPLE__)
44 #undef USE_INTERPOSE
45 #define USE_INTERPOSE 1
46
47 typedef struct interpose_t {
48 void *new_func;
49 void *orig_func;
50 } interpose_t;
51
52 #define MAC_INTERPOSE_PAIR(newf, oldf) {(void *)newf, (void *)oldf}
53 #define MAC_INTERPOSE_SINGLE(newf, oldf) \
54 __attribute__((used)) static const interpose_t macinterpose##newf##oldf \
55 __attribute__((section("__DATA, __interpose"))) = \
56 MAC_INTERPOSE_PAIR(newf, oldf)
57
58 #endif
59
60 #if !defined(_WIN32) && !defined(__APPLE__)
61 #undef USE_IMPLEMENT
62 #undef USE_ALIAS
63 #define USE_IMPLEMENT 0
64 #define USE_ALIAS 1
65 #endif
66
67 #ifdef _MSC_VER
68 #pragma warning(disable : 4100)
69 #undef malloc
70 #undef free
71 #undef calloc
72 #define RPMALLOC_RESTRICT __declspec(restrict)
73 #else
74 #define RPMALLOC_RESTRICT
75 #endif
76
77 #if ENABLE_OVERRIDE
78
79 typedef struct rp_nothrow_t {
80 int __dummy;
81 } rp_nothrow_t;
82
83 #if USE_IMPLEMENT
84
malloc(size_t size)85 extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL malloc(size_t size) {
86 return rpmalloc(size);
87 }
calloc(size_t count,size_t size)88 extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL calloc(size_t count,
89 size_t size) {
90 return rpcalloc(count, size);
91 }
realloc(void * ptr,size_t size)92 extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL realloc(void *ptr,
93 size_t size) {
94 return rprealloc(ptr, size);
95 }
reallocf(void * ptr,size_t size)96 extern inline void *RPMALLOC_CDECL reallocf(void *ptr, size_t size) {
97 return rprealloc(ptr, size);
98 }
aligned_alloc(size_t alignment,size_t size)99 extern inline void *RPMALLOC_CDECL aligned_alloc(size_t alignment,
100 size_t size) {
101 return rpaligned_alloc(alignment, size);
102 }
memalign(size_t alignment,size_t size)103 extern inline void *RPMALLOC_CDECL memalign(size_t alignment, size_t size) {
104 return rpmemalign(alignment, size);
105 }
posix_memalign(void ** memptr,size_t alignment,size_t size)106 extern inline int RPMALLOC_CDECL posix_memalign(void **memptr, size_t alignment,
107 size_t size) {
108 return rpposix_memalign(memptr, alignment, size);
109 }
free(void * ptr)110 extern inline void RPMALLOC_CDECL free(void *ptr) { rpfree(ptr); }
cfree(void * ptr)111 extern inline void RPMALLOC_CDECL cfree(void *ptr) { rpfree(ptr); }
malloc_usable_size(void * ptr)112 extern inline size_t RPMALLOC_CDECL malloc_usable_size(void *ptr) {
113 return rpmalloc_usable_size(ptr);
114 }
malloc_size(void * ptr)115 extern inline size_t RPMALLOC_CDECL malloc_size(void *ptr) {
116 return rpmalloc_usable_size(ptr);
117 }
118
119 #ifdef _WIN32
_malloc_base(size_t size)120 extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL _malloc_base(size_t size) {
121 return rpmalloc(size);
122 }
_free_base(void * ptr)123 extern inline void RPMALLOC_CDECL _free_base(void *ptr) { rpfree(ptr); }
_calloc_base(size_t count,size_t size)124 extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL _calloc_base(size_t count,
125 size_t size) {
126 return rpcalloc(count, size);
127 }
_msize(void * ptr)128 extern inline size_t RPMALLOC_CDECL _msize(void *ptr) {
129 return rpmalloc_usable_size(ptr);
130 }
_msize_base(void * ptr)131 extern inline size_t RPMALLOC_CDECL _msize_base(void *ptr) {
132 return rpmalloc_usable_size(ptr);
133 }
134 extern inline RPMALLOC_RESTRICT void *RPMALLOC_CDECL
_realloc_base(void * ptr,size_t size)135 _realloc_base(void *ptr, size_t size) {
136 return rprealloc(ptr, size);
137 }
138 #endif
139
140 #ifdef _WIN32
141 // For Windows, #include <rpnew.h> in one source file to get the C++ operator
142 // overrides implemented in your module
143 #else
144 // Overload the C++ operators using the mangled names
145 // (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling) operators
146 // delete and delete[]
147 #define RPDEFVIS __attribute__((visibility("default")))
148 extern void _ZdlPv(void *p);
_ZdlPv(void * p)149 void RPDEFVIS _ZdlPv(void *p) { rpfree(p); }
150 extern void _ZdaPv(void *p);
_ZdaPv(void * p)151 void RPDEFVIS _ZdaPv(void *p) { rpfree(p); }
152 #if ARCH_64BIT
153 // 64-bit operators new and new[], normal and aligned
154 extern void *_Znwm(uint64_t size);
_Znwm(uint64_t size)155 void *RPDEFVIS _Znwm(uint64_t size) { return rpmalloc(size); }
156 extern void *_Znam(uint64_t size);
_Znam(uint64_t size)157 void *RPDEFVIS _Znam(uint64_t size) { return rpmalloc(size); }
158 extern void *_Znwmm(uint64_t size, uint64_t align);
_Znwmm(uint64_t size,uint64_t align)159 void *RPDEFVIS _Znwmm(uint64_t size, uint64_t align) {
160 return rpaligned_alloc(align, size);
161 }
162 extern void *_Znamm(uint64_t size, uint64_t align);
_Znamm(uint64_t size,uint64_t align)163 void *RPDEFVIS _Znamm(uint64_t size, uint64_t align) {
164 return rpaligned_alloc(align, size);
165 }
166 extern void *_ZnwmSt11align_val_t(uint64_t size, uint64_t align);
_ZnwmSt11align_val_t(uint64_t size,uint64_t align)167 void *RPDEFVIS _ZnwmSt11align_val_t(uint64_t size, uint64_t align) {
168 return rpaligned_alloc(align, size);
169 }
170 extern void *_ZnamSt11align_val_t(uint64_t size, uint64_t align);
_ZnamSt11align_val_t(uint64_t size,uint64_t align)171 void *RPDEFVIS _ZnamSt11align_val_t(uint64_t size, uint64_t align) {
172 return rpaligned_alloc(align, size);
173 }
174 extern void *_ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t);
_ZnwmRKSt9nothrow_t(uint64_t size,rp_nothrow_t t)175 void *RPDEFVIS _ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) {
176 (void)sizeof(t);
177 return rpmalloc(size);
178 }
179 extern void *_ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t);
_ZnamRKSt9nothrow_t(uint64_t size,rp_nothrow_t t)180 void *RPDEFVIS _ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) {
181 (void)sizeof(t);
182 return rpmalloc(size);
183 }
184 extern void *_ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
185 rp_nothrow_t t);
_ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size,uint64_t align,rp_nothrow_t t)186 void *RPDEFVIS _ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
187 rp_nothrow_t t) {
188 (void)sizeof(t);
189 return rpaligned_alloc(align, size);
190 }
191 extern void *_ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
192 rp_nothrow_t t);
_ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size,uint64_t align,rp_nothrow_t t)193 void *RPDEFVIS _ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align,
194 rp_nothrow_t t) {
195 (void)sizeof(t);
196 return rpaligned_alloc(align, size);
197 }
198 // 64-bit operators sized delete and delete[], normal and aligned
199 extern void _ZdlPvm(void *p, uint64_t size);
_ZdlPvm(void * p,uint64_t size)200 void RPDEFVIS _ZdlPvm(void *p, uint64_t size) {
201 rpfree(p);
202 (void)sizeof(size);
203 }
204 extern void _ZdaPvm(void *p, uint64_t size);
_ZdaPvm(void * p,uint64_t size)205 void RPDEFVIS _ZdaPvm(void *p, uint64_t size) {
206 rpfree(p);
207 (void)sizeof(size);
208 }
209 extern void _ZdlPvSt11align_val_t(void *p, uint64_t align);
_ZdlPvSt11align_val_t(void * p,uint64_t align)210 void RPDEFVIS _ZdlPvSt11align_val_t(void *p, uint64_t align) {
211 rpfree(p);
212 (void)sizeof(align);
213 }
214 extern void _ZdaPvSt11align_val_t(void *p, uint64_t align);
_ZdaPvSt11align_val_t(void * p,uint64_t align)215 void RPDEFVIS _ZdaPvSt11align_val_t(void *p, uint64_t align) {
216 rpfree(p);
217 (void)sizeof(align);
218 }
219 extern void _ZdlPvmSt11align_val_t(void *p, uint64_t size, uint64_t align);
_ZdlPvmSt11align_val_t(void * p,uint64_t size,uint64_t align)220 void RPDEFVIS _ZdlPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
221 rpfree(p);
222 (void)sizeof(size);
223 (void)sizeof(align);
224 }
225 extern void _ZdaPvmSt11align_val_t(void *p, uint64_t size, uint64_t align);
_ZdaPvmSt11align_val_t(void * p,uint64_t size,uint64_t align)226 void RPDEFVIS _ZdaPvmSt11align_val_t(void *p, uint64_t size, uint64_t align) {
227 rpfree(p);
228 (void)sizeof(size);
229 (void)sizeof(align);
230 }
231 #else
232 // 32-bit operators new and new[], normal and aligned
233 extern void *_Znwj(uint32_t size);
_Znwj(uint32_t size)234 void *RPDEFVIS _Znwj(uint32_t size) { return rpmalloc(size); }
235 extern void *_Znaj(uint32_t size);
_Znaj(uint32_t size)236 void *RPDEFVIS _Znaj(uint32_t size) { return rpmalloc(size); }
237 extern void *_Znwjj(uint32_t size, uint32_t align);
_Znwjj(uint32_t size,uint32_t align)238 void *RPDEFVIS _Znwjj(uint32_t size, uint32_t align) {
239 return rpaligned_alloc(align, size);
240 }
241 extern void *_Znajj(uint32_t size, uint32_t align);
_Znajj(uint32_t size,uint32_t align)242 void *RPDEFVIS _Znajj(uint32_t size, uint32_t align) {
243 return rpaligned_alloc(align, size);
244 }
245 extern void *_ZnwjSt11align_val_t(size_t size, size_t align);
_ZnwjSt11align_val_t(size_t size,size_t align)246 void *RPDEFVIS _ZnwjSt11align_val_t(size_t size, size_t align) {
247 return rpaligned_alloc(align, size);
248 }
249 extern void *_ZnajSt11align_val_t(size_t size, size_t align);
_ZnajSt11align_val_t(size_t size,size_t align)250 void *RPDEFVIS _ZnajSt11align_val_t(size_t size, size_t align) {
251 return rpaligned_alloc(align, size);
252 }
253 extern void *_ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t);
_ZnwjRKSt9nothrow_t(size_t size,rp_nothrow_t t)254 void *RPDEFVIS _ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
255 (void)sizeof(t);
256 return rpmalloc(size);
257 }
258 extern void *_ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t);
_ZnajRKSt9nothrow_t(size_t size,rp_nothrow_t t)259 void *RPDEFVIS _ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t) {
260 (void)sizeof(t);
261 return rpmalloc(size);
262 }
263 extern void *_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
264 rp_nothrow_t t);
_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size,size_t align,rp_nothrow_t t)265 void *RPDEFVIS _ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
266 rp_nothrow_t t) {
267 (void)sizeof(t);
268 return rpaligned_alloc(align, size);
269 }
270 extern void *_ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
271 rp_nothrow_t t);
_ZnajSt11align_val_tRKSt9nothrow_t(size_t size,size_t align,rp_nothrow_t t)272 void *RPDEFVIS _ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align,
273 rp_nothrow_t t) {
274 (void)sizeof(t);
275 return rpaligned_alloc(align, size);
276 }
277 // 32-bit operators sized delete and delete[], normal and aligned
278 extern void _ZdlPvj(void *p, uint64_t size);
_ZdlPvj(void * p,uint64_t size)279 void RPDEFVIS _ZdlPvj(void *p, uint64_t size) {
280 rpfree(p);
281 (void)sizeof(size);
282 }
283 extern void _ZdaPvj(void *p, uint64_t size);
_ZdaPvj(void * p,uint64_t size)284 void RPDEFVIS _ZdaPvj(void *p, uint64_t size) {
285 rpfree(p);
286 (void)sizeof(size);
287 }
288 extern void _ZdlPvSt11align_val_t(void *p, uint32_t align);
_ZdlPvSt11align_val_t(void * p,uint64_t a)289 void RPDEFVIS _ZdlPvSt11align_val_t(void *p, uint64_t a) {
290 rpfree(p);
291 (void)sizeof(align);
292 }
293 extern void _ZdaPvSt11align_val_t(void *p, uint32_t align);
_ZdaPvSt11align_val_t(void * p,uint64_t a)294 void RPDEFVIS _ZdaPvSt11align_val_t(void *p, uint64_t a) {
295 rpfree(p);
296 (void)sizeof(align);
297 }
298 extern void _ZdlPvjSt11align_val_t(void *p, uint32_t size, uint32_t align);
_ZdlPvjSt11align_val_t(void * p,uint64_t size,uint64_t align)299 void RPDEFVIS _ZdlPvjSt11align_val_t(void *p, uint64_t size, uint64_t align) {
300 rpfree(p);
301 (void)sizeof(size);
302 (void)sizeof(a);
303 }
304 extern void _ZdaPvjSt11align_val_t(void *p, uint32_t size, uint32_t align);
_ZdaPvjSt11align_val_t(void * p,uint64_t size,uint64_t align)305 void RPDEFVIS _ZdaPvjSt11align_val_t(void *p, uint64_t size, uint64_t align) {
306 rpfree(p);
307 (void)sizeof(size);
308 (void)sizeof(a);
309 }
310 #endif
311 #endif
312 #endif
313
314 #if USE_INTERPOSE || USE_ALIAS
315
rpmalloc_nothrow(size_t size,rp_nothrow_t t)316 static void *rpmalloc_nothrow(size_t size, rp_nothrow_t t) {
317 (void)sizeof(t);
318 return rpmalloc(size);
319 }
rpaligned_alloc_reverse(size_t size,size_t align)320 static void *rpaligned_alloc_reverse(size_t size, size_t align) {
321 return rpaligned_alloc(align, size);
322 }
rpaligned_alloc_reverse_nothrow(size_t size,size_t align,rp_nothrow_t t)323 static void *rpaligned_alloc_reverse_nothrow(size_t size, size_t align,
324 rp_nothrow_t t) {
325 (void)sizeof(t);
326 return rpaligned_alloc(align, size);
327 }
rpfree_size(void * p,size_t size)328 static void rpfree_size(void *p, size_t size) {
329 (void)sizeof(size);
330 rpfree(p);
331 }
rpfree_aligned(void * p,size_t align)332 static void rpfree_aligned(void *p, size_t align) {
333 (void)sizeof(align);
334 rpfree(p);
335 }
rpfree_size_aligned(void * p,size_t size,size_t align)336 static void rpfree_size_aligned(void *p, size_t size, size_t align) {
337 (void)sizeof(size);
338 (void)sizeof(align);
339 rpfree(p);
340 }
341
342 #endif
343
344 #if USE_INTERPOSE
345
346 __attribute__((used)) static const interpose_t macinterpose_malloc[]
347 __attribute__((section("__DATA, __interpose"))) = {
348 // new and new[]
349 MAC_INTERPOSE_PAIR(rpmalloc, _Znwm),
350 MAC_INTERPOSE_PAIR(rpmalloc, _Znam),
351 MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _Znwmm),
352 MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _Znamm),
353 MAC_INTERPOSE_PAIR(rpmalloc_nothrow, _ZnwmRKSt9nothrow_t),
354 MAC_INTERPOSE_PAIR(rpmalloc_nothrow, _ZnamRKSt9nothrow_t),
355 MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _ZnwmSt11align_val_t),
356 MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _ZnamSt11align_val_t),
357 MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse_nothrow,
358 _ZnwmSt11align_val_tRKSt9nothrow_t),
359 MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse_nothrow,
360 _ZnamSt11align_val_tRKSt9nothrow_t),
361 // delete and delete[]
362 MAC_INTERPOSE_PAIR(rpfree, _ZdlPv), MAC_INTERPOSE_PAIR(rpfree, _ZdaPv),
363 MAC_INTERPOSE_PAIR(rpfree_size, _ZdlPvm),
364 MAC_INTERPOSE_PAIR(rpfree_size, _ZdaPvm),
365 MAC_INTERPOSE_PAIR(rpfree_aligned, _ZdlPvSt11align_val_t),
366 MAC_INTERPOSE_PAIR(rpfree_aligned, _ZdaPvSt11align_val_t),
367 MAC_INTERPOSE_PAIR(rpfree_size_aligned, _ZdlPvmSt11align_val_t),
368 MAC_INTERPOSE_PAIR(rpfree_size_aligned, _ZdaPvmSt11align_val_t),
369 // libc entry points
370 MAC_INTERPOSE_PAIR(rpmalloc, malloc),
371 MAC_INTERPOSE_PAIR(rpmalloc, calloc),
372 MAC_INTERPOSE_PAIR(rprealloc, realloc),
373 MAC_INTERPOSE_PAIR(rprealloc, reallocf),
374 #if defined(__MAC_10_15) && __MAC_OS_X_VERSION_MIN_REQUIRED >= __MAC_10_15
375 MAC_INTERPOSE_PAIR(rpaligned_alloc, aligned_alloc),
376 #endif
377 MAC_INTERPOSE_PAIR(rpmemalign, memalign),
378 MAC_INTERPOSE_PAIR(rpposix_memalign, posix_memalign),
379 MAC_INTERPOSE_PAIR(rpfree, free), MAC_INTERPOSE_PAIR(rpfree, cfree),
380 MAC_INTERPOSE_PAIR(rpmalloc_usable_size, malloc_usable_size),
381 MAC_INTERPOSE_PAIR(rpmalloc_usable_size, malloc_size)};
382
383 #endif
384
385 #if USE_ALIAS
386
387 #define RPALIAS(fn) __attribute__((alias(#fn), used, visibility("default")));
388
389 // Alias the C++ operators using the mangled names
390 // (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling)
391
392 // operators delete and delete[]
_ZdlPv(void * p)393 void _ZdlPv(void *p) RPALIAS(rpfree) void _ZdaPv(void *p) RPALIAS(rpfree)
394
395 #if ARCH_64BIT
396 // 64-bit operators new and new[], normal and aligned
397 void *_Znwm(uint64_t size) RPMALLOC_ATTRIB_MALLOC
398 RPMALLOC_ATTRIB_ALLOC_SIZE(1)
399 RPALIAS(rpmalloc) void *_Znam(uint64_t size) RPMALLOC_ATTRIB_MALLOC
400 RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc) void *_Znwmm(uint64_t size,
401 uint64_t align)
402 RPALIAS(rpaligned_alloc_reverse) void *_Znamm(uint64_t size,
403 uint64_t align)
404 RPALIAS(rpaligned_alloc_reverse) void *_ZnwmSt11align_val_t(
405 size_t size, size_t align)
406 RPALIAS(rpaligned_alloc_reverse) void *_ZnamSt11align_val_t(
407 size_t size, size_t align)
408 RPALIAS(rpaligned_alloc_reverse) void *_ZnwmRKSt9nothrow_t(
409 size_t size, rp_nothrow_t t)
410 RPALIAS(rpmalloc_nothrow) void *_ZnamRKSt9nothrow_t(
411 size_t size,
412 rp_nothrow_t t) RPALIAS(rpmalloc_nothrow) void
413 *_ZnwmSt11align_val_tRKSt9nothrow_t(size_t size,
414 size_t align,
415 rp_nothrow_t t)
416 RPALIAS(rpaligned_alloc_reverse_nothrow) void
417 *_ZnamSt11align_val_tRKSt9nothrow_t(
418 size_t size, size_t align,
419 rp_nothrow_t t)
420 RPALIAS(rpaligned_alloc_reverse_nothrow)
421 // 64-bit operators delete and delete[], sized and aligned
422 void _ZdlPvm(void *p, size_t n) RPALIAS(rpfree_size) void _ZdaPvm(void *p,
423 size_t n)
424 RPALIAS(rpfree_size) void _ZdlPvSt11align_val_t(void *p, size_t a)
425 RPALIAS(rpfree_aligned) void _ZdaPvSt11align_val_t(void *p,
426 size_t a)
427 RPALIAS(rpfree_aligned) void _ZdlPvmSt11align_val_t(void *p,
428 size_t n,
429 size_t a)
430 RPALIAS(rpfree_size_aligned) void _ZdaPvmSt11align_val_t(
431 void *p, size_t n, size_t a)
432 RPALIAS(rpfree_size_aligned)
433 #else
434 // 32-bit operators new and new[], normal and aligned
435 void *_Znwj(uint32_t size) RPMALLOC_ATTRIB_MALLOC
436 RPMALLOC_ATTRIB_ALLOC_SIZE(1)
437 RPALIAS(rpmalloc) void *_Znaj(uint32_t size) RPMALLOC_ATTRIB_MALLOC
438 RPMALLOC_ATTRIB_ALLOC_SIZE(1) RPALIAS(rpmalloc) void *_Znwjj(uint32_t size,
439 uint32_t align)
440 RPALIAS(rpaligned_alloc_reverse) void *_Znajj(uint32_t size,
441 uint32_t align)
442 RPALIAS(rpaligned_alloc_reverse) void *_ZnwjSt11align_val_t(
443 size_t size, size_t align)
444 RPALIAS(rpaligned_alloc_reverse) void *_ZnajSt11align_val_t(
445 size_t size, size_t align)
446 RPALIAS(rpaligned_alloc_reverse) void *_ZnwjRKSt9nothrow_t(
447 size_t size, rp_nothrow_t t)
448 RPALIAS(rpmalloc_nothrow) void *_ZnajRKSt9nothrow_t(
449 size_t size,
450 rp_nothrow_t t) RPALIAS(rpmalloc_nothrow) void
451 *_ZnwjSt11align_val_tRKSt9nothrow_t(size_t size,
452 size_t align,
453 rp_nothrow_t t)
454 RPALIAS(rpaligned_alloc_reverse_nothrow) void
455 *_ZnajSt11align_val_tRKSt9nothrow_t(
456 size_t size, size_t align,
457 rp_nothrow_t t)
458 RPALIAS(rpaligned_alloc_reverse_nothrow)
459 // 32-bit operators delete and delete[], sized and aligned
460 void _ZdlPvj(void *p, size_t n) RPALIAS(rpfree_size) void _ZdaPvj(void *p,
461 size_t n)
462 RPALIAS(rpfree_size) void _ZdlPvSt11align_val_t(void *p, size_t a)
463 RPALIAS(rpfree_aligned) void _ZdaPvSt11align_val_t(void *p,
464 size_t a)
465 RPALIAS(rpfree_aligned) void _ZdlPvjSt11align_val_t(void *p,
466 size_t n,
467 size_t a)
468 RPALIAS(rpfree_size_aligned) void _ZdaPvjSt11align_val_t(
469 void *p, size_t n, size_t a)
470 RPALIAS(rpfree_size_aligned)
471 #endif
472
473 void *malloc(size_t size) RPMALLOC_ATTRIB_MALLOC
474 RPMALLOC_ATTRIB_ALLOC_SIZE(1)
475 RPALIAS(rpmalloc) void *calloc(size_t count, size_t size)
476 RPALIAS(rpcalloc) void *realloc(void *ptr, size_t size)
477 RPALIAS(rprealloc) void *reallocf(void *ptr, size_t size)
478 RPMALLOC_ATTRIB_MALLOC
479 RPMALLOC_ATTRIB_ALLOC_SIZE(2)
480 RPALIAS(rprealloc) void *aligned_alloc(size_t alignment, size_t size)
481 RPALIAS(rpaligned_alloc) void *memalign(
482 size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC
483 RPMALLOC_ATTRIB_ALLOC_SIZE(2)
484 RPALIAS(rpmemalign) int posix_memalign(void **memptr, size_t alignment,
485 size_t size)
486 RPALIAS(rpposix_memalign) void free(void *ptr)
487 RPALIAS(rpfree) void cfree(void *ptr) RPALIAS(rpfree)
488 #if defined(__ANDROID__) || defined(__FreeBSD__)
489 size_t
490 malloc_usable_size(const void *ptr) RPALIAS(rpmalloc_usable_size)
491 #else
492 size_t
493 malloc_usable_size(void *ptr) RPALIAS(rpmalloc_usable_size)
494 #endif
495 size_t malloc_size(void *ptr) RPALIAS(rpmalloc_usable_size)
496
497 #endif
498
499 static inline size_t _rpmalloc_page_size(void) {
500 return _memory_page_size;
501 }
502
503 extern void *RPMALLOC_CDECL reallocarray(void *ptr, size_t count, size_t size);
504
reallocarray(void * ptr,size_t count,size_t size)505 extern void *RPMALLOC_CDECL reallocarray(void *ptr, size_t count, size_t size) {
506 size_t total;
507 #if ENABLE_VALIDATE_ARGS
508 #ifdef _MSC_VER
509 int err = SizeTMult(count, size, &total);
510 if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
511 errno = EINVAL;
512 return 0;
513 }
514 #else
515 int err = __builtin_umull_overflow(count, size, &total);
516 if (err || (total >= MAX_ALLOC_SIZE)) {
517 errno = EINVAL;
518 return 0;
519 }
520 #endif
521 #else
522 total = count * size;
523 #endif
524 return realloc(ptr, total);
525 }
526
valloc(size_t size)527 extern inline void *RPMALLOC_CDECL valloc(size_t size) {
528 get_thread_heap();
529 return rpaligned_alloc(_rpmalloc_page_size(), size);
530 }
531
pvalloc(size_t size)532 extern inline void *RPMALLOC_CDECL pvalloc(size_t size) {
533 get_thread_heap();
534 const size_t page_size = _rpmalloc_page_size();
535 const size_t aligned_size = ((size + page_size - 1) / page_size) * page_size;
536 #if ENABLE_VALIDATE_ARGS
537 if (aligned_size < size) {
538 errno = EINVAL;
539 return 0;
540 }
541 #endif
542 return rpaligned_alloc(_rpmalloc_page_size(), aligned_size);
543 }
544
545 #endif // ENABLE_OVERRIDE
546
547 #if ENABLE_PRELOAD
548
549 #ifdef _WIN32
550
551 #if defined(BUILD_DYNAMIC_LINK) && BUILD_DYNAMIC_LINK
552
553 extern __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE instance,
554 DWORD reason, LPVOID reserved);
555
DllMain(HINSTANCE instance,DWORD reason,LPVOID reserved)556 extern __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE instance,
557 DWORD reason,
558 LPVOID reserved) {
559 (void)sizeof(reserved);
560 (void)sizeof(instance);
561 if (reason == DLL_PROCESS_ATTACH)
562 rpmalloc_initialize();
563 else if (reason == DLL_PROCESS_DETACH)
564 rpmalloc_finalize();
565 else if (reason == DLL_THREAD_ATTACH)
566 rpmalloc_thread_initialize();
567 else if (reason == DLL_THREAD_DETACH)
568 rpmalloc_thread_finalize(1);
569 return TRUE;
570 }
571
572 // end BUILD_DYNAMIC_LINK
573 #else
574
_global_rpmalloc_init(void)575 extern void _global_rpmalloc_init(void) {
576 rpmalloc_set_main_thread();
577 rpmalloc_initialize();
578 }
579
580 #if defined(__clang__) || defined(__GNUC__)
581
initializer(void)582 static void __attribute__((constructor)) initializer(void) {
583 _global_rpmalloc_init();
584 }
585
586 #elif defined(_MSC_VER)
587
_global_rpmalloc_xib(void)588 static int _global_rpmalloc_xib(void) {
589 _global_rpmalloc_init();
590 return 0;
591 }
592
593 #pragma section(".CRT$XIB", read)
594 __declspec(allocate(".CRT$XIB")) void (*_rpmalloc_module_init)(void) =
595 _global_rpmalloc_xib;
596 #if defined(_M_IX86) || defined(__i386__)
597 #pragma comment(linker, "/include:" \
598 "__rpmalloc_module_init")
599 #else
600 #pragma comment(linker, "/include:" \
601 "_rpmalloc_module_init")
602 #endif
603
604 #endif
605
606 // end !BUILD_DYNAMIC_LINK
607 #endif
608
609 #else
610
611 #include <pthread.h>
612 #include <stdint.h>
613 #include <stdlib.h>
614 #include <unistd.h>
615
616 extern void rpmalloc_set_main_thread(void);
617
618 static pthread_key_t destructor_key;
619
620 static void thread_destructor(void *);
621
initializer(void)622 static void __attribute__((constructor)) initializer(void) {
623 rpmalloc_set_main_thread();
624 rpmalloc_initialize();
625 pthread_key_create(&destructor_key, thread_destructor);
626 }
627
finalizer(void)628 static void __attribute__((destructor)) finalizer(void) { rpmalloc_finalize(); }
629
630 typedef struct {
631 void *(*real_start)(void *);
632 void *real_arg;
633 } thread_starter_arg;
634
thread_starter(void * argptr)635 static void *thread_starter(void *argptr) {
636 thread_starter_arg *arg = argptr;
637 void *(*real_start)(void *) = arg->real_start;
638 void *real_arg = arg->real_arg;
639 rpmalloc_thread_initialize();
640 rpfree(argptr);
641 pthread_setspecific(destructor_key, (void *)1);
642 return (*real_start)(real_arg);
643 }
644
thread_destructor(void * value)645 static void thread_destructor(void *value) {
646 (void)sizeof(value);
647 rpmalloc_thread_finalize(1);
648 }
649
650 #ifdef __APPLE__
651
pthread_create_proxy(pthread_t * thread,const pthread_attr_t * attr,void * (* start_routine)(void *),void * arg)652 static int pthread_create_proxy(pthread_t *thread, const pthread_attr_t *attr,
653 void *(*start_routine)(void *), void *arg) {
654 rpmalloc_initialize();
655 thread_starter_arg *starter_arg = rpmalloc(sizeof(thread_starter_arg));
656 starter_arg->real_start = start_routine;
657 starter_arg->real_arg = arg;
658 return pthread_create(thread, attr, thread_starter, starter_arg);
659 }
660
661 MAC_INTERPOSE_SINGLE(pthread_create_proxy, pthread_create);
662
663 #else
664
665 #include <dlfcn.h>
666
pthread_create(pthread_t * thread,const pthread_attr_t * attr,void * (* start_routine)(void *),void * arg)667 int pthread_create(pthread_t *thread, const pthread_attr_t *attr,
668 void *(*start_routine)(void *), void *arg) {
669 #if defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \
670 defined(__NetBSD__) || defined(__DragonFly__) || defined(__APPLE__) || \
671 defined(__HAIKU__)
672 char fname[] = "pthread_create";
673 #else
674 char fname[] = "_pthread_create";
675 #endif
676 void *real_pthread_create = dlsym(RTLD_NEXT, fname);
677 rpmalloc_thread_initialize();
678 thread_starter_arg *starter_arg = rpmalloc(sizeof(thread_starter_arg));
679 starter_arg->real_start = start_routine;
680 starter_arg->real_arg = arg;
681 return (*(int (*)(pthread_t *, const pthread_attr_t *, void *(*)(void *),
682 void *))real_pthread_create)(thread, attr, thread_starter,
683 starter_arg);
684 }
685
686 #endif
687
688 #endif
689
690 #endif
691
692 #if ENABLE_OVERRIDE
693
694 #if defined(__GLIBC__) && defined(__linux__)
695
696 void *__libc_malloc(size_t size) RPMALLOC_ATTRIB_MALLOC
697 RPMALLOC_ATTRIB_ALLOC_SIZE(1)
698 RPALIAS(rpmalloc) void *__libc_calloc(size_t count, size_t size)
699 RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(1, 2)
700 RPALIAS(rpcalloc) void *__libc_realloc(void *p, size_t size)
701 RPMALLOC_ATTRIB_MALLOC
702 RPMALLOC_ATTRIB_ALLOC_SIZE(2) RPALIAS(rprealloc) void __libc_free(void *p)
703 RPALIAS(rpfree) void __libc_cfree(void *p)
704 RPALIAS(rpfree) void *__libc_memalign(size_t align, size_t size)
705 RPMALLOC_ATTRIB_MALLOC
706 RPMALLOC_ATTRIB_ALLOC_SIZE(2)
707 RPALIAS(rpmemalign) int __posix_memalign(void **p, size_t align,
708 size_t size)
709 RPALIAS(rpposix_memalign)
710
711 extern void *__libc_valloc(size_t size);
712 extern void *__libc_pvalloc(size_t size);
713
__libc_valloc(size_t size)714 void *__libc_valloc(size_t size) { return valloc(size); }
715
__libc_pvalloc(size_t size)716 void *__libc_pvalloc(size_t size) { return pvalloc(size); }
717
718 #endif
719
720 #endif
721
722 #if (defined(__GNUC__) || defined(__clang__))
723 #pragma GCC visibility pop
724 #endif
725