1 #include <mutex> 2 #include <new> 3 4 #define JEMALLOC_CPP_CPP_ 5 #ifdef __cplusplus 6 extern "C" { 7 #endif 8 9 #include "jemalloc/internal/jemalloc_preamble.h" 10 #include "jemalloc/internal/jemalloc_internal_includes.h" 11 12 #ifdef __cplusplus 13 } 14 #endif 15 16 // All operators in this file are exported. 17 18 // Possibly alias hidden versions of malloc and sdallocx to avoid an extra plt 19 // thunk? 20 // 21 // extern __typeof (sdallocx) sdallocx_int 22 // __attribute ((alias ("sdallocx"), 23 // visibility ("hidden"))); 24 // 25 // ... but it needs to work with jemalloc namespaces. 26 27 void *operator new(std::size_t size); 28 void *operator new[](std::size_t size); 29 void *operator new(std::size_t size, const std::nothrow_t &) noexcept; 30 void *operator new[](std::size_t size, const std::nothrow_t &) noexcept; 31 void operator delete(void *ptr) noexcept; 32 void operator delete[](void *ptr) noexcept; 33 void operator delete(void *ptr, const std::nothrow_t &) noexcept; 34 void operator delete[](void *ptr, const std::nothrow_t &) noexcept; 35 36 #if __cpp_sized_deallocation >= 201309 37 /* C++14's sized-delete operators. */ 38 void operator delete(void *ptr, std::size_t size) noexcept; 39 void operator delete[](void *ptr, std::size_t size) noexcept; 40 #endif 41 42 #if __cpp_aligned_new >= 201606 43 /* C++17's over-aligned operators. */ 44 void *operator new(std::size_t size, std::align_val_t); 45 void *operator new(std::size_t size, std::align_val_t, const std::nothrow_t &) noexcept; 46 void *operator new[](std::size_t size, std::align_val_t); 47 void *operator new[](std::size_t size, std::align_val_t, const std::nothrow_t &) noexcept; 48 void operator delete(void* ptr, std::align_val_t) noexcept; 49 void operator delete(void* ptr, std::align_val_t, const std::nothrow_t &) noexcept; 50 void operator delete(void* ptr, std::size_t size, std::align_val_t al) noexcept; 51 void operator delete[](void* ptr, std::align_val_t) noexcept; 52 void operator delete[](void* ptr, std::align_val_t, const std::nothrow_t &) noexcept; 53 void operator delete[](void* ptr, std::size_t size, std::align_val_t al) noexcept; 54 #endif 55 56 JEMALLOC_NOINLINE 57 static void * 58 handleOOM(std::size_t size, bool nothrow) { 59 if (opt_experimental_infallible_new) { 60 safety_check_fail("<jemalloc>: Allocation failed and " 61 "opt.experimental_infallible_new is true. Aborting.\n"); 62 return nullptr; 63 } 64 65 void *ptr = nullptr; 66 67 while (ptr == nullptr) { 68 std::new_handler handler; 69 // GCC-4.8 and clang 4.0 do not have std::get_new_handler. 70 { 71 static std::mutex mtx; 72 std::lock_guard<std::mutex> lock(mtx); 73 74 handler = std::set_new_handler(nullptr); 75 std::set_new_handler(handler); 76 } 77 if (handler == nullptr) 78 break; 79 80 try { 81 handler(); 82 } catch (const std::bad_alloc &) { 83 break; 84 } 85 86 ptr = je_malloc(size); 87 } 88 89 if (ptr == nullptr && !nothrow) 90 std::__throw_bad_alloc(); 91 return ptr; 92 } 93 94 template <bool IsNoExcept> 95 JEMALLOC_NOINLINE 96 static void * 97 fallback_impl(std::size_t size) noexcept(IsNoExcept) { 98 void *ptr = malloc_default(size); 99 if (likely(ptr != nullptr)) { 100 return ptr; 101 } 102 return handleOOM(size, IsNoExcept); 103 } 104 105 template <bool IsNoExcept> 106 JEMALLOC_ALWAYS_INLINE 107 void * 108 newImpl(std::size_t size) noexcept(IsNoExcept) { 109 return imalloc_fastpath(size, &fallback_impl<IsNoExcept>); 110 } 111 112 void * 113 operator new(std::size_t size) { 114 return newImpl<false>(size); 115 } 116 117 void * 118 operator new[](std::size_t size) { 119 return newImpl<false>(size); 120 } 121 122 void * 123 operator new(std::size_t size, const std::nothrow_t &) noexcept { 124 return newImpl<true>(size); 125 } 126 127 void * 128 operator new[](std::size_t size, const std::nothrow_t &) noexcept { 129 return newImpl<true>(size); 130 } 131 132 #if __cpp_aligned_new >= 201606 133 134 template <bool IsNoExcept> 135 JEMALLOC_ALWAYS_INLINE 136 void * 137 alignedNewImpl(std::size_t size, std::align_val_t alignment) noexcept(IsNoExcept) { 138 void *ptr = je_aligned_alloc(static_cast<std::size_t>(alignment), size); 139 if (likely(ptr != nullptr)) { 140 return ptr; 141 } 142 143 return handleOOM(size, IsNoExcept); 144 } 145 146 void * 147 operator new(std::size_t size, std::align_val_t alignment) { 148 return alignedNewImpl<false>(size, alignment); 149 } 150 151 void * 152 operator new[](std::size_t size, std::align_val_t alignment) { 153 return alignedNewImpl<false>(size, alignment); 154 } 155 156 void * 157 operator new(std::size_t size, std::align_val_t alignment, const std::nothrow_t &) noexcept { 158 return alignedNewImpl<true>(size, alignment); 159 } 160 161 void * 162 operator new[](std::size_t size, std::align_val_t alignment, const std::nothrow_t &) noexcept { 163 return alignedNewImpl<true>(size, alignment); 164 } 165 166 #endif // __cpp_aligned_new 167 168 void 169 operator delete(void *ptr) noexcept { 170 je_free(ptr); 171 } 172 173 void 174 operator delete[](void *ptr) noexcept { 175 je_free(ptr); 176 } 177 178 void 179 operator delete(void *ptr, const std::nothrow_t &) noexcept { 180 je_free(ptr); 181 } 182 183 void operator delete[](void *ptr, const std::nothrow_t &) noexcept { 184 je_free(ptr); 185 } 186 187 #if __cpp_sized_deallocation >= 201309 188 189 JEMALLOC_ALWAYS_INLINE 190 void 191 sizedDeleteImpl(void* ptr, std::size_t size) noexcept { 192 if (unlikely(ptr == nullptr)) { 193 return; 194 } 195 je_sdallocx_noflags(ptr, size); 196 } 197 198 void 199 operator delete(void *ptr, std::size_t size) noexcept { 200 sizedDeleteImpl(ptr, size); 201 } 202 203 void 204 operator delete[](void *ptr, std::size_t size) noexcept { 205 sizedDeleteImpl(ptr, size); 206 } 207 208 #endif // __cpp_sized_deallocation 209 210 #if __cpp_aligned_new >= 201606 211 212 JEMALLOC_ALWAYS_INLINE 213 void 214 alignedSizedDeleteImpl(void* ptr, std::size_t size, std::align_val_t alignment) noexcept { 215 if (config_debug) { 216 assert(((size_t)alignment & ((size_t)alignment - 1)) == 0); 217 } 218 if (unlikely(ptr == nullptr)) { 219 return; 220 } 221 je_sdallocx(ptr, size, MALLOCX_ALIGN(alignment)); 222 } 223 224 void 225 operator delete(void* ptr, std::align_val_t) noexcept { 226 je_free(ptr); 227 } 228 229 void 230 operator delete[](void* ptr, std::align_val_t) noexcept { 231 je_free(ptr); 232 } 233 234 void 235 operator delete(void* ptr, std::align_val_t, const std::nothrow_t&) noexcept { 236 je_free(ptr); 237 } 238 239 void 240 operator delete[](void* ptr, std::align_val_t, const std::nothrow_t&) noexcept { 241 je_free(ptr); 242 } 243 244 void 245 operator delete(void* ptr, std::size_t size, std::align_val_t alignment) noexcept { 246 alignedSizedDeleteImpl(ptr, size, alignment); 247 } 248 249 void 250 operator delete[](void* ptr, std::size_t size, std::align_val_t alignment) noexcept { 251 alignedSizedDeleteImpl(ptr, size, alignment); 252 } 253 254 #endif // __cpp_aligned_new 255