1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 #ifdef _WIN32 5 # ifdef _WIN64 6 # define FMT64_PREFIX "ll" 7 # define FMTPTR_PREFIX "ll" 8 # else 9 # define FMT64_PREFIX "ll" 10 # define FMTPTR_PREFIX "" 11 # endif 12 # define FMTd32 "d" 13 # define FMTu32 "u" 14 # define FMTx32 "x" 15 # define FMTd64 FMT64_PREFIX "d" 16 # define FMTu64 FMT64_PREFIX "u" 17 # define FMTx64 FMT64_PREFIX "x" 18 # define FMTdPTR FMTPTR_PREFIX "d" 19 # define FMTuPTR FMTPTR_PREFIX "u" 20 # define FMTxPTR FMTPTR_PREFIX "x" 21 #else 22 # include <inttypes.h> 23 # define FMTd32 PRId32 24 # define FMTu32 PRIu32 25 # define FMTx32 PRIx32 26 # define FMTd64 PRId64 27 # define FMTu64 PRIu64 28 # define FMTx64 PRIx64 29 # define FMTdPTR PRIdPTR 30 # define FMTuPTR PRIuPTR 31 # define FMTxPTR PRIxPTR 32 #endif 33 34 /* Size of stack-allocated buffer passed to buferror(). */ 35 #define BUFERROR_BUF 64 36 37 /* 38 * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be 39 * large enough for all possible uses within jemalloc. 40 */ 41 #define MALLOC_PRINTF_BUFSIZE 4096 42 43 /* Junk fill patterns. */ 44 #define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5) 45 #define JEMALLOC_FREE_JUNK ((uint8_t)0x5a) 46 47 /* 48 * Wrap a cpp argument that contains commas such that it isn't broken up into 49 * multiple arguments. 50 */ 51 #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ 52 53 /* 54 * Silence compiler warnings due to uninitialized values. This is used 55 * wherever the compiler fails to recognize that the variable is never used 56 * uninitialized. 57 */ 58 #ifdef JEMALLOC_CC_SILENCE 59 # define JEMALLOC_CC_SILENCE_INIT(v) = v 60 #else 61 # define JEMALLOC_CC_SILENCE_INIT(v) 62 #endif 63 64 #define JEMALLOC_GNUC_PREREQ(major, minor) \ 65 (!defined(__clang__) && \ 66 (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))) 67 #ifndef __has_builtin 68 # define __has_builtin(builtin) (0) 69 #endif 70 #define JEMALLOC_CLANG_HAS_BUILTIN(builtin) \ 71 (defined(__clang__) && __has_builtin(builtin)) 72 73 #ifdef __GNUC__ 74 # define likely(x) __builtin_expect(!!(x), 1) 75 # define unlikely(x) __builtin_expect(!!(x), 0) 76 # if JEMALLOC_GNUC_PREREQ(4, 6) || \ 77 JEMALLOC_CLANG_HAS_BUILTIN(__builtin_unreachable) 78 # define unreachable() __builtin_unreachable() 79 # else 80 # define unreachable() abort() 81 # endif 82 #else 83 # define likely(x) !!(x) 84 # define unlikely(x) !!(x) 85 # define unreachable() abort() 86 #endif 87 88 #include "jemalloc/internal/assert.h" 89 90 /* Use to assert a particular configuration, e.g., cassert(config_debug). */ 91 #define cassert(c) do { \ 92 if (unlikely(!(c))) \ 93 not_reached(); \ 94 } while (0) 95 96 #endif /* JEMALLOC_H_TYPES */ 97 /******************************************************************************/ 98 #ifdef JEMALLOC_H_STRUCTS 99 100 #endif /* JEMALLOC_H_STRUCTS */ 101 /******************************************************************************/ 102 #ifdef JEMALLOC_H_EXTERNS 103 104 int buferror(int err, char *buf, size_t buflen); 105 uintmax_t malloc_strtoumax(const char *restrict nptr, 106 char **restrict endptr, int base); 107 void malloc_write(const char *s); 108 109 /* 110 * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating 111 * point math. 112 */ 113 size_t malloc_vsnprintf(char *str, size_t size, const char *format, 114 va_list ap); 115 size_t malloc_snprintf(char *str, size_t size, const char *format, ...) 116 JEMALLOC_FORMAT_PRINTF(3, 4); 117 void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, 118 const char *format, va_list ap); 119 void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, 120 const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4); 121 void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2); 122 123 #endif /* JEMALLOC_H_EXTERNS */ 124 /******************************************************************************/ 125 #ifdef JEMALLOC_H_INLINES 126 127 #ifndef JEMALLOC_ENABLE_INLINE 128 unsigned ffs_llu(unsigned long long bitmap); 129 unsigned ffs_lu(unsigned long bitmap); 130 unsigned ffs_u(unsigned bitmap); 131 unsigned ffs_zu(size_t bitmap); 132 unsigned ffs_u64(uint64_t bitmap); 133 unsigned ffs_u32(uint32_t bitmap); 134 uint64_t pow2_ceil_u64(uint64_t x); 135 uint32_t pow2_ceil_u32(uint32_t x); 136 size_t pow2_ceil_zu(size_t x); 137 unsigned lg_floor(size_t x); 138 void set_errno(int errnum); 139 int get_errno(void); 140 #endif 141 142 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) 143 144 /* Sanity check. */ 145 #if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \ 146 || !defined(JEMALLOC_INTERNAL_FFS) 147 # error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure 148 #endif 149 150 JEMALLOC_ALWAYS_INLINE unsigned 151 ffs_llu(unsigned long long bitmap) 152 { 153 154 return (JEMALLOC_INTERNAL_FFSLL(bitmap)); 155 } 156 157 JEMALLOC_ALWAYS_INLINE unsigned 158 ffs_lu(unsigned long bitmap) 159 { 160 161 return (JEMALLOC_INTERNAL_FFSL(bitmap)); 162 } 163 164 JEMALLOC_ALWAYS_INLINE unsigned 165 ffs_u(unsigned bitmap) 166 { 167 168 return (JEMALLOC_INTERNAL_FFS(bitmap)); 169 } 170 171 JEMALLOC_ALWAYS_INLINE unsigned 172 ffs_zu(size_t bitmap) 173 { 174 175 #if LG_SIZEOF_PTR == LG_SIZEOF_INT 176 return (ffs_u(bitmap)); 177 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG 178 return (ffs_lu(bitmap)); 179 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG 180 return (ffs_llu(bitmap)); 181 #else 182 #error No implementation for size_t ffs() 183 #endif 184 } 185 186 JEMALLOC_ALWAYS_INLINE unsigned 187 ffs_u64(uint64_t bitmap) 188 { 189 190 #if LG_SIZEOF_LONG == 3 191 return (ffs_lu(bitmap)); 192 #elif LG_SIZEOF_LONG_LONG == 3 193 return (ffs_llu(bitmap)); 194 #else 195 #error No implementation for 64-bit ffs() 196 #endif 197 } 198 199 JEMALLOC_ALWAYS_INLINE unsigned 200 ffs_u32(uint32_t bitmap) 201 { 202 203 #if LG_SIZEOF_INT == 2 204 return (ffs_u(bitmap)); 205 #else 206 #error No implementation for 32-bit ffs() 207 #endif 208 return (ffs_u(bitmap)); 209 } 210 211 JEMALLOC_INLINE uint64_t 212 pow2_ceil_u64(uint64_t x) 213 { 214 215 x--; 216 x |= x >> 1; 217 x |= x >> 2; 218 x |= x >> 4; 219 x |= x >> 8; 220 x |= x >> 16; 221 x |= x >> 32; 222 x++; 223 return (x); 224 } 225 226 JEMALLOC_INLINE uint32_t 227 pow2_ceil_u32(uint32_t x) 228 { 229 230 x--; 231 x |= x >> 1; 232 x |= x >> 2; 233 x |= x >> 4; 234 x |= x >> 8; 235 x |= x >> 16; 236 x++; 237 return (x); 238 } 239 240 /* Compute the smallest power of 2 that is >= x. */ 241 JEMALLOC_INLINE size_t 242 pow2_ceil_zu(size_t x) 243 { 244 245 #if (LG_SIZEOF_PTR == 3) 246 return (pow2_ceil_u64(x)); 247 #else 248 return (pow2_ceil_u32(x)); 249 #endif 250 } 251 252 #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) 253 JEMALLOC_INLINE unsigned 254 lg_floor(size_t x) 255 { 256 size_t ret; 257 258 assert(x != 0); 259 260 asm ("bsr %1, %0" 261 : "=r"(ret) // Outputs. 262 : "r"(x) // Inputs. 263 ); 264 assert(ret < UINT_MAX); 265 return ((unsigned)ret); 266 } 267 #elif (defined(_MSC_VER)) 268 JEMALLOC_INLINE unsigned 269 lg_floor(size_t x) 270 { 271 unsigned long ret; 272 273 assert(x != 0); 274 275 #if (LG_SIZEOF_PTR == 3) 276 _BitScanReverse64(&ret, x); 277 #elif (LG_SIZEOF_PTR == 2) 278 _BitScanReverse(&ret, x); 279 #else 280 # error "Unsupported type size for lg_floor()" 281 #endif 282 assert(ret < UINT_MAX); 283 return ((unsigned)ret); 284 } 285 #elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) 286 JEMALLOC_INLINE unsigned 287 lg_floor(size_t x) 288 { 289 290 assert(x != 0); 291 292 #if (LG_SIZEOF_PTR == LG_SIZEOF_INT) 293 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x)); 294 #elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG) 295 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x)); 296 #else 297 # error "Unsupported type size for lg_floor()" 298 #endif 299 } 300 #else 301 JEMALLOC_INLINE unsigned 302 lg_floor(size_t x) 303 { 304 305 assert(x != 0); 306 307 x |= (x >> 1); 308 x |= (x >> 2); 309 x |= (x >> 4); 310 x |= (x >> 8); 311 x |= (x >> 16); 312 #if (LG_SIZEOF_PTR == 3) 313 x |= (x >> 32); 314 #endif 315 if (x == SIZE_T_MAX) 316 return ((8 << LG_SIZEOF_PTR) - 1); 317 x++; 318 return (ffs_zu(x) - 2); 319 } 320 #endif 321 322 /* Set error code. */ 323 JEMALLOC_INLINE void 324 set_errno(int errnum) 325 { 326 327 #ifdef _WIN32 328 SetLastError(errnum); 329 #else 330 errno = errnum; 331 #endif 332 } 333 334 /* Get last error code. */ 335 JEMALLOC_INLINE int 336 get_errno(void) 337 { 338 339 #ifdef _WIN32 340 return (GetLastError()); 341 #else 342 return (errno); 343 #endif 344 } 345 #endif 346 347 #endif /* JEMALLOC_H_INLINES */ 348 /******************************************************************************/ 349