1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 #ifdef _WIN32 5 # ifdef _WIN64 6 # define FMT64_PREFIX "ll" 7 # define FMTPTR_PREFIX "ll" 8 # else 9 # define FMT64_PREFIX "ll" 10 # define FMTPTR_PREFIX "" 11 # endif 12 # define FMTd32 "d" 13 # define FMTu32 "u" 14 # define FMTx32 "x" 15 # define FMTd64 FMT64_PREFIX "d" 16 # define FMTu64 FMT64_PREFIX "u" 17 # define FMTx64 FMT64_PREFIX "x" 18 # define FMTdPTR FMTPTR_PREFIX "d" 19 # define FMTuPTR FMTPTR_PREFIX "u" 20 # define FMTxPTR FMTPTR_PREFIX "x" 21 #else 22 # include <inttypes.h> 23 # define FMTd32 PRId32 24 # define FMTu32 PRIu32 25 # define FMTx32 PRIx32 26 # define FMTd64 PRId64 27 # define FMTu64 PRIu64 28 # define FMTx64 PRIx64 29 # define FMTdPTR PRIdPTR 30 # define FMTuPTR PRIuPTR 31 # define FMTxPTR PRIxPTR 32 #endif 33 34 /* Size of stack-allocated buffer passed to buferror(). */ 35 #define BUFERROR_BUF 64 36 37 /* 38 * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be 39 * large enough for all possible uses within jemalloc. 40 */ 41 #define MALLOC_PRINTF_BUFSIZE 4096 42 43 /* Junk fill patterns. */ 44 #define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5) 45 #define JEMALLOC_FREE_JUNK ((uint8_t)0x5a) 46 47 /* 48 * Wrap a cpp argument that contains commas such that it isn't broken up into 49 * multiple arguments. 50 */ 51 #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ 52 53 /* 54 * Silence compiler warnings due to uninitialized values. This is used 55 * wherever the compiler fails to recognize that the variable is never used 56 * uninitialized. 57 */ 58 #ifdef JEMALLOC_CC_SILENCE 59 # define JEMALLOC_CC_SILENCE_INIT(v) = v 60 #else 61 # define JEMALLOC_CC_SILENCE_INIT(v) 62 #endif 63 64 #ifdef __GNUC__ 65 # define likely(x) __builtin_expect(!!(x), 1) 66 # define unlikely(x) __builtin_expect(!!(x), 0) 67 #else 68 # define likely(x) !!(x) 69 # define unlikely(x) !!(x) 70 #endif 71 72 #if !defined(JEMALLOC_INTERNAL_UNREACHABLE) 73 # error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure 74 #endif 75 76 #define unreachable() JEMALLOC_INTERNAL_UNREACHABLE() 77 78 #include "jemalloc/internal/assert.h" 79 80 /* Use to assert a particular configuration, e.g., cassert(config_debug). */ 81 #define cassert(c) do { \ 82 if (unlikely(!(c))) \ 83 not_reached(); \ 84 } while (0) 85 86 #endif /* JEMALLOC_H_TYPES */ 87 /******************************************************************************/ 88 #ifdef JEMALLOC_H_STRUCTS 89 90 #endif /* JEMALLOC_H_STRUCTS */ 91 /******************************************************************************/ 92 #ifdef JEMALLOC_H_EXTERNS 93 94 int buferror(int err, char *buf, size_t buflen); 95 uintmax_t malloc_strtoumax(const char *restrict nptr, 96 char **restrict endptr, int base); 97 void malloc_write(const char *s); 98 99 /* 100 * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating 101 * point math. 102 */ 103 size_t malloc_vsnprintf(char *str, size_t size, const char *format, 104 va_list ap); 105 size_t malloc_snprintf(char *str, size_t size, const char *format, ...) 106 JEMALLOC_FORMAT_PRINTF(3, 4); 107 void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, 108 const char *format, va_list ap); 109 void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, 110 const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4); 111 void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2); 112 113 #endif /* JEMALLOC_H_EXTERNS */ 114 /******************************************************************************/ 115 #ifdef JEMALLOC_H_INLINES 116 117 #ifndef JEMALLOC_ENABLE_INLINE 118 unsigned ffs_llu(unsigned long long bitmap); 119 unsigned ffs_lu(unsigned long bitmap); 120 unsigned ffs_u(unsigned bitmap); 121 unsigned ffs_zu(size_t bitmap); 122 unsigned ffs_u64(uint64_t bitmap); 123 unsigned ffs_u32(uint32_t bitmap); 124 uint64_t pow2_ceil_u64(uint64_t x); 125 uint32_t pow2_ceil_u32(uint32_t x); 126 size_t pow2_ceil_zu(size_t x); 127 unsigned lg_floor(size_t x); 128 void set_errno(int errnum); 129 int get_errno(void); 130 #endif 131 132 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) 133 134 /* Sanity check. */ 135 #if !defined(JEMALLOC_INTERNAL_FFSLL) || !defined(JEMALLOC_INTERNAL_FFSL) \ 136 || !defined(JEMALLOC_INTERNAL_FFS) 137 # error JEMALLOC_INTERNAL_FFS{,L,LL} should have been defined by configure 138 #endif 139 140 JEMALLOC_ALWAYS_INLINE unsigned 141 ffs_llu(unsigned long long bitmap) 142 { 143 144 return (JEMALLOC_INTERNAL_FFSLL(bitmap)); 145 } 146 147 JEMALLOC_ALWAYS_INLINE unsigned 148 ffs_lu(unsigned long bitmap) 149 { 150 151 return (JEMALLOC_INTERNAL_FFSL(bitmap)); 152 } 153 154 JEMALLOC_ALWAYS_INLINE unsigned 155 ffs_u(unsigned bitmap) 156 { 157 158 return (JEMALLOC_INTERNAL_FFS(bitmap)); 159 } 160 161 JEMALLOC_ALWAYS_INLINE unsigned 162 ffs_zu(size_t bitmap) 163 { 164 165 #if LG_SIZEOF_PTR == LG_SIZEOF_INT 166 return (ffs_u(bitmap)); 167 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG 168 return (ffs_lu(bitmap)); 169 #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG 170 return (ffs_llu(bitmap)); 171 #else 172 #error No implementation for size_t ffs() 173 #endif 174 } 175 176 JEMALLOC_ALWAYS_INLINE unsigned 177 ffs_u64(uint64_t bitmap) 178 { 179 180 #if LG_SIZEOF_LONG == 3 181 return (ffs_lu(bitmap)); 182 #elif LG_SIZEOF_LONG_LONG == 3 183 return (ffs_llu(bitmap)); 184 #else 185 #error No implementation for 64-bit ffs() 186 #endif 187 } 188 189 JEMALLOC_ALWAYS_INLINE unsigned 190 ffs_u32(uint32_t bitmap) 191 { 192 193 #if LG_SIZEOF_INT == 2 194 return (ffs_u(bitmap)); 195 #else 196 #error No implementation for 32-bit ffs() 197 #endif 198 return (ffs_u(bitmap)); 199 } 200 201 JEMALLOC_INLINE uint64_t 202 pow2_ceil_u64(uint64_t x) 203 { 204 205 x--; 206 x |= x >> 1; 207 x |= x >> 2; 208 x |= x >> 4; 209 x |= x >> 8; 210 x |= x >> 16; 211 x |= x >> 32; 212 x++; 213 return (x); 214 } 215 216 JEMALLOC_INLINE uint32_t 217 pow2_ceil_u32(uint32_t x) 218 { 219 220 x--; 221 x |= x >> 1; 222 x |= x >> 2; 223 x |= x >> 4; 224 x |= x >> 8; 225 x |= x >> 16; 226 x++; 227 return (x); 228 } 229 230 /* Compute the smallest power of 2 that is >= x. */ 231 JEMALLOC_INLINE size_t 232 pow2_ceil_zu(size_t x) 233 { 234 235 #if (LG_SIZEOF_PTR == 3) 236 return (pow2_ceil_u64(x)); 237 #else 238 return (pow2_ceil_u32(x)); 239 #endif 240 } 241 242 #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) 243 JEMALLOC_INLINE unsigned 244 lg_floor(size_t x) 245 { 246 size_t ret; 247 248 assert(x != 0); 249 250 asm ("bsr %1, %0" 251 : "=r"(ret) // Outputs. 252 : "r"(x) // Inputs. 253 ); 254 assert(ret < UINT_MAX); 255 return ((unsigned)ret); 256 } 257 #elif (defined(_MSC_VER)) 258 JEMALLOC_INLINE unsigned 259 lg_floor(size_t x) 260 { 261 unsigned long ret; 262 263 assert(x != 0); 264 265 #if (LG_SIZEOF_PTR == 3) 266 _BitScanReverse64(&ret, x); 267 #elif (LG_SIZEOF_PTR == 2) 268 _BitScanReverse(&ret, x); 269 #else 270 # error "Unsupported type size for lg_floor()" 271 #endif 272 assert(ret < UINT_MAX); 273 return ((unsigned)ret); 274 } 275 #elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) 276 JEMALLOC_INLINE unsigned 277 lg_floor(size_t x) 278 { 279 280 assert(x != 0); 281 282 #if (LG_SIZEOF_PTR == LG_SIZEOF_INT) 283 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x)); 284 #elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG) 285 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x)); 286 #else 287 # error "Unsupported type size for lg_floor()" 288 #endif 289 } 290 #else 291 JEMALLOC_INLINE unsigned 292 lg_floor(size_t x) 293 { 294 295 assert(x != 0); 296 297 x |= (x >> 1); 298 x |= (x >> 2); 299 x |= (x >> 4); 300 x |= (x >> 8); 301 x |= (x >> 16); 302 #if (LG_SIZEOF_PTR == 3) 303 x |= (x >> 32); 304 #endif 305 if (x == SIZE_T_MAX) 306 return ((8 << LG_SIZEOF_PTR) - 1); 307 x++; 308 return (ffs_zu(x) - 2); 309 } 310 #endif 311 312 /* Set error code. */ 313 JEMALLOC_INLINE void 314 set_errno(int errnum) 315 { 316 317 #ifdef _WIN32 318 SetLastError(errnum); 319 #else 320 errno = errnum; 321 #endif 322 } 323 324 /* Get last error code. */ 325 JEMALLOC_INLINE int 326 get_errno(void) 327 { 328 329 #ifdef _WIN32 330 return (GetLastError()); 331 #else 332 return (errno); 333 #endif 334 } 335 #endif 336 337 #endif /* JEMALLOC_H_INLINES */ 338 /******************************************************************************/ 339