1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 #ifdef _WIN32 5 # ifdef _WIN64 6 # define FMT64_PREFIX "ll" 7 # define FMTPTR_PREFIX "ll" 8 # else 9 # define FMT64_PREFIX "ll" 10 # define FMTPTR_PREFIX "" 11 # endif 12 # define FMTd32 "d" 13 # define FMTu32 "u" 14 # define FMTx32 "x" 15 # define FMTd64 FMT64_PREFIX "d" 16 # define FMTu64 FMT64_PREFIX "u" 17 # define FMTx64 FMT64_PREFIX "x" 18 # define FMTdPTR FMTPTR_PREFIX "d" 19 # define FMTuPTR FMTPTR_PREFIX "u" 20 # define FMTxPTR FMTPTR_PREFIX "x" 21 #else 22 # include <inttypes.h> 23 # define FMTd32 PRId32 24 # define FMTu32 PRIu32 25 # define FMTx32 PRIx32 26 # define FMTd64 PRId64 27 # define FMTu64 PRIu64 28 # define FMTx64 PRIx64 29 # define FMTdPTR PRIdPTR 30 # define FMTuPTR PRIuPTR 31 # define FMTxPTR PRIxPTR 32 #endif 33 34 /* Size of stack-allocated buffer passed to buferror(). */ 35 #define BUFERROR_BUF 64 36 37 /* 38 * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be 39 * large enough for all possible uses within jemalloc. 40 */ 41 #define MALLOC_PRINTF_BUFSIZE 4096 42 43 /* 44 * Wrap a cpp argument that contains commas such that it isn't broken up into 45 * multiple arguments. 46 */ 47 #define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ 48 49 /* 50 * Silence compiler warnings due to uninitialized values. This is used 51 * wherever the compiler fails to recognize that the variable is never used 52 * uninitialized. 53 */ 54 #ifdef JEMALLOC_CC_SILENCE 55 # define JEMALLOC_CC_SILENCE_INIT(v) = v 56 #else 57 # define JEMALLOC_CC_SILENCE_INIT(v) 58 #endif 59 60 #define JEMALLOC_GNUC_PREREQ(major, minor) \ 61 (!defined(__clang__) && \ 62 (__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor)))) 63 #ifndef __has_builtin 64 # define __has_builtin(builtin) (0) 65 #endif 66 #define JEMALLOC_CLANG_HAS_BUILTIN(builtin) \ 67 (defined(__clang__) && __has_builtin(builtin)) 68 69 #ifdef __GNUC__ 70 # define likely(x) __builtin_expect(!!(x), 1) 71 # define unlikely(x) __builtin_expect(!!(x), 0) 72 # if JEMALLOC_GNUC_PREREQ(4, 6) || \ 73 JEMALLOC_CLANG_HAS_BUILTIN(__builtin_unreachable) 74 # define unreachable() __builtin_unreachable() 75 # else 76 # define unreachable() 77 # endif 78 #else 79 # define likely(x) !!(x) 80 # define unlikely(x) !!(x) 81 # define unreachable() 82 #endif 83 84 /* 85 * Define a custom assert() in order to reduce the chances of deadlock during 86 * assertion failure. 87 */ 88 #ifndef assert 89 #define assert(e) do { \ 90 if (unlikely(config_debug && !(e))) { \ 91 malloc_printf( \ 92 "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", \ 93 __FILE__, __LINE__, #e); \ 94 abort(); \ 95 } \ 96 } while (0) 97 #endif 98 99 #ifndef not_reached 100 #define not_reached() do { \ 101 if (config_debug) { \ 102 malloc_printf( \ 103 "<jemalloc>: %s:%d: Unreachable code reached\n", \ 104 __FILE__, __LINE__); \ 105 abort(); \ 106 } \ 107 unreachable(); \ 108 } while (0) 109 #endif 110 111 #ifndef not_implemented 112 #define not_implemented() do { \ 113 if (config_debug) { \ 114 malloc_printf("<jemalloc>: %s:%d: Not implemented\n", \ 115 __FILE__, __LINE__); \ 116 abort(); \ 117 } \ 118 } while (0) 119 #endif 120 121 #ifndef assert_not_implemented 122 #define assert_not_implemented(e) do { \ 123 if (unlikely(config_debug && !(e))) \ 124 not_implemented(); \ 125 } while (0) 126 #endif 127 128 /* Use to assert a particular configuration, e.g., cassert(config_debug). */ 129 #define cassert(c) do { \ 130 if (unlikely(!(c))) \ 131 not_reached(); \ 132 } while (0) 133 134 #endif /* JEMALLOC_H_TYPES */ 135 /******************************************************************************/ 136 #ifdef JEMALLOC_H_STRUCTS 137 138 #endif /* JEMALLOC_H_STRUCTS */ 139 /******************************************************************************/ 140 #ifdef JEMALLOC_H_EXTERNS 141 142 int buferror(int err, char *buf, size_t buflen); 143 uintmax_t malloc_strtoumax(const char *restrict nptr, 144 char **restrict endptr, int base); 145 void malloc_write(const char *s); 146 147 /* 148 * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating 149 * point math. 150 */ 151 int malloc_vsnprintf(char *str, size_t size, const char *format, 152 va_list ap); 153 int malloc_snprintf(char *str, size_t size, const char *format, ...) 154 JEMALLOC_FORMAT_PRINTF(3, 4); 155 void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, 156 const char *format, va_list ap); 157 void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, 158 const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4); 159 void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2); 160 161 #endif /* JEMALLOC_H_EXTERNS */ 162 /******************************************************************************/ 163 #ifdef JEMALLOC_H_INLINES 164 165 #ifndef JEMALLOC_ENABLE_INLINE 166 int jemalloc_ffsl(long bitmap); 167 int jemalloc_ffs(int bitmap); 168 size_t pow2_ceil(size_t x); 169 size_t lg_floor(size_t x); 170 void set_errno(int errnum); 171 int get_errno(void); 172 #endif 173 174 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) 175 176 /* Sanity check. */ 177 #if !defined(JEMALLOC_INTERNAL_FFSL) || !defined(JEMALLOC_INTERNAL_FFS) 178 # error Both JEMALLOC_INTERNAL_FFSL && JEMALLOC_INTERNAL_FFS should have been defined by configure 179 #endif 180 181 JEMALLOC_ALWAYS_INLINE int 182 jemalloc_ffsl(long bitmap) 183 { 184 185 return (JEMALLOC_INTERNAL_FFSL(bitmap)); 186 } 187 188 JEMALLOC_ALWAYS_INLINE int 189 jemalloc_ffs(int bitmap) 190 { 191 192 return (JEMALLOC_INTERNAL_FFS(bitmap)); 193 } 194 195 /* Compute the smallest power of 2 that is >= x. */ 196 JEMALLOC_INLINE size_t 197 pow2_ceil(size_t x) 198 { 199 200 x--; 201 x |= x >> 1; 202 x |= x >> 2; 203 x |= x >> 4; 204 x |= x >> 8; 205 x |= x >> 16; 206 #if (LG_SIZEOF_PTR == 3) 207 x |= x >> 32; 208 #endif 209 x++; 210 return (x); 211 } 212 213 #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) 214 JEMALLOC_INLINE size_t 215 lg_floor(size_t x) 216 { 217 size_t ret; 218 219 assert(x != 0); 220 221 asm ("bsr %1, %0" 222 : "=r"(ret) // Outputs. 223 : "r"(x) // Inputs. 224 ); 225 return (ret); 226 } 227 #elif (defined(_MSC_VER)) 228 JEMALLOC_INLINE size_t 229 lg_floor(size_t x) 230 { 231 unsigned long ret; 232 233 assert(x != 0); 234 235 #if (LG_SIZEOF_PTR == 3) 236 _BitScanReverse64(&ret, x); 237 #elif (LG_SIZEOF_PTR == 2) 238 _BitScanReverse(&ret, x); 239 #else 240 # error "Unsupported type sizes for lg_floor()" 241 #endif 242 return (ret); 243 } 244 #elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) 245 JEMALLOC_INLINE size_t 246 lg_floor(size_t x) 247 { 248 249 assert(x != 0); 250 251 #if (LG_SIZEOF_PTR == LG_SIZEOF_INT) 252 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x)); 253 #elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG) 254 return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x)); 255 #else 256 # error "Unsupported type sizes for lg_floor()" 257 #endif 258 } 259 #else 260 JEMALLOC_INLINE size_t 261 lg_floor(size_t x) 262 { 263 264 assert(x != 0); 265 266 x |= (x >> 1); 267 x |= (x >> 2); 268 x |= (x >> 4); 269 x |= (x >> 8); 270 x |= (x >> 16); 271 #if (LG_SIZEOF_PTR == 3 && LG_SIZEOF_PTR == LG_SIZEOF_LONG) 272 x |= (x >> 32); 273 if (x == KZU(0xffffffffffffffff)) 274 return (63); 275 x++; 276 return (jemalloc_ffsl(x) - 2); 277 #elif (LG_SIZEOF_PTR == 2) 278 if (x == KZU(0xffffffff)) 279 return (31); 280 x++; 281 return (jemalloc_ffs(x) - 2); 282 #else 283 # error "Unsupported type sizes for lg_floor()" 284 #endif 285 } 286 #endif 287 288 /* Set error code. */ 289 JEMALLOC_INLINE void 290 set_errno(int errnum) 291 { 292 293 #ifdef _WIN32 294 SetLastError(errnum); 295 #else 296 errno = errnum; 297 #endif 298 } 299 300 /* Get last error code. */ 301 JEMALLOC_INLINE int 302 get_errno(void) 303 { 304 305 #ifdef _WIN32 306 return (GetLastError()); 307 #else 308 return (errno); 309 #endif 310 } 311 #endif 312 313 #endif /* JEMALLOC_H_INLINES */ 314 /******************************************************************************/ 315