1 #include "jemalloc/internal/jemalloc_preamble.h" 2 #include "jemalloc/internal/jemalloc_internal_includes.h" 3 4 #include "jemalloc/internal/malloc_io.h" 5 #include "jemalloc/internal/util.h" 6 7 #ifdef assert 8 # undef assert 9 #endif 10 #ifdef not_reached 11 # undef not_reached 12 #endif 13 #ifdef not_implemented 14 # undef not_implemented 15 #endif 16 #ifdef assert_not_implemented 17 # undef assert_not_implemented 18 #endif 19 20 /* 21 * Define simple versions of assertion macros that won't recurse in case 22 * of assertion failures in malloc_*printf(). 23 */ 24 #define assert(e) do { \ 25 if (config_debug && !(e)) { \ 26 malloc_write("<jemalloc>: Failed assertion\n"); \ 27 abort(); \ 28 } \ 29 } while (0) 30 31 #define not_reached() do { \ 32 if (config_debug) { \ 33 malloc_write("<jemalloc>: Unreachable code reached\n"); \ 34 abort(); \ 35 } \ 36 unreachable(); \ 37 } while (0) 38 39 #define not_implemented() do { \ 40 if (config_debug) { \ 41 malloc_write("<jemalloc>: Not implemented\n"); \ 42 abort(); \ 43 } \ 44 } while (0) 45 46 #define assert_not_implemented(e) do { \ 47 if (unlikely(config_debug && !(e))) { \ 48 not_implemented(); \ 49 } \ 50 } while (0) 51 52 /******************************************************************************/ 53 /* Function prototypes for non-inline static functions. */ 54 55 #define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1) 56 static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s, 57 size_t *slen_p); 58 #define D2S_BUFSIZE (1 + U2S_BUFSIZE) 59 static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p); 60 #define O2S_BUFSIZE (1 + U2S_BUFSIZE) 61 static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p); 62 #define X2S_BUFSIZE (2 + U2S_BUFSIZE) 63 static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, 64 size_t *slen_p); 65 66 /******************************************************************************/ 67 68 /* malloc_message() setup. */ 69 void 70 wrtmessage(void *cbopaque, const char *s) { 71 malloc_write_fd(STDERR_FILENO, s, strlen(s)); 72 } 73 74 JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); 75 76 /* 77 * Wrapper around malloc_message() that avoids the need for 78 * je_malloc_message(...) throughout the code. 79 */ 80 void 81 malloc_write(const char *s) { 82 if (je_malloc_message != NULL) { 83 je_malloc_message(NULL, s); 84 } else { 85 wrtmessage(NULL, s); 86 } 87 } 88 89 /* 90 * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so 91 * provide a wrapper. 92 */ 93 int 94 buferror(int err, char *buf, size_t buflen) { 95 #ifdef _WIN32 96 FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0, 97 (LPSTR)buf, (DWORD)buflen, NULL); 98 return 0; 99 #elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) && defined(_GNU_SOURCE) 100 char *b = strerror_r(err, buf, buflen); 101 if (b != buf) { 102 strncpy(buf, b, buflen); 103 buf[buflen-1] = '\0'; 104 } 105 return 0; 106 #else 107 return strerror_r(err, buf, buflen); 108 #endif 109 } 110 111 uintmax_t 112 malloc_strtoumax(const char *restrict nptr, const char **restrict endptr, int base) { 113 uintmax_t ret, digit; 114 unsigned b; 115 bool neg; 116 const char *p, *ns; 117 118 p = nptr; 119 if (base < 0 || base == 1 || base > 36) { 120 ns = p; 121 set_errno(EINVAL); 122 ret = UINTMAX_MAX; 123 goto label_return; 124 } 125 b = base; 126 127 /* Swallow leading whitespace and get sign, if any. */ 128 neg = false; 129 while (true) { 130 switch (*p) { 131 case '\t': case '\n': case '\v': case '\f': case '\r': case ' ': 132 p++; 133 break; 134 case '-': 135 neg = true; 136 JEMALLOC_FALLTHROUGH; 137 case '+': 138 p++; 139 JEMALLOC_FALLTHROUGH; 140 default: 141 goto label_prefix; 142 } 143 } 144 145 /* Get prefix, if any. */ 146 label_prefix: 147 /* 148 * Note where the first non-whitespace/sign character is so that it is 149 * possible to tell whether any digits are consumed (e.g., " 0" vs. 150 * " -x"). 151 */ 152 ns = p; 153 if (*p == '0') { 154 switch (p[1]) { 155 case '0': case '1': case '2': case '3': case '4': case '5': 156 case '6': case '7': 157 if (b == 0) { 158 b = 8; 159 } 160 if (b == 8) { 161 p++; 162 } 163 break; 164 case 'X': case 'x': 165 switch (p[2]) { 166 case '0': case '1': case '2': case '3': case '4': 167 case '5': case '6': case '7': case '8': case '9': 168 case 'A': case 'B': case 'C': case 'D': case 'E': 169 case 'F': 170 case 'a': case 'b': case 'c': case 'd': case 'e': 171 case 'f': 172 if (b == 0) { 173 b = 16; 174 } 175 if (b == 16) { 176 p += 2; 177 } 178 break; 179 default: 180 break; 181 } 182 break; 183 default: 184 p++; 185 ret = 0; 186 goto label_return; 187 } 188 } 189 if (b == 0) { 190 b = 10; 191 } 192 193 /* Convert. */ 194 ret = 0; 195 while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b) 196 || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b) 197 || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) { 198 uintmax_t pret = ret; 199 ret *= b; 200 ret += digit; 201 if (ret < pret) { 202 /* Overflow. */ 203 set_errno(ERANGE); 204 ret = UINTMAX_MAX; 205 goto label_return; 206 } 207 p++; 208 } 209 if (neg) { 210 ret = (uintmax_t)(-((intmax_t)ret)); 211 } 212 213 if (p == ns) { 214 /* No conversion performed. */ 215 set_errno(EINVAL); 216 ret = UINTMAX_MAX; 217 goto label_return; 218 } 219 220 label_return: 221 if (endptr != NULL) { 222 if (p == ns) { 223 /* No characters were converted. */ 224 *endptr = nptr; 225 } else { 226 *endptr = p; 227 } 228 } 229 return ret; 230 } 231 232 static char * 233 u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) { 234 unsigned i; 235 236 i = U2S_BUFSIZE - 1; 237 s[i] = '\0'; 238 switch (base) { 239 case 10: 240 do { 241 i--; 242 s[i] = "0123456789"[x % (uint64_t)10]; 243 x /= (uint64_t)10; 244 } while (x > 0); 245 break; 246 case 16: { 247 const char *digits = (uppercase) 248 ? "0123456789ABCDEF" 249 : "0123456789abcdef"; 250 251 do { 252 i--; 253 s[i] = digits[x & 0xf]; 254 x >>= 4; 255 } while (x > 0); 256 break; 257 } default: { 258 const char *digits = (uppercase) 259 ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 260 : "0123456789abcdefghijklmnopqrstuvwxyz"; 261 262 assert(base >= 2 && base <= 36); 263 do { 264 i--; 265 s[i] = digits[x % (uint64_t)base]; 266 x /= (uint64_t)base; 267 } while (x > 0); 268 }} 269 270 *slen_p = U2S_BUFSIZE - 1 - i; 271 return &s[i]; 272 } 273 274 static char * 275 d2s(intmax_t x, char sign, char *s, size_t *slen_p) { 276 bool neg; 277 278 if ((neg = (x < 0))) { 279 x = -x; 280 } 281 s = u2s(x, 10, false, s, slen_p); 282 if (neg) { 283 sign = '-'; 284 } 285 switch (sign) { 286 case '-': 287 if (!neg) { 288 break; 289 } 290 JEMALLOC_FALLTHROUGH; 291 case ' ': 292 case '+': 293 s--; 294 (*slen_p)++; 295 *s = sign; 296 break; 297 default: not_reached(); 298 } 299 return s; 300 } 301 302 static char * 303 o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) { 304 s = u2s(x, 8, false, s, slen_p); 305 if (alt_form && *s != '0') { 306 s--; 307 (*slen_p)++; 308 *s = '0'; 309 } 310 return s; 311 } 312 313 static char * 314 x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) { 315 s = u2s(x, 16, uppercase, s, slen_p); 316 if (alt_form) { 317 s -= 2; 318 (*slen_p) += 2; 319 memcpy(s, uppercase ? "0X" : "0x", 2); 320 } 321 return s; 322 } 323 324 JEMALLOC_COLD 325 size_t 326 malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { 327 size_t i; 328 const char *f; 329 330 #define APPEND_C(c) do { \ 331 if (i < size) { \ 332 str[i] = (c); \ 333 } \ 334 i++; \ 335 } while (0) 336 #define APPEND_S(s, slen) do { \ 337 if (i < size) { \ 338 size_t cpylen = (slen <= size - i) ? slen : size - i; \ 339 memcpy(&str[i], s, cpylen); \ 340 } \ 341 i += slen; \ 342 } while (0) 343 #define APPEND_PADDED_S(s, slen, width, left_justify) do { \ 344 /* Left padding. */ \ 345 size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ? \ 346 (size_t)width - slen : 0); \ 347 if (!left_justify && pad_len != 0) { \ 348 size_t j; \ 349 for (j = 0; j < pad_len; j++) { \ 350 if (pad_zero) { \ 351 APPEND_C('0'); \ 352 } else { \ 353 APPEND_C(' '); \ 354 } \ 355 } \ 356 } \ 357 /* Value. */ \ 358 APPEND_S(s, slen); \ 359 /* Right padding. */ \ 360 if (left_justify && pad_len != 0) { \ 361 size_t j; \ 362 for (j = 0; j < pad_len; j++) { \ 363 APPEND_C(' '); \ 364 } \ 365 } \ 366 } while (0) 367 #define GET_ARG_NUMERIC(val, len) do { \ 368 switch ((unsigned char)len) { \ 369 case '?': \ 370 val = va_arg(ap, int); \ 371 break; \ 372 case '?' | 0x80U: \ 373 val = va_arg(ap, unsigned int); \ 374 break; \ 375 case 'l': \ 376 val = va_arg(ap, long); \ 377 break; \ 378 case 'l' | 0x80U: \ 379 val = va_arg(ap, unsigned long); \ 380 break; \ 381 case 'q': \ 382 val = va_arg(ap, long long); \ 383 break; \ 384 case 'q' | 0x80U: \ 385 val = va_arg(ap, unsigned long long); \ 386 break; \ 387 case 'j': \ 388 val = va_arg(ap, intmax_t); \ 389 break; \ 390 case 'j' | 0x80U: \ 391 val = va_arg(ap, uintmax_t); \ 392 break; \ 393 case 't': \ 394 val = va_arg(ap, ptrdiff_t); \ 395 break; \ 396 case 'z': \ 397 val = va_arg(ap, ssize_t); \ 398 break; \ 399 case 'z' | 0x80U: \ 400 val = va_arg(ap, size_t); \ 401 break; \ 402 case 'p': /* Synthetic; used for %p. */ \ 403 val = va_arg(ap, uintptr_t); \ 404 break; \ 405 default: \ 406 not_reached(); \ 407 val = 0; \ 408 } \ 409 } while (0) 410 411 i = 0; 412 f = format; 413 while (true) { 414 switch (*f) { 415 case '\0': goto label_out; 416 case '%': { 417 bool alt_form = false; 418 bool left_justify = false; 419 bool plus_space = false; 420 bool plus_plus = false; 421 int prec = -1; 422 int width = -1; 423 unsigned char len = '?'; 424 char *s; 425 size_t slen; 426 bool first_width_digit = true; 427 bool pad_zero = false; 428 429 f++; 430 /* Flags. */ 431 while (true) { 432 switch (*f) { 433 case '#': 434 assert(!alt_form); 435 alt_form = true; 436 break; 437 case '-': 438 assert(!left_justify); 439 left_justify = true; 440 break; 441 case ' ': 442 assert(!plus_space); 443 plus_space = true; 444 break; 445 case '+': 446 assert(!plus_plus); 447 plus_plus = true; 448 break; 449 default: goto label_width; 450 } 451 f++; 452 } 453 /* Width. */ 454 label_width: 455 switch (*f) { 456 case '*': 457 width = va_arg(ap, int); 458 f++; 459 if (width < 0) { 460 left_justify = true; 461 width = -width; 462 } 463 break; 464 case '0': 465 if (first_width_digit) { 466 pad_zero = true; 467 } 468 JEMALLOC_FALLTHROUGH; 469 case '1': case '2': case '3': case '4': 470 case '5': case '6': case '7': case '8': case '9': { 471 uintmax_t uwidth; 472 set_errno(0); 473 uwidth = malloc_strtoumax(f, &f, 10); 474 assert(uwidth != UINTMAX_MAX || get_errno() != 475 ERANGE); 476 width = (int)uwidth; 477 first_width_digit = false; 478 break; 479 } default: 480 break; 481 } 482 /* Width/precision separator. */ 483 if (*f == '.') { 484 f++; 485 } else { 486 goto label_length; 487 } 488 /* Precision. */ 489 switch (*f) { 490 case '*': 491 prec = va_arg(ap, int); 492 f++; 493 break; 494 case '0': case '1': case '2': case '3': case '4': 495 case '5': case '6': case '7': case '8': case '9': { 496 uintmax_t uprec; 497 set_errno(0); 498 uprec = malloc_strtoumax(f, &f, 10); 499 assert(uprec != UINTMAX_MAX || get_errno() != 500 ERANGE); 501 prec = (int)uprec; 502 break; 503 } 504 default: break; 505 } 506 /* Length. */ 507 label_length: 508 switch (*f) { 509 case 'l': 510 f++; 511 if (*f == 'l') { 512 len = 'q'; 513 f++; 514 } else { 515 len = 'l'; 516 } 517 break; 518 case 'q': case 'j': case 't': case 'z': 519 len = *f; 520 f++; 521 break; 522 default: break; 523 } 524 /* Conversion specifier. */ 525 switch (*f) { 526 case '%': 527 /* %% */ 528 APPEND_C(*f); 529 f++; 530 break; 531 case 'd': case 'i': { 532 intmax_t val JEMALLOC_CC_SILENCE_INIT(0); 533 char buf[D2S_BUFSIZE]; 534 535 /* 536 * Outputting negative, zero-padded numbers 537 * would require a nontrivial rework of the 538 * interaction between the width and padding 539 * (since 0 padding goes between the '-' and the 540 * number, while ' ' padding goes either before 541 * the - or after the number. Since we 542 * currently don't ever need 0-padded negative 543 * numbers, just don't bother supporting it. 544 */ 545 assert(!pad_zero); 546 547 GET_ARG_NUMERIC(val, len); 548 s = d2s(val, (plus_plus ? '+' : (plus_space ? 549 ' ' : '-')), buf, &slen); 550 APPEND_PADDED_S(s, slen, width, left_justify); 551 f++; 552 break; 553 } case 'o': { 554 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); 555 char buf[O2S_BUFSIZE]; 556 557 GET_ARG_NUMERIC(val, len | 0x80); 558 s = o2s(val, alt_form, buf, &slen); 559 APPEND_PADDED_S(s, slen, width, left_justify); 560 f++; 561 break; 562 } case 'u': { 563 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); 564 char buf[U2S_BUFSIZE]; 565 566 GET_ARG_NUMERIC(val, len | 0x80); 567 s = u2s(val, 10, false, buf, &slen); 568 APPEND_PADDED_S(s, slen, width, left_justify); 569 f++; 570 break; 571 } case 'x': case 'X': { 572 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); 573 char buf[X2S_BUFSIZE]; 574 575 GET_ARG_NUMERIC(val, len | 0x80); 576 s = x2s(val, alt_form, *f == 'X', buf, &slen); 577 APPEND_PADDED_S(s, slen, width, left_justify); 578 f++; 579 break; 580 } case 'c': { 581 unsigned char val; 582 char buf[2]; 583 584 assert(len == '?' || len == 'l'); 585 assert_not_implemented(len != 'l'); 586 val = va_arg(ap, int); 587 buf[0] = val; 588 buf[1] = '\0'; 589 APPEND_PADDED_S(buf, 1, width, left_justify); 590 f++; 591 break; 592 } case 's': 593 assert(len == '?' || len == 'l'); 594 assert_not_implemented(len != 'l'); 595 s = va_arg(ap, char *); 596 slen = (prec < 0) ? strlen(s) : (size_t)prec; 597 APPEND_PADDED_S(s, slen, width, left_justify); 598 f++; 599 break; 600 case 'p': { 601 uintmax_t val; 602 char buf[X2S_BUFSIZE]; 603 604 GET_ARG_NUMERIC(val, 'p'); 605 s = x2s(val, true, false, buf, &slen); 606 APPEND_PADDED_S(s, slen, width, left_justify); 607 f++; 608 break; 609 } default: not_reached(); 610 } 611 break; 612 } default: { 613 APPEND_C(*f); 614 f++; 615 break; 616 }} 617 } 618 label_out: 619 if (i < size) { 620 str[i] = '\0'; 621 } else { 622 str[size - 1] = '\0'; 623 } 624 625 #undef APPEND_C 626 #undef APPEND_S 627 #undef APPEND_PADDED_S 628 #undef GET_ARG_NUMERIC 629 return i; 630 } 631 632 JEMALLOC_FORMAT_PRINTF(3, 4) 633 size_t 634 malloc_snprintf(char *str, size_t size, const char *format, ...) { 635 size_t ret; 636 va_list ap; 637 638 va_start(ap, format); 639 ret = malloc_vsnprintf(str, size, format, ap); 640 va_end(ap); 641 642 return ret; 643 } 644 645 void 646 malloc_vcprintf(write_cb_t *write_cb, void *cbopaque, const char *format, 647 va_list ap) { 648 char buf[MALLOC_PRINTF_BUFSIZE]; 649 650 if (write_cb == NULL) { 651 /* 652 * The caller did not provide an alternate write_cb callback 653 * function, so use the default one. malloc_write() is an 654 * inline function, so use malloc_message() directly here. 655 */ 656 write_cb = (je_malloc_message != NULL) ? je_malloc_message : 657 wrtmessage; 658 } 659 660 malloc_vsnprintf(buf, sizeof(buf), format, ap); 661 write_cb(cbopaque, buf); 662 } 663 664 /* 665 * Print to a callback function in such a way as to (hopefully) avoid memory 666 * allocation. 667 */ 668 JEMALLOC_FORMAT_PRINTF(3, 4) 669 void 670 malloc_cprintf(write_cb_t *write_cb, void *cbopaque, const char *format, ...) { 671 va_list ap; 672 673 va_start(ap, format); 674 malloc_vcprintf(write_cb, cbopaque, format, ap); 675 va_end(ap); 676 } 677 678 /* Print to stderr in such a way as to avoid memory allocation. */ 679 JEMALLOC_FORMAT_PRINTF(1, 2) 680 void 681 malloc_printf(const char *format, ...) { 682 va_list ap; 683 684 va_start(ap, format); 685 malloc_vcprintf(NULL, NULL, format, ap); 686 va_end(ap); 687 } 688 689 /* 690 * Restore normal assertion macros, in order to make it possible to compile all 691 * C files as a single concatenation. 692 */ 693 #undef assert 694 #undef not_reached 695 #undef not_implemented 696 #undef assert_not_implemented 697 #include "jemalloc/internal/assert.h" 698