1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2019 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stddef.h> 7 #include <stdio.h> 8 #include <string.h> 9 #include <sys/queue.h> 10 11 #include <rte_errno.h> 12 #include <rte_memcpy.h> 13 #include <rte_memory.h> 14 #include <rte_eal.h> 15 #include <rte_eal_memconfig.h> 16 #include <rte_branch_prediction.h> 17 #include <rte_debug.h> 18 #include <rte_launch.h> 19 #include <rte_per_lcore.h> 20 #include <rte_lcore.h> 21 #include <rte_common.h> 22 #include <rte_spinlock.h> 23 24 #include <rte_eal_trace.h> 25 26 #include <rte_malloc.h> 27 #include "malloc_elem.h" 28 #include "malloc_heap.h" 29 #include "eal_memalloc.h" 30 #include "eal_memcfg.h" 31 #include "eal_private.h" 32 33 34 /* Free the memory space back to heap */ 35 static void 36 mem_free(void *addr, const bool trace_ena) 37 { 38 if (trace_ena) 39 rte_eal_trace_mem_free(addr); 40 41 if (addr == NULL) return; 42 if (malloc_heap_free(malloc_elem_from_data(addr)) < 0) 43 RTE_LOG(ERR, EAL, "Error: Invalid memory\n"); 44 } 45 46 void 47 rte_free(void *addr) 48 { 49 return mem_free(addr, true); 50 } 51 52 void 53 eal_free_no_trace(void *addr) 54 { 55 return mem_free(addr, false); 56 } 57 58 static void * 59 malloc_socket(const char *type, size_t size, unsigned int align, 60 int socket_arg, const bool trace_ena) 61 { 62 void *ptr; 63 64 /* return NULL if size is 0 or alignment is not power-of-2 */ 65 if (size == 0 || (align && !rte_is_power_of_2(align))) 66 return NULL; 67 68 /* if there are no hugepages and if we are not allocating from an 69 * external heap, use memory from any socket available. checking for 70 * socket being external may return -1 in case of invalid socket, but 71 * that's OK - if there are no hugepages, it doesn't matter. 72 */ 73 if (rte_malloc_heap_socket_is_external(socket_arg) != 1 && 74 !rte_eal_has_hugepages()) 75 socket_arg = SOCKET_ID_ANY; 76 77 ptr = malloc_heap_alloc(type, size, socket_arg, 0, 78 align == 0 ? 1 : align, 0, false); 79 80 if (trace_ena) 81 rte_eal_trace_mem_malloc(type, size, align, socket_arg, ptr); 82 return ptr; 83 } 84 85 /* 86 * Allocate memory on specified heap. 87 */ 88 void * 89 rte_malloc_socket(const char *type, size_t size, unsigned int align, 90 int socket_arg) 91 { 92 return malloc_socket(type, size, align, socket_arg, true); 93 } 94 95 void * 96 eal_malloc_no_trace(const char *type, size_t size, unsigned int align) 97 { 98 return malloc_socket(type, size, align, SOCKET_ID_ANY, false); 99 } 100 101 /* 102 * Allocate memory on default heap. 103 */ 104 void * 105 rte_malloc(const char *type, size_t size, unsigned align) 106 { 107 return rte_malloc_socket(type, size, align, SOCKET_ID_ANY); 108 } 109 110 /* 111 * Allocate zero'd memory on specified heap. 112 */ 113 void * 114 rte_zmalloc_socket(const char *type, size_t size, unsigned align, int socket) 115 { 116 void *ptr = rte_malloc_socket(type, size, align, socket); 117 118 if (ptr != NULL) { 119 struct malloc_elem *elem = malloc_elem_from_data(ptr); 120 121 if (elem->dirty) { 122 memset(ptr, 0, size); 123 } else { 124 #ifdef RTE_MALLOC_DEBUG 125 /* 126 * If DEBUG is enabled, then freed memory is marked 127 * with a poison value and set to zero on allocation. 128 * If DEBUG is disabled then memory is already zeroed. 129 */ 130 memset(ptr, 0, size); 131 #endif 132 } 133 } 134 135 rte_eal_trace_mem_zmalloc(type, size, align, socket, ptr); 136 return ptr; 137 } 138 139 /* 140 * Allocate zero'd memory on default heap. 141 */ 142 void * 143 rte_zmalloc(const char *type, size_t size, unsigned align) 144 { 145 return rte_zmalloc_socket(type, size, align, SOCKET_ID_ANY); 146 } 147 148 /* 149 * Allocate zero'd memory on specified heap. 150 */ 151 void * 152 rte_calloc_socket(const char *type, size_t num, size_t size, unsigned align, int socket) 153 { 154 return rte_zmalloc_socket(type, num * size, align, socket); 155 } 156 157 /* 158 * Allocate zero'd memory on default heap. 159 */ 160 void * 161 rte_calloc(const char *type, size_t num, size_t size, unsigned align) 162 { 163 return rte_zmalloc(type, num * size, align); 164 } 165 166 /* 167 * Resize allocated memory on specified heap. 168 */ 169 void * 170 rte_realloc_socket(void *ptr, size_t size, unsigned int align, int socket) 171 { 172 size_t user_size; 173 174 if (ptr == NULL) 175 return rte_malloc_socket(NULL, size, align, socket); 176 177 struct malloc_elem *elem = malloc_elem_from_data(ptr); 178 if (elem == NULL) { 179 RTE_LOG(ERR, EAL, "Error: memory corruption detected\n"); 180 return NULL; 181 } 182 183 user_size = size; 184 185 size = RTE_CACHE_LINE_ROUNDUP(size), align = RTE_CACHE_LINE_ROUNDUP(align); 186 187 /* check requested socket id and alignment matches first, and if ok, 188 * see if we can resize block 189 */ 190 if ((socket == SOCKET_ID_ANY || 191 (unsigned int)socket == elem->heap->socket_id) && 192 RTE_PTR_ALIGN(ptr, align) == ptr && 193 malloc_heap_resize(elem, size) == 0) { 194 rte_eal_trace_mem_realloc(size, align, socket, ptr); 195 196 asan_set_redzone(elem, user_size); 197 198 return ptr; 199 } 200 201 /* either requested socket id doesn't match, alignment is off 202 * or we have no room to expand, 203 * so move the data. 204 */ 205 void *new_ptr = rte_malloc_socket(NULL, size, align, socket); 206 if (new_ptr == NULL) 207 return NULL; 208 /* elem: |pad|data_elem|data|trailer| */ 209 const size_t old_size = old_malloc_size(elem); 210 rte_memcpy(new_ptr, ptr, old_size < size ? old_size : size); 211 rte_free(ptr); 212 213 rte_eal_trace_mem_realloc(size, align, socket, new_ptr); 214 return new_ptr; 215 } 216 217 /* 218 * Resize allocated memory. 219 */ 220 void * 221 rte_realloc(void *ptr, size_t size, unsigned int align) 222 { 223 return rte_realloc_socket(ptr, size, align, SOCKET_ID_ANY); 224 } 225 226 int 227 rte_malloc_validate(const void *ptr, size_t *size) 228 { 229 const struct malloc_elem *elem = malloc_elem_from_data(ptr); 230 if (!malloc_elem_cookies_ok(elem)) 231 return -1; 232 if (size != NULL) 233 *size = elem->size - elem->pad - MALLOC_ELEM_OVERHEAD; 234 return 0; 235 } 236 237 /* 238 * Function to retrieve data for heap on given socket 239 */ 240 int 241 rte_malloc_get_socket_stats(int socket, 242 struct rte_malloc_socket_stats *socket_stats) 243 { 244 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 245 int heap_idx; 246 247 heap_idx = malloc_socket_to_heap_id(socket); 248 if (heap_idx < 0) 249 return -1; 250 251 return malloc_heap_get_stats(&mcfg->malloc_heaps[heap_idx], 252 socket_stats); 253 } 254 255 /* 256 * Function to dump contents of all heaps 257 */ 258 void 259 rte_malloc_dump_heaps(FILE *f) 260 { 261 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 262 unsigned int idx; 263 264 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 265 fprintf(f, "Heap id: %u\n", idx); 266 malloc_heap_dump(&mcfg->malloc_heaps[idx], f); 267 } 268 } 269 270 int 271 rte_malloc_heap_get_socket(const char *name) 272 { 273 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 274 struct malloc_heap *heap = NULL; 275 unsigned int idx; 276 int ret; 277 278 if (name == NULL || 279 strnlen(name, RTE_HEAP_NAME_MAX_LEN) == 0 || 280 strnlen(name, RTE_HEAP_NAME_MAX_LEN) == 281 RTE_HEAP_NAME_MAX_LEN) { 282 rte_errno = EINVAL; 283 return -1; 284 } 285 rte_mcfg_mem_read_lock(); 286 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 287 struct malloc_heap *tmp = &mcfg->malloc_heaps[idx]; 288 289 if (!strncmp(name, tmp->name, RTE_HEAP_NAME_MAX_LEN)) { 290 heap = tmp; 291 break; 292 } 293 } 294 295 if (heap != NULL) { 296 ret = heap->socket_id; 297 } else { 298 rte_errno = ENOENT; 299 ret = -1; 300 } 301 rte_mcfg_mem_read_unlock(); 302 303 return ret; 304 } 305 306 int 307 rte_malloc_heap_socket_is_external(int socket_id) 308 { 309 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 310 unsigned int idx; 311 int ret = -1; 312 313 if (socket_id == SOCKET_ID_ANY) 314 return 0; 315 316 rte_mcfg_mem_read_lock(); 317 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 318 struct malloc_heap *tmp = &mcfg->malloc_heaps[idx]; 319 320 if ((int)tmp->socket_id == socket_id) { 321 /* external memory always has large socket ID's */ 322 ret = tmp->socket_id >= RTE_MAX_NUMA_NODES; 323 break; 324 } 325 } 326 rte_mcfg_mem_read_unlock(); 327 328 return ret; 329 } 330 331 /* 332 * Print stats on memory type. If type is NULL, info on all types is printed 333 */ 334 void 335 rte_malloc_dump_stats(FILE *f, __rte_unused const char *type) 336 { 337 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 338 unsigned int heap_id; 339 struct rte_malloc_socket_stats sock_stats; 340 341 /* Iterate through all initialised heaps */ 342 for (heap_id = 0; heap_id < RTE_MAX_HEAPS; heap_id++) { 343 struct malloc_heap *heap = &mcfg->malloc_heaps[heap_id]; 344 345 malloc_heap_get_stats(heap, &sock_stats); 346 347 fprintf(f, "Heap id:%u\n", heap_id); 348 fprintf(f, "\tHeap name:%s\n", heap->name); 349 fprintf(f, "\tHeap_size:%zu,\n", sock_stats.heap_totalsz_bytes); 350 fprintf(f, "\tFree_size:%zu,\n", sock_stats.heap_freesz_bytes); 351 fprintf(f, "\tAlloc_size:%zu,\n", sock_stats.heap_allocsz_bytes); 352 fprintf(f, "\tGreatest_free_size:%zu,\n", 353 sock_stats.greatest_free_size); 354 fprintf(f, "\tAlloc_count:%u,\n",sock_stats.alloc_count); 355 fprintf(f, "\tFree_count:%u,\n", sock_stats.free_count); 356 } 357 return; 358 } 359 360 /* 361 * TODO: Set limit to memory that can be allocated to memory type 362 */ 363 int 364 rte_malloc_set_limit(__rte_unused const char *type, 365 __rte_unused size_t max) 366 { 367 return 0; 368 } 369 370 /* 371 * Return the IO address of a virtual address obtained through rte_malloc 372 */ 373 rte_iova_t 374 rte_malloc_virt2iova(const void *addr) 375 { 376 const struct rte_memseg *ms; 377 struct malloc_elem *elem = malloc_elem_from_data(addr); 378 379 if (elem == NULL) 380 return RTE_BAD_IOVA; 381 382 if (!elem->msl->external && rte_eal_iova_mode() == RTE_IOVA_VA) 383 return (uintptr_t) addr; 384 385 ms = rte_mem_virt2memseg(addr, elem->msl); 386 if (ms == NULL) 387 return RTE_BAD_IOVA; 388 389 if (ms->iova == RTE_BAD_IOVA) 390 return RTE_BAD_IOVA; 391 392 return ms->iova + RTE_PTR_DIFF(addr, ms->addr); 393 } 394 395 static struct malloc_heap * 396 find_named_heap(const char *name) 397 { 398 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 399 unsigned int i; 400 401 for (i = 0; i < RTE_MAX_HEAPS; i++) { 402 struct malloc_heap *heap = &mcfg->malloc_heaps[i]; 403 404 if (!strncmp(name, heap->name, RTE_HEAP_NAME_MAX_LEN)) 405 return heap; 406 } 407 return NULL; 408 } 409 410 int 411 rte_malloc_heap_memory_add(const char *heap_name, void *va_addr, size_t len, 412 rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz) 413 { 414 struct malloc_heap *heap = NULL; 415 struct rte_memseg_list *msl; 416 unsigned int n; 417 int ret; 418 419 if (heap_name == NULL || va_addr == NULL || 420 page_sz == 0 || !rte_is_power_of_2(page_sz) || 421 RTE_ALIGN(len, page_sz) != len || 422 !rte_is_aligned(va_addr, page_sz) || 423 ((len / page_sz) != n_pages && iova_addrs != NULL) || 424 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 425 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 426 RTE_HEAP_NAME_MAX_LEN) { 427 rte_errno = EINVAL; 428 return -1; 429 } 430 rte_mcfg_mem_write_lock(); 431 432 /* find our heap */ 433 heap = find_named_heap(heap_name); 434 if (heap == NULL) { 435 rte_errno = ENOENT; 436 ret = -1; 437 goto unlock; 438 } 439 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 440 /* cannot add memory to internal heaps */ 441 rte_errno = EPERM; 442 ret = -1; 443 goto unlock; 444 } 445 n = len / page_sz; 446 447 msl = malloc_heap_create_external_seg(va_addr, iova_addrs, n, page_sz, 448 heap_name, heap->socket_id); 449 if (msl == NULL) { 450 ret = -1; 451 goto unlock; 452 } 453 454 rte_spinlock_lock(&heap->lock); 455 ret = malloc_heap_add_external_memory(heap, msl); 456 msl->heap = 1; /* mark it as heap segment */ 457 rte_spinlock_unlock(&heap->lock); 458 459 unlock: 460 rte_mcfg_mem_write_unlock(); 461 462 return ret; 463 } 464 465 int 466 rte_malloc_heap_memory_remove(const char *heap_name, void *va_addr, size_t len) 467 { 468 struct malloc_heap *heap = NULL; 469 struct rte_memseg_list *msl; 470 int ret; 471 472 if (heap_name == NULL || va_addr == NULL || len == 0 || 473 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 474 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 475 RTE_HEAP_NAME_MAX_LEN) { 476 rte_errno = EINVAL; 477 return -1; 478 } 479 rte_mcfg_mem_write_lock(); 480 /* find our heap */ 481 heap = find_named_heap(heap_name); 482 if (heap == NULL) { 483 rte_errno = ENOENT; 484 ret = -1; 485 goto unlock; 486 } 487 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 488 /* cannot remove memory from internal heaps */ 489 rte_errno = EPERM; 490 ret = -1; 491 goto unlock; 492 } 493 494 msl = malloc_heap_find_external_seg(va_addr, len); 495 if (msl == NULL) { 496 ret = -1; 497 goto unlock; 498 } 499 500 rte_spinlock_lock(&heap->lock); 501 ret = malloc_heap_remove_external_memory(heap, va_addr, len); 502 rte_spinlock_unlock(&heap->lock); 503 if (ret != 0) 504 goto unlock; 505 506 ret = malloc_heap_destroy_external_seg(msl); 507 508 unlock: 509 rte_mcfg_mem_write_unlock(); 510 511 return ret; 512 } 513 514 static int 515 sync_memory(const char *heap_name, void *va_addr, size_t len, bool attach) 516 { 517 struct malloc_heap *heap = NULL; 518 struct rte_memseg_list *msl; 519 int ret; 520 521 if (heap_name == NULL || va_addr == NULL || len == 0 || 522 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 523 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 524 RTE_HEAP_NAME_MAX_LEN) { 525 rte_errno = EINVAL; 526 return -1; 527 } 528 rte_mcfg_mem_read_lock(); 529 530 /* find our heap */ 531 heap = find_named_heap(heap_name); 532 if (heap == NULL) { 533 rte_errno = ENOENT; 534 ret = -1; 535 goto unlock; 536 } 537 /* we shouldn't be able to sync to internal heaps */ 538 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 539 rte_errno = EPERM; 540 ret = -1; 541 goto unlock; 542 } 543 544 /* find corresponding memseg list to sync to */ 545 msl = malloc_heap_find_external_seg(va_addr, len); 546 if (msl == NULL) { 547 ret = -1; 548 goto unlock; 549 } 550 551 if (attach) { 552 ret = rte_fbarray_attach(&msl->memseg_arr); 553 if (ret == 0) { 554 /* notify all subscribers that a new memory area was 555 * added. 556 */ 557 eal_memalloc_mem_event_notify(RTE_MEM_EVENT_ALLOC, 558 va_addr, len); 559 } else { 560 ret = -1; 561 goto unlock; 562 } 563 } else { 564 /* notify all subscribers that a memory area is about to 565 * be removed. 566 */ 567 eal_memalloc_mem_event_notify(RTE_MEM_EVENT_FREE, 568 msl->base_va, msl->len); 569 ret = rte_fbarray_detach(&msl->memseg_arr); 570 if (ret < 0) { 571 ret = -1; 572 goto unlock; 573 } 574 } 575 unlock: 576 rte_mcfg_mem_read_unlock(); 577 return ret; 578 } 579 580 int 581 rte_malloc_heap_memory_attach(const char *heap_name, void *va_addr, size_t len) 582 { 583 return sync_memory(heap_name, va_addr, len, true); 584 } 585 586 int 587 rte_malloc_heap_memory_detach(const char *heap_name, void *va_addr, size_t len) 588 { 589 return sync_memory(heap_name, va_addr, len, false); 590 } 591 592 int 593 rte_malloc_heap_create(const char *heap_name) 594 { 595 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 596 struct malloc_heap *heap = NULL; 597 int i, ret; 598 599 if (heap_name == NULL || 600 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 601 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 602 RTE_HEAP_NAME_MAX_LEN) { 603 rte_errno = EINVAL; 604 return -1; 605 } 606 /* check if there is space in the heap list, or if heap with this name 607 * already exists. 608 */ 609 rte_mcfg_mem_write_lock(); 610 611 for (i = 0; i < RTE_MAX_HEAPS; i++) { 612 struct malloc_heap *tmp = &mcfg->malloc_heaps[i]; 613 /* existing heap */ 614 if (strncmp(heap_name, tmp->name, 615 RTE_HEAP_NAME_MAX_LEN) == 0) { 616 RTE_LOG(ERR, EAL, "Heap %s already exists\n", 617 heap_name); 618 rte_errno = EEXIST; 619 ret = -1; 620 goto unlock; 621 } 622 /* empty heap */ 623 if (strnlen(tmp->name, RTE_HEAP_NAME_MAX_LEN) == 0) { 624 heap = tmp; 625 break; 626 } 627 } 628 if (heap == NULL) { 629 RTE_LOG(ERR, EAL, "Cannot create new heap: no space\n"); 630 rte_errno = ENOSPC; 631 ret = -1; 632 goto unlock; 633 } 634 635 /* we're sure that we can create a new heap, so do it */ 636 ret = malloc_heap_create(heap, heap_name); 637 unlock: 638 rte_mcfg_mem_write_unlock(); 639 640 return ret; 641 } 642 643 int 644 rte_malloc_heap_destroy(const char *heap_name) 645 { 646 struct malloc_heap *heap = NULL; 647 int ret; 648 649 if (heap_name == NULL || 650 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 651 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 652 RTE_HEAP_NAME_MAX_LEN) { 653 rte_errno = EINVAL; 654 return -1; 655 } 656 rte_mcfg_mem_write_lock(); 657 658 /* start from non-socket heaps */ 659 heap = find_named_heap(heap_name); 660 if (heap == NULL) { 661 RTE_LOG(ERR, EAL, "Heap %s not found\n", heap_name); 662 rte_errno = ENOENT; 663 ret = -1; 664 goto unlock; 665 } 666 /* we shouldn't be able to destroy internal heaps */ 667 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 668 rte_errno = EPERM; 669 ret = -1; 670 goto unlock; 671 } 672 /* sanity checks done, now we can destroy the heap */ 673 rte_spinlock_lock(&heap->lock); 674 ret = malloc_heap_destroy(heap); 675 676 /* if we failed, lock is still active */ 677 if (ret < 0) 678 rte_spinlock_unlock(&heap->lock); 679 unlock: 680 rte_mcfg_mem_write_unlock(); 681 682 return ret; 683 } 684