1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2019 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stddef.h> 7 #include <stdio.h> 8 #include <string.h> 9 #include <sys/queue.h> 10 11 #include <rte_errno.h> 12 #include <rte_memcpy.h> 13 #include <rte_memory.h> 14 #include <rte_eal.h> 15 #include <rte_eal_memconfig.h> 16 #include <rte_branch_prediction.h> 17 #include <rte_debug.h> 18 #include <rte_launch.h> 19 #include <rte_per_lcore.h> 20 #include <rte_lcore.h> 21 #include <rte_common.h> 22 #include <rte_spinlock.h> 23 24 #include <rte_eal_trace.h> 25 26 #include <rte_malloc.h> 27 #include "malloc_elem.h" 28 #include "malloc_heap.h" 29 #include "eal_memalloc.h" 30 #include "eal_memcfg.h" 31 #include "eal_private.h" 32 33 34 /* Free the memory space back to heap */ 35 static void 36 mem_free(void *addr, const bool trace_ena) 37 { 38 if (trace_ena) 39 rte_eal_trace_mem_free(addr); 40 41 if (addr == NULL) return; 42 if (malloc_heap_free(malloc_elem_from_data(addr)) < 0) 43 RTE_LOG(ERR, EAL, "Error: Invalid memory\n"); 44 } 45 46 void 47 rte_free(void *addr) 48 { 49 return mem_free(addr, true); 50 } 51 52 void 53 eal_free_no_trace(void *addr) 54 { 55 return mem_free(addr, false); 56 } 57 58 static void * 59 malloc_socket(const char *type, size_t size, unsigned int align, 60 int socket_arg, const bool trace_ena) 61 { 62 void *ptr; 63 64 /* return NULL if size is 0 or alignment is not power-of-2 */ 65 if (size == 0 || (align && !rte_is_power_of_2(align))) 66 return NULL; 67 68 /* if there are no hugepages and if we are not allocating from an 69 * external heap, use memory from any socket available. checking for 70 * socket being external may return -1 in case of invalid socket, but 71 * that's OK - if there are no hugepages, it doesn't matter. 72 */ 73 if (rte_malloc_heap_socket_is_external(socket_arg) != 1 && 74 !rte_eal_has_hugepages()) 75 socket_arg = SOCKET_ID_ANY; 76 77 ptr = malloc_heap_alloc(type, size, socket_arg, 0, 78 align == 0 ? 1 : align, 0, false); 79 80 if (trace_ena) 81 rte_eal_trace_mem_malloc(type, size, align, socket_arg, ptr); 82 return ptr; 83 } 84 85 /* 86 * Allocate memory on specified heap. 87 */ 88 void * 89 rte_malloc_socket(const char *type, size_t size, unsigned int align, 90 int socket_arg) 91 { 92 return malloc_socket(type, size, align, socket_arg, true); 93 } 94 95 void * 96 eal_malloc_no_trace(const char *type, size_t size, unsigned int align) 97 { 98 return malloc_socket(type, size, align, SOCKET_ID_ANY, false); 99 } 100 101 /* 102 * Allocate memory on default heap. 103 */ 104 void * 105 rte_malloc(const char *type, size_t size, unsigned align) 106 { 107 return rte_malloc_socket(type, size, align, SOCKET_ID_ANY); 108 } 109 110 /* 111 * Allocate zero'd memory on specified heap. 112 */ 113 void * 114 rte_zmalloc_socket(const char *type, size_t size, unsigned align, int socket) 115 { 116 void *ptr = rte_malloc_socket(type, size, align, socket); 117 118 #ifdef RTE_MALLOC_DEBUG 119 /* 120 * If DEBUG is enabled, then freed memory is marked with poison 121 * value and set to zero on allocation. 122 * If DEBUG is not enabled then memory is already zeroed. 123 */ 124 if (ptr != NULL) 125 memset(ptr, 0, size); 126 #endif 127 128 rte_eal_trace_mem_zmalloc(type, size, align, socket, ptr); 129 return ptr; 130 } 131 132 /* 133 * Allocate zero'd memory on default heap. 134 */ 135 void * 136 rte_zmalloc(const char *type, size_t size, unsigned align) 137 { 138 return rte_zmalloc_socket(type, size, align, SOCKET_ID_ANY); 139 } 140 141 /* 142 * Allocate zero'd memory on specified heap. 143 */ 144 void * 145 rte_calloc_socket(const char *type, size_t num, size_t size, unsigned align, int socket) 146 { 147 return rte_zmalloc_socket(type, num * size, align, socket); 148 } 149 150 /* 151 * Allocate zero'd memory on default heap. 152 */ 153 void * 154 rte_calloc(const char *type, size_t num, size_t size, unsigned align) 155 { 156 return rte_zmalloc(type, num * size, align); 157 } 158 159 /* 160 * Resize allocated memory on specified heap. 161 */ 162 void * 163 rte_realloc_socket(void *ptr, size_t size, unsigned int align, int socket) 164 { 165 size_t user_size; 166 167 if (ptr == NULL) 168 return rte_malloc_socket(NULL, size, align, socket); 169 170 struct malloc_elem *elem = malloc_elem_from_data(ptr); 171 if (elem == NULL) { 172 RTE_LOG(ERR, EAL, "Error: memory corruption detected\n"); 173 return NULL; 174 } 175 176 user_size = size; 177 178 size = RTE_CACHE_LINE_ROUNDUP(size), align = RTE_CACHE_LINE_ROUNDUP(align); 179 180 /* check requested socket id and alignment matches first, and if ok, 181 * see if we can resize block 182 */ 183 if ((socket == SOCKET_ID_ANY || 184 (unsigned int)socket == elem->heap->socket_id) && 185 RTE_PTR_ALIGN(ptr, align) == ptr && 186 malloc_heap_resize(elem, size) == 0) { 187 rte_eal_trace_mem_realloc(size, align, socket, ptr); 188 189 asan_set_redzone(elem, user_size); 190 191 return ptr; 192 } 193 194 /* either requested socket id doesn't match, alignment is off 195 * or we have no room to expand, 196 * so move the data. 197 */ 198 void *new_ptr = rte_malloc_socket(NULL, size, align, socket); 199 if (new_ptr == NULL) 200 return NULL; 201 /* elem: |pad|data_elem|data|trailer| */ 202 const size_t old_size = old_malloc_size(elem); 203 rte_memcpy(new_ptr, ptr, old_size < size ? old_size : size); 204 rte_free(ptr); 205 206 rte_eal_trace_mem_realloc(size, align, socket, new_ptr); 207 return new_ptr; 208 } 209 210 /* 211 * Resize allocated memory. 212 */ 213 void * 214 rte_realloc(void *ptr, size_t size, unsigned int align) 215 { 216 return rte_realloc_socket(ptr, size, align, SOCKET_ID_ANY); 217 } 218 219 int 220 rte_malloc_validate(const void *ptr, size_t *size) 221 { 222 const struct malloc_elem *elem = malloc_elem_from_data(ptr); 223 if (!malloc_elem_cookies_ok(elem)) 224 return -1; 225 if (size != NULL) 226 *size = elem->size - elem->pad - MALLOC_ELEM_OVERHEAD; 227 return 0; 228 } 229 230 /* 231 * Function to retrieve data for heap on given socket 232 */ 233 int 234 rte_malloc_get_socket_stats(int socket, 235 struct rte_malloc_socket_stats *socket_stats) 236 { 237 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 238 int heap_idx; 239 240 heap_idx = malloc_socket_to_heap_id(socket); 241 if (heap_idx < 0) 242 return -1; 243 244 return malloc_heap_get_stats(&mcfg->malloc_heaps[heap_idx], 245 socket_stats); 246 } 247 248 /* 249 * Function to dump contents of all heaps 250 */ 251 void 252 rte_malloc_dump_heaps(FILE *f) 253 { 254 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 255 unsigned int idx; 256 257 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 258 fprintf(f, "Heap id: %u\n", idx); 259 malloc_heap_dump(&mcfg->malloc_heaps[idx], f); 260 } 261 } 262 263 int 264 rte_malloc_heap_get_socket(const char *name) 265 { 266 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 267 struct malloc_heap *heap = NULL; 268 unsigned int idx; 269 int ret; 270 271 if (name == NULL || 272 strnlen(name, RTE_HEAP_NAME_MAX_LEN) == 0 || 273 strnlen(name, RTE_HEAP_NAME_MAX_LEN) == 274 RTE_HEAP_NAME_MAX_LEN) { 275 rte_errno = EINVAL; 276 return -1; 277 } 278 rte_mcfg_mem_read_lock(); 279 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 280 struct malloc_heap *tmp = &mcfg->malloc_heaps[idx]; 281 282 if (!strncmp(name, tmp->name, RTE_HEAP_NAME_MAX_LEN)) { 283 heap = tmp; 284 break; 285 } 286 } 287 288 if (heap != NULL) { 289 ret = heap->socket_id; 290 } else { 291 rte_errno = ENOENT; 292 ret = -1; 293 } 294 rte_mcfg_mem_read_unlock(); 295 296 return ret; 297 } 298 299 int 300 rte_malloc_heap_socket_is_external(int socket_id) 301 { 302 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 303 unsigned int idx; 304 int ret = -1; 305 306 if (socket_id == SOCKET_ID_ANY) 307 return 0; 308 309 rte_mcfg_mem_read_lock(); 310 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 311 struct malloc_heap *tmp = &mcfg->malloc_heaps[idx]; 312 313 if ((int)tmp->socket_id == socket_id) { 314 /* external memory always has large socket ID's */ 315 ret = tmp->socket_id >= RTE_MAX_NUMA_NODES; 316 break; 317 } 318 } 319 rte_mcfg_mem_read_unlock(); 320 321 return ret; 322 } 323 324 /* 325 * Print stats on memory type. If type is NULL, info on all types is printed 326 */ 327 void 328 rte_malloc_dump_stats(FILE *f, __rte_unused const char *type) 329 { 330 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 331 unsigned int heap_id; 332 struct rte_malloc_socket_stats sock_stats; 333 334 /* Iterate through all initialised heaps */ 335 for (heap_id = 0; heap_id < RTE_MAX_HEAPS; heap_id++) { 336 struct malloc_heap *heap = &mcfg->malloc_heaps[heap_id]; 337 338 malloc_heap_get_stats(heap, &sock_stats); 339 340 fprintf(f, "Heap id:%u\n", heap_id); 341 fprintf(f, "\tHeap name:%s\n", heap->name); 342 fprintf(f, "\tHeap_size:%zu,\n", sock_stats.heap_totalsz_bytes); 343 fprintf(f, "\tFree_size:%zu,\n", sock_stats.heap_freesz_bytes); 344 fprintf(f, "\tAlloc_size:%zu,\n", sock_stats.heap_allocsz_bytes); 345 fprintf(f, "\tGreatest_free_size:%zu,\n", 346 sock_stats.greatest_free_size); 347 fprintf(f, "\tAlloc_count:%u,\n",sock_stats.alloc_count); 348 fprintf(f, "\tFree_count:%u,\n", sock_stats.free_count); 349 } 350 return; 351 } 352 353 /* 354 * TODO: Set limit to memory that can be allocated to memory type 355 */ 356 int 357 rte_malloc_set_limit(__rte_unused const char *type, 358 __rte_unused size_t max) 359 { 360 return 0; 361 } 362 363 /* 364 * Return the IO address of a virtual address obtained through rte_malloc 365 */ 366 rte_iova_t 367 rte_malloc_virt2iova(const void *addr) 368 { 369 const struct rte_memseg *ms; 370 struct malloc_elem *elem = malloc_elem_from_data(addr); 371 372 if (elem == NULL) 373 return RTE_BAD_IOVA; 374 375 if (!elem->msl->external && rte_eal_iova_mode() == RTE_IOVA_VA) 376 return (uintptr_t) addr; 377 378 ms = rte_mem_virt2memseg(addr, elem->msl); 379 if (ms == NULL) 380 return RTE_BAD_IOVA; 381 382 if (ms->iova == RTE_BAD_IOVA) 383 return RTE_BAD_IOVA; 384 385 return ms->iova + RTE_PTR_DIFF(addr, ms->addr); 386 } 387 388 static struct malloc_heap * 389 find_named_heap(const char *name) 390 { 391 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 392 unsigned int i; 393 394 for (i = 0; i < RTE_MAX_HEAPS; i++) { 395 struct malloc_heap *heap = &mcfg->malloc_heaps[i]; 396 397 if (!strncmp(name, heap->name, RTE_HEAP_NAME_MAX_LEN)) 398 return heap; 399 } 400 return NULL; 401 } 402 403 int 404 rte_malloc_heap_memory_add(const char *heap_name, void *va_addr, size_t len, 405 rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz) 406 { 407 struct malloc_heap *heap = NULL; 408 struct rte_memseg_list *msl; 409 unsigned int n; 410 int ret; 411 412 if (heap_name == NULL || va_addr == NULL || 413 page_sz == 0 || !rte_is_power_of_2(page_sz) || 414 RTE_ALIGN(len, page_sz) != len || 415 !rte_is_aligned(va_addr, page_sz) || 416 ((len / page_sz) != n_pages && iova_addrs != NULL) || 417 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 418 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 419 RTE_HEAP_NAME_MAX_LEN) { 420 rte_errno = EINVAL; 421 return -1; 422 } 423 rte_mcfg_mem_write_lock(); 424 425 /* find our heap */ 426 heap = find_named_heap(heap_name); 427 if (heap == NULL) { 428 rte_errno = ENOENT; 429 ret = -1; 430 goto unlock; 431 } 432 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 433 /* cannot add memory to internal heaps */ 434 rte_errno = EPERM; 435 ret = -1; 436 goto unlock; 437 } 438 n = len / page_sz; 439 440 msl = malloc_heap_create_external_seg(va_addr, iova_addrs, n, page_sz, 441 heap_name, heap->socket_id); 442 if (msl == NULL) { 443 ret = -1; 444 goto unlock; 445 } 446 447 rte_spinlock_lock(&heap->lock); 448 ret = malloc_heap_add_external_memory(heap, msl); 449 msl->heap = 1; /* mark it as heap segment */ 450 rte_spinlock_unlock(&heap->lock); 451 452 unlock: 453 rte_mcfg_mem_write_unlock(); 454 455 return ret; 456 } 457 458 int 459 rte_malloc_heap_memory_remove(const char *heap_name, void *va_addr, size_t len) 460 { 461 struct malloc_heap *heap = NULL; 462 struct rte_memseg_list *msl; 463 int ret; 464 465 if (heap_name == NULL || va_addr == NULL || len == 0 || 466 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 467 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 468 RTE_HEAP_NAME_MAX_LEN) { 469 rte_errno = EINVAL; 470 return -1; 471 } 472 rte_mcfg_mem_write_lock(); 473 /* find our heap */ 474 heap = find_named_heap(heap_name); 475 if (heap == NULL) { 476 rte_errno = ENOENT; 477 ret = -1; 478 goto unlock; 479 } 480 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 481 /* cannot remove memory from internal heaps */ 482 rte_errno = EPERM; 483 ret = -1; 484 goto unlock; 485 } 486 487 msl = malloc_heap_find_external_seg(va_addr, len); 488 if (msl == NULL) { 489 ret = -1; 490 goto unlock; 491 } 492 493 rte_spinlock_lock(&heap->lock); 494 ret = malloc_heap_remove_external_memory(heap, va_addr, len); 495 rte_spinlock_unlock(&heap->lock); 496 if (ret != 0) 497 goto unlock; 498 499 ret = malloc_heap_destroy_external_seg(msl); 500 501 unlock: 502 rte_mcfg_mem_write_unlock(); 503 504 return ret; 505 } 506 507 static int 508 sync_memory(const char *heap_name, void *va_addr, size_t len, bool attach) 509 { 510 struct malloc_heap *heap = NULL; 511 struct rte_memseg_list *msl; 512 int ret; 513 514 if (heap_name == NULL || va_addr == NULL || len == 0 || 515 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 516 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 517 RTE_HEAP_NAME_MAX_LEN) { 518 rte_errno = EINVAL; 519 return -1; 520 } 521 rte_mcfg_mem_read_lock(); 522 523 /* find our heap */ 524 heap = find_named_heap(heap_name); 525 if (heap == NULL) { 526 rte_errno = ENOENT; 527 ret = -1; 528 goto unlock; 529 } 530 /* we shouldn't be able to sync to internal heaps */ 531 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 532 rte_errno = EPERM; 533 ret = -1; 534 goto unlock; 535 } 536 537 /* find corresponding memseg list to sync to */ 538 msl = malloc_heap_find_external_seg(va_addr, len); 539 if (msl == NULL) { 540 ret = -1; 541 goto unlock; 542 } 543 544 if (attach) { 545 ret = rte_fbarray_attach(&msl->memseg_arr); 546 if (ret == 0) { 547 /* notify all subscribers that a new memory area was 548 * added. 549 */ 550 eal_memalloc_mem_event_notify(RTE_MEM_EVENT_ALLOC, 551 va_addr, len); 552 } else { 553 ret = -1; 554 goto unlock; 555 } 556 } else { 557 /* notify all subscribers that a memory area is about to 558 * be removed. 559 */ 560 eal_memalloc_mem_event_notify(RTE_MEM_EVENT_FREE, 561 msl->base_va, msl->len); 562 ret = rte_fbarray_detach(&msl->memseg_arr); 563 if (ret < 0) { 564 ret = -1; 565 goto unlock; 566 } 567 } 568 unlock: 569 rte_mcfg_mem_read_unlock(); 570 return ret; 571 } 572 573 int 574 rte_malloc_heap_memory_attach(const char *heap_name, void *va_addr, size_t len) 575 { 576 return sync_memory(heap_name, va_addr, len, true); 577 } 578 579 int 580 rte_malloc_heap_memory_detach(const char *heap_name, void *va_addr, size_t len) 581 { 582 return sync_memory(heap_name, va_addr, len, false); 583 } 584 585 int 586 rte_malloc_heap_create(const char *heap_name) 587 { 588 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 589 struct malloc_heap *heap = NULL; 590 int i, ret; 591 592 if (heap_name == NULL || 593 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 594 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 595 RTE_HEAP_NAME_MAX_LEN) { 596 rte_errno = EINVAL; 597 return -1; 598 } 599 /* check if there is space in the heap list, or if heap with this name 600 * already exists. 601 */ 602 rte_mcfg_mem_write_lock(); 603 604 for (i = 0; i < RTE_MAX_HEAPS; i++) { 605 struct malloc_heap *tmp = &mcfg->malloc_heaps[i]; 606 /* existing heap */ 607 if (strncmp(heap_name, tmp->name, 608 RTE_HEAP_NAME_MAX_LEN) == 0) { 609 RTE_LOG(ERR, EAL, "Heap %s already exists\n", 610 heap_name); 611 rte_errno = EEXIST; 612 ret = -1; 613 goto unlock; 614 } 615 /* empty heap */ 616 if (strnlen(tmp->name, RTE_HEAP_NAME_MAX_LEN) == 0) { 617 heap = tmp; 618 break; 619 } 620 } 621 if (heap == NULL) { 622 RTE_LOG(ERR, EAL, "Cannot create new heap: no space\n"); 623 rte_errno = ENOSPC; 624 ret = -1; 625 goto unlock; 626 } 627 628 /* we're sure that we can create a new heap, so do it */ 629 ret = malloc_heap_create(heap, heap_name); 630 unlock: 631 rte_mcfg_mem_write_unlock(); 632 633 return ret; 634 } 635 636 int 637 rte_malloc_heap_destroy(const char *heap_name) 638 { 639 struct malloc_heap *heap = NULL; 640 int ret; 641 642 if (heap_name == NULL || 643 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 644 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 645 RTE_HEAP_NAME_MAX_LEN) { 646 rte_errno = EINVAL; 647 return -1; 648 } 649 rte_mcfg_mem_write_lock(); 650 651 /* start from non-socket heaps */ 652 heap = find_named_heap(heap_name); 653 if (heap == NULL) { 654 RTE_LOG(ERR, EAL, "Heap %s not found\n", heap_name); 655 rte_errno = ENOENT; 656 ret = -1; 657 goto unlock; 658 } 659 /* we shouldn't be able to destroy internal heaps */ 660 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 661 rte_errno = EPERM; 662 ret = -1; 663 goto unlock; 664 } 665 /* sanity checks done, now we can destroy the heap */ 666 rte_spinlock_lock(&heap->lock); 667 ret = malloc_heap_destroy(heap); 668 669 /* if we failed, lock is still active */ 670 if (ret < 0) 671 rte_spinlock_unlock(&heap->lock); 672 unlock: 673 rte_mcfg_mem_write_unlock(); 674 675 return ret; 676 } 677