1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2019 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stddef.h> 7 #include <stdio.h> 8 #include <string.h> 9 #include <sys/queue.h> 10 11 #include <rte_errno.h> 12 #include <rte_memcpy.h> 13 #include <rte_memory.h> 14 #include <rte_eal.h> 15 #include <rte_eal_memconfig.h> 16 #include <rte_common.h> 17 #include <rte_spinlock.h> 18 19 #include <rte_eal_trace.h> 20 21 #include <rte_malloc.h> 22 #include "malloc_elem.h" 23 #include "malloc_heap.h" 24 #include "eal_memalloc.h" 25 #include "eal_memcfg.h" 26 #include "eal_private.h" 27 28 29 /* Free the memory space back to heap */ 30 static void 31 mem_free(void *addr, const bool trace_ena) 32 { 33 if (trace_ena) 34 rte_eal_trace_mem_free(addr); 35 36 if (addr == NULL) return; 37 if (malloc_heap_free(malloc_elem_from_data(addr)) < 0) 38 RTE_LOG(ERR, EAL, "Error: Invalid memory\n"); 39 } 40 41 void 42 rte_free(void *addr) 43 { 44 return mem_free(addr, true); 45 } 46 47 void 48 eal_free_no_trace(void *addr) 49 { 50 return mem_free(addr, false); 51 } 52 53 static void * 54 malloc_socket(const char *type, size_t size, unsigned int align, 55 int socket_arg, const bool trace_ena) 56 { 57 void *ptr; 58 59 /* return NULL if size is 0 or alignment is not power-of-2 */ 60 if (size == 0 || (align && !rte_is_power_of_2(align))) 61 return NULL; 62 63 /* if there are no hugepages and if we are not allocating from an 64 * external heap, use memory from any socket available. checking for 65 * socket being external may return -1 in case of invalid socket, but 66 * that's OK - if there are no hugepages, it doesn't matter. 67 */ 68 if (rte_malloc_heap_socket_is_external(socket_arg) != 1 && 69 !rte_eal_has_hugepages()) 70 socket_arg = SOCKET_ID_ANY; 71 72 ptr = malloc_heap_alloc(type, size, socket_arg, 0, 73 align == 0 ? 1 : align, 0, false); 74 75 if (trace_ena) 76 rte_eal_trace_mem_malloc(type, size, align, socket_arg, ptr); 77 return ptr; 78 } 79 80 /* 81 * Allocate memory on specified heap. 82 */ 83 void * 84 rte_malloc_socket(const char *type, size_t size, unsigned int align, 85 int socket_arg) 86 { 87 return malloc_socket(type, size, align, socket_arg, true); 88 } 89 90 void * 91 eal_malloc_no_trace(const char *type, size_t size, unsigned int align) 92 { 93 return malloc_socket(type, size, align, SOCKET_ID_ANY, false); 94 } 95 96 /* 97 * Allocate memory on default heap. 98 */ 99 void * 100 rte_malloc(const char *type, size_t size, unsigned align) 101 { 102 return rte_malloc_socket(type, size, align, SOCKET_ID_ANY); 103 } 104 105 /* 106 * Allocate zero'd memory on specified heap. 107 */ 108 void * 109 rte_zmalloc_socket(const char *type, size_t size, unsigned align, int socket) 110 { 111 void *ptr = rte_malloc_socket(type, size, align, socket); 112 113 if (ptr != NULL) { 114 struct malloc_elem *elem = malloc_elem_from_data(ptr); 115 116 if (elem->dirty) { 117 memset(ptr, 0, size); 118 } else { 119 #ifdef RTE_MALLOC_DEBUG 120 /* 121 * If DEBUG is enabled, then freed memory is marked 122 * with a poison value and set to zero on allocation. 123 * If DEBUG is disabled then memory is already zeroed. 124 */ 125 memset(ptr, 0, size); 126 #endif 127 } 128 } 129 130 rte_eal_trace_mem_zmalloc(type, size, align, socket, ptr); 131 return ptr; 132 } 133 134 /* 135 * Allocate zero'd memory on default heap. 136 */ 137 void * 138 rte_zmalloc(const char *type, size_t size, unsigned align) 139 { 140 return rte_zmalloc_socket(type, size, align, SOCKET_ID_ANY); 141 } 142 143 /* 144 * Allocate zero'd memory on specified heap. 145 */ 146 void * 147 rte_calloc_socket(const char *type, size_t num, size_t size, unsigned align, int socket) 148 { 149 return rte_zmalloc_socket(type, num * size, align, socket); 150 } 151 152 /* 153 * Allocate zero'd memory on default heap. 154 */ 155 void * 156 rte_calloc(const char *type, size_t num, size_t size, unsigned align) 157 { 158 return rte_zmalloc(type, num * size, align); 159 } 160 161 /* 162 * Resize allocated memory on specified heap. 163 */ 164 void * 165 rte_realloc_socket(void *ptr, size_t size, unsigned int align, int socket) 166 { 167 size_t user_size; 168 169 if (ptr == NULL) 170 return rte_malloc_socket(NULL, size, align, socket); 171 172 struct malloc_elem *elem = malloc_elem_from_data(ptr); 173 if (elem == NULL) { 174 RTE_LOG(ERR, EAL, "Error: memory corruption detected\n"); 175 return NULL; 176 } 177 178 user_size = size; 179 180 size = RTE_CACHE_LINE_ROUNDUP(size), align = RTE_CACHE_LINE_ROUNDUP(align); 181 182 /* check requested socket id and alignment matches first, and if ok, 183 * see if we can resize block 184 */ 185 if ((socket == SOCKET_ID_ANY || 186 (unsigned int)socket == elem->heap->socket_id) && 187 RTE_PTR_ALIGN(ptr, align) == ptr && 188 malloc_heap_resize(elem, size) == 0) { 189 rte_eal_trace_mem_realloc(size, align, socket, ptr); 190 191 asan_set_redzone(elem, user_size); 192 193 return ptr; 194 } 195 196 /* either requested socket id doesn't match, alignment is off 197 * or we have no room to expand, 198 * so move the data. 199 */ 200 void *new_ptr = rte_malloc_socket(NULL, size, align, socket); 201 if (new_ptr == NULL) 202 return NULL; 203 /* elem: |pad|data_elem|data|trailer| */ 204 const size_t old_size = old_malloc_size(elem); 205 rte_memcpy(new_ptr, ptr, old_size < size ? old_size : size); 206 rte_free(ptr); 207 208 rte_eal_trace_mem_realloc(size, align, socket, new_ptr); 209 return new_ptr; 210 } 211 212 /* 213 * Resize allocated memory. 214 */ 215 void * 216 rte_realloc(void *ptr, size_t size, unsigned int align) 217 { 218 return rte_realloc_socket(ptr, size, align, SOCKET_ID_ANY); 219 } 220 221 int 222 rte_malloc_validate(const void *ptr, size_t *size) 223 { 224 const struct malloc_elem *elem = malloc_elem_from_data(ptr); 225 if (!malloc_elem_cookies_ok(elem)) 226 return -1; 227 if (size != NULL) 228 *size = elem->size - elem->pad - MALLOC_ELEM_OVERHEAD; 229 return 0; 230 } 231 232 /* 233 * Function to retrieve data for heap on given socket 234 */ 235 int 236 rte_malloc_get_socket_stats(int socket, 237 struct rte_malloc_socket_stats *socket_stats) 238 { 239 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 240 int heap_idx; 241 242 heap_idx = malloc_socket_to_heap_id(socket); 243 if (heap_idx < 0) 244 return -1; 245 246 return malloc_heap_get_stats(&mcfg->malloc_heaps[heap_idx], 247 socket_stats); 248 } 249 250 /* 251 * Function to dump contents of all heaps 252 */ 253 void 254 rte_malloc_dump_heaps(FILE *f) 255 { 256 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 257 unsigned int idx; 258 259 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 260 fprintf(f, "Heap id: %u\n", idx); 261 malloc_heap_dump(&mcfg->malloc_heaps[idx], f); 262 } 263 } 264 265 int 266 rte_malloc_heap_get_socket(const char *name) 267 { 268 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 269 struct malloc_heap *heap = NULL; 270 unsigned int idx; 271 int ret; 272 273 if (name == NULL || 274 strnlen(name, RTE_HEAP_NAME_MAX_LEN) == 0 || 275 strnlen(name, RTE_HEAP_NAME_MAX_LEN) == 276 RTE_HEAP_NAME_MAX_LEN) { 277 rte_errno = EINVAL; 278 return -1; 279 } 280 rte_mcfg_mem_read_lock(); 281 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 282 struct malloc_heap *tmp = &mcfg->malloc_heaps[idx]; 283 284 if (!strncmp(name, tmp->name, RTE_HEAP_NAME_MAX_LEN)) { 285 heap = tmp; 286 break; 287 } 288 } 289 290 if (heap != NULL) { 291 ret = heap->socket_id; 292 } else { 293 rte_errno = ENOENT; 294 ret = -1; 295 } 296 rte_mcfg_mem_read_unlock(); 297 298 return ret; 299 } 300 301 int 302 rte_malloc_heap_socket_is_external(int socket_id) 303 { 304 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 305 unsigned int idx; 306 int ret = -1; 307 308 if (socket_id == SOCKET_ID_ANY) 309 return 0; 310 311 rte_mcfg_mem_read_lock(); 312 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 313 struct malloc_heap *tmp = &mcfg->malloc_heaps[idx]; 314 315 if ((int)tmp->socket_id == socket_id) { 316 /* external memory always has large socket ID's */ 317 ret = tmp->socket_id >= RTE_MAX_NUMA_NODES; 318 break; 319 } 320 } 321 rte_mcfg_mem_read_unlock(); 322 323 return ret; 324 } 325 326 /* 327 * Print stats on memory type. If type is NULL, info on all types is printed 328 */ 329 void 330 rte_malloc_dump_stats(FILE *f, __rte_unused const char *type) 331 { 332 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 333 unsigned int heap_id; 334 struct rte_malloc_socket_stats sock_stats; 335 336 /* Iterate through all initialised heaps */ 337 for (heap_id = 0; heap_id < RTE_MAX_HEAPS; heap_id++) { 338 struct malloc_heap *heap = &mcfg->malloc_heaps[heap_id]; 339 340 malloc_heap_get_stats(heap, &sock_stats); 341 342 fprintf(f, "Heap id:%u\n", heap_id); 343 fprintf(f, "\tHeap name:%s\n", heap->name); 344 fprintf(f, "\tHeap_size:%zu,\n", sock_stats.heap_totalsz_bytes); 345 fprintf(f, "\tFree_size:%zu,\n", sock_stats.heap_freesz_bytes); 346 fprintf(f, "\tAlloc_size:%zu,\n", sock_stats.heap_allocsz_bytes); 347 fprintf(f, "\tGreatest_free_size:%zu,\n", 348 sock_stats.greatest_free_size); 349 fprintf(f, "\tAlloc_count:%u,\n",sock_stats.alloc_count); 350 fprintf(f, "\tFree_count:%u,\n", sock_stats.free_count); 351 } 352 return; 353 } 354 355 /* 356 * TODO: Set limit to memory that can be allocated to memory type 357 */ 358 int 359 rte_malloc_set_limit(__rte_unused const char *type, 360 __rte_unused size_t max) 361 { 362 return 0; 363 } 364 365 /* 366 * Return the IO address of a virtual address obtained through rte_malloc 367 */ 368 rte_iova_t 369 rte_malloc_virt2iova(const void *addr) 370 { 371 const struct rte_memseg *ms; 372 struct malloc_elem *elem = malloc_elem_from_data(addr); 373 374 if (elem == NULL) 375 return RTE_BAD_IOVA; 376 377 if (!elem->msl->external && rte_eal_iova_mode() == RTE_IOVA_VA) 378 return (uintptr_t) addr; 379 380 ms = rte_mem_virt2memseg(addr, elem->msl); 381 if (ms == NULL) 382 return RTE_BAD_IOVA; 383 384 if (ms->iova == RTE_BAD_IOVA) 385 return RTE_BAD_IOVA; 386 387 return ms->iova + RTE_PTR_DIFF(addr, ms->addr); 388 } 389 390 static struct malloc_heap * 391 find_named_heap(const char *name) 392 { 393 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 394 unsigned int i; 395 396 for (i = 0; i < RTE_MAX_HEAPS; i++) { 397 struct malloc_heap *heap = &mcfg->malloc_heaps[i]; 398 399 if (!strncmp(name, heap->name, RTE_HEAP_NAME_MAX_LEN)) 400 return heap; 401 } 402 return NULL; 403 } 404 405 int 406 rte_malloc_heap_memory_add(const char *heap_name, void *va_addr, size_t len, 407 rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz) 408 { 409 struct malloc_heap *heap = NULL; 410 struct rte_memseg_list *msl; 411 unsigned int n; 412 int ret; 413 414 if (heap_name == NULL || va_addr == NULL || 415 page_sz == 0 || !rte_is_power_of_2(page_sz) || 416 RTE_ALIGN(len, page_sz) != len || 417 !rte_is_aligned(va_addr, page_sz) || 418 ((len / page_sz) != n_pages && iova_addrs != NULL) || 419 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 420 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 421 RTE_HEAP_NAME_MAX_LEN) { 422 rte_errno = EINVAL; 423 return -1; 424 } 425 rte_mcfg_mem_write_lock(); 426 427 /* find our heap */ 428 heap = find_named_heap(heap_name); 429 if (heap == NULL) { 430 rte_errno = ENOENT; 431 ret = -1; 432 goto unlock; 433 } 434 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 435 /* cannot add memory to internal heaps */ 436 rte_errno = EPERM; 437 ret = -1; 438 goto unlock; 439 } 440 n = len / page_sz; 441 442 msl = malloc_heap_create_external_seg(va_addr, iova_addrs, n, page_sz, 443 heap_name, heap->socket_id); 444 if (msl == NULL) { 445 ret = -1; 446 goto unlock; 447 } 448 449 rte_spinlock_lock(&heap->lock); 450 ret = malloc_heap_add_external_memory(heap, msl); 451 msl->heap = 1; /* mark it as heap segment */ 452 rte_spinlock_unlock(&heap->lock); 453 454 unlock: 455 rte_mcfg_mem_write_unlock(); 456 457 return ret; 458 } 459 460 int 461 rte_malloc_heap_memory_remove(const char *heap_name, void *va_addr, size_t len) 462 { 463 struct malloc_heap *heap = NULL; 464 struct rte_memseg_list *msl; 465 int ret; 466 467 if (heap_name == NULL || va_addr == NULL || len == 0 || 468 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 469 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 470 RTE_HEAP_NAME_MAX_LEN) { 471 rte_errno = EINVAL; 472 return -1; 473 } 474 rte_mcfg_mem_write_lock(); 475 /* find our heap */ 476 heap = find_named_heap(heap_name); 477 if (heap == NULL) { 478 rte_errno = ENOENT; 479 ret = -1; 480 goto unlock; 481 } 482 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 483 /* cannot remove memory from internal heaps */ 484 rte_errno = EPERM; 485 ret = -1; 486 goto unlock; 487 } 488 489 msl = malloc_heap_find_external_seg(va_addr, len); 490 if (msl == NULL) { 491 ret = -1; 492 goto unlock; 493 } 494 495 rte_spinlock_lock(&heap->lock); 496 ret = malloc_heap_remove_external_memory(heap, va_addr, len); 497 rte_spinlock_unlock(&heap->lock); 498 if (ret != 0) 499 goto unlock; 500 501 ret = malloc_heap_destroy_external_seg(msl); 502 503 unlock: 504 rte_mcfg_mem_write_unlock(); 505 506 return ret; 507 } 508 509 static int 510 sync_memory(const char *heap_name, void *va_addr, size_t len, bool attach) 511 { 512 struct malloc_heap *heap = NULL; 513 struct rte_memseg_list *msl; 514 int ret; 515 516 if (heap_name == NULL || va_addr == NULL || len == 0 || 517 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 518 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 519 RTE_HEAP_NAME_MAX_LEN) { 520 rte_errno = EINVAL; 521 return -1; 522 } 523 rte_mcfg_mem_read_lock(); 524 525 /* find our heap */ 526 heap = find_named_heap(heap_name); 527 if (heap == NULL) { 528 rte_errno = ENOENT; 529 ret = -1; 530 goto unlock; 531 } 532 /* we shouldn't be able to sync to internal heaps */ 533 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 534 rte_errno = EPERM; 535 ret = -1; 536 goto unlock; 537 } 538 539 /* find corresponding memseg list to sync to */ 540 msl = malloc_heap_find_external_seg(va_addr, len); 541 if (msl == NULL) { 542 ret = -1; 543 goto unlock; 544 } 545 546 if (attach) { 547 ret = rte_fbarray_attach(&msl->memseg_arr); 548 if (ret == 0) { 549 /* notify all subscribers that a new memory area was 550 * added. 551 */ 552 eal_memalloc_mem_event_notify(RTE_MEM_EVENT_ALLOC, 553 va_addr, len); 554 } else { 555 ret = -1; 556 goto unlock; 557 } 558 } else { 559 /* notify all subscribers that a memory area is about to 560 * be removed. 561 */ 562 eal_memalloc_mem_event_notify(RTE_MEM_EVENT_FREE, 563 msl->base_va, msl->len); 564 ret = rte_fbarray_detach(&msl->memseg_arr); 565 if (ret < 0) { 566 ret = -1; 567 goto unlock; 568 } 569 } 570 unlock: 571 rte_mcfg_mem_read_unlock(); 572 return ret; 573 } 574 575 int 576 rte_malloc_heap_memory_attach(const char *heap_name, void *va_addr, size_t len) 577 { 578 return sync_memory(heap_name, va_addr, len, true); 579 } 580 581 int 582 rte_malloc_heap_memory_detach(const char *heap_name, void *va_addr, size_t len) 583 { 584 return sync_memory(heap_name, va_addr, len, false); 585 } 586 587 int 588 rte_malloc_heap_create(const char *heap_name) 589 { 590 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 591 struct malloc_heap *heap = NULL; 592 int i, ret; 593 594 if (heap_name == NULL || 595 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 596 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 597 RTE_HEAP_NAME_MAX_LEN) { 598 rte_errno = EINVAL; 599 return -1; 600 } 601 /* check if there is space in the heap list, or if heap with this name 602 * already exists. 603 */ 604 rte_mcfg_mem_write_lock(); 605 606 for (i = 0; i < RTE_MAX_HEAPS; i++) { 607 struct malloc_heap *tmp = &mcfg->malloc_heaps[i]; 608 /* existing heap */ 609 if (strncmp(heap_name, tmp->name, 610 RTE_HEAP_NAME_MAX_LEN) == 0) { 611 RTE_LOG(ERR, EAL, "Heap %s already exists\n", 612 heap_name); 613 rte_errno = EEXIST; 614 ret = -1; 615 goto unlock; 616 } 617 /* empty heap */ 618 if (strnlen(tmp->name, RTE_HEAP_NAME_MAX_LEN) == 0) { 619 heap = tmp; 620 break; 621 } 622 } 623 if (heap == NULL) { 624 RTE_LOG(ERR, EAL, "Cannot create new heap: no space\n"); 625 rte_errno = ENOSPC; 626 ret = -1; 627 goto unlock; 628 } 629 630 /* we're sure that we can create a new heap, so do it */ 631 ret = malloc_heap_create(heap, heap_name); 632 unlock: 633 rte_mcfg_mem_write_unlock(); 634 635 return ret; 636 } 637 638 int 639 rte_malloc_heap_destroy(const char *heap_name) 640 { 641 struct malloc_heap *heap = NULL; 642 int ret; 643 644 if (heap_name == NULL || 645 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 646 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 647 RTE_HEAP_NAME_MAX_LEN) { 648 rte_errno = EINVAL; 649 return -1; 650 } 651 rte_mcfg_mem_write_lock(); 652 653 /* start from non-socket heaps */ 654 heap = find_named_heap(heap_name); 655 if (heap == NULL) { 656 RTE_LOG(ERR, EAL, "Heap %s not found\n", heap_name); 657 rte_errno = ENOENT; 658 ret = -1; 659 goto unlock; 660 } 661 /* we shouldn't be able to destroy internal heaps */ 662 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 663 rte_errno = EPERM; 664 ret = -1; 665 goto unlock; 666 } 667 /* sanity checks done, now we can destroy the heap */ 668 rte_spinlock_lock(&heap->lock); 669 ret = malloc_heap_destroy(heap); 670 671 /* if we failed, lock is still active */ 672 if (ret < 0) 673 rte_spinlock_unlock(&heap->lock); 674 unlock: 675 rte_mcfg_mem_write_unlock(); 676 677 return ret; 678 } 679