1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2019 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stddef.h> 7 #include <stdio.h> 8 #include <string.h> 9 #include <sys/queue.h> 10 11 #include <rte_errno.h> 12 #include <rte_memcpy.h> 13 #include <rte_memory.h> 14 #include <rte_eal.h> 15 #include <rte_eal_memconfig.h> 16 #include <rte_branch_prediction.h> 17 #include <rte_debug.h> 18 #include <rte_launch.h> 19 #include <rte_per_lcore.h> 20 #include <rte_lcore.h> 21 #include <rte_common.h> 22 #include <rte_spinlock.h> 23 24 #include <rte_eal_trace.h> 25 26 #include <rte_malloc.h> 27 #include "malloc_elem.h" 28 #include "malloc_heap.h" 29 #include "eal_memalloc.h" 30 #include "eal_memcfg.h" 31 #include "eal_private.h" 32 33 34 /* Free the memory space back to heap */ 35 static void 36 mem_free(void *addr, const bool trace_ena) 37 { 38 if (trace_ena) 39 rte_eal_trace_mem_free(addr); 40 41 if (addr == NULL) return; 42 if (malloc_heap_free(malloc_elem_from_data(addr)) < 0) 43 RTE_LOG(ERR, EAL, "Error: Invalid memory\n"); 44 } 45 46 void 47 rte_free(void *addr) 48 { 49 return mem_free(addr, true); 50 } 51 52 void 53 eal_free_no_trace(void *addr) 54 { 55 return mem_free(addr, false); 56 } 57 58 static void * 59 malloc_socket(const char *type, size_t size, unsigned int align, 60 int socket_arg, const bool trace_ena) 61 { 62 void *ptr; 63 64 /* return NULL if size is 0 or alignment is not power-of-2 */ 65 if (size == 0 || (align && !rte_is_power_of_2(align))) 66 return NULL; 67 68 /* if there are no hugepages and if we are not allocating from an 69 * external heap, use memory from any socket available. checking for 70 * socket being external may return -1 in case of invalid socket, but 71 * that's OK - if there are no hugepages, it doesn't matter. 72 */ 73 if (rte_malloc_heap_socket_is_external(socket_arg) != 1 && 74 !rte_eal_has_hugepages()) 75 socket_arg = SOCKET_ID_ANY; 76 77 ptr = malloc_heap_alloc(type, size, socket_arg, 0, 78 align == 0 ? 1 : align, 0, false); 79 80 if (trace_ena) 81 rte_eal_trace_mem_malloc(type, size, align, socket_arg, ptr); 82 return ptr; 83 } 84 85 /* 86 * Allocate memory on specified heap. 87 */ 88 void * 89 rte_malloc_socket(const char *type, size_t size, unsigned int align, 90 int socket_arg) 91 { 92 return malloc_socket(type, size, align, socket_arg, true); 93 } 94 95 void * 96 eal_malloc_no_trace(const char *type, size_t size, unsigned int align) 97 { 98 return malloc_socket(type, size, align, SOCKET_ID_ANY, false); 99 } 100 101 /* 102 * Allocate memory on default heap. 103 */ 104 void * 105 rte_malloc(const char *type, size_t size, unsigned align) 106 { 107 return rte_malloc_socket(type, size, align, SOCKET_ID_ANY); 108 } 109 110 /* 111 * Allocate zero'd memory on specified heap. 112 */ 113 void * 114 rte_zmalloc_socket(const char *type, size_t size, unsigned align, int socket) 115 { 116 void *ptr = rte_malloc_socket(type, size, align, socket); 117 118 #ifdef RTE_MALLOC_DEBUG 119 /* 120 * If DEBUG is enabled, then freed memory is marked with poison 121 * value and set to zero on allocation. 122 * If DEBUG is not enabled then memory is already zeroed. 123 */ 124 if (ptr != NULL) 125 memset(ptr, 0, size); 126 #endif 127 128 rte_eal_trace_mem_zmalloc(type, size, align, socket, ptr); 129 return ptr; 130 } 131 132 /* 133 * Allocate zero'd memory on default heap. 134 */ 135 void * 136 rte_zmalloc(const char *type, size_t size, unsigned align) 137 { 138 return rte_zmalloc_socket(type, size, align, SOCKET_ID_ANY); 139 } 140 141 /* 142 * Allocate zero'd memory on specified heap. 143 */ 144 void * 145 rte_calloc_socket(const char *type, size_t num, size_t size, unsigned align, int socket) 146 { 147 return rte_zmalloc_socket(type, num * size, align, socket); 148 } 149 150 /* 151 * Allocate zero'd memory on default heap. 152 */ 153 void * 154 rte_calloc(const char *type, size_t num, size_t size, unsigned align) 155 { 156 return rte_zmalloc(type, num * size, align); 157 } 158 159 /* 160 * Resize allocated memory on specified heap. 161 */ 162 void * 163 rte_realloc_socket(void *ptr, size_t size, unsigned int align, int socket) 164 { 165 if (ptr == NULL) 166 return rte_malloc_socket(NULL, size, align, socket); 167 168 struct malloc_elem *elem = malloc_elem_from_data(ptr); 169 if (elem == NULL) { 170 RTE_LOG(ERR, EAL, "Error: memory corruption detected\n"); 171 return NULL; 172 } 173 174 size = RTE_CACHE_LINE_ROUNDUP(size), align = RTE_CACHE_LINE_ROUNDUP(align); 175 176 /* check requested socket id and alignment matches first, and if ok, 177 * see if we can resize block 178 */ 179 if ((socket == SOCKET_ID_ANY || 180 (unsigned int)socket == elem->heap->socket_id) && 181 RTE_PTR_ALIGN(ptr, align) == ptr && 182 malloc_heap_resize(elem, size) == 0) { 183 rte_eal_trace_mem_realloc(size, align, socket, ptr); 184 return ptr; 185 } 186 187 /* either requested socket id doesn't match, alignment is off 188 * or we have no room to expand, 189 * so move the data. 190 */ 191 void *new_ptr = rte_malloc_socket(NULL, size, align, socket); 192 if (new_ptr == NULL) 193 return NULL; 194 /* elem: |pad|data_elem|data|trailer| */ 195 const size_t old_size = elem->size - elem->pad - MALLOC_ELEM_OVERHEAD; 196 rte_memcpy(new_ptr, ptr, old_size < size ? old_size : size); 197 rte_free(ptr); 198 199 rte_eal_trace_mem_realloc(size, align, socket, new_ptr); 200 return new_ptr; 201 } 202 203 /* 204 * Resize allocated memory. 205 */ 206 void * 207 rte_realloc(void *ptr, size_t size, unsigned int align) 208 { 209 return rte_realloc_socket(ptr, size, align, SOCKET_ID_ANY); 210 } 211 212 int 213 rte_malloc_validate(const void *ptr, size_t *size) 214 { 215 const struct malloc_elem *elem = malloc_elem_from_data(ptr); 216 if (!malloc_elem_cookies_ok(elem)) 217 return -1; 218 if (size != NULL) 219 *size = elem->size - elem->pad - MALLOC_ELEM_OVERHEAD; 220 return 0; 221 } 222 223 /* 224 * Function to retrieve data for heap on given socket 225 */ 226 int 227 rte_malloc_get_socket_stats(int socket, 228 struct rte_malloc_socket_stats *socket_stats) 229 { 230 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 231 int heap_idx; 232 233 heap_idx = malloc_socket_to_heap_id(socket); 234 if (heap_idx < 0) 235 return -1; 236 237 return malloc_heap_get_stats(&mcfg->malloc_heaps[heap_idx], 238 socket_stats); 239 } 240 241 /* 242 * Function to dump contents of all heaps 243 */ 244 void 245 rte_malloc_dump_heaps(FILE *f) 246 { 247 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 248 unsigned int idx; 249 250 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 251 fprintf(f, "Heap id: %u\n", idx); 252 malloc_heap_dump(&mcfg->malloc_heaps[idx], f); 253 } 254 } 255 256 int 257 rte_malloc_heap_get_socket(const char *name) 258 { 259 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 260 struct malloc_heap *heap = NULL; 261 unsigned int idx; 262 int ret; 263 264 if (name == NULL || 265 strnlen(name, RTE_HEAP_NAME_MAX_LEN) == 0 || 266 strnlen(name, RTE_HEAP_NAME_MAX_LEN) == 267 RTE_HEAP_NAME_MAX_LEN) { 268 rte_errno = EINVAL; 269 return -1; 270 } 271 rte_mcfg_mem_read_lock(); 272 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 273 struct malloc_heap *tmp = &mcfg->malloc_heaps[idx]; 274 275 if (!strncmp(name, tmp->name, RTE_HEAP_NAME_MAX_LEN)) { 276 heap = tmp; 277 break; 278 } 279 } 280 281 if (heap != NULL) { 282 ret = heap->socket_id; 283 } else { 284 rte_errno = ENOENT; 285 ret = -1; 286 } 287 rte_mcfg_mem_read_unlock(); 288 289 return ret; 290 } 291 292 int 293 rte_malloc_heap_socket_is_external(int socket_id) 294 { 295 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 296 unsigned int idx; 297 int ret = -1; 298 299 if (socket_id == SOCKET_ID_ANY) 300 return 0; 301 302 rte_mcfg_mem_read_lock(); 303 for (idx = 0; idx < RTE_MAX_HEAPS; idx++) { 304 struct malloc_heap *tmp = &mcfg->malloc_heaps[idx]; 305 306 if ((int)tmp->socket_id == socket_id) { 307 /* external memory always has large socket ID's */ 308 ret = tmp->socket_id >= RTE_MAX_NUMA_NODES; 309 break; 310 } 311 } 312 rte_mcfg_mem_read_unlock(); 313 314 return ret; 315 } 316 317 /* 318 * Print stats on memory type. If type is NULL, info on all types is printed 319 */ 320 void 321 rte_malloc_dump_stats(FILE *f, __rte_unused const char *type) 322 { 323 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 324 unsigned int heap_id; 325 struct rte_malloc_socket_stats sock_stats; 326 327 /* Iterate through all initialised heaps */ 328 for (heap_id = 0; heap_id < RTE_MAX_HEAPS; heap_id++) { 329 struct malloc_heap *heap = &mcfg->malloc_heaps[heap_id]; 330 331 malloc_heap_get_stats(heap, &sock_stats); 332 333 fprintf(f, "Heap id:%u\n", heap_id); 334 fprintf(f, "\tHeap name:%s\n", heap->name); 335 fprintf(f, "\tHeap_size:%zu,\n", sock_stats.heap_totalsz_bytes); 336 fprintf(f, "\tFree_size:%zu,\n", sock_stats.heap_freesz_bytes); 337 fprintf(f, "\tAlloc_size:%zu,\n", sock_stats.heap_allocsz_bytes); 338 fprintf(f, "\tGreatest_free_size:%zu,\n", 339 sock_stats.greatest_free_size); 340 fprintf(f, "\tAlloc_count:%u,\n",sock_stats.alloc_count); 341 fprintf(f, "\tFree_count:%u,\n", sock_stats.free_count); 342 } 343 return; 344 } 345 346 /* 347 * TODO: Set limit to memory that can be allocated to memory type 348 */ 349 int 350 rte_malloc_set_limit(__rte_unused const char *type, 351 __rte_unused size_t max) 352 { 353 return 0; 354 } 355 356 /* 357 * Return the IO address of a virtual address obtained through rte_malloc 358 */ 359 rte_iova_t 360 rte_malloc_virt2iova(const void *addr) 361 { 362 const struct rte_memseg *ms; 363 struct malloc_elem *elem = malloc_elem_from_data(addr); 364 365 if (elem == NULL) 366 return RTE_BAD_IOVA; 367 368 if (!elem->msl->external && rte_eal_iova_mode() == RTE_IOVA_VA) 369 return (uintptr_t) addr; 370 371 ms = rte_mem_virt2memseg(addr, elem->msl); 372 if (ms == NULL) 373 return RTE_BAD_IOVA; 374 375 if (ms->iova == RTE_BAD_IOVA) 376 return RTE_BAD_IOVA; 377 378 return ms->iova + RTE_PTR_DIFF(addr, ms->addr); 379 } 380 381 static struct malloc_heap * 382 find_named_heap(const char *name) 383 { 384 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 385 unsigned int i; 386 387 for (i = 0; i < RTE_MAX_HEAPS; i++) { 388 struct malloc_heap *heap = &mcfg->malloc_heaps[i]; 389 390 if (!strncmp(name, heap->name, RTE_HEAP_NAME_MAX_LEN)) 391 return heap; 392 } 393 return NULL; 394 } 395 396 int 397 rte_malloc_heap_memory_add(const char *heap_name, void *va_addr, size_t len, 398 rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz) 399 { 400 struct malloc_heap *heap = NULL; 401 struct rte_memseg_list *msl; 402 unsigned int n; 403 int ret; 404 405 if (heap_name == NULL || va_addr == NULL || 406 page_sz == 0 || !rte_is_power_of_2(page_sz) || 407 RTE_ALIGN(len, page_sz) != len || 408 !rte_is_aligned(va_addr, page_sz) || 409 ((len / page_sz) != n_pages && iova_addrs != NULL) || 410 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 411 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 412 RTE_HEAP_NAME_MAX_LEN) { 413 rte_errno = EINVAL; 414 return -1; 415 } 416 rte_mcfg_mem_write_lock(); 417 418 /* find our heap */ 419 heap = find_named_heap(heap_name); 420 if (heap == NULL) { 421 rte_errno = ENOENT; 422 ret = -1; 423 goto unlock; 424 } 425 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 426 /* cannot add memory to internal heaps */ 427 rte_errno = EPERM; 428 ret = -1; 429 goto unlock; 430 } 431 n = len / page_sz; 432 433 msl = malloc_heap_create_external_seg(va_addr, iova_addrs, n, page_sz, 434 heap_name, heap->socket_id); 435 if (msl == NULL) { 436 ret = -1; 437 goto unlock; 438 } 439 440 rte_spinlock_lock(&heap->lock); 441 ret = malloc_heap_add_external_memory(heap, msl); 442 msl->heap = 1; /* mark it as heap segment */ 443 rte_spinlock_unlock(&heap->lock); 444 445 unlock: 446 rte_mcfg_mem_write_unlock(); 447 448 return ret; 449 } 450 451 int 452 rte_malloc_heap_memory_remove(const char *heap_name, void *va_addr, size_t len) 453 { 454 struct malloc_heap *heap = NULL; 455 struct rte_memseg_list *msl; 456 int ret; 457 458 if (heap_name == NULL || va_addr == NULL || len == 0 || 459 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 460 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 461 RTE_HEAP_NAME_MAX_LEN) { 462 rte_errno = EINVAL; 463 return -1; 464 } 465 rte_mcfg_mem_write_lock(); 466 /* find our heap */ 467 heap = find_named_heap(heap_name); 468 if (heap == NULL) { 469 rte_errno = ENOENT; 470 ret = -1; 471 goto unlock; 472 } 473 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 474 /* cannot remove memory from internal heaps */ 475 rte_errno = EPERM; 476 ret = -1; 477 goto unlock; 478 } 479 480 msl = malloc_heap_find_external_seg(va_addr, len); 481 if (msl == NULL) { 482 ret = -1; 483 goto unlock; 484 } 485 486 rte_spinlock_lock(&heap->lock); 487 ret = malloc_heap_remove_external_memory(heap, va_addr, len); 488 rte_spinlock_unlock(&heap->lock); 489 if (ret != 0) 490 goto unlock; 491 492 ret = malloc_heap_destroy_external_seg(msl); 493 494 unlock: 495 rte_mcfg_mem_write_unlock(); 496 497 return ret; 498 } 499 500 static int 501 sync_memory(const char *heap_name, void *va_addr, size_t len, bool attach) 502 { 503 struct malloc_heap *heap = NULL; 504 struct rte_memseg_list *msl; 505 int ret; 506 507 if (heap_name == NULL || va_addr == NULL || len == 0 || 508 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 509 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 510 RTE_HEAP_NAME_MAX_LEN) { 511 rte_errno = EINVAL; 512 return -1; 513 } 514 rte_mcfg_mem_read_lock(); 515 516 /* find our heap */ 517 heap = find_named_heap(heap_name); 518 if (heap == NULL) { 519 rte_errno = ENOENT; 520 ret = -1; 521 goto unlock; 522 } 523 /* we shouldn't be able to sync to internal heaps */ 524 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 525 rte_errno = EPERM; 526 ret = -1; 527 goto unlock; 528 } 529 530 /* find corresponding memseg list to sync to */ 531 msl = malloc_heap_find_external_seg(va_addr, len); 532 if (msl == NULL) { 533 ret = -1; 534 goto unlock; 535 } 536 537 if (attach) { 538 ret = rte_fbarray_attach(&msl->memseg_arr); 539 if (ret == 0) { 540 /* notify all subscribers that a new memory area was 541 * added. 542 */ 543 eal_memalloc_mem_event_notify(RTE_MEM_EVENT_ALLOC, 544 va_addr, len); 545 } else { 546 ret = -1; 547 goto unlock; 548 } 549 } else { 550 /* notify all subscribers that a memory area is about to 551 * be removed. 552 */ 553 eal_memalloc_mem_event_notify(RTE_MEM_EVENT_FREE, 554 msl->base_va, msl->len); 555 ret = rte_fbarray_detach(&msl->memseg_arr); 556 if (ret < 0) { 557 ret = -1; 558 goto unlock; 559 } 560 } 561 unlock: 562 rte_mcfg_mem_read_unlock(); 563 return ret; 564 } 565 566 int 567 rte_malloc_heap_memory_attach(const char *heap_name, void *va_addr, size_t len) 568 { 569 return sync_memory(heap_name, va_addr, len, true); 570 } 571 572 int 573 rte_malloc_heap_memory_detach(const char *heap_name, void *va_addr, size_t len) 574 { 575 return sync_memory(heap_name, va_addr, len, false); 576 } 577 578 int 579 rte_malloc_heap_create(const char *heap_name) 580 { 581 struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config; 582 struct malloc_heap *heap = NULL; 583 int i, ret; 584 585 if (heap_name == NULL || 586 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 587 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 588 RTE_HEAP_NAME_MAX_LEN) { 589 rte_errno = EINVAL; 590 return -1; 591 } 592 /* check if there is space in the heap list, or if heap with this name 593 * already exists. 594 */ 595 rte_mcfg_mem_write_lock(); 596 597 for (i = 0; i < RTE_MAX_HEAPS; i++) { 598 struct malloc_heap *tmp = &mcfg->malloc_heaps[i]; 599 /* existing heap */ 600 if (strncmp(heap_name, tmp->name, 601 RTE_HEAP_NAME_MAX_LEN) == 0) { 602 RTE_LOG(ERR, EAL, "Heap %s already exists\n", 603 heap_name); 604 rte_errno = EEXIST; 605 ret = -1; 606 goto unlock; 607 } 608 /* empty heap */ 609 if (strnlen(tmp->name, RTE_HEAP_NAME_MAX_LEN) == 0) { 610 heap = tmp; 611 break; 612 } 613 } 614 if (heap == NULL) { 615 RTE_LOG(ERR, EAL, "Cannot create new heap: no space\n"); 616 rte_errno = ENOSPC; 617 ret = -1; 618 goto unlock; 619 } 620 621 /* we're sure that we can create a new heap, so do it */ 622 ret = malloc_heap_create(heap, heap_name); 623 unlock: 624 rte_mcfg_mem_write_unlock(); 625 626 return ret; 627 } 628 629 int 630 rte_malloc_heap_destroy(const char *heap_name) 631 { 632 struct malloc_heap *heap = NULL; 633 int ret; 634 635 if (heap_name == NULL || 636 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 || 637 strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 638 RTE_HEAP_NAME_MAX_LEN) { 639 rte_errno = EINVAL; 640 return -1; 641 } 642 rte_mcfg_mem_write_lock(); 643 644 /* start from non-socket heaps */ 645 heap = find_named_heap(heap_name); 646 if (heap == NULL) { 647 RTE_LOG(ERR, EAL, "Heap %s not found\n", heap_name); 648 rte_errno = ENOENT; 649 ret = -1; 650 goto unlock; 651 } 652 /* we shouldn't be able to destroy internal heaps */ 653 if (heap->socket_id < RTE_MAX_NUMA_NODES) { 654 rte_errno = EPERM; 655 ret = -1; 656 goto unlock; 657 } 658 /* sanity checks done, now we can destroy the heap */ 659 rte_spinlock_lock(&heap->lock); 660 ret = malloc_heap_destroy(heap); 661 662 /* if we failed, lock is still active */ 663 if (ret < 0) 664 rte_spinlock_unlock(&heap->lock); 665 unlock: 666 rte_mcfg_mem_write_unlock(); 667 668 return ret; 669 } 670