1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. 3 * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. 4 * All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 #include "spdk/util.h" 9 #include "spdk/env_dpdk.h" 10 #include "spdk/log.h" 11 12 #include "env_internal.h" 13 14 #include <rte_config.h> 15 #include <rte_cycles.h> 16 #include <rte_malloc.h> 17 #include <rte_mempool.h> 18 #include <rte_memzone.h> 19 #include <rte_version.h> 20 21 static __thread bool g_is_thread_unaffinitized; 22 23 static uint64_t 24 virt_to_phys(void *vaddr) 25 { 26 uint64_t ret; 27 28 ret = rte_malloc_virt2iova(vaddr); 29 if (ret != RTE_BAD_IOVA) { 30 return ret; 31 } 32 33 return spdk_vtophys(vaddr, NULL); 34 } 35 36 void * 37 spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 38 { 39 void *buf; 40 41 if (flags == 0) { 42 return NULL; 43 } 44 45 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 46 buf = rte_malloc_socket(NULL, size, align, socket_id); 47 if (buf && phys_addr) { 48 #ifdef DEBUG 49 SPDK_ERRLOG("phys_addr param in spdk_malloc() is deprecated\n"); 50 #endif 51 *phys_addr = virt_to_phys(buf); 52 } 53 return buf; 54 } 55 56 void * 57 spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 58 { 59 void *buf; 60 61 if (flags == 0) { 62 return NULL; 63 } 64 65 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 66 buf = rte_zmalloc_socket(NULL, size, align, socket_id); 67 if (buf && phys_addr) { 68 #ifdef DEBUG 69 SPDK_ERRLOG("phys_addr param in spdk_zmalloc() is deprecated\n"); 70 #endif 71 *phys_addr = virt_to_phys(buf); 72 } 73 return buf; 74 } 75 76 void * 77 spdk_realloc(void *buf, size_t size, size_t align) 78 { 79 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 80 return rte_realloc(buf, size, align); 81 } 82 83 void 84 spdk_free(void *buf) 85 { 86 rte_free(buf); 87 } 88 89 void * 90 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 91 { 92 return spdk_malloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 93 } 94 95 void * 96 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 97 { 98 return spdk_zmalloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 99 } 100 101 void * 102 spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr) 103 { 104 return spdk_dma_malloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 105 } 106 107 void * 108 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr) 109 { 110 return spdk_dma_zmalloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 111 } 112 113 void * 114 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr) 115 { 116 void *new_buf; 117 118 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 119 new_buf = rte_realloc(buf, size, align); 120 if (new_buf && phys_addr) { 121 *phys_addr = virt_to_phys(new_buf); 122 } 123 return new_buf; 124 } 125 126 void 127 spdk_dma_free(void *buf) 128 { 129 spdk_free(buf); 130 } 131 132 void * 133 spdk_memzone_reserve_aligned(const char *name, size_t len, int socket_id, 134 unsigned flags, unsigned align) 135 { 136 const struct rte_memzone *mz; 137 unsigned dpdk_flags = 0; 138 139 if ((flags & SPDK_MEMZONE_NO_IOVA_CONTIG) == 0) { 140 dpdk_flags |= RTE_MEMZONE_IOVA_CONTIG; 141 } 142 143 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 144 socket_id = SOCKET_ID_ANY; 145 } 146 147 mz = rte_memzone_reserve_aligned(name, len, socket_id, dpdk_flags, align); 148 149 if (mz != NULL) { 150 memset(mz->addr, 0, len); 151 return mz->addr; 152 } else { 153 return NULL; 154 } 155 } 156 157 void * 158 spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags) 159 { 160 return spdk_memzone_reserve_aligned(name, len, socket_id, flags, 161 RTE_CACHE_LINE_SIZE); 162 } 163 164 void * 165 spdk_memzone_lookup(const char *name) 166 { 167 const struct rte_memzone *mz = rte_memzone_lookup(name); 168 169 if (mz != NULL) { 170 return mz->addr; 171 } else { 172 return NULL; 173 } 174 } 175 176 int 177 spdk_memzone_free(const char *name) 178 { 179 const struct rte_memzone *mz = rte_memzone_lookup(name); 180 181 if (mz != NULL) { 182 return rte_memzone_free(mz); 183 } 184 185 return -1; 186 } 187 188 void 189 spdk_memzone_dump(FILE *f) 190 { 191 rte_memzone_dump(f); 192 } 193 194 struct spdk_mempool * 195 spdk_mempool_create_ctor(const char *name, size_t count, 196 size_t ele_size, size_t cache_size, int socket_id, 197 spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg) 198 { 199 struct rte_mempool *mp; 200 size_t tmp; 201 202 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 203 socket_id = SOCKET_ID_ANY; 204 } 205 206 /* No more than half of all elements can be in cache */ 207 tmp = (count / 2) / rte_lcore_count(); 208 if (cache_size > tmp) { 209 cache_size = tmp; 210 } 211 212 if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) { 213 cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; 214 } 215 216 mp = rte_mempool_create(name, count, ele_size, cache_size, 217 0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg, 218 socket_id, 0); 219 220 return (struct spdk_mempool *)mp; 221 } 222 223 224 struct spdk_mempool * 225 spdk_mempool_create(const char *name, size_t count, 226 size_t ele_size, size_t cache_size, int socket_id) 227 { 228 return spdk_mempool_create_ctor(name, count, ele_size, cache_size, socket_id, 229 NULL, NULL); 230 } 231 232 char * 233 spdk_mempool_get_name(struct spdk_mempool *mp) 234 { 235 return ((struct rte_mempool *)mp)->name; 236 } 237 238 void 239 spdk_mempool_free(struct spdk_mempool *mp) 240 { 241 rte_mempool_free((struct rte_mempool *)mp); 242 } 243 244 void * 245 spdk_mempool_get(struct spdk_mempool *mp) 246 { 247 void *ele = NULL; 248 int rc; 249 250 rc = rte_mempool_get((struct rte_mempool *)mp, &ele); 251 if (rc != 0) { 252 return NULL; 253 } 254 return ele; 255 } 256 257 int 258 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 259 { 260 return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count); 261 } 262 263 void 264 spdk_mempool_put(struct spdk_mempool *mp, void *ele) 265 { 266 rte_mempool_put((struct rte_mempool *)mp, ele); 267 } 268 269 void 270 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 271 { 272 rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count); 273 } 274 275 size_t 276 spdk_mempool_count(const struct spdk_mempool *pool) 277 { 278 return rte_mempool_avail_count((struct rte_mempool *)pool); 279 } 280 281 uint32_t 282 spdk_mempool_obj_iter(struct spdk_mempool *mp, spdk_mempool_obj_cb_t obj_cb, 283 void *obj_cb_arg) 284 { 285 return rte_mempool_obj_iter((struct rte_mempool *)mp, (rte_mempool_obj_cb_t *)obj_cb, 286 obj_cb_arg); 287 } 288 289 struct env_mempool_mem_iter_ctx { 290 spdk_mempool_mem_cb_t *user_cb; 291 void *user_arg; 292 }; 293 294 static void 295 mempool_mem_iter_remap(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr, 296 unsigned mem_idx) 297 { 298 struct env_mempool_mem_iter_ctx *ctx = opaque; 299 300 ctx->user_cb((struct spdk_mempool *)mp, ctx->user_arg, memhdr->addr, memhdr->iova, memhdr->len, 301 mem_idx); 302 } 303 304 uint32_t 305 spdk_mempool_mem_iter(struct spdk_mempool *mp, spdk_mempool_mem_cb_t mem_cb, 306 void *mem_cb_arg) 307 { 308 struct env_mempool_mem_iter_ctx ctx = { 309 .user_cb = mem_cb, 310 .user_arg = mem_cb_arg 311 }; 312 313 return rte_mempool_mem_iter((struct rte_mempool *)mp, mempool_mem_iter_remap, &ctx); 314 } 315 316 struct spdk_mempool * 317 spdk_mempool_lookup(const char *name) 318 { 319 return (struct spdk_mempool *)rte_mempool_lookup(name); 320 } 321 322 bool 323 spdk_process_is_primary(void) 324 { 325 return (rte_eal_process_type() == RTE_PROC_PRIMARY); 326 } 327 328 uint64_t 329 spdk_get_ticks(void) 330 { 331 return rte_get_timer_cycles(); 332 } 333 334 uint64_t 335 spdk_get_ticks_hz(void) 336 { 337 return rte_get_timer_hz(); 338 } 339 340 void 341 spdk_delay_us(unsigned int us) 342 { 343 rte_delay_us(us); 344 } 345 346 void 347 spdk_pause(void) 348 { 349 rte_pause(); 350 } 351 352 void 353 spdk_unaffinitize_thread(void) 354 { 355 rte_cpuset_t new_cpuset; 356 long num_cores, i; 357 358 if (g_is_thread_unaffinitized) { 359 return; 360 } 361 362 CPU_ZERO(&new_cpuset); 363 364 num_cores = sysconf(_SC_NPROCESSORS_CONF); 365 366 /* Create a mask containing all CPUs */ 367 for (i = 0; i < num_cores; i++) { 368 CPU_SET(i, &new_cpuset); 369 } 370 371 rte_thread_set_affinity(&new_cpuset); 372 g_is_thread_unaffinitized = true; 373 } 374 375 void * 376 spdk_call_unaffinitized(void *cb(void *arg), void *arg) 377 { 378 rte_cpuset_t orig_cpuset; 379 void *ret; 380 381 if (cb == NULL) { 382 return NULL; 383 } 384 385 if (g_is_thread_unaffinitized) { 386 ret = cb(arg); 387 } else { 388 rte_thread_get_affinity(&orig_cpuset); 389 spdk_unaffinitize_thread(); 390 391 ret = cb(arg); 392 393 rte_thread_set_affinity(&orig_cpuset); 394 g_is_thread_unaffinitized = false; 395 } 396 397 return ret; 398 } 399 400 struct spdk_ring * 401 spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id) 402 { 403 char ring_name[64]; 404 static uint32_t ring_num = 0; 405 unsigned flags = RING_F_EXACT_SZ; 406 407 switch (type) { 408 case SPDK_RING_TYPE_SP_SC: 409 flags |= RING_F_SP_ENQ | RING_F_SC_DEQ; 410 break; 411 case SPDK_RING_TYPE_MP_SC: 412 flags |= RING_F_SC_DEQ; 413 break; 414 case SPDK_RING_TYPE_MP_MC: 415 flags |= 0; 416 break; 417 default: 418 return NULL; 419 } 420 421 snprintf(ring_name, sizeof(ring_name), "ring_%u_%d", 422 __atomic_fetch_add(&ring_num, 1, __ATOMIC_RELAXED), getpid()); 423 424 return (struct spdk_ring *)rte_ring_create(ring_name, count, socket_id, flags); 425 } 426 427 void 428 spdk_ring_free(struct spdk_ring *ring) 429 { 430 rte_ring_free((struct rte_ring *)ring); 431 } 432 433 size_t 434 spdk_ring_count(struct spdk_ring *ring) 435 { 436 return rte_ring_count((struct rte_ring *)ring); 437 } 438 439 size_t 440 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count, 441 size_t *free_space) 442 { 443 return rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count, 444 (unsigned int *)free_space); 445 } 446 447 size_t 448 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count) 449 { 450 return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count, NULL); 451 } 452 453 void 454 spdk_env_dpdk_dump_mem_stats(FILE *file) 455 { 456 fprintf(file, "DPDK memory size %" PRIu64 "\n", rte_eal_get_physmem_size()); 457 fprintf(file, "DPDK memory layout\n"); 458 rte_dump_physmem_layout(file); 459 fprintf(file, "DPDK memzones.\n"); 460 rte_memzone_dump(file); 461 fprintf(file, "DPDK mempools.\n"); 462 rte_mempool_list_dump(file); 463 fprintf(file, "DPDK malloc stats.\n"); 464 rte_malloc_dump_stats(file, NULL); 465 fprintf(file, "DPDK malloc heaps.\n"); 466 rte_malloc_dump_heaps(file); 467 } 468