1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. 3 * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. 4 * All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 #include "spdk/util.h" 9 #include "spdk/env_dpdk.h" 10 #include "spdk/log.h" 11 12 #include "env_internal.h" 13 14 #include <rte_config.h> 15 #include <rte_cycles.h> 16 #include <rte_malloc.h> 17 #include <rte_mempool.h> 18 #include <rte_memzone.h> 19 #include <rte_version.h> 20 21 static uint64_t 22 virt_to_phys(void *vaddr) 23 { 24 uint64_t ret; 25 26 ret = rte_malloc_virt2iova(vaddr); 27 if (ret != RTE_BAD_IOVA) { 28 return ret; 29 } 30 31 return spdk_vtophys(vaddr, NULL); 32 } 33 34 void * 35 spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 36 { 37 void *buf; 38 39 if (flags == 0) { 40 return NULL; 41 } 42 43 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 44 buf = rte_malloc_socket(NULL, size, align, socket_id); 45 if (buf && phys_addr) { 46 #ifdef DEBUG 47 SPDK_ERRLOG("phys_addr param in spdk_malloc() is deprecated\n"); 48 #endif 49 *phys_addr = virt_to_phys(buf); 50 } 51 return buf; 52 } 53 54 void * 55 spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 56 { 57 void *buf; 58 59 if (flags == 0) { 60 return NULL; 61 } 62 63 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 64 buf = rte_zmalloc_socket(NULL, size, align, socket_id); 65 if (buf && phys_addr) { 66 #ifdef DEBUG 67 SPDK_ERRLOG("phys_addr param in spdk_zmalloc() is deprecated\n"); 68 #endif 69 *phys_addr = virt_to_phys(buf); 70 } 71 return buf; 72 } 73 74 void * 75 spdk_realloc(void *buf, size_t size, size_t align) 76 { 77 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 78 return rte_realloc(buf, size, align); 79 } 80 81 void 82 spdk_free(void *buf) 83 { 84 rte_free(buf); 85 } 86 87 void * 88 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 89 { 90 return spdk_malloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 91 } 92 93 void * 94 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 95 { 96 return spdk_zmalloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 97 } 98 99 void * 100 spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr) 101 { 102 return spdk_dma_malloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 103 } 104 105 void * 106 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr) 107 { 108 return spdk_dma_zmalloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 109 } 110 111 void * 112 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr) 113 { 114 void *new_buf; 115 116 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 117 new_buf = rte_realloc(buf, size, align); 118 if (new_buf && phys_addr) { 119 *phys_addr = virt_to_phys(new_buf); 120 } 121 return new_buf; 122 } 123 124 void 125 spdk_dma_free(void *buf) 126 { 127 spdk_free(buf); 128 } 129 130 void * 131 spdk_memzone_reserve_aligned(const char *name, size_t len, int socket_id, 132 unsigned flags, unsigned align) 133 { 134 const struct rte_memzone *mz; 135 unsigned dpdk_flags = 0; 136 137 if ((flags & SPDK_MEMZONE_NO_IOVA_CONTIG) == 0) { 138 dpdk_flags |= RTE_MEMZONE_IOVA_CONTIG; 139 } 140 141 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 142 socket_id = SOCKET_ID_ANY; 143 } 144 145 mz = rte_memzone_reserve_aligned(name, len, socket_id, dpdk_flags, align); 146 147 if (mz != NULL) { 148 memset(mz->addr, 0, len); 149 return mz->addr; 150 } else { 151 return NULL; 152 } 153 } 154 155 void * 156 spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags) 157 { 158 return spdk_memzone_reserve_aligned(name, len, socket_id, flags, 159 RTE_CACHE_LINE_SIZE); 160 } 161 162 void * 163 spdk_memzone_lookup(const char *name) 164 { 165 const struct rte_memzone *mz = rte_memzone_lookup(name); 166 167 if (mz != NULL) { 168 return mz->addr; 169 } else { 170 return NULL; 171 } 172 } 173 174 int 175 spdk_memzone_free(const char *name) 176 { 177 const struct rte_memzone *mz = rte_memzone_lookup(name); 178 179 if (mz != NULL) { 180 return rte_memzone_free(mz); 181 } 182 183 return -1; 184 } 185 186 void 187 spdk_memzone_dump(FILE *f) 188 { 189 rte_memzone_dump(f); 190 } 191 192 struct spdk_mempool * 193 spdk_mempool_create_ctor(const char *name, size_t count, 194 size_t ele_size, size_t cache_size, int socket_id, 195 spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg) 196 { 197 struct rte_mempool *mp; 198 size_t tmp; 199 200 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 201 socket_id = SOCKET_ID_ANY; 202 } 203 204 /* No more than half of all elements can be in cache */ 205 tmp = (count / 2) / rte_lcore_count(); 206 if (cache_size > tmp) { 207 cache_size = tmp; 208 } 209 210 if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) { 211 cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; 212 } 213 214 mp = rte_mempool_create(name, count, ele_size, cache_size, 215 0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg, 216 socket_id, 0); 217 218 return (struct spdk_mempool *)mp; 219 } 220 221 222 struct spdk_mempool * 223 spdk_mempool_create(const char *name, size_t count, 224 size_t ele_size, size_t cache_size, int socket_id) 225 { 226 return spdk_mempool_create_ctor(name, count, ele_size, cache_size, socket_id, 227 NULL, NULL); 228 } 229 230 char * 231 spdk_mempool_get_name(struct spdk_mempool *mp) 232 { 233 return ((struct rte_mempool *)mp)->name; 234 } 235 236 void 237 spdk_mempool_free(struct spdk_mempool *mp) 238 { 239 rte_mempool_free((struct rte_mempool *)mp); 240 } 241 242 void * 243 spdk_mempool_get(struct spdk_mempool *mp) 244 { 245 void *ele = NULL; 246 int rc; 247 248 rc = rte_mempool_get((struct rte_mempool *)mp, &ele); 249 if (rc != 0) { 250 return NULL; 251 } 252 return ele; 253 } 254 255 int 256 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 257 { 258 return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count); 259 } 260 261 void 262 spdk_mempool_put(struct spdk_mempool *mp, void *ele) 263 { 264 rte_mempool_put((struct rte_mempool *)mp, ele); 265 } 266 267 void 268 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 269 { 270 rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count); 271 } 272 273 size_t 274 spdk_mempool_count(const struct spdk_mempool *pool) 275 { 276 return rte_mempool_avail_count((struct rte_mempool *)pool); 277 } 278 279 uint32_t 280 spdk_mempool_obj_iter(struct spdk_mempool *mp, spdk_mempool_obj_cb_t obj_cb, 281 void *obj_cb_arg) 282 { 283 return rte_mempool_obj_iter((struct rte_mempool *)mp, (rte_mempool_obj_cb_t *)obj_cb, 284 obj_cb_arg); 285 } 286 287 struct env_mempool_mem_iter_ctx { 288 spdk_mempool_mem_cb_t *user_cb; 289 void *user_arg; 290 }; 291 292 static void 293 mempool_mem_iter_remap(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr, 294 unsigned mem_idx) 295 { 296 struct env_mempool_mem_iter_ctx *ctx = opaque; 297 298 ctx->user_cb((struct spdk_mempool *)mp, ctx->user_arg, memhdr->addr, memhdr->iova, memhdr->len, 299 mem_idx); 300 } 301 302 uint32_t 303 spdk_mempool_mem_iter(struct spdk_mempool *mp, spdk_mempool_mem_cb_t mem_cb, 304 void *mem_cb_arg) 305 { 306 struct env_mempool_mem_iter_ctx ctx = { 307 .user_cb = mem_cb, 308 .user_arg = mem_cb_arg 309 }; 310 311 return rte_mempool_mem_iter((struct rte_mempool *)mp, mempool_mem_iter_remap, &ctx); 312 } 313 314 struct spdk_mempool * 315 spdk_mempool_lookup(const char *name) 316 { 317 return (struct spdk_mempool *)rte_mempool_lookup(name); 318 } 319 320 bool 321 spdk_process_is_primary(void) 322 { 323 return (rte_eal_process_type() == RTE_PROC_PRIMARY); 324 } 325 326 uint64_t 327 spdk_get_ticks(void) 328 { 329 return rte_get_timer_cycles(); 330 } 331 332 uint64_t 333 spdk_get_ticks_hz(void) 334 { 335 return rte_get_timer_hz(); 336 } 337 338 void 339 spdk_delay_us(unsigned int us) 340 { 341 rte_delay_us(us); 342 } 343 344 void 345 spdk_pause(void) 346 { 347 rte_pause(); 348 } 349 350 void 351 spdk_unaffinitize_thread(void) 352 { 353 rte_cpuset_t new_cpuset; 354 long num_cores, i; 355 356 CPU_ZERO(&new_cpuset); 357 358 num_cores = sysconf(_SC_NPROCESSORS_CONF); 359 360 /* Create a mask containing all CPUs */ 361 for (i = 0; i < num_cores; i++) { 362 CPU_SET(i, &new_cpuset); 363 } 364 365 rte_thread_set_affinity(&new_cpuset); 366 } 367 368 void * 369 spdk_call_unaffinitized(void *cb(void *arg), void *arg) 370 { 371 rte_cpuset_t orig_cpuset; 372 void *ret; 373 374 if (cb == NULL) { 375 return NULL; 376 } 377 378 rte_thread_get_affinity(&orig_cpuset); 379 380 spdk_unaffinitize_thread(); 381 382 ret = cb(arg); 383 384 rte_thread_set_affinity(&orig_cpuset); 385 386 return ret; 387 } 388 389 struct spdk_ring * 390 spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id) 391 { 392 char ring_name[64]; 393 static uint32_t ring_num = 0; 394 unsigned flags = RING_F_EXACT_SZ; 395 396 switch (type) { 397 case SPDK_RING_TYPE_SP_SC: 398 flags |= RING_F_SP_ENQ | RING_F_SC_DEQ; 399 break; 400 case SPDK_RING_TYPE_MP_SC: 401 flags |= RING_F_SC_DEQ; 402 break; 403 case SPDK_RING_TYPE_MP_MC: 404 flags |= 0; 405 break; 406 default: 407 return NULL; 408 } 409 410 snprintf(ring_name, sizeof(ring_name), "ring_%u_%d", 411 __atomic_fetch_add(&ring_num, 1, __ATOMIC_RELAXED), getpid()); 412 413 return (struct spdk_ring *)rte_ring_create(ring_name, count, socket_id, flags); 414 } 415 416 void 417 spdk_ring_free(struct spdk_ring *ring) 418 { 419 rte_ring_free((struct rte_ring *)ring); 420 } 421 422 size_t 423 spdk_ring_count(struct spdk_ring *ring) 424 { 425 return rte_ring_count((struct rte_ring *)ring); 426 } 427 428 size_t 429 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count, 430 size_t *free_space) 431 { 432 return rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count, 433 (unsigned int *)free_space); 434 } 435 436 size_t 437 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count) 438 { 439 return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count, NULL); 440 } 441 442 void 443 spdk_env_dpdk_dump_mem_stats(FILE *file) 444 { 445 fprintf(file, "DPDK memory size %" PRIu64 "\n", rte_eal_get_physmem_size()); 446 fprintf(file, "DPDK memory layout\n"); 447 rte_dump_physmem_layout(file); 448 fprintf(file, "DPDK memzones.\n"); 449 rte_memzone_dump(file); 450 fprintf(file, "DPDK mempools.\n"); 451 rte_mempool_list_dump(file); 452 fprintf(file, "DPDK malloc stats.\n"); 453 rte_malloc_dump_stats(file, NULL); 454 fprintf(file, "DPDK malloc heaps.\n"); 455 rte_malloc_dump_heaps(file); 456 } 457