1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. 3 * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. 4 * All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 #include "spdk/util.h" 9 #include "spdk/env_dpdk.h" 10 #include "spdk/log.h" 11 12 #include "env_internal.h" 13 14 #include <rte_config.h> 15 #include <rte_cycles.h> 16 #include <rte_malloc.h> 17 #include <rte_mempool.h> 18 #include <rte_memzone.h> 19 #include <rte_version.h> 20 21 static __thread bool g_is_thread_unaffinitized; 22 23 void * 24 spdk_malloc(size_t size, size_t align, uint64_t *unused, int socket_id, uint32_t flags) 25 { 26 if (flags == 0 || unused != NULL) { 27 return NULL; 28 } 29 30 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 31 return rte_malloc_socket(NULL, size, align, socket_id); 32 } 33 34 void * 35 spdk_zmalloc(size_t size, size_t align, uint64_t *unused, int socket_id, uint32_t flags) 36 { 37 if (flags == 0 || unused != NULL) { 38 return NULL; 39 } 40 41 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 42 return rte_zmalloc_socket(NULL, size, align, socket_id); 43 } 44 45 void * 46 spdk_realloc(void *buf, size_t size, size_t align) 47 { 48 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 49 return rte_realloc(buf, size, align); 50 } 51 52 void 53 spdk_free(void *buf) 54 { 55 rte_free(buf); 56 } 57 58 void * 59 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *unused, int socket_id) 60 { 61 return spdk_malloc(size, align, unused, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 62 } 63 64 void * 65 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *unused, int socket_id) 66 { 67 return spdk_zmalloc(size, align, unused, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 68 } 69 70 void * 71 spdk_dma_malloc(size_t size, size_t align, uint64_t *unused) 72 { 73 return spdk_dma_malloc_socket(size, align, unused, SPDK_ENV_SOCKET_ID_ANY); 74 } 75 76 void * 77 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *unused) 78 { 79 return spdk_dma_zmalloc_socket(size, align, unused, SPDK_ENV_SOCKET_ID_ANY); 80 } 81 82 void * 83 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *unused) 84 { 85 if (unused != NULL) { 86 return NULL; 87 } 88 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 89 return rte_realloc(buf, size, align); 90 } 91 92 void 93 spdk_dma_free(void *buf) 94 { 95 spdk_free(buf); 96 } 97 98 void * 99 spdk_memzone_reserve_aligned(const char *name, size_t len, int socket_id, 100 unsigned flags, unsigned align) 101 { 102 const struct rte_memzone *mz; 103 unsigned dpdk_flags = 0; 104 105 if ((flags & SPDK_MEMZONE_NO_IOVA_CONTIG) == 0) { 106 dpdk_flags |= RTE_MEMZONE_IOVA_CONTIG; 107 } 108 109 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 110 socket_id = SOCKET_ID_ANY; 111 } 112 113 mz = rte_memzone_reserve_aligned(name, len, socket_id, dpdk_flags, align); 114 115 if (mz != NULL) { 116 memset(mz->addr, 0, len); 117 return mz->addr; 118 } else { 119 return NULL; 120 } 121 } 122 123 void * 124 spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags) 125 { 126 return spdk_memzone_reserve_aligned(name, len, socket_id, flags, 127 RTE_CACHE_LINE_SIZE); 128 } 129 130 void * 131 spdk_memzone_lookup(const char *name) 132 { 133 const struct rte_memzone *mz = rte_memzone_lookup(name); 134 135 if (mz != NULL) { 136 return mz->addr; 137 } else { 138 return NULL; 139 } 140 } 141 142 int 143 spdk_memzone_free(const char *name) 144 { 145 const struct rte_memzone *mz = rte_memzone_lookup(name); 146 147 if (mz != NULL) { 148 return rte_memzone_free(mz); 149 } 150 151 return -1; 152 } 153 154 void 155 spdk_memzone_dump(FILE *f) 156 { 157 rte_memzone_dump(f); 158 } 159 160 struct spdk_mempool * 161 spdk_mempool_create_ctor(const char *name, size_t count, 162 size_t ele_size, size_t cache_size, int socket_id, 163 spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg) 164 { 165 struct rte_mempool *mp; 166 size_t tmp; 167 168 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 169 socket_id = SOCKET_ID_ANY; 170 } 171 172 /* No more than half of all elements can be in cache */ 173 tmp = (count / 2) / rte_lcore_count(); 174 if (cache_size > tmp) { 175 cache_size = tmp; 176 } 177 178 if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) { 179 cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; 180 } 181 182 mp = rte_mempool_create(name, count, ele_size, cache_size, 183 0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg, 184 socket_id, 0); 185 186 return (struct spdk_mempool *)mp; 187 } 188 189 190 struct spdk_mempool * 191 spdk_mempool_create(const char *name, size_t count, 192 size_t ele_size, size_t cache_size, int socket_id) 193 { 194 return spdk_mempool_create_ctor(name, count, ele_size, cache_size, socket_id, 195 NULL, NULL); 196 } 197 198 char * 199 spdk_mempool_get_name(struct spdk_mempool *mp) 200 { 201 return ((struct rte_mempool *)mp)->name; 202 } 203 204 void 205 spdk_mempool_free(struct spdk_mempool *mp) 206 { 207 rte_mempool_free((struct rte_mempool *)mp); 208 } 209 210 void * 211 spdk_mempool_get(struct spdk_mempool *mp) 212 { 213 void *ele = NULL; 214 int rc; 215 216 rc = rte_mempool_get((struct rte_mempool *)mp, &ele); 217 if (rc != 0) { 218 return NULL; 219 } 220 return ele; 221 } 222 223 int 224 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 225 { 226 return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count); 227 } 228 229 void 230 spdk_mempool_put(struct spdk_mempool *mp, void *ele) 231 { 232 rte_mempool_put((struct rte_mempool *)mp, ele); 233 } 234 235 void 236 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 237 { 238 rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count); 239 } 240 241 size_t 242 spdk_mempool_count(const struct spdk_mempool *pool) 243 { 244 return rte_mempool_avail_count((struct rte_mempool *)pool); 245 } 246 247 uint32_t 248 spdk_mempool_obj_iter(struct spdk_mempool *mp, spdk_mempool_obj_cb_t obj_cb, 249 void *obj_cb_arg) 250 { 251 return rte_mempool_obj_iter((struct rte_mempool *)mp, (rte_mempool_obj_cb_t *)obj_cb, 252 obj_cb_arg); 253 } 254 255 struct env_mempool_mem_iter_ctx { 256 spdk_mempool_mem_cb_t *user_cb; 257 void *user_arg; 258 }; 259 260 static void 261 mempool_mem_iter_remap(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr, 262 unsigned mem_idx) 263 { 264 struct env_mempool_mem_iter_ctx *ctx = opaque; 265 266 ctx->user_cb((struct spdk_mempool *)mp, ctx->user_arg, memhdr->addr, memhdr->iova, memhdr->len, 267 mem_idx); 268 } 269 270 uint32_t 271 spdk_mempool_mem_iter(struct spdk_mempool *mp, spdk_mempool_mem_cb_t mem_cb, 272 void *mem_cb_arg) 273 { 274 struct env_mempool_mem_iter_ctx ctx = { 275 .user_cb = mem_cb, 276 .user_arg = mem_cb_arg 277 }; 278 279 return rte_mempool_mem_iter((struct rte_mempool *)mp, mempool_mem_iter_remap, &ctx); 280 } 281 282 struct spdk_mempool * 283 spdk_mempool_lookup(const char *name) 284 { 285 return (struct spdk_mempool *)rte_mempool_lookup(name); 286 } 287 288 bool 289 spdk_process_is_primary(void) 290 { 291 return (rte_eal_process_type() == RTE_PROC_PRIMARY); 292 } 293 294 uint64_t 295 spdk_get_ticks(void) 296 { 297 return rte_get_timer_cycles(); 298 } 299 300 uint64_t 301 spdk_get_ticks_hz(void) 302 { 303 return rte_get_timer_hz(); 304 } 305 306 void 307 spdk_delay_us(unsigned int us) 308 { 309 rte_delay_us(us); 310 } 311 312 void 313 spdk_pause(void) 314 { 315 rte_pause(); 316 } 317 318 void 319 spdk_unaffinitize_thread(void) 320 { 321 rte_cpuset_t new_cpuset; 322 long num_cores, i; 323 324 if (g_is_thread_unaffinitized) { 325 return; 326 } 327 328 CPU_ZERO(&new_cpuset); 329 330 num_cores = sysconf(_SC_NPROCESSORS_CONF); 331 332 /* Create a mask containing all CPUs */ 333 for (i = 0; i < num_cores; i++) { 334 CPU_SET(i, &new_cpuset); 335 } 336 337 rte_thread_set_affinity(&new_cpuset); 338 g_is_thread_unaffinitized = true; 339 } 340 341 void * 342 spdk_call_unaffinitized(void *cb(void *arg), void *arg) 343 { 344 rte_cpuset_t orig_cpuset; 345 void *ret; 346 347 if (cb == NULL) { 348 return NULL; 349 } 350 351 if (g_is_thread_unaffinitized) { 352 ret = cb(arg); 353 } else { 354 rte_thread_get_affinity(&orig_cpuset); 355 spdk_unaffinitize_thread(); 356 357 ret = cb(arg); 358 359 rte_thread_set_affinity(&orig_cpuset); 360 g_is_thread_unaffinitized = false; 361 } 362 363 return ret; 364 } 365 366 struct spdk_ring * 367 spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id) 368 { 369 char ring_name[64]; 370 static uint32_t ring_num = 0; 371 unsigned flags = RING_F_EXACT_SZ; 372 373 switch (type) { 374 case SPDK_RING_TYPE_SP_SC: 375 flags |= RING_F_SP_ENQ | RING_F_SC_DEQ; 376 break; 377 case SPDK_RING_TYPE_MP_SC: 378 flags |= RING_F_SC_DEQ; 379 break; 380 case SPDK_RING_TYPE_MP_MC: 381 flags |= 0; 382 break; 383 default: 384 return NULL; 385 } 386 387 snprintf(ring_name, sizeof(ring_name), "ring_%u_%d", 388 __atomic_fetch_add(&ring_num, 1, __ATOMIC_RELAXED), getpid()); 389 390 return (struct spdk_ring *)rte_ring_create(ring_name, count, socket_id, flags); 391 } 392 393 void 394 spdk_ring_free(struct spdk_ring *ring) 395 { 396 rte_ring_free((struct rte_ring *)ring); 397 } 398 399 size_t 400 spdk_ring_count(struct spdk_ring *ring) 401 { 402 return rte_ring_count((struct rte_ring *)ring); 403 } 404 405 size_t 406 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count, 407 size_t *free_space) 408 { 409 return rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count, 410 (unsigned int *)free_space); 411 } 412 413 size_t 414 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count) 415 { 416 return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count, NULL); 417 } 418 419 void 420 spdk_env_dpdk_dump_mem_stats(FILE *file) 421 { 422 fprintf(file, "DPDK memory size %" PRIu64 "\n", rte_eal_get_physmem_size()); 423 fprintf(file, "DPDK memory layout\n"); 424 rte_dump_physmem_layout(file); 425 fprintf(file, "DPDK memzones.\n"); 426 rte_memzone_dump(file); 427 fprintf(file, "DPDK mempools.\n"); 428 rte_mempool_list_dump(file); 429 fprintf(file, "DPDK malloc stats.\n"); 430 rte_malloc_dump_stats(file, NULL); 431 fprintf(file, "DPDK malloc heaps.\n"); 432 rte_malloc_dump_heaps(file); 433 } 434