1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. 3 * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. 4 * All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 #include "spdk/util.h" 9 #include "spdk/env_dpdk.h" 10 #include "spdk/log.h" 11 12 #include "env_internal.h" 13 14 #include <rte_config.h> 15 #include <rte_cycles.h> 16 #include <rte_malloc.h> 17 #include <rte_mempool.h> 18 #include <rte_memzone.h> 19 #include <rte_version.h> 20 #include <rte_eal.h> 21 22 static __thread bool g_is_thread_unaffinitized; 23 24 void * 25 spdk_malloc(size_t size, size_t align, uint64_t *unused, int socket_id, uint32_t flags) 26 { 27 if (flags == 0 || unused != NULL) { 28 return NULL; 29 } 30 31 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 32 return rte_malloc_socket(NULL, size, align, socket_id); 33 } 34 35 void * 36 spdk_zmalloc(size_t size, size_t align, uint64_t *unused, int socket_id, uint32_t flags) 37 { 38 if (flags == 0 || unused != NULL) { 39 return NULL; 40 } 41 42 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 43 return rte_zmalloc_socket(NULL, size, align, socket_id); 44 } 45 46 void * 47 spdk_realloc(void *buf, size_t size, size_t align) 48 { 49 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 50 return rte_realloc(buf, size, align); 51 } 52 53 void 54 spdk_free(void *buf) 55 { 56 rte_free(buf); 57 } 58 59 void * 60 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *unused, int socket_id) 61 { 62 return spdk_malloc(size, align, unused, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 63 } 64 65 void * 66 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *unused, int socket_id) 67 { 68 return spdk_zmalloc(size, align, unused, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 69 } 70 71 void * 72 spdk_dma_malloc(size_t size, size_t align, uint64_t *unused) 73 { 74 return spdk_dma_malloc_socket(size, align, unused, SPDK_ENV_SOCKET_ID_ANY); 75 } 76 77 void * 78 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *unused) 79 { 80 return spdk_dma_zmalloc_socket(size, align, unused, SPDK_ENV_SOCKET_ID_ANY); 81 } 82 83 void * 84 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *unused) 85 { 86 if (unused != NULL) { 87 return NULL; 88 } 89 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 90 return rte_realloc(buf, size, align); 91 } 92 93 void 94 spdk_dma_free(void *buf) 95 { 96 spdk_free(buf); 97 } 98 99 void * 100 spdk_memzone_reserve_aligned(const char *name, size_t len, int socket_id, 101 unsigned flags, unsigned align) 102 { 103 const struct rte_memzone *mz; 104 unsigned dpdk_flags = 0; 105 106 if ((flags & SPDK_MEMZONE_NO_IOVA_CONTIG) == 0) { 107 dpdk_flags |= RTE_MEMZONE_IOVA_CONTIG; 108 } 109 110 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 111 socket_id = SOCKET_ID_ANY; 112 } 113 114 mz = rte_memzone_reserve_aligned(name, len, socket_id, dpdk_flags, align); 115 116 if (mz != NULL) { 117 memset(mz->addr, 0, len); 118 return mz->addr; 119 } else { 120 return NULL; 121 } 122 } 123 124 void * 125 spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags) 126 { 127 return spdk_memzone_reserve_aligned(name, len, socket_id, flags, 128 RTE_CACHE_LINE_SIZE); 129 } 130 131 void * 132 spdk_memzone_lookup(const char *name) 133 { 134 const struct rte_memzone *mz = rte_memzone_lookup(name); 135 136 if (mz != NULL) { 137 return mz->addr; 138 } else { 139 return NULL; 140 } 141 } 142 143 int 144 spdk_memzone_free(const char *name) 145 { 146 const struct rte_memzone *mz = rte_memzone_lookup(name); 147 148 if (mz != NULL) { 149 return rte_memzone_free(mz); 150 } 151 152 return -1; 153 } 154 155 void 156 spdk_memzone_dump(FILE *f) 157 { 158 rte_memzone_dump(f); 159 } 160 161 struct spdk_mempool * 162 spdk_mempool_create_ctor(const char *name, size_t count, 163 size_t ele_size, size_t cache_size, int socket_id, 164 spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg) 165 { 166 struct rte_mempool *mp; 167 size_t tmp; 168 169 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 170 socket_id = SOCKET_ID_ANY; 171 } 172 173 /* No more than half of all elements can be in cache */ 174 tmp = (count / 2) / rte_lcore_count(); 175 if (cache_size > tmp) { 176 cache_size = tmp; 177 } 178 179 if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) { 180 cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; 181 } 182 183 mp = rte_mempool_create(name, count, ele_size, cache_size, 184 0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg, 185 socket_id, 0); 186 187 return (struct spdk_mempool *)mp; 188 } 189 190 191 struct spdk_mempool * 192 spdk_mempool_create(const char *name, size_t count, 193 size_t ele_size, size_t cache_size, int socket_id) 194 { 195 return spdk_mempool_create_ctor(name, count, ele_size, cache_size, socket_id, 196 NULL, NULL); 197 } 198 199 char * 200 spdk_mempool_get_name(struct spdk_mempool *mp) 201 { 202 return ((struct rte_mempool *)mp)->name; 203 } 204 205 void 206 spdk_mempool_free(struct spdk_mempool *mp) 207 { 208 rte_mempool_free((struct rte_mempool *)mp); 209 } 210 211 void * 212 spdk_mempool_get(struct spdk_mempool *mp) 213 { 214 void *ele = NULL; 215 int rc; 216 217 rc = rte_mempool_get((struct rte_mempool *)mp, &ele); 218 if (rc != 0) { 219 return NULL; 220 } 221 return ele; 222 } 223 224 int 225 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 226 { 227 return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count); 228 } 229 230 void 231 spdk_mempool_put(struct spdk_mempool *mp, void *ele) 232 { 233 rte_mempool_put((struct rte_mempool *)mp, ele); 234 } 235 236 void 237 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 238 { 239 rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count); 240 } 241 242 size_t 243 spdk_mempool_count(const struct spdk_mempool *pool) 244 { 245 return rte_mempool_avail_count((struct rte_mempool *)pool); 246 } 247 248 uint32_t 249 spdk_mempool_obj_iter(struct spdk_mempool *mp, spdk_mempool_obj_cb_t obj_cb, 250 void *obj_cb_arg) 251 { 252 return rte_mempool_obj_iter((struct rte_mempool *)mp, (rte_mempool_obj_cb_t *)obj_cb, 253 obj_cb_arg); 254 } 255 256 struct env_mempool_mem_iter_ctx { 257 spdk_mempool_mem_cb_t *user_cb; 258 void *user_arg; 259 }; 260 261 static void 262 mempool_mem_iter_remap(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr, 263 unsigned mem_idx) 264 { 265 struct env_mempool_mem_iter_ctx *ctx = opaque; 266 267 ctx->user_cb((struct spdk_mempool *)mp, ctx->user_arg, memhdr->addr, memhdr->iova, memhdr->len, 268 mem_idx); 269 } 270 271 uint32_t 272 spdk_mempool_mem_iter(struct spdk_mempool *mp, spdk_mempool_mem_cb_t mem_cb, 273 void *mem_cb_arg) 274 { 275 struct env_mempool_mem_iter_ctx ctx = { 276 .user_cb = mem_cb, 277 .user_arg = mem_cb_arg 278 }; 279 280 return rte_mempool_mem_iter((struct rte_mempool *)mp, mempool_mem_iter_remap, &ctx); 281 } 282 283 struct spdk_mempool * 284 spdk_mempool_lookup(const char *name) 285 { 286 return (struct spdk_mempool *)rte_mempool_lookup(name); 287 } 288 289 bool 290 spdk_process_is_primary(void) 291 { 292 return (rte_eal_process_type() == RTE_PROC_PRIMARY); 293 } 294 295 uint64_t 296 spdk_get_ticks(void) 297 { 298 return rte_get_timer_cycles(); 299 } 300 301 uint64_t 302 spdk_get_ticks_hz(void) 303 { 304 return rte_get_timer_hz(); 305 } 306 307 void 308 spdk_delay_us(unsigned int us) 309 { 310 rte_delay_us(us); 311 } 312 313 void 314 spdk_pause(void) 315 { 316 rte_pause(); 317 } 318 319 void 320 spdk_unaffinitize_thread(void) 321 { 322 rte_cpuset_t new_cpuset; 323 long num_cores, i; 324 325 if (g_is_thread_unaffinitized) { 326 return; 327 } 328 329 CPU_ZERO(&new_cpuset); 330 331 num_cores = sysconf(_SC_NPROCESSORS_CONF); 332 333 /* Create a mask containing all CPUs */ 334 for (i = 0; i < num_cores; i++) { 335 CPU_SET(i, &new_cpuset); 336 } 337 338 rte_thread_set_affinity(&new_cpuset); 339 g_is_thread_unaffinitized = true; 340 } 341 342 void * 343 spdk_call_unaffinitized(void *cb(void *arg), void *arg) 344 { 345 rte_cpuset_t orig_cpuset; 346 void *ret; 347 348 if (cb == NULL) { 349 return NULL; 350 } 351 352 if (g_is_thread_unaffinitized) { 353 ret = cb(arg); 354 } else { 355 rte_thread_get_affinity(&orig_cpuset); 356 spdk_unaffinitize_thread(); 357 358 ret = cb(arg); 359 360 rte_thread_set_affinity(&orig_cpuset); 361 g_is_thread_unaffinitized = false; 362 } 363 364 return ret; 365 } 366 367 struct spdk_ring * 368 spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id) 369 { 370 char ring_name[64]; 371 static uint32_t ring_num = 0; 372 unsigned flags = RING_F_EXACT_SZ; 373 374 switch (type) { 375 case SPDK_RING_TYPE_SP_SC: 376 flags |= RING_F_SP_ENQ | RING_F_SC_DEQ; 377 break; 378 case SPDK_RING_TYPE_MP_SC: 379 flags |= RING_F_SC_DEQ; 380 break; 381 case SPDK_RING_TYPE_MP_MC: 382 flags |= 0; 383 break; 384 default: 385 return NULL; 386 } 387 388 snprintf(ring_name, sizeof(ring_name), "ring_%u_%d", 389 __atomic_fetch_add(&ring_num, 1, __ATOMIC_RELAXED), getpid()); 390 391 return (struct spdk_ring *)rte_ring_create(ring_name, count, socket_id, flags); 392 } 393 394 void 395 spdk_ring_free(struct spdk_ring *ring) 396 { 397 rte_ring_free((struct rte_ring *)ring); 398 } 399 400 size_t 401 spdk_ring_count(struct spdk_ring *ring) 402 { 403 return rte_ring_count((struct rte_ring *)ring); 404 } 405 406 size_t 407 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count, 408 size_t *free_space) 409 { 410 return rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count, 411 (unsigned int *)free_space); 412 } 413 414 size_t 415 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count) 416 { 417 return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count, NULL); 418 } 419 420 void 421 spdk_env_dpdk_dump_mem_stats(FILE *file) 422 { 423 fprintf(file, "DPDK memory size %" PRIu64 "\n", rte_eal_get_physmem_size()); 424 fprintf(file, "DPDK memory layout\n"); 425 rte_dump_physmem_layout(file); 426 fprintf(file, "DPDK memzones.\n"); 427 rte_memzone_dump(file); 428 fprintf(file, "DPDK mempools.\n"); 429 rte_mempool_list_dump(file); 430 fprintf(file, "DPDK malloc stats.\n"); 431 rte_malloc_dump_stats(file, NULL); 432 fprintf(file, "DPDK malloc heaps.\n"); 433 rte_malloc_dump_heaps(file); 434 } 435 436 int 437 spdk_get_tid(void) 438 { 439 return rte_sys_gettid(); 440 } 441