1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/util.h" 8 #include "spdk/env_dpdk.h" 9 #include "spdk/log.h" 10 11 #include "env_internal.h" 12 13 #include <rte_config.h> 14 #include <rte_cycles.h> 15 #include <rte_malloc.h> 16 #include <rte_mempool.h> 17 #include <rte_memzone.h> 18 #include <rte_version.h> 19 20 static uint64_t 21 virt_to_phys(void *vaddr) 22 { 23 uint64_t ret; 24 25 ret = rte_malloc_virt2iova(vaddr); 26 if (ret != RTE_BAD_IOVA) { 27 return ret; 28 } 29 30 return spdk_vtophys(vaddr, NULL); 31 } 32 33 void * 34 spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 35 { 36 void *buf; 37 38 if (flags == 0) { 39 return NULL; 40 } 41 42 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 43 buf = rte_malloc_socket(NULL, size, align, socket_id); 44 if (buf && phys_addr) { 45 #ifdef DEBUG 46 SPDK_ERRLOG("phys_addr param in spdk_malloc() is deprecated\n"); 47 #endif 48 *phys_addr = virt_to_phys(buf); 49 } 50 return buf; 51 } 52 53 void * 54 spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 55 { 56 void *buf; 57 58 if (flags == 0) { 59 return NULL; 60 } 61 62 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 63 buf = rte_zmalloc_socket(NULL, size, align, socket_id); 64 if (buf && phys_addr) { 65 #ifdef DEBUG 66 SPDK_ERRLOG("phys_addr param in spdk_zmalloc() is deprecated\n"); 67 #endif 68 *phys_addr = virt_to_phys(buf); 69 } 70 return buf; 71 } 72 73 void * 74 spdk_realloc(void *buf, size_t size, size_t align) 75 { 76 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 77 return rte_realloc(buf, size, align); 78 } 79 80 void 81 spdk_free(void *buf) 82 { 83 rte_free(buf); 84 } 85 86 void * 87 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 88 { 89 return spdk_malloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 90 } 91 92 void * 93 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 94 { 95 return spdk_zmalloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 96 } 97 98 void * 99 spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr) 100 { 101 return spdk_dma_malloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 102 } 103 104 void * 105 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr) 106 { 107 return spdk_dma_zmalloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 108 } 109 110 void * 111 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr) 112 { 113 void *new_buf; 114 115 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 116 new_buf = rte_realloc(buf, size, align); 117 if (new_buf && phys_addr) { 118 *phys_addr = virt_to_phys(new_buf); 119 } 120 return new_buf; 121 } 122 123 void 124 spdk_dma_free(void *buf) 125 { 126 spdk_free(buf); 127 } 128 129 void * 130 spdk_memzone_reserve_aligned(const char *name, size_t len, int socket_id, 131 unsigned flags, unsigned align) 132 { 133 const struct rte_memzone *mz; 134 unsigned dpdk_flags = 0; 135 136 if ((flags & SPDK_MEMZONE_NO_IOVA_CONTIG) == 0) { 137 dpdk_flags |= RTE_MEMZONE_IOVA_CONTIG; 138 } 139 140 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 141 socket_id = SOCKET_ID_ANY; 142 } 143 144 mz = rte_memzone_reserve_aligned(name, len, socket_id, dpdk_flags, align); 145 146 if (mz != NULL) { 147 memset(mz->addr, 0, len); 148 return mz->addr; 149 } else { 150 return NULL; 151 } 152 } 153 154 void * 155 spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags) 156 { 157 return spdk_memzone_reserve_aligned(name, len, socket_id, flags, 158 RTE_CACHE_LINE_SIZE); 159 } 160 161 void * 162 spdk_memzone_lookup(const char *name) 163 { 164 const struct rte_memzone *mz = rte_memzone_lookup(name); 165 166 if (mz != NULL) { 167 return mz->addr; 168 } else { 169 return NULL; 170 } 171 } 172 173 int 174 spdk_memzone_free(const char *name) 175 { 176 const struct rte_memzone *mz = rte_memzone_lookup(name); 177 178 if (mz != NULL) { 179 return rte_memzone_free(mz); 180 } 181 182 return -1; 183 } 184 185 void 186 spdk_memzone_dump(FILE *f) 187 { 188 rte_memzone_dump(f); 189 } 190 191 struct spdk_mempool * 192 spdk_mempool_create_ctor(const char *name, size_t count, 193 size_t ele_size, size_t cache_size, int socket_id, 194 spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg) 195 { 196 struct rte_mempool *mp; 197 size_t tmp; 198 199 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 200 socket_id = SOCKET_ID_ANY; 201 } 202 203 /* No more than half of all elements can be in cache */ 204 tmp = (count / 2) / rte_lcore_count(); 205 if (cache_size > tmp) { 206 cache_size = tmp; 207 } 208 209 if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) { 210 cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; 211 } 212 213 mp = rte_mempool_create(name, count, ele_size, cache_size, 214 0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg, 215 socket_id, 0); 216 217 return (struct spdk_mempool *)mp; 218 } 219 220 221 struct spdk_mempool * 222 spdk_mempool_create(const char *name, size_t count, 223 size_t ele_size, size_t cache_size, int socket_id) 224 { 225 return spdk_mempool_create_ctor(name, count, ele_size, cache_size, socket_id, 226 NULL, NULL); 227 } 228 229 char * 230 spdk_mempool_get_name(struct spdk_mempool *mp) 231 { 232 return ((struct rte_mempool *)mp)->name; 233 } 234 235 void 236 spdk_mempool_free(struct spdk_mempool *mp) 237 { 238 rte_mempool_free((struct rte_mempool *)mp); 239 } 240 241 void * 242 spdk_mempool_get(struct spdk_mempool *mp) 243 { 244 void *ele = NULL; 245 int rc; 246 247 rc = rte_mempool_get((struct rte_mempool *)mp, &ele); 248 if (rc != 0) { 249 return NULL; 250 } 251 return ele; 252 } 253 254 int 255 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 256 { 257 return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count); 258 } 259 260 void 261 spdk_mempool_put(struct spdk_mempool *mp, void *ele) 262 { 263 rte_mempool_put((struct rte_mempool *)mp, ele); 264 } 265 266 void 267 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 268 { 269 rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count); 270 } 271 272 size_t 273 spdk_mempool_count(const struct spdk_mempool *pool) 274 { 275 return rte_mempool_avail_count((struct rte_mempool *)pool); 276 } 277 278 uint32_t 279 spdk_mempool_obj_iter(struct spdk_mempool *mp, spdk_mempool_obj_cb_t obj_cb, 280 void *obj_cb_arg) 281 { 282 return rte_mempool_obj_iter((struct rte_mempool *)mp, (rte_mempool_obj_cb_t *)obj_cb, 283 obj_cb_arg); 284 } 285 286 struct spdk_mempool * 287 spdk_mempool_lookup(const char *name) 288 { 289 return (struct spdk_mempool *)rte_mempool_lookup(name); 290 } 291 292 bool 293 spdk_process_is_primary(void) 294 { 295 return (rte_eal_process_type() == RTE_PROC_PRIMARY); 296 } 297 298 uint64_t 299 spdk_get_ticks(void) 300 { 301 return rte_get_timer_cycles(); 302 } 303 304 uint64_t 305 spdk_get_ticks_hz(void) 306 { 307 return rte_get_timer_hz(); 308 } 309 310 void 311 spdk_delay_us(unsigned int us) 312 { 313 rte_delay_us(us); 314 } 315 316 void 317 spdk_pause(void) 318 { 319 rte_pause(); 320 } 321 322 void 323 spdk_unaffinitize_thread(void) 324 { 325 rte_cpuset_t new_cpuset; 326 long num_cores, i; 327 328 CPU_ZERO(&new_cpuset); 329 330 num_cores = sysconf(_SC_NPROCESSORS_CONF); 331 332 /* Create a mask containing all CPUs */ 333 for (i = 0; i < num_cores; i++) { 334 CPU_SET(i, &new_cpuset); 335 } 336 337 rte_thread_set_affinity(&new_cpuset); 338 } 339 340 void * 341 spdk_call_unaffinitized(void *cb(void *arg), void *arg) 342 { 343 rte_cpuset_t orig_cpuset; 344 void *ret; 345 346 if (cb == NULL) { 347 return NULL; 348 } 349 350 rte_thread_get_affinity(&orig_cpuset); 351 352 spdk_unaffinitize_thread(); 353 354 ret = cb(arg); 355 356 rte_thread_set_affinity(&orig_cpuset); 357 358 return ret; 359 } 360 361 struct spdk_ring * 362 spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id) 363 { 364 char ring_name[64]; 365 static uint32_t ring_num = 0; 366 unsigned flags = RING_F_EXACT_SZ; 367 368 switch (type) { 369 case SPDK_RING_TYPE_SP_SC: 370 flags |= RING_F_SP_ENQ | RING_F_SC_DEQ; 371 break; 372 case SPDK_RING_TYPE_MP_SC: 373 flags |= RING_F_SC_DEQ; 374 break; 375 case SPDK_RING_TYPE_MP_MC: 376 flags |= 0; 377 break; 378 default: 379 return NULL; 380 } 381 382 snprintf(ring_name, sizeof(ring_name), "ring_%u_%d", 383 __atomic_fetch_add(&ring_num, 1, __ATOMIC_RELAXED), getpid()); 384 385 return (struct spdk_ring *)rte_ring_create(ring_name, count, socket_id, flags); 386 } 387 388 void 389 spdk_ring_free(struct spdk_ring *ring) 390 { 391 rte_ring_free((struct rte_ring *)ring); 392 } 393 394 size_t 395 spdk_ring_count(struct spdk_ring *ring) 396 { 397 return rte_ring_count((struct rte_ring *)ring); 398 } 399 400 size_t 401 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count, 402 size_t *free_space) 403 { 404 return rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count, 405 (unsigned int *)free_space); 406 } 407 408 size_t 409 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count) 410 { 411 return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count, NULL); 412 } 413 414 void 415 spdk_env_dpdk_dump_mem_stats(FILE *file) 416 { 417 fprintf(file, "DPDK memory size %" PRIu64 "\n", rte_eal_get_physmem_size()); 418 fprintf(file, "DPDK memory layout\n"); 419 rte_dump_physmem_layout(file); 420 fprintf(file, "DPDK memzones.\n"); 421 rte_memzone_dump(file); 422 fprintf(file, "DPDK mempools.\n"); 423 rte_mempool_list_dump(file); 424 fprintf(file, "DPDK malloc stats.\n"); 425 rte_malloc_dump_stats(file, NULL); 426 fprintf(file, "DPDK malloc heaps.\n"); 427 rte_malloc_dump_heaps(file); 428 } 429