1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 #include "spdk/util.h" 36 #include "spdk/env_dpdk.h" 37 38 #include "env_internal.h" 39 40 #include <rte_config.h> 41 #include <rte_cycles.h> 42 #include <rte_malloc.h> 43 #include <rte_mempool.h> 44 #include <rte_memzone.h> 45 #include <rte_version.h> 46 47 static uint64_t 48 virt_to_phys(void *vaddr) 49 { 50 uint64_t ret; 51 52 ret = rte_malloc_virt2iova(vaddr); 53 if (ret != RTE_BAD_IOVA) { 54 return ret; 55 } 56 57 return spdk_vtophys(vaddr, NULL); 58 } 59 60 void * 61 spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 62 { 63 void *buf; 64 65 if (flags == 0) { 66 return NULL; 67 } 68 69 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 70 buf = rte_malloc_socket(NULL, size, align, socket_id); 71 if (buf && phys_addr) { 72 #ifdef DEBUG 73 fprintf(stderr, "phys_addr param in spdk_*malloc() is deprecated\n"); 74 #endif 75 *phys_addr = virt_to_phys(buf); 76 } 77 return buf; 78 } 79 80 void * 81 spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 82 { 83 void *buf = spdk_malloc(size, align, phys_addr, socket_id, flags); 84 if (buf) { 85 memset(buf, 0, size); 86 } 87 return buf; 88 } 89 90 void * 91 spdk_realloc(void *buf, size_t size, size_t align) 92 { 93 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 94 return rte_realloc(buf, size, align); 95 } 96 97 void 98 spdk_free(void *buf) 99 { 100 rte_free(buf); 101 } 102 103 void * 104 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 105 { 106 return spdk_malloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 107 } 108 109 void * 110 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 111 { 112 return spdk_zmalloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 113 } 114 115 void * 116 spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr) 117 { 118 return spdk_dma_malloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 119 } 120 121 void * 122 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr) 123 { 124 return spdk_dma_zmalloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 125 } 126 127 void * 128 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr) 129 { 130 void *new_buf; 131 132 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 133 new_buf = rte_realloc(buf, size, align); 134 if (new_buf && phys_addr) { 135 *phys_addr = virt_to_phys(new_buf); 136 } 137 return new_buf; 138 } 139 140 void 141 spdk_dma_free(void *buf) 142 { 143 spdk_free(buf); 144 } 145 146 void * 147 spdk_memzone_reserve_aligned(const char *name, size_t len, int socket_id, 148 unsigned flags, unsigned align) 149 { 150 const struct rte_memzone *mz; 151 unsigned dpdk_flags = 0; 152 153 #if RTE_VERSION >= RTE_VERSION_NUM(18, 05, 0, 0) 154 /* Older DPDKs do not offer such flag since their 155 * memzones are iova-contiguous by default. 156 */ 157 if ((flags & SPDK_MEMZONE_NO_IOVA_CONTIG) == 0) { 158 dpdk_flags |= RTE_MEMZONE_IOVA_CONTIG; 159 } 160 #endif 161 162 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 163 socket_id = SOCKET_ID_ANY; 164 } 165 166 mz = rte_memzone_reserve_aligned(name, len, socket_id, dpdk_flags, align); 167 168 if (mz != NULL) { 169 memset(mz->addr, 0, len); 170 return mz->addr; 171 } else { 172 return NULL; 173 } 174 } 175 176 void * 177 spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags) 178 { 179 return spdk_memzone_reserve_aligned(name, len, socket_id, flags, 180 RTE_CACHE_LINE_SIZE); 181 } 182 183 void * 184 spdk_memzone_lookup(const char *name) 185 { 186 const struct rte_memzone *mz = rte_memzone_lookup(name); 187 188 if (mz != NULL) { 189 return mz->addr; 190 } else { 191 return NULL; 192 } 193 } 194 195 int 196 spdk_memzone_free(const char *name) 197 { 198 const struct rte_memzone *mz = rte_memzone_lookup(name); 199 200 if (mz != NULL) { 201 return rte_memzone_free(mz); 202 } 203 204 return -1; 205 } 206 207 void 208 spdk_memzone_dump(FILE *f) 209 { 210 rte_memzone_dump(f); 211 } 212 213 struct spdk_mempool * 214 spdk_mempool_create_ctor(const char *name, size_t count, 215 size_t ele_size, size_t cache_size, int socket_id, 216 spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg) 217 { 218 struct rte_mempool *mp; 219 size_t tmp; 220 221 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 222 socket_id = SOCKET_ID_ANY; 223 } 224 225 /* No more than half of all elements can be in cache */ 226 tmp = (count / 2) / rte_lcore_count(); 227 if (cache_size > tmp) { 228 cache_size = tmp; 229 } 230 231 if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) { 232 cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; 233 } 234 235 mp = rte_mempool_create(name, count, ele_size, cache_size, 236 0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg, 237 socket_id, MEMPOOL_F_NO_PHYS_CONTIG); 238 239 return (struct spdk_mempool *)mp; 240 } 241 242 243 struct spdk_mempool * 244 spdk_mempool_create(const char *name, size_t count, 245 size_t ele_size, size_t cache_size, int socket_id) 246 { 247 return spdk_mempool_create_ctor(name, count, ele_size, cache_size, socket_id, 248 NULL, NULL); 249 } 250 251 char * 252 spdk_mempool_get_name(struct spdk_mempool *mp) 253 { 254 return ((struct rte_mempool *)mp)->name; 255 } 256 257 void 258 spdk_mempool_free(struct spdk_mempool *mp) 259 { 260 rte_mempool_free((struct rte_mempool *)mp); 261 } 262 263 void * 264 spdk_mempool_get(struct spdk_mempool *mp) 265 { 266 void *ele = NULL; 267 int rc; 268 269 rc = rte_mempool_get((struct rte_mempool *)mp, &ele); 270 if (rc != 0) { 271 return NULL; 272 } 273 return ele; 274 } 275 276 int 277 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 278 { 279 return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count); 280 } 281 282 void 283 spdk_mempool_put(struct spdk_mempool *mp, void *ele) 284 { 285 rte_mempool_put((struct rte_mempool *)mp, ele); 286 } 287 288 void 289 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 290 { 291 rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count); 292 } 293 294 size_t 295 spdk_mempool_count(const struct spdk_mempool *pool) 296 { 297 return rte_mempool_avail_count((struct rte_mempool *)pool); 298 } 299 300 uint32_t 301 spdk_mempool_obj_iter(struct spdk_mempool *mp, spdk_mempool_obj_cb_t obj_cb, 302 void *obj_cb_arg) 303 { 304 return rte_mempool_obj_iter((struct rte_mempool *)mp, (rte_mempool_obj_cb_t *)obj_cb, 305 obj_cb_arg); 306 } 307 308 struct spdk_mempool * 309 spdk_mempool_lookup(const char *name) 310 { 311 return (struct spdk_mempool *)rte_mempool_lookup(name); 312 } 313 314 bool 315 spdk_process_is_primary(void) 316 { 317 return (rte_eal_process_type() == RTE_PROC_PRIMARY); 318 } 319 320 uint64_t spdk_get_ticks(void) 321 { 322 return rte_get_timer_cycles(); 323 } 324 325 uint64_t spdk_get_ticks_hz(void) 326 { 327 return rte_get_timer_hz(); 328 } 329 330 void spdk_delay_us(unsigned int us) 331 { 332 rte_delay_us(us); 333 } 334 335 void spdk_pause(void) 336 { 337 rte_pause(); 338 } 339 340 void 341 spdk_unaffinitize_thread(void) 342 { 343 rte_cpuset_t new_cpuset, orig_cpuset; 344 long num_cores, i, orig_num_cores; 345 346 CPU_ZERO(&new_cpuset); 347 348 num_cores = sysconf(_SC_NPROCESSORS_CONF); 349 350 /* Create a mask containing all CPUs */ 351 for (i = 0; i < num_cores; i++) { 352 CPU_SET(i, &new_cpuset); 353 } 354 355 rte_thread_get_affinity(&orig_cpuset); 356 orig_num_cores = CPU_COUNT(&orig_cpuset); 357 if (orig_num_cores < num_cores) { 358 for (i = 0; i < orig_num_cores; i++) { 359 if (CPU_ISSET(i, &orig_cpuset)) { 360 CPU_CLR(i, &new_cpuset); 361 } 362 } 363 } 364 365 rte_thread_set_affinity(&new_cpuset); 366 } 367 368 void * 369 spdk_call_unaffinitized(void *cb(void *arg), void *arg) 370 { 371 rte_cpuset_t orig_cpuset; 372 void *ret; 373 374 if (cb == NULL) { 375 return NULL; 376 } 377 378 rte_thread_get_affinity(&orig_cpuset); 379 380 spdk_unaffinitize_thread(); 381 382 ret = cb(arg); 383 384 rte_thread_set_affinity(&orig_cpuset); 385 386 return ret; 387 } 388 389 struct spdk_ring * 390 spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id) 391 { 392 char ring_name[64]; 393 static uint32_t ring_num = 0; 394 unsigned flags = RING_F_EXACT_SZ; 395 396 switch (type) { 397 case SPDK_RING_TYPE_SP_SC: 398 flags |= RING_F_SP_ENQ | RING_F_SC_DEQ; 399 break; 400 case SPDK_RING_TYPE_MP_SC: 401 flags |= RING_F_SC_DEQ; 402 break; 403 case SPDK_RING_TYPE_MP_MC: 404 flags |= 0; 405 break; 406 default: 407 return NULL; 408 } 409 410 snprintf(ring_name, sizeof(ring_name), "ring_%u_%d", 411 __atomic_fetch_add(&ring_num, 1, __ATOMIC_RELAXED), getpid()); 412 413 return (struct spdk_ring *)rte_ring_create(ring_name, count, socket_id, flags); 414 } 415 416 void 417 spdk_ring_free(struct spdk_ring *ring) 418 { 419 rte_ring_free((struct rte_ring *)ring); 420 } 421 422 size_t 423 spdk_ring_count(struct spdk_ring *ring) 424 { 425 return rte_ring_count((struct rte_ring *)ring); 426 } 427 428 size_t 429 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count, 430 size_t *free_space) 431 { 432 return rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count, 433 (unsigned int *)free_space); 434 } 435 436 size_t 437 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count) 438 { 439 return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count, NULL); 440 } 441 442 void 443 spdk_env_dpdk_dump_mem_stats(FILE *file) 444 { 445 fprintf(file, "DPDK memory size %lu\n", rte_eal_get_physmem_size()); 446 fprintf(file, "DPDK memory layout\n"); 447 rte_dump_physmem_layout(file); 448 fprintf(file, "DPDK memzones.\n"); 449 rte_memzone_dump(file); 450 fprintf(file, "DPDK mempools.\n"); 451 rte_mempool_list_dump(file); 452 fprintf(file, "DPDK malloc stats.\n"); 453 rte_malloc_dump_stats(file, NULL); 454 fprintf(file, "DPDK malloc heaps.\n"); 455 rte_malloc_dump_heaps(file); 456 } 457