1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 #include "spdk/util.h" 36 #include "spdk/env_dpdk.h" 37 #include "spdk/log.h" 38 39 #include "env_internal.h" 40 41 #include <rte_config.h> 42 #include <rte_cycles.h> 43 #include <rte_malloc.h> 44 #include <rte_mempool.h> 45 #include <rte_memzone.h> 46 #include <rte_version.h> 47 48 static uint64_t 49 virt_to_phys(void *vaddr) 50 { 51 uint64_t ret; 52 53 ret = rte_malloc_virt2iova(vaddr); 54 if (ret != RTE_BAD_IOVA) { 55 return ret; 56 } 57 58 return spdk_vtophys(vaddr, NULL); 59 } 60 61 void * 62 spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 63 { 64 void *buf; 65 66 if (flags == 0) { 67 return NULL; 68 } 69 70 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 71 buf = rte_malloc_socket(NULL, size, align, socket_id); 72 if (buf && phys_addr) { 73 #ifdef DEBUG 74 SPDK_ERRLOG("phys_addr param in spdk_malloc() is deprecated\n"); 75 #endif 76 *phys_addr = virt_to_phys(buf); 77 } 78 return buf; 79 } 80 81 void * 82 spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 83 { 84 void *buf; 85 86 if (flags == 0) { 87 return NULL; 88 } 89 90 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 91 buf = rte_zmalloc_socket(NULL, size, align, socket_id); 92 if (buf && phys_addr) { 93 #ifdef DEBUG 94 SPDK_ERRLOG("phys_addr param in spdk_zmalloc() is deprecated\n"); 95 #endif 96 *phys_addr = virt_to_phys(buf); 97 } 98 return buf; 99 } 100 101 void * 102 spdk_realloc(void *buf, size_t size, size_t align) 103 { 104 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 105 return rte_realloc(buf, size, align); 106 } 107 108 void 109 spdk_free(void *buf) 110 { 111 rte_free(buf); 112 } 113 114 void * 115 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 116 { 117 return spdk_malloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 118 } 119 120 void * 121 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 122 { 123 return spdk_zmalloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 124 } 125 126 void * 127 spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr) 128 { 129 return spdk_dma_malloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 130 } 131 132 void * 133 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr) 134 { 135 return spdk_dma_zmalloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 136 } 137 138 void * 139 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr) 140 { 141 void *new_buf; 142 143 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 144 new_buf = rte_realloc(buf, size, align); 145 if (new_buf && phys_addr) { 146 *phys_addr = virt_to_phys(new_buf); 147 } 148 return new_buf; 149 } 150 151 void 152 spdk_dma_free(void *buf) 153 { 154 spdk_free(buf); 155 } 156 157 void * 158 spdk_memzone_reserve_aligned(const char *name, size_t len, int socket_id, 159 unsigned flags, unsigned align) 160 { 161 const struct rte_memzone *mz; 162 unsigned dpdk_flags = 0; 163 164 if ((flags & SPDK_MEMZONE_NO_IOVA_CONTIG) == 0) { 165 dpdk_flags |= RTE_MEMZONE_IOVA_CONTIG; 166 } 167 168 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 169 socket_id = SOCKET_ID_ANY; 170 } 171 172 mz = rte_memzone_reserve_aligned(name, len, socket_id, dpdk_flags, align); 173 174 if (mz != NULL) { 175 memset(mz->addr, 0, len); 176 return mz->addr; 177 } else { 178 return NULL; 179 } 180 } 181 182 void * 183 spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags) 184 { 185 return spdk_memzone_reserve_aligned(name, len, socket_id, flags, 186 RTE_CACHE_LINE_SIZE); 187 } 188 189 void * 190 spdk_memzone_lookup(const char *name) 191 { 192 const struct rte_memzone *mz = rte_memzone_lookup(name); 193 194 if (mz != NULL) { 195 return mz->addr; 196 } else { 197 return NULL; 198 } 199 } 200 201 int 202 spdk_memzone_free(const char *name) 203 { 204 const struct rte_memzone *mz = rte_memzone_lookup(name); 205 206 if (mz != NULL) { 207 return rte_memzone_free(mz); 208 } 209 210 return -1; 211 } 212 213 void 214 spdk_memzone_dump(FILE *f) 215 { 216 rte_memzone_dump(f); 217 } 218 219 struct spdk_mempool * 220 spdk_mempool_create_ctor(const char *name, size_t count, 221 size_t ele_size, size_t cache_size, int socket_id, 222 spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg) 223 { 224 struct rte_mempool *mp; 225 size_t tmp; 226 unsigned dpdk_flags = 0; 227 228 #if RTE_VERSION >= RTE_VERSION_NUM(21, 11, 0, 0) 229 dpdk_flags |= RTE_MEMPOOL_F_NO_IOVA_CONTIG; 230 #else 231 dpdk_flags |= MEMPOOL_F_NO_IOVA_CONTIG; 232 #endif 233 234 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 235 socket_id = SOCKET_ID_ANY; 236 } 237 238 /* No more than half of all elements can be in cache */ 239 tmp = (count / 2) / rte_lcore_count(); 240 if (cache_size > tmp) { 241 cache_size = tmp; 242 } 243 244 if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) { 245 cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; 246 } 247 248 mp = rte_mempool_create(name, count, ele_size, cache_size, 249 0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg, 250 socket_id, dpdk_flags); 251 252 return (struct spdk_mempool *)mp; 253 } 254 255 256 struct spdk_mempool * 257 spdk_mempool_create(const char *name, size_t count, 258 size_t ele_size, size_t cache_size, int socket_id) 259 { 260 return spdk_mempool_create_ctor(name, count, ele_size, cache_size, socket_id, 261 NULL, NULL); 262 } 263 264 char * 265 spdk_mempool_get_name(struct spdk_mempool *mp) 266 { 267 return ((struct rte_mempool *)mp)->name; 268 } 269 270 void 271 spdk_mempool_free(struct spdk_mempool *mp) 272 { 273 rte_mempool_free((struct rte_mempool *)mp); 274 } 275 276 void * 277 spdk_mempool_get(struct spdk_mempool *mp) 278 { 279 void *ele = NULL; 280 int rc; 281 282 rc = rte_mempool_get((struct rte_mempool *)mp, &ele); 283 if (rc != 0) { 284 return NULL; 285 } 286 return ele; 287 } 288 289 int 290 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 291 { 292 return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count); 293 } 294 295 void 296 spdk_mempool_put(struct spdk_mempool *mp, void *ele) 297 { 298 rte_mempool_put((struct rte_mempool *)mp, ele); 299 } 300 301 void 302 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 303 { 304 rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count); 305 } 306 307 size_t 308 spdk_mempool_count(const struct spdk_mempool *pool) 309 { 310 return rte_mempool_avail_count((struct rte_mempool *)pool); 311 } 312 313 uint32_t 314 spdk_mempool_obj_iter(struct spdk_mempool *mp, spdk_mempool_obj_cb_t obj_cb, 315 void *obj_cb_arg) 316 { 317 return rte_mempool_obj_iter((struct rte_mempool *)mp, (rte_mempool_obj_cb_t *)obj_cb, 318 obj_cb_arg); 319 } 320 321 struct spdk_mempool * 322 spdk_mempool_lookup(const char *name) 323 { 324 return (struct spdk_mempool *)rte_mempool_lookup(name); 325 } 326 327 bool 328 spdk_process_is_primary(void) 329 { 330 return (rte_eal_process_type() == RTE_PROC_PRIMARY); 331 } 332 333 uint64_t spdk_get_ticks(void) 334 { 335 return rte_get_timer_cycles(); 336 } 337 338 uint64_t spdk_get_ticks_hz(void) 339 { 340 return rte_get_timer_hz(); 341 } 342 343 void spdk_delay_us(unsigned int us) 344 { 345 rte_delay_us(us); 346 } 347 348 void spdk_pause(void) 349 { 350 rte_pause(); 351 } 352 353 void 354 spdk_unaffinitize_thread(void) 355 { 356 rte_cpuset_t new_cpuset; 357 long num_cores, i; 358 359 CPU_ZERO(&new_cpuset); 360 361 num_cores = sysconf(_SC_NPROCESSORS_CONF); 362 363 /* Create a mask containing all CPUs */ 364 for (i = 0; i < num_cores; i++) { 365 CPU_SET(i, &new_cpuset); 366 } 367 368 rte_thread_set_affinity(&new_cpuset); 369 } 370 371 void * 372 spdk_call_unaffinitized(void *cb(void *arg), void *arg) 373 { 374 rte_cpuset_t orig_cpuset; 375 void *ret; 376 377 if (cb == NULL) { 378 return NULL; 379 } 380 381 rte_thread_get_affinity(&orig_cpuset); 382 383 spdk_unaffinitize_thread(); 384 385 ret = cb(arg); 386 387 rte_thread_set_affinity(&orig_cpuset); 388 389 return ret; 390 } 391 392 struct spdk_ring * 393 spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id) 394 { 395 char ring_name[64]; 396 static uint32_t ring_num = 0; 397 unsigned flags = RING_F_EXACT_SZ; 398 399 switch (type) { 400 case SPDK_RING_TYPE_SP_SC: 401 flags |= RING_F_SP_ENQ | RING_F_SC_DEQ; 402 break; 403 case SPDK_RING_TYPE_MP_SC: 404 flags |= RING_F_SC_DEQ; 405 break; 406 case SPDK_RING_TYPE_MP_MC: 407 flags |= 0; 408 break; 409 default: 410 return NULL; 411 } 412 413 snprintf(ring_name, sizeof(ring_name), "ring_%u_%d", 414 __atomic_fetch_add(&ring_num, 1, __ATOMIC_RELAXED), getpid()); 415 416 return (struct spdk_ring *)rte_ring_create(ring_name, count, socket_id, flags); 417 } 418 419 void 420 spdk_ring_free(struct spdk_ring *ring) 421 { 422 rte_ring_free((struct rte_ring *)ring); 423 } 424 425 size_t 426 spdk_ring_count(struct spdk_ring *ring) 427 { 428 return rte_ring_count((struct rte_ring *)ring); 429 } 430 431 size_t 432 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count, 433 size_t *free_space) 434 { 435 return rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count, 436 (unsigned int *)free_space); 437 } 438 439 size_t 440 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count) 441 { 442 return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count, NULL); 443 } 444 445 void 446 spdk_env_dpdk_dump_mem_stats(FILE *file) 447 { 448 fprintf(file, "DPDK memory size %" PRIu64 "\n", rte_eal_get_physmem_size()); 449 fprintf(file, "DPDK memory layout\n"); 450 rte_dump_physmem_layout(file); 451 fprintf(file, "DPDK memzones.\n"); 452 rte_memzone_dump(file); 453 fprintf(file, "DPDK mempools.\n"); 454 rte_mempool_list_dump(file); 455 fprintf(file, "DPDK malloc stats.\n"); 456 rte_malloc_dump_stats(file, NULL); 457 fprintf(file, "DPDK malloc heaps.\n"); 458 rte_malloc_dump_heaps(file); 459 } 460