1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 #include "spdk/util.h" 36 #include "spdk/env_dpdk.h" 37 #include "spdk/log.h" 38 39 #include "env_internal.h" 40 41 #include <rte_config.h> 42 #include <rte_cycles.h> 43 #include <rte_malloc.h> 44 #include <rte_mempool.h> 45 #include <rte_memzone.h> 46 #include <rte_version.h> 47 48 static uint64_t 49 virt_to_phys(void *vaddr) 50 { 51 uint64_t ret; 52 53 ret = rte_malloc_virt2iova(vaddr); 54 if (ret != RTE_BAD_IOVA) { 55 return ret; 56 } 57 58 return spdk_vtophys(vaddr, NULL); 59 } 60 61 void * 62 spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 63 { 64 void *buf; 65 66 if (flags == 0) { 67 return NULL; 68 } 69 70 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 71 buf = rte_malloc_socket(NULL, size, align, socket_id); 72 if (buf && phys_addr) { 73 #ifdef DEBUG 74 SPDK_ERRLOG("phys_addr param in spdk_*malloc() is deprecated\n"); 75 #endif 76 *phys_addr = virt_to_phys(buf); 77 } 78 return buf; 79 } 80 81 void * 82 spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 83 { 84 void *buf = spdk_malloc(size, align, phys_addr, socket_id, flags); 85 if (buf) { 86 memset(buf, 0, size); 87 } 88 return buf; 89 } 90 91 void * 92 spdk_realloc(void *buf, size_t size, size_t align) 93 { 94 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 95 return rte_realloc(buf, size, align); 96 } 97 98 void 99 spdk_free(void *buf) 100 { 101 struct spdk_mem_region *region, *tmp; 102 103 rte_free(buf); 104 pthread_mutex_lock(&g_spdk_mem_region_mutex); 105 g_spdk_mem_do_not_notify = true; 106 /* Perform memory unregister previously postponed in memory_hotplug_cb() */ 107 TAILQ_FOREACH_SAFE(region, &g_spdk_mem_regions, tailq, tmp) { 108 spdk_mem_unregister(region->addr, region->len); 109 TAILQ_REMOVE(&g_spdk_mem_regions, region, tailq); 110 free(region); 111 } 112 g_spdk_mem_do_not_notify = false; 113 pthread_mutex_unlock(&g_spdk_mem_region_mutex); 114 } 115 116 void * 117 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 118 { 119 return spdk_malloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 120 } 121 122 void * 123 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 124 { 125 return spdk_zmalloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 126 } 127 128 void * 129 spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr) 130 { 131 return spdk_dma_malloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 132 } 133 134 void * 135 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr) 136 { 137 return spdk_dma_zmalloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 138 } 139 140 void * 141 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr) 142 { 143 void *new_buf; 144 145 align = spdk_max(align, RTE_CACHE_LINE_SIZE); 146 new_buf = rte_realloc(buf, size, align); 147 if (new_buf && phys_addr) { 148 *phys_addr = virt_to_phys(new_buf); 149 } 150 return new_buf; 151 } 152 153 void 154 spdk_dma_free(void *buf) 155 { 156 spdk_free(buf); 157 } 158 159 void * 160 spdk_memzone_reserve_aligned(const char *name, size_t len, int socket_id, 161 unsigned flags, unsigned align) 162 { 163 const struct rte_memzone *mz; 164 unsigned dpdk_flags = 0; 165 166 if ((flags & SPDK_MEMZONE_NO_IOVA_CONTIG) == 0) { 167 dpdk_flags |= RTE_MEMZONE_IOVA_CONTIG; 168 } 169 170 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 171 socket_id = SOCKET_ID_ANY; 172 } 173 174 mz = rte_memzone_reserve_aligned(name, len, socket_id, dpdk_flags, align); 175 176 if (mz != NULL) { 177 memset(mz->addr, 0, len); 178 return mz->addr; 179 } else { 180 return NULL; 181 } 182 } 183 184 void * 185 spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags) 186 { 187 return spdk_memzone_reserve_aligned(name, len, socket_id, flags, 188 RTE_CACHE_LINE_SIZE); 189 } 190 191 void * 192 spdk_memzone_lookup(const char *name) 193 { 194 const struct rte_memzone *mz = rte_memzone_lookup(name); 195 196 if (mz != NULL) { 197 return mz->addr; 198 } else { 199 return NULL; 200 } 201 } 202 203 int 204 spdk_memzone_free(const char *name) 205 { 206 const struct rte_memzone *mz = rte_memzone_lookup(name); 207 208 if (mz != NULL) { 209 return rte_memzone_free(mz); 210 } 211 212 return -1; 213 } 214 215 void 216 spdk_memzone_dump(FILE *f) 217 { 218 rte_memzone_dump(f); 219 } 220 221 struct spdk_mempool * 222 spdk_mempool_create_ctor(const char *name, size_t count, 223 size_t ele_size, size_t cache_size, int socket_id, 224 spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg) 225 { 226 struct rte_mempool *mp; 227 size_t tmp; 228 229 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 230 socket_id = SOCKET_ID_ANY; 231 } 232 233 /* No more than half of all elements can be in cache */ 234 tmp = (count / 2) / rte_lcore_count(); 235 if (cache_size > tmp) { 236 cache_size = tmp; 237 } 238 239 if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) { 240 cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; 241 } 242 243 mp = rte_mempool_create(name, count, ele_size, cache_size, 244 0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg, 245 socket_id, MEMPOOL_F_NO_IOVA_CONTIG); 246 247 return (struct spdk_mempool *)mp; 248 } 249 250 251 struct spdk_mempool * 252 spdk_mempool_create(const char *name, size_t count, 253 size_t ele_size, size_t cache_size, int socket_id) 254 { 255 return spdk_mempool_create_ctor(name, count, ele_size, cache_size, socket_id, 256 NULL, NULL); 257 } 258 259 char * 260 spdk_mempool_get_name(struct spdk_mempool *mp) 261 { 262 return ((struct rte_mempool *)mp)->name; 263 } 264 265 void 266 spdk_mempool_free(struct spdk_mempool *mp) 267 { 268 rte_mempool_free((struct rte_mempool *)mp); 269 } 270 271 void * 272 spdk_mempool_get(struct spdk_mempool *mp) 273 { 274 void *ele = NULL; 275 int rc; 276 277 rc = rte_mempool_get((struct rte_mempool *)mp, &ele); 278 if (rc != 0) { 279 return NULL; 280 } 281 return ele; 282 } 283 284 int 285 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 286 { 287 return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count); 288 } 289 290 void 291 spdk_mempool_put(struct spdk_mempool *mp, void *ele) 292 { 293 rte_mempool_put((struct rte_mempool *)mp, ele); 294 } 295 296 void 297 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 298 { 299 rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count); 300 } 301 302 size_t 303 spdk_mempool_count(const struct spdk_mempool *pool) 304 { 305 return rte_mempool_avail_count((struct rte_mempool *)pool); 306 } 307 308 uint32_t 309 spdk_mempool_obj_iter(struct spdk_mempool *mp, spdk_mempool_obj_cb_t obj_cb, 310 void *obj_cb_arg) 311 { 312 return rte_mempool_obj_iter((struct rte_mempool *)mp, (rte_mempool_obj_cb_t *)obj_cb, 313 obj_cb_arg); 314 } 315 316 struct spdk_mempool * 317 spdk_mempool_lookup(const char *name) 318 { 319 return (struct spdk_mempool *)rte_mempool_lookup(name); 320 } 321 322 bool 323 spdk_process_is_primary(void) 324 { 325 return (rte_eal_process_type() == RTE_PROC_PRIMARY); 326 } 327 328 uint64_t spdk_get_ticks(void) 329 { 330 return rte_get_timer_cycles(); 331 } 332 333 uint64_t spdk_get_ticks_hz(void) 334 { 335 return rte_get_timer_hz(); 336 } 337 338 void spdk_delay_us(unsigned int us) 339 { 340 rte_delay_us(us); 341 } 342 343 void spdk_pause(void) 344 { 345 rte_pause(); 346 } 347 348 void 349 spdk_unaffinitize_thread(void) 350 { 351 rte_cpuset_t new_cpuset, orig_cpuset; 352 long num_cores, i, orig_num_cores; 353 354 CPU_ZERO(&new_cpuset); 355 356 num_cores = sysconf(_SC_NPROCESSORS_CONF); 357 358 /* Create a mask containing all CPUs */ 359 for (i = 0; i < num_cores; i++) { 360 CPU_SET(i, &new_cpuset); 361 } 362 363 rte_thread_get_affinity(&orig_cpuset); 364 orig_num_cores = CPU_COUNT(&orig_cpuset); 365 if (orig_num_cores < num_cores) { 366 for (i = 0; i < orig_num_cores; i++) { 367 if (CPU_ISSET(i, &orig_cpuset)) { 368 CPU_CLR(i, &new_cpuset); 369 } 370 } 371 } 372 373 rte_thread_set_affinity(&new_cpuset); 374 } 375 376 void * 377 spdk_call_unaffinitized(void *cb(void *arg), void *arg) 378 { 379 rte_cpuset_t orig_cpuset; 380 void *ret; 381 382 if (cb == NULL) { 383 return NULL; 384 } 385 386 rte_thread_get_affinity(&orig_cpuset); 387 388 spdk_unaffinitize_thread(); 389 390 ret = cb(arg); 391 392 rte_thread_set_affinity(&orig_cpuset); 393 394 return ret; 395 } 396 397 struct spdk_ring * 398 spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id) 399 { 400 char ring_name[64]; 401 static uint32_t ring_num = 0; 402 unsigned flags = RING_F_EXACT_SZ; 403 404 switch (type) { 405 case SPDK_RING_TYPE_SP_SC: 406 flags |= RING_F_SP_ENQ | RING_F_SC_DEQ; 407 break; 408 case SPDK_RING_TYPE_MP_SC: 409 flags |= RING_F_SC_DEQ; 410 break; 411 case SPDK_RING_TYPE_MP_MC: 412 flags |= 0; 413 break; 414 default: 415 return NULL; 416 } 417 418 snprintf(ring_name, sizeof(ring_name), "ring_%u_%d", 419 __atomic_fetch_add(&ring_num, 1, __ATOMIC_RELAXED), getpid()); 420 421 return (struct spdk_ring *)rte_ring_create(ring_name, count, socket_id, flags); 422 } 423 424 void 425 spdk_ring_free(struct spdk_ring *ring) 426 { 427 rte_ring_free((struct rte_ring *)ring); 428 } 429 430 size_t 431 spdk_ring_count(struct spdk_ring *ring) 432 { 433 return rte_ring_count((struct rte_ring *)ring); 434 } 435 436 size_t 437 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count, 438 size_t *free_space) 439 { 440 return rte_ring_enqueue_bulk((struct rte_ring *)ring, objs, count, 441 (unsigned int *)free_space); 442 } 443 444 size_t 445 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count) 446 { 447 return rte_ring_dequeue_burst((struct rte_ring *)ring, objs, count, NULL); 448 } 449 450 void 451 spdk_env_dpdk_dump_mem_stats(FILE *file) 452 { 453 fprintf(file, "DPDK memory size %" PRIu64 "\n", rte_eal_get_physmem_size()); 454 fprintf(file, "DPDK memory layout\n"); 455 rte_dump_physmem_layout(file); 456 fprintf(file, "DPDK memzones.\n"); 457 rte_memzone_dump(file); 458 fprintf(file, "DPDK mempools.\n"); 459 rte_mempool_list_dump(file); 460 fprintf(file, "DPDK malloc stats.\n"); 461 rte_malloc_dump_stats(file, NULL); 462 fprintf(file, "DPDK malloc heaps.\n"); 463 rte_malloc_dump_heaps(file); 464 } 465