1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/env.h" 37 38 #include <rte_config.h> 39 #include <rte_cycles.h> 40 #include <rte_malloc.h> 41 #include <rte_mempool.h> 42 #include <rte_memzone.h> 43 #include <rte_version.h> 44 45 static uint64_t 46 virt_to_phys(void *vaddr) 47 { 48 uint64_t ret; 49 50 #if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3) 51 ret = rte_malloc_virt2iova(vaddr); 52 if (ret != RTE_BAD_IOVA) { 53 return ret; 54 } 55 #else 56 ret = rte_malloc_virt2phy(vaddr); 57 if (ret != RTE_BAD_PHYS_ADDR) { 58 return ret; 59 } 60 #endif 61 62 return spdk_vtophys(vaddr); 63 } 64 65 void * 66 spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 67 { 68 if (flags == 0) { 69 return NULL; 70 } 71 72 void *buf = rte_malloc_socket(NULL, size, align, socket_id); 73 if (buf && phys_addr) { 74 *phys_addr = virt_to_phys(buf); 75 } 76 return buf; 77 } 78 79 void * 80 spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 81 { 82 void *buf = spdk_malloc(size, align, phys_addr, socket_id, flags); 83 if (buf) { 84 memset(buf, 0, size); 85 } 86 return buf; 87 } 88 89 void 90 spdk_free(void *buf) 91 { 92 rte_free(buf); 93 } 94 95 void * 96 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 97 { 98 return spdk_malloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 99 } 100 101 void * 102 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 103 { 104 return spdk_zmalloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 105 } 106 107 void * 108 spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr) 109 { 110 return spdk_dma_malloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 111 } 112 113 void * 114 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr) 115 { 116 return spdk_dma_zmalloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 117 } 118 119 void * 120 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr) 121 { 122 void *new_buf = rte_realloc(buf, size, align); 123 if (new_buf && phys_addr) { 124 *phys_addr = virt_to_phys(new_buf); 125 } 126 return new_buf; 127 } 128 129 void 130 spdk_dma_free(void *buf) 131 { 132 spdk_free(buf); 133 } 134 135 void * 136 spdk_memzone_reserve_aligned(const char *name, size_t len, int socket_id, 137 unsigned flags, unsigned align) 138 { 139 const struct rte_memzone *mz; 140 unsigned dpdk_flags = 0; 141 142 #if RTE_VERSION >= RTE_VERSION_NUM(18, 05, 0, 0) 143 /* Older DPDKs do not offer such flag since their 144 * memzones are iova-contiguous by default. 145 */ 146 if ((flags & SPDK_MEMZONE_NO_IOVA_CONTIG) == 0) { 147 dpdk_flags |= RTE_MEMZONE_IOVA_CONTIG; 148 } 149 #endif 150 151 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 152 socket_id = SOCKET_ID_ANY; 153 } 154 155 mz = rte_memzone_reserve_aligned(name, len, socket_id, dpdk_flags, align); 156 157 if (mz != NULL) { 158 memset(mz->addr, 0, len); 159 return mz->addr; 160 } else { 161 return NULL; 162 } 163 } 164 165 void * 166 spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags) 167 { 168 return spdk_memzone_reserve_aligned(name, len, socket_id, flags, 169 RTE_CACHE_LINE_SIZE); 170 } 171 172 void * 173 spdk_memzone_lookup(const char *name) 174 { 175 const struct rte_memzone *mz = rte_memzone_lookup(name); 176 177 if (mz != NULL) { 178 return mz->addr; 179 } else { 180 return NULL; 181 } 182 } 183 184 int 185 spdk_memzone_free(const char *name) 186 { 187 const struct rte_memzone *mz = rte_memzone_lookup(name); 188 189 if (mz != NULL) { 190 return rte_memzone_free(mz); 191 } 192 193 return -1; 194 } 195 196 void 197 spdk_memzone_dump(FILE *f) 198 { 199 rte_memzone_dump(f); 200 } 201 202 struct spdk_mempool * 203 spdk_mempool_create_ctor(const char *name, size_t count, 204 size_t ele_size, size_t cache_size, int socket_id, 205 spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg) 206 { 207 struct rte_mempool *mp; 208 size_t tmp; 209 210 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 211 socket_id = SOCKET_ID_ANY; 212 } 213 214 /* No more than half of all elements can be in cache */ 215 tmp = (count / 2) / rte_lcore_count(); 216 if (cache_size > tmp) { 217 cache_size = tmp; 218 } 219 220 if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) { 221 cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; 222 } 223 224 mp = rte_mempool_create(name, count, ele_size, cache_size, 225 0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg, 226 socket_id, MEMPOOL_F_NO_PHYS_CONTIG); 227 228 return (struct spdk_mempool *)mp; 229 } 230 231 232 struct spdk_mempool * 233 spdk_mempool_create(const char *name, size_t count, 234 size_t ele_size, size_t cache_size, int socket_id) 235 { 236 return spdk_mempool_create_ctor(name, count, ele_size, cache_size, socket_id, 237 NULL, NULL); 238 } 239 240 char * 241 spdk_mempool_get_name(struct spdk_mempool *mp) 242 { 243 return ((struct rte_mempool *)mp)->name; 244 } 245 246 void 247 spdk_mempool_free(struct spdk_mempool *mp) 248 { 249 #if RTE_VERSION >= RTE_VERSION_NUM(16, 7, 0, 1) 250 rte_mempool_free((struct rte_mempool *)mp); 251 #endif 252 } 253 254 void * 255 spdk_mempool_get(struct spdk_mempool *mp) 256 { 257 void *ele = NULL; 258 int rc; 259 260 rc = rte_mempool_get((struct rte_mempool *)mp, &ele); 261 if (rc != 0) { 262 return NULL; 263 } 264 return ele; 265 } 266 267 int 268 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 269 { 270 return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count); 271 } 272 273 void 274 spdk_mempool_put(struct spdk_mempool *mp, void *ele) 275 { 276 rte_mempool_put((struct rte_mempool *)mp, ele); 277 } 278 279 void 280 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 281 { 282 rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count); 283 } 284 285 size_t 286 spdk_mempool_count(const struct spdk_mempool *pool) 287 { 288 #if RTE_VERSION < RTE_VERSION_NUM(16, 7, 0, 1) 289 return rte_mempool_count((struct rte_mempool *)pool); 290 #else 291 return rte_mempool_avail_count((struct rte_mempool *)pool); 292 #endif 293 } 294 295 bool 296 spdk_process_is_primary(void) 297 { 298 return (rte_eal_process_type() == RTE_PROC_PRIMARY); 299 } 300 301 uint64_t spdk_get_ticks(void) 302 { 303 return rte_get_timer_cycles(); 304 } 305 306 uint64_t spdk_get_ticks_hz(void) 307 { 308 return rte_get_timer_hz(); 309 } 310 311 void spdk_delay_us(unsigned int us) 312 { 313 rte_delay_us(us); 314 } 315 316 void 317 spdk_unaffinitize_thread(void) 318 { 319 rte_cpuset_t new_cpuset; 320 long num_cores, i; 321 322 CPU_ZERO(&new_cpuset); 323 324 num_cores = sysconf(_SC_NPROCESSORS_CONF); 325 326 /* Create a mask containing all CPUs */ 327 for (i = 0; i < num_cores; i++) { 328 CPU_SET(i, &new_cpuset); 329 } 330 331 rte_thread_set_affinity(&new_cpuset); 332 } 333 334 void * 335 spdk_call_unaffinitized(void *cb(void *arg), void *arg) 336 { 337 rte_cpuset_t orig_cpuset; 338 void *ret; 339 340 if (cb == NULL) { 341 return NULL; 342 } 343 344 rte_thread_get_affinity(&orig_cpuset); 345 346 spdk_unaffinitize_thread(); 347 348 ret = cb(arg); 349 350 rte_thread_set_affinity(&orig_cpuset); 351 352 return ret; 353 } 354 355 struct spdk_ring * 356 spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id) 357 { 358 char ring_name[64]; 359 static uint32_t ring_num = 0; 360 unsigned flags = 0; 361 362 switch (type) { 363 case SPDK_RING_TYPE_SP_SC: 364 flags = RING_F_SP_ENQ | RING_F_SC_DEQ; 365 break; 366 case SPDK_RING_TYPE_MP_SC: 367 flags = RING_F_SC_DEQ; 368 break; 369 default: 370 return NULL; 371 } 372 373 snprintf(ring_name, sizeof(ring_name), "ring_%u_%d", 374 __sync_fetch_and_add(&ring_num, 1), getpid()); 375 376 return (struct spdk_ring *)rte_ring_create(ring_name, count, socket_id, flags); 377 } 378 379 void 380 spdk_ring_free(struct spdk_ring *ring) 381 { 382 rte_ring_free((struct rte_ring *)ring); 383 } 384 385 size_t 386 spdk_ring_count(struct spdk_ring *ring) 387 { 388 return rte_ring_count((struct rte_ring *)ring); 389 } 390 391 size_t 392 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count) 393 { 394 int rc; 395 #if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0) 396 rc = rte_ring_mp_enqueue_bulk((struct rte_ring *)ring, objs, count); 397 if (rc == 0) { 398 return count; 399 } 400 401 return 0; 402 #else 403 rc = rte_ring_mp_enqueue_bulk((struct rte_ring *)ring, objs, count, NULL); 404 return rc; 405 #endif 406 } 407 408 size_t 409 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count) 410 { 411 #if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0) 412 return rte_ring_sc_dequeue_burst((struct rte_ring *)ring, objs, count); 413 #else 414 return rte_ring_sc_dequeue_burst((struct rte_ring *)ring, objs, count, NULL); 415 #endif 416 } 417