1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/env.h" 37 38 #include <rte_config.h> 39 #include <rte_cycles.h> 40 #include <rte_malloc.h> 41 #include <rte_mempool.h> 42 #include <rte_memzone.h> 43 #include <rte_version.h> 44 45 static uint64_t 46 virt_to_phys(void *vaddr) 47 { 48 uint64_t ret; 49 50 #if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3) 51 ret = rte_malloc_virt2iova(vaddr); 52 if (ret != RTE_BAD_IOVA) { 53 return ret; 54 } 55 #else 56 ret = rte_malloc_virt2phy(vaddr); 57 if (ret != RTE_BAD_PHYS_ADDR) { 58 return ret; 59 } 60 #endif 61 62 return spdk_vtophys(vaddr); 63 } 64 65 void * 66 spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 67 { 68 if (flags == 0) { 69 return NULL; 70 } 71 72 void *buf = rte_malloc_socket(NULL, size, align, socket_id); 73 if (buf && phys_addr) { 74 *phys_addr = virt_to_phys(buf); 75 } 76 return buf; 77 } 78 79 void * 80 spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 81 { 82 void *buf = spdk_malloc(size, align, phys_addr, socket_id, flags); 83 if (buf) { 84 memset(buf, 0, size); 85 } 86 return buf; 87 } 88 89 void 90 spdk_free(void *buf) 91 { 92 rte_free(buf); 93 } 94 95 void * 96 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 97 { 98 return spdk_malloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 99 } 100 101 void * 102 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 103 { 104 return spdk_zmalloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 105 } 106 107 void * 108 spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr) 109 { 110 return spdk_dma_malloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 111 } 112 113 void * 114 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr) 115 { 116 return spdk_dma_zmalloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 117 } 118 119 void * 120 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr) 121 { 122 void *new_buf = rte_realloc(buf, size, align); 123 if (new_buf && phys_addr) { 124 *phys_addr = virt_to_phys(new_buf); 125 } 126 return new_buf; 127 } 128 129 void 130 spdk_dma_free(void *buf) 131 { 132 spdk_free(buf); 133 } 134 135 void * 136 spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags) 137 { 138 const struct rte_memzone *mz; 139 unsigned dpdk_flags = 0; 140 141 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 142 socket_id = SOCKET_ID_ANY; 143 } 144 145 mz = rte_memzone_reserve(name, len, socket_id, dpdk_flags); 146 147 if (mz != NULL) { 148 memset(mz->addr, 0, len); 149 return mz->addr; 150 } else { 151 return NULL; 152 } 153 } 154 155 void * 156 spdk_memzone_lookup(const char *name) 157 { 158 const struct rte_memzone *mz = rte_memzone_lookup(name); 159 160 if (mz != NULL) { 161 return mz->addr; 162 } else { 163 return NULL; 164 } 165 } 166 167 int 168 spdk_memzone_free(const char *name) 169 { 170 const struct rte_memzone *mz = rte_memzone_lookup(name); 171 172 if (mz != NULL) { 173 return rte_memzone_free(mz); 174 } 175 176 return -1; 177 } 178 179 void 180 spdk_memzone_dump(FILE *f) 181 { 182 rte_memzone_dump(f); 183 } 184 185 struct spdk_mempool * 186 spdk_mempool_create_ctor(const char *name, size_t count, 187 size_t ele_size, size_t cache_size, int socket_id, 188 spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg) 189 { 190 struct rte_mempool *mp; 191 size_t tmp; 192 193 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 194 socket_id = SOCKET_ID_ANY; 195 } 196 197 /* No more than half of all elements can be in cache */ 198 tmp = (count / 2) / rte_lcore_count(); 199 if (cache_size > tmp) { 200 cache_size = tmp; 201 } 202 203 if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) { 204 cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; 205 } 206 207 mp = rte_mempool_create(name, count, ele_size, cache_size, 208 0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg, 209 socket_id, MEMPOOL_F_NO_PHYS_CONTIG); 210 211 return (struct spdk_mempool *)mp; 212 } 213 214 215 struct spdk_mempool * 216 spdk_mempool_create(const char *name, size_t count, 217 size_t ele_size, size_t cache_size, int socket_id) 218 { 219 return spdk_mempool_create_ctor(name, count, ele_size, cache_size, socket_id, 220 NULL, NULL); 221 } 222 223 char * 224 spdk_mempool_get_name(struct spdk_mempool *mp) 225 { 226 return ((struct rte_mempool *)mp)->name; 227 } 228 229 void 230 spdk_mempool_free(struct spdk_mempool *mp) 231 { 232 #if RTE_VERSION >= RTE_VERSION_NUM(16, 7, 0, 1) 233 rte_mempool_free((struct rte_mempool *)mp); 234 #endif 235 } 236 237 void * 238 spdk_mempool_get(struct spdk_mempool *mp) 239 { 240 void *ele = NULL; 241 int rc; 242 243 rc = rte_mempool_get((struct rte_mempool *)mp, &ele); 244 if (rc != 0) { 245 return NULL; 246 } 247 return ele; 248 } 249 250 int 251 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 252 { 253 return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count); 254 } 255 256 void 257 spdk_mempool_put(struct spdk_mempool *mp, void *ele) 258 { 259 rte_mempool_put((struct rte_mempool *)mp, ele); 260 } 261 262 void 263 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 264 { 265 rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count); 266 } 267 268 size_t 269 spdk_mempool_count(const struct spdk_mempool *pool) 270 { 271 #if RTE_VERSION < RTE_VERSION_NUM(16, 7, 0, 1) 272 return rte_mempool_count((struct rte_mempool *)pool); 273 #else 274 return rte_mempool_avail_count((struct rte_mempool *)pool); 275 #endif 276 } 277 278 bool 279 spdk_process_is_primary(void) 280 { 281 return (rte_eal_process_type() == RTE_PROC_PRIMARY); 282 } 283 284 uint64_t spdk_get_ticks(void) 285 { 286 return rte_get_timer_cycles(); 287 } 288 289 uint64_t spdk_get_ticks_hz(void) 290 { 291 return rte_get_timer_hz(); 292 } 293 294 void spdk_delay_us(unsigned int us) 295 { 296 rte_delay_us(us); 297 } 298 299 void 300 spdk_unaffinitize_thread(void) 301 { 302 rte_cpuset_t new_cpuset; 303 long num_cores, i; 304 305 CPU_ZERO(&new_cpuset); 306 307 num_cores = sysconf(_SC_NPROCESSORS_CONF); 308 309 /* Create a mask containing all CPUs */ 310 for (i = 0; i < num_cores; i++) { 311 CPU_SET(i, &new_cpuset); 312 } 313 314 rte_thread_set_affinity(&new_cpuset); 315 } 316 317 void * 318 spdk_call_unaffinitized(void *cb(void *arg), void *arg) 319 { 320 rte_cpuset_t orig_cpuset; 321 void *ret; 322 323 if (cb == NULL) { 324 return NULL; 325 } 326 327 rte_thread_get_affinity(&orig_cpuset); 328 329 spdk_unaffinitize_thread(); 330 331 ret = cb(arg); 332 333 rte_thread_set_affinity(&orig_cpuset); 334 335 return ret; 336 } 337 338 struct spdk_ring * 339 spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id) 340 { 341 char ring_name[64]; 342 static uint32_t ring_num = 0; 343 unsigned flags = 0; 344 345 switch (type) { 346 case SPDK_RING_TYPE_SP_SC: 347 flags = RING_F_SP_ENQ | RING_F_SC_DEQ; 348 break; 349 case SPDK_RING_TYPE_MP_SC: 350 flags = RING_F_SC_DEQ; 351 break; 352 default: 353 return NULL; 354 } 355 356 snprintf(ring_name, sizeof(ring_name), "ring_%u_%d", 357 __sync_fetch_and_add(&ring_num, 1), getpid()); 358 359 return (struct spdk_ring *)rte_ring_create(ring_name, count, socket_id, flags); 360 } 361 362 void 363 spdk_ring_free(struct spdk_ring *ring) 364 { 365 rte_ring_free((struct rte_ring *)ring); 366 } 367 368 size_t 369 spdk_ring_count(struct spdk_ring *ring) 370 { 371 return rte_ring_count((struct rte_ring *)ring); 372 } 373 374 size_t 375 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count) 376 { 377 int rc; 378 #if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0) 379 rc = rte_ring_mp_enqueue_bulk((struct rte_ring *)ring, objs, count); 380 if (rc == 0) { 381 return count; 382 } 383 384 return 0; 385 #else 386 rc = rte_ring_mp_enqueue_bulk((struct rte_ring *)ring, objs, count, NULL); 387 return rc; 388 #endif 389 } 390 391 size_t 392 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count) 393 { 394 #if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0) 395 return rte_ring_sc_dequeue_burst((struct rte_ring *)ring, objs, count); 396 #else 397 return rte_ring_sc_dequeue_burst((struct rte_ring *)ring, objs, count, NULL); 398 #endif 399 } 400