1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/env.h" 37 38 #include <rte_config.h> 39 #include <rte_cycles.h> 40 #include <rte_malloc.h> 41 #include <rte_mempool.h> 42 #include <rte_memzone.h> 43 #include <rte_version.h> 44 45 static uint64_t 46 virt_to_phys(void *vaddr) 47 { 48 uint64_t ret; 49 50 #if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 3) 51 ret = rte_malloc_virt2iova(vaddr); 52 if (ret != RTE_BAD_IOVA) { 53 return ret; 54 } 55 #else 56 ret = rte_malloc_virt2phy(vaddr); 57 if (ret != RTE_BAD_PHYS_ADDR) { 58 return ret; 59 } 60 #endif 61 62 return spdk_vtophys(vaddr); 63 } 64 65 void * 66 spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 67 { 68 if (flags == 0) { 69 return NULL; 70 } 71 72 void *buf = rte_malloc_socket(NULL, size, align, socket_id); 73 if (buf && phys_addr) { 74 *phys_addr = virt_to_phys(buf); 75 } 76 return buf; 77 } 78 79 void * 80 spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags) 81 { 82 void *buf = spdk_malloc(size, align, phys_addr, socket_id, flags); 83 if (buf) { 84 memset(buf, 0, size); 85 } 86 return buf; 87 } 88 89 void 90 spdk_free(void *buf) 91 { 92 rte_free(buf); 93 } 94 95 void * 96 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 97 { 98 return spdk_malloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 99 } 100 101 void * 102 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id) 103 { 104 return spdk_zmalloc(size, align, phys_addr, socket_id, (SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE)); 105 } 106 107 void * 108 spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr) 109 { 110 return spdk_dma_malloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 111 } 112 113 void * 114 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr) 115 { 116 return spdk_dma_zmalloc_socket(size, align, phys_addr, SPDK_ENV_SOCKET_ID_ANY); 117 } 118 119 void * 120 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr) 121 { 122 void *new_buf = rte_realloc(buf, size, align); 123 if (new_buf && phys_addr) { 124 *phys_addr = virt_to_phys(new_buf); 125 } 126 return new_buf; 127 } 128 129 void 130 spdk_dma_free(void *buf) 131 { 132 spdk_free(buf); 133 } 134 135 void * 136 spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags) 137 { 138 const struct rte_memzone *mz; 139 140 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 141 socket_id = SOCKET_ID_ANY; 142 } 143 144 mz = rte_memzone_reserve(name, len, socket_id, flags); 145 146 if (mz != NULL) { 147 memset(mz->addr, 0, len); 148 return mz->addr; 149 } else { 150 return NULL; 151 } 152 } 153 154 void * 155 spdk_memzone_lookup(const char *name) 156 { 157 const struct rte_memzone *mz = rte_memzone_lookup(name); 158 159 if (mz != NULL) { 160 return mz->addr; 161 } else { 162 return NULL; 163 } 164 } 165 166 int 167 spdk_memzone_free(const char *name) 168 { 169 const struct rte_memzone *mz = rte_memzone_lookup(name); 170 171 if (mz != NULL) { 172 return rte_memzone_free(mz); 173 } 174 175 return -1; 176 } 177 178 void 179 spdk_memzone_dump(FILE *f) 180 { 181 rte_memzone_dump(f); 182 } 183 184 struct spdk_mempool * 185 spdk_mempool_create_ctor(const char *name, size_t count, 186 size_t ele_size, size_t cache_size, int socket_id, 187 spdk_mempool_obj_cb_t *obj_init, void *obj_init_arg) 188 { 189 struct rte_mempool *mp; 190 size_t tmp; 191 192 if (socket_id == SPDK_ENV_SOCKET_ID_ANY) { 193 socket_id = SOCKET_ID_ANY; 194 } 195 196 /* No more than half of all elements can be in cache */ 197 tmp = (count / 2) / rte_lcore_count(); 198 if (cache_size > tmp) { 199 cache_size = tmp; 200 } 201 202 if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE) { 203 cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; 204 } 205 206 mp = rte_mempool_create(name, count, ele_size, cache_size, 207 0, NULL, NULL, (rte_mempool_obj_cb_t *)obj_init, obj_init_arg, 208 socket_id, MEMPOOL_F_NO_PHYS_CONTIG); 209 210 return (struct spdk_mempool *)mp; 211 } 212 213 214 struct spdk_mempool * 215 spdk_mempool_create(const char *name, size_t count, 216 size_t ele_size, size_t cache_size, int socket_id) 217 { 218 return spdk_mempool_create_ctor(name, count, ele_size, cache_size, socket_id, 219 NULL, NULL); 220 } 221 222 char * 223 spdk_mempool_get_name(struct spdk_mempool *mp) 224 { 225 return ((struct rte_mempool *)mp)->name; 226 } 227 228 void 229 spdk_mempool_free(struct spdk_mempool *mp) 230 { 231 #if RTE_VERSION >= RTE_VERSION_NUM(16, 7, 0, 1) 232 rte_mempool_free((struct rte_mempool *)mp); 233 #endif 234 } 235 236 void * 237 spdk_mempool_get(struct spdk_mempool *mp) 238 { 239 void *ele = NULL; 240 int rc; 241 242 rc = rte_mempool_get((struct rte_mempool *)mp, &ele); 243 if (rc != 0) { 244 return NULL; 245 } 246 return ele; 247 } 248 249 int 250 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 251 { 252 return rte_mempool_get_bulk((struct rte_mempool *)mp, ele_arr, count); 253 } 254 255 void 256 spdk_mempool_put(struct spdk_mempool *mp, void *ele) 257 { 258 rte_mempool_put((struct rte_mempool *)mp, ele); 259 } 260 261 void 262 spdk_mempool_put_bulk(struct spdk_mempool *mp, void *const *ele_arr, size_t count) 263 { 264 rte_mempool_put_bulk((struct rte_mempool *)mp, ele_arr, count); 265 } 266 267 size_t 268 spdk_mempool_count(const struct spdk_mempool *pool) 269 { 270 #if RTE_VERSION < RTE_VERSION_NUM(16, 7, 0, 1) 271 return rte_mempool_count((struct rte_mempool *)pool); 272 #else 273 return rte_mempool_avail_count((struct rte_mempool *)pool); 274 #endif 275 } 276 277 bool 278 spdk_process_is_primary(void) 279 { 280 return (rte_eal_process_type() == RTE_PROC_PRIMARY); 281 } 282 283 uint64_t spdk_get_ticks(void) 284 { 285 return rte_get_timer_cycles(); 286 } 287 288 uint64_t spdk_get_ticks_hz(void) 289 { 290 return rte_get_timer_hz(); 291 } 292 293 void spdk_delay_us(unsigned int us) 294 { 295 rte_delay_us(us); 296 } 297 298 void 299 spdk_unaffinitize_thread(void) 300 { 301 rte_cpuset_t new_cpuset; 302 long num_cores, i; 303 304 CPU_ZERO(&new_cpuset); 305 306 num_cores = sysconf(_SC_NPROCESSORS_CONF); 307 308 /* Create a mask containing all CPUs */ 309 for (i = 0; i < num_cores; i++) { 310 CPU_SET(i, &new_cpuset); 311 } 312 313 rte_thread_set_affinity(&new_cpuset); 314 } 315 316 void * 317 spdk_call_unaffinitized(void *cb(void *arg), void *arg) 318 { 319 rte_cpuset_t orig_cpuset; 320 void *ret; 321 322 if (cb == NULL) { 323 return NULL; 324 } 325 326 rte_thread_get_affinity(&orig_cpuset); 327 328 spdk_unaffinitize_thread(); 329 330 ret = cb(arg); 331 332 rte_thread_set_affinity(&orig_cpuset); 333 334 return ret; 335 } 336 337 struct spdk_ring * 338 spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id) 339 { 340 char ring_name[64]; 341 static uint32_t ring_num = 0; 342 unsigned flags = 0; 343 344 switch (type) { 345 case SPDK_RING_TYPE_SP_SC: 346 flags = RING_F_SP_ENQ | RING_F_SC_DEQ; 347 break; 348 case SPDK_RING_TYPE_MP_SC: 349 flags = RING_F_SC_DEQ; 350 break; 351 default: 352 return NULL; 353 } 354 355 snprintf(ring_name, sizeof(ring_name), "ring_%u_%d", 356 __sync_fetch_and_add(&ring_num, 1), getpid()); 357 358 return (struct spdk_ring *)rte_ring_create(ring_name, count, socket_id, flags); 359 } 360 361 void 362 spdk_ring_free(struct spdk_ring *ring) 363 { 364 rte_ring_free((struct rte_ring *)ring); 365 } 366 367 size_t 368 spdk_ring_count(struct spdk_ring *ring) 369 { 370 return rte_ring_count((struct rte_ring *)ring); 371 } 372 373 size_t 374 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count) 375 { 376 int rc; 377 #if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0) 378 rc = rte_ring_mp_enqueue_bulk((struct rte_ring *)ring, objs, count); 379 if (rc == 0) { 380 return count; 381 } 382 383 return 0; 384 #else 385 rc = rte_ring_mp_enqueue_bulk((struct rte_ring *)ring, objs, count, NULL); 386 return rc; 387 #endif 388 } 389 390 size_t 391 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count) 392 { 393 #if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0) 394 return rte_ring_sc_dequeue_burst((struct rte_ring *)ring, objs, count); 395 #else 396 return rte_ring_sc_dequeue_burst((struct rte_ring *)ring, objs, count, NULL); 397 #endif 398 } 399