1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2010-2015 Intel Corporation 4 * Copyright (c) 2007,2008 Kip Macy kmacy@freebsd.org 5 * All rights reserved. 6 * Derived from FreeBSD's bufring.h 7 * Used as BSD-3 Licensed with permission from Kip Macy. 8 */ 9 10 #include <stdio.h> 11 #include <stdarg.h> 12 #include <string.h> 13 #include <stdint.h> 14 #include <inttypes.h> 15 #include <errno.h> 16 #include <sys/queue.h> 17 18 #include <rte_common.h> 19 #include <rte_log.h> 20 #include <rte_memory.h> 21 #include <rte_memzone.h> 22 #include <rte_malloc.h> 23 #include <rte_launch.h> 24 #include <rte_eal.h> 25 #include <rte_eal_memconfig.h> 26 #include <rte_atomic.h> 27 #include <rte_per_lcore.h> 28 #include <rte_lcore.h> 29 #include <rte_branch_prediction.h> 30 #include <rte_errno.h> 31 #include <rte_string_fns.h> 32 #include <rte_spinlock.h> 33 #include <rte_tailq.h> 34 35 #include "rte_ring.h" 36 #include "rte_ring_elem.h" 37 38 TAILQ_HEAD(rte_ring_list, rte_tailq_entry); 39 40 static struct rte_tailq_elem rte_ring_tailq = { 41 .name = RTE_TAILQ_RING_NAME, 42 }; 43 EAL_REGISTER_TAILQ(rte_ring_tailq) 44 45 /* mask of all valid flag values to ring_create() */ 46 #define RING_F_MASK (RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ | \ 47 RING_F_MP_RTS_ENQ | RING_F_MC_RTS_DEQ | \ 48 RING_F_MP_HTS_ENQ | RING_F_MC_HTS_DEQ) 49 50 /* true if x is a power of 2 */ 51 #define POWEROF2(x) ((((x)-1) & (x)) == 0) 52 53 /* by default set head/tail distance as 1/8 of ring capacity */ 54 #define HTD_MAX_DEF 8 55 56 /* return the size of memory occupied by a ring */ 57 ssize_t 58 rte_ring_get_memsize_elem(unsigned int esize, unsigned int count) 59 { 60 ssize_t sz; 61 62 /* Check if element size is a multiple of 4B */ 63 if (esize % 4 != 0) { 64 RTE_LOG(ERR, RING, "element size is not a multiple of 4\n"); 65 66 return -EINVAL; 67 } 68 69 /* count must be a power of 2 */ 70 if ((!POWEROF2(count)) || (count > RTE_RING_SZ_MASK )) { 71 RTE_LOG(ERR, RING, 72 "Requested number of elements is invalid, must be power of 2, and not exceed %u\n", 73 RTE_RING_SZ_MASK); 74 75 return -EINVAL; 76 } 77 78 sz = sizeof(struct rte_ring) + count * esize; 79 sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE); 80 return sz; 81 } 82 83 /* return the size of memory occupied by a ring */ 84 ssize_t 85 rte_ring_get_memsize(unsigned int count) 86 { 87 return rte_ring_get_memsize_elem(sizeof(void *), count); 88 } 89 90 /* 91 * internal helper function to reset prod/cons head-tail values. 92 */ 93 static void 94 reset_headtail(void *p) 95 { 96 struct rte_ring_headtail *ht; 97 struct rte_ring_hts_headtail *ht_hts; 98 struct rte_ring_rts_headtail *ht_rts; 99 100 ht = p; 101 ht_hts = p; 102 ht_rts = p; 103 104 switch (ht->sync_type) { 105 case RTE_RING_SYNC_MT: 106 case RTE_RING_SYNC_ST: 107 ht->head = 0; 108 ht->tail = 0; 109 break; 110 case RTE_RING_SYNC_MT_RTS: 111 ht_rts->head.raw = 0; 112 ht_rts->tail.raw = 0; 113 break; 114 case RTE_RING_SYNC_MT_HTS: 115 ht_hts->ht.raw = 0; 116 break; 117 default: 118 /* unknown sync mode */ 119 RTE_ASSERT(0); 120 } 121 } 122 123 void 124 rte_ring_reset(struct rte_ring *r) 125 { 126 reset_headtail(&r->prod); 127 reset_headtail(&r->cons); 128 } 129 130 /* 131 * helper function, calculates sync_type values for prod and cons 132 * based on input flags. Returns zero at success or negative 133 * errno value otherwise. 134 */ 135 static int 136 get_sync_type(uint32_t flags, enum rte_ring_sync_type *prod_st, 137 enum rte_ring_sync_type *cons_st) 138 { 139 static const uint32_t prod_st_flags = 140 (RING_F_SP_ENQ | RING_F_MP_RTS_ENQ | RING_F_MP_HTS_ENQ); 141 static const uint32_t cons_st_flags = 142 (RING_F_SC_DEQ | RING_F_MC_RTS_DEQ | RING_F_MC_HTS_DEQ); 143 144 switch (flags & prod_st_flags) { 145 case 0: 146 *prod_st = RTE_RING_SYNC_MT; 147 break; 148 case RING_F_SP_ENQ: 149 *prod_st = RTE_RING_SYNC_ST; 150 break; 151 case RING_F_MP_RTS_ENQ: 152 *prod_st = RTE_RING_SYNC_MT_RTS; 153 break; 154 case RING_F_MP_HTS_ENQ: 155 *prod_st = RTE_RING_SYNC_MT_HTS; 156 break; 157 default: 158 return -EINVAL; 159 } 160 161 switch (flags & cons_st_flags) { 162 case 0: 163 *cons_st = RTE_RING_SYNC_MT; 164 break; 165 case RING_F_SC_DEQ: 166 *cons_st = RTE_RING_SYNC_ST; 167 break; 168 case RING_F_MC_RTS_DEQ: 169 *cons_st = RTE_RING_SYNC_MT_RTS; 170 break; 171 case RING_F_MC_HTS_DEQ: 172 *cons_st = RTE_RING_SYNC_MT_HTS; 173 break; 174 default: 175 return -EINVAL; 176 } 177 178 return 0; 179 } 180 181 int 182 rte_ring_init(struct rte_ring *r, const char *name, unsigned int count, 183 unsigned int flags) 184 { 185 int ret; 186 187 /* compilation-time checks */ 188 RTE_BUILD_BUG_ON((sizeof(struct rte_ring) & 189 RTE_CACHE_LINE_MASK) != 0); 190 RTE_BUILD_BUG_ON((offsetof(struct rte_ring, cons) & 191 RTE_CACHE_LINE_MASK) != 0); 192 RTE_BUILD_BUG_ON((offsetof(struct rte_ring, prod) & 193 RTE_CACHE_LINE_MASK) != 0); 194 195 RTE_BUILD_BUG_ON(offsetof(struct rte_ring_headtail, sync_type) != 196 offsetof(struct rte_ring_hts_headtail, sync_type)); 197 RTE_BUILD_BUG_ON(offsetof(struct rte_ring_headtail, tail) != 198 offsetof(struct rte_ring_hts_headtail, ht.pos.tail)); 199 200 RTE_BUILD_BUG_ON(offsetof(struct rte_ring_headtail, sync_type) != 201 offsetof(struct rte_ring_rts_headtail, sync_type)); 202 RTE_BUILD_BUG_ON(offsetof(struct rte_ring_headtail, tail) != 203 offsetof(struct rte_ring_rts_headtail, tail.val.pos)); 204 205 /* future proof flags, only allow supported values */ 206 if (flags & ~RING_F_MASK) { 207 RTE_LOG(ERR, RING, 208 "Unsupported flags requested %#x\n", flags); 209 return -EINVAL; 210 } 211 212 /* init the ring structure */ 213 memset(r, 0, sizeof(*r)); 214 ret = strlcpy(r->name, name, sizeof(r->name)); 215 if (ret < 0 || ret >= (int)sizeof(r->name)) 216 return -ENAMETOOLONG; 217 r->flags = flags; 218 ret = get_sync_type(flags, &r->prod.sync_type, &r->cons.sync_type); 219 if (ret != 0) 220 return ret; 221 222 if (flags & RING_F_EXACT_SZ) { 223 r->size = rte_align32pow2(count + 1); 224 r->mask = r->size - 1; 225 r->capacity = count; 226 } else { 227 if ((!POWEROF2(count)) || (count > RTE_RING_SZ_MASK)) { 228 RTE_LOG(ERR, RING, 229 "Requested size is invalid, must be power of 2, and not exceed the size limit %u\n", 230 RTE_RING_SZ_MASK); 231 return -EINVAL; 232 } 233 r->size = count; 234 r->mask = count - 1; 235 r->capacity = r->mask; 236 } 237 238 /* set default values for head-tail distance */ 239 if (flags & RING_F_MP_RTS_ENQ) 240 rte_ring_set_prod_htd_max(r, r->capacity / HTD_MAX_DEF); 241 if (flags & RING_F_MC_RTS_DEQ) 242 rte_ring_set_cons_htd_max(r, r->capacity / HTD_MAX_DEF); 243 244 return 0; 245 } 246 247 /* create the ring for a given element size */ 248 struct rte_ring * 249 rte_ring_create_elem(const char *name, unsigned int esize, unsigned int count, 250 int socket_id, unsigned int flags) 251 { 252 char mz_name[RTE_MEMZONE_NAMESIZE]; 253 struct rte_ring *r; 254 struct rte_tailq_entry *te; 255 const struct rte_memzone *mz; 256 ssize_t ring_size; 257 int mz_flags = 0; 258 struct rte_ring_list* ring_list = NULL; 259 const unsigned int requested_count = count; 260 int ret; 261 262 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 263 264 /* for an exact size ring, round up from count to a power of two */ 265 if (flags & RING_F_EXACT_SZ) 266 count = rte_align32pow2(count + 1); 267 268 ring_size = rte_ring_get_memsize_elem(esize, count); 269 if (ring_size < 0) { 270 rte_errno = ring_size; 271 return NULL; 272 } 273 274 ret = snprintf(mz_name, sizeof(mz_name), "%s%s", 275 RTE_RING_MZ_PREFIX, name); 276 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 277 rte_errno = ENAMETOOLONG; 278 return NULL; 279 } 280 281 te = rte_zmalloc("RING_TAILQ_ENTRY", sizeof(*te), 0); 282 if (te == NULL) { 283 RTE_LOG(ERR, RING, "Cannot reserve memory for tailq\n"); 284 rte_errno = ENOMEM; 285 return NULL; 286 } 287 288 rte_mcfg_tailq_write_lock(); 289 290 /* reserve a memory zone for this ring. If we can't get rte_config or 291 * we are secondary process, the memzone_reserve function will set 292 * rte_errno for us appropriately - hence no check in this this function */ 293 mz = rte_memzone_reserve_aligned(mz_name, ring_size, socket_id, 294 mz_flags, __alignof__(*r)); 295 if (mz != NULL) { 296 r = mz->addr; 297 /* no need to check return value here, we already checked the 298 * arguments above */ 299 rte_ring_init(r, name, requested_count, flags); 300 301 te->data = (void *) r; 302 r->memzone = mz; 303 304 TAILQ_INSERT_TAIL(ring_list, te, next); 305 } else { 306 r = NULL; 307 RTE_LOG(ERR, RING, "Cannot reserve memory\n"); 308 rte_free(te); 309 } 310 rte_mcfg_tailq_write_unlock(); 311 312 return r; 313 } 314 315 /* create the ring */ 316 struct rte_ring * 317 rte_ring_create(const char *name, unsigned int count, int socket_id, 318 unsigned int flags) 319 { 320 return rte_ring_create_elem(name, sizeof(void *), count, socket_id, 321 flags); 322 } 323 324 /* free the ring */ 325 void 326 rte_ring_free(struct rte_ring *r) 327 { 328 struct rte_ring_list *ring_list = NULL; 329 struct rte_tailq_entry *te; 330 331 if (r == NULL) 332 return; 333 334 /* 335 * Ring was not created with rte_ring_create, 336 * therefore, there is no memzone to free. 337 */ 338 if (r->memzone == NULL) { 339 RTE_LOG(ERR, RING, 340 "Cannot free ring, not created with rte_ring_create()\n"); 341 return; 342 } 343 344 if (rte_memzone_free(r->memzone) != 0) { 345 RTE_LOG(ERR, RING, "Cannot free memory\n"); 346 return; 347 } 348 349 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 350 rte_mcfg_tailq_write_lock(); 351 352 /* find out tailq entry */ 353 TAILQ_FOREACH(te, ring_list, next) { 354 if (te->data == (void *) r) 355 break; 356 } 357 358 if (te == NULL) { 359 rte_mcfg_tailq_write_unlock(); 360 return; 361 } 362 363 TAILQ_REMOVE(ring_list, te, next); 364 365 rte_mcfg_tailq_write_unlock(); 366 367 rte_free(te); 368 } 369 370 /* dump the status of the ring on the console */ 371 void 372 rte_ring_dump(FILE *f, const struct rte_ring *r) 373 { 374 fprintf(f, "ring <%s>@%p\n", r->name, r); 375 fprintf(f, " flags=%x\n", r->flags); 376 fprintf(f, " size=%"PRIu32"\n", r->size); 377 fprintf(f, " capacity=%"PRIu32"\n", r->capacity); 378 fprintf(f, " ct=%"PRIu32"\n", r->cons.tail); 379 fprintf(f, " ch=%"PRIu32"\n", r->cons.head); 380 fprintf(f, " pt=%"PRIu32"\n", r->prod.tail); 381 fprintf(f, " ph=%"PRIu32"\n", r->prod.head); 382 fprintf(f, " used=%u\n", rte_ring_count(r)); 383 fprintf(f, " avail=%u\n", rte_ring_free_count(r)); 384 } 385 386 /* dump the status of all rings on the console */ 387 void 388 rte_ring_list_dump(FILE *f) 389 { 390 const struct rte_tailq_entry *te; 391 struct rte_ring_list *ring_list; 392 393 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 394 395 rte_mcfg_tailq_read_lock(); 396 397 TAILQ_FOREACH(te, ring_list, next) { 398 rte_ring_dump(f, (struct rte_ring *) te->data); 399 } 400 401 rte_mcfg_tailq_read_unlock(); 402 } 403 404 /* search a ring from its name */ 405 struct rte_ring * 406 rte_ring_lookup(const char *name) 407 { 408 struct rte_tailq_entry *te; 409 struct rte_ring *r = NULL; 410 struct rte_ring_list *ring_list; 411 412 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 413 414 rte_mcfg_tailq_read_lock(); 415 416 TAILQ_FOREACH(te, ring_list, next) { 417 r = (struct rte_ring *) te->data; 418 if (strncmp(name, r->name, RTE_RING_NAMESIZE) == 0) 419 break; 420 } 421 422 rte_mcfg_tailq_read_unlock(); 423 424 if (te == NULL) { 425 rte_errno = ENOENT; 426 return NULL; 427 } 428 429 return r; 430 } 431