1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2010-2015 Intel Corporation 4 * Copyright (c) 2007,2008 Kip Macy kmacy@freebsd.org 5 * All rights reserved. 6 * Derived from FreeBSD's bufring.h 7 * Used as BSD-3 Licensed with permission from Kip Macy. 8 */ 9 10 #include <stdio.h> 11 #include <string.h> 12 #include <stdint.h> 13 #include <inttypes.h> 14 #include <errno.h> 15 #include <sys/queue.h> 16 17 #include <rte_common.h> 18 #include <rte_log.h> 19 #include <rte_memzone.h> 20 #include <rte_malloc.h> 21 #include <rte_eal_memconfig.h> 22 #include <rte_errno.h> 23 #include <rte_string_fns.h> 24 #include <rte_tailq.h> 25 26 #include "rte_ring.h" 27 #include "rte_ring_elem.h" 28 29 TAILQ_HEAD(rte_ring_list, rte_tailq_entry); 30 31 static struct rte_tailq_elem rte_ring_tailq = { 32 .name = RTE_TAILQ_RING_NAME, 33 }; 34 EAL_REGISTER_TAILQ(rte_ring_tailq) 35 36 /* mask of all valid flag values to ring_create() */ 37 #define RING_F_MASK (RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ | \ 38 RING_F_MP_RTS_ENQ | RING_F_MC_RTS_DEQ | \ 39 RING_F_MP_HTS_ENQ | RING_F_MC_HTS_DEQ) 40 41 /* true if x is a power of 2 */ 42 #define POWEROF2(x) ((((x)-1) & (x)) == 0) 43 44 /* by default set head/tail distance as 1/8 of ring capacity */ 45 #define HTD_MAX_DEF 8 46 47 /* return the size of memory occupied by a ring */ 48 ssize_t 49 rte_ring_get_memsize_elem(unsigned int esize, unsigned int count) 50 { 51 ssize_t sz; 52 53 /* Check if element size is a multiple of 4B */ 54 if (esize % 4 != 0) { 55 RTE_LOG(ERR, RING, "element size is not a multiple of 4\n"); 56 57 return -EINVAL; 58 } 59 60 /* count must be a power of 2 */ 61 if ((!POWEROF2(count)) || (count > RTE_RING_SZ_MASK )) { 62 RTE_LOG(ERR, RING, 63 "Requested number of elements is invalid, must be power of 2, and not exceed %u\n", 64 RTE_RING_SZ_MASK); 65 66 return -EINVAL; 67 } 68 69 sz = sizeof(struct rte_ring) + (ssize_t)count * esize; 70 sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE); 71 return sz; 72 } 73 74 /* return the size of memory occupied by a ring */ 75 ssize_t 76 rte_ring_get_memsize(unsigned int count) 77 { 78 return rte_ring_get_memsize_elem(sizeof(void *), count); 79 } 80 81 /* 82 * internal helper function to reset prod/cons head-tail values. 83 */ 84 static void 85 reset_headtail(void *p) 86 { 87 struct rte_ring_headtail *ht; 88 struct rte_ring_hts_headtail *ht_hts; 89 struct rte_ring_rts_headtail *ht_rts; 90 91 ht = p; 92 ht_hts = p; 93 ht_rts = p; 94 95 switch (ht->sync_type) { 96 case RTE_RING_SYNC_MT: 97 case RTE_RING_SYNC_ST: 98 ht->head = 0; 99 ht->tail = 0; 100 break; 101 case RTE_RING_SYNC_MT_RTS: 102 ht_rts->head.raw = 0; 103 ht_rts->tail.raw = 0; 104 break; 105 case RTE_RING_SYNC_MT_HTS: 106 ht_hts->ht.raw = 0; 107 break; 108 default: 109 /* unknown sync mode */ 110 RTE_ASSERT(0); 111 } 112 } 113 114 void 115 rte_ring_reset(struct rte_ring *r) 116 { 117 reset_headtail(&r->prod); 118 reset_headtail(&r->cons); 119 } 120 121 /* 122 * helper function, calculates sync_type values for prod and cons 123 * based on input flags. Returns zero at success or negative 124 * errno value otherwise. 125 */ 126 static int 127 get_sync_type(uint32_t flags, enum rte_ring_sync_type *prod_st, 128 enum rte_ring_sync_type *cons_st) 129 { 130 static const uint32_t prod_st_flags = 131 (RING_F_SP_ENQ | RING_F_MP_RTS_ENQ | RING_F_MP_HTS_ENQ); 132 static const uint32_t cons_st_flags = 133 (RING_F_SC_DEQ | RING_F_MC_RTS_DEQ | RING_F_MC_HTS_DEQ); 134 135 switch (flags & prod_st_flags) { 136 case 0: 137 *prod_st = RTE_RING_SYNC_MT; 138 break; 139 case RING_F_SP_ENQ: 140 *prod_st = RTE_RING_SYNC_ST; 141 break; 142 case RING_F_MP_RTS_ENQ: 143 *prod_st = RTE_RING_SYNC_MT_RTS; 144 break; 145 case RING_F_MP_HTS_ENQ: 146 *prod_st = RTE_RING_SYNC_MT_HTS; 147 break; 148 default: 149 return -EINVAL; 150 } 151 152 switch (flags & cons_st_flags) { 153 case 0: 154 *cons_st = RTE_RING_SYNC_MT; 155 break; 156 case RING_F_SC_DEQ: 157 *cons_st = RTE_RING_SYNC_ST; 158 break; 159 case RING_F_MC_RTS_DEQ: 160 *cons_st = RTE_RING_SYNC_MT_RTS; 161 break; 162 case RING_F_MC_HTS_DEQ: 163 *cons_st = RTE_RING_SYNC_MT_HTS; 164 break; 165 default: 166 return -EINVAL; 167 } 168 169 return 0; 170 } 171 172 int 173 rte_ring_init(struct rte_ring *r, const char *name, unsigned int count, 174 unsigned int flags) 175 { 176 int ret; 177 178 /* compilation-time checks */ 179 RTE_BUILD_BUG_ON((sizeof(struct rte_ring) & 180 RTE_CACHE_LINE_MASK) != 0); 181 RTE_BUILD_BUG_ON((offsetof(struct rte_ring, cons) & 182 RTE_CACHE_LINE_MASK) != 0); 183 RTE_BUILD_BUG_ON((offsetof(struct rte_ring, prod) & 184 RTE_CACHE_LINE_MASK) != 0); 185 186 RTE_BUILD_BUG_ON(offsetof(struct rte_ring_headtail, sync_type) != 187 offsetof(struct rte_ring_hts_headtail, sync_type)); 188 RTE_BUILD_BUG_ON(offsetof(struct rte_ring_headtail, tail) != 189 offsetof(struct rte_ring_hts_headtail, ht.pos.tail)); 190 191 RTE_BUILD_BUG_ON(offsetof(struct rte_ring_headtail, sync_type) != 192 offsetof(struct rte_ring_rts_headtail, sync_type)); 193 RTE_BUILD_BUG_ON(offsetof(struct rte_ring_headtail, tail) != 194 offsetof(struct rte_ring_rts_headtail, tail.val.pos)); 195 196 /* future proof flags, only allow supported values */ 197 if (flags & ~RING_F_MASK) { 198 RTE_LOG(ERR, RING, 199 "Unsupported flags requested %#x\n", flags); 200 return -EINVAL; 201 } 202 203 /* init the ring structure */ 204 memset(r, 0, sizeof(*r)); 205 ret = strlcpy(r->name, name, sizeof(r->name)); 206 if (ret < 0 || ret >= (int)sizeof(r->name)) 207 return -ENAMETOOLONG; 208 r->flags = flags; 209 ret = get_sync_type(flags, &r->prod.sync_type, &r->cons.sync_type); 210 if (ret != 0) 211 return ret; 212 213 if (flags & RING_F_EXACT_SZ) { 214 r->size = rte_align32pow2(count + 1); 215 r->mask = r->size - 1; 216 r->capacity = count; 217 } else { 218 if ((!POWEROF2(count)) || (count > RTE_RING_SZ_MASK)) { 219 RTE_LOG(ERR, RING, 220 "Requested size is invalid, must be power of 2, and not exceed the size limit %u\n", 221 RTE_RING_SZ_MASK); 222 return -EINVAL; 223 } 224 r->size = count; 225 r->mask = count - 1; 226 r->capacity = r->mask; 227 } 228 229 /* set default values for head-tail distance */ 230 if (flags & RING_F_MP_RTS_ENQ) 231 rte_ring_set_prod_htd_max(r, r->capacity / HTD_MAX_DEF); 232 if (flags & RING_F_MC_RTS_DEQ) 233 rte_ring_set_cons_htd_max(r, r->capacity / HTD_MAX_DEF); 234 235 return 0; 236 } 237 238 /* create the ring for a given element size */ 239 struct rte_ring * 240 rte_ring_create_elem(const char *name, unsigned int esize, unsigned int count, 241 int socket_id, unsigned int flags) 242 { 243 char mz_name[RTE_MEMZONE_NAMESIZE]; 244 struct rte_ring *r; 245 struct rte_tailq_entry *te; 246 const struct rte_memzone *mz; 247 ssize_t ring_size; 248 int mz_flags = 0; 249 struct rte_ring_list* ring_list = NULL; 250 const unsigned int requested_count = count; 251 int ret; 252 253 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 254 255 /* for an exact size ring, round up from count to a power of two */ 256 if (flags & RING_F_EXACT_SZ) 257 count = rte_align32pow2(count + 1); 258 259 ring_size = rte_ring_get_memsize_elem(esize, count); 260 if (ring_size < 0) { 261 rte_errno = -ring_size; 262 return NULL; 263 } 264 265 ret = snprintf(mz_name, sizeof(mz_name), "%s%s", 266 RTE_RING_MZ_PREFIX, name); 267 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 268 rte_errno = ENAMETOOLONG; 269 return NULL; 270 } 271 272 te = rte_zmalloc("RING_TAILQ_ENTRY", sizeof(*te), 0); 273 if (te == NULL) { 274 RTE_LOG(ERR, RING, "Cannot reserve memory for tailq\n"); 275 rte_errno = ENOMEM; 276 return NULL; 277 } 278 279 rte_mcfg_tailq_write_lock(); 280 281 /* reserve a memory zone for this ring. If we can't get rte_config or 282 * we are secondary process, the memzone_reserve function will set 283 * rte_errno for us appropriately - hence no check in this this function */ 284 mz = rte_memzone_reserve_aligned(mz_name, ring_size, socket_id, 285 mz_flags, __alignof__(*r)); 286 if (mz != NULL) { 287 r = mz->addr; 288 /* no need to check return value here, we already checked the 289 * arguments above */ 290 rte_ring_init(r, name, requested_count, flags); 291 292 te->data = (void *) r; 293 r->memzone = mz; 294 295 TAILQ_INSERT_TAIL(ring_list, te, next); 296 } else { 297 r = NULL; 298 RTE_LOG(ERR, RING, "Cannot reserve memory\n"); 299 rte_free(te); 300 } 301 rte_mcfg_tailq_write_unlock(); 302 303 return r; 304 } 305 306 /* create the ring */ 307 struct rte_ring * 308 rte_ring_create(const char *name, unsigned int count, int socket_id, 309 unsigned int flags) 310 { 311 return rte_ring_create_elem(name, sizeof(void *), count, socket_id, 312 flags); 313 } 314 315 /* free the ring */ 316 void 317 rte_ring_free(struct rte_ring *r) 318 { 319 struct rte_ring_list *ring_list = NULL; 320 struct rte_tailq_entry *te; 321 322 if (r == NULL) 323 return; 324 325 /* 326 * Ring was not created with rte_ring_create, 327 * therefore, there is no memzone to free. 328 */ 329 if (r->memzone == NULL) { 330 RTE_LOG(ERR, RING, 331 "Cannot free ring, not created with rte_ring_create()\n"); 332 return; 333 } 334 335 if (rte_memzone_free(r->memzone) != 0) { 336 RTE_LOG(ERR, RING, "Cannot free memory\n"); 337 return; 338 } 339 340 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 341 rte_mcfg_tailq_write_lock(); 342 343 /* find out tailq entry */ 344 TAILQ_FOREACH(te, ring_list, next) { 345 if (te->data == (void *) r) 346 break; 347 } 348 349 if (te == NULL) { 350 rte_mcfg_tailq_write_unlock(); 351 return; 352 } 353 354 TAILQ_REMOVE(ring_list, te, next); 355 356 rte_mcfg_tailq_write_unlock(); 357 358 rte_free(te); 359 } 360 361 /* dump the status of the ring on the console */ 362 void 363 rte_ring_dump(FILE *f, const struct rte_ring *r) 364 { 365 fprintf(f, "ring <%s>@%p\n", r->name, r); 366 fprintf(f, " flags=%x\n", r->flags); 367 fprintf(f, " size=%"PRIu32"\n", r->size); 368 fprintf(f, " capacity=%"PRIu32"\n", r->capacity); 369 fprintf(f, " ct=%"PRIu32"\n", r->cons.tail); 370 fprintf(f, " ch=%"PRIu32"\n", r->cons.head); 371 fprintf(f, " pt=%"PRIu32"\n", r->prod.tail); 372 fprintf(f, " ph=%"PRIu32"\n", r->prod.head); 373 fprintf(f, " used=%u\n", rte_ring_count(r)); 374 fprintf(f, " avail=%u\n", rte_ring_free_count(r)); 375 } 376 377 /* dump the status of all rings on the console */ 378 void 379 rte_ring_list_dump(FILE *f) 380 { 381 const struct rte_tailq_entry *te; 382 struct rte_ring_list *ring_list; 383 384 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 385 386 rte_mcfg_tailq_read_lock(); 387 388 TAILQ_FOREACH(te, ring_list, next) { 389 rte_ring_dump(f, (struct rte_ring *) te->data); 390 } 391 392 rte_mcfg_tailq_read_unlock(); 393 } 394 395 /* search a ring from its name */ 396 struct rte_ring * 397 rte_ring_lookup(const char *name) 398 { 399 struct rte_tailq_entry *te; 400 struct rte_ring *r = NULL; 401 struct rte_ring_list *ring_list; 402 403 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 404 405 rte_mcfg_tailq_read_lock(); 406 407 TAILQ_FOREACH(te, ring_list, next) { 408 r = (struct rte_ring *) te->data; 409 if (strncmp(name, r->name, RTE_RING_NAMESIZE) == 0) 410 break; 411 } 412 413 rte_mcfg_tailq_read_unlock(); 414 415 if (te == NULL) { 416 rte_errno = ENOENT; 417 return NULL; 418 } 419 420 return r; 421 } 422