1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2010-2015 Intel Corporation 4 * Copyright (c) 2007,2008 Kip Macy kmacy@freebsd.org 5 * All rights reserved. 6 * Derived from FreeBSD's bufring.h 7 * Used as BSD-3 Licensed with permission from Kip Macy. 8 */ 9 10 #include <stdalign.h> 11 #include <stdio.h> 12 #include <string.h> 13 #include <stdint.h> 14 #include <inttypes.h> 15 #include <errno.h> 16 #include <sys/queue.h> 17 18 #include <rte_common.h> 19 #include <rte_log.h> 20 #include <rte_memzone.h> 21 #include <rte_malloc.h> 22 #include <rte_eal_memconfig.h> 23 #include <rte_errno.h> 24 #include <rte_string_fns.h> 25 #include <rte_tailq.h> 26 27 #include "rte_ring.h" 28 #include "rte_ring_elem.h" 29 30 RTE_LOG_REGISTER_DEFAULT(ring_logtype, INFO); 31 #define RTE_LOGTYPE_RING ring_logtype 32 #define RING_LOG(level, ...) \ 33 RTE_LOG_LINE(level, RING, "" __VA_ARGS__) 34 35 TAILQ_HEAD(rte_ring_list, rte_tailq_entry); 36 37 static struct rte_tailq_elem rte_ring_tailq = { 38 .name = RTE_TAILQ_RING_NAME, 39 }; 40 EAL_REGISTER_TAILQ(rte_ring_tailq) 41 42 /* mask of all valid flag values to ring_create() */ 43 #define RING_F_MASK (RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ | \ 44 RING_F_MP_RTS_ENQ | RING_F_MC_RTS_DEQ | \ 45 RING_F_MP_HTS_ENQ | RING_F_MC_HTS_DEQ) 46 47 /* true if x is a power of 2 */ 48 #define POWEROF2(x) ((((x)-1) & (x)) == 0) 49 50 /* by default set head/tail distance as 1/8 of ring capacity */ 51 #define HTD_MAX_DEF 8 52 53 /* return the size of memory occupied by a ring */ 54 ssize_t 55 rte_ring_get_memsize_elem(unsigned int esize, unsigned int count) 56 { 57 ssize_t sz; 58 59 /* Check if element size is a multiple of 4B */ 60 if (esize % 4 != 0) { 61 RING_LOG(ERR, "element size is not a multiple of 4"); 62 63 return -EINVAL; 64 } 65 66 /* count must be a power of 2 */ 67 if ((!POWEROF2(count)) || (count > RTE_RING_SZ_MASK )) { 68 RING_LOG(ERR, 69 "Requested number of elements is invalid, must be power of 2, and not exceed %u", 70 RTE_RING_SZ_MASK); 71 72 return -EINVAL; 73 } 74 75 sz = sizeof(struct rte_ring) + (ssize_t)count * esize; 76 sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE); 77 return sz; 78 } 79 80 /* return the size of memory occupied by a ring */ 81 ssize_t 82 rte_ring_get_memsize(unsigned int count) 83 { 84 return rte_ring_get_memsize_elem(sizeof(void *), count); 85 } 86 87 /* 88 * internal helper function to reset prod/cons head-tail values. 89 */ 90 static void 91 reset_headtail(void *p) 92 { 93 struct rte_ring_headtail *ht; 94 struct rte_ring_hts_headtail *ht_hts; 95 struct rte_ring_rts_headtail *ht_rts; 96 97 ht = p; 98 ht_hts = p; 99 ht_rts = p; 100 101 switch (ht->sync_type) { 102 case RTE_RING_SYNC_MT: 103 case RTE_RING_SYNC_ST: 104 ht->head = 0; 105 ht->tail = 0; 106 break; 107 case RTE_RING_SYNC_MT_RTS: 108 ht_rts->head.raw = 0; 109 ht_rts->tail.raw = 0; 110 break; 111 case RTE_RING_SYNC_MT_HTS: 112 ht_hts->ht.raw = 0; 113 break; 114 default: 115 /* unknown sync mode */ 116 RTE_ASSERT(0); 117 } 118 } 119 120 void 121 rte_ring_reset(struct rte_ring *r) 122 { 123 reset_headtail(&r->prod); 124 reset_headtail(&r->cons); 125 } 126 127 /* 128 * helper function, calculates sync_type values for prod and cons 129 * based on input flags. Returns zero at success or negative 130 * errno value otherwise. 131 */ 132 static int 133 get_sync_type(uint32_t flags, enum rte_ring_sync_type *prod_st, 134 enum rte_ring_sync_type *cons_st) 135 { 136 static const uint32_t prod_st_flags = 137 (RING_F_SP_ENQ | RING_F_MP_RTS_ENQ | RING_F_MP_HTS_ENQ); 138 static const uint32_t cons_st_flags = 139 (RING_F_SC_DEQ | RING_F_MC_RTS_DEQ | RING_F_MC_HTS_DEQ); 140 141 switch (flags & prod_st_flags) { 142 case 0: 143 *prod_st = RTE_RING_SYNC_MT; 144 break; 145 case RING_F_SP_ENQ: 146 *prod_st = RTE_RING_SYNC_ST; 147 break; 148 case RING_F_MP_RTS_ENQ: 149 *prod_st = RTE_RING_SYNC_MT_RTS; 150 break; 151 case RING_F_MP_HTS_ENQ: 152 *prod_st = RTE_RING_SYNC_MT_HTS; 153 break; 154 default: 155 return -EINVAL; 156 } 157 158 switch (flags & cons_st_flags) { 159 case 0: 160 *cons_st = RTE_RING_SYNC_MT; 161 break; 162 case RING_F_SC_DEQ: 163 *cons_st = RTE_RING_SYNC_ST; 164 break; 165 case RING_F_MC_RTS_DEQ: 166 *cons_st = RTE_RING_SYNC_MT_RTS; 167 break; 168 case RING_F_MC_HTS_DEQ: 169 *cons_st = RTE_RING_SYNC_MT_HTS; 170 break; 171 default: 172 return -EINVAL; 173 } 174 175 return 0; 176 } 177 178 int 179 rte_ring_init(struct rte_ring *r, const char *name, unsigned int count, 180 unsigned int flags) 181 { 182 int ret; 183 184 /* compilation-time checks */ 185 RTE_BUILD_BUG_ON((sizeof(struct rte_ring) & 186 RTE_CACHE_LINE_MASK) != 0); 187 RTE_BUILD_BUG_ON((offsetof(struct rte_ring, cons) & 188 RTE_CACHE_LINE_MASK) != 0); 189 RTE_BUILD_BUG_ON((offsetof(struct rte_ring, prod) & 190 RTE_CACHE_LINE_MASK) != 0); 191 192 RTE_BUILD_BUG_ON(offsetof(struct rte_ring_headtail, sync_type) != 193 offsetof(struct rte_ring_hts_headtail, sync_type)); 194 RTE_BUILD_BUG_ON(offsetof(struct rte_ring_headtail, tail) != 195 offsetof(struct rte_ring_hts_headtail, ht.pos.tail)); 196 197 RTE_BUILD_BUG_ON(offsetof(struct rte_ring_headtail, sync_type) != 198 offsetof(struct rte_ring_rts_headtail, sync_type)); 199 RTE_BUILD_BUG_ON(offsetof(struct rte_ring_headtail, tail) != 200 offsetof(struct rte_ring_rts_headtail, tail.val.pos)); 201 202 /* future proof flags, only allow supported values */ 203 if (flags & ~RING_F_MASK) { 204 RING_LOG(ERR, 205 "Unsupported flags requested %#x", flags); 206 return -EINVAL; 207 } 208 209 /* init the ring structure */ 210 memset(r, 0, sizeof(*r)); 211 ret = strlcpy(r->name, name, sizeof(r->name)); 212 if (ret < 0 || ret >= (int)sizeof(r->name)) 213 return -ENAMETOOLONG; 214 r->flags = flags; 215 ret = get_sync_type(flags, &r->prod.sync_type, &r->cons.sync_type); 216 if (ret != 0) 217 return ret; 218 219 if (flags & RING_F_EXACT_SZ) { 220 r->size = rte_align32pow2(count + 1); 221 r->mask = r->size - 1; 222 r->capacity = count; 223 } else { 224 if ((!POWEROF2(count)) || (count > RTE_RING_SZ_MASK)) { 225 RING_LOG(ERR, 226 "Requested size is invalid, must be power of 2, and not exceed the size limit %u", 227 RTE_RING_SZ_MASK); 228 return -EINVAL; 229 } 230 r->size = count; 231 r->mask = count - 1; 232 r->capacity = r->mask; 233 } 234 235 /* set default values for head-tail distance */ 236 if (flags & RING_F_MP_RTS_ENQ) 237 rte_ring_set_prod_htd_max(r, r->capacity / HTD_MAX_DEF); 238 if (flags & RING_F_MC_RTS_DEQ) 239 rte_ring_set_cons_htd_max(r, r->capacity / HTD_MAX_DEF); 240 241 return 0; 242 } 243 244 /* create the ring for a given element size */ 245 struct rte_ring * 246 rte_ring_create_elem(const char *name, unsigned int esize, unsigned int count, 247 int socket_id, unsigned int flags) 248 { 249 char mz_name[RTE_MEMZONE_NAMESIZE]; 250 struct rte_ring *r; 251 struct rte_tailq_entry *te; 252 const struct rte_memzone *mz; 253 ssize_t ring_size; 254 int mz_flags = 0; 255 struct rte_ring_list* ring_list = NULL; 256 const unsigned int requested_count = count; 257 int ret; 258 259 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 260 261 /* for an exact size ring, round up from count to a power of two */ 262 if (flags & RING_F_EXACT_SZ) 263 count = rte_align32pow2(count + 1); 264 265 ring_size = rte_ring_get_memsize_elem(esize, count); 266 if (ring_size < 0) { 267 rte_errno = -ring_size; 268 return NULL; 269 } 270 271 ret = snprintf(mz_name, sizeof(mz_name), "%s%s", 272 RTE_RING_MZ_PREFIX, name); 273 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 274 rte_errno = ENAMETOOLONG; 275 return NULL; 276 } 277 278 te = rte_zmalloc("RING_TAILQ_ENTRY", sizeof(*te), 0); 279 if (te == NULL) { 280 RING_LOG(ERR, "Cannot reserve memory for tailq"); 281 rte_errno = ENOMEM; 282 return NULL; 283 } 284 285 rte_mcfg_tailq_write_lock(); 286 287 /* reserve a memory zone for this ring. If we can't get rte_config or 288 * we are secondary process, the memzone_reserve function will set 289 * rte_errno for us appropriately - hence no check in this function 290 */ 291 mz = rte_memzone_reserve_aligned(mz_name, ring_size, socket_id, 292 mz_flags, alignof(typeof(*r))); 293 if (mz != NULL) { 294 r = mz->addr; 295 /* no need to check return value here, we already checked the 296 * arguments above */ 297 rte_ring_init(r, name, requested_count, flags); 298 299 te->data = (void *) r; 300 r->memzone = mz; 301 302 TAILQ_INSERT_TAIL(ring_list, te, next); 303 } else { 304 r = NULL; 305 RING_LOG(ERR, "Cannot reserve memory"); 306 rte_free(te); 307 } 308 rte_mcfg_tailq_write_unlock(); 309 310 return r; 311 } 312 313 /* create the ring */ 314 struct rte_ring * 315 rte_ring_create(const char *name, unsigned int count, int socket_id, 316 unsigned int flags) 317 { 318 return rte_ring_create_elem(name, sizeof(void *), count, socket_id, 319 flags); 320 } 321 322 /* free the ring */ 323 void 324 rte_ring_free(struct rte_ring *r) 325 { 326 struct rte_ring_list *ring_list = NULL; 327 struct rte_tailq_entry *te; 328 329 if (r == NULL) 330 return; 331 332 /* 333 * Ring was not created with rte_ring_create, 334 * therefore, there is no memzone to free. 335 */ 336 if (r->memzone == NULL) { 337 RING_LOG(ERR, 338 "Cannot free ring, not created with rte_ring_create()"); 339 return; 340 } 341 342 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 343 rte_mcfg_tailq_write_lock(); 344 345 /* find out tailq entry */ 346 TAILQ_FOREACH(te, ring_list, next) { 347 if (te->data == (void *) r) 348 break; 349 } 350 351 if (te == NULL) { 352 rte_mcfg_tailq_write_unlock(); 353 return; 354 } 355 356 TAILQ_REMOVE(ring_list, te, next); 357 358 rte_mcfg_tailq_write_unlock(); 359 360 if (rte_memzone_free(r->memzone) != 0) 361 RING_LOG(ERR, "Cannot free memory"); 362 363 rte_free(te); 364 } 365 366 /* dump the status of the ring on the console */ 367 void 368 rte_ring_dump(FILE *f, const struct rte_ring *r) 369 { 370 fprintf(f, "ring <%s>@%p\n", r->name, r); 371 fprintf(f, " flags=%x\n", r->flags); 372 fprintf(f, " size=%"PRIu32"\n", r->size); 373 fprintf(f, " capacity=%"PRIu32"\n", r->capacity); 374 fprintf(f, " ct=%"PRIu32"\n", r->cons.tail); 375 fprintf(f, " ch=%"PRIu32"\n", r->cons.head); 376 fprintf(f, " pt=%"PRIu32"\n", r->prod.tail); 377 fprintf(f, " ph=%"PRIu32"\n", r->prod.head); 378 fprintf(f, " used=%u\n", rte_ring_count(r)); 379 fprintf(f, " avail=%u\n", rte_ring_free_count(r)); 380 } 381 382 /* dump the status of all rings on the console */ 383 void 384 rte_ring_list_dump(FILE *f) 385 { 386 const struct rte_tailq_entry *te; 387 struct rte_ring_list *ring_list; 388 389 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 390 391 rte_mcfg_tailq_read_lock(); 392 393 TAILQ_FOREACH(te, ring_list, next) { 394 rte_ring_dump(f, (struct rte_ring *) te->data); 395 } 396 397 rte_mcfg_tailq_read_unlock(); 398 } 399 400 /* search a ring from its name */ 401 struct rte_ring * 402 rte_ring_lookup(const char *name) 403 { 404 struct rte_tailq_entry *te; 405 struct rte_ring *r = NULL; 406 struct rte_ring_list *ring_list; 407 408 ring_list = RTE_TAILQ_CAST(rte_ring_tailq.head, rte_ring_list); 409 410 rte_mcfg_tailq_read_lock(); 411 412 TAILQ_FOREACH(te, ring_list, next) { 413 r = (struct rte_ring *) te->data; 414 if (strncmp(name, r->name, RTE_RING_NAMESIZE) == 0) 415 break; 416 } 417 418 rte_mcfg_tailq_read_unlock(); 419 420 if (te == NULL) { 421 rte_errno = ENOENT; 422 return NULL; 423 } 424 425 return r; 426 } 427