1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2010-2020 Intel Corporation 4 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org 5 * All rights reserved. 6 * Derived from FreeBSD's bufring.h 7 * Used as BSD-3 Licensed with permission from Kip Macy. 8 */ 9 10 #ifndef _RTE_RING_CORE_H_ 11 #define _RTE_RING_CORE_H_ 12 13 /** 14 * @file 15 * This file contains definition of RTE ring structure itself, 16 * init flags and some related macros. 17 * For majority of DPDK entities, it is not recommended to include 18 * this file directly, use include <rte_ring.h> or <rte_ring_elem.h> 19 * instead. 20 */ 21 22 #ifdef __cplusplus 23 extern "C" { 24 #endif 25 26 #include <stdalign.h> 27 #include <stdio.h> 28 #include <stdint.h> 29 #include <string.h> 30 #include <errno.h> 31 #include <rte_common.h> 32 #include <rte_config.h> 33 #include <rte_memory.h> 34 #include <rte_lcore.h> 35 #include <rte_atomic.h> 36 #include <rte_branch_prediction.h> 37 #include <rte_memzone.h> 38 #include <rte_pause.h> 39 #include <rte_debug.h> 40 41 #define RTE_TAILQ_RING_NAME "RTE_RING" 42 43 /** enqueue/dequeue behavior types */ 44 enum rte_ring_queue_behavior { 45 /** Enq/Deq a fixed number of items from a ring */ 46 RTE_RING_QUEUE_FIXED = 0, 47 /** Enq/Deq as many items as possible from ring */ 48 RTE_RING_QUEUE_VARIABLE 49 }; 50 51 #define RTE_RING_MZ_PREFIX "RG_" 52 /** The maximum length of a ring name. */ 53 #define RTE_RING_NAMESIZE (RTE_MEMZONE_NAMESIZE - \ 54 sizeof(RTE_RING_MZ_PREFIX) + 1) 55 56 /** prod/cons sync types */ 57 enum rte_ring_sync_type { 58 RTE_RING_SYNC_MT, /**< multi-thread safe (default mode) */ 59 RTE_RING_SYNC_ST, /**< single thread only */ 60 RTE_RING_SYNC_MT_RTS, /**< multi-thread relaxed tail sync */ 61 RTE_RING_SYNC_MT_HTS, /**< multi-thread head/tail sync */ 62 }; 63 64 /** 65 * structures to hold a pair of head/tail values and other metadata. 66 * Depending on sync_type format of that structure might be different, 67 * but offset for *sync_type* and *tail* values should remain the same. 68 */ 69 struct rte_ring_headtail { 70 volatile RTE_ATOMIC(uint32_t) head; /**< prod/consumer head. */ 71 volatile RTE_ATOMIC(uint32_t) tail; /**< prod/consumer tail. */ 72 union { 73 /** sync type of prod/cons */ 74 enum rte_ring_sync_type sync_type; 75 /** deprecated - True if single prod/cons */ 76 uint32_t single; 77 }; 78 }; 79 80 union __rte_ring_rts_poscnt { 81 /** raw 8B value to read/write *cnt* and *pos* as one atomic op */ 82 alignas(sizeof(uint64_t)) RTE_ATOMIC(uint64_t) raw; 83 struct { 84 uint32_t cnt; /**< head/tail reference counter */ 85 uint32_t pos; /**< head/tail position */ 86 } val; 87 }; 88 89 struct rte_ring_rts_headtail { 90 volatile union __rte_ring_rts_poscnt tail; 91 enum rte_ring_sync_type sync_type; /**< sync type of prod/cons */ 92 uint32_t htd_max; /**< max allowed distance between head/tail */ 93 volatile union __rte_ring_rts_poscnt head; 94 }; 95 96 union __rte_ring_hts_pos { 97 /** raw 8B value to read/write *head* and *tail* as one atomic op */ 98 alignas(sizeof(uint64_t)) RTE_ATOMIC(uint64_t) raw; 99 struct { 100 RTE_ATOMIC(uint32_t) head; /**< head position */ 101 RTE_ATOMIC(uint32_t) tail; /**< tail position */ 102 } pos; 103 }; 104 105 struct rte_ring_hts_headtail { 106 volatile union __rte_ring_hts_pos ht; 107 enum rte_ring_sync_type sync_type; /**< sync type of prod/cons */ 108 }; 109 110 /** 111 * An RTE ring structure. 112 * 113 * The producer and the consumer have a head and a tail index. The particularity 114 * of these index is that they are not between 0 and size(ring)-1. These indexes 115 * are between 0 and 2^32 -1, and we mask their value when we access the ring[] 116 * field. Thanks to this assumption, we can do subtractions between 2 index 117 * values in a modulo-32bit base: that's why the overflow of the indexes is not 118 * a problem. 119 */ 120 struct rte_ring { 121 alignas(RTE_CACHE_LINE_SIZE) char name[RTE_RING_NAMESIZE]; 122 /**< Name of the ring. */ 123 int flags; /**< Flags supplied at creation. */ 124 const struct rte_memzone *memzone; 125 /**< Memzone, if any, containing the rte_ring */ 126 uint32_t size; /**< Size of ring. */ 127 uint32_t mask; /**< Mask (size-1) of ring. */ 128 uint32_t capacity; /**< Usable size of ring */ 129 130 RTE_CACHE_GUARD; 131 132 /** Ring producer status. */ 133 union { 134 struct rte_ring_headtail prod; 135 struct rte_ring_hts_headtail hts_prod; 136 struct rte_ring_rts_headtail rts_prod; 137 } __rte_cache_aligned; 138 139 RTE_CACHE_GUARD; 140 141 /** Ring consumer status. */ 142 union { 143 struct rte_ring_headtail cons; 144 struct rte_ring_hts_headtail hts_cons; 145 struct rte_ring_rts_headtail rts_cons; 146 } __rte_cache_aligned; 147 148 RTE_CACHE_GUARD; 149 }; 150 151 #define RING_F_SP_ENQ 0x0001 /**< The default enqueue is "single-producer". */ 152 #define RING_F_SC_DEQ 0x0002 /**< The default dequeue is "single-consumer". */ 153 /** 154 * Ring is to hold exactly requested number of entries. 155 * Without this flag set, the ring size requested must be a power of 2, and the 156 * usable space will be that size - 1. With the flag, the requested size will 157 * be rounded up to the next power of two, but the usable space will be exactly 158 * that requested. Worst case, if a power-of-2 size is requested, half the 159 * ring space will be wasted. 160 */ 161 #define RING_F_EXACT_SZ 0x0004 162 #define RTE_RING_SZ_MASK (0x7fffffffU) /**< Ring size mask */ 163 164 #define RING_F_MP_RTS_ENQ 0x0008 /**< The default enqueue is "MP RTS". */ 165 #define RING_F_MC_RTS_DEQ 0x0010 /**< The default dequeue is "MC RTS". */ 166 167 #define RING_F_MP_HTS_ENQ 0x0020 /**< The default enqueue is "MP HTS". */ 168 #define RING_F_MC_HTS_DEQ 0x0040 /**< The default dequeue is "MC HTS". */ 169 170 #ifdef __cplusplus 171 } 172 #endif 173 174 #endif /* _RTE_RING_CORE_H_ */ 175