1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2010-2020 Intel Corporation 4 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org 5 * All rights reserved. 6 * Derived from FreeBSD's bufring.h 7 * Used as BSD-3 Licensed with permission from Kip Macy. 8 */ 9 10 #ifndef _RTE_RING_CORE_H_ 11 #define _RTE_RING_CORE_H_ 12 13 /** 14 * @file 15 * This file contains definition of RTE ring structure itself, 16 * init flags and some related macros. 17 * For majority of DPDK entities, it is not recommended to include 18 * this file directly, use include <rte_ring.h> or <rte_ring_elem.h> 19 * instead. 20 */ 21 22 #include <stdalign.h> 23 #include <stdio.h> 24 #include <stdint.h> 25 #include <string.h> 26 #include <errno.h> 27 #include <rte_common.h> 28 #include <rte_config.h> 29 #include <rte_memory.h> 30 #include <rte_lcore.h> 31 #include <rte_atomic.h> 32 #include <rte_branch_prediction.h> 33 #include <rte_memzone.h> 34 #include <rte_pause.h> 35 #include <rte_debug.h> 36 37 #define RTE_TAILQ_RING_NAME "RTE_RING" 38 39 /** enqueue/dequeue behavior types */ 40 enum rte_ring_queue_behavior { 41 /** Enq/Deq a fixed number of items from a ring */ 42 RTE_RING_QUEUE_FIXED = 0, 43 /** Enq/Deq as many items as possible from ring */ 44 RTE_RING_QUEUE_VARIABLE 45 }; 46 47 #define RTE_RING_MZ_PREFIX "RG_" 48 /** The maximum length of a ring name. */ 49 #define RTE_RING_NAMESIZE (RTE_MEMZONE_NAMESIZE - \ 50 sizeof(RTE_RING_MZ_PREFIX) + 1) 51 52 /** prod/cons sync types */ 53 enum rte_ring_sync_type { 54 RTE_RING_SYNC_MT, /**< multi-thread safe (default mode) */ 55 RTE_RING_SYNC_ST, /**< single thread only */ 56 RTE_RING_SYNC_MT_RTS, /**< multi-thread relaxed tail sync */ 57 RTE_RING_SYNC_MT_HTS, /**< multi-thread head/tail sync */ 58 }; 59 60 /** 61 * structures to hold a pair of head/tail values and other metadata. 62 * Depending on sync_type format of that structure might be different, 63 * but offset for *sync_type* and *tail* values should remain the same. 64 */ 65 struct rte_ring_headtail { 66 volatile RTE_ATOMIC(uint32_t) head; /**< prod/consumer head. */ 67 volatile RTE_ATOMIC(uint32_t) tail; /**< prod/consumer tail. */ 68 union { 69 /** sync type of prod/cons */ 70 enum rte_ring_sync_type sync_type; 71 /** deprecated - True if single prod/cons */ 72 uint32_t single; 73 }; 74 }; 75 76 union __rte_ring_rts_poscnt { 77 /** raw 8B value to read/write *cnt* and *pos* as one atomic op */ 78 alignas(sizeof(uint64_t)) RTE_ATOMIC(uint64_t) raw; 79 struct { 80 uint32_t cnt; /**< head/tail reference counter */ 81 uint32_t pos; /**< head/tail position */ 82 } val; 83 }; 84 85 struct rte_ring_rts_headtail { 86 volatile union __rte_ring_rts_poscnt tail; 87 enum rte_ring_sync_type sync_type; /**< sync type of prod/cons */ 88 uint32_t htd_max; /**< max allowed distance between head/tail */ 89 volatile union __rte_ring_rts_poscnt head; 90 }; 91 92 union __rte_ring_hts_pos { 93 /** raw 8B value to read/write *head* and *tail* as one atomic op */ 94 alignas(sizeof(uint64_t)) RTE_ATOMIC(uint64_t) raw; 95 struct { 96 RTE_ATOMIC(uint32_t) head; /**< head position */ 97 RTE_ATOMIC(uint32_t) tail; /**< tail position */ 98 } pos; 99 }; 100 101 struct rte_ring_hts_headtail { 102 volatile union __rte_ring_hts_pos ht; 103 enum rte_ring_sync_type sync_type; /**< sync type of prod/cons */ 104 }; 105 106 /** 107 * An RTE ring structure. 108 * 109 * The producer and the consumer have a head and a tail index. The particularity 110 * of these index is that they are not between 0 and size(ring)-1. These indexes 111 * are between 0 and 2^32 -1, and we mask their value when we access the ring[] 112 * field. Thanks to this assumption, we can do subtractions between 2 index 113 * values in a modulo-32bit base: that's why the overflow of the indexes is not 114 * a problem. 115 */ 116 struct rte_ring { 117 alignas(RTE_CACHE_LINE_SIZE) char name[RTE_RING_NAMESIZE]; 118 /**< Name of the ring. */ 119 int flags; /**< Flags supplied at creation. */ 120 const struct rte_memzone *memzone; 121 /**< Memzone, if any, containing the rte_ring */ 122 uint32_t size; /**< Size of ring. */ 123 uint32_t mask; /**< Mask (size-1) of ring. */ 124 uint32_t capacity; /**< Usable size of ring */ 125 126 RTE_CACHE_GUARD; 127 128 /** Ring producer status. */ 129 union __rte_cache_aligned { 130 struct rte_ring_headtail prod; 131 struct rte_ring_hts_headtail hts_prod; 132 struct rte_ring_rts_headtail rts_prod; 133 }; 134 135 RTE_CACHE_GUARD; 136 137 /** Ring consumer status. */ 138 union __rte_cache_aligned { 139 struct rte_ring_headtail cons; 140 struct rte_ring_hts_headtail hts_cons; 141 struct rte_ring_rts_headtail rts_cons; 142 }; 143 144 RTE_CACHE_GUARD; 145 }; 146 147 #define RING_F_SP_ENQ 0x0001 /**< The default enqueue is "single-producer". */ 148 #define RING_F_SC_DEQ 0x0002 /**< The default dequeue is "single-consumer". */ 149 /** 150 * Ring is to hold exactly requested number of entries. 151 * Without this flag set, the ring size requested must be a power of 2, and the 152 * usable space will be that size - 1. With the flag, the requested size will 153 * be rounded up to the next power of two, but the usable space will be exactly 154 * that requested. Worst case, if a power-of-2 size is requested, half the 155 * ring space will be wasted. 156 */ 157 #define RING_F_EXACT_SZ 0x0004 158 #define RTE_RING_SZ_MASK (0x7fffffffU) /**< Ring size mask */ 159 160 #define RING_F_MP_RTS_ENQ 0x0008 /**< The default enqueue is "MP RTS". */ 161 #define RING_F_MC_RTS_DEQ 0x0010 /**< The default dequeue is "MC RTS". */ 162 163 #define RING_F_MP_HTS_ENQ 0x0020 /**< The default enqueue is "MP HTS". */ 164 #define RING_F_MC_HTS_DEQ 0x0040 /**< The default dequeue is "MC HTS". */ 165 166 #endif /* _RTE_RING_CORE_H_ */ 167