1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2010-2020 Intel Corporation 4 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org 5 * All rights reserved. 6 * Derived from FreeBSD's bufring.h 7 * Used as BSD-3 Licensed with permission from Kip Macy. 8 */ 9 10 #ifndef _RTE_RING_PEEK_H_ 11 #define _RTE_RING_PEEK_H_ 12 13 /** 14 * @file 15 * It is not recommended to include this file directly. 16 * Please include <rte_ring_elem.h> instead. 17 * 18 * Ring Peek API 19 * Introduction of rte_ring with serialized producer/consumer (HTS sync mode) 20 * makes possible to split public enqueue/dequeue API into two phases: 21 * - enqueue/dequeue start 22 * - enqueue/dequeue finish 23 * That allows user to inspect objects in the ring without removing them 24 * from it (aka MT safe peek). 25 * Note that right now this new API is available only for two sync modes: 26 * 1) Single Producer/Single Consumer (RTE_RING_SYNC_ST) 27 * 2) Serialized Producer/Serialized Consumer (RTE_RING_SYNC_MT_HTS). 28 * It is a user responsibility to create/init ring with appropriate sync 29 * modes selected. 30 * As an example: 31 * // read 1 elem from the ring: 32 * n = rte_ring_dequeue_bulk_start(ring, &obj, 1, NULL); 33 * if (n != 0) { 34 * //examine object 35 * if (object_examine(obj) == KEEP) 36 * //decided to keep it in the ring. 37 * rte_ring_dequeue_finish(ring, 0); 38 * else 39 * //decided to remove it from the ring. 40 * rte_ring_dequeue_finish(ring, n); 41 * } 42 * Note that between _start_ and _finish_ none other thread can proceed 43 * with enqueue(/dequeue) operation till _finish_ completes. 44 */ 45 46 #include <rte_ring_peek_elem_pvt.h> 47 48 #ifdef __cplusplus 49 extern "C" { 50 #endif 51 52 /** 53 * Start to enqueue several objects on the ring. 54 * Note that no actual objects are put in the queue by this function, 55 * it just reserves for user such ability. 56 * User has to call appropriate enqueue_elem_finish() to copy objects into the 57 * queue and complete given enqueue operation. 58 * 59 * @param r 60 * A pointer to the ring structure. 61 * @param n 62 * The number of objects to add in the ring from the obj_table. 63 * @param free_space 64 * if non-NULL, returns the amount of space in the ring after the 65 * enqueue operation has finished. 66 * @return 67 * The number of objects that can be enqueued, either 0 or n 68 */ 69 static __rte_always_inline unsigned int 70 rte_ring_enqueue_bulk_elem_start(struct rte_ring *r, unsigned int n, 71 unsigned int *free_space) 72 { 73 return __rte_ring_do_enqueue_start(r, n, RTE_RING_QUEUE_FIXED, 74 free_space); 75 } 76 77 /** 78 * Start to enqueue several objects on the ring. 79 * Note that no actual objects are put in the queue by this function, 80 * it just reserves for user such ability. 81 * User has to call appropriate enqueue_finish() to copy objects into the 82 * queue and complete given enqueue operation. 83 * 84 * @param r 85 * A pointer to the ring structure. 86 * @param n 87 * The number of objects to add in the ring from the obj_table. 88 * @param free_space 89 * if non-NULL, returns the amount of space in the ring after the 90 * enqueue operation has finished. 91 * @return 92 * The number of objects that can be enqueued, either 0 or n 93 */ 94 static __rte_always_inline unsigned int 95 rte_ring_enqueue_bulk_start(struct rte_ring *r, unsigned int n, 96 unsigned int *free_space) 97 { 98 return rte_ring_enqueue_bulk_elem_start(r, n, free_space); 99 } 100 101 /** 102 * Start to enqueue several objects on the ring. 103 * Note that no actual objects are put in the queue by this function, 104 * it just reserves for user such ability. 105 * User has to call appropriate enqueue_elem_finish() to copy objects into the 106 * queue and complete given enqueue operation. 107 * 108 * @param r 109 * A pointer to the ring structure. 110 * @param n 111 * The number of objects to add in the ring from the obj_table. 112 * @param free_space 113 * if non-NULL, returns the amount of space in the ring after the 114 * enqueue operation has finished. 115 * @return 116 * Actual number of objects that can be enqueued. 117 */ 118 static __rte_always_inline unsigned int 119 rte_ring_enqueue_burst_elem_start(struct rte_ring *r, unsigned int n, 120 unsigned int *free_space) 121 { 122 return __rte_ring_do_enqueue_start(r, n, RTE_RING_QUEUE_VARIABLE, 123 free_space); 124 } 125 126 /** 127 * Start to enqueue several objects on the ring. 128 * Note that no actual objects are put in the queue by this function, 129 * it just reserves for user such ability. 130 * User has to call appropriate enqueue_finish() to copy objects into the 131 * queue and complete given enqueue operation. 132 * 133 * @param r 134 * A pointer to the ring structure. 135 * @param n 136 * The number of objects to add in the ring from the obj_table. 137 * @param free_space 138 * if non-NULL, returns the amount of space in the ring after the 139 * enqueue operation has finished. 140 * @return 141 * Actual number of objects that can be enqueued. 142 */ 143 static __rte_always_inline unsigned int 144 rte_ring_enqueue_burst_start(struct rte_ring *r, unsigned int n, 145 unsigned int *free_space) 146 { 147 return rte_ring_enqueue_burst_elem_start(r, n, free_space); 148 } 149 150 /** 151 * Complete to enqueue several objects on the ring. 152 * Note that number of objects to enqueue should not exceed previous 153 * enqueue_start return value. 154 * 155 * @param r 156 * A pointer to the ring structure. 157 * @param obj_table 158 * A pointer to a table of objects. 159 * @param esize 160 * The size of ring element, in bytes. It must be a multiple of 4. 161 * This must be the same value used while creating the ring. Otherwise 162 * the results are undefined. 163 * @param n 164 * The number of objects to add to the ring from the obj_table. 165 */ 166 static __rte_always_inline void 167 rte_ring_enqueue_elem_finish(struct rte_ring *r, const void *obj_table, 168 unsigned int esize, unsigned int n) 169 { 170 uint32_t tail; 171 172 switch (r->prod.sync_type) { 173 case RTE_RING_SYNC_ST: 174 n = __rte_ring_st_get_tail(&r->prod, &tail, n); 175 if (n != 0) 176 __rte_ring_enqueue_elems(r, tail, obj_table, esize, n); 177 __rte_ring_st_set_head_tail(&r->prod, tail, n, 1); 178 break; 179 case RTE_RING_SYNC_MT_HTS: 180 n = __rte_ring_hts_get_tail(&r->hts_prod, &tail, n); 181 if (n != 0) 182 __rte_ring_enqueue_elems(r, tail, obj_table, esize, n); 183 __rte_ring_hts_set_head_tail(&r->hts_prod, tail, n, 1); 184 break; 185 case RTE_RING_SYNC_MT: 186 case RTE_RING_SYNC_MT_RTS: 187 default: 188 /* unsupported mode, shouldn't be here */ 189 RTE_ASSERT(0); 190 } 191 } 192 193 /** 194 * Complete to enqueue several objects on the ring. 195 * Note that number of objects to enqueue should not exceed previous 196 * enqueue_start return value. 197 * 198 * @param r 199 * A pointer to the ring structure. 200 * @param obj_table 201 * A pointer to a table of objects. 202 * @param n 203 * The number of objects to add to the ring from the obj_table. 204 */ 205 static __rte_always_inline void 206 rte_ring_enqueue_finish(struct rte_ring *r, void * const *obj_table, 207 unsigned int n) 208 { 209 rte_ring_enqueue_elem_finish(r, obj_table, sizeof(uintptr_t), n); 210 } 211 212 /** 213 * Start to dequeue several objects from the ring. 214 * Note that user has to call appropriate dequeue_finish() 215 * to complete given dequeue operation and actually remove objects the ring. 216 * 217 * @param r 218 * A pointer to the ring structure. 219 * @param obj_table 220 * A pointer to a table of objects that will be filled. 221 * @param esize 222 * The size of ring element, in bytes. It must be a multiple of 4. 223 * This must be the same value used while creating the ring. Otherwise 224 * the results are undefined. 225 * @param n 226 * The number of objects to dequeue from the ring to the obj_table. 227 * @param available 228 * If non-NULL, returns the number of remaining ring entries after the 229 * dequeue has finished. 230 * @return 231 * The number of objects dequeued, either 0 or n. 232 */ 233 static __rte_always_inline unsigned int 234 rte_ring_dequeue_bulk_elem_start(struct rte_ring *r, void *obj_table, 235 unsigned int esize, unsigned int n, unsigned int *available) 236 { 237 return __rte_ring_do_dequeue_start(r, obj_table, esize, n, 238 RTE_RING_QUEUE_FIXED, available); 239 } 240 241 /** 242 * Start to dequeue several objects from the ring. 243 * Note that user has to call appropriate dequeue_finish() 244 * to complete given dequeue operation and actually remove objects the ring. 245 * 246 * @param r 247 * A pointer to the ring structure. 248 * @param obj_table 249 * A pointer to a table of void * pointers (objects) that will be filled. 250 * @param n 251 * The number of objects to dequeue from the ring to the obj_table. 252 * @param available 253 * If non-NULL, returns the number of remaining ring entries after the 254 * dequeue has finished. 255 * @return 256 * Actual number of objects dequeued. 257 */ 258 static __rte_always_inline unsigned int 259 rte_ring_dequeue_bulk_start(struct rte_ring *r, void **obj_table, 260 unsigned int n, unsigned int *available) 261 { 262 return rte_ring_dequeue_bulk_elem_start(r, obj_table, sizeof(uintptr_t), 263 n, available); 264 } 265 266 /** 267 * Start to dequeue several objects from the ring. 268 * Note that user has to call appropriate dequeue_finish() 269 * to complete given dequeue operation and actually remove objects the ring. 270 * 271 * @param r 272 * A pointer to the ring structure. 273 * @param obj_table 274 * A pointer to a table of objects that will be filled. 275 * @param esize 276 * The size of ring element, in bytes. It must be a multiple of 4. 277 * This must be the same value used while creating the ring. Otherwise 278 * the results are undefined. 279 * @param n 280 * The number of objects to dequeue from the ring to the obj_table. 281 * @param available 282 * If non-NULL, returns the number of remaining ring entries after the 283 * dequeue has finished. 284 * @return 285 * The actual number of objects dequeued. 286 */ 287 static __rte_always_inline unsigned int 288 rte_ring_dequeue_burst_elem_start(struct rte_ring *r, void *obj_table, 289 unsigned int esize, unsigned int n, unsigned int *available) 290 { 291 return __rte_ring_do_dequeue_start(r, obj_table, esize, n, 292 RTE_RING_QUEUE_VARIABLE, available); 293 } 294 295 /** 296 * Start to dequeue several objects from the ring. 297 * Note that user has to call appropriate dequeue_finish() 298 * to complete given dequeue operation and actually remove objects the ring. 299 * 300 * @param r 301 * A pointer to the ring structure. 302 * @param obj_table 303 * A pointer to a table of void * pointers (objects) that will be filled. 304 * @param n 305 * The number of objects to dequeue from the ring to the obj_table. 306 * @param available 307 * If non-NULL, returns the number of remaining ring entries after the 308 * dequeue has finished. 309 * @return 310 * The actual number of objects dequeued. 311 */ 312 static __rte_always_inline unsigned int 313 rte_ring_dequeue_burst_start(struct rte_ring *r, void **obj_table, 314 unsigned int n, unsigned int *available) 315 { 316 return rte_ring_dequeue_burst_elem_start(r, obj_table, 317 sizeof(uintptr_t), n, available); 318 } 319 320 /** 321 * Complete to dequeue several objects from the ring. 322 * Note that number of objects to dequeue should not exceed previous 323 * dequeue_start return value. 324 * 325 * @param r 326 * A pointer to the ring structure. 327 * @param n 328 * The number of objects to remove from the ring. 329 */ 330 static __rte_always_inline void 331 rte_ring_dequeue_elem_finish(struct rte_ring *r, unsigned int n) 332 { 333 uint32_t tail; 334 335 switch (r->cons.sync_type) { 336 case RTE_RING_SYNC_ST: 337 n = __rte_ring_st_get_tail(&r->cons, &tail, n); 338 __rte_ring_st_set_head_tail(&r->cons, tail, n, 0); 339 break; 340 case RTE_RING_SYNC_MT_HTS: 341 n = __rte_ring_hts_get_tail(&r->hts_cons, &tail, n); 342 __rte_ring_hts_set_head_tail(&r->hts_cons, tail, n, 0); 343 break; 344 case RTE_RING_SYNC_MT: 345 case RTE_RING_SYNC_MT_RTS: 346 default: 347 /* unsupported mode, shouldn't be here */ 348 RTE_ASSERT(0); 349 } 350 } 351 352 /** 353 * Complete to dequeue several objects from the ring. 354 * Note that number of objects to dequeue should not exceed previous 355 * dequeue_start return value. 356 * 357 * @param r 358 * A pointer to the ring structure. 359 * @param n 360 * The number of objects to remove from the ring. 361 */ 362 static __rte_always_inline void 363 rte_ring_dequeue_finish(struct rte_ring *r, unsigned int n) 364 { 365 rte_ring_dequeue_elem_finish(r, n); 366 } 367 368 #ifdef __cplusplus 369 } 370 #endif 371 372 #endif /* _RTE_RING_PEEK_H_ */ 373