1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2010-2017 Intel Corporation 4 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org 5 * All rights reserved. 6 * Derived from FreeBSD's bufring.h 7 * Used as BSD-3 Licensed with permission from Kip Macy. 8 */ 9 10 #ifndef _RTE_RING_GENERIC_PVT_H_ 11 #define _RTE_RING_GENERIC_PVT_H_ 12 13 /** 14 * @file rte_ring_generic_pvt.h 15 * It is not recommended to include this file directly, 16 * include <rte_ring.h> instead. 17 * Contains internal helper functions for MP/SP and MC/SC ring modes. 18 * For more information please refer to <rte_ring.h>. 19 */ 20 21 /** 22 * @internal This function updates tail values. 23 */ 24 static __rte_always_inline void 25 __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val, 26 uint32_t new_val, uint32_t single, uint32_t enqueue) 27 { 28 if (enqueue) 29 rte_smp_wmb(); 30 else 31 rte_smp_rmb(); 32 /* 33 * If there are other enqueues/dequeues in progress that preceded us, 34 * we need to wait for them to complete 35 */ 36 if (!single) 37 rte_wait_until_equal_32((volatile uint32_t *)(uintptr_t)&ht->tail, old_val, 38 rte_memory_order_relaxed); 39 40 ht->tail = new_val; 41 } 42 43 /** 44 * @internal This is a helper function that moves the producer/consumer head 45 * 46 * @param d 47 * A pointer to the headtail structure with head value to be moved 48 * @param s 49 * A pointer to the counter-part headtail structure. Note that this 50 * function only reads tail value from it 51 * @param capacity 52 * Either ring capacity value (for producer), or zero (for consumer) 53 * @param is_st 54 * Indicates whether multi-thread safe path is needed or not 55 * @param n 56 * The number of elements we want to move head value on 57 * @param behavior 58 * RTE_RING_QUEUE_FIXED: Move on a fixed number of items 59 * RTE_RING_QUEUE_VARIABLE: Move on as many items as possible 60 * @param old_head 61 * Returns head value as it was before the move 62 * @param new_head 63 * Returns the new head value 64 * @param entries 65 * Returns the number of ring entries available BEFORE head was moved 66 * @return 67 * Actual number of objects the head was moved on 68 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only 69 */ 70 static __rte_always_inline unsigned int 71 __rte_ring_headtail_move_head(struct rte_ring_headtail *d, 72 const struct rte_ring_headtail *s, uint32_t capacity, 73 unsigned int is_st, unsigned int n, 74 enum rte_ring_queue_behavior behavior, 75 uint32_t *old_head, uint32_t *new_head, uint32_t *entries) 76 { 77 unsigned int max = n; 78 int success; 79 80 do { 81 /* Reset n to the initial burst count */ 82 n = max; 83 84 *old_head = d->head; 85 86 /* add rmb barrier to avoid load/load reorder in weak 87 * memory model. It is noop on x86 88 */ 89 rte_smp_rmb(); 90 91 /* 92 * The subtraction is done between two unsigned 32bits value 93 * (the result is always modulo 32 bits even if we have 94 * *old_head > s->tail). So 'entries' is always between 0 95 * and capacity (which is < size). 96 */ 97 *entries = (capacity + s->tail - *old_head); 98 99 /* check that we have enough room in ring */ 100 if (unlikely(n > *entries)) 101 n = (behavior == RTE_RING_QUEUE_FIXED) ? 102 0 : *entries; 103 104 if (n == 0) 105 return 0; 106 107 *new_head = *old_head + n; 108 if (is_st) { 109 d->head = *new_head; 110 success = 1; 111 } else 112 success = rte_atomic32_cmpset( 113 (uint32_t *)(uintptr_t)&d->head, 114 *old_head, *new_head); 115 } while (unlikely(success == 0)); 116 return n; 117 } 118 119 #endif /* _RTE_RING_GENERIC_PVT_H_ */ 120