1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2017,2018 HXT-semitech Corporation. 4 * Copyright (c) 2007-2009 Kip Macy kmacy@freebsd.org 5 * Copyright (c) 2021 Arm Limited 6 * All rights reserved. 7 * Derived from FreeBSD's bufring.h 8 * Used as BSD-3 Licensed with permission from Kip Macy. 9 */ 10 11 #ifndef _RTE_RING_C11_PVT_H_ 12 #define _RTE_RING_C11_PVT_H_ 13 14 /** 15 * @file rte_ring_c11_pvt.h 16 * It is not recommended to include this file directly, 17 * include <rte_ring.h> instead. 18 * Contains internal helper functions for MP/SP and MC/SC ring modes. 19 * For more information please refer to <rte_ring.h>. 20 */ 21 22 /** 23 * @internal This function updates tail values. 24 */ 25 static __rte_always_inline void 26 __rte_ring_update_tail(struct rte_ring_headtail *ht, uint32_t old_val, 27 uint32_t new_val, uint32_t single, uint32_t enqueue) 28 { 29 RTE_SET_USED(enqueue); 30 31 /* 32 * If there are other enqueues/dequeues in progress that preceded us, 33 * we need to wait for them to complete 34 */ 35 if (!single) 36 rte_wait_until_equal_32((uint32_t *)(uintptr_t)&ht->tail, old_val, 37 rte_memory_order_relaxed); 38 39 rte_atomic_store_explicit(&ht->tail, new_val, rte_memory_order_release); 40 } 41 42 /** 43 * @internal This is a helper function that moves the producer/consumer head 44 * 45 * @param d 46 * A pointer to the headtail structure with head value to be moved 47 * @param s 48 * A pointer to the counter-part headtail structure. Note that this 49 * function only reads tail value from it 50 * @param capacity 51 * Either ring capacity value (for producer), or zero (for consumer) 52 * @param is_st 53 * Indicates whether multi-thread safe path is needed or not 54 * @param n 55 * The number of elements we want to move head value on 56 * @param behavior 57 * RTE_RING_QUEUE_FIXED: Move on a fixed number of items 58 * RTE_RING_QUEUE_VARIABLE: Move on as many items as possible 59 * @param old_head 60 * Returns head value as it was before the move 61 * @param new_head 62 * Returns the new head value 63 * @param entries 64 * Returns the number of ring entries available BEFORE head was moved 65 * @return 66 * Actual number of objects the head was moved on 67 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only 68 */ 69 static __rte_always_inline unsigned int 70 __rte_ring_headtail_move_head(struct rte_ring_headtail *d, 71 const struct rte_ring_headtail *s, uint32_t capacity, 72 unsigned int is_st, unsigned int n, 73 enum rte_ring_queue_behavior behavior, 74 uint32_t *old_head, uint32_t *new_head, uint32_t *entries) 75 { 76 uint32_t stail; 77 int success; 78 unsigned int max = n; 79 80 *old_head = rte_atomic_load_explicit(&d->head, 81 rte_memory_order_relaxed); 82 do { 83 /* Reset n to the initial burst count */ 84 n = max; 85 86 /* Ensure the head is read before tail */ 87 rte_atomic_thread_fence(rte_memory_order_acquire); 88 89 /* load-acquire synchronize with store-release of ht->tail 90 * in update_tail. 91 */ 92 stail = rte_atomic_load_explicit(&s->tail, 93 rte_memory_order_acquire); 94 95 /* The subtraction is done between two unsigned 32bits value 96 * (the result is always modulo 32 bits even if we have 97 * *old_head > s->tail). So 'entries' is always between 0 98 * and capacity (which is < size). 99 */ 100 *entries = (capacity + stail - *old_head); 101 102 /* check that we have enough room in ring */ 103 if (unlikely(n > *entries)) 104 n = (behavior == RTE_RING_QUEUE_FIXED) ? 105 0 : *entries; 106 107 if (n == 0) 108 return 0; 109 110 *new_head = *old_head + n; 111 if (is_st) { 112 d->head = *new_head; 113 success = 1; 114 } else 115 /* on failure, *old_head is updated */ 116 success = rte_atomic_compare_exchange_strong_explicit( 117 &d->head, old_head, *new_head, 118 rte_memory_order_relaxed, 119 rte_memory_order_relaxed); 120 } while (unlikely(success == 0)); 121 return n; 122 } 123 124 #endif /* _RTE_RING_C11_PVT_H_ */ 125