1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 * Copyright(c) 2019 Arm Limited 4 */ 5 6 #ifndef _RTE_PAUSE_H_ 7 #define _RTE_PAUSE_H_ 8 9 /** 10 * @file 11 * 12 * CPU pause operation. 13 */ 14 15 #include <stdint.h> 16 #include <assert.h> 17 18 #include <rte_common.h> 19 #include <rte_atomic.h> 20 #include <rte_stdatomic.h> 21 22 #ifdef __cplusplus 23 extern "C" { 24 #endif 25 26 /** 27 * Pause CPU execution for a short while 28 * 29 * This call is intended for tight loops which poll a shared resource or wait 30 * for an event. A short pause within the loop may reduce the power consumption. 31 */ 32 static inline void rte_pause(void); 33 34 /** 35 * Wait for *addr to be updated with a 16-bit expected value, with a relaxed 36 * memory ordering model meaning the loads around this API can be reordered. 37 * 38 * @param addr 39 * A pointer to the memory location. 40 * @param expected 41 * A 16-bit expected value to be in the memory location. 42 * @param memorder 43 * Two different memory orders that can be specified: 44 * rte_memory_order_acquire and rte_memory_order_relaxed. 45 */ 46 static __rte_always_inline void 47 rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, 48 rte_memory_order memorder); 49 50 /** 51 * Wait for *addr to be updated with a 32-bit expected value, with a relaxed 52 * memory ordering model meaning the loads around this API can be reordered. 53 * 54 * @param addr 55 * A pointer to the memory location. 56 * @param expected 57 * A 32-bit expected value to be in the memory location. 58 * @param memorder 59 * Two different memory orders that can be specified: 60 * rte_memory_order_acquire and rte_memory_order_relaxed. 61 */ 62 static __rte_always_inline void 63 rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected, 64 rte_memory_order memorder); 65 66 /** 67 * Wait for *addr to be updated with a 64-bit expected value, with a relaxed 68 * memory ordering model meaning the loads around this API can be reordered. 69 * 70 * @param addr 71 * A pointer to the memory location. 72 * @param expected 73 * A 64-bit expected value to be in the memory location. 74 * @param memorder 75 * Two different memory orders that can be specified: 76 * rte_memory_order_acquire and rte_memory_order_relaxed. 77 */ 78 static __rte_always_inline void 79 rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected, 80 rte_memory_order memorder); 81 82 #ifndef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED 83 static __rte_always_inline void 84 rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, 85 rte_memory_order memorder) 86 { 87 assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed); 88 89 while (rte_atomic_load_explicit((volatile __rte_atomic uint16_t *)addr, memorder) 90 != expected) 91 rte_pause(); 92 } 93 94 static __rte_always_inline void 95 rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected, 96 rte_memory_order memorder) 97 { 98 assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed); 99 100 while (rte_atomic_load_explicit((volatile __rte_atomic uint32_t *)addr, memorder) 101 != expected) 102 rte_pause(); 103 } 104 105 static __rte_always_inline void 106 rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected, 107 rte_memory_order memorder) 108 { 109 assert(memorder == rte_memory_order_acquire || memorder == rte_memory_order_relaxed); 110 111 while (rte_atomic_load_explicit((volatile __rte_atomic uint64_t *)addr, memorder) 112 != expected) 113 rte_pause(); 114 } 115 116 /* 117 * Wait until *addr & mask makes the condition true. With a relaxed memory 118 * ordering model, the loads around this helper can be reordered. 119 * 120 * @param addr 121 * A pointer to the memory location. 122 * @param mask 123 * A mask of value bits in interest. 124 * @param cond 125 * A symbol representing the condition. 126 * @param expected 127 * An expected value to be in the memory location. 128 * @param memorder 129 * Two different memory orders that can be specified: 130 * rte_memory_order_acquire and rte_memory_order_relaxed. 131 */ 132 #define RTE_WAIT_UNTIL_MASKED(addr, mask, cond, expected, memorder) do { \ 133 RTE_BUILD_BUG_ON(!__builtin_constant_p(memorder)); \ 134 RTE_BUILD_BUG_ON((memorder) != rte_memory_order_acquire && \ 135 (memorder) != rte_memory_order_relaxed); \ 136 typeof(*(addr)) expected_value = (expected); \ 137 while (!((rte_atomic_load_explicit((addr), (memorder)) & (mask)) \ 138 cond expected_value)) \ 139 rte_pause(); \ 140 } while (0) 141 #endif /* ! RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED */ 142 143 #ifdef __cplusplus 144 } 145 #endif 146 147 #endif /* _RTE_PAUSE_H_ */ 148