1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2015 Cavium, Inc 3 * Copyright(c) 2020 Arm Limited 4 */ 5 6 #ifndef _RTE_ATOMIC_ARM64_H_ 7 #define _RTE_ATOMIC_ARM64_H_ 8 9 #ifndef RTE_FORCE_INTRINSICS 10 # error Platform must be built with RTE_FORCE_INTRINSICS 11 #endif 12 13 #include "generic/rte_atomic.h" 14 #include <rte_branch_prediction.h> 15 #include <rte_debug.h> 16 17 #ifdef __cplusplus 18 extern "C" { 19 #endif 20 21 #define rte_mb() asm volatile("dmb osh" : : : "memory") 22 23 #define rte_wmb() asm volatile("dmb oshst" : : : "memory") 24 25 #define rte_rmb() asm volatile("dmb oshld" : : : "memory") 26 27 #define rte_smp_mb() asm volatile("dmb ish" : : : "memory") 28 29 #define rte_smp_wmb() asm volatile("dmb ishst" : : : "memory") 30 31 #define rte_smp_rmb() asm volatile("dmb ishld" : : : "memory") 32 33 #define rte_io_mb() rte_mb() 34 35 #define rte_io_wmb() rte_wmb() 36 37 #define rte_io_rmb() rte_rmb() 38 39 static __rte_always_inline void 40 rte_atomic_thread_fence(rte_memory_order memorder) 41 { 42 __rte_atomic_thread_fence(memorder); 43 } 44 45 /*------------------------ 128 bit atomic operations -------------------------*/ 46 47 #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS) 48 49 #if defined(RTE_CC_CLANG) 50 #define __ATOMIC128_CAS_OP(cas_op_name, op_string) \ 51 static __rte_noinline void \ 52 cas_op_name(rte_int128_t *dst, rte_int128_t *old, rte_int128_t updated) \ 53 { \ 54 /* caspX instructions register pair must start from even-numbered 55 * register at operand 1. 56 * So, specify registers for local variables here. 57 */ \ 58 register uint64_t x0 __asm("x0") = (uint64_t)old->val[0]; \ 59 register uint64_t x1 __asm("x1") = (uint64_t)old->val[1]; \ 60 register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0]; \ 61 register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1]; \ 62 asm volatile( \ 63 ".arch armv8-a+lse\n" \ 64 op_string " %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" \ 65 : [old0] "+r" (x0), \ 66 [old1] "+r" (x1) \ 67 : [upd0] "r" (x2), \ 68 [upd1] "r" (x3), \ 69 [dst] "r" (dst) \ 70 : "memory"); \ 71 old->val[0] = x0; \ 72 old->val[1] = x1; \ 73 } 74 #else 75 #define __ATOMIC128_CAS_OP(cas_op_name, op_string) \ 76 static __rte_always_inline void \ 77 cas_op_name(rte_int128_t *dst, rte_int128_t *old, rte_int128_t updated) \ 78 { \ 79 asm volatile( \ 80 op_string " %[old], %H[old], %[upd], %H[upd], [%[dst]]" \ 81 : [old] "+r"(old->int128) \ 82 : [upd] "r"(updated.int128), [dst] "r"(dst) \ 83 : "memory"); \ 84 } 85 #endif 86 87 __ATOMIC128_CAS_OP(__cas_128_relaxed, "casp") 88 __ATOMIC128_CAS_OP(__cas_128_acquire, "caspa") 89 __ATOMIC128_CAS_OP(__cas_128_release, "caspl") 90 __ATOMIC128_CAS_OP(__cas_128_acq_rel, "caspal") 91 92 #undef __ATOMIC128_CAS_OP 93 94 #endif 95 96 static inline int 97 rte_atomic128_cmp_exchange(rte_int128_t *dst, rte_int128_t *exp, 98 const rte_int128_t *src, unsigned int weak, int success, 99 int failure) 100 { 101 /* Always do strong CAS */ 102 RTE_SET_USED(weak); 103 /* Ignore memory ordering for failure, memory order for 104 * success must be stronger or equal 105 */ 106 RTE_SET_USED(failure); 107 /* Find invalid memory order */ 108 RTE_ASSERT(success == rte_memory_order_relaxed || 109 success == rte_memory_order_acquire || 110 success == rte_memory_order_release || 111 success == rte_memory_order_acq_rel || 112 success == rte_memory_order_seq_cst); 113 114 rte_int128_t expected = *exp; 115 rte_int128_t desired = *src; 116 rte_int128_t old; 117 118 #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS) 119 if (success == rte_memory_order_relaxed) 120 __cas_128_relaxed(dst, exp, desired); 121 else if (success == rte_memory_order_acquire) 122 __cas_128_acquire(dst, exp, desired); 123 else if (success == rte_memory_order_release) 124 __cas_128_release(dst, exp, desired); 125 else 126 __cas_128_acq_rel(dst, exp, desired); 127 old = *exp; 128 #else 129 #define __HAS_ACQ(mo) ((mo) != rte_memory_order_relaxed && (mo) != rte_memory_order_release) 130 #define __HAS_RLS(mo) ((mo) == rte_memory_order_release || (mo) == rte_memory_order_acq_rel || \ 131 (mo) == rte_memory_order_seq_cst) 132 133 int ldx_mo = __HAS_ACQ(success) ? rte_memory_order_acquire : rte_memory_order_relaxed; 134 int stx_mo = __HAS_RLS(success) ? rte_memory_order_release : rte_memory_order_relaxed; 135 136 #undef __HAS_ACQ 137 #undef __HAS_RLS 138 139 uint32_t ret = 1; 140 141 /* ldx128 can not guarantee atomic, 142 * Must write back src or old to verify atomicity of ldx128; 143 */ 144 do { 145 146 #define __LOAD_128(op_string, src, dst) { \ 147 asm volatile( \ 148 op_string " %0, %1, %2" \ 149 : "=&r" (dst.val[0]), \ 150 "=&r" (dst.val[1]) \ 151 : "Q" (src->val[0]) \ 152 : "memory"); } 153 154 if (ldx_mo == rte_memory_order_relaxed) 155 __LOAD_128("ldxp", dst, old) 156 else 157 __LOAD_128("ldaxp", dst, old) 158 159 #undef __LOAD_128 160 161 #define __STORE_128(op_string, dst, src, ret) { \ 162 asm volatile( \ 163 op_string " %w0, %1, %2, %3" \ 164 : "=&r" (ret) \ 165 : "r" (src.val[0]), \ 166 "r" (src.val[1]), \ 167 "Q" (dst->val[0]) \ 168 : "memory"); } 169 170 if (likely(old.int128 == expected.int128)) { 171 if (stx_mo == rte_memory_order_relaxed) 172 __STORE_128("stxp", dst, desired, ret) 173 else 174 __STORE_128("stlxp", dst, desired, ret) 175 } else { 176 /* In the failure case (since 'weak' is ignored and only 177 * weak == 0 is implemented), expected should contain 178 * the atomically read value of dst. This means, 'old' 179 * needs to be stored back to ensure it was read 180 * atomically. 181 */ 182 if (stx_mo == rte_memory_order_relaxed) 183 __STORE_128("stxp", dst, old, ret) 184 else 185 __STORE_128("stlxp", dst, old, ret) 186 } 187 188 #undef __STORE_128 189 190 } while (unlikely(ret)); 191 192 /* Unconditionally updating the value of exp removes an 'if' statement. 193 * The value of exp should already be in register if not in the cache. 194 */ 195 *exp = old; 196 #endif 197 198 return (old.int128 == expected.int128); 199 } 200 201 #ifdef __cplusplus 202 } 203 #endif 204 205 #endif /* _RTE_ATOMIC_ARM64_H_ */ 206