1 /* $NetBSD: lock.h,v 1.23 2022/04/09 23:43:20 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Wayne Knowles and Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Machine-dependent spin lock operations for MIPS processors. 34 * 35 * Note: R2000/R3000 doesn't have any atomic update instructions; this 36 * will cause problems for user applications using this header. 37 */ 38 39 #ifndef _MIPS_LOCK_H_ 40 #define _MIPS_LOCK_H_ 41 42 #include <sys/param.h> 43 44 #include <sys/atomic.h> 45 46 static __inline int 47 __SIMPLELOCK_LOCKED_P(const __cpu_simple_lock_t *__ptr) 48 { 49 return *__ptr != __SIMPLELOCK_UNLOCKED; 50 } 51 52 static __inline int 53 __SIMPLELOCK_UNLOCKED_P(const __cpu_simple_lock_t *__ptr) 54 { 55 return *__ptr == __SIMPLELOCK_UNLOCKED; 56 } 57 58 static __inline void 59 __cpu_simple_lock_clear(__cpu_simple_lock_t *__ptr) 60 { 61 *__ptr = __SIMPLELOCK_UNLOCKED; 62 } 63 64 static __inline void 65 __cpu_simple_lock_set(__cpu_simple_lock_t *__ptr) 66 { 67 *__ptr = __SIMPLELOCK_LOCKED; 68 } 69 70 #ifndef _HARDKERNEL 71 72 static __inline int 73 __cpu_simple_lock_try(__cpu_simple_lock_t *lp) 74 { 75 unsigned long t0, v0; 76 77 __asm volatile( 78 "# -- BEGIN __cpu_simple_lock_try\n" 79 " .set push \n" 80 " .set mips2 \n" 81 "1: ll %0, %4 \n" 82 " bnez %0, 2f \n" 83 " nop \n" 84 " li %0, %3 \n" 85 " sc %0, %2 \n" 86 " beqz %0, 2f \n" 87 " nop \n" 88 " li %1, 1 \n" 89 " sync \n" 90 " j 3f \n" 91 " nop \n" 92 " nop \n" 93 "2: li %1, 0 \n" 94 "3: \n" 95 " .set pop \n" 96 "# -- END __cpu_simple_lock_try \n" 97 : "=r" (t0), "=r" (v0), "+m" (*lp) 98 : "i" (__SIMPLELOCK_LOCKED), "m" (*lp)); 99 100 return (v0 != 0); 101 } 102 103 #else /* !_HARDKERNEL */ 104 105 u_int _atomic_cas_uint(volatile u_int *, u_int, u_int); 106 u_long _atomic_cas_ulong(volatile u_long *, u_long, u_long); 107 void * _atomic_cas_ptr(volatile void *, void *, void *); 108 109 static __inline int 110 __cpu_simple_lock_try(__cpu_simple_lock_t *lp) 111 { 112 113 /* 114 * Successful _atomic_cas_uint functions as a load-acquire -- 115 * on MP systems, it issues sync after the LL/SC CAS succeeds; 116 * on non-MP systems every load is a load-acquire so it's moot. 117 * This pairs with the membar_release and store sequence in 118 * __cpu_simple_unlock that functions as a store-release 119 * operation. 120 * 121 * NOTE: This applies only to _atomic_cas_uint (with the 122 * underscore), in sys/arch/mips/mips/lock_stubs_*.S. Not true 123 * for atomic_cas_uint (without the underscore), from 124 * common/lib/libc/arch/mips/atomic/atomic_cas.S which does not 125 * imply a load-acquire. It is unclear why these disagree. 126 */ 127 return _atomic_cas_uint(lp, 128 __SIMPLELOCK_UNLOCKED, __SIMPLELOCK_LOCKED) == 129 __SIMPLELOCK_UNLOCKED; 130 } 131 132 #endif /* _HARDKERNEL */ 133 134 static __inline void 135 __cpu_simple_lock_init(__cpu_simple_lock_t *lp) 136 { 137 138 *lp = __SIMPLELOCK_UNLOCKED; 139 } 140 141 static __inline void 142 __cpu_simple_lock(__cpu_simple_lock_t *lp) 143 { 144 145 while (!__cpu_simple_lock_try(lp)) { 146 while (*lp == __SIMPLELOCK_LOCKED) 147 /* spin */; 148 } 149 } 150 151 static __inline void 152 __cpu_simple_unlock(__cpu_simple_lock_t *lp) 153 { 154 155 /* 156 * The membar_release and then store functions as a 157 * store-release operation that pairs with the load-acquire 158 * operation in successful __cpu_simple_lock_try. 159 * 160 * Can't use atomic_store_release here because that's not 161 * available in userland at the moment. 162 */ 163 membar_release(); 164 *lp = __SIMPLELOCK_UNLOCKED; 165 166 #ifdef _MIPS_ARCH_OCTEONP 167 /* 168 * On Cavium's recommendation, we issue an extra SYNCW that is 169 * not necessary for correct ordering because apparently stores 170 * can get stuck in Octeon store buffers for hundreds of 171 * thousands of cycles, according to the following note: 172 * 173 * Programming Notes: 174 * [...] 175 * Core A (writer) 176 * SW R1, DATA 177 * LI R2, 1 178 * SYNCW 179 * SW R2, FLAG 180 * SYNCW 181 * [...] 182 * 183 * The second SYNCW instruction executed by core A is not 184 * necessary for correctness, but has very important 185 * performance effects on OCTEON. Without it, the store 186 * to FLAG may linger in core A's write buffer before it 187 * becomes visible to other cores. (If core A is not 188 * performing many stores, this may add hundreds of 189 * thousands of cycles to the flag release time since the 190 * OCTEON core normally retains stores to attempt to merge 191 * them before sending the store on the CMB.) 192 * Applications should include this second SYNCW 193 * instruction after flag or lock releases. 194 * 195 * Cavium Networks OCTEON Plus CN50XX Hardware Reference 196 * Manual, July 2008, Appendix A, p. 943. 197 * https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/hactive/CN50XX-HRM-V0.99E.pdf 198 * 199 * XXX It might be prudent to put this into 200 * atomic_store_release itself. 201 */ 202 __asm volatile("syncw" ::: "memory"); 203 #endif 204 } 205 206 #endif /* _MIPS_LOCK_H_ */ 207