1 /* $NetBSD: kern_lock.c,v 1.137 2008/04/01 19:49:31 drochner Exp $ */ 2 3 /*- 4 * Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the NetBSD 22 * Foundation, Inc. and its contributors. 23 * 4. Neither the name of The NetBSD Foundation nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 29 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 30 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 37 * POSSIBILITY OF SUCH DAMAGE. 38 */ 39 40 #include <sys/cdefs.h> 41 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.137 2008/04/01 19:49:31 drochner Exp $"); 42 43 #include "opt_multiprocessor.h" 44 45 #include <sys/param.h> 46 #include <sys/proc.h> 47 #include <sys/lock.h> 48 #include <sys/systm.h> 49 #include <sys/kernel.h> 50 #include <sys/lockdebug.h> 51 #include <sys/cpu.h> 52 #include <sys/syslog.h> 53 #include <sys/atomic.h> 54 55 #include <machine/stdarg.h> 56 #include <machine/lock.h> 57 58 #include <dev/lockstat.h> 59 60 #define RETURN_ADDRESS (uintptr_t)__builtin_return_address(0) 61 62 bool kernel_lock_dodebug; 63 64 __cpu_simple_lock_t kernel_lock[CACHE_LINE_SIZE / sizeof(__cpu_simple_lock_t)] 65 __aligned(CACHE_LINE_SIZE); 66 67 #if defined(DEBUG) || defined(LKM) 68 void 69 assert_sleepable(void) 70 { 71 #if !defined(_RUMPKERNEL) 72 const char *reason; 73 74 if (panicstr != NULL) { 75 return; 76 } 77 78 LOCKDEBUG_BARRIER(kernel_lock, 1); 79 80 reason = NULL; 81 if (CURCPU_IDLE_P() && !cold) { 82 reason = "idle"; 83 } 84 if (cpu_intr_p()) { 85 reason = "interrupt"; 86 } 87 if ((curlwp->l_pflag & LP_INTR) != 0) { 88 reason = "softint"; 89 } 90 91 if (reason) { 92 panic("%s: %s caller=%p", __func__, reason, 93 (void *)RETURN_ADDRESS); 94 } 95 #endif /* !defined(_RUMPKERNEL) */ 96 } 97 #endif /* defined(DEBUG) || defined(LKM) */ 98 99 /* 100 * rump doesn't need the kernel lock so force it out. We cannot 101 * currently easily include it for compilation because of 102 * a) SPINLOCK_* b) membar_producer(). They are defined in different 103 * places / way for each arch, so just simply do not bother to 104 * fight a lot for no gain (i.e. pain but still no gain). 105 */ 106 #ifndef _RUMPKERNEL 107 /* 108 * Functions for manipulating the kernel_lock. We put them here 109 * so that they show up in profiles. 110 */ 111 112 #define _KERNEL_LOCK_ABORT(msg) \ 113 LOCKDEBUG_ABORT(kernel_lock, &_kernel_lock_ops, __func__, msg) 114 115 #ifdef LOCKDEBUG 116 #define _KERNEL_LOCK_ASSERT(cond) \ 117 do { \ 118 if (!(cond)) \ 119 _KERNEL_LOCK_ABORT("assertion failed: " #cond); \ 120 } while (/* CONSTCOND */ 0) 121 #else 122 #define _KERNEL_LOCK_ASSERT(cond) /* nothing */ 123 #endif 124 125 void _kernel_lock_dump(volatile void *); 126 127 lockops_t _kernel_lock_ops = { 128 "Kernel lock", 129 0, 130 _kernel_lock_dump 131 }; 132 133 /* 134 * Initialize the kernel lock. 135 */ 136 void 137 kernel_lock_init(void) 138 { 139 140 KASSERT(CACHE_LINE_SIZE >= sizeof(__cpu_simple_lock_t)); 141 __cpu_simple_lock_init(kernel_lock); 142 kernel_lock_dodebug = LOCKDEBUG_ALLOC(kernel_lock, &_kernel_lock_ops, 143 RETURN_ADDRESS); 144 } 145 146 /* 147 * Print debugging information about the kernel lock. 148 */ 149 void 150 _kernel_lock_dump(volatile void *junk) 151 { 152 struct cpu_info *ci = curcpu(); 153 154 (void)junk; 155 156 printf_nolog("curcpu holds : %18d wanted by: %#018lx\n", 157 ci->ci_biglock_count, (long)ci->ci_biglock_wanted); 158 } 159 160 /* 161 * Acquire 'nlocks' holds on the kernel lock. If 'l' is non-null, the 162 * acquisition is from process context. 163 */ 164 void 165 _kernel_lock(int nlocks) 166 { 167 struct cpu_info *ci = curcpu(); 168 LOCKSTAT_TIMER(spintime); 169 LOCKSTAT_FLAG(lsflag); 170 struct lwp *owant; 171 u_int spins; 172 int s; 173 struct lwp *l = curlwp; 174 175 _KERNEL_LOCK_ASSERT(nlocks > 0); 176 177 if (ci->ci_biglock_count != 0) { 178 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock)); 179 ci->ci_biglock_count += nlocks; 180 l->l_blcnt += nlocks; 181 return; 182 } 183 184 _KERNEL_LOCK_ASSERT(l->l_blcnt == 0); 185 LOCKDEBUG_WANTLOCK(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS, 186 0); 187 188 s = splvm(); 189 if (__cpu_simple_lock_try(kernel_lock)) { 190 ci->ci_biglock_count = nlocks; 191 l->l_blcnt = nlocks; 192 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, 193 RETURN_ADDRESS, 0); 194 splx(s); 195 return; 196 } 197 198 /* 199 * To remove the ordering constraint between adaptive mutexes 200 * and kernel_lock we must make it appear as if this thread is 201 * blocking. For non-interlocked mutex release, a store fence 202 * is required to ensure that the result of any mutex_exit() 203 * by the current LWP becomes visible on the bus before the set 204 * of ci->ci_biglock_wanted becomes visible. 205 */ 206 membar_producer(); 207 owant = ci->ci_biglock_wanted; 208 ci->ci_biglock_wanted = l; 209 210 /* 211 * Spin until we acquire the lock. Once we have it, record the 212 * time spent with lockstat. 213 */ 214 LOCKSTAT_ENTER(lsflag); 215 LOCKSTAT_START_TIMER(lsflag, spintime); 216 217 spins = 0; 218 do { 219 splx(s); 220 while (__SIMPLELOCK_LOCKED_P(kernel_lock)) { 221 if (SPINLOCK_SPINOUT(spins)) { 222 extern volatile int start_init_exec; 223 if (!start_init_exec) 224 _KERNEL_LOCK_ABORT("spinout"); 225 } 226 SPINLOCK_BACKOFF_HOOK; 227 SPINLOCK_SPIN_HOOK; 228 } 229 s = splvm(); 230 } while (!__cpu_simple_lock_try(kernel_lock)); 231 232 ci->ci_biglock_count = nlocks; 233 l->l_blcnt = nlocks; 234 LOCKSTAT_STOP_TIMER(lsflag, spintime); 235 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS, 0); 236 if (owant == NULL) { 237 LOCKSTAT_EVENT_RA(lsflag, kernel_lock, 238 LB_KERNEL_LOCK | LB_SPIN, 1, spintime, RETURN_ADDRESS); 239 } 240 LOCKSTAT_EXIT(lsflag); 241 splx(s); 242 243 /* 244 * Now that we have kernel_lock, reset ci_biglock_wanted. This 245 * store must be unbuffered (immediately visible on the bus) in 246 * order for non-interlocked mutex release to work correctly. 247 * It must be visible before a mutex_exit() can execute on this 248 * processor. 249 * 250 * Note: only where CAS is available in hardware will this be 251 * an unbuffered write, but non-interlocked release cannot be 252 * done on CPUs without CAS in hardware. 253 */ 254 (void)atomic_swap_ptr(&ci->ci_biglock_wanted, owant); 255 256 /* 257 * Issue a memory barrier as we have acquired a lock. This also 258 * prevents stores from a following mutex_exit() being reordered 259 * to occur before our store to ci_biglock_wanted above. 260 */ 261 membar_enter(); 262 } 263 264 /* 265 * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release 266 * all holds. If 'l' is non-null, the release is from process context. 267 */ 268 void 269 _kernel_unlock(int nlocks, int *countp) 270 { 271 struct cpu_info *ci = curcpu(); 272 u_int olocks; 273 int s; 274 struct lwp *l = curlwp; 275 276 _KERNEL_LOCK_ASSERT(nlocks < 2); 277 278 olocks = l->l_blcnt; 279 280 if (olocks == 0) { 281 _KERNEL_LOCK_ASSERT(nlocks <= 0); 282 if (countp != NULL) 283 *countp = 0; 284 return; 285 } 286 287 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock)); 288 289 if (nlocks == 0) 290 nlocks = olocks; 291 else if (nlocks == -1) { 292 nlocks = 1; 293 _KERNEL_LOCK_ASSERT(olocks == 1); 294 } 295 296 _KERNEL_LOCK_ASSERT(ci->ci_biglock_count >= l->l_blcnt); 297 298 l->l_blcnt -= nlocks; 299 if (ci->ci_biglock_count == nlocks) { 300 s = splvm(); 301 LOCKDEBUG_UNLOCKED(kernel_lock_dodebug, kernel_lock, 302 RETURN_ADDRESS, 0); 303 ci->ci_biglock_count = 0; 304 __cpu_simple_unlock(kernel_lock); 305 splx(s); 306 } else 307 ci->ci_biglock_count -= nlocks; 308 309 if (countp != NULL) 310 *countp = olocks; 311 } 312 #endif /* !_RUMPKERNEL */ 313