1 /* $NetBSD: kern_lock.c,v 1.140 2008/04/28 20:24:03 martin Exp $ */ 2 3 /*- 4 * Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.140 2008/04/28 20:24:03 martin Exp $"); 35 36 #include "opt_multiprocessor.h" 37 38 #include <sys/param.h> 39 #include <sys/proc.h> 40 #include <sys/lock.h> 41 #include <sys/systm.h> 42 #include <sys/kernel.h> 43 #include <sys/lockdebug.h> 44 #include <sys/cpu.h> 45 #include <sys/syslog.h> 46 #include <sys/atomic.h> 47 48 #include <machine/stdarg.h> 49 #include <machine/lock.h> 50 51 #include <dev/lockstat.h> 52 53 #define RETURN_ADDRESS (uintptr_t)__builtin_return_address(0) 54 55 bool kernel_lock_dodebug; 56 57 __cpu_simple_lock_t kernel_lock[CACHE_LINE_SIZE / sizeof(__cpu_simple_lock_t)] 58 __aligned(CACHE_LINE_SIZE); 59 60 #if defined(DEBUG) || defined(LKM) 61 void 62 assert_sleepable(void) 63 { 64 #if !defined(_RUMPKERNEL) 65 const char *reason; 66 67 if (panicstr != NULL) { 68 return; 69 } 70 71 LOCKDEBUG_BARRIER(kernel_lock, 1); 72 73 reason = NULL; 74 if (CURCPU_IDLE_P() && !cold) { 75 reason = "idle"; 76 } 77 if (cpu_intr_p()) { 78 reason = "interrupt"; 79 } 80 if ((curlwp->l_pflag & LP_INTR) != 0) { 81 reason = "softint"; 82 } 83 84 if (reason) { 85 panic("%s: %s caller=%p", __func__, reason, 86 (void *)RETURN_ADDRESS); 87 } 88 #endif /* !defined(_RUMPKERNEL) */ 89 } 90 #endif /* defined(DEBUG) || defined(LKM) */ 91 92 /* 93 * rump doesn't need the kernel lock so force it out. We cannot 94 * currently easily include it for compilation because of 95 * a) SPINLOCK_* b) membar_producer(). They are defined in different 96 * places / way for each arch, so just simply do not bother to 97 * fight a lot for no gain (i.e. pain but still no gain). 98 */ 99 #ifndef _RUMPKERNEL 100 /* 101 * Functions for manipulating the kernel_lock. We put them here 102 * so that they show up in profiles. 103 */ 104 105 #define _KERNEL_LOCK_ABORT(msg) \ 106 LOCKDEBUG_ABORT(kernel_lock, &_kernel_lock_ops, __func__, msg) 107 108 #ifdef LOCKDEBUG 109 #define _KERNEL_LOCK_ASSERT(cond) \ 110 do { \ 111 if (!(cond)) \ 112 _KERNEL_LOCK_ABORT("assertion failed: " #cond); \ 113 } while (/* CONSTCOND */ 0) 114 #else 115 #define _KERNEL_LOCK_ASSERT(cond) /* nothing */ 116 #endif 117 118 void _kernel_lock_dump(volatile void *); 119 120 lockops_t _kernel_lock_ops = { 121 "Kernel lock", 122 0, 123 _kernel_lock_dump 124 }; 125 126 /* 127 * Initialize the kernel lock. 128 */ 129 void 130 kernel_lock_init(void) 131 { 132 133 KASSERT(CACHE_LINE_SIZE >= sizeof(__cpu_simple_lock_t)); 134 __cpu_simple_lock_init(kernel_lock); 135 kernel_lock_dodebug = LOCKDEBUG_ALLOC(kernel_lock, &_kernel_lock_ops, 136 RETURN_ADDRESS); 137 } 138 139 /* 140 * Print debugging information about the kernel lock. 141 */ 142 void 143 _kernel_lock_dump(volatile void *junk) 144 { 145 struct cpu_info *ci = curcpu(); 146 147 (void)junk; 148 149 printf_nolog("curcpu holds : %18d wanted by: %#018lx\n", 150 ci->ci_biglock_count, (long)ci->ci_biglock_wanted); 151 } 152 153 /* 154 * Acquire 'nlocks' holds on the kernel lock. If 'l' is non-null, the 155 * acquisition is from process context. 156 */ 157 void 158 _kernel_lock(int nlocks) 159 { 160 struct cpu_info *ci; 161 LOCKSTAT_TIMER(spintime); 162 LOCKSTAT_FLAG(lsflag); 163 struct lwp *owant; 164 u_int spins; 165 int s; 166 struct lwp *l = curlwp; 167 168 _KERNEL_LOCK_ASSERT(nlocks > 0); 169 170 s = splvm(); 171 ci = curcpu(); 172 if (ci->ci_biglock_count != 0) { 173 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock)); 174 ci->ci_biglock_count += nlocks; 175 l->l_blcnt += nlocks; 176 splx(s); 177 return; 178 } 179 180 _KERNEL_LOCK_ASSERT(l->l_blcnt == 0); 181 LOCKDEBUG_WANTLOCK(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS, 182 0); 183 184 if (__cpu_simple_lock_try(kernel_lock)) { 185 ci->ci_biglock_count = nlocks; 186 l->l_blcnt = nlocks; 187 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, 188 RETURN_ADDRESS, 0); 189 splx(s); 190 return; 191 } 192 193 /* 194 * To remove the ordering constraint between adaptive mutexes 195 * and kernel_lock we must make it appear as if this thread is 196 * blocking. For non-interlocked mutex release, a store fence 197 * is required to ensure that the result of any mutex_exit() 198 * by the current LWP becomes visible on the bus before the set 199 * of ci->ci_biglock_wanted becomes visible. 200 */ 201 membar_producer(); 202 owant = ci->ci_biglock_wanted; 203 ci->ci_biglock_wanted = l; 204 205 /* 206 * Spin until we acquire the lock. Once we have it, record the 207 * time spent with lockstat. 208 */ 209 LOCKSTAT_ENTER(lsflag); 210 LOCKSTAT_START_TIMER(lsflag, spintime); 211 212 spins = 0; 213 do { 214 splx(s); 215 while (__SIMPLELOCK_LOCKED_P(kernel_lock)) { 216 if (SPINLOCK_SPINOUT(spins)) { 217 extern volatile int start_init_exec; 218 if (!start_init_exec) 219 _KERNEL_LOCK_ABORT("spinout"); 220 } 221 SPINLOCK_BACKOFF_HOOK; 222 SPINLOCK_SPIN_HOOK; 223 } 224 s = splvm(); 225 } while (!__cpu_simple_lock_try(kernel_lock)); 226 227 ci->ci_biglock_count = nlocks; 228 l->l_blcnt = nlocks; 229 LOCKSTAT_STOP_TIMER(lsflag, spintime); 230 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS, 0); 231 if (owant == NULL) { 232 LOCKSTAT_EVENT_RA(lsflag, kernel_lock, 233 LB_KERNEL_LOCK | LB_SPIN, 1, spintime, RETURN_ADDRESS); 234 } 235 LOCKSTAT_EXIT(lsflag); 236 splx(s); 237 238 /* 239 * Now that we have kernel_lock, reset ci_biglock_wanted. This 240 * store must be unbuffered (immediately visible on the bus) in 241 * order for non-interlocked mutex release to work correctly. 242 * It must be visible before a mutex_exit() can execute on this 243 * processor. 244 * 245 * Note: only where CAS is available in hardware will this be 246 * an unbuffered write, but non-interlocked release cannot be 247 * done on CPUs without CAS in hardware. 248 */ 249 (void)atomic_swap_ptr(&ci->ci_biglock_wanted, owant); 250 251 /* 252 * Issue a memory barrier as we have acquired a lock. This also 253 * prevents stores from a following mutex_exit() being reordered 254 * to occur before our store to ci_biglock_wanted above. 255 */ 256 membar_enter(); 257 } 258 259 /* 260 * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release 261 * all holds. If 'l' is non-null, the release is from process context. 262 */ 263 void 264 _kernel_unlock(int nlocks, int *countp) 265 { 266 struct cpu_info *ci; 267 u_int olocks; 268 int s; 269 struct lwp *l = curlwp; 270 271 _KERNEL_LOCK_ASSERT(nlocks < 2); 272 273 olocks = l->l_blcnt; 274 275 if (olocks == 0) { 276 _KERNEL_LOCK_ASSERT(nlocks <= 0); 277 if (countp != NULL) 278 *countp = 0; 279 return; 280 } 281 282 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock)); 283 284 if (nlocks == 0) 285 nlocks = olocks; 286 else if (nlocks == -1) { 287 nlocks = 1; 288 _KERNEL_LOCK_ASSERT(olocks == 1); 289 } 290 s = splvm(); 291 ci = curcpu(); 292 _KERNEL_LOCK_ASSERT(ci->ci_biglock_count >= l->l_blcnt); 293 if (ci->ci_biglock_count == nlocks) { 294 LOCKDEBUG_UNLOCKED(kernel_lock_dodebug, kernel_lock, 295 RETURN_ADDRESS, 0); 296 ci->ci_biglock_count = 0; 297 __cpu_simple_unlock(kernel_lock); 298 l->l_blcnt -= nlocks; 299 splx(s); 300 if (l->l_dopreempt) 301 kpreempt(0); 302 } else { 303 ci->ci_biglock_count -= nlocks; 304 l->l_blcnt -= nlocks; 305 splx(s); 306 } 307 308 if (countp != NULL) 309 *countp = olocks; 310 } 311 #endif /* !_RUMPKERNEL */ 312