1 /* $NetBSD: kern_lock.c,v 1.146 2008/07/02 14:47:34 matt Exp $ */ 2 3 /*- 4 * Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.146 2008/07/02 14:47:34 matt Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/proc.h> 38 #include <sys/lock.h> 39 #include <sys/systm.h> 40 #include <sys/kernel.h> 41 #include <sys/lockdebug.h> 42 #include <sys/cpu.h> 43 #include <sys/syslog.h> 44 #include <sys/atomic.h> 45 46 #include <machine/stdarg.h> 47 #include <machine/lock.h> 48 49 #include <dev/lockstat.h> 50 51 #define RETURN_ADDRESS (uintptr_t)__builtin_return_address(0) 52 53 bool kernel_lock_dodebug; 54 55 __cpu_simple_lock_t kernel_lock[CACHE_LINE_SIZE / sizeof(__cpu_simple_lock_t)] 56 __aligned(CACHE_LINE_SIZE); 57 58 #if defined(DEBUG) || defined(LKM) 59 void 60 assert_sleepable(void) 61 { 62 const char *reason; 63 64 if (panicstr != NULL) { 65 return; 66 } 67 68 LOCKDEBUG_BARRIER(kernel_lock, 1); 69 70 reason = NULL; 71 if (CURCPU_IDLE_P() && !cold) { 72 reason = "idle"; 73 } 74 if (cpu_intr_p()) { 75 reason = "interrupt"; 76 } 77 if ((curlwp->l_pflag & LP_INTR) != 0) { 78 reason = "softint"; 79 } 80 81 if (reason) { 82 panic("%s: %s caller=%p", __func__, reason, 83 (void *)RETURN_ADDRESS); 84 } 85 } 86 #endif /* defined(DEBUG) || defined(LKM) */ 87 88 /* 89 * Functions for manipulating the kernel_lock. We put them here 90 * so that they show up in profiles. 91 */ 92 93 #define _KERNEL_LOCK_ABORT(msg) \ 94 LOCKDEBUG_ABORT(kernel_lock, &_kernel_lock_ops, __func__, msg) 95 96 #ifdef LOCKDEBUG 97 #define _KERNEL_LOCK_ASSERT(cond) \ 98 do { \ 99 if (!(cond)) \ 100 _KERNEL_LOCK_ABORT("assertion failed: " #cond); \ 101 } while (/* CONSTCOND */ 0) 102 #else 103 #define _KERNEL_LOCK_ASSERT(cond) /* nothing */ 104 #endif 105 106 void _kernel_lock_dump(volatile void *); 107 108 lockops_t _kernel_lock_ops = { 109 "Kernel lock", 110 LOCKOPS_SPIN, 111 _kernel_lock_dump 112 }; 113 114 /* 115 * Initialize the kernel lock. 116 */ 117 void 118 kernel_lock_init(void) 119 { 120 121 CTASSERT(CACHE_LINE_SIZE >= sizeof(__cpu_simple_lock_t)); 122 __cpu_simple_lock_init(kernel_lock); 123 kernel_lock_dodebug = LOCKDEBUG_ALLOC(kernel_lock, &_kernel_lock_ops, 124 RETURN_ADDRESS); 125 } 126 127 /* 128 * Print debugging information about the kernel lock. 129 */ 130 void 131 _kernel_lock_dump(volatile void *junk) 132 { 133 struct cpu_info *ci = curcpu(); 134 135 (void)junk; 136 137 printf_nolog("curcpu holds : %18d wanted by: %#018lx\n", 138 ci->ci_biglock_count, (long)ci->ci_biglock_wanted); 139 } 140 141 /* 142 * Acquire 'nlocks' holds on the kernel lock. If 'l' is non-null, the 143 * acquisition is from process context. 144 */ 145 void 146 _kernel_lock(int nlocks) 147 { 148 struct cpu_info *ci; 149 LOCKSTAT_TIMER(spintime); 150 LOCKSTAT_FLAG(lsflag); 151 struct lwp *owant; 152 u_int spins; 153 int s; 154 struct lwp *l = curlwp; 155 156 _KERNEL_LOCK_ASSERT(nlocks > 0); 157 158 s = splvm(); 159 ci = curcpu(); 160 if (ci->ci_biglock_count != 0) { 161 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock)); 162 ci->ci_biglock_count += nlocks; 163 l->l_blcnt += nlocks; 164 splx(s); 165 return; 166 } 167 168 _KERNEL_LOCK_ASSERT(l->l_blcnt == 0); 169 LOCKDEBUG_WANTLOCK(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS, 170 false, false); 171 172 if (__cpu_simple_lock_try(kernel_lock)) { 173 ci->ci_biglock_count = nlocks; 174 l->l_blcnt = nlocks; 175 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL, 176 RETURN_ADDRESS, 0); 177 splx(s); 178 return; 179 } 180 181 /* 182 * To remove the ordering constraint between adaptive mutexes 183 * and kernel_lock we must make it appear as if this thread is 184 * blocking. For non-interlocked mutex release, a store fence 185 * is required to ensure that the result of any mutex_exit() 186 * by the current LWP becomes visible on the bus before the set 187 * of ci->ci_biglock_wanted becomes visible. 188 */ 189 membar_producer(); 190 owant = ci->ci_biglock_wanted; 191 ci->ci_biglock_wanted = l; 192 193 /* 194 * Spin until we acquire the lock. Once we have it, record the 195 * time spent with lockstat. 196 */ 197 LOCKSTAT_ENTER(lsflag); 198 LOCKSTAT_START_TIMER(lsflag, spintime); 199 200 spins = 0; 201 do { 202 splx(s); 203 while (__SIMPLELOCK_LOCKED_P(kernel_lock)) { 204 if (SPINLOCK_SPINOUT(spins)) { 205 extern int start_init_exec; 206 if (!start_init_exec) 207 _KERNEL_LOCK_ABORT("spinout"); 208 } 209 SPINLOCK_BACKOFF_HOOK; 210 SPINLOCK_SPIN_HOOK; 211 } 212 s = splvm(); 213 } while (!__cpu_simple_lock_try(kernel_lock)); 214 215 ci->ci_biglock_count = nlocks; 216 l->l_blcnt = nlocks; 217 LOCKSTAT_STOP_TIMER(lsflag, spintime); 218 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL, 219 RETURN_ADDRESS, 0); 220 if (owant == NULL) { 221 LOCKSTAT_EVENT_RA(lsflag, kernel_lock, 222 LB_KERNEL_LOCK | LB_SPIN, 1, spintime, RETURN_ADDRESS); 223 } 224 LOCKSTAT_EXIT(lsflag); 225 splx(s); 226 227 /* 228 * Now that we have kernel_lock, reset ci_biglock_wanted. This 229 * store must be unbuffered (immediately visible on the bus) in 230 * order for non-interlocked mutex release to work correctly. 231 * It must be visible before a mutex_exit() can execute on this 232 * processor. 233 * 234 * Note: only where CAS is available in hardware will this be 235 * an unbuffered write, but non-interlocked release cannot be 236 * done on CPUs without CAS in hardware. 237 */ 238 (void)atomic_swap_ptr(&ci->ci_biglock_wanted, owant); 239 240 /* 241 * Issue a memory barrier as we have acquired a lock. This also 242 * prevents stores from a following mutex_exit() being reordered 243 * to occur before our store to ci_biglock_wanted above. 244 */ 245 membar_enter(); 246 } 247 248 /* 249 * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release 250 * all holds. If 'l' is non-null, the release is from process context. 251 */ 252 void 253 _kernel_unlock(int nlocks, int *countp) 254 { 255 struct cpu_info *ci; 256 u_int olocks; 257 int s; 258 struct lwp *l = curlwp; 259 260 _KERNEL_LOCK_ASSERT(nlocks < 2); 261 262 olocks = l->l_blcnt; 263 264 if (olocks == 0) { 265 _KERNEL_LOCK_ASSERT(nlocks <= 0); 266 if (countp != NULL) 267 *countp = 0; 268 return; 269 } 270 271 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock)); 272 273 if (nlocks == 0) 274 nlocks = olocks; 275 else if (nlocks == -1) { 276 nlocks = 1; 277 _KERNEL_LOCK_ASSERT(olocks == 1); 278 } 279 s = splvm(); 280 ci = curcpu(); 281 _KERNEL_LOCK_ASSERT(ci->ci_biglock_count >= l->l_blcnt); 282 if (ci->ci_biglock_count == nlocks) { 283 LOCKDEBUG_UNLOCKED(kernel_lock_dodebug, kernel_lock, 284 RETURN_ADDRESS, 0); 285 ci->ci_biglock_count = 0; 286 __cpu_simple_unlock(kernel_lock); 287 l->l_blcnt -= nlocks; 288 splx(s); 289 if (l->l_dopreempt) 290 kpreempt(0); 291 } else { 292 ci->ci_biglock_count -= nlocks; 293 l->l_blcnt -= nlocks; 294 splx(s); 295 } 296 297 if (countp != NULL) 298 *countp = olocks; 299 } 300