1 /* $NetBSD: kern_lock.c,v 1.153 2012/08/30 02:23:14 matt Exp $ */ 2 3 /*- 4 * Copyright (c) 2002, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, and by Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.153 2012/08/30 02:23:14 matt Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/proc.h> 38 #include <sys/lock.h> 39 #include <sys/systm.h> 40 #include <sys/kernel.h> 41 #include <sys/lockdebug.h> 42 #include <sys/cpu.h> 43 #include <sys/syslog.h> 44 #include <sys/atomic.h> 45 #include <sys/lwp.h> 46 47 #include <machine/lock.h> 48 49 #include <dev/lockstat.h> 50 51 #define RETURN_ADDRESS (uintptr_t)__builtin_return_address(0) 52 53 bool kernel_lock_dodebug; 54 55 __cpu_simple_lock_t kernel_lock[CACHE_LINE_SIZE / sizeof(__cpu_simple_lock_t)] 56 __cacheline_aligned; 57 58 void 59 assert_sleepable(void) 60 { 61 const char *reason; 62 uint64_t pctr; 63 bool idle; 64 65 if (panicstr != NULL) { 66 return; 67 } 68 69 LOCKDEBUG_BARRIER(kernel_lock, 1); 70 71 /* 72 * Avoid disabling/re-enabling preemption here since this 73 * routine may be called in delicate situations. 74 */ 75 do { 76 pctr = lwp_pctr(); 77 idle = CURCPU_IDLE_P(); 78 } while (pctr != lwp_pctr()); 79 80 reason = NULL; 81 if (idle && !cold) { 82 reason = "idle"; 83 } 84 if (cpu_intr_p()) { 85 reason = "interrupt"; 86 } 87 if (cpu_softintr_p()) { 88 reason = "softint"; 89 } 90 91 if (reason) { 92 panic("%s: %s caller=%p", __func__, reason, 93 (void *)RETURN_ADDRESS); 94 } 95 } 96 97 /* 98 * Functions for manipulating the kernel_lock. We put them here 99 * so that they show up in profiles. 100 */ 101 102 #define _KERNEL_LOCK_ABORT(msg) \ 103 LOCKDEBUG_ABORT(kernel_lock, &_kernel_lock_ops, __func__, msg) 104 105 #ifdef LOCKDEBUG 106 #define _KERNEL_LOCK_ASSERT(cond) \ 107 do { \ 108 if (!(cond)) \ 109 _KERNEL_LOCK_ABORT("assertion failed: " #cond); \ 110 } while (/* CONSTCOND */ 0) 111 #else 112 #define _KERNEL_LOCK_ASSERT(cond) /* nothing */ 113 #endif 114 115 void _kernel_lock_dump(volatile void *); 116 117 lockops_t _kernel_lock_ops = { 118 "Kernel lock", 119 LOCKOPS_SPIN, 120 _kernel_lock_dump 121 }; 122 123 /* 124 * Initialize the kernel lock. 125 */ 126 void 127 kernel_lock_init(void) 128 { 129 130 CTASSERT(CACHE_LINE_SIZE >= sizeof(__cpu_simple_lock_t)); 131 __cpu_simple_lock_init(kernel_lock); 132 kernel_lock_dodebug = LOCKDEBUG_ALLOC(kernel_lock, &_kernel_lock_ops, 133 RETURN_ADDRESS); 134 } 135 136 /* 137 * Print debugging information about the kernel lock. 138 */ 139 void 140 _kernel_lock_dump(volatile void *junk) 141 { 142 struct cpu_info *ci = curcpu(); 143 144 (void)junk; 145 146 printf_nolog("curcpu holds : %18d wanted by: %#018lx\n", 147 ci->ci_biglock_count, (long)ci->ci_biglock_wanted); 148 } 149 150 /* 151 * Acquire 'nlocks' holds on the kernel lock. 152 */ 153 void 154 _kernel_lock(int nlocks) 155 { 156 struct cpu_info *ci; 157 LOCKSTAT_TIMER(spintime); 158 LOCKSTAT_FLAG(lsflag); 159 struct lwp *owant; 160 u_int spins; 161 int s; 162 struct lwp *l = curlwp; 163 164 _KERNEL_LOCK_ASSERT(nlocks > 0); 165 166 s = splvm(); 167 ci = curcpu(); 168 if (ci->ci_biglock_count != 0) { 169 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock)); 170 ci->ci_biglock_count += nlocks; 171 l->l_blcnt += nlocks; 172 splx(s); 173 return; 174 } 175 176 _KERNEL_LOCK_ASSERT(l->l_blcnt == 0); 177 LOCKDEBUG_WANTLOCK(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS, 178 false, false); 179 180 if (__cpu_simple_lock_try(kernel_lock)) { 181 ci->ci_biglock_count = nlocks; 182 l->l_blcnt = nlocks; 183 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL, 184 RETURN_ADDRESS, 0); 185 splx(s); 186 return; 187 } 188 189 /* 190 * To remove the ordering constraint between adaptive mutexes 191 * and kernel_lock we must make it appear as if this thread is 192 * blocking. For non-interlocked mutex release, a store fence 193 * is required to ensure that the result of any mutex_exit() 194 * by the current LWP becomes visible on the bus before the set 195 * of ci->ci_biglock_wanted becomes visible. 196 */ 197 membar_producer(); 198 owant = ci->ci_biglock_wanted; 199 ci->ci_biglock_wanted = l; 200 201 /* 202 * Spin until we acquire the lock. Once we have it, record the 203 * time spent with lockstat. 204 */ 205 LOCKSTAT_ENTER(lsflag); 206 LOCKSTAT_START_TIMER(lsflag, spintime); 207 208 spins = 0; 209 do { 210 splx(s); 211 while (__SIMPLELOCK_LOCKED_P(kernel_lock)) { 212 if (SPINLOCK_SPINOUT(spins)) { 213 extern int start_init_exec; 214 if (!start_init_exec) 215 _KERNEL_LOCK_ABORT("spinout"); 216 } 217 SPINLOCK_BACKOFF_HOOK; 218 SPINLOCK_SPIN_HOOK; 219 } 220 s = splvm(); 221 } while (!__cpu_simple_lock_try(kernel_lock)); 222 223 ci->ci_biglock_count = nlocks; 224 l->l_blcnt = nlocks; 225 LOCKSTAT_STOP_TIMER(lsflag, spintime); 226 LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL, 227 RETURN_ADDRESS, 0); 228 if (owant == NULL) { 229 LOCKSTAT_EVENT_RA(lsflag, kernel_lock, 230 LB_KERNEL_LOCK | LB_SPIN, 1, spintime, RETURN_ADDRESS); 231 } 232 LOCKSTAT_EXIT(lsflag); 233 splx(s); 234 235 /* 236 * Now that we have kernel_lock, reset ci_biglock_wanted. This 237 * store must be unbuffered (immediately visible on the bus) in 238 * order for non-interlocked mutex release to work correctly. 239 * It must be visible before a mutex_exit() can execute on this 240 * processor. 241 * 242 * Note: only where CAS is available in hardware will this be 243 * an unbuffered write, but non-interlocked release cannot be 244 * done on CPUs without CAS in hardware. 245 */ 246 (void)atomic_swap_ptr(&ci->ci_biglock_wanted, owant); 247 248 /* 249 * Issue a memory barrier as we have acquired a lock. This also 250 * prevents stores from a following mutex_exit() being reordered 251 * to occur before our store to ci_biglock_wanted above. 252 */ 253 membar_enter(); 254 } 255 256 /* 257 * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release 258 * all holds. 259 */ 260 void 261 _kernel_unlock(int nlocks, int *countp) 262 { 263 struct cpu_info *ci; 264 u_int olocks; 265 int s; 266 struct lwp *l = curlwp; 267 268 _KERNEL_LOCK_ASSERT(nlocks < 2); 269 270 olocks = l->l_blcnt; 271 272 if (olocks == 0) { 273 _KERNEL_LOCK_ASSERT(nlocks <= 0); 274 if (countp != NULL) 275 *countp = 0; 276 return; 277 } 278 279 _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock)); 280 281 if (nlocks == 0) 282 nlocks = olocks; 283 else if (nlocks == -1) { 284 nlocks = 1; 285 _KERNEL_LOCK_ASSERT(olocks == 1); 286 } 287 s = splvm(); 288 ci = curcpu(); 289 _KERNEL_LOCK_ASSERT(ci->ci_biglock_count >= l->l_blcnt); 290 if (ci->ci_biglock_count == nlocks) { 291 LOCKDEBUG_UNLOCKED(kernel_lock_dodebug, kernel_lock, 292 RETURN_ADDRESS, 0); 293 ci->ci_biglock_count = 0; 294 __cpu_simple_unlock(kernel_lock); 295 l->l_blcnt -= nlocks; 296 splx(s); 297 if (l->l_dopreempt) 298 kpreempt(0); 299 } else { 300 ci->ci_biglock_count -= nlocks; 301 l->l_blcnt -= nlocks; 302 splx(s); 303 } 304 305 if (countp != NULL) 306 *countp = olocks; 307 } 308 309 bool 310 _kernel_locked_p(void) 311 { 312 return __SIMPLELOCK_LOCKED_P(kernel_lock); 313 } 314