1 /* $OpenBSD: kern_lock.c,v 1.52 2017/12/04 09:51:03 mpi Exp $ */ 2 3 /* 4 * Copyright (c) 1995 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code contains ideas from software contributed to Berkeley by 8 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 9 * System project at Carnegie-Mellon University. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. Neither the name of the University nor the names of its contributors 20 * may be used to endorse or promote products derived from this software 21 * without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 * 35 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 36 */ 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/sched.h> 41 #include <sys/atomic.h> 42 #include <sys/witness.h> 43 44 #include <ddb/db_output.h> 45 46 #ifdef MP_LOCKDEBUG 47 #ifndef DDB 48 #error "MP_LOCKDEBUG requires DDB" 49 #endif 50 51 /* CPU-dependent timing, this needs to be settable from ddb. */ 52 int __mp_lock_spinout = 200000000; 53 #endif /* MP_LOCKDEBUG */ 54 55 #if defined(MULTIPROCESSOR) || defined(WITNESS) 56 struct __mp_lock kernel_lock; 57 #endif 58 59 #ifdef MULTIPROCESSOR 60 61 /* 62 * Functions for manipulating the kernel_lock. We put them here 63 * so that they show up in profiles. 64 */ 65 66 void 67 _kernel_lock_init(void) 68 { 69 __mp_lock_init(&kernel_lock); 70 } 71 72 /* 73 * Acquire/release the kernel lock. Intended for use in the scheduler 74 * and the lower half of the kernel. 75 */ 76 77 void 78 _kernel_lock(const char *file, int line) 79 { 80 SCHED_ASSERT_UNLOCKED(); 81 #ifdef WITNESS 82 ___mp_lock(&kernel_lock, file, line); 83 #else 84 __mp_lock(&kernel_lock); 85 #endif 86 } 87 88 void 89 _kernel_unlock(void) 90 { 91 __mp_unlock(&kernel_lock); 92 } 93 94 int 95 _kernel_lock_held(void) 96 { 97 if (panicstr) 98 return 1; 99 return (__mp_lock_held(&kernel_lock, curcpu())); 100 } 101 102 #ifdef __USE_MI_MPLOCK 103 104 /* Ticket lock implementation */ 105 /* 106 * Copyright (c) 2014 David Gwynne <dlg@openbsd.org> 107 * 108 * Permission to use, copy, modify, and distribute this software for any 109 * purpose with or without fee is hereby granted, provided that the above 110 * copyright notice and this permission notice appear in all copies. 111 * 112 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 113 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 114 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 115 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 116 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 117 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 118 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 119 */ 120 121 #include <machine/cpu.h> 122 123 void 124 ___mp_lock_init(struct __mp_lock *mpl, struct lock_type *type) 125 { 126 memset(mpl->mpl_cpus, 0, sizeof(mpl->mpl_cpus)); 127 mpl->mpl_users = 0; 128 mpl->mpl_ticket = 1; 129 130 #ifdef WITNESS 131 mpl->mpl_lock_obj.lo_name = type->lt_name; 132 mpl->mpl_lock_obj.lo_type = type; 133 if (mpl == &kernel_lock) 134 mpl->mpl_lock_obj.lo_flags = LO_WITNESS | LO_INITIALIZED | 135 LO_SLEEPABLE | (LO_CLASS_KERNEL_LOCK << LO_CLASSSHIFT); 136 else if (mpl == &sched_lock) 137 mpl->mpl_lock_obj.lo_flags = LO_WITNESS | LO_INITIALIZED | 138 LO_RECURSABLE | (LO_CLASS_SCHED_LOCK << LO_CLASSSHIFT); 139 WITNESS_INIT(&mpl->mpl_lock_obj, type); 140 #endif 141 } 142 143 static __inline void 144 __mp_lock_spin(struct __mp_lock *mpl, u_int me) 145 { 146 #ifndef MP_LOCKDEBUG 147 while (mpl->mpl_ticket != me) 148 CPU_BUSY_CYCLE(); 149 #else 150 int nticks = __mp_lock_spinout; 151 152 while (mpl->mpl_ticket != me) { 153 CPU_BUSY_CYCLE(); 154 155 if (--nticks <= 0) { 156 db_printf("__mp_lock(%p): lock spun out", mpl); 157 db_enter(); 158 nticks = __mp_lock_spinout; 159 } 160 } 161 #endif 162 } 163 164 void 165 ___mp_lock(struct __mp_lock *mpl LOCK_FL_VARS) 166 { 167 struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()]; 168 unsigned long s; 169 170 #ifdef WITNESS 171 if (!__mp_lock_held(mpl, curcpu())) 172 WITNESS_CHECKORDER(&mpl->mpl_lock_obj, 173 LOP_EXCLUSIVE | LOP_NEWORDER, file, line, NULL); 174 #endif 175 176 s = intr_disable(); 177 if (cpu->mplc_depth++ == 0) 178 cpu->mplc_ticket = atomic_inc_int_nv(&mpl->mpl_users); 179 intr_restore(s); 180 181 __mp_lock_spin(mpl, cpu->mplc_ticket); 182 membar_enter_after_atomic(); 183 184 WITNESS_LOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line); 185 } 186 187 void 188 ___mp_unlock(struct __mp_lock *mpl LOCK_FL_VARS) 189 { 190 struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()]; 191 unsigned long s; 192 193 #ifdef MP_LOCKDEBUG 194 if (!__mp_lock_held(mpl, curcpu())) { 195 db_printf("__mp_unlock(%p): not held lock\n", mpl); 196 db_enter(); 197 } 198 #endif 199 200 WITNESS_UNLOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line); 201 202 s = intr_disable(); 203 if (--cpu->mplc_depth == 0) { 204 membar_exit(); 205 mpl->mpl_ticket++; 206 } 207 intr_restore(s); 208 } 209 210 int 211 ___mp_release_all(struct __mp_lock *mpl LOCK_FL_VARS) 212 { 213 struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()]; 214 unsigned long s; 215 int rv; 216 #ifdef WITNESS 217 int i; 218 #endif 219 220 s = intr_disable(); 221 rv = cpu->mplc_depth; 222 #ifdef WITNESS 223 for (i = 0; i < rv; i++) 224 WITNESS_UNLOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line); 225 #endif 226 cpu->mplc_depth = 0; 227 membar_exit(); 228 mpl->mpl_ticket++; 229 intr_restore(s); 230 231 return (rv); 232 } 233 234 int 235 ___mp_release_all_but_one(struct __mp_lock *mpl LOCK_FL_VARS) 236 { 237 struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()]; 238 int rv = cpu->mplc_depth - 1; 239 #ifdef WITNESS 240 int i; 241 242 for (i = 0; i < rv; i++) 243 WITNESS_UNLOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line); 244 #endif 245 246 #ifdef MP_LOCKDEBUG 247 if (!__mp_lock_held(mpl, curcpu())) { 248 db_printf("__mp_release_all_but_one(%p): not held lock\n", mpl); 249 db_enter(); 250 } 251 #endif 252 253 cpu->mplc_depth = 1; 254 255 return (rv); 256 } 257 258 void 259 ___mp_acquire_count(struct __mp_lock *mpl, int count LOCK_FL_VARS) 260 { 261 while (count--) 262 ___mp_lock(mpl LOCK_FL_ARGS); 263 } 264 265 int 266 __mp_lock_held(struct __mp_lock *mpl, struct cpu_info *ci) 267 { 268 struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[CPU_INFO_UNIT(ci)]; 269 270 return (cpu->mplc_ticket == mpl->mpl_ticket && cpu->mplc_depth > 0); 271 } 272 273 #endif /* __USE_MI_MPLOCK */ 274 275 #endif /* MULTIPROCESSOR */ 276