1 /* $NetBSD: subr_pserialize.c,v 1.19 2022/11/15 10:29:56 macallan Exp $ */ 2 3 /*- 4 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * Passive serialization. 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: subr_pserialize.c,v 1.19 2022/11/15 10:29:56 macallan Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/atomic.h> 38 #include <sys/cpu.h> 39 #include <sys/kernel.h> 40 #include <sys/evcnt.h> 41 #include <sys/kmem.h> 42 #include <sys/mutex.h> 43 #include <sys/pserialize.h> 44 #include <sys/xcall.h> 45 46 struct pserialize { 47 char psz_dummy; 48 }; 49 50 static kmutex_t psz_lock __cacheline_aligned; 51 static struct evcnt psz_ev_excl __cacheline_aligned = 52 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pserialize", "exclusive access"); 53 EVCNT_ATTACH_STATIC(psz_ev_excl); 54 55 /* 56 * pserialize_init: 57 * 58 * Initialize passive serialization structures. 59 */ 60 void 61 pserialize_init(void) 62 { 63 64 mutex_init(&psz_lock, MUTEX_DEFAULT, IPL_NONE); 65 } 66 67 /* 68 * pserialize_create: 69 * 70 * Create and initialize a passive serialization object. 71 */ 72 pserialize_t 73 pserialize_create(void) 74 { 75 pserialize_t psz; 76 77 psz = kmem_zalloc(sizeof(*psz), KM_SLEEP); 78 return psz; 79 } 80 81 /* 82 * pserialize_destroy: 83 * 84 * Destroy a passive serialization object. 85 */ 86 void 87 pserialize_destroy(pserialize_t psz) 88 { 89 90 kmem_free(psz, sizeof(*psz)); 91 } 92 93 /* 94 * pserialize_perform: 95 * 96 * Perform the write side of passive serialization. 97 */ 98 void 99 pserialize_perform(pserialize_t psz) 100 { 101 102 KASSERT(!cpu_intr_p()); 103 KASSERT(!cpu_softintr_p()); 104 105 if (__predict_false(panicstr != NULL)) { 106 return; 107 } 108 109 if (__predict_false(mp_online == false)) { 110 psz_ev_excl.ev_count++; 111 return; 112 } 113 114 /* 115 * Broadcast a NOP to all CPUs and wait until all of them complete. 116 */ 117 xc_barrier(XC_HIGHPRI); 118 119 mutex_enter(&psz_lock); 120 psz_ev_excl.ev_count++; 121 mutex_exit(&psz_lock); 122 } 123 124 int 125 pserialize_read_enter(void) 126 { 127 int s; 128 129 s = splsoftserial(); 130 curcpu()->ci_psz_read_depth++; 131 __insn_barrier(); 132 return s; 133 } 134 135 void 136 pserialize_read_exit(int s) 137 { 138 139 KASSERT((cold || kpreempt_disabled())); 140 141 __insn_barrier(); 142 if (__predict_false(curcpu()->ci_psz_read_depth-- == 0)) 143 panic("mismatching pserialize_read_exit()"); 144 splx(s); 145 } 146 147 /* 148 * pserialize_in_read_section: 149 * 150 * True if the caller is in a pserialize read section. To be used 151 * only for diagnostic assertions where we want to guarantee the 152 * condition like: 153 * 154 * KASSERT(pserialize_in_read_section()); 155 */ 156 bool 157 pserialize_in_read_section(void) 158 { 159 160 return kpreempt_disabled() && curcpu()->ci_psz_read_depth > 0; 161 } 162 163 /* 164 * pserialize_not_in_read_section: 165 * 166 * True if the caller is not in a pserialize read section. To be 167 * used only for diagnostic assertions where we want to guarantee 168 * the condition like: 169 * 170 * KASSERT(pserialize_not_in_read_section()); 171 */ 172 bool 173 pserialize_not_in_read_section(void) 174 { 175 bool notin; 176 177 kpreempt_disable(); 178 notin = (curcpu()->ci_psz_read_depth == 0); 179 kpreempt_enable(); 180 181 return notin; 182 } 183