1 /* $NetBSD: subr_pcu.c,v 1.3 2011/02/19 20:19:54 matt Exp $ */ 2 3 /*- 4 * Copyright (c) 2011 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Per CPU Unit (PCU) - is an interface to manage synchronization of any 34 * per CPU context (unit) tied with LWP context. Typical use: FPU state. 35 * 36 * Concurrency notes: 37 * 38 * PCU state may be loaded only by the current LWP, that is, curlwp. 39 * Therefore, only LWP itself can set a CPU for lwp_t::l_pcu_cpu[id]. 40 * 41 * Request for a PCU release can be from owner LWP (whether PCU state 42 * is on current CPU or remote CPU) or any other LWP running on that 43 * CPU (in such case, owner LWP is on a remote CPU or sleeping). 44 * 45 * In any case, PCU state can only be changed from the running CPU. 46 * If said PCU state is on the remote CPU, a cross-call will be sent 47 * by the owner LWP. Therefore struct cpu_info::ci_pcu_curlwp[id] 48 * may only be changed by current CPU, and lwp_t::l_pcu_cpu[id] may 49 * only be unset by the CPU which has PCU state loaded. 50 * 51 * There is a race condition: LWP may have a PCU state on a remote CPU, 52 * which it requests to be released via cross-call. At the same time, 53 * other LWP on remote CPU might release existing PCU state and load 54 * its own one. Cross-call may arrive after this and release different 55 * PCU state than intended. In such case, such LWP would re-load its 56 * PCU state again. 57 */ 58 59 #include <sys/cdefs.h> 60 __KERNEL_RCSID(0, "$NetBSD: subr_pcu.c,v 1.3 2011/02/19 20:19:54 matt Exp $"); 61 62 #include <sys/param.h> 63 #include <sys/cpu.h> 64 #include <sys/lwp.h> 65 #include <sys/pcu.h> 66 #include <sys/xcall.h> 67 68 #if PCU_UNIT_COUNT > 0 69 70 #define PCU_SAVE 0x01 /* Save PCU state to the LWP. */ 71 #define PCU_RELEASE 0x02 /* Release PCU state on the CPU. */ 72 73 #if 0 74 /* 75 * pcu_init_lwp: initialize PCU structures for LWP. 76 */ 77 void 78 pcu_init_lwp(lwp_t *l) 79 { 80 81 memset(l->l_pcu_cpu, 0, sizeof(uint32_t) * PCU_UNIT_COUNT); 82 l->l_pcu_used = 0; 83 } 84 #endif 85 86 /* 87 * pcu_cpu_op: save/release PCU state on the current CPU. 88 * 89 * => Must be called at IPL_SOFTCLOCK or from the soft-interrupt. 90 */ 91 static void 92 pcu_cpu_op(const pcu_ops_t *pcu, const int flags) 93 { 94 const u_int id = pcu->pcu_id; 95 struct cpu_info *ci = curcpu(); 96 lwp_t *l = ci->ci_pcu_curlwp[id]; 97 98 /* If no state - nothing to do. */ 99 if (l == NULL) { 100 return; 101 } 102 if (flags & PCU_SAVE) { 103 pcu->pcu_state_save(l, (flags & PCU_RELEASE) != 0); 104 } 105 if (flags & PCU_RELEASE) { 106 ci->ci_pcu_curlwp[id] = NULL; 107 l->l_pcu_cpu[id] = NULL; 108 } 109 } 110 111 /* 112 * pcu_lwp_op: perform PCU state save, release or both operations on LWP. 113 */ 114 static void 115 pcu_lwp_op(const pcu_ops_t *pcu, lwp_t *l, int flags) 116 { 117 const u_int id = pcu->pcu_id; 118 struct cpu_info *ci; 119 uint64_t where; 120 int s; 121 122 /* 123 * Caller should have re-checked if there is any state to manage. 124 * Block the interrupts and inspect again, since cross-call sent 125 * by remote CPU could have changed the state. 126 */ 127 s = splsoftclock(); 128 ci = l->l_pcu_cpu[id]; 129 if (ci == curcpu()) { 130 /* 131 * State is on the current CPU - just perform the operations. 132 */ 133 KASSERT(ci->ci_pcu_curlwp[id] == l); 134 135 if (flags & PCU_SAVE) { 136 pcu->pcu_state_save(l, (flags & PCU_RELEASE) != 0); 137 } 138 if (flags & PCU_RELEASE) { 139 ci->ci_pcu_curlwp[id] = NULL; 140 l->l_pcu_cpu[id] = NULL; 141 } 142 splx(s); 143 return; 144 } 145 splx(s); 146 147 if (__predict_false(ci == NULL)) { 148 /* Cross-call has won the race - no state to manage. */ 149 return; 150 } 151 152 /* 153 * State is on the remote CPU - perform the operations there. 154 * Note: there is a race condition; see description in the top. 155 */ 156 where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op, 157 __UNCONST(pcu), (void *)(uintptr_t)flags, ci); 158 xc_wait(where); 159 160 KASSERT((flags & PCU_RELEASE) == 0 || l->l_pcu_cpu[id] == NULL); 161 } 162 163 /* 164 * pcu_load: load/initialize the PCU state of current LWP on current CPU. 165 */ 166 void 167 pcu_load(const pcu_ops_t *pcu) 168 { 169 const u_int id = pcu->pcu_id; 170 struct cpu_info *ci, *curci; 171 lwp_t *l = curlwp; 172 uint64_t where; 173 int s; 174 175 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 176 177 s = splsoftclock(); 178 curci = curcpu(); 179 ci = l->l_pcu_cpu[id]; 180 181 /* Does this CPU already have our PCU state loaded? */ 182 if (ci == curci) { 183 KASSERT(curci->ci_pcu_curlwp[id] == l); 184 splx(s); 185 return; 186 } 187 188 /* If PCU state of this LWP is on the remote CPU - save it there. */ 189 if (ci) { 190 splx(s); 191 /* Note: there is a race; see description in the top. */ 192 where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op, 193 __UNCONST(pcu), (void *)(PCU_SAVE | PCU_RELEASE), ci); 194 xc_wait(where); 195 196 /* Enter IPL_SOFTCLOCK and re-fetch the current CPU. */ 197 s = splsoftclock(); 198 curci = curcpu(); 199 } 200 KASSERT(l->l_pcu_cpu[id] == NULL); 201 202 /* Save the PCU state on the current CPU, if there is any. */ 203 pcu_cpu_op(pcu, PCU_SAVE | PCU_RELEASE); 204 KASSERT(curci->ci_pcu_curlwp[id] == NULL); 205 206 /* 207 * Finally, load the state for this LWP on this CPU. Indicate to 208 * load function whether PCU was used before. Note the usage. 209 */ 210 pcu->pcu_state_load(l, ((1 << id) & l->l_pcu_used) != 0); 211 curci->ci_pcu_curlwp[id] = l; 212 l->l_pcu_cpu[id] = curci; 213 l->l_pcu_used |= (1 << id); 214 splx(s); 215 } 216 217 /* 218 * pcu_discard: discard the PCU state of current LWP. 219 */ 220 void 221 pcu_discard(const pcu_ops_t *pcu) 222 { 223 const u_int id = pcu->pcu_id; 224 lwp_t *l = curlwp; 225 226 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 227 228 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 229 return; 230 } 231 pcu_lwp_op(pcu, l, PCU_RELEASE); 232 l->l_pcu_used &= ~(1 << id); 233 } 234 235 /* 236 * pcu_save_lwp: save PCU state to the given LWP. 237 */ 238 void 239 pcu_save_lwp(const pcu_ops_t *pcu, lwp_t *l) 240 { 241 const u_int id = pcu->pcu_id; 242 243 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 244 245 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 246 return; 247 } 248 pcu_lwp_op(pcu, l, PCU_SAVE | PCU_RELEASE); 249 } 250 251 /* 252 * pcu_used: return true if PCU was used (pcu_load() case) by the LWP. 253 */ 254 bool 255 pcu_used(const pcu_ops_t *pcu, lwp_t *l) 256 { 257 const u_int id = pcu->pcu_id; 258 259 return l->l_pcu_used & (1 << id); 260 } 261 262 #endif /* PCU_UNIT_COUNT > 0 */ 263