1 /* $NetBSD: subr_pcu.c,v 1.12 2012/08/30 02:24:48 matt Exp $ */ 2 3 /*- 4 * Copyright (c) 2011 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Per CPU Unit (PCU) - is an interface to manage synchronization of any 34 * per CPU context (unit) tied with LWP context. Typical use: FPU state. 35 * 36 * Concurrency notes: 37 * 38 * PCU state may be loaded only by the current LWP, that is, curlwp. 39 * Therefore, only LWP itself can set a CPU for lwp_t::l_pcu_cpu[id]. 40 * 41 * Request for a PCU release can be from owner LWP (whether PCU state 42 * is on current CPU or remote CPU) or any other LWP running on that 43 * CPU (in such case, owner LWP is on a remote CPU or sleeping). 44 * 45 * In any case, PCU state can only be changed from the running CPU. 46 * If said PCU state is on the remote CPU, a cross-call will be sent 47 * by the owner LWP. Therefore struct cpu_info::ci_pcu_curlwp[id] 48 * may only be changed by current CPU, and lwp_t::l_pcu_cpu[id] may 49 * only be unset by the CPU which has PCU state loaded. 50 * 51 * There is a race condition: LWP may have a PCU state on a remote CPU, 52 * which it requests to be released via cross-call. At the same time, 53 * other LWP on remote CPU might release existing PCU state and load 54 * its own one. Cross-call may arrive after this and release different 55 * PCU state than intended. In such case, such LWP would re-load its 56 * PCU state again. 57 */ 58 59 #include <sys/cdefs.h> 60 __KERNEL_RCSID(0, "$NetBSD: subr_pcu.c,v 1.12 2012/08/30 02:24:48 matt Exp $"); 61 62 #include <sys/param.h> 63 #include <sys/cpu.h> 64 #include <sys/lwp.h> 65 #include <sys/pcu.h> 66 #include <sys/xcall.h> 67 68 #if PCU_UNIT_COUNT > 0 69 70 static void pcu_lwp_op(const pcu_ops_t *, lwp_t *, int); 71 72 #define PCU_SAVE 0x01 /* Save PCU state to the LWP. */ 73 #define PCU_RELEASE 0x02 /* Release PCU state on the CPU. */ 74 75 /* XXX */ 76 extern const pcu_ops_t * const pcu_ops_md_defs[]; 77 78 /* 79 * pcu_switchpoint: release PCU state if the LWP is being run on another CPU. 80 * 81 * On each context switches, called by mi_switch() with IPL_SCHED. 82 * 'l' is an LWP which is just we switched to. (the new curlwp) 83 */ 84 85 void 86 pcu_switchpoint(lwp_t *l) 87 { 88 const uint32_t pcu_inuse = l->l_pcu_used; 89 u_int id; 90 /* int s; */ 91 92 KASSERTMSG(l == curlwp, "l %p != curlwp %p", l, curlwp); 93 94 if (__predict_true(pcu_inuse == 0)) { 95 /* PCUs are not in use. */ 96 return; 97 } 98 /* commented out as we know we are already at IPL_SCHED */ 99 /* s = splsoftclock(); */ 100 for (id = 0; id < PCU_UNIT_COUNT; id++) { 101 if ((pcu_inuse & (1 << id)) == 0) { 102 continue; 103 } 104 struct cpu_info * const pcu_ci = l->l_pcu_cpu[id]; 105 if (pcu_ci == NULL || pcu_ci == l->l_cpu) { 106 continue; 107 } 108 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 109 pcu->pcu_state_release(l); 110 } 111 /* splx(s); */ 112 } 113 114 /* 115 * pcu_discard_all: discard PCU state of the given LWP. 116 * 117 * Used by exec and LWP exit. 118 */ 119 120 void 121 pcu_discard_all(lwp_t *l) 122 { 123 const uint32_t pcu_inuse = l->l_pcu_used; 124 125 KASSERT(l == curlwp || ((l->l_flag & LW_SYSTEM) && pcu_inuse == 0)); 126 127 if (__predict_true(pcu_inuse == 0)) { 128 /* PCUs are not in use. */ 129 return; 130 } 131 const int s = splsoftclock(); 132 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 133 if ((pcu_inuse & (1 << id)) == 0) { 134 continue; 135 } 136 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 137 continue; 138 } 139 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 140 /* 141 * We aren't releasing since this LWP isn't giving up PCU, 142 * just saving it. 143 */ 144 pcu_lwp_op(pcu, l, PCU_RELEASE); 145 } 146 l->l_pcu_used = 0; 147 splx(s); 148 } 149 150 /* 151 * pcu_save_all: save PCU state of the given LWP so that eg. coredump can 152 * examine it. 153 */ 154 155 void 156 pcu_save_all(lwp_t *l) 157 { 158 const uint32_t pcu_inuse = l->l_pcu_used; 159 /* 160 * Unless LW_WCORE, we aren't releasing since this LWP isn't giving 161 * up PCU, just saving it. 162 */ 163 const int flags = PCU_SAVE | (l->l_flag & LW_WCORE ? PCU_RELEASE : 0); 164 165 /* 166 * Normally we save for the current LWP, but sometimes we get called 167 * with a different LWP (forking a system LWP or doing a coredump of 168 * a process with multiple threads) and we need to deal with that. 169 */ 170 KASSERT(l == curlwp 171 || (((l->l_flag & LW_SYSTEM) 172 || (curlwp->l_proc == l->l_proc && l->l_stat == LSSUSPENDED)) 173 && pcu_inuse == 0)); 174 175 if (__predict_true(pcu_inuse == 0)) { 176 /* PCUs are not in use. */ 177 return; 178 } 179 const int s = splsoftclock(); 180 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 181 if ((pcu_inuse & (1 << id)) == 0) { 182 continue; 183 } 184 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 185 continue; 186 } 187 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 188 pcu_lwp_op(pcu, l, flags); 189 } 190 splx(s); 191 } 192 193 /* 194 * pcu_do_op: save/release PCU state on the current CPU. 195 * 196 * => Must be called at IPL_SOFTCLOCK or from the soft-interrupt. 197 */ 198 static inline void 199 pcu_do_op(const pcu_ops_t *pcu, lwp_t * const l, const int flags) 200 { 201 struct cpu_info * const ci = curcpu(); 202 const u_int id = pcu->pcu_id; 203 204 KASSERT(l->l_pcu_cpu[id] == ci); 205 206 if (flags & PCU_SAVE) { 207 pcu->pcu_state_save(l); 208 } 209 if (flags & PCU_RELEASE) { 210 pcu->pcu_state_release(l); 211 ci->ci_pcu_curlwp[id] = NULL; 212 l->l_pcu_cpu[id] = NULL; 213 } 214 } 215 216 /* 217 * pcu_cpu_op: helper routine to call pcu_do_op() via xcall(9) or 218 * by pcu_load. 219 */ 220 static void 221 pcu_cpu_op(const pcu_ops_t *pcu, const int flags) 222 { 223 const u_int id = pcu->pcu_id; 224 lwp_t * const l = curcpu()->ci_pcu_curlwp[id]; 225 226 //KASSERT(cpu_softintr_p()); 227 228 /* If no state - nothing to do. */ 229 if (l == NULL) { 230 return; 231 } 232 pcu_do_op(pcu, l, flags); 233 } 234 235 /* 236 * pcu_lwp_op: perform PCU state save, release or both operations on LWP. 237 */ 238 static void 239 pcu_lwp_op(const pcu_ops_t *pcu, lwp_t *l, int flags) 240 { 241 const u_int id = pcu->pcu_id; 242 struct cpu_info *ci; 243 uint64_t where; 244 int s; 245 246 /* 247 * Caller should have re-checked if there is any state to manage. 248 * Block the interrupts and inspect again, since cross-call sent 249 * by remote CPU could have changed the state. 250 */ 251 s = splsoftclock(); 252 ci = l->l_pcu_cpu[id]; 253 if (ci == curcpu()) { 254 /* 255 * State is on the current CPU - just perform the operations. 256 */ 257 KASSERTMSG(ci->ci_pcu_curlwp[id] == l, 258 "%s: cpu%u: pcu_curlwp[%u] (%p) != l (%p)", 259 __func__, cpu_index(ci), id, ci->ci_pcu_curlwp[id], l); 260 pcu_do_op(pcu, l, flags); 261 splx(s); 262 return; 263 } 264 splx(s); 265 266 if (__predict_false(ci == NULL)) { 267 /* Cross-call has won the race - no state to manage. */ 268 return; 269 } 270 271 /* 272 * State is on the remote CPU - perform the operations there. 273 * Note: there is a race condition; see description in the top. 274 */ 275 where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op, 276 __UNCONST(pcu), (void *)(uintptr_t)flags, ci); 277 xc_wait(where); 278 279 KASSERT((flags & PCU_RELEASE) == 0 || l->l_pcu_cpu[id] == NULL); 280 } 281 282 /* 283 * pcu_load: load/initialize the PCU state of current LWP on current CPU. 284 */ 285 void 286 pcu_load(const pcu_ops_t *pcu) 287 { 288 const u_int id = pcu->pcu_id; 289 struct cpu_info *ci, *curci; 290 lwp_t * const l = curlwp; 291 uint64_t where; 292 int s; 293 294 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 295 296 s = splsoftclock(); 297 curci = curcpu(); 298 ci = l->l_pcu_cpu[id]; 299 300 /* Does this CPU already have our PCU state loaded? */ 301 if (ci == curci) { 302 KASSERT(curci->ci_pcu_curlwp[id] == l); 303 splx(s); 304 return; 305 } 306 307 /* If PCU state of this LWP is on the remote CPU - save it there. */ 308 if (ci) { 309 splx(s); 310 /* Note: there is a race; see description in the top. */ 311 where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op, 312 __UNCONST(pcu), (void *)(PCU_SAVE | PCU_RELEASE), ci); 313 xc_wait(where); 314 315 /* Enter IPL_SOFTCLOCK and re-fetch the current CPU. */ 316 s = splsoftclock(); 317 curci = curcpu(); 318 } 319 KASSERT(l->l_pcu_cpu[id] == NULL); 320 321 /* Save the PCU state on the current CPU, if there is any. */ 322 pcu_cpu_op(pcu, PCU_SAVE | PCU_RELEASE); 323 KASSERT(curci->ci_pcu_curlwp[id] == NULL); 324 325 /* 326 * Finally, load the state for this LWP on this CPU. Indicate to 327 * load function whether PCU was used before. Note the usage. 328 */ 329 pcu->pcu_state_load(l, ((1 << id) & l->l_pcu_used) != 0); 330 curci->ci_pcu_curlwp[id] = l; 331 l->l_pcu_cpu[id] = curci; 332 l->l_pcu_used |= (1 << id); 333 splx(s); 334 } 335 336 /* 337 * pcu_discard: discard the PCU state of current LWP. 338 */ 339 void 340 pcu_discard(const pcu_ops_t *pcu) 341 { 342 const u_int id = pcu->pcu_id; 343 lwp_t * const l = curlwp; 344 345 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 346 347 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 348 return; 349 } 350 pcu_lwp_op(pcu, l, PCU_RELEASE); 351 l->l_pcu_used &= ~(1 << id); 352 } 353 354 /* 355 * pcu_save_lwp: save PCU state to the given LWP. 356 */ 357 void 358 pcu_save(const pcu_ops_t *pcu) 359 { 360 const u_int id = pcu->pcu_id; 361 lwp_t * const l = curlwp; 362 363 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 364 365 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 366 return; 367 } 368 pcu_lwp_op(pcu, l, PCU_SAVE | PCU_RELEASE); 369 } 370 371 /* 372 * pcu_used: return true if PCU was used (pcu_load() case) by the LWP. 373 */ 374 bool 375 pcu_used_p(const pcu_ops_t *pcu) 376 { 377 const u_int id = pcu->pcu_id; 378 lwp_t * const l = curlwp; 379 380 return l->l_pcu_used & (1 << id); 381 } 382 383 #endif /* PCU_UNIT_COUNT > 0 */ 384