1 /* $NetBSD: subr_pcu.c,v 1.22 2020/06/06 18:13:01 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 2011, 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Per CPU Unit (PCU) - is an interface to manage synchronization of any 34 * per CPU context (unit) tied with LWP context. Typical use: FPU state. 35 * 36 * Concurrency notes: 37 * 38 * PCU state may be loaded only by the current LWP, that is, curlwp. 39 * Therefore, only LWP itself can set a CPU for lwp_t::l_pcu_cpu[id]. 40 * 41 * There are some important rules about operation calls. The request 42 * for a PCU release can be from a) the owner LWP (regardless whether 43 * the PCU state is on the current CPU or remote CPU) b) any other LWP 44 * running on that CPU (in such case, the owner LWP is on a remote CPU 45 * or sleeping). 46 * 47 * In any case, the PCU state can *only* be changed from the current 48 * CPU. If said PCU state is on the remote CPU, a cross-call will be 49 * sent by the owner LWP. Therefore struct cpu_info::ci_pcu_curlwp[id] 50 * may only be changed by the current CPU and lwp_t::l_pcu_cpu[id] may 51 * only be cleared by the CPU which has the PCU state loaded. 52 */ 53 54 #include <sys/cdefs.h> 55 __KERNEL_RCSID(0, "$NetBSD: subr_pcu.c,v 1.22 2020/06/06 18:13:01 thorpej Exp $"); 56 57 #include <sys/param.h> 58 #include <sys/cpu.h> 59 #include <sys/lwp.h> 60 #include <sys/pcu.h> 61 #include <sys/ipi.h> 62 63 #if PCU_UNIT_COUNT > 0 64 65 static inline void pcu_do_op(const pcu_ops_t *, lwp_t * const, const int); 66 static void pcu_lwp_op(const pcu_ops_t *, lwp_t *, const int); 67 68 /* 69 * Internal PCU commands for the pcu_do_op() function. 70 */ 71 #define PCU_CMD_SAVE 0x01 /* save PCU state to the LWP */ 72 #define PCU_CMD_RELEASE 0x02 /* release PCU state on the CPU */ 73 74 /* 75 * Message structure for another CPU passed via ipi(9). 76 */ 77 typedef struct { 78 const pcu_ops_t *pcu; 79 lwp_t * owner; 80 const int flags; 81 } pcu_ipi_msg_t; 82 83 /* 84 * PCU IPIs run at IPL_HIGH (aka IPL_PCU in this code). 85 */ 86 #define splpcu splhigh 87 88 /* PCU operations structure provided by the MD code. */ 89 extern const pcu_ops_t * const pcu_ops_md_defs[]; 90 91 /* 92 * pcu_switchpoint: release PCU state if the LWP is being run on another CPU. 93 * This routine is called on each context switch by by mi_switch(). 94 */ 95 void 96 pcu_switchpoint(lwp_t *l) 97 { 98 const uint32_t pcu_valid = l->l_pcu_valid; 99 int s; 100 101 KASSERTMSG(l == curlwp, "l %p != curlwp %p", l, curlwp); 102 103 if (__predict_true(pcu_valid == 0)) { 104 /* PCUs are not in use. */ 105 return; 106 } 107 s = splpcu(); 108 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 109 if ((pcu_valid & (1U << id)) == 0) { 110 continue; 111 } 112 struct cpu_info * const pcu_ci = l->l_pcu_cpu[id]; 113 if (pcu_ci == l->l_cpu) { 114 KASSERT(pcu_ci->ci_pcu_curlwp[id] == l); 115 continue; 116 } 117 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 118 pcu->pcu_state_release(l); 119 } 120 splx(s); 121 } 122 123 /* 124 * pcu_discard_all: discard PCU state of the given LWP. 125 * 126 * Used by exec and LWP exit. 127 */ 128 void 129 pcu_discard_all(lwp_t *l) 130 { 131 const uint32_t pcu_valid = l->l_pcu_valid; 132 133 /* 134 * The check for LSIDL here is to catch the case where the LWP exits 135 * due to an error in the LWP creation path before it ever runs. 136 */ 137 KASSERT(l == curlwp || l->l_stat == LSIDL || 138 ((l->l_flag & LW_SYSTEM) && pcu_valid == 0)); 139 140 if (__predict_true(pcu_valid == 0)) { 141 /* PCUs are not in use. */ 142 return; 143 } 144 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 145 if ((pcu_valid & (1U << id)) == 0) { 146 continue; 147 } 148 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 149 continue; 150 } 151 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 152 pcu_lwp_op(pcu, l, PCU_CMD_RELEASE); 153 } 154 l->l_pcu_valid = 0; 155 } 156 157 /* 158 * pcu_save_all: save PCU state of the given LWP so that eg. coredump can 159 * examine it. 160 */ 161 void 162 pcu_save_all(lwp_t *l) 163 { 164 const uint32_t pcu_valid = l->l_pcu_valid; 165 int flags = PCU_CMD_SAVE; 166 167 /* If LW_WCORE, we are also releasing the state. */ 168 if (__predict_false(l->l_flag & LW_WCORE)) { 169 flags |= PCU_CMD_RELEASE; 170 } 171 172 /* 173 * Normally we save for the current LWP, but sometimes we get called 174 * with a different LWP (forking a system LWP or doing a coredump of 175 * a process with multiple threads) and we need to deal with that. 176 */ 177 KASSERT(l == curlwp || (((l->l_flag & LW_SYSTEM) || 178 (curlwp->l_proc == l->l_proc && l->l_stat == LSSUSPENDED)) && 179 pcu_valid == 0)); 180 181 if (__predict_true(pcu_valid == 0)) { 182 /* PCUs are not in use. */ 183 return; 184 } 185 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 186 if ((pcu_valid & (1U << id)) == 0) { 187 continue; 188 } 189 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 190 continue; 191 } 192 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 193 pcu_lwp_op(pcu, l, flags); 194 } 195 } 196 197 /* 198 * pcu_do_op: save/release PCU state on the current CPU. 199 * 200 * => Must be called at IPL_PCU or from the interrupt. 201 */ 202 static inline void 203 pcu_do_op(const pcu_ops_t *pcu, lwp_t * const l, const int flags) 204 { 205 struct cpu_info * const ci = curcpu(); 206 const u_int id = pcu->pcu_id; 207 208 KASSERT(l->l_pcu_cpu[id] == ci); 209 210 if (flags & PCU_CMD_SAVE) { 211 pcu->pcu_state_save(l); 212 } 213 if (flags & PCU_CMD_RELEASE) { 214 pcu->pcu_state_release(l); 215 ci->ci_pcu_curlwp[id] = NULL; 216 l->l_pcu_cpu[id] = NULL; 217 } 218 } 219 220 /* 221 * pcu_cpu_ipi: helper routine to call pcu_do_op() via ipi(9). 222 */ 223 static void 224 pcu_cpu_ipi(void *arg) 225 { 226 const pcu_ipi_msg_t *pcu_msg = arg; 227 const pcu_ops_t *pcu = pcu_msg->pcu; 228 const u_int id = pcu->pcu_id; 229 lwp_t *l = pcu_msg->owner; 230 231 KASSERT(pcu_msg->owner != NULL); 232 233 if (curcpu()->ci_pcu_curlwp[id] != l) { 234 /* 235 * Different ownership: another LWP raced with us and 236 * perform save and release. There is nothing to do. 237 */ 238 KASSERT(l->l_pcu_cpu[id] == NULL); 239 return; 240 } 241 pcu_do_op(pcu, l, pcu_msg->flags); 242 } 243 244 /* 245 * pcu_lwp_op: perform PCU state save, release or both operations on LWP. 246 */ 247 static void 248 pcu_lwp_op(const pcu_ops_t *pcu, lwp_t *l, const int flags) 249 { 250 const u_int id = pcu->pcu_id; 251 struct cpu_info *ci; 252 int s; 253 254 /* 255 * Caller should have re-checked if there is any state to manage. 256 * Block the interrupts and inspect again, since cross-call sent 257 * by remote CPU could have changed the state. 258 */ 259 s = splpcu(); 260 ci = l->l_pcu_cpu[id]; 261 if (ci == curcpu()) { 262 /* 263 * State is on the current CPU - just perform the operations. 264 */ 265 KASSERTMSG(ci->ci_pcu_curlwp[id] == l, 266 "%s: cpu%u: pcu_curlwp[%u] (%p) != l (%p)", 267 __func__, cpu_index(ci), id, ci->ci_pcu_curlwp[id], l); 268 pcu_do_op(pcu, l, flags); 269 splx(s); 270 return; 271 } 272 if (__predict_false(ci == NULL)) { 273 /* Cross-call has won the race - no state to manage. */ 274 splx(s); 275 return; 276 } 277 278 /* 279 * The state is on the remote CPU: perform the operation(s) there. 280 */ 281 pcu_ipi_msg_t pcu_msg = { .pcu = pcu, .owner = l, .flags = flags }; 282 ipi_msg_t ipi_msg = { .func = pcu_cpu_ipi, .arg = &pcu_msg }; 283 ipi_unicast(&ipi_msg, ci); 284 splx(s); 285 286 /* Wait for completion. */ 287 ipi_wait(&ipi_msg); 288 289 KASSERT((flags & PCU_CMD_RELEASE) == 0 || l->l_pcu_cpu[id] == NULL); 290 } 291 292 /* 293 * pcu_load: load/initialize the PCU state of current LWP on current CPU. 294 */ 295 void 296 pcu_load(const pcu_ops_t *pcu) 297 { 298 lwp_t *oncpu_lwp, * const l = curlwp; 299 const u_int id = pcu->pcu_id; 300 struct cpu_info *ci, *curci; 301 int s; 302 303 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 304 305 s = splpcu(); 306 curci = curcpu(); 307 ci = l->l_pcu_cpu[id]; 308 309 /* Does this CPU already have our PCU state loaded? */ 310 if (ci == curci) { 311 /* 312 * Fault reoccurred while the PCU state is loaded and 313 * therefore PCU should be re‐enabled. This happens 314 * if LWP is context switched to another CPU and then 315 * switched back to the original CPU while the state 316 * on that CPU has not been changed by other LWPs. 317 * 318 * It may also happen due to instruction "bouncing" on 319 * some architectures. 320 */ 321 KASSERT(curci->ci_pcu_curlwp[id] == l); 322 KASSERT(pcu_valid_p(pcu, l)); 323 pcu->pcu_state_load(l, PCU_VALID | PCU_REENABLE); 324 splx(s); 325 return; 326 } 327 328 /* If PCU state of this LWP is on the remote CPU - save it there. */ 329 if (ci) { 330 pcu_ipi_msg_t pcu_msg = { .pcu = pcu, .owner = l, 331 .flags = PCU_CMD_SAVE | PCU_CMD_RELEASE }; 332 ipi_msg_t ipi_msg = { .func = pcu_cpu_ipi, .arg = &pcu_msg }; 333 ipi_unicast(&ipi_msg, ci); 334 splx(s); 335 336 /* 337 * Wait for completion, re-enter IPL_PCU and re-fetch 338 * the current CPU. 339 */ 340 ipi_wait(&ipi_msg); 341 s = splpcu(); 342 curci = curcpu(); 343 } 344 KASSERT(l->l_pcu_cpu[id] == NULL); 345 346 /* Save the PCU state on the current CPU, if there is any. */ 347 if ((oncpu_lwp = curci->ci_pcu_curlwp[id]) != NULL) { 348 pcu_do_op(pcu, oncpu_lwp, PCU_CMD_SAVE | PCU_CMD_RELEASE); 349 KASSERT(curci->ci_pcu_curlwp[id] == NULL); 350 } 351 352 /* 353 * Finally, load the state for this LWP on this CPU. Indicate to 354 * the load function whether PCU state was valid before this call. 355 */ 356 const bool valid = ((1U << id) & l->l_pcu_valid) != 0; 357 pcu->pcu_state_load(l, valid ? PCU_VALID : 0); 358 curci->ci_pcu_curlwp[id] = l; 359 l->l_pcu_cpu[id] = curci; 360 l->l_pcu_valid |= (1U << id); 361 splx(s); 362 } 363 364 /* 365 * pcu_discard: discard the PCU state of the given LWP. If "valid" 366 * parameter is true, then keep considering the PCU state as valid. 367 */ 368 void 369 pcu_discard(const pcu_ops_t *pcu, lwp_t *l, bool valid) 370 { 371 const u_int id = pcu->pcu_id; 372 373 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 374 375 if (__predict_false(valid)) { 376 l->l_pcu_valid |= (1U << id); 377 } else { 378 l->l_pcu_valid &= ~(1U << id); 379 } 380 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 381 return; 382 } 383 pcu_lwp_op(pcu, l, PCU_CMD_RELEASE); 384 } 385 386 /* 387 * pcu_save_lwp: save PCU state to the given LWP. 388 */ 389 void 390 pcu_save(const pcu_ops_t *pcu, lwp_t *l) 391 { 392 const u_int id = pcu->pcu_id; 393 394 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 395 396 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 397 return; 398 } 399 pcu_lwp_op(pcu, l, PCU_CMD_SAVE | PCU_CMD_RELEASE); 400 } 401 402 /* 403 * pcu_save_all_on_cpu: save all PCU states on the current CPU. 404 */ 405 void 406 pcu_save_all_on_cpu(void) 407 { 408 int s; 409 410 s = splpcu(); 411 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 412 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 413 lwp_t *l; 414 415 if ((l = curcpu()->ci_pcu_curlwp[id]) != NULL) { 416 pcu_do_op(pcu, l, PCU_CMD_SAVE | PCU_CMD_RELEASE); 417 } 418 } 419 splx(s); 420 } 421 422 /* 423 * pcu_valid_p: return true if PCU state is considered valid. Generally, 424 * it always becomes "valid" when pcu_load() is called. 425 */ 426 bool 427 pcu_valid_p(const pcu_ops_t *pcu, const lwp_t *l) 428 { 429 const u_int id = pcu->pcu_id; 430 431 return (l->l_pcu_valid & (1U << id)) != 0; 432 } 433 434 #endif /* PCU_UNIT_COUNT > 0 */ 435