1 /* $NetBSD: subr_pcu.c,v 1.24 2020/08/07 18:46:00 christos Exp $ */ 2 3 /*- 4 * Copyright (c) 2011, 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Per CPU Unit (PCU) - is an interface to manage synchronization of any 34 * per CPU context (unit) tied with LWP context. Typical use: FPU state. 35 * 36 * Concurrency notes: 37 * 38 * PCU state may be loaded only by the current LWP, that is, curlwp. 39 * Therefore, only LWP itself can set a CPU for lwp_t::l_pcu_cpu[id]. 40 * 41 * There are some important rules about operation calls. The request 42 * for a PCU release can be from a) the owner LWP (regardless whether 43 * the PCU state is on the current CPU or remote CPU) b) any other LWP 44 * running on that CPU (in such case, the owner LWP is on a remote CPU 45 * or sleeping). 46 * 47 * In any case, the PCU state can *only* be changed from the current 48 * CPU. If said PCU state is on the remote CPU, a cross-call will be 49 * sent by the owner LWP. Therefore struct cpu_info::ci_pcu_curlwp[id] 50 * may only be changed by the current CPU and lwp_t::l_pcu_cpu[id] may 51 * only be cleared by the CPU which has the PCU state loaded. 52 */ 53 54 #include <sys/cdefs.h> 55 __KERNEL_RCSID(0, "$NetBSD: subr_pcu.c,v 1.24 2020/08/07 18:46:00 christos Exp $"); 56 57 #include <sys/param.h> 58 #include <sys/cpu.h> 59 #include <sys/lwp.h> 60 #include <sys/pcu.h> 61 #include <sys/ipi.h> 62 63 #if PCU_UNIT_COUNT > 0 64 65 static inline void pcu_do_op(const pcu_ops_t *, lwp_t * const, const int); 66 static void pcu_lwp_op(const pcu_ops_t *, lwp_t *, const int); 67 68 /* 69 * Internal PCU commands for the pcu_do_op() function. 70 */ 71 #define PCU_CMD_SAVE 0x01 /* save PCU state to the LWP */ 72 #define PCU_CMD_RELEASE 0x02 /* release PCU state on the CPU */ 73 74 /* 75 * Message structure for another CPU passed via ipi(9). 76 */ 77 typedef struct { 78 const pcu_ops_t *pcu; 79 lwp_t * owner; 80 const int flags; 81 } pcu_ipi_msg_t; 82 83 /* 84 * PCU IPIs run at IPL_HIGH (aka IPL_PCU in this code). 85 */ 86 #define splpcu splhigh 87 88 /* PCU operations structure provided by the MD code. */ 89 extern const pcu_ops_t * const pcu_ops_md_defs[]; 90 91 #ifdef DIAGNOSTIC 92 /* 93 * pcu_available_p: true if lwp is allowed to use PCU state. 94 */ 95 static inline bool 96 pcu_available_p(struct lwp *l) 97 { 98 99 /* XXX Not sure this is safe unless l is locked! */ 100 return (l->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) != LW_SYSTEM; 101 } 102 #endif 103 104 /* 105 * pcu_switchpoint: release PCU state if the LWP is being run on another CPU. 106 * This routine is called on each context switch by by mi_switch(). 107 */ 108 void 109 pcu_switchpoint(lwp_t *l) 110 { 111 const uint32_t pcu_valid = l->l_pcu_valid; 112 int s; 113 114 KASSERTMSG(l == curlwp, "l %p != curlwp %p", l, curlwp); 115 116 if (__predict_true(pcu_valid == 0)) { 117 /* PCUs are not in use. */ 118 return; 119 } 120 s = splpcu(); 121 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 122 if ((pcu_valid & (1U << id)) == 0) { 123 continue; 124 } 125 struct cpu_info * const pcu_ci = l->l_pcu_cpu[id]; 126 if (pcu_ci == l->l_cpu) { 127 KASSERT(pcu_ci->ci_pcu_curlwp[id] == l); 128 continue; 129 } 130 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 131 pcu->pcu_state_release(l); 132 } 133 splx(s); 134 } 135 136 /* 137 * pcu_discard_all: discard PCU state of the given LWP. 138 * 139 * Used by exec and LWP exit. 140 */ 141 void 142 pcu_discard_all(lwp_t *l) 143 { 144 const uint32_t pcu_valid = l->l_pcu_valid; 145 146 /* 147 * The check for LSIDL here is to catch the case where the LWP exits 148 * due to an error in the LWP creation path before it ever runs. 149 */ 150 KASSERT(l == curlwp || l->l_stat == LSIDL || 151 (!pcu_available_p(l) && pcu_valid == 0)); 152 153 if (__predict_true(pcu_valid == 0)) { 154 /* PCUs are not in use. */ 155 return; 156 } 157 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 158 if ((pcu_valid & (1U << id)) == 0) { 159 continue; 160 } 161 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 162 continue; 163 } 164 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 165 pcu_lwp_op(pcu, l, PCU_CMD_RELEASE); 166 } 167 l->l_pcu_valid = 0; 168 } 169 170 /* 171 * pcu_save_all: save PCU state of the given LWP so that eg. coredump can 172 * examine it. 173 */ 174 void 175 pcu_save_all(lwp_t *l) 176 { 177 const uint32_t pcu_valid = l->l_pcu_valid; 178 int flags = PCU_CMD_SAVE; 179 180 /* If LW_WCORE, we are also releasing the state. */ 181 if (__predict_false(l->l_flag & LW_WCORE)) { 182 flags |= PCU_CMD_RELEASE; 183 } 184 185 /* 186 * Normally we save for the current LWP, but sometimes we get called 187 * with a different LWP (forking a system LWP or doing a coredump of 188 * a process with multiple threads) and we need to deal with that. 189 */ 190 KASSERT(l == curlwp || ((!pcu_available_p(l) || 191 (curlwp->l_proc == l->l_proc && l->l_stat == LSSUSPENDED)) && 192 pcu_valid == 0)); 193 194 if (__predict_true(pcu_valid == 0)) { 195 /* PCUs are not in use. */ 196 return; 197 } 198 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 199 if ((pcu_valid & (1U << id)) == 0) { 200 continue; 201 } 202 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 203 continue; 204 } 205 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 206 pcu_lwp_op(pcu, l, flags); 207 } 208 } 209 210 /* 211 * pcu_do_op: save/release PCU state on the current CPU. 212 * 213 * => Must be called at IPL_PCU or from the interrupt. 214 */ 215 static inline void 216 pcu_do_op(const pcu_ops_t *pcu, lwp_t * const l, const int flags) 217 { 218 struct cpu_info * const ci = curcpu(); 219 const u_int id = pcu->pcu_id; 220 221 KASSERT(l->l_pcu_cpu[id] == ci); 222 223 if (flags & PCU_CMD_SAVE) { 224 pcu->pcu_state_save(l); 225 } 226 if (flags & PCU_CMD_RELEASE) { 227 pcu->pcu_state_release(l); 228 ci->ci_pcu_curlwp[id] = NULL; 229 l->l_pcu_cpu[id] = NULL; 230 } 231 } 232 233 /* 234 * pcu_cpu_ipi: helper routine to call pcu_do_op() via ipi(9). 235 */ 236 static void 237 pcu_cpu_ipi(void *arg) 238 { 239 const pcu_ipi_msg_t *pcu_msg = arg; 240 const pcu_ops_t *pcu = pcu_msg->pcu; 241 const u_int id = pcu->pcu_id; 242 lwp_t *l = pcu_msg->owner; 243 244 KASSERT(pcu_msg->owner != NULL); 245 246 if (curcpu()->ci_pcu_curlwp[id] != l) { 247 /* 248 * Different ownership: another LWP raced with us and 249 * perform save and release. There is nothing to do. 250 */ 251 KASSERT(l->l_pcu_cpu[id] == NULL); 252 return; 253 } 254 pcu_do_op(pcu, l, pcu_msg->flags); 255 } 256 257 /* 258 * pcu_lwp_op: perform PCU state save, release or both operations on LWP. 259 */ 260 static void 261 pcu_lwp_op(const pcu_ops_t *pcu, lwp_t *l, const int flags) 262 { 263 const u_int id = pcu->pcu_id; 264 struct cpu_info *ci; 265 int s; 266 267 /* 268 * Caller should have re-checked if there is any state to manage. 269 * Block the interrupts and inspect again, since cross-call sent 270 * by remote CPU could have changed the state. 271 */ 272 s = splpcu(); 273 ci = l->l_pcu_cpu[id]; 274 if (ci == curcpu()) { 275 /* 276 * State is on the current CPU - just perform the operations. 277 */ 278 KASSERTMSG(ci->ci_pcu_curlwp[id] == l, 279 "%s: cpu%u: pcu_curlwp[%u] (%p) != l (%p)", 280 __func__, cpu_index(ci), id, ci->ci_pcu_curlwp[id], l); 281 pcu_do_op(pcu, l, flags); 282 splx(s); 283 return; 284 } 285 if (__predict_false(ci == NULL)) { 286 /* Cross-call has won the race - no state to manage. */ 287 splx(s); 288 return; 289 } 290 291 /* 292 * The state is on the remote CPU: perform the operation(s) there. 293 */ 294 pcu_ipi_msg_t pcu_msg = { .pcu = pcu, .owner = l, .flags = flags }; 295 ipi_msg_t ipi_msg = { .func = pcu_cpu_ipi, .arg = &pcu_msg }; 296 ipi_unicast(&ipi_msg, ci); 297 splx(s); 298 299 /* Wait for completion. */ 300 ipi_wait(&ipi_msg); 301 302 KASSERT((flags & PCU_CMD_RELEASE) == 0 || l->l_pcu_cpu[id] == NULL); 303 } 304 305 /* 306 * pcu_load: load/initialize the PCU state of current LWP on current CPU. 307 */ 308 void 309 pcu_load(const pcu_ops_t *pcu) 310 { 311 lwp_t *oncpu_lwp, * const l = curlwp; 312 const u_int id = pcu->pcu_id; 313 struct cpu_info *ci, *curci; 314 int s; 315 316 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 317 318 s = splpcu(); 319 curci = curcpu(); 320 ci = l->l_pcu_cpu[id]; 321 322 /* Does this CPU already have our PCU state loaded? */ 323 if (ci == curci) { 324 /* 325 * Fault reoccurred while the PCU state is loaded and 326 * therefore PCU should be re‐enabled. This happens 327 * if LWP is context switched to another CPU and then 328 * switched back to the original CPU while the state 329 * on that CPU has not been changed by other LWPs. 330 * 331 * It may also happen due to instruction "bouncing" on 332 * some architectures. 333 */ 334 KASSERT(curci->ci_pcu_curlwp[id] == l); 335 KASSERT(pcu_valid_p(pcu, l)); 336 pcu->pcu_state_load(l, PCU_VALID | PCU_REENABLE); 337 splx(s); 338 return; 339 } 340 341 /* If PCU state of this LWP is on the remote CPU - save it there. */ 342 if (ci) { 343 pcu_ipi_msg_t pcu_msg = { .pcu = pcu, .owner = l, 344 .flags = PCU_CMD_SAVE | PCU_CMD_RELEASE }; 345 ipi_msg_t ipi_msg = { .func = pcu_cpu_ipi, .arg = &pcu_msg }; 346 ipi_unicast(&ipi_msg, ci); 347 splx(s); 348 349 /* 350 * Wait for completion, re-enter IPL_PCU and re-fetch 351 * the current CPU. 352 */ 353 ipi_wait(&ipi_msg); 354 s = splpcu(); 355 curci = curcpu(); 356 } 357 KASSERT(l->l_pcu_cpu[id] == NULL); 358 359 /* Save the PCU state on the current CPU, if there is any. */ 360 if ((oncpu_lwp = curci->ci_pcu_curlwp[id]) != NULL) { 361 pcu_do_op(pcu, oncpu_lwp, PCU_CMD_SAVE | PCU_CMD_RELEASE); 362 KASSERT(curci->ci_pcu_curlwp[id] == NULL); 363 } 364 365 /* 366 * Finally, load the state for this LWP on this CPU. Indicate to 367 * the load function whether PCU state was valid before this call. 368 */ 369 const bool valid = ((1U << id) & l->l_pcu_valid) != 0; 370 pcu->pcu_state_load(l, valid ? PCU_VALID : 0); 371 curci->ci_pcu_curlwp[id] = l; 372 l->l_pcu_cpu[id] = curci; 373 l->l_pcu_valid |= (1U << id); 374 splx(s); 375 } 376 377 /* 378 * pcu_discard: discard the PCU state of the given LWP. If "valid" 379 * parameter is true, then keep considering the PCU state as valid. 380 */ 381 void 382 pcu_discard(const pcu_ops_t *pcu, lwp_t *l, bool valid) 383 { 384 const u_int id = pcu->pcu_id; 385 386 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 387 388 if (__predict_false(valid)) { 389 l->l_pcu_valid |= (1U << id); 390 } else { 391 l->l_pcu_valid &= ~(1U << id); 392 } 393 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 394 return; 395 } 396 pcu_lwp_op(pcu, l, PCU_CMD_RELEASE); 397 } 398 399 /* 400 * pcu_save_lwp: save PCU state to the given LWP. 401 */ 402 void 403 pcu_save(const pcu_ops_t *pcu, lwp_t *l) 404 { 405 const u_int id = pcu->pcu_id; 406 407 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 408 409 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 410 return; 411 } 412 pcu_lwp_op(pcu, l, PCU_CMD_SAVE | PCU_CMD_RELEASE); 413 } 414 415 /* 416 * pcu_save_all_on_cpu: save all PCU states on the current CPU. 417 */ 418 void 419 pcu_save_all_on_cpu(void) 420 { 421 int s; 422 423 s = splpcu(); 424 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 425 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 426 lwp_t *l; 427 428 if ((l = curcpu()->ci_pcu_curlwp[id]) != NULL) { 429 pcu_do_op(pcu, l, PCU_CMD_SAVE | PCU_CMD_RELEASE); 430 } 431 } 432 splx(s); 433 } 434 435 /* 436 * pcu_valid_p: return true if PCU state is considered valid. Generally, 437 * it always becomes "valid" when pcu_load() is called. 438 */ 439 bool 440 pcu_valid_p(const pcu_ops_t *pcu, const lwp_t *l) 441 { 442 const u_int id = pcu->pcu_id; 443 444 return (l->l_pcu_valid & (1U << id)) != 0; 445 } 446 447 #endif /* PCU_UNIT_COUNT > 0 */ 448