1 /* $NetBSD: subr_pcu.c,v 1.19 2014/05/25 14:53:55 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2011, 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Per CPU Unit (PCU) - is an interface to manage synchronization of any 34 * per CPU context (unit) tied with LWP context. Typical use: FPU state. 35 * 36 * Concurrency notes: 37 * 38 * PCU state may be loaded only by the current LWP, that is, curlwp. 39 * Therefore, only LWP itself can set a CPU for lwp_t::l_pcu_cpu[id]. 40 * 41 * There are some important rules about operation calls. The request 42 * for a PCU release can be from a) the owner LWP (regardless whether 43 * the PCU state is on the current CPU or remote CPU) b) any other LWP 44 * running on that CPU (in such case, the owner LWP is on a remote CPU 45 * or sleeping). 46 * 47 * In any case, the PCU state can *only* be changed from the current 48 * CPU. If said PCU state is on the remote CPU, a cross-call will be 49 * sent by the owner LWP. Therefore struct cpu_info::ci_pcu_curlwp[id] 50 * may only be changed by the current CPU and lwp_t::l_pcu_cpu[id] may 51 * only be cleared by the CPU which has the PCU state loaded. 52 */ 53 54 #include <sys/cdefs.h> 55 __KERNEL_RCSID(0, "$NetBSD: subr_pcu.c,v 1.19 2014/05/25 14:53:55 rmind Exp $"); 56 57 #include <sys/param.h> 58 #include <sys/cpu.h> 59 #include <sys/lwp.h> 60 #include <sys/pcu.h> 61 #include <sys/ipi.h> 62 63 #if PCU_UNIT_COUNT > 0 64 65 static inline void pcu_do_op(const pcu_ops_t *, lwp_t * const, const int); 66 static void pcu_lwp_op(const pcu_ops_t *, lwp_t *, const int); 67 68 /* 69 * Internal PCU commands for the pcu_do_op() function. 70 */ 71 #define PCU_CMD_SAVE 0x01 /* save PCU state to the LWP */ 72 #define PCU_CMD_RELEASE 0x02 /* release PCU state on the CPU */ 73 74 /* 75 * Message structure for another CPU passed via ipi(9). 76 */ 77 typedef struct { 78 const pcu_ops_t *pcu; 79 lwp_t * owner; 80 const int flags; 81 } pcu_ipi_msg_t; 82 83 /* 84 * PCU IPIs run at IPL_HIGH (aka IPL_PCU in this code). 85 */ 86 #define splpcu splhigh 87 88 /* PCU operations structure provided by the MD code. */ 89 extern const pcu_ops_t * const pcu_ops_md_defs[]; 90 91 /* 92 * pcu_switchpoint: release PCU state if the LWP is being run on another CPU. 93 * This routine is called on each context switch by by mi_switch(). 94 */ 95 void 96 pcu_switchpoint(lwp_t *l) 97 { 98 const uint32_t pcu_valid = l->l_pcu_valid; 99 int s; 100 101 KASSERTMSG(l == curlwp, "l %p != curlwp %p", l, curlwp); 102 103 if (__predict_true(pcu_valid == 0)) { 104 /* PCUs are not in use. */ 105 return; 106 } 107 s = splpcu(); 108 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 109 if ((pcu_valid & (1U << id)) == 0) { 110 continue; 111 } 112 struct cpu_info * const pcu_ci = l->l_pcu_cpu[id]; 113 if (pcu_ci == NULL || pcu_ci == l->l_cpu) { 114 continue; 115 } 116 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 117 pcu->pcu_state_release(l); 118 } 119 splx(s); 120 } 121 122 /* 123 * pcu_discard_all: discard PCU state of the given LWP. 124 * 125 * Used by exec and LWP exit. 126 */ 127 void 128 pcu_discard_all(lwp_t *l) 129 { 130 const uint32_t pcu_valid = l->l_pcu_valid; 131 132 KASSERT(l == curlwp || ((l->l_flag & LW_SYSTEM) && pcu_valid == 0)); 133 134 if (__predict_true(pcu_valid == 0)) { 135 /* PCUs are not in use. */ 136 return; 137 } 138 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 139 if ((pcu_valid & (1U << id)) == 0) { 140 continue; 141 } 142 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 143 continue; 144 } 145 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 146 pcu_lwp_op(pcu, l, PCU_CMD_RELEASE); 147 } 148 l->l_pcu_valid = 0; 149 } 150 151 /* 152 * pcu_save_all: save PCU state of the given LWP so that eg. coredump can 153 * examine it. 154 */ 155 void 156 pcu_save_all(lwp_t *l) 157 { 158 const uint32_t pcu_valid = l->l_pcu_valid; 159 int flags = PCU_CMD_SAVE; 160 161 /* If LW_WCORE, we are also releasing the state. */ 162 if (__predict_false(l->l_flag & LW_WCORE)) { 163 flags |= PCU_CMD_RELEASE; 164 } 165 166 /* 167 * Normally we save for the current LWP, but sometimes we get called 168 * with a different LWP (forking a system LWP or doing a coredump of 169 * a process with multiple threads) and we need to deal with that. 170 */ 171 KASSERT(l == curlwp || (((l->l_flag & LW_SYSTEM) || 172 (curlwp->l_proc == l->l_proc && l->l_stat == LSSUSPENDED)) && 173 pcu_valid == 0)); 174 175 if (__predict_true(pcu_valid == 0)) { 176 /* PCUs are not in use. */ 177 return; 178 } 179 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 180 if ((pcu_valid & (1U << id)) == 0) { 181 continue; 182 } 183 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 184 continue; 185 } 186 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 187 pcu_lwp_op(pcu, l, flags); 188 } 189 } 190 191 /* 192 * pcu_do_op: save/release PCU state on the current CPU. 193 * 194 * => Must be called at IPL_PCU or from the interrupt. 195 */ 196 static inline void 197 pcu_do_op(const pcu_ops_t *pcu, lwp_t * const l, const int flags) 198 { 199 struct cpu_info * const ci = curcpu(); 200 const u_int id = pcu->pcu_id; 201 202 KASSERT(l->l_pcu_cpu[id] == ci); 203 204 if (flags & PCU_CMD_SAVE) { 205 pcu->pcu_state_save(l); 206 } 207 if (flags & PCU_CMD_RELEASE) { 208 pcu->pcu_state_release(l); 209 ci->ci_pcu_curlwp[id] = NULL; 210 l->l_pcu_cpu[id] = NULL; 211 } 212 } 213 214 /* 215 * pcu_cpu_ipi: helper routine to call pcu_do_op() via ipi(9). 216 */ 217 static void 218 pcu_cpu_ipi(void *arg) 219 { 220 const pcu_ipi_msg_t *pcu_msg = arg; 221 const pcu_ops_t *pcu = pcu_msg->pcu; 222 const u_int id = pcu->pcu_id; 223 lwp_t *l = pcu_msg->owner; 224 225 KASSERT(pcu_msg->owner != NULL); 226 227 if (curcpu()->ci_pcu_curlwp[id] != l) { 228 /* 229 * Different ownership: another LWP raced with us and 230 * perform save and release. There is nothing to do. 231 */ 232 KASSERT(l->l_pcu_cpu[id] == NULL); 233 return; 234 } 235 pcu_do_op(pcu, l, pcu_msg->flags); 236 } 237 238 /* 239 * pcu_lwp_op: perform PCU state save, release or both operations on LWP. 240 */ 241 static void 242 pcu_lwp_op(const pcu_ops_t *pcu, lwp_t *l, const int flags) 243 { 244 const u_int id = pcu->pcu_id; 245 struct cpu_info *ci; 246 int s; 247 248 /* 249 * Caller should have re-checked if there is any state to manage. 250 * Block the interrupts and inspect again, since cross-call sent 251 * by remote CPU could have changed the state. 252 */ 253 s = splpcu(); 254 ci = l->l_pcu_cpu[id]; 255 if (ci == curcpu()) { 256 /* 257 * State is on the current CPU - just perform the operations. 258 */ 259 KASSERTMSG(ci->ci_pcu_curlwp[id] == l, 260 "%s: cpu%u: pcu_curlwp[%u] (%p) != l (%p)", 261 __func__, cpu_index(ci), id, ci->ci_pcu_curlwp[id], l); 262 pcu_do_op(pcu, l, flags); 263 splx(s); 264 return; 265 } 266 if (__predict_false(ci == NULL)) { 267 /* Cross-call has won the race - no state to manage. */ 268 splx(s); 269 return; 270 } 271 272 /* 273 * The state is on the remote CPU: perform the operation(s) there. 274 */ 275 pcu_ipi_msg_t pcu_msg = { .pcu = pcu, .owner = l, .flags = flags }; 276 ipi_msg_t ipi_msg = { .func = pcu_cpu_ipi, .arg = &pcu_msg }; 277 ipi_unicast(&ipi_msg, ci); 278 splx(s); 279 280 /* Wait for completion. */ 281 ipi_wait(&ipi_msg); 282 283 KASSERT((flags & PCU_CMD_RELEASE) == 0 || l->l_pcu_cpu[id] == NULL); 284 } 285 286 /* 287 * pcu_load: load/initialize the PCU state of current LWP on current CPU. 288 */ 289 void 290 pcu_load(const pcu_ops_t *pcu) 291 { 292 lwp_t *oncpu_lwp, * const l = curlwp; 293 const u_int id = pcu->pcu_id; 294 struct cpu_info *ci, *curci; 295 int s; 296 297 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 298 299 s = splpcu(); 300 curci = curcpu(); 301 ci = l->l_pcu_cpu[id]; 302 303 /* Does this CPU already have our PCU state loaded? */ 304 if (ci == curci) { 305 /* 306 * Fault reoccurred while the PCU state is loaded and 307 * therefore PCU should be re‐enabled. This happens 308 * if LWP is context switched to another CPU and then 309 * switched back to the original CPU while the state 310 * on that CPU has not been changed by other LWPs. 311 * 312 * It may also happen due to instruction "bouncing" on 313 * some architectures. 314 */ 315 KASSERT(curci->ci_pcu_curlwp[id] == l); 316 KASSERT(pcu_valid_p(pcu)); 317 pcu->pcu_state_load(l, PCU_VALID | PCU_REENABLE); 318 splx(s); 319 return; 320 } 321 322 /* If PCU state of this LWP is on the remote CPU - save it there. */ 323 if (ci) { 324 pcu_ipi_msg_t pcu_msg = { .pcu = pcu, .owner = l, 325 .flags = PCU_CMD_SAVE | PCU_CMD_RELEASE }; 326 ipi_msg_t ipi_msg = { .func = pcu_cpu_ipi, .arg = &pcu_msg }; 327 ipi_unicast(&ipi_msg, ci); 328 splx(s); 329 330 /* 331 * Wait for completion, re-enter IPL_PCU and re-fetch 332 * the current CPU. 333 */ 334 ipi_wait(&ipi_msg); 335 s = splpcu(); 336 curci = curcpu(); 337 } 338 KASSERT(l->l_pcu_cpu[id] == NULL); 339 340 /* Save the PCU state on the current CPU, if there is any. */ 341 if ((oncpu_lwp = curci->ci_pcu_curlwp[id]) != NULL) { 342 pcu_do_op(pcu, oncpu_lwp, PCU_CMD_SAVE | PCU_CMD_RELEASE); 343 KASSERT(curci->ci_pcu_curlwp[id] == NULL); 344 } 345 346 /* 347 * Finally, load the state for this LWP on this CPU. Indicate to 348 * the load function whether PCU state was valid before this call. 349 */ 350 const bool valid = ((1U << id) & l->l_pcu_valid) != 0; 351 pcu->pcu_state_load(l, valid ? PCU_VALID : 0); 352 curci->ci_pcu_curlwp[id] = l; 353 l->l_pcu_cpu[id] = curci; 354 l->l_pcu_valid |= (1U << id); 355 splx(s); 356 } 357 358 /* 359 * pcu_discard: discard the PCU state of current LWP. If "valid" 360 * parameter is true, then keep considering the PCU state as valid. 361 */ 362 void 363 pcu_discard(const pcu_ops_t *pcu, bool valid) 364 { 365 const u_int id = pcu->pcu_id; 366 lwp_t * const l = curlwp; 367 368 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 369 370 if (__predict_false(valid)) { 371 l->l_pcu_valid |= (1U << id); 372 } else { 373 l->l_pcu_valid &= ~(1U << id); 374 } 375 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 376 return; 377 } 378 pcu_lwp_op(pcu, l, PCU_CMD_RELEASE); 379 } 380 381 /* 382 * pcu_save_lwp: save PCU state to the given LWP. 383 */ 384 void 385 pcu_save(const pcu_ops_t *pcu) 386 { 387 const u_int id = pcu->pcu_id; 388 lwp_t * const l = curlwp; 389 390 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 391 392 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 393 return; 394 } 395 pcu_lwp_op(pcu, l, PCU_CMD_SAVE | PCU_CMD_RELEASE); 396 } 397 398 /* 399 * pcu_save_all_on_cpu: save all PCU states on the current CPU. 400 */ 401 void 402 pcu_save_all_on_cpu(void) 403 { 404 int s; 405 406 s = splpcu(); 407 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 408 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 409 lwp_t *l; 410 411 if ((l = curcpu()->ci_pcu_curlwp[id]) != NULL) { 412 pcu_do_op(pcu, l, PCU_CMD_SAVE | PCU_CMD_RELEASE); 413 } 414 } 415 splx(s); 416 } 417 418 /* 419 * pcu_valid_p: return true if PCU state is considered valid. Generally, 420 * it always becomes "valid" when pcu_load() is called. 421 */ 422 bool 423 pcu_valid_p(const pcu_ops_t *pcu) 424 { 425 const u_int id = pcu->pcu_id; 426 lwp_t * const l = curlwp; 427 428 return (l->l_pcu_valid & (1U << id)) != 0; 429 } 430 431 #endif /* PCU_UNIT_COUNT > 0 */ 432