1 /* $NetBSD: subr_pcu.c,v 1.18 2014/05/16 00:48:41 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2011, 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Per CPU Unit (PCU) - is an interface to manage synchronization of any 34 * per CPU context (unit) tied with LWP context. Typical use: FPU state. 35 * 36 * Concurrency notes: 37 * 38 * PCU state may be loaded only by the current LWP, that is, curlwp. 39 * Therefore, only LWP itself can set a CPU for lwp_t::l_pcu_cpu[id]. 40 * 41 * There are some important rules about operation calls. The request 42 * for a PCU release can be from a) the owner LWP (regardless whether 43 * the PCU state is on the current CPU or remote CPU) b) any other LWP 44 * running on that CPU (in such case, the owner LWP is on a remote CPU 45 * or sleeping). 46 * 47 * In any case, the PCU state can *only* be changed from the current 48 * CPU. If said PCU state is on the remote CPU, a cross-call will be 49 * sent by the owner LWP. Therefore struct cpu_info::ci_pcu_curlwp[id] 50 * may only be changed by the current CPU and lwp_t::l_pcu_cpu[id] may 51 * only be cleared by the CPU which has the PCU state loaded. 52 */ 53 54 #include <sys/cdefs.h> 55 __KERNEL_RCSID(0, "$NetBSD: subr_pcu.c,v 1.18 2014/05/16 00:48:41 rmind Exp $"); 56 57 #include <sys/param.h> 58 #include <sys/cpu.h> 59 #include <sys/lwp.h> 60 #include <sys/pcu.h> 61 #include <sys/xcall.h> 62 63 #if PCU_UNIT_COUNT > 0 64 65 static inline void pcu_do_op(const pcu_ops_t *, lwp_t * const, const int); 66 static void pcu_lwp_op(const pcu_ops_t *, lwp_t *, const int); 67 68 /* 69 * Internal PCU commands for the pcu_do_op() function. 70 */ 71 #define PCU_CMD_SAVE 0x01 /* save PCU state to the LWP */ 72 #define PCU_CMD_RELEASE 0x02 /* release PCU state on the CPU */ 73 74 /* 75 * Message structure for another CPU passed via xcall(9). 76 */ 77 typedef struct { 78 const pcu_ops_t *pcu; 79 lwp_t * owner; 80 const int flags; 81 } pcu_xcall_msg_t; 82 83 /* PCU operations structure provided by the MD code. */ 84 extern const pcu_ops_t * const pcu_ops_md_defs[]; 85 86 /* 87 * pcu_switchpoint: release PCU state if the LWP is being run on another CPU. 88 * 89 * On each context switches, called by mi_switch() with IPL_SCHED. 90 * 'l' is an LWP which is just we switched to. (the new curlwp) 91 */ 92 void 93 pcu_switchpoint(lwp_t *l) 94 { 95 const uint32_t pcu_valid = l->l_pcu_valid; 96 /* int s; */ 97 98 KASSERTMSG(l == curlwp, "l %p != curlwp %p", l, curlwp); 99 100 if (__predict_true(pcu_valid == 0)) { 101 /* PCUs are not in use. */ 102 return; 103 } 104 /* commented out as we know we are already at IPL_SCHED */ 105 /* s = splsoftserial(); */ 106 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 107 if ((pcu_valid & (1U << id)) == 0) { 108 continue; 109 } 110 struct cpu_info * const pcu_ci = l->l_pcu_cpu[id]; 111 if (pcu_ci == NULL || pcu_ci == l->l_cpu) { 112 continue; 113 } 114 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 115 pcu->pcu_state_release(l); 116 } 117 /* splx(s); */ 118 } 119 120 /* 121 * pcu_discard_all: discard PCU state of the given LWP. 122 * 123 * Used by exec and LWP exit. 124 */ 125 void 126 pcu_discard_all(lwp_t *l) 127 { 128 const uint32_t pcu_valid = l->l_pcu_valid; 129 130 KASSERT(l == curlwp || ((l->l_flag & LW_SYSTEM) && pcu_valid == 0)); 131 132 if (__predict_true(pcu_valid == 0)) { 133 /* PCUs are not in use. */ 134 return; 135 } 136 const int s = splsoftserial(); 137 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 138 if ((pcu_valid & (1U << id)) == 0) { 139 continue; 140 } 141 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 142 continue; 143 } 144 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 145 pcu_lwp_op(pcu, l, PCU_CMD_RELEASE); 146 } 147 l->l_pcu_valid = 0; 148 splx(s); 149 } 150 151 /* 152 * pcu_save_all: save PCU state of the given LWP so that eg. coredump can 153 * examine it. 154 */ 155 void 156 pcu_save_all(lwp_t *l) 157 { 158 const uint32_t pcu_valid = l->l_pcu_valid; 159 int flags = PCU_CMD_SAVE; 160 161 /* If LW_WCORE, we are also releasing the state. */ 162 if (__predict_false(l->l_flag & LW_WCORE)) { 163 flags |= PCU_CMD_RELEASE; 164 } 165 166 /* 167 * Normally we save for the current LWP, but sometimes we get called 168 * with a different LWP (forking a system LWP or doing a coredump of 169 * a process with multiple threads) and we need to deal with that. 170 */ 171 KASSERT(l == curlwp || (((l->l_flag & LW_SYSTEM) || 172 (curlwp->l_proc == l->l_proc && l->l_stat == LSSUSPENDED)) && 173 pcu_valid == 0)); 174 175 if (__predict_true(pcu_valid == 0)) { 176 /* PCUs are not in use. */ 177 return; 178 } 179 const int s = splsoftserial(); 180 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 181 if ((pcu_valid & (1U << id)) == 0) { 182 continue; 183 } 184 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 185 continue; 186 } 187 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 188 pcu_lwp_op(pcu, l, flags); 189 } 190 splx(s); 191 } 192 193 /* 194 * pcu_do_op: save/release PCU state on the current CPU. 195 * 196 * => Must be called at IPL_SOFTSERIAL or from the soft-interrupt. 197 */ 198 static inline void 199 pcu_do_op(const pcu_ops_t *pcu, lwp_t * const l, const int flags) 200 { 201 struct cpu_info * const ci = curcpu(); 202 const u_int id = pcu->pcu_id; 203 204 KASSERT(l->l_pcu_cpu[id] == ci); 205 206 if (flags & PCU_CMD_SAVE) { 207 pcu->pcu_state_save(l); 208 } 209 if (flags & PCU_CMD_RELEASE) { 210 pcu->pcu_state_release(l); 211 ci->ci_pcu_curlwp[id] = NULL; 212 l->l_pcu_cpu[id] = NULL; 213 } 214 } 215 216 /* 217 * pcu_cpu_xcall: helper routine to call pcu_do_op() via xcall(9). 218 */ 219 static void 220 pcu_cpu_xcall(void *arg1, void *arg2 __unused) 221 { 222 const pcu_xcall_msg_t *pcu_msg = arg1; 223 const pcu_ops_t *pcu = pcu_msg->pcu; 224 const u_int id = pcu->pcu_id; 225 lwp_t *l = pcu_msg->owner; 226 227 KASSERT(cpu_softintr_p()); 228 KASSERT(pcu_msg->owner != NULL); 229 230 if (curcpu()->ci_pcu_curlwp[id] != l) { 231 /* 232 * Different ownership: another LWP raced with us and 233 * perform save and release. There is nothing to do. 234 */ 235 KASSERT(l->l_pcu_cpu[id] == NULL); 236 return; 237 } 238 pcu_do_op(pcu, l, pcu_msg->flags); 239 } 240 241 /* 242 * pcu_lwp_op: perform PCU state save, release or both operations on LWP. 243 */ 244 static void 245 pcu_lwp_op(const pcu_ops_t *pcu, lwp_t *l, const int flags) 246 { 247 const u_int id = pcu->pcu_id; 248 struct cpu_info *ci; 249 uint64_t where; 250 int s; 251 252 /* 253 * Caller should have re-checked if there is any state to manage. 254 * Block the interrupts and inspect again, since cross-call sent 255 * by remote CPU could have changed the state. 256 */ 257 s = splsoftserial(); 258 ci = l->l_pcu_cpu[id]; 259 if (ci == curcpu()) { 260 /* 261 * State is on the current CPU - just perform the operations. 262 */ 263 KASSERTMSG(ci->ci_pcu_curlwp[id] == l, 264 "%s: cpu%u: pcu_curlwp[%u] (%p) != l (%p)", 265 __func__, cpu_index(ci), id, ci->ci_pcu_curlwp[id], l); 266 pcu_do_op(pcu, l, flags); 267 splx(s); 268 return; 269 } 270 splx(s); 271 272 if (__predict_false(ci == NULL)) { 273 /* Cross-call has won the race - no state to manage. */ 274 return; 275 } 276 277 /* 278 * The state is on the remote CPU: perform the operation(s) there. 279 */ 280 pcu_xcall_msg_t pcu_msg = { .pcu = pcu, .owner = l, .flags = flags }; 281 where = xc_unicast(XC_HIGHPRI, pcu_cpu_xcall, &pcu_msg, NULL, ci); 282 xc_wait(where); 283 284 KASSERT((flags & PCU_CMD_RELEASE) == 0 || l->l_pcu_cpu[id] == NULL); 285 } 286 287 /* 288 * pcu_load: load/initialize the PCU state of current LWP on current CPU. 289 */ 290 void 291 pcu_load(const pcu_ops_t *pcu) 292 { 293 lwp_t *oncpu_lwp, * const l = curlwp; 294 const u_int id = pcu->pcu_id; 295 struct cpu_info *ci, *curci; 296 uint64_t where; 297 int s; 298 299 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 300 301 s = splsoftserial(); 302 curci = curcpu(); 303 ci = l->l_pcu_cpu[id]; 304 305 /* Does this CPU already have our PCU state loaded? */ 306 if (ci == curci) { 307 /* Fault happen: indicate re-enable. */ 308 KASSERT(curci->ci_pcu_curlwp[id] == l); 309 KASSERT(pcu_valid_p(pcu)); 310 pcu->pcu_state_load(l, PCU_VALID | PCU_REENABLE); 311 splx(s); 312 return; 313 } 314 315 /* If PCU state of this LWP is on the remote CPU - save it there. */ 316 if (ci) { 317 splx(s); 318 319 pcu_xcall_msg_t pcu_msg = { .pcu = pcu, .owner = l, 320 .flags = PCU_CMD_SAVE | PCU_CMD_RELEASE }; 321 where = xc_unicast(XC_HIGHPRI, pcu_cpu_xcall, 322 &pcu_msg, NULL, ci); 323 xc_wait(where); 324 325 /* Enter IPL_SOFTSERIAL and re-fetch the current CPU. */ 326 s = splsoftserial(); 327 curci = curcpu(); 328 } 329 KASSERT(l->l_pcu_cpu[id] == NULL); 330 331 /* Save the PCU state on the current CPU, if there is any. */ 332 if ((oncpu_lwp = curci->ci_pcu_curlwp[id]) != NULL) { 333 pcu_do_op(pcu, oncpu_lwp, PCU_CMD_SAVE | PCU_CMD_RELEASE); 334 KASSERT(curci->ci_pcu_curlwp[id] == NULL); 335 } 336 337 /* 338 * Finally, load the state for this LWP on this CPU. Indicate to 339 * the load function whether PCU state was valid before this call. 340 */ 341 const bool valid = ((1U << id) & l->l_pcu_valid) != 0; 342 pcu->pcu_state_load(l, valid ? PCU_VALID : 0); 343 curci->ci_pcu_curlwp[id] = l; 344 l->l_pcu_cpu[id] = curci; 345 l->l_pcu_valid |= (1U << id); 346 splx(s); 347 } 348 349 /* 350 * pcu_discard: discard the PCU state of current LWP. If "valid" 351 * parameter is true, then keep considering the PCU state as valid. 352 */ 353 void 354 pcu_discard(const pcu_ops_t *pcu, bool valid) 355 { 356 const u_int id = pcu->pcu_id; 357 lwp_t * const l = curlwp; 358 359 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 360 361 if (__predict_false(valid)) { 362 l->l_pcu_valid |= (1U << id); 363 } else { 364 l->l_pcu_valid &= ~(1U << id); 365 } 366 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 367 return; 368 } 369 pcu_lwp_op(pcu, l, PCU_CMD_RELEASE); 370 } 371 372 /* 373 * pcu_save_lwp: save PCU state to the given LWP. 374 */ 375 void 376 pcu_save(const pcu_ops_t *pcu) 377 { 378 const u_int id = pcu->pcu_id; 379 lwp_t * const l = curlwp; 380 381 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 382 383 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 384 return; 385 } 386 pcu_lwp_op(pcu, l, PCU_CMD_SAVE | PCU_CMD_RELEASE); 387 } 388 389 /* 390 * pcu_save_all_on_cpu: save all PCU states on the current CPU. 391 */ 392 void 393 pcu_save_all_on_cpu(void) 394 { 395 int s; 396 397 s = splsoftserial(); 398 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 399 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 400 lwp_t *l; 401 402 if ((l = curcpu()->ci_pcu_curlwp[id]) != NULL) { 403 pcu_do_op(pcu, l, PCU_CMD_SAVE | PCU_CMD_RELEASE); 404 } 405 } 406 splx(s); 407 } 408 409 /* 410 * pcu_valid_p: return true if PCU state is considered valid. Generally, 411 * it always becomes "valid" when pcu_load() is called. 412 */ 413 bool 414 pcu_valid_p(const pcu_ops_t *pcu) 415 { 416 const u_int id = pcu->pcu_id; 417 lwp_t * const l = curlwp; 418 419 return (l->l_pcu_valid & (1U << id)) != 0; 420 } 421 422 #endif /* PCU_UNIT_COUNT > 0 */ 423