1 /* $NetBSD: subr_xcall.c,v 1.26 2018/02/07 04:25:09 ozaki-r Exp $ */ 2 3 /*- 4 * Copyright (c) 2007-2010 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran and Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Cross call support 34 * 35 * Background 36 * 37 * Sometimes it is necessary to modify hardware state that is tied 38 * directly to individual CPUs (such as a CPU's local timer), and 39 * these updates can not be done remotely by another CPU. The LWP 40 * requesting the update may be unable to guarantee that it will be 41 * running on the CPU where the update must occur, when the update 42 * occurs. 43 * 44 * Additionally, it's sometimes necessary to modify per-CPU software 45 * state from a remote CPU. Where these update operations are so 46 * rare or the access to the per-CPU data so frequent that the cost 47 * of using locking or atomic operations to provide coherency is 48 * prohibitive, another way must be found. 49 * 50 * Cross calls help to solve these types of problem by allowing 51 * any CPU in the system to request that an arbitrary function be 52 * executed on any other CPU. 53 * 54 * Implementation 55 * 56 * A slow mechanism for making 'low priority' cross calls is 57 * provided. The function to be executed runs on the remote CPU 58 * within a bound kthread. No queueing is provided, and the 59 * implementation uses global state. The function being called may 60 * block briefly on locks, but in doing so must be careful to not 61 * interfere with other cross calls in the system. The function is 62 * called with thread context and not from a soft interrupt, so it 63 * can ensure that it is not interrupting other code running on the 64 * CPU, and so has exclusive access to the CPU. Since this facility 65 * is heavyweight, it's expected that it will not be used often. 66 * 67 * Cross calls must not allocate memory, as the pagedaemon uses 68 * them (and memory allocation may need to wait on the pagedaemon). 69 * 70 * A low-overhead mechanism for high priority calls (XC_HIGHPRI) is 71 * also provided. The function to be executed runs on a software 72 * interrupt context, at IPL_SOFTSERIAL level, and is expected to 73 * be very lightweight, e.g. avoid blocking. 74 */ 75 76 #include <sys/cdefs.h> 77 __KERNEL_RCSID(0, "$NetBSD: subr_xcall.c,v 1.26 2018/02/07 04:25:09 ozaki-r Exp $"); 78 79 #include <sys/types.h> 80 #include <sys/param.h> 81 #include <sys/xcall.h> 82 #include <sys/mutex.h> 83 #include <sys/condvar.h> 84 #include <sys/evcnt.h> 85 #include <sys/kthread.h> 86 #include <sys/cpu.h> 87 88 #ifdef _RUMPKERNEL 89 #include "rump_private.h" 90 #endif 91 92 /* Cross-call state box. */ 93 typedef struct { 94 kmutex_t xc_lock; 95 kcondvar_t xc_busy; 96 xcfunc_t xc_func; 97 void * xc_arg1; 98 void * xc_arg2; 99 uint64_t xc_headp; 100 uint64_t xc_donep; 101 unsigned int xc_ipl; 102 } xc_state_t; 103 104 /* Bit indicating high (1) or low (0) priority. */ 105 #define XC_PRI_BIT (1ULL << 63) 106 107 /* Low priority xcall structures. */ 108 static xc_state_t xc_low_pri __cacheline_aligned; 109 110 /* High priority xcall structures. */ 111 static xc_state_t xc_high_pri __cacheline_aligned; 112 static void * xc_sihs[4] __cacheline_aligned; 113 114 /* Event counters. */ 115 static struct evcnt xc_unicast_ev __cacheline_aligned; 116 static struct evcnt xc_broadcast_ev __cacheline_aligned; 117 118 static void xc_init(void); 119 static void xc_thread(void *); 120 121 static inline uint64_t xc_highpri(xcfunc_t, void *, void *, struct cpu_info *, 122 unsigned int); 123 static inline uint64_t xc_lowpri(xcfunc_t, void *, void *, struct cpu_info *); 124 125 /* The internal form of IPL */ 126 #define XC_IPL_MASK 0xff00 127 /* 128 * Assign 0 to XC_IPL_SOFTSERIAL to treat IPL_SOFTSERIAL as the default value 129 * (just XC_HIGHPRI). 130 */ 131 #define XC_IPL_SOFTSERIAL 0 132 #define XC_IPL_SOFTNET 1 133 #define XC_IPL_SOFTBIO 2 134 #define XC_IPL_SOFTCLOCK 3 135 #define XC_IPL_MAX XC_IPL_SOFTCLOCK 136 137 CTASSERT(XC_IPL_MAX <= __arraycount(xc_sihs)); 138 139 /* 140 * xc_init: 141 * 142 * Initialize low and high priority cross-call structures. 143 */ 144 static void 145 xc_init(void) 146 { 147 xc_state_t *xclo = &xc_low_pri, *xchi = &xc_high_pri; 148 149 memset(xclo, 0, sizeof(xc_state_t)); 150 mutex_init(&xclo->xc_lock, MUTEX_DEFAULT, IPL_NONE); 151 cv_init(&xclo->xc_busy, "xclocv"); 152 153 memset(xchi, 0, sizeof(xc_state_t)); 154 mutex_init(&xchi->xc_lock, MUTEX_DEFAULT, IPL_SOFTSERIAL); 155 cv_init(&xchi->xc_busy, "xchicv"); 156 157 /* Set up a softint for each IPL_SOFT*. */ 158 #define SETUP_SOFTINT(xipl, sipl) do { \ 159 xc_sihs[(xipl)] = softint_establish( (sipl) | SOFTINT_MPSAFE,\ 160 xc__highpri_intr, NULL); \ 161 KASSERT(xc_sihs[(xipl)] != NULL); \ 162 } while (0) 163 164 SETUP_SOFTINT(XC_IPL_SOFTSERIAL, SOFTINT_SERIAL); 165 /* 166 * If a IPL_SOFTXXX have the same value of the previous, we don't use 167 * the IPL (see xc_encode_ipl). So we don't need to allocate a softint 168 * for it. 169 */ 170 #if IPL_SOFTNET != IPL_SOFTSERIAL 171 SETUP_SOFTINT(XC_IPL_SOFTNET, SOFTINT_NET); 172 #endif 173 #if IPL_SOFTBIO != IPL_SOFTNET 174 SETUP_SOFTINT(XC_IPL_SOFTBIO, SOFTINT_BIO); 175 #endif 176 #if IPL_SOFTCLOCK != IPL_SOFTBIO 177 SETUP_SOFTINT(XC_IPL_SOFTCLOCK, SOFTINT_CLOCK); 178 #endif 179 180 #undef SETUP_SOFTINT 181 182 evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL, 183 "crosscall", "unicast"); 184 evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL, 185 "crosscall", "broadcast"); 186 } 187 188 /* 189 * Encode an IPL to a form that can be embedded into flags of xc_broadcast 190 * or xc_unicast. 191 */ 192 unsigned int 193 xc_encode_ipl(int ipl) 194 { 195 196 switch (ipl) { 197 case IPL_SOFTSERIAL: 198 return __SHIFTIN(XC_IPL_SOFTSERIAL, XC_IPL_MASK); 199 /* IPL_SOFT* can be the same value (e.g., on sparc or mips). */ 200 #if IPL_SOFTNET != IPL_SOFTSERIAL 201 case IPL_SOFTNET: 202 return __SHIFTIN(XC_IPL_SOFTNET, XC_IPL_MASK); 203 #endif 204 #if IPL_SOFTBIO != IPL_SOFTNET 205 case IPL_SOFTBIO: 206 return __SHIFTIN(XC_IPL_SOFTBIO, XC_IPL_MASK); 207 #endif 208 #if IPL_SOFTCLOCK != IPL_SOFTBIO 209 case IPL_SOFTCLOCK: 210 return __SHIFTIN(XC_IPL_SOFTCLOCK, XC_IPL_MASK); 211 #endif 212 } 213 214 panic("Invalid IPL: %d", ipl); 215 } 216 217 /* 218 * Extract an XC_IPL from flags of xc_broadcast or xc_unicast. 219 */ 220 static inline unsigned int 221 xc_extract_ipl(unsigned int flags) 222 { 223 224 return __SHIFTOUT(flags, XC_IPL_MASK); 225 } 226 227 /* 228 * xc_init_cpu: 229 * 230 * Initialize the cross-call subsystem. Called once for each CPU 231 * in the system as they are attached. 232 */ 233 void 234 xc_init_cpu(struct cpu_info *ci) 235 { 236 static bool again = false; 237 int error __diagused; 238 239 if (!again) { 240 /* Autoconfiguration will prevent re-entry. */ 241 xc_init(); 242 again = true; 243 } 244 cv_init(&ci->ci_data.cpu_xcall, "xcall"); 245 error = kthread_create(PRI_XCALL, KTHREAD_MPSAFE, ci, xc_thread, 246 NULL, NULL, "xcall/%u", ci->ci_index); 247 KASSERT(error == 0); 248 } 249 250 /* 251 * xc_broadcast: 252 * 253 * Trigger a call on all CPUs in the system. 254 */ 255 uint64_t 256 xc_broadcast(unsigned int flags, xcfunc_t func, void *arg1, void *arg2) 257 { 258 259 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 260 ASSERT_SLEEPABLE(); 261 262 if ((flags & XC_HIGHPRI) != 0) { 263 int ipl = xc_extract_ipl(flags); 264 return xc_highpri(func, arg1, arg2, NULL, ipl); 265 } else { 266 return xc_lowpri(func, arg1, arg2, NULL); 267 } 268 } 269 270 /* 271 * xc_unicast: 272 * 273 * Trigger a call on one CPU. 274 */ 275 uint64_t 276 xc_unicast(unsigned int flags, xcfunc_t func, void *arg1, void *arg2, 277 struct cpu_info *ci) 278 { 279 280 KASSERT(ci != NULL); 281 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 282 ASSERT_SLEEPABLE(); 283 284 if ((flags & XC_HIGHPRI) != 0) { 285 int ipl = xc_extract_ipl(flags); 286 return xc_highpri(func, arg1, arg2, ci, ipl); 287 } else { 288 return xc_lowpri(func, arg1, arg2, ci); 289 } 290 } 291 292 /* 293 * xc_wait: 294 * 295 * Wait for a cross call to complete. 296 */ 297 void 298 xc_wait(uint64_t where) 299 { 300 xc_state_t *xc; 301 302 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 303 ASSERT_SLEEPABLE(); 304 305 /* Determine whether it is high or low priority cross-call. */ 306 if ((where & XC_PRI_BIT) != 0) { 307 xc = &xc_high_pri; 308 where &= ~XC_PRI_BIT; 309 } else { 310 xc = &xc_low_pri; 311 } 312 313 /* Fast path, if already done. */ 314 if (xc->xc_donep >= where) { 315 return; 316 } 317 318 /* Slow path: block until awoken. */ 319 mutex_enter(&xc->xc_lock); 320 while (xc->xc_donep < where) { 321 cv_wait(&xc->xc_busy, &xc->xc_lock); 322 } 323 mutex_exit(&xc->xc_lock); 324 } 325 326 /* 327 * xc_lowpri: 328 * 329 * Trigger a low priority call on one or more CPUs. 330 */ 331 static inline uint64_t 332 xc_lowpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci) 333 { 334 xc_state_t *xc = &xc_low_pri; 335 CPU_INFO_ITERATOR cii; 336 uint64_t where; 337 338 mutex_enter(&xc->xc_lock); 339 while (xc->xc_headp != xc->xc_donep) { 340 cv_wait(&xc->xc_busy, &xc->xc_lock); 341 } 342 xc->xc_arg1 = arg1; 343 xc->xc_arg2 = arg2; 344 xc->xc_func = func; 345 if (ci == NULL) { 346 xc_broadcast_ev.ev_count++; 347 for (CPU_INFO_FOREACH(cii, ci)) { 348 if ((ci->ci_schedstate.spc_flags & SPCF_RUNNING) == 0) 349 continue; 350 xc->xc_headp += 1; 351 ci->ci_data.cpu_xcall_pending = true; 352 cv_signal(&ci->ci_data.cpu_xcall); 353 } 354 } else { 355 xc_unicast_ev.ev_count++; 356 xc->xc_headp += 1; 357 ci->ci_data.cpu_xcall_pending = true; 358 cv_signal(&ci->ci_data.cpu_xcall); 359 } 360 KASSERT(xc->xc_donep < xc->xc_headp); 361 where = xc->xc_headp; 362 mutex_exit(&xc->xc_lock); 363 364 /* Return a low priority ticket. */ 365 KASSERT((where & XC_PRI_BIT) == 0); 366 return where; 367 } 368 369 /* 370 * xc_thread: 371 * 372 * One thread per-CPU to dispatch low priority calls. 373 */ 374 static void 375 xc_thread(void *cookie) 376 { 377 struct cpu_info *ci = curcpu(); 378 xc_state_t *xc = &xc_low_pri; 379 void *arg1, *arg2; 380 xcfunc_t func; 381 382 mutex_enter(&xc->xc_lock); 383 for (;;) { 384 while (!ci->ci_data.cpu_xcall_pending) { 385 if (xc->xc_headp == xc->xc_donep) { 386 cv_broadcast(&xc->xc_busy); 387 } 388 cv_wait(&ci->ci_data.cpu_xcall, &xc->xc_lock); 389 KASSERT(ci == curcpu()); 390 } 391 ci->ci_data.cpu_xcall_pending = false; 392 func = xc->xc_func; 393 arg1 = xc->xc_arg1; 394 arg2 = xc->xc_arg2; 395 mutex_exit(&xc->xc_lock); 396 397 KASSERT(func != NULL); 398 (*func)(arg1, arg2); 399 400 mutex_enter(&xc->xc_lock); 401 xc->xc_donep++; 402 } 403 /* NOTREACHED */ 404 } 405 406 /* 407 * xc_ipi_handler: 408 * 409 * Handler of cross-call IPI. 410 */ 411 void 412 xc_ipi_handler(void) 413 { 414 xc_state_t *xc = & xc_high_pri; 415 416 KASSERT(xc->xc_ipl < __arraycount(xc_sihs)); 417 KASSERT(xc_sihs[xc->xc_ipl] != NULL); 418 419 /* Executes xc__highpri_intr() via software interrupt. */ 420 softint_schedule(xc_sihs[xc->xc_ipl]); 421 } 422 423 /* 424 * xc__highpri_intr: 425 * 426 * A software interrupt handler for high priority calls. 427 */ 428 void 429 xc__highpri_intr(void *dummy) 430 { 431 xc_state_t *xc = &xc_high_pri; 432 void *arg1, *arg2; 433 xcfunc_t func; 434 435 KASSERTMSG(!cpu_intr_p(), "high priority xcall for function %p", 436 xc->xc_func); 437 /* 438 * Lock-less fetch of function and its arguments. 439 * Safe since it cannot change at this point. 440 */ 441 KASSERT(xc->xc_donep < xc->xc_headp); 442 func = xc->xc_func; 443 arg1 = xc->xc_arg1; 444 arg2 = xc->xc_arg2; 445 446 KASSERT(func != NULL); 447 (*func)(arg1, arg2); 448 449 /* 450 * Note the request as done, and if we have reached the head, 451 * cross-call has been processed - notify waiters, if any. 452 */ 453 mutex_enter(&xc->xc_lock); 454 if (++xc->xc_donep == xc->xc_headp) { 455 cv_broadcast(&xc->xc_busy); 456 } 457 mutex_exit(&xc->xc_lock); 458 } 459 460 /* 461 * xc_highpri: 462 * 463 * Trigger a high priority call on one or more CPUs. 464 */ 465 static inline uint64_t 466 xc_highpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci, 467 unsigned int ipl) 468 { 469 xc_state_t *xc = &xc_high_pri; 470 uint64_t where; 471 472 mutex_enter(&xc->xc_lock); 473 while (xc->xc_headp != xc->xc_donep) { 474 cv_wait(&xc->xc_busy, &xc->xc_lock); 475 } 476 xc->xc_func = func; 477 xc->xc_arg1 = arg1; 478 xc->xc_arg2 = arg2; 479 xc->xc_headp += (ci ? 1 : ncpu); 480 xc->xc_ipl = ipl; 481 where = xc->xc_headp; 482 mutex_exit(&xc->xc_lock); 483 484 /* 485 * Send the IPI once lock is released. 486 * Note: it will handle the local CPU case. 487 */ 488 489 #ifdef _RUMPKERNEL 490 rump_xc_highpri(ci); 491 #else 492 #ifdef MULTIPROCESSOR 493 kpreempt_disable(); 494 if (curcpu() == ci) { 495 /* Unicast: local CPU. */ 496 xc_ipi_handler(); 497 } else if (ci) { 498 /* Unicast: remote CPU. */ 499 xc_send_ipi(ci); 500 } else { 501 /* Broadcast: all, including local. */ 502 xc_send_ipi(NULL); 503 xc_ipi_handler(); 504 } 505 kpreempt_enable(); 506 #else 507 KASSERT(ci == NULL || curcpu() == ci); 508 xc_ipi_handler(); 509 #endif 510 #endif 511 512 /* Indicate a high priority ticket. */ 513 return (where | XC_PRI_BIT); 514 } 515