1 /* $NetBSD: subr_xcall.c,v 1.18 2013/11/26 21:13:05 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2007-2010 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran and Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Cross call support 34 * 35 * Background 36 * 37 * Sometimes it is necessary to modify hardware state that is tied 38 * directly to individual CPUs (such as a CPU's local timer), and 39 * these updates can not be done remotely by another CPU. The LWP 40 * requesting the update may be unable to guarantee that it will be 41 * running on the CPU where the update must occur, when the update 42 * occurs. 43 * 44 * Additionally, it's sometimes necessary to modify per-CPU software 45 * state from a remote CPU. Where these update operations are so 46 * rare or the access to the per-CPU data so frequent that the cost 47 * of using locking or atomic operations to provide coherency is 48 * prohibitive, another way must be found. 49 * 50 * Cross calls help to solve these types of problem by allowing 51 * any CPU in the system to request that an arbitrary function be 52 * executed on any other CPU. 53 * 54 * Implementation 55 * 56 * A slow mechanism for making 'low priority' cross calls is 57 * provided. The function to be executed runs on the remote CPU 58 * within a bound kthread. No queueing is provided, and the 59 * implementation uses global state. The function being called may 60 * block briefly on locks, but in doing so must be careful to not 61 * interfere with other cross calls in the system. The function is 62 * called with thread context and not from a soft interrupt, so it 63 * can ensure that it is not interrupting other code running on the 64 * CPU, and so has exclusive access to the CPU. Since this facility 65 * is heavyweight, it's expected that it will not be used often. 66 * 67 * Cross calls must not allocate memory, as the pagedaemon uses 68 * them (and memory allocation may need to wait on the pagedaemon). 69 * 70 * A low-overhead mechanism for high priority calls (XC_HIGHPRI) is 71 * also provided. The function to be executed runs on a software 72 * interrupt context, at IPL_SOFTSERIAL level, and is expected to 73 * be very lightweight, e.g. avoid blocking. 74 */ 75 76 #include <sys/cdefs.h> 77 __KERNEL_RCSID(0, "$NetBSD: subr_xcall.c,v 1.18 2013/11/26 21:13:05 rmind Exp $"); 78 79 #include <sys/types.h> 80 #include <sys/param.h> 81 #include <sys/xcall.h> 82 #include <sys/mutex.h> 83 #include <sys/condvar.h> 84 #include <sys/evcnt.h> 85 #include <sys/kthread.h> 86 #include <sys/cpu.h> 87 88 #ifdef _RUMPKERNEL 89 #include "rump_private.h" 90 #endif 91 92 /* Cross-call state box. */ 93 typedef struct { 94 kmutex_t xc_lock; 95 kcondvar_t xc_busy; 96 xcfunc_t xc_func; 97 void * xc_arg1; 98 void * xc_arg2; 99 uint64_t xc_headp; 100 uint64_t xc_donep; 101 } xc_state_t; 102 103 /* Bit indicating high (1) or low (0) priority. */ 104 #define XC_PRI_BIT (1ULL << 63) 105 106 /* Low priority xcall structures. */ 107 static xc_state_t xc_low_pri __cacheline_aligned; 108 static uint64_t xc_tailp __cacheline_aligned; 109 110 /* High priority xcall structures. */ 111 static xc_state_t xc_high_pri __cacheline_aligned; 112 static void * xc_sih __cacheline_aligned; 113 114 /* Event counters. */ 115 static struct evcnt xc_unicast_ev __cacheline_aligned; 116 static struct evcnt xc_broadcast_ev __cacheline_aligned; 117 118 static void xc_init(void); 119 static void xc_thread(void *); 120 121 static inline uint64_t xc_highpri(xcfunc_t, void *, void *, struct cpu_info *); 122 static inline uint64_t xc_lowpri(xcfunc_t, void *, void *, struct cpu_info *); 123 124 /* 125 * xc_init: 126 * 127 * Initialize low and high priority cross-call structures. 128 */ 129 static void 130 xc_init(void) 131 { 132 xc_state_t *xclo = &xc_low_pri, *xchi = &xc_high_pri; 133 134 memset(xclo, 0, sizeof(xc_state_t)); 135 mutex_init(&xclo->xc_lock, MUTEX_DEFAULT, IPL_NONE); 136 cv_init(&xclo->xc_busy, "xclocv"); 137 xc_tailp = 0; 138 139 memset(xchi, 0, sizeof(xc_state_t)); 140 mutex_init(&xchi->xc_lock, MUTEX_DEFAULT, IPL_SOFTSERIAL); 141 cv_init(&xchi->xc_busy, "xchicv"); 142 xc_sih = softint_establish(SOFTINT_SERIAL | SOFTINT_MPSAFE, 143 xc__highpri_intr, NULL); 144 KASSERT(xc_sih != NULL); 145 146 evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL, 147 "crosscall", "unicast"); 148 evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL, 149 "crosscall", "broadcast"); 150 } 151 152 /* 153 * xc_init_cpu: 154 * 155 * Initialize the cross-call subsystem. Called once for each CPU 156 * in the system as they are attached. 157 */ 158 void 159 xc_init_cpu(struct cpu_info *ci) 160 { 161 static bool again = false; 162 int error __diagused; 163 164 if (!again) { 165 /* Autoconfiguration will prevent re-entry. */ 166 xc_init(); 167 again = true; 168 } 169 cv_init(&ci->ci_data.cpu_xcall, "xcall"); 170 error = kthread_create(PRI_XCALL, KTHREAD_MPSAFE, ci, xc_thread, 171 NULL, NULL, "xcall/%u", ci->ci_index); 172 KASSERT(error == 0); 173 } 174 175 /* 176 * xc_broadcast: 177 * 178 * Trigger a call on all CPUs in the system. 179 */ 180 uint64_t 181 xc_broadcast(u_int flags, xcfunc_t func, void *arg1, void *arg2) 182 { 183 184 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 185 186 if ((flags & XC_HIGHPRI) != 0) { 187 return xc_highpri(func, arg1, arg2, NULL); 188 } else { 189 return xc_lowpri(func, arg1, arg2, NULL); 190 } 191 } 192 193 /* 194 * xc_unicast: 195 * 196 * Trigger a call on one CPU. 197 */ 198 uint64_t 199 xc_unicast(u_int flags, xcfunc_t func, void *arg1, void *arg2, 200 struct cpu_info *ci) 201 { 202 203 KASSERT(ci != NULL); 204 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 205 206 if ((flags & XC_HIGHPRI) != 0) { 207 return xc_highpri(func, arg1, arg2, ci); 208 } else { 209 return xc_lowpri(func, arg1, arg2, ci); 210 } 211 } 212 213 /* 214 * xc_wait: 215 * 216 * Wait for a cross call to complete. 217 */ 218 void 219 xc_wait(uint64_t where) 220 { 221 xc_state_t *xc; 222 223 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 224 225 /* Determine whether it is high or low priority cross-call. */ 226 if ((where & XC_PRI_BIT) != 0) { 227 xc = &xc_high_pri; 228 where &= ~XC_PRI_BIT; 229 } else { 230 xc = &xc_low_pri; 231 } 232 233 /* Fast path, if already done. */ 234 if (xc->xc_donep >= where) { 235 return; 236 } 237 238 /* Slow path: block until awoken. */ 239 mutex_enter(&xc->xc_lock); 240 while (xc->xc_donep < where) { 241 cv_wait(&xc->xc_busy, &xc->xc_lock); 242 } 243 mutex_exit(&xc->xc_lock); 244 } 245 246 /* 247 * xc_lowpri: 248 * 249 * Trigger a low priority call on one or more CPUs. 250 */ 251 static inline uint64_t 252 xc_lowpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci) 253 { 254 xc_state_t *xc = &xc_low_pri; 255 CPU_INFO_ITERATOR cii; 256 uint64_t where; 257 258 mutex_enter(&xc->xc_lock); 259 while (xc->xc_headp != xc_tailp) { 260 cv_wait(&xc->xc_busy, &xc->xc_lock); 261 } 262 xc->xc_arg1 = arg1; 263 xc->xc_arg2 = arg2; 264 xc->xc_func = func; 265 if (ci == NULL) { 266 xc_broadcast_ev.ev_count++; 267 for (CPU_INFO_FOREACH(cii, ci)) { 268 if ((ci->ci_schedstate.spc_flags & SPCF_RUNNING) == 0) 269 continue; 270 xc->xc_headp += 1; 271 ci->ci_data.cpu_xcall_pending = true; 272 cv_signal(&ci->ci_data.cpu_xcall); 273 } 274 } else { 275 xc_unicast_ev.ev_count++; 276 xc->xc_headp += 1; 277 ci->ci_data.cpu_xcall_pending = true; 278 cv_signal(&ci->ci_data.cpu_xcall); 279 } 280 KASSERT(xc_tailp < xc->xc_headp); 281 where = xc->xc_headp; 282 mutex_exit(&xc->xc_lock); 283 284 /* Return a low priority ticket. */ 285 KASSERT((where & XC_PRI_BIT) == 0); 286 return where; 287 } 288 289 /* 290 * xc_thread: 291 * 292 * One thread per-CPU to dispatch low priority calls. 293 */ 294 static void 295 xc_thread(void *cookie) 296 { 297 struct cpu_info *ci = curcpu(); 298 xc_state_t *xc = &xc_low_pri; 299 void *arg1, *arg2; 300 xcfunc_t func; 301 302 mutex_enter(&xc->xc_lock); 303 for (;;) { 304 while (!ci->ci_data.cpu_xcall_pending) { 305 if (xc->xc_headp == xc_tailp) { 306 cv_broadcast(&xc->xc_busy); 307 } 308 cv_wait(&ci->ci_data.cpu_xcall, &xc->xc_lock); 309 KASSERT(ci == curcpu()); 310 } 311 ci->ci_data.cpu_xcall_pending = false; 312 func = xc->xc_func; 313 arg1 = xc->xc_arg1; 314 arg2 = xc->xc_arg2; 315 xc_tailp++; 316 mutex_exit(&xc->xc_lock); 317 318 KASSERT(func != NULL); 319 (*func)(arg1, arg2); 320 321 mutex_enter(&xc->xc_lock); 322 xc->xc_donep++; 323 } 324 /* NOTREACHED */ 325 } 326 327 /* 328 * xc_ipi_handler: 329 * 330 * Handler of cross-call IPI. 331 */ 332 void 333 xc_ipi_handler(void) 334 { 335 /* Executes xc__highpri_intr() via software interrupt. */ 336 softint_schedule(xc_sih); 337 } 338 339 /* 340 * xc__highpri_intr: 341 * 342 * A software interrupt handler for high priority calls. 343 */ 344 void 345 xc__highpri_intr(void *dummy) 346 { 347 xc_state_t *xc = &xc_high_pri; 348 void *arg1, *arg2; 349 xcfunc_t func; 350 351 KASSERT(!cpu_intr_p()); 352 /* 353 * Lock-less fetch of function and its arguments. 354 * Safe since it cannot change at this point. 355 */ 356 KASSERT(xc->xc_donep < xc->xc_headp); 357 func = xc->xc_func; 358 arg1 = xc->xc_arg1; 359 arg2 = xc->xc_arg2; 360 361 KASSERT(func != NULL); 362 (*func)(arg1, arg2); 363 364 /* 365 * Note the request as done, and if we have reached the head, 366 * cross-call has been processed - notify waiters, if any. 367 */ 368 mutex_enter(&xc->xc_lock); 369 if (++xc->xc_donep == xc->xc_headp) { 370 cv_broadcast(&xc->xc_busy); 371 } 372 mutex_exit(&xc->xc_lock); 373 } 374 375 /* 376 * xc_highpri: 377 * 378 * Trigger a high priority call on one or more CPUs. 379 */ 380 static inline uint64_t 381 xc_highpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci) 382 { 383 xc_state_t *xc = &xc_high_pri; 384 uint64_t where; 385 386 mutex_enter(&xc->xc_lock); 387 while (xc->xc_headp != xc->xc_donep) { 388 cv_wait(&xc->xc_busy, &xc->xc_lock); 389 } 390 xc->xc_func = func; 391 xc->xc_arg1 = arg1; 392 xc->xc_arg2 = arg2; 393 xc->xc_headp += (ci ? 1 : ncpu); 394 where = xc->xc_headp; 395 mutex_exit(&xc->xc_lock); 396 397 /* 398 * Send the IPI once lock is released. 399 * Note: it will handle the local CPU case. 400 */ 401 402 #ifdef _RUMPKERNEL 403 rump_xc_highpri(ci); 404 #else 405 #ifdef MULTIPROCESSOR 406 kpreempt_disable(); 407 if (curcpu() == ci) { 408 /* Unicast: local CPU. */ 409 xc_ipi_handler(); 410 } else if (ci) { 411 /* Unicast: remote CPU. */ 412 xc_send_ipi(ci); 413 } else { 414 /* Broadcast: all, including local. */ 415 xc_send_ipi(NULL); 416 xc_ipi_handler(); 417 } 418 kpreempt_enable(); 419 #else 420 KASSERT(ci == NULL || curcpu() == ci); 421 xc_ipi_handler(); 422 #endif 423 #endif 424 425 /* Indicate a high priority ticket. */ 426 return (where | XC_PRI_BIT); 427 } 428