1 /* $NetBSD: subr_xcall.c,v 1.5 2007/11/06 00:42:44 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Cross call support 41 * 42 * Background 43 * 44 * Sometimes it is necessary to modify hardware state that is tied 45 * directly to individual CPUs (such as a CPU's local timer), and 46 * these updates can not be done remotely by another CPU. The LWP 47 * requesting the update may be unable to guarantee that it will be 48 * running on the CPU where the update must occur, when the update 49 * occurs. 50 * 51 * Additionally, it's sometimes necessary to modify per-CPU software 52 * state from a remote CPU. Where these update operations are so 53 * rare or the access to the per-CPU data so frequent that the cost 54 * of using locking or atomic operations to provide coherency is 55 * prohibitive, another way must be found. 56 * 57 * Cross calls help to solve these types of problem by allowing 58 * any CPU in the system to request that an arbitrary function be 59 * executed on any other CPU. 60 * 61 * Implementation 62 * 63 * A slow mechanism for making 'low priority' cross calls is 64 * provided. The function to be executed runs on the remote CPU 65 * within a bound kthread. No queueing is provided, and the 66 * implementation uses global state. The function being called may 67 * block briefly on locks, but in doing so must be careful to not 68 * interfere with other cross calls in the system. The function is 69 * called with thread context and not from a soft interrupt, so it 70 * can ensure that it is not interrupting other code running on the 71 * CPU, and so has exclusive access to the CPU. Since this facility 72 * is heavyweight, it's expected that it will not be used often. 73 * 74 * Cross calls must not allocate memory, as the pagedaemon uses 75 * them (and memory allocation may need to wait on the pagedaemon). 76 * 77 * Future directions 78 * 79 * Add a low-overhead mechanism to run cross calls in interrupt 80 * context (XC_HIGHPRI). 81 */ 82 83 #include <sys/cdefs.h> 84 __KERNEL_RCSID(0, "$NetBSD: subr_xcall.c,v 1.5 2007/11/06 00:42:44 ad Exp $"); 85 86 #include <sys/types.h> 87 #include <sys/param.h> 88 #include <sys/xcall.h> 89 #include <sys/mutex.h> 90 #include <sys/condvar.h> 91 #include <sys/evcnt.h> 92 #include <sys/kthread.h> 93 #include <sys/cpu.h> 94 95 static void xc_thread(void *); 96 static uint64_t xc_lowpri(u_int, xcfunc_t, void *, void *, struct cpu_info *); 97 98 static kmutex_t xc_lock; 99 static xcfunc_t xc_func; 100 static void *xc_arg1; 101 static void *xc_arg2; 102 static kcondvar_t xc_busy; 103 static struct evcnt xc_unicast_ev; 104 static struct evcnt xc_broadcast_ev; 105 static uint64_t xc_headp; 106 static uint64_t xc_tailp; 107 static uint64_t xc_donep; 108 109 /* 110 * xc_init_cpu: 111 * 112 * Initialize the cross-call subsystem. Called once for each CPU 113 * in the system as they are attached. 114 */ 115 void 116 xc_init_cpu(struct cpu_info *ci) 117 { 118 static bool again; 119 int error; 120 121 if (!again) { 122 /* Autoconfiguration will prevent re-entry. */ 123 again = true; 124 mutex_init(&xc_lock, MUTEX_DEFAULT, IPL_NONE); 125 cv_init(&xc_busy, "xcallbsy"); 126 evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL, 127 "crosscall", "unicast"); 128 evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL, 129 "crosscall", "broadcast"); 130 } 131 132 cv_init(&ci->ci_data.cpu_xcall, "xcall"); 133 error = kthread_create(PRI_XCALL, KTHREAD_MPSAFE, ci, xc_thread, 134 NULL, NULL, "xcall/%d", (int)ci->ci_cpuid); 135 if (error != 0) 136 panic("xc_init_cpu: error %d", error); 137 } 138 139 /* 140 * xc_unicast: 141 * 142 * Trigger a call on all CPUs in the system. 143 */ 144 uint64_t 145 xc_broadcast(u_int flags, xcfunc_t func, void *arg1, void *arg2) 146 { 147 148 if ((flags & XC_HIGHPRI) != 0) { 149 panic("xc_unicast: no high priority crosscalls yet"); 150 } else { 151 return xc_lowpri(flags, func, arg1, arg2, NULL); 152 } 153 } 154 155 /* 156 * xc_unicast: 157 * 158 * Trigger a call on one CPU. 159 */ 160 uint64_t 161 xc_unicast(u_int flags, xcfunc_t func, void *arg1, void *arg2, 162 struct cpu_info *ci) 163 { 164 165 if ((flags & XC_HIGHPRI) != 0) { 166 panic("xc_unicast: no high priority crosscalls yet"); 167 } else { 168 KASSERT(ci != NULL); 169 return xc_lowpri(flags, func, arg1, arg2, ci); 170 } 171 } 172 173 /* 174 * xc_lowpri: 175 * 176 * Trigger a low priority call on one or more CPUs. 177 */ 178 static uint64_t 179 xc_lowpri(u_int flags, xcfunc_t func, void *arg1, void *arg2, 180 struct cpu_info *ci) 181 { 182 CPU_INFO_ITERATOR cii; 183 u_int where; 184 185 mutex_enter(&xc_lock); 186 while (xc_headp != xc_tailp) 187 cv_wait(&xc_busy, &xc_lock); 188 xc_arg1 = arg1; 189 xc_arg2 = arg2; 190 xc_func = func; 191 if (ci == NULL) { 192 xc_broadcast_ev.ev_count++; 193 for (CPU_INFO_FOREACH(cii, ci)) { 194 xc_headp += 1; 195 ci->ci_data.cpu_xcall_pending = true; 196 cv_signal(&ci->ci_data.cpu_xcall); 197 } 198 } else { 199 xc_unicast_ev.ev_count++; 200 xc_headp += 1; 201 ci->ci_data.cpu_xcall_pending = true; 202 cv_signal(&ci->ci_data.cpu_xcall); 203 } 204 KASSERT(xc_tailp < xc_headp); 205 where = xc_headp; 206 mutex_exit(&xc_lock); 207 208 return where; 209 } 210 211 /* 212 * xc_wait: 213 * 214 * Wait for a cross call to complete. 215 */ 216 void 217 xc_wait(uint64_t where) 218 { 219 220 if (xc_donep >= where) 221 return; 222 223 mutex_enter(&xc_lock); 224 while (xc_donep < where) 225 cv_wait(&xc_busy, &xc_lock); 226 mutex_exit(&xc_lock); 227 } 228 229 /* 230 * xc_thread: 231 * 232 * One thread per-CPU to dispatch low priority calls. 233 */ 234 static void 235 xc_thread(void *cookie) 236 { 237 void *arg1, *arg2; 238 struct cpu_info *ci; 239 xcfunc_t func; 240 241 ci = curcpu(); 242 243 mutex_enter(&xc_lock); 244 for (;;) { 245 while (!ci->ci_data.cpu_xcall_pending) { 246 if (xc_headp == xc_tailp) 247 cv_broadcast(&xc_busy); 248 cv_wait(&ci->ci_data.cpu_xcall, &xc_lock); 249 KASSERT(ci == curcpu()); 250 } 251 ci->ci_data.cpu_xcall_pending = false; 252 func = xc_func; 253 arg1 = xc_arg1; 254 arg2 = xc_arg2; 255 xc_tailp++; 256 mutex_exit(&xc_lock); 257 258 (*func)(arg1, arg2); 259 260 mutex_enter(&xc_lock); 261 xc_donep++; 262 } 263 /* NOTREACHED */ 264 } 265