1 /* $NetBSD: subr_ipi.c,v 1.3 2015/01/18 23:16:35 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Inter-processor interrupt (IPI) interface: asynchronous IPIs to 34 * invoke functions with a constant argument and synchronous IPIs 35 * with the cross-call support. 36 */ 37 38 #include <sys/cdefs.h> 39 __KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.3 2015/01/18 23:16:35 rmind Exp $"); 40 41 #include <sys/param.h> 42 #include <sys/types.h> 43 44 #include <sys/atomic.h> 45 #include <sys/evcnt.h> 46 #include <sys/cpu.h> 47 #include <sys/ipi.h> 48 #include <sys/intr.h> 49 #include <sys/kcpuset.h> 50 #include <sys/kmem.h> 51 #include <sys/lock.h> 52 #include <sys/mutex.h> 53 54 /* 55 * An array of the IPI handlers used for asynchronous invocation. 56 * The lock protects the slot allocation. 57 */ 58 59 typedef struct { 60 ipi_func_t func; 61 void * arg; 62 } ipi_intr_t; 63 64 static kmutex_t ipi_mngmt_lock; 65 static ipi_intr_t ipi_intrs[IPI_MAXREG] __cacheline_aligned; 66 67 /* 68 * Per-CPU mailbox for IPI messages: it is a single cache line storing 69 * up to IPI_MSG_MAX messages. This interface is built on top of the 70 * synchronous IPIs. 71 */ 72 73 #define IPI_MSG_SLOTS (CACHE_LINE_SIZE / sizeof(ipi_msg_t *)) 74 #define IPI_MSG_MAX IPI_MSG_SLOTS 75 76 typedef struct { 77 ipi_msg_t * msg[IPI_MSG_SLOTS]; 78 } ipi_mbox_t; 79 80 81 /* Mailboxes for the synchronous IPIs. */ 82 static ipi_mbox_t * ipi_mboxes __read_mostly; 83 static struct evcnt ipi_mboxfull_ev __cacheline_aligned; 84 static void ipi_msg_cpu_handler(void *); 85 86 /* Handler for the synchronous IPIs - it must be zero. */ 87 #define IPI_SYNCH_ID 0 88 89 #ifndef MULTIPROCESSOR 90 #define cpu_ipi(ci) KASSERT(ci == NULL) 91 #endif 92 93 void 94 ipi_sysinit(void) 95 { 96 const size_t len = ncpu * sizeof(ipi_mbox_t); 97 98 /* Initialise the per-CPU bit fields. */ 99 for (u_int i = 0; i < ncpu; i++) { 100 struct cpu_info *ci = cpu_lookup(i); 101 memset(&ci->ci_ipipend, 0, sizeof(ci->ci_ipipend)); 102 } 103 mutex_init(&ipi_mngmt_lock, MUTEX_DEFAULT, IPL_NONE); 104 memset(ipi_intrs, 0, sizeof(ipi_intrs)); 105 106 /* Allocate per-CPU IPI mailboxes. */ 107 ipi_mboxes = kmem_zalloc(len, KM_SLEEP); 108 KASSERT(ipi_mboxes != NULL); 109 110 /* 111 * Register the handler for synchronous IPIs. This mechanism 112 * is built on top of the asynchronous interface. Slot zero is 113 * reserved permanently; it is also handy to use zero as a failure 114 * for other registers (as it is potentially less error-prone). 115 */ 116 ipi_intrs[IPI_SYNCH_ID].func = ipi_msg_cpu_handler; 117 118 evcnt_attach_dynamic(&ipi_mboxfull_ev, EVCNT_TYPE_MISC, NULL, 119 "ipi", "full"); 120 } 121 122 /* 123 * ipi_register: register an asynchronous IPI handler. 124 * 125 * => Returns IPI ID which is greater than zero; on failure - zero. 126 */ 127 u_int 128 ipi_register(ipi_func_t func, void *arg) 129 { 130 mutex_enter(&ipi_mngmt_lock); 131 for (u_int i = 0; i < IPI_MAXREG; i++) { 132 if (ipi_intrs[i].func == NULL) { 133 /* Register the function. */ 134 ipi_intrs[i].func = func; 135 ipi_intrs[i].arg = arg; 136 mutex_exit(&ipi_mngmt_lock); 137 138 KASSERT(i != IPI_SYNCH_ID); 139 return i; 140 } 141 } 142 mutex_exit(&ipi_mngmt_lock); 143 printf("WARNING: ipi_register: table full, increase IPI_MAXREG\n"); 144 return 0; 145 } 146 147 /* 148 * ipi_unregister: release the IPI handler given the ID. 149 */ 150 void 151 ipi_unregister(u_int ipi_id) 152 { 153 ipi_msg_t ipimsg = { .func = (ipi_func_t)nullop }; 154 155 KASSERT(ipi_id != IPI_SYNCH_ID); 156 KASSERT(ipi_id < IPI_MAXREG); 157 158 /* Release the slot. */ 159 mutex_enter(&ipi_mngmt_lock); 160 KASSERT(ipi_intrs[ipi_id].func != NULL); 161 ipi_intrs[ipi_id].func = NULL; 162 163 /* Ensure that there are no IPIs in flight. */ 164 kpreempt_disable(); 165 ipi_broadcast(&ipimsg); 166 ipi_wait(&ipimsg); 167 kpreempt_enable(); 168 mutex_exit(&ipi_mngmt_lock); 169 } 170 171 /* 172 * ipi_trigger: asynchronously send an IPI to the specified CPU. 173 */ 174 void 175 ipi_trigger(u_int ipi_id, struct cpu_info *ci) 176 { 177 const u_int i = ipi_id >> IPI_BITW_SHIFT; 178 const uint32_t bitm = 1U << (ipi_id & IPI_BITW_MASK); 179 180 KASSERT(ipi_id < IPI_MAXREG); 181 KASSERT(kpreempt_disabled()); 182 KASSERT(curcpu() != ci); 183 184 /* Mark as pending and send an IPI. */ 185 if (membar_consumer(), (ci->ci_ipipend[i] & bitm) == 0) { 186 atomic_or_32(&ci->ci_ipipend[i], bitm); 187 cpu_ipi(ci); 188 } 189 } 190 191 /* 192 * ipi_trigger_multi: same as ipi_trigger() but sends to the multiple 193 * CPUs given the target CPU set. 194 */ 195 void 196 ipi_trigger_multi(u_int ipi_id, const kcpuset_t *target) 197 { 198 const cpuid_t selfid = cpu_index(curcpu()); 199 CPU_INFO_ITERATOR cii; 200 struct cpu_info *ci; 201 202 KASSERT(kpreempt_disabled()); 203 KASSERT(target != NULL); 204 205 for (CPU_INFO_FOREACH(cii, ci)) { 206 const cpuid_t cpuid = cpu_index(ci); 207 208 if (!kcpuset_isset(target, cpuid) || cpuid == selfid) { 209 continue; 210 } 211 ipi_trigger(ipi_id, ci); 212 } 213 if (kcpuset_isset(target, selfid)) { 214 int s = splhigh(); 215 ipi_cpu_handler(); 216 splx(s); 217 } 218 } 219 220 /* 221 * put_msg: insert message into the mailbox. 222 */ 223 static inline void 224 put_msg(ipi_mbox_t *mbox, ipi_msg_t *msg) 225 { 226 int count = SPINLOCK_BACKOFF_MIN; 227 again: 228 for (u_int i = 0; i < IPI_MSG_MAX; i++) { 229 if (__predict_true(mbox->msg[i] == NULL) && 230 atomic_cas_ptr(&mbox->msg[i], NULL, msg) == NULL) { 231 return; 232 } 233 } 234 235 /* All slots are full: we have to spin-wait. */ 236 ipi_mboxfull_ev.ev_count++; 237 SPINLOCK_BACKOFF(count); 238 goto again; 239 } 240 241 /* 242 * ipi_cpu_handler: the IPI handler. 243 */ 244 void 245 ipi_cpu_handler(void) 246 { 247 struct cpu_info * const ci = curcpu(); 248 249 /* 250 * Handle asynchronous IPIs: inspect per-CPU bit field, extract 251 * IPI ID numbers and execute functions in those slots. 252 */ 253 for (u_int i = 0; i < IPI_BITWORDS; i++) { 254 uint32_t pending, bit; 255 256 if (ci->ci_ipipend[i] == 0) { 257 continue; 258 } 259 pending = atomic_swap_32(&ci->ci_ipipend[i], 0); 260 #ifndef __HAVE_ATOMIC_AS_MEMBAR 261 membar_producer(); 262 #endif 263 while ((bit = ffs(pending)) != 0) { 264 const u_int ipi_id = (i << IPI_BITW_SHIFT) | --bit; 265 ipi_intr_t *ipi_hdl = &ipi_intrs[ipi_id]; 266 267 pending &= ~(1U << bit); 268 KASSERT(ipi_hdl->func != NULL); 269 ipi_hdl->func(ipi_hdl->arg); 270 } 271 } 272 } 273 274 /* 275 * ipi_msg_cpu_handler: handle synchronous IPIs - iterate mailbox, 276 * execute the passed functions and acknowledge the messages. 277 */ 278 static void 279 ipi_msg_cpu_handler(void *arg __unused) 280 { 281 const struct cpu_info * const ci = curcpu(); 282 ipi_mbox_t *mbox = &ipi_mboxes[cpu_index(ci)]; 283 284 for (u_int i = 0; i < IPI_MSG_MAX; i++) { 285 ipi_msg_t *msg; 286 287 /* Get the message. */ 288 if ((msg = mbox->msg[i]) == NULL) { 289 continue; 290 } 291 mbox->msg[i] = NULL; 292 293 /* Execute the handler. */ 294 KASSERT(msg->func); 295 msg->func(msg->arg); 296 297 /* Ack the request. */ 298 atomic_dec_uint(&msg->_pending); 299 } 300 } 301 302 /* 303 * ipi_unicast: send an IPI to a single CPU. 304 * 305 * => The CPU must be remote; must not be local. 306 * => The caller must ipi_wait() on the message for completion. 307 */ 308 void 309 ipi_unicast(ipi_msg_t *msg, struct cpu_info *ci) 310 { 311 const cpuid_t id = cpu_index(ci); 312 313 KASSERT(msg->func != NULL); 314 KASSERT(kpreempt_disabled()); 315 KASSERT(curcpu() != ci); 316 317 msg->_pending = 1; 318 membar_producer(); 319 320 put_msg(&ipi_mboxes[id], msg); 321 ipi_trigger(IPI_SYNCH_ID, ci); 322 } 323 324 /* 325 * ipi_multicast: send an IPI to each CPU in the specified set. 326 * 327 * => The caller must ipi_wait() on the message for completion. 328 */ 329 void 330 ipi_multicast(ipi_msg_t *msg, const kcpuset_t *target) 331 { 332 const struct cpu_info * const self = curcpu(); 333 CPU_INFO_ITERATOR cii; 334 struct cpu_info *ci; 335 u_int local; 336 337 KASSERT(msg->func != NULL); 338 KASSERT(kpreempt_disabled()); 339 340 local = !!kcpuset_isset(target, cpu_index(self)); 341 msg->_pending = kcpuset_countset(target) - local; 342 membar_producer(); 343 344 for (CPU_INFO_FOREACH(cii, ci)) { 345 cpuid_t id; 346 347 if (__predict_false(ci == self)) { 348 continue; 349 } 350 id = cpu_index(ci); 351 if (!kcpuset_isset(target, id)) { 352 continue; 353 } 354 put_msg(&ipi_mboxes[id], msg); 355 ipi_trigger(IPI_SYNCH_ID, ci); 356 } 357 if (local) { 358 msg->func(msg->arg); 359 } 360 } 361 362 /* 363 * ipi_broadcast: send an IPI to all CPUs. 364 * 365 * => The caller must ipi_wait() on the message for completion. 366 */ 367 void 368 ipi_broadcast(ipi_msg_t *msg) 369 { 370 const struct cpu_info * const self = curcpu(); 371 CPU_INFO_ITERATOR cii; 372 struct cpu_info *ci; 373 374 KASSERT(msg->func != NULL); 375 KASSERT(kpreempt_disabled()); 376 377 msg->_pending = ncpu - 1; 378 membar_producer(); 379 380 /* Broadcast IPIs for remote CPUs. */ 381 for (CPU_INFO_FOREACH(cii, ci)) { 382 cpuid_t id; 383 384 if (__predict_false(ci == self)) { 385 continue; 386 } 387 id = cpu_index(ci); 388 put_msg(&ipi_mboxes[id], msg); 389 ipi_trigger(IPI_SYNCH_ID, ci); 390 } 391 392 /* Finally, execute locally. */ 393 msg->func(msg->arg); 394 } 395 396 /* 397 * ipi_wait: spin-wait until the message is processed. 398 */ 399 void 400 ipi_wait(ipi_msg_t *msg) 401 { 402 int count = SPINLOCK_BACKOFF_MIN; 403 404 while (msg->_pending) { 405 KASSERT(msg->_pending < ncpu); 406 SPINLOCK_BACKOFF(count); 407 } 408 } 409