1 /* $NetBSD: subr_ipi.c,v 1.4 2019/04/06 02:59:05 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Inter-processor interrupt (IPI) interface: asynchronous IPIs to 34 * invoke functions with a constant argument and synchronous IPIs 35 * with the cross-call support. 36 */ 37 38 #include <sys/cdefs.h> 39 __KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.4 2019/04/06 02:59:05 thorpej Exp $"); 40 41 #include <sys/param.h> 42 #include <sys/types.h> 43 44 #include <sys/atomic.h> 45 #include <sys/evcnt.h> 46 #include <sys/cpu.h> 47 #include <sys/ipi.h> 48 #include <sys/intr.h> 49 #include <sys/kcpuset.h> 50 #include <sys/kmem.h> 51 #include <sys/lock.h> 52 #include <sys/mutex.h> 53 54 /* 55 * An array of the IPI handlers used for asynchronous invocation. 56 * The lock protects the slot allocation. 57 */ 58 59 typedef struct { 60 ipi_func_t func; 61 void * arg; 62 } ipi_intr_t; 63 64 static kmutex_t ipi_mngmt_lock; 65 static ipi_intr_t ipi_intrs[IPI_MAXREG] __cacheline_aligned; 66 67 /* 68 * Per-CPU mailbox for IPI messages: it is a single cache line storing 69 * up to IPI_MSG_MAX messages. This interface is built on top of the 70 * synchronous IPIs. 71 */ 72 73 #define IPI_MSG_SLOTS (CACHE_LINE_SIZE / sizeof(ipi_msg_t *)) 74 #define IPI_MSG_MAX IPI_MSG_SLOTS 75 76 typedef struct { 77 ipi_msg_t * msg[IPI_MSG_SLOTS]; 78 } ipi_mbox_t; 79 80 81 /* Mailboxes for the synchronous IPIs. */ 82 static ipi_mbox_t * ipi_mboxes __read_mostly; 83 static struct evcnt ipi_mboxfull_ev __cacheline_aligned; 84 static void ipi_msg_cpu_handler(void *); 85 86 /* Handler for the synchronous IPIs - it must be zero. */ 87 #define IPI_SYNCH_ID 0 88 89 #ifndef MULTIPROCESSOR 90 #define cpu_ipi(ci) KASSERT(ci == NULL) 91 #endif 92 93 void 94 ipi_sysinit(void) 95 { 96 const size_t len = ncpu * sizeof(ipi_mbox_t); 97 98 /* Initialise the per-CPU bit fields. */ 99 for (u_int i = 0; i < ncpu; i++) { 100 struct cpu_info *ci = cpu_lookup(i); 101 memset(&ci->ci_ipipend, 0, sizeof(ci->ci_ipipend)); 102 } 103 mutex_init(&ipi_mngmt_lock, MUTEX_DEFAULT, IPL_NONE); 104 memset(ipi_intrs, 0, sizeof(ipi_intrs)); 105 106 /* Allocate per-CPU IPI mailboxes. */ 107 ipi_mboxes = kmem_zalloc(len, KM_SLEEP); 108 KASSERT(ipi_mboxes != NULL); 109 110 /* 111 * Register the handler for synchronous IPIs. This mechanism 112 * is built on top of the asynchronous interface. Slot zero is 113 * reserved permanently; it is also handy to use zero as a failure 114 * for other registers (as it is potentially less error-prone). 115 */ 116 ipi_intrs[IPI_SYNCH_ID].func = ipi_msg_cpu_handler; 117 118 evcnt_attach_dynamic(&ipi_mboxfull_ev, EVCNT_TYPE_MISC, NULL, 119 "ipi", "full"); 120 } 121 122 /* 123 * ipi_register: register an asynchronous IPI handler. 124 * 125 * => Returns IPI ID which is greater than zero; on failure - zero. 126 */ 127 u_int 128 ipi_register(ipi_func_t func, void *arg) 129 { 130 mutex_enter(&ipi_mngmt_lock); 131 for (u_int i = 0; i < IPI_MAXREG; i++) { 132 if (ipi_intrs[i].func == NULL) { 133 /* Register the function. */ 134 ipi_intrs[i].func = func; 135 ipi_intrs[i].arg = arg; 136 mutex_exit(&ipi_mngmt_lock); 137 138 KASSERT(i != IPI_SYNCH_ID); 139 return i; 140 } 141 } 142 mutex_exit(&ipi_mngmt_lock); 143 printf("WARNING: ipi_register: table full, increase IPI_MAXREG\n"); 144 return 0; 145 } 146 147 /* 148 * ipi_unregister: release the IPI handler given the ID. 149 */ 150 void 151 ipi_unregister(u_int ipi_id) 152 { 153 ipi_msg_t ipimsg = { .func = (ipi_func_t)nullop }; 154 155 KASSERT(ipi_id != IPI_SYNCH_ID); 156 KASSERT(ipi_id < IPI_MAXREG); 157 158 /* Release the slot. */ 159 mutex_enter(&ipi_mngmt_lock); 160 KASSERT(ipi_intrs[ipi_id].func != NULL); 161 ipi_intrs[ipi_id].func = NULL; 162 163 /* Ensure that there are no IPIs in flight. */ 164 kpreempt_disable(); 165 ipi_broadcast(&ipimsg, false); 166 ipi_wait(&ipimsg); 167 kpreempt_enable(); 168 mutex_exit(&ipi_mngmt_lock); 169 } 170 171 /* 172 * ipi_mark_pending: internal routine to mark an IPI pending on the 173 * specified CPU (which might be curcpu()). 174 */ 175 static bool 176 ipi_mark_pending(u_int ipi_id, struct cpu_info *ci) 177 { 178 const u_int i = ipi_id >> IPI_BITW_SHIFT; 179 const uint32_t bitm = 1U << (ipi_id & IPI_BITW_MASK); 180 181 KASSERT(ipi_id < IPI_MAXREG); 182 KASSERT(kpreempt_disabled()); 183 184 /* Mark as pending and send an IPI. */ 185 if (membar_consumer(), (ci->ci_ipipend[i] & bitm) == 0) { 186 atomic_or_32(&ci->ci_ipipend[i], bitm); 187 return true; 188 } 189 return false; 190 } 191 192 /* 193 * ipi_trigger: asynchronously send an IPI to the specified CPU. 194 */ 195 void 196 ipi_trigger(u_int ipi_id, struct cpu_info *ci) 197 { 198 199 KASSERT(curcpu() != ci); 200 if (ipi_mark_pending(ipi_id, ci)) { 201 cpu_ipi(ci); 202 } 203 } 204 205 /* 206 * ipi_trigger_multi_internal: the guts of ipi_trigger_multi() and 207 * ipi_trigger_broadcast(). 208 */ 209 static void 210 ipi_trigger_multi_internal(u_int ipi_id, const kcpuset_t *target, 211 bool skip_self) 212 { 213 const cpuid_t selfid = cpu_index(curcpu()); 214 CPU_INFO_ITERATOR cii; 215 struct cpu_info *ci; 216 217 KASSERT(kpreempt_disabled()); 218 KASSERT(target != NULL); 219 220 for (CPU_INFO_FOREACH(cii, ci)) { 221 const cpuid_t cpuid = cpu_index(ci); 222 223 if (!kcpuset_isset(target, cpuid) || cpuid == selfid) { 224 continue; 225 } 226 ipi_trigger(ipi_id, ci); 227 } 228 if (!skip_self && kcpuset_isset(target, selfid)) { 229 ipi_mark_pending(ipi_id, curcpu()); 230 int s = splhigh(); 231 ipi_cpu_handler(); 232 splx(s); 233 } 234 } 235 236 /* 237 * ipi_trigger_multi: same as ipi_trigger() but sends to the multiple 238 * CPUs given the target CPU set. 239 */ 240 void 241 ipi_trigger_multi(u_int ipi_id, const kcpuset_t *target) 242 { 243 ipi_trigger_multi_internal(ipi_id, target, false); 244 } 245 246 /* 247 * ipi_trigger_broadcast: same as ipi_trigger_multi() to kcpuset_attached, 248 * optionally skipping the sending CPU. 249 */ 250 void 251 ipi_trigger_broadcast(u_int ipi_id, bool skip_self) 252 { 253 ipi_trigger_multi_internal(ipi_id, kcpuset_attached, skip_self); 254 } 255 256 /* 257 * put_msg: insert message into the mailbox. 258 */ 259 static inline void 260 put_msg(ipi_mbox_t *mbox, ipi_msg_t *msg) 261 { 262 int count = SPINLOCK_BACKOFF_MIN; 263 again: 264 for (u_int i = 0; i < IPI_MSG_MAX; i++) { 265 if (__predict_true(mbox->msg[i] == NULL) && 266 atomic_cas_ptr(&mbox->msg[i], NULL, msg) == NULL) { 267 return; 268 } 269 } 270 271 /* All slots are full: we have to spin-wait. */ 272 ipi_mboxfull_ev.ev_count++; 273 SPINLOCK_BACKOFF(count); 274 goto again; 275 } 276 277 /* 278 * ipi_cpu_handler: the IPI handler. 279 */ 280 void 281 ipi_cpu_handler(void) 282 { 283 struct cpu_info * const ci = curcpu(); 284 285 /* 286 * Handle asynchronous IPIs: inspect per-CPU bit field, extract 287 * IPI ID numbers and execute functions in those slots. 288 */ 289 for (u_int i = 0; i < IPI_BITWORDS; i++) { 290 uint32_t pending, bit; 291 292 if (ci->ci_ipipend[i] == 0) { 293 continue; 294 } 295 pending = atomic_swap_32(&ci->ci_ipipend[i], 0); 296 #ifndef __HAVE_ATOMIC_AS_MEMBAR 297 membar_producer(); 298 #endif 299 while ((bit = ffs(pending)) != 0) { 300 const u_int ipi_id = (i << IPI_BITW_SHIFT) | --bit; 301 ipi_intr_t *ipi_hdl = &ipi_intrs[ipi_id]; 302 303 pending &= ~(1U << bit); 304 KASSERT(ipi_hdl->func != NULL); 305 ipi_hdl->func(ipi_hdl->arg); 306 } 307 } 308 } 309 310 /* 311 * ipi_msg_cpu_handler: handle synchronous IPIs - iterate mailbox, 312 * execute the passed functions and acknowledge the messages. 313 */ 314 static void 315 ipi_msg_cpu_handler(void *arg __unused) 316 { 317 const struct cpu_info * const ci = curcpu(); 318 ipi_mbox_t *mbox = &ipi_mboxes[cpu_index(ci)]; 319 320 for (u_int i = 0; i < IPI_MSG_MAX; i++) { 321 ipi_msg_t *msg; 322 323 /* Get the message. */ 324 if ((msg = mbox->msg[i]) == NULL) { 325 continue; 326 } 327 mbox->msg[i] = NULL; 328 329 /* Execute the handler. */ 330 KASSERT(msg->func); 331 msg->func(msg->arg); 332 333 /* Ack the request. */ 334 atomic_dec_uint(&msg->_pending); 335 } 336 } 337 338 /* 339 * ipi_unicast: send an IPI to a single CPU. 340 * 341 * => The CPU must be remote; must not be local. 342 * => The caller must ipi_wait() on the message for completion. 343 */ 344 void 345 ipi_unicast(ipi_msg_t *msg, struct cpu_info *ci) 346 { 347 const cpuid_t id = cpu_index(ci); 348 349 KASSERT(msg->func != NULL); 350 KASSERT(kpreempt_disabled()); 351 KASSERT(curcpu() != ci); 352 353 msg->_pending = 1; 354 membar_producer(); 355 356 put_msg(&ipi_mboxes[id], msg); 357 ipi_trigger(IPI_SYNCH_ID, ci); 358 } 359 360 /* 361 * ipi_multicast: send an IPI to each CPU in the specified set. 362 * 363 * => The caller must ipi_wait() on the message for completion. 364 */ 365 void 366 ipi_multicast(ipi_msg_t *msg, const kcpuset_t *target) 367 { 368 const struct cpu_info * const self = curcpu(); 369 CPU_INFO_ITERATOR cii; 370 struct cpu_info *ci; 371 u_int local; 372 373 KASSERT(msg->func != NULL); 374 KASSERT(kpreempt_disabled()); 375 376 local = !!kcpuset_isset(target, cpu_index(self)); 377 msg->_pending = kcpuset_countset(target) - local; 378 membar_producer(); 379 380 for (CPU_INFO_FOREACH(cii, ci)) { 381 cpuid_t id; 382 383 if (__predict_false(ci == self)) { 384 continue; 385 } 386 id = cpu_index(ci); 387 if (!kcpuset_isset(target, id)) { 388 continue; 389 } 390 put_msg(&ipi_mboxes[id], msg); 391 ipi_trigger(IPI_SYNCH_ID, ci); 392 } 393 if (local) { 394 msg->func(msg->arg); 395 } 396 } 397 398 /* 399 * ipi_broadcast: send an IPI to all CPUs. 400 * 401 * => The caller must ipi_wait() on the message for completion. 402 */ 403 void 404 ipi_broadcast(ipi_msg_t *msg, bool skip_self) 405 { 406 const struct cpu_info * const self = curcpu(); 407 CPU_INFO_ITERATOR cii; 408 struct cpu_info *ci; 409 410 KASSERT(msg->func != NULL); 411 KASSERT(kpreempt_disabled()); 412 413 msg->_pending = ncpu - 1; 414 membar_producer(); 415 416 /* Broadcast IPIs for remote CPUs. */ 417 for (CPU_INFO_FOREACH(cii, ci)) { 418 cpuid_t id; 419 420 if (__predict_false(ci == self)) { 421 continue; 422 } 423 id = cpu_index(ci); 424 put_msg(&ipi_mboxes[id], msg); 425 ipi_trigger(IPI_SYNCH_ID, ci); 426 } 427 428 if (!skip_self) { 429 /* Finally, execute locally. */ 430 msg->func(msg->arg); 431 } 432 } 433 434 /* 435 * ipi_wait: spin-wait until the message is processed. 436 */ 437 void 438 ipi_wait(ipi_msg_t *msg) 439 { 440 int count = SPINLOCK_BACKOFF_MIN; 441 442 while (msg->_pending) { 443 KASSERT(msg->_pending < ncpu); 444 SPINLOCK_BACKOFF(count); 445 } 446 } 447